diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..3fc4c73e2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Shell scripts are executed and inspected verbatim on every CI host. +*.sh text eol=lf +# Original Go test drivers and embedded ASM/C retain byte-exact hash guards. +*.go text eol=lf +# Tracked ASM conformance sources retain their original whole-program hashes. +*.s text eol=lf diff --git a/.github/actions/publish-windows-llvm22/action.yml b/.github/actions/publish-windows-llvm22/action.yml new file mode 100644 index 000000000..61a524d00 --- /dev/null +++ b/.github/actions/publish-windows-llvm22/action.yml @@ -0,0 +1,9 @@ +name: Publish Windows LLVM 22 +description: Validate and publish one installed LLVM prefix without retry-dependent paths. +inputs: + attempt: + description: The successful setup-msys2 attempt (1, 2, or 3). + required: true +runs: + using: node24 + main: index.cjs diff --git a/.github/actions/publish-windows-llvm22/index.cjs b/.github/actions/publish-windows-llvm22/index.cjs new file mode 100644 index 000000000..b24b99f94 --- /dev/null +++ b/.github/actions/publish-windows-llvm22/index.cjs @@ -0,0 +1,157 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const {execFileSync} = require('node:child_process'); + +function requireThat(condition, message) { + if (!condition) throw new Error(message); +} + +function absent(file) { + try { fs.lstatSync(file); } catch (error) { + if (error.code === 'ENOENT') return; + throw error; + } + throw new Error('publication target already exists: ' + file); +} + +function directory(file) { + const stat = fs.lstatSync(file); + requireThat(stat.isDirectory() && !stat.isSymbolicLink(), 'not an owned directory: ' + file); + return stat; +} + +function noLinks(file) { + const stat = fs.lstatSync(file); + requireThat(!stat.isSymbolicLink() && (stat.isFile() || stat.isDirectory()), + 'redirected or unsupported prefix entry: ' + file); + if (stat.isDirectory()) { + for (const name of fs.readdirSync(file)) noLinks(path.join(file, name)); + } +} + +function publishPrefix({runnerTemp, attempt, systemRoot, run}) { + requireThat(typeof attempt === 'string' && /^[123]$/.test(attempt), 'unknown successful bootstrap attempt'); + for (const value of [runnerTemp, systemRoot]) { + requireThat(typeof value === 'string' && path.isAbsolute(value) && + path.normalize(value) === value && !/[\x00-\x1f]/.test(value), 'invalid absolute system path'); + } + for (let ancestor = runnerTemp; ; ancestor = path.dirname(ancestor)) { + directory(ancestor); + if (ancestor === path.dirname(ancestor)) break; + } + const source = path.join(runnerTemp, 'plan9asm-msys2-attempt-' + attempt, 'msys64', 'clang64'); + const target = path.join(runnerTemp, 'plan9asm-llvm22'); + const proof = path.join(runnerTemp, 'plan9asm-llvm22-proof'); + directory(path.dirname(path.dirname(source))); + directory(path.dirname(source)); + const original = directory(source); + absent(target); + absent(proof); + noLinks(source); + for (const relative of ['bin/llvm-config.exe', 'bin/clang.exe', 'bin/clang++.exe', + 'include/_mingw.h', 'lib/libkernel32.a']) { + requireThat(fs.lstatSync(path.join(source, relative)).isFile(), 'missing prefix input: ' + relative); + } + + // lstat covers symlinks/junctions; Windows attributes also reject other + // reparse tags. Inspect before descending and never follow a redirected node. + const checkReparse = ` +$ErrorActionPreference = 'Stop' +$root = Get-Item -Force -LiteralPath $env:PLAN9ASM_LLVM22_CHECK +for ($item = $root; $null -ne $item; $item = $item.Parent) { + if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) { throw 'reparse ancestor' } +} +$pending = [Collections.Generic.Stack[IO.FileSystemInfo]]::new() +$pending.Push($root) +while ($pending.Count) { + $item = $pending.Pop() + if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) { throw 'reparse prefix entry' } + if ($item -is [IO.DirectoryInfo]) { + foreach ($child in $item.GetFileSystemInfos()) { $pending.Push($child) } + } +}`; + run(path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'), + ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', checkReparse], + {env: {...process.env, PLAN9ASM_LLVM22_CHECK: source}}); + const checked = directory(source); + requireThat(original.dev === checked.dev && original.ino === checked.ino, 'prefix replaced during validation'); + for (let ancestor = path.dirname(source); ; ancestor = path.dirname(ancestor)) { + directory(ancestor); + if (ancestor === path.dirname(ancestor)) break; + } + noLinks(source); + absent(target); + // Windows directory rename is same-volume and cannot replace an existing + // file/directory. Do not copy, merge, remove failed attempts, or retry a move. + fs.renameSync(source, target); + const moved = directory(target); + requireThat(original.dev === moved.dev && original.ino === moved.ino, 'published prefix identity changed'); + + const bin = path.join(target, 'bin'); + // No bootstrap PATH: the relocation probe must resolve its DLLs, linker, + // headers and runtime from the moved prefix, not a retained old installation. + const env = {...process.env, PATH: [bin, path.join(systemRoot, 'System32'), systemRoot].join(path.delimiter)}; + if (process.platform === 'win32') { + for (const name of Object.keys(env)) if (name.toUpperCase() === 'PATH' && name !== 'PATH') delete env[name]; + } + const query = (file, args) => { + const result = run(file, args, {env}); + requireThat(typeof result === 'string' && !result.includes('\0'), 'invalid tool output'); + return result; + }; + const config = path.join(bin, 'llvm-config.exe'); + const clang = path.join(bin, 'clang.exe'); + const line = value => { + requireThat(/^[^\r\n]+\r?\n?$/.test(value), 'expected one tool output line'); + return value.replace(/\r?\n$/, ''); + }; + const samePath = (value, expected) => path.normalize(line(value).replace(/[\\/]/g, path.sep)) === expected; + requireThat(line(query(config, ['--version'])) === '22.1.8', 'expected LLVM 22.1.8'); + for (const [flag, relative] of [['--prefix', ''], ['--bindir', 'bin'], ['--includedir', 'include'], ['--libdir', 'lib']]) { + requireThat(samePath(query(config, [flag]), path.join(target, relative)), 'LLVM prefix did not relocate: ' + flag); + } + requireThat(/^clang version 22\.1\.8(?:\s|$)/.test(query(clang, ['--version'])), 'expected Clang 22.1.8'); + const resources = path.join(target, 'lib', 'clang', '22'); + requireThat(samePath(query(clang, ['--print-resource-dir']), resources), 'Clang resources did not relocate'); + directory(resources); + const flags = args => { + const value = query(config, args).replace(/\r/g, '').replace(/\n/g, ' ').trim(); + requireThat(value.length > 0 && !/[\x00-\x1f]/.test(value) && + !/plan9asm-msys2-attempt-/i.test(value), 'invalid or attempt-dependent LLVM flags'); + return value; + }; + const environment = { + CGO_CFLAGS: flags(['--cflags']), + CGO_CXXFLAGS: flags(['--cxxflags']), + CGO_LDFLAGS: flags(['--ldflags', '--libs', 'all', '--system-libs']), + LLVM_CONFIG: config, CC: clang, CXX: path.join(bin, 'clang++.exe'), + CGO_ENABLED: '1', GOFLAGS: '-tags=byollvm', + }; + fs.mkdirSync(proof); // Exclusive, retained on failure; never overwrite a probe. + const input = path.join(proof, 'publication.c'); + const output = path.join(proof, 'publication.exe'); + fs.writeFileSync(input, '#include \nint main(void) { return sizeof(uintptr_t) == 8 ? 0 : 1; }\n', {flag: 'wx'}); + query(clang, [input, '-o', output]); + requireThat(fs.lstatSync(output).isFile(), 'compiler did not produce the native probe'); + query(output, []); + return {environment, path: bin}; +} + +module.exports = {publishPrefix}; +if (require.main === module) { + try { + requireThat(process.platform === 'win32', 'Windows LLVM publication requires Windows'); + const result = publishPrefix({runnerTemp: process.env.RUNNER_TEMP, attempt: process.env.INPUT_ATTEMPT, + systemRoot: process.env.SystemRoot || process.env.SYSTEMROOT, + run: (file, args, options) => execFileSync(file, args, + {...options, encoding: 'utf8', timeout: 120000, maxBuffer: 1024 * 1024, windowsHide: true})}); + fs.appendFileSync(process.env.GITHUB_ENV, Object.entries(result.environment).map(([key, value]) => `${key}=${value}\n`).join('')); + fs.appendFileSync(process.env.GITHUB_PATH, result.path + '\n'); + console.log('Published and executed LLVM 22.1.8 from ' + result.path); + } catch (error) { + console.error(error.stack || error.message); + process.exitCode = 1; + } +} diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index e46e57771..dfde4216a 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -3,12 +3,8 @@ description: Set up Go with writable caches on hosted and self-hosted runners inputs: go-version: - description: Go version accepted by actions/setup-go + description: Exact stable Go version or a registered compatibility label required: true - cache: - description: Whether actions/setup-go should cache dependencies - required: false - default: "true" runs: using: composite @@ -21,8 +17,84 @@ runs: echo "GOMODCACHE=${RUNNER_TEMP}/go/pkg/mod" >> "$GITHUB_ENV" echo "GOCACHE=${RUNNER_TEMP}/go-build-cache" >> "$GITHUB_ENV" + # Compatibility labels are stable job identities. Freeze their actual SDKs + # in source so different runner caches or a mid-run release cannot split + # one group across compiler/source versions. Update these pins explicitly + # from https://go.dev/dl/?mode=json&include=all. + - name: Resolve exact Go version + id: resolve_go + shell: bash + env: + REQUESTED_GO_VERSION: ${{ inputs.go-version }} + run: | + set -euo pipefail + + case "$REQUESTED_GO_VERSION" in + 1.20.x) resolved_go=1.20.14 ;; + 1.21.x) resolved_go=1.21.13 ;; + 1.22.x) resolved_go=1.22.12 ;; + 1.23.x) resolved_go=1.23.12 ;; + 1.24.x) resolved_go=1.24.13 ;; + 1.25.x) resolved_go=1.25.14 ;; + 1.26.x) resolved_go=1.26.10 ;; + 1.27.x) resolved_go=1.27.3 ;; + *) + if [[ ! "$REQUESTED_GO_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Go selector has no frozen exact SDK: $REQUESTED_GO_VERSION" >&2 + exit 1 + fi + resolved_go="$REQUESTED_GO_VERSION" + ;; + esac + printf 'go-version=%s\n' "$resolved_go" >> "$GITHUB_OUTPUT" + - name: Set up Go + id: install_go uses: actions/setup-go@v7 with: - go-version: ${{ inputs.go-version }} - cache: ${{ inputs.cache }} + go-version: ${{ steps.resolve_go.outputs.go-version }} + # Account cache egress exhaustion must not block compiler installation. + # The writable job-local Go caches above remain available. + cache: false + + # Some self-hosted runner interruptions report a successful action before + # its download completes. Neither that outcome nor an ambient Go is proof + # that the requested compiler was installed. + - name: Verify Go installation + shell: bash + env: + REQUESTED_GO_VERSION: ${{ inputs.go-version }} + RESOLVED_GO_VERSION: ${{ steps.resolve_go.outputs.go-version }} + INSTALLED_GO_VERSION: ${{ steps.install_go.outputs.go-version }} + run: | + set -euo pipefail + + if [[ ! "$INSTALLED_GO_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Go installer did not report a complete stable version" >&2 + exit 1 + fi + + if [[ ! "$RESOLVED_GO_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ || + "$INSTALLED_GO_VERSION" != "$RESOLVED_GO_VERSION" ]]; then + echo "Resolved Go $RESOLVED_GO_VERSION, installed $INSTALLED_GO_VERSION" >&2 + exit 1 + fi + + if [[ "$REQUESTED_GO_VERSION" =~ ^[0-9]+\.[0-9]+\.x$ ]]; then + minor="${REQUESTED_GO_VERSION%.x}" + if [[ "$INSTALLED_GO_VERSION" != "$minor."* ]]; then + echo "Requested Go $REQUESTED_GO_VERSION, installed $INSTALLED_GO_VERSION" >&2 + exit 1 + fi + elif [[ "$REQUESTED_GO_VERSION" != "$INSTALLED_GO_VERSION" ]]; then + echo "Requested Go $REQUESTED_GO_VERSION, installed $INSTALLED_GO_VERSION" >&2 + exit 1 + fi + + actual="$(GOTOOLCHAIN=local go env GOVERSION)" + if [[ "$actual" != "go$INSTALLED_GO_VERSION" ]]; then + echo "Installed Go $INSTALLED_GO_VERSION, but PATH selects $actual" >&2 + exit 1 + fi + printf 'Verified Go %s: %s\n' "$INSTALLED_GO_VERSION" "$(command -v go)" + GOTOOLCHAIN=local go version diff --git a/.github/actions/setup-node22/action.yml b/.github/actions/setup-node22/action.yml new file mode 100644 index 000000000..c32dc0b0a --- /dev/null +++ b/.github/actions/setup-node22/action.yml @@ -0,0 +1,34 @@ +name: Setup Node.js 22 +description: Install the exact Node.js runtime shared by CI producers and verifiers + +runs: + using: composite + steps: + - name: Install Node.js 22 + id: install_node22 + uses: actions/setup-node@v6 + with: + node-version: '22.23.4' + + # An installer step marked successful without its output must not borrow + # the runner image's ambient Node or produce an incompatible WASM scope. + - name: Verify Node.js installation + shell: bash + env: + EXPECTED_NODE_VERSION: 'v22.23.4' + INSTALLED_NODE_VERSION: ${{ steps.install_node22.outputs.node-version }} + run: | + set -euo pipefail + + if [[ "$INSTALLED_NODE_VERSION" != "$EXPECTED_NODE_VERSION" ]]; then + echo "Expected Node $EXPECTED_NODE_VERSION, installer reported '$INSTALLED_NODE_VERSION'" >&2 + exit 1 + fi + + actual="$(node --version)" + actual="${actual%$'\r'}" + if [[ "$actual" != "$EXPECTED_NODE_VERSION" ]]; then + echo "Installed Node $INSTALLED_NODE_VERSION, but PATH selects '$actual'" >&2 + exit 1 + fi + printf 'Verified Node %s: %s\n' "$actual" "$(command -v node)" diff --git a/.github/actions/setup-windows-llvm22/action.yml b/.github/actions/setup-windows-llvm22/action.yml index 6134a7f85..01ea699a9 100644 --- a/.github/actions/setup-windows-llvm22/action.yml +++ b/.github/actions/setup-windows-llvm22/action.yml @@ -5,10 +5,42 @@ runs: using: composite steps: - name: Set up MSYS2 for LLVM 22 + id: msys2_attempt_1 + continue-on-error: true uses: msys2/setup-msys2@v2 with: msystem: CLANG64 path-type: inherit + release: true + location: ${{ runner.temp }}/plan9asm-msys2-attempt-1 + update: true + + # These are bounded installer retries, not test retries or LLVM fallbacks. + # Only the first two bootstrap failures may proceed to another attempt. + # setup-msys2 refuses an existing installation, including a failed update's + # directory. Keep each fresh attempt separate; the action rewrites its + # msys2 shell wrapper to the successful installation without deleting any. + - name: Retry MSYS2 bootstrap (2 of 3) + id: msys2_attempt_2 + if: ${{ steps.msys2_attempt_1.outcome == 'failure' }} + continue-on-error: true + uses: msys2/setup-msys2@v2 + with: + msystem: CLANG64 + path-type: inherit + release: true + location: ${{ runner.temp }}/plan9asm-msys2-attempt-2 + update: true + + - name: Retry MSYS2 bootstrap (3 of 3, failure is fatal) + id: msys2_attempt_3 + if: ${{ steps.msys2_attempt_1.outcome == 'failure' && steps.msys2_attempt_2.outcome == 'failure' }} + uses: msys2/setup-msys2@v2 + with: + msystem: CLANG64 + path-type: inherit + release: true + location: ${{ runner.temp }}/plan9asm-msys2-attempt-3 update: true - name: Install LLVM 22 @@ -39,27 +71,31 @@ runs: # libc++ supplies the compiler runtime. Keep the exact LLVM component # set mutually consistent while acknowledging the cc-libs virtual. - pacman --noconfirm -U \ - --assume-installed "$prefix-cc-libs=$LLVM_VERSION" \ - "${urls[@]}" + install_status=1 + for attempt in 1 2 3; do + if pacman --noconfirm -U \ + --assume-installed "$prefix-cc-libs=$LLVM_VERSION" \ + "${urls[@]}"; then + install_status=0 + break + else + install_status=$? + fi + if (( attempt < 3 )); then + echo "Pinned LLVM install attempt $attempt failed ($install_status); retrying in 5 seconds" >&2 + sleep 5 + fi + done + if (( install_status != 0 )); then + exit "$install_status" + fi if [[ "$(llvm-config --version)" != "$LLVM_VERSION" ]]; then echo "expected LLVM $LLVM_VERSION, got $(llvm-config --version)" >&2 exit 1 fi - flatten() { - local value=$1 - value="${value//$'\r'/}" - value="${value//$'\n'/ }" - printf '%s' "$value" - } - echo "CGO_CFLAGS=$(flatten "$(llvm-config --cflags)")" >> "$GITHUB_ENV" - echo "CGO_CXXFLAGS=$(flatten "$(llvm-config --cxxflags)")" >> "$GITHUB_ENV" - echo "CGO_LDFLAGS=$(flatten "$(llvm-config --ldflags --libs all --system-libs)")" >> "$GITHUB_ENV" - clang_bin="$(cygpath -w /clang64/bin)" - echo "CC=$clang_bin\\clang.exe" >> "$GITHUB_ENV" - echo "CXX=$clang_bin\\clang++.exe" >> "$GITHUB_ENV" - echo 'CGO_ENABLED=1' >> "$GITHUB_ENV" - echo 'GOFLAGS=-tags=byollvm' >> "$GITHUB_ENV" - echo "$clang_bin" >> "$GITHUB_PATH" + - name: Publish stable LLVM 22 prefix + uses: ./.github/actions/publish-windows-llvm22 + with: + attempt: ${{ steps.msys2_attempt_1.outcome == 'success' && '1' || steps.msys2_attempt_2.outcome == 'success' && '2' || steps.msys2_attempt_3.outcome == 'success' && '3' || '' }} diff --git a/.github/scripts/audit-arm64-fallback.cjs b/.github/scripts/audit-arm64-fallback.cjs new file mode 100644 index 000000000..95a0d9dea --- /dev/null +++ b/.github/scripts/audit-arm64-fallback.cjs @@ -0,0 +1,46 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); + +// The exhaustive runner separately verifies the complete source, tools, plan, +// commands and raw event protocol. This additional contract prevents removal +// of an execution child from silently shrinking this required runtime matrix. +function auditARM64Fallback(raw) { + const root = 'TestFallbackARM64WideFrameCompilesWithLLVM22'; + const expected = [root, ...['darwin', 'linux', 'windows', 'native_complete-argument_oracle'] + .map(child => `${root}/${child}`)].sort(); + const started = new Set(); + const passed = new Set(); + let packagePassed = false; + for (const line of raw.split(/\r?\n/).filter(Boolean)) { + const event = JSON.parse(line); + if (event.Action === 'build-output' && typeof event.ImportPath === 'string' && event.ImportPath && !event.Test) { + continue; + } + assert.equal(event.Package, 'github.com/xgo-dev/plan9asm/cmd/plan9asmll'); + assert.ok(!packagePassed, 'event after package terminal'); + assert.ok(!['fail', 'skip', 'build-fail'].includes(event.Action), 'required execution did not pass'); + if (event.Action === 'run') { + assert.ok(expected.includes(event.Test) && !started.has(event.Test), 'foreign or duplicate test'); + started.add(event.Test); + } + if (event.Action === 'pass') { + if (event.Test) { + assert.ok(started.has(event.Test) && !passed.has(event.Test), 'unstarted or duplicate terminal'); + passed.add(event.Test); + } else { + packagePassed = true; + } + } + } + assert.ok(packagePassed, 'package did not finish'); + assert.deepEqual([...started].sort(), expected, 'missing required execution axis'); + assert.deepEqual([...passed].sort(), expected, 'incomplete required execution axis'); + return { roots: 1, children: 4, passed: 5, skipped: 0, incomplete: 0 }; +} + +module.exports = { auditARM64Fallback }; + +if (require.main === module) { + assert.equal(process.argv.length, 3, 'one original runner trace required'); + process.stdout.write(JSON.stringify(auditARM64Fallback(fs.readFileSync(process.argv[2], 'utf8')), null, 2) + '\n'); +} diff --git a/.github/scripts/audit-blend-minpos-native.cjs b/.github/scripts/audit-blend-minpos-native.cjs new file mode 100644 index 000000000..bdb73ee02 --- /dev/null +++ b/.github/scripts/audit-blend-minpos-native.cjs @@ -0,0 +1,148 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = ['TestCrossLinuxRuntimeMatrixX86BlendMinposOriginalFaultsAMD64', + 'TestCrossLinuxRuntimeMatrixX86BlendMinposOriginalFaults386']; +const axes = ['backend_O0', 'backend_O2', 'IR_O2_backend_O2']; +const forms = [['BLENDVPS', 'X'], ['VBLENDVPS', 'X'], ['VBLENDVPS', 'Y'], + ['BLENDVPD', 'X'], ['VBLENDVPD', 'X'], ['VBLENDVPD', 'Y'], + ['PBLENDVB', 'X'], ['VPBLENDVB', 'X'], ['VPBLENDVB', 'Y'], + ['PHMINPOSUW', 'X'], ['VPHMINPOSUW', 'X']]; +const parents = roots.flatMap(root => axes.map(axis => root + '/' + axis)); +const leaves = parents.flatMap(parent => forms.flatMap(([op, bank]) => ['live', 'dead'].flatMap(role => + [0, 1].map(zero => `${parent}/${op}/${bank}/${role}/maskzero=${zero}`)))); +const children = [...parents, ...leaves]; +const leafSet = new Set(leaves), parentSet = new Set(parents); +const marker = 'blend/minimum original Go bytes and LLVM native alignment/access/CF/MXCSR=0x1f80/canary probes 20 failures 0\n'; +const preflight = 'blend/minimum actual native SSE4.1=1 AVX2=1\n'; +const hostPrefix = 'IA32 direct host:', sdkPrefix = 'actual LEA source/object receipt:'; +const imagePrefix = 'native oracle static IA32 ELF'; +const host = /IA32 direct host: linux\/amd64 kernel=x86_64 CPUsha=[a-f0-9]{64} mapsha=[a-f0-9]{64}\b/g; +const image = /native oracle static IA32 ELF bytes=[1-9][0-9]* sha256=[a-f0-9]{64}\b/g; + +// These 22 minimum-ISA functions and 44 live/dead/mask cases are actual +// original-byte fault execution, not the 78-program private mathematical +// companion or its virtual Z/Y/X alias observations. +function auditBlendMinposNative(inventory, raw) { + const events = raw.split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line)); + // The shared owner checker closes the root/axis topology. Keep the 264 + // fixed leaves in the stricter linear protocol below: feeding every leaf + // into its all-ancestors loop would make exhaustive negative audits O(n²). + const ancestry = events.filter(event => !leafSet.has(event.Test)); + auditNativeOwners(inventory, ancestry.map(event => JSON.stringify(event)).join('\n'), roots, parents); + const outputs = new Map([...roots, ...children].map(name => [name, ''])); + const states = new Map(), passed = new Map(); + const spans = new Map(roots.map(root => [root, []])); + let started = false, complete = false, driver = '', kernelExecutions = 0; + + function rootObservations(root, final) { + const output = outputs.get(root), arch = root.endsWith('386') ? '386' : 'amd64'; + const capture = new RegExp('actual LEA source/object receipt: SDK=go1\\.27\\.2 target=linux/' + arch + + ' source=[a-f0-9]{64} driver=([a-f0-9]{64}) object=[a-f0-9]{64}\\b', 'g'); + const captures = [...output.matchAll(capture)], hosts = [...output.matchAll(host)]; + assert.equal(hosts.length, final ? 2 : 1, 'missing or duplicated direct native host bracket'); + assert.equal(output.split(hostPrefix).length - 1, hosts.length, 'malformed extra native host'); + assert.equal(captures.length, 1, 'missing unique actual Go1.27.2 source/object receipt'); + assert.equal(output.split(sdkPrefix).length - 1, 1, 'malformed extra SDK source/object receipt'); + if (!driver) driver = captures[0][1]; + assert.equal(captures[0][1], driver, 'native owners used different actual Go drivers'); + assert.equal(output.split(preflight).length - 1, 1, 'missing complete actual SSE4.1/AVX2 preflight'); + assert.equal(output.split('blend/minimum actual native').length - 1, 1, 'malformed extra native ISA preflight'); + assert.ok(hosts[0].index < output.indexOf(preflight) && output.indexOf(preflight) < captures[0].index, + 'ISA/source capture escaped the actual native host bracket'); + assert.ok(!output.includes(marker) && !output.includes(imagePrefix), + 'root summary cannot replace actual leaf execution or axis-owned images'); + if (final) { + assert.ok(captures[0].index < hosts[1].index, 'final host preceded original source capture'); + const span = spans.get(root).filter(value => value.offset <= hosts[1].index).pop(); + for (const axis of axes) { + assert.ok(passed.has(root + '/' + axis), 'native root lacks a complete optimization axis'); + assert.ok(span && span.sequence > passed.get(root + '/' + axis), 'final host preceded an execution axis'); + } + } + } + + for (const [sequence, event] of events.entries()) { + assert.ok(!complete, 'event after complete blend/minimum package'); + if (event.Action === 'build-output' && event.ImportPath && !event.Test) continue; + assert.equal(event.Package, 'github.com/xgo-dev/plan9asm', 'foreign native package'); + if (!event.Test) { + assert.ok(['start', 'output', 'pass'].includes(event.Action), 'unknown native package lifecycle'); + if (event.Action === 'start') { + assert.ok(!started, 'duplicate native package start'); + started = true; + } else { + assert.ok(started, 'observation before native package start'); + if (event.Action === 'output') assert.equal(typeof event.Output, 'string', 'nontext package output'); + if (event.Action === 'pass') complete = true; + } + continue; + } + assert.ok(started, 'native test before package start'); + assert.ok(['run', 'output', 'pass'].includes(event.Action), 'unknown native test lifecycle'); + const root = event.Test.split('/')[0]; + if (leafSet.has(event.Test)) { + const parent = event.Test.split('/').slice(0, 2).join('/'); + assert.equal(states.get(root), 'running', 'leaf outside its live root'); + assert.equal(states.get(parent), 'running', 'leaf outside its live optimization axis'); + } + if (event.Action === 'run') { + assert.ok(!states.has(event.Test), 'native test lifecycle restarted'); + states.set(event.Test, 'running'); + if (event.Test !== root) rootObservations(root, false); + if (leafSet.has(event.Test) && root.endsWith('386')) { + const parent = event.Test.split('/').slice(0, 2).join('/'); + assert.equal([...outputs.get(parent).matchAll(image)].length, 1, + 'IA32 leaf started before its complete static image inspection'); + } + continue; + } + assert.equal(states.get(event.Test), 'running', 'observation outside its live native owner'); + if (event.Action === 'output') { + assert.equal(typeof event.Output, 'string', 'nontext native output'); + if (event.Test === root) spans.get(root).push({offset: outputs.get(root).length, sequence}); + outputs.set(event.Test, outputs.get(event.Test) + event.Output); + continue; + } + states.set(event.Test, 'passed'); + passed.set(event.Test, sequence); + if (event.Test === root) { + rootObservations(root, true); + continue; + } + const output = outputs.get(event.Test); + for (const foreign of [hostPrefix, sdkPrefix, 'blend/minimum actual native']) { + assert.ok(!output.includes(foreign), 'child cannot replace actual root host/SDK/ISA ownership'); + } + if (parentSet.has(event.Test)) { + assert.ok(!output.includes(marker), 'axis summary cannot replace actual completed kernels'); + const count = root.endsWith('386') ? 1 : 0; + assert.equal([...output.matchAll(image)].length, count, 'axis lacks its exact native static image'); + assert.equal(output.split(imagePrefix).length - 1, count, 'malformed extra IA32 static image'); + for (const leaf of leaves.filter(name => name.startsWith(event.Test + '/'))) { + assert.ok(passed.has(leaf), 'axis ended before its complete opcode/width/live/dead/mask family'); + } + } else { + assert.ok(!output.includes(imagePrefix), 'leaf cannot replace its axis-owned IA32 image'); + assert.equal(output.split(marker).length - 1, 1, 'leaf lacks exactly one complete original/translated 20-probe observation'); + assert.equal(output.split('blend/minimum original Go bytes').length - 1, 1, 'malformed extra leaf completion'); + kernelExecutions++; + } + } + assert.equal(passed.size, roots.length + children.length, 'incomplete actual root/axis/leaf inventory'); + return {roots: roots.length, children: children.length, passed: passed.size, skipped: 0, incomplete: 0, + nativeHosts: 4, sdkCaptures: 2, isaPreflights: 2, staticIA32Images: 3, + kernelExecutions, pairedProbes: kernelExecutions * 20, directProbeProcesses: kernelExecutions * 40}; +} + +module.exports = {auditBlendMinposNative}; +if (require.main === module) { + const directory = process.argv[2]; + process.stdout.write(JSON.stringify(auditBlendMinposNative( + fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8'))) + '\n'); +} diff --git a/.github/scripts/audit-duplicate-native-families.cjs b/.github/scripts/audit-duplicate-native-families.cjs new file mode 100644 index 000000000..1832b2c91 --- /dev/null +++ b/.github/scripts/audit-duplicate-native-families.cjs @@ -0,0 +1,169 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); +const {requireIA32Images, requireSDKCommands} = require('./audit-integer-native-families.cjs'); + +const roots = [ + 'TestCrossLinuxRuntimeMatrixX86DuplicateMoveOriginalGoAndNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86DuplicateMoveOriginalGoAndNative386', +]; +const nativeAxes = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2']; +const axes = [...nativeAxes, 'OriginalGo']; +const children = roots.flatMap(root => axes.map(axis => root + '/' + axis)); +const nativeHost = /IA32 direct host: linux\/amd64 kernel=x86_64 CPUsha=[a-f0-9]{64} mapsha=[a-f0-9]{64}\b/g; +const avxPreflight = /actual guest CPUID\/XCR0 preflight: AVX=1 AVX2=[01] AVX512F=[01]\r?\n/g; +const originalArgs = '["-test.run=^TestOriginal$" "-test.count=1" "-test.v"]'; +const originalEvidence = 'actual original Go legacy/VEX: 18 rows × 16 IEEE/bit patterns × 6 masks = 1728 whole32/FLAGS/DF/MXCSR/canary outcomes; no EVEX execution claim\n'; + +function pipelineEvidence(arch) { + return `duplicate native arch=${arch} live=60 dead=30 whole64-outcomes=5760 fault-processes=712; legacy original opcode/all mod16, E4NF/E5NF independent full-span references\n`; +} + +function requireExecutableRoles(output, arch, original, commands, observed) { + const receipts = [...output.matchAll(/actual executable=(\/[^\r\n]+?) sha256=([a-f0-9]{64}) argv=(\[[^\r\n]*\])\r?\n/g)]; + assert.equal(receipts.length, output.split('actual executable=').length - 1, + 'malformed actual executable receipt'); + assert.equal(receipts.length, (original ? 2 : 0) + (arch === '386' ? 1 : 0), + 'actual executable receipts escaped their original Go or native pipeline role'); + if (original) { + assert.deepEqual(receipts.slice(0, 2).map(receipt => receipt.slice(1)), + commands.map(command => command.slice(1)), 'SDK commands lost their explicit OriginalGo ownership'); + } + if (arch !== '386') return; + + const executable = receipts[original ? 2 : 0]; + if (original) { + const binary = / "-o" "([^"\r\n]+)" /.exec(commands[1][3]); + assert.ok(binary, 'original Go build lost its exact output executable'); + assert.equal(executable[1], binary[1], 'original execution differs from the actual Go build output'); + } else { + assert.equal(path.posix.basename(executable[1]), 'comparison', + 'native pipeline cannot substitute original Go compilation or a foreign executable'); + } + assert.equal(executable[2], observed.images[0][2], 'actual executable differs from inspected IA32 image'); + assert.equal(executable[3], original ? originalArgs : '[]', 'actual executable arguments changed'); + assert.ok(observed.images[0].index < executable.index && executable.index < observed.direct[0].index, + 'actual executable escaped its inspected image/direct execution bracket'); +} + +// Each first-party wrapper supplies its own fixed inventory and exact program +// completion witnesses. Reports can never choose the denominator or family. +function auditSSEAVXNativeFamilies(inventory, raw, roots, pipelineEvidence, originalEvidence) { + const children = roots.flatMap(root => axes.map(axis => root + '/' + axis)); + const result = auditNativeOwners(inventory, raw, roots, children); + const outputs = new Map([...roots, ...children].map(name => [name, ''])); + const outputSpans = new Map(roots.map(root => [root, []])); + const states = new Map(); + const childPasses = new Map(); + let packageStarted = false, packageComplete = false; + let nativeHosts = 0, avxPreflights = 0, sdkBuilds = 0; + let staticIA32Images = 0, directIA32Executions = 0; + + const events = raw.split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line)); + for (const [sequence, event] of events.entries()) { + assert.ok(!packageComplete, 'event after the complete duplicate package'); + if (event.Action === 'build-output' && event.ImportPath && !event.Test) continue; + if (!event.Test) { + assert.ok(['start', 'output', 'pass'].includes(event.Action), 'unknown duplicate package action'); + if (event.Action === 'start') { + assert.ok(!packageStarted, 'duplicate native package start'); + packageStarted = true; + } else { + assert.ok(packageStarted, 'duplicate observation before package start'); + if (event.Action === 'output') assert.equal(typeof event.Output, 'string', 'package output must be text'); + if (event.Action === 'pass') packageComplete = true; + } + continue; + } + assert.ok(packageStarted, 'duplicate test before package start'); + assert.ok(['run', 'output', 'pass'].includes(event.Action), 'unknown duplicate test action'); + const root = event.Test.split('/')[0]; + const arch = root.endsWith('386') ? '386' : 'amd64'; + const original = event.Test === root + '/OriginalGo'; + if (event.Action === 'run') { + states.set(event.Test, 'running'); + if (event.Test !== root) { + const output = outputs.get(root); + const hosts = [...output.matchAll(nativeHost)], preflights = [...output.matchAll(avxPreflight)]; + assert.equal(hosts.length, 1, 'duplicate axis lacks its preceding actual native host observation'); + assert.equal(preflights.length, 1, 'duplicate axis lacks actual AVX CPUID/XCR0 permission'); + assert.ok(hosts[0].index < preflights[0].index, 'duplicate AVX preflight preceded native host identity'); + assert.ok(!output.includes('actual executable='), 'original Go receipts must belong to its explicit child'); + if (original) { + for (const axis of nativeAxes) { + assert.equal(states.get(root + '/' + axis), 'passed', 'original Go preceded a complete native axis'); + } + } + } + continue; + } + assert.equal(states.get(event.Test), 'running', 'duplicate observation after its terminal'); + if (event.Action === 'output') { + assert.equal(typeof event.Output, 'string', 'duplicate output must be text'); + if (event.Test === root) outputSpans.get(root).push({offset: outputs.get(root).length, sequence}); + outputs.set(event.Test, outputs.get(event.Test) + event.Output); + continue; + } + states.set(event.Test, 'passed'); + const output = outputs.get(event.Test); + if (event.Test !== root) { + childPasses.set(event.Test, sequence); + assert.equal([...output.matchAll(nativeHost)].length, 0, 'child host cannot replace root native ownership'); + assert.equal([...output.matchAll(avxPreflight)].length, 0, 'child preflight cannot replace root AVX permission'); + const evidence = original ? originalEvidence : pipelineEvidence(arch); + assert.equal(output.split(evidence).length - 1, 1, 'missing exact complete duplicate execution evidence'); + if (original) { + const commands = requireSDKCommands(output); + assert.ok(commands[1].index < output.indexOf(evidence), 'original Go terminal preceded actual SDK build'); + sdkBuilds++; + let observed; + if (arch === '386') { + observed = requireIA32Images(output, 1, originalArgs); + assert.ok(commands[1].index < observed.images[0].index && + observed.direct[0].index < output.indexOf(evidence), 'original Go IA32 execution escaped SDK/terminal bracket'); + staticIA32Images++; + directIA32Executions++; + } + requireExecutableRoles(output, arch, true, commands, observed); + } else { + let observed; + if (arch === '386') { + observed = requireIA32Images(output, 2, '[]'); + assert.ok(observed.images[1].index < output.indexOf(evidence), 'fault image observation followed complete execution'); + staticIA32Images += 2; + directIA32Executions++; + } + requireExecutableRoles(output, arch, false, undefined, observed); + } + continue; + } + + const hosts = [...output.matchAll(nativeHost)], preflights = [...output.matchAll(avxPreflight)]; + assert.equal(hosts.length, 2, 'duplicate root must retain pre/post actual native host observations'); + assert.equal(preflights.length, 1, 'duplicate root must retain its single actual AVX preflight'); + assert.ok(!output.includes('actual executable='), 'SDK receipts escaped their OriginalGo child'); + for (const axis of axes) assert.equal(states.get(root + '/' + axis), 'passed', 'root preceded required child completion'); + const spans = outputSpans.get(root).filter(span => span.offset <= hosts[1].index); + assert.ok(spans[spans.length - 1].sequence > childPasses.get(root + '/OriginalGo'), + 'final native host observation preceded complete original Go execution'); + // These are live host receipts, not immutable CPU MHz or map snapshots. + nativeHosts += 2; + avxPreflights++; + } + return {...result, nativeHosts, avxPreflights, sdkBuilds, staticIA32Images, directIA32Executions}; +} + +function auditDuplicateNativeFamilies(inventory, raw) { + return auditSSEAVXNativeFamilies(inventory, raw, roots, pipelineEvidence, originalEvidence); +} + +module.exports = {auditDuplicateNativeFamilies, auditSSEAVXNativeFamilies, roots, children}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditDuplicateNativeFamilies(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-floating-compare.cjs b/.github/scripts/audit-floating-compare.cjs new file mode 100644 index 000000000..dde879a45 --- /dev/null +++ b/.github/scripts/audit-floating-compare.cjs @@ -0,0 +1,25 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = [ + 'TestCrossLinuxRuntimeMatrixX86FloatingCompareIndependentIEEEAndMXCSR', + 'TestCrossLinuxRuntimeMatrixX86FloatingCompareDeadResultEffects', + 'TestCrossLinuxRuntimeMatrixX86FloatingCompareFPScalarByteSpans', + 'TestCrossLinuxRuntimeMatrixX86FloatingCompareLegacyUpperAliases', +]; +const children = [ + ...['CMPPS', 'CMPPD', 'CMPSS', 'CMPSD'].map(opcode => roots[0] + '/' + opcode), + ...['-O0', '-O2'].map(level => roots[1] + '/' + level), +]; +const auditFloatingCompare = (inventory, raw) => auditNativeOwners(inventory, raw, roots, children); + +module.exports = {auditFloatingCompare, roots, children}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditFloatingCompare(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-floating-helpers.cjs b/.github/scripts/audit-floating-helpers.cjs new file mode 100644 index 000000000..e03aaef2e --- /dev/null +++ b/.github/scripts/audit-floating-helpers.cjs @@ -0,0 +1,50 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = [ + 'TestAMD64GetExpRuntimeSemantics', + 'TestAMD64GetMantRuntimeSemantics', + 'TestAMD64ScaleFRuntimeSemantics', + 'TestAMD64FloatingRangeRuntimeSemantics', + 'TestAMD64FloatingRangeDoubleAndScalarRuntimeSemantics', + 'TestAMD64FixupImmediateRuntimeSemantics', + 'TestAMD64ScaledRoundRuntimeSemantics', + 'TestAMD64FPClassRuntimeSemantics', +]; +const axes = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2']; +const children = roots.flatMap(root => axes.map(axis => root + '/' + axis)); + +function auditFloatingHelpers(inventory, raw) { + const result = auditNativeOwners(inventory, raw, roots, children); + const output = new Map(roots.map(root => [root, ''])); + const preflights = new Set(); + for (const line of raw.split(/\r?\n/).filter(Boolean)) { + const event = JSON.parse(line); + if (event.Action === 'output' && output.has(event.Test)) { + output.set(event.Test, output.get(event.Test) + event.Output); + } + if (event.Action !== 'run' || !children.includes(event.Test)) continue; + const root = event.Test.split('/')[0]; + const observation = output.get(root); + const flags = observation.match(/AVX512F=1 AVX512DQ=1 AVX512VL=1 OSXSAVE=1 XCR0=([0-9a-f]+)\b/); + assert.ok(flags, 'missing current native F/DQ/VL and OS-enabled state: ' + root); + assert.equal(BigInt('0x' + flags[1]) & 0xe6n, 0xe6n, 'incomplete native XCR0: ' + root); + assert.ok(observation.includes('FLOATING_HELPER_NATIVE_DQ_OK'), + 'original native DQ opcode did not complete before its runtime: ' + root); + preflights.add(root); + } + assert.equal(preflights.size, 8, 'missing actual native floating-helper preflight'); + return {...result, preflights: preflights.size}; +} + +module.exports = {auditFloatingHelpers, roots, children}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditFloatingHelpers(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-integer-native-families.cjs b/.github/scripts/audit-integer-native-families.cjs new file mode 100644 index 000000000..e218c53d0 --- /dev/null +++ b/.github/scripts/audit-integer-native-families.cjs @@ -0,0 +1,191 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = [ + 'TestCrossLinuxRuntimeMatrixX86MOVNTIOriginalFlagsAndFaults', + 'TestCrossLinuxRuntimeMatrixX86PackedAlignLegacyNativeFaults', + 'TestCrossLinuxRuntimeMatrixX86ImplicitDivideOriginalGoAndNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86ImplicitDivideOriginalGoAndNative386', + 'TestCrossLinuxRuntimeMatrixX86SegmentQueryOriginalFlagsAliasesAndFaults', + 'TestCrossLinuxRuntimeMatrixX86MoveHalfOriginalGoFlagsDFAliases', + 'TestCrossLinuxRuntimeMatrixX86ImplicitDivideLogicalFPOriginalGoAndNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86ImplicitDivideLogicalFPOriginalGoAndNative386', +]; +const axes = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2']; +const children = roots.flatMap(root => axes.map(axis => root + '/' + axis)); +const nativeHost = /IA32 direct host: linux\/amd64 kernel=x86_64 CPUsha=[a-f0-9]{64} mapsha=[a-f0-9]{64}\b/g; +const originalArgs = '["-test.run=^TestOriginal$" "-test.count=1" "-test.v"]'; +const segmentEvidence = '48 original-Go opcode rows × 16 inputs; full GP aliases, six status flags, canaries; 160 live/dead m16 fault processes\n'; +const halfPreflight = 'half native CPUID/XCR0 AVX512F=1; original EVEX transport opcode executed\n'; +const halfEvidence = 'half native full64/FLAGS/DF=1/canary outcomes=512; complete original STD/CLD source and all eight named half moves\n'; +const halfOriginal = 'half actual original Go full64/FLAGS/DF=1/canary outcomes=512\n'; + +function logicalFPCompletion(ia32, original) { + const arch = ia32 ? '386' : 'amd64'; + const rows = ia32 ? 6 : 8; + const values = ia32 ? 96 : 128; + const faults = ia32 ? 18 : 24; + return original + ? `logical FP actual original Go arch=${arch} rows=${rows} values=${values}; complete original FP divisor source and whole AX/DX/DF/canary observations\n` + : `logical FP native arch=${arch} rows=${rows} values=${values} dead-division-faults=${faults}; whole AX/DX, DF and canaries; zero/overflow and signed-min/-1 SIGFPE\n`; +} + +function requireIA32Images(output, staticCount, args) { + const images = [...output.matchAll(/native oracle static IA32 ELF bytes=([1-9][0-9]*) sha256=([a-f0-9]{64})\b/g)]; + const direct = [...output.matchAll(/IA32 direct ELF bytes=([1-9][0-9]*) sha256=([a-f0-9]{64}) argv=(\[[^\r\n]*\])/g)]; + assert.equal(images.length, staticCount, 'missing complete static IA32 observations'); + assert.equal(direct.length, 1, 'missing direct IA32 execution'); + assert.equal(direct[0][3], args, 'wrong direct IA32 execution arguments'); + assert.equal(direct[0][1], images[0][1], 'direct IA32 image size changed'); + assert.equal(direct[0][2], images[0][2], 'direct IA32 image hash changed'); + assert.ok(images[0].index < direct[0].index, 'direct IA32 execution preceded image inspection'); + if (staticCount === 2) { + assert.ok(direct[0].index < images[1].index, 'fault image inspection preceded value execution'); + } + return {images, direct}; +} + +function requireSDKCommands(output) { + const commands = [...output.matchAll(/actual executable=([^\r\n]+\/bin\/go) sha256=([a-f0-9]{64}) argv=(\[[^\r\n]*\])/g)]; + assert.equal(commands.length, 2, 'must observe the actual SDK version and original Go build'); + assert.equal(commands[0][1], commands[1][1], 'SDK executable changed'); + assert.equal(commands[0][2], commands[1][2], 'SDK hash changed'); + assert.equal(commands[0][3], '["version"]', 'actual SDK version was not observed'); + assert.match(commands[1][3], /^\["test" "-p=1" "-vet=off" "-c" "-o" "\/[^"\r\n]+\/original\.test" "\."\]$/, + 'original Go compile did not retain its complete actual SDK command'); + assert.ok(commands[0].index < commands[1].index, 'original Go build preceded actual SDK version'); + return commands; +} + +function auditIntegerNativeFamilies(inventory, raw) { + const result = auditNativeOwners(inventory, raw, roots, children); + const states = new Map(); + const outputs = new Map([...roots, ...children].map(name => [name, ''])); + const spans = new Map(roots.map(root => [root, []])); + const lastChildPass = new Map(); + let packageStarted = false, packageComplete = false; + let nativeHosts = 0, sdkBuilds = 0, staticIA32Images = 0, directIA32Executions = 0; + + const events = raw.split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line)); + for (const [sequence, event] of events.entries()) { + assert.ok(!packageComplete, 'event after the complete native package'); + if (event.Action === 'build-output' && event.ImportPath && !event.Test) continue; + if (!event.Test) { + assert.ok(['start', 'output', 'pass'].includes(event.Action), 'unknown native package action'); + if (event.Action === 'start') { + assert.ok(!packageStarted, 'duplicate native package start'); + packageStarted = true; + } else { + assert.ok(packageStarted, 'native event before package start'); + if (event.Action === 'output') assert.equal(typeof event.Output, 'string', 'package output must be text'); + if (event.Action === 'pass') packageComplete = true; + } + continue; + } + assert.ok(packageStarted, 'native test before package start'); + assert.ok(['run', 'output', 'pass'].includes(event.Action), 'unknown native test action'); + const root = event.Test.split('/')[0]; + const half = root.includes('MoveHalf'); + const logicalFP = root.includes('ImplicitDivideLogicalFP'); + const observed = root.includes('ImplicitDivide') || root.includes('SegmentQuery') || half; + const ia32 = root.endsWith('386'); + if (event.Action === 'run') { + states.set(event.Test, 'running'); + if (observed && event.Test !== root) { + assert.equal([...outputs.get(root).matchAll(nativeHost)].length, 1, + 'integer axis lacks its preceding actual native host observation'); + assert.ok(!outputs.get(root).includes('actual executable='), + 'original Go compilation must follow all three native axes'); + if (half) { + assert.equal(outputs.get(root).split(halfPreflight).length - 1, 1, + 'half axis lacks actual CPUID/XCR0 and native EVEX transport preflight'); + } + } + continue; + } + assert.equal(states.get(event.Test), 'running', 'native observation after its terminal'); + if (event.Action === 'output') { + assert.equal(typeof event.Output, 'string', 'native output must be text'); + if (event.Test === root) { + spans.get(root).push({offset: outputs.get(root).length, sequence}); + } + outputs.set(event.Test, outputs.get(event.Test) + event.Output); + continue; + } + states.set(event.Test, 'passed'); + if (event.Test !== root) { + lastChildPass.set(root, sequence); + if (root.includes('SegmentQuery')) { + assert.equal(outputs.get(event.Test).split(segmentEvidence).length - 1, 1, + 'segment query axis lacks its complete opcode flags alias and fault evidence'); + } + if (half) { + assert.equal(outputs.get(event.Test).split(halfEvidence).length - 1, 1, + 'half axis lacks all 512 full64/FLAGS/DF/canary observations'); + } + if (logicalFP) { + assert.equal(outputs.get(event.Test).split(logicalFPCompletion(ia32, false)).length - 1, 1, + 'logical FP axis lacks complete native whole-register values and dead #DE execution'); + } + if (ia32) { + requireIA32Images(outputs.get(event.Test), 2, '[]'); + staticIA32Images += 2; + directIA32Executions++; + } + continue; + } + if (!observed) continue; + + const output = outputs.get(root); + const hosts = [...output.matchAll(nativeHost)]; + const commands = requireSDKCommands(output); + assert.equal(hosts.length, 2, 'integer root must retain pre/post actual native host observations'); + assert.ok(hosts[0].index < commands[0].index && commands[0].index < commands[1].index && + commands[1].index < hosts[1].index, 'integer native/SDK observation order changed'); + if (half) { + assert.equal(output.split(halfPreflight).length - 1, 1, 'duplicate or absent half native CPU preflight'); + assert.equal(output.split(halfOriginal).length - 1, 1, 'missing complete actual original Go DF observations'); + assert.ok(commands[1].index < output.indexOf(halfOriginal) && output.indexOf(halfOriginal) < hosts[1].index, + 'original Go DF terminal escaped its SDK/native host bracket'); + } + if (logicalFP) { + const completion = logicalFPCompletion(ia32, true); + assert.equal(output.split(completion).length - 1, 1, + 'logical FP root lacks complete actual original Go value execution'); + assert.ok(commands[1].index < output.indexOf(completion) && output.indexOf(completion) < hosts[1].index, + 'original Go logical FP completion escaped its SDK/native host bracket'); + } + + // The SDK and final host observations occur after every execution child. + // A printed root PASS before original Go execution cannot certify it. + for (const observation of [commands[0], commands[1], hosts[1]]) { + const preceding = spans.get(root).filter(span => span.offset <= observation.index); + assert.ok(preceding[preceding.length - 1].sequence > lastChildPass.get(root), + 'original Go or final native host observation preceded the complete native axes'); + } + if (ia32) { + const observed = requireIA32Images(output, 1, originalArgs); + assert.ok(commands[1].index < observed.images[0].index && + observed.direct[0].index < hosts[1].index, 'original Go IA32 execution escaped its native/SDK bracket'); + staticIA32Images++; + directIA32Executions++; + } + // /proc/cpuinfo and maps are live observations, not stable identities. + // CPU MHz and mappings may change; require both valid hashes, not equality. + nativeHosts += hosts.length; + sdkBuilds++; + } + return {...result, nativeHosts, sdkBuilds, staticIA32Images, directIA32Executions}; +} + +module.exports = {auditIntegerNativeFamilies, requireIA32Images, requireSDKCommands, roots, children}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditIntegerNativeFamilies(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-legacy-abs-sign-native.cjs b/.github/scripts/audit-legacy-abs-sign-native.cjs new file mode 100644 index 000000000..b38989bff --- /dev/null +++ b/.github/scripts/audit-legacy-abs-sign-native.cjs @@ -0,0 +1,127 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = ['TestCrossLinuxRuntimeMatrixX86LegacyAbsSignOriginalAlignmentFaultsAMD64', + 'TestCrossLinuxRuntimeMatrixX86LegacyAbsSignOriginalAlignmentFaults386']; +const axes = ['backend_O0', 'backend_O2', 'IR_O2_backend_O2']; +const operations = ['PABSB', 'PABSW', 'PABSD', 'PSIGNB', 'PSIGNW', 'PSIGND', + 'VPABSB', 'VPABSW', 'VPABSD', 'VPSIGNB', 'VPSIGNW', 'VPSIGND']; +const roles = ['live', 'dead']; +const parents = roots.flatMap(root => axes.map(axis => root + '/' + axis)); +const leaves = parents.flatMap(parent => operations.flatMap(op => roles.map(role => parent + '/' + op + '/' + role))); +const children = [...parents, ...leaves]; +const marker = 'ABS/sign original Go bytes and LLVM native alignment/access/CF/MXCSR=0x1f80/canary probes 20 failures 0\n'; +const preflight = 'ABS/sign actual native SSSE3=1 AVX=1\n'; +const hostPrefix = 'IA32 direct host:'; +const sdkPrefix = 'actual LEA source/object receipt:'; +const imagePrefix = 'native oracle static IA32 ELF'; +const host = /IA32 direct host: linux\/amd64 kernel=x86_64 CPUsha=[a-f0-9]{64} mapsha=[a-f0-9]{64}\b/g; +const image = /native oracle static IA32 ELF bytes=[1-9][0-9]* sha256=[a-f0-9]{64}\b/g; + +function auditLegacyAbsSignNative(inventory, raw) { + const result = auditNativeOwners(inventory, raw, roots, children); + const outputs = new Map([...roots, ...children].map(name => [name, ''])); + const states = new Map(), passed = new Map(); + const spans = new Map(roots.map(root => [root, []])); + let started = false, complete = false, kernelExecutions = 0; + + function rootObservations(root, final) { + const output = outputs.get(root), arch = root.endsWith('386') ? '386' : 'amd64'; + const captures = [...output.matchAll(new RegExp('actual LEA source/object receipt: SDK=go[1-9][0-9]*\\.[0-9]+(?:\\.[0-9]+)?(?:beta[0-9]+|rc[0-9]+)? target=linux/' + arch + + ' source=[a-f0-9]{64} driver=[a-f0-9]{64} object=[a-f0-9]{64}\\b', 'g'))]; + const hosts = [...output.matchAll(host)]; + assert.equal(hosts.length, final ? 2 : 1, 'missing or duplicated actual native host bracket'); + assert.equal(output.split(hostPrefix).length - 1, hosts.length, 'malformed extra native host'); + assert.equal(captures.length, 1, 'missing unique actual original Go SDK/source/object capture'); + assert.equal(output.split(sdkPrefix).length - 1, 1, 'malformed extra SDK/source/object capture'); + assert.equal(output.split(preflight).length - 1, 1, 'missing or duplicated actual SSSE3/AVX preflight'); + assert.equal(output.split('ABS/sign actual native').length - 1, 1, 'malformed extra CPU preflight'); + assert.ok(hosts[0].index < output.indexOf(preflight) && output.indexOf(preflight) < captures[0].index, + 'CPU or original source capture escaped the actual native host bracket'); + assert.ok(!output.includes(marker) && !output.includes(imagePrefix), 'root summaries cannot replace executed kernels or axis-owned images'); + if (final) { + assert.ok(captures[0].index < hosts[1].index, 'final native host preceded original source capture'); + const span = spans.get(root).filter(value => value.offset <= hosts[1].index).pop(); + for (const axis of axes) { + assert.ok(passed.has(root + '/' + axis), 'root lacks its complete native optimization axis'); + assert.ok(span && span.sequence > passed.get(root + '/' + axis), 'final native host preceded an execution axis'); + } + } + } + + for (const [sequence, line] of raw.split(/\r?\n/).filter(Boolean).entries()) { + const event = JSON.parse(line); + assert.ok(!complete, 'event after complete native ABS/sign package'); + if (event.Action === 'build-output' && event.ImportPath && !event.Test) continue; + if (!event.Test) { + assert.ok(['start', 'output', 'pass'].includes(event.Action), 'unknown ABS/sign package lifecycle'); + if (event.Action === 'start') { + assert.ok(!started, 'duplicate ABS/sign package start'); + started = true; + } else { + assert.ok(started, 'package observation before actual start'); + if (event.Action === 'output') assert.equal(typeof event.Output, 'string', 'nontext package output'); + if (event.Action === 'pass') complete = true; + } + continue; + } + assert.ok(started, 'native ABS/sign test before package start'); + assert.ok(['run', 'output', 'pass'].includes(event.Action), 'unknown ABS/sign test lifecycle'); + const root = event.Test.split('/')[0]; + if (event.Action === 'run') { + assert.ok(!states.has(event.Test), 'native ABS/sign lifecycle restarted'); + states.set(event.Test, 'running'); + if (event.Test !== root) rootObservations(root, false); + if (leaves.includes(event.Test) && root.endsWith('386')) { + const parent = event.Test.split('/').slice(0, 2).join('/'); + assert.equal([...outputs.get(parent).matchAll(image)].length, 1, + 'IA32 kernel started before its complete static image inspection'); + } + continue; + } + assert.equal(states.get(event.Test), 'running', 'ABS/sign observation outside its live owner'); + if (event.Action === 'output') { + assert.equal(typeof event.Output, 'string', 'nontext ABS/sign test output'); + if (event.Test === root) spans.get(root).push({offset: outputs.get(root).length, sequence}); + outputs.set(event.Test, outputs.get(event.Test) + event.Output); + continue; + } + states.set(event.Test, 'passed'); + passed.set(event.Test, sequence); + if (event.Test === root) { + rootObservations(root, true); + continue; + } + const output = outputs.get(event.Test); + for (const foreign of [hostPrefix, sdkPrefix, 'ABS/sign actual native']) { + assert.ok(!output.includes(foreign), 'child cannot replace actual root host/SDK/ISA ownership'); + } + if (parents.includes(event.Test)) { + assert.ok(!output.includes(marker), 'axis summary cannot replace actual kernel execution'); + const count = root.endsWith('386') ? 1 : 0; + assert.equal([...output.matchAll(image)].length, count, 'axis lacks its exact complete native static image'); + assert.equal(output.split(imagePrefix).length - 1, count, 'malformed extra IA32 static image'); + for (const op of operations) for (const role of roles) { + assert.ok(passed.has(event.Test + '/' + op + '/' + role), 'axis ended before its entire live/dead opcode family'); + } + } else { + assert.ok(!output.includes(imagePrefix), 'kernel cannot replace axis-owned image authentication'); + assert.equal(output.split(marker).length - 1, 1, 'kernel lacks exactly one actual complete 20-probe observation'); + kernelExecutions++; + } + } + return {...result, nativeHosts: 4, sdkCaptures: 2, isaPreflights: 2, staticIA32Images: 3, + kernelExecutions, directProbeProcesses: kernelExecutions * 20 * 2}; +} + +module.exports = {auditLegacyAbsSignNative}; +if (require.main === module) { + const directory = process.argv[2]; + process.stdout.write(JSON.stringify(auditLegacyAbsSignNative( + fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8'))) + '\n'); +} diff --git a/.github/scripts/audit-lock-native.cjs b/.github/scripts/audit-lock-native.cjs new file mode 100644 index 000000000..97f9b274b --- /dev/null +++ b/.github/scripts/audit-lock-native.cjs @@ -0,0 +1,20 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = ['TestCrossLinuxRuntimeMatrixX86LockScalarRMWNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86LockScalarRMWNative386']; +const children = roots.flatMap(root => ['-O0', '-O2'].map(level => root + '/' + level)); +function auditNativeLock(inventory, raw) { + return auditNativeOwners(inventory, raw, roots, children); +} +module.exports = {auditNativeLock, roots, children}; + +if (require.main === module) { + assert.equal(process.argv.length, 3, 'one actual native LOCK report directory required'); + const directory = process.argv[2]; + const result = auditNativeLock(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result, null, 2) + '\n'); +} diff --git a/.github/scripts/audit-mmx-streaming.cjs b/.github/scripts/audit-mmx-streaming.cjs new file mode 100644 index 000000000..e1f24a48a --- /dev/null +++ b/.github/scripts/audit-mmx-streaming.cjs @@ -0,0 +1,18 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = ['TestCrossLinuxRuntimeMatrixX86MMXStreamingOriginalAndLLVM']; +const children = ['original-go', 'llvm-default-O0', 'llvm-default-O2', 'llvm-O2-O0', 'llvm-O2-O2'] + .map(name => roots[0] + '/' + name); +const auditMMXStreaming = (inventory, raw) => auditNativeOwners(inventory, raw, roots, children); + +module.exports = {auditMMXStreaming, roots, children}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditMMXStreaming(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-native-leaw.cjs b/.github/scripts/audit-native-leaw.cjs new file mode 100644 index 000000000..c5c054ba7 --- /dev/null +++ b/.github/scripts/audit-native-leaw.cjs @@ -0,0 +1,103 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const root = 'TestCrossLinuxRuntimeMatrixX86LEAWRetainedPointerOriginalBytesAndNativeDF'; +const apis = ['text', 'module', 'Go']; +const pipelines = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2']; +const groups = apis.map(api => root + '/' + api); +const leaves = groups.flatMap(group => pipelines.map(pipeline => group + '/' + pipeline)); +const original = root + '/OriginalGoBytes'; +const children = [...groups, ...leaves, original]; +const llvmMarker = 'native retained LEAW LLVM execution: LEAW_RETAINED_POINTER_300_VALUES_STATUS_CANARIES_PASS\n'; +const originalMarker = 'native retained LEAW actual Go bytes execution: LEAW_RETAINED_POINTER_300_VALUES_STATUS_CANARIES_PASS\n'; +const hostPrefix = 'IA32 direct host:'; +const sdkPrefix = 'actual LEA source/object receipt:'; +const host = /IA32 direct host: linux\/amd64 kernel=x86_64 CPUsha=[a-f0-9]{64} mapsha=[a-f0-9]{64}\b/g; +const sdk = /actual LEA source\/object receipt: SDK=go[1-9][0-9]*\.[0-9]+(?:\.[0-9]+)?(?:beta[0-9]+|rc[0-9]+)? target=linux\/amd64 source=[a-f0-9]{64} driver=[a-f0-9]{64} object=[a-f0-9]{64}\b/g; + +function auditNativeLEAW(inventory, raw) { + const result = auditNativeOwners(inventory, raw, [root], children); + const outputs = new Map([root, ...children].map(name => [name, ''])); + const passed = new Map(), states = new Map(), spans = []; + let started = false, complete = false; + for (const [sequence, line] of raw.split(/\r?\n/).filter(Boolean).entries()) { + const event = JSON.parse(line); + assert.ok(!complete, 'event after the complete LEAW package'); + if (event.Action === 'build-output' && event.ImportPath && !event.Test) continue; + if (!event.Test) { + assert.ok(['start', 'output', 'pass'].includes(event.Action), 'unknown LEAW package lifecycle action'); + if (event.Action === 'start') { + assert.ok(!started, 'duplicate LEAW package start'); + started = true; + } else { + assert.ok(started, 'LEAW package observation before its start'); + if (event.Action === 'output') assert.equal(typeof event.Output, 'string', 'package output must be text'); + if (event.Action === 'pass') complete = true; + } + continue; + } + assert.ok(started, 'LEAW test before its package start'); + assert.ok(['run', 'output', 'pass'].includes(event.Action), 'unknown LEAW test lifecycle action'); + if (event.Action === 'run') { + assert.ok(!states.has(event.Test), 'LEAW test restarted after its original lifecycle'); + states.set(event.Test, 'running'); + if (event.Test !== root) { + const output = outputs.get(root); + const hosts = [...output.matchAll(host)], captures = [...output.matchAll(sdk)]; + assert.equal(hosts.length, 1, 'LEAW child lacks its preceding actual native host'); + assert.equal(captures.length, 1, 'LEAW child lacks its preceding actual SDK/source/object capture'); + assert.ok(hosts[0].index < captures[0].index, 'SDK capture preceded actual native host identity'); + if (event.Test === original) { + for (const group of groups) assert.ok(passed.has(group), 'original bytes preceded a complete API group'); + } + } + continue; + } + assert.equal(states.get(event.Test), 'running', 'LEAW observation outside its live execution owner'); + if (event.Action === 'output') { + assert.equal(typeof event.Output, 'string', 'LEAW output must be text'); + if (event.Test === root) spans.push({offset: outputs.get(root).length, sequence}); + outputs.set(event.Test, outputs.get(event.Test) + event.Output); + continue; + } + states.set(event.Test, 'passed'); + passed.set(event.Test, sequence); + const output = outputs.get(event.Test); + if (event.Test !== root) { + assert.ok(!output.includes(hostPrefix) && !output.includes(sdkPrefix), 'child cannot replace root host/SDK ownership'); + if (groups.includes(event.Test)) { + for (const pipeline of pipelines) assert.ok(passed.has(event.Test + '/' + pipeline), 'API parent preceded an executed LLVM leaf'); + assert.ok(!output.includes(llvmMarker) && !output.includes(originalMarker), 'API parent cannot replace a leaf execution'); + } else { + const marker = event.Test === original ? originalMarker : llvmMarker; + const foreign = event.Test === original ? llvmMarker : originalMarker; + assert.equal(output.split(marker).length - 1, 1, 'missing or duplicated complete actual LEAW execution marker'); + assert.ok(!output.includes(foreign), 'LLVM and actual Go bytes are separate execution obligations'); + } + continue; + } + const hosts = [...output.matchAll(host)], captures = [...output.matchAll(sdk)]; + assert.equal(hosts.length, 2, 'LEAW requires actual native pre/post host observations'); + assert.equal(output.split(hostPrefix).length - 1, 2, 'malformed extra native host receipt'); + assert.equal(captures.length, 1, 'LEAW requires its unique actual SDK/source/object capture'); + assert.equal(output.split(sdkPrefix).length - 1, 1, 'malformed extra SDK capture'); + assert.ok(hosts[0].index < captures[0].index && captures[0].index < hosts[1].index, 'SDK capture escaped native host brackets'); + for (const child of children) assert.ok(passed.has(child), 'root preceded a required execution child'); + const finalSpan = spans.filter(span => span.offset <= hosts[1].index).pop(); + assert.ok(finalSpan && finalSpan.sequence > passed.get(original), 'final host preceded actual original-byte execution'); + assert.ok(!output.includes(llvmMarker) && !output.includes(originalMarker), 'root summary cannot replace leaf execution'); + } + return {...result, nativeHosts: 2, sdkCaptures: 1, llvmExecutions: 9, originalByteExecutions: 1}; +} + +module.exports = {auditNativeLEAW}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditNativeLEAW(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-native-owners.cjs b/.github/scripts/audit-native-owners.cjs new file mode 100644 index 000000000..36d6ceb56 --- /dev/null +++ b/.github/scripts/audit-native-owners.cjs @@ -0,0 +1,52 @@ +const assert = require('node:assert/strict'); + +// The caller supplies a fixed first-party inventory, never report metadata. +// Printed payloads and incomplete, duplicated or skipped test axes are not runs. +function auditNativeOwners(inventory, raw, roots, children) { + assert.ok(roots.length > 0, 'required native owners cannot be empty'); + assert.equal(new Set(roots).size, roots.length, 'duplicate required native owner'); + assert.ok(roots.every(root => /^Test[^/\s]+$/.test(root)), 'invalid required native owner'); + assert.ok(children.every(child => roots.some(root => child.startsWith(root + '/'))), + 'required native child has no owner'); + const discovered = inventory.split(/\r?\n/).filter(line => line.startsWith('Test')); + assert.deepEqual(discovered.slice().sort(), roots.slice().sort(), 'complete fresh native inventory changed'); + const expected = new Set([...roots, ...children]); + assert.equal(expected.size, roots.length + children.length, 'duplicate required native axis'); + const started = new Set(), passed = new Set(); + let complete = false; + for (const line of raw.split(/\r?\n/).filter(Boolean)) { + const event = JSON.parse(line); + if (event.Action === 'build-output' && typeof event.ImportPath === 'string' && event.ImportPath && !event.Test) continue; + assert.equal(event.Package, 'github.com/xgo-dev/plan9asm', 'foreign native package'); + assert.ok(!complete, 'event after native package terminal'); + assert.ok(!['fail', 'skip', 'build-fail'].includes(event.Action), 'required native execution failed or skipped'); + if (event.Test) { + const owner = event.Test.split('/')[0]; + assert.ok(expected.has(event.Test), 'foreign native root or hardware/ABI/backend child'); + if (event.Test !== owner) assert.ok(started.has(owner) && !passed.has(owner), 'child outside its live owner'); + for (const parent of expected) { + if (event.Test.startsWith(parent + '/')) { + assert.ok(started.has(parent) && !passed.has(parent), 'child outside a required live architecture parent'); + } + } + if (event.Action === 'run') { + assert.ok(!started.has(event.Test), 'duplicate native start'); + started.add(event.Test); + } else { + assert.ok(started.has(event.Test), 'event for an unstarted native owner'); + if (event.Action === 'pass') { + assert.ok(!passed.has(event.Test), 'duplicate native terminal'); + passed.add(event.Test); + } + } + } else if (event.Action === 'pass') { + assert.deepEqual([...started].sort(), [...expected].sort(), 'missing native execution axis'); + assert.deepEqual([...passed].sort(), [...expected].sort(), 'unfinished native execution axis'); + complete = true; + } + } + assert.ok(complete, 'native package did not finish successfully'); + return {roots: roots.length, children: children.length, passed: passed.size, skipped: 0, incomplete: 0}; +} + +module.exports = {auditNativeOwners}; diff --git a/.github/scripts/audit-native-priority.cjs b/.github/scripts/audit-native-priority.cjs new file mode 100644 index 000000000..6264b629c --- /dev/null +++ b/.github/scripts/audit-native-priority.cjs @@ -0,0 +1,172 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +function auditNativePriority(directory, io = fs) { + const inventories = { + 'static-read': ['TestX86RawReturnStaticReadNumericOracle'], + alias: [ + 'TestCrossLinuxRuntimeMatrixX86MOVQSourceAliasesOriginalAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86MOVQSourceAliasesCountedProvenance', + ], + bool: [ + 'TestCrossLinuxRuntimeMatrixAMD64RegisterCanonicalBoolInstructionBytes', + 'TestCrossLinuxRuntimeMatrixGoAMD64CanonicalBoolByteRepresentation', + ], + caller: [ + 'TestCrossLinuxRuntimeMatrixX86CallerAddressInstructionBytesAndLLVM', + 'TestCrossLinuxRuntimeMatrixGoX86CallerAddressABI0Representation', + ], + 'caller-cmp': [ + 'TestCrossLinuxRuntimeMatrixX86CallerAddressCMPInstructionBytesAndLLVM', + 'TestCrossLinuxRuntimeMatrixGoX86CallerAddressCMPABI0Representation', + ], + numerical: [ + 'TestCrossLinuxRuntimeMatrixX86NumericalSBReadMutableAMD64', + 'TestCrossLinuxRuntimeMatrixX86NumericalSBReadMutableLocalAMD64', + ], + 'numerical-lea': ['TestCrossLinuxRuntimeMatrixX86NumericalLEAOriginalBytesAndLLVM'], + 'direct-tail': [ + 'TestCrossLinuxRuntimeMatrixAMD64DirectTailWholeSourceOriginalGoAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86DirectTail386OriginalGoAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86DirectTailPCResultAliasesAndExternal', + ], + 'gp-simd-sparse': [ + 'TestCrossLinuxRuntimeMatrixX86GPSIMDSparseNativeAMD64OriginalGoAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86GPSIMDSparseNative386OriginalGoAndLLVM', + ], + 'bit-scan': [ + 'TestCrossLinuxRuntimeMatrixX86BitScanNativeAMD64OriginalGoAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86BitScanNative386OriginalGoAndLLVM', + ], + 'bool-386': ['TestCrossLinuxRuntimeMatrixX86ABI0Mode32CanonicalBoolOriginalGoAndLLVM'], + 'packed-string': ['TestCrossLinuxRuntimeMatrixX86PackedStringImplicitVectorNativeAVX'], + 'independent-fp': [ + 'TestCrossLinuxRuntimeMatrixX86InsertExtractIndependentFPFields', + 'TestCrossLinuxRuntimeMatrixX86LegacyPSIndependentFPFields', + ], + 'source-stack': ['TestCrossLinuxRuntimeMatrixX86RawSourceStackNumerical'], + continuation: ['TestCrossLinuxRuntimeMatrixX86ContinuationWholeCFGOriginalGoAndLLVM'], + 'packed-string-fault': ['TestCrossLinuxRuntimeMatrixX86PackedStringOriginalNativeMemoryFaults'], + 'byte-swap': ['TestCrossLinuxRuntimeMatrixX86ByteSwapOriginalGoAndLLVM'], + crc32: ['TestAMD64CRC32RuntimeSemanticsAndFlags'], + 'bit-test': ['TestCrossLinuxRuntimeMatrixX86BitTestNumericalAndAliases'], + }; + const actualFiles = io.readdirSync(directory).filter(name => name.endsWith('-inventory.log')).sort(); + assert.deepEqual(actualFiles, Object.keys(inventories).map(name => name + '-inventory.log').sort(), + 'missing or foreign independent native-axis inventories'); + const mandatory = new Set(); + for (const [inventory, expected] of Object.entries(inventories)) { + const names = io.readFileSync(path.join(directory, inventory + '-inventory.log'), 'utf8') + .split(/\r?\n/).filter(line => /^Test\S+$/.test(line)); + assert.deepEqual(names.slice().sort(), expected.slice().sort(), 'changed mandatory inventory: ' + inventory); + for (const name of names) { + assert.ok(!mandatory.has(name), 'duplicate mandatory owner: ' + name); + mandatory.add(name); + } + } + assert.equal(mandatory.size, 29); + const discovered = io.readFileSync(path.join(directory, 'all-roots.stdout'), 'utf8') + .split(/\r?\n/).filter(line => /^Test\S+$/.test(line)).sort(); + assert.equal(discovered.length, 66, 'fresh discovery denominator changed'); + assert.equal(new Set(discovered).size, 66, 'duplicate fresh discovery'); + const roots = new Set(), rootTerminals = new Set(); + const children = new Set(), childTerminals = new Set(); + let packageTerminal; + io.readFileSync(path.join(directory, 'native.stderr'), 'utf8'); + const raw = io.readFileSync(path.join(directory, 'native.jsonl'), 'utf8'); + for (const line of raw.split(/\r?\n/)) { + if (!line) continue; + const event = JSON.parse(line); + if (event.Package) assert.equal(event.Package, 'github.com/xgo-dev/plan9asm', 'foreign package event'); + if (!event.Test) { + assert.notEqual(event.Action, 'build-fail', 'native package build failed'); + if (['pass', 'fail', 'skip'].includes(event.Action)) { + assert.equal(event.Package, 'github.com/xgo-dev/plan9asm', 'package terminal lacks exact package identity'); + assert.equal(packageTerminal, undefined, 'duplicate package terminal'); + assert.equal(event.Action, 'pass', 'native package did not execute successfully'); + packageTerminal = { package: event.Package, elapsed: event.Elapsed }; + } + continue; + } + assert.equal(packageTerminal, undefined, 'test event after package terminal'); + assert.equal(event.Package, 'github.com/xgo-dev/plan9asm', 'test lacks exact package identity'); + const child = event.Test.includes('/'); + const root = event.Test.split('/')[0]; + const starts = child ? children : roots; + const terminals = child ? childTerminals : rootTerminals; + if (child) assert.ok(roots.has(root), 'foreign/unstarted child root: ' + event.Test); + if (event.Action === 'run') { + assert.ok(!starts.has(event.Test), 'duplicate test start: ' + event.Test); + if (child) { + // t.Run may put slashes in one literal name without intermediate + // parent events. The actual root must still have started. + assert.ok(!rootTerminals.has(root), 'child starts after root terminal'); + } + starts.add(event.Test); + } + if (['pass', 'fail', 'skip'].includes(event.Action)) { + assert.ok(starts.has(event.Test), 'test terminal lacks start: ' + event.Test); + assert.ok(!terminals.has(event.Test), 'duplicate test terminal: ' + event.Test); + if (child) assert.ok(!rootTerminals.has(root), 'child finishes after root terminal: ' + event.Test); + assert.equal(event.Action, 'pass', event.Test + ' did not execute successfully'); + terminals.add(event.Test); + } + } + assert.equal(roots.size, 66, 'exact native root denominator changed'); + assert.deepEqual([...roots].sort(), discovered, 'execution roots differ from fresh discovery'); + assert.deepEqual([...roots].sort(), [...rootTerminals].sort(), 'incomplete root terminals'); + assert.deepEqual([...children].sort(), [...childTerminals].sort(), 'incomplete child terminals'); + assert.ok(packageTerminal, 'missing native package terminal'); + for (const name of mandatory) { + assert.ok(roots.has(name) && rootTerminals.has(name), 'missing mandatory root: ' + name); + } + const sourceStack = 'TestCrossLinuxRuntimeMatrixX86RawSourceStackNumerical'; + const continuation = 'TestCrossLinuxRuntimeMatrixX86ContinuationWholeCFGOriginalGoAndLLVM'; + const requiredChildren = [sourceStack + '/amd64', sourceStack + '/386']; + for (const arch of ['amd64', '386']) { + for (const dispatch of ['memory', 'register']) { + requiredChildren.push(continuation + '/' + arch + '/' + dispatch); + } + } + const packedStringFault = 'TestCrossLinuxRuntimeMatrixX86PackedStringOriginalNativeMemoryFaults'; + for (const optimization of ['-O0', '-O2']) { + requiredChildren.push(packedStringFault + '/' + optimization); + } + const byteSwap = 'TestCrossLinuxRuntimeMatrixX86ByteSwapOriginalGoAndLLVM'; + for (const arch of ['amd64', '386']) { + requiredChildren.push(byteSwap + '/' + arch); + } + const directTail386 = 'TestCrossLinuxRuntimeMatrixX86DirectTail386OriginalGoAndLLVM'; + for (const carrier of ['SI', 'BP', 'SI-stack', 'BP-stack']) { + requiredChildren.push(directTail386 + '/' + carrier); + } + for (const dispatch of ['BP-memory', 'BP-register', 'BP-target']) { + requiredChildren.push(continuation + '/386/' + dispatch); + } + const bitTest = 'TestCrossLinuxRuntimeMatrixX86BitTestNumericalAndAliases'; + for (const api of ['text', 'module', 'Go']) { + for (const axis of ['backend_O0', 'IR_O2/backend_O2']) { + requiredChildren.push(`${bitTest}/${api}/${axis}`); + } + } + for (const child of requiredChildren) { + assert.ok(children.has(child) && childTerminals.has(child), 'missing required child: ' + child); + } + return { + protocol: 'go_test_json_native66', + roots: roots.size, passed: rootTerminals.size, + children: children.size, childPassed: childTerminals.size, + failed: 0, skipped: 0, incomplete: 0, duplicate: 0, + inventories: actualFiles.length, mandatory: [...mandatory].sort(), requiredChildren, + packageResult: packageTerminal, + }; +} + + +module.exports = { auditNativePriority }; + +if (require.main === module) { + assert.equal(process.argv.length, 3, 'one native artifact directory required'); + process.stdout.write(JSON.stringify(auditNativePriority(process.argv[2]), null, 2) + '\n'); +} diff --git a/.github/scripts/audit-packed-conversion-native.cjs b/.github/scripts/audit-packed-conversion-native.cjs new file mode 100644 index 000000000..4a2dbc5ef --- /dev/null +++ b/.github/scripts/audit-packed-conversion-native.cjs @@ -0,0 +1,146 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {execFileSync} = require('node:child_process'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const owners = 8; +const pipelines = ['backend_O0', 'backend_O2', 'IR_O2_backend_O2']; +const statusRoot = 'TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512Status'; +const faultRoot = 'TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaults'; +const independentRoot = 'TestCrossLinuxRuntimeMatrixX86PackedConversionNativeIndependentC'; + +function partition(root, owner) { + assert.ok(Number.isInteger(owner) && owner >= 0 && owner < owners, 'invalid native conversion owner'); + return root + (owner ? 'Part' + owner : ''); +} + +function batches(count, limit, owner) { + const selected = []; + for (let start = 0, batch = 0; start < count; start += limit, batch++) { + if (batch % owners === owner) selected.push([start, Math.min(start + limit, count)]); + } + assert.ok(selected.length > 0, 'empty native conversion partition'); + return selected; +} + +function required(owner) { + const roots = [partition(statusRoot, owner), partition(faultRoot, owner)]; + if (!owner) roots.push(independentRoot); + const children = []; + const values = batches(692, 64, owner); + const faults = batches(222, 16, owner); + for (const [root, selected, originals] of [ + [roots[0], values, ['original_go_named', 'original_go_raw']], + [roots[1], faults, ['original_go_opcode_reference']], + ]) { + for (const [start] of selected) { + const batch = root + '/batch-' + String(start).padStart(4, '0'); + children.push(batch, ...originals.map(original => batch + '/' + original)); + for (const spelling of ['named', 'raw']) { + const source = batch + '/' + spelling; + children.push(source); + for (const api of ['text', 'module', 'Go']) { + const publicPath = source + '/' + api; + children.push(publicPath, ...pipelines.map(pipeline => publicPath + '/' + pipeline)); + } + } + } + } + if (!owner) children.push(...pipelines.map(pipeline => independentRoot + '/' + pipeline)); + return {roots, children, values, faults}; +} + +function auditOwner(inventory, raw, owner) { + const expected = required(owner); + const result = auditNativeOwners(inventory, raw, expected.roots, expected.children); + const output = new Map(expected.roots.map(root => [root, ''])); + const preflights = new Set(); + for (const line of raw.split(/\r?\n/).filter(Boolean)) { + const event = JSON.parse(line); + if (event.Action === 'output' && output.has(event.Test)) { + assert.equal(typeof event.Output, 'string', 'invalid native CPU observation'); + output.set(event.Test, output.get(event.Test) + event.Output); + } + if (event.Action !== 'run' || !expected.children.includes(event.Test)) continue; + const root = event.Test.split('/')[0]; + const observation = output.get(root); + const flags = observation.match(/AVX512F=1 AVX512DQ=1 AVX512BW=1 AVX512VL=1 OSXSAVE=1 XCR0=([0-9a-f]+)\b/); + assert.ok(flags, 'missing current native F/DQ/BW/VL or OS state: ' + root); + assert.equal(BigInt('0x' + flags[1]) & 0xe6n, 0xe6n, 'incomplete native XCR0: ' + root); + assert.ok(observation.includes('PACKED_CONVERSION_NATIVE_F_DQ_BW_VL_OK'), + 'original native conversion opcodes did not complete before runtime: ' + root); + preflights.add(root); + } + assert.equal(preflights.size, expected.roots.length, 'missing actual native conversion preflight'); + return {...result, owner, values: expected.values, faults: expected.faults, preflights: preflights.size}; +} + +function readOwner(directory, owner, head) { + const context = JSON.parse(fs.readFileSync(path.join(directory, 'context.json'), 'utf8')); + assert.deepEqual(Object.keys(context).sort(), ['head', 'owner', 'schema'], 'unknown native owner context fields'); + assert.equal(context.schema, 1, 'invalid native owner context schema'); + assert.equal(context.owner, owner, 'native receipt borrowed another partition'); + assert.match(head, /^[0-9a-f]{40}$/, 'invalid current source identity'); + assert.equal(context.head, head, 'native receipt is not the current checked-out source'); + return {...auditOwner(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8'), owner), head}; +} + +function auditComplete(directories, head) { + assert.equal(directories.length, owners, 'missing complete native owner artifacts'); + const seen = new Set(), receipts = []; + for (const directory of directories) { + const context = JSON.parse(fs.readFileSync(path.join(directory, 'context.json'), 'utf8')); + assert.ok(Number.isInteger(context.owner) && context.owner >= 0 && context.owner < owners, + 'invalid complete native partition'); + assert.ok(!seen.has(context.owner), 'duplicated native partition artifact'); + seen.add(context.owner); + // Reaudit the complete raw stream and fresh inventory, not a printed + // success string, cached aggregate or self-declared receipt counts. + receipts.push(readOwner(directory, context.owner, head)); + } + assert.deepEqual([...seen].sort(), [0, 1, 2, 3, 4, 5, 6, 7], 'incomplete native owner union'); + for (const [field, count] of [['values', 692], ['faults', 222]]) { + const consumed = new Array(count).fill(0); + for (const receipt of receipts) { + for (const [start, end] of receipt[field]) { + for (let index = start; index < end; index++) consumed[index]++; + } + } + assert.ok(consumed.every(count => count === 1), 'native partitions overlap or omit ' + field); + } + return {head, owners, roots: receipts.reduce((sum, receipt) => sum + receipt.roots, 0), + children: receipts.reduce((sum, receipt) => sum + receipt.children, 0), + passed: receipts.reduce((sum, receipt) => sum + receipt.passed, 0), + valueKernels: 692, faultKernels: 222, skipped: 0, incomplete: 0}; +} + +function artifactDirectories(root) { + const found = []; + for (const entry of fs.readdirSync(root, {withFileTypes: true})) { + assert.ok(!entry.isSymbolicLink(), 'native artifacts cannot redirect source receipts'); + if (!entry.isDirectory()) continue; + const directory = path.join(root, entry.name); + assert.ok(fs.existsSync(path.join(directory, 'context.json')), 'unrecognized native owner artifact'); + found.push(directory); + } + return found; +} + +module.exports = {required, auditOwner, readOwner, auditComplete}; + +if (require.main === module) { + const [command, argument, directory] = process.argv.slice(2); + if (command === 'roots') { + process.stdout.write(required(Number(argument)).roots.join('|') + '\n'); + } else { + const head = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8'}).trim(); + const result = command === 'owner' ? readOwner(directory, Number(argument), head) : + command === 'complete' ? auditComplete(artifactDirectories(argument), head) : null; + assert.ok(result, 'unknown native conversion audit command'); + process.stdout.write(JSON.stringify(result) + '\n'); + } +} diff --git a/.github/scripts/audit-packed-reduction-native.cjs b/.github/scripts/audit-packed-reduction-native.cjs new file mode 100644 index 000000000..9ee127520 --- /dev/null +++ b/.github/scripts/audit-packed-reduction-native.cjs @@ -0,0 +1,134 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {execFileSync} = require('node:child_process'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const owners = 4; +const pipelines = ['backend_O0', 'backend_O2', 'IR_O2_backend_O2']; + +function batches(count, limit, owner) { + assert.ok(Number.isInteger(owner) && owner >= 0 && owner < owners, 'invalid native reduction owner'); + const selected = []; + for (let start = 0, batch = 0; start < count; start += limit, batch++) { + if (batch % owners === owner) selected.push([start, Math.min(start + limit, count)]); + } + assert.ok(selected.length > 0, 'empty native reduction partition'); + return selected; +} + +function required(owner) { + const values = batches(258, 64, owner), faults = batches(37, 8, owner); + const roots = ['Values', 'DeadFaults'].map(family => + 'TestCrossLinuxRuntimeMatrixX86PackedReductionNative' + family + 'Part' + owner); + const children = []; + for (const [root, selected, originals] of [ + [roots[0], values, ['original_go_named', 'original_go_raw']], + [roots[1], faults, ['original_go_opcode_reference']], + ]) { + for (const [start] of selected) { + const batch = root + '/batch-' + String(start).padStart(4, '0'); + children.push(batch, ...originals.map(original => batch + '/' + original)); + for (const spelling of ['named', 'raw']) { + const source = batch + '/' + spelling; + children.push(source); + for (const api of ['text', 'module', 'Go']) { + const publicPath = source + '/' + api; + children.push(publicPath, ...pipelines.map(pipeline => publicPath + '/' + pipeline)); + } + } + } + } + return {roots, children, values, faults}; +} + +function auditOwner(inventory, raw, owner) { + const expected = required(owner); + const result = auditNativeOwners(inventory, raw, expected.roots, expected.children); + const output = new Map(expected.roots.map(root => [root, ''])); + const preflights = new Set(); + for (const line of raw.split(/\r?\n/).filter(Boolean)) { + const event = JSON.parse(line); + if (event.Action === 'output' && output.has(event.Test)) { + assert.equal(typeof event.Output, 'string', 'invalid native reduction CPU observation'); + output.set(event.Test, output.get(event.Test) + event.Output); + } + if (event.Action !== 'run' || !expected.children.includes(event.Test)) continue; + const root = event.Test.split('/')[0], observation = output.get(root); + const flags = observation.match(/SSE3=1 SSE41=1 AVX=1 OSXSAVE=1 XCR0=([0-9a-f]+)\b/); + assert.ok(flags, 'missing current native SSE3/SSE4.1/AVX or OS state: ' + root); + assert.equal(BigInt('0x' + flags[1]) & 6n, 6n, 'incomplete native XCR0: ' + root); + assert.ok(observation.includes('PACKED_REDUCTION_NATIVE_SSE3_SSE41_AVX_OK members=fff'), + 'all12 original native reduction opcodes did not complete before runtime: ' + root); + preflights.add(root); + } + assert.equal(preflights.size, expected.roots.length, 'missing actual native reduction preflight'); + return {...result, owner, values: expected.values, faults: expected.faults, preflights: preflights.size}; +} + +function readOwner(directory, owner, head) { + const context = JSON.parse(fs.readFileSync(path.join(directory, 'context.json'), 'utf8')); + assert.deepEqual(Object.keys(context).sort(), ['head', 'owner', 'schema'], 'unknown native context fields'); + assert.equal(context.schema, 1, 'invalid native context schema'); + assert.equal(context.owner, owner, 'native reduction receipt borrowed another partition'); + assert.match(head, /^[0-9a-f]{40}$/, 'invalid current source identity'); + assert.equal(context.head, head, 'native reduction receipt is not the current checked-out source'); + return {...auditOwner(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8'), owner), head}; +} + +function auditComplete(directories, head) { + assert.equal(directories.length, owners, 'missing complete native reduction artifacts'); + const seen = new Set(), receipts = []; + for (const directory of directories) { + const context = JSON.parse(fs.readFileSync(path.join(directory, 'context.json'), 'utf8')); + assert.ok(Number.isInteger(context.owner) && context.owner >= 0 && context.owner < owners, + 'invalid complete native reduction partition'); + assert.ok(!seen.has(context.owner), 'duplicate native reduction artifact'); + seen.add(context.owner); + receipts.push(readOwner(directory, context.owner, head)); + } + assert.deepEqual([...seen].sort(), [0, 1, 2, 3], 'incomplete native reduction union'); + for (const [field, count] of [['values', 258], ['faults', 37]]) { + const consumed = new Array(count).fill(0); + for (const receipt of receipts) { + for (const [start, end] of receipt[field]) { + for (let index = start; index < end; index++) consumed[index]++; + } + } + assert.ok(consumed.every(count => count === 1), 'native reduction partitions overlap or omit ' + field); + } + return {head, owners, roots: receipts.reduce((sum, receipt) => sum + receipt.roots, 0), + children: receipts.reduce((sum, receipt) => sum + receipt.children, 0), + passed: receipts.reduce((sum, receipt) => sum + receipt.passed, 0), + valueKernels: 258, faultKernels: 37, skipped: 0, incomplete: 0}; +} + +function artifactDirectories(root) { + const found = []; + for (const entry of fs.readdirSync(root, {withFileTypes: true})) { + assert.ok(!entry.isSymbolicLink(), 'native artifacts cannot redirect source receipts'); + if (!entry.isDirectory()) continue; + const directory = path.join(root, entry.name); + assert.ok(fs.existsSync(path.join(directory, 'context.json')), 'unrecognized native owner artifact'); + found.push(directory); + } + return found; +} + +module.exports = {required, auditOwner, readOwner, auditComplete}; + +if (require.main === module) { + const [command, argument, directory] = process.argv.slice(2); + if (command === 'roots') { + process.stdout.write(required(Number(argument)).roots.join('|') + '\n'); + } else { + const head = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8'}).trim(); + const result = command === 'owner' ? readOwner(directory, Number(argument), head) : + command === 'complete' ? auditComplete(artifactDirectories(argument), head) : null; + assert.ok(result, 'unknown native reduction audit command'); + process.stdout.write(JSON.stringify(result) + '\n'); + } +} diff --git a/.github/scripts/audit-private-leave-native.cjs b/.github/scripts/audit-private-leave-native.cjs new file mode 100644 index 000000000..7586d84db --- /dev/null +++ b/.github/scripts/audit-private-leave-native.cjs @@ -0,0 +1,112 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); +const {requireIA32Images} = require('./audit-integer-native-families.cjs'); + +const root = 'TestCrossLinuxRuntimeMatrixX86LeavePrivateEmitterWidthsAndFlags'; +const architectures = ['amd64', '386']; +const pipelines = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2']; +const groups = architectures.map(arch => root + '/' + arch); +const leaves = groups.flatMap(group => pipelines.map(pipeline => group + '/' + pipeline)); +const children = [...groups, ...leaves]; +const hostPrefix = 'IA32 direct host:'; +const sdkPrefix = 'actual LEA source/object receipt:'; +const markerPrefix = 'private LEAVE numerical completion arch='; +const host = /IA32 direct host: linux\/amd64 kernel=x86_64 CPUsha=[a-f0-9]{64} mapsha=[a-f0-9]{64}\b/g; + +function requireSDK(output, arch) { + const sdk = new RegExp('actual LEA source/object receipt: SDK=go[1-9][0-9]*\\.[0-9]+(?:\\.[0-9]+)?' + + '(?:beta[0-9]+|rc[0-9]+)? target=linux/' + arch + + ' source=[a-f0-9]{64} driver=[a-f0-9]{64} object=[a-f0-9]{64}\\b', 'g'); + assert.equal([...output.matchAll(sdk)].length, 1, 'private width lacks its actual SDK/source/object receipt'); + assert.equal(output.split(sdkPrefix).length - 1, 1, 'duplicate or malformed SDK capture'); +} + +// The fixed numerical contract is deliberately not an original Go/native +// stack ABI. Reuse the shared exact inventory, then bind every live owner to +// its real host, selected SDK, IA32 image and complete numerical execution. +function auditPrivateLEAVE(inventory, raw) { + const result = auditNativeOwners(inventory, raw, [root], children); + const outputs = new Map([root, ...children].map(name => [name, ''])); + const states = new Map(); + let started = false, complete = false; + for (const line of raw.split(/\r?\n/).filter(Boolean)) { + const event = JSON.parse(line); + assert.ok(!complete, 'private LEAVE event after complete package'); + if (event.Action === 'build-output' && event.ImportPath && !event.Test) continue; + if (!event.Test) { + assert.ok(['start', 'output', 'pass'].includes(event.Action), 'unknown private-width package action'); + if (event.Action === 'start') { + assert.ok(!started, 'duplicate private-width package start'); + started = true; + } else { + assert.ok(started, 'private-width output/terminal before package start'); + if (event.Action === 'output') assert.equal(typeof event.Output, 'string'); + if (event.Action === 'pass') complete = true; + } + continue; + } + assert.ok(started, 'private-width test before actual package start'); + assert.ok(['run', 'output', 'pass'].includes(event.Action), 'unknown private-width test action'); + const arch = event.Test.split('/')[1], group = root + '/' + arch; + if (event.Action === 'run') { + assert.ok(!states.has(event.Test), 'private-width test restarted after its first lifecycle'); + states.set(event.Test, 'running'); + if (event.Test !== root) { + assert.equal([...outputs.get(root).matchAll(host)].length, 1, 'actual native host must precede every width'); + if (leaves.includes(event.Test)) requireSDK(outputs.get(group), arch); + } + continue; + } + assert.equal(states.get(event.Test), 'running', 'private-width observation outside live owner'); + if (event.Action === 'output') { + assert.equal(typeof event.Output, 'string'); + outputs.set(event.Test, outputs.get(event.Test) + event.Output); + if (event.Test === root && [...outputs.get(root).matchAll(host)].length > 1) { + assert.ok(groups.every(group => states.get(group) === 'passed'), 'final host preceded a complete actual width'); + } + continue; + } + states.set(event.Test, 'passed'); + const output = outputs.get(event.Test); + if (event.Test === root) { + assert.equal([...output.matchAll(host)].length, 2, 'private LEAVE requires actual native host brackets'); + assert.equal(output.split(hostPrefix).length - 1, 2, 'duplicate or malformed actual host'); + assert.ok(children.every(child => states.get(child) === 'passed'), 'root preceded a required numerical child'); + assert.ok(!output.includes(sdkPrefix) && !output.includes(markerPrefix), 'root summary cannot replace SDK/leaf observations'); + } else if (groups.includes(event.Test)) { + requireSDK(output, arch); + assert.ok(pipelines.every(pipeline => states.get(group + '/' + pipeline) === 'passed'), 'width parent preceded a real LLVM execution'); + assert.ok(!output.includes(hostPrefix) && !output.includes(markerPrefix) && !output.includes('IA32 ELF'), + 'width parent cannot replace actual host or leaf execution'); + } else { + const marker = markerPrefix + arch + ': PRIVATE_LEAVE_128_WIDTH_VALUES_FLAGS_CANARIES_PASS\n'; + assert.equal(output.split(marker).length - 1, 1, 'missing or duplicate complete private-width execution'); + assert.equal(output.split(markerPrefix).length - 1, 1, 'foreign width or replacement marker'); + assert.ok(!output.includes(hostPrefix) && !output.includes(sdkPrefix), 'leaf cannot replace host or SDK owner'); + if (arch === '386') { + const {direct} = requireIA32Images(output, 1, '[]'); + const commands = [...output.matchAll(/actual executable=([^\r\n]+) sha256=([a-f0-9]{64}) argv=\[\]/g)]; + assert.equal(commands.length, 1, 'missing actual static IA32 executable observation'); + assert.equal(commands[0][2], direct[0][2], 'actual IA32 executable hash changed'); + assert.ok(commands[0].index < direct[0].index && direct[0].index < output.indexOf(marker), + 'private IA32 completion preceded actual image execution'); + } else { + assert.ok(!output.includes('IA32 ELF'), 'AMD64 execution cannot borrow an IA32 image'); + } + } + } + return {...result, nativeHosts: 2, sdkCaptures: 2, llvmExecutions: 6, + staticIA32Executions: 3, privateNumericalOnly: true}; +} + +module.exports = {auditPrivateLEAVE}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditPrivateLEAVE(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-product-cmov.cjs b/.github/scripts/audit-product-cmov.cjs new file mode 100644 index 000000000..44bce672f --- /dev/null +++ b/.github/scripts/audit-product-cmov.cjs @@ -0,0 +1,36 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +// Product and wide AES execution are portable LLVM, not native AVX512. +// CMOV additionally executes original Go opcode bytes. Keep all old programs. +// Vector tests retain their original independent C values and named/raw source. +// IFMA executes portable integer LLVM against its original independent C oracle, +// not the original AVX512 instructions. +const roots = [ + 'TestCrossLinuxRuntimeMatrixX86DwordProductIndependentWholeBytes', + 'TestCrossLinuxRuntimeMatrixX86CMOVIndependentWholeBytes', + 'TestCrossLinuxRuntimeMatrixX86AESFamilyWholeByteAliases', + 'TestAMD64VectorTestRuntimeSemantics', + 'TestAMD64IFMARuntimeSemantics', + 'TestAMD64QwordBitLookupRuntimeSemantics', + 'TestCrossLinuxRuntimeMatrixX86CMOVFPPartialCrossSlotWholeBytes', +]; +const children = ['-O0', '-O2', 'IR_O2/backend_O2'].map(axis => roots[0] + '/' + axis) + .concat(['backend_O0', 'backend_O2', 'IR_O2/backend_O2'].map(axis => roots[1] + '/' + axis)) + .concat(['-O0', '-O2', 'IR_O2/backend_O2'].map(axis => roots[2] + '/' + axis)) + .concat(['named', 'raw'].flatMap(encoding => + ['backend_O0', 'backend_O2', 'IR_O2/backend_O2'].map(axis => roots[3] + '/' + encoding + '/' + axis))) + .concat([4, 5, 6].flatMap(index => + ['backend_O0', 'backend_O2', 'IR_O2/backend_O2'].map(axis => roots[index] + '/' + axis))); +const auditProductCMOV = (inventory, raw) => auditNativeOwners(inventory, raw, roots, children); + +module.exports = {auditProductCMOV, roots, children}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditProductCMOV(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-scalar-bmi.cjs b/.github/scripts/audit-scalar-bmi.cjs new file mode 100644 index 000000000..5c357e9ae --- /dev/null +++ b/.github/scripts/audit-scalar-bmi.cjs @@ -0,0 +1,20 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = [ + 'TestAMD64BLSRuntimeSemantics', + 'TestAMD64BEXTRRuntimeSemanticsAndFlags', + 'TestAMD64BZHIRuntimeSemanticsAndFlags', +]; +const auditScalarBMI = (inventory, raw) => auditNativeOwners(inventory, raw, roots, []); + +module.exports = {auditScalarBMI, roots}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditScalarBMI(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-scalar-reciprocal-native.cjs b/.github/scripts/audit-scalar-reciprocal-native.cjs new file mode 100644 index 000000000..41cacaa8c --- /dev/null +++ b/.github/scripts/audit-scalar-reciprocal-native.cjs @@ -0,0 +1,27 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const {auditSSEAVXNativeFamilies} = require('./audit-duplicate-native-families.cjs'); + +const roots = [ + 'TestCrossLinuxRuntimeMatrixX86ScalarReciprocalOriginalGoAndNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86ScalarReciprocalOriginalGoAndNative386', +]; +const originalEvidence = 'actual original Go scalar estimates: 20 aliases × 24 bit patterns × 64 MXCSR profiles = 30720 whole32/FLAGS/DF/MXCSR/canary outcomes; independent scalar references, no AVX512/Z execution claim\n'; + +function pipelineEvidence(arch) { + return `scalar estimates native arch=${arch} live=20 dead=4 whole32-outcomes=30720 fault-processes=40; SSE/AVX actual scalar opcodes, no AVX512/Z transport\n`; +} + +function auditScalarReciprocalNative(inventory, raw) { + return auditSSEAVXNativeFamilies(inventory, raw, roots, pipelineEvidence, originalEvidence); +} + +module.exports = {auditScalarReciprocalNative}; +if (require.main === module) { + const directory = process.argv[2]; + const result = auditScalarReciprocalNative(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/audit-unlocked-native.cjs b/.github/scripts/audit-unlocked-native.cjs new file mode 100644 index 000000000..a647114d9 --- /dev/null +++ b/.github/scripts/audit-unlocked-native.cjs @@ -0,0 +1,21 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = ['TestCrossLinuxRuntimeMatrixX86UnlockedScalarRMWNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86UnlockedScalarRMWNative386']; +const children = roots.flatMap(root => ['text', 'module', 'Go'].map(api => root + '/' + api)); + +function auditNativeUnlocked(inventory, raw) { + return auditNativeOwners(inventory, raw, roots, children); +} +module.exports = {auditNativeUnlocked, roots, children}; + +if (require.main === module) { + assert.equal(process.argv.length, 3, 'one actual unlocked native report directory required'); + const directory = process.argv[2]; + const result = auditNativeUnlocked(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result, null, 2) + '\n'); +} diff --git a/.github/scripts/audit-value-fault-priority.cjs b/.github/scripts/audit-value-fault-priority.cjs new file mode 100644 index 000000000..08c6acc16 --- /dev/null +++ b/.github/scripts/audit-value-fault-priority.cjs @@ -0,0 +1,27 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +// These execute portable LLVM memory semantics on a real Linux host. They do +// not claim original-Go or physical AVX512 instruction execution. +const roots = ['TestX86VectorValueNativePortableFaults', 'TestGoX86NumericalSBReadRuntimeFaultCorePolicy']; +const children = ['text', 'module', 'Go'].map(api => roots[0]+'/'+api) + .concat(['386/mutable', '386/width', 'amd64/mutable', 'amd64/width', 'amd64/local'] + .map(role => roots[1]+'/'+role)); + +function auditValueFaultPriority(inventory, raw) { + return auditNativeOwners(inventory, raw, roots, children); +} + +module.exports = {auditValueFaultPriority, roots, children}; + +if (require.main === module) { + assert.equal(process.argv.length, 3, 'one actual portable fault report directory required'); + const directory = process.argv[2]; + const result = auditValueFaultPriority(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result, null, 2)+'\n'); +} diff --git a/.github/scripts/audit-x87-native.cjs b/.github/scripts/audit-x87-native.cjs new file mode 100644 index 000000000..47934e347 --- /dev/null +++ b/.github/scripts/audit-x87-native.cjs @@ -0,0 +1,43 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {auditNativeOwners} = require('./audit-native-owners.cjs'); + +const roots = [ + 'TestCrossLinuxRuntimeMatrixX86X87SignedI64IdentityOriginalGoAndLLVM', + 'TestAMD64X87ConditionalMoveRuntimeConditions', + 'TestX86RawX87ClearExceptionRuntimeSemantics', + 'TestX86RawX87SoftwareClearPreservesAllStatusFields', + 'TestCrossLinuxRuntimeMatrixX86X87ScopedRoundingOriginalGoAndLLVM', +]; +const children = []; +for (const arch of ['amd64', '386']) { + const parent = `${roots[0]}/${arch}`; + children.push(parent); + for (const api of ['text', 'module', 'Go']) { + for (const optimized of [false, true]) { + for (const backend of [0, 2]) children.push(`${parent}/${api}/IR_O2=${optimized}/backend-O${backend}`); + } + } +} +for (const api of ['text', 'module', 'Go']) { + const parent = `${roots[4]}/${api}`; + children.push(parent); + for (const optimized of [false, true]) { + for (const backend of [0, 2]) children.push(`${parent}/IR_O2=${optimized}/backend-O${backend}`); + } +} + +function auditNativeX87(inventory, raw) { + return auditNativeOwners(inventory, raw, roots, children); +} + +module.exports = {auditNativeX87, roots, children}; + +if (require.main === module) { + assert.equal(process.argv.length, 3, 'one actual native x87 report directory required'); + const directory = process.argv[2]; + const result = auditNativeX87(fs.readFileSync(path.join(directory, 'inventory.log'), 'utf8'), + fs.readFileSync(path.join(directory, 'raw.jsonl'), 'utf8')); + process.stdout.write(JSON.stringify(result, null, 2) + '\n'); +} diff --git a/.github/scripts/diagnostic-qualification.cjs b/.github/scripts/diagnostic-qualification.cjs new file mode 100644 index 000000000..317c4c74d --- /dev/null +++ b/.github/scripts/diagnostic-qualification.cjs @@ -0,0 +1,35 @@ +'use strict'; + +const assert = require('node:assert/strict'); +// Retained to audit receipts from the retired fork diagnostic workflow. +const finalRequiredNeeds = [ + 'ownership', 'native-x86-oracles', 'sdk-priority', 'arm64-fallback-cross-priority', + 'lock-native-priority', 'unlocked-rmw-native-priority', 'vector-value-fault-priority', + 'packed-conversion-native-complete', 'packed-reduction-native-complete', 'complete-windows', +]; + +// Scheduling Windows after a failed predecessor discovers its real failures; +// it cannot supply success for that predecessor. GitHub's current-run needs +// observations remain the only input, never a job's self-reported outputs. +function qualifyDiagnosticNeeds(needs) { + assert.ok(needs && typeof needs === 'object' && !Array.isArray(needs), + 'missing actual diagnostic dependency observations'); + assert.deepEqual(Object.keys(needs).sort(), finalRequiredNeeds.slice().sort(), + 'diagnostic qualification requires every original gate and complete Windows'); + for (const name of finalRequiredNeeds) { + const observation = needs[name]; + assert.ok(observation && typeof observation === 'object' && !Array.isArray(observation), + 'malformed actual dependency result: ' + name); + assert.ok(Object.hasOwn(observation, 'result'), 'absent actual dependency result: ' + name); + assert.equal(observation.result, 'success', 'required diagnostic gate did not succeed: ' + name); + } + return {schema: 1, successfulJobs: finalRequiredNeeds.slice()}; +} + +module.exports = {qualifyDiagnosticNeeds}; +if (require.main === module) { + const input = process.env.DIAGNOSTIC_NEEDS_JSON; + assert.equal(typeof input, 'string', 'actual GitHub needs context is required'); + const result = qualifyDiagnosticNeeds(JSON.parse(input)); + process.stdout.write(JSON.stringify(result) + '\n'); +} diff --git a/.github/scripts/node22-setup.cjs b/.github/scripts/node22-setup.cjs new file mode 100644 index 000000000..7871609c2 --- /dev/null +++ b/.github/scripts/node22-setup.cjs @@ -0,0 +1,16 @@ +'use strict'; + +const assert = require('node:assert/strict'); + +function requireNode22Setup(step) { + assert.equal((step.match(/^(?: | - )uses: \.\/\.github\/actions\/setup-node22$/gm) || []).length, 1, + 'one exact verified Node 22 installer is required'); + assert.equal((step.match(/^(?: | - )uses:/gm) || []).length, 1, + 'another installer cannot share or replace the required Node step'); + assert.equal((step.match(/^ timeout-minutes: 10$/gm) || []).length, 1, + 'Node installation has its own bounded deadline'); + assert.doesNotMatch(step, /^(?: | - )(?:if|continue-on-error|with):|\|\|\s*true|node-version:/m, + 'the required exact runtime cannot be skipped, weakened or overridden'); +} + +module.exports = {requireNode22Setup}; diff --git a/.github/scripts/priority-aes-family-fresh-leases.test.cjs b/.github/scripts/priority-aes-family-fresh-leases.test.cjs new file mode 100644 index 000000000..e7afc8cb5 --- /dev/null +++ b/.github/scripts/priority-aes-family-fresh-leases.test.cjs @@ -0,0 +1,55 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const {test} = require('node:test'); + +const source = fs.readFileSync('amd64_aes_family_fixed_local_test.go', 'utf8'); + +test('the complete AES bound-consumer root prepares the process-owned compiler cache', () => { + assert.match(source, /func TestX86AESFamilyActualBoundFixedLocalConsumers\(t \*testing\.T\) \{\s*goAMD64FramePrepareOwnedToolCache\(t\)/); +}); + +test('the cache helper selects owned artifacts and prepares tools before source leases', () => { + const tools = fs.readFileSync('amd64_fixed_local_conversion_tool_cache_test.go', 'utf8'); + const helper = tools.match(/^func goAMD64FramePrepareOwnedToolCache\(t \*testing\.T\) \{\n([\s\S]*?)^\}/m); + assert.ok(helper, 'the real cache owner must exist'); + // Check the preparation boundary. Actual ownership, cleanup, source + // independence and concurrent close are exercised by the Go lifecycle tests. + let previous = -1; + for (const call of [ + 'fixedLocalProcessToolCache.directory()', + 't.Setenv("GOCACHE", cache)', + 'goAMD64FrameSourceFixture(t)', + 'fixedLocalConversionPrepareToolCache(t, request)', + ]) { + const offset = helper[1].indexOf(call); + assert.ok(offset > previous, `missing or reordered preparation: ${call}`); + previous = offset; + } + assert.match(helper[1], /cache, err := fixedLocalProcessToolCache\.directory\(\)\s*if err != nil \{\s*t\.Fatal\(err\)\s*\}/); +}); + +test('all AES targets consume independent original source leases for all three APIs', () => { + assert.match(source, /fixedLocalFreshAPIs\(t, pkg, request, options, \[\]string\{"Translate", "Module", "GoModule"\}/); + assert.doesNotMatch(source, /requireGoAMD64FrameCapture\(|runtimeOracleStageContext\(|\.BindFixedLocal\(|fixedLocalAPIs\(/); +}); + +test('AES retains exactly twelve original names, two source addresses, two targets and one object path', () => { + const oracle = fs.readFileSync('amd64_aes_family_effects_test.go', 'utf8'); + const names = oracle.match(/var x86AESOracleNames = \[\]string\{([^}]+)\}/); + assert.ok(names); + assert.deepEqual([...names[1].matchAll(/"([^"]+)"/g)].map(match => match[1]), + ['AESENC', 'AESENCLAST', 'AESDEC', 'AESDECLAST', 'AESIMC', 'AESKEYGENASSIST', + 'VAESENC', 'VAESENCLAST', 'VAESDEC', 'VAESDECLAST', 'VAESIMC', 'VAESKEYGENASSIST']); + const builder = source.slice(source.indexOf('\tvar assembly, declarations'), source.indexOf('\tfor _, target :=')); + assert.equal(crypto.createHash('sha256').update(builder).digest('hex'), + 'ccc5fe309b626943f3f92ef9ecc847e3f009aa66521eb395a4adcc6b6a63ace9'); + assert.match(builder, /\[\]string\{"16\(SP\)", "tmp-144\(SP\)"\}/); + assert.deepEqual([...source.matchAll(/\{"([^"\n]+\/amd64)", "([^"\n]+)"\}/g)] + .map(match => match.slice(1)), [['linux/amd64', 'x86_64-unknown-linux-gnu'], ['darwin/amd64', 'x86_64-apple-darwin']]); + assert.equal((source.match(/compileLLVMToObject\(/g) || []).length, 1); + assert.match(source, /compileLLVMToObject\(t, llc, target\.triple, "aes-native-local\.ll", "aes-native-local\.o", ir\)/); + assert.doesNotMatch(source, /x86MulDivRoundOptimize|"-O[0123]"|t\.Skip|t\.Parallel/); +}); diff --git a/.github/scripts/priority-arm64-fallback.test.cjs b/.github/scripts/priority-arm64-fallback.test.cjs new file mode 100644 index 000000000..e430dff59 --- /dev/null +++ b/.github/scripts/priority-arm64-fallback.test.cjs @@ -0,0 +1,85 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); +const { auditARM64Fallback } = require('./audit-arm64-fallback.cjs'); + +const root = 'TestFallbackARM64WideFrameCompilesWithLLVM22'; +const pkg = 'github.com/xgo-dev/plan9asm/cmd/plan9asmll'; +const names = [root, ...['darwin', 'linux', 'windows', 'native_complete-argument_oracle'] + .map(child => `${root}/${child}`)]; +const events = [ + ...names.map(Test => ({ Action: 'run', Test, Package: pkg })), + ...names.slice().reverse().map(Test => ({ Action: 'pass', Test, Package: pkg })), + { Action: 'pass', Package: pkg }, +]; +const raw = value => value.map(event => JSON.stringify(event)).join('\n') + '\n'; + +test('mandatory ARM64 runtime axes supplement the exhaustive runner audit', () => { + assert.deepEqual(auditARM64Fallback(raw(events)), { + roots: 1, children: 4, passed: 5, skipped: 0, incomplete: 0, + }); +}); + +test('successful Go dependency build output is not a failed runtime axis', () => { + const dependency = { Action: 'build-output', ImportPath: 'example.invalid/dependency', Output: 'compiler diagnostic\n' }; + assert.equal(auditARM64Fallback(raw([dependency, ...events])).children, 4); +}); + +test('the selected cross scope binds the actual Node auditor through the shared observer', () => { + const source = fs.readFileSync('cmd/plan9asmtest/ci_scope.go', 'utf8'); + assert.match(source, /if os\.Getenv\("PLAN9ASM_CROSS_EXEC"\) == "1" && !cfg\.Module \{\s*if err := captureCINodeDependency\(ctx, cfg, stem, origins\); err != nil/); +}); + +test('missing, skipped, duplicated or incomplete execution cannot count as covered', () => { + const native = names[4]; + const mutations = [ + events.filter(event => event.Test !== native), + events.filter(event => !(event.Test === native && event.Action === 'pass')), + events.filter(event => event.Test), + events.map(event => event.Test === native && event.Action === 'pass' ? { ...event, Action: 'skip' } : event), + events.map(event => !event.Test ? { ...event, Action: 'fail' } : event), + events.map(event => ({ ...event, Package: pkg + '/foreign' })), + [events[0], ...events], + [...events.slice(0, -1), events[8], events.at(-1)], + [...events, events[0]], + [{ Action: 'build-output', ImportPath: '', Output: 'unidentified' }, ...events], + [{ Action: 'build-output', ImportPath: 'dep', Test: root }, ...events], + [{ Action: 'build-fail', ImportPath: 'dep' }, ...events], + ]; + for (const mutation of mutations) { + assert.throws(() => auditARM64Fallback(raw(mutation))); + } + assert.throws(() => auditARM64Fallback('{not JSON}\n')); + assert.throws(() => auditARM64Fallback('')); +}); + +function requireDriver(source) { + assert.match(source, /PLAN9ASM_CROSS_EXEC:-\} != 1/); + assert.match(source, /root=TestFallbackARM64WideFrameCompilesWithLLVM22\n/); + assert.match(source, /go run \.\/cmd\/plan9asmtest -dir cmd\/plan9asmll -out/); + assert.match(source, /-ci-selector "\^\$\{root\}\$" -ci-isolate-root "\$root"/); + assert.match(source, /-shards 2 -timeout 5m/); + assert.match(source, /for owner in 0 1; do/); + assert.match(source, /-verify-ci-partitions "\$base\/owners"/); + assert.match(source, /node \.github\/scripts\/audit-arm64-fallback\.cjs/); + assert.match(source, /cross-runtime-owner-0\/root-0000\.stdout/); + assert.doesNotMatch(source, /continue-on-error|\|\| true|-run[= ]|timeout (?:10|20|30)m/); +} + +test('ARM64 driver keeps actual nested-module discovery and strict fresh verification', () => { + const source = fs.readFileSync('scripts/test-arm64-fallback-ci.sh', 'utf8'); + requireDriver(source); + for (const [before, after] of [ + ['-dir cmd/plan9asmll', '-dir .'], + ['^${root}$', '${root}'], + ['for owner in 0 1;', 'for owner in 0;'], + ['-verify-ci-partitions', '-verify-removed'], + ['audit-arm64-fallback.cjs', 'audit-removed.cjs'], + ['PLAN9ASM_CROSS_EXEC:-} != 1', 'PLAN9ASM_CROSS_EXEC:-} != 0'], + ['-timeout 5m', '-timeout 20m'], + ]) { + const changed = source.replace(before, after); + assert.notEqual(changed, source); + assert.throws(() => requireDriver(changed)); + } +}); diff --git a/.github/scripts/priority-blend-minpos-native.test.cjs b/.github/scripts/priority-blend-minpos-native.test.cjs new file mode 100644 index 000000000..d9d6c6f83 --- /dev/null +++ b/.github/scripts/priority-blend-minpos-native.test.cjs @@ -0,0 +1,187 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); + +// Independent of the auditor and production instruction maps. +const roots = ['TestCrossLinuxRuntimeMatrixX86BlendMinposOriginalFaultsAMD64', + 'TestCrossLinuxRuntimeMatrixX86BlendMinposOriginalFaults386']; +const axes = ['backend_O0', 'backend_O2', 'IR_O2_backend_O2']; +const forms = [['BLENDVPS', 'X'], ['VBLENDVPS', 'X'], ['VBLENDVPS', 'Y'], + ['BLENDVPD', 'X'], ['VBLENDVPD', 'X'], ['VBLENDVPD', 'Y'], + ['PBLENDVB', 'X'], ['VPBLENDVB', 'X'], ['VPBLENDVB', 'Y'], + ['PHMINPOSUW', 'X'], ['VPHMINPOSUW', 'X']]; +const parents = roots.flatMap(root => axes.map(axis => root + '/' + axis)); +const leaves = parents.flatMap(parent => forms.flatMap(([op, bank]) => ['live', 'dead'].flatMap(role => + [0, 1].map(zero => `${parent}/${op}/${bank}/${role}/maskzero=${zero}`)))); +const names = [...roots, ...parents, ...leaves]; +const inventory = roots.join('\n') + '\nok\tgithub.com/xgo-dev/plan9asm\n'; +const digest = 'a'.repeat(64); +const host = `IA32 direct host: linux/amd64 kernel=x86_64 CPUsha=${digest} mapsha=${digest}\n`; +const preflight = 'blend/minimum actual native SSE4.1=1 AVX2=1\n'; +const marker = 'blend/minimum original Go bytes and LLVM native alignment/access/CF/MXCSR=0x1f80/canary probes 20 failures 0\n'; +const image = `native oracle static IA32 ELF bytes=4096 sha256=${digest}\n`; +const sdk = arch => `actual LEA source/object receipt: SDK=go1.27.2 target=linux/${arch} source=${digest} driver=${digest} object=${digest}\n`; +const event = (Action, Test, Output) => ({Action, Package: 'github.com/xgo-dev/plan9asm', + ...(Test ? {Test} : {}), ...(Output !== undefined ? {Output} : {})}); +const serialize = events => events.map(value => JSON.stringify(value)).join('\n'); + +function completeEvents() { + const events = [event('start')]; + for (const root of roots) { + const arch = root.endsWith('386') ? '386' : 'amd64'; + events.push(event('run', root), event('output', root, host + preflight + sdk(arch))); + for (const axis of axes) { + const parent = root + '/' + axis; + events.push(event('run', parent)); + if (arch === '386') events.push(event('output', parent, image)); + for (const leaf of leaves.filter(name => name.startsWith(parent + '/'))) { + events.push(event('run', leaf), event('output', leaf, marker), event('pass', leaf)); + } + events.push(event('pass', parent)); + } + events.push(event('output', root, host), event('pass', root)); + } + events.push(event('pass')); + return events; +} + +function audit(events = completeEvents(), discovered = inventory) { + assert.ok(fs.existsSync('.github/scripts/audit-blend-minpos-native.cjs'), + 'original blend/minimum native scope lacks an explicit strict auditor'); + return require('./audit-blend-minpos-native.cjs').auditBlendMinposNative(discovered, serialize(events)); +} + +test('blend/minimum requires two roots, 264 complete native kernels and 5280 paired probes', () => { + assert.deepEqual(audit(), {roots: 2, children: 270, passed: 272, skipped: 0, incomplete: 0, + nativeHosts: 4, sdkCaptures: 2, isaPreflights: 2, staticIA32Images: 3, + kernelExecutions: 264, pairedProbes: 5280, directProbeProcesses: 10560}); +}); + +test('blend/minimum rejects every missing duplicate failed skipped or incomplete lifecycle', () => { + audit(); + for (const name of names) { + for (const Action of ['fail', 'skip']) { + assert.throws(() => audit(completeEvents().map(value => value.Test === name && value.Action === 'pass' + ? {...value, Action} : value)), name + Action); + } + for (const Action of ['run', 'pass']) { + const original = completeEvents(); + const at = original.findIndex(value => value.Test === name && value.Action === Action); + assert.throws(() => audit(original.filter((_, index) => index !== at)), name + ' missing ' + Action); + original.splice(at, 0, original[at]); + assert.throws(() => audit(original), name + ' duplicate ' + Action); + } + } +}); + +test('blend/minimum binds all residues and zero-mask dead reads to their own actual completion', () => { + audit(); + for (const leaf of leaves) { + for (const replacement of ['', marker + marker, marker.replace('probes 20', 'probes 19')]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === leaf && value.Output + ? {...value, Output: replacement} : value)), leaf); + } + } + for (const replacement of [marker.replace('failures 0', 'failures 1'), marker.replace('0x1f80', 'inherited'), + marker.replace('0x1f80', '0x21f80'), host + marker, preflight + marker, sdk('amd64') + marker, image + marker]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === leaves[0] && value.Output + ? {...value, Output: replacement} : value))); + } + for (const target of [roots[0], parents[0]]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === leaves[0] && value.Output + ? {...value, Test: target} : value)), 'summary is not actual leaf execution'); + } +}); + +test('blend/minimum requires native host ISA and exact original Go source before every axis', () => { + audit(); + for (const root of roots) { + const arch = root.endsWith('386') ? '386' : 'amd64'; + for (const needle of [host, preflight, sdk(arch)]) { + for (const replacement of ['', needle + needle, needle.replace(digest, 'invalid'), + needle.replace('kernel=x86_64', 'kernel=aarch64'), needle.replace('SSE4.1=1', 'SSE4.1=0'), + needle.replace('AVX2=1', 'AVX2=0'), needle.replace('go1.27.2', 'go1.20.14'), + needle.replace('linux/' + arch, 'linux/arm64')]) { + if (replacement === needle) continue; + assert.throws(() => audit(completeEvents().map(value => value.Test === root && value.Output + ? {...value, Output: value.Output.replace(needle, replacement)} : value)), root + needle); + } + } + const lateSDK = completeEvents(); + const at = lateSDK.findIndex(value => value.Test === root && value.Output?.includes(preflight)); + lateSDK[at] = event('output', root, host + preflight); + lateSDK.splice(at + 2, 0, event('output', root, sdk(arch))); + assert.throws(() => audit(lateSDK), 'source capture after axis start'); + const earlyPost = completeEvents(); + const post = earlyPost.findLastIndex(value => value.Test === root && value.Output === host); + const [observation] = earlyPost.splice(post, 1); + const terminal = earlyPost.findIndex(value => value.Test === root + '/' + axes[2] && value.Action === 'pass'); + earlyPost.splice(terminal, 0, observation); + assert.throws(() => audit(earlyPost), 'host bracket before final axis'); + } + const otherSDK = completeEvents().map(value => value.Test === roots[1] && value.Output?.includes(' driver=') + ? {...value, Output: value.Output.replace(' driver=' + digest, ' driver=' + 'b'.repeat(64))} : value); + assert.throws(() => audit(otherSDK), 'the two owners must use the same actual Go driver'); +}); + +test('blend/minimum IA32 images precede every actual leaf and cannot be emulated summaries', () => { + audit(); + for (const parent of parents.filter(name => name.startsWith(roots[1] + '/'))) { + for (const replacement of ['', image + image, image.replace('4096', '0'), image.replace(digest, 'invalid'), + image.replace('static IA32', 'QEMU translated')]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === parent && value.Output + ? {...value, Output: replacement} : value)), parent); + } + const late = completeEvents(); + const at = late.findIndex(value => value.Test === parent && value.Output === image); + const [observation] = late.splice(at, 1); + late.splice(at + 2, 0, observation); + assert.throws(() => audit(late), 'image inspected after first execution'); + } +}); + +test('blend/minimum rejects foreign inventories events packages and out-of-lifecycle output', () => { + audit(); + for (const discovered of ['', inventory + roots[0] + '\n', inventory.replace(roots[0], 'TestObjectsOnly')]) { + assert.throws(() => audit(completeEvents(), discovered)); + } + for (const events of [completeEvents().slice(1), completeEvents().slice(0, -1), + [event('start'), ...completeEvents()], [...completeEvents(), event('pass')], + completeEvents().map(value => ({...value, Package: 'foreign/package'})), + [event('output', undefined, '264 kernels passed')]]) assert.throws(() => audit(events)); + for (const name of [roots[0], parents[0], leaves[0]]) { + const late = completeEvents(); + const at = late.findIndex(value => value.Test === name && value.Action === 'pass'); + late.splice(at + 1, 0, event('output', name, marker)); + assert.throws(() => audit(late)); + } +}); + +test('blend/minimum fixed leaves retain their own package and live root/axis lifecycle', () => { + audit(); + for (const leaf of [leaves[0], leaves.at(-1)]) { + for (const Action of ['run', 'output', 'pass']) { + assert.throws(() => audit(completeEvents().map(value => value.Test === leaf && value.Action === Action + ? {...value, Package: 'foreign/package'} : value)), 'foreign leaf ' + Action); + assert.throws(() => audit(completeEvents().map(value => value.Test === leaf && value.Action === Action + ? {...value, Action: 'unrecognized'} : value)), 'unknown leaf ' + Action); + } + const parent = leaf.split('/').slice(0, 2).join('/'); + for (const before of [leaf.split('/')[0], parent]) { + const events = completeEvents(); + const own = events.filter(value => value.Test === leaf); + const others = events.filter(value => value.Test !== leaf); + const at = others.findIndex(value => value.Test === before && value.Action === 'run'); + others.splice(at, 0, ...own); + assert.throws(() => audit(others), 'leaf before actual parent ' + before); + } + const events = completeEvents(); + events.push(event('output', leaf, marker)); + assert.throws(() => audit(events), 'leaf after completed package'); + } + const foreign = completeEvents(); + foreign.splice(4, 0, event('run', parents[0] + '/unknown/opcode')); + assert.throws(() => audit(foreign), 'unknown leaf cannot evade fixed ancestry'); +}); diff --git a/.github/scripts/priority-checkout-source-eol.test.cjs b/.github/scripts/priority-checkout-source-eol.test.cjs new file mode 100644 index 000000000..14629cbc4 --- /dev/null +++ b/.github/scripts/priority-checkout-source-eol.test.cjs @@ -0,0 +1,106 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); +const test = require('node:test'); + +const attributes = fs.readFileSync('.gitattributes', 'utf8'); +const source = 'package fixture\n\nconst assembly = `TEXT original(SB),4,$0-0\nRET\n`\n' + + 'const oracle = `int main(void) { return 0; }\n`\nconst escaped = "\\r\\n"\n'; +const fixtures = { + 'original.go': source, + 'nested/original_test.go': source, + 'original.s': 'TEXT original(SB),4,$0-0\nRET\n', + 'nested/conformance.s': fs.readFileSync('testdata/conformance/amd64/conformance_amd64.s', 'utf8'), + 'script.sh': '#!/bin/sh\nprintf "original\\n"\n', + 'ordinary.txt': 'ordinary text\nsecond line\n', + 'ordinary.asm': 'TEXT ordinary(SB),4,$0-0\nRET\n', + 'downloaded/original.s': 'TEXT downloaded(SB),4,$0-0\r\nRET\r\n', +}; +const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); + +function checkoutFixture(rules) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'plan9asm-checkout-eol-')); + const environment = {...process.env}; + for (const key of Object.keys(environment)) { + if (key.startsWith('GIT_')) delete environment[key]; + } + Object.assign(environment, { + GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: os.devNull, GIT_ATTR_NOSYSTEM: '1', + }); + try { + const git = args => { + const result = spawnSync('git', ['-c', 'core.attributesFile=' + os.devNull, + '-c', 'core.hooksPath=' + os.devNull, '-c', 'commit.gpgsign=false', + '-c', 'user.name=Checkout fixture', '-c', 'user.email=fixture@example.invalid', + '-C', directory, ...args], { + env: environment, encoding: 'utf8', timeout: 5000, + }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + return result.stdout; + }; + git(['init', '--quiet']); + git(['config', '--local', 'core.autocrlf', 'true']); + git(['config', '--local', 'core.safecrlf', 'false']); + fs.writeFileSync(path.join(directory, '.gitattributes'), rules); + fs.writeFileSync(path.join(directory, '.gitignore'), 'downloaded/\n'); + for (const [name, text] of Object.entries(fixtures)) { + const target = path.join(directory, name); + fs.mkdirSync(path.dirname(target), {recursive: true}); + fs.writeFileSync(target, text); + } + git(['add', '.']); + git(['commit', '--quiet', '-m', 'original fixture']); + for (const name of Object.keys(fixtures)) { + if (!name.startsWith('downloaded/')) fs.unlinkSync(path.join(directory, name)); + } + // This actually restores bytes through Git's worktree conversion, not a + // handcrafted CRLF replacement or a global user configuration change. + git(['checkout-index', '--all', '--force']); + return Object.fromEntries(Object.keys(fixtures).map(name => + [name, fs.readFileSync(path.join(directory, name))])); + } finally { + fs.rmSync(directory, {recursive: true, force: true}); + } +} + +function requireOriginalSourceBytes(observed) { + for (const name of ['original.go', 'nested/original_test.go', 'original.s', 'nested/conformance.s']) { + assert.equal(digest(observed[name]), digest(Buffer.from(fixtures[name])), + 'checkout must preserve complete original repository Go/ASM/C bytes: ' + name); + assert.equal(observed[name].includes(Buffer.from('\r\n')), false, name); + } + assert.equal(observed['script.sh'].toString(), fixtures['script.sh']); + for (const name of ['ordinary.txt', 'ordinary.asm']) { + assert.equal(observed[name].toString(), fixtures[name].replaceAll('\n', '\r\n'), + 'control must prove actual autocrlf conversion outside the source policy: ' + name); + } + assert.equal(digest(observed['downloaded/original.s']), digest(Buffer.from(fixtures['downloaded/original.s'])), + 'Git source policy must not normalize ignored external authority bytes'); +} + +test('actual autocrlf checkout preserves complete Go and ASM hashes without altering downloads', () => { + requireOriginalSourceBytes(checkoutFixture(attributes)); +}); + +test('checkout policy rejects missing substituted or CRLF Go and ASM rules using real Git controls', () => { + const complete = '*.sh text eol=lf\n*.go text eol=lf\n*.s text eol=lf\n'; + requireOriginalSourceBytes(checkoutFixture(complete)); + for (const changed of [ + complete.replace('*.go text eol=lf\n', ''), + complete.replace('*.go text', '*.notgo text'), + complete.replace('*.go text eol=lf', '*.go text eol=crlf'), + complete.replace('*.s text eol=lf\n', ''), + complete.replace('*.s text', '*.notasm text'), + complete.replace('*.s text eol=lf', '*.s text eol=crlf'), + ]) { + const observed = checkoutFixture(changed); + assert.equal(observed['ordinary.txt'].toString(), fixtures['ordinary.txt'].replaceAll('\n', '\r\n')); + assert.throws(() => requireOriginalSourceBytes(observed)); + } +}); diff --git a/.github/scripts/priority-completion.test.cjs b/.github/scripts/priority-completion.test.cjs new file mode 100644 index 000000000..939885df3 --- /dev/null +++ b/.github/scripts/priority-completion.test.cjs @@ -0,0 +1,200 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); +const { test } = require('node:test'); +const {jobBody} = require('./workflow-needs.cjs'); + +const workflow = fs.readFileSync(path.join(__dirname, '../workflows/go-ci.yml'), 'utf8'); + +function jobSection(name, source = workflow) { + return jobBody(source, name); +} + +function policySelector(source) { + const steps = jobSection('ci_policy', source).split(/(?=^ - )/m); + const selected = steps.filter(step => step.startsWith(' - name: CI policy tests\n')); + assert.equal(selected.length, 1, 'one actual Go policy execution step required'); + assert.doesNotMatch(selected[0], /^ (?:if|continue-on-error):/m); + const commands = [...selected[0].matchAll(/^ run: go test \. -run '([^']+)' -count=1$/gm)]; + assert.equal(commands.length, 1, 'one uncached, unmasked root-package policy command required'); + return commands[0][1]; +} + +function corpusBaselineStep(source) { + const steps = jobSection('ci_policy', source).split(/(?=^ - )/m); + const selected = steps.filter(step => step.startsWith(' - name: Official corpus form baselines\n')); + assert.equal(selected.length, 1, 'one actual corpus baseline execution step required'); + const step = selected[0]; + assert.match(step, /^ timeout-minutes: 10$/m); + assert.match(step, /^ shell: bash$/m); + assert.doesNotMatch(step, /^ (?:if|continue-on-error):/m); + const marker = ' run: |\n'; + assert.equal(step.split(marker).length, 2, 'one literal baseline run body required'); + const lines = step.slice(step.indexOf(marker) + marker.length).trimEnd().split('\n'); + assert.ok(lines.every(line => line.startsWith(' ')), 'complete literal shell body required'); + const script = lines.map(line => line.slice(10)).join('\n') + '\n'; + assert.deepEqual(script.trimEnd().split('\n'), [ + 'set -euo pipefail', + 'scripts/check-go-asm-coverage.sh', + 'scripts/check-arm64-plan9-corpus.sh', + ]); + return {step, script}; +} + +function requireCompletePolicy(source) { + assert.equal(policySelector(source), + '^(TestCI|TestFixedLocal|TestGoAMD64FixedLocal(Packed|Conversion)RuntimeOwnership$|TestX86PackedConversion(RequiredNativeOwners|CompleteNativeRuntimeOwnership)$|TestX86FloatingCompareRequiredLinuxBuildSelection$|TestRuntimeOracleCommand(ConsumersRetainOwnedExecution|AuditDistinguishesTestingRun)$)', + 'preflight must select complete static-contract prefixes, not a manually enumerated subset'); + const policy = jobSection('ci_policy', source); + assert.match(policy, /^ timeout-minutes: 20$/m); + assert.match(policy, /^ go-version: '1\.27\.1'$/m); + assert.match(policy, /^ run: bash scripts\/install-ci-llvm22-ubuntu\.sh$/m); + assert.match(policy, /^ uses: \.\/\.github\/actions\/setup-node22$/m); + assert.doesNotMatch(policy, /^ (?:needs|if|continue-on-error):/m); + corpusBaselineStep(source); +} + +test('translator jobs require policy preflight, not external compatibility', () => { + for (const name of [ + 'fmt', 'build', + 'stdlib-corpus', 'cross-runtime', 'test', 'nested-cli', 'arm-scan', 'race', 'coverage', 'nested-cli-coverage', + ]) { + const section = jobSection(name); + assert.match(section, /^ needs: ci_policy$/m, name); + // Native needs success gating must not be bypassed by always()/!cancelled(). + assert.doesNotMatch(section, /^ if:/m, name); + } + assert.doesNotMatch(workflow, /discovered_library|reported_library|testdata\/discovery/); +}); + +test('all translator matrix failures remain failures', () => { + for (const name of ['test', 'nested-cli', 'race', 'coverage', 'nested-cli-coverage', 'cross-runtime']) { + const job = jobSection(name); + assert.match(job, /^ fail-fast: false$/m); + assert.doesNotMatch(job, /continue-on-error:/); + } +}); + +test('cheap CI policy checks pass before instruction and official corpus jobs', () => { + const policy = jobSection('ci_policy'); + assert.match(policy, /bash scripts\/install-ci-llvm22-ubuntu\.sh/); + assert.match(policy, /node --test \.github\/scripts\/priority-\*\.test\.cjs/); + requireCompletePolicy(workflow); + assert.doesNotMatch(policy, /^ (needs|if):/m); + assert.ok(workflow.indexOf('\n ci_policy:\n') < + workflow.indexOf('\n test:\n')); + + // Exercise the actual workflow body with owned protocol scripts, never Go/LLVM. + const {script} = corpusBaselineStep(workflow); + for (const [first, second, status, expected] of [ + [0, 0, 0, ['go-asm', 'arm64-plan9']], + [17, 0, 17, ['go-asm']], + [0, 19, 19, ['go-asm', 'arm64-plan9']], + ]) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'plan9asm-baseline-preflight-')); + try { + fs.mkdirSync(path.join(directory, 'scripts')); + for (const [file, name, exit] of [ + ['check-go-asm-coverage.sh', 'go-asm', first], + ['check-arm64-plan9-corpus.sh', 'arm64-plan9', second], + ]) { + fs.writeFileSync(path.join(directory, 'scripts', file), + `#!/usr/bin/env bash\nprintf '%s\\n' '${name}' >> calls\nexit ${exit}\n`, {mode: 0o755}); + } + const result = spawnSync('bash', ['--noprofile', '--norc', '-c', script], { + cwd: directory, encoding: 'utf8', timeout: 5000, + }); + assert.ifError(result.error); + assert.equal(result.signal, null); + assert.equal(result.status, status, result.stderr); + assert.deepEqual(fs.readFileSync(path.join(directory, 'calls'), 'utf8').trimEnd().split('\n'), expected); + } finally { + fs.rmSync(directory, {recursive: true, force: true}); + } + } +}); + +test('PR preflight selects every existing CI and FixedLocal static contract', () => { + requireCompletePolicy(workflow); + const selected = new RegExp(policySelector(workflow)); + const root = path.join(__dirname, '../..'); + const names = fs.readdirSync(root).filter(file => file.endsWith('_test.go')).flatMap(file => + [...fs.readFileSync(path.join(root, file), 'utf8').matchAll( + /^func (Test(?:CI|FixedLocal)[A-Za-z0-9_]+|TestGoAMD64FixedLocal(?:Packed|Conversion)RuntimeOwnership|TestX86PackedConversion(?:RequiredNativeOwners|CompleteNativeRuntimeOwnership)|TestX86FloatingCompareRequiredLinuxBuildSelection|TestRuntimeOracleCommand(?:ConsumersRetainOwnedExecution|AuditDistinguishesTestingRun))\(t \*testing\.T\)/gm, + )].map(match => match[1])); + for (const guard of [ + 'TestFixedLocalWideCompileOnlyOwnersDoNotRequireRuntimeTransport', + 'TestFixedLocalCompileOnlyToolsRetainExactLLVMAndSDKWithoutExecution', + 'TestFixedLocalSharedHelpersBuildOnAllHosts', + 'TestGoAMD64FixedLocalPackedRuntimeOwnership', + 'TestGoAMD64FixedLocalConversionRuntimeOwnership', + 'TestX86PackedConversionRequiredNativeOwners', + 'TestX86PackedConversionCompleteNativeRuntimeOwnership', + 'TestX86FloatingCompareRequiredLinuxBuildSelection', + 'TestRuntimeOracleCommandConsumersRetainOwnedExecution', + 'TestRuntimeOracleCommandAuditDistinguishesTestingRun', + ]) assert.ok(names.includes(guard), 'independently required source guard missing: ' + guard); + for (const name of names) assert.ok(selected.test(name), 'preflight omitted existing static contract: ' + name); +}); + +test('preflight includes future static contracts without selecting expensive execution owners', () => { + const selected = new RegExp(policySelector(workflow)); + for (const name of ['TestCIFutureSchedulingContract', 'TestFixedLocalFutureSourceOwnershipContract', + 'TestGoAMD64FixedLocalPackedRuntimeOwnership']) { + assert.ok(selected.test(name), 'new static contracts must not need another suffix allowlist: ' + name); + } + for (const name of [ + 'TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWideBatch00', + 'TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWideBatch05', + 'TestGoAMD64FixedLocalPackedWideRuntimeOracleCompileOnly', + 'TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch00', + 'TestGoAMD64FixedLocalReadonlyDataWindowsRegisteredConsumers', + 'TestX86AESFamilyActualBoundFixedLocalConsumers', + 'TestRuntimeOracleCommandRequiresBoundContextAndOutput', + 'TestRuntimeOracleCommandConsumersRetainOwnedExecutionNative', + 'TestRuntimeOracleCommandAuditDistinguishesTestingRunNative', + 'TestGoAMD64FixedLocalConversionRuntimeOwnershipNative', + 'TestX86PackedConversionCompleteNativeRuntimeOwnershipNative', + 'TestX86FloatingCompareRequiredLinuxBuildSelectionNative', + ]) assert.ok(!selected.test(name), 'cheap preflight acquired an actual native/object owner: ' + name); +}); + +test('preflight rejects stale suffix allowlists, missing contract families and failure masking', () => { + requireCompletePolicy(workflow); + const command = "run: go test . -run '" + policySelector(workflow) + "' -count=1"; + for (const selector of [ + '^TestCI', + '^(TestCI|TestFixedLocal(APIsHaveIndependentBoundedCaptureLeases|WideCompileOnlyHasThreeCompleteAPIOwners)|TestGoAMD64FixedLocalPackedRuntimeOwnership)', + '^(TestFixedLocal|TestGoAMD64FixedLocalPackedRuntimeOwnership)', + '^(TestCI|TestFixedLocal)', + '^(TestCI|TestFixedLocal|TestGoAMD64FixedLocalPackedRuntimeOwnership)', + '^(TestCI|TestFixedLocal|TestGoAMD64FixedLocalPackedRuntimeOwnership|TestRuntimeOracleCommand(ConsumersRetainOwnedExecution|AuditDistinguishesTestingRun)$)', + '^Test', + ]) assert.throws(() => requireCompletePolicy(workflow.replace(command, + () => "run: go test . -run '" + selector + "' -count=1"))); + for (const changed of ['run: echo policy', command + ' || true', + "if: false\n " + command]) { + assert.throws(() => requireCompletePolicy(workflow.replace(command, () => changed))); + } + + const {step} = corpusBaselineStep(workflow); + for (const changed of [ + '', step + step, + step.replace('timeout-minutes: 10', 'timeout-minutes: 0'), + step.replace('shell: bash', 'shell: python'), + step.replace(' shell: bash', ' if: false\n shell: bash'), + step.replace(' shell: bash', ' continue-on-error: true\n shell: bash'), + step.replace(' scripts/check-go-asm-coverage.sh\n', ''), + step.replace(' scripts/check-arm64-plan9-corpus.sh\n', ''), + step.replace('scripts/check-go-asm-coverage.sh', 'echo scripts/check-go-asm-coverage.sh'), + step.replace('scripts/check-arm64-plan9-corpus.sh', '# scripts/check-arm64-plan9-corpus.sh'), + step.replace('set -euo pipefail', 'set +e'), + step.replace('scripts/check-go-asm-coverage.sh', 'scripts/check-go-asm-coverage.sh || true'), + step.replace('scripts/check-arm64-plan9-corpus.sh', 'scripts/check-arm64-plan9-corpus.sh &'), + ]) { + assert.notEqual(changed, step, 'negative control must alter the actual baseline step'); + assert.throws(() => requireCompletePolicy(workflow.replace(step, () => changed))); + } +}); diff --git a/.github/scripts/priority-cross-runtime.test.cjs b/.github/scripts/priority-cross-runtime.test.cjs new file mode 100644 index 000000000..53bc7b1be --- /dev/null +++ b/.github/scripts/priority-cross-runtime.test.cjs @@ -0,0 +1,141 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); + +const workflow = fs.readFileSync('.github/workflows/go-ci.yml', 'utf8'); +const nested = fs.readFileSync('.github/workflows/linux-cli-module.yml', 'utf8'); +function job(name, source = workflow) { + const marker = `\n ${name}:\n`; + const start = source.indexOf(marker); + assert.notEqual(start, -1, `missing ${name}`); + const body = source.slice(start + marker.length); + const end = body.search(/^ [a-zA-Z][a-zA-Z0-9_-]*:\s*$/m); + return end < 0 ? body : body.slice(0, end); +} + +test('required cross-runtime inventory has sixteen exhaustive owners, not a concurrency cap', () => { + const body = job('cross-runtime'); + assert.match(body, /fail-fast: false/); + assert.doesNotMatch(body, /max-parallel:|continue-on-error:/); + assert.match(body, /owner: \[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15\]/); + assert.match(body, /-ci-isolate-root TestCrossLinuxRuntimeMatrix/); + assert.match(body, /-ci-owner \$\{\{ matrix.owner \}\}/); + assert.match(body, /-shards 16 -timeout 20m/); + assert.match(body, /\^\(TestCrossLinuxRuntimeMatrix\|TestARM64Conformance\|TestARM64RawSQSHLURuntimeSemantics\|TestARMIntegerMemoryRuntimeCross\|TestARMRawSaturationRuntimeCross\)/); + assert.match(body, /PLAN9ASM_CROSS_EXEC: '1'/); + assert.match(body, /bash scripts\/install-ci-qemu.sh/); + assert.match(body, /actions\/upload-artifact@v6/); + assert.match(body, /if: \$\{\{ !cancelled\(\) \}\}/); +}); + +test('strict completion runs even after a failed owner and keeps original tool requirements', () => { + const body = job('cross-runtime-verify'); + assert.match(body, /needs: cross-runtime/); + assert.match(body, /if: \$\{\{ !cancelled\(\) \}\}/); + assert.match(body, /-verify-ci-partitions _out\/cross-runtime-partitions/); + assert.match(body, /-shards 16 -timeout 20m/); + assert.match(body, /-ci-isolate-root TestCrossLinuxRuntimeMatrix/); + assert.match(body, /actions\/download-artifact@v8/); + assert.match(body, /gcc-i686-linux-gnu gcc-arm-linux-gnueabihf/); + assert.match(body, /gcc-aarch64-linux-gnu/); + assert.match(body, /bash scripts\/install-ci-qemu.sh/); + assert.doesNotMatch(body, /continue-on-error:|PLAN9ASM_CROSS_EXEC: '0'/); +}); + +function requireARM64FallbackOwner(source) { + const owner = job('cross-runtime', source); + assert.match(owner, /- name: Execute rich CLI ARM64 fallback oracle\n if: \$\{\{ !cancelled\(\) && matrix\.owner == 15 \}\}\n run: bash scripts\/test-arm64-fallback-ci\.sh execute/); + assert.equal((source.match(/- name: Execute rich CLI ARM64 fallback oracle/g) || []).length, 1); + const reader = job('cross-runtime-verify', source); + assert.match(reader, /- name: Verify rich CLI ARM64 fallback execution\n if: \$\{\{ !cancelled\(\) \}\}\n run: bash scripts\/test-arm64-fallback-ci\.sh complete/); +} + +test('rich CLI ARM64 execution has a unique cross owner and a fresh completion reader', () => { + requireARM64FallbackOwner(workflow); +}); + +test('ARM64 execution cannot lose its unique owner, success guard or reader', () => { + for (const [before, after] of [ + ['Execute rich CLI ARM64 fallback oracle', 'Removed runtime oracle'], + ['matrix.owner == 15', 'matrix.owner == 14'], + ['!cancelled() && matrix.owner == 15', 'success()'], + ['test-arm64-fallback-ci.sh execute', 'test-arm64-fallback-ci.sh complete'], + ['Verify rich CLI ARM64 fallback execution', 'Removed execution reader'], + ]) { + const changed = workflow.replace(before, after); + assert.notEqual(changed, workflow); + assert.throws(() => requireARM64FallbackOwner(changed)); + } + assert.throws(() => requireARM64FallbackOwner(workflow + '\n - name: Execute rich CLI ARM64 fallback oracle\n')); +}); + +const moduleExecutionJobs = [ + ...['test', 'race', 'coverage', 'module-completion'].map(name => [workflow, name]), + ...['root-parts', 'complete'].map(name => [nested, name]), +]; +const moduleCompilerInstall = 'bash scripts/install-ci-llvm22-ubuntu.sh ' + + 'gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu'; + +// The unchanged portable FMA access oracle executes AArch64 on Linux/amd64 +// in ordinary module owners too. Its linker cannot be supplied solely by the +// separate cross-runtime job, nor can missing infrastructure become a skip. +function requireModuleCrossTools(source, name) { + const body = job(name, source); + const installs = body.split(/(?=^ - )/m).filter(step => + step.includes('run: bash scripts/install-ci-llvm22-ubuntu.sh')); + assert.equal(installs.length, 1, `${name}: one LLVM/cross-compiler install`); + assert.match(installs[0], /^ timeout-minutes: 14$/m); + assert.ok(installs[0].includes(` run: ${moduleCompilerInstall}\n`), + `${name}: both 386 and AArch64 module execution dependencies required`); + assert.doesNotMatch(installs[0], /continue-on-error:|\|\|\s*true/); + assert.match(body, /^ run: bash scripts\/install-ci-qemu\.sh$/m); +} + +// The intended complete installation is an independent positive control; the +// real workflow below must satisfy exactly the same requirement on its own. +for (const [source, name] of moduleExecutionJobs) { + const moduleToolsFixture = source.replaceAll( + 'bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross\n', + moduleCompilerInstall + '\n'); + test(`${name}: real module execution owns its required cross tools`, () => { + requireModuleCrossTools(source, name); + }); + + test(`${name}: complete cross tools reject missing dependencies or swallowed errors`, () => { + requireModuleCrossTools(moduleToolsFixture, name); + const original = job(name, moduleToolsFixture); + for (const [before, after] of [ + [' gcc-aarch64-linux-gnu', ''], + [' gcc-i686-linux-gnu', ''], + [' libc6-dev-i386-cross', ''], + ['install-ci-llvm22-ubuntu.sh', 'install-unpinned-llvm.sh'], + [moduleCompilerInstall, moduleCompilerInstall + ' || true'], + ['install-ci-qemu.sh', 'install-unpinned-qemu.sh'], + ]) { + const mutated = original.replace(before, after); + assert.notEqual(mutated, original, `${name}: mutation changed its own job`); + // Shell regexes can contain $'; a function keeps replacement bytes literal. + const changed = moduleToolsFixture.replace(original, () => mutated); + assert.equal(job(name, changed), mutated, 'mutation must retain its exact job boundary'); + assert.throws(() => requireModuleCrossTools(changed, name), undefined, + `${name}: reject ${before}`); + } + }); +} + +test('ARM64 main driver retains the complete strict oracle and its four mandatory children', () => { + const driver = fs.readFileSync('scripts/test-arm64-fallback-ci.sh', 'utf8'); + assert.match(driver, /root=TestFallbackARM64WideFrameCompilesWithLLVM22\n/); + assert.match(driver, /-dir cmd\/plan9asmll/); + assert.match(driver, /-ci-selector "\^\$\{root\}\$" -ci-isolate-root "\$root"/); + assert.match(driver, /-shards 2 -timeout 5m/); + assert.match(driver, /for owner in 0 1; do/); + assert.match(driver, /-verify-ci-partitions "\$base\/owners"/); + assert.match(driver, /node \.github\/scripts\/audit-arm64-fallback\.cjs/); + const auditor = fs.readFileSync('.github/scripts/audit-arm64-fallback.cjs', 'utf8'); + assert.match(auditor, /\['darwin', 'linux', 'windows', 'native_complete-argument_oracle'\]/); + assert.match(auditor, /roots: 1, children: 4, passed: 5, skipped: 0, incomplete: 0/); + const qemu = fs.readFileSync('scripts/install-ci-qemu.sh', 'utf8'); + assert.match(qemu, /^version=10\.2\.3$/m); + assert.match(qemu, /sha256sum --check --status/); +}); diff --git a/.github/scripts/priority-current-sdk-scope.test.cjs b/.github/scripts/priority-current-sdk-scope.test.cjs new file mode 100644 index 000000000..a5432a143 --- /dev/null +++ b/.github/scripts/priority-current-sdk-scope.test.cjs @@ -0,0 +1,152 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const {jobBody} = require('./workflow-needs.cjs'); + +const source = fs.readFileSync(path.join(__dirname, '../workflows/go-ci.yml'), 'utf8'); +const nested = fs.readFileSync(path.join(__dirname, '../workflows/linux-cli-module.yml'), 'utf8'); +const exactSDK = '1.27.2'; +const minors = ['1.20.x', '1.21.x', '1.22.x', '1.23.x', '1.24.x', + '1.25.x', '1.26.x', exactSDK]; +const testCreationOrder = [exactSDK, minors[0], ...minors.slice(1, -1).reverse()]; +const matrixJobs = new Set(['stdlib-corpus', 'test', 'nested-cli', 'module-completion']); +// The independent SDK/runtime integration workflow has its own exact-SDK +// policy. These remain the required PR instruction and module owners. +const jobs = ['fmt', 'build', 'stdlib-corpus', 'cross-runtime', + 'cross-runtime-verify', 'test', 'nested-cli', 'nested-cli-coverage', + 'arm-scan', 'race', 'coverage', 'module-completion']; + +function requireSetups(workflow) { + for (const name of jobs) { + const body = jobBody(workflow, name); + if (name === 'nested-cli' || name === 'nested-cli-coverage') { + assert.match(body, /^ uses: \.\/\.github\/workflows\/linux-cli-module\.yml$/m); + const expected = name === 'nested-cli' ? '${{ matrix.go-version }}' : `'${exactSDK}'`; + assert.ok(body.includes(' go-version: ' + expected + '\n')); + continue; + } + const setups = body.split(/(?=^ - )/m).filter(step => + /^ uses: \.\/\.github\/actions\/setup-go$/m.test(step)); + assert.equal(setups.length, name === 'stdlib-corpus' ? 2 : 1, + name + ': every actual Go installation must be covered'); + const expected = matrixJobs.has(name) ? '${{ matrix.go-version }}' : `'${exactSDK}'`; + const versions = setups.map(step => { + assert.match(step, /^ with:$/m); + assert.doesNotMatch(step, /go-version-file:|check-latest:/); + const fields = [...step.matchAll(/^ go-version: (.+)$/gm)]; + assert.equal(fields.length, 1, name + ': one explicit SDK selection required'); + return fields[0][1]; + }); + assert.deepEqual(versions, name === 'stdlib-corpus' ? [`'${exactSDK}'`, expected] : [expected], + name + ': current SDK must use the registered exact producer'); + assert.doesNotMatch(body, /uses: actions\/setup-go@/); + } + for (const name of ['root-parts', 'complete']) { + const body = jobBody(nested, name); + const setups = body.split(/(?=^ - )/m).filter(step => + /^ uses: \.\/\.github\/actions\/setup-go$/m.test(step)); + assert.equal(setups.length, 1, 'one actual reusable SDK setup: ' + name); + assert.match(setups[0], /^ go-version: \$\{\{ inputs\.go-version \}\}$/m); + assert.doesNotMatch(body, /uses: actions\/setup-go@|go-version-file:|check-latest:/); + } +} + +function requireMatrices(workflow) { + const regular = jobBody(workflow, 'test'); + const versions = regular.match(/^ go-version: \[(.+)\]$/m); + assert.ok(versions, 'complete test SDK matrix required'); + assert.deepEqual([...versions[1].matchAll(/'([^']+)'/g)].map(match => match[1]), testCreationOrder); + const nested = jobBody(workflow, 'nested-cli').match(/^ go-version: \[(.+)\]$/m); + assert.ok(nested, 'complete independent CLI SDK matrix required'); + assert.deepEqual([...nested[1].matchAll(/'([^']+)'/g)].map(match => match[1]), minors.slice(4)); + + const corpus = [...jobBody(workflow, 'stdlib-corpus').matchAll( + /^ - \{os: ([^,]+), .*go-version: '([^']+)', .*targets: '([^']+)'\}$/gm)]; + assert.equal(corpus.length, 10, 'eight Linux SDK lanes and both native hosts remain required'); + assert.deepEqual(corpus.slice(0, 8).map(match => [match[1], match[2]]), + minors.map(version => ['ubuntu-latest', version])); + assert.deepEqual(corpus.slice(8).map(match => [match[1], match[2]]), + [['macos-latest', exactSDK], ['windows-latest', exactSDK]]); + + const complete = [...jobBody(workflow, 'module-completion').matchAll( + /^ - \{flavor: ([^,]+), go-version: '([^']+)'\}$/gm)]; + assert.deepEqual(complete.map(match => [match[1], match[2]]), + [...minors.map(version => ['test', version]), ['race', exactSDK], ['coverage', exactSDK]]); +} + +function requireCurrentFixture(workflow) { + const body = jobBody(workflow, 'test'); + const step = body.split(/(?=^ - )/m).find(step => + step.startsWith(' - name: Required exact downloaded AMD64 numerical fixture\n')); + assert.ok(step, 'current-only numerical owner must remain present'); + assert.ok(step.includes("if: runner.os == 'Linux' && matrix.go-version == '" + + exactSDK + "' && matrix.module-owner == 2\n"), + 'the exact current matrix must still activate its numerical owner'); + assert.match(step, /go test \.\/gotoolprofile -run="\$scope" -count=1 -timeout=10m/); +} + +function requireArtifactScope(workflow) { + for (const flavor of ['race', 'coverage']) { + assert.ok(jobBody(workflow, flavor).includes( + 'name: module-' + flavor + '-' + exactSDK + '-${{ matrix.module-role }}\n'), + flavor + ': artifact identity must match the exact completion matrix'); + } + assert.ok(jobBody(workflow, 'windows-module').includes( + 'name: test (windows-latest, ' + exactSDK + ', ${{ matrix.module }})\n')); + assert.doesNotMatch(workflow, /1\.27\.x/, 'no current floating SDK selector or stale scope remains'); +} + +function requireCurrentScope(workflow) { + requireSetups(workflow); + requireMatrices(workflow); + requireCurrentFixture(workflow); + requireArtifactScope(workflow); +} + +// Negative controls start from a complete positive fixture while the actual +// workflow is RED. This does not relax compiler-observation admission. +const fixture = source.replaceAll('1.27.x', exactSDK); + +test('formal current Go owners use the audited exact SDK', () => { + requireSetups(source); +}); + +test('all eight minor lanes and exact current completion scopes remain present', () => { + requireMatrices(source); +}); + +test('the exact current matrix still executes its required numerical fixture', () => { + requireCurrentFixture(source); +}); + +test('current artifacts and Windows labels identify the selected exact SDK', () => { + requireArtifactScope(source); +}); + +test('every current owner rejects a foreign patch or floating selector', () => { + requireCurrentScope(fixture); + for (const name of jobs) { + const before = jobBody(fixture, name); + for (const version of ['1.27.3', '1.27.x', '1.27', 'stable']) { + const after = before.replaceAll(exactSDK, version); + assert.notEqual(after, before, name + ': mutation must change the actual owner'); + assert.throws(() => requireCurrentScope(fixture.replace(before, () => after)), name + ': ' + version); + } + } +}); + +test('neither stale current-only conditions nor artifact names can borrow another pin', () => { + requireCurrentScope(fixture); + for (const [before, after] of [ + ["matrix.go-version == '1.27.2'", "matrix.go-version == '1.27.x'"], + ['module-race-1.27.2-', 'module-race-1.27.x-'], + ['module-coverage-1.27.2-', 'module-coverage-1.27.3-'], + ["{flavor: coverage, go-version: '1.27.2'}", "{flavor: coverage, go-version: '1.27.3'}"], + ]) { + assert.ok(fixture.includes(before), 'mutation must affect the actual workflow'); + assert.throws(() => requireCurrentScope(fixture.replace(before, after))); + } +}); diff --git a/.github/scripts/priority-diagnostic-qualification.test.cjs b/.github/scripts/priority-diagnostic-qualification.test.cjs new file mode 100644 index 000000000..54ab1d81d --- /dev/null +++ b/.github/scripts/priority-diagnostic-qualification.test.cjs @@ -0,0 +1,99 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); + +// Independent inventory for auditing historical diagnostic receipts. +const {qualifyDiagnosticNeeds} = require('./diagnostic-qualification.cjs'); +const allNeeds = [ + 'ownership', 'native-x86-oracles', 'sdk-priority', 'arm64-fallback-cross-priority', + 'lock-native-priority', 'unlocked-rmw-native-priority', 'vector-value-fault-priority', + 'packed-conversion-native-complete', 'packed-reduction-native-complete', 'complete-windows', +]; + +function successfulNeeds() { + return Object.fromEntries(allNeeds.map(name => [name, {result: 'success', outputs: {}}])); +} + +function clone(value) { + return JSON.parse(JSON.stringify(value)); +} + +test('only all ten genuine success results qualify; Windows cannot replace an original gate', () => { + const {qualifyDiagnosticNeeds} = require('./diagnostic-qualification.cjs'); + assert.deepEqual(qualifyDiagnosticNeeds(successfulNeeds()), {schema: 1, successfulJobs: allNeeds}); + for (const name of allNeeds) { + for (const result of ['failure', 'skipped', 'cancelled', 'queued', 'in_progress', '', true, null]) { + const needs = successfulNeeds(); + needs[name].result = result; + assert.throws(() => qualifyDiagnosticNeeds(needs), undefined, name + ': ' + result); + } + const missing = successfulNeeds(); + delete missing[name]; + assert.throws(() => qualifyDiagnosticNeeds(missing), undefined, 'missing ' + name); + for (const malformed of [null, [], 'success', {outputs: {result: 'success'}}]) { + const needs = successfulNeeds(); + needs[name] = malformed; + assert.throws(() => qualifyDiagnosticNeeds(needs), undefined, 'malformed ' + name); + } + } + for (const needs of [null, [], true, 'success', {}, + {...successfulNeeds(), extra: {result: 'success', outputs: {}}}]) { + assert.throws(() => qualifyDiagnosticNeeds(needs)); + } + const forged = clone(successfulNeeds()); + forged['native-x86-oracles'] = {result: 'failure', outputs: {result: 'success'}}; + forged['complete-windows'].outputs = {qualification: 'success'}; + assert.throws(() => qualifyDiagnosticNeeds(forged)); +}); + +test('the actual qualification command exits nonzero for failed, skipped, absent or malformed needs', () => { + const script = path.join(__dirname, 'diagnostic-qualification.cjs'); + const execute = input => { + const env = {...process.env}; + if (input === undefined) delete env.DIAGNOSTIC_NEEDS_JSON; + else env.DIAGNOSTIC_NEEDS_JSON = input; + return spawnSync(process.execPath, [script], {encoding: 'utf8', timeout: 5000, env}); + }; + const rejects = (input, label) => { + const failed = execute(input); + assert.equal(failed.error, undefined, label); + assert.equal(failed.signal, null, label); + assert.ok(Number.isInteger(failed.status) && failed.status > 0, label); + assert.equal(failed.stdout, '', label); + }; + const success = execute(JSON.stringify(successfulNeeds())); + assert.equal(success.error, undefined); + assert.equal(success.status, 0, success.stderr); + assert.deepEqual(JSON.parse(success.stdout), {schema: 1, successfulJobs: allNeeds}); + for (const name of allNeeds) { + for (const result of ['failure', 'skipped', 'cancelled']) { + const needs = successfulNeeds(); + needs[name].result = result; + rejects(JSON.stringify(needs), name + ': ' + result); + } + const missing = successfulNeeds(); + delete missing[name]; + rejects(JSON.stringify(missing), 'missing ' + name); + } + for (const input of [undefined, '', '{', 'null', '[]', '{}']) { + rejects(input, 'malformed actual context: ' + input); + } +}); + +test('early Windows completion cannot qualify a pending failed or absent original owner', () => { + const successful = () => Object.fromEntries(allNeeds.map(name => [name, {result: 'success', outputs: {}}])); + assert.deepEqual(qualifyDiagnosticNeeds(successful()), {schema: 1, successfulJobs: allNeeds}); + for (const name of allNeeds) { + for (const result of ['failure', 'cancelled', 'skipped', 'queued', 'in_progress']) { + const changed = successful(); + changed[name] = {result, outputs: {result: 'success', qualification: 'success'}}; + assert.throws(() => qualifyDiagnosticNeeds(changed), undefined, name + '/' + result); + } + const changed = successful(); + delete changed[name]; + assert.throws(() => qualifyDiagnosticNeeds(changed), undefined, 'absent ' + name); + } +}); diff --git a/.github/scripts/priority-duplicate-native-families.test.cjs b/.github/scripts/priority-duplicate-native-families.test.cjs new file mode 100644 index 000000000..c12ae729c --- /dev/null +++ b/.github/scripts/priority-duplicate-native-families.test.cjs @@ -0,0 +1,187 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const {auditDuplicateNativeFamilies} = require('./audit-duplicate-native-families.cjs'); + +// This denominator is independent of the production auditor's inventory. +const roots = [ + 'TestCrossLinuxRuntimeMatrixX86DuplicateMoveOriginalGoAndNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86DuplicateMoveOriginalGoAndNative386', +]; +const axes = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2', 'OriginalGo']; +const packageName = 'github.com/xgo-dev/plan9asm'; +const inventory = roots.join('\n') + '\nok\t' + packageName + '\n'; +const hash = 'a'.repeat(64), imageHash = 'b'.repeat(64); +const host = `IA32 direct host: linux/amd64 kernel=x86_64 CPUsha=${hash} mapsha=${hash}\n`; +const avx = 'actual guest CPUID/XCR0 preflight: AVX=1 AVX2=0 AVX512F=0\n'; +const staticImage = `native oracle static IA32 ELF bytes=4096 sha256=${imageHash}\n`; +const directImage = args => `IA32 direct ELF bytes=4096 sha256=${imageHash} argv=${args}\n`; +const sdk = args => `actual executable=/sdk/bin/go sha256=${hash} argv=${args}\n`; +const executable = (filename, args) => `actual executable=/tmp/owned/${filename} sha256=${imageHash} argv=${args}\n`; +const originalArgs = '["-test.run=^TestOriginal$" "-test.count=1" "-test.v"]'; +const compileArgs = '["test" "-p=1" "-vet=off" "-c" "-o" "/tmp/owned/original.test" "."]'; +const pipelineEvidence = arch => `duplicate native arch=${arch} live=60 dead=30 whole64-outcomes=5760 fault-processes=712; legacy original opcode/all mod16, E4NF/E5NF independent full-span references\n`; +const originalEvidence = 'actual original Go legacy/VEX: 18 rows × 16 IEEE/bit patterns × 6 masks = 1728 whole32/FLAGS/DF/MXCSR/canary outcomes; no EVEX execution claim\n'; +const event = (Action, Test, Output) => ({Action, Package: packageName, + ...(Test ? {Test} : {}), ...(Output !== undefined ? {Output} : {})}); +const serialize = events => events.map(value => JSON.stringify(value)).join('\n'); + +function completeEvents() { + const events = [event('start')]; + for (const root of roots) { + const arch = root.endsWith('386') ? '386' : 'amd64'; + events.push(event('run', root), event('output', root, host + avx)); + for (const axis of axes) { + const child = root + '/' + axis; + events.push(event('run', child)); + if (axis === 'OriginalGo') { + events.push(event('output', child, sdk('["version"]') + sdk(compileArgs))); + if (arch === '386') events.push(event('output', child, + staticImage + executable('original.test', originalArgs) + directImage(originalArgs))); + events.push(event('output', child, originalEvidence)); + } else { + if (arch === '386') events.push(event('output', child, + staticImage + executable('comparison', '[]') + directImage('[]') + staticImage)); + events.push(event('output', child, pipelineEvidence(arch))); + } + events.push(event('pass', child)); + } + events.push(event('output', root, host), event('pass', root)); + } + events.push(event('pass')); + return events; +} + +test('duplicate moves independently require two native roots and all four execution children', () => { + assert.deepEqual(auditDuplicateNativeFamilies(inventory, serialize(completeEvents())), + {roots: 2, children: 8, passed: 10, skipped: 0, incomplete: 0, + nativeHosts: 4, avxPreflights: 2, sdkBuilds: 2, staticIA32Images: 7, directIA32Executions: 4}); +}); + +test('duplicate moves reject every absent duplicated failed or skipped required terminal', () => { + for (const name of roots.flatMap(root => [root, ...axes.map(axis => root + '/' + axis)])) { + for (const Action of ['fail', 'skip']) { + assert.throws(() => auditDuplicateNativeFamilies(inventory, + serialize(completeEvents().map(value => value.Action === 'pass' && value.Test === name + ? {...value, Action} : value))), name + Action); + } + for (const Action of ['run', 'pass']) { + const events = completeEvents(); + const at = events.findIndex(value => value.Test === name && value.Action === Action); + assert.throws(() => auditDuplicateNativeFamilies(inventory, + serialize(events.filter((_, index) => index !== at))), name + '/missing ' + Action); + events.splice(at, 0, events[at]); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), name + '/duplicate ' + Action); + } + } +}); + +test('duplicate moves reject stale inventory malformed lifecycle foreign children and printed PASS', () => { + for (const changed of ['', inventory + roots[0] + '\n', inventory.replace(roots[0] + '\n', '')]) { + assert.throws(() => auditDuplicateNativeFamilies(changed, serialize(completeEvents()))); + } + for (const changed of [ + [event('output', undefined, '10 passed')], completeEvents().slice(1), completeEvents().slice(0, -1), + [event('start'), ...completeEvents()], [...completeEvents(), event('pass')], + [...completeEvents(), {Action: 'build-output', ImportPath: 'runtime', Output: 'late'}], + [event('run', roots[0] + '/objects-only'), ...completeEvents()], + completeEvents().map(value => ({...value, Package: 'foreign/package'})), + ]) assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(changed))); + for (const changed of [event('unknown'), event('output', roots[0], 'late')]) { + const events = completeEvents(); + events.splice(-1, 0, changed); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events))); + } + assert.throws(() => auditDuplicateNativeFamilies(inventory, '{not JSON}')); +}); + +test('duplicate moves require native host brackets AVX and complete per-axis evidence', () => { + for (const root of roots) { + const arch = root.endsWith('386') ? '386' : 'amd64'; + for (const needle of [host, avx]) { + for (const replacement of ['', needle.replace('linux/amd64', 'linux/arm64'), + needle.replace('AVX=1', 'AVX=0'), needle.replace(hash, 'invalid')]) { + if (replacement === needle) continue; + const events = completeEvents().map(value => value.Test === root && value.Output + ? {...value, Output: value.Output.replace(needle, replacement)} : value); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), needle); + } + } + for (const axis of axes) { + const evidence = axis === 'OriginalGo' ? originalEvidence : pipelineEvidence(arch); + for (const replacement of ['', evidence.replace('1728', '1727').replace('712', '711'), evidence + evidence]) { + const events = completeEvents().map(value => value.Test === root + '/' + axis && value.Output + ? {...value, Output: value.Output.replace(evidence, replacement)} : value); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), axis); + } + } + const events = completeEvents(); + const original = events.filter(value => value.Test === root + '/OriginalGo'); + const earlier = events.filter(value => value.Test !== root + '/OriginalGo'); + const at = earlier.findIndex(value => value.Test === root + '/' + axes[2] && value.Action === 'pass'); + earlier.splice(at, 0, ...original); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(earlier)), 'Go before final native axis'); + } +}); + +test('duplicate original Go requires actual SDK and 386 direct image identity in the OriginalGo child', () => { + for (const root of roots) { + const child = root + '/OriginalGo'; + for (const needle of [sdk('["version"]'), sdk(compileArgs)]) { + const events = completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replace(needle, '')} : value); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), needle); + } + const events = completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replace(sdk(compileArgs), sdk(compileArgs).replace(hash, imageHash))} : value); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), 'changed SDK identity'); + } + for (const axis of axes) { + const child = roots[1] + '/' + axis; + const args = axis === 'OriginalGo' ? originalArgs : '[]'; + for (const needle of [staticImage, directImage(args)]) { + const events = completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replace(needle, '')} : value); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), child + needle); + } + for (const replacement of ['bytes=0', 'bytes=-1', 'bytes=guest']) { + const events = completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replaceAll('bytes=4096', replacement)} : value); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), replacement); + } + const changedImage = completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replace(directImage(args), directImage(args).replace(imageHash, hash))} : value); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(changedImage)), 'changed actual image'); + } +}); + +test('duplicate actual executable receipts retain precise pipeline and original Go roles', () => { + auditDuplicateNativeFamilies(inventory, serialize(completeEvents())); + for (const axis of axes) { + const child = roots[1] + '/' + axis; + const original = axis === 'OriginalGo'; + const receipt = executable(original ? 'original.test' : 'comparison', original ? originalArgs : '[]'); + for (const replacement of ['', receipt + receipt, + receipt.replace(imageHash, hash), receipt.replace('sha256=' + imageHash, 'sha256=invalid'), + receipt.replace(original ? 'original.test' : 'comparison', 'unrelated'), + receipt.replace(original ? originalArgs : '[]', '["unobserved"]'), sdk(compileArgs)]) { + const events = completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replace(receipt, replacement)} : value); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), axis + '/foreign or absent executable'); + } + const events = completeEvents(); + const at = events.findIndex(value => value.Test === child && value.Output?.includes(receipt)); + events[at].Output = events[at].Output.replace(receipt, ''); + const first = events.findIndex(value => value.Test === child && value.Action === 'output'); + events.splice(first, 0, event('output', child, receipt)); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), axis + '/execution before inspected image'); + } + const events = completeEvents(); + const child = roots[1] + '/OriginalGo'; + const at = events.findIndex(value => value.Test === child && value.Output?.includes(sdk(compileArgs))); + events[at].Output = events[at].Output.replace(sdk(compileArgs), ''); + const pipeline = events.findIndex(value => value.Test === roots[1] + '/backend_O0' && value.Action === 'output'); + events.splice(pipeline, 0, event('output', roots[1] + '/backend_O0', sdk(compileArgs))); + assert.throws(() => auditDuplicateNativeFamilies(inventory, serialize(events)), 'pipeline cannot own original Go compilation'); +}); diff --git a/.github/scripts/priority-floating-compare.test.cjs b/.github/scripts/priority-floating-compare.test.cjs new file mode 100644 index 000000000..e4cd86207 --- /dev/null +++ b/.github/scripts/priority-floating-compare.test.cjs @@ -0,0 +1,33 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const {auditFloatingCompare, roots, children} = require('./audit-floating-compare.cjs'); + +test('floating execution auditing retains all result, dead, FP-field, alias and backend owners', () => { + const event = (Action, Test) => ({Action, Package: 'github.com/xgo-dev/plan9asm', ...(Test ? {Test} : {})}); + const events = [event('start')]; + for (const root of roots) { + events.push(event('run', root)); + for (const child of children.filter(child => child.startsWith(root + '/'))) { + events.push(event('run', child), event('pass', child)); + } + events.push(event('pass', root)); + } + events.push(event('pass')); + const raw = values => values.map(JSON.stringify).join('\n') + '\n'; + const inventory = roots.join('\n') + '\n'; + assert.deepEqual(auditFloatingCompare(inventory, raw(events)), + {roots:4, children:6, passed:10, skipped:0, incomplete:0}); + for (const name of [...roots, ...children]) { + assert.throws(() => auditFloatingCompare(inventory, raw(events.filter(event => event.Test !== name)))); + assert.throws(() => auditFloatingCompare(inventory, raw(events.map(event => + event.Test === name && event.Action === 'pass' ? {...event, Action:'skip'} : event)))); + assert.throws(() => auditFloatingCompare(inventory, raw(events.map(event => + event.Test === name ? {...event, Test:name+'-compile-only'} : event)))); + } + assert.throws(() => auditFloatingCompare(roots.slice(1).join('\n')+'\n', raw(events))); + assert.throws(() => auditFloatingCompare(inventory, raw([ + event('start'), {...event('output'), Output:'PASS '+roots.join('\nPASS ')}, event('pass'), + ]))); +}); diff --git a/.github/scripts/priority-floating-helpers.test.cjs b/.github/scripts/priority-floating-helpers.test.cjs new file mode 100644 index 000000000..d8be239b0 --- /dev/null +++ b/.github/scripts/priority-floating-helpers.test.cjs @@ -0,0 +1,90 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const {auditFloatingHelpers} = require('./audit-floating-helpers.cjs'); + +// Keep this denominator independent of the implementation/auditor exports. +const owners = [ + 'TestAMD64GetExpRuntimeSemantics', + 'TestAMD64GetMantRuntimeSemantics', + 'TestAMD64ScaleFRuntimeSemantics', + 'TestAMD64FloatingRangeRuntimeSemantics', + 'TestAMD64FloatingRangeDoubleAndScalarRuntimeSemantics', + 'TestAMD64FixupImmediateRuntimeSemantics', + 'TestAMD64ScaledRoundRuntimeSemantics', + 'TestAMD64FPClassRuntimeSemantics', +]; +const axes = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2']; +const packageName = 'github.com/xgo-dev/plan9asm'; +const event = (Action, Test, Output) => ({Action, Package: packageName, + ...(Test ? {Test} : {}), ...(Output ? {Output} : {})}); +const inventory = owners.join('\n') + '\nok\t' + packageName + '\n'; +const native = 'AVX512F=1 AVX512DQ=1 AVX512VL=1 OSXSAVE=1 XCR0=e7\nFLOATING_HELPER_NATIVE_DQ_OK\n'; + +function completeEvents() { + const events = [event('start')]; + for (const owner of owners) { + events.push(event('run', owner), event('output', owner, native)); + for (const axis of axes) { + events.push(event('run', owner + '/' + axis), event('pass', owner + '/' + axis)); + } + events.push(event('pass', owner)); + } + events.push(event('pass')); + return events; +} +const serialize = events => events.map(value => JSON.stringify(value)).join('\n'); + +test('native helper audit requires all eight originals and 24 execution children', () => { + assert.deepEqual(auditFloatingHelpers(inventory, serialize(completeEvents())), + {roots: 8, children: 24, passed: 32, skipped: 0, incomplete: 0, preflights: 8}); +}); + +test('native helper audit rejects every absent, failed, skipped or incomplete owner/axis', () => { + const expected = owners.flatMap(owner => [owner, ...axes.map(axis => owner + '/' + axis)]); + for (const name of expected) { + const original = completeEvents(); + for (const action of ['fail', 'skip']) { + const changed = original.map(value => value.Action === 'pass' && value.Test === name + ? {...value, Action: action} : value); + assert.throws(() => auditFloatingHelpers(inventory, serialize(changed)), name + '/' + action); + } + assert.throws(() => auditFloatingHelpers(inventory, + serialize(original.filter(value => value.Test !== name))), 'missing ' + name); + assert.throws(() => auditFloatingHelpers(inventory, + serialize(original.filter(value => value.Action !== 'pass' || value.Test !== name))), 'unfinished ' + name); + const started = original.find(value => value.Action === 'run' && value.Test === name); + assert.throws(() => auditFloatingHelpers(inventory, + serialize([started, ...original])), 'duplicate ' + name); + } +}); + +test('native helper audit rejects missing, late or incomplete actual CPU/opcode preflight', () => { + for (const owner of owners) { + for (const text of ['', native.replace('DQ=1', 'DQ=0'), native.replace('F=1', 'F=0'), + native.replace('VL=1', 'VL=0'), native.replace('OSXSAVE=1', 'OSXSAVE=0'), + native.replace('XCR0=e7', 'XCR0=7'), native.replace('FLOATING_HELPER_NATIVE_DQ_OK', 'object-only')]) { + const changed = completeEvents().map(value => value.Action === 'output' && value.Test === owner + ? {...value, Output: text} : value); + assert.throws(() => auditFloatingHelpers(inventory, serialize(changed)), owner); + } + const changed = completeEvents(); + const at = changed.findIndex(value => value.Action === 'output' && value.Test === owner); + const [late] = changed.splice(at, 1); + changed.splice(at + 2, 0, late); + assert.throws(() => auditFloatingHelpers(inventory, serialize(changed)), 'late ' + owner); + } +}); + +test('native helper audit rejects changed inventories, foreign paths and printed success', () => { + for (const changed of ['', inventory + owners[0] + '\n', inventory.replace(owners[0] + '\n', '')]) { + assert.throws(() => auditFloatingHelpers(changed, serialize(completeEvents()))); + } + for (const changed of [ + [event('output', undefined, '32 passed')], + completeEvents().slice(0, -1), + [event('run', owners[0] + '/sample'), ...completeEvents()], + [...completeEvents(), event('pass')], + ]) assert.throws(() => auditFloatingHelpers(inventory, serialize(changed))); +}); diff --git a/.github/scripts/priority-go-installation.test.cjs b/.github/scripts/priority-go-installation.test.cjs new file mode 100644 index 000000000..a7b7cb299 --- /dev/null +++ b/.github/scripts/priority-go-installation.test.cjs @@ -0,0 +1,218 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); +const {test} = require('node:test'); + +const action = fs.readFileSync('.github/actions/setup-go/action.yml', 'utf8'); + +test('Go setup keeps writable local caches without accessing the account cache quota', () => { + assert.match(action, /^ cache: false$/m); + assert.doesNotMatch(action, /^ cache:/m); + for (const name of ['go-ci.yml', 'windows-module.yml', 'runtime-integration.yml', 'linux-cli-module.yml']) { + const source = fs.readFileSync(path.join('.github/workflows', name), 'utf8'); + const steps = source.split(/(?=^ - )/m).filter(step => + /uses: \.\/\.github\/actions\/setup-go\n/.test(step)); + assert.ok(steps.length > 0, name); + for (const step of steps) { + assert.doesNotMatch(step, /^ cache:/m, name); + } + } + for (const variable of ['GOPATH', 'GOBIN', 'GOMODCACHE', 'GOCACHE']) { + assert.match(action, new RegExp('echo "' + variable + '=\\$\\{RUNNER_TEMP\\}/')); + } +}); + +test('formal Go setup has its own bounded installation deadline', () => { + for (const name of ['go-ci.yml', 'windows-module.yml', 'runtime-integration.yml', 'linux-cli-module.yml']) { + const source = fs.readFileSync(path.join('.github/workflows', name), 'utf8'); + const steps = source.split(/(?=^ - )/m).filter(step => + /uses: \.\/\.github\/actions\/setup-go\n/.test(step)); + assert.ok(steps.length > 0, name); + for (const step of steps) { + assert.match(step, /^ timeout-minutes: 10$/m, + name + ': downloads cannot consume the complete test job budget'); + } + } +}); + +function verificationScript() { + const step = action.split(/(?=^ - name:)/m).find(part => + part.startsWith(' - name: Verify Go installation\n')); + assert.ok(step, 'setup must verify the actual Go command after the installer'); + assert.match(step, /REQUESTED_GO_VERSION: \$\{\{ inputs\.go-version \}\}/); + assert.match(step, /INSTALLED_GO_VERSION: \$\{\{ steps\.install_go\.outputs\.go-version \}\}/); + assert.doesNotMatch(step, /continue-on-error:|if:|\|\| true/); + const start = step.indexOf(' run: |\n'); + assert.ok(start >= 0, 'the actual verification script is required'); + return step.slice(start + ' run: |\n'.length) + .split('\n').map(line => line.startsWith(' ') ? line.slice(8) : line) + .join('\n'); +} + +function verify(options = {}) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'plan9asm-go-install-')); + try { + if (!options.missingGo) { + fs.writeFileSync(path.join(directory, 'go'), `#!/bin/sh +[ "$GOTOOLCHAIN" = local ] || exit 93 +case "$*" in + 'env GOVERSION') printf 'go%s\\n' "$MOCK_GO_VERSION" ;; + 'version') printf 'go version go%s linux/amd64\\n' "$MOCK_GO_VERSION" ;; + *) exit 92 ;; +esac +`, {mode: 0o755}); + } + return spawnSync('/bin/bash', ['--noprofile', '--norc', '-c', verificationScript()], { + encoding: 'utf8', + timeout: 5000, + env: { + PATH: directory, + REQUESTED_GO_VERSION: options.requested ?? '1.27.2', + RESOLVED_GO_VERSION: options.resolved ?? options.installed ?? '1.27.2', + INSTALLED_GO_VERSION: options.installed ?? '1.27.2', + MOCK_GO_VERSION: options.actual ?? '1.27.2', + GOTOOLCHAIN: 'auto', + }, + }); + } finally { + fs.rmSync(directory, {recursive: true, force: true}); + } +} + +function resolve(requested, toolCache = 'unused-runner-cache') { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'plan9asm-go-resolve-')); + try { + const step = action.split(/(?=^ - name:)/m).find(part => + part.startsWith(' - name: Resolve exact Go version\n')); + assert.ok(step, 'the installer needs a deterministic SDK resolver'); + const start = step.indexOf(' run: |\n'); + assert.ok(start >= 0); + const script = step.slice(start + ' run: |\n'.length) + .split('\n').map(line => line.startsWith(' ') ? line.slice(8) : line) + .join('\n'); + const output = path.join(directory, 'output'); + fs.writeFileSync(output, ''); + const result = spawnSync('/bin/bash', ['--noprofile', '--norc', '-c', script], { + encoding: 'utf8', + timeout: 5000, + env: { + PATH: directory, + REQUESTED_GO_VERSION: requested, + RUNNER_TOOL_CACHE: toolCache, + GITHUB_OUTPUT: output, + }, + }); + return {...result, output: fs.readFileSync(output, 'utf8')}; + } finally { + fs.rmSync(directory, {recursive: true, force: true}); + } +} + +test('setup verifies a real selected command without downloading another toolchain', () => { + assert.match(action, /- name: Set up Go\n id: install_go\n uses: actions\/setup-go@v7/); + assert.match(verificationScript(), /GOTOOLCHAIN=local go env GOVERSION/); + const result = verify(); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Verified Go 1\.27\.2/); +}); + +test('a runner cache cannot substitute another patch for the frozen SDK', () => { + const result = verify({ + requested: '1.26.x', resolved: '1.26.10', + installed: '1.26.9', actual: '1.26.9', + }); + assert.ifError(result.error); + assert.notEqual(result.status, 0, 'the actual mixed-SDK CI group must fail closed'); + assert.doesNotMatch(result.stdout, /Verified Go/); +}); + +test('the installer receives the same repository-frozen exact SDK in every job', () => { + const resolver = action.split(/(?=^ - name:)/m).find(part => + part.startsWith(' - name: Resolve exact Go version\n')); + assert.ok(resolver, 'wildcard SDK labels need one deterministic tracked mapping'); + assert.ok(action.indexOf(resolver) < action.indexOf(' - name: Set up Go\n')); + const installer = action.split(/(?=^ - name:)/m).find(part => + part.startsWith(' - name: Set up Go\n')); + assert.match(installer, /go-version: \$\{\{ steps\.resolve_go\.outputs\.go-version \}\}/); + assert.match(action, /RESOLVED_GO_VERSION: \$\{\{ steps\.resolve_go\.outputs\.go-version \}\}/); +}); + +test('every compatibility selector is independent of runner cache and release timing', () => { + const pins = { + '1.20.x': '1.20.14', '1.21.x': '1.21.13', + '1.22.x': '1.22.12', '1.23.x': '1.23.12', + '1.24.x': '1.24.13', '1.25.x': '1.25.14', + '1.26.x': '1.26.10', '1.27.x': '1.27.3', + }; + for (const file of fs.readdirSync('.github/workflows').filter(name => name.endsWith('.yml'))) { + const workflow = fs.readFileSync(path.join('.github/workflows', file), 'utf8'); + for (const match of workflow.matchAll(/\b1\.[0-9]+\.x\b/g)) { + assert.ok(Object.hasOwn(pins, match[0]), file + ': missing frozen SDK selector'); + } + } + for (const [requested, version] of Object.entries(pins)) { + for (const cache of ['image-with-old-sdk', 'image-with-new-sdk']) { + const result = resolve(requested, cache); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.output, `go-version=${version}\n`); + } + } +}); + +test('an explicit stable SDK is preserved without a floating lookup', () => { + for (const version of ['1.20.14', '1.24.11', '1.27.1', '1.27.2', '1.27.3']) { + const result = resolve(version); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.output, `go-version=${version}\n`); + } +}); + +test('unregistered or malformed SDK selectors fail before installation', () => { + for (const version of ['', 'stable', '1.28.x', '1.26', '1.26.*', '1.26.10rc1', + '1.26.x\n1.27.2', '1.26.x; echo unexpected']) { + const result = resolve(version); + assert.ifError(result.error); + assert.notEqual(result.status, 0, version); + assert.equal(result.output, ''); + } +}); + +test('historical minor selectors verify their own resolved stable patch', () => { + for (const [requested, installed] of [['1.20.x', '1.20.14'], ['1.26.x', '1.26.10']]) { + const result = verify({requested, installed, actual: installed}); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + } +}); + +test('an interrupted installer cannot borrow a preinstalled Go or an empty output', () => { + for (const options of [ + {missingGo: true}, + {installed: ''}, + {installed: '1.27.1'}, + {actual: '1.27.1'}, + {actual: '1.28.0'}, + {installed: '1.27.2rc1', actual: '1.27.2rc1'}, + {installed: '1.27.2\n1.27.2'}, + {resolved: ''}, + {resolved: '1.27.2rc1'}, + {resolved: '1.27.2\n1.27.2'}, + {resolved: '1.27.1'}, + {actual: '1.27.2\n1.27.2'}, + {requested: '1.26.x'}, + {requested: ''}, + {requested: 'stable'}, + ]) { + const result = verify(options); + assert.ifError(result.error); + assert.notEqual(result.status, 0, JSON.stringify(options)); + assert.doesNotMatch(result.stdout, /Verified Go/); + } +}); diff --git a/.github/scripts/priority-integer-native-families.test.cjs b/.github/scripts/priority-integer-native-families.test.cjs new file mode 100644 index 000000000..882b245b0 --- /dev/null +++ b/.github/scripts/priority-integer-native-families.test.cjs @@ -0,0 +1,243 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const {auditIntegerNativeFamilies} = require('./audit-integer-native-families.cjs'); + +// Independent owner/axis inventory: do not import the auditor's denominator. +const owners = [ + 'TestCrossLinuxRuntimeMatrixX86MOVNTIOriginalFlagsAndFaults', + 'TestCrossLinuxRuntimeMatrixX86PackedAlignLegacyNativeFaults', + 'TestCrossLinuxRuntimeMatrixX86ImplicitDivideOriginalGoAndNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86ImplicitDivideOriginalGoAndNative386', + 'TestCrossLinuxRuntimeMatrixX86SegmentQueryOriginalFlagsAliasesAndFaults', + 'TestCrossLinuxRuntimeMatrixX86MoveHalfOriginalGoFlagsDFAliases', + 'TestCrossLinuxRuntimeMatrixX86ImplicitDivideLogicalFPOriginalGoAndNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86ImplicitDivideLogicalFPOriginalGoAndNative386', +]; +const axes = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2']; +const packageName = 'github.com/xgo-dev/plan9asm'; +const hash = 'a'.repeat(64); +const imageHash = 'b'.repeat(64); +const inventory = owners.join('\n') + '\nok\t' + packageName + '\n'; +const host = `IA32 direct host: linux/amd64 kernel=x86_64 CPUsha=${hash} mapsha=${hash}\n`; +const staticImage = `native oracle static IA32 ELF bytes=4096 sha256=${imageHash}\n`; +const directImage = args => `IA32 direct ELF bytes=4096 sha256=${imageHash} argv=${args}\n`; +const sdk = args => `actual executable=/sdk/bin/go sha256=${hash} argv=${args}\n`; +const originalArgs = '["-test.run=^TestOriginal$" "-test.count=1" "-test.v"]'; +const compileArgs = '["test" "-p=1" "-vet=off" "-c" "-o" "/tmp/owned/original.test" "."]'; +const segmentEvidence = '48 original-Go opcode rows × 16 inputs; full GP aliases, six status flags, canaries; 160 live/dead m16 fault processes\n'; +const halfPreflight = 'half native CPUID/XCR0 AVX512F=1; original EVEX transport opcode executed\n'; +const halfEvidence = 'half native full64/FLAGS/DF=1/canary outcomes=512; complete original STD/CLD source and all eight named half moves\n'; +const halfOriginal = 'half actual original Go full64/FLAGS/DF=1/canary outcomes=512\n'; +function logicalFPEvidence(owner, original = false) { + const arch = owner.endsWith('386') ? '386' : 'amd64'; + const rows = arch === '386' ? 6 : 8; + const values = arch === '386' ? 96 : 128; + const faults = arch === '386' ? 18 : 24; + return original + ? `logical FP actual original Go arch=${arch} rows=${rows} values=${values}; complete original FP divisor source and whole AX/DX/DF/canary observations\n` + : `logical FP native arch=${arch} rows=${rows} values=${values} dead-division-faults=${faults}; whole AX/DX, DF and canaries; zero/overflow and signed-min/-1 SIGFPE\n`; +} +const event = (Action, Test, Output) => ({Action, Package: packageName, + ...(Test ? {Test} : {}), ...(Output !== undefined ? {Output} : {})}); +const serialize = events => events.map(value => JSON.stringify(value)).join('\n'); + +function completeEvents() { + const events = [event('start')]; + for (const owner of owners) { + const observed = owner.includes('ImplicitDivide') || owner.includes('SegmentQuery') || owner.includes('MoveHalf'); + const ia32 = owner.endsWith('386'); + events.push(event('run', owner)); + if (observed) events.push(event('output', owner, host)); + if (owner.includes('MoveHalf')) events.push(event('output', owner, halfPreflight)); + for (const axis of axes) { + const child = owner + '/' + axis; + events.push(event('run', child)); + if (ia32) { + events.push(event('output', child, staticImage + directImage('[]') + staticImage)); + } + if (owner.includes('SegmentQuery')) events.push(event('output', child, segmentEvidence)); + if (owner.includes('MoveHalf')) events.push(event('output', child, halfEvidence)); + if (owner.includes('LogicalFP')) events.push(event('output', child, logicalFPEvidence(owner))); + events.push(event('pass', child)); + } + if (observed) { + events.push(event('output', owner, sdk('["version"]') + sdk(compileArgs))); + if (ia32) events.push(event('output', owner, staticImage + directImage(originalArgs))); + if (owner.includes('MoveHalf')) events.push(event('output', owner, halfOriginal)); + if (owner.includes('LogicalFP')) events.push(event('output', owner, logicalFPEvidence(owner, true))); + events.push(event('output', owner, host)); + } + events.push(event('pass', owner)); + } + events.push(event('pass')); + return events; +} + +test('native integer families require all eight roots and twenty-four complete execution axes', () => { + assert.deepEqual(auditIntegerNativeFamilies(inventory, serialize(completeEvents())), + {roots: 8, children: 24, passed: 32, skipped: 0, incomplete: 0, + nativeHosts: 12, sdkBuilds: 6, staticIA32Images: 14, directIA32Executions: 8}); +}); + +test('logical FP division requires every actual native and original Go completion', () => { + for (const owner of owners.filter(name => name.includes('LogicalFP'))) { + for (const name of [owner, ...axes.map(axis => owner + '/' + axis)]) { + const marker = logicalFPEvidence(owner, name === owner); + for (const replacement of ['', marker + marker, marker.replace('rows=', 'partial-rows='), + marker.replace('arch=', 'emulated-arch='), marker.replace('values=', 'ignored-values=')]) { + const changed = completeEvents().map(value => value.Test === name && value.Output + ? {...value, Output: value.Output.replace(marker, replacement)} : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed)), name); + } + const early = completeEvents(); + const at = early.findIndex(value => value.Test === name && value.Output?.includes(marker)); + const [completion] = early.splice(at, 1); + const beforeRun = early.findIndex(value => value.Test === name && value.Action === 'run'); + early.splice(beforeRun, 0, completion); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(early)), 'completion before run'); + const late = completeEvents(); + const terminal = late.findIndex(value => value.Test === name && value.Action === 'pass'); + late.splice(terminal + 1, 0, event('output', name, marker)); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(late)), 'completion after terminal'); + } + } +}); + +test('native integer families reject each missing, failed, skipped or duplicate terminal', () => { + const names = owners.flatMap(owner => [owner, ...axes.map(axis => owner + '/' + axis)]); + for (const name of names) { + const original = completeEvents(); + for (const Action of ['fail', 'skip']) { + assert.throws(() => auditIntegerNativeFamilies(inventory, + serialize(original.map(value => value.Action === 'pass' && value.Test === name + ? {...value, Action} : value))), name + '/' + Action); + } + for (const Action of ['run', 'pass']) { + const at = original.findIndex(value => value.Action === Action && value.Test === name); + assert.throws(() => auditIntegerNativeFamilies(inventory, + serialize(original.filter((_, index) => index !== at))), name + '/missing ' + Action); + const duplicate = original.slice(); + duplicate.splice(at, 0, original[at]); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(duplicate)), + name + '/duplicate ' + Action); + } + } +}); + +test('native integer families reject malformed packages, inventory changes and printed success', () => { + for (const changed of ['', inventory + owners[0] + '\n', inventory.replace(owners[0] + '\n', '')]) { + assert.throws(() => auditIntegerNativeFamilies(changed, serialize(completeEvents()))); + } + for (const changed of [ + [event('output', undefined, '16 passed')], + completeEvents().slice(1), + completeEvents().slice(0, -1), + [event('start'), ...completeEvents()], + [...completeEvents(), event('pass')], + [...completeEvents(), {Action: 'build-output', ImportPath: 'runtime', Output: 'late'}], + [event('run', owners[0] + '/objects-only'), ...completeEvents()], + completeEvents().map(value => ({...value, Package: 'foreign/package'})), + ]) assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed))); + for (const value of [event('unknown'), event('output', owners[0], 'late output')]) { + const changed = completeEvents(); + changed.splice(-1, 0, value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed))); + } + assert.throws(() => auditIntegerNativeFamilies(inventory, '{not JSON}')); +}); + +test('division and segment native hosts and actual Go SDK builds bracket all three axes', () => { + for (const owner of owners.slice(2)) { + for (const replacement of ['', host.replace('linux/amd64', 'linux/arm64'), + host.replace('kernel=x86_64', 'kernel=aarch64'), host.replace(hash, 'not-a-digest')]) { + const changed = completeEvents().map(value => value.Test === owner && value.Output === host + ? {...value, Output: replacement} : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed)), owner); + } + for (const needle of ['["version"]', compileArgs]) { + const changed = completeEvents().map(value => value.Test === owner && value.Output + ? {...value, Output: value.Output.replace(sdk(needle), '')} : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed)), needle); + } + const changed = completeEvents(); + const before = changed.findIndex(value => value.Test === owner && value.Output === host); + const [late] = changed.splice(before, 1); + changed.splice(before + 2, 0, late); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed)), 'late preflight'); + const wrongTool = completeEvents().map(value => value.Test === owner && value.Output + ? {...value, Output: value.Output.replaceAll('/sdk/bin/go', '/other/bin/go')} : value); + // A different actual SDK path is valid if both commands bind the same tool. + assert.doesNotThrow(() => auditIntegerNativeFamilies(inventory, serialize(wrongTool))); + const splitTool = completeEvents().map(value => value.Test === owner && value.Output + ? {...value, Output: value.Output.replace(sdk(compileArgs), sdk(compileArgs).replace(hash, imageHash))} + : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(splitTool)), 'changed SDK'); + const liveHostHashes = completeEvents(); + const lastHost = liveHostHashes.findLastIndex(value => value.Test === owner && value.Output === host); + liveHostHashes[lastHost] = {...liveHostHashes[lastHost], Output: host.replaceAll(hash, imageHash)}; + assert.doesNotThrow(() => auditIntegerNativeFamilies(inventory, serialize(liveHostHashes))); + const unboundArgs = completeEvents().map(value => value.Test === owner && value.Output + ? {...value, Output: value.Output.replace('"-p=1" "-vet=off"', '"-p=8"')} : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(unboundArgs)), 'changed build'); + const missingPost = completeEvents(); + const post = missingPost.findLastIndex(value => value.Test === owner && value.Output === host); + missingPost.splice(post, 1); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(missingPost)), 'missing final host'); + const earlyGo = completeEvents(); + const goAt = earlyGo.findIndex(value => value.Test === owner && value.Output?.includes('actual executable=')); + const [originalGo] = earlyGo.splice(goAt, 1); + const finalChild = earlyGo.findIndex(value => value.Test === owner + '/' + axes[2] && value.Action === 'pass'); + earlyGo.splice(finalChild, 0, originalGo); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(earlyGo)), 'Go before final axis'); + } +}); + +test('segment query axes require their complete actual opcode flags alias and fault evidence', () => { + const owner = owners[4]; + for (const axis of axes) { + for (const replacement of ['', segmentEvidence.replace('160', '159'), + segmentEvidence.replace('six status flags', 'ZF only')]) { + const changed = completeEvents().map(value => value.Test === owner + '/' + axis && value.Output + ? {...value, Output: value.Output.replace(segmentEvidence, replacement)} : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed)), axis); + } + } +}); + +test('native half moves require actual CPU original Go and each complete DF=1 axis', () => { + const owner = owners[5]; + for (const name of [owner, ...axes.map(axis => owner + '/' + axis)]) { + const needles = name === owner ? [halfPreflight, halfOriginal] : [halfEvidence]; + for (const needle of needles) { + for (const replacement of ['', needle + needle, needle.replace('=512', '=511').replace('AVX512F=1', 'AVX512F=0')]) { + if (replacement === needle) continue; + const changed = completeEvents().map(value => value.Test === name && value.Output + ? {...value, Output: value.Output.replace(needle, replacement)} : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed)), name); + } + } + } +}); + +test('division IA32 requires actual static and direct images in every axis and original Go', () => { + for (const owner of [owners[3], owners[7]]) { + const required = [owner, ...axes.map(axis => owner + '/' + axis)]; + for (const name of required) { + for (const needle of [staticImage, directImage(name === owner ? originalArgs : '[]')]) { + const changed = completeEvents().map(value => value.Test === name && value.Output + ? {...value, Output: value.Output.replace(needle, '')} : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed)), name + needle); + } + } + } + for (const replacement of ['bytes=0', 'bytes=-1', 'bytes=guest']) { + const changed = completeEvents().map(value => value.Output + ? {...value, Output: value.Output.replaceAll('bytes=4096', replacement)} : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(changed)), replacement); + } + const forged = completeEvents().map(value => value.Output + ? {...value, Output: value.Output.replaceAll('IA32 direct ELF', 'QEMU translated ELF')} : value); + assert.throws(() => auditIntegerNativeFamilies(inventory, serialize(forged))); +}); diff --git a/.github/scripts/priority-legacy-abs-sign-native.test.cjs b/.github/scripts/priority-legacy-abs-sign-native.test.cjs new file mode 100644 index 000000000..e469aa3cb --- /dev/null +++ b/.github/scripts/priority-legacy-abs-sign-native.test.cjs @@ -0,0 +1,152 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); + +// Independent execution denominator, not imported from the auditor. +const roots = ['TestCrossLinuxRuntimeMatrixX86LegacyAbsSignOriginalAlignmentFaultsAMD64', + 'TestCrossLinuxRuntimeMatrixX86LegacyAbsSignOriginalAlignmentFaults386']; +const axes = ['backend_O0', 'backend_O2', 'IR_O2_backend_O2']; +const operations = ['PABSB', 'PABSW', 'PABSD', 'PSIGNB', 'PSIGNW', 'PSIGND', + 'VPABSB', 'VPABSW', 'VPABSD', 'VPSIGNB', 'VPSIGNW', 'VPSIGND']; +const roles = ['live', 'dead']; +const children = roots.flatMap(root => axes.flatMap(axis => [root + '/' + axis, + ...operations.flatMap(op => roles.map(role => root + '/' + axis + '/' + op + '/' + role))])); +const inventory = roots.join('\n') + '\nok\tgithub.com/xgo-dev/plan9asm\n'; +const digest = 'a'.repeat(64); +const host = `IA32 direct host: linux/amd64 kernel=x86_64 CPUsha=${digest} mapsha=${digest}\n`; +const preflight = 'ABS/sign actual native SSSE3=1 AVX=1\n'; +const marker = 'ABS/sign original Go bytes and LLVM native alignment/access/CF/MXCSR=0x1f80/canary probes 20 failures 0\n'; +const image = `native oracle static IA32 ELF bytes=4096 sha256=${digest}\n`; +const sdk = arch => `actual LEA source/object receipt: SDK=go1.27.2 target=linux/${arch} source=${digest} driver=${digest} object=${digest}\n`; +const event = (Action, Test, Output) => ({Action, Package: 'github.com/xgo-dev/plan9asm', + ...(Test ? {Test} : {}), ...(Output !== undefined ? {Output} : {})}); +const serialize = events => events.map(value => JSON.stringify(value)).join('\n'); + +function completeEvents() { + const events = [event('start')]; + for (const root of roots) { + const arch = root.endsWith('386') ? '386' : 'amd64'; + events.push(event('run', root), event('output', root, host + preflight + sdk(arch))); + for (const axis of axes) { + const parent = root + '/' + axis; + events.push(event('run', parent)); + if (arch === '386') events.push(event('output', parent, image)); + for (const op of operations) for (const role of roles) { + const leaf = parent + '/' + op + '/' + role; + events.push(event('run', leaf), event('output', leaf, marker), event('pass', leaf)); + } + events.push(event('pass', parent)); + } + events.push(event('output', root, host), event('pass', root)); + } + events.push(event('pass')); + return events; +} + +function audit(events = completeEvents(), discovered = inventory) { + const filename = '.github/scripts/audit-legacy-abs-sign-native.cjs'; + assert.ok(fs.existsSync(filename), 'original legacy alignment faults lack their required exact native auditor'); + return require('./audit-legacy-abs-sign-native.cjs').auditLegacyAbsSignNative(discovered, serialize(events)); +} + +test('legacy ABS/sign native owner requires 144 complete live/dead kernel executions and 5760 direct probes', () => { + assert.deepEqual(audit(), {roots: 2, children: 150, passed: 152, skipped: 0, incomplete: 0, + nativeHosts: 4, sdkCaptures: 2, isaPreflights: 2, staticIA32Images: 3, + kernelExecutions: 144, directProbeProcesses: 5760}); +}); + +test('legacy ABS/sign rejects every missing duplicated failed skipped or unfinished lifecycle', () => { + audit(); + for (const name of [...roots, ...children]) { + for (const Action of ['fail', 'skip']) { + assert.throws(() => audit(completeEvents().map(value => value.Test === name && value.Action === 'pass' + ? {...value, Action} : value)), name + Action); + } + for (const Action of ['run', 'pass']) { + const original = completeEvents(); + const at = original.findIndex(value => value.Action === Action && value.Test === name); + assert.throws(() => audit(original.filter((_, index) => index !== at)), name + ' missing ' + Action); + original.splice(at, 0, original[at]); + assert.throws(() => audit(original), name + ' duplicate ' + Action); + } + const late = completeEvents(); + const at = late.findIndex(value => value.Action === 'pass' && value.Test === name); + late.splice(at + 1, 0, event('output', name, 'late output\n')); + assert.throws(() => audit(late), name + ' output after terminal'); + } +}); + +test('legacy ABS/sign rejects changed native inventory package lifecycle and printed summaries', () => { + audit(); + for (const discovered of ['', inventory + roots[0] + '\n', inventory.replace(roots[1], 'TestObjectOnly')]) { + assert.throws(() => audit(completeEvents(), discovered)); + } + for (const events of [completeEvents().slice(1), completeEvents().slice(0, -1), + [...completeEvents(), event('pass')], [event('start'), ...completeEvents()], + [event('output', undefined, '144 kernels passed')], + completeEvents().map(value => ({...value, Package: 'foreign/package'}))]) assert.throws(() => audit(events)); +}); + +test('legacy ABS/sign binds each actual 20-probe completion marker to its own live kernel', () => { + audit(); + for (const name of children.filter(name => name.endsWith('/live') || name.endsWith('/dead'))) { + for (const replacement of ['', marker + marker, marker.replace('20', '19'), marker.replace('failures 0', 'failures 1'), + marker.replace('MXCSR=0x1f80', 'MXCSR=inherited'), marker.replace('MXCSR=0x1f80', 'MXCSR=0x21f80'), + host + marker, preflight + marker, sdk('amd64') + marker, image + marker]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === name && value.Output + ? {...value, Output: value.Output.replace(marker, replacement)} : value)), name); + } + } + const misplaced = completeEvents().map(value => value.Test === roots[0] + '/backend_O0/PABSB/live' && value.Output + ? {...value, Test: roots[0] + '/backend_O0'} : value); + assert.throws(() => audit(misplaced), 'axis summary cannot replace an executed kernel'); +}); + +test('legacy ABS/sign requires actual host ISA and original SDK object before every axis', () => { + audit(); + for (const root of roots) { + const arch = root.endsWith('386') ? '386' : 'amd64'; + for (const needle of [host, preflight, sdk(arch)]) { + for (const replacement of ['', needle + needle, needle.replace('kernel=x86_64', 'kernel=aarch64'), + needle.replace('SSSE3=1', 'SSSE3=0'), needle.replace('AVX=1', 'AVX=0'), + needle.replace(digest, 'invalid'), needle.replace('linux/' + arch, 'linux/arm64')]) { + if (needle === replacement) continue; + assert.throws(() => audit(completeEvents().map(value => value.Test === root && value.Output + ? {...value, Output: value.Output.replace(needle, replacement)} : value)), needle); + } + } + const earlyPost = completeEvents(); + const post = earlyPost.findLastIndex(value => value.Test === root && value.Output === host); + const [observation] = earlyPost.splice(post, 1); + const beforeLast = earlyPost.findIndex(value => value.Test === root + '/' + axes[2] && value.Action === 'pass'); + earlyPost.splice(beforeLast, 0, observation); + assert.throws(() => audit(earlyPost), 'final native host must follow all complete axes'); + const earlyAxis = completeEvents(); + const first = earlyAxis.findIndex(value => value.Test === root && value.Output?.includes(preflight)); + const output = earlyAxis[first].Output; + earlyAxis[first] = event('output', root, host + preflight); + earlyAxis.splice(first + 2, 0, event('output', root, sdk(arch))); + assert.throws(() => audit(earlyAxis), 'SDK capture cannot follow an axis start'); + assert.ok(output.includes(sdk(arch))); + } +}); + +test('legacy ABS/sign every mode32 axis binds a complete static native image before its first kernel', () => { + audit(); + const root = roots[1]; + for (const axis of axes) { + const parent = root + '/' + axis; + for (const replacement of ['', image + image, image.replace('4096', '0'), + image.replace(digest, 'invalid'), image.replace('static IA32', 'QEMU translated')]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === parent && value.Output + ? {...value, Output: value.Output.replace(image, replacement)} : value)), parent); + } + const late = completeEvents(); + const at = late.findIndex(value => value.Test === parent && value.Output === image); + const [observation] = late.splice(at, 1); + late.splice(at + 2, 0, observation); + assert.throws(() => audit(late), 'IA32 image cannot be first inspected after execution'); + } +}); diff --git a/.github/scripts/priority-linux-cli-partitions.test.cjs b/.github/scripts/priority-linux-cli-partitions.test.cjs new file mode 100644 index 000000000..2884a2bd0 --- /dev/null +++ b/.github/scripts/priority-linux-cli-partitions.test.cjs @@ -0,0 +1,261 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const {spawnSync} = require('node:child_process'); +const test = require('node:test'); +const {jobBody, jobNeeds} = require('./workflow-needs.cjs'); + +const root = path.join(__dirname, '../..'); +const workflow = fs.readFileSync(path.join(root, '.github/workflows/go-ci.yml'), 'utf8'); +function optional(name) { + return fs.existsSync(path.join(root, name)) ? fs.readFileSync(path.join(root, name), 'utf8') : ''; +} +const nested = optional('.github/workflows/linux-cli-module.yml'); +const driver = optional('scripts/test-linux-cli-module-ci.sh'); +const route = "${{ github.repository_owner == 'xgo-dev' && " + + "fromJSON('[\"qiniu\", \"ubuntu-24.04-large\"]') || 'ubuntu-latest' }}"; +const identity = 'linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}'; +const directory = '_out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}'; + +function step(body, title) { + const selected = body.split(/(?=^ - )/m).filter(item => + item.startsWith(' - name: ' + title + '\n')); + assert.equal(selected.length, 1, 'one actual step: ' + title); + return selected[0]; +} + +function requireCalls(source) { + for (const [name, flavor, version] of [ + ['nested-cli', 'test', '${{ matrix.go-version }}'], + ['nested-cli-coverage', 'coverage', "'1.27.2'"], + ]) { + const body = jobBody(source, name); + assert.deepEqual(jobNeeds(source, name), ['ci_policy']); + assert.match(body, /^ uses: \.\/\.github\/workflows\/linux-cli-module\.yml$/m, + name + ': one serial full-package command still exhausts its shared binary deadline'); + assert.ok(body.includes(' module: ${{ matrix.module }}\n')); + assert.ok(body.includes(' go-version: ' + version + '\n')); + assert.ok(body.includes(' flavor: ' + flavor + '\n')); + assert.match(body, /^ module: \[plan9asm, plan9asmll\]$/m); + assert.match(body, /^ fail-fast: false$/m); + assert.doesNotMatch(body, /^ if:|continue-on-error:|max-parallel:|\bgo test\b|^ steps:/m); + } + assert.match(jobBody(source, 'nested-cli'), + /^ go-version: \['1\.24\.x', '1\.25\.x', '1\.26\.x', '1\.27\.2'\]$/m); +} + +function requireRoles(source) { + for (const input of ['module', 'go-version', 'flavor']) { + assert.ok(source.includes(' ' + input + ':\n required: true\n type: string\n')); + } + for (const name of ['root-parts', 'complete']) { + const body = jobBody(source, name); + assert.ok(body.includes(' runs-on: ' + route + '\n'), name + ': same explicit runner route'); + assert.doesNotMatch(body, /continue-on-error:|max-parallel:|\|\|\s*true/); + assert.match(body, new RegExp('^ timeout-minutes: ' + (name === 'complete' ? 30 : 60) + '$', 'm')); + const setup = step(body, 'Setup Go'); + assert.match(setup, /^ timeout-minutes: 10$/m); + assert.match(setup, /^ uses: \.\/\.github\/actions\/setup-go$/m); + assert.match(setup, /^ go-version: \$\{\{ inputs\.go-version \}\}$/m); + assert.doesNotMatch(setup, /^ cache:/m); + const compiler = step(body, 'Install LLVM 22 and cross compilers'); + assert.match(compiler, /^ timeout-minutes: 14$/m); + assert.match(compiler, /^ run: bash scripts\/install-ci-llvm22-ubuntu\.sh gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu$/m); + const qemu = step(body, 'Install required execution runtimes'); + assert.match(qemu, /^ timeout-minutes: 5$/m); + assert.match(qemu, /^ run: bash scripts\/install-ci-qemu\.sh$/m); + const node = step(body, 'Setup required wasm execution runtime'); + assert.match(node, /^ timeout-minutes: 10$/m); + assert.match(node, /^ uses: \.\/\.github\/actions\/setup-node22$/m); + assert.doesNotMatch(node, /node-version:|uses: actions\/setup-node@/); + for (const item of [setup, compiler, qemu, node]) { + assert.doesNotMatch(item, /^ if:|continue-on-error:/m); + } + assert.ok(body.indexOf(compiler) < body.indexOf(qemu)); + assert.ok(body.indexOf(qemu) < body.indexOf(node)); + } + const roots = jobBody(source, 'root-parts'); + assert.doesNotMatch(roots, /^ if:/m); + assert.match(roots, /^ part: \[0, 1, 2, 3\]$/m); + assert.match(roots, /^ fail-fast: false$/m); + assert.match(roots, /test-linux-cli-module-ci\.sh \$\{\{ inputs\.module \}\} \$\{\{ inputs\.flavor \}\} execute 0 \$\{\{ matrix\.part \}\}/); + assert.doesNotMatch(step(roots, 'Execute disjoint root assignment'), /^ if:/m); + const retain = step(roots, 'Retain actual root evidence'); + assert.match(retain, /^ if: \$\{\{ always\(\) \}\}$/m); + assert.match(retain, /^ uses: actions\/upload-artifact@v6$/m); + assert.match(retain, /^ if-no-files-found: error$/m); + assert.ok(retain.includes(' name: ' + identity + '-root-${{ matrix.part }}\n')); + assert.ok(retain.includes(' path: ' + directory + '-root-${{ matrix.part }}/\n')); + for (const [title, owner] of [['profile', 1], ['remaining-package', 2]]) { + const execution = step(roots, 'Execute complete ' + title + ' owner'); + assert.match(execution, /^ if: \$\{\{ !cancelled\(\) && matrix\.part == 0 \}\}$/m); + assert.ok(execution.includes(' run: bash scripts/test-linux-cli-module-ci.sh ' + + '${{ inputs.module }} ${{ inputs.flavor }} execute ' + owner + ' -1\n')); + const artifact = step(roots, 'Retain actual ' + title + ' evidence'); + assert.match(artifact, /^ if: \$\{\{ always\(\) && matrix\.part == 0 \}\}$/m); + assert.match(artifact, /^ uses: actions\/upload-artifact@v6$/m); + assert.match(artifact, /^ if-no-files-found: error$/m); + assert.ok(artifact.includes(' name: ' + identity + '-owner-' + owner + '\n')); + assert.ok(artifact.includes(' path: ' + directory + '-owner-' + owner + '/\n')); + } + const complete = jobBody(source, 'complete'); + assert.deepEqual(jobNeeds(source, 'complete'), ['root-parts']); + assert.match(complete, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + for (let part = 0; part < 4; part++) { + const download = step(complete, 'Download root part ' + part); + assert.match(download, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + assert.match(download, /^ uses: actions\/download-artifact@v8$/m); + assert.ok(download.includes(' name: ' + identity + '-root-' + part + '\n')); + assert.ok(download.includes(' path: ' + directory + '-owners/owner-0000/part-000' + part + '\n')); + } + for (const [title, owner] of [['Download profile owner', 1], ['Download remaining-package owner', 2]]) { + const download = step(complete, title); + assert.match(download, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + assert.match(download, /^ uses: actions\/download-artifact@v8$/m); + assert.ok(download.includes(' name: ' + identity + '-owner-' + owner + '\n')); + assert.ok(download.includes(' path: ' + directory + '-owners/owner-000' + owner + '\n')); + } + const verify = step(complete, 'Verify complete CLI module'); + assert.match(verify, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + assert.match(verify, /test-linux-cli-module-ci\.sh \$\{\{ inputs\.module \}\} \$\{\{ inputs\.flavor \}\} complete/); + assert.doesNotMatch(complete, /merge-multiple:|pattern:|codecov\/codecov-action/); + const success = step(complete, 'Require successful CLI owners'); + assert.match(success, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + assert.match(success, /^ run: test '\$\{\{ needs\.root-parts\.result \}\}' = success$/m); + const summary = step(complete, 'Show complete CLI coverage'); + assert.match(summary, /^ if: inputs\.flavor == 'coverage'$/m); + assert.match(summary, /^ run: go tool cover -func=coverage-\$\{\{ inputs\.module \}\}\.out \| tail -n 1$/m); + assert.match(summary, /^ working-directory: cmd\/\$\{\{ inputs\.module \}\}$/m); + assert.ok(complete.indexOf(verify) < complete.indexOf(summary)); + assert.ok(complete.indexOf(summary) < complete.indexOf(success)); +} + +function requireDriver(source) { + for (const text of ['plan9asm|plan9asmll)', 'test) flags=()', + 'coverage) flags=("-coverprofile=coverage-$module.out")', + '-module', '-dir=cmd/$module', '-shards=8', '-workers=2', '-timeout=30m', + '-module-root-partitions=4', '-verify-module=_out/linux-cli-$module-$flavor-owners', + 'GOFLAGS= GOENV=off GOWORK=off GOTOOLCHAIN=local GOMAXPROCS=2', + 'go run ./cmd/plan9asmtest']) { + assert.ok(source.includes(text), 'complete original module protocol: ' + text); + } + assert.doesNotMatch(source, /(?:^|\s)-(?:run|skip|short|list)(?:=|\s)|timeout=(?:45m|60m)|\|\|\s*true/); +} + +test('all CLI SDK/module invocations require strict complete partition workflows', () => requireCalls(workflow)); +test('each Linux CLI fragment and fresh aggregate owns exact tools, budgets and evidence', () => requireRoles(nested)); +test('Linux CLI driver retains full inventory, original flags and exact module protocol', () => requireDriver(driver)); + +test('closed ownership rejects missing pieces, skipped collection and borrowed coverage', () => { + requireCalls(workflow); + requireRoles(nested); + requireDriver(driver); + for (const [before, after] of [ + ['part: [0, 1, 2, 3]', 'part: [0, 1, 2]'], + ['execute 2 -1', 'execute 1 -1'], + ['needs: root-parts', 'needs: unrelated'], + [' if: ${{ !cancelled() }}', ' if: success()'], + ['timeout-minutes: 60', 'timeout-minutes: 90'], + ['-root-3\n', '-root-2\n'], + ['part-0003\n', 'part-0002\n'], + ['uses: ./.github/actions/setup-node22', 'uses: actions/setup-node@v6'], + ['timeout-minutes: 10', 'timeout-minutes: 30'], + [' gcc-aarch64-linux-gnu', ''], + [' gcc-i686-linux-gnu', ''], + [' libc6-dev-i386-cross', ''], + ['install-ci-qemu.sh', 'install-unpinned-qemu.sh'], + ['install-ci-llvm22-ubuntu.sh', 'install-llvm23.sh'], + [route, "'ubuntu-latest'"], + [identity, 'linux-cli-wrong-module'], + [directory + '-root-${{ matrix.part }}/', directory + '-all-roots/'], + ["test '${{ needs.root-parts.result }}' = success", 'true'], + ['!cancelled() && matrix.part == 0', 'success() && matrix.part == 0'], + ['inputs.flavor == \'coverage\'', 'false'], + ]) { + const changed = nested.replace(before, () => after); + assert.notEqual(changed, nested, 'negative control must affect the actual source: ' + before); + assert.throws(() => requireRoles(changed), undefined, before); + } + for (const [before, after] of [ + ['-timeout=30m', '-timeout=45m'], ['-shards=8', '-shards=4'], + ['-module-root-partitions=4', '-module-root-partitions=2'], + ['-verify-module=', '-replay-unverified='], + ['-dir=cmd/$module', '-dir=cmd/plan9asm'], + ['go run ./cmd/plan9asmtest', 'go run ./cmd/plan9asmtest -run=TestOnly'], + ['go run ./cmd/plan9asmtest', 'go run ./cmd/plan9asmtest -skip=TestSlow'], + ['coverage-$module.out', 'coverage-root.out'], + ]) { + const changed = driver.replace(before, () => after); + assert.notEqual(changed, driver); + assert.throws(() => requireDriver(changed), undefined, before); + } +}); + +test('actual Linux CLI shell transports every module/flavor/owner and rejects partial grammar', t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'linux-cli-transport-')); + t.after(() => fs.rmSync(directory, {recursive: true, force: true})); + // This is explicitly a shell transport fixture, not Go or instruction execution. + fs.writeFileSync(path.join(directory, 'go'), '#!/bin/bash\n' + + 'printf "SYNTHETIC_ENV=%s/%s/%s/%s/%s\\n" "$GOFLAGS" "$GOENV" "$GOWORK" "$GOTOOLCHAIN" "$GOMAXPROCS"\n' + + 'printf "%s\\n" SYNTHETIC_GO "$@"\n', {mode: 0o700}); + const run = (...args) => spawnSync('bash', [path.join(root, 'scripts/test-linux-cli-module-ci.sh'), ...args], { + cwd: directory, + env: {...process.env, PATH: directory + path.delimiter + process.env.PATH, GOFLAGS: ''}, + encoding: 'utf8', timeout: 5000, + }); + for (const module of ['plan9asm', 'plan9asmll']) { + for (const flavor of ['test', 'coverage']) { + const outputs = new Set(); + for (const [owner, part] of [['0', '0'], ['0', '1'], ['0', '2'], ['0', '3'], ['1', '-1'], ['2', '-1']]) { + const actual = run(module, flavor, 'execute', owner, part); + assert.equal(actual.status, 0, actual.stderr || String(actual.error)); + const fields = actual.stdout.trim().split('\n'); + assert.ok(fields.includes('SYNTHETIC_ENV=/off/off/local/2')); + for (const field of ['run', './cmd/plan9asmtest', '-module', '-dir=cmd/' + module, + '-module-owner=' + owner, '-module-root-partition=' + part, '-shards=8', + '-workers=2', '-timeout=30m', '-module-root-partitions=4']) { + assert.ok(fields.includes(field), 'actual driver lost ' + field); + } + assert.equal(fields.includes('-coverprofile=coverage-' + module + '.out'), flavor === 'coverage'); + const output = fields.filter(field => field.startsWith('-out=')); + assert.equal(output.length, 1); + assert.ok(!outputs.has(output[0]), 'owners cannot share an output directory'); + outputs.add(output[0]); + } + if (flavor === 'coverage') { + assert.equal(run(module, flavor, 'complete').status, 1, 'missing complete profile must fail'); + const profile = path.join(directory, 'cmd', module, 'coverage-' + module + '.out'); + fs.mkdirSync(path.dirname(profile), {recursive: true}); + fs.writeFileSync(profile, 'synthetic profile, not coverage evidence\n'); + } + const complete = run(module, flavor, 'complete'); + assert.equal(complete.status, 0, complete.stderr); + assert.ok(complete.stdout.includes('-verify-module=_out/linux-cli-' + module + '-' + flavor + '-owners\n')); + for (const role of [ + ['execute', '0', '4'], ['execute', '0', '-1'], ['execute', '0', '01'], + ['execute', '0', '+1'], ['execute', '0', '1+1'], ['execute', '3', '-1'], + ['execute', '1', '0'], ['execute', '2', '0'], ['execute', '0'], + ['execute', '0', '0', 'extra'], ['complete', '0'], ['unknown'], + ]) { + const invalid = run(module, flavor, ...role); + assert.equal(invalid.status, 1, [module, flavor, ...role].join('/')); + assert.doesNotMatch(invalid.stdout, /SYNTHETIC_GO/); + } + } + } + for (const args of [['root', 'test', 'complete'], ['plan9asm', 'race', 'complete']]) { + const invalid = run(...args); + assert.equal(invalid.status, 1); + assert.doesNotMatch(invalid.stdout, /SYNTHETIC_GO/); + } + const flags = spawnSync('bash', [path.join(root, 'scripts/test-linux-cli-module-ci.sh'), + 'plan9asmll', 'test', 'execute', '0', '0'], { + cwd: directory, env: {...process.env, PATH: directory + path.delimiter + process.env.PATH, GOFLAGS: '-run=Only'}, + encoding: 'utf8', timeout: 5000, + }); + assert.equal(flags.status, 1); + assert.doesNotMatch(flags.stdout, /SYNTHETIC_GO/); +}); diff --git a/.github/scripts/priority-lock-native.test.cjs b/.github/scripts/priority-lock-native.test.cjs new file mode 100644 index 000000000..a4622ddb3 --- /dev/null +++ b/.github/scripts/priority-lock-native.test.cjs @@ -0,0 +1,39 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const roots = ['TestCrossLinuxRuntimeMatrixX86LockScalarRMWNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86LockScalarRMWNative386']; + +test('LOCK execution auditing requires original owners and all four completed backend children', () => { + const {auditNativeLock, roots: actual, children} = require('./audit-lock-native.cjs'); + assert.deepEqual(actual, roots); + assert.deepEqual(children, roots.flatMap(root => ['-O0', '-O2'].map(level => root + '/' + level))); + const event = (Action, Test) => ({Action, Package: 'github.com/xgo-dev/plan9asm', ...(Test ? {Test} : {})}); + const events = [event('start')]; + for (const root of roots) { + events.push(event('run', root)); + for (const child of children.filter(name => name.startsWith(root + '/'))) { + events.push(event('run', child), event('pass', child)); + } + events.push(event('pass', root)); + } + events.push(event('pass')); + const inventory = roots.join('\n') + '\n'; + const encode = values => values.map(JSON.stringify).join('\n') + '\n'; + assert.deepEqual(auditNativeLock(inventory, encode(events)), + {roots: 2, children: 4, passed: 6, skipped: 0, incomplete: 0}); + for (const child of children) { + for (const changed of [ + events.filter(value => value.Test !== child), + events.filter(value => value.Test !== child || value.Action !== 'pass'), + events.map(value => value.Test === child && value.Action === 'pass' ? {...value, Action: 'skip'} : value), + events.map(value => value.Test === child ? {...value, Test: child + '-object-only'} : value), + ]) assert.throws(() => auditNativeLock(inventory, encode(changed))); + } + for (const changed of [events.slice(0, -1), [...events, event('pass')], + [event('start'), {...event('output'), Output: encode(events)}, event('pass')]]) { + assert.throws(() => auditNativeLock(inventory, encode(changed))); + } +}); diff --git a/.github/scripts/priority-mask-vector-inputs.test.cjs b/.github/scripts/priority-mask-vector-inputs.test.cjs new file mode 100644 index 000000000..4821bb2dd --- /dev/null +++ b/.github/scripts/priority-mask-vector-inputs.test.cjs @@ -0,0 +1,33 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); + +test('portable fault audit requires both actual owners and every completed child', () => { + const {auditValueFaultPriority, roots, children} = require('./audit-value-fault-priority.cjs'); + const event = (Action, Test) => ({Action, Package: 'github.com/xgo-dev/plan9asm', ...(Test ? {Test} : {})}); + const events = [event('start')]; + for (const root of roots) { + events.push(event('run', root)); + for (const child of children.filter(child => child.startsWith(root+'/'))) { + events.push(event('run', child), event('pass', child)); + } + events.push(event('pass', root)); + } + events.push(event('pass')); + const inventory = roots.join('\n')+'\nok\tgithub.com/xgo-dev/plan9asm\n'; + const encode = values => values.map(value => JSON.stringify(value)).join('\n')+'\n'; + assert.deepEqual(auditValueFaultPriority(inventory, encode(events)), + {roots:2, children:8, passed:10, skipped:0, incomplete:0}); + for (const changed of [ + events.filter(value => value.Test !== children[0]), + events.filter(value => value.Test !== children[0] || value.Action !== 'pass'), + events.map(value => value.Test === children[0] && value.Action === 'pass' ? {...value, Action:'skip'} : value), + events.map(value => value.Test === children[0] && value.Action === 'pass' ? {...value, Action:'fail'} : value), + events.map(value => value.Test === children[0] ? {...value, Test:children[0]+'/sample'} : value), + events.slice(0,-1), + [...events, event('pass')], + [event('start'), {...event('output'), Output:'PASS '+children.join('\nPASS ')}, event('pass')], + ]) assert.throws(() => auditValueFaultPriority(inventory, encode(changed))); + assert.throws(() => auditValueFaultPriority(inventory.replace(roots[0], 'TestUnrelatedOwner'), encode(events))); +}); diff --git a/.github/scripts/priority-mmx-streaming.test.cjs b/.github/scripts/priority-mmx-streaming.test.cjs new file mode 100644 index 000000000..8a2393d71 --- /dev/null +++ b/.github/scripts/priority-mmx-streaming.test.cjs @@ -0,0 +1,26 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const {auditMMXStreaming, roots, children} = require('./audit-mmx-streaming.cjs'); + +test('MMX streaming audit rejects omitted, skipped or substituted original/backend execution', () => { + const event = (Action, Test) => ({Action, Package: 'github.com/xgo-dev/plan9asm', ...(Test ? {Test} : {})}); + const events = [event('start'), event('run', roots[0])]; + for (const child of children) events.push(event('run', child), event('pass', child)); + events.push(event('pass', roots[0]), event('pass')); + const encode = values => values.map(JSON.stringify).join('\n') + '\n'; + assert.deepEqual(auditMMXStreaming(roots[0] + '\n', encode(events)), + {roots:1, children:5, passed:6, skipped:0, incomplete:0}); + for (const child of children) { + for (const changed of [ + events.filter(value => value.Test !== child), + events.map(value => value.Test === child && value.Action === 'pass' ? {...value, Action:'skip'} : value), + events.map(value => value.Test === child ? {...value, Test:child+'-object-only'} : value), + ]) assert.throws(() => auditMMXStreaming(roots[0]+'\n', encode(changed))); + } + assert.throws(() => auditMMXStreaming('TestOtherOwner\n', encode(events))); + assert.throws(() => auditMMXStreaming(roots[0]+'\n', encode([ + event('start'), {...event('output'), Output:'PASS '+children.join('\nPASS ')}, event('pass'), + ]))); +}); diff --git a/.github/scripts/priority-module-root-parts.test.cjs b/.github/scripts/priority-module-root-parts.test.cjs new file mode 100644 index 000000000..6e5714337 --- /dev/null +++ b/.github/scripts/priority-module-root-parts.test.cjs @@ -0,0 +1,350 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const { spawnSync } = require('node:child_process'); +const { test } = require('node:test'); +const {requireJobNeeds} = require('./workflow-needs.cjs'); + +const workflow = fs.readFileSync( + path.join(__dirname, '../workflows/go-ci.yml'), 'utf8', +); +const script = fs.readFileSync( + path.join(__dirname, '../../scripts/test-module-ci.sh'), 'utf8', +); + +function job(name, source = workflow) { + const marker = `\n ${name}:\n`; + const start = source.indexOf(marker); + assert.notEqual(start, -1, `missing job ${name}`); + const body = source.slice(start + marker.length); + const end = body.search(/^ [a-zA-Z][a-zA-Z0-9_-]*:\s*$/m); + return end < 0 ? body : body.slice(0, end); +} + +function step(body, name) { + const marker = ` - name: ${name}\n`; + const start = body.indexOf(marker); + assert.notEqual(start, -1, `missing step ${name}`); + const end = body.indexOf('\n - name:', start + marker.length); + return body.slice(start, end < 0 ? body.length : end); +} + +test('every Unix root owner uses one two-shard wave with complete flavor-specific roles', () => { + for (const name of ['test', 'race', 'coverage']) { + const count = name === 'race' ? 32 : 16; + const roles = [ + ...Array.from({ length: count }, (_, index) => `root-${index}`), + 'profile', 'other', + ]; + const body = job(name); + const match = body.match(/^ module-role: \[([^\]]+)\]$/m); + assert.ok(match, `${name}: missing explicit module scheduling roles`); + assert.deepEqual(match[1].split(',').map(value => value.trim()), roles); + + const mappings = Array.from(body.matchAll( + /^ - \{module-role: (root-\d+|profile|other), module-owner: ([012]), root-partition: (-?\d+)\}$/gm, + ), ([, role, owner, part]) => [role, Number(owner), Number(part)]); + assert.deepEqual(mappings, [ + ...Array.from({ length: count }, (_, index) => [`root-${index}`, 0, index]), + ['profile', 1, -1], + ['other', 2, -1], + ], `${name}: role mapping repeats or loses package/shard ownership`); + assert.doesNotMatch(body, /^ module-owner: \[/m, + `${name}: Cartesian ownership repeats profile/rest`); + assert.match(body, /^ fail-fast: false$/m); + assert.doesNotMatch(body, /max-parallel:|continue-on-error:/); + } +}); + +test('root job fanout retains original workers, per-command and job budgets', () => { + assert.match(script, /test\) timeout=45m ;;/); + assert.match(script, /race\) timeout=60m; flags=\(-race\); root_partitions=32; shards=64 ;;/); + assert.match(script, /coverage\) timeout=45m; flags=\(-coverprofile=coverage-root\.out\) ;;/); + assert.match(script, /-shards=\$shards -workers=2/); + assert.match(script, /GOMAXPROCS=2/); + for (const [name, minutes] of [['test', 60], ['race', 75], ['coverage', 60]]) { + assert.match(job(name), new RegExp(`^ timeout-minutes: ${minutes}$`, 'm')); + } + assert.match(job('module-completion'), /^ timeout-minutes: 30$/m); +}); + +test('executor and completion both request the same explicit root partition count', () => { + assert.match(script, /^root_partitions=16$/m); + assert.match(script, /-module-root-partitions=\$root_partitions/); + assert.match(script, /-module-root-partition=\$4/); + assert.match(script, /\[\[ \$# != 4/); + for (const [name, flavor, title] of [ + ['test', 'test', 'Go test (Ubuntu)'], + ['race', 'race', 'Go test (race)'], + ['coverage', 'coverage', 'Go test with coverage'], + ]) { + assert.match(step(job(name), title), new RegExp( + `bash scripts/test-module-ci\\.sh ${flavor} execute \\$\\{\\{ matrix\\.module-owner \\}\\} \\$\\{\\{ matrix\\.root-partition \\}\\}`, + )); + } + assert.match(step(job('module-completion'), 'Check complete actual module inventory and coverage'), + /bash scripts\/test-module-ci\.sh \$\{\{ matrix\.flavor \}\} complete/); +}); + +test('actual shell dispatch validates flavor-specific part bounds before launching Go', t => { + // This is only an argument-transport fixture, never compiler/test evidence. + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'module-ci-args-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + fs.writeFileSync(path.join(directory, 'go'), + '#!/usr/bin/env bash\nprintf "%s\\n" SYNTHETIC_GO "$@"\n', { mode: 0o700 }); + const run = (...args) => spawnSync('bash', [ + path.join(__dirname, '../../scripts/test-module-ci.sh'), ...args, + ], { + cwd: directory, + env: { ...process.env, PATH: directory + path.delimiter + process.env.PATH, GOFLAGS: '' }, + encoding: 'utf8', + timeout: 5000, + }); + for (const [flavor, count, shards] of [['test', 16, 32], ['coverage', 16, 32], ['race', 32, 64]]) { + for (const args of [['execute', '0', String(count - 1)], ['execute', '2', '-1'], ['complete']]) { + const actual = run(flavor, ...args); + assert.equal(actual.status, 0, actual.stderr || String(actual.error)); + const fields = actual.stdout.trim().split('\n'); + assert.equal(fields[0], 'SYNTHETIC_GO'); + assert.ok(fields.includes(`-module-root-partitions=${count}`)); + assert.ok(fields.includes(`-shards=${shards}`)); + assert.ok(fields.includes('-workers=2')); + } + for (const part of [String(count), '-1', '01', '1+1', '999999999999999999999']) { + const actual = run(flavor, 'execute', '0', part); + assert.equal(actual.status, 1, `${flavor}/${part}: ${actual.stderr}`); + assert.doesNotMatch(actual.stdout, /SYNTHETIC_GO/); + } + } +}); + +test('each root part has an exact separate artifact and strict download destination', () => { + for (const name of ['test', 'race', 'coverage']) { + const body = job(name); + assert.match(body, /name: module-(?:test|race|coverage)-.*\$\{\{ matrix\.module-role \}\}/); + assert.match(body, /if-no-files-found: error/); + } + const aggregate = job('module-completion'); + for (let part = 0; part < 32; part++) { + const download = step(aggregate, `Download root part ${part}`); + assert.match(download, new RegExp( + `^ name: module-\\$\\{\\{ matrix\\.flavor \\}\\}-\\$\\{\\{ matrix\\.go-version \\}\\}-root-${part}$`, 'm', + )); + assert.match(download, new RegExp( + `^ path: _out/module-\\$\\{\\{ matrix\\.flavor \\}\\}-owners/owner-0000/part-${String(part).padStart(4, '0')}$`, 'm', + )); + if (part >= 16) { + assert.match(download, /if: \$\{\{ !cancelled\(\) && matrix\.flavor == 'race' \}\}/); + } else { + assert.doesNotMatch(download, /matrix\.flavor ==/); + } + } + assert.match(step(aggregate, 'Download profile owner'), /-profile$/m); + assert.match(step(aggregate, 'Download remaining-package owner'), /-other$/m); + assert.doesNotMatch(aggregate, /merge-multiple:|pattern:/); + requireJobNeeds(workflow, 'module-completion', ['test', 'nested-cli', 'race', 'coverage', 'nested-cli-coverage']); + assert.match(aggregate, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); +}); + +test('owner-two numerical fixture runs once and CLIs have independent complete modules', () => { + const body = job('test'); + assert.match(step(body, 'Required exact downloaded AMD64 numerical fixture'), /matrix\.module-owner == 2/); + assert.doesNotMatch(body, /nested_modules|working-directory: cmd\/plan9asm/); + requireJobNeeds(workflow, 'nested-cli', ['ci_policy']); + assert.match(job('nested-cli'), /^ module: \[plan9asm, plan9asmll\]$/m); + assert.doesNotMatch(body, /os: windows-latest/); + assert.match(job('windows-module'), /uses: \.\/\.github\/workflows\/windows-module\.yml/); + assert.match(body, + /go-version: \['1\.27\.2', '1\.20\.x', '1\.26\.x', '1\.25\.x', '1\.24\.x', '1\.23\.x', '1\.22\.x', '1\.21\.x'\]/); +}); + +test('selected native CI remains a separate sixteen-owner serial protocol', () => { + const body = job('cross-runtime'); + assert.match(body, /owner: \[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15\]/); + assert.match(body, /-ci-owner \$\{\{ matrix.owner \}\} -shards 16 -timeout 20m/); + assert.doesNotMatch(body, /module-root-partition|module-owner|(?:^|\s)-workers(?:=|\s)/); +}); + +test('Windows modules retain disjoint scheduling and complete coverage readback', () => { + const body = job('windows-module'); + assert.match(body, /^ needs: ci_policy$/m); + assert.match(body, /^ fail-fast: false$/m); + assert.match(body, /^ module: \[root, plan9asm, plan9asmll\]$/m); + assert.doesNotMatch(body, /max-parallel:|continue-on-error:/); + + const native = fs.readFileSync(path.join(__dirname, '../workflows/windows-module.yml'), 'utf8'); + const roots = job('root-parts', native); + assert.match(roots, /inputs.module == 'root' && '\[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\]' \|\| '\[0,1,2,3\]'/); + assert.match(roots, /execute 0 \$\{\{ matrix.part \}\}/); + const others = job('other-owners', native); + assert.match(others, /owner: \[1, 2\]/); + assert.match(others, /execute \$\{\{ matrix.owner \}\} -1/); + for (const owner of [roots, others]) { + assert.match(owner, /fail-fast: false/); + assert.match(owner, /timeout-minutes: 60/); + assert.match(owner, /if-no-files-found: error/); + assert.doesNotMatch(owner, /max-parallel:|continue-on-error:|codecov-action/); + } + const complete = job('complete', native); + requireJobNeeds(native, 'complete', ['root-parts', 'other-owners']); + assert.match(complete, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + assert.doesNotMatch(complete, /merge-multiple:|pattern:/); + assert.deepEqual(Array.from(complete.matchAll( + /^ - name: Download root part (\d+)$/gm, + ), ([, part]) => Number(part)), Array.from({ length: 16 }, (_, part) => part)); + for (let part = 0; part < 16; part++) { + const download = step(complete, `Download root part ${part}`); + assert.match(download, new RegExp(`-root-${part}$`, 'm')); + assert.match(download, new RegExp(`owner-0000/part-${String(part).padStart(4, '0')}$`, 'm')); + if (part >= 4) assert.match(download, /if: \$\{\{ inputs.module == 'root' \}\}/); + else assert.doesNotMatch(download, /if:/); + } + assert.match(step(complete, 'Download profile owner'), /owner-0001$/m); + assert.match(step(complete, 'Download remaining-package owner'), /owner-0002$/m); + assert.match(step(complete, 'Verify complete module and publish coverage'), + /test-windows-module-ci.sh \$\{\{ inputs.module \}\} complete/); + assert.match(step(complete, 'Upload verified complete Windows module coverage'), + /files: \.\/coverage-windows-\$\{\{ inputs.module \}\}\.out/); +}); + +function windowsModuleShellFixture(t) { + // Argument transport only; native Go execution is covered by the Windows + // module fixture, not by this synthetic command. + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'windows-module-ci-args-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const tools = Object.fromEntries(['go', 'llvm', 'node', 'python', 'jq'].map(name => { + const bin = path.join(directory, name, 'bin'); + fs.mkdirSync(bin, { recursive: true }); + return [name, bin]; + })); + fs.writeFileSync(path.join(tools.go, 'go'), `#!/bin/bash +if [[ "$*" == 'env GOHOSTOS' ]]; then + echo windows +elif [[ "$*" == 'env GOROOT' ]]; then + printf '%s\\n' '${path.dirname(tools.go)}' +else + printf 'SYNTHETIC_PATH=%s\\n' "$PATH" + printf '%s\\n' SYNTHETIC_GO "$@" +fi +`, { mode: 0o700 }); + const commands = { + [path.join(directory, 'cygpath')]: 'if [[ "$1" == -m ]]; then echo C:/fixture; else printf "%s\\n" "$2"; fi', + [path.join(tools.llvm, 'llvm-config')]: `if [[ "$1" == --version ]]; then echo 22.1.8; else echo '${tools.llvm}'; fi`, + [path.join(tools.node, 'node')]: `echo '${tools.node}'`, + [path.join(tools.python, 'python')]: `echo '${tools.python}'`, + [path.join(tools.jq, 'jq')]: 'exit 0', + }; + for (const [file, body] of Object.entries(commands)) { + fs.writeFileSync(file, `#!/usr/bin/env bash\n${body}\n`, { mode: 0o700 }); + } + const environment = {}; + const runAmbient = (ambient, flags, ...args) => spawnSync('bash', [ + path.join(__dirname, '../../scripts/test-windows-module-ci.sh'), ...args, + ], { + cwd: directory, + env: { + ...process.env, + PATH: [directory, ...Object.values(tools), ambient, process.env.PATH].join(path.delimiter), + GOFLAGS: flags, + LLVM_CONFIG: '', + SYSTEMROOT: path.join(directory, 'Windows'), + ...environment, + }, + encoding: 'utf8', + timeout: 5000, + }); + return { directory, tools, environment, runAmbient, run: (flags, ...args) => runAmbient('', flags, ...args) }; +} + +test('Windows shell transports complete module roles and rejects hidden build flags', t => { + const { directory, run } = windowsModuleShellFixture(t); + for (const [module, parts, shards] of [['root', 16, 64], ['plan9asm', 4, 8], ['plan9asmll', 4, 8]]) { + const rootRoles = Array.from({ length: parts }, (_, part) => ['execute', '0', String(part)]); + for (const args of [...rootRoles, ['execute', '1', '-1'], ['execute', '2', '-1']]) { + const actual = run('-tags=byollvm', module, ...args); + assert.equal(actual.status, 0, actual.stderr || String(actual.error)); + const fields = actual.stdout.trim().split('\n'); + assert.ok(fields.includes(`-module-owner=${args[1]}`)); + assert.ok(fields.includes(`-module-root-partition=${args[2]}`)); + for (const field of ['-module', '-timeout=30m', '-workers=2', '-tags=byollvm', + `-shards=${shards}`, `-module-root-partitions=${parts}`, `-coverprofile=C:/fixture/coverage-windows-${module}.out`]) { + assert.ok(fields.includes(field), `missing ${field}: ${actual.stdout}`); + } + assert.doesNotMatch(actual.stdout, /-run[=\s]|-skip[=\s]/); + } + const invalidParts = [String(parts), '-1', '-0', '+1', '01', '08', '1+1', '999999999999999999999']; + for (const args of [ + ...invalidParts.map(part => ['execute', '0', part]), + ['execute', '1', '0'], ['complete', '0'], + ]) { + const actual = run('-tags=byollvm', module, ...args); + assert.equal(actual.status, 1, `${module}/${args}: ${actual.stderr}`); + assert.doesNotMatch(actual.stdout, /SYNTHETIC_GO/); + } + const missing = run('-tags=byollvm', module, 'complete'); + assert.equal(missing.status, 1, 'completion without an actual profile must fail'); + fs.writeFileSync(path.join(directory, `coverage-windows-${module}.out`), 'fixture\n'); + const complete = run('-tags=byollvm', module, 'complete'); + assert.equal(complete.status, 0, complete.stderr); + assert.match(complete.stdout, new RegExp(`-verify-module=_out/windows-module-${module}-owners`)); + } + for (const flags of ['', '-tags=foreign', '-tags=byollvm -run=TestOnly']) { + const actual = run(flags, 'root', 'execute', '0', '0'); + assert.equal(actual.status, 1, actual.stderr); + assert.doesNotMatch(actual.stdout, /SYNTHETIC_GO/); + } +}); + +test('Windows module execution and aggregation exclude ambient Git and WSL paths', t => { + const { directory, tools, runAmbient } = windowsModuleShellFixture(t); + const paths = []; + for (const [git, wsl] of [['ucrt64', true], ['mingw64', false]]) { + const ambient = [path.join(directory, 'Git', git, 'bin')]; + if (wsl) ambient.push(path.join(directory, 'WSL')); + for (const role of [['execute', '0', '0'], ['complete']]) { + fs.writeFileSync(path.join(directory, 'coverage-windows-root.out'), 'fixture\n'); + const actual = runAmbient(ambient.join(path.delimiter), '-tags=byollvm', 'root', ...role); + assert.equal(actual.status, 0, actual.stderr); + const observed = actual.stdout.split('\n').find(line => line.startsWith('SYNTHETIC_PATH=')); + assert.ok(observed, 'the actual driver did not reach the synthetic command'); + paths.push(observed.slice('SYNTHETIC_PATH='.length)); + } + } + assert.equal(new Set(paths).size, 1, 'cross-owner tool search still depends on the hosted image'); + const selected = paths[0].split(path.delimiter); + for (const bin of Object.values(tools)) assert.ok(selected.includes(bin), `lost selected ${bin}`); + assert.doesNotMatch(paths[0], /ucrt64|mingw64|WSL/); + assert.deepEqual(selected, [ + tools.go, tools.llvm, tools.node, tools.python, tools.jq, + '/cmd', '/usr/bin', path.join(directory, 'Windows', 'System32'), path.join(directory, 'Windows'), + ]); +}); + +test('Windows module tool selection fails without falling back from explicit LLVM', t => { + const { directory, tools, environment, run } = windowsModuleShellFixture(t); + const configured = path.join(directory, 'explicit-llvm-config'); + environment.LLVM_CONFIG = configured; + for (const version of ['23.1.0', '21.1.8', '', '22.1.8\nforeign']) { + fs.writeFileSync(configured, `#!/usr/bin/env bash\nprintf '%s\\n' '${version}'\n`, { mode: 0o700 }); + const actual = run('-tags=byollvm', 'root', 'execute', '0', '0'); + assert.equal(actual.status, 1, actual.stderr); + assert.match(actual.stderr, /selected LLVM 22/); + assert.doesNotMatch(actual.stdout, /SYNTHETIC_GO/); + } + environment.LLVM_CONFIG = path.join(directory, 'missing-llvm-config'); + const missing = run('-tags=byollvm', 'root', 'execute', '0', '0'); + assert.notEqual(missing.status, 0); + assert.doesNotMatch(missing.stdout, /SYNTHETIC_GO/); + environment.LLVM_CONFIG = ''; + for (const name of ['node', 'python']) { + const file = path.join(tools[name], name); + const original = fs.readFileSync(file); + fs.writeFileSync(file, '#!/usr/bin/env bash\nexit 17\n', { mode: 0o700 }); + const actual = run('-tags=byollvm', 'root', 'execute', '0', '0'); + assert.equal(actual.status, 1, actual.stderr); + assert.match(actual.stderr, /one nonempty path/); + assert.doesNotMatch(actual.stdout, /SYNTHETIC_GO/); + fs.writeFileSync(file, original, { mode: 0o700 }); + } +}); diff --git a/.github/scripts/priority-native-auditor.test.cjs b/.github/scripts/priority-native-auditor.test.cjs new file mode 100644 index 000000000..e10afd64d --- /dev/null +++ b/.github/scripts/priority-native-auditor.test.cjs @@ -0,0 +1,300 @@ +const assert = require('node:assert/strict'); +const path = require('node:path'); +const test = require('node:test'); +const { auditNativePriority } = require('./audit-native-priority.cjs'); + +// Synthetic machine-event controls are not native execution receipts. +const inventories = { + 'static-read': ['TestX86RawReturnStaticReadNumericOracle'], + alias: ['TestCrossLinuxRuntimeMatrixX86MOVQSourceAliasesOriginalAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86MOVQSourceAliasesCountedProvenance'], + bool: ['TestCrossLinuxRuntimeMatrixAMD64RegisterCanonicalBoolInstructionBytes', + 'TestCrossLinuxRuntimeMatrixGoAMD64CanonicalBoolByteRepresentation'], + caller: ['TestCrossLinuxRuntimeMatrixX86CallerAddressInstructionBytesAndLLVM', + 'TestCrossLinuxRuntimeMatrixGoX86CallerAddressABI0Representation'], + 'caller-cmp': ['TestCrossLinuxRuntimeMatrixX86CallerAddressCMPInstructionBytesAndLLVM', + 'TestCrossLinuxRuntimeMatrixGoX86CallerAddressCMPABI0Representation'], + numerical: ['TestCrossLinuxRuntimeMatrixX86NumericalSBReadMutableAMD64', + 'TestCrossLinuxRuntimeMatrixX86NumericalSBReadMutableLocalAMD64'], + 'numerical-lea': ['TestCrossLinuxRuntimeMatrixX86NumericalLEAOriginalBytesAndLLVM'], + 'direct-tail': ['TestCrossLinuxRuntimeMatrixAMD64DirectTailWholeSourceOriginalGoAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86DirectTail386OriginalGoAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86DirectTailPCResultAliasesAndExternal'], + 'gp-simd-sparse': ['TestCrossLinuxRuntimeMatrixX86GPSIMDSparseNativeAMD64OriginalGoAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86GPSIMDSparseNative386OriginalGoAndLLVM'], + 'bit-scan': ['TestCrossLinuxRuntimeMatrixX86BitScanNativeAMD64OriginalGoAndLLVM', + 'TestCrossLinuxRuntimeMatrixX86BitScanNative386OriginalGoAndLLVM'], + 'bool-386': ['TestCrossLinuxRuntimeMatrixX86ABI0Mode32CanonicalBoolOriginalGoAndLLVM'], + 'packed-string': ['TestCrossLinuxRuntimeMatrixX86PackedStringImplicitVectorNativeAVX'], + 'independent-fp': ['TestCrossLinuxRuntimeMatrixX86InsertExtractIndependentFPFields', + 'TestCrossLinuxRuntimeMatrixX86LegacyPSIndependentFPFields'], + 'source-stack': ['TestCrossLinuxRuntimeMatrixX86RawSourceStackNumerical'], + continuation: ['TestCrossLinuxRuntimeMatrixX86ContinuationWholeCFGOriginalGoAndLLVM'], + 'packed-string-fault': ['TestCrossLinuxRuntimeMatrixX86PackedStringOriginalNativeMemoryFaults'], + 'byte-swap': ['TestCrossLinuxRuntimeMatrixX86ByteSwapOriginalGoAndLLVM'], + crc32: ['TestAMD64CRC32RuntimeSemanticsAndFlags'], + 'bit-test': ['TestCrossLinuxRuntimeMatrixX86BitTestNumericalAndAliases'], +}; +const sourceStack = inventories['source-stack'][0]; +const continuation = inventories.continuation[0]; +const packedStringFault = inventories['packed-string-fault'][0]; +const byteSwap = inventories['byte-swap'][0]; +const crc32 = inventories.crc32[0]; +const bitTest = inventories['bit-test'][0]; +const directTail386 = inventories['direct-tail'][1]; +const mode32Children = [ + ...['SI', 'BP', 'SI-stack', 'BP-stack'].map(carrier => directTail386 + '/' + carrier), + ...['BP-memory', 'BP-register', 'BP-target'].map(dispatch => continuation + '/386/' + dispatch), +]; +const requiredChildren = [sourceStack + '/amd64', sourceStack + '/386', + ...['amd64', '386'].flatMap(arch => ['memory', 'register'].map(dispatch => + continuation + '/' + arch + '/' + dispatch)), + packedStringFault + '/-O0', packedStringFault + '/-O2', + byteSwap + '/amd64', byteSwap + '/386', ...mode32Children, + ...['text', 'module', 'Go'].flatMap(api => ['backend_O0', 'IR_O2/backend_O2'] + .map(axis => `${bitTest}/${api}/${axis}`))]; + +function fixture() { + const files = new Map(); + for (const [name, roots] of Object.entries(inventories)) { + files.set(name + '-inventory.log', roots.join('\n') + '\n'); + } + const roots = Object.values(inventories).flat(); + for (let index = 0; index < 37; index++) roots.push('TestSyntheticExtraOwner' + index); + const events = []; + const emit = (Action, Test) => events.push({ + Action, Package: 'github.com/xgo-dev/plan9asm', ...(Test ? { Test } : {}), Elapsed: 1, + }); + for (const root of roots) { + emit('run', root); + for (const child of requiredChildren.filter(name => name.startsWith(root + '/'))) { + emit('run', child); + emit('pass', child); + } + if (root === 'TestSyntheticExtraOwner0') { + // A literal slash in one t.Run name does not create intermediate parents. + emit('run', root + '/one/literal/name'); + emit('pass', root + '/one/literal/name'); + } + emit('pass', root); + } + events.push({ Action: 'output', Package: 'github.com/xgo-dev/plan9asm', + Output: '--- PASS: TestOriginal (1.00s)\n{"Action":"pass","Test":"TestForgedPayload"}\n' }); + emit('pass'); + files.set('all-roots.stdout', roots.join('\n') + '\n'); + files.set('native.jsonl', events.map(JSON.stringify).join('\n') + '\n'); + files.set('native.stderr', ''); + return files; +} + +function events(files) { + return files.get('native.jsonl').trim().split('\n').map(JSON.parse); +} + +function replaceEvents(files, rows) { + files.set('native.jsonl', rows.map(JSON.stringify).join('\n') + '\n'); +} + +function audit(files) { + return auditNativePriority('synthetic-fixture', { + readdirSync() { return [...files.keys()]; }, + readFileSync(filename) { + const name = path.basename(filename); + assert.ok(files.has(name), 'missing synthetic file: ' + name); + return files.get(name); + }, + }); +} + +test('synthetic66 requires nineteen inventories,29 owners and23 source/ISA/fault children', () => { + const result = audit(fixture()); + assert.equal(result.protocol, 'go_test_json_native66'); + assert.equal(result.roots, 66); + assert.equal(result.inventories, 19); + assert.equal(result.mandatory.length, 29); + assert.ok(result.mandatory.includes(inventories['static-read'][0])); + assert.ok(result.mandatory.includes(crc32)); + assert.ok(result.mandatory.includes(bitTest)); + assert.equal(result.requiredChildren.length, 23); + // Only this synthetic fixture has24 children; real execution is counted. + assert.equal(result.children, 24); +}); + +test('static-read native root cannot be replaced by a same-count nonmandatory owner', () => { + const files = fixture(); + const root = inventories['static-read'][0]; + files.set('all-roots.stdout', files.get('all-roots.stdout').replace(root, 'TestUnrelatedStaticRead')); + files.set('native.jsonl', files.get('native.jsonl').replaceAll(root, 'TestUnrelatedStaticRead')); + assert.throws(() => audit(files), /mandatory root/); +}); + +test('CRC32 native root cannot be replaced by a same-count nonmandatory owner', () => { + const files = fixture(); + files.set('all-roots.stdout', files.get('all-roots.stdout').replace(crc32, 'TestUnrelatedCRC32Owner')); + files.set('native.jsonl', files.get('native.jsonl').replaceAll(crc32, 'TestUnrelatedCRC32Owner')); + assert.throws(() => audit(files), /mandatory root/); +}); + +test('CRC32 native root cannot be supplied by printed run and PASS payloads', () => { + const files = fixture(); + const rows = events(files).filter(event => event.Test !== crc32); + rows.splice(rows.length - 1, 0, { Action: 'output', Package: 'github.com/xgo-dev/plan9asm', + Output: JSON.stringify({ Action: 'run', Test: crc32 }) + '\n' + + JSON.stringify({ Action: 'pass', Test: crc32 }) + '\n' }); + replaceEvents(files, rows); + assert.throws(() => audit(files), /denominator|mandatory root/); +}); + +test('fresh full discovery rejects same-count nonmandatory root substitution', () => { + const files = fixture(); + files.set('native.jsonl', files.get('native.jsonl').replaceAll( + 'TestSyntheticExtraOwner0', 'TestUnrelatedSameCountOwner')); + assert.throws(() => audit(files), /discovery/); +}); + +test('missing, duplicate or renamed independent inventory cannot certify execution', () => { + for (const inventory of Object.keys(inventories)) { + const absent = fixture(); + absent.delete(inventory + '-inventory.log'); + assert.throws(() => audit(absent), /inventories/); + const renamed = fixture(); + renamed.set(inventory + '-inventory.log', 'TestUnrelatedNativeOwner\n'); + assert.throws(() => audit(renamed), /inventory/); + const duplicate = fixture(); + duplicate.set(inventory + '-inventory.log', duplicate.get(inventory + '-inventory.log').repeat(2)); + assert.throws(() => audit(duplicate), /inventory/); + } +}); + +test('each required source-stack ISA, continuation dispatch, fault and byte-swap axis must finish', () => { + for (const child of requiredChildren) { + const files = fixture(); + replaceEvents(files, events(files).filter(event => event.Test !== child)); + assert.throws(() => audit(files), /required child/); + } +}); + +test('both byte-swap ISAs reject same-count child substitution', () => { + for (const arch of ['amd64', '386']) { + const files = fixture(); + files.set('native.jsonl', files.get('native.jsonl').replaceAll( + byteSwap + '/' + arch, byteSwap + '/unrelated-' + arch)); + assert.throws(() => audit(files), /required child/); + } +}); + +test('all seven ordinary386 carrier, private-stack and code-target children reject substitution', () => { + assert.equal(mode32Children.length, 7); + for (const child of mode32Children) { + const files = fixture(); + replaceEvents(files, events(files).map(event => event.Test === child ? + { ...event, Test: child + '-unrelated' } : event)); + assert.throws(() => audit(files), /required child/); + } +}); + +test('root or child SKIP and FAIL are never PASS', () => { + for (const name of [...Object.values(inventories).flat(), ...requiredChildren]) { + for (const action of ['skip', 'fail']) { + const files = fixture(); + const rows = events(files); + rows.find(event => event.Action === 'pass' && event.Test === name).Action = action; + replaceEvents(files, rows); + assert.throws(() => audit(files), /execute successfully/); + } + } +}); + +test('required children cannot finish after their successful root terminal', () => { + for (const child of requiredChildren.filter(name => + name.startsWith(packedStringFault + '/') || name.startsWith(byteSwap + '/') || mode32Children.includes(name))) { + const files = fixture(); + const rows = events(files); + const index = rows.findIndex(event => event.Action === 'pass' && event.Test === child); + const [terminal] = rows.splice(index, 1); + const parent = rows.findIndex(event => event.Action === 'pass' && event.Test === child.split('/')[0]); + rows.splice(parent + 1, 0, terminal); + replaceEvents(files, rows); + assert.throws(() => audit(files), /child.*root terminal/); + } +}); + +test('duplicate or missing root and child events cannot close the protocol', () => { + for (const name of [sourceStack, requiredChildren[0], byteSwap, byteSwap + '/amd64', byteSwap + '/386', crc32, + bitTest, ...requiredChildren.filter(child => child.startsWith(bitTest + '/')), ...mode32Children]) { + for (const action of ['run', 'pass']) { + const duplicate = fixture(); + const rows = events(duplicate); + const index = rows.findIndex(event => event.Action === action && event.Test === name); + rows.splice(index + 1, 0, { ...rows[index] }); + replaceEvents(duplicate, rows); + assert.throws(() => audit(duplicate), /duplicate/); + const missing = fixture(); + replaceEvents(missing, events(missing).filter(event => event.Action !== action || event.Test !== name)); + assert.throws(() => audit(missing)); + } + } +}); + +test('tool-output text cannot replace missing machine events', () => { + const files = fixture(); + const rows = events(files).filter(event => event.Action !== 'pass' || event.Test !== requiredChildren[1]); + rows.splice(rows.length - 1, 0, { Action: 'output', Package: 'github.com/xgo-dev/plan9asm', + Output: JSON.stringify({ Action: 'pass', Test: requiredChildren[1] }) + '\n' }); + replaceEvents(files, rows); + assert.throws(() => audit(files), /child terminals/); +}); + +test('ordinary386 carrier execution cannot be supplied by printed PASS payloads', () => { + for (const child of mode32Children) { + const files = fixture(); + const rows = events(files).filter(event => event.Test !== child); + rows.splice(rows.length - 1, 0, { Action: 'output', Package: 'github.com/xgo-dev/plan9asm', + Output: JSON.stringify({ Action: 'run', Test: child }) + '\n' + + JSON.stringify({ Action: 'pass', Test: child }) + '\n' }); + replaceEvents(files, rows); + assert.throws(() => audit(files), /required child/); + } +}); + +test('missing package pass, failed package and malformed JSON remain failures', () => { + const incomplete = fixture(); + replaceEvents(incomplete, events(incomplete).slice(0, -1)); + assert.throws(() => audit(incomplete), /package terminal/); + const failed = fixture(); + const rows = events(failed); + rows[rows.length - 1].Action = 'fail'; + replaceEvents(failed, rows); + assert.throws(() => audit(failed), /execute successfully/); + const malformed = fixture(); + malformed.set('native.jsonl', malformed.get('native.jsonl') + '{truncated\n'); + assert.throws(() => audit(malformed)); +}); + +test('foreign package, unstarted child or duplicate package terminals are rejected', () => { + const foreign = fixture(); + foreign.set('native.jsonl', foreign.get('native.jsonl').replaceAll( + 'github.com/xgo-dev/plan9asm', 'example.invalid/foreign')); + assert.throws(() => audit(foreign), /package/); + const child = fixture(); + const rows = events(child); + rows.splice(rows.length - 1, 0, { Action: 'pass', Package: 'github.com/xgo-dev/plan9asm', + Test: 'TestUnstarted/child' }); + replaceEvents(child, rows); + assert.throws(() => audit(child), /unstarted child/); + const duplicate = fixture(); + const repeated = events(duplicate); + repeated.push({ ...repeated[repeated.length - 1] }); + replaceEvents(duplicate, repeated); + assert.throws(() => audit(duplicate), /duplicate package/); +}); + +test('package completion requires its own exact package identity', () => { + for (const identity of [undefined, null, '']) { + const files = fixture(); + const rows = events(files); + rows[rows.length - 1].Package = identity; + replaceEvents(files, rows); + assert.throws(() => audit(files), /package terminal.*identity/); + } +}); diff --git a/.github/scripts/priority-native-leaw.test.cjs b/.github/scripts/priority-native-leaw.test.cjs new file mode 100644 index 000000000..461597e2b --- /dev/null +++ b/.github/scripts/priority-native-leaw.test.cjs @@ -0,0 +1,107 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); + +const root = 'TestCrossLinuxRuntimeMatrixX86LEAWRetainedPointerOriginalBytesAndNativeDF'; +const apis = ['text', 'module', 'Go']; +const paths = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2']; +const children = [...apis.flatMap(api => [root + '/' + api, + ...paths.map(path => root + '/' + api + '/' + path)]), root + '/OriginalGoBytes']; +const inventory = root + '\nok\tgithub.com/xgo-dev/plan9asm\n'; +const hash = 'a'.repeat(64); +const host = `IA32 direct host: linux/amd64 kernel=x86_64 CPUsha=${hash} mapsha=${hash}\n`; +const sdk = `actual LEA source/object receipt: SDK=go1.27.2 target=linux/amd64 source=${hash} driver=${hash} object=${hash}\n`; +const completion = 'LEAW_RETAINED_POINTER_300_VALUES_STATUS_CANARIES_PASS\n'; +const llvm = 'native retained LEAW LLVM execution: ' + completion; +const original = 'native retained LEAW actual Go bytes execution: ' + completion; +const event = (Action, Test, Output) => ({Action, Package: 'github.com/xgo-dev/plan9asm', + ...(Test ? {Test} : {}), ...(Output !== undefined ? {Output} : {})}); +const serialize = events => events.map(value => JSON.stringify(value)).join('\n'); + +function completeEvents() { + const events = [event('start'), event('run', root), event('output', root, host + sdk)]; + for (const api of apis) { + const group = root + '/' + api; + events.push(event('run', group)); + for (const path of paths) { + const leaf = group + '/' + path; + events.push(event('run', leaf), event('output', leaf, llvm), event('pass', leaf)); + } + events.push(event('pass', group)); + } + events.push(event('run', root + '/OriginalGoBytes'), event('output', root + '/OriginalGoBytes', original), + event('pass', root + '/OriginalGoBytes'), event('output', root, host), event('pass', root), event('pass')); + return events; +} + +function audit(events = completeEvents(), discovered = inventory) { + assert.ok(fs.existsSync('.github/scripts/audit-native-leaw.cjs'), 'LEAW lacks a required exact native completion auditor'); + return require('./audit-native-leaw.cjs').auditNativeLEAW(discovered, serialize(events)); +} + +test('native LEAW requires its unique root, three API parents, nine LLVM leaves and actual Go bytes', () => { + assert.deepEqual(audit(), {roots: 1, children: 13, passed: 14, skipped: 0, incomplete: 0, + nativeHosts: 2, sdkCaptures: 1, llvmExecutions: 9, originalByteExecutions: 1}); +}); + +test('native LEAW rejects missing duplicated failed skipped and incomplete lifecycle terminals', () => { + audit(); + for (const name of [root, ...children]) { + for (const Action of ['fail', 'skip']) { + assert.throws(() => audit(completeEvents().map(value => value.Test === name && value.Action === 'pass' + ? {...value, Action} : value)), name + Action); + } + for (const Action of ['run', 'pass']) { + const events = completeEvents(); + const at = events.findIndex(value => value.Test === name && value.Action === Action); + assert.throws(() => audit(events.filter((_, index) => index !== at)), name + ' missing ' + Action); + events.splice(at, 0, events[at]); + assert.throws(() => audit(events), name + ' duplicate ' + Action); + } + const late = completeEvents(); + const at = late.findIndex(value => value.Test === name && value.Action === 'pass'); + late.splice(at + 1, 0, event('output', name, 'late output after the terminal\n')); + assert.throws(() => audit(late), name + ' output after pass'); + } + for (const discovered of ['', inventory + root + '\n', inventory.replace(root, 'TestUnrelated')]) { + assert.throws(() => audit(completeEvents(), discovered)); + } + for (const events of [completeEvents().slice(1), completeEvents().slice(0, -1), + [...completeEvents(), event('pass')], [event('start'), ...completeEvents()], + [event('output', undefined, '14 passed')], + completeEvents().map(value => ({...value, Package: 'foreign/package'}))]) assert.throws(() => audit(events)); +}); + +test('native LEAW binds real host brackets SDK capture and each actual execution marker to its own live owner', () => { + audit(); + for (const needle of [host, sdk]) { + for (const replacement of ['', needle + needle, needle.replace('linux/amd64', 'linux/arm64'), + needle.replace(hash, 'invalid')]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === root && value.Output + ? {...value, Output: value.Output.replace(needle, replacement)} : value)), needle); + } + } + for (const name of children.filter(name => !apis.some(api => name === root + '/' + api))) { + const marker = name.endsWith('/OriginalGoBytes') ? original : llvm; + for (const replacement of ['', marker + marker, marker.replace('300', '299'), + marker === original ? llvm : original, sdk + marker, host + marker]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === name && value.Output + ? {...value, Output: value.Output.replace(marker, replacement)} : value)), name); + } + } + const early = completeEvents(); + const originalEvents = early.filter(value => value.Test === root + '/OriginalGoBytes'); + const others = early.filter(value => value.Test !== root + '/OriginalGoBytes'); + others.splice(3, 0, ...originalEvents); + assert.throws(() => audit(others), 'original bytes cannot precede complete API groups'); + const misplaced = completeEvents().map(value => value.Test === root + '/text/backend_O0' && value.Output + ? {...value, Test: root + '/text'} : value); + assert.throws(() => audit(misplaced), 'an API parent cannot replace an executed leaf'); + const late = completeEvents(); + const at = late.findIndex(value => value.Action === 'output' && value.Test === root); + late[at] = event('output', root, host); + late.splice(5, 0, event('output', root, sdk)); + assert.throws(() => audit(late), 'SDK capture must precede the API run'); +}); diff --git a/.github/scripts/priority-nested-cli-coverage-isolation.test.cjs b/.github/scripts/priority-nested-cli-coverage-isolation.test.cjs new file mode 100644 index 000000000..38c07095c --- /dev/null +++ b/.github/scripts/priority-nested-cli-coverage-isolation.test.cjs @@ -0,0 +1,220 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const {jobBody, jobNeeds, requireJobNeeds} = require('./workflow-needs.cjs'); +const {requireNode22Setup} = require('./node22-setup.cjs'); + +const workflow = fs.readFileSync(path.join(__dirname, '../workflows/go-ci.yml'), 'utf8'); +const nested = fs.readFileSync(path.join(__dirname, '../workflows/linux-cli-module.yml'), 'utf8'); +const driver = fs.readFileSync(path.join(__dirname, '../../scripts/test-linux-cli-module-ci.sh'), 'utf8'); +const producer = 'nested-cli-coverage'; +const required = ['test', 'nested-cli', 'race', 'coverage', producer]; +const compilerInstall = 'bash scripts/install-ci-llvm22-ubuntu.sh ' + + 'gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu'; +const completeCondition = " if: ${{ !cancelled() && matrix.flavor == 'coverage' }}"; +const completeCommand = " run: test '${{ needs['nested-cli-coverage'].result }}' = success"; + +function steps(body) { + return body.split(/(?=^ - )/m).slice(1); +} + +function step(body, title) { + const selected = steps(body).filter(item => item.startsWith(` - name: ${title}\n`)); + assert.equal(selected.length, 1, 'one actual step required: ' + title); + return selected[0]; +} + +function requireIndependentOwners(source) { + const body = jobBody(source, producer); + assert.deepEqual(jobNeeds(source, producer), ['ci_policy']); + assert.match(body, /^ fail-fast: false$/m); + assert.doesNotMatch(body, /^ if:|continue-on-error:|max-parallel:|^ steps:/m); + const axes = [...body.matchAll(/^ ([a-z-]+): \[([^\]]+)\]$/gm)]; + assert.deepEqual(axes.map(([, key, values]) => [key, values.split(',').map(value => value.trim())]), + [['module', ['plan9asm', 'plan9asmll']]], + 'one independent current-SDK job for each complete CLI, not a serial module loop'); + assert.doesNotMatch(body, /^ (?:include|exclude):|\bgo test\b/m); + assert.match(body, /^ uses: \.\/\.github\/workflows\/linux-cli-module\.yml$/m); + assert.match(body, /^ module: \$\{\{ matrix\.module \}\}$/m); + assert.match(body, /^ go-version: '1\.27\.2'$/m); + assert.match(body, /^ flavor: coverage$/m); + return body; +} + +function requireCompleteCommands(source) { + requireIndependentOwners(source); + assert.ok(driver.includes('coverage) flags=("-coverprofile=coverage-$module.out")')); + assert.ok(driver.includes('-module-root-partitions=4')); + assert.ok(driver.includes('-verify-module=_out/linux-cli-$module-$flavor-owners')); + assert.ok(driver.includes('-timeout=30m')); + assert.doesNotMatch(driver, /(?:^|\s)-(?:run|skip|short|list)(?:=|\s)/m); + const complete = jobBody(nested, 'complete'); + const summary = step(complete, 'Show complete CLI coverage'); + assert.match(summary, /^ if: inputs\.flavor == 'coverage'$/m); + assert.match(summary, /^ run: go tool cover -func=coverage-\$\{\{ inputs\.module \}\}\.out \| tail -n 1$/m); + assert.match(summary, /^ working-directory: cmd\/\$\{\{ inputs\.module \}\}$/m); + assert.ok(complete.indexOf('Verify complete CLI module') < complete.indexOf(summary)); +} + +function requireTools(source) { + for (const name of ['root-parts', 'complete']) { + const body = jobBody(source, name); + const checkout = step(body, 'Checkout'); + assert.match(checkout, /^ uses: actions\/checkout@v7$/m); + const setup = step(body, 'Setup Go'); + assert.match(setup, /^ timeout-minutes: 10$/m); + assert.match(setup, /^ uses: \.\/\.github\/actions\/setup-go$/m); + assert.match(setup, /^ go-version: \$\{\{ inputs\.go-version \}\}$/m); + assert.doesNotMatch(setup, /^ cache:/m); + assert.doesNotMatch(body, /uses: actions\/setup-go@|go-version-file:|check-latest:/); + const compiler = step(body, 'Install LLVM 22 and cross compilers'); + assert.ok(compiler.includes(' run: ' + compilerInstall + '\n')); + assert.match(compiler, /^ timeout-minutes: 14$/m); + const qemu = step(body, 'Install required execution runtimes'); + assert.match(qemu, /^ run: bash scripts\/install-ci-qemu\.sh$/m); + assert.match(qemu, /^ timeout-minutes: 5$/m); + const info = step(body, 'LLVM tools info'); + for (const command of ['command -v llc', 'llc --version', 'command -v clang', 'clang --version']) { + assert.ok(info.includes(' ' + command + '\n')); + } + const node = step(body, 'Setup required wasm execution runtime'); + requireNode22Setup(node); + for (const item of [checkout, setup, compiler, qemu, info, node]) { + assert.doesNotMatch(item, /^ if:|continue-on-error:|\|\|\s*true/m); + assert.ok(body.indexOf(item) < body.indexOf(' run: bash scripts/test-linux-cli-module-ci.sh ')); + } + assert.ok(body.indexOf(qemu) < body.indexOf(node)); + } +} + +function requireOriginalOwners(source) { + for (const flavor of ['test', 'race', 'coverage']) { + const body = jobBody(source, flavor); + assert.doesNotMatch(body, /working-directory: cmd\/|coverage-plan9asm|nested_modules/, + flavor + ': root/profile/rest owners must not serialize complete nested CLI suites'); + } + const body = jobBody(source, 'coverage'); + assert.match(body, /^ timeout-minutes: 60$/m); + assert.match(step(body, 'Go test with coverage'), + /^ run: bash scripts\/test-module-ci\.sh coverage execute \$\{\{ matrix\.module-owner \}\} \$\{\{ matrix\.root-partition \}\}$/m); + const evidence = step(body, 'Retain complete module coverage owner evidence'); + assert.match(evidence, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + assert.match(evidence, /^ name: module-coverage-1\.27\.2-\$\{\{ matrix\.module-role \}\}$/m); + assert.match(evidence, /^ path: _out\/module-coverage\/$/m); + assert.match(evidence, /^ if-no-files-found: error$/m); +} + +function requireCompletion(source) { + requireJobNeeds(source, 'module-completion', required); + const body = jobBody(source, 'module-completion'); + assert.match(body, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + for (const title of ['Check complete actual module inventory and coverage', + 'Retain independent aggregate evidence']) { + assert.match(step(body, title), /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + } + const summary = step(body, 'Show complete module coverage'); + assert.match(summary, /^ if: matrix\.flavor == 'coverage'$/m); + assert.match(summary, /^ run: go tool cover -func=coverage-root\.out \| tail -n 1$/m); + const upload = step(body, 'Upload coverage to Codecov'); + assert.match(upload, /^ if: matrix\.flavor == 'coverage'$/m); + assert.match(upload, /^ uses: codecov\/codecov-action@v7$/m); + for (const field of ['use_oidc: true', 'files: ./coverage-root.out', 'disable_search: true', + 'flags: unittests', 'name: ubuntu-go1.27', 'fail_ci_if_error: false', 'verbose: true']) { + assert.ok(upload.includes(' ' + field + '\n')); + } + assert.doesNotMatch(upload, /coverage-\$\{\{|coverage-plan9asm/, + 'nested modules are not main-module coverage inputs'); + const success = step(body, 'Require complete CLI coverage'); + assert.ok(success.includes(completeCondition + '\n')); + assert.ok(success.includes(completeCommand + '\n')); + assert.doesNotMatch(success, /continue-on-error:|\|\|\s*true/); + assert.ok(body.indexOf(success) > body.indexOf(upload), + 'a failed CLI producer must not suppress the original root aggregate and upload'); +} + +test('both complete CLI coverage modules own independent current-SDK job budgets', () => { + requireIndependentOwners(workflow); + requireCompleteCommands(workflow); +}); + +test('independent CLI coverage retains exact Go, LLVM and required execution dependencies', () => { + requireIndependentOwners(workflow); + requireTools(nested); +}); + +test('root coverage no longer serializes CLI coverage and race has no invented CLI matrix', () => { + requireOriginalOwners(workflow); +}); + +test('CLI coverage failure remains a completion failure after original root evidence collection', () => { + requireCompletion(workflow); +}); + +test('CLI coverage policy rejects reduced, duplicated, serial, filtered or over-budget owners', () => { + requireCompleteCommands(workflow); + const body = jobBody(workflow, producer); + for (const [before, after] of [ + ['module: [plan9asm, plan9asmll]', 'module: [plan9asm]'], + ['module: [plan9asm, plan9asmll]', 'module: [plan9asmll]'], + ['module: [plan9asm, plan9asmll]', 'module: [plan9asm, plan9asm]'], + ['needs: ci_policy', 'needs: coverage'], + ['fail-fast: false', 'fail-fast: true'], + ['fail-fast: false', 'fail-fast: false\n max-parallel: 1'], + ['needs: ci_policy', 'needs: ci_policy\n continue-on-error: true'], + ['linux-cli-module.yml', 'windows-module.yml'], + ['module: ${{ matrix.module }}', 'module: plan9asm'], + ["go-version: '1.27.2'", "go-version: '1.27.x'"], + ['flavor: coverage', 'flavor: test'], + ]) { + const changed = body.replace(before, () => after); + assert.notEqual(changed, body, before); + assert.throws(() => requireCompleteCommands(workflow.replace(body, () => changed)), undefined, before); + } +}); + +test('CLI coverage dependencies cannot borrow skipped, missing or foreign tool installations', () => { + requireTools(nested); + for (const name of ['root-parts', 'complete']) { + const body = jobBody(nested, name); + for (const [before, after] of [ + ['go-version: ${{ inputs.go-version }}', "go-version: '1.27.2'"], + [' gcc-i686-linux-gnu', ''], + [' libc6-dev-i386-cross', ''], + [' gcc-aarch64-linux-gnu', ''], + ['install-ci-llvm22-ubuntu.sh', 'install-llvm23.sh'], + ['install-ci-qemu.sh', 'install-unpinned-qemu.sh'], + ['uses: ./.github/actions/setup-node22', 'uses: actions/setup-node@v6'], + [' run: ' + compilerInstall, ' if: false\n run: ' + compilerInstall], + ]) { + const changed = body.replace(before, () => after); + assert.notEqual(changed, body, before); + assert.throws(() => requireTools(nested.replace(body, () => changed)), undefined, before); + } + } +}); + +test('CLI coverage completion rejects missing needs, skipped collection and borrowed success', () => { + requireCompletion(workflow); + const body = jobBody(workflow, 'module-completion'); + for (const gate of required) { + const changed = body.replace(/^ needs:.*$/m, + () => ' needs: [' + required.filter(name => name !== gate).join(', ') + ']'); + assert.notEqual(changed, body, gate); + assert.throws(() => requireCompletion(workflow.replace(body, () => changed)), undefined, gate); + } + for (const [before, after] of [ + [' if: ${{ !cancelled() }}', ' if: success()'], + [completeCondition, " if: matrix.flavor == 'coverage'"], + [completeCondition, " if: ${{ !cancelled() && matrix.flavor == 'test' }}"], + [completeCommand, " run: test '${{ needs['nested-cli'].result }}' = success"], + [completeCommand, completeCommand + ' || true'], + [completeCommand, ' run: true'], + ]) { + const changed = body.replace(before, () => after); + assert.notEqual(changed, body, before); + assert.throws(() => requireCompletion(workflow.replace(body, () => changed)), undefined, before); + } +}); diff --git a/.github/scripts/priority-nested-cli-isolation.test.cjs b/.github/scripts/priority-nested-cli-isolation.test.cjs new file mode 100644 index 000000000..62ddd78cf --- /dev/null +++ b/.github/scripts/priority-nested-cli-isolation.test.cjs @@ -0,0 +1,163 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const {jobBody, jobNeeds, requireJobNeeds} = require('./workflow-needs.cjs'); +const {requireNode22Setup} = require('./node22-setup.cjs'); + +const workflow = fs.readFileSync(path.join(__dirname, '../workflows/go-ci.yml'), 'utf8'); +const nested = fs.readFileSync(path.join(__dirname, '../workflows/linux-cli-module.yml'), 'utf8'); +const versions = ['1.24.x', '1.25.x', '1.26.x', '1.27.2']; +const modules = ['plan9asm', 'plan9asmll']; +const compilerInstall = 'bash scripts/install-ci-llvm22-ubuntu.sh ' + + 'gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu'; + +function steps(body) { + return body.split(/(?=^ - )/m).slice(1); +} + +function step(body, title) { + const selected = steps(body).filter(item => item.startsWith(` - name: ${title}\n`)); + assert.equal(selected.length, 1, 'one actual step required: ' + title); + return selected[0]; +} + +function matrixValues(body, key) { + const values = [...body.matchAll(new RegExp(`^ ${key}: \\[([^\\]]+)\\]$`, 'gm'))]; + assert.equal(values.length, 1, 'one complete matrix axis required: ' + key); + return values[0][1].split(',').map(value => value.trim().replace(/^'(.*)'$/, '$1')); +} + +function requireIndependentCLI(source) { + const body = jobBody(source, 'nested-cli'); + assert.deepEqual(jobNeeds(source, 'nested-cli'), ['ci_policy']); + assert.doesNotMatch(body, /^ if:|continue-on-error:|max-parallel:/m); + assert.match(body, /^ fail-fast: false$/m); + assert.deepEqual(matrixValues(body, 'go-version'), versions); + assert.deepEqual(matrixValues(body, 'module'), modules); + assert.doesNotMatch(body, /^ (?:include|exclude):|^ steps:|\bgo test\b/m); + assert.match(body, /^ uses: \.\/\.github\/workflows\/linux-cli-module\.yml$/m); + assert.match(body, /^ module: \$\{\{ matrix\.module \}\}$/m); + assert.match(body, /^ go-version: \$\{\{ matrix\.go-version \}\}$/m); + assert.match(body, /^ flavor: test$/m); + assert.doesNotMatch(body, /scripts\/test-module-ci\.sh|Required exact downloaded/); + requireJobNeeds(source, 'module-completion', ['test', 'nested-cli', 'race', 'coverage', 'nested-cli-coverage']); + assert.match(jobBody(source, 'module-completion'), /^ if: \$\{\{ !cancelled\(\) \}\}$/m, + 'completion must still collect original owners after a failed CLI job'); +} + +function requireCLITools(source) { + for (const name of ['root-parts', 'complete']) { + const body = jobBody(source, name); + const checkout = step(body, 'Checkout'); + assert.match(checkout, /^ uses: actions\/checkout@v7$/m); + const setup = step(body, 'Setup Go'); + assert.match(setup, /^ timeout-minutes: 10$/m); + assert.match(setup, /^ uses: \.\/\.github\/actions\/setup-go$/m); + assert.match(setup, /^ go-version: \$\{\{ inputs\.go-version \}\}$/m); + assert.doesNotMatch(setup, /^ cache:/m); + assert.doesNotMatch(body, /uses: actions\/setup-go@|go-version-file:|check-latest:/); + const compiler = step(body, 'Install LLVM 22 and cross compilers'); + assert.ok(compiler.includes(` run: ${compilerInstall}\n`)); + assert.match(compiler, /^ timeout-minutes: 14$/m); + const qemu = step(body, 'Install required execution runtimes'); + assert.match(qemu, /^ run: bash scripts\/install-ci-qemu\.sh$/m); + assert.match(qemu, /^ timeout-minutes: 5$/m); + const node = step(body, 'Setup required wasm execution runtime'); + requireNode22Setup(node); + for (const item of [checkout, setup, compiler, qemu, node]) { + assert.doesNotMatch(item, /^ if:|continue-on-error:|\|\|\s*true/m); + assert.ok(body.indexOf(item) < body.indexOf(' run: bash scripts/test-linux-cli-module-ci.sh '), + 'every CLI dependency must be installed before execution'); + } + assert.ok(body.indexOf(qemu) < body.indexOf(node), 'retain QEMU/Node PATH provisioning order'); + } +} + +function requireOriginalOtherOwner(source) { + const body = jobBody(source, 'test'); + assert.match(body, /^ timeout-minutes: 60$/m); + assert.match(step(body, 'Go test (Ubuntu)'), + /^ run: bash scripts\/test-module-ci\.sh test execute \$\{\{ matrix\.module-owner \}\} \$\{\{ matrix\.root-partition \}\}$/m); + assert.doesNotMatch(body, /nested_modules|working-directory: cmd\/plan9asm|Go test \(cmd\/plan9asm/, + 'remaining-package owners cannot serialize complete CLI suites'); + const numerical = step(body, 'Required exact downloaded AMD64 numerical fixture'); + assert.match(numerical, + /^ if: runner\.os == 'Linux' && matrix\.go-version == '1\.27\.2' && matrix\.module-owner == 2$/m); + assert.match(numerical, /^ shell: bash$/m); + assert.match(numerical, /^ set -euo pipefail$/m); + assert.match(numerical, /^ test "\$\(go env GOHOSTOS\)\/\$\(go env GOHOSTARCH\)" = linux\/amd64$/m); + assert.match(numerical, /^ scope='\^TestAMD64DownloadedFixedLocalLLVM22Numerical\$'$/m); + assert.match(numerical, + /^ go test \.\/gotoolprofile -list="\$scope" \| grep -Fx 'TestAMD64DownloadedFixedLocalLLVM22Numerical'$/m); + assert.match(numerical, /^ go test \.\/gotoolprofile -run="\$scope" -count=1 -timeout=10m$/m); +} + +test('both complete CLIs own independent jobs for every supported nested SDK', () => { + requireIndependentCLI(workflow); +}); + +test('every independent CLI owner retains its complete tool and execution runtimes', () => { + requireCLITools(nested); +}); + +test('root other owner retains the exact numerical fixture without serial CLI suites', () => { + requireOriginalOtherOwner(workflow); +}); + +test('CLI ownership rejects reduced matrices, serial scheduling and weakened success gates', () => { + requireIndependentCLI(workflow); + const body = jobBody(workflow, 'nested-cli'); + for (const [before, after] of [ + ["'1.24.x', ", ''], + ["'1.25.x', ", ''], + ["'1.26.x', ", ''], + ["'1.27.2'", "'1.27.x'"], + ['module: [plan9asm, plan9asmll]', 'module: [plan9asm]'], + ['module: [plan9asm, plan9asmll]', 'module: [plan9asmll]'], + ['fail-fast: false', 'fail-fast: true'], + ['fail-fast: false', 'fail-fast: false\n max-parallel: 1'], + ['needs: ci_policy', 'needs: test'], + ['linux-cli-module.yml', 'windows-module.yml'], + ['module: ${{ matrix.module }}', 'module: plan9asm'], + ['go-version: ${{ matrix.go-version }}', "go-version: '1.27.2'"], + ['flavor: test', 'flavor: coverage'], + ]) { + const changed = body.replace(before, () => after); + assert.notEqual(changed, body, 'ownership mutation must change its actual job: ' + before); + assert.throws(() => requireIndependentCLI(workflow.replace(body, () => changed)), undefined, before); + } + const complete = jobBody(workflow, 'module-completion'); + const required = ['test', 'nested-cli', 'race', 'coverage', 'nested-cli-coverage']; + for (const gate of required) { + const reduced = required.filter(name => name !== gate); + const changed = complete.replace(/^ needs:.*$/m, ' needs: [' + reduced.join(', ') + ']'); + assert.throws(() => requireIndependentCLI(workflow.replace(complete, changed)), undefined, gate); + } + const skipped = complete.replace('if: ${{ !cancelled() }}', 'if: success()'); + assert.notEqual(skipped, complete); + assert.throws(() => requireIndependentCLI(workflow.replace(complete, skipped))); +}); + +test('CLI tool policies reject missing infrastructure and skipped setup', () => { + requireCLITools(nested); + for (const name of ['root-parts', 'complete']) { + const body = jobBody(nested, name); + for (const [before, after] of [ + [' gcc-i686-linux-gnu', ''], + [' libc6-dev-i386-cross', ''], + [' gcc-aarch64-linux-gnu', ''], + ['install-ci-llvm22-ubuntu.sh', 'install-llvm23.sh'], + ['install-ci-qemu.sh', 'install-unpinned-qemu.sh'], + ['uses: ./.github/actions/setup-node22', 'uses: actions/setup-node@v6'], + ['go-version: ${{ inputs.go-version }}', "go-version: '1.27.2'"], + [' run: ' + compilerInstall, ' if: false\n run: ' + compilerInstall], + ]) { + const changed = body.replace(before, () => after); + assert.notEqual(changed, body, 'tool mutation must change its actual job: ' + before); + assert.throws(() => requireCLITools(nested.replace(body, () => changed)), undefined, before); + } + } +}); diff --git a/.github/scripts/priority-node22-installation.test.cjs b/.github/scripts/priority-node22-installation.test.cjs new file mode 100644 index 000000000..d83215d7c --- /dev/null +++ b/.github/scripts/priority-node22-installation.test.cjs @@ -0,0 +1,92 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); +const {test} = require('node:test'); + +const action = fs.readFileSync('.github/actions/setup-node22/action.yml', 'utf8'); + +function verificationScript() { + const step = action.split(/(?=^ - name:)/m).find(part => + part.startsWith(' - name: Verify Node.js installation\n')); + if (!step) return ''; + const marker = ' run: |\n'; + assert.ok(step.includes(marker), 'verification requires the actual script'); + return step.slice(step.indexOf(marker) + marker.length) + .split('\n').map(line => line.startsWith(' ') ? line.slice(8) : line).join('\n'); +} + +function verify(options = {}) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'plan9asm-node22-install-')); + try { + if (!options.missingNode) { + fs.writeFileSync(path.join(directory, 'node'), `#!/bin/sh +[ "$*" = '--version' ] || exit 92 +printf '%s\\n' "$MOCK_NODE_VERSION" +exit "$MOCK_NODE_EXIT" +`, {mode: 0o755}); + } + return spawnSync('/bin/bash', ['--noprofile', '--norc', '-c', verificationScript()], { + encoding: 'utf8', timeout: 5000, + env: { + PATH: directory, + EXPECTED_NODE_VERSION: 'v22.23.4', + INSTALLED_NODE_VERSION: options.installed ?? 'v22.23.4', + MOCK_NODE_VERSION: options.actual ?? 'v22.23.4', + MOCK_NODE_EXIT: String(options.exit ?? 0), + }, + }); + } finally { + fs.rmSync(directory, {recursive: true, force: true}); + } +} + +test('Node setup pins one exact runtime and verifies its official installer output', () => { + assert.match(action, /id: install_node22\n uses: actions\/setup-node@v6\n with:\n node-version: '22\.23\.4'/); + assert.match(action, /INSTALLED_NODE_VERSION: \$\{\{ steps\.install_node22\.outputs\.node-version \}\}/); + assert.match(action, /EXPECTED_NODE_VERSION: 'v22\.23\.4'/); + assert.doesNotMatch(action, /continue-on-error:|\n if:|\|\|\s*true|node-version-file:|mirror:/); + assert.ok(verificationScript().includes('set -euo pipefail')); +}); + +test('a complete exact Node installation selects its actual PATH runtime', () => { + for (const actual of ['v22.23.4', 'v22.23.4\r']) { + const result = verify({actual}); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Verified Node v22\.23\.4: .*\/node/); + } +}); + +test('an interrupted Node download cannot borrow an ambient runtime or empty output', () => { + for (const options of [ + {installed: ''}, + {installed: '', actual: 'v22.23.3'}, + {installed: '', actual: 'v22.23.4'}, + {missingNode: true}, + {actual: 'v22.23.3'}, + {actual: 'v24.0.0'}, + {actual: '', exit: 1}, + {actual: 'v22.23.4', exit: 1}, + ]) { + const result = verify(options); + assert.ifError(result.error); + assert.notEqual(result.status, 0, JSON.stringify(options)); + assert.doesNotMatch(result.stdout, /Verified Node/); + } +}); + +test('Node verification rejects wrong patches, nonstable versions and malformed output', () => { + for (const value of ['v22.23.3', '22.23.4', 'v22.23.4-rc.1', 'v24.0.0', + 'v22.23.4\nv22.23.4', ' v22.23.4', 'v22.23.4 ']) { + for (const options of [{installed: value}, {actual: value}]) { + const result = verify(options); + assert.ifError(result.error); + assert.notEqual(result.status, 0, JSON.stringify(options)); + assert.doesNotMatch(result.stdout, /Verified Node/); + } + } +}); diff --git a/.github/scripts/priority-node22-workflows.test.cjs b/.github/scripts/priority-node22-workflows.test.cjs new file mode 100644 index 000000000..75a5e61d2 --- /dev/null +++ b/.github/scripts/priority-node22-workflows.test.cjs @@ -0,0 +1,175 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const {test} = require('node:test'); +const {jobBody} = require('./workflow-needs.cjs'); +const {requireNode22Setup} = require('./node22-setup.cjs'); + +const roles = { + 'go-ci.yml': ['ci_policy', 'cross-runtime', 'cross-runtime-verify', 'test', 'nested-cli', + 'nested-cli-coverage', 'race', 'coverage', 'module-completion'], + 'windows-module.yml': ['root-parts', 'other-owners', 'complete'], + 'runtime-integration.yml': ['ci_policy'], +}; + +// Both complete layouts retain the same two CLI installation obligations. +// Once the reusable producer exists, verify its real roles and exact callers; +// a missing callee cannot make the old inline obligations disappear. +const cliWorkflow = '.github/workflows/linux-cli-module.yml'; +const usesReusableCLI = fs.existsSync(cliWorkflow); +if (usesReusableCLI) { + roles['go-ci.yml'] = ['ci_policy', 'cross-runtime', 'cross-runtime-verify', 'test', + 'race', 'coverage', 'module-completion']; + roles['linux-cli-module.yml'] = ['root-parts', 'complete']; +} + +function requireCLICaller(body, coverage) { + assert.equal((body.match(/^ uses: \.\/\.github\/workflows\/linux-cli-module\.yml$/gm) || []).length, 1); + assert.match(body, /^ needs: ci_policy$/m); + assert.match(body, /^ fail-fast: false$/m); + assert.match(body, /^ module: \[plan9asm, plan9asmll\]$/m); + assert.match(body, /^ module: \$\{\{ matrix\.module \}\}$/m); + if (coverage) { + assert.match(body, /^ go-version: '1\.27\.2'$/m); + assert.doesNotMatch(body, /^ go-version:/m, + 'coverage has one exact current SDK, not an additional matrix'); + } else { + assert.ok(body.includes(" go-version: ['1.24.x', '1.25.x', '1.26.x', '1.27.2']\n")); + assert.match(body, /^ go-version: \$\{\{ matrix\.go-version \}\}$/m); + } + assert.ok(body.includes(' flavor: ' + (coverage ? 'coverage' : 'test') + '\n')); + assert.doesNotMatch(body, /^ (?:if|runs-on|steps|continue-on-error):|max-parallel:/m); +} + +if (usesReusableCLI) { + test('both complete CLI callers preserve their exact reusable runtime obligations', () => { + const workflow = fs.readFileSync('.github/workflows/go-ci.yml', 'utf8'); + for (const [name, coverage] of [['nested-cli', false], ['nested-cli-coverage', true]]) { + const body = jobBody(workflow, name); + requireCLICaller(body, coverage); + for (const [before, after] of [ + ['uses: ./.github/workflows/linux-cli-module.yml', 'uses: ./.github/workflows/foreign.yml'], + ['module: [plan9asm, plan9asmll]', 'module: [plan9asm]'], + ["'1.27.2'", "'1.27.x'"], + ['flavor: ' + (coverage ? 'coverage' : 'test'), 'flavor: race'], + ['fail-fast: false', 'fail-fast: true'], + ]) { + const changed = body.replace(before, after); + assert.notEqual(changed, body); + assert.throws(() => requireCLICaller(changed, coverage)); + } + } + }); +} + +function nodeStep(body) { + const steps = body.split(/(?=^ - )/m).slice(1).filter(step => + step.includes('uses: ./.github/actions/setup-node22')); + assert.equal(steps.length, 1, 'each required role has exactly one verified Node setup'); + requireNode22Setup(steps[0]); + assert.doesNotMatch(body, /uses: actions\/setup-node@|node-version:/, + 'another runtime cannot replace or follow the verified exact installer'); + return steps[0]; +} + +// Derive runtime users separately from the closed role census. A new cross or +// complete-module consumer must not borrow ambient Node just because its name +// was omitted from the list above. +function requireRuntimeNodeDependencies(source) { + const start = source.indexOf('\njobs:\n'); + assert.ok(start >= 0, 'a formal reusable workflow must retain its job graph'); + const jobs = source.slice(start); + const observed = []; + for (const [, name] of jobs.matchAll(/^ ([a-zA-Z0-9_-]+):$/gm)) { + const body = jobBody(source, name); + const runtime = /PLAN9ASM_CROSS_EXEC\s*[:=]\s*['"]?1(?:['"]|\s|$)/.test(body) || + /scripts\/test-(?:arm64-fallback|(?:linux-cli-|windows-)?module)-ci\.sh/.test(body) || + /^ +(?:run: )?node\s/m.test(body) || + /^ +run: go test\s/m.test(body); + if (runtime) { + nodeStep(body); + observed.push(name); + } + } + return observed.sort(); +} + +function requireFormalRuntimeDependencies(readWorkflow) { + const pending = ['go-ci.yml']; + const observed = new Map(); + while (pending.length) { + const name = pending.pop(); + if (observed.has(name)) continue; + const source = readWorkflow(name); + observed.set(name, requireRuntimeNodeDependencies(source)); + for (const [, callee] of source.matchAll( + /^ uses: \.\/\.github\/workflows\/([a-zA-Z0-9_.-]+\.yml)$/gm)) { + pending.push(callee); + } + } + return observed; +} + +test('actual cross-runtime and module consumers independently require verified Node', () => { + const workflow = fs.readFileSync('.github/workflows/go-ci.yml', 'utf8'); + const readWorkflow = name => fs.readFileSync('.github/workflows/' + name, 'utf8'); + const observed = requireFormalRuntimeDependencies(readWorkflow); + assert.deepEqual(observed.get('go-ci.yml'), + ['ci_policy', 'coverage', 'cross-runtime', 'cross-runtime-verify', + 'module-completion', 'race', 'test']); + const added = workflow + '\n new-cross-consumer:\n env:\n' + + " PLAN9ASM_CROSS_EXEC: '1'\n steps:\n - run: go run ./cmd/plan9asmtest\n"; + assert.throws(() => requireRuntimeNodeDependencies(added), + 'a newly added consumer cannot disappear from Node coverage'); + assert.deepEqual(observed.get('windows-module.yml'), ['complete', 'other-owners', 'root-parts']); + assert.deepEqual(observed.get('linux-cli-module.yml'), ['complete', 'root-parts']); + + const consumer = "name: New consumer\njobs:\n new-cross-consumer:\n env:\n" + + " PLAN9ASM_CROSS_EXEC: '1'\n steps:\n - run: go run ./cmd/plan9asmtest\n"; + const caller = workflow + '\n new-reusable-consumer:\n' + + ' uses: ./.github/workflows/new-consumer.yml\n'; + const fixture = source => name => name === 'go-ci.yml' ? caller + : name === 'new-consumer.yml' ? source : readWorkflow(name); + assert.throws(() => requireFormalRuntimeDependencies(fixture(consumer)), + /exactly one verified Node setup/); + const guarded = consumer.replace(' steps:\n', ' steps:\n' + + nodeStep(jobBody(workflow, 'cross-runtime'))); + assert.deepEqual(requireFormalRuntimeDependencies(fixture(guarded)).get('new-consumer.yml'), + ['new-cross-consumer']); +}); + +for (const [file, jobs] of Object.entries(roles)) { + const workflow = fs.readFileSync('.github/workflows/' + file, 'utf8'); + test(file + ' preserves every required verified Node role and its setup deadline', () => { + assert.equal((workflow.match(/uses: \.\/\.github\/actions\/setup-node22$/gm) || []).length, jobs.length); + for (const name of jobs) nodeStep(jobBody(workflow, name)); + }); + test(file + ' rejects missing, skipped, duplicate, ambient or unbounded runtime setup', () => { + for (const name of jobs) { + const body = jobBody(workflow, name), step = nodeStep(body); + for (const replacement of [ + '', step + step, + step.replace('uses: ./.github/actions/setup-node22', 'uses: actions/setup-node@v6'), + step.replace('timeout-minutes: 10', 'timeout-minutes: 60'), + step.replace('timeout-minutes: 10', 'if: false\n timeout-minutes: 10'), + step.replace('timeout-minutes: 10', 'continue-on-error: true\n timeout-minutes: 10'), + ...(step.includes(' - uses:') + ? [step.replace(' - uses:', ' - if: false\n uses:')] : []), + step + ' uses: actions/setup-node@v6\n', + step + " with:\n node-version: '22'\n", + ]) { + assert.notEqual(replacement, step, 'negative control must change its actual setup'); + assert.throws(() => nodeStep(body.replace(step, replacement)), undefined, name); + } + } + }); +} + +test('policy checks execute only after their own verified Node installation', () => { + for (const file of ['go-ci.yml', 'runtime-integration.yml']) { + const body = jobBody(fs.readFileSync('.github/workflows/' + file, 'utf8'), 'ci_policy'); + const setup = nodeStep(body); + assert.ok(body.indexOf(setup) < body.indexOf('run: node --test ')); + } +}); diff --git a/.github/scripts/priority-optional-avx512.test.cjs b/.github/scripts/priority-optional-avx512.test.cjs new file mode 100644 index 000000000..cb6ba0b23 --- /dev/null +++ b/.github/scripts/priority-optional-avx512.test.cjs @@ -0,0 +1,12 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); + +test('Windows execution and completion share the explicit optional AVX512 policy', () => { + const workflow = fs.readFileSync('.github/workflows/windows-module.yml', 'utf8'); + assert.match(workflow, /^env:\n PLAN9ASM_AVX512_RUNTIME: auto\n/m); + assert.equal((workflow.match(/PLAN9ASM_AVX512_RUNTIME/g) || []).length, 1, + 'individual owners cannot override the common execution environment'); +}); diff --git a/.github/scripts/priority-packed-conversions-native.test.cjs b/.github/scripts/priority-packed-conversions-native.test.cjs new file mode 100644 index 000000000..0c0769956 --- /dev/null +++ b/.github/scripts/priority-packed-conversions-native.test.cjs @@ -0,0 +1,148 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const auditor = require('./audit-packed-conversion-native.cjs'); + +test('native conversion receipts require a strict current-head owner and complete-union auditor', () => { + const path = '.github/scripts/audit-packed-conversion-native.cjs'; + assert.ok(fs.existsSync(path), 'missing complete native conversion receipt auditor'); + const source = fs.readFileSync(path, 'utf8'); + for (const guard of ['auditNativeOwners', 'AVX512F=1 AVX512DQ=1 AVX512BW=1 AVX512VL=1 OSXSAVE=1', + 'PACKED_CONVERSION_NATIVE_F_DQ_BW_VL_OK', '0xe6n', 'original_go_named', 'original_go_raw']) { + assert.ok(source.includes(guard), 'missing strict native conversion guard: ' + guard); + } +}); + +// This independently constructs the expected semantic partitions and all +// actual run/pass events. Neither auditor metadata nor a success payload can +// supply the original Go, public API, backend or native hardware axes. +function fixture(owner) { + const suffix = owner ? 'Part' + owner : ''; + const roots = [ + 'TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512Status' + suffix, + 'TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaults' + suffix, + ]; + if (!owner) roots.push('TestCrossLinuxRuntimeMatrixX86PackedConversionNativeIndependentC'); + const children = []; + for (let family = 0; family < 2; family++) { + const size = family ? 222 : 692, limit = family ? 16 : 64; + for (let start = owner * limit; start < size; start += 8 * limit) { + const batch = roots[family] + '/batch-' + String(start).padStart(4, '0'); + children.push(batch); + const originals = family ? ['original_go_opcode_reference'] : ['original_go_named', 'original_go_raw']; + children.push(...originals.map(original => batch + '/' + original)); + for (const spelling of ['named', 'raw']) { + const source = batch + '/' + spelling; + children.push(source); + for (const api of ['text', 'module', 'Go']) { + const route = source + '/' + api; + children.push(route, ...['backend_O0', 'backend_O2', 'IR_O2_backend_O2'].map(axis => route + '/' + axis)); + } + } + } + } + if (!owner) children.push(...['backend_O0', 'backend_O2', 'IR_O2_backend_O2'].map(axis => roots[2] + '/' + axis)); + const events = []; + const packageName = 'github.com/xgo-dev/plan9asm'; + const emit = (action, name, output) => { + const event = {Action: action, Package: packageName}; + if (name) event.Test = name; + if (output !== undefined) event.Output = output; + events.push(event); + }; + function execute(name) { + emit('run', name); + if (roots.includes(name)) { + emit('output', name, 'AVX512F=1 AVX512DQ=1 AVX512BW=1 AVX512VL=1 OSXSAVE=1 XCR0=e7\n'); + emit('output', name, 'PACKED_CONVERSION_NATIVE_F_DQ_BW_VL_OK\n'); + } + for (const child of children.filter(child => child.slice(0, child.lastIndexOf('/')) === name)) execute(child); + emit('pass', name); + } + for (const root of roots) execute(root); + emit('pass'); + return {roots, children, inventory: roots.join('\n') + '\n', events}; +} + +function raw(events) { + return events.map(event => JSON.stringify(event)).join('\n') + '\n'; +} + +test('all eight owners consume every complete original/public/native execution axis', () => { + let roots = 0, children = 0; + for (let owner = 0; owner < 8; owner++) { + const independent = fixture(owner), expected = auditor.required(owner); + assert.deepEqual(expected.roots, independent.roots); + assert.deepEqual(expected.children, independent.children); + const result = auditor.auditOwner(independent.inventory, raw(independent.events), owner); + assert.equal(result.preflights, independent.roots.length); + assert.equal(result.passed, independent.roots.length + independent.children.length); + roots += result.roots; + children += result.children; + } + assert.equal(roots, 17); + assert.equal(children, 714); +}); + +test('failed skipped missing duplicate foreign incomplete and printed native receipts fail', () => { + const base = fixture(0); + const leaf = base.children.find(child => child.endsWith('/original_go_raw')); + for (const action of ['fail', 'skip']) { + const events = base.events.map(event => event.Test === leaf && event.Action === 'pass' ? {...event, Action: action} : event); + assert.throws(() => auditor.auditOwner(base.inventory, raw(events), 0)); + } + for (const alter of [ + events => events.filter(event => event.Test !== leaf), + events => events.filter(event => !(event.Test === leaf && event.Action === 'pass')), + events => [...events.slice(0, -1), {Action: 'pass', Package: 'github.com/xgo-dev/plan9asm', Test: leaf}, events.at(-1)], + events => events.map(event => event.Test === leaf ? {...event, Test: event.Test + '/foreign'} : event), + events => events.slice(0, -1), + events => [{Action: 'output', Package: 'github.com/xgo-dev/plan9asm', Output: 'all native conversions passed\n'}, events.at(-1)], + ]) assert.throws(() => auditor.auditOwner(base.inventory, raw(alter(base.events)), 0)); + assert.throws(() => auditor.auditOwner(base.inventory + 'TestForeign\n', raw(base.events), 0)); + assert.throws(() => auditor.auditOwner(base.inventory, raw(base.events), 1)); +}); + +test('each current owner must observe all native features and enabled state before execution', () => { + const base = fixture(1); + for (const [before, after] of [ + ['AVX512F=1', 'AVX512F=0'], ['AVX512DQ=1', 'AVX512DQ=0'], + ['AVX512BW=1', 'AVX512BW=0'], ['AVX512VL=1', 'AVX512VL=0'], + ['OSXSAVE=1', 'OSXSAVE=0'], ['XCR0=e7', 'XCR0=c7'], + ['PACKED_CONVERSION_NATIVE_F_DQ_BW_VL_OK', 'opcode_not_run'], + ]) { + const events = base.events.map(event => event.Output ? {...event, Output: event.Output.replace(before, after)} : event); + assert.throws(() => auditor.auditOwner(base.inventory, raw(events), 1), before); + } + const late = base.events.filter(event => !event.Output); + late.splice(late.length - 2, 0, ...base.events.filter(event => event.Output)); + assert.throws(() => auditor.auditOwner(base.inventory, raw(late), 1)); +}); + +test('complete audit reopens all eight current-source raw streams instead of trusting summaries', t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'packed-native-audit-')); + t.after(() => fs.rmSync(directory, {recursive: true, force: true})); + const head = 'a'.repeat(40), directories = []; + for (let owner = 0; owner < 8; owner++) { + const data = fixture(owner), child = path.join(directory, String(owner)); + fs.mkdirSync(child); + fs.writeFileSync(path.join(child, 'context.json'), JSON.stringify({schema: 1, head, owner})); + fs.writeFileSync(path.join(child, 'inventory.log'), data.inventory); + fs.writeFileSync(path.join(child, 'raw.jsonl'), raw(data.events)); + directories.push(child); + } + assert.deepEqual(auditor.auditComplete(directories, head), { + head, owners: 8, roots: 17, children: 714, passed: 731, + valueKernels: 692, faultKernels: 222, skipped: 0, incomplete: 0, + }); + assert.throws(() => auditor.auditComplete(directories.slice(0, -1), head)); + assert.throws(() => auditor.auditComplete([...directories.slice(0, -1), directories[0]], head)); + assert.throws(() => auditor.auditComplete(directories, 'b'.repeat(40))); + const bad = path.join(directories[7], 'raw.jsonl'); + fs.writeFileSync(bad, '{"allPassed":true}\n'); + assert.throws(() => auditor.auditComplete(directories, head)); +}); diff --git a/.github/scripts/priority-packed-reductions-native.test.cjs b/.github/scripts/priority-packed-reductions-native.test.cjs new file mode 100644 index 000000000..6e40295d1 --- /dev/null +++ b/.github/scripts/priority-packed-reductions-native.test.cjs @@ -0,0 +1,141 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const auditor = require('./audit-packed-reduction-native.cjs'); + +test('native reduction receipts require all current source execution and CPU evidence', () => { + const path = '.github/scripts/audit-packed-reduction-native.cjs'; + assert.ok(fs.existsSync(path), 'missing complete native reduction receipt auditor'); + const source = fs.readFileSync(path, 'utf8'); + for (const required of [ + 'auditNativeOwners', 'SSE3=1 SSE41=1 AVX=1 OSXSAVE=1', + 'PACKED_REDUCTION_NATIVE_SSE3_SSE41_AVX_OK', '6n', + 'original_go_named', 'original_go_raw', 'original_go_opcode_reference', + 'backend_O0', 'backend_O2', 'IR_O2_backend_O2', + ]) { + assert.ok(source.includes(required), 'missing strict native reduction guard: ' + required); + } +}); + +// Synthetic raw streams test only the auditor. Actual CPU/Go/LLVM execution +// remains required in the separate native jobs; this is not such a receipt. +function fixture(owner) { + const roots = ['Values', 'DeadFaults'].map(family => + 'TestCrossLinuxRuntimeMatrixX86PackedReductionNative' + family + 'Part' + owner); + const children = []; + for (let family = 0; family < 2; family++) { + const count = family ? 37 : 258, limit = family ? 8 : 64; + for (let start = owner * limit; start < count; start += 4 * limit) { + const batch = roots[family] + '/batch-' + String(start).padStart(4, '0'); + const originals = family ? ['original_go_opcode_reference'] : ['original_go_named', 'original_go_raw']; + children.push(batch, ...originals.map(original => batch + '/' + original)); + for (const spelling of ['named', 'raw']) { + const source = batch + '/' + spelling; + children.push(source); + for (const api of ['text', 'module', 'Go']) { + const route = source + '/' + api; + children.push(route, ...['backend_O0', 'backend_O2', 'IR_O2_backend_O2'].map(axis => route + '/' + axis)); + } + } + } + } + const events = []; + const emit = (Action, Test, Output) => { + const event = {Action, Package: 'github.com/xgo-dev/plan9asm'}; + if (Test) event.Test = Test; + if (Output !== undefined) event.Output = Output; + events.push(event); + }; + function execute(name) { + emit('run', name); + if (roots.includes(name)) { + emit('output', name, 'SSE3=1 SSE41=1 AVX=1 OSXSAVE=1 XCR0=7\n'); + emit('output', name, 'PACKED_REDUCTION_NATIVE_SSE3_SSE41_AVX_OK members=fff\n'); + } + for (const child of children.filter(child => child.slice(0, child.lastIndexOf('/')) === name)) execute(child); + emit('pass', name); + } + for (const root of roots) execute(root); + emit('pass'); + return {roots, children, inventory: roots.join('\n') + '\n', events}; +} + +function raw(events) { + return events.map(event => JSON.stringify(event)).join('\n') + '\n'; +} + +test('all four owner pairs retain the complete Go source, public API and pipeline axes', () => { + let roots = 0, children = 0; + for (let owner = 0; owner < 4; owner++) { + const independent = fixture(owner), expected = auditor.required(owner); + assert.deepEqual(expected.roots, independent.roots); + assert.deepEqual(expected.children, independent.children); + const result = auditor.auditOwner(independent.inventory, raw(independent.events), owner); + assert.equal(result.preflights, 2); + assert.equal(result.passed, independent.roots.length + independent.children.length); + roots += result.roots; + children += result.children; + } + assert.equal(roots, 8); + assert.equal(children, 285); +}); + +test('failed skipped missing duplicated foreign incomplete and printed receipts fail', () => { + const base = fixture(0), leaf = base.children.find(child => child.endsWith('/original_go_raw')); + for (const action of ['fail', 'skip']) { + const events = base.events.map(event => event.Test === leaf && event.Action === 'pass' ? {...event, Action: action} : event); + assert.throws(() => auditor.auditOwner(base.inventory, raw(events), 0)); + } + for (const change of [ + events => events.filter(event => event.Test !== leaf), + events => events.filter(event => !(event.Test === leaf && event.Action === 'pass')), + events => [...events.slice(0, -1), {Action: 'pass', Package: 'github.com/xgo-dev/plan9asm', Test: leaf}, events.at(-1)], + events => events.map(event => event.Test === leaf ? {...event, Test: leaf + '/foreign'} : event), + events => events.slice(0, -1), + events => [{Action: 'output', Package: 'github.com/xgo-dev/plan9asm', Output: 'all reductions passed\n'}, events.at(-1)], + ]) assert.throws(() => auditor.auditOwner(base.inventory, raw(change(base.events)), 0)); + assert.throws(() => auditor.auditOwner(base.inventory, raw(base.events), 1)); + assert.throws(() => auditor.auditOwner(base.inventory + 'TestForeign\n', raw(base.events), 0)); +}); + +test('all12 actual opcodes and enabled native state must precede every runtime child', () => { + const base = fixture(1); + for (const [before, after] of [ + ['SSE3=1', 'SSE3=0'], ['SSE41=1', 'SSE41=0'], ['AVX=1', 'AVX=0'], + ['OSXSAVE=1', 'OSXSAVE=0'], ['XCR0=7', 'XCR0=3'], + ['members=fff', 'members=7ff'], ['PACKED_REDUCTION_NATIVE_SSE3_SSE41_AVX_OK', 'not_executed'], + ]) { + const events = base.events.map(event => event.Output ? {...event, Output: event.Output.replace(before, after)} : event); + assert.throws(() => auditor.auditOwner(base.inventory, raw(events), 1), before); + } + const late = base.events.filter(event => !event.Output); + late.splice(late.length - 2, 0, ...base.events.filter(event => event.Output)); + assert.throws(() => auditor.auditOwner(base.inventory, raw(late), 1)); +}); + +test('complete reduction audit reopens all four current-head raw streams', t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'reduction-native-audit-')); + t.after(() => fs.rmSync(directory, {recursive: true, force: true})); + const head = 'a'.repeat(40), directories = []; + for (let owner = 0; owner < 4; owner++) { + const data = fixture(owner), child = path.join(directory, String(owner)); + fs.mkdirSync(child); + fs.writeFileSync(path.join(child, 'context.json'), JSON.stringify({schema: 1, head, owner})); + fs.writeFileSync(path.join(child, 'inventory.log'), data.inventory); + fs.writeFileSync(path.join(child, 'raw.jsonl'), raw(data.events)); + directories.push(child); + } + assert.deepEqual(auditor.auditComplete(directories, head), { + head, owners: 4, roots: 8, children: 285, passed: 293, + valueKernels: 258, faultKernels: 37, skipped: 0, incomplete: 0, + }); + assert.throws(() => auditor.auditComplete(directories.slice(0, -1), head)); + assert.throws(() => auditor.auditComplete([...directories.slice(0, -1), directories[0]], head)); + assert.throws(() => auditor.auditComplete(directories, 'b'.repeat(40))); + fs.writeFileSync(path.join(directories[3], 'raw.jsonl'), '{"allPassed":true}\n'); + assert.throws(() => auditor.auditComplete(directories, head)); +}); diff --git a/.github/scripts/priority-packed-string-partitions.test.cjs b/.github/scripts/priority-packed-string-partitions.test.cjs new file mode 100644 index 000000000..c4962f6a8 --- /dev/null +++ b/.github/scripts/priority-packed-string-partitions.test.cjs @@ -0,0 +1,39 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); + +const source = fs.readFileSync(path.join(__dirname, '../../amd64_packed_string_vector_test.go'), 'utf8'); + +for (const [name, arch, triple] of [ + ['Linux386', '386', 'i386-unknown-linux-gnu'], + ['Windows386', '386', 'i686-pc-windows-msvc'], + ['DarwinAMD64', 'amd64', 'x86_64-apple-darwin'], + ['LinuxAMD64', 'amd64', 'x86_64-unknown-linux-gnu'], + ['WindowsAMD64', 'amd64', 'x86_64-pc-windows-msvc'], +]) { + test('packed-string objects retain independent complete owner ' + name, () => { + const declaration = `func TestX86PackedStringObjects${name}(t *testing.T) {\n` + + `\tx86PackedStringTargetObjects(t, "${arch}", "${triple}")\n}`; + assert.ok(source.includes(declaration), 'missing independent target owner'); + }); +} + +test('packed-string partitioning retains every original row and object axis', () => { + assert.ok(source.includes('func TestX86PackedStringObjectBatchInventory(t *testing.T)')); + assert.ok(!source.includes('func TestX86PackedStringThreeAPIsFiveTargetsO0O2Objects(')); + assert.ok(source.includes('const maxFunctions = 64')); + assert.ok(source.includes('reflect.DeepEqual(joined, original)')); + const helper = source.slice(source.indexOf('func x86PackedStringTargetObjects('), + source.indexOf('\nfunc TestX86PackedStringObjectsLinux386(')); + assert.ok(helper.startsWith('func x86PackedStringTargetObjects(')); + assert.ok(helper.includes('x86PackedStringObjectBatches(x86PackedStringVectorCases())')); + assert.ok(helper.includes('[]string{"text", "module", "Go"}')); + assert.ok(helper.includes('[]string{"-O0", "-O2"}')); + assert.ok(helper.includes('x86LEAActualTargetGoObject(t, arch, triple, source)')); + assert.ok(helper.includes('input = x86UndefinedTrapOptimizedIR(t, input)')); + assert.ok(helper.includes('x86PackedStringCompileObject(t, llc, triple, input, level)')); + assert.ok(!helper.includes('t.Skip')); +}); diff --git a/.github/scripts/priority-private-leave-native.test.cjs b/.github/scripts/priority-private-leave-native.test.cjs new file mode 100644 index 000000000..7e45613e1 --- /dev/null +++ b/.github/scripts/priority-private-leave-native.test.cjs @@ -0,0 +1,100 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); + +const root = 'TestCrossLinuxRuntimeMatrixX86LeavePrivateEmitterWidthsAndFlags'; +const architectures = ['amd64', '386']; +const paths = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2']; +const groups = architectures.map(arch => root + '/' + arch); +const leaves = groups.flatMap(group => paths.map(path => group + '/' + path)); +const names = [root, ...groups, ...leaves]; +const hash = 'a'.repeat(64); +const host = `IA32 direct host: linux/amd64 kernel=x86_64 CPUsha=${hash} mapsha=${hash}\n`; +const sdk = arch => `actual LEA source/object receipt: SDK=go1.27.2 target=linux/${arch} source=${hash} driver=${hash} object=${hash}\n`; +const marker = arch => `private LEAVE numerical completion arch=${arch}: PRIVATE_LEAVE_128_WIDTH_VALUES_FLAGS_CANARIES_PASS\n`; +const image = `native oracle static IA32 ELF bytes=512 sha256=${hash}\n`; +const command = `actual executable=/tmp/private-leave sha256=${hash} argv=[]\n`; +const direct = `IA32 direct ELF bytes=512 sha256=${hash} argv=[]\n`; +const event = (Action, Test, Output) => ({Action, Package: 'github.com/xgo-dev/plan9asm', + ...(Test ? {Test} : {}), ...(Output !== undefined ? {Output} : {})}); + +function completeEvents() { + const events = [event('start'), event('run', root), event('output', root, host)]; + for (const arch of architectures) { + const group = root + '/' + arch; + events.push(event('run', group), event('output', group, sdk(arch))); + for (const path of paths) { + const leaf = group + '/' + path; + events.push(event('run', leaf), event('output', leaf, + (arch === '386' ? image + command + direct : '') + marker(arch)), event('pass', leaf)); + } + events.push(event('pass', group)); + } + events.push(event('output', root, host), event('pass', root), event('pass')); + return events; +} + +function audit(events = completeEvents(), inventory = root + '\n') { + assert.ok(fs.existsSync('.github/scripts/audit-private-leave-native.cjs'), 'private LEAVE lacks its exact native-width auditor'); + return require('./audit-private-leave-native.cjs').auditPrivateLEAVE(inventory, + events.map(value => JSON.stringify(value)).join('\n')); +} + +test('private LEAVE requires all four width values with six actual pipelines, not a native Go ABI claim', () => { + assert.deepEqual(audit(), {roots: 1, children: 8, passed: 9, skipped: 0, incomplete: 0, + nativeHosts: 2, sdkCaptures: 2, llvmExecutions: 6, staticIA32Executions: 3, privateNumericalOnly: true}); +}); + +test('private LEAVE rejects missing duplicate failed skipped incomplete and late lifecycle evidence', () => { + audit(); + for (const name of names) { + for (const Action of ['fail', 'skip']) assert.throws(() => audit(completeEvents().map(value => + value.Test === name && value.Action === 'pass' ? {...value, Action} : value)), name + Action); + for (const Action of ['run', 'pass']) { + const events = completeEvents(); + const at = events.findIndex(value => value.Test === name && value.Action === Action); + assert.throws(() => audit(events.filter((_, index) => index !== at)), name + ' missing ' + Action); + events.splice(at, 0, events[at]); + assert.throws(() => audit(events), name + ' duplicate ' + Action); + } + const events = completeEvents(); + const at = events.findIndex(value => value.Test === name && value.Action === 'pass'); + events.splice(at + 1, 0, event('output', name, 'late completion\n')); + assert.throws(() => audit(events), name + ' output outside running owner'); + } + for (const inventory of ['', root + '\n' + root + '\n', 'TestUnrelated\n']) assert.throws(() => audit(completeEvents(), inventory)); + for (const events of [completeEvents().slice(1), completeEvents().slice(0, -1), + [event('output', undefined, 'all widths passed')], [event('start'), ...completeEvents()], + [...completeEvents(), event('pass')], completeEvents().map(value => ({...value, Package: 'foreign'}))]) { + assert.throws(() => audit(events)); + } +}); + +test('private LEAVE binds actual host SDK static ELF hash and every complete execution to its own live owner', () => { + audit(); + for (const needle of [host, ...architectures.map(sdk), image, command, direct, ...architectures.map(marker)]) { + for (const replacement of ['', needle + needle, needle.replace(hash, 'invalid'), + needle.replace('128', '127').replace('linux/amd64', 'linux/arm64').replace('arch=386', 'arch=amd64')]) { + if (replacement === needle) continue; + assert.throws(() => audit(completeEvents().map(value => value.Output + ? {...value, Output: value.Output.replace(needle, replacement)} : value)), needle); + } + } + for (const needle of [image, command, direct]) { + assert.throws(() => audit(completeEvents().map(value => value.Output + ? {...value, Output: value.Output.replace(needle, needle.replace(hash, 'b'.repeat(64)))} : value))); + } + const lateSDK = completeEvents(); + const capture = lateSDK.splice(lateSDK.findIndex(value => value.Output === sdk('amd64')), 1)[0]; + lateSDK.splice(lateSDK.findIndex(value => value.Test === root + '/amd64/backend_O0' && value.Action === 'pass'), 0, capture); + assert.throws(() => audit(lateSDK), 'SDK must precede actual compilation/execution'); + const earlyHost = completeEvents(); + const post = earlyHost.splice(earlyHost.length - 3, 1)[0]; + earlyHost.splice(4, 0, post); + assert.throws(() => audit(earlyHost), 'final host must follow both complete architectures'); + const borrowed = completeEvents().map(value => value.Test === root + '/386/backend_O0' && value.Output + ? {...value, Test: root + '/386'} : value); + assert.throws(() => audit(borrowed), 'a parent cannot replace a leaf execution'); +}); diff --git a/.github/scripts/priority-product-cmov.test.cjs b/.github/scripts/priority-product-cmov.test.cjs new file mode 100644 index 000000000..c662c39e1 --- /dev/null +++ b/.github/scripts/priority-product-cmov.test.cjs @@ -0,0 +1,40 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const {auditProductCMOV, roots, children} = require('./audit-product-cmov.cjs'); + +test('product and CMOV execution retains every root and optimization child without skips', () => { + assert.deepEqual(roots, [ + 'TestCrossLinuxRuntimeMatrixX86DwordProductIndependentWholeBytes', + 'TestCrossLinuxRuntimeMatrixX86CMOVIndependentWholeBytes', + 'TestCrossLinuxRuntimeMatrixX86AESFamilyWholeByteAliases', + 'TestAMD64VectorTestRuntimeSemantics', + 'TestAMD64IFMARuntimeSemantics', + 'TestAMD64QwordBitLookupRuntimeSemantics', + 'TestCrossLinuxRuntimeMatrixX86CMOVFPPartialCrossSlotWholeBytes', + ]); + const event = (Action, Test) => ({Action, Package: 'github.com/xgo-dev/plan9asm', ...(Test ? {Test} : {})}); + const events = [event('start')]; + for (const root of roots) { + events.push(event('run', root)); + for (const child of children.filter(name => name.startsWith(root + '/'))) { + events.push(event('run', child), event('pass', child)); + } + events.push(event('pass', root)); + } + events.push(event('pass')); + const raw = values => values.map(JSON.stringify).join('\n') + '\n'; + const inventory = roots.join('\n') + '\n'; + assert.deepEqual(auditProductCMOV(inventory, raw(events)), + {roots: 7, children: 24, passed: 31, skipped: 0, incomplete: 0}); + for (const name of [...roots, ...children]) { + assert.throws(() => auditProductCMOV(inventory, raw(events.filter(event => event.Test !== name)))); + for (const Action of ['skip', 'fail']) { + assert.throws(() => auditProductCMOV(inventory, raw(events.map(event => + event.Test === name && event.Action === 'pass' ? {...event, Action} : event)))); + } + } + assert.throws(() => auditProductCMOV(inventory, raw(events.slice(0, -1)))); + assert.throws(() => auditProductCMOV(inventory, raw([event('start'), event('pass')]))); +}); diff --git a/.github/scripts/priority-race-shard-layout.test.cjs b/.github/scripts/priority-race-shard-layout.test.cjs new file mode 100644 index 000000000..358e3b301 --- /dev/null +++ b/.github/scripts/priority-race-shard-layout.test.cjs @@ -0,0 +1,94 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const {jobBody, requireJobNeeds} = require('./workflow-needs.cjs'); + +const workflow = fs.readFileSync(path.join(__dirname, '../workflows/go-ci.yml'), 'utf8'); +const script = fs.readFileSync(path.join(__dirname, '../../scripts/test-module-ci.sh'), 'utf8'); + +function step(body, title) { + const selected = body.split(/(?=^ - )/m).filter(item => item.startsWith(` - name: ${title}\n`)); + assert.equal(selected.length, 1, 'one exact complete-owner step required: ' + title); + return selected[0]; +} + +function requireRaceOwners(source) { + const body = jobBody(source, 'race'); + const roles = Array.from({length: 32}, (_, index) => 'root-' + index).concat('profile', 'other'); + const axis = body.match(/^ module-role: \[([^\]]+)\]$/m); + assert.ok(axis, 'explicit complete race scheduling owners'); + assert.deepEqual(axis[1].split(',').map(value => value.trim()), roles); + const mappings = [...body.matchAll( + /^ - \{module-role: (root-\d+|profile|other), module-owner: ([012]), root-partition: (-?\d+)\}$/gm, + )].map(([, role, owner, part]) => [role, Number(owner), Number(part)]); + assert.deepEqual(mappings, [ + ...Array.from({length: 32}, (_, index) => ['root-' + index, 0, index]), + ['profile', 1, -1], ['other', 2, -1], + ]); + assert.match(body, /^ needs: ci_policy$/m); + assert.match(body, /^ timeout-minutes: 75$/m); + assert.match(body, /^ fail-fast: false$/m); + assert.doesNotMatch(body, /^ if:|max-parallel:|continue-on-error:/m); + assert.match(step(body, 'Go test (race)'), + /^ run: bash scripts\/test-module-ci\.sh race execute \$\{\{ matrix\.module-owner \}\} \$\{\{ matrix\.root-partition \}\}$/m); +} + +function requireCompleteRaceDownloads(source) { + const body = jobBody(source, 'module-completion'); + requireJobNeeds(source, 'module-completion', ['test', 'nested-cli', 'race', 'coverage', 'nested-cli-coverage']); + assert.match(body, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + assert.deepEqual([...body.matchAll(/^ - name: Download root part (\d+)$/gm)].map(match => Number(match[1])), + Array.from({length: 32}, (_, index) => index)); + for (let index = 0; index < 32; index++) { + const download = step(body, 'Download root part ' + index); + assert.ok(download.includes('name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-' + index + '\n')); + assert.ok(download.includes('path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-' + + String(index).padStart(4, '0') + '\n')); + if (index >= 16) assert.match(download, /^ if: \$\{\{ !cancelled\(\) && matrix\.flavor == 'race' \}\}$/m); + else assert.doesNotMatch(download, /^ if:.*matrix\.flavor/m); + } + assert.doesNotMatch(body, /merge-multiple:|pattern:/); +} + +test('race owns 64 complete shards through 32 independent two-shard jobs', () => { + requireRaceOwners(workflow); +}); + +test('complete aggregation retains every new race owner without raising budgets', () => { + requireCompleteRaceDownloads(workflow); +}); + +test('race script selects its complete layout for execution and independent completion', () => { + assert.match(script, /^root_partitions=16$/m); + assert.match(script, /^shards=32$/m); + assert.ok(script.includes('race) timeout=60m; flags=(-race); root_partitions=32; shards=64 ;;')); + assert.equal([...script.matchAll(/-shards=\$shards -workers=2/g)].length, 2); + assert.match(script, /GOMAXPROCS=2/); + assert.match(script, /test\) timeout=45m ;;/); + assert.match(script, /coverage\) timeout=45m; flags=\(-coverprofile=coverage-root\.out\) ;;/); +}); + +test('race layout rejects an omitted owner, four-shard owner, masked failure or incomplete readback', () => { + requireRaceOwners(workflow); + requireCompleteRaceDownloads(workflow); + const race = jobBody(workflow, 'race'); + for (const [before, after] of [ + ['root-31,', ''], + ['root-partition: 31', 'root-partition: 30'], + ['timeout-minutes: 75', 'timeout-minutes: 90'], + ['fail-fast: false', 'fail-fast: true'], + ['timeout-minutes: 75', 'timeout-minutes: 75\n continue-on-error: true'], + ]) { + const changed = race.replace(before, after); + assert.notEqual(changed, race); + assert.throws(() => requireRaceOwners(workflow.replace(race, () => changed))); + } + const complete = jobBody(workflow, 'module-completion'); + for (const index of [8, 16, 24, 31]) { + const download = step(complete, 'Download root part ' + index); + assert.throws(() => requireCompleteRaceDownloads(workflow.replace(download, ''))); + } +}); diff --git a/.github/scripts/priority-readonly-data-fresh-leases.test.cjs b/.github/scripts/priority-readonly-data-fresh-leases.test.cjs new file mode 100644 index 000000000..84727c55f --- /dev/null +++ b/.github/scripts/priority-readonly-data-fresh-leases.test.cjs @@ -0,0 +1,82 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const {test} = require('node:test'); + +const read = file => fs.readFileSync(file, 'utf8'); +const sha = value => crypto.createHash('sha256').update(value).digest('hex'); +const matrix = read('amd64_fixed_local_data_matrix_go127_test.go'); +const objects = read('amd64_fixed_local_data_go127_test.go'); +const runtime = read('amd64_fixed_local_data_runtime_go127_test.go'); +const frame = read('amd64_fixed_local_frame_test.go'); +const fresh = frame.slice(frame.indexOf('func fixedLocalFreshAPIs('), frame.indexOf('\nfunc fixedLocalAPI(')); +const single = frame.slice(frame.indexOf('func fixedLocalAPI('), frame.indexOf('\nfunc fixedLocalRequireContext(')); + +function requireFreshOwner(source) { + assert.match(source, /fixedLocalFreshAPIs\(t, pkg, request, options, \[\]string\{"Translate", "Module", "GoModule"\}/); + assert.doesNotMatch(source, /requireGoAMD64FrameCapture\(|runtimeOracleStageContext\(|source\.BindFixedLocal\(|fixedLocalDataAPIs\(/); +} + +function originalCacheSetup(source, owners) { + // Cache policy checks owner selection and preparation order; Go lifecycle + // tests exercise the process-owned cache and independent source workspaces. + // Normalize only this setup call, keeping the original numerical and + // mutation/Close fingerprints unchanged. + const prepared = '\tgoAMD64FramePrepareOwnedToolCache(t)\n'; + assert.equal(source.split(prepared).length - 1, owners); + return source.replaceAll(prepared, '\tt.Setenv("GOCACHE", t.TempDir())\n'); +} + +test('every complete readonly DATA consumer owns a fresh lease per API', () => { + requireFreshOwner(matrix.slice(matrix.indexOf('func fixedLocalDataTargetConsumers('), matrix.indexOf('\nfunc TestGoAMD64FixedLocalReadonlyDataRejectsUnownedAccesses('))); + requireFreshOwner(objects.slice(objects.indexOf('func TestGoAMD64FixedLocalReadonlyDataThreeTargetObjects('), objects.indexOf('\nfunc fixedLocalDataAPIs('))); + requireFreshOwner(runtime); +}); + +test('the shared fresh owner passes complete preprocessed DATA to the single API implementation', () => { + assert.match(fresh, /fixedLocalAPI\(t, pkg, file, bound, api, check, expanded\)/); + assert.match(single, /assembly \.\.\.string/); + assert.match(single, /len\(assembly\) > 1/); + assert.match(single, /len\(assembly\) == 1 && assembly\[0\] == ""/); + assert.match(single, /if len\(assembly\) == 1 \{\s*source\.WriteString\(assembly\[0\]\)/); + assert.match(single, /TranslateGoModule\(pkg, \[\]byte\(source\.String\(\)\)/); + assert.match(fresh, /defer cancel\(\)/); + assert.match(fresh, /defer func\(\) \{\s*if err := source\.Close\(\)/); + assert.match(single, /defer ctx\.Dispose\(\)/); + assert.match(single, /defer module\.Dispose\(\)/); + assert.match(single, /defer translation\.Module\.Dispose\(\)/); +}); + +test('the complete literal matrix retains all original 119 operations and four source batches on every target', () => { + const builder = matrix.slice(matrix.indexOf('func fixedLocalDataOperations('), matrix.indexOf('\nfunc TestGoAMD64FixedLocalReadonlyDataLinuxRegisteredConsumers(')); + assert.equal(sha(builder), 'e34a575231dceb014395766eba83308b6a8e078a57082b90ccc6ed92b5b3d3d9'); + for (const [target, triple] of [['linux/amd64', 'x86_64-unknown-linux-gnu'], ['darwin/amd64', 'x86_64-apple-darwin'], ['windows/amd64', 'x86_64-pc-windows-msvc']]) { + assert.ok(matrix.includes(`fixedLocalDataTargetConsumers(t, "${target}", "${triple}")`)); + } + assert.match(matrix, /len\(operations\) != 119/); + assert.match(matrix, /for start := 0; start < len\(operations\); start \+= 32/); + assert.match(matrix, /for index := start; index < end; index\+\+/); + assert.match(matrix, /assembly\.WriteString\(strings\.Replace\(fixedLocalLiteralData, "\$32", "\$64", 1\)\)/); + assert.match(matrix, /compileLLVMToObject\(t, llc, triple, "literal-consumers\.ll"/); + assert.match(matrix, /compileLLVMToObject\(t, llc, triple, "literal-consumers-opt\.ll"/); +}); + +test('original runtime values are unchanged outside independent tool preparation', () => { + const original = originalCacheSetup(runtime.slice(0, runtime.indexOf('\tpkg, request,')), 1); + assert.equal(sha(original), '85e040739a6a7623e4ef1797241e4ee9ab93bca60203975369762e6e68b16eda'); + assert.match(runtime, /fixedLocalPackedRunOriginal\(t, execution, assembly\.String\(\), declarations\.String\(\), goOracle, true\)/); + assert.match(runtime, /\[\]string\{"-O0", "-O2"\}/); +}); + +test('negative shared-binding mutation/Close controls are not renewed', () => { + const guards = read('amd64_fixed_local_data_guards_go127_test.go'); + const additional = guards.indexOf('\nfunc TestGoAMD64FixedLocalReadonlyDataCLIEmissionIdentity('); + assert.ok(additional > 0, 'the independent emission-identity regression must remain present'); + // Appending another independent owner must not change the original controls' + // fingerprint or require pretending it shares their cache/observation. + const original = originalCacheSetup(guards.slice(0, additional), 2); + assert.equal(sha(original), '3730f8eaf2f8485989658c94056002b6508f8cae74a14ba56c7c6c956f472d1a'); + originalCacheSetup(guards.slice(additional), 1); +}); diff --git a/.github/scripts/priority-runtime-integration.test.cjs b/.github/scripts/priority-runtime-integration.test.cjs new file mode 100644 index 000000000..47bacbc35 --- /dev/null +++ b/.github/scripts/priority-runtime-integration.test.cjs @@ -0,0 +1,293 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const {jobBody, jobNeeds} = require('./workflow-needs.cjs'); + +const directory = path.join(__dirname, '../workflows'); +const goCI = fs.readFileSync(path.join(directory, 'go-ci.yml'), 'utf8'); +const runtimeFile = path.join(directory, 'runtime-integration.yml'); +const runtime = fs.existsSync(runtimeFile) ? fs.readFileSync(runtimeFile, 'utf8') : ''; +const benchmarkScript = fs.readFileSync(path.join(__dirname, '../../scripts/benchmark-compile.sh'), 'utf8'); +const instructionJobs = ['ci_policy', 'fmt', 'build', 'stdlib-corpus', 'cross-runtime', + 'cross-runtime-verify', 'test', 'nested-cli', 'windows-module', 'arm-scan', + 'race', 'coverage', 'nested-cli-coverage', 'module-completion']; +const linuxTargets = ['linux/386', 'linux/amd64', 'linux/arm', 'linux/arm64', + 'darwin/amd64', 'darwin/arm64', 'windows/386', 'windows/amd64', 'windows/arm64', 'js/wasm']; +const corpusTargets = [linuxTargets, + ...Array.from({length: 7}, () => [...linuxTargets, 'wasip1/wasm']), + ['darwin/amd64', 'darwin/arm64'], ['windows/386', 'windows/amd64', 'windows/arm64']]; + +function jobNames(source) { + const sections = source.split('\njobs:\n'); + assert.equal(sections.length, 2, 'one explicit jobs section'); + return [...sections[1].matchAll(/^ ([a-zA-Z_][a-zA-Z0-9_-]*):\s*$/gm)].map(match => match[1]); +} + +function requireJobSet(source, expected) { + assert.deepEqual(jobNames(source).sort(), expected.slice().sort(), 'complete, unique job ownership'); +} + +function requireExactSDK(body) { + const setup = body.split(/(?=^ - )/m).filter(step => + /^ uses: \.\/\.github\/actions\/setup-go$/m.test(step)); + assert.equal(setup.length, 1, 'one observed SDK setup'); + assert.match(setup[0], /^ go-version: '1\.27\.2'$/m); + assert.doesNotMatch(body, /go-version-file:|check-latest:|uses: actions\/setup-go@/); +} + +function requireRuntimeIntegration(source) { + assert.match(source, /^name: SDK\/runtime integration$/m); + const triggers = source.match(/^on:\n([\s\S]*?)(?=^\S)/m); + assert.ok(triggers, 'explicit standalone workflow triggers'); + assert.deepEqual([...triggers[1].matchAll(/^ ([a-z_]+):[^\n]*$/gm)].map(match => match[1]), + ['workflow_dispatch', 'schedule']); + assert.match(triggers[1], /^ - cron: '17 3 \* \* 1'$/m, 'one weekly schedule'); + assert.equal([...triggers[1].matchAll(/cron:/g)].length, 1); + assert.match(source, /^ cancel-in-progress: false$/m); + assert.match(source, /^permissions:\n contents: read$/m); + requireJobSet(source, ['ci_policy', 'benchmark', 'stdlib-corpus']); + assert.doesNotMatch(source, /continue-on-error:|\|\|\s*true|\bexit 0\b/); + assert.doesNotMatch(source, /PLAN9ASM_BENCHMARK_(?:TARGETS|REPORT_DIR):/); + + const policy = jobBody(source, 'ci_policy'); + assert.doesNotMatch(policy, /^ (?:needs|if):/m); + assert.match(policy, /^ timeout-minutes: 20$/m); + assert.match(policy, /^ run: bash scripts\/install-ci-llvm22-ubuntu\.sh$/m); + assert.match(policy, /^ run: node --test \.github\/scripts\/priority-\*\.test\.cjs$/m); + assert.match(policy, /^ run: go test \. -run '\^TestCI' -count=1$/m); + requireExactSDK(policy); + + const job = jobBody(source, 'benchmark'); + assert.deepEqual(jobNeeds(source, 'benchmark'), ['ci_policy']); + assert.match(job, /^ name: complete standard-library SDK\/runtime compile \(Go 1\.27, LLVM 22\)$/m); + assert.match(job, /^ timeout-minutes: 35$/m); + assert.doesNotMatch(job, /^ if:/m); + requireExactSDK(job); + const steps = job.split(/(?=^ - )/m).slice(1); + assert.equal(steps.length, 5, 'retain every original execution and report step'); + assert.match(steps[0], /^ uses: actions\/checkout@v7$/m); + assert.match(steps[1], /^ run: bash scripts\/install-ci-llvm22-ubuntu\.sh$/m); + assert.match(steps[1], /^ timeout-minutes: 14$/m); + for (const step of steps.slice(0, 4)) assert.doesNotMatch(step, /^ if:/m); + assert.match(steps[3], /^ run: scripts\/benchmark-compile\.sh$/m); + const environment = [...steps[3].matchAll(/^ ([A-Z0-9_]+): (.+)$/gm)]; + assert.deepEqual(environment.map(match => [match[1], match[2]]), [ + ['PLAN9ASM_BENCHMARK_MAX_TARGET_SECONDS', "'300'"], + ['PLAN9ASM_BENCHMARK_MAX_TOTAL_SECONDS', "'900'"], + ]); + assert.match(steps[4], /^ if: always\(\)$/m); + assert.match(steps[4], /^ uses: actions\/upload-artifact@v6$/m); + assert.match(steps[4], /^ name: compile-benchmark$/m); + assert.match(steps[4], /^ path: _out\/compile-benchmark$/m); + assert.match(steps[4], /^ if-no-files-found: error$/m); + + const corpus = jobBody(source, 'stdlib-corpus'); + assert.deepEqual(jobNeeds(source, 'stdlib-corpus'), ['ci_policy']); + assert.match(corpus, /^ name: complete stdlib source\/object corpus \(/m); + assert.doesNotMatch(corpus, /^ if:|--instructions-only/m); + assert.match(corpus, /^ scripts\/check-stdlib-corpus\.sh$/m); + requireCorpusTargets(corpus); + // The complete matrix, tools and prebuild transport are still the original + // corpus job. Only its instruction-only call and redundant official-table + // step differ from the required PR instruction owner. + assert.equal(corpusBody(corpus), corpusBody(jobBody(goCI, 'stdlib-corpus')), + 'full SDK/OS/architecture-profile matrix and tool requirements must match'); +} + +function corpusBody(body) { + return body.replace(/^ name: [^\n]+\n/m, '') + .replace('scripts/check-stdlib-corpus.sh --instructions-only', 'scripts/check-stdlib-corpus.sh') + .split('\n - name: Official Go assembler coverage regression\n')[0].trimEnd(); +} + +function requireCorpusTargets(body) { + const actual = [...body.matchAll(/targets: '([^']+)'/g)].map(match => match[1].split(',')); + assert.deepEqual(actual, corpusTargets, + 'every SDK/host lane must retain its complete, independently known OS/architecture inventory'); +} + +function requireInstructionOwners(source) { + requireJobSet(source, instructionJobs); + assert.doesNotMatch(source, /runtime-integration\.yml|scripts\/benchmark-compile\.sh/); + for (const name of ['fmt', 'build', 'stdlib-corpus', 'cross-runtime', 'test', 'nested-cli', + 'windows-module', 'arm-scan', 'race', 'coverage', 'nested-cli-coverage']) { + assert.deepEqual(jobNeeds(source, name), ['ci_policy'], name); + assert.doesNotMatch(jobBody(source, name), /^ if:|continue-on-error:/m, name); + } + assert.deepEqual(jobNeeds(source, 'cross-runtime-verify'), ['cross-runtime']); + assert.deepEqual(jobNeeds(source, 'module-completion'), + ['test', 'nested-cli', 'race', 'coverage', 'nested-cli-coverage']); + assert.match(jobBody(source, 'build'), /run: scripts\/check-arm64-plan9-corpus\.sh/); + const corpus = jobBody(source, 'stdlib-corpus'); + assert.match(corpus, /^ name: stdlib instruction corpus \(/m); + assert.match(corpus, /^ scripts\/check-stdlib-corpus\.sh --instructions-only$/m); + assert.match(corpus, /scripts\/check-go-asm-coverage\.sh/); + requireCorpusTargets(corpus); + assert.match(jobBody(source, 'cross-runtime'), /scripts\/install-ci-qemu\.sh/); + assert.match(jobBody(source, 'windows-module'), /uses: \.\/\.github\/workflows\/windows-module\.yml/); + assert.match(jobBody(source, 'coverage'), /bash scripts\/test-module-ci\.sh coverage execute/); + assert.match(jobBody(source, 'module-completion'), /files: \.\/coverage-root\.out/); +} + +function withoutJob(source, name) { + return source.replace('\n ' + name + ':\n' + jobBody(source, name), '\n'); +} + +// The positive mutation fixture is the intended move of the real existing +// job, not a claimed execution result. Actual workflow tests below never use +// this fallback to hide a missing integration workflow. +let instructionFixture = /\n benchmark:\n/.test(goCI) ? withoutJob(goCI, 'benchmark') : goCI; +instructionFixture = instructionFixture.replace('name: stdlib-corpus (', 'name: stdlib instruction corpus (') + .replace(' scripts/check-stdlib-corpus.sh\n', ' scripts/check-stdlib-corpus.sh --instructions-only\n'); +let runtimeFixture = runtime; +if (!runtimeFixture) { + const policy = jobBody(goCI, 'ci_policy').replace("go-version: '1.27.1'", "go-version: '1.27.2'"); + const benchmark = jobBody(goCI, 'benchmark').replace('name: compile benchmark (Go 1.27, LLVM 22)', + 'name: complete standard-library SDK/runtime compile (Go 1.27, LLVM 22)'); + const corpus = jobBody(goCI, 'stdlib-corpus') + .replace('name: stdlib-corpus (', 'name: complete stdlib source/object corpus (') + .split('\n - name: Official Go assembler coverage regression\n')[0]; + runtimeFixture = "name: SDK/runtime integration\n\non:\n workflow_dispatch:\n schedule:\n" + + " - cron: '17 3 * * 1'\n\nconcurrency:\n" + + " group: ${{ github.workflow }}-${{ github.ref }}\n cancel-in-progress: false\n\n" + + 'permissions:\n contents: read\n\njobs:\n ci_policy:\n' + policy + '\n benchmark:\n' + benchmark + + '\n stdlib-corpus:\n' + corpus; +} + +test('complete SDK/runtime integration has independent manual and weekly strict ownership', () => { + requireRuntimeIntegration(runtime); +}); + +test('PR Go CI retains every instruction owner without a successful runtime placeholder', () => { + requireInstructionOwners(goCI); +}); + +test('the unchanged strict benchmark still requires full five-target whole-source compilation', () => { + assert.ok(benchmarkScript.includes('targets_csv=${PLAN9ASM_BENCHMARK_TARGETS:-linux/386,linux/amd64,linux/arm,linux/arm64,js/wasm}')); + for (const flag of ['-patterns=std', '-compile', '-strict-load', '-keep-going=true']) { + assert.ok(benchmarkScript.includes(' ' + flag + ' \\'), flag); + } + assert.match(benchmarkScript, /\.failed == 0 and \.not_applicable == 0 and \.success == \.total_asm/); + assert.match(benchmarkScript, /if \(\( budget_failed != 0 \|\| compile_failed != 0 \)\); then\n exit 1/); + assert.match(benchmarkScript, /benchmark reports already exist; select a fresh PLAN9ASM_BENCHMARK_REPORT_DIR/); +}); + +test('runtime policy rejects trigger, compiler, budget, source, report and failure masking drift', async t => { + requireRuntimeIntegration(runtimeFixture); + const changes = [ + ['PR trigger', ' workflow_dispatch:\n', ' workflow_dispatch:\n pull_request:\n'], + ['inline PR trigger', ' workflow_dispatch:\n', ' workflow_dispatch:\n pull_request: {}\n'], + ['push trigger', ' workflow_dispatch:\n', ' workflow_dispatch:\n push:\n'], + ['inline push trigger', ' workflow_dispatch:\n', ' workflow_dispatch:\n push: {}\n'], + ['call trigger', ' workflow_dispatch:\n', ' workflow_dispatch:\n workflow_call:\n'], + ['duplicate manual trigger', ' workflow_dispatch:\n', ' workflow_dispatch:\n workflow_dispatch: {}\n'], + ['duplicate schedule trigger', ' schedule:\n', ' schedule:\n schedule:\n'], + ['missing manual', ' workflow_dispatch:\n', ''], + ['daily schedule', "cron: '17 3 * * 1'", "cron: '17 3 * * *'"], + ['cancel prior evidence', 'cancel-in-progress: false', 'cancel-in-progress: true'], + ['missing policy command', "run: go test . -run '^TestCI' -count=1", 'run: echo policy'], + ['foreign SDK', "go-version: '1.27.2'", "go-version: '1.27.3'"], + ['floating SDK', "go-version: '1.27.2'", "go-version: '1.27.x'"], + ['wrong LLVM', 'run: bash scripts/install-ci-llvm22-ubuntu.sh', 'run: echo LLVM23'], + ['job budget', 'timeout-minutes: 35', 'timeout-minutes: 45'], + ['target budget', "PLAN9ASM_BENCHMARK_MAX_TARGET_SECONDS: '300'", "PLAN9ASM_BENCHMARK_MAX_TARGET_SECONDS: '600'"], + ['total budget', "PLAN9ASM_BENCHMARK_MAX_TOTAL_SECONDS: '900'", "PLAN9ASM_BENCHMARK_MAX_TOTAL_SECONDS: '1800'"], + ['partial target override', "PLAN9ASM_BENCHMARK_MAX_TOTAL_SECONDS: '900'", "PLAN9ASM_BENCHMARK_MAX_TOTAL_SECONDS: '900'\n PLAN9ASM_BENCHMARK_TARGETS: linux/amd64"], + ['successful placeholder', 'run: scripts/benchmark-compile.sh', 'run: echo success'], + ['arguments substitute sources', 'run: scripts/benchmark-compile.sh', 'run: scripts/benchmark-compile.sh -patterns=math'], + ['hidden command failure', 'run: scripts/benchmark-compile.sh', 'run: scripts/benchmark-compile.sh || true'], + ['hidden job failure', ' timeout-minutes: 35', ' timeout-minutes: 35\n continue-on-error: true'], + ['skipped job', ' timeout-minutes: 35', ' timeout-minutes: 35\n if: false'], + ['skipped compile', ' - name: Translate and compile standard-library assembly', ' - name: Translate and compile standard-library assembly\n if: false'], + ['lost failure report', 'if: always()', 'if: success()'], + ['missing report', 'if-no-files-found: error', 'if-no-files-found: ignore'], + ['wrong report', 'path: _out/compile-benchmark', 'path: _out/partial-benchmark'], + ['runtime corpus reduced to scanning', ' scripts/check-stdlib-corpus.sh\n', ' scripts/check-stdlib-corpus.sh --instructions-only\n'], + ['missing corpus target', 'linux/arm64,', ''], + ['missing corpus prebuild', 'go build -C cmd/plan9asm -o "$tool" .', 'echo no-tool'], + ]; + for (const [name, before, after] of changes) { + await t.test(name, () => { + assert.ok(runtimeFixture.includes(before), 'mutation must apply'); + assert.throws(() => requireRuntimeIntegration(runtimeFixture.replaceAll(before, after))); + }); + } + await t.test('missing benchmark policy needs', () => { + assert.throws(() => requireRuntimeIntegration(runtimeFixture.replace(' needs: ci_policy\n', ''))); + }); + await t.test('duplicate benchmark owner', () => { + assert.throws(() => requireRuntimeIntegration(runtimeFixture + '\n benchmark:\n run: true\n')); + }); + await t.test('missing full source/object corpus owner', () => { + assert.throws(() => requireRuntimeIntegration(withoutJob(runtimeFixture, 'stdlib-corpus'))); + }); + for (const line of jobBody(runtimeFixture, 'stdlib-corpus').split('\n').filter(line => + /^ - \{os: /.test(line))) { + await t.test('missing full corpus lane ' + line.trim(), () => { + assert.throws(() => requireRuntimeIntegration(runtimeFixture.replace(line + '\n', ''))); + }); + } +}); + +test('responsibility split cannot delete or replace a required instruction owner', async t => { + requireInstructionOwners(instructionFixture); + for (const name of instructionJobs) { + await t.test('missing ' + name, () => { + assert.throws(() => requireInstructionOwners(withoutJob(instructionFixture, name))); + }); + } + for (const [before, after] of [ + ['scripts/check-go-asm-coverage.sh', 'echo official-coverage'], + ['scripts/check-stdlib-corpus.sh', 'echo corpus'], + ['scripts/check-stdlib-corpus.sh --instructions-only', 'scripts/check-stdlib-corpus.sh'], + ['scripts/check-arm64-plan9-corpus.sh', 'echo arm64-corpus'], + ['linux/arm64,', ''], + ['scripts/install-ci-qemu.sh', 'echo native'], + ['files: ./coverage-root.out', 'files: ./partial-coverage.out'], + [' needs: [test, nested-cli, race, coverage, nested-cli-coverage]', + ' needs: [test, nested-cli, race, nested-cli-coverage]'], + [' needs: [test, nested-cli, race, coverage, nested-cli-coverage]', + ' needs: [test, nested-cli, race, coverage]'], + ['uses: ./.github/workflows/windows-module.yml', 'uses: ./.github/workflows/runtime-integration.yml'], + ]) { + await t.test('replace ' + before, () => { + assert.ok(instructionFixture.includes(before), 'mutation must apply'); + assert.throws(() => requireInstructionOwners(instructionFixture.replaceAll(before, after))); + }); + } + await t.test('successful old benchmark identity', () => { + assert.throws(() => requireInstructionOwners(instructionFixture + '\n benchmark:\n run: echo success\n')); + }); +}); + +test('both corpus copies independently reject every missing or duplicate OS/architecture target', async t => { + requireCorpusTargets(jobBody(goCI, 'stdlib-corpus')); + requireRuntimeIntegration(runtimeFixture); + requireInstructionOwners(instructionFixture); + function replaceTargets(source, lane, targets) { + const body = jobBody(source, 'stdlib-corpus'); + const field = [...body.matchAll(/targets: '([^']+)'/g)][lane]; + const changed = body.slice(0, field.index) + "targets: '" + targets.join(',') + "'" + + body.slice(field.index + field[0].length); + assert.notEqual(changed, body, 'target mutation must apply'); + return source.replace(body, changed); + } + for (const [lane, targets] of corpusTargets.entries()) { + for (const [index, target] of targets.entries()) { + for (const operation of ['missing', 'duplicate']) { + await t.test('lane ' + lane + '/' + target + '/' + operation, () => { + const changed = targets.slice(); + if (operation === 'missing') changed.splice(index, 1); + else changed.splice(index, 0, target); + // Change both copies together: cross-workflow equality alone must + // never make a jointly reduced or duplicated matrix acceptable. + assert.throws(() => requireInstructionOwners(replaceTargets(instructionFixture, lane, changed))); + assert.throws(() => requireRuntimeIntegration(replaceTargets(runtimeFixture, lane, changed))); + }); + } + } + } +}); diff --git a/.github/scripts/priority-scalar-bmi.test.cjs b/.github/scripts/priority-scalar-bmi.test.cjs new file mode 100644 index 000000000..29173d301 --- /dev/null +++ b/.github/scripts/priority-scalar-bmi.test.cjs @@ -0,0 +1,22 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const {auditScalarBMI, roots} = require('./audit-scalar-bmi.cjs'); + +test('native scalar BMI execution cannot be replaced by skips, output text or an incomplete owner', () => { + const event = (Action, Test) => ({Action, Package:'github.com/xgo-dev/plan9asm', ...(Test ? {Test} : {})}); + const events = [event('start'), ...roots.flatMap(root => [event('run', root), event('pass', root)]), event('pass')]; + const raw = values => values.map(JSON.stringify).join('\n') + '\n'; + const inventory = roots.join('\n') + '\n'; + assert.deepEqual(auditScalarBMI(inventory, raw(events)), + {roots:3, children:0, passed:3, skipped:0, incomplete:0}); + for (const root of roots) { + assert.throws(() => auditScalarBMI(inventory, raw(events.filter(event => event.Test !== root)))); + assert.throws(() => auditScalarBMI(inventory, raw(events.map(event => + event.Test === root && event.Action === 'pass' ? {...event, Action:'skip'} : event)))); + } + assert.throws(() => auditScalarBMI(inventory, raw(events.slice(0, -1)))); + assert.throws(() => auditScalarBMI(inventory, + raw([event('start'), {...event('output'), Output:roots.join('\nPASS ')}, event('pass')]))); +}); diff --git a/.github/scripts/priority-scalar-reciprocal-native.test.cjs b/.github/scripts/priority-scalar-reciprocal-native.test.cjs new file mode 100644 index 000000000..96e83993c --- /dev/null +++ b/.github/scripts/priority-scalar-reciprocal-native.test.cjs @@ -0,0 +1,157 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); + +// Deliberately independent of the implementation's denominator and receipts. +const roots = [ + 'TestCrossLinuxRuntimeMatrixX86ScalarReciprocalOriginalGoAndNativeAMD64', + 'TestCrossLinuxRuntimeMatrixX86ScalarReciprocalOriginalGoAndNative386', +]; +const axes = ['backend_O0', 'backend_O2', 'IR_O2/backend_O2', 'OriginalGo']; +const hash = 'a'.repeat(64), imageHash = 'b'.repeat(64); +const host = `IA32 direct host: linux/amd64 kernel=x86_64 CPUsha=${hash} mapsha=${hash}\n`; +const avx = 'actual guest CPUID/XCR0 preflight: AVX=1 AVX2=0 AVX512F=0\n'; +const image = `native oracle static IA32 ELF bytes=4096 sha256=${imageHash}\n`; +const direct = args => `IA32 direct ELF bytes=4096 sha256=${imageHash} argv=${args}\n`; +const sdk = args => `actual executable=/sdk/bin/go sha256=${hash} argv=${args}\n`; +const executable = (filename, args) => `actual executable=/tmp/owned/${filename} sha256=${imageHash} argv=${args}\n`; +const originalArgs = '["-test.run=^TestOriginal$" "-test.count=1" "-test.v"]'; +const compileArgs = '["test" "-p=1" "-vet=off" "-c" "-o" "/tmp/owned/original.test" "."]'; +const evidence = arch => `scalar estimates native arch=${arch} live=20 dead=4 whole32-outcomes=30720 fault-processes=40; SSE/AVX actual scalar opcodes, no AVX512/Z transport\n`; +const originalEvidence = 'actual original Go scalar estimates: 20 aliases × 24 bit patterns × 64 MXCSR profiles = 30720 whole32/FLAGS/DF/MXCSR/canary outcomes; independent scalar references, no AVX512/Z execution claim\n'; +const inventory = roots.join('\n') + '\nok\tgithub.com/xgo-dev/plan9asm\n'; +const event = (Action, Test, Output) => ({Action, Package: 'github.com/xgo-dev/plan9asm', + ...(Test ? {Test} : {}), ...(Output !== undefined ? {Output} : {})}); +const serialize = events => events.map(value => JSON.stringify(value)).join('\n'); + +function completeEvents() { + const events = [event('start')]; + for (const root of roots) { + const arch = root.endsWith('386') ? '386' : 'amd64'; + events.push(event('run', root), event('output', root, host + avx)); + for (const axis of axes) { + const child = root + '/' + axis; + events.push(event('run', child)); + if (axis === 'OriginalGo') { + events.push(event('output', child, sdk('["version"]') + sdk(compileArgs))); + if (arch === '386') events.push(event('output', child, + image + executable('original.test', originalArgs) + direct(originalArgs))); + events.push(event('output', child, originalEvidence)); + } else { + if (arch === '386') events.push(event('output', child, + image + executable('comparison', '[]') + direct('[]') + image)); + events.push(event('output', child, evidence(arch))); + } + events.push(event('pass', child)); + } + events.push(event('output', root, host), event('pass', root)); + } + events.push(event('pass')); + return events; +} + +function audit(events = completeEvents(), discovered = inventory) { + const auditor = '.github/scripts/audit-scalar-reciprocal-native.cjs'; + assert.ok(fs.existsSync(auditor), 'scalar estimates lack an independent strict native completion auditor'); + return require('./audit-scalar-reciprocal-native.cjs').auditScalarReciprocalNative(discovered, serialize(events)); +} + +test('scalar estimate native ownership requires two roots, three optimized native axes and actual Go', () => { + assert.deepEqual(audit(), {roots: 2, children: 8, passed: 10, skipped: 0, incomplete: 0, + nativeHosts: 4, avxPreflights: 2, sdkBuilds: 2, staticIA32Images: 7, directIA32Executions: 4}); +}); + +test('scalar estimates reject every missing duplicate skipped failed or incomplete execution terminal', () => { + audit(); // A failing/missing auditor must not make the negative controls pass. + for (const name of roots.flatMap(root => [root, ...axes.map(axis => root + '/' + axis)])) { + for (const Action of ['fail', 'skip']) { + assert.throws(() => audit(completeEvents().map(value => value.Action === 'pass' && value.Test === name + ? {...value, Action} : value)), name + Action); + } + for (const Action of ['run', 'pass']) { + const events = completeEvents(); + const at = events.findIndex(value => value.Test === name && value.Action === Action); + assert.throws(() => audit(events.filter((_, index) => index !== at)), name + ' missing ' + Action); + events.splice(at, 0, events[at]); + assert.throws(() => audit(events), name + ' duplicate ' + Action); + } + } + for (const changed of ['', inventory + roots[0] + '\n', inventory.replace(roots[0] + '\n', '')]) { + assert.throws(() => audit(completeEvents(), changed)); + } + for (const events of [completeEvents().slice(1), completeEvents().slice(0, -1), + [...completeEvents(), event('pass')], [event('start'), ...completeEvents()], + completeEvents().map(value => ({...value, Package: 'foreign/package'}))]) assert.throws(() => audit(events)); +}); + +test('scalar estimates require real host brackets AVX OS permission SDK identities and direct static IA32', () => { + audit(); + for (const root of roots) { + const arch = root.endsWith('386') ? '386' : 'amd64'; + for (const needle of [host, avx]) { + for (const replacement of ['', needle + needle, needle.replace('linux/amd64', 'linux/arm64'), + needle.replace('AVX=1', 'AVX=0'), needle.replace(hash, 'invalid')]) { + if (replacement === needle) continue; + assert.throws(() => audit(completeEvents().map(value => value.Test === root && value.Output + ? {...value, Output: value.Output.replace(needle, replacement)} : value))); + } + } + for (const axis of axes) { + const child = root + '/' + axis; + const receipt = axis === 'OriginalGo' ? originalEvidence : evidence(arch); + for (const replacement of ['', receipt + receipt, receipt.replaceAll('30720', '30719')]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replace(receipt, replacement)} : value))); + } + const required = []; + if (axis === 'OriginalGo') required.push(sdk('["version"]'), sdk(compileArgs)); + if (arch === '386') required.push(image, direct(axis === 'OriginalGo' ? originalArgs : '[]')); + for (const needle of required) { + assert.throws(() => audit(completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replace(needle, '')} : value))); + } + } + } + const child = roots[1] + '/OriginalGo'; + for (const needle of [sdk(compileArgs), direct(originalArgs)]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replace(needle, needle.replace(needle.includes('actual executable') ? hash : imageHash, 'c'.repeat(64)))} : value))); + } + const events = completeEvents(); + const original = events.filter(value => value.Test === roots[0] + '/OriginalGo'); + const earlier = events.filter(value => value.Test !== roots[0] + '/OriginalGo'); + const at = earlier.findIndex(value => value.Test === roots[0] + '/' + axes[2] && value.Action === 'pass'); + earlier.splice(at, 0, ...original); + assert.throws(() => audit(earlier), 'actual Go cannot precede a finished native axis'); +}); + +test('scalar estimate executable receipts cannot replace or escape original Go ownership', () => { + audit(); + for (const axis of axes) { + const child = roots[1] + '/' + axis; + const original = axis === 'OriginalGo'; + const receipt = executable(original ? 'original.test' : 'comparison', original ? originalArgs : '[]'); + for (const replacement of ['', receipt + receipt, + receipt.replace(imageHash, hash), receipt.replace('sha256=' + imageHash, 'sha256=invalid'), + receipt.replace(original ? 'original.test' : 'comparison', 'unrelated'), + receipt.replace(original ? originalArgs : '[]', '["unobserved"]'), sdk(compileArgs)]) { + assert.throws(() => audit(completeEvents().map(value => value.Test === child && value.Output + ? {...value, Output: value.Output.replace(receipt, replacement)} : value)), axis + '/foreign or absent executable'); + } + const events = completeEvents(); + const at = events.findIndex(value => value.Test === child && value.Output?.includes(receipt)); + events[at].Output = events[at].Output.replace(receipt, ''); + const first = events.findIndex(value => value.Test === child && value.Action === 'output'); + events.splice(first, 0, event('output', child, receipt)); + assert.throws(() => audit(events), axis + '/execution before inspected image'); + } + const events = completeEvents(); + const child = roots[1] + '/OriginalGo'; + const at = events.findIndex(value => value.Test === child && value.Output?.includes(sdk(compileArgs))); + events[at].Output = events[at].Output.replace(sdk(compileArgs), ''); + const pipeline = events.findIndex(value => value.Test === roots[1] + '/backend_O0' && value.Action === 'output'); + events.splice(pipeline, 0, event('output', roots[1] + '/backend_O0', sdk(compileArgs))); + assert.throws(() => audit(events), 'pipeline cannot own original Go compilation'); +}); diff --git a/.github/scripts/priority-stdlib-corpus-mode.test.cjs b/.github/scripts/priority-stdlib-corpus-mode.test.cjs new file mode 100644 index 000000000..2461d7120 --- /dev/null +++ b/.github/scripts/priority-stdlib-corpus-mode.test.cjs @@ -0,0 +1,354 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); +const test = require('node:test'); + +// These executable protocol fixtures never invoke Go or LLVM. The real Python +// validator and actual checked-out shell script still decide scan failures. +const goFixture = [ + '#!/usr/bin/env bash', 'set -euo pipefail', + 'if [[ "$1" == env ]]; then printenv "$2"; exit; fi', + 'if [[ "$1" == version ]]; then printf "protocol Go version only\\n"; exit; fi', + '[[ "$1" == build ]] || exit 91', + 'while (( $# > 0 )); do', + ' if [[ "$1" == -o ]]; then', + ' case "$2" in', + ' *plan9asmscan|*plan9asmscan.exe) role=scanner; source=$CORPUS_PROTOCOL_SCANNER ;;', + ' *) role=cli; source=$CORPUS_PROTOCOL_TRANSLATOR ;;', + ' esac', + ' printf "build|%s\\n" "$role" >> "$CORPUS_PROTOCOL_TRACE"', + ' [[ "$CORPUS_PROTOCOL_MODE" != "build-$role" ]] || exit 42', + ' cp "$source" "$2"', ' exit', ' fi', ' shift', 'done', 'exit 92', +].join('\n') + '\n'; + +const scannerFixture = [ + '#!/usr/bin/env bash', 'set -euo pipefail', 'goos= goarch= output=', + 'while (( $# > 0 )); do', ' IFS== read -r key value <<< "$1"', + ' case "$key" in', + ' -goos) goos=$value ;; -goarch) goarch=$value ;;', + ' -out) shift; output=$1 ;;', ' esac', ' shift', 'done', + 'printf "scan|%s/%s|%s|%s|%s|%s|%s\\n" "$goos" "$goarch" "$GO386" "$GOAMD64" "$GOARM" "$GOARM64" "$GOWASM" >> "$CORPUS_PROTOCOL_TRACE"', + '[[ "$CORPUS_PROTOCOL_MODE" != scanner-exit ]] || { echo protocol-scanner-failure >&2; exit 43; }', + 'if [[ "$CORPUS_PROTOCOL_MODE" == malformed ]]; then printf "{broken\\n" > "$output"; exit; fi', + "unsupported='[]'; forms='[]'; errors='[]'", + 'case "$CORPUS_PROTOCOL_MODE" in', + " op) unsupported='[{\"op\":\"PROTOCOL_BAD\",\"count\":1}]' ;;", + " form) forms='[{\"form\":\"PROTOCOL_BAD register\",\"errors\":[\"protocol form rejection\"]}]' ;;", + " parse) errors='[{\"File\":\"protocol.s\",\"Err\":\"protocol parse rejection\"}]' ;;", + 'esac', + 'printf \'{"std_pkgs_with_sfile":1,"asm_files":1,"unsupported":%s,"unsupported_by_form":%s,"parse_errs":%s}\\n\' "$unsupported" "$forms" "$errors" > "$output"', +].join('\n') + '\n'; + +const translatorFixture = [ + '#!/usr/bin/env bash', 'set -euo pipefail', + '[[ "$1" == transpile && "$GOTOOLCHAIN" == local ]] || exit 93', 'shift', + 'goos= goarch= output= meta=', + 'while (( $# > 0 )); do', + ' IFS== read -r key value <<< "$1"', + ' case "$key" in', + ' -goos) goos=$value ;; -goarch) goarch=$value ;;', + ' -dir) shift; output=$1 ;; -meta) shift; meta=$1 ;;', + ' std) ;; *) exit 94 ;;', ' esac', ' shift', 'done', + 'printf "transpile|%s/%s|%s|%s|%s|%s|%s\\n" "$goos" "$goarch" "$GO386" "$GOAMD64" "$GOARM" "$GOARM64" "$GOWASM" >> "$CORPUS_PROTOCOL_TRACE"', + '[[ "$CORPUS_PROTOCOL_MODE" != transpile ]] || { echo protocol-transpile-failure >&2; exit 44; }', + 'mkdir -p "$output"', + '[[ "$CORPUS_PROTOCOL_MODE" != empty ]] || exit 0', + 'printf "; protocol fixture, not LLVM compilation evidence\\n" > "$output/protocol.ll"', + 'if [[ "$goarch" == wasm ]]; then printf "; second protocol input, not LLVM evidence\\n" > "$output/second.ll"; fi', + 'printf "{}\\n" > "$meta"', +].join('\n') + '\n'; + +const llvmFixture = [ + '#!/usr/bin/env bash', 'set -euo pipefail', + 'if [[ "$1" == --version ]]; then printf "LLVM version 22.1.8\\n"; exit; fi', + 'triple= output= input= object=false', + 'while (( $# > 0 )); do', ' IFS== read -r key value <<< "$1"', + ' case "$key" in', + ' -mtriple) triple=$value ;; -filetype) [[ "$value" == obj ]] || exit 95; object=true ;;', + ' -o) shift; output=$1 ;; *.ll) input=$1 ;;', ' esac', ' shift', 'done', + '[[ "$object" == true && -n "$triple" && -n "$output" && -f "$input" ]] || exit 96', + 'name=$(basename "$input")', + 'printf "object|%s|%s\\n" "$triple" "$name" >> "$CORPUS_PROTOCOL_TRACE"', + '[[ "$CORPUS_PROTOCOL_MODE" != object ]] || { echo protocol-object-failure >&2; exit 45; }', + 'if [[ "$CORPUS_PROTOCOL_MODE" == second-object && "$name" == second.ll ]]; then echo protocol-second-object-failure >&2; exit 47; fi', + 'printf "protocol fixture only, not a real object\\n" > "$output"', +].join('\n') + '\n'; + +const llvmConfigFixture = [ + '#!/usr/bin/env bash', 'set -euo pipefail', + 'case "$1" in', + ' --version) if [[ "$CORPUS_PROTOCOL_MODE" == wrong-llvm ]]; then echo 23.0.0; else echo 22.1.8; fi ;;', + ' --bindir) if [[ "$CORPUS_PROTOCOL_MODE" == missing-llvm ]]; then echo "$CORPUS_PROTOCOL_BIN/missing"; else echo "$CORPUS_PROTOCOL_BIN"; fi ;;', + ' *) exit 97 ;;', 'esac', +].join('\n') + '\n'; + +function observeNativeBashExecutable() { + const command = process.platform === 'win32' + ? 'if [[ "$BASH" == *.exe ]]; then cygpath -w -- "$BASH"; else cygpath -w -- "$BASH.exe"; fi' + : 'printf "%s\\n" "$BASH"'; + // Resolve only in the original native environment. Windows Node cannot + // resolve a command name through a child's colon-delimited MSYS PATH. + const result = spawnSync('bash', ['-c', command], {env: process.env, + encoding: 'utf8', timeout: 5000}); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ''); + const line = result.stdout.match(/^([^\r\n\0]+)\r?\n$/); + assert.ok(line, 'one complete native Bash executable path'); + const executable = line[1]; + assert.equal(executable, executable.trim(), 'an unpadded executable path'); + assert.ok(path.isAbsolute(executable), 'native absolute Bash executable'); + assert.ok(fs.statSync(executable).isFile(), 'Bash executable must be a regular file'); + fs.accessSync(executable, fs.constants.X_OK); + return executable; +} + +const bashExecutable = observeNativeBashExecutable(); + +function runCorpusShell(args, options) { + return spawnSync(bashExecutable, args, options); +} + +function shellPaths(names) { + if (process.platform !== 'win32') return names; + const result = runCorpusShell(['-c', + 'for name in "$@"; do cygpath -u -- "$name" || exit; done', 'corpus-paths', ...names], + {encoding: 'utf8', timeout: 5000}); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + const converted = result.stdout.trimEnd().split(/\r?\n/); + assert.equal(converted.length, names.length); + return converted; +} + +function fixture(t, mode = 'success', targets = '') { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'stdlib-corpus-protocol-')); + t.after(() => fs.rmSync(directory, {recursive: true, force: true})); + const bin = path.join(directory, 'bin'); + const scripts = path.join(directory, 'scripts'); + fs.mkdirSync(bin); + fs.mkdirSync(scripts); + for (const name of ['check-stdlib-corpus.sh', 'llvm22.sh']) { + fs.copyFileSync(path.join(__dirname, '../../scripts', name), path.join(scripts, name)); + } + for (const [name, source] of Object.entries({go: goFixture, scanner: scannerFixture, + translator: translatorFixture, llc: llvmFixture, 'llvm-config': llvmConfigFixture})) { + fs.writeFileSync(path.join(bin, name), source, {mode: 0o700}); + } + if (mode === 'broken-python') { + for (const name of ['python', 'python3']) { + fs.writeFileSync(path.join(bin, name), '#!/usr/bin/env bash\necho protocol-python-failure >&2\nexit 46\n', {mode: 0o700}); + } + } + const trace = path.join(directory, 'trace'); + const [shellBin, shellScript, scanner, translator, shellTrace, config] = shellPaths([ + bin, path.join(scripts, 'check-stdlib-corpus.sh'), path.join(bin, 'scanner'), + path.join(bin, 'translator'), trace, path.join(bin, 'llvm-config'), + ]); + const shellPath = runCorpusShell(['-c', 'printf "%s" "$PATH"'], + {encoding: 'utf8', timeout: 5000}); + assert.ifError(shellPath.error); + assert.equal(shellPath.status, 0, shellPath.stderr); + const environment = {...process.env, + PATH: shellBin + ':' + shellPath.stdout, LLVM_CONFIG: config, + PLAN9ASM_CMD: mode === 'missing-cli' ? shellBin + '/absent' : '', + PLAN9ASM_CORPUS_TARGETS: targets, PLAN9ASM_CORPUS_INCLUDE_WINDOWS: '1', + CORPUS_PROTOCOL_BIN: shellBin, CORPUS_PROTOCOL_SCANNER: scanner, + CORPUS_PROTOCOL_TRANSLATOR: translator, CORPUS_PROTOCOL_TRACE: shellTrace, + CORPUS_PROTOCOL_MODE: mode, + }; + return { + run(args = []) { + const result = runCorpusShell([shellScript, ...args], {cwd: directory, + env: environment, encoding: 'utf8', timeout: 15000, maxBuffer: 1024 * 1024}); + assert.ifError(result.error); + assert.equal(result.signal, null, result.stderr); + assert.equal(typeof result.status, 'number'); + const rows = fs.existsSync(trace) ? fs.readFileSync(trace, 'utf8').trimEnd().split('\n') : []; + return {...result, rows, output: result.stdout + result.stderr}; + }, + }; +} + +test('corpus Bash transport uses the observed executable even when child PATH has no Bash', t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'corpus-shell-transport-')); + t.after(() => fs.rmSync(directory, {recursive: true, force: true})); + const command = process.platform === 'win32' + ? 'if [[ "$BASH" == *.exe ]]; then cygpath -w -- "$BASH"; else cygpath -w -- "$BASH.exe"; fi' + : 'printf "%s\\n" "$BASH"'; + // Independent prior uses the original native PATH, never the child's POSIX + // search list. This is process transport, not a compiler/native-ISA oracle. + const observed = spawnSync('bash', ['-c', command], {env: process.env, + encoding: 'utf8', timeout: 5000}); + assert.ifError(observed.error); + assert.equal(observed.status, 0, observed.stderr); + const executable = observed.stdout.replace(/\r?\n$/, ''); + assert.ok(path.isAbsolute(executable)); + assert.doesNotMatch(executable, /[\r\n\0]/); + assert.ok(fs.statSync(executable).isFile()); + fs.accessSync(executable, fs.constants.X_OK); + + const options = {env: {...process.env, PATH: directory}, encoding: 'utf8', timeout: 5000}; + const args = ['-c', 'printf "CORPUS_SHELL_TRANSPORT_PASS\\n"']; + const named = spawnSync('bash', args, options); + assert.equal(named.error && named.error.code, 'ENOENT', 'negative must fail actual command lookup'); + const absolute = spawnSync(executable, args, options); + assert.ifError(absolute.error); + assert.equal(absolute.status, 0, absolute.stderr); + assert.equal(absolute.stdout, 'CORPUS_SHELL_TRANSPORT_PASS\n'); + + const actual = runCorpusShell(args, options); + assert.ifError(actual.error); + assert.equal(actual.status, 0, actual.stderr); + assert.equal(actual.stdout, absolute.stdout); +}); + +test('corpus rewritten-PATH consumers never resolve Bash by name', () => { + for (const consumer of [runCorpusShell, shellPaths, fixture]) { + assert.doesNotMatch(consumer.toString(), /spawnSync\(['"]bash['"]/, + consumer.name + ': only the original native-environment bootstrap may use a name'); + } +}); + +function expectedScans() { + const rows = []; + const add = (target, role = '', value = '') => { + const settings = ['GO386', 'GOAMD64', 'GOARM', 'GOARM64', 'GOWASM'].map(name => + name === role ? value : ''); + rows.push(['scan', target, ...settings].join('|')); + }; + for (const value of ['sse2', 'softfloat']) add('linux/386', 'GO386', value); + for (const value of ['v1', 'v2', 'v3', 'v4']) add('linux/amd64', 'GOAMD64', value); + for (const value of ['5', '5,hardfloat', '6', '6,softfloat', '7', '7,softfloat']) add('linux/arm', 'GOARM', value); + for (let index = 0; index < 10; index++) add('linux/arm64', 'GOARM64', 'v8.' + index); + for (let index = 0; index < 6; index++) add('linux/arm64', 'GOARM64', 'v9.' + index); + for (const value of ['v8.0,lse', 'v8.0,crypto', 'v8.0,lse,crypto']) add('linux/arm64', 'GOARM64', value); + add('darwin/amd64'); + add('darwin/arm64'); + for (const target of ['js/wasm', 'wasip1/wasm']) { + add(target); + for (const value of ['satconv', 'signext', 'satconv,signext']) add(target, 'GOWASM', value); + } + for (const target of ['windows/386', 'windows/amd64', 'windows/arm64']) add(target); + assert.equal(rows.length, 44, 'all original default profiles'); + assert.equal(new Set(rows).size, 44, 'each original profile exactly once'); + return rows; +} + +function wasmScanRows() { + const rows = expectedScans().filter(row => /^scan\|(?:js|wasip1)\/wasm\|/.test(row)); + assert.equal(rows.length, 8, 'complete JS/WASI plus all GOWASM profiles'); + return rows; +} + +function wasmObjectRows(scan) { + const triple = scan.split('|')[1] === 'js/wasm' ? 'wasm32-unknown-unknown' : 'wasm32-unknown-wasi'; + return ['protocol.ll', 'second.ll'].map(name => 'object|' + triple + '|' + name); +} + +test('instruction-only corpus scans all 44 profiles and compiles every full contextual WASM source', t => { + const result = fixture(t).run(['--instructions-only']); + assert.equal(result.status, 0, result.output); + assert.deepEqual(result.rows.filter(row => row.startsWith('scan|')), expectedScans()); + const wasm = new Set(wasmScanRows()); + const expected = ['build|scanner', 'build|cli']; + for (const scan of expectedScans()) { + expected.push(scan); + if (wasm.has(scan)) expected.push(scan.replace('scan|', 'transpile|'), ...wasmObjectRows(scan)); + } + assert.deepEqual(result.rows, expected, + 'WASM needs whole typed-stack source and every LLVM object; other architectures remain scan-only'); +}); + +test('default corpus retains all 44 whole-source and object compilation protocols', t => { + const result = fixture(t).run(); + assert.equal(result.status, 0, result.output); + assert.deepEqual(result.rows.filter(row => row.startsWith('scan|')), expectedScans()); + const translations = result.rows.filter(row => row.startsWith('transpile|')); + assert.deepEqual(translations, expectedScans().map(row => row.replace('scan|', 'transpile|'))); + assert.equal(result.rows.filter(row => row.startsWith('object|')).length, 52, + 'all 44 targets plus the second generated file in each of eight WASM profiles'); +}); + +test('both corpus modes keep scan and required-tool failures fatal', async t => { + const failures = { + op: /unsupported ops remain: PROTOCOL_BAD/, + form: /unsupported operand forms remain: PROTOCOL_BAD/, + parse: /parse errors remain: protocol.s: protocol parse rejection/, + malformed: /JSONDecodeError/, + 'scanner-exit': /protocol-scanner-failure/, + 'build-scanner': /build/, + 'build-cli': /build/, + 'wrong-llvm': /LLVM_CONFIG must select LLVM 22/, + 'missing-llvm': /LLVM 22 llc not found/, + 'missing-cli': /PLAN9ASM_CMD is not executable/, + 'broken-python': /protocol-python-failure/, + }; + for (const args of [[], ['--instructions-only']]) { + for (const [mode, diagnostic] of Object.entries(failures)) { + await t.test((args.length ? 'instructions/' : 'default/') + mode, t => { + const result = fixture(t, mode, 'windows/arm64').run(args); + assert.notEqual(result.status, 0, result.output); + if (mode.startsWith('build-')) { + assert.ok(result.rows.includes('build|' + mode.slice('build-'.length)), 'actual failed tool build'); + } else { + assert.match(result.output, diagnostic); + } + }); + } + } +}); + +test('instruction-only mode never starts a failing whole-runtime transpiler', t => { + const result = fixture(t, 'transpile', 'windows/arm64').run(['--instructions-only']); + assert.equal(result.status, 0, result.output); + assert.deepEqual(result.rows.filter(row => row.startsWith('scan|')), ['scan|windows/arm64|||||']); + assert.deepEqual(result.rows.filter(row => /^(transpile|object)\|/.test(row)), []); +}); + +test('instruction-only mode cannot hide any contextual WASM source or object failure', async t => { + for (const scan of wasmScanRows()) { + const fields = scan.split('|'); + const target = fields[1] + (fields[6] ? '/' + fields[6].replaceAll(',', '+') : ''); + for (const mode of ['transpile', 'empty', 'object', 'second-object']) { + await t.test(target + '/' + mode, t => { + const result = fixture(t, mode, target).run(['--instructions-only']); + assert.notEqual(result.status, 0, 'WASM Context was accepted without its complete source/object companion'); + assert.match(result.output, mode === 'empty' ? /no \.ll files generated/ : + new RegExp('protocol-' + mode + '-failure')); + assert.deepEqual(result.rows.filter(row => row.startsWith('scan|')), [scan]); + assert.deepEqual(result.rows.filter(row => row.startsWith('transpile|')), + [scan.replace('scan|', 'transpile|')]); + if (mode === 'second-object') { + assert.deepEqual(result.rows.filter(row => row.startsWith('object|')), wasmObjectRows(scan)); + } + }); + } + } +}); + +test('default corpus retains transpile, empty-output and object failures', async t => { + for (const mode of ['transpile', 'empty', 'object']) { + await t.test(mode, t => { + const result = fixture(t, mode, 'windows/arm64').run(); + assert.notEqual(result.status, 0, result.output); + assert.match(result.output, mode === 'empty' ? /no \.ll files generated/ : + new RegExp('protocol-' + mode + '-failure')); + }); + } +}); + +test('corpus mode arguments reject unknown, duplicate and extra operands', async t => { + for (const args of [['--unknown'], ['--instructions-only', '--instructions-only'], + ['--instructions-only', 'std'], ['--instructions-only=true'], ['std']]) { + await t.test(args.join(' '), t => { + const result = fixture(t, 'success', 'windows/arm64').run(args); + assert.notEqual(result.status, 0, 'unrecognized mode silently ran the default protocol'); + assert.deepEqual(result.rows, [], 'invalid arguments must fail before any tool invocation'); + }); + } +}); diff --git a/.github/scripts/priority-ubuntu-llvm-install.test.cjs b/.github/scripts/priority-ubuntu-llvm-install.test.cjs new file mode 100644 index 000000000..8568ef2e8 --- /dev/null +++ b/.github/scripts/priority-ubuntu-llvm-install.test.cjs @@ -0,0 +1,304 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); +const {test} = require('node:test'); + +const root = path.join(__dirname, '../..'); +const installer = path.join(root, 'scripts/install-ci-llvm22-ubuntu.sh'); +const configure = path.join(root, 'scripts/configure-ci-ubuntu-mirrors.py'); +const badMirrors = 'https:/us.archive.ubuntu.com/ubuntu\n' + + 'https://mirrors.edge.kernel.org/ubuntu\n'; +const sources = 'Types: deb\nURIs: mirror+file:/etc/apt/apt-mirrors.txt\n' + + 'Suites: noble noble-updates noble-backports\n' + + 'Components: main universe restricted multiverse\n' + + 'Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n'; + +function fixture(t, mirrors = true) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'plan9asm-ubuntu-apt-')); + t.after(() => fs.rmSync(directory, {recursive: true, force: true})); + if (mirrors) { + for (const file of ['apt-mirrors.txt', 'apt-mirrors-security.txt']) { + fs.writeFileSync(path.join(directory, file), badMirrors); + } + } + fs.writeFileSync(path.join(directory, 'ubuntu.sources'), sources); + fs.writeFileSync(path.join(directory, 'custom-mirrors.txt'), 'https://apt.llvm.org/noble/\n'); + return directory; +} + +function runConfigure(directory, architecture) { + const result = spawnSync('python3', [configure, directory, architecture], { + encoding: 'utf8', timeout: 5000, + env: {...process.env, PYTHONDONTWRITEBYTECODE: '1'}, + }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + return result.stdout; +} + +function assertUnchangedSources(directory) { + assert.equal(fs.readFileSync(path.join(directory, 'ubuntu.sources'), 'utf8'), sources); + assert.equal(fs.readFileSync(path.join(directory, 'custom-mirrors.txt'), 'utf8'), + 'https://apt.llvm.org/noble/\n'); +} + +test('Ubuntu mirror configuration repairs the actual malformed runner lists only', t => { + for (const [architecture, archive, security] of [ + ['amd64', 'https://archive.ubuntu.com/ubuntu/', 'https://security.ubuntu.com/ubuntu/'], + ['i386', 'https://archive.ubuntu.com/ubuntu/', 'https://security.ubuntu.com/ubuntu/'], + ['arm64', 'https://ports.ubuntu.com/ubuntu-ports/', 'https://ports.ubuntu.com/ubuntu-ports/'], + ['armhf', 'https://ports.ubuntu.com/ubuntu-ports/', 'https://ports.ubuntu.com/ubuntu-ports/'], + ]) { + const directory = fixture(t); + assert.equal(runConfigure(directory, architecture), archive + '\n'); + assert.equal(runConfigure(directory, architecture), archive + '\n'); + assert.equal(fs.readFileSync(path.join(directory, 'apt-mirrors.txt'), 'utf8'), archive + '\n'); + assert.equal(fs.readFileSync(path.join(directory, 'apt-mirrors-security.txt'), 'utf8'), security + '\n'); + assertUnchangedSources(directory); + } +}); + +test('Ubuntu direct sources do not acquire new mirror lists or unrelated configuration', t => { + const directory = fixture(t, false); + runConfigure(directory, 'amd64'); + assert.deepEqual(fs.readdirSync(directory).sort(), ['custom-mirrors.txt', 'ubuntu.sources']); + assertUnchangedSources(directory); +}); + +// Execute the actual Bash installer. Every privileged/network/package command +// is intercepted; only the real narrow helper runs against this test's directory. +function runInstaller(t, options = {}) { + const directory = fixture(t); + const environment = path.join(directory, 'environment'); + const script = ` +updates=0 +installs=0 +IFS=, read -r -a update_statuses <<< "$MOCK_UPDATES" +IFS=, read -r -a install_statuses <<< "$MOCK_INSTALLS" +dpkg() { + case "$1" in + -s) return "$MOCK_INSTALLED" ;; + --print-architecture) + printf 'DPKG_ARCH\\n' >&2 + if [[ "$MOCK_ARCH_EXIT" != 0 ]]; then return "$MOCK_ARCH_EXIT"; fi + printf '%s\\n' "$MOCK_ARCH" ;; + *) return 99 ;; + esac +} +lsb_release() { test "$1" = -cs && printf 'noble\\n'; } +curl() { + printf 'CURL'; printf ' <%s>' "$@"; printf '\\n' >&2 + printf 'test signing key\\n' +} +tee() { + case "$1" in + /etc/apt/trusted.gpg.d/llvm-snapshot.asc|/etc/apt/sources.list.d/llvm.list|/etc/apt/sources.list.d/plan9asm-ci-fallback.list) + command tee "$MOCK_APT/$(basename "$1")" ;; + *) return 99 ;; + esac +} +env() { + test "$1" = DEBIAN_FRONTEND=noninteractive || return 99 + shift + "$@" +} +timeout() { + test "$1" = -k && test "$2" = 10s || return 99 + case "$3" in 2m|4m) ;; *) return 99 ;; esac + printf 'TIMEOUT %s\\n' "$3" + shift 3 + "$@" +} +apt-get() { + printf 'APT_BEGIN\\n' >&2 + printf 'APT'; printf ' <%s>' "$@"; printf '\\n' + for arg in "$@"; do + case "$arg" in + update) + updates=$((updates + 1)) + return "\${update_statuses[updates - 1]:-99}" ;; + install) + installs=$((installs + 1)) + return "\${install_statuses[installs - 1]:-99}" ;; + esac + done + return 99 +} +sudo() { + case "$1" in + tee|timeout) "$@" ;; + python3) + test "$#" = 4 && test "$3" = /etc/apt || return 99 + printf 'CONFIGURE %s\\n' "$4" >&2 + if [[ "$MOCK_CONFIG_EXIT" != 0 ]]; then return "$MOCK_CONFIG_EXIT"; fi + if [[ "$MOCK_CONFIG_OUTPUT" != __actual_helper__ ]]; then + printf '%s' "$MOCK_CONFIG_OUTPUT" + else + command python3 "$2" "$MOCK_APT" "$4" + fi ;; + *) return 99 ;; + esac +} +llvm-config-22() { printf '%s\\n' "$MOCK_LLVM_VERSION"; } +clang-22() { return 0; } +llc-22() { return 0; } +source "$1" gcc-i686-linux-gnu libc6-dev-i386-cross +`; + const result = spawnSync('bash', ['--noprofile', '--norc', '-c', script, 'installer-test', installer], { + encoding: 'utf8', timeout: 5000, + env: { + ...process.env, PYTHONDONTWRITEBYTECODE: '1', + MOCK_APT: directory, MOCK_ARCH: options.architecture || 'amd64', + MOCK_ARCH_EXIT: String(options.architectureExit || 0), + MOCK_INSTALLED: options.installed ? '0' : '1', + MOCK_UPDATES: (options.updates || [0]).join(','), + MOCK_INSTALLS: (options.installs || [0]).join(','), + MOCK_CONFIG_EXIT: String(options.configurationExit || 0), + MOCK_CONFIG_OUTPUT: options.configurationOutput === undefined + ? '__actual_helper__' : options.configurationOutput, + MOCK_LLVM_VERSION: options.version || '22.1.8', GITHUB_ENV: environment, + }, + }); + assert.ifError(result.error); + return {...result, directory, environment: fs.existsSync(environment) + ? fs.readFileSync(environment, 'utf8') : ''}; +} + +test('actual Ubuntu installer configures mirrors before strict IPv4 update and install', t => { + const result = runInstaller(t); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.stderr.indexOf('CONFIGURE amd64\n') >= 0, result.stderr); + assert.ok(result.stderr.indexOf('CONFIGURE amd64\n') < result.stderr.indexOf('APT_BEGIN\n'), result.stderr); + for (const command of result.stdout.split('\n').filter(line => line.startsWith('APT '))) { + for (const option of ['Acquire::ForceIPv4=true', 'APT::Update::Error-Mode=any', + 'Acquire::Retries=2', 'Acquire::http::Timeout=30', 'Acquire::https::Timeout=30']) { + assert.ok(command.includes('<' + option + '>'), command); + } + } + assert.match(result.stdout, /TIMEOUT 2m\n/); + assert.match(result.stdout, /TIMEOUT 4m\n/); + assert.match(result.stdout, / <-y> /); + assert.equal(fs.readFileSync(path.join(result.directory, 'apt-mirrors.txt'), 'utf8'), + 'https://archive.ubuntu.com/ubuntu/\n'); + assertUnchangedSources(result.directory); + assert.match(result.environment, /^PATH=\/usr\/lib\/llvm-22\/bin:/); +}); + +test('already installed LLVM packages need no mirror change, update or download', t => { + const result = runInstaller(t, {installed: true}); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stdout + result.stderr, /DPKG_ARCH|CONFIGURE|APT |CURL|TIMEOUT/); + assert.equal(fs.readFileSync(path.join(result.directory, 'apt-mirrors.txt'), 'utf8'), badMirrors); + assertUnchangedSources(result.directory); + assert.match(result.environment, /^PATH=\/usr\/lib\/llvm-22\/bin:/); +}); + +test('two exhausted updates retain exit 124 without installation or PATH publication', t => { + const result = runInstaller(t, {updates: [124, 124]}); + assert.equal(result.status, 124, result.stderr); + const commands = result.stdout.split('\n').filter(line => line.startsWith('APT ')); + assert.equal(commands.length, 2); + assert.ok(commands.every(line => line.endsWith(''))); + assert.match(commands[1], //); + assert.match(commands[1], //); + assert.equal(result.environment, ''); +}); + +test('fallback keeps strict network options, the identical package transaction and original budgets', t => { + const result = runInstaller(t, {updates: [0, 0], installs: [100, 0]}); + assert.equal(result.status, 0, result.stderr); + const commands = result.stdout.split('\n').filter(line => line.startsWith('APT ')); + assert.equal(commands.length, 4); + for (const command of commands) { + assert.match(command, //); + assert.match(command, //); + } + assert.equal(commands[1].slice(commands[1].indexOf('')), + commands[3].slice(commands[3].indexOf(''))); + assert.equal((result.stdout.match(/^TIMEOUT 2m$/gm) || []).length, 2); + assert.equal((result.stdout.match(/^TIMEOUT 4m$/gm) || []).length, 2); +}); + +test('isolated fallback uses the same official Ubuntu architecture route as the repaired mirror lists', async t => { + for (const [architecture, archive] of [ + ['amd64', 'https://archive.ubuntu.com/ubuntu/'], + ['i386', 'https://archive.ubuntu.com/ubuntu/'], + ['arm64', 'https://ports.ubuntu.com/ubuntu-ports/'], + ['armhf', 'https://ports.ubuntu.com/ubuntu-ports/'], + ]) { + await t.test(architecture, child => { + const result = runInstaller(child, {architecture, updates: [124, 0]}); + assert.equal(result.status, 0, result.stderr); + const key = '[signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg]'; + const components = 'main universe restricted multiverse'; + assert.equal(fs.readFileSync(path.join(result.directory, 'plan9asm-ci-fallback.list'), 'utf8'), + `deb ${key} ${archive} noble ${components}\n` + + `deb ${key} ${archive} noble-updates ${components}\n` + + 'deb https://apt.llvm.org/noble/ llvm-toolchain-noble-22 main\n'); + assert.equal(fs.readFileSync(path.join(result.directory, 'apt-mirrors.txt'), 'utf8'), archive + '\n'); + const commands = result.stdout.split('\n').filter(line => line.startsWith('APT ')); + assert.equal(commands.length, 3); + assert.equal((result.stderr.match(/^DPKG_ARCH$/gm) || []).length, 1); + assert.ok(commands[0].endsWith('')); + assert.ok(commands[1].endsWith('')); + assert.ok(commands[2].includes('')); + assertUnchangedSources(result.directory); + }); + } +}); + +test('failed fallback update cannot fall through to missing-package installation or publish PATH', t => { + const result = runInstaller(t, {updates: [124, 100]}); + assert.equal(result.status, 100, result.stderr); + const commands = result.stdout.split('\n').filter(line => line.startsWith('APT ')); + assert.equal(commands.length, 2); + for (const command of commands) { + assert.ok(command.endsWith('')); + assert.match(command, //); + } + assert.equal(result.environment, ''); +}); + +test('invalid mirror helper output or helper failure cannot reach APT or publish PATH', async t => { + for (const [name, configurationOutput] of [ + ['empty', ''], + ['foreign URL', 'https://example.invalid/ubuntu/\n'], + ['diagnostic prefix', 'CONFIGURE amd64\nhttps://archive.ubuntu.com/ubuntu/\n'], + ['extra source', 'https://archive.ubuntu.com/ubuntu/\nextra-source\n'], + ]) { + await t.test(name, child => { + const result = runInstaller(child, {configurationOutput}); + assert.equal(result.status, 1, result.stderr); + assert.match(result.stderr, /invalid Ubuntu archive mirror/); + assert.doesNotMatch(result.stdout, /APT |TIMEOUT/); + assert.equal(result.environment, ''); + }); + } + await t.test('failure exit', child => { + const result = runInstaller(child, {configurationExit: 68}); + assert.equal(result.status, 68, result.stderr); + assert.doesNotMatch(result.stdout, /APT |TIMEOUT/); + assert.equal(result.environment, ''); + }); +}); + +test('failed architecture query cannot silently select the ports route', t => { + const result = runInstaller(t, {architectureExit: 67}); + assert.equal(result.status, 67, result.stderr); + assert.doesNotMatch(result.stdout + result.stderr, /CONFIGURE|APT |TIMEOUT/); + assert.equal(result.environment, ''); +}); + +test('failed package installation and wrong LLVM version cannot publish success', t => { + const failed = runInstaller(t, {updates: [0, 0], installs: [100, 101]}); + assert.equal(failed.status, 101, failed.stderr); + assert.equal(failed.environment, ''); + for (const installed of [false, true]) { + const wrong = runInstaller(t, {installed, version: '23.0.0'}); + assert.equal(wrong.status, 1, wrong.stderr); + assert.match(wrong.stderr, /does not select LLVM 22/); + assert.equal(wrong.environment, ''); + } +}); diff --git a/.github/scripts/priority-unix-root-owner-budget.test.cjs b/.github/scripts/priority-unix-root-owner-budget.test.cjs new file mode 100644 index 000000000..fad315a83 --- /dev/null +++ b/.github/scripts/priority-unix-root-owner-budget.test.cjs @@ -0,0 +1,103 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const {jobBody, requireJobNeeds} = require('./workflow-needs.cjs'); + +const workflow = fs.readFileSync(path.join(__dirname, '../workflows/go-ci.yml'), 'utf8'); +const script = fs.readFileSync(path.join(__dirname, '../../scripts/test-module-ci.sh'), 'utf8'); + +function step(body, title) { + const matches = body.split(/(?=^ - )/m).filter(value => value.startsWith(` - name: ${title}\n`)); + assert.equal(matches.length, 1, 'one exact step required: ' + title); + return matches[0]; +} + +function requireOwners(source, name) { + const body = jobBody(source, name); + const roles = Array.from({length: 16}, (_, index) => 'root-' + index).concat('profile', 'other'); + const axis = body.match(/^ module-role: \[([^\]]+)\]$/m); + assert.ok(axis, name + ': explicit complete scheduling roles'); + assert.deepEqual(axis[1].split(',').map(value => value.trim()), roles); + assert.deepEqual([...body.matchAll( + /^ - \{module-role: (root-\d+|profile|other), module-owner: ([012]), root-partition: (-?\d+)\}$/gm, + )].map(([, role, owner, part]) => [role, Number(owner), Number(part)]), [ + ...Array.from({length: 16}, (_, index) => ['root-' + index, 0, index]), + ['profile', 1, -1], ['other', 2, -1], + ]); + assert.match(body, /^ needs: ci_policy$/m); + assert.match(body, /^ timeout-minutes: 60$/m); + assert.match(body, /^ fail-fast: false$/m); + assert.doesNotMatch(body, /^ if:|max-parallel:|continue-on-error:|module-owner: \[/m); + const title = name === 'test' ? 'Go test (Ubuntu)' : 'Go test with coverage'; + assert.ok(step(body, title).includes('run: bash scripts/test-module-ci.sh ' + name + + ' execute ${{ matrix.module-owner }} ${{ matrix.root-partition }}\n')); +} + +function requireDownloads(source) { + const body = jobBody(source, 'module-completion'); + requireJobNeeds(source, 'module-completion', ['test', 'nested-cli', 'race', 'coverage', 'nested-cli-coverage']); + assert.match(body, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + for (let index = 0; index < 32; index++) { + const download = step(body, 'Download root part ' + index); + assert.ok(download.includes('name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-' + index + '\n')); + assert.ok(download.includes('path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-' + + String(index).padStart(4, '0') + '\n')); + if (index > 0 && index < 16) assert.match(download, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); + if (index < 16) assert.doesNotMatch(download, /^ if:.*matrix\.flavor/m); + if (index >= 16) assert.match(download, /^ if: \$\{\{ !cancelled\(\) && matrix\.flavor == 'race' \}\}$/m); + } + assert.doesNotMatch(body, /merge-multiple:|pattern:/); +} + +test('test and coverage retain all roots in sixteen independently budgeted owners', () => { + requireOwners(workflow, 'test'); + requireOwners(workflow, 'coverage'); +}); + +test('Unix root owners use one two-worker wave without raising any original budget', () => { + assert.match(script, /^root_partitions=16$/m); + assert.match(script, /^shards=32$/m); + assert.match(script, /test\) timeout=45m ;;/); + assert.match(script, /coverage\) timeout=45m; flags=\(-coverprofile=coverage-root\.out\) ;;/); + assert.match(script, /race\) timeout=60m; flags=\(-race\); root_partitions=32; shards=64 ;;/); + assert.equal([...script.matchAll(/-shards=\$shards -workers=2/g)].length, 2); + assert.equal([...script.matchAll(/-module-root-partitions=\$root_partitions/g)].length, 2); + assert.match(script, /GOMAXPROCS=2/); + assert.match(jobBody(workflow, 'race'), /^ timeout-minutes: 75$/m); + assert.match(jobBody(workflow, 'module-completion'), /^ timeout-minutes: 30$/m); +}); + +test('all flavors require exact root fragments zero through fifteen before complete readback', () => { + requireDownloads(workflow); +}); + +test('lost or duplicated owners, stale eight-part readback and relaxed budgets are rejected', () => { + for (const name of ['test', 'coverage']) { + requireOwners(workflow, name); + const body = jobBody(workflow, name); + for (const [before, after] of [ + ['root-15,', ''], + ['root-partition: 15', 'root-partition: 14'], + ['timeout-minutes: 60', 'timeout-minutes: 75'], + ['fail-fast: false', 'fail-fast: true'], + ['timeout-minutes: 60', 'timeout-minutes: 60\n continue-on-error: true'], + ]) { + const changed = body.replace(before, after); + assert.notEqual(changed, body); + assert.throws(() => requireOwners(workflow.replace(body, () => changed), name)); + } + } + requireDownloads(workflow); + const complete = jobBody(workflow, 'module-completion'); + for (const index of [8, 15]) { + const original = step(complete, 'Download root part ' + index); + assert.throws(() => requireDownloads(workflow.replace(original, ''))); + const stale = original.replace('if: ${{ !cancelled() }}', + "if: ${{ !cancelled() && matrix.flavor == 'race' }}"); + assert.notEqual(stale, original); + assert.throws(() => requireDownloads(workflow.replace(original, () => stale))); + } +}); diff --git a/.github/scripts/priority-unlocked-native.test.cjs b/.github/scripts/priority-unlocked-native.test.cjs new file mode 100644 index 000000000..41f57e104 --- /dev/null +++ b/.github/scripts/priority-unlocked-native.test.cjs @@ -0,0 +1,41 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const {auditNativeUnlocked, roots, children} = require('./audit-unlocked-native.cjs'); + +test('unlocked audit requires both executed ISAs and all three public API children', () => { + assert.equal(roots.length, 2); + assert.equal(children.length, 6); + const event = (Action, Test) => ({Action, Package: 'github.com/xgo-dev/plan9asm', ...(Test ? {Test} : {})}); + const events = [event('start')]; + for (const root of roots) { + events.push(event('run', root)); + for (const child of children.filter(name => name.startsWith(root + '/'))) { + events.push(event('run', child), event('pass', child)); + } + events.push(event('pass', root)); + } + events.push(event('pass')); + const inventory = roots.join('\n') + '\n'; + const encode = rows => rows.map(JSON.stringify).join('\n') + '\n'; + assert.deepEqual(auditNativeUnlocked(inventory, encode(events)), + {roots: 2, children: 6, passed: 8, skipped: 0, incomplete: 0}); + for (const child of children) { + for (const altered of [ + events.filter(row => row.Test !== child), + events.map(row => row.Test === child && row.Action === 'pass' ? {...row, Action: 'skip'} : row), + events.map(row => row.Test === child ? {...row, Test: child + '-object-only'} : row), + ]) assert.throws(() => auditNativeUnlocked(inventory, encode(altered))); + } + assert.throws(() => auditNativeUnlocked(inventory, + encode([event('start'), {...event('output'), Output: encode(events)}, event('pass')]))); +}); + +test('unlocked native sources retain real host and backend requirements', () => { + const native = fs.readFileSync('amd64_scalar_memory_rmw_linux_amd64_test.go', 'utf8'); + assert.equal((native.match(/x87IA32RequireNativeHost\(t\)/g) || []).length, 4); + assert.equal((native.match(/\[\]string\{"-O0", "-O2"\}/g) || []).length, 2); + assert.doesNotMatch(native, /\.Skip|qemu|rosetta|PLAN9ASM_CROSS_EXEC/); +}); diff --git a/.github/scripts/priority-windows-cancelled-evidence.test.cjs b/.github/scripts/priority-windows-cancelled-evidence.test.cjs new file mode 100644 index 000000000..e2a9b65c5 --- /dev/null +++ b/.github/scripts/priority-windows-cancelled-evidence.test.cjs @@ -0,0 +1,55 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); + +const source = fs.readFileSync(path.join(__dirname, '../workflows/windows-module.yml'), 'utf8'); +const labels = ['Retain actual root evidence', 'Retain actual package evidence']; + +function retention(source, label) { + const start = source.indexOf(' - name: ' + label + '\n'); + assert.notEqual(start, -1, label); + const tail = source.slice(start); + const next = tail.search(/\n(?: - | [a-z-]+:)/); + return tail.slice(0, next < 0 ? undefined : next); +} + +function verify(source) { + for (const label of labels) { + const step = retention(source, label); + assert.match(step, /^ if: \$\{\{ always\(\) \}\}$/m, label); + assert.match(step, /uses: actions\/upload-artifact@v6/); + assert.match(step, /if-no-files-found: error/); + } +} + +function condition(source, label, replacement) { + const step = retention(source, label); + const changed = step.replace(/^ if: .+\n/m, replacement); + assert.notEqual(changed, step, label); + return source.replace(step, changed); +} + +// Mutations start from a complete positive fixture even while the real +// workflow is RED; missing or cancellation-dependent retention must fail. +let fixture = source; +for (const label of labels) { + const step = retention(fixture, label); + fixture = fixture.replace(step, step.replace(/^ if: .+$/m, ' if: ${{ always() }}')); +} + +test('cancelled Windows owners retain diagnostic evidence, not successful coverage', () => { + verify(source); +}); + +for (const label of labels) { + test(label + ' cannot lose its cancellation-independent condition', () => { + verify(fixture); + assert.throws(() => verify(condition(fixture, label, ''))); + }); + test(label + ' cannot suppress reports after cancellation', () => { + assert.throws(() => verify(condition(fixture, label, ' if: ${{ !cancelled() }}\n'))); + }); +} diff --git a/.github/scripts/priority-windows-codecov-python.test.cjs b/.github/scripts/priority-windows-codecov-python.test.cjs new file mode 100644 index 000000000..c26746784 --- /dev/null +++ b/.github/scripts/priority-windows-codecov-python.test.cjs @@ -0,0 +1,148 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const {jobBody} = require('./workflow-needs.cjs'); + +const source = fs.readFileSync(path.join(__dirname, '../workflows/windows-module.yml'), 'utf8'); +const setup = ` - name: Set up compatible Python for coverage upload + id: codecov_python + uses: actions/setup-python@v7 + with: + python-version: '3.13' +`; +const beforeUpload = ` - name: Check Codecov Python and pip ownership + shell: bash + env: + CODECOV_PYTHON: \${{ steps.codecov_python.outputs.python-path }} + run: | + python - <<'PY' + import os + import shutil + import subprocess + import sys + if sys.version_info[:2] != (3, 13): + raise SystemExit("Codecov upload requires Python 3.13") + if not os.path.samefile(sys.executable, os.environ["CODECOV_PYTHON"]): + raise SystemExit("Codecov Python differs from setup-python output") + pip = shutil.which("pip") + expected = os.path.join(os.path.dirname(sys.executable), "Scripts", "pip.exe") + if not pip or not os.path.samefile(pip, expected): + raise SystemExit("Codecov pip is not owned by the selected Python") + print("Codecov upload Python:", sys.version, sys.executable, flush=True) + print("Codecov upload pip:", pip, flush=True) + subprocess.run([pip, "--version"], check=True) + PY +`; +const afterUpload = ` - name: Check installed Codecov CLI ownership + shell: bash + env: + CODECOV_PYTHON: \${{ steps.codecov_python.outputs.python-path }} + run: | + python - <<'PY' + import os + import shutil + import subprocess + import sys + if not os.path.samefile(sys.executable, os.environ["CODECOV_PYTHON"]): + raise SystemExit("Codecov Python changed during upload") + cli = shutil.which("codecovcli") + expected = os.path.join(os.path.dirname(sys.executable), "Scripts", "codecovcli.exe") + if not cli or not os.path.samefile(cli, expected): + raise SystemExit("Codecov CLI is not owned by the selected Python") + print("Codecov upload CLI:", cli, flush=True) + subprocess.run([cli, "--help"], check=True) + PY +`; + +function oneStep(body, marker) { + const found = body.split(/(?=^ - )/m).filter(step => step.includes(marker)); + assert.equal(found.length, 1, 'one actual step required: ' + marker); + return found[0]; +} + +function requirePhaseBoundary(workflow) { + for (const name of ['root-parts', 'other-owners']) { + assert.doesNotMatch(jobBody(workflow, name), /setup-python|codecov-action|CODECOV_PYTHON/); + } + const body = jobBody(workflow, 'complete'); + const verify = body.indexOf('- name: Verify complete module and publish coverage\n'); + const retain = body.indexOf('- name: Retain independent completion evidence\n'); + const python = body.indexOf('uses: actions/setup-python@v7\n'); + const upload = body.indexOf('- name: Upload verified complete Windows module coverage\n'); + assert.ok(verify >= 0 && retain > verify && python > retain && upload > python, + 'the uploader must not alter Python/PATH before module qualification and evidence retention'); + assert.equal((body.match(/uses: actions\/setup-python@/g) || []).length, 1); + const action = oneStep(body, '- name: Upload verified complete Windows module coverage\n'); + for (const input of ['use_pypi: true', 'use_oidc: true', 'disable_search: true', + 'fail_ci_if_error: true', 'files: ./coverage-windows-${{ inputs.module }}.out']) { + assert.ok(action.includes(input), 'retain actual verified coverage upload input: ' + input); + } + assert.doesNotMatch(action, /continue-on-error:|if:|skip_validation:|PYO3_USE_ABI3_FORWARD_COMPATIBILITY/); +} + +function requireCompatiblePython(workflow) { + requirePhaseBoundary(workflow); + const body = jobBody(workflow, 'complete'); + const python = oneStep(body, 'uses: actions/setup-python@v7\n'); + assert.match(python, /^ id: codecov_python$/m); + assert.match(python, /^ python-version: '3\.13'$/m); + assert.doesNotMatch(python, /python-version-file:|update-environment: false|allow-prereleases: true/); +} + +function requireBindings(workflow) { + const body = jobBody(workflow, 'complete'); + const before = oneStep(body, '- name: Check Codecov Python and pip ownership\n'); + const after = oneStep(body, '- name: Check installed Codecov CLI ownership\n'); + for (const step of [before, after]) { + assert.match(step, /^ shell: bash$/m); + assert.ok(step.includes('CODECOV_PYTHON: ${{ steps.codecov_python.outputs.python-path }}\n')); + assert.ok(step.includes('os.path.samefile(sys.executable, os.environ["CODECOV_PYTHON"])')); + assert.doesNotMatch(step, /continue-on-error:|if:|GITHUB_PATH|GITHUB_ENV|PYO3_USE_ABI3_FORWARD_COMPATIBILITY/); + } + for (const text of ['sys.version_info[:2] != (3, 13)', 'shutil.which("pip")', + '"Scripts", "pip.exe"', 'not os.path.samefile(pip, expected)', + 'subprocess.run([pip, "--version"], check=True)']) { + assert.ok(before.includes(text), 'the action executes PATH pip, not a python action input: ' + text); + } + for (const text of ['shutil.which("codecovcli")', '"Scripts", "codecovcli.exe"', + 'not os.path.samefile(cli, expected)', 'subprocess.run([cli, "--help"], check=True)']) { + assert.ok(after.includes(text), 'the actual installed CLI must be observed: ' + text); + } + const upload = body.indexOf('- name: Upload verified complete Windows module coverage\n'); + assert.ok(body.indexOf(before) < upload && body.indexOf(after) > upload); +} + +const fixture = source.replace(/ - uses: actions\/setup-python@v7\n with:\n python-version: '3\.x'\n/, + setup + beforeUpload); +const completeFixture = fixture.endsWith(afterUpload) ? fixture : fixture + afterUpload; + +test('Windows coverage uploader uses a compatible Python after qualification', () => requireCompatiblePython(source)); +test('PATH pip and installed Codecov CLI are actually bound to the selected Python', () => requireBindings(source)); +test('upload setup cannot change a live module toolbank or suppress an upload error', () => requirePhaseBoundary(source)); + +test('another Python minor or disabled PATH update cannot borrow the compatible pin', () => { + requireCompatiblePython(completeFixture); + requireBindings(completeFixture); + for (const value of ['3.x', '3.15', '3.14', '${{ matrix.python }}']) { + assert.throws(() => requireCompatiblePython(completeFixture.replace("python-version: '3.13'", `python-version: '${value}'`))); + } + assert.throws(() => requireCompatiblePython(completeFixture.replace("python-version: '3.13'", "python-version: '3.13'\n update-environment: false"))); +}); + +test('Python, pip and installed CLI binding checks cannot be omitted or softened', () => { + for (const text of ['os.path.samefile(sys.executable, os.environ["CODECOV_PYTHON"])', + 'not os.path.samefile(pip, expected)', 'not os.path.samefile(cli, expected)', + 'subprocess.run([pip, "--version"], check=True)', 'subprocess.run([cli, "--help"], check=True)']) { + assert.ok(completeFixture.includes(text)); + assert.throws(() => requireBindings(completeFixture.replace(text, 'REMOVED_CHECK'))); + } +}); + +test('compatible upload Python cannot move before module evidence is sealed', () => { + const moved = completeFixture.replace(setup, '').replace(' - name: Verify complete module and publish coverage\n', + setup + ' - name: Verify complete module and publish coverage\n'); + assert.throws(() => requirePhaseBoundary(moved)); +}); diff --git a/.github/scripts/priority-windows-llvm-prefix.test.cjs b/.github/scripts/priority-windows-llvm-prefix.test.cjs new file mode 100644 index 000000000..61293d673 --- /dev/null +++ b/.github/scripts/priority-windows-llvm-prefix.test.cjs @@ -0,0 +1,158 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const {test} = require('node:test'); + +const action = fs.readFileSync('.github/actions/setup-windows-llvm22/action.yml', 'utf8'); +const publisherFile = path.resolve('.github/actions/publish-windows-llvm22/index.cjs'); +function publisher() { + assert.ok(fs.existsSync(publisherFile), 'successful attempts need one stable, guarded native publication'); + return require(publisherFile).publishPrefix; +} + +function fixture(t, attempt = '1') { + const root = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'llvm22 prefix ')); + t.after(() => fs.rmSync(root, {recursive: true, force: true})); + const source = path.join(root, 'plan9asm-msys2-attempt-' + attempt, 'msys64', 'clang64'); + for (const relative of ['bin/llvm-config.exe', 'bin/clang.exe', 'bin/clang++.exe', + 'include/stdint.h', 'include/_mingw.h', 'lib/libkernel32.a', 'lib/clang/22/include/stddef.h']) { + const file = path.join(source, relative); + fs.mkdirSync(path.dirname(file), {recursive: true}); + fs.writeFileSync(file, 'synthetic transport input, not executable LLVM\n', {flag: 'wx'}); + } + const target = path.join(root, 'plan9asm-llvm22'); + const commands = []; + const run = (file, args, options) => { + commands.push({file, args}); + const name = path.basename(file); + if (name === 'powershell.exe') { + assert.ok(args.join(' ').includes('ReparsePoint')); + assert.ok(options.env.PLAN9ASM_LLVM22_CHECK); + return ''; + } + assert.ok(file.startsWith(target + path.sep) || file.startsWith(path.join(root, 'plan9asm-llvm22-proof') + path.sep)); + if (name === 'llvm-config.exe') { + switch (args.join(' ')) { + case '--version': return '22.1.8\r\n'; + case '--prefix': return target + '\r\n'; + case '--bindir': return path.join(target, 'bin') + '\n'; + case '--includedir': return path.join(target, 'include') + '\n'; + case '--libdir': return path.join(target, 'lib') + '\n'; + case '--cflags': return '-I' + path.join(target, 'include') + '\n'; + case '--cxxflags': return '-I' + path.join(target, 'include') + ' -std=c++17\n'; + case '--ldflags --libs all --system-libs': return '-L' + path.join(target, 'lib') + ' -lLLVM\n'; + default: assert.fail('unexpected real publisher query: ' + args.join(' ')); + } + } + if (name === 'clang.exe') { + if (args.join(' ') === '--version') return 'clang version 22.1.8\nTarget: x86_64-w64-windows-gnu\n'; + if (args.join(' ') === '--print-resource-dir') return path.join(target, 'lib', 'clang', '22') + '\n'; + const output = args[args.indexOf('-o') + 1]; + assert.ok(output.startsWith(path.join(root, 'plan9asm-llvm22-proof') + path.sep)); + fs.writeFileSync(output, 'synthetic link result, never native execution\n', {flag: 'wx'}); + return ''; + } + assert.equal(name, 'publication.exe'); + assert.equal(fs.existsSync(file), true); + return ''; + }; + return {root, source, target, commands, run, options: {runnerTemp: root, attempt, + systemRoot: path.join(root, 'Windows'), run}}; +} + +test('different successful bootstrap attempts publish identical strict tool bindings', t => { + const publish = publisher(); + const f = fixture(t); + const first = publish(f.options); + assert.equal(fs.existsSync(f.source), false); + assert.equal(first.path, path.join(f.target, 'bin')); + assert.equal(first.environment.LLVM_CONFIG, path.join(f.target, 'bin', 'llvm-config.exe')); + assert.equal(first.environment.CC, path.join(f.target, 'bin', 'clang.exe')); + assert.equal(first.environment.CXX, path.join(f.target, 'bin', 'clang++.exe')); + assert.equal(first.environment.CGO_ENABLED, '1'); + assert.equal(first.environment.GOFLAGS, '-tags=byollvm'); + assert.doesNotMatch(JSON.stringify(first), /plan9asm-msys2-attempt-/); + fs.renameSync(f.target, path.join(f.root, 'retained-success-1')); + fs.renameSync(path.join(f.root, 'plan9asm-llvm22-proof'), path.join(f.root, 'retained-proof-1')); + const secondSource = path.join(f.root, 'plan9asm-msys2-attempt-2', 'msys64', 'clang64'); + fs.mkdirSync(path.dirname(secondSource), {recursive: true}); + fs.cpSync(path.join(f.root, 'retained-success-1'), secondSource, {recursive: true, errorOnExist: true}); + const failed = path.join(f.root, 'plan9asm-msys2-attempt-1', 'failed-update.txt'); + fs.writeFileSync(failed, 'preserved original bootstrap failure\n'); + const second = publish({...f.options, attempt: '2'}); + assert.deepEqual(second, first, 'strict compiler environments must not depend on which attempt succeeded'); + assert.equal(fs.readFileSync(failed, 'utf8'), 'preserved original bootstrap failure\n'); +}); + +test('publication never covers an existing target or a redirected source', t => { + const publish = publisher(); + for (const kind of ['file', 'directory', 'symlink', 'source-link', 'nested-link']) { + const f = fixture(t); + if (kind === 'file') fs.writeFileSync(f.target, 'keep target'); + if (kind === 'directory') fs.mkdirSync(f.target); + if (kind === 'symlink') fs.symlinkSync(f.source, f.target, 'dir'); + if (kind === 'source-link') { + const retained = path.join(f.root, 'original-source'); + fs.renameSync(f.source, retained); + fs.symlinkSync(retained, f.source, 'dir'); + } + if (kind === 'nested-link') fs.symlinkSync(path.join(f.source, 'include'), path.join(f.source, 'redirect'), 'dir'); + assert.throws(() => publish(f.options), undefined, kind); + assert.equal(f.commands.filter(c => path.basename(c.file) === 'clang.exe').length, 0); + assert.equal(fs.existsSync(f.source), true, 'refused publication cannot remove its source'); + } +}); + +test('unknown attempts, missing inputs and replacement during checks fail closed', t => { + const publish = publisher(); + for (const attempt of ['', '0', '4', '01', '../1']) { + const f = fixture(t); + assert.throws(() => publish({...f.options, attempt})); + assert.equal(fs.existsSync(f.target), false); + } + const missing = fixture(t); + fs.unlinkSync(path.join(missing.source, 'bin', 'clang.exe')); + assert.throws(() => publish(missing.options)); + const replaced = fixture(t); + let changed = false; + assert.throws(() => publish({...replaced.options, run(file, args, options) { + if (!changed && path.basename(file) === 'powershell.exe') { + changed = true; + fs.renameSync(replaced.source, path.join(replaced.root, 'retained-replaced-source')); + fs.mkdirSync(replaced.source); + } + return replaced.run(file, args, options); + }})); + assert.equal(fs.existsSync(replaced.target), false); +}); + +test('relocation, version, resource, compile and execution failures cannot publish bindings', t => { + const publish = publisher(); + for (const kind of ['version', 'prefix', 'resources', 'flags', 'compile', 'execute', 'reparse']) { + const f = fixture(t); + assert.throws(() => publish({...f.options, run(file, args, options) { + const name = path.basename(file), command = args.join(' '); + if (kind === 'reparse' && name === 'powershell.exe') throw Error('native reparse point'); + if (kind === 'version' && name === 'llvm-config.exe' && command === '--version') return '23.0.0\n'; + if (kind === 'prefix' && command === '--prefix') return f.source + '\n'; + if (kind === 'resources' && command === '--print-resource-dir') return path.join(f.source, 'lib/clang/22') + '\n'; + if (kind === 'flags' && command === '--cflags') return '-I' + path.join(f.source, 'include') + '\n'; + if (kind === 'compile' && name === 'clang.exe' && args.includes('-o')) throw Error('compile/link failed'); + if (kind === 'execute' && name === 'publication.exe') throw Error('native execution failed'); + return f.run(file, args, options); + }}), undefined, kind); + } +}); + +test('the actual action installs first and publishes only through the stable native action', () => { + const install = action.slice(action.indexOf(' - name: Install LLVM 22')); + assert.match(install, /uses: \.\/\.github\/actions\/publish-windows-llvm22/); + assert.doesNotMatch(install, />> "\$GITHUB_ENV"|>> "\$GITHUB_PATH"/); + assert.match(install, /steps\.msys2_attempt_1\.outcome == 'success'.*'1'/); + const metadata = fs.readFileSync('.github/actions/publish-windows-llvm22/action.yml', 'utf8'); + assert.match(metadata, /using: node24/); + assert.doesNotMatch(metadata, /continue-on-error:|\n\s+if:/); +}); diff --git a/.github/scripts/priority-windows-llvm.test.cjs b/.github/scripts/priority-windows-llvm.test.cjs new file mode 100644 index 000000000..0a523847c --- /dev/null +++ b/.github/scripts/priority-windows-llvm.test.cjs @@ -0,0 +1,175 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { test } = require('node:test'); + +const action = fs.readFileSync('.github/actions/setup-windows-llvm22/action.yml', 'utf8'); + +function bootstrapSteps() { + const bootstrap = action.slice(action.indexOf(' steps:'), action.indexOf(' - name: Install LLVM 22')); + return bootstrap.split(/(?= - name:)/).filter(step => step.includes('uses: msys2/setup-msys2@v2')); +} + +// Model the observed setup-msys2 implementation at ec48f7c5447b3140e2b088413ae3a55687bccb6e: +// release installs into join(location || RUNNER_TEMP, 'msys64'), refuses an +// existing directory, and writes the msys2 shell before updating packages. +// A failed download/extraction/update can therefore leave the directory behind. +function bootstrap(updateOutcomes) { + const runnerTemp = 'D:\\runner temp'; + const directories = new Set(); + const attempts = []; + let shellInstallation; + for (const [index, step] of bootstrapSteps().entries()) { + if (index > 0) { + const condition = step.match(/if: \$\{\{ (.+) \}\}/)?.[1]; + const expected = Array.from({ length: index }, (_, prior) => + `steps.msys2_attempt_${prior + 1}.outcome == 'failure'`).join(' && '); + assert.equal(condition, expected, 'retry only after every prior failure'); + if (!attempts.every(attempt => attempt.outcome === 'failure')) break; + } + const location = step.match(/\n location: (.+)/)?.[1] + .replace('${{ runner.temp }}', runnerTemp) || runnerTemp; + const installation = path.win32.join(location, 'msys64'); + const collision = directories.has(installation); + let outcome = 'failure'; + if (!collision) { + directories.add(installation); + shellInstallation = installation; + outcome = updateOutcomes[index] === 0 ? 'success' : 'failure'; + } + attempts.push({ installation, collision, outcome }); + if (outcome === 'success') break; + if (!/continue-on-error: true/.test(step)) { + return { attempts, directories, shellInstallation, installerMayRun: false }; + } + } + return { + attempts, directories, shellInstallation, + installerMayRun: attempts.at(-1)?.outcome === 'success', + }; +} + +test('MSYS2 bootstrap retries are bounded and the final failure remains fatal', () => { + const attempts = bootstrapSteps(); + assert.equal(attempts.length, 3); + for (const [index, step] of attempts.entries()) { + assert.match(step, new RegExp(`id: msys2_attempt_${index + 1}`)); + assert.match(step, /msystem: CLANG64/); + assert.match(step, /path-type: inherit/); + assert.match(step, /update: true/); + if (index < 2) { + assert.match(step, /continue-on-error: true/); + } else { + assert.doesNotMatch(step, /continue-on-error:/); + } + if (index > 0) { + for (let prior = 1; prior <= index; prior++) { + assert.match(step, new RegExp(`steps\\.msys2_attempt_${prior}\\.outcome == 'failure'`)); + } + } + } +}); + +test('a partially installed MSYS2 directory cannot poison subsequent bootstrap attempts', () => { + for (const outcomes of [[1, 0], [1, 1, 0]]) { + const result = bootstrap(outcomes); + assert.equal(result.installerMayRun, true, 'a later successful update must reach LLVM installation'); + assert.equal(result.attempts.length, outcomes.length); + assert.ok(result.attempts.every(attempt => !attempt.collision), 'each retry must start a fresh installation'); + assert.equal(result.directories.size, outcomes.length, 'failed bootstrap directories are retained, not deleted'); + assert.equal(result.shellInstallation, result.attempts.at(-1).installation, + 'the official msys2 wrapper must select the successful attempt, not the first failed installation'); + } +}); + +test('all three bootstrap attempts explicitly own distinct fresh runner-temp locations', () => { + const locations = bootstrapSteps().map((step, index) => { + assert.match(step, /\n release: true\n/); + const location = step.match(/\n location: (.+)/)?.[1]; + assert.equal(location, `\${{ runner.temp }}/plan9asm-msys2-attempt-${index + 1}`); + return location; + }); + assert.equal(new Set(locations).size, 3); + assert.doesNotMatch(action, /release: false|Remove-Item|rm\s+-[rf]|SigLevel\s*=|--nosignature|--skippgpcheck/); +}); + +test('bootstrap success short-circuits retries and exhausted bootstrap remains fatal', () => { + const first = bootstrap([0]); + assert.equal(first.attempts.length, 1); + assert.equal(first.installerMayRun, true); + assert.equal(first.shellInstallation, first.attempts[0].installation); + const exhausted = bootstrap([1, 1, 1]); + assert.equal(exhausted.attempts.length, 3); + assert.ok(exhausted.attempts.every(attempt => !attempt.collision)); + assert.equal(exhausted.installerMayRun, false, 'no LLVM transaction or build environment after final failure'); +}); + +function install(outcomes, version = '22.1.8') { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'plan9asm-llvm-install-')); + try { + const installStep = action.slice(action.indexOf(' - name: Install LLVM 22')).split(/\n - name:/)[0]; + const run = installStep.slice(installStep.indexOf(' run: |\n') + ' run: |\n'.length) + .split('\n').map(line => line.startsWith(' ') ? line.slice(8) : line).join('\n'); + const script = ` +mock_pacman_calls=0 +outcomes=(${outcomes.join(' ')}) +pacman() { + mock_pacman_calls=$((mock_pacman_calls + 1)) + printf 'PACMAN:' + printf ' %s' "$@" + printf '\\n' + return "\${outcomes[mock_pacman_calls - 1]}" +} +sleep() { test "$1" = 5; printf 'RETRY_WAIT\\n'; } +llvm-config() { + if [[ "$1" == --version ]]; then printf '%s\\n' "$MOCK_LLVM_VERSION"; + else printf 'checked-flags\\n'; fi +} +cygpath() { printf 'checked-clang64-bin\\n'; } +${run}`; + const result = spawnSync('bash', ['-c', script], { + encoding: 'utf8', timeout: 5000, + env: { + ...process.env, + LLVM_VERSION: '22.1.8', LLVM_PACKAGE_VERSION: '22.1.8-2', + LLVM_RUNTIME_PACKAGE_VERSION: '22.1.8-1', MOCK_LLVM_VERSION: version, + GITHUB_ENV: path.join(directory, 'environment'), GITHUB_PATH: path.join(directory, 'path'), + }, + }); + assert.ifError(result.error); + return { + ...result, + environment: fs.existsSync(path.join(directory, 'environment')) + ? fs.readFileSync(path.join(directory, 'environment'), 'utf8') : '', + }; + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +} + +test('actual installer script retries the identical pinned transaction only', () => { + for (const outcomes of [[0], [67, 0], [67, 68, 0]]) { + const result = install(outcomes); + assert.equal(result.status, 0, result.stderr); + const transactions = result.stdout.split('\n').filter(line => line.startsWith('PACMAN:')); + assert.equal(transactions.length, outcomes.length); + assert.equal(new Set(transactions).size, 1); + assert.match(transactions[0], /llvm-22\.1\.8-2-any\.pkg\.tar\.zst/); + assert.doesNotMatch(transactions[0], /llvm-23/); + assert.equal((result.stdout.match(/RETRY_WAIT/g) || []).length, outcomes.length - 1); + assert.equal(result.environment, '', 'installation alone cannot publish attempt-dependent bindings'); + } +}); + +test('exhausted installation and wrong LLVM version publish no build environment', () => { + const exhausted = install([67, 68, 69]); + assert.equal(exhausted.status, 69, exhausted.stderr); + assert.equal((exhausted.stdout.match(/PACMAN:/g) || []).length, 3); + assert.equal(exhausted.environment, ''); + const wrongVersion = install([0], '23.0.0'); + assert.equal(wrongVersion.status, 1); + assert.match(wrongVersion.stderr, /expected LLVM 22\.1\.8, got 23\.0\.0/); + assert.equal(wrongVersion.environment, ''); +}); diff --git a/.github/scripts/priority-windows-sdk-scope.test.cjs b/.github/scripts/priority-windows-sdk-scope.test.cjs new file mode 100644 index 000000000..b52483d73 --- /dev/null +++ b/.github/scripts/priority-windows-sdk-scope.test.cjs @@ -0,0 +1,95 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const {jobBody} = require('./workflow-needs.cjs'); + +const source = fs.readFileSync(path.join(__dirname, '../workflows/windows-module.yml'), 'utf8'); +const jobs = ['root-parts', 'other-owners', 'complete']; +const exactSDK = '1.27.2'; + +// These independent jobs exchange one source/tool/environment-bound plan. +// A floating minor selector can resolve to a different SDK between execution +// and fresh completion. Pin the shared scope, never normalize its real hashes. +function requireExactSDK(workflow) { + for (const name of jobs) { + const body = jobBody(workflow, name); + const steps = body.split(/(?=^ - )/m).slice(1); + const setup = steps.filter(step => + /^ - uses: \.\/\.github\/actions\/setup-go$/m.test(step)); + assert.equal(setup.length, 1, name + ': one actual owned Go setup required'); + assert.doesNotMatch(body, /uses: actions\/setup-go@/, + name + ': another Go installer cannot replace the owned setup'); + const versions = [...setup[0].matchAll(/^ go-version: (.+)$/gm)]; + assert.equal(versions.length, 1, name + ': one explicit Go pin required'); + assert.match(setup[0], /^ with:$/m, name + ': setup input scope missing'); + assert.ok(versions[0][1] === `'${exactSDK}'` || versions[0][1] === `"${exactSDK}"`, + name + ': execution and completion require the same exact ' + exactSDK + ' SDK'); + assert.doesNotMatch(setup[0], /go-version-file:|check-latest:/, + name + ': no alternative source may override the exact SDK'); + } + assert.equal((workflow.match(/uses: \.\/\.github\/actions\/setup-go$/gm) || []).length, 3, + 'all shared Windows Go setup roles must be covered'); +} + +function replaceJob(workflow, name, change) { + const before = jobBody(workflow, name); + const after = change(before); + assert.notEqual(after, before, 'negative control must mutate its actual job'); + return workflow.replace(before, after); +} + +// Controls begin from a complete positive fixture even while the real source +// is RED. A matching pin in another job or run string is never a substitute. +const fixture = source.replace(/^ go-version: .+$/gm, ` go-version: '${exactSDK}'`); + +test('shared Windows execution and fresh completion pin one exact SDK', () => { + requireExactSDK(source); +}); + +test('main Windows caller retains the shared executor and aggregate scope', () => { + for (const [file, name] of [ + ['go-ci.yml', 'windows-module'], + ]) { + const workflow = fs.readFileSync(path.join(__dirname, '../workflows', file), 'utf8'); + const body = jobBody(workflow, name); + assert.match(body, /^ uses: \.\/\.github\/workflows\/windows-module\.yml$/m); + assert.match(body, /^ module: \$\{\{ matrix\.module \}\}$/m); + assert.doesNotMatch(body, /go-version:/, + name + ': the shared scope must not inherit a floating caller selector'); + } + requireExactSDK(fixture); +}); + +for (const name of jobs) { + test(name + ' cannot use another patch, floating minor or dynamic SDK selector', () => { + requireExactSDK(fixture); + for (const version of ['1.27.3', '1.27.x', '1.27', 'stable', '${{ matrix.go }}']) { + assert.throws(() => requireExactSDK(replaceJob(fixture, name, body => + body.replace(`go-version: '${exactSDK}'`, `go-version: '${version}'`)))); + } + }); + + test(name + ' cannot omit, duplicate or borrow its SDK pin', () => { + const pin = ` go-version: '${exactSDK}'\n`; + for (const replacement of ['', pin + pin, ' go-version-file: go.mod\n']) { + assert.throws(() => requireExactSDK(replaceJob(fixture, name, body => + body.replace(pin, replacement)))); + } + assert.throws(() => requireExactSDK(replaceJob(fixture, name, body => + body.replace(pin, '') + ` - run: echo "go-version: '${exactSDK}'"\n`))); + }); + + test(name + ' cannot lose or repeat the actual Go installer', () => { + const setup = ' - uses: ./.github/actions/setup-go\n'; + assert.throws(() => requireExactSDK(replaceJob(fixture, name, body => + body.replace(setup, ' - uses: actions/setup-go@v7\n')))); + assert.throws(() => requireExactSDK(replaceJob(fixture, name, body => + body.replace(setup, setup + ` with:\n go-version: '${exactSDK}'\n` + setup)))); + assert.throws(() => requireExactSDK(replaceJob(fixture, name, body => + body.replace(` go-version: '${exactSDK}'\n`, + ` go-version: '${exactSDK}'\n check-latest: true\n`)))); + }); +} diff --git a/.github/scripts/priority-workflow-needs.test.cjs b/.github/scripts/priority-workflow-needs.test.cjs new file mode 100644 index 000000000..6d1cc18b5 --- /dev/null +++ b/.github/scripts/priority-workflow-needs.test.cjs @@ -0,0 +1,57 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const {jobBody, jobNeeds, requireJobNeeds} = require('./workflow-needs.cjs'); + +function replaceNeeds(source, name, declaration) { + const body = jobBody(source, name); + const changed = body.replace(/^ needs:.*$/m, ' needs: ' + declaration); + assert.notEqual(body, changed, 'negative/format control must change its actual job'); + return source.replace(body, changed); +} + +test('concrete dependencies reject duplicates, expressions, self references and absent jobs', () => { + const source = 'jobs:\n first:\n runs-on: ubuntu-latest\n' + + ' second:\n runs-on: ubuntu-latest\n' + + ' complete:\n needs: [first, second]\n'; + const requireComplete = value => requireJobNeeds(value, 'complete', ['first', 'second']); + assert.deepEqual(requireComplete(source), ['first', 'second']); + assert.deepEqual(requireComplete(replaceNeeds(source, 'complete', '["second", "first"]')), ['second', 'first']); + for (const declaration of [ + '[first, first]', '[first, absent]', '[first, complete]', + '${{ needs.from_another_job }}', '[]', '[first,]', '[first]', '[second]', + ]) assert.throws(() => requireComplete(replaceNeeds(source, 'complete', declaration))); + const body = jobBody(source, 'complete'); + assert.throws(() => requireComplete(source.replace(body, + body.replace(/^ needs:.*$/m, ' needs: first\n needs: second')))); + assert.throws(() => requireComplete(source.replace(body, body.replace(/^ needs:.*\n/m, '')))); + assert.throws(() => requireComplete(source + '\n complete:\n')); + assert.throws(() => requireComplete(source.replace('\n first:\n', '\n absent:\n'))); +}); + +test('main and reusable CLI completion retain every actual owner gate', () => { + for (const [file, name, required] of [ + ['go-ci.yml', 'module-completion', ['test', 'nested-cli', 'race', 'coverage', 'nested-cli-coverage']], + ['windows-module.yml', 'complete', ['root-parts', 'other-owners']], + ['linux-cli-module.yml', 'complete', ['root-parts']], + ]) { + const source = fs.readFileSync('.github/workflows/' + file, 'utf8'); + requireJobNeeds(source, name, required); + requireJobNeeds(replaceNeeds(source, name, '[' + required.slice().reverse().join(', ') + ']'), name, required); + for (const missing of required) { + const reduced = '[' + required.filter(gate => gate !== missing).join(', ') + ']'; + assert.throws(() => requireJobNeeds(replaceNeeds(source, name, reduced), name, required)); + } + } +}); + +test('concrete block and scalar needs retain the same owned success requirements', () => { + const source = 'jobs:\n first:\n runs-on: ubuntu-latest\n' + + ' second:\n needs: first\n' + + ' complete:\n needs:\n - first\n - second\n'; + assert.deepEqual(jobNeeds(source, 'second'), ['first']); + assert.deepEqual(requireJobNeeds(source, 'complete', ['second', 'first']), ['first', 'second']); + assert.throws(() => requireJobNeeds(source.replace(' - second\n', ''), 'complete', ['first', 'second'])); +}); diff --git a/.github/scripts/priority-x87-native.test.cjs b/.github/scripts/priority-x87-native.test.cjs new file mode 100644 index 000000000..aa18de3a3 --- /dev/null +++ b/.github/scripts/priority-x87-native.test.cjs @@ -0,0 +1,86 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); + +const roots = [ + 'TestCrossLinuxRuntimeMatrixX86X87SignedI64IdentityOriginalGoAndLLVM', + 'TestAMD64X87ConditionalMoveRuntimeConditions', + 'TestX86RawX87ClearExceptionRuntimeSemantics', + 'TestX86RawX87SoftwareClearPreservesAllStatusFields', + 'TestCrossLinuxRuntimeMatrixX86X87ScopedRoundingOriginalGoAndLLVM', +]; + +test('native x87 auditor requires actual complete ordered terminals, not printed PASS text', () => { + const {auditNativeX87, roots: actualRoots, children} = require('./audit-x87-native.cjs'); + assert.deepEqual(actualRoots, roots); + const originalChildren = []; + for (const arch of ['amd64', '386']) { + const parent = `${roots[0]}/${arch}`; + originalChildren.push(parent); + for (const api of ['text', 'module', 'Go']) { + for (const optimized of [false, true]) { + for (const backend of [0, 2]) originalChildren.push(`${parent}/${api}/IR_O2=${optimized}/backend-O${backend}`); + } + } + } + const roundingChildren = []; + for (const api of ['text', 'module', 'Go']) { + const parent = `${roots[4]}/${api}`; + roundingChildren.push(parent); + for (const optimized of [false, true]) { + for (const backend of [0, 2]) roundingChildren.push(`${parent}/IR_O2=${optimized}/backend-O${backend}`); + } + } + assert.equal(originalChildren.length, 26, 'retain all original native axes'); + assert.equal(roundingChildren.length, 15, 'three rounding API parents and twelve complete pipelines'); + assert.deepEqual(children, [...originalChildren, ...roundingChildren]); + const event = (Action, Test) => ({Action, Package: 'github.com/xgo-dev/plan9asm', ...(Test ? {Test} : {})}); + const events = [event('start')]; + for (const root of roots) { + events.push(event('run', root)); + for (const parent of children.filter(name => name.startsWith(root + '/') && name.split('/').length === 2)) { + events.push(event('run', parent)); + for (const child of children.filter(name => name.startsWith(parent + '/'))) { + events.push(event('run', child), event('pass', child)); + } + events.push(event('pass', parent)); + } + events.push(event('pass', root)); + } + events.push(event('pass')); + const inventory = roots.join('\n') + '\nok\tgithub.com/xgo-dev/plan9asm\n'; + const encode = values => values.map(value => JSON.stringify(value)).join('\n') + '\n'; + assert.deepEqual(auditNativeX87(inventory, encode(events)), {roots: 5, children: 41, passed: 46, skipped: 0, incomplete: 0}); + const child = children[1]; + const earlyParent = events.filter(value => value.Test !== children[0] || value.Action !== 'pass'); + earlyParent.splice(earlyParent.findIndex(value => value.Test === child), 0, event('pass', children[0])); + for (const changed of [ + earlyParent, + events.filter(value => value.Test !== child), + events.filter(value => value.Test !== child || value.Action !== 'pass'), + events.map(value => value.Test === child && value.Action === 'pass' ? {...value, Action: 'skip'} : value), + events.map(value => value.Test === child && value.Action === 'pass' ? {...value, Action: 'fail'} : value), + events.map(value => value.Test === child ? {...value, Test: child + '-object-only'} : value), + events.slice(0, -1), + [...events, event('pass')], + [event('start'), {...event('output'), Output: events.map(value => 'PASS ' + (value.Test || 'package')).join('\n')}, event('pass')], + ]) assert.throws(() => auditNativeX87(inventory, encode(changed))); + for (const child of roundingChildren) { + for (const action of ['skip', 'fail']) { + assert.throws(() => auditNativeX87(inventory, encode(events.map(value => + value.Test === child && value.Action === 'pass' ? {...value, Action: action} : value)))); + } + assert.throws(() => auditNativeX87(inventory, encode(events.filter(value => value.Test !== child)))); + assert.throws(() => auditNativeX87(inventory, encode(events.filter(value => + value.Test !== child || value.Action !== 'pass')))); + } + assert.throws(() => auditNativeX87(inventory.replace(roots[0], 'TestObjectOnly'), encode(events))); +}); + +test('scoped rounding emits explicit live API parents for all twelve native pipelines', () => { + const source = fs.readFileSync('x86_x87_scoped_rounding_linux_amd64_test.go', 'utf8'); + assert.match(source, /func TestCrossLinuxRuntimeMatrixX86X87ScopedRoundingOriginalGoAndLLVM\(t \*testing\.T\)/); + assert.match(source, /for _, api := range \[\]string\{"text", "module", "Go"\} \{\n\s+t\.Run\(api, func\(t \*testing\.T\) \{/); + assert.match(source, /t\.Run\(fmt\.Sprintf\("IR_O2=%t\/backend%s", optimized, level\)/); + assert.doesNotMatch(source, /t\.Skip|exec\.Command|qemu|rosetta/); +}); diff --git a/.github/scripts/workflow-needs.cjs b/.github/scripts/workflow-needs.cjs new file mode 100644 index 000000000..0b9fc9890 --- /dev/null +++ b/.github/scripts/workflow-needs.cjs @@ -0,0 +1,61 @@ +'use strict'; + +const assert = require('node:assert/strict'); + +function jobBody(source, name) { + assert.match(name, /^[a-zA-Z_][a-zA-Z0-9_-]*$/, 'invalid workflow job identity'); + const marker = '\n ' + name + ':\n'; + assert.equal(source.split(marker).length - 1, 1, 'one exact workflow job required: ' + name); + const rest = source.slice(source.indexOf(marker) + marker.length); + const end = rest.search(/^ [a-zA-Z_][a-zA-Z0-9_-]*:\s*$/m); + return end < 0 ? rest : rest.slice(0, end); +} + +// This intentionally accepts only concrete job identities, not an expression, +// comment or a matching line in a different job. Whitespace/order and adding +// another real dependency do not change whether each required gate is present. +function jobNeeds(source, name) { + const lines = jobBody(source, name).split('\n'); + const positions = lines.flatMap((line, index) => /^ needs:/.test(line) ? [index] : []); + assert.equal(positions.length, 1, 'one actual needs declaration required: ' + name); + const index = positions[0], value = lines[index].slice(' needs:'.length).trim(); + let needs; + if (value.startsWith('[')) { + assert.ok(value.endsWith(']'), 'incomplete concrete needs array: ' + name); + needs = value.slice(1, -1).split(',').map(value => value.trim()); + } else if (value) { + needs = [value]; + } else { + needs = []; + for (const line of lines.slice(index + 1)) { + if (!/^ - /.test(line)) break; + needs.push(line.slice(' - '.length).trim()); + } + } + assert.ok(needs.length > 0, 'empty workflow needs: ' + name); + needs = needs.map(dependency => { + if ((dependency.startsWith('"') && dependency.endsWith('"')) || + (dependency.startsWith("'") && dependency.endsWith("'"))) { + return dependency.slice(1, -1); + } + return dependency; + }); + assert.equal(new Set(needs).size, needs.length, 'duplicate workflow dependency: ' + name); + for (const dependency of needs) { + assert.match(dependency, /^[a-zA-Z_][a-zA-Z0-9_-]*$/, 'non-concrete workflow dependency: ' + name); + assert.notEqual(dependency, name, 'workflow job cannot depend on itself'); + jobBody(source, dependency); + } + return needs; +} + +function requireJobNeeds(source, name, required) { + const actual = jobNeeds(source, name); + assert.equal(new Set(required).size, required.length, 'duplicate required dependency policy'); + for (const dependency of required) { + assert.ok(actual.includes(dependency), name + ' omitted required success gate: ' + dependency); + } + return actual; +} + +module.exports = {jobBody, jobNeeds, requireJobNeeds}; diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 39dd98fc6..fd13182ef 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -18,18 +18,60 @@ permissions: contents: read jobs: + ci_policy: + name: CI policy preflight + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install LLVM 22 + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh + + - name: Setup Go + timeout-minutes: 10 + uses: ./.github/actions/setup-go + with: + go-version: '1.27.1' + + - name: Setup verified policy runtime + uses: ./.github/actions/setup-node22 + timeout-minutes: 10 + + - name: Workflow scheduling regression tests + run: node --test .github/scripts/priority-*.test.cjs + + - name: CI policy tests + run: go test . -run '^(TestCI|TestFixedLocal|TestGoAMD64FixedLocal(Packed|Conversion)RuntimeOwnership$|TestX86PackedConversion(RequiredNativeOwners|CompleteNativeRuntimeOwnership)$|TestX86FloatingCompareRequiredLinuxBuildSelection$|TestRuntimeOracleCommand(ConsumersRetainOwnedExecution|AuditDistinguishesTestingRun)$)' -count=1 + + - name: Compile explicit native AVX512 owners and verify their inventory + run: go test . -tags=plan9asm_native_avx512 -run '^TestCIAVX512NativeQualificationInventory$' -count=1 + + - name: Official corpus form baselines + timeout-minutes: 10 + shell: bash + run: | + set -euo pipefail + scripts/check-go-asm-coverage.sh + scripts/check-arm64-plan9-corpus.sh + + fmt: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v7 - name: Setup Go + timeout-minutes: 10 uses: ./.github/actions/setup-go with: - go-version: '1.27.x' - cache: true + go-version: '1.27.2' - name: Go fmt check run: | @@ -41,25 +83,22 @@ jobs: fi build: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v7 - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh - name: Setup Go + timeout-minutes: 10 uses: ./.github/actions/setup-go with: - go-version: '1.27.x' - cache: true + go-version: '1.27.2' - name: Go build run: go build ./... @@ -75,61 +114,10 @@ jobs: - name: ARM64 Plan 9 reference corpus run: scripts/check-arm64-plan9-corpus.sh - reported_library_list: - name: Third-party corpus matrix - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} - outputs: - libraries: ${{ steps.matrix.outputs.libraries }} - - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Read third-party library manifest - id: matrix - run: | - manifest=testdata/corpus/reported-libraries.json - if ! jq -e '.libraries | length > 0 and all(.[]; (.id | test("^[a-z0-9]+(-[a-z0-9]+)*$")))' "$manifest" >/dev/null; then - echo "third-party library ids must match ^[a-z0-9]+(-[a-z0-9]+)*$" >&2 - exit 1 - fi - libraries=$(jq -c '[.libraries[].id]' "$manifest") - printf '%s\n' "libraries=$libraries" >> "$GITHUB_OUTPUT" - - reported_library_corpus: - name: third-party-corpus (${{ matrix.library }}) - needs: reported_library_list - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} - strategy: - fail-fast: false - matrix: - library: ${{ fromJSON(needs.reported_library_list.outputs.libraries) }} - - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" - - - name: Setup Go - uses: ./.github/actions/setup-go - with: - go-version: '1.27.x' - cache: true - - - name: Scan latest module and compile complete asm corpus - env: - REPORTED_LIBRARY: ${{ matrix.library }} - run: scripts/check-reported-library-corpus.sh "$REPORTED_LIBRARY" stdlib-corpus: - name: stdlib-corpus (${{ matrix.os }}, Go ${{ matrix.go-version }}) + needs: ci_policy + name: stdlib instruction corpus (${{ matrix.os }}, Go ${{ matrix.go-version }}) runs-on: ${{ github.repository_owner == 'xgo-dev' && matrix.runner || matrix.os }} strategy: fail-fast: false @@ -145,10 +133,10 @@ jobs: - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.24.x', targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.25.x', targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.26.x', targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.27.x', targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.27.2', targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} # Native hosts are auxiliary integration checks on the latest Go. - - {os: macos-latest, go-version: '1.27.x', targets: 'darwin/amd64,darwin/arm64'} - - {os: windows-latest, go-version: '1.27.x', targets: 'windows/386,windows/amd64,windows/arm64'} + - {os: macos-latest, go-version: '1.27.2', targets: 'darwin/amd64,darwin/arm64'} + - {os: windows-latest, go-version: '1.27.2', targets: 'windows/386,windows/amd64,windows/arm64'} steps: - name: Checkout @@ -156,12 +144,8 @@ jobs: - name: Install LLVM 22 (Ubuntu) if: runner.os == 'Linux' - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh - name: Install LLVM 22 (macOS) if: runner.os == 'macOS' @@ -181,10 +165,10 @@ jobs: - name: Setup Go for corpus tool build if: matrix.prebuild_tool + timeout-minutes: 10 uses: ./.github/actions/setup-go with: - go-version: '1.27.x' - cache: false + go-version: '1.27.2' - name: Build corpus tool for older Go lane if: matrix.prebuild_tool @@ -195,10 +179,10 @@ jobs: echo "PLAN9ASM_CMD=$tool" >> "$GITHUB_ENV" - name: Setup Go + timeout-minutes: 10 uses: ./.github/actions/setup-go with: go-version: ${{ matrix.go-version }} - cache: true - name: Stdlib asm corpus gate shell: bash @@ -206,7 +190,7 @@ jobs: PLAN9ASM_CORPUS_TARGETS: ${{ matrix.targets }} run: | chmod +x scripts/check-stdlib-corpus.sh - scripts/check-stdlib-corpus.sh + scripts/check-stdlib-corpus.sh --instructions-only - name: Official Go assembler coverage regression shell: bash @@ -215,37 +199,133 @@ jobs: scripts/check-go-asm-coverage.sh cross-runtime: + needs: ci_policy name: cross compile, link, and run (Linux) - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + strategy: + # Every selected root is owned exactly once; owner zero isolates the + # giant matrix root. A failure must not cancel other owners. + fail-fast: false + matrix: + owner: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + + env: + GOFLAGS: '' + GOENV: 'off' + GOWORK: 'off' + GOTOOLCHAIN: local + PLAN9ASM_CROSS_EXEC: '1' steps: - name: Checkout uses: actions/checkout@v7 - - name: Install LLVM, cross compilers, and QEMU - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y \ - llvm-22-dev clang-22 lld-22 \ - gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu \ - qemu-user - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + - name: Install LLVM and cross compilers + timeout-minutes: 14 + run: >- + bash scripts/install-ci-llvm22-ubuntu.sh + gcc-i686-linux-gnu gcc-arm-linux-gnueabihf + gcc-aarch64-linux-gnu + + - name: Install pinned QEMU user-mode emulators + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh - name: Setup Go + timeout-minutes: 10 uses: ./.github/actions/setup-go with: - go-version: '1.27.x' - cache: true + go-version: '1.27.2' + + - name: Setup required wasm execution runtime + timeout-minutes: 10 + uses: ./.github/actions/setup-node22 - name: Cross transpile, compile, link, and run - env: - PLAN9ASM_CROSS_EXEC: '1' - run: go test . -run '^(TestCrossLinuxRuntimeMatrix|TestARM64Conformance)' -count=1 -v + run: | + mkdir -p _out + go run ./cmd/plan9asmtest -out _out/cross-runtime-owner \ + -ci-selector '^(TestCrossLinuxRuntimeMatrix|TestARM64Conformance|TestARM64RawSQSHLURuntimeSemantics|TestARMIntegerMemoryRuntimeCross|TestARMRawSaturationRuntimeCross)' \ + -ci-isolate-root TestCrossLinuxRuntimeMatrix \ + -ci-owner ${{ matrix.owner }} -shards 16 -timeout 20m + + - name: Execute rich CLI ARM64 fallback oracle + if: ${{ !cancelled() && matrix.owner == 15 }} + run: bash scripts/test-arm64-fallback-ci.sh execute + + - name: Retain every owner receipt, including failures + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: cross-runtime-owner-${{ matrix.owner }} + path: _out/cross-runtime-owner + if-no-files-found: error + + cross-runtime-verify: + name: cross-runtime exhaustive completion + needs: cross-runtime + if: ${{ !cancelled() }} + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + env: + GOFLAGS: '' + GOENV: 'off' + GOWORK: 'off' + GOTOOLCHAIN: local + PLAN9ASM_CROSS_EXEC: '1' + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install LLVM and cross compilers + timeout-minutes: 14 + run: >- + bash scripts/install-ci-llvm22-ubuntu.sh + gcc-i686-linux-gnu gcc-arm-linux-gnueabihf + gcc-aarch64-linux-gnu + + - name: Install pinned QEMU user-mode emulators + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + + - name: Setup Go + timeout-minutes: 10 + uses: ./.github/actions/setup-go + with: + go-version: '1.27.2' + + - name: Setup required wasm execution runtime + timeout-minutes: 10 + uses: ./.github/actions/setup-node22 + + - name: Download all owner evidence + uses: actions/download-artifact@v8 + with: + pattern: cross-runtime-owner-* + path: _out/cross-runtime-partitions + + - name: Verify the fresh complete five-prefix inventory + run: | + go run ./cmd/plan9asmtest -out _out/cross-runtime-aggregate \ + -verify-ci-partitions _out/cross-runtime-partitions \ + -ci-selector '^(TestCrossLinuxRuntimeMatrix|TestARM64Conformance|TestARM64RawSQSHLURuntimeSemantics|TestARMIntegerMemoryRuntimeCross|TestARMRawSaturationRuntimeCross)' \ + -ci-isolate-root TestCrossLinuxRuntimeMatrix -shards 16 -timeout 20m + + - name: Verify rich CLI ARM64 fallback execution + if: ${{ !cancelled() }} + run: bash scripts/test-arm64-fallback-ci.sh complete + + - name: Retain completion evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: cross-runtime-completion + path: _out/cross-runtime-aggregate + if-no-files-found: error test: - name: test (${{ matrix.os }}, ${{ matrix.go-version }}, ${{ matrix.nested_modules }}) + needs: ci_policy + name: test (${{ matrix.os }}, ${{ matrix.go-version }}, ${{ matrix.module-role }}) + timeout-minutes: 60 permissions: contents: read id-token: write @@ -254,26 +334,42 @@ jobs: # Keep both matrix variants visible even if one fails. fail-fast: false matrix: + os: [ubuntu-latest] + # Create current-SDK and minimum-SDK jobs before historical lanes. + # Runner scheduling is external; retain every version and owner. + go-version: ['1.27.2', '1.20.x', '1.26.x', '1.25.x', '1.24.x', '1.23.x', '1.22.x', '1.21.x'] + module-role: [root-0, root-1, root-2, root-3, root-4, root-5, root-6, root-7, + root-8, root-9, root-10, root-11, root-12, root-13, root-14, root-15, profile, other] include: - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.20.x', nested_modules: false} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.21.x', nested_modules: false} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.22.x', nested_modules: false} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.23.x', nested_modules: false} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.24.x', nested_modules: true} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.25.x', nested_modules: true} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.26.x', nested_modules: true} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.27.x', nested_modules: true} - - {os: windows-latest, go-version: '1.27.x', nested_modules: true} + - {module-role: root-0, module-owner: 0, root-partition: 0} + - {module-role: root-1, module-owner: 0, root-partition: 1} + - {module-role: root-2, module-owner: 0, root-partition: 2} + - {module-role: root-3, module-owner: 0, root-partition: 3} + - {module-role: root-4, module-owner: 0, root-partition: 4} + - {module-role: root-5, module-owner: 0, root-partition: 5} + - {module-role: root-6, module-owner: 0, root-partition: 6} + - {module-role: root-7, module-owner: 0, root-partition: 7} + - {module-role: root-8, module-owner: 0, root-partition: 8} + - {module-role: root-9, module-owner: 0, root-partition: 9} + - {module-role: root-10, module-owner: 0, root-partition: 10} + - {module-role: root-11, module-owner: 0, root-partition: 11} + - {module-role: root-12, module-owner: 0, root-partition: 12} + - {module-role: root-13, module-owner: 0, root-partition: 13} + - {module-role: root-14, module-owner: 0, root-partition: 14} + - {module-role: root-15, module-owner: 0, root-partition: 15} + - {module-role: profile, module-owner: 1, root-partition: -1} + - {module-role: other, module-owner: 2, root-partition: -1} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large]} steps: - name: Checkout uses: actions/checkout@v7 - name: Setup Go + timeout-minutes: 10 uses: ./.github/actions/setup-go with: go-version: ${{ matrix.go-version }} - cache: true - name: Setup Python (Windows) if: runner.os == 'Windows' @@ -283,17 +379,18 @@ jobs: - name: Install LLVM 22 (Ubuntu) if: runner.os == 'Linux' - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu - name: Install LLVM 22 (Windows) if: runner.os == 'Windows' uses: ./.github/actions/setup-windows-llvm22 + - name: Install required 386 execution runtime + if: runner.os == 'Linux' + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + - name: LLVM tools info shell: bash run: | @@ -302,52 +399,69 @@ jobs: command -v clang clang --version + - name: Setup required wasm execution runtime + uses: ./.github/actions/setup-node22 + timeout-minutes: 10 + - name: Go test (Ubuntu) if: runner.os != 'Windows' - run: go test ./... + run: bash scripts/test-module-ci.sh test execute ${{ matrix.module-owner }} ${{ matrix.root-partition }} - - name: Go test (cmd/plan9asm, Ubuntu) - if: runner.os != 'Windows' && matrix.nested_modules - run: go test ./... - working-directory: cmd/plan9asm - - - name: Go test (cmd/plan9asmll, Ubuntu) - if: runner.os != 'Windows' && matrix.nested_modules - run: go test ./... - working-directory: cmd/plan9asmll - - - name: Go test with coverage (Windows) - if: runner.os == 'Windows' && matrix.go-version == '1.27.x' + - name: Required exact downloaded AMD64 numerical fixture + if: runner.os == 'Linux' && matrix.go-version == '1.27.2' && matrix.module-owner == 2 shell: bash run: | set -euo pipefail - go test ./... -coverprofile=coverage-windows-root.out - (cd cmd/plan9asm && go test ./... -coverprofile=../../coverage-windows-plan9asm.out) - (cd cmd/plan9asmll && go test ./... -coverprofile=../../coverage-windows-plan9asmll.out) + test "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" = linux/amd64 + scope='^TestAMD64DownloadedFixedLocalLLVM22Numerical$' + go test ./gotoolprofile -list="$scope" | grep -Fx 'TestAMD64DownloadedFixedLocalLLVM22Numerical' + go test ./gotoolprofile -run="$scope" -count=1 -timeout=10m + + - name: Retain complete module owner evidence (Ubuntu) + if: ${{ !cancelled() && runner.os != 'Windows' }} + uses: actions/upload-artifact@v6 + with: + name: module-test-${{ matrix.go-version }}-${{ matrix.module-role }} + path: _out/module-test/ + if-no-files-found: error + + # Complete CLI suites need independent job budgets instead of sharing the + # remaining-package owner's root suite and current numerical fixture. + nested-cli: + needs: ci_policy + name: test (ubuntu-latest, ${{ matrix.go-version }}, ${{ matrix.module }}) + strategy: + fail-fast: false + matrix: + go-version: ['1.24.x', '1.25.x', '1.26.x', '1.27.2'] + module: [plan9asm, plan9asmll] - - name: Go test (Windows) - if: runner.os == 'Windows' && matrix.go-version != '1.27.x' - shell: bash - run: | - set -euo pipefail - go test ./... - (cd cmd/plan9asm && go test ./...) - (cd cmd/plan9asmll && go test ./...) + uses: ./.github/workflows/linux-cli-module.yml + with: + module: ${{ matrix.module }} + go-version: ${{ matrix.go-version }} + flavor: test - - name: Upload Windows coverage to Codecov - if: runner.os == 'Windows' && matrix.go-version == '1.27.x' - uses: codecov/codecov-action@v7 - with: - use_oidc: true - files: ./coverage-windows-root.out,./coverage-windows-plan9asm.out,./coverage-windows-plan9asmll.out - disable_search: true - flags: windows - name: windows-go1.27 - fail_ci_if_error: true - verbose: true + + # Each module independently executes its complete inventory; only its strict + # root/profile/remaining-package aggregate may publish coverage. + windows-module: + needs: ci_policy + name: test (windows-latest, 1.27.2, ${{ matrix.module }}) + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + module: [root, plan9asm, plan9asmll] + uses: ./.github/workflows/windows-module.yml + with: + module: ${{ matrix.module }} arm-scan: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} strategy: fail-fast: false matrix: @@ -364,23 +478,14 @@ jobs: uses: actions/checkout@v7 - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - for attempt in 1 2 3; do - sudo apt-get update && sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev && break - if [ "$attempt" -eq 3 ]; then - exit 1 - fi - sleep 5 - done - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh - name: Setup Go + timeout-minutes: 10 uses: ./.github/actions/setup-go with: - go-version: '1.27.x' - cache: true + go-version: '1.27.2' - name: ARM scan gate env: @@ -407,31 +512,116 @@ jobs: PY race: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + name: race (${{ matrix.module-role }}) + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + module-role: [root-0, root-1, root-2, root-3, root-4, root-5, root-6, root-7, + root-8, root-9, root-10, root-11, root-12, root-13, root-14, root-15, + root-16, root-17, root-18, root-19, root-20, root-21, root-22, root-23, + root-24, root-25, root-26, root-27, root-28, root-29, root-30, root-31, + profile, other] + include: + - {module-role: root-0, module-owner: 0, root-partition: 0} + - {module-role: root-1, module-owner: 0, root-partition: 1} + - {module-role: root-2, module-owner: 0, root-partition: 2} + - {module-role: root-3, module-owner: 0, root-partition: 3} + - {module-role: root-4, module-owner: 0, root-partition: 4} + - {module-role: root-5, module-owner: 0, root-partition: 5} + - {module-role: root-6, module-owner: 0, root-partition: 6} + - {module-role: root-7, module-owner: 0, root-partition: 7} + - {module-role: root-8, module-owner: 0, root-partition: 8} + - {module-role: root-9, module-owner: 0, root-partition: 9} + - {module-role: root-10, module-owner: 0, root-partition: 10} + - {module-role: root-11, module-owner: 0, root-partition: 11} + - {module-role: root-12, module-owner: 0, root-partition: 12} + - {module-role: root-13, module-owner: 0, root-partition: 13} + - {module-role: root-14, module-owner: 0, root-partition: 14} + - {module-role: root-15, module-owner: 0, root-partition: 15} + - {module-role: root-16, module-owner: 0, root-partition: 16} + - {module-role: root-17, module-owner: 0, root-partition: 17} + - {module-role: root-18, module-owner: 0, root-partition: 18} + - {module-role: root-19, module-owner: 0, root-partition: 19} + - {module-role: root-20, module-owner: 0, root-partition: 20} + - {module-role: root-21, module-owner: 0, root-partition: 21} + - {module-role: root-22, module-owner: 0, root-partition: 22} + - {module-role: root-23, module-owner: 0, root-partition: 23} + - {module-role: root-24, module-owner: 0, root-partition: 24} + - {module-role: root-25, module-owner: 0, root-partition: 25} + - {module-role: root-26, module-owner: 0, root-partition: 26} + - {module-role: root-27, module-owner: 0, root-partition: 27} + - {module-role: root-28, module-owner: 0, root-partition: 28} + - {module-role: root-29, module-owner: 0, root-partition: 29} + - {module-role: root-30, module-owner: 0, root-partition: 30} + - {module-role: root-31, module-owner: 0, root-partition: 31} + - {module-role: profile, module-owner: 1, root-partition: -1} + - {module-role: other, module-owner: 2, root-partition: -1} steps: - name: Checkout uses: actions/checkout@v7 - name: Setup Go + timeout-minutes: 10 uses: ./.github/actions/setup-go with: - go-version: '1.27.x' - cache: true + go-version: '1.27.2' - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu + + - name: Install required 386 execution runtime + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + + - name: Setup required wasm execution runtime + uses: ./.github/actions/setup-node22 + timeout-minutes: 10 - name: Go test (race) - run: go test -race ./... + run: bash scripts/test-module-ci.sh race execute ${{ matrix.module-owner }} ${{ matrix.root-partition }} + + - name: Retain complete module race owner evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: module-race-1.27.2-${{ matrix.module-role }} + path: _out/module-race/ + if-no-files-found: error coverage: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + name: coverage (${{ matrix.module-role }}) + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + module-role: [root-0, root-1, root-2, root-3, root-4, root-5, root-6, root-7, + root-8, root-9, root-10, root-11, root-12, root-13, root-14, root-15, profile, other] + include: + - {module-role: root-0, module-owner: 0, root-partition: 0} + - {module-role: root-1, module-owner: 0, root-partition: 1} + - {module-role: root-2, module-owner: 0, root-partition: 2} + - {module-role: root-3, module-owner: 0, root-partition: 3} + - {module-role: root-4, module-owner: 0, root-partition: 4} + - {module-role: root-5, module-owner: 0, root-partition: 5} + - {module-role: root-6, module-owner: 0, root-partition: 6} + - {module-role: root-7, module-owner: 0, root-partition: 7} + - {module-role: root-8, module-owner: 0, root-partition: 8} + - {module-role: root-9, module-owner: 0, root-partition: 9} + - {module-role: root-10, module-owner: 0, root-partition: 10} + - {module-role: root-11, module-owner: 0, root-partition: 11} + - {module-role: root-12, module-owner: 0, root-partition: 12} + - {module-role: root-13, module-owner: 0, root-partition: 13} + - {module-role: root-14, module-owner: 0, root-partition: 14} + - {module-role: root-15, module-owner: 0, root-partition: 15} + - {module-role: profile, module-owner: 1, root-partition: -1} + - {module-role: other, module-owner: 2, root-partition: -1} permissions: contents: read id-token: write @@ -441,37 +631,307 @@ jobs: uses: actions/checkout@v7 - name: Setup Go + timeout-minutes: 10 uses: ./.github/actions/setup-go with: - go-version: '1.27.x' - cache: true + go-version: '1.27.2' - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu - - name: Go test with coverage - run: | - go test ./... -coverprofile=coverage-root.out - go tool cover -func=coverage-root.out | tail -n 1 + - name: Install required 386 execution runtime + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh - - name: Go test with coverage (cmd/plan9asm) - run: | - go test ./... -coverprofile=coverage-plan9asm.out - go tool cover -func=coverage-plan9asm.out | tail -n 1 - working-directory: cmd/plan9asm + - name: Setup required wasm execution runtime + uses: ./.github/actions/setup-node22 + timeout-minutes: 10 - - name: Go test with coverage (cmd/plan9asmll) - run: | - go test ./... -coverprofile=coverage-plan9asmll.out - go tool cover -func=coverage-plan9asmll.out | tail -n 1 - working-directory: cmd/plan9asmll + - name: Go test with coverage + run: bash scripts/test-module-ci.sh coverage execute ${{ matrix.module-owner }} ${{ matrix.root-partition }} + - name: Retain complete module coverage owner evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: module-coverage-1.27.2-${{ matrix.module-role }} + path: _out/module-coverage/ + if-no-files-found: error + + # Coverage-instrumented CLI suites retain their own complete command budget, + # independent of the main module and of the ordinary nested CLI matrix. + nested-cli-coverage: + needs: ci_policy + name: coverage (ubuntu-latest, 1.27.2, ${{ matrix.module }}) + strategy: + fail-fast: false + matrix: + module: [plan9asm, plan9asmll] + + uses: ./.github/workflows/linux-cli-module.yml + with: + module: ${{ matrix.module }} + go-version: '1.27.2' + flavor: coverage + + module-completion: + needs: [test, nested-cli, race, coverage, nested-cli-coverage] + if: ${{ !cancelled() }} + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 30 + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + include: + - {flavor: test, go-version: '1.20.x'} + - {flavor: test, go-version: '1.21.x'} + - {flavor: test, go-version: '1.22.x'} + - {flavor: test, go-version: '1.23.x'} + - {flavor: test, go-version: '1.24.x'} + - {flavor: test, go-version: '1.25.x'} + - {flavor: test, go-version: '1.26.x'} + - {flavor: test, go-version: '1.27.2'} + - {flavor: race, go-version: '1.27.2'} + - {flavor: coverage, go-version: '1.27.2'} + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Setup Go + timeout-minutes: 10 + uses: ./.github/actions/setup-go + with: + go-version: ${{ matrix.go-version }} + - name: Install LLVM 22 + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu + - name: Install required 386 execution runtime + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + - name: Setup required wasm execution runtime + uses: ./.github/actions/setup-node22 + timeout-minutes: 10 + - name: Download root part 0 + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-0 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0000 + - name: Download root part 1 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-1 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0001 + - name: Download root part 2 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-2 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0002 + - name: Download root part 3 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-3 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0003 + - name: Download root part 4 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-4 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0004 + - name: Download root part 5 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-5 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0005 + - name: Download root part 6 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-6 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0006 + - name: Download root part 7 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-7 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0007 + - name: Download root part 8 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-8 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0008 + - name: Download root part 9 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-9 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0009 + - name: Download root part 10 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-10 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0010 + - name: Download root part 11 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-11 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0011 + - name: Download root part 12 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-12 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0012 + - name: Download root part 13 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-13 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0013 + - name: Download root part 14 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-14 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0014 + - name: Download root part 15 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-15 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0015 + - name: Download root part 16 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-16 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0016 + - name: Download root part 17 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-17 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0017 + - name: Download root part 18 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-18 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0018 + - name: Download root part 19 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-19 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0019 + - name: Download root part 20 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-20 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0020 + - name: Download root part 21 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-21 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0021 + - name: Download root part 22 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-22 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0022 + - name: Download root part 23 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-23 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0023 + - name: Download root part 24 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-24 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0024 + - name: Download root part 25 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-25 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0025 + - name: Download root part 26 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-26 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0026 + - name: Download root part 27 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-27 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0027 + - name: Download root part 28 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-28 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0028 + - name: Download root part 29 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-29 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0029 + - name: Download root part 30 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-30 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0030 + - name: Download root part 31 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-31 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0031 + - name: Download profile owner + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-profile + path: _out/module-${{ matrix.flavor }}-owners/owner-0001 + - name: Download remaining-package owner + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-other + path: _out/module-${{ matrix.flavor }}-owners/owner-0002 + - name: Check complete actual module inventory and coverage + if: ${{ !cancelled() }} + run: bash scripts/test-module-ci.sh ${{ matrix.flavor }} complete + - name: Retain independent aggregate evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: module-verified-${{ matrix.flavor }}-${{ matrix.go-version }} + path: _out/module-${{ matrix.flavor }}-verified/ + if-no-files-found: error + - name: Show complete module coverage + if: matrix.flavor == 'coverage' + run: go tool cover -func=coverage-root.out | tail -n 1 - name: Upload coverage to Codecov + if: matrix.flavor == 'coverage' uses: codecov/codecov-action@v7 with: use_oidc: true @@ -481,3 +941,6 @@ jobs: name: ubuntu-go1.27 fail_ci_if_error: false verbose: true + - name: Require complete CLI coverage + if: ${{ !cancelled() && matrix.flavor == 'coverage' }} + run: test '${{ needs['nested-cli-coverage'].result }}' = success diff --git a/.github/workflows/linux-cli-module.yml b/.github/workflows/linux-cli-module.yml new file mode 100644 index 000000000..aad38ca85 --- /dev/null +++ b/.github/workflows/linux-cli-module.yml @@ -0,0 +1,166 @@ +name: Complete Linux CLI module + +on: + workflow_call: + inputs: + module: + required: true + type: string + go-version: + required: true + type: string + flavor: + required: true + type: string + +permissions: + contents: read + +jobs: + root-parts: + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + part: [0, 1, 2, 3] + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Setup Go + timeout-minutes: 10 + uses: ./.github/actions/setup-go + with: + go-version: ${{ inputs.go-version }} + - name: Install LLVM 22 and cross compilers + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu + - name: Install required execution runtimes + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + - name: LLVM tools info + shell: bash + run: | + command -v llc + llc --version + command -v clang + clang --version + - name: Setup required wasm execution runtime + timeout-minutes: 10 + uses: ./.github/actions/setup-node22 + - name: Execute disjoint root assignment + run: bash scripts/test-linux-cli-module-ci.sh ${{ inputs.module }} ${{ inputs.flavor }} execute 0 ${{ matrix.part }} + - name: Retain actual root evidence + if: ${{ always() }} + uses: actions/upload-artifact@v6 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-root-${{ matrix.part }} + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-root-${{ matrix.part }}/ + if-no-files-found: error + + # Both CLIs currently have only their root package. Keep the two real + # empty-role receipts without allocating separate empty runner jobs. + # Any future profile/rest packages still execute, never invent a pass. + - name: Execute complete profile owner + if: ${{ !cancelled() && matrix.part == 0 }} + run: bash scripts/test-linux-cli-module-ci.sh ${{ inputs.module }} ${{ inputs.flavor }} execute 1 -1 + - name: Retain actual profile evidence + if: ${{ always() && matrix.part == 0 }} + uses: actions/upload-artifact@v6 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-owner-1 + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-owner-1/ + if-no-files-found: error + - name: Execute complete remaining-package owner + if: ${{ !cancelled() && matrix.part == 0 }} + run: bash scripts/test-linux-cli-module-ci.sh ${{ inputs.module }} ${{ inputs.flavor }} execute 2 -1 + - name: Retain actual remaining-package evidence + if: ${{ always() && matrix.part == 0 }} + uses: actions/upload-artifact@v6 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-owner-2 + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-owner-2/ + if-no-files-found: error + + complete: + needs: root-parts + if: ${{ !cancelled() }} + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Setup Go + timeout-minutes: 10 + uses: ./.github/actions/setup-go + with: + go-version: ${{ inputs.go-version }} + - name: Install LLVM 22 and cross compilers + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross gcc-aarch64-linux-gnu + - name: Install required execution runtimes + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + - name: LLVM tools info + shell: bash + run: | + command -v llc + llc --version + command -v clang + clang --version + - name: Setup required wasm execution runtime + timeout-minutes: 10 + uses: ./.github/actions/setup-node22 + - name: Download root part 0 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-root-0 + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-owners/owner-0000/part-0000 + - name: Download root part 1 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-root-1 + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-owners/owner-0000/part-0001 + - name: Download root part 2 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-root-2 + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-owners/owner-0000/part-0002 + - name: Download root part 3 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-root-3 + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-owners/owner-0000/part-0003 + - name: Download profile owner + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-owner-1 + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-owners/owner-0001 + - name: Download remaining-package owner + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-owner-2 + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-owners/owner-0002 + - name: Verify complete CLI module + if: ${{ !cancelled() }} + run: bash scripts/test-linux-cli-module-ci.sh ${{ inputs.module }} ${{ inputs.flavor }} complete + - name: Retain independent completion evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: linux-cli-${{ inputs.module }}-${{ inputs.go-version }}-${{ inputs.flavor }}-completion + path: _out/linux-cli-${{ inputs.module }}-${{ inputs.flavor }}-verified/ + if-no-files-found: error + - name: Show complete CLI coverage + if: inputs.flavor == 'coverage' + run: go tool cover -func=coverage-${{ inputs.module }}.out | tail -n 1 + working-directory: cmd/${{ inputs.module }} + - name: Require successful CLI owners + if: ${{ !cancelled() }} + run: test '${{ needs.root-parts.result }}' = success diff --git a/.github/workflows/runtime-integration.yml b/.github/workflows/runtime-integration.yml new file mode 100644 index 000000000..8a5725fba --- /dev/null +++ b/.github/workflows/runtime-integration.yml @@ -0,0 +1,154 @@ +name: SDK/runtime integration + +on: + workflow_dispatch: + schedule: + - cron: '17 3 * * 1' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + ci_policy: + name: CI policy preflight + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install LLVM 22 + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh + + - name: Setup Go + timeout-minutes: 10 + uses: ./.github/actions/setup-go + with: + go-version: '1.27.2' + + - name: Setup verified policy runtime + uses: ./.github/actions/setup-node22 + timeout-minutes: 10 + + - name: Workflow scheduling regression tests + run: node --test .github/scripts/priority-*.test.cjs + + - name: CI policy tests + run: go test . -run '^TestCI' -count=1 + + benchmark: + needs: ci_policy + name: complete standard-library SDK/runtime compile (Go 1.27, LLVM 22) + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 35 + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install LLVM 22 + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh + + - name: Setup Go + timeout-minutes: 10 + uses: ./.github/actions/setup-go + with: + go-version: '1.27.2' + + - name: Translate and compile standard-library assembly + env: + PLAN9ASM_BENCHMARK_MAX_TARGET_SECONDS: '300' + PLAN9ASM_BENCHMARK_MAX_TOTAL_SECONDS: '900' + run: scripts/benchmark-compile.sh + + - name: Upload benchmark report + if: always() + uses: actions/upload-artifact@v6 + with: + name: compile-benchmark + path: _out/compile-benchmark + if-no-files-found: error + + stdlib-corpus: + needs: ci_policy + name: complete stdlib source/object corpus (${{ matrix.os }}, Go ${{ matrix.go-version }}) + runs-on: ${{ github.repository_owner == 'xgo-dev' && matrix.runner || matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # Linux is the authoritative cross-target corpus gate. Each + # unversioned Linux architecture and wasm target expands to every + # setting supported by that Go toolchain in check-stdlib-corpus.sh. + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.20.x', prebuild_tool: true, targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm'} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.21.x', prebuild_tool: true, targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.22.x', prebuild_tool: true, targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.23.x', prebuild_tool: true, targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.24.x', targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.25.x', targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.26.x', targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.27.2', targets: 'linux/386,linux/amd64,linux/arm,linux/arm64,darwin/amd64,darwin/arm64,windows/386,windows/amd64,windows/arm64,js/wasm,wasip1/wasm'} + # Native hosts are auxiliary integration checks on the latest Go. + - {os: macos-latest, go-version: '1.27.2', targets: 'darwin/amd64,darwin/arm64'} + - {os: windows-latest, go-version: '1.27.2', targets: 'windows/386,windows/amd64,windows/arm64'} + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install LLVM 22 (Ubuntu) + if: runner.os == 'Linux' + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh + + - name: Install LLVM 22 (macOS) + if: runner.os == 'macOS' + run: | + brew install llvm@22 + echo "PATH=$(brew --prefix llvm@22)/bin:$PATH" >> "$GITHUB_ENV" + + - name: Setup Python (Windows) + if: runner.os == 'Windows' + uses: actions/setup-python@v7 + with: + python-version: '3.x' + + - name: Install LLVM 22 (Windows) + if: runner.os == 'Windows' + uses: ./.github/actions/setup-windows-llvm22 + + - name: Setup Go for corpus tool build + if: matrix.prebuild_tool + timeout-minutes: 10 + uses: ./.github/actions/setup-go + with: + go-version: '1.27.2' + + - name: Build corpus tool for older Go lane + if: matrix.prebuild_tool + shell: bash + run: | + tool="$RUNNER_TEMP/plan9asm" + go build -C cmd/plan9asm -o "$tool" . + echo "PLAN9ASM_CMD=$tool" >> "$GITHUB_ENV" + + - name: Setup Go + timeout-minutes: 10 + uses: ./.github/actions/setup-go + with: + go-version: ${{ matrix.go-version }} + + - name: Stdlib asm corpus gate + shell: bash + env: + PLAN9ASM_CORPUS_TARGETS: ${{ matrix.targets }} + run: | + chmod +x scripts/check-stdlib-corpus.sh + scripts/check-stdlib-corpus.sh diff --git a/.github/workflows/windows-module.yml b/.github/workflows/windows-module.yml new file mode 100644 index 000000000..a1109694a --- /dev/null +++ b/.github/workflows/windows-module.yml @@ -0,0 +1,256 @@ +name: Complete Windows module + +on: + workflow_call: + inputs: + module: + required: true + type: string + +permissions: + contents: read + +env: + PLAN9ASM_AVX512_RUNTIME: auto + +jobs: + root-parts: + runs-on: windows-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + part: ${{ fromJSON(inputs.module == 'root' && '[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]' || '[0,1,2,3]') }} + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-go + with: + go-version: '1.27.2' + timeout-minutes: 10 + - uses: ./.github/actions/setup-windows-llvm22 + - uses: ./.github/actions/setup-node22 + timeout-minutes: 10 + - name: Execute disjoint root assignment + shell: bash + run: bash scripts/test-windows-module-ci.sh ${{ inputs.module }} execute 0 ${{ matrix.part }} + - name: Retain actual root evidence + if: ${{ always() }} + uses: actions/upload-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-${{ matrix.part }} + path: _out/windows-module-${{ inputs.module }}/ + if-no-files-found: error + + other-owners: + runs-on: windows-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + owner: [1, 2] + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-go + with: + go-version: '1.27.2' + timeout-minutes: 10 + - uses: ./.github/actions/setup-windows-llvm22 + - uses: ./.github/actions/setup-node22 + timeout-minutes: 10 + - name: Execute complete profile or remaining packages + shell: bash + run: bash scripts/test-windows-module-ci.sh ${{ inputs.module }} execute ${{ matrix.owner }} -1 + - name: Retain actual package evidence + if: ${{ always() }} + uses: actions/upload-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-owner-${{ matrix.owner }} + path: _out/windows-module-${{ inputs.module }}/ + if-no-files-found: error + + complete: + needs: [root-parts, other-owners] + if: ${{ !cancelled() }} + runs-on: windows-latest + timeout-minutes: 30 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/setup-go + with: + go-version: '1.27.2' + timeout-minutes: 10 + - uses: ./.github/actions/setup-windows-llvm22 + - uses: ./.github/actions/setup-node22 + timeout-minutes: 10 + - name: Download root part 0 + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-0 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0000 + - name: Download root part 1 + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-1 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0001 + - name: Download root part 2 + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-2 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0002 + - name: Download root part 3 + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-3 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0003 + - name: Download root part 4 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-4 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0004 + - name: Download root part 5 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-5 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0005 + - name: Download root part 6 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-6 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0006 + - name: Download root part 7 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-7 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0007 + - name: Download root part 8 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-8 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0008 + - name: Download root part 9 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-9 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0009 + - name: Download root part 10 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-10 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0010 + - name: Download root part 11 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-11 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0011 + - name: Download root part 12 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-12 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0012 + - name: Download root part 13 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-13 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0013 + - name: Download root part 14 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-14 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0014 + - name: Download root part 15 + if: ${{ inputs.module == 'root' }} + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-root-15 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0000/part-0015 + - name: Download profile owner + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-owner-1 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0001 + - name: Download remaining-package owner + uses: actions/download-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-owner-2 + path: _out/windows-module-${{ inputs.module }}-owners/owner-0002 + - name: Verify complete module and publish coverage + shell: bash + run: bash scripts/test-windows-module-ci.sh ${{ inputs.module }} complete + - name: Retain independent completion evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: windows-module-${{ inputs.module }}-completion + path: _out/windows-module-${{ inputs.module }}-verified/ + if-no-files-found: error + - name: Set up compatible Python for coverage upload + id: codecov_python + uses: actions/setup-python@v7 + with: + python-version: '3.13' + - name: Check Codecov Python and pip ownership + shell: bash + env: + CODECOV_PYTHON: ${{ steps.codecov_python.outputs.python-path }} + run: | + python - <<'PY' + import os + import shutil + import subprocess + import sys + if sys.version_info[:2] != (3, 13): + raise SystemExit("Codecov upload requires Python 3.13") + if not os.path.samefile(sys.executable, os.environ["CODECOV_PYTHON"]): + raise SystemExit("Codecov Python differs from setup-python output") + pip = shutil.which("pip") + expected = os.path.join(os.path.dirname(sys.executable), "Scripts", "pip.exe") + if not pip or not os.path.samefile(pip, expected): + raise SystemExit("Codecov pip is not owned by the selected Python") + print("Codecov upload Python:", sys.version, sys.executable, flush=True) + print("Codecov upload pip:", pip, flush=True) + subprocess.run([pip, "--version"], check=True) + PY + - name: Upload verified complete Windows module coverage + uses: codecov/codecov-action@v7 + with: + use_pypi: true + use_oidc: true + files: ./coverage-windows-${{ inputs.module }}.out + disable_search: true + flags: windows + name: windows-go1.27-${{ inputs.module }} + fail_ci_if_error: true + verbose: true + - name: Check installed Codecov CLI ownership + shell: bash + env: + CODECOV_PYTHON: ${{ steps.codecov_python.outputs.python-path }} + run: | + python - <<'PY' + import os + import shutil + import subprocess + import sys + if not os.path.samefile(sys.executable, os.environ["CODECOV_PYTHON"]): + raise SystemExit("Codecov Python changed during upload") + cli = shutil.which("codecovcli") + expected = os.path.join(os.path.dirname(sys.executable), "Scripts", "codecovcli.exe") + if not cli or not os.path.samefile(cli, expected): + raise SystemExit("Codecov CLI is not owned by the selected Python") + print("Codecov upload CLI:", cli, flush=True) + subprocess.run([cli, "--help"], check=True) + PY diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..bbe7e64e2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/_out/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..dfffd6c89 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,61 @@ +# plan9asm development + +plan9asm translates Go Plan 9 assembly to LLVM IR. This repository owns the +translator, shared Go source/tool observations, official instruction coverage +and independent regression tests. External scanning, ledgers and compatibility +batches live in [llgo-compat/plan9asm](https://github.com/llgo-compat/plan9asm). + +## Start here + +1. Inspect `git status --short`, `git worktree list` and `git remote -v`. + Preserve existing work and reuse a persistent idle worktree. +2. Read [architecture](docs/development/architecture.md), [current work](docs/development/current-work.md), + [instruction families](docs/development/instructions.md) and + [validation](docs/development/validation.md) for the task. + +## Rules + +- Push only to an allowed `cpunion` or `dastai` fork. Create/update the + contribution PR; do not directly push, merge, approve or close upstream PRs. +- Use current Go 1.27 and LLVM 22. Never fall back to another LLVM version; + missing required tools/backends fail, never skip. +- Support all five architectures: 386, amd64, arm, arm64 and wasm. + Host-inapplicable runtime tests require an explicit cross-runtime counterpart. + Never generate LLVM `blockaddress` for wasm. +- Use TDD. Consult complete official Go tables and implement typed instruction + families and operand formats. Each ecosystem-discovered defect must have an + independent regression and a rule-class fix, not a library-specific patch. +- Run tests with Go while llgo support is incomplete; do not add `!llgo` tags. +- Keep rejection, source context, translation, LLVM compilation and executed + semantics distinct. Infrastructure failures never establish source rejection. +- Use `apply_patch`, readable multi-line code and moderate blank lines, then + `gofmt`. Release LLVM objects and owned subprocess resources promptly. +- Do not commit private paths/hosts, package caches, binaries or reports. + Keep evidence under ignored `_out/`; preserve failed reports as failed. +- Freeze source and tools during validation, even documentation. Never edit a + running snapshot or reuse its report to certify a changed revision. +- Commit verified batches promptly. Stage repairs in the allowed fork; retain + draft status until current-head tests, CI, review and coverage pass. +- Before publishing test-helper or workflow changes, run the complete Node + policy suite and the Go `ci_policy` selector described in validation. + +## Commands + +After selecting the tools documented in validation: + +```sh +go test . -run '' -count=1 +go test ./... -count=1 -timeout=45m +(cd cmd/plan9asm && go test ./... -count=1) +(cd cmd/plan9asmll && go test ./... -count=1) +scripts/check-go-asm-coverage.sh +scripts/check-arm64-plan9-corpus.sh +scripts/check-stdlib-corpus.sh --instructions-only +``` + +Instruction checks retain complete WASM source/object compilation. Default +full-source corpus and strict benchmark belong to the independent +[SDK/runtime integration workflow](docs/development/validation.md#official-forms-and-benchmark). + +Do not reintroduce external discovery jobs or ledger gates into translator CI. +Keep this entry point short; durable procedures belong in linked documents. diff --git a/README.md b/README.md index 4eb8494f1..bcdd983c6 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,9 @@ Plan 9 assembly parser and LLVM IR translator, extracted as an independent modul releases are intentionally rejected instead of used as fallbacks. - `TranslateModule` builds an in-memory `llvm.Module` (`github.com/xgo-dev/llvm`). - `Translate` keeps compatibility and returns textual IR from that module. -- Root module dependency stays small (`goplus/llvm`). -- `golang.org/x/tools/go/packages` is used only in `cmd/plan9asmll` submodule. +- The root module uses `github.com/xgo-dev/llvm` for LLVM bindings. +- Both `cmd/plan9asm` and `cmd/plan9asmll` use `golang.org/x/tools/go/packages` + in their separate modules. - `cmd/plan9asm` does not depend on `llgo/internal/build` or `llgo/internal/packages`. ## Quick test @@ -120,6 +121,18 @@ go run -C cmd/plan9asmll . \ -report /tmp/plan9asmll-x86.json ``` +For an ordinary external module, `-feature-profile` explicitly consumes a +private `actual_go_cpu_profile_consumer_v1` invocation artifact. It re-observes +the actual Go marker/environment/tool identity, loads that same environment, +checks selected source hashes and package-role macros, and records CPP/LLVM +output hashes. Source directories and module replacements must match the +artifact's exact source root. Special and test-only package roles remain +outside this mode. Without the flag, existing CLI behavior is unchanged. +The external runner in [llgo-compat/plan9asm](https://github.com/llgo-compat/plan9asm) +owns module discovery, ledger records and third-party compatibility CI. It pins +this translator and the shared `gotoolprofile` package; translation/object +checks remain distinct from operand-form and runtime coverage. + ## Output behavior - Every asm file is printed with explicit status (`OK` or `FAIL`). diff --git a/amd64_abi.go b/amd64_abi.go new file mode 100644 index 000000000..c227c591d --- /dev/null +++ b/amd64_abi.go @@ -0,0 +1,90 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) abi0CallStackPtr(off int64) (string, error) { + sp, err := c.loadReg(SP) + if err != nil { + return "", fmt.Errorf("ABI0 call stack: %w", err) + } + addr := sp + if off != 0 { + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %d\n", next, sp, off) + addr = "%" + next + } + ptr := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", ptr, addr) + return "%" + ptr, nil +} + +func (c *amd64Ctx) abi0CallArgs(callee string, sig FuncSig) ([]string, error) { + slotsByArg := make([][]FrameSlot, len(sig.Args)) + for _, slot := range sig.Frame.Params { + if slot.Index < 0 || slot.Index >= len(sig.Args) { + return nil, fmt.Errorf("amd64 call %q: ABI0 parameter at +%d has invalid argument index %d", callee, slot.Offset, slot.Index) + } + slotsByArg[slot.Index] = append(slotsByArg[slot.Index], slot) + } + args := make([]string, 0, len(sig.Args)) + for argIndex, argType := range sig.Args { + slots := slotsByArg[argIndex] + if len(slots) == 0 { + return nil, fmt.Errorf("amd64 call %q: ABI0 argument %d has no frame slot", callee, argIndex) + } + if len(slots) == 1 && len(frameSlotFields(slots[0])) == 0 { + if slots[0].Type != argType { + return nil, fmt.Errorf("amd64 call %q: ABI0 argument %d frame type %s does not match %s", callee, argIndex, slots[0].Type, argType) + } + ptr, err := c.abi0CallStackPtr(slots[0].Offset) + if err != nil { + return nil, err + } + value := c.loadFPStorage(argType, ptr, ", align 1") + args = append(args, fmt.Sprintf("%s %s", argType, value)) + continue + } + aggregate := "undef" + for _, slot := range slots { + if len(frameSlotFields(slot)) == 0 { + return nil, fmt.Errorf("amd64 call %q: ABI0 aggregate argument %d has a scalar frame slot", callee, argIndex) + } + ptr, err := c.abi0CallStackPtr(slot.Offset) + if err != nil { + return nil, err + } + value := c.loadFPStorage(slot.Type, ptr, ", align 1") + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertvalue %s %s, %s %s%s\n", inserted, argType, aggregate, slot.Type, value, frameSlotExtractSuffix(slot)) + aggregate = "%" + inserted + } + args = append(args, fmt.Sprintf("%s %s", argType, aggregate)) + } + return args, nil +} + +func (c *amd64Ctx) storeABI0CallResult(callee string, sig FuncSig, result string) error { + if len(sig.Frame.Results) == 0 { + return fmt.Errorf("amd64 call %q: ABI0 result %s has no frame slot", callee, sig.Ret) + } + fields, aggregate := parseLiteralStructFields(sig.Ret) + for _, slot := range sig.Frame.Results { + value := result + if aggregate { + if slot.Index < 0 || slot.Index >= len(fields) || fields[slot.Index] != slot.Type { + return fmt.Errorf("amd64 call %q: ABI0 result slot %d does not match %s", callee, slot.Index, sig.Ret) + } + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", extracted, sig.Ret, result, slot.Index) + value = "%" + extracted + } else if slot.Index != 0 || slot.Type != sig.Ret { + return fmt.Errorf("amd64 call %q: ABI0 scalar result frame does not match %s", callee, sig.Ret) + } + ptr, err := c.abi0CallStackPtr(slot.Offset) + if err != nil { + return err + } + c.storeFPStorage(slot.Type, value, ptr, ", align 1", "") + } + return nil +} diff --git a/amd64_abi0_anchor_test.go b/amd64_abi0_anchor_test.go new file mode 100644 index 000000000..87668c8cd --- /dev/null +++ b/amd64_abi0_anchor_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func TestAMD64ABI0AnchorPreservesTailJumpRelocation(t *testing.T) { + args := make([]LLVMType, 15) + for i := range args { + args[i] = I32 + } + for _, anchorName := range []string{"gcasmABI0Keep", "relocationKeep"} { + source := fmt.Sprintf("TEXT ·%s(SB),NOSPLIT,$0-0\n", anchorName) + + "JMP ·Fn(SB)\nJMP ·FnTwo(SB)\nJMP ·FnThree(SB)\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{ + "example." + anchorName: {Name: "example." + anchorName, Ret: Void}, + "example.Fn": {Name: "example.Fn", Args: args, Ret: I32}, + "example.FnTwo": {Name: "example.FnTwo", Args: args, Ret: I32}, + "example.FnThree": {Name: "example.FnThree", Args: args, Ret: I32}, + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i386-apple-darwin"}, + {"386", "i686-pc-windows-msvc"}, + } { + triple := target.triple + t.Run(anchorName+"/"+triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: triple, + ResolveSym: testResolveSym("example"), Sigs: sigs, + }) + if err != nil { + t.Fatal(err) + } + for _, symbol := range []string{"example.Fn", "example.FnTwo", "example.FnThree"} { + if !strings.Contains(ir, "ptr @"+symbol) { + t.Fatalf("ABI0 anchor omitted direct symbol relocation for %s", symbol) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + objdump := findLLVM22Tool("llvm-objdump") + if objdump == "" { + t.Fatal("LLVM 22 llvm-objdump not found") + } + dir := t.TempDir() + llPath := filepath.Join(dir, "anchor.ll") + object := filepath.Join(dir, "anchor.o") + if err := os.WriteFile(llPath, []byte(ir), 0644); err != nil { + t.Fatal(err) + } + if output, err := exec.Command(llc, "-mtriple="+triple, "-filetype=obj", llPath, "-o", object).CombinedOutput(); err != nil { + t.Fatalf("LLVM 22 object compile: %v\n%s", err, output) + } + output, err := exec.Command(objdump, "-dr", object).CombinedOutput() + if err != nil { + t.Fatalf("LLVM 22 object dump: %v\n%s", err, output) + } + if !regexp.MustCompile(`(?m)^\s*0: e9 `).Match(output) { + t.Fatalf("anchor is not an entry-point tail jump with target relocation:\n%s", output) + } + for _, symbol := range []string{"example.Fn", "example.FnTwo", "example.FnThree"} { + if !strings.Contains(string(output), symbol) { + t.Fatalf("object omitted relocation for %s:\n%s", symbol, output) + } + } + }) + } + } +} + +func TestAMD64NonAnchorStillRejectsUnmappedTailArguments(t *testing.T) { + const source = "TEXT ·forward(SB),NOSPLIT,$0-0\nJMP ·Fn(SB)\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + args := make([]LLVMType, 7) + for i := range args { + args[i] = I32 + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: testResolveSym("example"), + Sigs: map[string]FuncSig{ + "example.forward": {Name: "example.forward", Ret: Void}, + "example.Fn": {Name: "example.Fn", Args: args, Ret: I32}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "missing arg reg") { + t.Fatalf("unmapped non-anchor tail arguments unexpectedly accepted: %v", err) + } +} diff --git a/amd64_abi0_forward.go b/amd64_abi0_forward.go new file mode 100644 index 000000000..2410abcfe --- /dev/null +++ b/amd64_abi0_forward.go @@ -0,0 +1,129 @@ +package plan9asm + +// inferX86ABI0Forwarder proves the complete straight-line adapter shape before +// assigning a typed ABI0 signature to an indirect CALL. The final incoming +// word is the code pointer, not an argument of the callback. This is structural +// inference: no library, function or parameter names participate. +func inferX86ABI0Forwarder(fn Func, sig FuncSig, goarch string) *FuncSig { + if goarch != "amd64" || len(sig.ArgRegs) != 0 || len(sig.Args) == 0 { + return nil + } + codeIndex := len(sig.Args) - 1 + if sig.Args[codeIndex] != I64 && sig.Args[codeIndex] != Ptr { + return nil + } + var code FrameSlot + var params []FrameSlot + codeSlots := 0 + for _, slot := range sig.Frame.Params { + if slot.Index == codeIndex { + code = slot + codeSlots++ + } else { + params = append(params, slot) + } + } + if codeSlots != 1 || len(frameSlotFields(code)) != 0 { + return nil + } + + instructions := x86ABI0ForwardInstructions(fn) + // Two instructions per copied slot, code-pointer load, CALL, then RET. + if len(instructions) != 2*len(params)+2*len(sig.Frame.Results)+3 { + return nil + } + for i, slot := range params { + load, store := instructions[2*i], instructions[2*i+1] + if !x86ABI0ForwardCopy(load, store, slot, true) || + store.Args[1].Mem.Off != slot.Offset || slot.Offset >= code.Offset { + return nil + } + } + callIndex := 2*len(params) + 1 + load, call := instructions[callIndex-1], instructions[callIndex] + if load.Op != "MOVQ" || len(load.Args) != 2 || load.Args[0].Kind != OpFP || + load.Args[0].FPOffset != code.Offset || load.Args[1].Kind != OpReg || + call.Op != "CALL" || len(call.Args) != 1 || call.Args[0].Kind != OpReg || + call.Args[0].Reg != load.Args[1].Reg { + return nil + } + results := append([]FrameSlot(nil), sig.Frame.Results...) + for i, slot := range results { + load, store := instructions[callIndex+1+2*i], instructions[callIndex+2+2*i] + if !x86ABI0ForwardCopy(load, store, slot, false) { + return nil + } + // Removing the trailing pointer shifts the callback's results one + // pointer word earlier. Do not infer a signature from arbitrary loads. + if load.Args[0].Mem.Off != slot.Offset-8 || slot.Offset-8 < code.Offset { + return nil + } + results[i].Offset -= 8 + } + last := instructions[len(instructions)-1] + if last.Op != "RET" || len(last.Args) != 0 { + return nil + } + return &FuncSig{ + Args: append([]LLVMType(nil), sig.Args[:codeIndex]...), Ret: sig.Ret, + Frame: FrameLayout{Params: params, Results: results}, + } +} + +func x86ABI0ForwardCopy(load, store Instr, slot FrameSlot, incoming bool) bool { + if len(load.Args) != 2 || len(store.Args) != 2 || + load.Args[1].Kind != OpReg || store.Args[0].Kind != OpReg || + load.Args[1].Reg != store.Args[0].Reg || load.Args[1].Reg == SP || load.Args[1].Reg == BP { + return false + } + width := x86ABI0ForwardMoveBytes(load, 0) + if width == 0 || width != x86ABI0ForwardMoveBytes(store, 1) || width != frameTypeSize(slot.Type, 8) { + return false + } + fp, sp := load.Args[0], store.Args[1] + if !incoming { + fp, sp = store.Args[1], load.Args[0] + } + return fp.Kind == OpFP && fp.FPOffset == slot.Offset && + sp.Kind == OpMem && sp.Mem.Base == SP && sp.Mem.Index == "" && sp.Mem.Sym == "" && + sp.Mem.OffRaw == "" && sp.Mem.IndexExt == "" && sp.Mem.Segment == "" && sp.Mem.Scale == 0 +} + +func x86ABI0ForwardInstructions(fn Func) []Instr { + var instructions []Instr + for _, ins := range fn.Instrs { + switch ins.Op { + case "TEXT", "PCDATA", "FUNCDATA", "GO_ARGS", "NO_LOCAL_POINTERS": + continue + } + instructions = append(instructions, ins) + } + return instructions +} + +// Widening MOVs retain their low source bytes. Only copying those exact bytes +// back is forwarding; arithmetic, masks and partial-width stores do not qualify. +func x86ABI0ForwardMoveBytes(ins Instr, index int) int64 { + if len(ins.Args) != 2 || !x86ResultNamedPredicateInstruction(ins) { + return 0 + } + if x86RawReturnScalarMoveBytes(x86SourceMoveInstruction(ins, "amd64").Op) == 0 { + if form, known, err := amd64ScalarVectorMoveProperties(ins, "amd64"); known { + if err != nil || form.mask >= 0 || form.memory != index || form.spec.rawOnly { + return 0 + } + } else if _, known, err := x86ScalarExtensionProperties(ins.Op, ins, "amd64"); index != 0 || !known || err != nil { + return 0 + } + } + // This temporary projection queries the shared width grammar only. It is + // never emitted and cannot create a logical FP or native storage role. + projected := ins + projected.Args = append([]Operand(nil), ins.Args...) + projected.Args[index] = Operand{Kind: OpFP} + width, write, known := X86FPAccessBytes(projected, index, "amd64") + if !known || write != (index == 1) { + return 0 + } + return width +} diff --git a/amd64_abi0_forward_test.go b/amd64_abi0_forward_test.go new file mode 100644 index 000000000..a1c4b1781 --- /dev/null +++ b/amd64_abi0_forward_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" + "testing" +) + +func TestX86ABI0ForwardCopyScalarWidths(t *testing.T) { + for _, test := range []struct { + op, register string + typ LLVMType + }{ + {"MOVB", "AX", I8}, + {"MOVW", "AX", I16}, + {"MOVL", "AX", I32}, + {"MOVQ", "AX", I64}, + {"MOVSS", "X0", "float"}, + {"MOVSD", "X0", "double"}, + } { + t.Run(test.op, func(t *testing.T) { + source := fmt.Sprintf(`TEXT copy(SB),$16-16 + %s arg+8(FP), %s + %s %s, 8(SP) + %s 8(SP), %s + %s %s, ret+8(FP) + RET +`, test.op, test.register, test.op, test.register, + test.op, test.register, test.op, test.register) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + var instructions []Instr + for _, ins := range file.Funcs[0].Instrs { + if ins.Op != "TEXT" { + instructions = append(instructions, ins) + } + } + slot := FrameSlot{Offset: 8, Type: test.typ} + if !x86ABI0ForwardCopy(instructions[0], instructions[1], slot, true) || + !x86ABI0ForwardCopy(instructions[2], instructions[3], slot, false) { + t.Fatal("rejected equal-width scalar frame copies") + } + slot.Type = I64 + if test.typ == I64 || test.typ == "double" { + slot.Type = I32 + } + if x86ABI0ForwardCopy(instructions[0], instructions[1], slot, true) { + t.Fatal("accepted a copy with the wrong frame-slot width") + } + }) + } +} + +func TestX86ABI0ForwarderRequiresCompleteTypedShape(t *testing.T) { + tc := issue44Cases[3] + pkg := mustGoPackage(t, "test/forward", "package forward\n"+tc.declarations) + fn := pkg.Types.Scope().Lookup("Y").(*types.Func) + sig, err := goFuncSigForDeclaredFunc("Y", fn, "amd64", nil, nil, true) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name, old, replacement string + valid bool + }{ + {name: "exact", valid: true}, + {name: "renamed_code_parameter", old: "code+", replacement: "function_pointer+", valid: true}, + {name: "wrong_argument_offset", old: "AX, 8(SP)", replacement: "AX, 0(SP)"}, + {name: "wrong_argument_width", old: "MOVQ b+8(FP)", replacement: "MOVL b+8(FP)"}, + {name: "missing_argument", old: "MOVQ AX, 8(SP)\n", replacement: ""}, + {name: "wrong_code_slot", old: "code+16(FP)", replacement: "code+8(FP)"}, + {name: "wrong_code_register", old: "CALL *AX", replacement: "CALL *BX"}, + {name: "wrong_result_offset", old: "16(SP), AX", replacement: "8(SP), AX"}, + {name: "wrong_result_destination", old: "ret+24(FP)", replacement: "ret+16(FP)"}, + {name: "mutation", old: "CALL *AX", replacement: "INCQ 0(SP)\nCALL *AX"}, + {name: "second_call", old: "CALL *AX", replacement: "CALL *AX\nCALL *AX"}, + {name: "branch", old: "CALL *AX", replacement: "JMP end\nCALL *AX\nend:"}, + {name: "tail_jump", old: "RET", replacement: "JMP *AX"}, + } { + t.Run(test.name, func(t *testing.T) { + source := tc.assembly + if test.old != "" { + source = strings.Replace(source, test.old, test.replacement, 1) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + got := inferX86ABI0Forwarder(file.Funcs[0], sig, "amd64") + if (got != nil) != test.valid { + t.Fatalf("inferred %+v, want valid=%v", got, test.valid) + } + if got != nil && (len(got.Args) != 2 || got.Ret != I64 || got.Frame.Results[0].Offset != 16) { + t.Fatalf("wrong callback signature: %+v", got) + } + }) + } +} + +func TestGoABI0NestedAggregateFrameLayout(t *testing.T) { + pkg := mustGoPackage(t, "test/layout", `package layout +type Mixed struct { Byte uint8; Pair [2]uint64; Z complex128 } +func Y(a Mixed, b complex64) (r Mixed) +`) + fn := pkg.Types.Scope().Lookup("Y").(*types.Func) + for _, arch := range []string{"386", "amd64", "arm", "arm64", "wasm"} { + t.Run(arch, func(t *testing.T) { + sig, err := goFuncSigForDeclaredFunc("Y", fn, arch, nil, nil, true) + if err != nil { + t.Fatal(err) + } + if sig.Args[0] != "{ i8, [2 x i64], { double, double } }" || len(sig.Frame.Params) != 7 || len(sig.Frame.Results) != 5 { + t.Fatalf("incorrect aggregate layout: %+v", sig) + } + word := int64(goWordSize(arch)) + wantOffsets := []int64{0, word, word + 8, word + 16, word + 24, word + 32, word + 36} + wantPaths := []string{", 0", ", 1, 0", ", 1, 1", ", 2, 0", ", 2, 1", ", 0", ", 1"} + for i, slot := range sig.Frame.Params { + if slot.Offset != wantOffsets[i] || frameSlotExtractSuffix(slot) != wantPaths[i] { + t.Fatalf("slot %d = %+v, want offset=%d path=%s", i, slot, wantOffsets[i], wantPaths[i]) + } + } + }) + } +} diff --git a/amd64_abi0_outgoing_candidate.go b/amd64_abi0_outgoing_candidate.go new file mode 100644 index 000000000..7f59bdf74 --- /dev/null +++ b/amd64_abi0_outgoing_candidate.go @@ -0,0 +1,143 @@ +package plan9asm + +import "strings" + +// A complete typed forwarding adapter uses the existing ABI0 call allocation. +// This only chooses that existing route. It grants no initialized bytes, FLAGS, +// native backing, caller extent or source authority to its later translation. +func x86ABI0OutgoingForwarder(fn Func, options Options) bool { + if options.Goarch != "amd64" || options.ResolveSym == nil || + fn.FrameSize <= 0 || fn.FrameSize > 4096 || fn.ArgSize < 0 || fn.ArgSize > 4096 || + len(fn.Instrs) > 8192 || len(fn.X86RawText) != 0 || strings.Contains(fn.Sym, "<") { + return false + } + resolvedCaller := options.ResolveSym(fn.Sym) + caller, found := options.Sigs[resolvedCaller] + if !found || caller.Name != "" && caller.Name != resolvedCaller || !x86ABI0ForwardLayout(caller, fn.ArgSize) { + return false + } + sourceFrames := [2]map[int64]FrameSlot{make(map[int64]FrameSlot), make(map[int64]FrameSlot)} + for group, slots := range [][]FrameSlot{caller.Frame.Params, caller.Frame.Results} { + for _, slot := range slots { + sourceFrames[group][slot.Offset] = slot + } + } + instructions := x86ABI0ForwardInstructions(fn) + if len(instructions) == 0 || instructions[len(instructions)-1].Op != OpRET || + len(instructions[len(instructions)-1].Args) != 0 { + return false + } + start, calls := 0, 0 + for start < len(instructions)-1 { + at := start + for at < len(instructions) && instructions[at].Op != OpCALL { + at++ + } + if at == len(instructions) || len(instructions[at].Args) != 1 || instructions[at].Args[0].Kind != OpSym { + return false + } + call := instructions[at] + name, offset, valid := parseSBRef(call.Args[0].Sym) + if !valid || offset != 0 || strings.Contains(name, "<") || !x86ResultNamedPredicateInstruction(call) { + return false + } + resolved := options.ResolveSym(name) + callee, declared := options.Sigs[resolved] + if !declared || callee.Name != "" && callee.Name != resolved || !x86ABI0ForwardLayout(callee, fn.FrameSize) || + at-start != 2*len(callee.Frame.Params) || at+1+2*len(callee.Frame.Results) >= len(instructions) || + len(caller.Frame.Results) != len(callee.Frame.Results) { + return false + } + writtenResults := make(map[int64]bool, len(caller.Frame.Results)) + for group, slots := range [][]FrameSlot{callee.Frame.Params, callee.Frame.Results} { + incoming, begin := group == 0, start + if !incoming { + begin = at + 1 + } + for index, slot := range slots { + load, store := instructions[begin+2*index], instructions[begin+2*index+1] + if len(load.Args) != 2 || len(store.Args) != 2 { + return false + } + fp, sp, memoryIndex := load.Args[0], store, 1 + if !incoming { + fp, sp, memoryIndex = store.Args[1], load, 0 + } + memory, err := fixedLocalMemoryAddress(sp, memoryIndex, fn.FrameSize) + if err != nil || memory != sp.Args[memoryIndex].Mem || memory.Off != slot.Offset { + return false + } + source, matched := sourceFrames[group][fp.FPOffset] + if !matched || fp.Kind != OpFP || slot.Type != source.Type || !x86ABI0ForwardCopy(load, store, source, incoming) { + return false + } + if !incoming { + if writtenResults[source.Offset] { + return false + } + writtenResults[source.Offset] = true + } + } + } + start = at + 1 + 2*len(callee.Frame.Results) + calls++ + } + return calls != 0 && start == len(instructions)-1 +} + +func x86ABI0ForwardLayout(sig FuncSig, limit int64) bool { + return x86ABI0FrameLayout(sig, limit, "amd64") +} + +func x86ABI0FrameLayout(sig FuncSig, limit int64, arch string) bool { + if len(sig.ArgRegs) != 0 || sig.ARMEntry != nil || sig.WASMContext != "" || + sig.ARM64GoRegisterABI != nil || sig.ARM64ClosureABI != nil || + len(sig.Frame.Params)+len(sig.Frame.Results) > 4096 { + return false + } + results := []LLVMType{sig.Ret} + if sig.Ret == Void { + results = nil + } else if fields, aggregate := parseLiteralStructFields(sig.Ret); aggregate { + results = fields + } + if len(results) != len(sig.Frame.Results) { + return false + } + end := int64(0) + roles := make(map[string]bool) + for group, slots := range [][]FrameSlot{sig.Frame.Params, sig.Frame.Results} { + for index, slot := range slots { + width := x86FrameScalarBytes(slot.Type, arch) + if width == 0 || slot.Offset < end || slot.Offset > limit-width { + return false + } + if group == 0 { + role, valid := x86ResultParameterRole(sig.Args, slot) + if !valid || roles[role] { + return false + } + roles[role] = true + } else if slot.Index != index || !x86RawReturnSlotMatchesSignature(slot, 1, sig) { + return false + } + end = slot.Offset + width + } + } + remaining := len(sig.Frame.Params) + for _, argument := range sig.Args { + // The existing role check proves every supplied leaf is unique. Count + // all actual leaves as well, so missing aggregate fields cannot pass. + err := goLLVMScalarLeaves(argument, nil, uint64(len(sig.Frame.Params)), func(LLVMType, []int) error { + if remaining == 0 { + return ErrProbeNeedsContext + } + remaining-- + return nil + }) + if err != nil { + return false + } + } + return remaining == 0 +} diff --git a/amd64_abi0_outgoing_candidate_test.go b/amd64_abi0_outgoing_candidate_test.go new file mode 100644 index 000000000..6e5089c91 --- /dev/null +++ b/amd64_abi0_outgoing_candidate_test.go @@ -0,0 +1,300 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "go/types" + "strings" + "testing" +) + +func requireX86OutgoingCandidateGoSource(t *testing.T, source string) { + t.Helper() + requireX86GoAssemblerResult(t, "amd64", source, true) + t.Logf("original Go assembler accepted source sha256=%x", sha256.Sum256([]byte(source))) +} + +func x86OutgoingCandidateFixture(t *testing.T, parameters, load string, calls []string) (*File, Options, string) { + t.Helper() + declarations := "package outgoing\nfunc Caller(" + parameters + ") float64\n" + + "func First(a uint8) float64\nfunc Callee(" + parameters + ") float64\n" + pkg := mustGoPackage(t, "outgoing", declarations) + options := Options{Goarch: "amd64", Sigs: make(map[string]FuncSig), + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "outgoing.", 1) }} + for _, name := range []string{"Caller", "First", "Callee"} { + declaration := pkg.Types.Scope().Lookup(name).(*types.Func) + sig, err := DeriveGoFuncABI0Signature("outgoing."+name, declaration, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + options.Sigs[sig.Name] = sig + } + caller := options.Sigs["outgoing.Caller"] + var source strings.Builder + fmt.Fprintf(&source, "TEXT ·Caller(SB),4,$64-%d\n", caller.Frame.Results[0].Offset+8) + for _, name := range calls { + callee := options.Sigs["outgoing."+name] + for index, slot := range callee.Frame.Params { + input := caller.Frame.Params[index] + move, register := "MOVQ", "AX" + switch slot.Type { + case I8: + move = "MOVB" + case I16: + move = "MOVW" + case I32: + move = "MOVL" + case "float": + move, register = "MOVSS", "X0" + case "double": + move, register = "MOVSD", "X0" + } + read := move + if index == 0 && load != "" { + read = load + } + inputName := input.Name + if inputName == "" { + inputName = fmt.Sprintf("arg%d", index) + } + fmt.Fprintf(&source, "%s %s+%d(FP),%s\n%s %s,%d(SP)\n", + read, inputName, input.Offset, register, move, register, slot.Offset) + } + fmt.Fprintf(&source, "CALL ·%s(SB)\nMOVSD %d(SP),X1\nMOVSD X1,ret+%d(FP)\n", + name, callee.Frame.Results[0].Offset, caller.Frame.Results[0].Offset) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + return file, options, source.String() +} + +func TestGoAMD64FixedLocalOutgoingABI0ForwardCandidates(t *testing.T) { + for _, parameters := range []string{ + "a uint8", "a uint16", "a uint32", "a uint64", "a float32", "a float64", + "a uint8,b uint64", "a struct { X uint8; Y uint64 }", + "a []byte", "a [3]uint64", "a struct { X [2]uint16; Y struct { Z uint64 } }", + } { + for _, rounds := range []int{1, 2} { + t.Run(fmt.Sprintf("%s/%d", parameters, rounds), func(t *testing.T) { + calls := []string{"Callee"} + if rounds == 2 { + calls = append(calls, "Callee") + } + file, options, source := x86OutgoingCandidateFixture(t, parameters, "", calls) + requireX86OutgoingCandidateGoSource(t, source) + t.Logf("original untyped source candidate=%t", GoAMD64FixedLocalFrameBindingNeeded(file, "amd64")) + if GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", options) { + t.Fatal("complete typed outgoing frame was routed to the unrelated native-frame factory") + } + }) + } + } + t.Run("different callee layouts", func(t *testing.T) { + file, options, source := x86OutgoingCandidateFixture(t, "a uint8,b uint64", "MOVBLZX", []string{"First", "Callee"}) + requireX86OutgoingCandidateGoSource(t, source) + if GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", options) { + t.Fatal("independent typed call rounds lost their actual parameter/result offsets") + } + }) + t.Run("multiple callers", func(t *testing.T) { + _, options, source := x86OutgoingCandidateFixture(t, "a uint64", "", []string{"Callee"}) + sig := options.Sigs["outgoing.Caller"] + sig.Name = "outgoing.Other" + options.Sigs[sig.Name] = sig + second := strings.Replace(source, "·Caller", "·Other", 1) + complete := source + second + requireX86OutgoingCandidateGoSource(t, complete) + file, err := Parse(ArchAMD64, complete) + if err != nil { + t.Fatal(err) + } + if GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", options) { + t.Fatal("a second complete typed caller changed the source role") + } + file, err = Parse(ArchAMD64, source+strings.Replace(second, "RET\n", "MOVQ AX,48(SP)\nRET\n", 1)) + if err != nil { + t.Fatal(err) + } + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", options) { + t.Fatal("one valid caller hid another function's private access") + } + }) +} + +func TestGoAMD64FixedLocalOutgoingABI0ForwardGuards(t *testing.T) { + for _, test := range []struct { + name, old, replacement string + }{ + {"padding store", "CALL ·Callee(SB)", "MOVB AX,1(SP)\nCALL ·Callee(SB)"}, + {"private store", "CALL ·Callee(SB)", "MOVQ AX,48(SP)\nCALL ·Callee(SB)"}, + {"extra vector access", "CALL ·Callee(SB)", "PADDL 32(SP),X0\nCALL ·Callee(SB)"}, + {"overlapping source slots", "MOVQ AX,8(SP)", "MOVQ AX,0(SP)"}, + {"wrong width", "MOVQ AX,8(SP)", "MOVL AX,8(SP)"}, + {"wrong register", "MOVQ AX,8(SP)", "MOVQ BX,8(SP)"}, + {"missing input", "MOVQ AX,8(SP)\n", ""}, + {"old result", "MOVSD 16(SP),X1", "MOVSD 8(SP),X1"}, + {"SP escape", "CALL ·Callee(SB)", "MOVQ SP,R11\nCALL ·Callee(SB)"}, + {"SP write", "CALL ·Callee(SB)", "ADDQ $16,SP\nCALL ·Callee(SB)"}, + {"BP alias", "MOVQ AX,8(SP)", "MOVQ AX,-56(BP)"}, + {"indexed access", "MOVQ AX,8(SP)", "MOVQ AX,8(SP)(CX*1)"}, + {"flags across call", "CALL ·Callee(SB)", "CMPQ AX,AX\nCALL ·Callee(SB)\nSETCS CL"}, + {"branch", "CALL ·Callee(SB)", "JMP next\nnext:\nCALL ·Callee(SB)"}, + {"indirect", "CALL ·Callee(SB)", "CALL AX"}, + {"internal ABI", "CALL ·Callee(SB)", "CALL ·Callee(SB)"}, + {"missing callee", "CALL ·Callee(SB)", "CALL ·Missing(SB)"}, + {"frame overflow", "$64-24", "$16-24"}, + } { + t.Run(test.name, func(t *testing.T) { + _, options, source := x86OutgoingCandidateFixture(t, "a uint8,b uint64", "MOVBLZX", []string{"First", "Callee"}) + if !strings.Contains(source, test.old) { + t.Fatalf("negative control did not mutate its source: %q", test.old) + } + source = strings.Replace(source, test.old, test.replacement, 1) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") || !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", options) { + t.Fatal("unclosed frame or call silently bypassed its original candidate") + } + }) + } + for _, change := range []string{"zero options", "duplicate options", "custom caller", "custom callee", "overlapping callee", "missing slot"} { + t.Run(change, func(t *testing.T) { + file, options, _ := x86OutgoingCandidateFixture(t, "a uint8,b uint64", "MOVBLZX", []string{"Callee"}) + provided := []Options{options} + switch change { + case "zero options": + provided = []Options{{}} + case "duplicate options": + provided = append(provided, options) + case "custom caller", "custom callee": + name := "outgoing.Callee" + if change == "custom caller" { + name = "outgoing.Caller" + } + sig := options.Sigs[name] + sig.ArgRegs = []Reg{AX, BX} + options.Sigs[name] = sig + case "overlapping callee": + sig := options.Sigs["outgoing.Callee"] + sig.Frame.Params[1].Offset = 0 + options.Sigs[sig.Name] = sig + case "missing slot": + sig := options.Sigs["outgoing.Callee"] + sig.Frame.Params = sig.Frame.Params[:1] + options.Sigs[sig.Name] = sig + } + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", provided...) { + t.Fatal("ambiguous or nonordinary declarations disabled the native-frame candidate") + } + }) + } +} + +func TestGoAMD64FixedLocalOutgoingExtensionFamily(t *testing.T) { + for _, opcode := range []Op{ + "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", "MOVBQSX", "MOVBQZX", + "MOVWLSX", "MOVWLZX", "MOVWQSX", "MOVWQZX", "MOVLQSX", "MOVLQZX", "MOVSWW", "MOVZWW", + } { + t.Run(string(opcode), func(t *testing.T) { + spec := amd64ScalarExtensionMoveSpecs[opcode] + parameters := fmt.Sprintf("a uint%d", spec.sourceBits) + file, options, source := x86OutgoingCandidateFixture(t, parameters, string(opcode), []string{"Callee"}) + requireX86OutgoingCandidateGoSource(t, source) + if GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", options) { + t.Fatal("shared widening move did not preserve its exact outgoing low bytes") + } + }) + } +} + +func TestGoAMD64FixedLocalOutgoingCompleteSlotGuards(t *testing.T) { + t.Run("missing aggregate field", func(t *testing.T) { + _, options, source := x86OutgoingCandidateFixture(t, "a struct { X uint8; Y uint64 }", "", []string{"Callee"}) + for _, name := range []string{"Caller", "Callee"} { + sig := options.Sigs["outgoing."+name] + sig.Frame.Params = sig.Frame.Params[:1] + options.Sigs[sig.Name] = sig + } + const missing = "MOVQ arg1+8(FP),AX\nMOVQ AX,8(SP)\n" + if !strings.Contains(source, missing) { + t.Fatal("aggregate negative did not remove its second field") + } + source = strings.Replace(source, missing, "", 1) + requireX86OutgoingCandidateGoSource(t, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", options) { + t.Fatal("one aggregate leaf was mistaken for its complete argument") + } + }) + t.Run("duplicate result destination", func(t *testing.T) { + options, source := x86OutgoingMultipleResultsFixture(t, 8, 8) + requireX86OutgoingCandidateGoSource(t, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", options) { + t.Fatal("two stores to one result were mistaken for complete result forwarding") + } + }) +} + +func x86OutgoingMultipleResultsFixture(t *testing.T, firstOffset, secondOffset int) (Options, string) { + t.Helper() + _, options, source := x86OutgoingCandidateFixture(t, "a uint64", "", []string{"Callee"}) + for _, name := range []string{"Caller", "Callee"} { + sig := options.Sigs["outgoing."+name] + sig.Ret = "{ double, double }" + second := sig.Frame.Results[0] + second.Index, second.Offset = 1, 16 + sig.Frame.Results = append(sig.Frame.Results, second) + options.Sigs[sig.Name] = sig + } + source = strings.Replace(source, "$64-16", "$64-24", 1) + source = strings.Replace(source, "ret+8(FP)", fmt.Sprintf("ret+%d(FP)", firstOffset), 1) + source = strings.Replace(source, "RET\n", fmt.Sprintf("MOVSD 16(SP),X1\nMOVSD X1,ret+%d(FP)\nRET\n", secondOffset), 1) + return options, source +} + +func TestGoAMD64FixedLocalOutgoingMultipleResults(t *testing.T) { + for _, offsets := range [][2]int{{8, 16}, {16, 8}} { + t.Run(fmt.Sprint(offsets), func(t *testing.T) { + options, source := x86OutgoingMultipleResultsFixture(t, offsets[0], offsets[1]) + requireX86OutgoingCandidateGoSource(t, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if GoAMD64FixedLocalFrameBindingNeeded(file, "amd64", options) { + t.Fatal("complete distinct result destinations lost their ordinary call route") + } + }) + } +} + +func TestGoAMD64FixedLocalOutgoingLayoutBoundaries(t *testing.T) { + for _, typ := range []LLVMType{"{}", "[0 x i64]", "%unknown"} { + t.Run(string(typ), func(t *testing.T) { + if x86ABI0ForwardLayout(FuncSig{Args: []LLVMType{typ}, Ret: Void}, 8) { + t.Fatal("unsupported empty or named type acquired a complete frame role") + } + }) + } + for _, typ := range []LLVMType{"[2 x i64]", "[18446744073709551615 x i64]"} { + t.Run(string(typ), func(t *testing.T) { + sig := FuncSig{Args: []LLVMType{typ}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Type: I64, Index: 0, Field: 0}}}} + if x86ABI0ForwardLayout(sig, 8) { + t.Fatal("one array element was mistaken for the complete bounded inventory") + } + }) + } +} diff --git a/amd64_accumulator_sign_extend_forms.go b/amd64_accumulator_sign_extend_forms.go new file mode 100644 index 000000000..9efd606fc --- /dev/null +++ b/amd64_accumulator_sign_extend_forms.go @@ -0,0 +1,53 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86AccumulatorSignExtendSpec struct { + sourceBytes, destinationBytes int64 + destination Reg +} + +// Complete x86 ynone / 98,99 family. The prefix selects the operand width; the +// destination distinguishes sign extension from the sign-only high half. +var x86AccumulatorSignExtendSpecs = map[Op]x86AccumulatorSignExtendSpec{ + "CBW": {1, 2, AX}, + "CWDE": {2, 4, AX}, + "CDQE": {4, 8, AX}, + "CWD": {2, 2, DX}, + "CDQ": {4, 4, DX}, + "CQO": {8, 8, DX}, +} + +func x86AccumulatorSignExtendProperties(op Op, ins Instr, arch string) (x86AccumulatorSignExtendSpec, bool, error) { + name := strings.ToUpper(string(op)) + base, _, dotted := strings.Cut(name, ".") + spec, known := x86AccumulatorSignExtendSpecs[Op(base)] + if !known { + return spec, false, nil + } + if arch == "" { + arch = "amd64" + } + if arch != "amd64" && arch != "386" || arch == "386" && spec.destinationBytes == 8 || + dotted || len(ins.Args) != 0 || ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return spec, true, fmt.Errorf("%s %s requires its original zero-operand ynone row: %q", arch, op, ins.Raw) + } + return spec, true, nil +} + +func x86TypedAccumulatorSignExtendByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + if !x86ResultNamedPredicateInstruction(ins) { + return x86TypedByteEffect{}, false, nil + } + spec, known, err := x86AccumulatorSignExtendProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + return x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: -1, operand: Operand{Kind: OpReg, Reg: AX}, width: spec.sourceBytes, read: true}, + {index: -1, operand: Operand{Kind: OpReg, Reg: spec.destination}, width: spec.destinationBytes, write: true}, + }}, true, nil +} diff --git a/amd64_address_memory_class_test.go b/amd64_address_memory_class_test.go new file mode 100644 index 000000000..8d020485b --- /dev/null +++ b/amd64_address_memory_class_test.go @@ -0,0 +1,373 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86MemoryClassKeepsAddressImmediatesDistinct(t *testing.T) { + for _, source := range []string{ + "$·cell(SB)", "$·cell+8(SB)", "$16(AX)", "$16(AX)(CX*4)", + } { + t.Run(source, func(t *testing.T) { + operand, err := parseOperandForArch(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if operand.Kind != OpSym || !strings.HasPrefix(operand.Sym, "$") { + t.Fatalf("source address identity lost: %#v", operand) + } + if isAMD64MemoryOperand(operand) { + t.Fatalf("Go TYPE_ADDR became a memory operand: %#v", operand) + } + // A caller-built operand must obey the same lexical boundary. + operand.Sym = " \t" + operand.Sym + if isAMD64MemoryOperand(operand) { + t.Fatal("whitespace granted address-immediate memory permission") + } + }) + } + + for _, source := range []string{ + "·cell(SB)", "·cell+8(SB)", "(AX)", "16(AX)(CX*4)", + "arg+0(FP)", "local-8(SP)", "(FS)", "(GS)", + } { + t.Run("memory/"+source, func(t *testing.T) { + operand, err := parseOperandForArch(ArchAMD64, source) + if err != nil || !isAMD64MemoryOperand(operand) { + t.Fatalf("ordinary memory class changed: %#v: %v", operand, err) + } + }) + } +} + +func TestX86AddressImmediateMemoryOnlyFormsRejectAcrossAPIs(t *testing.T) { + // Independent witnesses from Go's Ym/Ymb/Yml, yxchg, SSE memory, + // scalar-memory and non-temporal rows. The source contains a TYPE_ADDR + // where a real memory operand is required; relocation is not a load/store. + rows := []string{ + "XADDB AL,%s", "XADDW CX,%s", "XADDL CX,%s", + "XCHGB AL,%s", "XCHGW CX,%s", "XCHGL CX,%s", + "INCB %s", "INCW %s", "INCL %s", + "DECB %s", "DECW %s", "DECL %s", + "NEGB %s", "NEGW %s", "NEGL %s", + "NOTB %s", "NOTW %s", "NOTL %s", + "BTCL $1,%s", "BTRL $1,%s", "BTSL $1,%s", + "STMXCSR %s", "LDMXCSR %s", "CLFLUSH %s", + "MOVNTIL CX,%s", "MOVNTO X0,%s", "MOVNTPS X0,%s", + "ADDSUBPS %s,X0", "ADDSUBPD %s,X0", + "AESENC %s,X0", "PSHUFB %s,X0", "LDDQU %s,X0", + } + for _, arch := range []string{"386", "amd64"} { + for _, row := range rows { + for _, address := range []string{"$·cell(SB)", "$16(AX)"} { + instruction := fmt.Sprintf(row, address) + t.Run(arch+"/"+instruction, func(t *testing.T) { + memorySource := "TEXT ·Value(SB),4,$0-0\n" + + fmt.Sprintf(row, strings.TrimPrefix(address, "$")) + "\nRET\n" + goAssemblerOracleOutput(t, arch, memorySource, "", true) + source := "TEXT ·Value(SB),4,$0-0\n" + instruction + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + for _, target := range x86AddressMemoryTargets(arch) { + for _, api := range []string{"text", "module", "go"} { + t.Run(target+"/"+api, func(t *testing.T) { + _, err := x86AddressMemoryAPI(t, api, source, arch, target) + if err == nil { + t.Fatal("Go-rejected address immediate acquired memory semantics") + } + }) + } + } + }) + } + } + } +} + +func TestX86AddressClassActualGoRulesExcludeMemory(t *testing.T) { + // Read the actual selected assembler's complete classification/coverage + // relation. This oracle does not use the translator's operand predicate. + file, err := parser.ParseFile(token.NewFileSet(), filepath.Join(runtime.GOROOT(), + "src/cmd/internal/obj/x86/asm6.go"), nil, 0) + if err != nil { + t.Fatal(err) + } + classes := make(map[string]bool) + foundAddress, foundConstant := false, false + edges := make(map[string][]string) + ast.Inspect(file, func(node ast.Node) bool { + switch node := node.(type) { + case *ast.FuncDecl: + if node.Name.Name != "oclass" { + return true + } + ast.Inspect(node.Body, func(child ast.Node) bool { + clause, ok := child.(*ast.CaseClause) + if !ok { + return true + } + for _, value := range clause.List { + name, ok := value.(*ast.SelectorExpr) + if !ok || name.Sel.Name != "TYPE_ADDR" && name.Sel.Name != "TYPE_CONST" { + continue + } + foundAddress = foundAddress || name.Sel.Name == "TYPE_ADDR" + foundConstant = foundConstant || name.Sel.Name == "TYPE_CONST" + for _, statement := range clause.Body { + ast.Inspect(statement, func(inside ast.Node) bool { + if result, ok := inside.(*ast.ReturnStmt); ok { + if len(result.Results) != 1 { + t.Fatal("unexpected Go operand classification return") + } + ident, ok := result.Results[0].(*ast.Ident) + if !ok || !strings.HasPrefix(ident.Name, "Y") { + t.Fatal("unregistered Go address/constant classifier") + } + classes[ident.Name] = true + } + return true + }) + } + } + return true + }) + return false + case *ast.AssignStmt: + for _, left := range node.Lhs { + index, ok := left.(*ast.IndexExpr) + if !ok { + continue + } + name, ok := index.X.(*ast.Ident) + if !ok || name.Name != "ycover" { + continue + } + sum, ok := index.Index.(*ast.BinaryExpr) + if !ok || sum.Op != token.ADD { + t.Fatal("unregistered Go class coverage index") + } + product, ok := sum.X.(*ast.BinaryExpr) + if !ok || product.Op != token.MUL { + t.Fatal("unregistered Go class coverage product") + } + from, fromOK := product.X.(*ast.Ident) + size, sizeOK := product.Y.(*ast.Ident) + to, toOK := sum.Y.(*ast.Ident) + if !fromOK || !sizeOK || !toOK || size.Name != "Ymax" { + t.Fatal("unregistered Go class coverage operands") + } + if from.Name == "i" && to.Name == "i" { + continue // The actual diagonal only relates a class to itself. + } + if !strings.HasPrefix(from.Name, "Y") || !strings.HasPrefix(to.Name, "Y") { + t.Fatal("unregistered computed Go class relation") + } + edges[from.Name] = append(edges[from.Name], to.Name) + } + } + return true + }) + if !foundAddress || !foundConstant || !classes["Yiauto"] || !classes["Yi32"] || len(edges) == 0 { + t.Fatal("incomplete actual Go TYPE_ADDR/constant/coverage inventory") + } + for changed := true; changed; { + changed = false + for from := range classes { + for _, to := range edges[from] { + if !classes[to] { + classes[to], changed = true, true + } + } + } + } + for _, memory := range []string{"Ym", "Ymb", "Yml", "Ymm", "Yxm", "Yxvm", "Yyvm", "Yzvm"} { + if classes[memory] { + t.Fatalf("actual Go address immediate acquired memory-class reachability: %s", memory) + } + } +} + +func TestX86AddressClassPreservesActualMemoryAndRelocationObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"386", "amd64"} { + move := "MOVQ" + if arch == "386" { + move = "MOVL" + } + for _, row := range []struct { + body string + needsContext bool + }{ + {move + " $·cell(SB),AX", false}, + {move + " $16(AX),BX", false}, + {"XADDL CX,·cell(SB)", true}, + {"XCHGL CX,16(AX)", true}, + {"INCL ·cell(SB)", true}, + {"NEGL 16(AX)", true}, + {"STMXCSR ·cell(SB)", true}, + {"LDMXCSR 16(AX)", true}, + {"PSHUFB ·cell(SB),X0", false}, + {"MOVNTPS X0,16(AX)", true}, + } { + source := "TEXT ·Value(SB),4,$0-0\n" + row.body + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + for _, target := range x86AddressMemoryTargets(arch) { + if row.needsContext { + // Keep every original memory/relocation object. A Void + // format fixture supplies neither initialized AX nor the + // external cell's storage role to a public source proof. + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir := x86ScalarFormatObjectUnit(t, file, + map[string]FuncSig{"Value": {Name: "Value", Ret: Void}}, arch, target, 0) + x86ResultInitializationObject(t, llc, target, ir, "original-address-format") + } + for _, api := range []string{"text", "module", "go"} { + t.Run(arch+"/"+row.body+"/"+target+"/"+api, func(t *testing.T) { + ir, err := x86AddressMemoryAPI(t, api, source, arch, target) + if row.needsContext { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound format acquired public storage/input authority", err) + } + return + } + if err != nil { + t.Fatal("legitimate memory/address operand lost:", err) + } + x86ResultInitializationObject(t, llc, target, ir, "address-class") + }) + } + } + } + } +} + +func TestX86AddressClassInitializedExchangeObjects(t *testing.T) { + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + // Keep the discovering operands. The static counterpart supplies + // its actual mutable DATA; the pointer counterpart binds its entry + // parameter. Both independently define the numerical CX input. + fixtures := []x86OriginalGPInitializedFormat{ + { + source: "DATA ·cell+0(SB)/4,$0\nGLOBL ·cell(SB),16,$4\n" + + "TEXT ·AddressStatic(SB),4,$0-0\nMOVL $7,CX\nXADDL CX,·cell(SB)\nRET\n", + declaration: "func AddressStatic()", + }, + { + source: fmt.Sprintf("TEXT ·AddressPointer(SB),4,$0-%d\n%s p+0(FP),AX\n"+ + "MOVL $7,CX\nXCHGL CX,16(AX)\nRET\n", word, move), + declaration: "func AddressPointer(p *[32]byte)", + }, + } + x86OriginalGPInitializedFormatObjects(t, fixtures, target.arch, target.triple) + }) + } +} + +func TestX86AddressClassInitializedNonTemporalStoreObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + // Keep the discovering row byte-for-byte, including displacement. + // The new entry explicitly supplies its pointer and vector inputs; + // neither the declaration nor LLVM zeroed slots supplies them. + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\n%s p+0(FP),AX\n"+ + "PXOR X0,X0\nMOVNTPS X0,16(AX)\nRET\n", word, move) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[32]byte)", target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("initialized original non-temporal address form rejected", err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + if !strings.Contains(text, "asm sideeffect") || !strings.Contains(text, "movntps") { + t.Fatal("actual non-temporal store/alignment effect disappeared") + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + }) + } + }) + } +} + +func x86AddressMemoryTargets(arch string) []string { + if arch == "386" { + return []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + return []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } +} + +func x86AddressMemoryAPI(t *testing.T, api, source, arch, target string) (string, error) { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + options := Options{ + Goarch: arch, TargetTriple: target, ResolveSym: resolve, + Sigs: map[string]FuncSig{"Value": {Name: "Value", Ret: Void}}, + } + if api == "text" { + return Translate(file, options) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + if api == "module" { + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + return "", err + } + return module.String(), nil + } + if api != "go" { + t.Fatalf("unregistered API %q", api) + } + pkg := mustGoPackage(t, "example.invalid/addressclass", "package addressclass\nfunc Value()\n") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: arch, TargetTriple: target, ResolveSym: resolve, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + return "", err + } + return translated.Module.String(), nil +} diff --git a/amd64_address_memory_sinks_test.go b/amd64_address_memory_sinks_test.go new file mode 100644 index 000000000..64e4387cc --- /dev/null +++ b/amd64_address_memory_sinks_test.go @@ -0,0 +1,156 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86SymbolMemoryPointerRejectsAddressIdentityBeforeEmission(t *testing.T) { + for _, source := range []string{ + "$·cell(SB)", "$·cell+8(SB)", " \t$·cell-8(SB)", + } { + t.Run(source, func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before, temporary := output.String(), ctx.tmp + pointer, err := ctx.ptrFromSB(source) + if err == nil || pointer != "" || !strings.Contains(err.Error(), "address immediate") { + t.Fatalf("address acquired a memory pointer: %q: %v", pointer, err) + } + if output.String() != before || ctx.tmp != temporary { + t.Fatal("rejected address emitted IR or consumed a temporary") + } + }) + } + + for _, source := range []string{"·cell(SB)", "·cell+8(SB)", "·cell-8(SB)"} { + t.Run("memory/"+source, func(t *testing.T) { + ctx, _ := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + if pointer, err := ctx.ptrFromSB(source); err != nil || pointer == "" { + t.Fatalf("ordinary symbolic memory lost: %q: %v", pointer, err) + } + }) + } +} + +func x86AddressMemorySinkRows(arch string) []string { + // These ordinary stores and x87 loads use direct OpSym dispatch, not the + // shared memory-class predicate. Their Go encoder rows still require actual + // memory, including the complete scalar and x87 storage widths below. + rows := []string{ + "MOVB AL,%s", "MOVW CX,%s", "MOVL CX,%s", + "MOVSS X0,%s", "MOVSD X0,%s", "MOVOU X0,%s", + "FMOVF F0,%s", "FMOVFP F0,%s", "FMOVD F0,%s", "FMOVDP F0,%s", + "FMOVW F0,%s", "FMOVWP F0,%s", "FMOVL F0,%s", "FMOVLP F0,%s", + "FMOVVP F0,%s", "FMOVBP F0,%s", "FMOVXP F0,%s", + "FMOVF %s,F0", "FMOVD %s,F0", "FMOVW %s,F0", "FMOVL %s,F0", + "FMOVV %s,F0", "FMOVB %s,F0", "FMOVX %s,F0", + } + if arch == "amd64" { + rows = append(rows, "MOVQ CX,%s") + } + return rows +} + +func x86AddressMemorySinkSource(row, operand string) string { + // These are compile-only syntax/operand tests. Initialize explicit register + // inputs, but do not claim execution of the external cell or x87 state. + return "TEXT ·Value(SB),4,$0-0\nMOVL $7,AX\nMOVL $9,CX\n" + + "PXOR X0,X0\nFLDZ\n" + fmt.Sprintf(row, operand) + "\nRET\n" +} + +func TestX86AddressImmediateDirectMemorySinksRejectAcrossAPIs(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86AddressMemorySinkRows(arch) { + for _, operand := range []string{"$·cell(SB)", "$·cell+8(SB)"} { + t.Run(arch+"/"+fmt.Sprintf(row, operand), func(t *testing.T) { + ordinary := x86AddressMemorySinkSource(row, strings.TrimPrefix(operand, "$")) + goAssemblerOracleOutput(t, arch, ordinary, "", true) + source := x86AddressMemorySinkSource(row, operand) + goAssemblerOracleOutput(t, arch, source, "", false) + for _, target := range x86AddressMemoryTargets(arch) { + for _, api := range []string{"text", "module", "go"} { + t.Run(target+"/"+api, func(t *testing.T) { + if _, err := x86AddressMemoryAPI(t, api, source, arch, target); err == nil { + t.Fatal("Go-rejected address bypassed memory-role validation") + } + }) + } + } + }) + } + } + } +} + +func TestX86DirectSymbolMemorySinkObjectsRemainSupported(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86AddressMemorySinkRows(arch) { + source := x86AddressMemorySinkSource(row, "·cell+8(SB)") + goAssemblerOracleOutput(t, arch, source, "", true) + for _, target := range x86AddressMemoryTargets(arch) { + for _, api := range []string{"text", "module", "go"} { + t.Run(arch+"/"+row+"/"+target+"/"+api, func(t *testing.T) { + ir, err := x86AddressMemoryAPI(t, api, source, arch, target) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("format-only external sink acquired storage or x87 state", err) + } + if api == "text" { + x86LEAActualTargetGoObject(t, arch, target, source) + unit := x86ZeroArgumentLowererObjectUnit(t, mustParseX86RawReturn(t, source), arch, target, "ExternalSinkEmitterOnly", + func(consumer *amd64Ctx, instruction Instr) (bool, bool, error) { + if known, terminated, err := consumer.lowerMov(instruction.Op, instruction); known { + return known, terminated, err + } + if known, terminated, err := consumer.lowerVec(instruction.Op, instruction); known { + return known, terminated, err + } + if known, terminated, err := consumer.lowerFP(instruction.Op, instruction); known { + return known, terminated, err + } + return consumer.lowerX87(instruction.Op, instruction) + }) + x86ResultInitializationObject(t, llc, target, unit, "original-external-sink-emitter-only") + } + if counterpart, integer := x86AddressIntegerSinkCounterpart(row); integer { + declaration := fmt.Sprintf("func Value(input %s)", counterpart.inputType) + body := x86X87IntegerInputCases(counterpart)[0].body + bounded := x86X87IntegerInputSource(t, arch, declaration, body, false, "") + boundedAPI := api + if api == "go" { + boundedAPI = "Go" + } + ir, err = x86X87InitializedWordAPI(t, bounded, declaration, arch, target, boundedAPI, X87Software) + if err != nil || ir == "" { + t.Fatal("independent initialized and balanced integer-sink counterpart failed", err) + } + x86ResultInitializationObject(t, llc, target, ir, "memory-sink") + } + }) + } + } + } + } +} + +// Match the independent complete eight integer memory directions, never the +// implementation's permission table. Their original external operands remain +// unbound; the separate counterpart owns actual typed input/private storage. +func x86AddressIntegerSinkCounterpart(row string) (x86X87IntegerInputRow, bool) { + for _, counterpart := range x86X87IntegerInputRows { + if counterpart.name == "round" { + continue + } + instruction := strings.ReplaceAll(counterpart.instruction, "input+0(FP)", "memory") + instruction = strings.ReplaceAll(instruction, "converted-8(SP)", "memory") + if fmt.Sprintf(row, "memory") == instruction { + return counterpart, true + } + } + return x86X87IntegerInputRow{}, false +} diff --git a/amd64_addsub_round_closure_test.go b/amd64_addsub_round_closure_test.go new file mode 100644 index 000000000..c43faf9de --- /dev/null +++ b/amd64_addsub_round_closure_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// Each complete original VEX row has register and memory counterparts. +// This is bounded numerical availability/object evidence, not native FP, +// MXCSR, fault execution or a certificate for the input pointee's extent. +func TestX86AlternatingRoundInitializedOfficialRowsThreeAPIObjects(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + word, move, output := 8, "MOVQ", "VMOVUPS Z2,(DX)" + if target.arch == "386" { + word, move, output = 4, "MOVL", "VMOVUPS Y2,(DX)" + } + var source, declarations, refusedSource, refusedDeclarations strings.Builder + rows, functions, refusedRows, refusedFunctions := 0, 0, 0, 0 + for _, row := range rules { + if row.Opcode != "VADDSUBPS" && row.Opcode != "VADDSUBPD" && row.Opcode != "VROUNDPS" && row.Opcode != "VROUNDPD" && row.Opcode != "VROUNDSS" && row.Opcode != "VROUNDSD" { + continue + } + rows++ + goRefused := target.arch == "386" && (row.Opcode == "VROUNDSS" || row.Opcode == "VROUNDSD") + if goRefused { + refusedRows++ + } + bank := map[string]string{"Yxr": "X", "Yyr": "Y"}[row.Classes[len(row.Classes)-1]] + if bank == "" { + t.Fatal("original row lost its VEX carrier", row) + } + immediate := "$7" + if row.Classes[0] == "Yi8" { + immediate = "$-1" + } + for _, memory := range []bool{false, true} { + name := fmt.Sprint("AlternatingRound", functions) + functions++ + var function strings.Builder + declaration := fmt.Sprintf("func %s(out *[64]byte,a,b *[32]byte)\n", name) + fmt.Fprintf(&function, "TEXT ·%s(SB),4,$0-%d\n%s out+0(FP),DX\n%s a+%d(FP),AX\n", name, 3*word, move, move, word) + first := bank + "0" + if memory { + first = "(AX)" + } else { + function.WriteString("VMOVUPS (AX),Y0\n") + } + if strings.HasPrefix(row.Opcode, "VADDSUB") || strings.HasSuffix(row.Opcode, "SS") || strings.HasSuffix(row.Opcode, "SD") { + fmt.Fprintf(&function, "%s b+%d(FP),BX\nVMOVUPS (BX),Y1\n", move, 2*word) + } + switch { + case strings.HasPrefix(row.Opcode, "VADDSUB"): + fmt.Fprintf(&function, "%s %s,%s1,%s2\n", row.Opcode, first, bank, bank) + case strings.HasSuffix(row.Opcode, "SS") || strings.HasSuffix(row.Opcode, "SD"): + fmt.Fprintf(&function, "%s %s,%s,X1,X2\n", row.Opcode, immediate, first) + default: + fmt.Fprintf(&function, "%s %s,%s,%s2\n", row.Opcode, immediate, first, bank) + } + function.WriteString(output + "\nRET\n") + if goRefused { + refusedFunctions++ + refusedSource.WriteString(function.String()) + refusedDeclarations.WriteString(declaration) + } else { + source.WriteString(function.String()) + declarations.WriteString(declaration) + } + } + } + if rows != 14 || functions != 28 { + t.Fatal("complete original row/source union changed", rows, functions) + } + if target.arch == "386" { + if refusedRows != 2 || refusedFunctions != 4 || rows-refusedRows != 12 || functions-refusedFunctions != 24 { + t.Fatal("mode32 ordinary/refused source union changed", rows, functions, refusedRows, refusedFunctions) + } + x86RoundRejectedGoSource(t, refusedSource.String(), refusedDeclarations.String(), target.arch, target.triple) + } else if refusedRows != 0 || refusedFunctions != 0 { + t.Fatal("AMD64 lost an original canonical row") + } + t.Logf("original rows=%d functions=%d selected rows=%d functions=%d refused rows=%d functions=%d", rows, functions, rows-refusedRows, functions-refusedFunctions, refusedRows, refusedFunctions) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("complete independently initialized alternating/round rows rejected", err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + }) + } + }) + } + +} diff --git a/amd64_addsub_round_forms.go b/amd64_addsub_round_forms.go new file mode 100644 index 000000000..ce08cc12a --- /dev/null +++ b/amd64_addsub_round_forms.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Alternating add/sub uses the existing floating dependency record, with +// its own complete legacy/VEX grammar and no EVEX mask or rounding forms. +func x86AlternatingFloatProperties(ins Instr, arch string) (x86FloatBinaryForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + f := x86FloatBinaryForm{first: 0, second: 1, destination: len(ins.Args) - 1, mask: -1, memory: -1, byteWidth: 16} + switch name { + case "ADDSUBPS": + f.bits, f.legacy = 32, true + case "ADDSUBPD": + f.bits, f.legacy = 64, true + case "VADDSUBPS": + f.bits = 32 + case "VADDSUBPD": + f.bits = 64 + default: + return f, false, nil + } + if dotted { + return f, true, fmt.Errorf("%s %s has no suffixed forms in Go 1.27: %q", arch, name, ins.Raw) + } + legacyRegister := func(operand Operand) bool { + index, valid := amd64ParseXReg(operand.Reg) + return operand.Kind == OpReg && valid && index < 16 && (arch != "386" || index < 8) + } + if f.legacy { + if len(ins.Args) != 2 || !legacyRegister(ins.Args[1]) { + return f, true, fmt.Errorf("%s %s expects X/m128, X using Go 1.27's yxm table: %q", arch, name, ins.Raw) + } + if ins.Args[0].Kind == OpReg { + if !legacyRegister(ins.Args[0]) { + return f, true, fmt.Errorf("%s %s source is outside the legacy X class: %q", arch, name, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return f, true, fmt.Errorf("%s %s source must be X or memory: %q", arch, name, ins.Raw) + } + } else { + if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { + return f, true, fmt.Errorf("%s %s expects src1, src2, destination: %q", arch, name, ins.Raw) + } + f.byteWidth = amd64VectorByteWidth(ins.Args[2].Reg) + if f.byteWidth != 16 && f.byteWidth != 32 || !amd64VEXVectorRegister(ins.Args[1], f.byteWidth) || !amd64VEXVectorRegister(ins.Args[2], f.byteWidth) { + return f, true, fmt.Errorf("%s %s second source and destination must be same-width VEX X/Y registers: %q", arch, name, ins.Raw) + } + if ins.Args[0].Kind == OpReg { + if !amd64VEXVectorRegister(ins.Args[0], f.byteWidth) { + return f, true, fmt.Errorf("%s %s first source must match the destination width: %q", arch, name, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return f, true, fmt.Errorf("%s %s first source must be a vector register or memory: %q", arch, name, ins.Raw) + } + } + if isAMD64MemoryOperand(ins.Args[0]) { + f.memory, f.memoryBytes = 0, f.byteWidth + } + return f, true, nil +} + +type x86RoundForm struct { + spec amd64RoundSpec + source, upper, destination Operand + byteWidth, memoryBytes int + immediate uint8 + memory bool +} + +// The original single round spec remains the only operation selector. Scalar +// source bytes and upper-copy bytes stay separate from the X/Y carrier. +func x86RoundProperties(ins Instr, arch string) (x86RoundForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := amd64RoundSpecs[Op(name)] + f := x86RoundForm{spec: spec} + if !known { + return f, false, nil + } + if dotted { + return f, true, fmt.Errorf("%s %s does not accept instruction suffixes: %q", arch, name, ins.Raw) + } + want := 3 + if spec.vector && spec.scalar { + want = 4 + } + if len(ins.Args) != want { + return f, true, fmt.Errorf("%s %s expects immediate, source, %sdestination: %q", arch, name, map[bool]string{true: "upper-lane source, "}[spec.vector && spec.scalar], ins.Raw) + } + // I386's ordinary Go frontend does not accept this four-operand form. + // A normalized encoded instruction retains its separate physical grammar. + if arch == "386" && spec.vector && spec.scalar && !ins.x86Encoded && ins.x86SourceRaw == nil { + return f, true, fmt.Errorf("%s %s four-operand form is not accepted by Go's frontend: %q", arch, name, ins.Raw) + } + immediate := ins.Args[0] + if !amd64UnsignedImmediate(immediate, 8) && + !(spec.vector && !spec.scalar && amd64VectorByteShiftImmediate(immediate)) { + return f, true, fmt.Errorf("%s %s expects an integer immediate within Go's 8-bit table: %q", arch, name, ins.Raw) + } + imm := immediate.Imm + f.immediate, f.source, f.destination = uint8(imm), ins.Args[1], ins.Args[len(ins.Args)-1] + if f.destination.Kind != OpReg { + return f, true, fmt.Errorf("%s %s destination must be a vector register: %q", arch, name, ins.Raw) + } + f.byteWidth = amd64VectorByteWidth(f.destination.Reg) + register := func(operand Operand, width int, memory bool) bool { + if memory && isAMD64MemoryOperand(operand) { + return true + } + index, valid := amd64VectorRegisterIndex(operand.Reg, width) + return amd64VEXVectorRegister(operand, width) && valid && (arch != "386" || index < 8) + } + if spec.scalar { + if f.byteWidth != 16 || !register(f.destination, 16, false) { + return f, true, fmt.Errorf("%s %s destination must be an X register in the Go VEX class: %q", arch, name, ins.Raw) + } + } else if !spec.vector && f.byteWidth != 16 || spec.vector && f.byteWidth != 16 && f.byteWidth != 32 || !register(f.destination, f.byteWidth, false) { + return f, true, fmt.Errorf("%s %s destination has a width outside Go 1.27's table: %q", arch, name, ins.Raw) + } + if !register(f.source, f.byteWidth, true) { + return f, true, fmt.Errorf("%s %s source must be same-width X/Y or memory: %q", arch, name, ins.Raw) + } + f.upper = f.destination + if spec.vector && spec.scalar { + f.upper = ins.Args[2] + if !register(f.upper, 16, false) { + return f, true, fmt.Errorf("%s %s upper-lane source must be an X register: %q", arch, name, ins.Raw) + } + } + f.memory, f.memoryBytes = isAMD64MemoryOperand(f.source), f.byteWidth + if spec.scalar { + f.memoryBytes = spec.laneBits / 8 + } + return f, true, nil +} + +func x86TypedAlternatingFloatByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + local := ins + local.Op = op + f, known, err := x86AlternatingFloatProperties(local, arch) + if known && ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + err = fmt.Errorf("alternating floating dispatch/source opcode mismatch") + } + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: f.first, operand: ins.Args[f.first], width: int64(f.byteWidth), read: true}, + {index: f.second, operand: ins.Args[f.second], width: int64(f.byteWidth), read: true}, + {index: f.destination, operand: ins.Args[f.destination], width: int64(f.byteWidth), write: true}, + }} + effect.vector.upperWrite = x86TypedVectorUpperZero + if f.legacy { + effect.vector.upperWrite = x86TypedVectorUpperPreserve + } + return effect, true, nil +} + +func x86TypedRoundByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + local := ins + local.Op = op + f, known, err := x86RoundProperties(local, arch) + if known && ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + err = fmt.Errorf("round dispatch/source opcode mismatch") + } + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: 1, read: true}, + {index: 1, operand: f.source, width: int64(f.memoryBytes), read: true}, + }} + if f.spec.scalar { + upper := len(ins.Args) - 1 + if f.spec.vector { + upper = 2 + } + effect.operands = append(effect.operands, x86TypedByteOperand{index: upper, operand: f.upper, width: 16, read: true, requiredBytes: 0xffff &^ x86CallerVectorByteMask(int64(f.memoryBytes))}) + } + effect.operands = append(effect.operands, x86TypedByteOperand{index: len(ins.Args) - 1, operand: f.destination, width: int64(f.byteWidth), write: true}) + effect.vector.upperWrite = x86TypedVectorUpperZero + if !f.spec.vector { + effect.vector.upperWrite = x86TypedVectorUpperPreserve + } + return effect, true, nil +} diff --git a/amd64_addsub_round_memory.go b/amd64_addsub_round_memory.go new file mode 100644 index 000000000..7a82cea67 --- /dev/null +++ b/amd64_addsub_round_memory.go @@ -0,0 +1,29 @@ +package plan9asm + +import "fmt" + +// Only these normalized VEX family forms select an observable read. Legacy, +// registers and logical FP retain their existing evaluator and alias behavior. +// This is emission, never a source, address, extent or caller certificate. +func (c *amd64Ctx) loadAlternatingRoundFloat(operand Operand, width, bits int, observable bool) (string, error) { + if observable && (operand.Kind == OpMem || operand.Kind == OpSym) { + bytes, err := c.loadX86PackedShiftBytes(operand, width, true) + if err != nil { + return "", err + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", result, width, bytes, amd64FMA3LLVMType(width*8/bits, bits)) + return "%" + result, nil + } + if width*8 == bits { + return c.evalScalarFloatForInteger(operand, bits) + } + return c.loadHorizontalFloatingOperand(operand, width, bits) +} + +func (c *amd64Ctx) loadRoundPackedBytes(form x86RoundForm) (string, error) { + if form.spec.vector && (form.source.Kind == OpMem || form.source.Kind == OpSym) { + return c.loadX86PackedShiftBytes(form.source, form.byteWidth, true) + } + return c.loadPackedCompareBytes(form.source, form.byteWidth) +} diff --git a/amd64_adx_forms.go b/amd64_adx_forms.go new file mode 100644 index 000000000..aad936e12 --- /dev/null +++ b/amd64_adx_forms.go @@ -0,0 +1,119 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" +) + +type x86ADXForm struct { + spec amd64ADXSpec + source, destination Operand +} + +// All four yml_rl rows share one complete grammar. Their independent CF/OF +// chains are effects, not source-memory, native-entry or numerical authority. +func x86ADXProperties(op Op, ins Instr, arch string) (x86ADXForm, bool, error) { + name, _, suffix := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64ADXSpecs[Op(name)] + form := x86ADXForm{spec: spec} + if !known { + return form, false, nil + } + if arch == "" { + arch = "amd64" + } + fail := func() (x86ADXForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside Go ADX yml_rl grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || arch == "386" && spec.bits == 64 || suffix || len(ins.Args) != 2 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + form.source, form.destination = ins.Args[0], ins.Args[1] + if form.destination.Kind != OpReg || !isX86YrlRegisterForArch(form.destination.Reg, arch) { + return fail() + } + if form.source.Kind == OpReg { + if !isX86YrlRegisterForArch(form.source.Reg, arch) { + return fail() + } + } else if !x86ScalarCompareMemoryValid(form.source, arch) { + if ins.x86Encoded && ins.x86RIPLiteral && form.source.Kind == OpImm { + return form, true, fmt.Errorf("%w: physical ADX literal needs its original native-read role", ErrProbeNeedsContext) + } + return fail() + } + return form, true, nil +} + +func (form x86ADXForm) carryFlag() uint8 { + if form.spec.carry == amd64ADXCarryOF { + return x86ByteFlagOF + } + return x86ByteFlagCF +} + +func x86TypedADXByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + if !x86ResultNamedPredicateInstruction(ins) { + return x86TypedByteEffect{}, false, nil + } + form, known, err := x86ADXProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.spec.bits / 8) + flag := form.carryFlag() + return x86TypedByteEffect{requiresResultProof: true, flagsRead: flag, flagsDefined: flag, flagsKilled: flag, + operands: []x86TypedByteOperand{ + {index: 0, operand: form.source, width: width, read: true}, + {index: 1, operand: form.destination, width: width, read: true, write: true}, + }}, true, nil +} + +func x86RawADXSource(ins Instr, arch string) bool { + _, known, err := x86ADXProperties(ins.Op, ins, arch) + origin := ins.x86SourceRaw + if !known || err != nil || origin == nil || origin.op != ins.Op || !reflect.DeepEqual(origin.args, ins.Args) || + origin.vectorBytes != ins.x86VectorBytes || origin.addressBits != ins.x86AddressBits || + ins.x86AddressBits != goWordSize(arch)*8 || ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 || + ins.x86RawNearReturn != nil || ins.x86RIPLiteral || ins.x86NumericalRead != nil { + return false + } + source := ins.Args[0] + return source.Kind == OpReg || source.Kind == OpMem && x86OrdinaryNativeMemoryOperand(source, arch) +} + +func x86LoweredADXResultStep(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) (bool, bool) { + form, known, err := x86ADXProperties(ins.Op, ins, arch) + if !known { + return false, false + } + if err != nil { + return false, true + } + local := ins + if !x86ResultNamedPredicateInstruction(ins) { + if !x86RawADXSource(ins, arch) { + return false, true + } + local.x86Encoded, local.x86SourceRaw, local.x86AddressBits = false, nil, 0 + } + if !x86OrdinaryNativeMemoryOperand(form.source, arch) { + return x86ResultTransfer(local, state, owned, resultOffset, arch), true + } + effect, _, err := x86TypedADXByteEffect(local.Op, local, arch) + if err != nil || state.flags&effect.flagsRead != effect.flagsRead { + return false, true + } + for _, input := range effect.operands { + if input.read && !x86ResultReadWithNativeContinuation(local, input.operand, input.width, state, owned, arch, true) { + return false, true + } + } + if !x86ResultStore(form.destination, int64(form.spec.bits/8), state, owned, resultOffset, arch) { + return false, true + } + state.flags = state.flags&^effect.flagsKilled | effect.flagsDefined + return true, true +} diff --git a/amd64_aes_family_effects_test.go b/amd64_aes_family_effects_test.go new file mode 100644 index 000000000..a3c5e10f8 --- /dev/null +++ b/amd64_aes_family_effects_test.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +// Independent source inventory: six SSE names and their six AVX names. AES +// round rows are destructive only in their legacy spelling; no Go AES row +// has a K operand, broadcast or zeroing suffix. +var x86AESOracleNames = []string{"AESENC", "AESENCLAST", "AESDEC", "AESDECLAST", "AESIMC", "AESKEYGENASSIST", "VAESENC", "VAESENCLAST", "VAESDEC", "VAESDECLAST", "VAESIMC", "VAESKEYGENASSIST"} + +func x86AESOracleRow(name, source, bank string) (string, int) { + if strings.Contains(name, "KEYGEN") { + return name + " $27," + source + "," + bank + "2", 2 + } + if strings.HasSuffix(name, "IMC") { + return name + " " + source + "," + bank + "2", 2 + } + if strings.HasPrefix(name, "V") { + return name + " " + source + "," + bank + "1," + bank + "2", 2 + } + return name + " " + source + "," + bank + "1", 1 +} + +func TestX86AESFamilyInitialTypedAndPrivateResults(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames { + row, destination := x86AESOracleRow(name, "X0", "X") + t.Run(arch+"/"+name, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + ins := x86YMMReadSource(t, row) + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || effect.flagsRead != 0 || effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatalf("complete AES typed byte/flags effect absent: known=%t err=%v", known, err) + } + for _, transfer := range []struct { + name string + fn func(Instr, *x86ResultState, []bool, int64, string) bool + }{{"public", x86ResultTransfer}, {"private", x86ScalarFPResultTransfer}} { + state := x86ResultState{flags: x86ByteFlagsAll} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + if !transfer.fn(ins, &state, nil, 0, arch) || state.xmm[destination] != 0xffff || state.flags != x86ByteFlagsAll { + t.Fatal(transfer.name, "lost complete AES input/result or preserved flags", state) + } + } + }) + } + } +} + +func TestX86AESFamilyInitialCallerSourceEffects(t *testing.T) { + for _, name := range x86AESOracleNames { + row, destination := x86AESOracleRow(name, "X0", "X") + t.Run(name, func(t *testing.T) { + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\n" + row + fmt.Sprintf("\nVMOVSS X%d,(DX)", destination) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("actual Go accepted whole-family caller has no registered AES source effect", err) + } + }) + } +} + +func TestX86AESFamilyResultEveryNativeMemoryByte(t *testing.T) { + for _, name := range x86AESOracleNames { + row, destination := x86AESOracleRow(name, "a+0(FP)", "X") + ins := x86YMMReadSource(t, row) + for missing := -1; missing < 16; missing++ { + t.Run(fmt.Sprintf("%s/missing%d", name, missing), func(t *testing.T) { + state := x86ResultState{fp: make([]bool, 16), flags: x86ByteFlagsAll} + state.xmm[1] = 0xffff + owned := make([]bool, 16) + for i := range owned { + owned[i], state.fp[i] = true, i != missing + } + before := state.clone() + got := x86ScalarFPResultTransfer(ins, &state, owned, 16, "amd64") + if got != (missing == -1) || got && state.xmm[destination] != 0xffff || !got && !reflect.DeepEqual(state, before) { + t.Fatal("native m128 envelope/unknown byte must not be shrunk to numerical subwords", got, state) + } + }) + } + } +} + +func TestX86AESFamilyWholeSourceResultsCannotLaunderUndefinedInputs(t *testing.T) { + for _, name := range x86AESOracleNames { + row, destination := x86AESOracleRow(name, "X0", "X") + for _, target := range x86RawReturnTargets { + for _, api := range []string{"text", "module", "Go"} { + t.Run(name+"/"+target.triple+"/"+api, func(t *testing.T) { + source := fmt.Sprintf("TEXT ·F(SB),4,$0-4\n%s\nMOVL X%d,AX\nMOVL AX,ret+0(FP)\nRET\n", row, destination) + ir, err := x86ScalarFPBitViewAPI(t, source, "func F() uint32\n", target.arch, target.triple, api) + if target.arch == "amd64" { + x86VectorScratchCodegenControl(t, source, "func F() uint32\n", target.arch, target.triple, ir, err) + return + } + if err == nil { + t.Fatal("undefined AES byte escaped through GP result-store fallback") + } + }) + } + } + } +} diff --git a/amd64_aes_family_encoder_test.go b/amd64_aes_family_encoder_test.go new file mode 100644 index 000000000..55504de47 --- /dev/null +++ b/amd64_aes_family_encoder_test.go @@ -0,0 +1,531 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86AESFamilyNamedAndRawThreeAPIFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, target := range x86RawReturnTargets { + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/raw=%t", target.triple, raw), func(t *testing.T) { + var source, declarations strings.Builder + declarations.WriteString("package aesobjects\n") + count := 0 + for _, name := range x86AESOracleNames { + widths := []int{16} + if strings.HasPrefix(name, "V") && !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + widths = append(widths, 32, 64) + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, operand := range []string{bank + "0", fmt.Sprintf("%d(AX)(CX*2)", width)} { + row, _ := x86AESOracleRow(name, operand, bank) + code := x86OrderingHintGoBytes(t, target.arch, "TEXT actual(SB),4,$0-0\n"+row+"\nRET\n") + if raw { + row = x86DwordProductDirectives(code[:len(code)-1]) + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", count, row) + fmt.Fprintf(&declarations, "func F%d()\n", count) + count++ + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original AES formats invented native addresses or vector inputs", api, err) + } + }) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + file, err = normalizeX86RawFile(file, target.arch) + if err != nil { + t.Fatal(err) + } + signatures := make(map[string]FuncSig) + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + signatures[name] = FuncSig{Name: name, Ret: Void} + } + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.arch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerAES(ins.Op, ins) }) + for _, input := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, input, level) + } + } + }) + } + } +} + +func TestX86AESFamilyCompressedDisp8EntireTupleAxes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames[6:10] { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, disp := range []int{-128, -1, 1, 127} { + for _, address := range []string{fmt.Sprintf("%d(AX)", disp*width), fmt.Sprintf("%d(AX)(CX*2)", disp*width), fmt.Sprintf("%d(BP)", disp*width)} { + // X16 forces Go's EVEX X/Y rows. Their original bytes are + // never patched into a different register contract. Go accepts + // that spelling in mode32, but physical high banks remain Context. + row := name + " " + address + "," + bank + "1," + bank + "16" + if width == 64 { + row = name + " " + address + ",Z1,Z2" + } + t.Run(arch+"/"+row, func(t *testing.T) { + code := x86OrderingHintGoBytes(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + mode := 64 + if arch == "386" { + if width != 64 { + if _, _, known, err := decodedX86AESInstruction(code, 32); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("actual mode32 high destination acquired a low-bank identity", row, fmt.Sprintf("%x", code), err) + } + // Inspect exactly the same original bytes in physical + // mode64 for their tuple address, not mode32 permission. + } else { + mode = 32 + } + } + ins, length, known, err := decodedX86AESInstruction(code, mode) + if !known || err != nil || length != len(code)-1 || ins.Args[0].Mem.Off != int64(disp*width) { + t.Fatal("actual Go compressed full tuple did not retain exact native address", row, fmt.Sprintf("%x", code), ins, err) + } + }) + } + } + } + } + } +} + +func TestX86AESFamilyOriginalGoPhysicalRegisterExtensionAxes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames[6:10] { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for role := 0; role < 3; role++ { + for _, index := range []int{0, 7, 8, 15, 16, 31} { + args := []string{bank + "0", bank + "1", bank + "2"} + args[role] = fmt.Sprintf("%s%d", bank, index) + row := name + " " + strings.Join(args, ",") + t.Run(arch+"/"+row, func(t *testing.T) { + if arch == "386" && width == 64 && index >= 8 { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + if _, known, err := x86AESProperties(Op(name), x86YMMReadSource(t, row), arch); !known || err == nil { + t.Fatal("actual Go rejected mode32 Z bank acquired a valid grammar row", row, err) + } + return + } + code := x86OrderingHintGoBytes(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + mode := 64 + if arch == "386" { + mode = 32 + if index >= 8 { + if _, _, known, err := decodedX86AESInstruction(code, mode); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("actual Go high spelling acquired a different physical mode32 register", row, fmt.Sprintf("%x", code), err) + } + return + } + } + ins, length, known, err := decodedX86AESInstruction(code, mode) + if !known || err != nil || length != len(code)-1 || ins.Args[role].Reg != Reg(args[role]) { + t.Fatal("all original VEX/EVEX source/state/destination extension bits", row, fmt.Sprintf("%x", code), ins, err) + } + }) + } + } + } + } + } +} + +func TestX86AESFamilyMode32NamedHighAndRawReservedBoundaries(t *testing.T) { + for _, name := range x86AESOracleNames[6:10] { + for _, bank := range []string{"X", "Y"} { + row := name + " " + bank + "16," + bank + "1," + bank + "2" + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + if _, known, err := x86AESProperties(Op(name), x86YMMReadSource(t, row), "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go named high mode32 vector borrowed physical identity", row, err) + } + } + } + valid := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\nVAESENC 64(AX),Z1,Z2\nRET\n") + for _, mutation := range []func([]byte){func(b []byte) { b[2] &^= 4 }, func(b []byte) { b[3] |= 1 }, func(b []byte) { b[3] |= 0x80 }, func(b []byte) { b[3] |= 0x10 }, func(b []byte) { b[3] |= 0x60 }} { + changed := append([]byte(nil), valid...) + mutation(changed) + if _, _, known, err := decodedX86AESInstruction(changed, 64); !known || err == nil { + t.Fatal("reserved AES bytes acquired an unrelated raw effect", fmt.Sprintf("%x", changed), err) + } + } +} + +func TestX86AESFamilyPhysicalWIGIndependentLLVMDecoder(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM22 required") + } + for _, name := range x86AESOracleNames[6:] { + forms := []struct { + width int + destination int + }{{16, 2}} + if !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + forms = append(forms, struct { + width int + destination int + }{32, 2}, struct { + width int + destination int + }{16, 16}, struct { + width int + destination int + }{32, 16}, struct { + width int + destination int + }{64, 2}) + } + for _, form := range forms { + width := form.width + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + row, _ := x86AESOracleRow(name, fmt.Sprintf("%d(AX)", width), bank) + row = strings.TrimSuffix(row, bank+"2") + fmt.Sprintf("%s%d", bank, form.destination) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + // Intel's VEX and EVEX AES attributes are WIG. This is an + // explicit physical-attribute fixture, not a claimed Go named row. + for _, wide := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/W=%t", row, wide), func(t *testing.T) { + physical := append([]byte(nil), code[:len(code)-1]...) + if wide { + physical[2] |= 0x80 + } + var input strings.Builder + for _, b := range physical { + fmt.Fprintf(&input, "0x%02x ", b) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=x86_64-unknown-linux-gnu") + command.Stdin = strings.NewReader(input.String() + "\n") + output, err := runtimeOracleCombinedOutput(ctx, command) + if err != nil || !strings.Contains(strings.ToLower(string(output)), strings.ToLower(name)) || strings.Contains(string(output), "warning:") { + t.Fatal("independent LLVM22 physical WIG decoder", err, string(output)) + } + ins, length, known, err := decodedX86AESInstruction(physical, 64) + if !known || err != nil || length != len(physical) || ins.Op != Op(name) { + t.Fatal("complete physical AES WIG attribute lost", fmt.Sprintf("%x", physical), ins, err) + } + }) + } + } + } +} + +func TestX86AESFamilyLogicalFPNumericalSpanContracts(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, name := range x86AESOracleNames { + row, _ := x86AESOracleRow(name, "a+0(FP)", "X") + for _, contract := range []struct { + parameters string + accepted bool + }{{"a,b uint64", true}, {"a,b,c,d float32", true}, {"a,b float64", true}, + {"a uint8,b uint64", false}, {"a uint8,b uint64,c uint32", false}, {"a,b *byte", false}, {"a,b,c,d *byte", false}} { + t.Run(target.triple+"/"+name+"/"+contract.parameters, func(t *testing.T) { + size := 16 + switch contract.parameters { + case "a uint8,b uint64": + if target.arch == "386" { + size = 12 + } + case "a uint8,b uint64,c uint32": + if target.arch == "amd64" { + size = 20 + } + case "a,b *byte": + size = int(goWordSize(target.arch)) * 2 + case "a,b,c,d *byte": + size = int(goWordSize(target.arch)) * 4 + } + source := fmt.Sprintf("TEXT ·F(SB),4,$0-%d\n%s\nRET\n", size, row) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + initialized := strings.Replace(source, fmt.Sprintf("$0-%d\n", size), + fmt.Sprintf("$0-%d\nMOVL $1,AX\nMOVL AX,X1\n", size), 1) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + declaration := "func F(" + contract.parameters + ")\n" + if contract.accepted && strings.HasPrefix(name, "V") && !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + if ir == "" || err != nil { + t.Fatal("unused original FP round result cannot change an unrelated source contract", api, err) + } + } + ir, err := x86ScalarFPBitViewAPI(t, initialized, declaration, target.arch, target.triple, api) + accepted := contract.accepted && strings.HasPrefix(name, "V") + if (err == nil) != accepted || !accepted && ir != "" { + t.Fatal("numeric FP bytes cannot supply native legacy AES alignment, holes or pointer backing", accepted, err) + } + }) + } + }) + } + } + } +} + +func TestX86AESFamilyAbsoluteDisp32AndImmediateClasses(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames { + for _, address := range []int64{-2147483649, -2147483648, 2147483647, 4294967295, 4294967296} { + accepted := address == int64(int32(address)) || arch == "386" && address == int64(uint32(address)) + row, _ := x86AESOracleRow(name, fmt.Sprint(address), "X") + t.Run(arch+"/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", accepted) + _, known, err := x86AESProperties(Op(name), x86YMMReadSource(t, row), arch) + if !known || (err == nil) != accepted { + t.Fatal("physical Go disp32 extent differs from shared AES grammar", accepted, err) + } + }) + } + } + for _, name := range []string{"AESKEYGENASSIST", "VAESKEYGENASSIST"} { + var all strings.Builder + for immediate := -129; immediate <= 256; immediate++ { + accepted := immediate >= 0 && immediate <= 255 || name == "VAESKEYGENASSIST" && immediate >= -128 && immediate < 0 + row := fmt.Sprintf("%s $%d,X0,X2", name, immediate) + if accepted { + all.WriteString(row + "\n") + } else { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + } + if _, known, err := x86AESProperties(Op(name), x86YMMReadSource(t, row), arch); !known || (err == nil) != accepted { + t.Fatal("complete Yu8/Yi8 immediate classes", arch, row, accepted, err) + } + } + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+all.String()+"RET\n", "", true) + } + } +} + +func TestX86AESFamilyCompleteIndependentEncoderRows(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + data, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, table := range []struct{ name, body string }{{"yaes", "{Zlitm_r, 2, argList{Yxm, Yxr}},"}, {"yxshuf", "{Zibm_r, 2, argList{Yu8, Yxm, Yxr}},"}} { + start := strings.Index(string(data), "var "+table.name+" = []ytab{") + if start < 0 { + t.Fatal(table.name) + } + actual := strings.SplitN(strings.SplitN(string(data)[start:], "\n}", 2)[0], "[]ytab{", 2)[1] + a, err := x86SignedPackedFloatGoOpcodeTokens(actual) + if err != nil { + t.Fatal(err) + } + w, err := x86SignedPackedFloatGoOpcodeTokens(table.body) + if err != nil || !reflect.DeepEqual(a, w) { + t.Fatal("complete legacy zcase/zoffset/classes changed", table.name, actual) + } + } + legacyRows := regexp.MustCompile(`\{A(AES[A-Z]+), ([^,]+), ([^,]+), opBytes\{([^}]+)\}\}`).FindAllStringSubmatch(string(data), -1) + if len(legacyRows) != 6 || len(x86AESSpecs) != 12 { + t.Fatal("whole family inventory changed", len(legacyRows), len(x86AESSpecs)) + } + for i, name := range x86AESOracleNames[:6] { + table, escape, opcode := "yaes", "0x38", []string{"0xdc", "0xdd", "0xde", "0xdf", "0xdb", "0xdf"}[i] + if i == 5 { + table, escape = "yxshuf", "0x3a" + } + want := fmt.Sprintf("{A%s, %s, Pq, opBytes{%s, %s, 0}}", name, table, escape, opcode) + if !strings.Contains(string(data), want) { + t.Fatal("legacy complete prefix/opcode attributes changed", want) + } + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + counts := map[string]int{} + for _, rule := range rules { + if !strings.HasPrefix(rule.Opcode, "VAES") { + continue + } + index := counts[rule.Opcode] + counts[rule.Opcode]++ + isIMC, isKey := rule.Opcode == "VAESIMC", rule.Opcode == "VAESKEYGENASSIST" + opcode := map[string]string{"VAESENC": "0xDC", "VAESENCLAST": "0xDD", "VAESDEC": "0xDE", "VAESDECLAST": "0xDF", "VAESIMC": "0xDB", "VAESKEYGENASSIST": "0xDF"}[rule.Opcode] + if opcode == "" { + t.Fatal("unaccounted actual Go AES member", rule.Opcode) + } + table, classes, zcase := "_yvaesdec", []string{"Yxm,Yxr,Yxr", "Yym,Yyr,Yyr", "YxmEvex,YxrEvex,YxrEvex", "YymEvex,YyrEvex,YyrEvex", "Yzm,Yzr,Yzr"}, "Zvex_rm_v_r" + zoffset, offset := []int{2, 2, 3, 3, 3}, []int{0, 2, 4, 7, 10} + var opbytes []string + if isKey { + table, classes, zcase = "_yvaeskeygenassist", []string{"Yu8,Yxm,Yxr", "Yi8,Yxm,Yxr"}, "Zvex_i_rm_r" + zoffset, offset = []int{0, 2}, []int{0, 0} + opbytes = []string{"avxEscape | vex128 | vex66 | vex0F3A | vexW0", opcode} + } else if isIMC { + table, classes = "_yvaesimc", []string{"Yxm,Yxr"} + zoffset, offset = []int{2}, []int{0} + opbytes = []string{"avxEscape | vex128 | vex66 | vex0F38 | vexW0", opcode} + } else { + for _, width := range []int{128, 256} { + opbytes = append(opbytes, fmt.Sprintf("avxEscape | vex%d | vex66 | vex0F38 | vexW0", width), opcode) + } + for _, width := range []int{128, 256, 512} { + opbytes = append(opbytes, fmt.Sprintf("avxEscape | evex%d | evex66 | evex0F38 | evexW0", width), fmt.Sprintf("evexN%d", width/8), opcode) + } + if index >= 2 { + zcase = "Zevex_rm_v_r" + } + } + if index >= len(classes) || rule.RowIndex != index || rule.Ytab != table || rule.Prefix != "Pavx" || rule.ZCase != zcase || rule.ZOffset != zoffset[index] || rule.EncodingOffset != offset[index] || strings.Join(rule.Classes, ",") != classes[index] || !reflect.DeepEqual(rule.OpBytes, opbytes) { + t.Fatal("independent entire encoder row/attributes changed", rule) + } + } + for i, name := range x86AESOracleNames[6:] { + want := 5 + if i == 4 { + want = 1 + } + if i == 5 { + want = 2 + } + if counts[name] != want { + t.Fatal(name, counts, want) + } + } + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil || regexp.MustCompile(`instructions\["V?AES`).Match(aliases) { + t.Fatal("actual Go AES alias inventory changed", err) + } +} + +func TestX86AESFamilyActualGoRawEncoderForms(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames { + widths := []int{16} + if strings.HasPrefix(name, "V") && !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + widths = append(widths, 32, 64) + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, source := range []string{bank + "0", fmt.Sprintf("%d(AX)", width), fmt.Sprintf("-%d(AX)(CX*2)", width), "(FS)", "(GS)"} { + row, _ := x86AESOracleRow(name, source, bank) + t.Run(arch+"/"+row, func(t *testing.T) { + code := x86OrderingHintGoBytes(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+x86DwordProductDirectives(code[:len(code)-1])+"\nRET\n") + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal("actual Go AES physical row missing", row, fmt.Sprintf("%x", code), err) + } + ins := decoded.Instrs[1] + form, known, err := x86AESProperties(ins.Op, ins, arch) + if !known || err != nil || form.byteWidth != width { + t.Fatal("actual Go raw AES identity", ins, err) + } + named := x86YMMReadSource(t, row) + for i := range named.Args { + if named.Args[i].Kind == OpMem && (named.Args[i].Mem.Base == FS || named.Args[i].Mem.Base == GS) { + named.Args[i].Mem.Segment, named.Args[i].Mem.Base = named.Args[i].Mem.Base, "" + } + if named.Args[i].Kind == OpMem && named.Args[i].Mem.Index == "" { + named.Args[i].Mem.Scale = ins.Args[i].Mem.Scale // no-index scale is not an encoded input + } + if named.Args[i].Kind == OpMem && strings.Contains(name, "KEYGEN") { + named.Args[i].Mem.Segment = "" // actual From3 prefix omission, independently bound above + } + } + if ins.Op != named.Op || !reflect.DeepEqual(ins.Args, named.Args) { + t.Fatalf("physical original operand/tuple width changed %s code=%x actual=%#v named=%#v", row, code, ins.Args, named.Args) + } + }) + } + } + } + } + // Independent fixed byte witness: prefix, map, opcode and immediate. The + // source encoder is checked independently of our decoder/typed grammar. + for name, want := range map[string][]byte{"AESENC X0,X1": {0x66, 0x0f, 0x38, 0xdc, 0xc8, 0xc3}, "AESKEYGENASSIST $27,X0,X2": {0x66, 0x0f, 0x3a, 0xdf, 0xd0, 27, 0xc3}, "VAESENC X0,X1,X2": {0xc4, 0xe2, 0x71, 0xdc, 0xd0, 0xc3}, "VAESKEYGENASSIST $-128,X0,X2": {0xc4, 0xe3, 0x79, 0xdf, 0xd0, 128, 0xc3}} { + got := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+name+"\nRET\n") + if !bytes.Equal(got, want) { + t.Fatal("independent fixed AES prefix/opcode bytes", name, got, want) + } + } +} + +func TestX86AESFamilyThreeAPIsFiveTargetNumericResults(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, name := range x86AESOracleNames { + for _, memory := range []bool{false, true} { + first, params, offset := "X0", "", 0 + if memory { + first, params, offset = "a+0(FP)", "a,b uint64", 16 + } + row, destination := x86AESOracleRow(name, first, "X") + if memory && !strings.HasPrefix(name, "V") { + row, destination = x86AESOracleRow(name, "X3", "X") + row = "MOVOU a+0(FP),X3\n" + row + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-%d\nMOVL $1,AX\nMOVL AX,X0\nMOVL AX,X1\n%s\nMOVQ X%d,ret+%d(FP)\nRET\n", count, offset+8, row, destination, offset) + fmt.Fprintf(&declarations, "func F%d(%s) uint64\n", count, params) + count++ + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "aes-results-"+api+".ll", "aes-results-"+api+".o", ir) + }) + } + }) + } +} + +func TestX86AESFamilyPermissionAndEncodingBoundaries(t *testing.T) { + for _, name := range x86AESOracleNames { + for _, source := range []string{"(SP)", "(BP)", "tmp(SP)"} { + row, _ := x86AESOracleRow(name, source, "X") + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-16\n"+row+"\nRET\n", "", true) + ir, err := x86ScalarFPBitViewAPI(t, "TEXT ·F(SB),4,$0-16\n"+row+"\nRET\n", "func F(a,b uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("AES grammar granted unbound native stack storage", row, err) + } + } + } +} diff --git a/amd64_aes_family_fault_runtime_unix_test.go b/amd64_aes_family_fault_runtime_unix_test.go new file mode 100644 index 000000000..b218a5210 --- /dev/null +++ b/amd64_aes_family_fault_runtime_unix_test.go @@ -0,0 +1,143 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86AESFamilyLiveDeadFaultEnvelopes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, originals, rows strings.Builder + declarations.WriteString("package aesfault\n") + count, outcomes := 0, 0 + for _, name := range x86AESOracleNames { + legacy := !strings.HasPrefix(name, "V") + widths := []int{16} + if !legacy && !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + widths = append(widths, 32, 64) + } + for _, width := range widths { + for _, dead := range []bool{false, true} { + fn := fmt.Sprintf("F%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + row, destination := x86AESOracleRow(name, "(DI)", bank) + store := "" + if !dead { + store = fmt.Sprintf("MOVQ out+8(FP),SI\nVMOVUPS %s%d,(SI)\n", bank, destination) + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),DI\nVPXORD Z1,Z1,Z1\n%s\n%sRET\n", fn, row, store) + fmt.Fprintf(&declarations, "func %s(p *[64]byte,out *[64]byte)\n", fn) + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint8_t*);\n", fn) + original := "NULL" + legacyInt := 0 + if legacy { + legacyInt = 1 + original = "Original" + fn + nativeStore := "" + if !dead { + nativeStore = fmt.Sprintf("MOVOU X%d,(SI)\n", destination) + } + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\nPXOR X1,X1\n"+row+"\n"+nativeStore+"RET\n") + var encoded []string + for _, value := range actual { + encoded = append(encoded, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(encoded, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint8_t*);\n", original) + outcomes += 9 + } + outcomes += 9 + fmt.Fprintf(&rows, "{%s,%s,%d,%d},\n", fn, original, width, legacyInt) + } + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + ir := x86NarrowOracleIR(t, source.String(), declarations.String(), execution.triple) + // Rosetta is a differential diagnostic, not the native alignment oracle. + // The Linux AMD64 owner must additionally enforce the architectural fault. + nativeAlignment := runtime.GOOS == "linux" && runtime.GOARCH == "amd64" + reference := fmt.Sprintf("#define NATIVE_ALIGNMENT %d\n", x86FloatAddSubBoolInt(nativeAlignment)) + + fmt.Sprintf(x86AESFaultC, prototypes.String(), originals.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{{"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}} { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "aes-fault", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("aes live/dead kernels %d exact fault outcomes %d tripwires 4 failures 0\n", count, outcomes) + if string(output) != want { + t.Fatal("AES original/native-width/dead-read faults", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } +} + +const x86AESFaultC = ` +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef ALIGNMENT_OBSERVED +#define ALIGNMENT_OBSERVED(kernel,address,fault) ((void)0) +#endif +%s +%s +typedef void(*Program)(const uint8_t*,uint8_t*); +typedef struct {Program translated,original;unsigned width,legacy;} Kernel; +static Kernel kernels[]={%s}; +static int observe(Program p,unsigned width,unsigned address){ + pid_t child=fork();if(child<0)return -1; + if(child==0){ + struct rlimit no_core={0,0};if(setrlimit(RLIMIT_CORE,&no_core))_exit(120); + _Alignas(64) uint8_t data[128],out[64];memset(data,0xa7,sizeof(data));const uint8_t *input=data; + if(address==1)input++; + if(address==2)input=NULL; + if(address>=3){long page=sysconf(_SC_PAGESIZE);if(page<=0)_exit(121); + uint8_t *memory=mmap(NULL,2*(size_t)page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0);if(memory==MAP_FAILED)_exit(122); + memset(memory,0xa7,2*(size_t)page); + if(address<7){if(mprotect(memory+page,(size_t)page,PROT_NONE))_exit(123); + unsigned available[]={0,width-1,width,width+1};input=memory+page-available[address-3]; + }else{if(mprotect(memory,(size_t)page,PROT_NONE))_exit(123);input=memory+page-(address==7?4:12);} + } + p(input,out);_exit(0); + } + int status=0;if(waitpid(child,&status,0)!=child)return -2; + if(WIFEXITED(status))return WEXITSTATUS(status)?-3:0; + if(WIFSIGNALED(status)&&(WTERMSIG(status)==SIGSEGV||WTERMSIG(status)==SIGBUS))return 1; + return -4; +} +static void missing(const uint8_t*p,uint8_t*out){(void)p;(void)out;} +static void short_read(const uint8_t*p,uint8_t*out){out[0]=*(volatile const uint8_t*)p;} +static void over_read(const uint8_t*p,uint8_t*out){out[0]=*((volatile const uint8_t*)p+63);} +static void missing_prefix(const uint8_t*p,uint8_t*out){out[0]=*((volatile const uint8_t*)p+4)^*((volatile const uint8_t*)p+12);} +int main(void){unsigned count=0; + if(observe(missing,16,2)!=0||observe(short_read,64,4)!=0||observe(over_read,16,5)!=1||observe(missing_prefix,16,7)!=0)return 4; + for(unsigned i=0;i=7; + // Intel SDM Table 2-21: legacy SSE memory operands retain #GP for a + // non-16-byte-aligned address; VEX/EVEX do not inherit that exception. + int unaligned=k.legacy&&(a==1||a==6); + if(unaligned&&NATIVE_ALIGNMENT)want=1; + for(int original=1;original>=0;original--){if(original&&!k.original)continue; + int got=observe(original?k.original:k.translated,k.width,a);count++; + // Diagnostic hosts still require exact original/translated agreement. + if(original&&unaligned&&!NATIVE_ALIGNMENT&&(got==0||got==1)){ + want=got;ALIGNMENT_OBSERVED(i,a,got); + } + if(got!=want){fprintf(stderr,"AES kernel%%u width%%u address%%u original%%d got%%d want%%d\n",i,k.width,a,original,got,want);return 1;} + } + } + printf("aes live/dead kernels %%zu exact fault outcomes %%u tripwires 4 failures 0\n",sizeof(kernels)/sizeof(kernels[0]),count);return 0; +} +` diff --git a/amd64_aes_family_fixed_local_test.go b/amd64_aes_family_fixed_local_test.go new file mode 100644 index 000000000..3de68a9b9 --- /dev/null +++ b/amd64_aes_family_fixed_local_test.go @@ -0,0 +1,45 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86AESFamilyActualBoundFixedLocalConsumers(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + var assembly, declarations strings.Builder + count := 0 + for _, name := range x86AESOracleNames { + for _, source := range []string{"16(SP)", "tmp-144(SP)"} { + row, _ := x86AESOracleRow(name, source, "X") + if strings.HasPrefix(name, "V") && !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + row = name + " " + source + ",X0,X0" + } else if strings.Contains(name, "KEYGEN") { + row = name + " $27," + source + ",X0" + } else { + row = name + " " + source + ",X0" + } + body, _, _ := x86PrivateAlignedFrameAuditFixture(t, row) + fn := fmt.Sprintf("F%d", count) + count++ + assembly.WriteString(strings.Replace(body, "·Value", "·"+fn, 1)) + fmt.Fprintf(&declarations, "func %s(a,b,c,d uint64) uint64\n", fn) + } + } + for _, target := range []struct{ name, triple string }{{"linux/amd64", "x86_64-unknown-linux-gnu"}, {"darwin/amd64", "x86_64-apple-darwin"}} { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := x86CMOVFixedLocalFixture(t, assembly.String(), declarations.String(), target.name, target.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "aes-native-local.ll", "aes-native-local.o", ir) + }) + }) + } +} diff --git a/amd64_aes_family_forms.go b/amd64_aes_family_forms.go new file mode 100644 index 000000000..270d722fa --- /dev/null +++ b/amd64_aes_family_forms.go @@ -0,0 +1,236 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86AESOperation uint8 + +const ( + x86AESEncrypt x86AESOperation = iota + x86AESEncryptLast + x86AESDecrypt + x86AESDecryptLast + x86AESInverseMix + x86AESKeyAssist +) + +type x86AESSpec struct { + operation x86AESOperation + vector bool +} + +// All AES optab attributes are explicit. In particular _yvaesdec has EVEX +// full tuples but no mask, broadcast, zeroing, rounding or SAE capability. +var x86AESSpecs = map[Op]x86AESSpec{ + "AESENC": {x86AESEncrypt, false}, "AESENCLAST": {x86AESEncryptLast, false}, + "AESDEC": {x86AESDecrypt, false}, "AESDECLAST": {x86AESDecryptLast, false}, + "AESIMC": {x86AESInverseMix, false}, "AESKEYGENASSIST": {x86AESKeyAssist, false}, + "VAESENC": {x86AESEncrypt, true}, "VAESENCLAST": {x86AESEncryptLast, true}, + "VAESDEC": {x86AESDecrypt, true}, "VAESDECLAST": {x86AESDecryptLast, true}, + "VAESIMC": {x86AESInverseMix, true}, "VAESKEYGENASSIST": {x86AESKeyAssist, true}, +} + +type x86AESForm struct { + spec x86AESSpec + byteWidth, source, state, destination, memory int + sourceOperand Operand // effective Go source role; original instruction is immutable +} + +// This shared grammar supplies operand roles only. No bank initialization, +// native memory, source authentication, frame or pointer authority is created. +func x86AESProperties(op Op, ins Instr, arch string) (x86AESForm, bool, error) { + name, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86AESSpecs[Op(name)] + if !known { + return x86AESForm{}, false, nil + } + f := x86AESForm{spec: spec, byteWidth: 16, source: 0, state: -1, destination: len(ins.Args) - 1, memory: -1} + fail := func() (x86AESForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside complete Go AES grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || dotted || suffix != "" || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + count := 2 + if spec.operation == x86AESKeyAssist { + count, f.source = 3, 1 + if len(ins.Args) != count || ins.Args[0].ImmRaw != "" || ins.Args[0].ImmIsFloat || !amd64UnsignedImmediate(ins.Args[0], 8) && !(spec.vector && amd64VectorByteShiftImmediate(ins.Args[0])) { + return fail() + } + } else if spec.operation < x86AESInverseMix { + f.state = 1 + if spec.vector { + count = 3 + } + } + if len(ins.Args) != count { + return fail() + } + if spec.vector && spec.operation < x86AESInverseMix { + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + } + register := func(a Operand) bool { + if a.Kind != OpReg { + return false + } + if spec.vector && spec.operation < x86AESInverseMix { + return amd64GoPackedVectorMoveRegister(a, f.byteWidth, arch) + } + index, valid := amd64ParseXReg(a.Reg) + return valid && index < 16 && (spec.vector || arch != "386" || index < 8) + } + if !register(ins.Args[f.destination]) || f.state >= 0 && !register(ins.Args[f.state]) { + return fail() + } + source := ins.Args[f.source] + if isAMD64MemoryOperand(source) { + validation := source + if source.Kind == OpMem && arch == "386" && strings.HasSuffix(source.Mem.Sym, "(SB)") { + validation.Mem.Base = "" // actual Go ignores Reg, not Index, for named SB memory + } + if !x86PackedShiftMemoryValid(validation, arch) { + return fail() + } + if source.Kind == OpSym { + if address, err := parseInt(strings.TrimSpace(source.Sym)); err == nil && + address != int64(int32(address)) && !(arch == "386" && address == int64(uint32(address))) { + return fail() + } + } + if source.Kind == OpMem { + if !x86ConditionalMoveOffsetKnown(source.Mem) { + return fail() + } + if err := x86SymbolIndexedMemoryError(source.Mem, arch); err != nil { + return f, true, err + } + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return f, true, fmt.Errorf("%w: AES source has no bound native address-size override", ErrProbeNeedsContext) + } + f.memory = f.source + } else if !register(source) { + return fail() + } + if err := x86Mode32VectorRegisterContext(ins, arch); err != nil { + return f, true, err + } + f.sourceOperand = source + if spec.operation == x86AESKeyAssist && source.Kind == OpMem && !ins.x86Encoded && ins.x86SourceRaw == nil { + // cmd/asm puts this memory in From3; obj/x86 emits segment prefixes + // only from From and To. Its real memory bytes therefore omit this + // annotation. Raw physical instructions retain their actual prefix. + f.sourceOperand.Mem.Segment = "" + if source.Mem.Base == FS || source.Mem.Base == GS { + f.sourceOperand.Mem.Base = "" + } + } + return f, true, nil +} + +func x86TypedAESByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + f, known, err := x86AESProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + e := x86TypedByteEffect{requiresResultProof: true, + operands: []x86TypedByteOperand{{index: f.source, operand: f.sourceOperand, width: int64(f.byteWidth), read: true}}} + if f.state >= 0 && f.state != f.destination { + e.operands = append(e.operands, x86TypedByteOperand{index: f.state, operand: ins.Args[f.state], width: int64(f.byteWidth), read: true}) + } + e.operands = append(e.operands, x86TypedByteOperand{index: f.destination, operand: ins.Args[f.destination], width: int64(f.byteWidth), read: f.state == f.destination, write: true}) + return e, true, nil +} + +// Numerical dependencies are independent of the complete physical memory +// envelope. A SubWord/RotWord observes only source dwords 1 and 3; an AES round +// never has a zero idiom when the two sources alias. Wider lanes are 128-bit. +func x86AESDefinedBytes(f x86AESForm, source, state uint64) uint64 { + known := uint64(0) + for lane := 0; lane < f.byteWidth; lane += 16 { + for out := 0; out < 16; out++ { + var sourceMask, stateMask uint64 + switch f.spec.operation { + case x86AESKeyAssist: + at := out/8*8 + 4 + out%4 + if out%8 >= 4 { + at = out/8*8 + 4 + (out+1)%4 + } + sourceMask = uint64(1) << uint(lane+at) + case x86AESInverseMix: + sourceMask = uint64(15) << uint(lane+out/4*4) + default: + sourceMask = uint64(1) << uint(lane+out) + column, row := out/4, out%4 + first, last := 0, 4 + if f.spec.operation == x86AESEncryptLast || f.spec.operation == x86AESDecryptLast { + first, last = row, row+1 + } + for r := first; r < last; r++ { + shift := r + if f.spec.operation == x86AESDecrypt || f.spec.operation == x86AESDecryptLast { + shift = 4 - r + } + stateMask |= uint64(1) << uint(lane+((column+shift)%4)*4+r) + } + } + if source&sourceMask == sourceMask && state&stateMask == stateMask { + known |= uint64(1) << uint(lane+out) + } + } + } + return known +} + +func x86ResultAESStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + return x86ResultAESStepWithRead(ins, state, owned, arch, false) +} + +func x86ResultAESStepWithRead(ins Instr, state *x86ResultState, owned []bool, arch string, native bool) (bool, bool) { + f, known, err := x86AESProperties(ins.Op, ins, arch) + if !known { + return false, false + } + if err != nil { + return false, true + } + read := func(index int) (uint64, bool) { + a := ins.Args[index] + if index == f.source { + a = f.sourceOperand + } + if a.Kind == OpReg { + i, valid := amd64VectorRegisterIndex(a.Reg, f.byteWidth) + if !valid || i >= len(state.xmm) { + return 0, false + } + return state.vectorBytes(i), true + } + return x86CallerVectorByteMask(int64(f.byteWidth)), + x86ResultReadWithNativeContinuation(ins, a, int64(f.byteWidth), state, owned, arch, native && index == f.source) + } + source, valid := read(f.source) + if !valid { + return false, true + } + input := uint64(0) + if f.state >= 0 { + input, valid = read(f.state) + if !valid { + return false, true + } + } + destination, _ := amd64VectorRegisterIndex(ins.Args[f.destination].Reg, f.byteWidth) + if destination >= len(state.xmm) { + return false, true + } + defined := x86AESDefinedBytes(f, source, input) + // Preserve the existing nontrapping register NUM domain: an accepted + // transfer may leave result bytes unknown, never initialized to alloca + // zeroes. Every later result, store, address or control use still consumes + // its actual known bytes. Native memory retains its full read above. + state.writeVector(destination, f.byteWidth, defined, !f.spec.vector) + return true, true +} diff --git a/amd64_aes_family_runtime_test.go b/amd64_aes_family_runtime_test.go new file mode 100644 index 000000000..b332c274f --- /dev/null +++ b/amd64_aes_family_runtime_test.go @@ -0,0 +1,241 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// FIPS-197 byte arithmetic, independent of LLVM intrinsics, the source grammar +// and the production defined-byte model. Compute the S-box from GF(2^8), not +// from a table shared with any emitter. +const x86AESIndependentC = ` +#include +#include +#include +#include +static uint8_t mul(uint8_t a,uint8_t b){uint8_t v=0;for(int i=0;i<8;i++){if(b&1)v^=a; a=(uint8_t)((a<<1)^((a&128)?0x1b:0));b>>=1;}return v;} +static uint8_t rol(uint8_t a,int n){return (uint8_t)((a<>(8-n)));} +static uint8_t box(uint8_t a){uint8_t p=1,b=a;int n=254;if(!a)p=0;else while(n){if(n&1)p=mul(p,b);b=mul(b,b);n>>=1;}return p^rol(p,1)^rol(p,2)^rol(p,3)^rol(p,4)^0x63;} +static uint8_t inverse(uint8_t a){for(int i=0;i<256;i++)if(box((uint8_t)i)==a)return (uint8_t)i;return 0;} +static void mix(uint8_t *out,const uint8_t *in,int inv){for(int c=0;c<4;c++)for(int r=0;r<4;r++){ + const uint8_t forward[4]={2,3,1,1},backward[4]={14,11,13,9};out[c*4+r]=0; + for(int k=0;k<4;k++)out[c*4+r]^=mul(in[c*4+k],inv?backward[(k-r+4)%4]:forward[(k-r+4)%4]);}} +static void aes(uint8_t *out,const uint8_t *state,const uint8_t *key,int op,int imm){ + uint8_t shifted[16],temp[16]; + if(op==4){mix(out,key,1);return;} + if(op==5){for(int h=0;h<2;h++)for(int b=0;b<4;b++){out[h*8+b]=box(key[h*8+4+b]);out[h*8+4+b]=box(key[h*8+4+(b+1)%4])^(b==0?imm:0);}return;} + for(int c=0;c<4;c++)for(int r=0;r<4;r++){int source=((c+(op>=2?4-r:r))%4)*4+r;shifted[c*4+r]=op>=2?inverse(state[source]):box(state[source]);} + if(op==0||op==2)mix(temp,shifted,op==2);else memcpy(temp,shifted,16); + for(int i=0;i<16;i++)out[i]=temp[i]^key[i]; +} +typedef void(*Program)(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t); +struct Aligned {_Alignas(64) uint8_t before[64],value[80],after[64];}; +static int check(Program p,const char *name,int width,int op,int legacy,int memory,int alias,int imm){ + for(int sample=0;sample<37;sample++)for(int profile=0;profile<2;profile++){ + struct Aligned a,b,old,out;memset(&a,0xa7,sizeof(a));memset(&b,0xb9,sizeof(b));memset(&old,0x6d,sizeof(old));memset(&out,0xc3,sizeof(out)); + for(int i=0;i<64;i++){a.value[i]=(uint8_t)(sample*19+i*37);b.value[i]=(uint8_t)(sample*71+i*11);old.value[i]=(uint8_t)(sample*97+i*3);} + struct Aligned oa=a,ob=b,oo=old;uint64_t seed=profile?0x8d5:0;uint32_t csr=0x1f80|((sample%4)<<13)|(sample%64),saved=_mm_getcsr();_mm_setcsr(csr); + p(out.value,a.value,b.value,old.value,seed,csr);uint32_t after=_mm_getcsr();_mm_setcsr(saved); + uint64_t flags;uint32_t reported;memcpy(&flags,out.value+64,8);memcpy(&reported,out.value+72,4); + if((flags&0x8d5)!=seed||reported!=csr||after!=csr||memcmp(&a,&oa,sizeof(a))||memcmp(&b,&ob,sizeof(b))||memcmp(&old,&oo,sizeof(old)))return 1; + uint8_t expected[64];const uint8_t *retained=alias==0?a.value:alias==1?b.value:old.value; + memcpy(expected,legacy?retained:(const uint8_t[64]){0},64); + for(int lane=0;lane= 4 { + row = strings.TrimSuffix(row, "X2") + fmt.Sprintf("X%d", alias) + destination = alias + } + fn := fmt.Sprintf("NativeAES%d", count) + count++ + // SysV DI/SI/DX/CX/R8/R9 provide out/a/b/old/flags/csr. + body := fmt.Sprintf("TEXT actual(SB),4,$0-0\nMOVOU (SI),X0\nMOVOU (DX),X1\nMOVOU (CX),X2\nMOVL R9,76(DI)\nLDMXCSR 76(DI)\nPUSHQ R8\nPOPFQ\n%s\nMOVOU X%d,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,64(DI)\nSTMXCSR 72(DI)\nRET\n", row, destination) + code := x86OrderingHintGoBytes(t, "amd64", body) + var encoded []string + for _, b := range code { + encoded = append(encoded, fmt.Sprintf("0x%02x", b)) + } + symbol := fn + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + original.WriteString(x86OriginalSysVDeclaration("void", fn, "uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t")) + fmt.Fprintf(&original, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(encoded, ",")+"\n") + fmt.Fprintf(&checks, "if(nativecheck(%s,%q,%d))return 1;\n", fn, row, op) + } + } + } + reference := x86AESIndependentC + original.String() + x86AESNativeCheckC + "int main(void){\n" + checks.String() + fmt.Sprintf("printf(\"original Go AES X128 kernels %d samples %d full-byte failures 0\\n\");return 0;}\n", count, count*37*2) + output := execution.compileAndRunLLVM(t, "aes-native-go-bytes", "target triple = \""+execution.triple+"\"\n", reference, "-O2") + want := fmt.Sprintf("original Go AES X128 kernels %d samples %d full-byte failures 0\n", count, count*37*2) + if string(output) != want { + t.Fatal("actual Go full-X128 source-byte oracle", string(output)) + } + t.Log(strings.TrimSpace(string(output))) +} + +var x86AESNativeCheckC = x86OriginalSysVPointer("void", "OriginalProgram", + "uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t") + ` +static int nativecheck(OriginalProgram p,const char *name,int op){ + for(int sample=0;sample<37;sample++)for(int profile=0;profile<2;profile++){ + struct Aligned a,b,old,out;memset(&a,0xa7,sizeof(a));memset(&b,0xb9,sizeof(b));memset(&old,0x6d,sizeof(old));memset(&out,0xc3,sizeof(out)); + for(int i=0;i<64;i++){a.value[i]=(uint8_t)(sample*19+i*37);b.value[i]=(uint8_t)(sample*71+i*11);old.value[i]=(uint8_t)(sample*97+i*3);} + struct Aligned oa=a,ob=b,oo=old;uint64_t seed=profile?0x8d5:0;uint32_t csr=0x1f80|((sample%4)<<13)|(sample%64),saved=_mm_getcsr();_mm_setcsr(csr); + p(out.value,a.value,b.value,old.value,seed,csr);uint32_t after=_mm_getcsr();_mm_setcsr(saved); + uint64_t flags;uint32_t reported,scratch;memcpy(&flags,out.value+64,8);memcpy(&reported,out.value+72,4);memcpy(&scratch,out.value+76,4); + uint8_t expected[16];aes(expected,b.value,a.value,op,27); + if(memcmp(out.value,expected,16)||(flags&0x8d5)!=seed||reported!=csr||scratch!=csr||after!=csr||memcmp(&a,&oa,sizeof(a))||memcmp(&b,&ob,sizeof(b))||memcmp(&old,&oo,sizeof(old))){fprintf(stderr,"original %s sample%d full bytes/flags/CSR/input mismatch\n",name,sample);return 1;} + for(int i=16;i<64;i++)if(out.value[i]!=0xc3)return 2; + for(int i=0;i<64;i++)if(out.before[i]!=0xc3||out.after[i]!=0xc3)return 3; + }return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86AESFamilyLogicalFloatBitPayloads(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, reference, checks strings.Builder + declarations.WriteString("package aesbits\n") + reference.WriteString(x86AESIndependentC) + count := 0 + for op, name := range x86AESOracleNames { + for _, bits := range []int{32, 64} { + fn := fmt.Sprintf("FloatAES%d", count) + count++ + row, destination := x86AESOracleRow(name, "a+0(FP)", "X") + if !strings.HasPrefix(name, "V") { + // This is a logical bit-payload oracle, not native FP backing. + // Use Go's unaligned move before the legacy register operation; + // the separate memory oracle checks its alignment exceptions. + row, destination = x86AESOracleRow(name, "X3", "X") + row = "MOVOU a+0(FP),X3\n" + row + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-24\nVPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\n%s\nMOVQ out+16(FP),AX\nVMOVUPS Z%d,(AX)\nRET\n", fn, row, destination) + if bits == 32 { + checks.WriteString("for(unsigned j=0;j<4;j++){uint32_t v=payloads32[(sample+j)%8];memcpy(key+j*4,&v,4);}\n") + fmt.Fprintf(&declarations, "func %s(a,b,c,d float32,out *[64]byte)\n", fn) + fmt.Fprintf(&reference, "extern void %s(float,float,float,float,uint8_t*);\n", fn) + fmt.Fprintf(&checks, "{float f[4];memcpy(f,key,16);%s(f[0],f[1],f[2],f[3],out.value);}\n", fn) + } else { + checks.WriteString("for(unsigned j=0;j<2;j++){uint64_t v=payloads64[(sample+j)%8];memcpy(key+j*8,&v,8);}\n") + fmt.Fprintf(&declarations, "func %s(a,b float64,out *[64]byte)\n", fn) + fmt.Fprintf(&reference, "extern void %s(double,double,uint8_t*);\n", fn) + fmt.Fprintf(&checks, "{double f[2];memcpy(f,key,16);%s(f[0],f[1],out.value);}\n", fn) + } + fmt.Fprintf(&checks, "aes(expected,(const uint8_t[16]){0},key,%d,27);\n", op%6) + fmt.Fprintf(&checks, `if (memcmp(out.value,expected,64) || _mm_getcsr()!=csr) { + fprintf(stderr,"%s sample%%u csr%%x want%%x\n",sample,_mm_getcsr(),csr); + for(int j=0;j<64;j++) if(out.value[j]!=expected[j]) + fprintf(stderr,"byte%%d got%%x want%%x\n",j,out.value[j],expected[j]); + return 1; +} +`, fn) + checks.WriteString(`for(int j=0;j<64;j++) if(out.before[j]!=0xc3 || out.after[j]!=0xc3) return 2; +for(int j=64;j<80;j++) if(out.value[j]!=0xc3) return 3; +memset(&out,0xc3,sizeof(out)); +`) + } + } + reference.WriteString(` +int main(void){ + const uint32_t payloads32[]={0,0x80000000,0x7fc00013,0x7f800001,0xff800001,0x7f800000,0xff800000,1}; + const uint64_t payloads64[]={0,UINT64_C(0x8000000000000000),UINT64_C(0x7ff8000000000013),UINT64_C(0x7ff0000000000001),UINT64_C(0xfff0000000000001),UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000),1}; + for(unsigned sample=0;sample<8;sample++){ + uint8_t key[16],expected[64]={0};struct Aligned out;memset(&out,0xc3,sizeof(out)); + unsigned csr=0x1f80|((sample%4)<<13)|(sample%64),saved=_mm_getcsr();_mm_setcsr(csr); +` + checks.String() + `_mm_setcsr(saved); + }return 0; +} +`) + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + ir := x86NarrowOracleIR(t, source.String(), declarations.String(), execution.triple) + for _, axis := range []struct{ name, ir, backend string }{{"backend_O0", ir, "-O0"}, {"backend_O2", ir, "-O2"}, {"IR_O2/backend_O2", x86DwordProductIROptimize(t, ir), "-O2"}} { + t.Run(axis.name, func(t *testing.T) { + execution.compileAndRunLLVM(t, "aes-float-bits", axis.ir, reference.String(), axis.backend) + }) + } + t.Logf("%d complete F32/F64 logical bit-view kernels ×8 NaN/zero/infinity/denormal payloads on three axes; full64 bytes/MXCSR/canaries, no numeric float conversion or native-FP certificate", count) +} diff --git a/amd64_aes_family_sdk_test.go b/amd64_aes_family_sdk_test.go new file mode 100644 index 000000000..e71928f37 --- /dev/null +++ b/amd64_aes_family_sdk_test.go @@ -0,0 +1,99 @@ +package plan9asm + +import ( + "crypto/sha256" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// This is a tiny actual-SDK object/source fixture, not a local stdlib/corpus +// replay. Original AES instructions and selected function bodies are retained. +// Dynamic aeskeysched contents alone are not extent, pointer or initialization +// authority. A later global producer must authenticate its own source/layout. +func TestX86AESFamilyOriginalSDKMemhash386GlobalBoundary(t *testing.T) { + relative := "src/internal/runtime/maps/memhash_386.s" + names := []string{"·memHash32AES", "·memHash64AES"} + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), relative)) + if errors.Is(err, os.ErrNotExist) { + relative = "src/runtime/asm_386.s" + names = []string{"runtime·memhash32", "runtime·memhash64"} + data, err = os.ReadFile(filepath.Join(runtime.GOROOT(), relative)) + } + if err != nil { + t.Fatal(err) + } + headerPath := filepath.Join(runtime.GOROOT(), "pkg/include/textflag.h") + header, err := os.ReadFile(headerPath) + if err != nil { + t.Fatal(err) + } + for _, name := range names { + t.Run(name, func(t *testing.T) { + start := strings.Index(string(data), "TEXT "+name+"(SB)") + if start < 0 { + t.Fatal("actual SDK selected function disappeared", relative, name) + } + body := string(data[start:]) + if next := strings.Index(body[1:], "\nTEXT "); next >= 0 { + body = body[:next+1] + } + original := "#include \"textflag.h\"\n" + body + dir := t.TempDir() + input, object := filepath.Join(dir, "actual-sdk.s"), filepath.Join(dir, "actual-sdk.o") + if err := os.WriteFile(input, []byte(original), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "asm", "-I", filepath.Dir(headerPath), "-p", "actual-sdk", "-o", object, input) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=386") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), true, "386"); failure != nil { + t.Fatal("unchanged SDK function original object", failure, string(output)) + } + physical, err := os.ReadFile(object) + if err != nil { + t.Fatal(err) + } + processed, err := PreprocessAssemblySource(original, AssemblyPreprocessOptions{FileName: relative, ReadInclude: func(parent, name string) (string, []byte, error) { + if name != "textflag.h" { + return "", nil, fmt.Errorf("unselected header %s", name) + } + return "pkg/include/textflag.h", header, nil + }}) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, processed) // shared x86 parser; the typed consumer is explicitly mode386 + if err != nil || len(file.Funcs) != 1 { + t.Fatal("actual source/include parser", err) + } + aes := 0 + for _, ins := range file.Funcs[0].Instrs { + if ins.Op != "AESENC" { + continue + } + aes++ + form, known, err := x86AESProperties(ins.Op, ins, "386") + if !known || err != nil || form.byteWidth != 16 || form.memory != 0 { + t.Fatal("unchanged actual SDK AES source must have registered whole-family grammar", ins, err) + } + state := x86ResultState{} + state.xmm[0] = 0xffff + if ok, known := x86ResultAESStep(ins, &state, nil, "386"); !known || ok { + t.Fatal("dynamic SB numerical values borrowed unproved initialized/extent authority", ins) + } + } + if aes != 3 { + t.Fatal("actual SDK AES inventory changed", aes) + } + t.Logf("SDK=%s full-sourceSHA256=%x selected-originalSHA256=%x headerSHA256=%x original-objectSHA256=%x bytes=%d exactAES=%d; global producer absent, not a corpus/version pass", relative, sha256.Sum256(data), sha256.Sum256([]byte(original)), sha256.Sum256(header), sha256.Sum256(physical), len(physical), aes) + }) + } +} diff --git a/amd64_aes_scalar_integration_runtime_test.go b/amd64_aes_scalar_integration_runtime_test.go new file mode 100644 index 000000000..746118516 --- /dev/null +++ b/amd64_aes_scalar_integration_runtime_test.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Execute the exact original FP source, not a GP-source substitute. The tiny +// test-only native bridge owns an ABI0 argument/result area for those actual +// Go object bytes and retains the four bytes immediately after the result. +// Neither this bridge nor its initialized bytes grant production native-FP, +// fixed-local, caller or pointer permission. +func TestCrossLinuxRuntimeMatrixX86AESFamilyScalarInsertResult(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + const source = "TEXT ·view(SB),4,$0-12\nMOVL k+0(FP),X0\nPINSRD $1,k+0(FP),X0\nAESENC X0,X0\nMOVL X0,ret+8(FP)\nRET\n" + code := x86OrderingHintGoBytes(t, "amd64", source) + var encoded []string + for _, value := range code { + encoded = append(encoded, fmt.Sprintf("0x%02x", value)) + } + original, bridge := "OriginalGoAESScalar", "OriginalGoAESScalarBridge" + if strings.Contains(execution.triple, "apple") { + original, bridge = "_"+original, "_"+bridge + } + assembly := ".text\n.globl " + original + "\n" + original + ":\n.byte " + strings.Join(encoded, ",") + + "\n.globl " + bridge + "\n" + bridge + ":\nsubq $24,%rsp\nmovl %edi,0(%rsp)\nmovl %esi,4(%rsp)\nmovl $0xc3c3c3c3,8(%rsp)\nmovl $0x6d6d6d6d,12(%rsp)\ncall " + original + "\nmovq 8(%rsp),%rax\naddq $24,%rsp\nret\n" + // This reference uses independent FIPS-197 arithmetic. It does not reuse + // the production grammar, LLVM AES intrinsic or byte-definedness model. + reference := x86AESIndependentC + fmt.Sprintf("\n__asm__(%q);\n", assembly) + + x86OriginalSysVDeclaration("uint64_t", "OriginalGoAESScalarBridge", "uint32_t,uint32_t") + ` +extern uint32_t view(uint32_t,uint32_t); +int main(void){ + for(unsigned sample=0;sample<64;sample++){ + const uint32_t special[4]={0,0xffffffff,0x80000000,0x01234567}; + uint32_t k=sample<4?special[sample]:sample*0x9137a5c3u+0x87654321u,h=k^0x6b5917d3u; + uint8_t input[16]={0},expected[16];uint32_t want; + memcpy(input,&k,4);memcpy(input+4,&k,4);aes(expected,input,input,0,0);memcpy(&want,expected,4); + uint64_t original=OriginalGoAESScalarBridge(k,h);uint32_t translated=view(k,h); + if((uint32_t)original!=want||(original>>32)!=0x6d6d6d6du||translated!=want){ + fprintf(stderr,"mixed AES scalar sample%u original%llx translated%x expected%x\n",sample,(unsigned long long)original,translated,want);return 1; + } + } + puts("mixed AES scalar exact Go bytes/FIPS/translated samples 64 result-canary failures 0");return 0; +} +` + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(k,h uint32) uint32\n", "amd64", execution.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, axis := range []struct{ name, level string }{{"backend_O0", "-O0"}, {"backend_O2", "-O2"}, {"IR_O2/backend_O2", "-O2"}} { + t.Run(api+"/"+axis.name, func(t *testing.T) { + optimized := ir + if strings.HasPrefix(axis.name, "IR_O2/") { + optimized = x86DwordProductIROptimize(t, ir) + } + output := execution.compileAndRunLLVM(t, "aes-scalar-result", optimized, reference, axis.level) + if string(output) != "mixed AES scalar exact Go bytes/FIPS/translated samples 64 result-canary failures 0\n" { + t.Fatal("unbound mixed-source runtime result", string(output)) + } + }) + } + } + t.Logf("exact original Go FP source bytes %x; 3 translation APIs x3 explicit optimization axes x64 independent result/canary samples (host transport, not native Linux)", code) +} diff --git a/amd64_ah_flags.go b/amd64_ah_flags.go new file mode 100644 index 000000000..f89640a83 --- /dev/null +++ b/amd64_ah_flags.go @@ -0,0 +1,50 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86AHFlagsSpec struct { + toAH bool +} + +// The complete ynone/Px family in both Go x86 modes. AH is an implicit byte +// operand; OF, DF and all other FLAGS are outside this five-bit transfer. +var x86AHFlagsSpecs = map[Op]x86AHFlagsSpec{ + "LAHF": {toAH: true}, "SAHF": {}, +} + +const x86AHFlagsMask = x86ByteFlagCF | x86ByteFlagPF | x86ByteFlagAF | x86ByteFlagZF | x86ByteFlagSF + +func x86AHFlagsProperties(op Op, ins Instr, arch string) (x86AHFlagsSpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86AHFlagsSpecs[Op(name)] + if !known { + return spec, false, nil + } + if len(ins.Args) != 0 { + return spec, true, fmt.Errorf("%s %s takes no operands: %q", arch, op, ins.Raw) + } + if arch != "386" && arch != "amd64" || dotted || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return spec, true, fmt.Errorf("%s %s outside Go AH/FLAGS ynone grammar: %q", arch, op, ins.Raw) + } + return spec, true, nil +} + +func x86TypedAHFlagsEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + spec, known, err := x86AHFlagsProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: -1, operand: Operand{Kind: OpReg, Reg: AH}, width: 1, read: !spec.toAH, write: spec.toAH}, + }} + if spec.toAH { + effect.flagsRead = x86AHFlagsMask + } else { + effect.flagsDefined, effect.flagsKilled = x86AHFlagsMask, x86AHFlagsMask + } + return effect, true, nil +} diff --git a/amd64_andn_feature_identity_test.go b/amd64_andn_feature_identity_test.go new file mode 100644 index 000000000..062e7d32c --- /dev/null +++ b/amd64_andn_feature_identity_test.go @@ -0,0 +1,63 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +// Go's scalar BMI rows are ANDNL and ANDNQ. The 32 packed logical names +// are a separate family even when their source mnemonic starts with ANDN. +func TestX86ANDNFeatureIdentityDoesNotBorrowScalarBMI(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86ScalarFPLogicalOpcodes { + t.Run(arch+"/"+op, func(t *testing.T) { + row, _ := x86ScalarFPLogicalRow(op) + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + features := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, file.Funcs[0]) + for _, feature := range strings.Split(features, ",") { + if feature == "+bmi" || feature == "+bmi2" { + t.Errorf("packed opcode %s borrowed scalar ISA: %s", op, features) + } + } + }) + } + } +} + +func TestX86ANDNNativeMemoryKeepsItsOwnISAAndScalarControls(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, tc := range []struct{ row, want string }{ + {"ANDNPS (AX),X1", "+sse"}, + {"ANDNPD (AX),X1", "+sse2"}, + {"ANDNL AX,BX,CX", "+bmi"}, + } { + t.Run(arch+"/"+tc.row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + tc.row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, file.Funcs[0]); got != tc.want { + t.Errorf("native source ISA=%q want=%q", got, tc.want) + } + }) + } + } + // The Q row is independently retained, not accidentally dropped while + // removing packed names from the former broad prefix heuristic. + source := "TEXT f(SB),4,$0-0\nANDNQ AX,BX,CX\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", file.Funcs[0]); got != "+bmi" { + t.Fatalf("scalar Q BMI row lost its ISA: %q", got) + } +} diff --git a/amd64_arithmetic_shift_go_rules_test.go b/amd64_arithmetic_shift_go_rules_test.go new file mode 100644 index 000000000..e1c9bb11a --- /dev/null +++ b/amd64_arithmetic_shift_go_rules_test.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// Independent complete ordered rows, including zero-offset aliases. Encoding +// attributes are opcode-specific even when operand tables are shared. +func TestX86ArithmeticShiftCompleteGoEncoderRules(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + for _, spec := range []struct { + name, table, mapName, w, immediate, opcode, broadcast string + uniform, vex bool + }{ + {"VPSRAW", "_yvpslld", "vex0F", "vexW0", "0x71", "0xE1", "", true, true}, + {"VPSRAD", "_yvpslld", "vex0F", "vexW0", "0x72", "0xE2", "evexBcstN4", true, true}, + {"VPSRAQ", "_yvpsraq", "vex0F", "vexW1", "0x72", "0xE2", "evexBcstN8", true, false}, + {"VPSRAVD", "_yvandnpd", "vex0F38", "vexW0", "", "0x46", "evexBcstN4", false, true}, + {"VPSRAVQ", "_yvblendmpd", "vex0F38", "vexW1", "", "0x46", "evexBcstN8", false, false}, + {"VPSRAVW", "_yvblendmpd", "vex0F38", "vexW1", "", "0x11", "", false, false}, + } { + var expected []goasmrules.X86AVXRule + var bytes []string + offset := 0 + add := func(zcase string, advance int, classes string) { + expected = append(expected, goasmrules.X86AVXRule{ + Opcode: spec.name, Ytab: spec.table, Prefix: "Pavx", RowIndex: len(expected), + ZCase: zcase, ZOffset: advance, EncodingOffset: offset, Classes: strings.Split(classes, ","), + }) + offset += advance + } + if spec.vex { + if spec.uniform { + for _, bank := range []string{"Yxr", "Yyr"} { + add("Zvex_i_rm_vo", 0, "Yu8,"+bank+","+bank) + add("Zvex_i_rm_vo", 3, "Yi8,"+bank+","+bank) + } + } + for index, bank := range []string{"Yxr", "Yyr"} { + source := []string{"Yxm", "Yym"}[index] + if spec.uniform { + source = "Yxm" + } + add("Zvex_rm_v_r", 2, source+","+bank+","+bank) + } + for pass := 0; pass < 2; pass++ { + if pass == 0 && !spec.uniform { + continue + } + for _, width := range []int{128, 256} { + bytes = append(bytes, fmt.Sprintf("avxEscape | vex%d | vex66 | %s | %s", width, spec.mapName, spec.w)) + if pass == 0 { + bytes = append(bytes, spec.immediate, "04") + } else { + bytes = append(bytes, spec.opcode) + } + } + } + } + for pass := 0; pass < 2; pass++ { + if pass == 0 && !spec.uniform { + continue + } + for index, bank := range []string{"YxrEvex", "YyrEvex", "Yzr"} { + source := []string{"YxmEvex", "YymEvex", "Yzm"}[index] + if spec.uniform && pass == 1 { + source = "YxmEvex" + } + classes, zcase, masked, advance := source+","+bank+","+bank, "Zevex_rm_v_r", "Zevex_rm_v_k_r", 3 + if pass == 0 { + classes, zcase, masked, advance = "Yu8,"+source+","+bank, "Zevex_i_rm_vo", "Zevex_i_rm_k_vo", 4 + } + add(zcase, 0, classes) + parts := strings.Split(classes, ",") + add(masked, advance, strings.Join(parts[:len(parts)-1], ",")+",Yknot0,"+bank) + width := []int{128, 256, 512}[index] + prefix := strings.ReplaceAll(fmt.Sprintf("avxEscape | evex%d | evex66 | %s | %s", width, spec.mapName, spec.w), "vex0F", "evex0F") + prefix = strings.ReplaceAll(prefix, "vexW", "evexW") + tuple := 16 + if !spec.uniform || pass == 0 { + tuple = width / 8 + } + attributes := fmt.Sprintf("evexN%d", tuple) + if (!spec.uniform || pass == 0) && spec.broadcast != "" { + attributes += " | " + spec.broadcast + } + bytes = append(bytes, prefix, attributes+" | evexZeroingEnabled") + if pass == 0 { + bytes = append(bytes, spec.immediate, "04") + } else { + bytes = append(bytes, spec.opcode) + } + } + } + var actual []goasmrules.X86AVXRule + for _, rule := range rules { + if rule.Opcode == spec.name { + actual = append(actual, rule) + } + } + for index := range expected { + expected[index].OpBytes = bytes + } + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("complete ordered arithmetic encoder rules changed: %s\ngot %+v\nwant %+v", spec.name, actual, expected) + } + } +} + +func TestX86PackedShiftEVEXUnsignedImmediateGrammar(t *testing.T) { + for _, name := range []string{"VPSLLD", "VPSLLQ", "VPSRLD", "VPSRLQ", "VPSRAD", "VPSRAQ"} { + for _, count := range []int{-128, -1} { + row := fmt.Sprintf("%s.BCST $%d,(AX),X2", name, count) + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86PackedShiftProperties(ins.Op, ins, "amd64"); !known || err == nil { + t.Errorf("EVEX-only Yu8 broadcast accepted signed count: %s: %v", row, err) + } + } + } +} diff --git a/amd64_arithmetic_shift_result_test.go b/amd64_arithmetic_shift_result_test.go new file mode 100644 index 000000000..e87322775 --- /dev/null +++ b/amd64_arithmetic_shift_result_test.go @@ -0,0 +1,215 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "testing" +) + +// Independent of the production shift directory. Uniform counts select their +// low qword; variable counts have one complete unsigned integer per data lane. +func x86ArithmeticShiftTestNames() []string { + return []string{"PSRAW", "PSRAL", "VPSRAW", "VPSRAD", "VPSRAQ", "VPSRAVW", "VPSRAVD", "VPSRAVQ"} +} + +func x86ArithmeticShiftTestBits(name string) int { + switch name[len(name)-1] { + case 'W': + return 16 + case 'Q': + return 64 + default: + return 32 + } +} + +func TestX86ArithmeticShiftSharedTypedResultGrammar(t *testing.T) { + for _, name := range x86ArithmeticShiftTestNames() { + for _, bank := range []string{"X", "Y", "Z"} { + legacy := name[0] != 'V' + if legacy && bank != "X" { + continue + } + perLane := len(name) > 5 && name[5] == 'V' + count := "X0" + if perLane { + count = bank + "0" + } + row := name + " " + count + "," + bank + "1," + bank + "2" + if legacy { + row = name + " X0,X1" + } + ins := x86ScalarLaneInstruction(t, row) + for _, arch := range []string{"386", "amd64"} { + form, known, err := x86PackedShiftProperties(ins.Op, ins, arch) + if !known || err != nil || form.laneBits != x86ArithmeticShiftTestBits(name) { + t.Errorf("complete arithmetic family missing from shared result grammar: %s/%s: %v", arch, row, err) + continue + } + state := x86ResultState{flags: x86ByteFlagsAll} + state.writeVector(0, 64, ^uint64(0), false) + state.writeVector(1, 64, ^uint64(0), false) + if accepted, known := x86ResultPackedShiftStep(ins, &state, nil, arch); !known || !accepted { + t.Errorf("initialized arithmetic result rejected: %s/%s", arch, row) + } + } + } + } +} + +// Enumerate individual bits rather than borrowing the interval implementation. +// Oversized arithmetic counts select the sign bit, never a constant zero. +func TestX86ArithmeticShiftEverySignBitDependency(t *testing.T) { + for _, name := range x86ArithmeticShiftTestNames() { + legacy := name[0] != 'V' + perLane := len(name) > 5 && name[5] == 'V' + for _, width := range []int{16, 32, 64} { + if legacy && width != 16 { + continue + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + counts := []int{-1} + if !perLane { + counts = append(counts, 0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 127, 255) + } + for _, count := range counts { + if legacy && count == 255 { + continue + } + countSource := "X0" + if count >= 0 { + countSource = fmt.Sprintf("$%d", count) + } else if perLane { + countSource = bank + "0" + } + row := name + " " + countSource + "," + bank + "1," + bank + "2" + if legacy { + row = name + " " + countSource + ",X1" + } + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !known || err != nil { + t.Errorf("arithmetic dependency grammar missing: %s: %v", row, err) + continue + } + bits := x86ArithmeticShiftTestBits(name) + for output := 0; output < width; output++ { + var data, counter uint64 + for bit := output * 8; bit < output*8+8; bit++ { + first, end := count, count+1 + if count < 0 { + first, end = 0, bits+1 + } + for shift := first; shift < end; shift++ { + from := bit%bits + shift + if from >= bits { + from = bits - 1 + } + data |= uint64(1) << uint((bit/bits*bits+from)/8) + } + } + if count < 0 { + counter = 255 + if perLane { + counter = x86CallerVectorByteMask(int64(bits/8)) << uint(output/(bits/8)*(bits/8)) + } + } + gotData, gotCount := x86PackedShiftByteInputs(form, output) + if gotData != data || gotCount != counter { + t.Fatalf("%s byte%d: dependencies %x/%x want %x/%x", row, output, gotData, gotCount, data, counter) + } + } + } + } + } +} + +func TestX86ArithmeticShiftWholeSourceInputBoundaries(t *testing.T) { + for _, name := range x86ArithmeticShiftTestNames() { + row, memory, destination := name+" X0,X1", name+" (AX),X1", "X1" + if name[0] == 'V' { + row, memory, destination = name+" X0,X1,X2", name+" (AX),X1,X2", "X2" + } + for _, api := range []string{"text", "module", "Go"} { + // Partial numerical knowledge is intentional: an unobserved + // pure register result need not initialize every output byte. + // Ordinary ABI0 codegen may select allocated vector scratch. + // Native addresses and masks retain independent input checks. + source := "TEXT ·view(SB),4,$0-4\n" + row + "\nPEXTRD $0," + destination + ",AX\nMOVL AX,ret+0(FP)\nRET\n" + ir, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32", "amd64", "x86_64-unknown-linux-gnu", api) + x86VectorScratchCodegenControl(t, source, "func view() uint32", "amd64", "x86_64-unknown-linux-gnu", ir, err) + ir, err = x86ScalarFPBitViewAPI(t, "TEXT ·view(SB),4,$0-0\n"+memory+"\nRET\n", + "func view()", "amd64", "x86_64-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("unbound dead native count accepted: %s/%s: %v", memory, api, err) + } + if name[0] == 'V' { + masked := name + ".Z X0,X1,K1,X2" + ir, err = x86ScalarFPBitViewAPI(t, "TEXT ·view(SB),4,$0-0\n"+masked+"\nRET\n", + "func view()", "amd64", "x86_64-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("unknown dead mask accepted: %s/%s: %v", masked, api, err) + } + } + } + } +} + +func TestX86ArithmeticShiftSignBytesMasksAndUpperAliases(t *testing.T) { + for _, row := range []string{ + "PSRAW $127,X1", "PSRAL $127,X1", "VPSRAW $255,X1,X2", + "VPSRAD $255,Y1,Y2", "VPSRAQ $255,Z1,Z2", + "VPSRAW $255,X1,K1,X2", "VPSRAW.Z $255,X1,K1,X2", + "VPSRAD $255,Y1,K1,Y2", "VPSRAQ.Z $255,Z1,K1,Z2", + } { + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !known || err != nil { + t.Fatal(row, err) + } + var signs uint64 + for at := form.laneBits/8 - 1; at < form.byteWidth; at += form.laneBits / 8 { + signs |= uint64(1) << uint(at) + } + for hole := -1; hole < form.byteWidth; hole++ { + state := x86ResultState{flags: x86ByteFlagsAll, direction: x86ResultDirectionForward} + data := signs | 0xa55a000000000000 + if hole >= 0 { + data &^= uint64(1) << uint(hole) + } + state.writeVector(1, 64, data, false) + state.writeVector(2, 64, 0x55aa55aa55aa55aa, false) + state.masks[1] = true + before := state.clone() + if !x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("partial arithmetic shift transfer rejected", row, hole) + } + var expected uint64 + for output := 0; output < form.byteWidth; output++ { + sign := output/(form.laneBits/8)*(form.laneBits/8) + form.laneBits/8 - 1 + if before.vectorBytes(1)&(uint64(1)<= 0 && !form.zeroing { + expected &= before.vectorBytes(destination) + } + want := before.clone() + want.writeVector(destination, form.byteWidth, expected, form.legacy) + if !reflect.DeepEqual(state, want) { + t.Fatal("sign dependency, mask, upper alias or unrelated state changed", row, hole) + } + } + } + for _, row := range []string{"VPSRAQ $-1,X1,X2", "VPSRAD $1.0,X1,X2", "VPSRAW $undefined,X1,X2"} { + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86PackedShiftProperties(ins.Op, ins, "amd64"); !known || err == nil { + t.Fatal("invalid integer count acquired arithmetic grammar", row, err) + } + } +} diff --git a/amd64_arithmetic_shift_value_test.go b/amd64_arithmetic_shift_value_test.go new file mode 100644 index 000000000..e4edf35d7 --- /dev/null +++ b/amd64_arithmetic_shift_value_test.go @@ -0,0 +1,160 @@ +package plan9asm + +import ( + "regexp" + "runtime" + "strconv" + "strings" + "testing" +) + +func x86ArithmeticShiftValueRows(arch string) []struct{ op, bank, mode string } { + var rows []struct{ op, bank, mode string } + for _, name := range x86ArithmeticShiftTestNames() { + for _, bank := range []string{"X", "Y", "Z"} { + if name[0] != 'V' && bank != "X" { + continue + } + modes := []string{"plain"} + if arch == "amd64" && name[0] == 'V' { + modes = append(modes, "merge", "zero") + } + for _, mode := range modes { + rows = append(rows, struct{ op, bank, mode string }{name, bank, mode}) + } + } + } + return rows +} + +func x86ArithmeticShiftValueProgram(arch string) (string, string, string, string) { + return x86FixedBinaryValueRuntimeProgram(arch, false, "ArithmeticShift", x86ArithmeticShiftValueRows, + func(op string) (int, int) { + perLane := 0 + if strings.Contains(op, "RAV") { + perLane = 1 + } + return x86ArithmeticShiftTestBits(op), perLane + }) +} + +func TestX86ArithmeticShiftPortableBaselineTargetIR(t *testing.T) { + source, declarations, _, _ := x86ArithmeticShiftValueProgram("amd64") + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, api) + if err != nil { + t.Fatal(api, err) + } + if regexp.MustCompile(`"target-features"="[^"]*\+avx`).MatchString(ir) { + t.Errorf("portable numerical oracle retains native ISA requirements: %s/%s", triple, api) + } + } + } +} + +func TestX86ArithmeticShiftInitializedFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + if len(x86ArithmeticShiftValueRows("amd64")) != 56 || len(x86ArithmeticShiftValueRows("386")) != 20 { + t.Fatal("complete arithmetic opcode/width denominator changed") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + source, declarations, _, _ := x86ArithmeticShiftValueProgram(target.arch) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + } + }) + } +} + +// Portable LLVM numerical execution is not original AVX/AVX512 execution. +// Counts include each boundary and high unsigned bits; the reference builds +// individual output bits without signed C shifts or the production emitter. +func TestX86ArithmeticShiftPortableValuesAndCanaries(t *testing.T) { + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc and clang required") + } + arch, cWord := "amd64", "uint64_t" + if strconv.IntSize == 32 { + arch, cWord = "386", "uint32_t" + } + source, declarations, prototypes, rows := x86ArithmeticShiftValueProgram(arch) + driver := "#include \n#include \n#include \n" + prototypes + + "typedef " + cWord + " shift_word;\n" + ` +struct kernel {void (*fn)(uint8_t*,uint8_t*,uint8_t*,shift_word);unsigned width,bits,mode,legacy,per_lane;}; +static const struct kernel kernels[]={ +` + rows + `}; +int main(void) { + const uint64_t counts[]={0,1,7,8,15,16,31,32,63,64,127,255,256,UINT64_C(0x100000000),UINT64_MAX}; + const shift_word masks[]={0,1,(shift_word)0xa55a9c63f00fc33cULL,(shift_word)-1}; + for(unsigned k=0;k>1); + unsigned count_bytes=p->per_lane?p->bits/8:8; + for(unsigned lane=0;lane<64/count_bytes;lane++)for(unsigned byte=0;byte>(byte*8); + memcpy(a_before,a,67);memcpy(b_before,b,67);memcpy(want,b+1,64); + if(!p->legacy)memset(want+p->width,0,64-p->width); + for(unsigned lane=0;lanewidth/(p->bits/8);lane++) { + if(p->mode && !(masks[m]>>lane&1)) { + if(p->mode==2)memset(want+lane*(p->bits/8),0,p->bits/8); + continue; + } + unsigned at=p->per_lane?lane*count_bytes:0; + uint64_t shift=0; + for(unsigned byte=0;byte=p->bits)shift=p->bits-1; + for(unsigned bit=0;bitbits;bit++) { + unsigned from=bit+(unsigned)shift; + if(from>=p->bits)from=p->bits-1; + unsigned input=lane*p->bits+from,output=lane*p->bits+bit; + unsigned value=(b[1+input/8]>>(input%8))&1; + want[output/8]=(want[output/8]&~(1u<<(output%8)))|(value<<(output%8)); + } + } + p->fn(a+1,b+1,out+1,masks[m]); + if(memcmp(out+1,want,64)||out[65]!=1||out[0]!=0xc3||out[66]!=0xc3|| + memcmp(a,a_before,67)||memcmp(b,b_before,67)) { + fprintf(stderr,"arithmetic shift row=%u sample=%u count=%u width=%u bits=%u\n",k,sample,c,p->width,p->bits); + return 1; + } + } + } + return 0; +} +` + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, arch, triple, api) + if err != nil { + t.Fatal(api, err) + } + if regexp.MustCompile(`"target-features"="[^"]*\+avx`).MatchString(ir) { + t.Fatal("portable arithmetic oracle requires native AVX/AVX512 hardware") + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []int{0, 2} { + compileAndRunRuntimeTestWithCompiler(t, llc, []string{clang}, "arithmetic-shift-values", triple, text, driver, nil, level) + } + } + }) + } + t.Log("complete opcode/width kernels and Go-valid merge/zero masks; full 64-byte aliases, CF, input/output canaries and unsigned saturating counts; portable LLVM only") +} diff --git a/amd64_atomic_memory_test.go b/amd64_atomic_memory_test.go new file mode 100644 index 000000000..51ca561db --- /dev/null +++ b/amd64_atomic_memory_test.go @@ -0,0 +1,214 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +// An object-only success is insufficient: LLVM can legalize an under-aligned +// atomicrmw into an unresolved libatomic call, absent from the Windows CRT. +func TestX86AtomicExchangeMemoryNeedsNoRuntimeHelpers(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"386", "i686-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-w64-windows-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"amd64", "x86_64-w64-windows-gnu"}, + {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-apple-darwin"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT exchange(SB),$0-0\n") + for _, family := range []string{"XADD", "XCHG"} { + widths := "BWL" + if target.arch == "amd64" { + widths += "Q" + } + for _, width := range widths { + reg := "DX" + if width == 'B' { + reg = "DL" + } + fmt.Fprintf(&source, "%s%c %s, 1(BX)\n", family, width, reg) + fmt.Fprintf(&source, "%s%c %s, 1(FS)\n", family, width, reg) + fmt.Fprintf(&source, "%s%c %s, 1(GS)\n", family, width, reg) + } + } + source.WriteString("RET\n") + // Preserve every uninitialized original instruction and its Go + // object/public Context. This private emitter proves formats and + // absence of invented runtime helpers, not initialized execution. + file, signatures := x86UnboundVoidFormatSource(t, source.String(), "exchange", target.arch, target.triple) + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.arch, target.triple, 0, + func(c *amd64Ctx, instruction Instr) (bool, bool, error) { + spec, known := amd64ExchangeSpecs[instruction.Op] + if !known { + return false, false, fmt.Errorf("original exchange fixture changed family: %s", instruction.Raw) + } + return c.lowerExchange(instruction.Op, spec, instruction) + }) + dir := t.TempDir() + input := filepath.Join(dir, "exchange.ll") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + asm, err := exec.Command(llc, "-mtriple="+target.triple, "-filetype=asm", input, "-o", "-").CombinedOutput() + if err != nil { + t.Fatalf("LLVM 22: %v\n%s", err, asm) + } + if strings.Contains(string(asm), "__atomic") || strings.Contains(string(asm), "__sync") { + t.Fatalf("native x86 exchange acquired an external atomic helper:\n%s", asm) + } + for _, segment := range []string{"%fs:", "%gs:"} { + want := 6 + if target.arch == "amd64" { + want = 8 + } + if got := strings.Count(string(asm), segment); got != want { + t.Fatalf("atomic memory operand retained %d/%d %s overrides:\n%s", got, want, segment, asm) + } + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + // The complete original instruction tail is byte-exact. Only + // actual typed-pointer and numerical entry producers are added. + _, original, found := strings.Cut(source.String(), "\n") + if !found { + t.Fatal("original exchange fixture lost its TEXT header") + } + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + initialized := fmt.Sprintf("TEXT exchange(SB),4,$0-%d\n%s p+0(FP),BX\n%s $5,DX\n%s", + word, move, move, original) + if !strings.HasSuffix(initialized, original) { + t.Fatal("initialized exchange changed an original instruction or direction") + } + x86OriginalGPInitializedFormatObjects(t, []x86OriginalGPInitializedFormat{{ + source: initialized, declaration: "func exchange(p *[32]byte)", + }}, target.arch, target.triple) + }) + } +} + +func TestX86ExchangeGrammarMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`\{A((?:XADD|XCHG)[BWLQ]),\s*(\w+),`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 8 || len(rows) != len(amd64ExchangeSpecs) { + t.Fatalf("Go encoder has %d exchange opcodes; grammar has %d", len(rows), len(amd64ExchangeSpecs)) + } + for _, row := range rows { + spec, ok := amd64ExchangeSpecs[Op(row[1])] + bits := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[row[1][len(row[1])-1]] + kind, table := amd64ExchangeSwap, "yxchg" + if strings.HasPrefix(row[1], "XADD") { + kind, table = amd64ExchangeAdd, "yrl_ml" + if bits == 8 { + table = "yrb_mb" + } + } else if bits == 8 { + table = "yml_mb" + } + if !ok || spec.bits != bits || spec.kind != kind || row[2] != table { + t.Errorf("Go %s/%s disagrees with typed grammar: %+v", row[1], row[2], spec) + } + } +} + +func TestAMD64ExchangeUnalignedRuntimeSemantics(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution requires amd64 or Rosetta; the all-host object matrix remains required") + } + var source, declarations, cases strings.Builder + sigs := make(map[string]FuncSig) + for _, family := range []string{"XADD", "XCHG", "REVERSE"} { + for i, width := range "BWLQ" { + bits := []int{8, 16, 32, 64}[i] + name := fmt.Sprintf("exchange_%s_%d", family, bits) + reg := "AX" + if bits == 8 { + reg = "AL" + } + instruction := fmt.Sprintf("%s%c %s, 1(BX)", family, width, reg) + if family == "REVERSE" { + instruction = fmt.Sprintf("XCHG%c 1(BX), %s", width, reg) + } + fmt.Fprintf(&source, "TEXT %s(SB),$0-24\nMOVQ p+0(FP), BX\nMOVQ v+8(FP), AX\n%s\nMOVQ AX, ret+16(FP)\nRET\n", name, instruction) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, I64}, Ret: I64, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}, + Results: []FrameSlot{{Offset: 16, Type: I64, Index: 0, Field: -1}}, + }} + fmt.Fprintf(&declarations, "extern uint64_t %s(uint8_t *, uint64_t);\n", name) + add := 0 + if family == "XADD" { + add = 1 + } + fmt.Fprintf(&cases, "{%s, %d, %d},\n", name, bits, add) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple, runPrefix = "x86_64-apple-macosx", []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +` + declarations.String() + ` +int main(void) { + struct { uint64_t (*fn)(uint8_t *, uint64_t); int bits, add; } cases[] = { +` + cases.String() + ` + }; + const uint64_t values[] = {0, 1, UINT64_MAX, UINT64_C(0x8877665544332211), UINT64_C(0x8000000080008080)}; + for (unsigned c = 0; c < sizeof(cases)/sizeof(cases[0]); c++) { + int bits = cases[c].bits, bytes = bits/8; + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + for (unsigned offset = 0; offset < 8; offset++) { + for (unsigned a = 0; a < 5; a++) for (unsigned b = 0; b < 5; b++) { + _Alignas(64) uint8_t storage[64]; + memset(storage, 0x5a, sizeof(storage)); + uint64_t initial = values[a], value = values[b]; + memcpy(storage + offset + 1, &initial, bytes); + uint64_t got = cases[c].fn(storage + offset, value), memory = 0; + memcpy(&memory, storage + offset + 1, bytes); + uint64_t want = initial & mask; + if (bits < 32) want |= value & ~mask; + if (got != want || memory != ((cases[c].add ? initial + value : value) & mask)) return 1; + for (unsigned i = 0; i < sizeof(storage); i++) { + if ((i < offset + 1 || i >= offset + 1 + bytes) && storage[i] != 0x5a) return 2; + } + } + } + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "exchange_unaligned", triple, ir, mainC, runPrefix) +} diff --git a/amd64_auxiliary_carry_runtime_test.go b/amd64_auxiliary_carry_runtime_test.go new file mode 100644 index 000000000..3bd59f2fb --- /dev/null +++ b/amd64_auxiliary_carry_runtime_test.go @@ -0,0 +1,310 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// The reference computes carry/borrow using widened arithmetic, including the +// low nibble for AF; it does not repeat the lowerer's XOR-bit implementation. +const x86AuxiliaryGoReference = `package auxiliary + +import ( + "encoding/binary" + "math/bits" + "testing" +) + +type kernel struct { + fn func(*[64]byte, uint64, uint64, uint64) + op string + width, form int +} + +func expected(op string, width, form int, a, b, seed uint64) uint64 { + if width == 0 || (op == "CMPS" || op == "SCAS") && form != 0 { + return seed & 0x8d5 + } + mask := ^uint64(0) + if width < 64 { + mask = (uint64(1) << width) - 1 + } + a &= mask + b &= mask + if form == 3 { + b = 1 + } + carry := uint64(0) + if op == "ADC" || op == "SBB" { + carry = seed & 1 + } + add := op == "ADD" || op == "ADC" || op == "INC" || op == "XADD" + switch op { + case "INC", "DEC": + b = 1 + case "NEG": + b, a = a, 0 + case "CMPXCHG": + a, b = b, a + } + + result := a - b - carry + cf := a < b || a == b && carry != 0 + af := (a & 15) < (b & 15) + carry + if add { + var c uint64 + result, c = bits.Add64(a, b, carry) + cf = c != 0 || width < 64 && result > mask + af = (a & 15) + (b & 15) + carry > 15 + } + result &= mask + sign := uint64(1) << (width - 1) + overflow := ((a ^ b) & (a ^ result) & sign) != 0 + if add { + overflow = ((^(a ^ b)) & (a ^ result) & sign) != 0 + } + if op == "INC" || op == "DEC" { + cf = seed & 1 != 0 + } + + f := uint64(0) + if cf { f |= 1 } + if bits.OnesCount8(uint8(result)) % 2 == 0 { f |= 4 } + if af { f |= 16 } + if result == 0 { f |= 64 } + if result & sign != 0 { f |= 128 } + if overflow { f |= 2048 } + return f +} + +func check(t *testing.T, k kernel, a, b, seed uint64) { + t.Helper() + var out [64]byte + for i := range out { + out[i] = 0xa7 + } + k.fn(&out, a, b, seed) + want := expected(k.op, k.width, k.form, a, b, seed) + got := binary.LittleEndian.Uint64(out[24:32]) & 0x8d5 + if got != want || out[32] != byte(want) | 2 { + t.Fatalf("%s width=%d form=%d a=%x b=%x seed=%x flags=%x LAHF=%x want=%x", k.op, k.width, k.form, a, b, seed, got, out[32], want) + } + flags := []uint64{want & 1, (want >> 2) & 1, (want >> 6) & 1, (want >> 7) & 1, (want >> 11) & 1, ((want >> 7) ^ (want >> 11)) & 1} + for i, v := range flags { + if out[33+i] != byte(v) { + t.Fatalf("%s SETcc %d got=%d want=%d", k.op, i, out[33+i], v) + } + } + for i, v := range out { + if (i >= 16 && i < 24 || i >= 39) && v != 0xa7 { + t.Fatalf("%s byte canary %d", k.op, i) + } + } +} + +func TestOriginalArithmeticStatus(t *testing.T) { + edges := []uint64{0, 1, 15, 16, 17, 127, 128, 255, 256, 32767, 32768, 65535, 0x7fffffff, 0x80000000, 0xffffffff, 0x7fffffffffffffff, 0x8000000000000000, ^uint64(0)} + for _, k := range kernels { + for _, seed := range []uint64{0, 0x8d5} { + for a := uint64(0); a < 16; a++ { + for b := uint64(0); b < 16; b++ { + check(t, k, a, b, seed) + } + } + for i, a := range edges { + for _, b := range []uint64{edges[(i+1)%len(edges)], edges[(i+7)%len(edges)]} { + check(t, k, a, b, seed) + } + } + } + } +} +` + +func TestCrossLinuxRuntimeMatrixX86AuxiliaryCarryOriginalGo(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + var cases []x86AuxiliaryCase + for _, row := range x86AuxiliaryCases("amd64") { + if row.op != "CMPXCHG" { + cases = append(cases, row) + } + } + x86AuxiliaryOriginalGo(t, cases) +} + +func TestCrossLinuxRuntimeMatrixX86AuxiliaryCarryCMPXCHGNativeGo(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("original CMPXCHG flags require the native AMD64 cross-runtime job, not Rosetta") + } + var cases []x86AuxiliaryCase + for _, row := range x86AuxiliaryCases("amd64") { + if row.op == "CMPXCHG" { + cases = append(cases, row) + } + } + x86AuxiliaryOriginalGo(t, cases) +} + +func x86AuxiliaryOriginalGo(t *testing.T, cases []x86AuxiliaryCase) { + t.Helper() + var declarations, kernels strings.Builder + kernels.WriteString("var kernels=[]kernel{\n") + for _, row := range cases { + fmt.Fprintf(&declarations, "func %s(*[64]byte,uint64,uint64,uint64)\n", row.name) + fmt.Fprintf(&kernels, "{%s,%q,%d,%d},\n", row.name, row.op, row.bits, row.form) + } + kernels.WriteString("}\n") + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module auxiliary\n\ngo 1.20\n", + "auxiliary.go": "package auxiliary\n" + declarations.String(), + "auxiliary_amd64.s": x86AuxiliaryAssembly("amd64", cases), + "auxiliary_test.go": x86AuxiliaryGoReference + kernels.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("original Go complete arithmetic status: %v\n%s", err, output) + } + t.Logf("%d original Go kernels, exhaustive low nibbles, both CF/AF seeds and signed/unsigned width boundaries", len(cases)) +} + +const x86AuxiliaryCReference = ` +#include +#include +#include +typedef void (*kernel)(uint8_t *, uint64_t, uint64_t, uint64_t); + +static int check(kernel fn, const char *op, int width, int form, + uint64_t a, uint64_t b, uint64_t seed) { + _Alignas(8) uint8_t out[64]; + memset(out, 0xa7, sizeof(out)); + fn(out, a, b, seed); + + uint64_t want = seed & 0x8d5; + if (width && !((!strcmp(op, "CMPS") || !strcmp(op, "SCAS")) && form)) { + uint64_t mask = width == 64 ? UINT64_MAX : (UINT64_C(1) << width) - 1; + a &= mask; + b &= mask; + if (form == 3) { + b = 1; + } + unsigned carry = !strcmp(op, "ADC") || !strcmp(op, "SBB") ? seed & 1 : 0; + int add = !strcmp(op, "ADD") || !strcmp(op, "ADC") || + !strcmp(op, "INC") || !strcmp(op, "XADD"); + if (!strcmp(op, "INC") || !strcmp(op, "DEC")) { + b = 1; + } + if (!strcmp(op, "NEG")) { + b = a; + a = 0; + } + if (!strcmp(op, "CMPXCHG")) { + uint64_t swap = a; + a = b; + b = swap; + } + + __uint128_t wide = add ? (__uint128_t)a + b + carry : (__uint128_t)a - b - carry; + uint64_t result = (uint64_t)wide & mask; + unsigned cf = add ? wide > mask : (__uint128_t)a < (__uint128_t)b + carry; + unsigned af = add ? (a & 15) + (b & 15) + carry > 15 : (a & 15) < (b & 15) + carry; + uint64_t sign = UINT64_C(1) << (width - 1); + unsigned of = (add ? (~(a ^ b) & (a ^ result)) : (a ^ b) & (a ^ result)) & sign ? 1 : 0; + if (!strcmp(op, "INC") || !strcmp(op, "DEC")) { + cf = seed & 1; + } + unsigned ones = 0; + for (unsigned i = 0; i < 8; i++) { + ones += (result >> i) & 1; + } + want = cf | ((ones % 2 == 0) << 2) | (af << 4) | ((result == 0) << 6) | + ((!!(result & sign)) << 7) | (of << 11); + } + + uint64_t flags; + memcpy(&flags, out + 24, 8); + flags &= 0x8d5; + if (flags != want || out[32] != ((want & 255) | 2)) { + fprintf(stderr, "%s width=%d form=%d flags=%llx lahf=%x want=%llx\n", + op, width, form, (unsigned long long)flags, out[32], (unsigned long long)want); + return 1; + } + unsigned bits[] = {0, 2, 6, 7, 11}; + for (unsigned i = 0; i < 5; i++) { + if (out[33+i] != ((want >> bits[i]) & 1)) { + return 2; + } + } + if (out[38] != (((want >> 7) ^ (want >> 11)) & 1)) { + return 3; + } + for (unsigned i = 16; i < 24; i++) { + if (out[i] != 0xa7) { + return 4; + } + } + for (unsigned i = 39; i < 64; i++) { + if (out[i] != 0xa7) { + return 5; + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86AuxiliaryCarryLLVM(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + cases := x86AuxiliaryCases("amd64") + file, err := Parse(ArchAMD64, x86AuxiliaryAssembly("amd64", cases)) + if err != nil { + t.Fatal(err) + } + var declarations, checks strings.Builder + for _, row := range cases { + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,uint64_t,uint64_t,uint64_t);\n", row.name) + fmt.Fprintf(&checks, "if(check(%s,%q,%d,%d,a,b,seed))return 1;\n", row.name, row.op, row.bits, row.form) + } + main := x86AuxiliaryCReference + declarations.String() + ` +int main(void) { + const uint64_t edges[] = { + 0, 1, 15, 16, 17, 127, 128, 255, 256, 32767, 32768, 65535, + 0x7fffffff, 0x80000000, 0xffffffff, 0x7fffffffffffffff, + 0x8000000000000000, UINT64_MAX + }; + for (unsigned s = 0; s < 2; s++) { + for (unsigned i = 0; i < 292; i++) { + uint64_t seed = s ? 0x8d5 : 0, a, b; + if (i < 256) { + a = i / 16; + b = i % 16; + } else { + a = edges[(i-256)/2]; + b = edges[((i-256)/2 + (i%2 ? 7 : 1))%18]; + } +` + checks.String() + " }\n }\n return 0;\n}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + ir, err := Translate(file, x86AuxiliaryOptions("amd64", triple, cases)) + if err != nil { + t.Fatal(err) + } + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "auxiliary-status", triple, ir, main, prefix) + }) + } +} diff --git a/amd64_auxiliary_carry_test.go b/amd64_auxiliary_carry_test.go new file mode 100644 index 000000000..8245af1f3 --- /dev/null +++ b/amd64_auxiliary_carry_test.go @@ -0,0 +1,189 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +type x86AuxiliaryCase struct { + name string + op string + bits int + form int +} + +// Operand axes are tested independently of the existing complete encoder-row +// tests. All arithmetic flag producers, including exchange/string comparisons, +// must feed the same architectural AF bit consumed by LAHF and PUSHF. +func x86AuxiliaryCases(arch string) []x86AuxiliaryCase { + var cases []x86AuxiliaryCase + widths := []int{8, 16, 32} + if arch == "amd64" { + widths = append(widths, 64) + } + for _, op := range []string{"ADD", "SUB", "ADC", "SBB", "CMP", "INC", "DEC", "NEG", "XADD", "CMPXCHG", "CMPS", "SCAS"} { + forms := 4 // Register, memory source, memory destination, immediate. + switch op { + case "INC", "DEC", "NEG", "XADD", "CMPXCHG": + forms = 2 + case "CMPS", "SCAS": + forms = 3 // Single comparison; zero-count REP and REPN with NULL inputs. + } + for _, bits := range widths { + for form := 0; form < forms; form++ { + cases = append(cases, x86AuxiliaryCase{fmt.Sprintf("aux%d", len(cases)), op, bits, form}) + } + } + } + return append(cases, x86AuxiliaryCase{"auxpopf", "POPF", 0, 0}, x86AuxiliaryCase{"auxsahf", "SAHF", 0, 0}) +} + +func x86AuxiliaryAssembly(arch string, cases []x86AuxiliaryCase) string { + saved := "R8" + if arch == "386" { + saved = "BP" + } + return x86AuxiliaryAssemblyWithSavedRegister(arch, cases, saved) +} + +func x86AuxiliaryAssemblyWithSavedRegister(arch string, cases []x86AuxiliaryCase, saved string) string { + move, push, flags, slot := "Q", "Q", "Q", 8 + if arch == "386" { + move, push, flags, slot = "L", "L", "L", 4 + } + var source strings.Builder + for _, row := range cases { + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n", row.name, slot*4) + fmt.Fprintf(&source, "MOV%s out+0(FP),DI\nMOV%s a+%d(FP),BX\nMOV%s b+%d(FP),DX\nMOV%s seed+%d(FP),CX\n", move, move, slot, move, slot*2, move, slot*3) + fmt.Fprintf(&source, "MOV%s BX,0(DI)\nMOV%s DX,8(DI)\n", move, move) + if row.op == "SAHF" { + fmt.Fprintf(&source, "MOV%s CX,AX\nSHL%s $8,AX\n", move, move) + } + if row.op == "CMPXCHG" { + fmt.Fprintf(&source, "MOV%s DX,AX\n", move) + } + if row.op == "CMPS" || row.op == "SCAS" { + // Use an ordinary GP, not AMD64's independently owned BP frame. + fmt.Fprintf(&source, "PUSH%s %s\n", push, saved) + fmt.Fprintf(&source, "MOV%s DI,%s\nMOV%s DI,SI\nADD%s $8,DI\nMOV%s BX,AX\n", move, saved, move, move, move) + } + fmt.Fprintf(&source, "PUSH%s CX\nPOPF%s\n", push, flags) + suffix := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[row.bits] + opcode := row.op + suffix + switch row.op { + case "POPF": // The seed is already installed above. + case "SAHF": + source.WriteString("SAHF\n") + case "CMPS", "SCAS": + if row.form != 0 { + fmt.Fprintf(&source, "MOV%s $0,CX\nMOV%s $0,SI\nMOV%s $0,DI\n", move, move, move) + prefix := "REP" + if row.form == 2 { + prefix = "REPN" + } + fmt.Fprintf(&source, "%s\n", prefix) + } + fmt.Fprintf(&source, "%s\nMOV%s %s,DI\n", opcode, move, saved) + case "INC", "DEC", "NEG": + destination := "BX" + if row.form == 1 { + destination = "0(DI)" + } + fmt.Fprintf(&source, "%s %s\n", opcode, destination) + case "XADD", "CMPXCHG": + destination := "BX" + if row.form == 1 { + destination = "0(DI)" + } + fmt.Fprintf(&source, "%s DX,%s\n", opcode, destination) + case "CMP": + operands := []string{"BX,DX", "BX,8(DI)", "0(DI),DX", "BX,$1"} + fmt.Fprintf(&source, "%s %s\n", opcode, operands[row.form]) + default: + operands := []string{"DX,BX", "8(DI),BX", "DX,0(DI)", "$1,BX"} + fmt.Fprintf(&source, "%s %s\n", opcode, operands[row.form]) + } + fmt.Fprintf(&source, "PUSHF%s\nPOP%s CX\nMOV%s CX,24(DI)\nLAHF\nMOVB AH,32(DI)\n", flags, push, move) + source.WriteString("SETCS 33(DI)\nSETPS 34(DI)\nSETEQ 35(DI)\nSETMI 36(DI)\nSETOS 37(DI)\nSETLT 38(DI)\n") + if row.op == "CMPS" || row.op == "SCAS" { + fmt.Fprintf(&source, "POP%s %s\n", push, saved) + } + source.WriteString("RET\n") + } + return source.String() +} + +func TestX86AuxiliaryCarryOriginalPhysicalBPRemainsContext(t *testing.T) { + cases := []x86AuxiliaryCase{{name: "PhysicalFrame", op: "CMPS", bits: 8}} + source := x86AuxiliaryAssemblyWithSavedRegister("amd64", cases, "BP") + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue // The whole-family owner retains the ordinary mode32 BP form. + } + t.Run(target.triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, + "func PhysicalFrame(out *byte,a,b,seed uint64)", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("original BP source borrowed ordinary numerical GP transport", api, err) + } + } + }) + } +} + +func x86AuxiliaryOptions(arch, triple string, cases []x86AuxiliaryCase) Options { + typ, slot := I64, int64(8) + if arch == "386" { + typ, slot = I32, 4 + } + sigs := make(map[string]FuncSig, len(cases)) + for _, row := range cases { + sigs[row.name] = FuncSig{Name: row.name, Args: []LLVMType{Ptr, typ, typ, typ}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: slot, Type: typ, Index: 1, Field: -1}, + {Offset: slot * 2, Type: typ, Index: 2, Field: -1}, + {Offset: slot * 3, Type: typ, Index: 3, Field: -1}, + }}, + } + } + return Options{Goarch: arch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + } +} + +func TestX86AuxiliaryCarryWholeFamilyObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + cases := x86AuxiliaryCases(target.arch) + source := x86AuxiliaryAssembly(target.arch, cases) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86AuxiliaryOptions(target.arch, target.triple, cases)) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "auxiliary.ll", "auxiliary.o", ir) + if !strings.Contains(ir, "%flags_af = alloca i1") { + t.Fatal("arithmetic status consumers silently omit architectural AF") + } + }) + } +} diff --git a/amd64_bextr_forms.go b/amd64_bextr_forms.go new file mode 100644 index 000000000..c6317d4d9 --- /dev/null +++ b/amd64_bextr_forms.go @@ -0,0 +1,56 @@ +package plan9asm + +func x86BEXTRProperties(op Op, ins Instr, arch string) (x86ScalarBitFieldForm, bool, error) { + return x86ScalarBitFieldFamilyProperties(op, ins, arch, x86BitFieldExtract) +} + +func x86TypedScalarBitFieldByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86ScalarBitFieldProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.bits / 8) + return x86TypedByteEffect{ + operands: []x86TypedByteOperand{ + {index: 0, operand: form.control, width: form.spec.controlBytes, read: true}, + {index: 1, operand: form.source, width: width, read: true}, + {index: 2, operand: form.destination, width: width, write: true}, + }, + flagsDefined: form.spec.flagsDefined, + flagsKilled: x86ByteFlagsAll, + requiresResultProof: true, + }, true, nil +} + +// A paired native read supplies numerical source bytes only if it continues. +// It grants no caller, extent, raw-entry or reusable native-memory capability. +func x86ResultScalarBitFieldStep(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, + arch string, nativeRead bool) (bool, bool) { + effect, known, err := x86TypedScalarBitFieldByteEffect(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + if !x86ResultNamedPredicateInstruction(ins) { + return false, true + } + for _, operand := range effect.operands { + if !operand.read { + continue + } + if operand.operand.Kind == OpReg { + index, _, valid := x86ResultGP(operand.operand.Reg, operand.width, arch) + if !valid || state.pointerGP[index] { + return false, true + } + } + if !x86ResultReadWithNativeContinuation(ins, operand.operand, operand.width, state, owned, arch, nativeRead) { + return false, true + } + } + output := effect.operands[2] + if !x86ResultStore(output.operand, output.width, state, owned, resultOffset, arch) { + return false, true + } + state.flags = state.flags&^effect.flagsKilled | effect.flagsDefined + return true, true +} diff --git a/amd64_bittest_effects_test.go b/amd64_bittest_effects_test.go new file mode 100644 index 000000000..3331947cf --- /dev/null +++ b/amd64_bittest_effects_test.go @@ -0,0 +1,194 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86BitTestTypedByteAndFlagEffects(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86BitTestOracleNames(arch) { + width := int64(map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]]) + writes := len(op) != 3 + for _, index := range []string{"$1", "CX"} { + ins := x86PureGPUnknownInstruction(t, op+" "+index+",AX") + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 2 || effect.dynamicMemory || + effect.flagsDefined != x86ByteFlagCF || effect.flagsRead != 0 || + effect.flagsKilled != x86ByteFlagsAll&^(x86ByteFlagCF|x86ByteFlagZF) || + !effect.operands[1].read || effect.operands[1].write != writes { + t.Fatalf("incomplete %s/%s byte/flag effects: %+v %v", arch, ins.Raw, effect, err) + } + for flags := uint8(0); flags <= x86ByteFlagsAll; flags++ { + state := x86ResultState{flags: flags} + state.gp[0], state.gp[1] = 0xff, 0xff + if !x86ResultTransfer(ins, &state, nil, 0, arch) || state.flags != flags&x86ByteFlagZF|x86ByteFlagCF { + t.Fatalf("%s/%s invented incoming flag state %x: %x", arch, ins.Raw, flags, state.flags) + } + } + for missingOperand := 0; missingOperand < 2; missingOperand++ { + for missingByte := int64(-1); missingByte < width; missingByte++ { + state := x86ResultState{flags: x86ByteFlagsAll} + state.gp[0], state.gp[1] = uint8(1<= 0 && (missingOperand == 1 || index != "$1") + if missing { + register := 1 - missingOperand + state.gp[register] &^= 1 << uint(missingByte) + } + before := state.clone() + accepted := x86ResultTransfer(ins, &state, nil, 0, arch) + if accepted == missing { + t.Fatalf("%s/%s missing operand=%d byte=%d admitted=%v", arch, ins.Raw, missingOperand, missingByte, accepted) + } + if accepted { + want := before.gp[0] + if writes && width == 4 { + want |= 0xf0 + } + if state.gp[0] != want || state.gp[1] != before.gp[1] || state.flags != x86ByteFlagCF|x86ByteFlagZF { + t.Fatalf("%s/%s changed unwritten bytes or preserved undefined flags: %+v", arch, ins.Raw, state) + } + } + } + } + memory := x86PureGPUnknownInstruction(t, op+" "+index+",(DX)") + effect, known, err = x86TypedInstructionByteEffect(memory.Op, memory, arch) + if !known || err != nil || effect.dynamicMemory != (index == "CX") { + t.Fatal("memory bit string was mistaken for a fixed register word", memory, effect, err) + } + } + } + } +} + +func TestX86BitTestFlagsAndMemoryDoNotAcquireAuthority(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, op := range x86BitTestOracleNames(target.arch) { + for _, condition := range []struct { + branch string + valid bool + }{{"JCS", true}, {"JE", true}, {"JOS", false}, {"JMI", false}, {"JPS", false}} { + source := "TEXT ·value(SB),4,$0-4\nMOVL $1,AX\nCMPL AX,AX\n" + op + " $0,AX\n" + + condition.branch + " equal\nMOVL $7,AX\nMOVL AX,X0\nJMP done\nequal:\nPXOR X0,X0\ndone:\nPEXTRD $0,X0,AX\nMOVL AX,ret+0(FP)\nRET\n" + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, source, "func value() uint32", target.arch, target.triple, api) + if condition.valid && err != nil || !condition.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s/%s/%s/%s flag initialization differs: %v", target.triple, op, condition.branch, api, err) + } + } + } + for _, memory := range []string{"unbound(SB)", "(DX)", "8(DX)(FS*1)", "8(DX)(GS*1)"} { + source := "TEXT ·value(SB),4,$0-4\n" + op + " $0," + memory + "\nPXOR X0,X0\nPEXTRD $0,X0,AX\nMOVL AX,ret+0(FP)\nRET\n" + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source, "func value() uint32", target.arch, target.triple, api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("new native/source read authority: %s/%s/%s/%s: %v", target.triple, op, memory, api, err) + } + } + } + } + } +} + +func TestX86BitTestSegmentContinuationObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, op := range x86BitTestOracleNames(target.arch) { + for _, segment := range []string{"FS", "GS"} { + for _, index := range []string{"$0", "CX"} { + memory := "8(" + segment + ")" + ins := x86PureGPUnknownInstruction(t, op+" "+index+","+memory) + state := x86ResultState{} + state.gp[1] = 0xff + if x86ResultTransfer(ins, &state, nil, 0, target.arch) { + t.Fatal("standalone observer borrowed paired native segment continuation", ins.Raw) + } + + // Preserve the original bare-segment program. Only the + // register-index counterpart adds its required input. + name := fmt.Sprintf("segment%d", count) + count++ + setup := "" + if index == "CX" { + setup = "MOVL $1,CX\n" + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-4\n%s%s %s,%s\nPXOR X0,X0\nPEXTRD $0,X0,AX\nMOVL AX,ret+0(FP)\nRET\n", + name, setup, op, index, memory) + fmt.Fprintf(&declarations, "func %s() uint32\n", name) + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + if strings.Count(text, "load volatile") != count || + strings.Count(text, "store volatile") != count*3/4 || + !strings.Contains(text, "addrspace(256)") || !strings.Contains(text, "addrspace(257)") { + t.Fatal("dead segment bit operations lost their exact paired accesses", api) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + } + }) + } +} + +func TestX86BitTestFPCompleteByteSpans(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + var validSource, declarations strings.Builder + count := 0 + for _, op := range x86BitTestOracleNames(target.arch) { + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + for _, declaration := range []string{"func value(x uint64)", "func value(x [8]byte)", "func value(x float64)"} { + for offset := -1; offset <= 8; offset++ { + valid := offset >= 0 && offset+width <= 8 + source := fmt.Sprintf("TEXT ·value(SB),4,$0-8\n%s $-1,x%+d(FP)\nRET\n", op, offset) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + if valid && err != nil || !valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s/%s/%s/%s/%d byte boundary: %v", target.triple, op, declaration, api, offset, err) + } + if valid && x86CMOVFPHasNumericConversion(ir) { + t.Fatal("raw FP bits were numerically converted") + } + } + } + } + for _, declaration := range []string{"func value(x *uint64)", "func value(x uint8,y uint64)", "func value() uint64"} { + source := fmt.Sprintf("TEXT ·value(SB),4,$0-16\n%s $0,x+0(FP)\nRET\n", op) + if _, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, "text"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("hole/pointer/uninitialized result acquired FP bytes: %s/%s/%s: %v", target.triple, op, declaration, err) + } + } + name := fmt.Sprintf("span%d", count) + fmt.Fprintf(&validSource, "TEXT ·%s(SB),4,$0-8\n%s $1,x+0(FP)\nRET\n", name, op) + fmt.Fprintf(&declarations, "func %s(x [8]byte)\n", name) + count++ + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, validSource.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, validSource.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "bit-test-fp.ll", "bit-test-fp.o", ir) + } + } +} diff --git a/amd64_bittest_encoder_test.go b/amd64_bittest_encoder_test.go new file mode 100644 index 000000000..5c046d13f --- /dev/null +++ b/amd64_bittest_encoder_test.go @@ -0,0 +1,389 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86BitTestCompleteGoEncoderRows(t *testing.T) { + contents, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + text := string(contents) + start := strings.Index(text, "var ybtl = []ytab{") + if start < 0 { + t.Fatal("actual Go omitted the bit-string encoder table") + } + actual := strings.SplitN(strings.SplitN(text[start:], "\n}", 2)[0], "[]ytab{", 2)[1] + got, err := x86SignedPackedFloatGoOpcodeTokens(actual) + want, wantErr := x86SignedPackedFloatGoOpcodeTokens(`{Zibo_m,2,argList{Yi8,Yml}}, {Zr_m,1,argList{Yrl,Yml}},`) + if err != nil || wantErr != nil || !reflect.DeepEqual(got, want) { + t.Fatal("complete ordered ybtl rows/classes/zcase/offset changed") + } + rows := regexp.MustCompile(`(?m)^\s*\{A(BT(?:C|R|S)?[WLQ]),\s*(\w+),\s*(\w+),\s*opBytes\{([^}]+)\}\},`).FindAllStringSubmatch(text, -1) + if len(rows) != 12 { + t.Fatalf("complete Go family changed: %d rows", len(rows)) + } + seen := make(map[string]bool) + for _, row := range rows { + op, suffix := row[1], row[1][len(row[1])-1] + stem := op[:len(op)-1] + group := map[string]int{"BT": 4, "BTC": 7, "BTR": 6, "BTS": 5}[stem] + registerOpcode := map[string]int{"BT": 0xa3, "BTC": 0xbb, "BTR": 0xb3, "BTS": 0xab}[stem] + prefix := map[byte]string{'W': "Pq", 'L': "Pm", 'Q': "Pw"}[suffix] + encoding := fmt.Sprintf("0xba,0%d,0x%x", group, registerOpcode) + if suffix == 'Q' { + encoding = fmt.Sprintf("0x0f,0xba,0%d,0x0f,0x%x", group, registerOpcode) + } + got, err := x86SignedPackedFloatGoOpcodeTokens(row[4]) + want, wantErr := x86SignedPackedFloatGoOpcodeTokens(encoding) + if seen[op] || row[2] != "ybtl" || row[3] != prefix || err != nil || wantErr != nil || !reflect.DeepEqual(got, want) { + t.Fatal("complete Go family registration/prefix/opcode changed", row) + } + seen[op] = true + } + for _, op := range x86BitTestOracleNames("amd64") { + if !seen[op] { + t.Fatal("independent witness absent from actual Go optab", op) + } + } +} + +func TestX86BitTestAllImmediateBytesActualGo(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + var expected []byte + source.WriteString("TEXT f(SB),4,$0-0\n") + count := 0 + for _, op := range x86BitTestOracleNames(arch) { + stem, suffix := op[:len(op)-1], op[len(op)-1] + group := byte(map[string]int{"BT": 4, "BTC": 7, "BTR": 6, "BTS": 5}[stem]) + for immediate := -128; immediate < 128; immediate++ { + for _, destination := range []string{"AX", "7(DX)"} { + fmt.Fprintf(&source, "%s $%d,%s\n", op, immediate, destination) + if suffix == 'W' { + expected = append(expected, 0x66) + } else if suffix == 'Q' { + expected = append(expected, 0x48) + } + expected = append(expected, 0x0f, 0xba) + if destination == "AX" { + expected = append(expected, 0xc0|group<<3) + } else { + // Go emits exactly the source displacement, not a new + // displacement inferred from high immediate bits. + expected = append(expected, 0x42|group<<3, 7) + } + expected = append(expected, byte(immediate)) + ins := x86PureGPUnknownInstruction(t, fmt.Sprintf("%s $%d,%s", op, immediate, destination)) + form, known, err := x86BitTestProperties(ins.Op, ins, arch) + if !known || err != nil || form.index.Imm != int64(immediate) { + t.Fatal("typed immediate differs from independent original Go byte", ins, err) + } + count++ + } + } + } + source.WriteString("RET\n") + expected = append(expected, 0xc3) + actual := x86PackedShiftActualGoBytes(t, arch, source.String()) + if !bytes.Equal(actual, expected) { + t.Fatalf("%s complete original immediate bytes differ: got %d bytes, want %d", arch, len(actual), len(expected)) + } + t.Logf("%s: %d actual Go bit-string encodings, every imm8, register and memory", arch, count) + } +} + +func TestX86BitTestCompleteFormsFiveTargetsThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, deadRegisters strings.Builder + var initialized, unknownMemory []x86OriginalGPInitializedFormat + var deadDeclarations strings.Builder + privateOnly := 0 + count := 0 + for _, op := range x86BitTestOracleNames(target.arch) { + rows := []string{"$-128,AX", "$127,CX", "CX,AX", "AX,AX", "SP,BP", "CX,SP", + "$63,7(DX)", "CX,7(DX)", "CX,-3(DX)(SI*4)", "$1,data-3(SB)", "CX,data+3(SB)", + "$1,8(FS)", "CX,8(GS)", "$1,4096", "CX,4096"} + if target.arch == "amd64" { + rows = append(rows, "R8,R15", "R9,-7(R12)(R11*8)") + } else { + rows = append(rows, "$4294967295,AX", "CX,data(SB)(SP)", "$1,data(SB)(BX*4)") + } + registers := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} + if target.arch == "amd64" { + registers = append(registers, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + } + for _, register := range registers { + rows = append(rows, register+",AX", "$1,"+register, "AX,"+register) + } + for _, row := range rows { + body := op + " " + row + ins := x86PureGPUnknownInstruction(t, body) + form, known, err := x86BitTestProperties(ins.Op, ins, target.arch) + if !known || err != nil { + t.Fatalf("typed shape %s: %v", body, err) + } + name := fmt.Sprintf("form%d", count) + original := fmt.Sprintf("TEXT ·%s(SB),4,$0-0\n%s\nRET\n", name, body) + source.WriteString(original) + hardwareOnly := form.destination.Kind == OpMem && form.destination.Mem.Segment != "" && + form.destination.Mem.Base == "" && form.destination.Mem.Index == "" && form.index.Kind == OpImm + if form.destination.Kind != OpReg && !hardwareOnly { + unknownMemory = append(unknownMemory, x86OriginalGPInitializedFormat{ + source: original, declaration: "func " + name + "()", + }) + } + ordinaryScratch := form.destination.Kind == OpReg && + !x86OriginalGPPhysicalRegister(form.destination.Reg, target.arch) && + (form.index.Kind != OpReg || !x86OriginalGPPhysicalRegister(form.index.Reg, target.arch)) + if ordinaryScratch { + // A dead ordinary register scratch is still a valid + // public source; do not invent NUM to retain that domain. + deadRegisters.WriteString(original) + fmt.Fprintf(&deadDeclarations, "func %s()\n", name) + } + fixture, reason := x86BitTestOriginalInitializedFormat(target.arch, name, body, form) + if reason == "" { + initialized = append(initialized, fixture) + } else { + privateOnly++ + t.Logf("original row %d retains format/public-role boundary: %s: %s", count, body, reason) + } + count++ + } + } + want := map[string]int{"amd64": 780, "386": 336}[target.arch] + if count != want { + t.Fatal("original ordered bit-test inventory changed", count, want) + } + wantInitialized := map[string]int{"amd64": 636, "386": 248}[target.arch] + if len(initialized) != wantInitialized || privateOnly != want-wantInitialized { + t.Fatal("original bit-test input partition changed", len(initialized), privateOnly, wantInitialized) + } + // The complete unchanged source retains actual Go objects and + // three public Context observations. Emit every original row + // through the same typed family, with no public input claim. + file, signatures := x86UnboundVoidFormatSources(t, source.String(), target.arch, target.triple) + formats := *file + formats.Funcs = nil + physicalWrites := 0 + for _, function := range file.Funcs { + instruction := function.Instrs[1] + form, _, _ := x86BitTestProperties(instruction.Op, instruction, target.arch) + if target.arch != "386" || form.stem == "BT" || form.destination.Kind != OpReg || form.destination.Reg != SP { + formats.Funcs = append(formats.Funcs, function) + continue + } + // Go encodes these original physical ESP writes, but the + // ordinary emitter requires a modeled stack/return contract. + // Preserve its actual rejection rather than fabricate an object. + var partial strings.Builder + resolve := func(name string) string { return strings.TrimPrefix(name, "·") } + consumer := newX86Ctx(&partial, function, signatures[resolve(function.Sym)], resolve, + signatures, target.arch, target.triple, false) + if err := consumer.emitEntryAllocas(); err != nil { + t.Fatal(err) + } + known, terminal, err := consumer.lowerBitTestFamily(instruction.Op, instruction) + if !known || terminal || err == nil || !strings.Contains(err.Error(), "386 direct SP write is unsupported") { + t.Fatal("original physical ESP row lost its independent emitter boundary", instruction.Raw, err) + } + physicalWrites++ + } + if want := map[string]int{"amd64": 0, "386": 18}[target.arch]; physicalWrites != want { + t.Fatal("original physical ESP write inventory changed", physicalWrites, want) + } + ir := x86SingleInstructionFormatObjectUnits(t, &formats, signatures, target.arch, target.triple, + func(c *amd64Ctx, instruction Instr) (bool, bool, error) { + return c.lowerBitTestFamily(instruction.Op, instruction) + }) + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + for row, fixture := range unknownMemory { + t.Run(fmt.Sprintf("unbound-memory/row%d", row), func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, fixture.source, fixture.declaration, target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("unbound original memory format invented inputs", api, err) + } + } + }) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run("dead-registers/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, deadRegisters.String(), deadDeclarations.String(), target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("original pure dead-register domain changed", err) + } + }) + } + x86OriginalGPInitializedFormatObjects(t, initialized, target.arch, target.triple) + t.Logf("%d original Go formats; %d private LLVM formats; %d explicit physical-emitter rejections; %d exact-row initialized counterparts; %d independent physical/static/address roles", + count, len(formats.Funcs), physicalWrites, len(initialized), privateOnly) + }) + } +} + +// These test-only inputs name complete Go sources; they carry no product +// permission and are never substituted into the original format inventory. +type x86OriginalGPInitializedFormat struct { + source, declaration string +} + +// Keep original row identity, exact displacement, segment and operand order. +// A real NUM producer is separate from an opaque incoming pointer identity. +func x86BitTestOriginalInitializedFormat(arch, name, body string, form x86BitTestForm) (x86OriginalGPInitializedFormat, string) { + var numerical []Reg + if form.index.Kind == OpReg { + numerical = append(numerical, form.index.Reg) + } + pointer := Reg("") + switch form.destination.Kind { + case OpReg: + numerical = append(numerical, form.destination.Reg) + case OpMem: + memory := form.destination.Mem + if memory.Sym != "" { + return x86OriginalGPInitializedFormat{}, "static bit string needs its independently bound source/storage role" + } + if memory.Segment != "" { + // The hardware base is implicit; an explicit GP is its NUM + // offset, never an opaque incoming-pointer identity. + if memory.Base != "" { + numerical = append(numerical, memory.Base) + } + } else { + if memory.Base == "" { + return x86OriginalGPInitializedFormat{}, "absolute numerical address has no ordinary opaque-pointer entry role" + } + pointer = memory.Base + } + if memory.Index != "" { + numerical = append(numerical, memory.Index) + } + default: + return x86OriginalGPInitializedFormat{}, "absolute/SB operand keeps its independent source-bound address role" + } + if pointer != "" && x86OriginalGPPhysicalRegister(pointer, arch) { + return x86OriginalGPInitializedFormat{}, "physical source frame register needs an independent registered entry" + } + for _, register := range numerical { + if x86OriginalGPPhysicalRegister(register, arch) { + return x86OriginalGPInitializedFormat{}, "physical source frame register cannot be fabricated as ordinary NUM" + } + if register == pointer { + return x86OriginalGPInitializedFormat{}, "one opaque address identity cannot also provide numerical source/index bytes" + } + } + move, word := "MOVQ", 8 + if arch == "386" { + move, word = "MOVL", 4 + } + var prefix strings.Builder + arguments, declaration := 0, "func "+name+"()" + if pointer != "" { + arguments = word + declaration = "func " + name + "(p *[64]byte)" + fmt.Fprintf(&prefix, "%s p+0(FP),%s\n", move, pointer) + } + seen := make(map[Reg]bool) + for _, register := range numerical { + if !seen[register] { + fmt.Fprintf(&prefix, "%s $1,%s\n", move, register) + seen[register] = true + } + } + source := fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n%s%s\nRET\n", name, arguments, prefix.String(), body) + return x86OriginalGPInitializedFormat{source, declaration}, "" +} + +// This ordinary source observer has no physical SP/amd64BP entry role. +// BP remains an ordinary GP in Go's mode32 source contract. +func x86OriginalGPPhysicalRegister(register Reg, arch string) bool { + return register == SP || arch == "amd64" && register == BP +} + +// Batch complete original-format counterparts, never one LLVM process per +// row. All original bytes and private objects remain independently owned. +func x86OriginalGPInitializedFormatObjects(t *testing.T, fixtures []x86OriginalGPInitializedFormat, arch, triple string) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for start := 0; start < len(fixtures); start += 32 { + end := start + 32 + if end > len(fixtures) { + end = len(fixtures) + } + t.Run(fmt.Sprintf("initialized/batch%02d", start/32), func(t *testing.T) { + var source, declarations strings.Builder + for _, fixture := range fixtures[start:end] { + source.WriteString(fixture.source) + declarations.WriteString(fixture.declaration + "\n") + } + x86LEAActualTargetGoObject(t, arch, triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), arch, triple, api) + if err != nil || ir == "" { + t.Fatal("same original instruction with complete ordinary inputs", err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + }) + } + }) + } +} + +func TestX86BitTestMalformedAndModeForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86BitTestOracleNames(arch) { + for _, row := range []string{"$128,AX", "$-129,AX", "$1.0,AX", "AL,AX", "X0,AX", "AX,X0", "AX,$1", "AX", "(AX),BX", "$1,(AX)(SP*2)"} { + body := op + " " + row + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+body+"\nRET\n", "", false) + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86BitTestProperties(ins.Op, ins, arch); !known || err == nil { + t.Errorf("malformed original form admitted: %s %s", arch, body) + } + } + for _, mutate := range []func(*Instr){ + func(i *Instr) { i.Op += ".Z" }, + func(i *Instr) { i.Args[0].ImmRaw = "unknown" }, + func(i *Instr) { i.Args[1].Mem.OffRaw = "unknown" }, + func(i *Instr) { i.Args[1].Mem.Index = CX; i.Args[1].Mem.Scale = 3 }, + } { + ins := x86PureGPUnknownInstruction(t, op+" $1,(DX)") + mutate(&ins) + if _, known, err := x86BitTestProperties(ins.Op, ins, arch); !known || err == nil { + t.Errorf("typed malformed shape admitted: %s %+v", arch, ins) + } + } + } + } + for _, body := range []string{"BTQ AX,BX", "BTCQ $1,(AX)", "BTRQ AX,BX", "BTSQ AX,BX", "BTL R8,AX", "BTSL AX,R15"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+body+"\nRET\n", "", false) + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86BitTestProperties(ins.Op, ins, "386"); !known || err == nil { + t.Error("386 accepted unavailable native encoding", body) + } + } +} diff --git a/amd64_bittest_fault_runtime_unix_test.go b/amd64_bittest_fault_runtime_unix_test.go new file mode 100644 index 000000000..313681c37 --- /dev/null +++ b/amd64_bittest_fault_runtime_unix_test.go @@ -0,0 +1,127 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// This root is required by the Linux cross-runtime matrix. Local Rosetta +// observations are diagnostic, not a replacement for native AMD64 execution. +func TestCrossLinuxRuntimeMatrixX86BitTestDeadMemoryFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, native, prototypes, pairs strings.Builder + count := 0 + for _, op := range x86BitTestOracleNames("amd64") { + for _, index := range []string{"$0", "CX"} { + name, original := fmt.Sprintf("dead%d", count), fmt.Sprintf("original%d", count) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),AX\nMOVQ index+8(FP),CX\n%s %s,(AX)\nRET\n", name, op, index) + fmt.Fprintf(&declarations, "func %s(p *byte,index int64)\n", name) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\nMOVQ SI,CX\n"+op+" "+index+",(DI)\nRET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,int64_t),%s(uint8_t*,int64_t);\n", name, original) + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + indexed, writes := 0, 1 + if index == "CX" { + indexed = 1 + } + if len(op) == 3 { + writes = 0 + } + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d},\n", original, name, width, indexed, writes) + count++ + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + main := ` +#include +#include +#include +#include +#include +#include +#include +#include +` + prototypes.String() + native.String() + ` +typedef void (*program)(uint8_t*,int64_t); +static const struct { program original,translated; int width,indexed,writes; } pairs[]={ +` + pairs.String() + `}; +static int outcome(program fn,uint8_t *p,int64_t index) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + fn(p,index); _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS)) return 1; + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + uint8_t *data=mmap(0,3*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(data==MAP_FAILED || mprotect(data+2*page,page,PROT_NONE)) return 2; + memset(data,0xff,2*page); + for(unsigned i=0;i 127 { + return fail("bit index is outside Go's signed Yi8 class") + } + } else if form.index.Kind != OpReg || !isX86YrlRegisterForArch(form.index.Reg, arch) { + return fail("bit index is outside Go's Yi8/Yrl rows") + } + if form.destination.Kind == OpReg { + if !isX86YrlRegisterForArch(form.destination.Reg, arch) { + return fail("destination is outside Go's Yml register class") + } + } else if !x86ScalarCompareMemoryValid(form.destination, arch) { + if x86SourceTLSMemoryRole(form.destination, arch) { + return form, true, fmt.Errorf("%w: bit string needs its actual Go TLS address contract", ErrProbeNeedsContext) + } + return fail("destination is outside Go's encodable Yml memory class") + } + return form, true, nil +} + +func x86TypedBitTestByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86BitTestProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.bits / 8) + return x86TypedByteEffect{ + operands: []x86TypedByteOperand{ + {index: 0, operand: form.index, width: width, read: true}, + {index: 1, operand: form.destination, width: width, read: true, write: form.stem != "BT"}, + }, + flagsDefined: x86ByteFlagCF, + flagsKilled: x86ByteFlagOF | x86ByteFlagSF | x86ByteFlagAF | x86ByteFlagPF, + dynamicMemory: form.index.Kind == OpReg && form.destination.Kind != OpReg, + }, true, nil +} diff --git a/amd64_bittest_fp_runtime_test.go b/amd64_bittest_fp_runtime_test.go new file mode 100644 index 000000000..48351829c --- /dev/null +++ b/amd64_bittest_fp_runtime_test.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86BitTestFPPartialWriteback(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, checks, goChecks strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, op := range x86BitTestOracleNames("amd64") { + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + for offset := 0; offset+width <= 8; offset++ { + name := fmt.Sprintf("Partial%d", count) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\n%s $-1,x+%d(FP)\nMOVQ x+0(FP),AX\nMOVQ AX,ret+8(FP)\nRET\n", name, op, offset) + fmt.Fprintf(&declarations, "func %s(x uint64) uint64\n", name) + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint64_t);\n", name) + mask := uint64(1) << uint(8*(offset+width)-1) + cWant, goWant := "x", "x" + switch op[:len(op)-1] { + case "BTC": + cWant, goWant = fmt.Sprintf("(x^UINT64_C(0x%x))", mask), fmt.Sprintf("x^uint64(0x%x)", mask) + case "BTR": + cWant, goWant = fmt.Sprintf("(x&~UINT64_C(0x%x))", mask), fmt.Sprintf("x&^uint64(0x%x)", mask) + case "BTS": + cWant, goWant = fmt.Sprintf("(x|UINT64_C(0x%x))", mask), fmt.Sprintf("x|uint64(0x%x)", mask) + } + fmt.Fprintf(&checks, "if(%s(x)!=%s) {fprintf(stderr,\"FP partial %d mismatch\\n\");return 1;}\n", name, cWant, count) + fmt.Fprintf(&goChecks, "if got,want:=%s(x),%s;got!=want {t.Fatalf(\"FP partial %d got=%%x want=%%x\",got,want)}\n", name, goWant, count) + count++ + } + } + if count != 52 { + t.Fatalf("partial FP byte-span denominator changed: %d", count) + } + const patterns = "0,1,0x8000000000000000,0xffffffffffffffff,0x55aa0123cdef9876,0x7ff8000000000001" + original := "package oracle\nimport \"testing\"\nfunc TestOriginal(t *testing.T) { for _,x:=range []uint64{" + patterns + "} {\n" + goChecks.String() + "}}\n" + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), source.String(), original) + main := "#include \n#include \n" + prototypes.String() + + "int main(void) {const uint64_t patterns[]={" + patterns + "};\nfor(unsigned i=0;i<6;++i) {uint64_t x=patterns[i];\n" + checks.String() + + "}\nputs(\"52 partial FP programs x 6 complete values\");return 0;}\n" + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), strings.TrimPrefix(declarations.String(), "package oracle\n"), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, pipeline := range []struct{ level, ir string }{{"-O0", ir}, {"-O2", x86MulDivRoundOptimize(t, ir)}} { + output := execution.compileAndRunLLVM(t, "bit-test-fp-partial", pipeline.ir, main, pipeline.level) + if string(output) != "52 partial FP programs x 6 complete values\n" { + t.Fatalf("missing complete original/LLVM FP value oracle: %s", output) + } + } + }) + } +} diff --git a/amd64_bittest_runtime_test.go b/amd64_bittest_runtime_test.go new file mode 100644 index 000000000..e98214360 --- /dev/null +++ b/amd64_bittest_runtime_test.go @@ -0,0 +1,139 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The original Go machine bytes and this independent whole-byte reference +// agree separately with LLVM. CF and preserved ZF are compared; the ISA's +// undefined OF/SF/AF/PF are deliberately not assigned invented expectations. +func TestCrossLinuxRuntimeMatrixX86BitTestNumericalAndAliases(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, native, prototypes, pairs strings.Builder + count := 0 + for _, op := range x86BitTestOracleNames("amd64") { + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + operation := map[string]int{"BT": 0, "BTC": 1, "BTR": 2, "BTS": 3}[op[:len(op)-1]] + for _, memory := range []bool{false, true} { + for _, index := range []string{"CX", "$-128", "$-1", "$0", "$15", "$16", "$31", "$32", "$63", "$64", "$127", "DX"} { + if memory && index == "DX" { + continue // DX/DX is the separate register alias axis. + } + name, original := fmt.Sprintf("numeric%d", count), fmt.Sprintf("original%d", count) + destination, originalDestination, after := "DX", "DX", "MOVQ DX,(DI)\n" + memoryFlag, mode, immediate := 0, 0, 0 + if memory { + destination, originalDestination, after, memoryFlag = "(DI)", "(DI)", "", 1 + } + if strings.HasPrefix(index, "$") { + mode = 1 + if _, err := fmt.Sscanf(index, "$%d", &immediate); err != nil { + t.Fatal(err) + } + } else if index == "DX" { + mode = 2 + } + body := op + " " + index + "," + destination + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-32\nMOVQ p+0(FP),DI\nMOVQ index+8(FP),CX\nMOVQ value+16(FP),DX\nCMPQ DX,DX\n%s\nPUSHFQ\nPOPQ AX\nANDQ $65,AX\n%sMOVQ AX,ret+24(FP)\nRET\n", name, body, after) + fmt.Fprintf(&declarations, "func %s(p *byte,index int64,value uint64) uint64\n", name) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\nMOVQ SI,CX\nCMPQ DX,DX\n"+op+" "+index+","+originalDestination+"\nPUSHFQ\nPOPQ AX\nANDQ $65,AX\n"+after+"RET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint8_t*,int64_t,uint64_t);\n", name) + prototypes.WriteString(x86OriginalSysVDeclaration("uint64_t", original, "uint8_t*,int64_t,uint64_t")) + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d,%d,%d},\n", original, name, width, operation, memoryFlag, mode, immediate) + count++ + } + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + main := ` +#include +#include +#include +` + prototypes.String() + native.String() + + x86OriginalSysVPointer("uint64_t", "originalProgram", "uint8_t*,int64_t,uint64_t") + ` +typedef uint64_t (*program)(uint8_t*,int64_t,uint64_t); +static const struct { originalProgram original; program translated; int width,op,memory,mode,immediate; } pairs[]={ +` + pairs.String() + `}; +int main(void) { + const int64_t indexes[]={-255,-129,-128,-65,-64,-33,-32,-17,-16,-1,0,1,15,16,31,32,63,64,65,127,255}; + const uint64_t values[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000), + UINT64_C(0x55aa0123cdef9876),UINT64_C(0x12345678ffffffff)}; + for(unsigned i=0;i=0?signedIndex/bits:-1-((-1-signedIndex)/bits))*pairs[i].width; + } + uint64_t old=values[v]; + if(pairs[i].memory) { + old=0; + memcpy(&old,expected+257+offset,pairs[i].width); + } + uint64_t selected=UINT64_C(1)<<(index&(bits-1)); + uint64_t flags=64|!!(old&selected),result=old; + if(pairs[i].op==1) result^=selected; + if(pairs[i].op==2) result&=~selected; + if(pairs[i].op==3) result|=selected; + if(pairs[i].memory) { + if(pairs[i].op) memcpy(expected+257+offset,&result,pairs[i].width); + } else { + if(pairs[i].op && bits==32) result=(uint32_t)result; + memcpy(expected+257,&result,8); + } + uint64_t a=pairs[i].original(original+257,indexes[j],values[v]); + uint64_t b=pairs[i].translated(translated+257,indexes[j],values[v]); + if(a!=flags || b!=flags || memcmp(original,expected,sizeof(expected)) || memcmp(translated,expected,sizeof(expected))) { + fprintf(stderr,"bit test=%u index=%lld value=%llx expected flags=%llx Go=%llx LLVM=%llx\n", + i,(long long)indexes[j],(unsigned long long)values[v],(unsigned long long)flags,(unsigned long long)a,(unsigned long long)b); + return 1; + } + } + } + } + printf("%zu original Go/LLVM bit tests x 126 independent whole-memory, alias and CF/ZF checks\n",sizeof(pairs)/sizeof(*pairs)); + return 0; +} + +` + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, pipeline := range []struct{ name, level, ir string }{ + {"backend_O0", "-O0", ir}, + {"IR_O2/backend_O2", "-O2", x86MulDivRoundOptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "bit-test-numerical", pipeline.ir, main, pipeline.level) + want := fmt.Sprintf("%d original Go/LLVM bit tests x 126 independent whole-memory, alias and CF/ZF checks\n", count) + if string(output) != want { + t.Fatalf("missing independent original/LLVM numerical receipt: %s", output) + } + }) + } + }) + } +} diff --git a/amd64_bittest_typed_test.go b/amd64_bittest_typed_test.go new file mode 100644 index 000000000..eda656e7f --- /dev/null +++ b/amd64_bittest_typed_test.go @@ -0,0 +1,71 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func x86BitTestOracleNames(arch string) []string { + var names []string + for _, stem := range []string{"BT", "BTC", "BTR", "BTS"} { + for _, suffix := range "WLQ" { + if arch != "386" || suffix != 'Q' { + names = append(names, stem+string(suffix)) + } + } + } + return names +} + +func TestX86BitTestDeadReadSurvivesIROptimization(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + move, word, triple := "MOVQ", 8, "x86_64-unknown-linux-gnu" + if arch == "386" { + move, word, triple = "MOVL", 4, "i386-unknown-linux-gnu" + } + for _, op := range x86BitTestOracleNames(arch) { + t.Run(arch+"/"+op, func(t *testing.T) { + source := fmt.Sprintf("TEXT ·dead(SB),4,$0-%d\n%s p+0(FP),AX\n%s $0,(AX)\nRET\n", word, move, op) + goAssemblerOracleOutput(t, arch, source, "", true) + ir, err := x86ScalarFPBitViewAPI(t, source, "func dead(p *byte)", arch, triple, "text") + if err != nil { + t.Fatal(err) + } + optimized := x86MulDivRoundOptimize(t, ir) + if !strings.Contains(optimized, "load volatile") { + t.Fatalf("Go's required bit-string read/fault was removed by LLVM IR O2:\n%s", optimized) + } + if strings.HasPrefix(op, "BTC") || strings.HasPrefix(op, "BTR") || strings.HasPrefix(op, "BTS") { + if !strings.Contains(optimized, "store volatile") { + t.Fatal("native bit-string write is not independently observable") + } + } + }) + } + } +} + +func TestX86BitTest386ImmediateClassification(t *testing.T) { + for _, op := range x86BitTestOracleNames("386") { + // Go mode32 classifies its int32 value before selecting Yi8. This + // witness is legal despite the original source literal exceeding 32 bits. + source := "TEXT ·value(SB),4,$0-0\n" + op + " $4294967295,AX\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", true) + if _, err := x86ScalarFPBitViewAPI(t, source, "func value()", "386", "i386-unknown-linux-gnu", "text"); err != nil { + t.Errorf("actual Go signed-Yi8 classification %s: %v", op, err) + } + } +} + +func TestX86BitTestDynamicFPDoesNotBecomeModulo(t *testing.T) { + for _, op := range x86BitTestOracleNames("amd64") { + source := "TEXT ·value(SB),4,$0-16\nMOVQ index+8(FP),CX\n" + op + " CX,x+0(FP)\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + _, err := x86ScalarFPBitViewAPI(t, source, "func value(x uint64,index int64)", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("dynamic native bit string borrowed fixed FP slot/modulo: %s: %v", op, err) + } + } +} diff --git a/amd64_blend_minpos_forms.go b/amd64_blend_minpos_forms.go new file mode 100644 index 000000000..2cef580ef --- /dev/null +++ b/amd64_blend_minpos_forms.go @@ -0,0 +1,92 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// These two Go tables share a full vector source and a vector output, not +// their data dependencies. Blends read one sign bit per mask lane and two +// possible data bytes; minimum-position reads all eight unsigned words. +// Neither the grammar nor a native address establishes initialized storage. +type x86BlendMinposForm struct { + source, destination, mask, base int + byteWidth, laneBits int + legacy, minimum bool + memory bool +} + +func x86BlendMinposProperties(op Op, ins Instr, arch string) (x86BlendMinposForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + blend, isBlend := amd64VariableBlendSpecs[Op(name)] + minimum, isMinimum := amd64MinimumPositionSpecs[Op(name)] + form := x86BlendMinposForm{source: 0, destination: 1, mask: -1, base: -1, + byteWidth: 16, laneBits: 16, minimum: isMinimum} + if !isBlend && !isMinimum { + return form, false, nil + } + fail := func() (x86BlendMinposForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside Go blend/minimum-position operand grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || dotted || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + vector := isBlend && blend.vector || isMinimum && minimum.vector + form.legacy = !vector + if isBlend { + form.source, form.mask, form.base, form.destination = 1, 0, 2, 2 + form.laneBits = blend.laneBits + if vector { + form.destination = 3 + } + } + if len(ins.Args) != form.destination+1 { + return fail() + } + if arch == "386" && len(ins.Args) > 3 && !ins.x86Encoded { + // The ordinary Go frontend cannot express this VEX form in mode32. + // Only a validated physical decoder supplies its distinct grammar; + // original bytes and all producer inputs still need whole replay. + return fail() + } + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if form.byteWidth != 16 && (form.byteWidth != 32 || form.legacy || form.minimum) { + return fail() + } + register := func(operand Operand) bool { + if !vector { + index, valid := amd64ParseXReg(operand.Reg) + return operand.Kind == OpReg && valid && index < 16 && (arch != "386" || index < 8) + } + return amd64VEXVectorRegister(operand, form.byteWidth) + } + if !register(ins.Args[form.destination]) || + isBlend && (!register(ins.Args[form.mask]) || !register(ins.Args[form.base]) || + form.legacy && ins.Args[form.mask].Reg != "X0") { + return fail() + } + source := ins.Args[form.source] + form.memory = isAMD64MemoryOperand(source) + if !register(source) && (!form.memory || !x86PackedShiftMemoryValid(source, arch)) { + return fail() + } + return form, true, x86Mode32VectorRegisterContext(ins, arch) +} + +func x86TypedBlendMinposByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86BlendMinposProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.source, operand: ins.Args[form.source], width: int64(form.byteWidth), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), write: true}, + }} + if !form.minimum { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: int64(form.byteWidth), read: true}, + x86TypedByteOperand{index: form.base, operand: ins.Args[form.base], width: int64(form.byteWidth), read: true}) + } + return effect, true, nil +} diff --git a/amd64_blend_minpos_native.go b/amd64_blend_minpos_native.go new file mode 100644 index 000000000..d3be27dc8 --- /dev/null +++ b/amd64_blend_minpos_native.go @@ -0,0 +1,52 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// All four legacy m128 names use Intel Type4, unlike their VEX forms. +// An unaligned load or align16 metadata cannot retain the original #GP. +// One sideeffect opcode also retains the read for dead/zero-mask results. +func (c *amd64Ctx) lowerLegacyBlendMinposMemory(ins Instr, form x86BlendMinposForm) error { + source := ins.Args[form.source] + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || source.Kind == OpFP { + return fmt.Errorf("%w: legacy blend/minimum m128 needs its original native x86 storage and alignment", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + mnemonic := strings.ToLower(string(ins.Op)) + result := c.newTmp() + if form.minimum { + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(%s %s)\n", + result, mnemonic+" "+segment+"($1), $0", "=x,r,~{memory}", pointerType, pointer) + } else { + // The original implicit X0 is an independent input even when the + // source destination aliases X0. A temporary early-clobber output + // prevents LLVM's carrier allocation from overwriting that input. + mask, failure := c.loadPackedCompareBytes(ins.Args[form.mask], 16) + if failure != nil { + return failure + } + base, failure := c.loadPackedCompareBytes(ins.Args[form.base], 16) + if failure != nil { + return failure + } + maskCarrier, baseCarrier := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x float>\n", maskCarrier, mask) + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x float>\n", baseCarrier, base) + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(<4 x float> %%%s, %s %s, <4 x float> %%%s)\n", + result, mnemonic+" "+segment+"($2), $0", "=&x,0,r,{xmm0},~{memory}", baseCarrier, pointerType, pointer, maskCarrier) + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <16 x i8>\n", bytes, result) + return c.storeLegacyXViews(ins.Args[form.destination].Reg, "%"+bytes) +} diff --git a/amd64_blend_variable_runtime_test.go b/amd64_blend_variable_runtime_test.go new file mode 100644 index 000000000..583df9eba --- /dev/null +++ b/amd64_blend_variable_runtime_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86VariableBlendMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for _, family := range x86VariableBlendTestFamilies { + for _, width := range []int{0, 16, 32} { + for _, memory := range []bool{false, true} { + for dst := 0; dst < 4; dst++ { + name := fmt.Sprintf("blend%d", index) + index++ + reg, move, op, size := "X", "MOVOU", family.legacy, 16 + if width > 0 { + op, size = family.vector, width + } + if width == 32 { + reg, move = "Y", "VMOVUPS" + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ base+0(FP),AX\nMOVQ source+8(FP),BX\nMOVQ mask+16(FP),CX\nMOVQ out+24(FP),DI\n") + for r := 0; r < 4; r++ { + fmt.Fprintf(&source, "VMOVUPS (AX),Z%d\n", r) + } + fmt.Fprintf(&source, "%s (CX),%s0\n%s (BX),%s1\n", move, reg, move, reg) + operand := reg + "1" + if memory { + operand = "(BX)" + } + line := fmt.Sprintf("%s %s0,%s,%s%d", op, reg, operand, reg, dst) + if width > 0 { + line = fmt.Sprintf("%s %s0,%s,%s2,%s%d", op, reg, operand, reg, reg, dst) + } + emitX86GoEncodedTestInstruction(t, &source, line, raw) + fmt.Fprintf(&source, "VMOVUPS Z%d,(DI)\nVMOVUPS Y%d,64(DI)\nVMOVUPS X%d,96(DI)\nRET\n", dst, dst, dst) + sigs[name] = FuncSig{ + Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void*, const void*, const void*, void*);\n", name) + fmt.Fprintf(&checks, " if (check(%s, %d, %d, %d, %d)) {\n", name, size, family.bits/8, width, dst) + fmt.Fprintf(&checks, " fprintf(stderr, \"%s failed\\n\");\n return 1;\n }\n", name) + } + } + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source.String()) + // The unchanged C intentionally asks for unaligned numerical + // values. Execute its explicit private mathematical companion, + // not the original legacy memory opcode's different #GP contract. + file = x86BlendMinposPrivateMathCompanion(t, source.String(), 72, 12, 96) + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + ir, err = x86BlendMinposMathIR(file, ir) + if err != nil { + t.Fatal("private mathematical companion lost exact source/IR ownership", err) + } + body := "#include \n#include \n#include \n" + + x86TestGuardPagesC + declarations.String() + x86VariableBlendOracleC + + "\nint main(void) {\n if (setup_guard()) return 2;\n" + + checks.String() + " return free_guard();\n}\n" + x86ConflictFunnelPortableRuntime(t, llc, clang, "variable-blend-private-math", triple, ir, body) + }) + } +} + +const x86VariableBlendOracleC = ` +static int check(void (*fn)(const void*, const void*, const void*, void*), + int size, int lane, int vex, int dst) { + for (int n = 0; n < 256; n++) { + uint8_t old[65], storage[33], masks[33], out[112], want[112] = {0}; + uint8_t *input = storage + 1, *mask = masks + 1; + for (int i = 0; i < 65; i++) old[i] = (uint8_t)(i * 3 + n); + for (int i = 0; i < size; i++) { + input[i] = (uint8_t)(i * 7 + n + 83); + mask[i] = (uint8_t)(n + i * 31); + } + const uint8_t *base = old + 1; + if (!vex && dst == 0) base = mask; + if (!vex && dst == 1) base = input; + for (int i = 0; i < size; i += lane) { + int enabled = mask[i + lane - 1] >> 7; + memcpy(want + i, (enabled ? input : base) + i, (size_t)lane); + } + if (!vex) memcpy(want + 16, old + 1 + 16, 48); + memcpy(want + 64, want, 32); + memcpy(want + 96, want, 16); + + for (int edge = 0; edge < 2; edge++) { + const uint8_t *source = input; + if (edge) { + source = guard + page - size; + memcpy(guard + page - size, input, (size_t)size); + } + memset(out, 0xa5, sizeof(out)); + fn(old + 1, source, mask, out); + if (memcmp(out, want, sizeof(out))) { + fprintf(stderr, "size=%d lane=%d vex=%d dst=%d n=%d edge=%d\n", + size, lane, vex, dst, n, edge); + return 1; + } + } + } + return 0; +} +` diff --git a/amd64_blocks.go b/amd64_blocks.go index 19e9cba47..5d051cb05 100644 --- a/amd64_blocks.go +++ b/amd64_blocks.go @@ -6,38 +6,91 @@ import ( ) type amd64Block struct { - name string - instrs []Instr + name string + instrs []Instr + synthetic bool +} + +func isAMD64ConditionalBranch(op Op) bool { + op = Op(strings.ToUpper(string(op))) + if _, known := x86CounterControlPropertiesForOpcode(op); known { + return true + } + return x86BranchCondition(op) != "" +} + +// Branch lowering, byte dependencies and closed-loop predicates share the +// complete condition-alias family. Counter-zero branches have a different role. +func x86BranchCondition(op Op) string { + switch op { + case "JE", "JEQ", "JZ": + return "EQ" + case "JNE", "JNZ": + return "NE" + case "JL", "JLT", "JNGE": + return "LT" + case "JGE", "JNL": + return "GE" + case "JS", "JMI": + return "MI" + case "JNS", "JPL": + return "PL" + case "JLE", "JNG": + return "LE" + case "JG", "JGT", "JNLE": + return "GT" + case "JB", "JLO", "JC", "JCS", "JNAE": + return "CS" + case "JNC", "JCC", "JAE", "JHS", "JNB": + return "CC" + case "JBE", "JLS", "JNA": + return "LS" + case "JA", "JHI", "JNBE": + return "HI" + case "JO", "JOS": + return "OS" + case "JNO", "JOC": + return "OC" + case "JP", "JPE", "JPS": + return "PS" + case "JNP", "JPO", "JPC": + return "PC" + } + return "" +} + +func isAMD64CounterZeroBranch(op Op) bool { + spec, known := x86CounterControlPropertiesForOpcode(op) + return known && !spec.decrement } func amd64SplitBlocks(fn Func) []amd64Block { - blocks := []amd64Block{{name: "entry"}} + names := newAssemblyBlockNames(fn) + blocks := []amd64Block{{name: names.fresh("entry"), synthetic: true}} cur := 0 anon := 0 isPCRelTarget := func(ins Instr) (off int64, ok bool) { op := strings.ToUpper(string(ins.Op)) - switch Op(op) { - case "JMP", - "JE", "JEQ", "JZ", "JNE", "JNZ", - "JL", "JLT", "JLE", "JG", "JGT", "JGE", "JS", "JNS", - "JB", "JLO", "JBE", "JA", "JHI", "JAE", "JHS", "JLS", "JNA", - "JNC", "JC", "JCC": - default: + if op != "JMP" && op != "LOOP" && op != "XBEGIN" && !isAMD64ConditionalBranch(Op(op)) { return 0, false } - if len(ins.Args) != 1 || ins.Args[0].Kind != OpMem { + args := ins.Args + if op != "JMP" && len(args) == 2 && args[0].Kind == OpImm { + args = args[1:] + } + if len(args) != 1 || args[0].Kind != OpMem { return 0, false } - if !strings.EqualFold(string(ins.Args[0].Mem.Base), "PC") { + if !strings.EqualFold(string(args[0].Mem.Base), "PC") { return 0, false } - return ins.Args[0].Mem.Off, true + return args[0].Mem.Off, true } startAnon := func() { anon++ - blocks = append(blocks, amd64Block{name: fmt.Sprintf("anon_%d", anon)}) + blocks = append(blocks, amd64Block{name: names.fresh(fmt.Sprintf("anon_%d", anon)), synthetic: true}) cur = len(blocks) - 1 } @@ -46,16 +99,16 @@ func amd64SplitBlocks(fn Func) []amd64Block { return true } op := strings.ToUpper(string(ins.Op)) - switch Op(op) { - case "JMP", - "JE", "JEQ", "JZ", "JNE", "JNZ", - "JL", "JLT", "JLE", "JG", "JGT", "JGE", "JS", "JNS", - "JB", "JLO", "JBE", "JA", "JHI", "JAE", "JHS", "JLS", "JNA", - "JNC", "JC", "JCC": + if _, ok := amd64FarReturnSpecs[Op(op)]; ok { + return true + } + if spec, ok := amd64SystemTransferSpecs[Op(op)]; ok && spec.terminates() { + return true + } + if op == "JMP" || op == "LOOP" || op == "XBEGIN" || isAMD64ConditionalBranch(Op(op)) { return true - default: - return false } + return false } linear := make([]Instr, 0, len(fn.Instrs)) @@ -79,8 +132,9 @@ func amd64SplitBlocks(fn Func) []amd64Block { for _, ins := range fn.Instrs { if ins.Op == OpLABEL && len(ins.Args) == 1 && ins.Args[0].Kind == OpLabel { lbl := ins.Args[0].Sym - if len(blocks[cur].instrs) == 0 && strings.HasPrefix(blocks[cur].name, "anon_") { + if cur != 0 && len(blocks[cur].instrs) == 0 && blocks[cur].synthetic { blocks[cur].name = lbl + blocks[cur].synthetic = false continue } blocks = append(blocks, amd64Block{name: lbl}) @@ -97,15 +151,22 @@ func amd64SplitBlocks(fn Func) []amd64Block { } } - if len(blocks) > 1 && len(blocks[len(blocks)-1].instrs) == 0 && strings.HasPrefix(blocks[len(blocks)-1].name, "anon_") { + if len(blocks) > 1 && len(blocks[len(blocks)-1].instrs) == 0 && blocks[len(blocks)-1].synthetic { blocks = blocks[:len(blocks)-1] } return blocks } func amd64LLVMBlockName(src string) string { + return "bb_" + amd64LLVMSafeName(src) +} + +// Keep block names in a namespace distinct from SSA temporaries and register +// allocas. Go permits local labels such as x1 and t1, which otherwise collide +// with the translator's %x1 and %t1 values in LLVM IR. +func amd64LLVMSafeName(src string) string { if src == "" { - return "bb" + return "empty" } var out []byte for i := 0; i < len(src); i++ { @@ -115,16 +176,15 @@ func amd64LLVMBlockName(src string) string { continue } out = append(out, '_') + out = append(out, "0123456789abcdef"[ch>>4]) + out = append(out, "0123456789abcdef"[ch&0xf]) } s := string(out) if s == "" { - return "bb" + return "empty" } if s[0] >= '0' && s[0] <= '9' { - s = "bb_" + s - } - if s == "entry" { - s = "bb_entry" + s = "n_" + s } return s } diff --git a/amd64_bls_forms.go b/amd64_bls_forms.go new file mode 100644 index 000000000..f29c012ce --- /dev/null +++ b/amd64_bls_forms.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86BLSKind uint8 + +const ( + x86BLSIsolate x86BLSKind = iota + x86BLSMask + x86BLSReset +) + +type x86BLSSpec struct { + bits int + kind x86BLSKind + digit byte +} + +var x86BLSSpecs = map[Op]x86BLSSpec{ + "BLSIL": {32, x86BLSIsolate, 3}, + "BLSIQ": {64, x86BLSIsolate, 3}, + "BLSMSKL": {32, x86BLSMask, 2}, + "BLSMSKQ": {64, x86BLSMask, 2}, + "BLSRL": {32, x86BLSReset, 1}, + "BLSRQ": {64, x86BLSReset, 1}, +} + +type x86BLSForm struct { + spec x86BLSSpec + source, destination Operand +} + +// The complete Go Yml,Yrl family uses VEX.LZ/0F38/F3. W is ignored in +// mode32; retain the original Q spelling, but consume its physical width. +func x86BLSProperties(op Op, ins Instr, arch string) (x86BLSForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86BLSSpecs[Op(name)] + form := x86BLSForm{spec: spec} + if !known { + return form, false, nil + } + fail := func() (x86BLSForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go BLS grammar: %q", arch, op, ins.Raw) + } + if arch != "386" && arch != "amd64" || dotted || len(ins.Args) != 2 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return fail() + } + if arch == "386" { + form.spec.bits = 32 + } + form.source, form.destination = ins.Args[0], ins.Args[1] + if form.source.Kind == OpReg { + if !isX86YrlRegisterForArch(form.source.Reg, arch) { + return fail() + } + } else if !isAMD64MemoryOperand(form.source) || !x86PackedShiftMemoryValid(form.source, arch) || + x86NamedMultiplyMemoryError(form.source, arch) != nil { + return fail() + } + if form.destination.Kind != OpReg || !isX86YrlRegisterForArch(form.destination.Reg, arch) { + return fail() + } + return form, true, nil +} + +func x86TypedBLSByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86BLSProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.spec.bits / 8) + return x86TypedByteEffect{ + operands: []x86TypedByteOperand{ + {index: 0, operand: form.source, width: width, read: true}, + {index: 1, operand: form.destination, width: width, write: true}, + }, + flagsDefined: x86ByteFlagCF | x86ByteFlagOF | x86ByteFlagZF | x86ByteFlagSF, + flagsKilled: x86ByteFlagsAll, + requiresResultProof: true, + }, true, nil +} + +// Only the paired exact-width volatile emitter consumes this continuation. +// This creates no pointee, caller, frame or raw-entry capability. +func x86LoweredBLSNativeRead(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) (bool, bool) { + form, known, err := x86BLSProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + if form.source.Kind != OpMem || form.source.Mem.Sym != "" { + return false, false + } + width := int64(form.spec.bits / 8) + if !x86ResultReadWithNativeContinuation(ins, form.source, width, state, owned, arch, true) || + !x86ResultStore(form.destination, width, state, owned, resultOffset, arch) { + return false, true + } + effect, _, _ := x86TypedBLSByteEffect(ins.Op, ins, arch) + state.flags = state.flags&^effect.flagsKilled | effect.flagsDefined + return true, true +} + +func (c *amd64Ctx) evalBLSSource(ins Instr, form x86BLSForm, typ LLVMType) (string, error) { + if form.source.Kind == OpFP && x86ResultNamedPredicateInstruction(ins) { + if value, handled, err := c.loadIntegerFPBytesMode(form.source.FPOffset, typ, true); handled || err != nil { + return value, err + } + } + if isAMD64MemoryOperand(form.source) && form.source.Kind != OpFP && + (c.nativeMemoryOperand(ins, 0) || + x86CallerMemoryCertified(ins, 0) || c.fixedLocalMemoryCertified(ins, 0)) { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return "", fmt.Errorf("%w: BLS needs its original native address size", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.source) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", value, typ, pointerType, pointer) + return "%" + value, nil + } + return c.evalIntSized(form.source, typ) +} diff --git a/amd64_bmi2_shift_386_contract_test.go b/amd64_bmi2_shift_386_contract_test.go new file mode 100644 index 000000000..b74865610 --- /dev/null +++ b/amd64_bmi2_shift_386_contract_test.go @@ -0,0 +1,247 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "reflect" + "strconv" + "strings" + "testing" +) + +type x86BMI2Shift386Program struct { + name, source, declaration, destination string +} + +func x86BMI2Shift386Programs(t *testing.T, source, secondDestination string) []x86BMI2Shift386Program { + t.Helper() + parts := strings.Split(source, "TEXT ") + if len(parts) != 25 || parts[0] != "" { + t.Fatal("complete historical 24-TEXT inventory changed") + } + var programs []x86BMI2Shift386Program + for index, part := range parts[1:] { + name := fmt.Sprintf("shift386_%d", index) + if !strings.HasPrefix(part, name+"(SB),4,$0-12\n") { + t.Fatal("original TEXT order, frame or result extent changed", index) + } + destination := "CX" + if index%2 != 0 { + destination = secondDestination + } + programs = append(programs, x86BMI2Shift386Program{ + name: name, source: "TEXT " + part, destination: destination, + declaration: fmt.Sprintf("func %s(count uint32,source *uint32) uint32\n", name), + }) + } + return programs +} + +// These complete bytes are independent of the translator and raw decoder. +// In mode32 the Q spellings retain Go's W1 bytes, but hardware ignores W. +// The final 89 /r SIB 24 0c store is physically relative to the current ESP, +// even though the original source names the ABI0 result ret+8(FP). +func x86BMI2Shift386ExpectedCode(t *testing.T, index int, destination string) []byte { + t.Helper() + third := []byte{0x7a, 0xfa, 0x79, 0xf9, 0x7b, 0xfb}[index/4] + register := map[string]byte{"CX": 1, "SP": 4, "DX": 2}[destination] + if register == 0 { + t.Fatal("unknown independent destination", destination) + } + code := []byte{0x8b, 0x44, 0x24, 4, 0x8b, 0x5c, 0x24, 8} + code = append(code, 0xc4, 0xe2, third, 0xf7, register<<3|3) + return append(code, 0x89, 0x44|register<<3, 0x24, 12, 0xc3) +} + +func x86BMI2Shift386OriginalGoPrior(t *testing.T, source, secondDestination string) []x86BMI2Shift386Program { + t.Helper() + programs := x86BMI2Shift386Programs(t, source, secondDestination) + listing := x86InsertExtractGoListing(t, "386", source) + codes, instructions := map[string][]byte{}, map[string][]string{} + owner := "" + for _, line := range strings.Split(string(listing), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[1] == "STEXT" { + owner = fields[0] + if dot := strings.LastIndex(owner, "."); dot >= 0 { + owner = owner[dot+1:] + } + if _, duplicate := codes[owner]; duplicate { + t.Fatal("duplicate original Go TEXT owner", owner) + } + codes[owner] = nil + } + if chunk := x86InsertExtractListedBytes(line); len(chunk) != 0 { + pc, err := strconv.ParseInt(fields[0], 0, 64) + if owner == "" || err != nil || pc != int64(len(codes[owner])) { + t.Fatal("unbound or noncontiguous original Go bytes", line, err) + } + codes[owner] = append(codes[owner], chunk...) + } + if row := goAMD64FrameRow.FindStringSubmatch(line); row != nil && !strings.HasPrefix(row[3], "TEXT") { + instructions[owner] = append(instructions[owner], strings.Join(strings.Fields(row[3]), "")) + } + } + if len(codes) != 24 { + t.Fatal("original Go object lost a complete TEXT", len(codes)) + } + for index, program := range programs { + if !bytes.Equal(codes[program.name], x86BMI2Shift386ExpectedCode(t, index, program.destination)) { + t.Fatalf("complete original Go bytes differ for %s (%s): %x", program.name, program.destination, codes[program.name]) + } + rows := instructions[program.name] + if len(rows) < 4 || rows[len(rows)-1] != "RET" { + t.Fatal("original Go listing lost its real return", program.name, rows) + } + if index%4 < 2 && !strings.HasSuffix(rows[len(rows)-3], ","+program.destination) { + t.Fatal("original named instruction no longer writes its physical destination", program.name, rows) + } + } + t.Logf("actual complete original Go object: sourceSHA=%x, 24 functions; physical destination write, ESP+12 result store and RET bytes retained; no original program execution", sha256.Sum256([]byte(source))) + return programs +} + +func TestX86BMI2Shift386OriginalGoPhysicalSPAndReturnContract(t *testing.T) { + source, _, _, _ := x86BMI2Shift386Fixture(t, "SP") + programs := x86BMI2Shift386OriginalGoPrior(t, source, "SP") + stack, ordinary := 0, 0 + for _, program := range programs { + if program.destination == "SP" { + stack++ + } else { + ordinary++ + } + } + if stack != 12 || ordinary != 12 { + t.Fatal("original SP and CX coverage changed", stack, ordinary) + } + // The first original C sample is value=0/count=0. Every shift writes ESP=0; + // the next actual store targets address 12 and RET cannot recover the old PC. + // This is a source/object witness, not a claim that the invalid ABI ran. + if x86LogicalSourceFrameRegister(SP, "386") != true { + t.Fatal("physical SP became an ordinary numerical source register") + } +} + +func TestX86BMI2Shift386OriginalSPStaysContextAndCXWorks(t *testing.T) { + source, _, _, _ := x86BMI2Shift386Fixture(t, "SP") + programs := x86BMI2Shift386OriginalGoPrior(t, source, "SP") + var declarations strings.Builder + for _, program := range programs { + declarations.WriteString(program.declaration) + } + for _, target := range x86RawReturnTargets[:2] { + t.Run(target.triple, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + if ir, err := x86ScalarFPBitViewAPI(t, source, declarations.String(), "386", target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("whole original source acquired an unproved physical stack return", err) + } + for _, program := range programs { + t.Run(program.name, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, program.source, program.declaration, "386", target.triple, api) + if program.destination == "SP" { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("SP result replaced the physical ABI0 continuation with a virtual GP", err) + } + } else if err != nil || ir == "" { + t.Fatal("same complete initialized CX program must remain supported", err) + } + }) + } + }) + } + }) + } +} + +func TestX86BMI2Shift386SafeGPCompanionCompleteObjects(t *testing.T) { + source, signatures, cDeclarations, checks := x86BMI2Shift386Fixture(t, "DX") + programs := x86BMI2Shift386OriginalGoPrior(t, source, "DX") + _, originalSignatures, originalCDeclarations, originalChecks := x86BMI2Shift386Fixture(t, "SP") + if !reflect.DeepEqual(signatures, originalSignatures) || cDeclarations != originalCDeclarations || checks != originalChecks || strings.Count(checks, "if (check(") != 24 { + t.Fatal("safe GP companion changed original signatures, C calls or 24-kernel denominator") + } + var declarations strings.Builder + for _, program := range programs { + declarations.WriteString(program.declaration) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets[:2] { + t.Run(target.triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, "386", target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations.String(), "386", target.triple, api) + if err != nil || ir == "" { + t.Fatal("explicit initialized GP companion did not close", err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + x86PackedStringCompileObject(t, llc, target.triple, text, "-O0") + } + }) + } + }) + } +} + +func TestX86BMI2Shift386RequiredDriverChoosesSafeGPCompanion(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "amd64_bmi2_shift_cross_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + fixtures, driver, literals := 0, "", []string{} + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if !ok || function.Name.Name != "TestCrossLinuxRuntimeMatrixBMI2Shift386" { + continue + } + ast.Inspect(function.Body, func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok { + if name, ok := call.Fun.(*ast.Ident); ok && name.Name == "x86BMI2Shift386Fixture" && len(call.Args) == 2 { + fixtures++ + if literal, ok := call.Args[1].(*ast.BasicLit); ok && literal.Kind == token.STRING { + driver, err = strconv.Unquote(literal.Value) + if err != nil { + t.Fatal(err) + } + } + } + } + if assignment, ok := node.(*ast.AssignStmt); ok && len(assignment.Lhs) == 1 && len(assignment.Rhs) == 1 { + if name, ok := assignment.Lhs[0].(*ast.Ident); ok && name.Name == "mainC" { + ast.Inspect(assignment.Rhs[0], func(node ast.Node) bool { + if literal, ok := node.(*ast.BasicLit); ok && literal.Kind == token.STRING { + value, err := strconv.Unquote(literal.Value) + if err != nil { + t.Fatal(err) + } + literals = append(literals, value) + } + return true + }) + } + } + return true + }) + } + if len(literals) != 3 || fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join(literals, "\x00")))) != "40eda8daa32e89f9f9c077d2cf2ef23e41331377a7fc44f5992c46e9e9228718" { + t.Fatal("original complete C source, 256 samples or page-end/fault oracle changed") + } + if fixtures != 1 || driver != "DX" { + t.Fatalf("required runtime must explicitly execute the safe CX/DX companion, not the original physical-SP source: fixtures=%d role=%q", fixtures, driver) + } + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil || !strings.Contains(string(workflow), "TestCrossLinuxRuntimeMatrix") { + t.Fatal("safe companion lost the required complete cross-runtime owner", err) + } +} diff --git a/amd64_bmi2_shift_386_fixture_test.go b/amd64_bmi2_shift_386_fixture_test.go new file mode 100644 index 000000000..da051b4c6 --- /dev/null +++ b/amd64_bmi2_shift_386_fixture_test.go @@ -0,0 +1,67 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// SP preserves the complete historical source. DX is an explicitly separate +// executable GP companion: it does not certify the original physical-SP rows. +func x86BMI2Shift386Fixture(t *testing.T, secondDestination string) (string, map[string]FuncSig, string, string) { + t.Helper() + if secondDestination != "SP" && secondDestination != "DX" { + t.Fatal("unknown BMI2 386 fixture role", secondDestination) + } + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, op := range x86BMI2ShiftTestOps { + kind := 0 + if strings.HasPrefix(op, "SHRX") { + kind = 1 + } else if strings.HasPrefix(op, "SARX") { + kind = 2 + } + + for _, raw := range []bool{false, true} { + for _, destination := range []string{"CX", secondDestination} { + name := fmt.Sprintf("shift386_%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-12\n", name) + source.WriteString("MOVL count+0(FP),AX\nMOVL source+4(FP),BX\n") + line := fmt.Sprintf("%s AX,(BX),%s\n", op, destination) + if raw { + code := assembleX87ControlBytes(t, "386", "TEXT probe(SB),4,$0-0\n"+line+"RET\n") + for _, b := range code[:len(code)-1] { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + } else { + source.WriteString(line) + } + fmt.Fprintf(&source, "MOVL %s,ret+8(FP)\nRET\n", destination) + + sigs[name] = FuncSig{ + Name: name, + Args: []LLVMType{I32, Ptr}, + Ret: I32, + Frame: FrameLayout{ + Params: []FrameSlot{ + {Offset: 0, Type: I32, Index: 0, Field: -1}, + {Offset: 4, Type: Ptr, Index: 1, Field: -1}, + }, + Results: []FrameSlot{ + {Offset: 8, Type: I32, Index: 0, Field: -1}, + }, + }, + } + fmt.Fprintf(&declarations, "extern uint32_t %s(uint32_t, const uint32_t*);\n", name) + fmt.Fprintf(&checks, " if (check(%s, value, %d)) return %d;\n", name, kind, index) + } + } + } + if index != 24 { + t.Fatalf("generated %d functions, want 24", index) + } + return source.String(), sigs, declarations.String(), checks.String() +} diff --git a/amd64_bmi2_shift_cross_test.go b/amd64_bmi2_shift_cross_test.go new file mode 100644 index 000000000..84b4f6f7a --- /dev/null +++ b/amd64_bmi2_shift_cross_test.go @@ -0,0 +1,78 @@ +package plan9asm + +import ( + "os" + "os/exec" + "runtime" + "testing" +) + +// Keep this in the required CI cross-runtime regex, not an optional native test. +func TestCrossLinuxRuntimeMatrixBMI2Shift386(t *testing.T) { + if os.Getenv("PLAN9ASM_CROSS_EXEC") != "1" { + t.Skip("set PLAN9ASM_CROSS_EXEC=1 for the required Linux cross-runtime matrix") + } + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("cross-runtime driver requires linux/amd64") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, tool := range []string{"i686-linux-gnu-gcc", "qemu-i386"} { + if _, err := exec.LookPath(tool); err != nil { + t.Fatalf("required cross-runtime tool %s: %v", tool, err) + } + } + + // Execute the explicitly separate CX/DX companion. The historical CX/SP + // source remains covered by complete Go objects and public Context tests; + // its unrecovered physical ESP write is not a valid ABI0 return program. + source, sigs, declarations, checks := x86BMI2Shift386Fixture(t, "DX") + + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +#include +#include +` + declarations + x86BMI2ShiftOracleC + ` +static void memory_fault(int signal_number) { + (void)signal_number; + _exit(98); +} + +static int check(uint32_t (*fn)(uint32_t, const uint32_t*), uint32_t *value, int kind) { + uint32_t values[] = {0, 1, UINT32_MAX, 0x80000000U, 0x7fffffffU, 0xfedcba98U}; + for (unsigned n = 0; n < 256; n++) { + *value = values[n % 6]; + uint32_t count = n < 128 ? n : UINT32_MAX - n; + if (fn(count, value) != shift_reference(*value, count, 32, kind)) { + fprintf(stderr, "386 kind=%d case=%u\n", kind, n); + return 1; + } + } + return 0; +} + +int main(void) { + if (signal(SIGSEGV, memory_fault) == SIG_ERR) return 92; + long page = sysconf(_SC_PAGESIZE); + if (page < 4) return 90; + + uint8_t *memory = mmap(NULL, page * 2, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (memory == MAP_FAILED || mprotect(memory + page, page, PROT_NONE)) return 91; + uint32_t *value = (uint32_t *)(memory + page - 4); +` + checks + " return munmap(memory, page * 2) != 0;\n}\n" + compileAndRunRuntimeTestWithCompiler(t, llc, []string{"i686-linux-gnu-gcc"}, "bmi2_shift_386", + "i386-unknown-linux-gnu", ir, mainC, []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"}) +} diff --git a/amd64_bmi2_shift_runtime_test.go b/amd64_bmi2_shift_runtime_test.go new file mode 100644 index 000000000..7be9d70c1 --- /dev/null +++ b/amd64_bmi2_shift_runtime_test.go @@ -0,0 +1,136 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86BMI2ShiftRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } else if runtime.GOARCH != "amd64" { + t.Skip("execution requires amd64 or Rosetta; five-target objects remain required") + } + + source, sigs, declarations, checks := x86BMI2ShiftRuntimeFixture(t) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := x86BMI2ShiftRuntimeDriver(declarations, checks) + compileAndRunRuntimeTestForTarget(t, llc, clang, "bmi2_shift", triple, ir, mainC, prefix) +} + +func x86BMI2ShiftRuntimeFixture(t *testing.T) (string, map[string]FuncSig, string, string) { + t.Helper() + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, op := range x86BMI2ShiftTestOps { + bits, kind := 32, 0 + if strings.HasSuffix(op, "Q") { + bits = 64 + } + if strings.HasPrefix(op, "SHRX") { + kind = 1 + } else if strings.HasPrefix(op, "SARX") { + kind = 2 + } + + for _, raw := range []bool{false, true} { + for _, memory := range []bool{false, true} { + for _, destination := range []string{"CX", "AX", "DX"} { + name := fmt.Sprintf("bmi2shift%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\n", name) + source.WriteString("MOVQ count+0(FP),AX\nMOVQ source+8(FP),BX\nMOVQ out+16(FP),DI\n") + source.WriteString("MOVQ (BX),DX\nMOVQ $-1,CX\nCMPQ AX,AX\nSTC\n") + input := "DX" + if memory { + input = "(BX)" + } + line := fmt.Sprintf("%s AX,%s,%s", op, input, destination) + emitX86GoEncodedTestInstruction(t, &source, line, raw) + fmt.Fprintf(&source, "MOVQ %s,(DI)\n", destination) + source.WriteString("SETCS 8(DI)\nSETEQ 9(DI)\nSETPS 10(DI)\nSETMI 11(DI)\nSETOS 12(DI)\nRET\n") + + sigs[name] = FuncSig{ + Name: name, + Args: []LLVMType{I64, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(uint64_t, const void*, void*);\n", name) + fmt.Fprintf(&checks, " if (check(%s, %d, %d)) return %d;\n", name, bits, kind, index) + } + } + } + } + if index != 72 { + t.Fatalf("generated %d functions, want 72", index) + } + + return source.String(), sigs, declarations.String(), checks.String() +} + +func x86BMI2ShiftRuntimeDriver(declarations, checks string) string { + return "#include \n#include \n#include \n" + + declarations + x86BMI2ShiftOracleC + ` +static int check(void (*fn)(uint64_t, const void*, void*), int bits, int kind) { + uint64_t values[] = {0, 1, UINT64_MAX, UINT64_C(0x8000000080000000), + UINT64_C(0x7fffffff7fffffff), UINT64_C(0xfedcba9876543210)}; + for (unsigned n = 0; n < 256; n++) { + uint8_t input[9], out[13] = {0}; + uint64_t value = values[n % 6]; + uint64_t count = n < 128 ? n : UINT64_MAX - n; + memcpy(input + 1, &value, sizeof(value)); + + fn(count, input + 1, out); + uint64_t got; + memcpy(&got, out, sizeof(got)); + if (got != shift_reference(value, count, bits, kind) || + out[8] != 1 || out[9] != 1 || out[10] != 1 || out[11] || out[12]) { + fprintf(stderr, "bits=%d kind=%d case=%u\n", bits, kind, n); + return 1; + } + } + return 0; +} + +int main(void) { +` + checks + " return 0;\n}\n" +} + +const x86BMI2ShiftOracleC = ` +static uint64_t shift_reference(uint64_t value, uint64_t count, int bits, int kind) { + uint64_t mask = bits == 64 ? UINT64_MAX : UINT32_MAX; + unsigned shift = (unsigned)count & (bits - 1); + value &= mask; + if (kind == 0) { + return (value << shift) & mask; + } + + uint64_t result = value >> shift; + if (kind == 2 && shift && (value & (UINT64_C(1) << (bits - 1)))) { + result |= mask ^ (mask >> shift); + } + return result; +} +` diff --git a/amd64_broadcast_go_frame_test.go b/amd64_broadcast_go_frame_test.go new file mode 100644 index 000000000..2f1035980 --- /dev/null +++ b/amd64_broadcast_go_frame_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86PackedBroadcastGoFrameScalarWidths(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + pointerBytes, movePointer := 8, "MOVQ" + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + pointerBytes, movePointer = 4, "MOVL" + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, form := range []struct { + op string + bytes int + }{ + {"VPBROADCASTB", 1}, {"VPBROADCASTW", 2}, + {"VPBROADCASTD", 4}, {"VPBROADCASTQ", 8}, + } { + t.Run(arch+"/"+form.op, func(t *testing.T) { + outOffset := (form.bytes + pointerBytes - 1) &^ (pointerBytes - 1) + var source, declarations strings.Builder + declarations.WriteString("package broadcast\n") + for _, width := range []string{"X", "Y", "Z"} { + for index, masking := range []struct{ suffix, mask string }{ + {}, {mask: "K1,"}, {suffix: ".Z", mask: "K7,"}, + } { + name := fmt.Sprintf("Broadcast%s%d", width, index) + fmt.Fprintf(&declarations, "func %s(x uint%d, out *[64]byte)\n", name, form.bytes*8) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n", name, outOffset+pointerBytes) + if masking.mask != "" { + fmt.Fprintf(&source, "%s $1,AX\nKMOVQ AX,%s\n", movePointer, strings.TrimSuffix(masking.mask, ",")) + if masking.suffix == "" { + clear := map[string]string{"X": "PXOR X0,X0", "Y": "VPXOR Y0,Y0,Y0", "Z": "VPXORD Z0,Z0,Z0"}[width] + fmt.Fprintln(&source, clear) + } + } + fmt.Fprintf(&source, "%s%s x+0(FP),%s%s0\n", form.op, masking.suffix, masking.mask, width) + fmt.Fprintf(&source, "%s out+%d(FP),AX\n", movePointer, outOffset) + store := map[string]string{"X": "MOVOU", "Y": "VMOVDQU", "Z": "VMOVDQU64"}[width] + fmt.Fprintf(&source, "%s %s0,(AX)\nRET\n", store, width) + } + } + requireX86GoAssemblerResult(t, arch, source.String(), true) + sourceText := source.String() + if arch == "386" { + // KMOVQ's W bit is ignored for a GP transfer in mode32. + // Keep KMOVD as a reference and compile the original below. + sourceText = strings.ReplaceAll(sourceText, "KMOVQ AX,", "KMOVD AX,") + requireX86GoAssemblerResult(t, arch, sourceText, true) + } + file, err := Parse(ArchAMD64, sourceText) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "example.com/broadcast", declarations.String()) + resolve := testResolveSym(pkg.Path) + sigs, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range triples { + t.Run(triple, func(t *testing.T) { + if arch == "386" { + original, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + originalSigs, err := goSigsForAsmFile(pkg, original, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(original, Options{ + Goarch: arch, TargetTriple: triple, Sigs: originalSigs, ResolveSym: resolve, + }) + if err != nil || ir == "" { + t.Fatal("original mode32 KMOVQ must consume GP32 and zero-extend the mask", err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{ + Goarch: arch, TargetTriple: triple, Sigs: sigs, ResolveSym: resolve, + }) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "broadcast-frame.ll", "broadcast-frame.o", module.String()) + }) + } + }) + } + } +} diff --git a/amd64_broadcast_views_test.go b/amd64_broadcast_views_test.go new file mode 100644 index 000000000..7880db6c1 --- /dev/null +++ b/amd64_broadcast_views_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Independent axes from Go 1.27 _yvpbroadcastb: B/W/D/Q, X/Y/Z, +// memory/X/GP source, unmasked/merge/zero. Observe every overlapping view, +// including upper bits, and suppress masked-off scalar memory faults. +func integerBroadcastViews(t *testing.T, triple string) (string, string) { + t.Helper() + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for _, spec := range []struct { + op string + bits int + }{ + {"VPBROADCASTB", 8}, {"VPBROADCASTW", 16}, {"VPBROADCASTD", 32}, {"VPBROADCASTQ", 64}, + } { + for width, dst := range []string{"X0", "Y0", "Z0"} { + for input := 0; input < 3; input++ { + for mask := 0; mask < 3; mask++ { + name := fmt.Sprintf("integer_broadcast_%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ old+0(FP),BX\nMOVQ src+8(FP),AX\nMOVQ out+16(FP),DI\nMOVQ mask+24(FP),CX\nKMOVQ CX,K7\nVMOVUPS (BX),Z0\n", name) + operand := "(AX)" + if input == 1 { + source.WriteString("VMOVUPS (AX),X1\n") + operand = "X1" + } else if input == 2 { + source.WriteString("MOVQ (AX),R8\n") + operand = "R8" + } + op, predicate := spec.op, "" + if mask != 0 { + predicate = "K7," + } + if mask == 2 { + op += ".Z" + } + fmt.Fprintf(&source, "%s %s,%s%s\nVMOVUPS Z0,(DI)\nVMOVUPS Y0,64(DI)\nVMOVUPS X0,96(DI)\nRET\n", op, operand, predicate, dst) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *,uint64_t);\n", name) + fmt.Fprintf(&checks, "if (check(%s,%d,%d,%d,%d)) return 1;\n", name, 16< +#include +` + declarations.String() + ` +static int check(void (*fn)(const void *,const void *,void *,uint64_t),int width,int lane,int form,int input) { + const uint64_t masks[] = {0, 1, 2, UINT64_C(0xaaaaaaaaaaaaaaaa), UINT64_MAX, UINT64_C(1) << 63}; + for (unsigned m = 0; m < sizeof(masks)/sizeof(masks[0]); m++) { + uint8_t old[64], src[16], output[112]; + for (int i = 0; i < 64; i++) old[i] = (uint8_t)(197-i*3); + for (int i = 0; i < 16; i++) src[i] = (uint8_t)(17+i*7); + uint64_t relevant = width/lane == 64 ? UINT64_MAX : (UINT64_C(1) << (width/lane)) - 1; + int inactiveMemory = input == 0 && form != 0 && (masks[m] & relevant) == 0; + fn(old, inactiveMemory ? 0 : src, output, masks[m]); + for (int view = 0; view < 3; view++) { + int offset = view == 0 ? 0 : view == 1 ? 64 : 96; + for (int i = 0; i < (64 >> view); i++) { + int enabled = form == 0 || ((masks[m] >> (i/lane)) & 1); + uint8_t want = i >= width ? 0 : enabled ? src[i%lane] : form == 2 ? 0 : old[i]; + if (output[offset+i] != want) { + fprintf(stderr,"broadcast width=%d lane=%d form=%d input=%d mask=%u view=%d byte=%d got=%u want=%u\n",width,lane,form,input,m,view,i,output[offset+i],want); + return 1; + } + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + return ir, main +} + +func TestIntegerBroadcastRegisterViews(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"x86_64-apple-macosx", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, main := integerBroadcastViews(t, triple) + compileLLVMToObject(t, llc, triple, "integer_broadcast.ll", "integer_broadcast.o", ir) + var runner []string + execute := runtime.GOARCH == "amd64" && triple == testTargetTriple(runtime.GOOS, runtime.GOARCH) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && triple == "x86_64-apple-macosx" && rosettaAvailable() { + execute = true + runner = []string{"/usr/bin/arch", "-x86_64"} + } + if execute { + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("LLVM 22 clang not found") + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "integer_broadcast", triple, ir, main, runner) + } + }) + } +} diff --git a/amd64_bswap.go b/amd64_bswap.go new file mode 100644 index 000000000..f8bd54b47 --- /dev/null +++ b/amd64_bswap.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" +) + +// Go's complete ybswap row is a single Yrl destination encoded by Z_rp. +// There is no byte/word, memory, immediate, mask or suffix form. +var x86ByteSwapSpecs = map[Op]int{"BSWAPL": 32, "BSWAPQ": 64} + +func x86ByteSwapProperties(op Op, ins Instr, arch string) (int, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + bits, known := x86ByteSwapSpecs[Op(name)] + if !known { + return 0, false, nil + } + if arch != "amd64" && arch != "386" || arch == "386" && bits == 64 || dotted || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) || len(ins.Args) != 1 || + ins.Args[0].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[0].Reg, arch) { + return bits, true, fmt.Errorf("%s %s outside Go BSWAP register grammar: %q", arch, op, ins.Raw) + } + return bits, true, nil +} + +func x86TypedByteSwapEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + bits, known, err := x86ByteSwapProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: int64(bits / 8), read: true, write: true}, + }}, true, nil // BSWAP does not read or modify FLAGS, DF or any memory. +} + +// Byte facts are permuted, not widened to a full initialized word. The same +// dependency transports independently proved canonical bytes, never values, +// pointer identity, native backing or a new canonicality witness. +func x86ByteSwapByteMask(before uint8, bits int) uint8 { + var after uint8 + for index := 0; index < bits/8; index++ { + if before&(1<= 0 { + mutated.Args[form.mask].Reg = "K2" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K2"}) + } + } + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: updated} + if x86CallerMemoryCertified(mutated, index) { + t.Fatal("changed shuffle retained caller memory authority", change) + } + } + } + if !found { + t.Fatal("no shuffle consumed the caller proof") + } + } +} diff --git a/amd64_byte_shuffle_effects.go b/amd64_byte_shuffle_effects.go new file mode 100644 index 000000000..c7dfc1c35 --- /dev/null +++ b/amd64_byte_shuffle_effects.go @@ -0,0 +1,70 @@ +package plan9asm + +func x86ResultByteShuffleStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + return x86ResultByteShuffleStepWithRead(ins, state, owned, arch, false) +} + +func x86ResultByteShuffleStepWithRead(ins Instr, state *x86ResultState, owned []bool, arch string, nativeRead bool) (bool, bool) { + form, known, err := x86ByteShuffleProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false, true + } + } + control := uint64(0) + if form.memory >= 0 { + // Only the paired emitter may observe continuation past its complete + // native read. A mask cannot shrink E4NF, certify alignment or grant + // backing; public/caller observers keep their separate storage rules. + if !x86ResultReadWithNativeContinuation(ins, ins.Args[form.memory], int64(form.memoryBytes), state, owned, arch, nativeRead) { + return false, true + } + control = x86CallerVectorByteMask(int64(form.byteWidth)) + } else { + index, _ := amd64VectorRegisterIndex(ins.Args[form.first].Reg, form.byteWidth) + control = state.vectorBytes(index) + } + data, _ := amd64VectorRegisterIndex(ins.Args[form.second].Reg, form.byteWidth) + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + initialized := x86ByteShuffleDefinedBytes(form.byteWidth, control, state.vectorBytes(data)) + if form.mask >= 0 && !form.zeroing { + initialized &= state.vectorBytes(destination) + } + state.writeVector(destination, form.byteWidth, initialized, form.legacy) + return true, true +} + +func x86CallerByteShuffleStep(ins *Instr, form x86PackedBinaryForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + control := uint64(0) + if form.memory >= 0 { + // A bounded slice is not a witness for legacy m128 alignment. + // E4NF vector reads require the full tuple even with a zero mask. + if form.legacy || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + control = width + } else { + control = state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] + } + data := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + known := x86ByteShuffleDefinedBytes(form.byteWidth, control, data) + if form.mask >= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_byte_shuffle_fault_unix_test.go b/amd64_byte_shuffle_fault_unix_test.go new file mode 100644 index 000000000..c4f7aa213 --- /dev/null +++ b/amd64_byte_shuffle_fault_unix_test.go @@ -0,0 +1,177 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ByteShuffleMemoryFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, originals, entries strings.Builder + count := 0 + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for legacy := 0; legacy < 2; legacy++ { + if legacy != 0 && width != 16 { + continue + } + for masking := 0; masking < 3; masking++ { + if legacy != 0 && masking != 0 { + continue + } + for dead := 0; dead < 2; dead++ { + name := fmt.Sprint("Fault", count) + count++ + row := "VPSHUFB" + if legacy != 0 { + row = "PSHUFB" + } + if masking == 2 { + row += ".Z" + } + row += " (SI)," + bank + "1" + if masking != 0 { + row += ",K1" + } + if legacy == 0 { + row += "," + bank + "1" + } + fmt.Fprintf(&declarations, "func %s(out,control,data *[64]byte)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-24\nMOVQ out+0(FP),DI\nMOVQ control+8(FP),SI\nMOVQ data+16(FP),DX\nVMOVUPS (DX),Z1\nMOVL $0,AX\nKMOVQ AX,K1\n%s\n", name, row) + if dead == 0 { + fmt.Fprintf(&source, "VMOVUPS %s1,(DI)\n", bank) + } + source.WriteString("RET\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*);\n", name) + original := "0" + if legacy != 0 { + // Original Go bytes use the native C argument registers. + // This does not claim native VEX/EVEX execution. + original = "Original" + name + native := "TEXT native(SB),4,$0-0\nMOVOU (DX),X1\nPSHUFB (SI),X1\n" + if dead == 0 { + native += "MOVOU X1,(DI)\n" + } + code := x86OrderingHintGoBytes(t, "amd64", native+"RET\n") + var bytes []string + for _, value := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*);\n", original) + } + fmt.Fprintf(&entries, "{%s,%s,%d,%d,%d},\n", name, original, width, masking, dead) + } + } + } + } + if count != 20 { + t.Fatal("incomplete fault family", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf(x86ByteShuffleFaultReference, prototypes.String(), originals.String(), entries.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + text := ir + if level == "-O2" { + text = x86MulDivRoundOptimize(t, text) + } + output := execution.compileAndRunLLVM(t, "byte-shuffle-faults", text, driver, level) + if string(output) != "byte shuffle: 20 kernels, 110 address outcomes, failures 0\n" { + t.Fatal("incomplete fault oracle", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } +} + +const x86ByteShuffleFaultReference = ` +#include +#include +#include +#include +#include +#include +#include +#include +%s +%s +typedef void (*Program)(uint8_t*,const uint8_t*,const uint8_t*); +typedef struct { Program translated, original; int width, masking, dead; } Kernel; +static const Kernel kernels[] = {%s}; + +static int observe(Program run, Kernel k, const uint8_t *control) { + pid_t child = fork(); + if (child < 0) return -1; + if (child == 0) { + struct rlimit no_core = {0,0}; + if (setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + _Alignas(64) uint8_t data[64], out[66]; + for (int i=0;i<64;i++) data[i]=(uint8_t)(i*17+31); + memset(out,0xa7,sizeof(out)); + run(out+1,control,data); + for (int i=0;i<66;i++) { + unsigned want=0xa7; + if (!k.dead && i>=1 && i<=k.width) { + int offset=i-1; + if (k.masking) want=k.masking==2 ? 0 : data[offset]; + else want=control[offset]&0x80 ? 0 : data[offset/16*16+(control[offset]&15)]; + } + if (out[i]!=want) _exit(121); + } + _exit(0); + } + int status; + if (waitpid(child,&status,0)!=child) return -2; + if (WIFEXITED(status)) return WEXITSTATUS(status)==0 ? 0 : -3; + if (WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS)) return 1; + return -4; +} + +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if (page<128) return 1; + uint8_t *memory=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if (memory==MAP_FAILED || mprotect(memory+page,page,PROT_NONE)) return 2; + for (long i=0;i1 && j!=3 && want!=0)) return 3; + outcomes++; + } + if (got<0 || got!=want) { + fprintf(stderr,"kernel %%u address %%u got %%d want %%d\n",i,j,got,want); + return 4; + } + outcomes++; + } + } + if (munmap(memory,2*page)) return 5; + printf("byte shuffle: %%zu kernels, %%u address outcomes, failures 0\n",sizeof(kernels)/sizeof(kernels[0]),outcomes); + return 0; +} +` diff --git a/amd64_byte_shuffle_forms.go b/amd64_byte_shuffle_forms.go new file mode 100644 index 000000000..3a7cd74cc --- /dev/null +++ b/amd64_byte_shuffle_forms.go @@ -0,0 +1,63 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's ymshufb has only X/m128, X, not the ISA's separate MMX form. +// Both vector encodings share the binary grammar, with no broadcast or SAE. +var x86ByteShuffleSpecs = map[Op]x86PackedBinarySpec{ + "PSHUFB": {encoding: x86PackedLogicalLegacyFP}, + "VPSHUFB": {encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, +} + +func x86ByteShuffleProperties(op Op, ins Instr, arch string) (x86PackedBinaryForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86ByteShuffleSpecs[Op(base)] + if !known { + return x86PackedBinaryForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86PackedBinaryForm{}, true, fmt.Errorf("byte shuffle dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + grammarArch := arch + if arch == "386" && ins.x86Encoded && spec.encoding != x86PackedLogicalLegacyFP { + // Physical EVEX masking does not inherit cmd/asm's named operand cap. + grammarArch = "amd64" + } + form, err := x86PackedBinaryProperties(local, grammarArch, spec) + if err == nil && grammarArch != arch && form.memory >= 0 && !x86PackedShiftMemoryValid(local.Args[form.memory], arch) { + err = fmt.Errorf("386 byte shuffle physical source has an unbound address") + } + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && form.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: byte shuffle has no bound address-size override", ErrProbeNeedsContext) + } + return form, true, err +} + +// A dynamic control byte may select any byte of its own 128-bit data lane. +// This is numerical definedness, never a memory, alignment or caller grant. +func x86ByteShuffleDefinedBytes(width int, control, data uint64) uint64 { + var known uint64 + for start := 0; start < width; start += 16 { + lane := uint64(0xffff) << uint(start) + if data&lane == lane { + known |= control & lane + } + } + return known +} + +func x86TypedByteShuffleEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86ByteShuffleProperties(op, ins, arch) + if !known || err != nil || !form.legacy { + return x86TypedByteEffect{}, known && (err != nil || form.legacy), err + } + return x86PackedBinaryDependencyEffect(ins, form, ins.Args[form.first]), true, nil +} diff --git a/amd64_byte_shuffle_forms_test.go b/amd64_byte_shuffle_forms_test.go new file mode 100644 index 000000000..310ec0403 --- /dev/null +++ b/amd64_byte_shuffle_forms_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86ByteShuffleCompleteGoEncoderRows(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APSHUFB, ymshufb, Pq, opBytes{0x38, 0x00}}", + "var ymshufb = []ytab{\n\t{Zm2_r, 2, argList{Yxm, Yxr}},\n}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatal("actual legacy Go row changed", row) + } + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + if len(x86ByteShuffleSpecs) != 2 { + t.Fatal("byte shuffle family changed") + } + for _, row := range []struct{ begin, end, want string }{ + {"{as: AVPSHUFB,", "\n\t}},", `{as: AVPSHUFB, ytab: _yvandnpd, prefix: Pavx, op: opBytes{ + avxEscape | vex128 | vex66 | vex0F38 | vexW0, 0x00, + avxEscape | vex256 | vex66 | vex0F38 | vexW0, 0x00, + avxEscape | evex128 | evex66 | evex0F38 | evexW0, evexN16 | evexZeroingEnabled, 0x00, + avxEscape | evex256 | evex66 | evex0F38 | evexW0, evexN32 | evexZeroingEnabled, 0x00, + avxEscape | evex512 | evex66 | evex0F38 | evexW0, evexN64 | evexZeroingEnabled, 0x00,`}, + {"var _yvandnpd =", "\n}", `var _yvandnpd = []ytab{ + {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr, Yxr}}, + {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yym, Yyr, Yyr}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{YxmEvex, YxrEvex, YxrEvex}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{YxmEvex, YxrEvex, Yknot0, YxrEvex}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{YymEvex, YyrEvex, YyrEvex}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{YymEvex, YyrEvex, Yknot0, YyrEvex}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{Yzm, Yzr, Yzr}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{Yzm, Yzr, Yknot0, Yzr}},`}, + } { + start := strings.Index(string(avx), row.begin) + if start < 0 || strings.SplitN(string(avx)[start:], row.end, 2)[0] != row.want { + t.Fatal("actual Go encoder classes, offsets or attributes changed", row.begin) + } + } +} + +func x86ByteShuffleFormatRows(arch string) []string { + var rows []string + for _, source := range []string{"X0", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-16(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, "PSHUFB "+source+",X1") + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + for mode := 0; mode < 3; mode++ { + if arch == "386" && mode != 0 { + continue + } + name, args := "VPSHUFB", source+","+bank+"1" + if mode != 0 { + args += ",K7" + } + if mode == 2 { + name += ".Z" + } + rows = append(rows, name+" "+args+","+bank+"2") + } + } + } + return rows +} + +func TestX86ByteShuffleGoFormsAndMemoryShapes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86ByteShuffleFormatRows(arch) { + t.Run(arch+"/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$64-64\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86ByteShuffleProperties(ins.Op, ins, arch) + if !known || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatal("Go shape missing in a consumer", known, err) + } + if form.memory >= 0 { + width, write, known := x86InstructionMemoryShape(ins, form.memory, arch) + if !known || write || width != int64(form.byteWidth) { + t.Fatal("lost full E4NF control tuple", width, write, known) + } + } + }) + } + } +} + +func TestX86ByteShuffleMode32HighBanksRemainContext(t *testing.T) { + for _, bank := range []string{"X", "Y"} { + for _, index := range []int{8, 15, 16, 31} { + row := fmt.Sprintf("VPSHUFB %s%d,%s1,%s2", bank, index, bank, bank) + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86ByteShuffleProperties(ins.Op, ins, "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("source acceptance invented physical high-bank identity", row, err) + } + } + } +} + +func TestX86ByteShuffleInvalidFormsAndSourceIdentity(t *testing.T) { + for _, row := range []string{ + "PSHUFB M0,M1", "PSHUFB X16,X1", "PSHUFB.Z X0,X1", + "VPSHUFB X0,Y1,Y2", "VPSHUFB X0,(AX),X2", "VPSHUFB $1,X1,X2", + "VPSHUFB X0,X1,K0,X2", "VPSHUFB.BCST (AX),X1,X2", + "VPSHUFB.SAE X0,X1,X2", "VPSHUFB.RN_SAE X0,X1,X2", + "VPSHUFB (AX)(CX*3),X1,X2", "VPSHUFB 2147483648(AX),X1,X2", + } { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, parseErr := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if parseErr != nil { + // The original Go rejection above still owns this source axis. + continue + } + ins := file.Funcs[0].Instrs[1] + if _, known, err := x86ByteShuffleProperties(ins.Op, ins, "amd64"); !known || err == nil || x86CallerVectorMoveKnown(ins) { + t.Fatal("invalid Go form acquired an effect", row, known, err) + } + } + for _, row := range []string{"PSHUFB (AX),X1", "VPSHUFB (AX),X1,X2"} { + for _, mutation := range []string{"dispatch", "address-size", "unbound-override"} { + ins := x86ScalarLaneInstruction(t, row) + op := ins.Op + switch mutation { + case "dispatch": + ins.Op = "MOVOU" + case "address-size": + ins.x86AddressBits = 32 + case "unbound-override": + ins.x86UnboundAddressOverride = true + } + var out strings.Builder + ctx := amd64Ctx{goarch: "amd64", b: &out} + if known, _, err := ctx.lowerPackedByteShuffle(op, ins); !known || err == nil || out.Len() != 0 { + t.Fatal("changed source identity emitted IR", row, mutation, err, out.String()) + } + } + } +} + +func TestX86ByteShuffleVectorDispatcherUsesCompleteGrammar(t *testing.T) { + for _, row := range []string{ + "PSHUFB Y0,X0", "PSHUFB X0", "PSHUFB.Z X0,X1", + "VPSHUFB Y0,Y1,AX", "VPSHUFB X0,Y1,Y2", + "VPSHUFB X0,X1,K0,X2", "VPSHUFB.BCST (AX),X1,X2", + } { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + for _, keepOpcode := range []bool{false, true} { + ins := x86ScalarLaneInstruction(t, row) + op := ins.Op + if !keepOpcode { + ins.Op = "" + } + var output strings.Builder + ctx := amd64Ctx{goarch: "amd64", b: &output} + known, terminated, err := ctx.lowerVec(op, ins) + if !known || terminated || err == nil || output.Len() != 0 { + t.Fatalf("vector dispatcher bypassed byte-shuffle grammar for %q: (%v, %v, %v), IR %q", row, known, terminated, err, output.String()) + } + } + } + for _, row := range []string{ + "PSHUFB X0,X1", "VPSHUFB X0,X1,X2", "VPSHUFB Y0,Y1,Y2", + "VPSHUFB Z0,Z1,Z2", "VPSHUFB X0,X1,K7,X2", "VPSHUFB.Z Z0,Z1,K7,Z2", + } { + ins := x86ScalarLaneInstruction(t, row) + function := Func{Instrs: []Instr{ins}} + family, familyOutput := newAMD64CtxWithFuncForTest(t, function, FuncSig{Name: "shuffle", Ret: Void}, nil) + dispatch, dispatchOutput := newAMD64CtxWithFuncForTest(t, function, FuncSig{Name: "shuffle", Ret: Void}, nil) + if known, terminated, err := family.lowerPackedByteShuffle(ins.Op, ins); !known || terminated || err != nil { + t.Fatal("independent family entry", row, known, terminated, err) + } + if known, terminated, err := dispatch.lowerVec(ins.Op, ins); !known || terminated || err != nil { + t.Fatal("vector family entry", row, known, terminated, err) + } + if familyOutput.String() != dispatchOutput.String() { + t.Fatal("vector dispatcher changed the family's emitted effects", row) + } + } +} diff --git a/amd64_byte_shuffle_native_result_test.go b/amd64_byte_shuffle_native_result_test.go new file mode 100644 index 000000000..14208d048 --- /dev/null +++ b/amd64_byte_shuffle_native_result_test.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func x86ByteShuffleNativeResultRows(arch string) []string { + rows := []string{"PSHUFB (DX),X0"} + for _, bank := range []string{"X", "Y", "Z"} { + rows = append(rows, "VPSHUFB (DX),"+bank+"0,"+bank+"1") + if arch == "amd64" { + rows = append(rows, "VPSHUFB (DX),"+bank+"0,K1,"+bank+"1", + "VPSHUFB.Z (DX),"+bank+"0,K1,"+bank+"1") + } + } + return rows +} + +func TestX86ByteShuffleNativeResultsCompleteFamilyFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + var source, declarations strings.Builder + for index, row := range x86ByteShuffleNativeResultRows(target.arch) { + name, output := fmt.Sprint("ShuffleNative", index), "X1" + if strings.HasPrefix(row, "PSHUFB ") { + output = "X0" + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),DX\n"+ + "VPXORQ Z0,Z0,Z0\nVPXORQ Z1,Z1,Z1\nMOVL $1,AX\nKMOVW AX,K1\n%s\n"+ + "PEXTRD $0,%s,AX\nMOVL AX,ret+%d(FP)\nRET\n", name, word+4, move, row, output, word) + fmt.Fprintf(&declarations, "func %s(p *[64]byte) uint32\n", name) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("complete initialized family lost its paired native read", api, err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + } + }) + } +} + +func TestX86ByteShuffleNativeContinuationCannotCreateAddressOrMaskFacts(t *testing.T) { + for _, row := range x86ByteShuffleNativeResultRows("amd64") { + for _, mutation := range []string{"unknown-base", "pointer-index", "missing-mask", "numerical-address"} { + if mutation == "missing-mask" && !strings.Contains(row, "K1") { + continue + } + t.Run(row+"/"+mutation, func(t *testing.T) { + setup := "MOVQ p+0(FP),DX\nVPXORQ Z0,Z0,Z0\nVPXORQ Z1,Z1,Z1\nMOVL $1,AX\nKMOVW AX,K1\n" + instruction := row + switch mutation { + case "unknown-base": + setup = strings.Replace(setup, "MOVQ p+0(FP),DX\n", "", 1) + case "pointer-index": + instruction = strings.Replace(row, "(DX)", "(DX)(DX*1)", 1) + case "missing-mask": + setup = strings.Replace(setup, "KMOVW AX,K1\n", "", 1) + case "numerical-address": + setup += "MOVL $1,DX\n" + } + output := "X1" + if strings.HasPrefix(row, "PSHUFB ") { + output = "X0" + } + source := "TEXT ·Shuffle(SB),4,$0-12\n" + setup + instruction + "\nPEXTRD $0," + output + ",AX\nMOVL AX,ret+8(FP)\nRET\n" + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Shuffle(p *[64]byte) uint32", "amd64", "x86_64-unknown-linux-gnu", api) + if mutation == "numerical-address" { + x86RequireNumericNativeSourceContext(t, source, "func Shuffle(p *[64]byte) uint32", "amd64") + goAssemblerOracleOutput(t, "amd64", source, "", true) + width := 16 + if strings.Contains(row, "Y0") { + width = 32 + } else if strings.Contains(row, "Z0") { + width = 64 + } + read := strings.Contains(ir, fmt.Sprintf("load volatile <%d x i8>", width)) + if strings.HasPrefix(row, "PSHUFB ") { + read = strings.Contains(ir, "call <4 x float> asm sideeffect \"pshufb ") && strings.Contains(ir, "~{memory}") + } + if err != nil || !read { + t.Fatal("numerical address lost the original exact-width byte-shuffle read", err) + } + x86NumericNativeAddressObject(t, ir, "x86_64-unknown-linux-gnu") + } else if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("conditional native read manufactured independent facts", api, err) + } + } + }) + } + } +} diff --git a/amd64_byte_shuffle_objects_test.go b/amd64_byte_shuffle_objects_test.go new file mode 100644 index 000000000..6d4a3e831 --- /dev/null +++ b/amd64_byte_shuffle_objects_test.go @@ -0,0 +1,67 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86ByteShuffleThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + rows := x86ByteShuffleFormatRows(target.arch) + for begin := 0; begin < len(rows); begin += 64 { + end := begin + 64 + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprint(begin), func(t *testing.T) { + var source, declarations strings.Builder + for index, row := range rows[begin:end] { + if strings.HasPrefix(row, "PSHUFB ") && (strings.Contains(row, "(FP)") || strings.Contains(row, "(SP)")) { + // Explicit native-storage rejection controls below. + continue + } + fmt.Fprintf(&declarations, "func Shuffle%d(value [64]byte)\n", index) + fmt.Fprintf(&source, "TEXT ·Shuffle%d(SB),4,$64-64\n%s\nRET\n", index, row) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "byte-shuffle-"+api) + x86ResultInitializationObject(t, llc, target.triple, x86MulDivRoundOptimize(t, ir), "byte-shuffle-"+api+"-optimized") + } + }) + } + }) + } +} + +func TestX86ByteShuffleVirtualLegacyStorageRemainsContext(t *testing.T) { + for _, row := range []string{"PSHUFB value+0(FP),X1", "PSHUFB local-16(SP),X1", "PSHUFB (SP),X1", "PSHUFB (BP),X1"} { + for _, target := range x86RawReturnTargets { + t.Run(target.triple+"/"+row, func(t *testing.T) { + source := "TEXT ·Shuffle(SB),4,$64-64\nPXOR X1,X1\n" + row + "\nRET\n" + declaration := "func Shuffle(value [64]byte)" + if strings.Contains(row, "(BP)") { + source = x86OrdinaryBPNativeSource(target.arch, "Shuffle", row, "(BP)", 64) + declaration = "func Shuffle(p *byte)" + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + x86RequireOrdinaryBPNativeOutcome(t, target.arch, row, ir, err) + }) + } + }) + } + } +} diff --git a/amd64_byte_shuffle_raw_test.go b/amd64_byte_shuffle_raw_test.go new file mode 100644 index 000000000..e5da92d4e --- /dev/null +++ b/amd64_byte_shuffle_raw_test.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86ByteShuffleRawWIGAndPhysicalMasks(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, width := range []byte{0, 4} { + for _, w := range []byte{0, 0x80} { + code := []byte{0xc4, 0xe2, 0x71 | width | w, 0x00, 0xd0} + ins, size, known, err := decodedX86VPSHUFBInstruction(code, mode) + if !known || err != nil || size != len(code) || ins.Op != "VPSHUFB" { + t.Errorf("VEX.WIG mode=%d code=%x: size=%d known=%t err=%v", mode, code, size, known, err) + } + } + } + for width := byte(0); width < 3; width++ { + for mask := byte(0); mask < 8; mask++ { + for _, zero := range []byte{0, 0x80} { + if mask == 0 && zero != 0 { + continue + } + for _, w := range []byte{0, 0x80} { + code := []byte{0x62, 0xf2, 0x75 | w, 8 | width<<5 | mask | zero, 0x00, 0xd0} + t.Run(fmt.Sprintf("%d/%x", mode, code), func(t *testing.T) { + ins, size, known, err := decodedX86VPSHUFBInstruction(code, mode) + if !known || err != nil || size != len(code) { + t.Fatalf("physical EVEX inherited named operand limit or W0: size=%d known=%t err=%v", size, known, err) + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + if !ins.x86Encoded { + t.Fatal("decoder lost physical encoding identity") + } + if _, known, err := x86ByteShuffleProperties(ins.Op, ins, arch); !known || err != nil { + t.Fatal("typed physical consumer", known, err) + } + ins.x86Encoded = false + if _, _, err := x86ByteShuffleProperties(ins.Op, ins, arch); mode == 32 && mask != 0 && err == nil { + t.Fatal("named mode32 form acquired physical decoder authority") + } + }) + } + } + } + } + } +} + +func TestX86ByteShuffleRawThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var codes [][]byte + for _, width := range []byte{0, 4} { + for _, w := range []byte{0, 0x80} { + codes = append(codes, []byte{0xc4, 0xe2, 0x71 | width | w, 0x00, 0xd0}) + } + } + for width := byte(0); width < 3; width++ { + for _, mask := range []byte{1, 7} { + for _, zero := range []byte{0, 0x80} { + for _, w := range []byte{0, 0x80} { + codes = append(codes, []byte{0x62, 0xf2, 0x75 | w, 8 | width<<5 | mask | zero, 0x00, 0xd0}) + } + } + } + } + var source, declarations strings.Builder + for index, code := range codes { + fmt.Fprintf(&declarations, "func RawShuffle%d()\n", index) + fmt.Fprintf(&source, "TEXT ·RawShuffle%d(SB),4,$0-0\n", index) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("RET\n") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "raw-byte-shuffle-"+api) + } + }) + } +} + +func TestX86ByteShuffleReservedRawForms(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe2, 0x70, 0, 0xd0}, // Wrong mandatory prefix. + {0xc4, 0xe2, 0x71, 0}, // Truncated ModRM. + {0x62, 0xf2, 0x75, 0x68, 0, 0xd0}, // Reserved LL. + {0x62, 0xf2, 0x75, 0x18, 0, 0xd0}, // No broadcast/rounding. + {0x62, 0xf2, 0x75, 0x88, 0, 0xd0}, // Zeroing without mask. + {0x67, 0x62, 0xf2, 0x75, 0x08, 0, 0x10}, // Unbound address size. + } { + for _, mode := range []int{32, 64} { + if _, _, known, err := decodedX86VPSHUFBInstruction(code, mode); !known || err == nil { + t.Fatalf("reserved byte shuffle accepted: mode=%d code=%x known=%t err=%v", mode, code, known, err) + } + } + } +} diff --git a/amd64_byte_shuffle_result_test.go b/amd64_byte_shuffle_result_test.go new file mode 100644 index 000000000..997975159 --- /dev/null +++ b/amd64_byte_shuffle_result_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86ByteShuffleIndependentResultConsumers(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + const data = "DATA control<>+0(SB)/8,$0x0001020304050607\nDATA control<>+8(SB)/8,$0x08090a0b0c0d0e0f\nGLOBL control<>(SB),8,$16\n" + for _, row := range []string{ + "PSHUFB X0,X1", "VPSHUFB X0,X1,X1", + "PSHUFB control<>(SB),X1", "VPSHUFB control<>(SB),X1,X1", + } { + t.Run(row, func(t *testing.T) { + source := data + "TEXT ·view(SB),4,$0-20\nMOVQ a+0(FP),X0\nMOVQ b+8(FP),X1\n" + row + "\nPEXTRD $0,X1,AX\nMOVL AX,ret+16(FP)\nRET\n" + for _, target := range x86RawReturnTargets { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(a,b uint64) uint32\n", target.arch, target.triple, api) + if err != nil { + t.Fatalf("%s/%s complete numerical shuffle has no result consumer: %v", target.triple, api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "byte-shuffle-result-"+api) + } + } + }) + } +} + +func TestX86ByteShuffleDeadMemoryRetainsOriginalAccess(t *testing.T) { + for _, row := range []struct { + body string + width int + }{ + {"PSHUFB (DX),X0", 16}, + {"VPSHUFB (DX),X0,X1", 16}, + {"VPSHUFB (DX),Y0,Y1", 32}, + {"VPSHUFB (DX),Z0,Z1", 64}, + {"VPSHUFB.Z (DX),Z0,K1,Z1", 64}, + } { + t.Run(row.body, func(t *testing.T) { + source := "TEXT ·view(SB),4,$0-8\nMOVQ p+0(FP),DX\nVPXORQ Z0,Z0,Z0\nMOVL $0,AX\nKMOVW AX,K1\n" + row.body + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte)\n", "amd64", "x86_64-unknown-linux-gnu", "Go") + if err != nil { + t.Fatal(err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + want := fmt.Sprintf("load volatile <%d x i8>", row.width) + if !strings.HasPrefix(row.body, "V") { + want = "asm sideeffect \"pshufb " + } + if !strings.Contains(text, want) { + t.Errorf("dead shuffle lost its full native read/alignment effect: want %s", want) + } + } + }) + } +} + +func TestX86ByteShuffleSparseResultAndCallerBytes(t *testing.T) { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, legacy := range []bool{false, true} { + if legacy && width != 16 { + continue + } + for masking := 0; masking < 3; masking++ { + if legacy && masking != 0 { + continue + } + row, dst := "VPSHUFB", 2 + if legacy { + row, dst = "PSHUFB", 1 + } + if masking == 2 { + row += ".Z" + } + row += " " + bank + "0," + bank + "1" + if masking != 0 { + row += ",K1" + } + if !legacy { + row += "," + bank + "2" + } + ins := x86ScalarLaneInstruction(t, row) + for input := 0; input < 3; input++ { + for missing := -1; missing < 64; missing++ { + initial := [3]uint64{^uint64(0), ^uint64(0), ^uint64(0)} + if missing >= 0 { + initial[input] &^= uint64(1) << uint(missing) + } + result := x86ResultState{flags: 0x3f} + caller := x86SourceStackState{vectors: map[Reg]uint64{}, masks: map[Reg]bool{"K1": true}} + for i, value := range initial { + result.writeVector(i, 64, value, false) + caller.vectors[Reg(fmt.Sprintf("Z%d", i))] = value + } + result.masks[1] = true + if !x86ResultTransfer(ins, &result, nil, 0, "amd64") || !x86CallerVectorMoveStep(&ins, &caller, nil) { + t.Fatal("missing byte consumer", row) + } + var want uint64 + for output := 0; output < 64; output++ { + known := !legacy || initial[dst]>>uint(output)&1 != 0 + if output < width { + known = initial[0]>>uint(output)&1 != 0 + for selected := 0; selected < 16; selected++ { + known = known && initial[1]>>uint(output/16*16+selected)&1 != 0 + } + if masking == 1 { + known = known && initial[dst]>>uint(output)&1 != 0 + } + } + if known { + want |= uint64(1) << uint(output) + } + } + if result.vectorBytes(dst) != want || caller.vectors[Reg(fmt.Sprintf("Z%d", dst))] != want || result.flags != 0x3f { + t.Fatalf("%s input=%d missing=%d: result=%x caller=%x want=%x", row, input, missing, result.vectorBytes(dst), caller.vectors[Reg(fmt.Sprintf("Z%d", dst))], want) + } + } + } + } + } + } +} + +func TestX86ByteShuffleResultMemoryAndMaskBoundaries(t *testing.T) { + for _, bank := range []string{"X", "Y", "Z"} { + width := map[string]int{"X": 16, "Y": 32, "Z": 64}[bank] + row := "VPSHUFB.Z value+0(FP)," + bank + "1,K1," + bank + "2" + ins := x86ScalarLaneInstruction(t, row) + for missing := -1; missing < 64; missing++ { + for _, maskKnown := range []bool{false, true} { + state := x86ResultState{fp: make([]bool, 64)} + owned := make([]bool, 64) + for i := range owned { + owned[i], state.fp[i] = true, i != missing + } + state.writeVector(1, 64, ^uint64(0), false) + state.masks[1] = maskKnown + accepted := x86ResultTransfer(ins, &state, owned, 64, "amd64") + want := maskKnown && (missing < 0 || missing >= width) + if accepted != want || accepted && state.vectorBytes(2) != ^uint64(0) || len(ins.x86CallerMemory) != 0 { + t.Fatal("mask/tuple observation invented bytes or caller permission", row, missing, maskKnown, accepted) + } + } + } + } + for _, source := range []string{"(AX)", "·unknown(SB)", "value-1(FP)"} { + ins := x86ScalarLaneInstruction(t, "VPSHUFB.Z "+source+",X1,K1,X2") + state := x86ResultState{} + state.xmm[1], state.masks[1], state.pointerGP[0] = 0xffff, true, true + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("unproved memory acquired numerical authority", source) + } + } +} diff --git a/amd64_byte_shuffle_runtime_test.go b/amd64_byte_shuffle_runtime_test.go new file mode 100644 index 000000000..e2bb101d0 --- /dev/null +++ b/amd64_byte_shuffle_runtime_test.go @@ -0,0 +1,136 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// This observes portable LLVM semantics, not execution of original AVX512. +// Every output byte, including wider aliases and surrounding canaries, is +// compared with an independent scalar reference over all 256 control values. +func TestCrossLinuxRuntimeMatrixX86ByteShuffleSoftware(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, reference, checks strings.Builder + reference.WriteString(x86ByteShuffleReference) + count := 0 + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for legacy := 0; legacy < 2; legacy++ { + for memory := 0; memory < 2; memory++ { + for mask := 0; mask < 3; mask++ { + for dst := 0; dst < 3; dst++ { + if legacy != 0 && (width != 16 || mask != 0 || dst != 1) { + continue + } + name := fmt.Sprint("Shuffle", count) + count++ + row, control := "VPSHUFB", bank+"0" + if legacy != 0 { + row = "PSHUFB" + } + if memory != 0 { + control = "(DI)" + } + if mask == 2 { + row += ".Z" + } + row += " " + control + "," + bank + "1" + if mask != 0 { + row += ",K1" + } + if legacy == 0 { + row += fmt.Sprintf(",%s%d", bank, dst) + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed,mask uint64)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-56\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\n", name) + source.WriteString("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVQ mask+48(FP),AX\nKMOVQ AX,K1\nPUSHQ seed+40(FP)\nPOPFQ\n") + fmt.Fprintf(&source, "%s\nVMOVUPS Z%d,(DX)\nPUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n", row, dst) + fmt.Fprintf(&reference, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t);\n", name) + fmt.Fprintf(&checks, " if (check(%s,%q,%d,%d,%d,%d)) return 1;\n", name, row, width, legacy, mask, dst) + } + } + } + } + } + if count != 56 || count > 64 { + t.Fatal("incomplete or oversized runtime batch", count) + } + reference.WriteString("int main(void) {\n" + checks.String() + " return 0;\n}\n") + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "byte-shuffle", ir, reference.String(), level) + }) + } + }) + } +} + +const x86ByteShuffleReference = ` +#include +#include +#include +#include + +typedef void (*kernel)(uint8_t *, const uint8_t *, const uint8_t *, + uint64_t *, uint32_t *, uint64_t, uint64_t); + +static int check(kernel fn, const char *name, int width, int legacy, + int masking, int dst) { + _Alignas(64) uint8_t a[64], b[64], out[192], oldA[64], oldB[64]; + const uint64_t masks[] = {0, UINT64_MAX, UINT64_C(0x5555555555555555), UINT64_C(1) << 63}; + for (int sample = 0; sample < 256; ++sample) { + for (int i = 0; i < 64; ++i) { + a[i] = (uint8_t)(sample + i * 53); + b[i] = (uint8_t)(sample * 17 - i * 91); + } + memcpy(oldA, a, 64); + memcpy(oldB, b, 64); + for (int seedIndex = 0; seedIndex < 2; ++seedIndex) { + uint64_t seed = seedIndex ? 0x8d5 : 0; + for (unsigned mi = 0; mi < sizeof(masks)/sizeof(masks[0]); ++mi) { + uint64_t mask = masks[mi]; + struct { uint64_t before, value, after; } flags = {11, 0, 13}; + struct { uint32_t before, value, after; } csr = {17, 0, 19}; + uint32_t initialCSR = _mm_getcsr(); + memset(out, 0xa7, sizeof(out)); + fn(out + 64, a, b, &flags.value, &csr.value, seed, mask); + if ((flags.value & 0x8d5) != seed || flags.before != 11 || flags.after != 13 || + csr.value != initialCSR || csr.before != 17 || csr.after != 19 || + memcmp(a, oldA, 64) || memcmp(b, oldB, 64)) { + fprintf(stderr, "%s state/input/canary mismatch\n", name); + return 1; + } + for (int i = 0; i < 192; ++i) { + unsigned want = 0xa7; + int offset = i - 64; + if (offset >= 0 && offset < 64) { + const uint8_t *old = dst == 0 ? a : b; + want = legacy ? old[offset] : 0; + if (offset < width) { + if (!masking || ((mask >> offset) & 1)) { + want = a[offset] & 0x80 ? 0 : b[(offset / 16) * 16 + (a[offset] & 15)]; + } else { + want = masking == 2 ? 0 : old[offset]; + } + } + } + if (out[i] != want) { + fprintf(stderr, "%s sample=%d mask=%llx byte=%d got=%x want=%x\n", + name, sample, (unsigned long long)mask, i, out[i], want); + return 1; + } + } + } + } + } + return 0; +} +` diff --git a/amd64_carry_control.go b/amd64_carry_control.go new file mode 100644 index 000000000..e75e5d16b --- /dev/null +++ b/amd64_carry_control.go @@ -0,0 +1,41 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86CarryControlSpec struct { + set, complement bool +} + +// asm6.go's complete CLC/STC/CMC family is ynone/Px in both x86 modes. +// Only CF changes; none of the other arithmetic flags or DF is defined here. +var x86CarryControlSpecs = map[Op]x86CarryControlSpec{ + "CLC": {}, "STC": {set: true}, "CMC": {complement: true}, +} + +func x86CarryControlProperties(op Op, ins Instr, arch string) (x86CarryControlSpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86CarryControlSpecs[Op(name)] + if !known { + return spec, false, nil + } + if arch != "386" && arch != "amd64" || dotted || len(ins.Args) != 0 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return spec, true, fmt.Errorf("%s %s outside Go carry-control ynone grammar: %q", arch, op, ins.Raw) + } + return spec, true, nil +} + +func x86TypedCarryControlByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + spec, known, err := x86CarryControlProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + effect := x86TypedByteEffect{flagsKilled: x86ByteFlagCF, flagsDefined: x86ByteFlagCF} + if spec.complement { + effect.flagsRead = x86ByteFlagCF + } + return effect, true, nil +} diff --git a/amd64_clmul_definedness_test.go b/amd64_clmul_definedness_test.go new file mode 100644 index 000000000..7eeaa7c00 --- /dev/null +++ b/amd64_clmul_definedness_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "errors" + "fmt" + "testing" +) + +func TestX86CLMULSelectedQwordResultBytes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, vector := range []bool{false, true} { + if arch == "386" && vector { + continue // Stock Go rejects the four-operand source form. + } + for _, immediate := range []int{0, 1, 16, 17, 238, 239, 254, 255} { + name, tail := "PCLMULQDQ", "X0,X1" + if vector { + name, tail = "VPCLMULQDQ", "X0,X1,X2" + } + t.Run(fmt.Sprintf("%s/%s/%d", arch, name, immediate), func(t *testing.T) { + row := fmt.Sprintf("%s $%d,%s", name, immediate, tail) + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + state := x86ResultState{flags: x86ByteFlagCF | x86ByteFlagZF} + state.xmm[0] = uint16(255) << uint((immediate>>4&1)*8) + state.xmm[1] = uint16(255) << uint((immediate&1)*8) + if !x86ResultTransfer(ins, &state, nil, 0, arch) { + t.Fatal("complete selected qwords lack their registered result effect") + } + destination := 1 + if vector { + destination = 2 + } + if state.xmm[destination] != 0xffff || state.flags != x86ByteFlagCF|x86ByteFlagZF { + t.Fatal("carryless product did not define exactly the full X result while preserving flags") + } + }) + } + } + } +} + +func TestX86CLMULMissingByteAndAliasEffects(t *testing.T) { + for _, bank := range []string{"X", "Y", "Z"} { + for _, legacy := range []bool{true, false} { + if legacy && bank != "X" { + continue + } + for immediate := 0; immediate < 256; immediate++ { + row := fmt.Sprintf("PCLMULQDQ $%d,X0,X1", immediate) + if !legacy { + row = fmt.Sprintf("VPCLMULQDQ $%d,%s0,%s1,%s2", immediate, bank, bank, bank) + } + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86CLMULProperties(ins.Op, ins, "amd64") + if !known || err != nil { + t.Fatal(err) + } + for missing := -1; missing < form.byteWidth; missing++ { + for source := 0; source < 2; source++ { + inputs := [2]uint64{^uint64(0), ^uint64(0)} + if missing >= 0 { + inputs[source] &^= uint64(1) << uint(missing) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": inputs[0], "Z1": inputs[1], "Z2": 1 << 60}} + if !x86CallerVectorMoveKnown(ins) || !x86CallerVectorMoveStep(&ins, &state, nil) { + t.Fatal("complete register effect is unregistered", row) + } + want := ^uint64(0) + if missing >= 0 { + selector := immediate & 1 + if source == 0 { + selector = immediate >> 4 & 1 + } + if missing%16/8 == selector { + want &^= uint64(65535) << uint(missing/16*16) + } + } + destination := Reg("Z2") + if legacy { + destination = "Z1" + } + if state.vectors[destination] != want { + t.Fatalf("%s missing %d operand %d got %016x want %016x", row, missing, source, state.vectors[destination], want) + } + result := x86ResultState{} + result.writeVector(0, 64, inputs[0], false) + result.writeVector(1, 64, inputs[1], false) + result.writeVector(2, 64, 1<<60, false) + if !x86ResultTransfer(ins, &result, nil, 0, "amd64") { + t.Fatal("partial known bytes lost the effect registration") + } + index := 2 + if legacy { + index = 1 + } + if result.vectorBytes(index) != want { + t.Fatal("caller/result complete numerical byte selectors differ", row) + } + } + } + } + } + } +} + +func TestX86CLMULNoMemoryOrUnknownResultPermission(t *testing.T) { + for _, row := range []string{ + "PCLMULQDQ $0,(AX),X1", "PCLMULQDQ $0,·data(SB),X1", + "VPCLMULQDQ $0,(AX),X1,X2", "VPCLMULQDQ $0,(AX),Y1,Y2", "VPCLMULQDQ $0,(AX),Z1,Z2", + } { + ins := x86ScalarLaneInstruction(t, row) + state := x86SourceStackState{vectors: map[Reg]uint64{"Z1": ^uint64(0)}} + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, nil) || len(ins.x86CallerMemory) != 0 { + t.Fatal("shape or selected qword invented caller memory authority", row) + } + result := x86ResultState{pointerGP: [16]bool{true}} + result.xmm[1] = 0xffff + if x86ResultTransfer(ins, &result, nil, 0, "amd64") { + t.Fatal("pointer identity or SB spelling invented readable bytes", row) + } + } + for _, arch := range []string{"amd64", "386"} { + for _, missing := range []int{0, 1, 2, 3} { + body := "MOVL $7,AX\n" + for lane := 0; lane < 4; lane++ { + if lane != missing { + body += fmt.Sprintf("PINSRD $%d,AX,X%d\n", lane%2, lane/2) + } + } + body += "PCLMULQDQ $0,X0,X1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+0(FP)\nRET" + _, file, sig, source := x86ResultInitializationFixture(t, arch, "func Value() uint32", body) + goAssemblerOracleOutput(t, arch, source, "", true) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a partially missing selected qword acquired numerical result bytes", arch, missing, err) + } + } + } +} + +func TestX86CLMULScalarResultWholeSource(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, immediate := range []int{0, 1, 16, 17} { + t.Run(fmt.Sprintf("%s/%d", arch, immediate), func(t *testing.T) { + // Separate 32-bit insertions initialize only each selected qword; + // the opposite halves deliberately retain unknown input bytes. + body := "MOVL $7,AX\n" + for operand, selector := range []int{immediate >> 4 & 1, immediate & 1} { + for half := 0; half < 2; half++ { + body += fmt.Sprintf("PINSRD $%d,AX,X%d\n", selector*2+half, operand) + } + } + body += fmt.Sprintf("PCLMULQDQ $%d,X0,X1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+0(FP)\nRET", immediate) + _, file, sig, source := x86ResultInitializationFixture(t, arch, "func Value() uint32", body) + goAssemblerOracleOutput(t, arch, source, "", true) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, arch); err != nil { + t.Errorf("selected qword result failed: %v", err) + } + } + }) + } + } +} diff --git a/amd64_clmul_fault_runtime_test.go b/amd64_clmul_fault_runtime_test.go new file mode 100644 index 000000000..9c528265b --- /dev/null +++ b/amd64_clmul_fault_runtime_test.go @@ -0,0 +1,95 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86CLMULOriginalDeadReadFaults(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, references, pairs strings.Builder + sigs := map[string]FuncSig{} + for index, immediate := range []int{0, 1, 16, 17} { + name := fmt.Sprint("clmulfault", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-8\nMOVQ p+0(FP),AX\nPXOR X1,X1\nPCLMULQDQ $%d,(AX),X1\nRET\n", name, immediate) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + code := x86OrderingHintGoBytes(t, "amd64", fmt.Sprintf("TEXT f(SB),4,$0-0\nPCLMULQDQ $%d,(DI),X1\nRET\n", immediate)) + if len(code) != 7 || code[len(code)-1] != 0xc3 { + t.Fatal("original memory opcode or its RET changed", code) + } + var bytes []string + for _, value := range code[:len(code)-1] { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + fmt.Fprintf(&references, "extern void %s(const void*);\nstatic void original%d(const void *p) {\n", name, index) + fmt.Fprintf(&references, " __asm__ volatile(\"pxor %%%%xmm1, %%%%xmm1; .byte %s\"::\"D\"(p):\"xmm1\",\"memory\");\n}\n", strings.Join(bytes, ",")) + if index > 0 { + pairs.WriteString(",") + } + fmt.Fprintf(&pairs, "{original%d,%s}", index, name) + } + oracle := `#include +#include +#include +#include +#include +#include +#include +` + references.String() + ` +typedef void (*operation)(const void*); +struct pair {operation original,translated;}; +static const struct pair pairs[]={ ` + pairs.String() + ` }; +static int outcome(operation fn,const void *p) { + pid_t pid=fork(); if(pid<0) return -1; + if(!pid) { + struct rlimit limit={0,0}; if(setrlimit(RLIMIT_CORE,&limit)) _exit(120); + fn(p); _exit(0); + } + int status; if(waitpid(pid,&status,0)!=pid) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status)) return WTERMSIG(status); + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); if(page<128) return 1; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(region==MAP_FAILED || mprotect(region+page,page,PROT_NONE)) return 2; + for(unsigned op=0;op<4;op++) for(unsigned extent=0;extent<=32;extent++) { + const void *p=region+page-extent; + int original=outcome(pairs[op].original,p),translated=outcome(pairs[op].translated,p); + if(original<0 || original!=translated || (extent==32 && original!=0) || + (extent==0 && original!=SIGSEGV && original!=SIGBUS)) { + fprintf(stderr,"CLMUL selector=%u extent=%u original=%d translated=%d\n",op,extent,original,translated); + return 3; + } + } + for(unsigned op=0;op<4;op++) { + int original=outcome(pairs[op].original,0),translated=outcome(pairs[op].translated,0); + if(original!=translated || (original!=SIGSEGV && original!=SIGBUS)) return 4; + } + return munmap(region,2*page)!=0; +} +` + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "clmul-original-dead-faults", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_clmul_forms.go b/amd64_clmul_forms.go new file mode 100644 index 000000000..89f116953 --- /dev/null +++ b/amd64_clmul_forms.go @@ -0,0 +1,172 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's yxshuf and _yvpclmulqdq rows select one qword from each 128-bit +// lane. The complete memory tuple is distinct from those numerical inputs. +type x86CLMULForm struct { + legacy bool + byteWidth, source, second int + destination, memory int + immediate uint8 +} + +func x86CLMULProperties(op Op, ins Instr, arch string) (x86CLMULForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + if name != "PCLMULQDQ" && name != "VPCLMULQDQ" { + return x86CLMULForm{}, false, nil + } + f := x86CLMULForm{legacy: name == "PCLMULQDQ", source: 1, second: 2, destination: 3, memory: -1} + if f.legacy { + f.destination = 2 + } + fail := func() (x86CLMULForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside complete Go carryless-multiply grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || dotted || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || + len(ins.Args) != f.destination+1 || arch == "386" && !f.legacy && !ins.x86Encoded { + return fail() + } + constant, valid := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !valid || !amd64UnsignedImmediate(constant, 8) { + return fail() + } + f.immediate = uint8(constant.Imm) + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + if f.legacy && f.byteWidth != 16 || f.byteWidth != 16 && f.byteWidth != 32 && f.byteWidth != 64 { + return fail() + } + register := func(operand Operand) bool { + index, valid := amd64VectorRegisterIndex(operand.Reg, f.byteWidth) + limit := 32 + if f.legacy { + limit = 16 + } + if arch == "386" { + limit = 8 // A physical raw binding is not a high-bank or named-frontend permission. + } + return operand.Kind == OpReg && valid && index < limit + } + if !register(ins.Args[f.second]) || !register(ins.Args[f.destination]) { + return fail() + } + source := ins.Args[f.source] + if register(source) { + return f, true, nil + } + if !isAMD64MemoryOperand(source) || !x86PackedShiftMemoryValid(source, arch) { + return fail() + } + if source.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(source.Mem, arch); err != nil { + return f, true, err + } + if source.Mem.OffRaw != "" || source.Mem.IndexExt != "" { + return fail() + } + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return f, true, fmt.Errorf("%w: carryless multiply has no bound native address-size override", ErrProbeNeedsContext) + } + f.memory = f.source + return f, true, nil +} + +func x86TypedLegacyCLMULByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86CLMULProperties(op, ins, arch) + if !known || err != nil || !form.legacy { + return x86TypedByteEffect{}, known && (err != nil || form.legacy), err + } + return x86CLMULDependencyEffect(ins, form), true, nil +} + +func x86CLMULDependencyEffect(ins Instr, form x86CLMULForm) x86TypedByteEffect { + binary := x86PackedBinaryForm{ + byteWidth: form.byteWidth, first: form.source, second: form.second, destination: form.destination, + legacy: form.legacy, mask: -1, memory: form.memory, memoryBytes: form.byteWidth, + } + effect := x86PackedBinaryDependencyEffect(ins, binary, ins.Args[form.source]) + source, second := x86CLMULSelectedBytes(form) + if ins.Args[form.source].Kind == OpReg { + effect.operands[0].requiredBytes = source + } + effect.operands[1].requiredBytes = second + return effect +} + +// Selection controls numerical register inputs, never the physical memory +// tuple. Each lane consumes one qword from each of the two source operands. +func x86CLMULSelectedBytes(f x86CLMULForm) (source, second uint64) { + for lane := 0; lane < f.byteWidth; lane += 16 { + source |= uint64(255) << uint(lane+int(f.immediate>>4&1)*8) + second |= uint64(255) << uint(lane+int(f.immediate&1)*8) + } + return +} + +// Each product requires all bytes of exactly two selected qwords. No mask, +// unselected qword, destination alias or numerical zero supplies a missing input. +func x86CLMULDefinedBytes(f x86CLMULForm, source, second uint64) uint64 { + var known uint64 + sourceBytes, secondBytes := x86CLMULSelectedBytes(f) + for lane := 0; lane < f.byteWidth; lane += 16 { + laneMask := uint64(65535) << uint(lane) + sourceMask, secondMask := sourceBytes&laneMask, secondBytes&laneMask + if source&sourceMask == sourceMask && second&secondMask == secondMask { + known |= laneMask + } + } + return known +} + +func x86ResultCLMULStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + f, known, err := x86CLMULProperties(ins.Op, ins, arch) + if !known { + return false, false + } + if err != nil { + return false, true + } + return x86ResultCLMULTransfer(ins, f, state, owned, arch, false) +} + +// Keep the complete captured form across a source-local provenance projection. +// Memory always consumes the original full tuple; numerical availability still +// depends only on the two selected qwords of each independent 128-bit lane. +func x86ResultCLMULTransfer(ins Instr, f x86CLMULForm, state *x86ResultState, + owned []bool, arch string, nativeRead bool) (bool, bool) { + source := x86CallerVectorByteMask(int64(f.byteWidth)) + if f.memory >= 0 { + if !x86ResultReadWithNativeContinuation(ins, ins.Args[f.memory], int64(f.byteWidth), state, owned, arch, nativeRead) { + return false, true + } + } else { + index, _ := amd64VectorRegisterIndex(ins.Args[f.source].Reg, f.byteWidth) + source = state.vectorBytes(index) + } + second, _ := amd64VectorRegisterIndex(ins.Args[f.second].Reg, f.byteWidth) + destination, _ := amd64VectorRegisterIndex(ins.Args[f.destination].Reg, f.byteWidth) + state.writeVector(destination, f.byteWidth, x86CLMULDefinedBytes(f, source, state.vectorBytes(second)), f.legacy) + return true, true +} + +func x86CallerCLMULStep(ins Instr, f x86CLMULForm, state *x86SourceStackState) bool { + if f.memory >= 0 { + return false // A registered byte effect grants no new native read proof. + } + source := state.vectors[x86CallerVectorRoot(ins.Args[f.source].Reg)] + second := state.vectors[x86CallerVectorRoot(ins.Args[f.second].Reg)] + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + width := x86CallerVectorByteMask(int64(f.byteWidth)) + known := x86CLMULDefinedBytes(f, source, second) + if f.legacy { + state.vectors[destination] = state.vectors[destination]&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_clmul_forms_test.go b/amd64_clmul_forms_test.go new file mode 100644 index 000000000..17e874bc4 --- /dev/null +++ b/amd64_clmul_forms_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86CLMULIndependentCompleteGoRows(t *testing.T) { + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(legacy), "{APCLMULQDQ, yxshuf, Pq, opBytes{0x3a, 0x44, 0}}") { + t.Fatal("complete Go legacy prefix/map/opcode/offset changed") + } + start := strings.Index(string(legacy), "var yxshuf = []ytab{") + if start < 0 { + t.Fatal("Go omitted yxshuf") + } + actual := strings.SplitN(string(legacy)[start:], "\n}", 2)[0] + actual = strings.SplitN(actual, "[]ytab{", 2)[1] + if strings.Join(strings.Fields(actual), " ") != "{Zibm_r, 2, argList{Yu8, Yxm, Yxr}}," { + t.Fatal("complete Go legacy classes/zcase/zoffset changed", actual) + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + classes := []string{ + "Yu8,Yxm,Yxr,Yxr", "Yu8,Yym,Yyr,Yyr", + "Yu8,YxmEvex,YxrEvex,YxrEvex", "Yu8,YymEvex,YyrEvex,YyrEvex", "Yu8,Yzm,Yzr,Yzr", + } + opbytes := []string{ + "avxEscape | vex128 | vex66 | vex0F3A | vexW0", "0x44", + "avxEscape | vex256 | vex66 | vex0F3A | vexW0", "0x44", + "avxEscape | evex128 | evex66 | evex0F3A | evexW0", "evexN16", "0x44", + "avxEscape | evex256 | evex66 | evex0F3A | evexW0", "evexN32", "0x44", + "avxEscape | evex512 | evex66 | evex0F3A | evexW0", "evexN64", "0x44", + } + count := 0 + for _, rule := range rules { + if rule.Opcode != "VPCLMULQDQ" { + continue + } + zcase := "Zevex_i_rm_v_r" + if count < 2 { + zcase = "Zvex_i_rm_v_r" + } + if count >= 5 || rule.RowIndex != count || rule.Ytab != "_yvpclmulqdq" || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != []int{2, 2, 3, 3, 3}[count] || + rule.EncodingOffset != []int{0, 2, 4, 7, 10}[count] || + strings.Join(rule.Classes, ",") != classes[count] || !reflect.DeepEqual(rule.OpBytes, opbytes) { + t.Fatal("complete independent Go vector row changed", rule) + } + count++ + } + if count != 5 { + t.Fatal("incomplete vector family", count) + } +} + +func x86CLMULIndependentRows(arch string) []string { + var rows []string + limit := 16 + if arch == "386" { + limit = 8 + } + for immediate := 0; immediate < 256; immediate++ { + rows = append(rows, fmt.Sprintf("PCLMULQDQ $%d,X0,X1", immediate)) + } + for index := 0; index < limit; index++ { + rows = append(rows, fmt.Sprintf("PCLMULQDQ $17,X%d,X1", index), fmt.Sprintf("PCLMULQDQ $17,X0,X%d", index)) + } + for _, memory := range []string{"(AX)", "16(BX)(CX*2)", "value+0(FP)", "local-16(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, "PCLMULQDQ $255,"+memory+",X1") + } + if arch == "386" { + // cmd/asm classifies TYPE_CONST after mode32 signed truncation. + rows = append(rows, "PCLMULQDQ $4294967296,X0,X1", "PCLMULQDQ $4294967551,X0,X1") + return rows + } + for _, bank := range []string{"X", "Y", "Z"} { + for immediate := 0; immediate < 256; immediate++ { + rows = append(rows, fmt.Sprintf("VPCLMULQDQ $%d,%s0,%s1,%s2", immediate, bank, bank, bank)) + } + for index := 0; index < 32; index++ { + for operand := 0; operand < 3; operand++ { + registers := []string{bank + "0", bank + "1", bank + "2"} + registers[operand] = fmt.Sprintf("%s%d", bank, index) + rows = append(rows, "VPCLMULQDQ $255,"+strings.Join(registers, ",")) + } + } + for _, memory := range []string{"(AX)", "16(BX)(CX*2)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, "VPCLMULQDQ $17,"+memory+","+bank+"1,"+bank+"2") + } + } + return rows +} + +func TestX86CLMULCompleteGoOperandAxes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + rows := x86CLMULIndependentRows(arch) + var source strings.Builder + for index, row := range rows { + fmt.Fprintf(&source, "TEXT f%d(SB),4,$64-64\n%s\nRET\n", index, row) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for index, fn := range file.Funcs { + ins := fn.Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, known, err := x86CLMULProperties(ins.Op, ins, arch) + if !known || err != nil { + t.Fatal(rows[index], err) + } + for operand := range ins.Args { + width, write, memory := x86InstructionMemoryShape(ins, operand, arch) + if memory != (operand == form.memory) || memory && (write || width != int64(form.byteWidth)) { + t.Fatal("native tuple differs from complete operand form", rows[index], operand, width, write, memory) + } + if ins.Args[operand].Kind == OpFP { + width, write, known := X86FPAccessBytes(ins, operand, arch) + if !known || write || width != int64(form.byteWidth) { + t.Fatal("FP diagnostic lost full native tuple", rows[index], width, write, known) + } + } + } + if fmt.Sprintf("%#v", ins) != before { + t.Fatal("grammar changed its original source") + } + } + t.Logf("%d complete independent Go forms accepted", len(rows)) + }) + } +} + +func TestX86CLMULInvalidGrammarAndPrivateBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + rows := []string{ + "PCLMULQDQ $-1,X0,X1", "PCLMULQDQ $256,X0,X1", "PCLMULQDQ $1.5,X0,X1", + "PCLMULQDQ $1,X16,X0", "PCLMULQDQ $1,Y0,Y1", "PCLMULQDQ $1,M0,M1", + "PCLMULQDQ.Z $1,X0,X1", "PCLMULQDQ $1,X0,(AX)", "PCLMULQDQ $1,X0", + "VPCLMULQDQ $-1,X0,X1,X2", "VPCLMULQDQ $256,X0,X1,X2", + "VPCLMULQDQ.Z $1,X0,X1,X2", "VPCLMULQDQ.BCST $1,(AX),X1,X2", + "VPCLMULQDQ $1,X0,Y1,Y2", "VPCLMULQDQ $1,X0,X1,K1,X2", + "VPCLMULQDQ $1,X0,(AX),X2", "VPCLMULQDQ $1,X0,X1,(AX)", + } + if arch == "386" { + rows = append(rows, "PCLMULQDQ $1,X8,X0", "VPCLMULQDQ $1,X0,X1,X2", "VPCLMULQDQ $1,Z0,Z1,Z2") + } + for _, row := range rows { + goAssemblerOracleOutput(t, arch, "TEXT ·invalid(SB),4,$0-0\n"+row+"\nRET\n", "", false) + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86CLMULProperties(ins.Op, ins, arch); !known || err == nil { + t.Error("unbound grammar accepted", arch, row) + } + } + } + for _, mutation := range []string{"float", "unresolved", "dispatch", "address-size", "unknown-displacement", "unknown-index"} { + ins := x86ScalarLaneInstruction(t, "PCLMULQDQ $1,(AX),X1") + op := ins.Op + switch mutation { + case "float": + ins.Args[0].ImmIsFloat = true + case "unresolved": + ins.Args[0].ImmRaw = "missing" + case "dispatch": + ins.Op = "AESENC" + case "address-size": + ins.x86AddressBits = 32 + case "unknown-displacement": + ins.Args[1].Mem.OffRaw = "missing" + case "unknown-index": + ins.Args[1].Mem.IndexExt = "unknown" + } + if _, known, err := x86CLMULProperties(op, ins, "amd64"); !known || err == nil { + t.Error("constructed source bypassed complete grammar", mutation) + } + } +} diff --git a/amd64_clmul_objects_test.go b/amd64_clmul_objects_test.go new file mode 100644 index 000000000..03cd91c16 --- /dev/null +++ b/amd64_clmul_objects_test.go @@ -0,0 +1,44 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86CLMULThreeAPIFiveTargetResultObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc is required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + for _, vector := range []bool{false, true} { + if target.arch == "386" && vector { + continue // Source mode, independently checked against Go. + } + for _, selector := range []int{0, 1, 16, 17} { + var body strings.Builder + body.WriteString("MOVL a+0(FP),AX\nMOVL b+4(FP),BX\n") + for half := 0; half < 2; half++ { + fmt.Fprintf(&body, "PINSRD $%d,AX,X0\nPINSRD $%d,BX,X1\n", (selector>>4&1)*2+half, (selector&1)*2+half) + } + if vector { + fmt.Fprintf(&body, "VPCLMULQDQ $%d,X0,X1,X2\nPEXTRD $0,X2,AX\n", selector) + } else { + fmt.Fprintf(&body, "PCLMULQDQ $%d,X0,X1\nPEXTRD $0,X1,AX\n", selector) + } + source := "TEXT ·view(SB),4,$0-12\n" + body.String() + "MOVL AX,ret+8(FP)\nRET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(a,b uint32) uint32\n", target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal(api, vector, selector, err) + } + compileLLVMToObject(t, llc, target.triple, "clmul-result.ll", "clmul-result.o", ir) + } + } + } + }) + } +} diff --git a/amd64_clmul_runtime_test.go b/amd64_clmul_runtime_test.go new file mode 100644 index 000000000..2d6357b83 --- /dev/null +++ b/amd64_clmul_runtime_test.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Polynomial long multiplication is independent of the production selectors, +// descriptors and LLVM intrinsic. Both input qwords and both output qwords are +// observed, including the otherwise ignored immediate bits. +const x86CLMULIndependentC = `#include +#include +#include +#include +static void product(uint64_t a,uint64_t b,uint64_t *out) { + uint64_t lo=0,hi=0; + for(unsigned bit=0;bit<64;bit++) { + if((b>>bit)&1) { + lo^=a<>(64-bit); + } + } + out[0]=lo; out[1]=hi; +} +static const uint64_t edges[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000), + UINT64_C(0x123456789abcdef0),UINT64_C(0xfedcba9876543210), + UINT64_C(0x1111111111111111),UINT64_C(0xaaaaaaaaaaaaaaaa)}; +struct Buffer {_Alignas(64) uint8_t before[64],value[80],after[64];}; +typedef void (*Program)(uint8_t*,const uint8_t*,const uint8_t*); +static int check(Program program,const char *name,unsigned width,int legacy,int selector,int alias) { + for(unsigned sample=0;sample<73;sample++) { + struct Buffer a,b,out; + memset(&a,0xa7,sizeof(a)); memset(&b,0xb9,sizeof(b)); memset(&out,0xc3,sizeof(out)); + for(unsigned word=0;word<8;word++) { + uint64_t av=edges[(sample+word)%8],bv=edges[(sample/8+word*3)%8]; + if(sample>=64) {av^=(uint64_t)sample*713781;bv^=(uint64_t)sample*391531;} + memcpy(a.value+word*8,&av,8); memcpy(b.value+word*8,&bv,8); + } + struct Buffer originalA=a,originalB=b,want=out; + for(unsigned lane=0;lane>4)&1)*8,8); + memcpy(&bv,b.value+lane+(selector&1)*8,8); + product(bv,av,pair); memcpy(want.value+lane,pair,16); + } + memcpy(want.value+width,legacy?b.value+width:(uint8_t[64]){0},64-width); + want.value[64]=1; want.value[65]=0; + unsigned saved=_mm_getcsr(),profile=0x1f80|((sample%4)<<13)|(sample%64); + memcpy(want.value+68,&profile,4); _mm_setcsr(profile); + program(out.value,a.value,b.value); unsigned after=_mm_getcsr(); _mm_setcsr(saved); + if(after!=profile||memcmp(&a,&originalA,sizeof(a))||memcmp(&b,&originalB,sizeof(b))||memcmp(&out,&want,sizeof(out))) { + for(unsigned i=0;i= width) { + t.Fatal("each potential byte must exist, regardless of predicate", valid, width, effect) + } + if !valid && state.known[input] != before { + t.Fatal("failed proof washed unknown bytes") + } + result := x86ResultState{flags: x86ByteFlagsAll} + result.gp[0], result.gp[3] = 0xff, 0xff + index := 0 + if input == BX { + index = 3 + } + result.gp[index] &^= 1 << uint(missing) + beforeResult := result + validResult := x86ResultTransfer(file.Funcs[0].Instrs[1], &result, nil, 0, "amd64") + if validResult != (missing >= width) || !validResult && !reflect.DeepEqual(result, beforeResult) { + t.Fatal("result observer borrowed potential undefined bytes or mutated a failed proof", validResult) + } + if validResult && width == 4 && result.gp[3] != 0xff { + t.Fatal("L false-or-true writes did not define upper zero bytes") + } + }) + } + } + } +} + +func TestX86CMOVFamilyExactRequiredFlagsAndUnconditionalFrameBytes(t *testing.T) { + for _, name := range x86CMOVOracleNames("amd64") { + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[name[4]] + for _, source := range []string{"AX", "x+0(FP)", "local-8(SP)"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$8-8\n"+name+" "+source+",BX\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + flags := uint8(0) + for _, condition := range x86CMOVOracleConditions { + if strings.HasSuffix(name, condition.name) { + flags = condition.flags + } + } + for bit := uint8(1); bit <= x86ByteFlagOF; bit <<= 1 { + state := x86ResultState{flags: x86ByteFlagsAll &^ bit, fp: make([]bool, 8), auto: make([]bool, 8)} + state.gp[0], state.gp[3] = 0xff, 0xff + owned := make([]bool, 8) + for index := 0; index < 8; index++ { + state.fp[index], state.auto[index], owned[index] = true, true, true + } + valid := x86ResultTransfer(ins, &state, owned, 8, "amd64") + if valid != (flags&bit == 0) || valid && state.flags != x86ByteFlagsAll&^bit { + t.Fatal("CMOV exact flag dependency/preservation", name, source, bit, valid) + } + } + if source == "AX" { + continue + } + for missing := 0; missing < 8; missing++ { + state := x86ResultState{flags: x86ByteFlagsAll, fp: make([]bool, 8), auto: make([]bool, 8)} + state.gp[0], state.gp[3] = 0xff, 0xff + owned := make([]bool, 8) + for index := 0; index < 8; index++ { + state.fp[index], state.auto[index], owned[index] = true, true, true + } + state.fp[missing], state.auto[missing] = false, false + if valid := x86ResultTransfer(ins, &state, owned, 8, "amd64"); valid != (missing >= width) { + t.Fatal("CMOV condition cannot suppress a missing FP/AUTO source byte", name, source, missing, valid) + } + } + } + } +} + +func TestX86CMOVFamilyCountedComparisonVersionGuards(t *testing.T) { + base := x86CallerCountedLoopScalarAssembly(false) + for _, destination := range []string{"BX", "CX", "AX"} { + t.Run(destination, func(t *testing.T) { + assembly := strings.Replace(base, "CMPL CX,AX\nJNE again", "CMPL CX,AX\nCMOVLLE R11,"+destination+"\nJNE again", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("CMOV relaxed the exact adjacent comparison/latch or retained a stale counter/bound version", destination, err) + } + }) + } +} + +func TestX86CMOVFamilyCountedNumberObserver(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-0\nCMOVLLE BX,AX\nRET\n") + if err != nil { + t.Fatal(err) + } + flags := x86CallerCountedFlags{pc: 7, op: "CMPL", known: true, equal: true} + facts := x86CallerCountedNumbers{regs: map[Reg]x86CallerCountedNumber{ + AX: {version: 2, value: 17, known: true}, BX: {version: 3, value: 23, known: true}, + }, flags: flags, source: -1} + facts.step(8, 0, file.Funcs[0].Instrs[1], x86SourceStackState{}, nil) + if facts.flags != flags { + t.Fatal("CMOV changed existing numerical flag facts", facts.flags) + } + if _, retained := facts.regs[AX]; retained { + t.Fatal("CMOV invented an unconditional selected numerical value or retained a stale target version") + } + if !facts.regs[BX].known || facts.regs[BX].version != 3 { + t.Fatal("CMOV erased its unchanged source version") + } +} + +func TestX86CMOVFamilyIndependentResultByteEffect(t *testing.T) { + for _, row := range []string{"CMOVWCC AX,BX", "CMOVLLE AX,BX", "CMOVQPS AX,BX", "CMOVLLE AX,AX"} { + t.Run(row, func(t *testing.T) { + result := "BX" + if row == "CMOVLLE AX,AX" { + result = "AX" + } + body := "MOVQ x+0(FP),AX\nMOVQ $9,BX\nCMPQ AX,BX\n" + row + "\nMOVQ " + result + ",ret0+8(FP)\nRET" + _, file, sig, source := x86ResultInitializationFixture(t, "amd64", "func Value(x uint64) uint64", body) + goAssemblerOracleOutput(t, "amd64", source, "", true) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); err != nil { + t.Fatal("actual-Go conditional move has no complete result byte effect", err) + } + }) + } +} + +func TestX86CMOVFamilyIndependentDeadNativeRead(t *testing.T) { + for _, row := range []string{"CMOVWCC (DI),BX", "CMOVLLE (DI),BX", "CMOVQPS (DI),BX"} { + t.Run(row, func(t *testing.T) { + source := "TEXT ·Dead(SB),4,$0-0\nMOVQ $7,AX\nMOVQ $9,BX\nCMPQ AX,BX\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Dead()", "amd64", "x86_64-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original uninitialized DI acquired native memory permission", api, err) + } + } + initialized := strings.Replace(source, "$0-0\n", "$0-8\nMOVQ p+0(FP),DI\n", 1) + goAssemblerOracleOutput(t, "amd64", initialized, "", true) + ir := x86NarrowOracleIR(t, initialized, "package cmov\nfunc Dead(p *byte)\n", "x86_64-unknown-linux-gnu") + optimized := x86DwordProductIROptimize(t, ir) + t.Logf("original loads=%t; actual IR default retains any read=%t", strings.Contains(ir, " = load "), strings.Contains(optimized, " = load ")) + if !strings.Contains(ir, "load volatile") || !strings.Contains(optimized, "load volatile") { + t.Fatal("unconditional source read can disappear with a dead CMOV result under real IR default") + } + }) + } +} + +func TestX86CMOVFamilyExactSourceMemoryCertificates(t *testing.T) { + for _, name := range x86CMOVOracleNames("amd64") { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "CMPQ AX,AX\n"+name+" (DI),AX\nMOVL AX,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + if ins.Op != Op(name) { + continue + } + found = true + access, exists := ins.x86CallerMemory[0] + width := map[byte]int64{'W': 2, 'L': 4, 'Q': 8}[name[4]] + if !exists || access.width != width || access.write || !x86CallerMemoryCertified(ins, 0) { + t.Fatal("false condition shrank or omitted native source read", name, access) + } + for _, mutation := range []string{"opcode", "width", "direction", "offset", "destination", "source", "address-size", "segment"} { + changed, certificate := ins, access + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86CallerMemory = map[int]x86CallerMemoryAccess{0: access} + switch mutation { + case "opcode": + changed.Op = "CMOV" + Op(string(name[4])) + "CS" + if changed.Op == ins.Op { + changed.Op = "CMOV" + Op(string(name[4])) + "CC" + } + case "width": + certificate.width-- + changed.x86CallerMemory[0] = certificate + case "direction": + certificate.write = true + changed.x86CallerMemory[0] = certificate + case "offset": + changed.Args[0].Mem.Off++ + case "destination": + changed.Args[1].Reg = BX + case "source": + changed.Args[0].Mem.Base = SI + case "address-size": + changed.x86AddressBits = 32 + case "segment": + changed.Args[0].Mem.Segment = FS + } + if x86CallerMemoryCertified(changed, 0) { + t.Fatal("stale source or native extent survived", name, mutation) + } + } + } + if !found { + t.Fatal("source normalization omitted family instruction", name) + } + } +} diff --git a/amd64_cmov_fault_runtime_unix_test.go b/amd64_cmov_fault_runtime_unix_test.go new file mode 100644 index 000000000..127eb792b --- /dev/null +++ b/amd64_cmov_fault_runtime_unix_test.go @@ -0,0 +1,117 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86CMOVUnconditionalFaultEnvelopes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, prototypes, originals, rows strings.Builder + declarations.WriteString("package faultoracle\n") + count := 0 + for _, width := range []string{"W", "L", "Q"} { + for condition, row := range x86CMOVOracleConditions { + for _, dead := range []bool{false, true} { + fn := fmt.Sprintf("F%d", count) + count++ + name := "CMOV" + width + row.name + store, nativeStore := "MOVQ out+16(FP),DX\nMOVQ BX,(DX)\n", "MOVQ AX,(DX)\n" + if dead { + store, nativeStore = "", "" + } + fmt.Fprintf(&declarations, "func %s(p *[8]byte,seed uint64,out *uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-24\nMOVQ p+0(FP),DI\nMOVQ seed+8(FP),AX\nMOVQ $0x87654321fedcba98,BX\nPUSHQ AX\nPOPFQ\n%s (DI),BX\n%sRET\n", fn, name, store) + original := "Original" + fn + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\nMOVQ $0x87654321fedcba98,AX\nPUSHQ SI\nPOPFQ\n"+name+" (DI),AX\n"+nativeStore+"RET\n") + var bytes []string + for _, value := range actual { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint64_t,uint64_t*);\nextern void %s(const uint8_t*,uint64_t,uint64_t*);\n", fn, original) + fmt.Fprintf(&rows, "{%s,%s,%d,%d},\n", fn, original, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + reference := fmt.Sprintf(x86CMOVFaultC, prototypes.String(), originals.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-fault", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("cmov live/dead kernels %d translated/original fault outcomes %d false/live/dead tripwires 4 failures 0\n", count, count*2*7*2) + if string(output) != want { + t.Fatal("unconditional exact-width source fault disagreement", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } +} + +const x86CMOVFaultC = ` +#include +#include +#include +#include +#include +#include +#include +#include +` + x86CMOVConditionC + ` +%s +%s +typedef void(*Program)(const uint8_t*,uint64_t,uint64_t*); +typedef struct {Program translated,original;unsigned width,condition;} Kernel; +static Kernel kernels[]={%s}; +static int observe(Program p,unsigned width,unsigned address,uint64_t flags) { + pid_t child=fork();if(child<0) return -1; + if(child==0) { + struct rlimit no_core={0,0};if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + uint8_t data[16];memset(data,0xa7,sizeof(data));const uint8_t *input=data; + if(address==1) input++; + if(address==2) input=NULL; + if(address>=3) { + long page=sysconf(_SC_PAGESIZE);if(page<=0) _exit(121); + uint8_t *memory=mmap(NULL,2*(size_t)page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(memory==MAP_FAILED) _exit(122); + memset(memory,0xa7,(size_t)page); + if(mprotect(memory+page,(size_t)page,PROT_NONE)) _exit(123); + unsigned available[]={0,width-1,width,width+1};input=memory+page-available[address-3]; + } + uint64_t out=0;p(input,flags,&out);_exit(0); + } + int status=0;if(waitpid(child,&status,0)!=child) return -2; + if(WIFEXITED(status)) return WEXITSTATUS(status)?-3:0; + if(WIFSIGNALED(status)&&(WTERMSIG(status)==SIGSEGV||WTERMSIG(status)==SIGBUS)) return 1; + return -4; +} +static void missing_read(const uint8_t*p,uint64_t f,uint64_t*out) {(void)p;(void)f;(void)out;} +static void too_wide(const uint8_t*p,uint64_t f,uint64_t*out) {(void)f;*out=*(volatile const uint64_t*)p;} +int main(void) { + unsigned count=0; + // Negative observers prove faults are real, not a forged list of outcomes. + if(observe(missing_read,8,2,0)!=0||observe(missing_read,8,4,0)!=0||observe(too_wide,2,5,0)!=1||observe(too_wide,4,5,0)!=1) return 4; + for(unsigned i=0;i=0;native--) { + int want=address==2||address==3||address==4; + int got=observe(native?k.original:k.translated,k.width,address,flags);count++; + if(got!=want) {fprintf(stderr,"kernel%%u width%%u taken%%u address%%u native%%u got%%d want%%d\n",i,k.width,taken,address,native,got,want);return 1;} + } + } + printf("cmov live/dead kernels %%zu translated/original fault outcomes %%u false/live/dead tripwires 4 failures 0\n",sizeof(kernels)/sizeof(kernels[0]),count); + return 0; +} +` diff --git a/amd64_cmov_fixed_local_runtime_test.go b/amd64_cmov_fixed_local_runtime_test.go new file mode 100644 index 000000000..1991188e2 --- /dev/null +++ b/amd64_cmov_fixed_local_runtime_test.go @@ -0,0 +1,174 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86CMOVFixedLocalAUTOWholeBytesBatch00(t *testing.T) { + x86CMOVFixedLocalRuntimeBatch(t, 0) +} + +func TestCrossLinuxRuntimeMatrixX86CMOVFixedLocalAUTOWholeBytesBatch01(t *testing.T) { + x86CMOVFixedLocalRuntimeBatch(t, 1) +} + +func TestCrossLinuxRuntimeMatrixX86CMOVFixedLocalAUTOWholeBytesBatch02(t *testing.T) { + x86CMOVFixedLocalRuntimeBatch(t, 2) +} + +func TestCrossLinuxRuntimeMatrixX86CMOVFixedLocalAUTOWholeBytesBatch03(t *testing.T) { + x86CMOVFixedLocalRuntimeBatch(t, 3) +} + +func TestCrossLinuxRuntimeMatrixX86CMOVFixedLocalAUTOWholeBytesBatch04(t *testing.T) { + x86CMOVFixedLocalRuntimeBatch(t, 4) +} + +func TestCrossLinuxRuntimeMatrixX86CMOVFixedLocalAUTOWholeBytesBatch05(t *testing.T) { + x86CMOVFixedLocalRuntimeBatch(t, 5) +} + +func TestCrossLinuxRuntimeMatrixX86CMOVFixedLocalAUTOWholeBytesBatch06(t *testing.T) { + x86CMOVFixedLocalRuntimeBatch(t, 6) +} + +func x86CMOVFixedLocalRuntimeBatch(t *testing.T, batch int) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + target, ok := x86CMOVFixedLocalExecutionTarget(execution.triple) + if !ok { + t.Fatal("CMOV frame capture requires the actual execution target", execution.triple) + } + goAMD64FramePrepareOwnedToolCache(t) + triple := execution.triple + kernels := x86CMOVFixedLocalKernels(true) + start, end, ok := fixedLocalObjectBatchRange(len(kernels), batch) + if !ok { + t.Fatal("invalid complete CMOV runtime batch", batch, len(kernels)) + } + var assembly, declarations, prototypes, rows strings.Builder + prefix := "" + if target == "darwin/amd64" { + prefix = "_" + } + for _, kernel := range kernels[start:end] { + assembly.WriteString(kernel.assembly) + fmt.Fprintf(&declarations, "func %s(a,b uint64) uint64\n", kernel.name) + symbol := prefix + "example.invalid/framefixture." + kernel.name + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint64_t,uint64_t) __asm__(%s);\n", kernel.name, runtimeCAsmLabel(symbol)) + fmt.Fprintf(&rows, "{%s,%d,%d,%d},\n", kernel.name, kernel.width, kernel.condition, kernel.mode) + } + pkg, request, file, options := x86CMOVFixedLocalFixture(t, assembly.String(), declarations.String(), target, triple) + observer := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := observer.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, bound) + if err != nil { + t.Fatal(err) + } + count := end - start + reference := runtimeCAsmLabelPreamble + fmt.Sprintf(x86CMOVFixedLocalWholeBytesC, prototypes.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-fixed-local-auto", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("cmov/private GP/SIMD AUTO kernels %d whole-byte outcomes %d failures 0\n", count, count*64) + if string(output) != want { + t.Fatal("actual native AUTO displacement/predicate/writeback mismatch", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + x86WordMultiplyRunOriginalGo(t, execution, "package oracle\n"+declarations.String(), assembly.String(), fmt.Sprintf(x86CMOVFixedLocalOriginalGo, rows.String())) + t.Logf("batch%d: %d/%d independently executed actual Go ABI0 local CMOV/GP/SIMD kernels ×64 full-width input pairs; host native/Rosetta is not strict Linux evidence", batch, count, len(kernels)) +} + +func x86CMOVFixedLocalExecutionTarget(triple string) (string, bool) { + switch triple { + case "x86_64-unknown-linux-gnu": + return "linux/amd64", true + case "x86_64-apple-darwin", "x86_64-apple-macosx": + return "darwin/amd64", true + case "x86_64-pc-windows-msvc", "x86_64-w64-windows-gnu": + return "windows/amd64", true + default: + return "", false + } +} + +func TestX86CMOVFixedLocalExecutionTargets(t *testing.T) { + for _, row := range []struct{ triple, target string }{ + {"x86_64-unknown-linux-gnu", "linux/amd64"}, + {"x86_64-apple-darwin", "darwin/amd64"}, + {"x86_64-apple-macosx", "darwin/amd64"}, + {"x86_64-pc-windows-msvc", "windows/amd64"}, + {"x86_64-w64-windows-gnu", "windows/amd64"}, + {"aarch64-unknown-linux-gnu", ""}, + {"i686-pc-windows-msvc", ""}, + {"", ""}, + } { + actual, ok := x86CMOVFixedLocalExecutionTarget(row.triple) + if actual != row.target || ok != (row.target != "") { + t.Fatal("capture must retain the actual native execution platform", row, actual, ok) + } + } + for _, platform := range []string{"linux", "darwin", "windows"} { + triple := testTargetTriple(platform, "amd64") + if actual, ok := x86CMOVFixedLocalExecutionTarget(triple); !ok || actual != platform+"/amd64" { + t.Fatal("actual shared runtime triple lost its capture platform", triple, actual, ok) + } + } +} + +const x86CMOVFixedLocalWholeBytesC = ` +#include +#include +` + x86CMOVConditionC + ` +%s +typedef struct {uint64_t(*fn)(uint64_t,uint64_t);unsigned width,condition,mode;} Kernel; +static Kernel kernels[]={%s}; +int main(void) { + const uint64_t edges[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000),UINT64_C(0x7fffffffffffffff),UINT64_C(0x87654321fedcba98),UINT64_C(0x80000000),UINT64_C(0x7fc123457f800001)}; + unsigned count=0; + for(unsigned k=0;k>bit)&1; + uint64_t flags=(a>63)<<7)|((((a^b)&(a^difference))>>63)<<11); + uint64_t selected=predicate(kernel.condition,flags)?a:b; + uint64_t mask=kernel.width==8?UINT64_MAX:((UINT64_C(1)<<(kernel.width*8))-1); + uint64_t want=kernel.width==2?((b&~mask)|(selected&mask)):(selected&mask); + if(kernel.mode) want=kernel.mode==1?b:a; + uint64_t got=kernel.fn(a,b); + if(got!=want) {fprintf(stderr,"kernel%%u a%%llx b%%llx got%%llx want%%llx\n",k,(unsigned long long)a,(unsigned long long)b,(unsigned long long)got,(unsigned long long)want);return 1;} + count++; + } + printf("cmov/private GP/SIMD AUTO kernels %%zu whole-byte outcomes %%u failures 0\n",sizeof(kernels)/sizeof(kernels[0]),count); + return 0; +} +` + +const x86CMOVFixedLocalOriginalGo = `package oracle +import "testing" +func TestOriginal(t *testing.T) { + kernels:=[]struct{fn func(uint64,uint64)uint64;width,condition,mode int}{%s} + edges:=[]uint64{0,1,0xffffffffffffffff,0x8000000000000000,0x7fffffffffffffff,0x87654321fedcba98,0x80000000,0x7fc123457f800001} + count:=0 + for k,kernel:=range kernels {for _,a:=range edges {for _,b:=range edges { + d:=a-b;ones:=0;for bit:=uint(0);bit<8;bit++ {ones+=int(d>>bit&1)} + cf,pf,zf,sf,of:=a>63!=0,((a^b)&(a^d))>>63!=0 + predicates:=[]bool{!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf} + selected:=b;if predicates[kernel.condition] {selected=a} + mask:=uint64(0xffffffffffffffff);if kernel.width<8 {mask=(uint64(1)< end { + limit = end + } + units = append(units, x86CMOVFixedLocalSourceForRange(kernels, chunk, limit)) + } + return units, true +} + +func x86CMOVFixedLocalSourceForRange(kernels []x86CMOVFixedLocalKernel, start, end int) x86CMOVFixedLocalSourceUnit { + unit := x86CMOVFixedLocalSourceUnit{start: start, end: end} + var assembly, declarations strings.Builder + for _, kernel := range kernels[start:end] { + assembly.WriteString(kernel.assembly) + fmt.Fprintf(&declarations, "func %s(a,b uint64) uint64\n", kernel.name) + unit.names = append(unit.names, "example.invalid/framefixture."+kernel.name) + } + unit.assembly, unit.declarations = assembly.String(), declarations.String() + return unit +} + +func TestFixedLocalCMOVObjectSourcesPreserveCompleteOriginalUnion(t *testing.T) { + kernels := x86CMOVFixedLocalKernels(false) + runtimeKernels := x86CMOVFixedLocalKernels(true) + if len(kernels) != 384 || len(runtimeKernels) != 387 || + !reflect.DeepEqual(runtimeKernels[:len(kernels)], kernels) { + t.Fatal("complete CMOV grammar or original runtime GP/SIMD suffix changed") + } + // Independent axes prevent a cropped or duplicated producer table from + // validating its own source union merely by retaining the same count. + conditions := []string{ + "CC", "CS", "EQ", "GE", "GT", "HI", "LE", "LS", + "LT", "MI", "NE", "OC", "OS", "PC", "PL", "PS", + } + sources := []string{ + "AX", "8(SP)", "tmp(SP)", "UNKNOWN(SP)", + "tmp+0(SP)", "tmp+(0*4)(SP)", "tmp-8(SP)", "tmp-(2*4)(SP)", + } + if !reflect.DeepEqual(x86CMOVAUTOSources, sources[2:]) || len(x86CMOVOracleConditions) != len(conditions) { + t.Fatal("complete independent CMOV condition/source spelling inventory changed") + } + for index, condition := range x86CMOVOracleConditions { + if condition.name != conditions[index] { + t.Fatal("original CMOV condition ordering changed", index, condition) + } + } + var originalAssembly, originalDeclarations strings.Builder + for index, kernel := range kernels { + operation := "CMOV" + []string{"W", "L", "Q"}[index/128] + conditions[index/8%16] + if kernel.name != fmt.Sprintf("F%d", index) || kernel.width != []int{2, 4, 8}[index/128] || + kernel.condition != index/8%16 || kernel.mode != 0 || + kernel.assembly != x86CMOVFixedLocalAssembly(kernel.name, operation, sources[index%8]) || + !strings.HasPrefix(kernel.assembly, "TEXT ·"+kernel.name+"(SB),4,$16-24\n") { + t.Fatal("original condition/width/source ordinal, symbol or ABI changed", index, kernel) + } + originalAssembly.WriteString(kernel.assembly) + fmt.Fprintf(&originalDeclarations, "func F%d(a,b uint64) uint64\n", index) + } + for index, kernel := range runtimeKernels[384:] { + if kernel.name != fmt.Sprintf("F%d", 384+index) || kernel.width != 8 || kernel.mode != index+1 { + t.Fatal("original runtime private GP/SIMD kernel changed", index, kernel) + } + } + seen, sourceCount := make([]int, len(kernels)), 0 + var actualAssembly, actualDeclarations strings.Builder + for batch := 0; batch < 6; batch++ { + start, end, ok := fixedLocalObjectBatchRange(len(kernels), batch) + if !ok || end-start != 64 { + t.Fatal("the six complete scheduling owners changed", batch, start, end, ok) + } + units, ok := x86CMOVFixedLocalObjectSources(kernels, batch) + if !ok || len(units) == 0 { + t.Fatal("missing independently complete original source units", batch) + } + next := start + for _, unit := range units { + if unit.start != next || unit.end <= unit.start || unit.end > end { + t.Fatal("source units cropped, overlap or escape the original owner", batch, unit.start, unit.end, next) + } + if unit.end-unit.start > 32 { + t.Error("one five-minute API lease still carries more than 32 complete functions", batch, unit.end-unit.start) + } + file, err := Parse(ArchAMD64, unit.assembly) + if err != nil || file == nil || len(file.Funcs) != unit.end-unit.start || len(unit.names) != len(file.Funcs) { + t.Fatal("source unit is not an independently complete parsed program", batch, err) + } + for index := unit.start; index < unit.end; index++ { + local := index - unit.start + if file.Funcs[local].Sym != fmt.Sprintf("·F%d", index) || + unit.names[local] != fmt.Sprintf("example.invalid/framefixture.F%d", index) { + t.Fatal("source unit renamed, reordered or substituted an original function", index) + } + seen[index]++ + } + actualAssembly.WriteString(unit.assembly) + actualDeclarations.WriteString(unit.declarations) + next, sourceCount = unit.end, sourceCount+1 + } + if next != end { + t.Fatal("source units omitted the end of a complete scheduling owner", batch, next, end) + } + } + if sourceCount != 12 || actualAssembly.String() != originalAssembly.String() || + actualDeclarations.String() != originalDeclarations.String() { + t.Error("independent source units must reproduce all original ASM/declarations in twelve bounded programs", sourceCount) + } + for index, owners := range seen { + if owners != 1 { + t.Fatal("missing or duplicated original CMOV function", index, owners) + } + } + for _, batch := range []int{-1, 6} { + if units, ok := x86CMOVFixedLocalObjectSources(kernels, batch); ok || len(units) != 0 { + t.Fatal("invalid source owner accepted", batch) + } + } + if units, ok := x86CMOVFixedLocalObjectSources(nil, 0); ok || len(units) != 0 { + t.Fatal("empty original source inventory accepted") + } + + function := fixedLocalLeaseFunctions(t, "amd64_cmov_fixed_local_test.go")["x86CMOVFixedLocalActualObjects"] + if function == nil || fixedLocalLeaseCalls(function.Body)["x86CMOVFixedLocalObjectSources"] != 1 { + t.Error("actual object consumer still captures one unpartitioned scheduling source") + return + } + unitLoops := 0 + ast.Inspect(function.Body, func(node ast.Node) bool { + loop, ok := node.(*ast.RangeStmt) + if !ok || !fixedLocalLeaseIdent(loop.X, "units") { + return true + } + unitLoops++ + calls := fixedLocalLeaseCalls(loop.Body) + if calls["x86CMOVFixedLocalFixture"] != 1 || calls["fixedLocalFreshAPIs"] != 1 { + t.Error("each source unit requires its own typed namespace and fresh three-API lifecycle") + } + ast.Inspect(loop.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + switch fixedLocalLeaseCallName(call) { + case "x86CMOVFixedLocalFixture": + if len(call.Args) != 5 { + t.Fatal("source unit changed its complete typed fixture arguments") + } + fixedLocalLeaseRequireArguments(t, &ast.CallExpr{Fun: call.Fun, Args: call.Args[:1]}, "t") + fixedLocalLeaseRequireArguments(t, &ast.CallExpr{Fun: call.Fun, Args: call.Args[3:]}, "target", "triple") + for index, field := range []string{"assembly", "declarations"} { + selected, ok := call.Args[index+1].(*ast.SelectorExpr) + if !ok || !fixedLocalLeaseIdent(selected.X, "unit") || selected.Sel.Name != field { + t.Error("source unit substituted the complete original payload", field) + } + } + case "fixedLocalFreshAPIs": + if len(call.Args) != 6 { + t.Fatal("source unit lost its complete API consumer") + } + fixedLocalLeaseRequireArguments(t, &ast.CallExpr{Fun: call.Fun, Args: call.Args[:4]}, "t", "pkg", "request", "options") + apis, ok := call.Args[4].(*ast.CompositeLit) + if !ok || len(apis.Elts) != 3 { + t.Fatal("source unit lost an independent public API") + } + for index, want := range []string{"Translate", "Module", "GoModule"} { + literal, ok := apis.Elts[index].(*ast.BasicLit) + if !ok || literal.Value != fmt.Sprintf("%q", want) { + t.Error("source unit changed its complete three-API plan", want) + } + } + } + return true + }) + return true + }) + if unitLoops != 1 { + t.Error("source units must drive exactly one independent consumer loop", unitLoops) + } +} diff --git a/amd64_cmov_fixed_local_test.go b/amd64_cmov_fixed_local_test.go new file mode 100644 index 000000000..ab9ef065b --- /dev/null +++ b/amd64_cmov_fixed_local_test.go @@ -0,0 +1,389 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +var x86CMOVAUTOSources = []string{"tmp(SP)", "UNKNOWN(SP)", "tmp+0(SP)", "tmp+(0*4)(SP)", "tmp-8(SP)", "tmp-(2*4)(SP)"} + +func TestX86CMOVFamilyAUTOActualGoPhysicalRoles(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, width := range []string{"W", "L", "Q"} { + if arch == "386" && width == "Q" { + continue + } + for _, source := range x86CMOVAUTOSources { + t.Run(arch+"/"+width+"/"+source, func(t *testing.T) { + code := x86OrderingHintGoBytes(t, arch, "TEXT original(SB),4,$16-0\nCMOV"+width+"LE "+source+",AX\nRET\n") + mode := 64 + if arch == "386" { + mode = 32 + } + found := 0 + for len(code) > 0 { + ins, err := x86asm.Decode(code, mode) + if err != nil || ins.Len == 0 { + t.Fatal("independent actual Go object decode", err) + } + if ins.Op == x86asm.CMOVLE { + memory, ok := ins.Args[1].(x86asm.Mem) + want := int64(16) + if strings.Contains(source, "-") { + want = 8 + } + if !ok || memory.Disp != want || memory.Index != 0 || ins.MemBytes != map[string]int{"W": 2, "L": 4, "Q": 8}[width] { + t.Fatal("NAME_AUTO is frame+offset, not virtual/native zero", source, ins) + } + found++ + } + code = code[ins.Len:] + } + if found != 1 { + t.Fatal("actual Go did not emit one CMOV", found) + } + }) + } + } + } +} + +func TestX86CMOVFamilyAUTOGrammarThreeAPIFiveTargets(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for _, source := range x86CMOVAUTOSources { + var assembly strings.Builder + for _, name := range x86CMOVOracleNames(target.arch) { + fmt.Fprintf(&assembly, "%s %s,AX\n", name, source) + file, err := Parse(ArchAMD64, "TEXT Value(SB),4,$16-0\n"+name+" "+source+",AX\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + t.Run(target.triple+"/"+name+"/"+source, func(t *testing.T) { + if _, known, err := x86ConditionalMoveProperties(ins.Op, ins, target.arch); !known || err != nil { + t.Fatal("accepted NAME_AUTO descriptive offset is not an undefined macro", err) + } + }) + } + original := "TEXT ·Value(SB),4,$16-0\n" + assembly.String() + "RET\n" + goAssemblerOracleOutput(t, target.arch, original, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+source+"/"+api, func(t *testing.T) { + _, err := x86OrderingHintTranslateAPI(t, api, original, "package ordering\nfunc Value()\n", target.arch, target.triple) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("legal grammar is not original native storage authority", err) + } + }) + } + } + } +} + +func x86CMOVFixedLocalAssembly(name, op, source string) string { + shift, restore := "", "" + if source != "AX" && source != "8(SP)" && source != "tmp-8(SP)" && source != "tmp-(2*4)(SP)" { + shift, restore = "MOVQ SP,R11\nADDQ $-16,SP\n", "MOVQ R11,SP\n" + } + return fmt.Sprintf("TEXT ·%s(SB),4,$16-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ AX,0(SP)\nMOVQ AX,8(SP)\n%sCMPQ AX,BX\n%s %s,BX\n%sMOVQ BX,ret+16(FP)\nRET\n", name, shift, op, source, restore) +} + +func TestX86CMOVFamilyFixedLocalSourceRoleQueryNotPermission(t *testing.T) { + for _, op := range x86CMOVOracleNames("amd64") { + for _, source := range []string{"tmp(SP)", "UNKNOWN(SP)", "tmp-8(SP)", "tmp-(2*4)(SP)", "8(SP)", "(BP)", "AX", "a+0(FP)", "slot+UNKNOWN(SP)"} { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$16-24\n"+op+" "+source+",BX\nRET\n") + if err != nil { + t.Fatal(err) + } + want := source != "AX" && source != "a+0(FP)" && source != "slot+UNKNOWN(SP)" + if got := GoAMD64FixedLocalFrameBindingNeeded(file, "amd64"); got != want { + t.Errorf("typed source binding query %s %s got=%t want=%t", op, source, got, want) + } + } + } +} + +func x86CMOVFixedLocalFixture(t *testing.T, assembly, declarations, target, triple string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options) { + t.Helper() + pkg, request := goAMD64FrameTypedFixture(t, declarations, assembly) + request.Target = target + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, name := range pkg.Types.Scope().Names() { + sig, err := DeriveGoFuncABI0Signature(pkg.Path+"."+name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[sig.Name] = sig + } + return pkg, request, file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) }} +} + +type x86CMOVFixedLocalKernel struct { + name, assembly string + width, condition, mode int +} + +func x86CMOVFixedLocalKernels(includeMoves bool) []x86CMOVFixedLocalKernel { + var kernels []x86CMOVFixedLocalKernel + for _, width := range []string{"W", "L", "Q"} { + for condition, row := range x86CMOVOracleConditions { + for _, source := range append([]string{"AX", "8(SP)"}, x86CMOVAUTOSources...) { + name := fmt.Sprintf("F%d", len(kernels)) + kernels = append(kernels, x86CMOVFixedLocalKernel{ + name: name, + assembly: x86CMOVFixedLocalAssembly(name, "CMOV"+width+row.name, source), + width: map[string]int{"W": 2, "L": 4, "Q": 8}[width], + condition: condition, + }) + } + } + } + if includeMoves { + for mode, operation := range []string{ + "MOVQ tmp-8(SP),BX", + "MOVQ AX,tmp-8(SP)\nMOVQ tmp-8(SP),BX", + "MOVOU tmp-16(SP),X0\nMOVQ X0,BX", + } { + name := fmt.Sprintf("F%d", len(kernels)) + kernels = append(kernels, x86CMOVFixedLocalKernel{ + name: name, width: 8, mode: mode + 1, + assembly: fmt.Sprintf("TEXT ·%s(SB),4,$16-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ AX,0(SP)\nMOVQ BX,8(SP)\n%s\nMOVQ BX,ret+16(FP)\nRET\n", name, operation), + }) + } + } + return kernels +} + +func TestX86CMOVFixedLocalCompleteBatchPlan(t *testing.T) { + for _, includeMoves := range []bool{false, true} { + kernels := x86CMOVFixedLocalKernels(includeMoves) + count, batches := 384, 6 + if includeMoves { + count, batches = 387, 7 + } + if len(kernels) != count { + t.Fatal("complete original CMOV/GP/SIMD kernel inventory changed", includeMoves, len(kernels)) + } + seen := make([]int, count) + for batch := 0; batch < batches; batch++ { + start, end, ok := fixedLocalObjectBatchRange(count, batch) + if !ok || end-start > 64 || end <= start { + t.Fatal("invalid bounded CMOV batch", batch, start, end, ok) + } + for index := start; index < end; index++ { + seen[index]++ + kernel := kernels[index] + if kernel.name != fmt.Sprintf("F%d", index) || + !strings.HasPrefix(kernel.assembly, "TEXT ·"+kernel.name+"(SB),4,$16-24\n") { + t.Fatal("batch changed original source symbol or ABI0 frame", index, kernel) + } + if index < 384 { + width := []int{2, 4, 8}[index/128] + if kernel.width != width || kernel.condition != index/8%16 || kernel.mode != 0 { + t.Fatal("batch changed independent width/condition/source ordering", index, kernel) + } + } else if kernel.width != 8 || kernel.condition != 0 || kernel.mode != index-383 { + t.Fatal("batch lost original private GP/SIMD AUTO kernel", index, kernel) + } + } + } + for index, owners := range seen { + if owners != 1 { + t.Fatal("missing or duplicated original kernel", index, owners) + } + } + for _, batch := range []int{-1, batches} { + if _, _, ok := fixedLocalObjectBatchRange(count, batch); ok { + t.Fatal("out-of-range CMOV batch accepted", count, batch) + } + } + } +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsLinuxBatch00(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 0) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsLinuxBatch01(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 1) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsLinuxBatch02(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 2) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsLinuxBatch03(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 3) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsLinuxBatch04(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 4) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsLinuxBatch05(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 5) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsDarwinBatch00(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 0) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsDarwinBatch01(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 1) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsDarwinBatch02(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 2) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsDarwinBatch03(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 3) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsDarwinBatch04(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 4) +} + +func TestX86CMOVFamilyFixedLocalAUTOActualObjectsDarwinBatch05(t *testing.T) { + x86CMOVFixedLocalActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 5) +} + +func x86CMOVFixedLocalActualObjects(t *testing.T, target, triple string, batch int) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + kernels := x86CMOVFixedLocalKernels(false) + units, ok := x86CMOVFixedLocalObjectSources(kernels, batch) + if !ok { + t.Fatal("invalid complete CMOV object batch", batch, len(kernels)) + } + count := 0 + for index, unit := range units { + count += unit.end - unit.start + t.Run(fmt.Sprintf("source%d", index), func(t *testing.T) { + pkg, request, _, options := x86CMOVFixedLocalFixture(t, unit.assembly, unit.declarations, target, triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "fixed_local_native") { + t.Fatal("exact sealed original local storage", err) + } + for _, optimize := range []bool{false, true} { + input := ir + if optimize { + input = x86DwordProductIROptimize(t, ir) + } + x86CompactExpandRequireAllDefinitions(t, input, unit.names) + compileLLVMToObject(t, llc, triple, "cmov-fixed-local.ll", "cmov-fixed-local.o", input) + } + }) + }) + } + t.Logf("%s batch%d: %d/%d original condition/width/register/physical/AUTO kernels; %d independent sources, three APIs, original/optimized IR, llc-O0", + target, batch, count, len(kernels), len(units)) +} + +func TestX86CMOVFamilyFixedLocalEveryReadAndAuthorityNegatives(t *testing.T) { + for _, op := range x86CMOVOracleNames("amd64") { + base := x86CMOVFixedLocalAssembly("Value", op, "tmp(SP)") + pkg := mustGoPackage(t, "cmovlocal", "package cmovlocal\nfunc Value(a,b uint64) uint64\n") + sig := x86WordMultiplySignature(t, pkg, "Value") + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[4]] + mutations := map[string]string{ + "unknown flags": strings.Replace(base, "CMPQ AX,BX", "MOVQ AX,DX", 1), + "old target pointer": strings.Replace(base, "CMPQ AX,BX", "CMPQ AX,BX\nMOVQ SP,BX", 1), + "g target": strings.Replace(base, "tmp(SP),BX", "tmp(SP),R14", 1), + "BP target": strings.Replace(base, "tmp(SP),BX", "tmp(SP),BP", 1), + "AUTO saved BP": strings.Replace(base, "ADDQ $-16,SP", "ADDQ $0,SP", 1), + "AUTO return": strings.Replace(base, "tmp(SP)", "tmp+24(SP)", 1), + "undefined expression": strings.Replace(base, "tmp(SP)", "slot+UNKNOWN(SP)", 1), + "AUTO address input": strings.Replace(base, op+" tmp(SP),BX", "MOVQ $tmp(SP),AX\n"+op+" AX,BX", 1), + "FP pointer permission": strings.Replace(base, "tmp(SP)", "p+0(FP)", 1), + "source private pointer": strings.Replace(base, "MOVQ AX,0(SP)", "MOVQ SP,0(SP)", 1), + } + for missing := 0; missing < width; missing++ { + var stores, old strings.Builder + for byteIndex := 0; byteIndex < width; byteIndex++ { + if byteIndex != missing { + fmt.Fprintf(&stores, "MOVB $1,%d(SP)\n", byteIndex) + } + } + // Old destination reads are potential even when every flag seed + // makes the predicate true. Byte-state is checked independently. + old.WriteString("MOVQ b+8(FP),DX\n") + mutations[fmt.Sprintf("source byte %d", missing)] = strings.Replace(base, "MOVQ AX,0(SP)\n", stores.String(), 1) + changed, err := Parse(ArchAMD64, strings.Replace(base, "MOVQ b+8(FP),BX\n", old.String(), 1)) + if err != nil { + t.Fatal(err) + } + t.Run(op+fmt.Sprintf("/old byte %d", missing), func(t *testing.T) { + state := x86FixedLocalState{ + pointers: map[Reg]int64{SP: -16}, + known: map[Reg]uint8{AX: 255, BX: 255 &^ (1 << uint(missing))}, + bytes: newFixedLocalByteSet(16), flags: x86ByteFlagsAll, + } + for index := int64(0); index < 16; index++ { + state.bytes.set(index, true) + } + for _, ins := range changed.Funcs[0].Instrs { + if ins.Op == Op(op) { + if err := fixedLocalStep(&ins, &state, sig, &x86FixedLocalProof{frame: 16}, 0); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing old byte became a defined potential input", err) + } + } + } + }) + } + for name, source := range mutations { + t.Run(op+"/"+name, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("fixed local memory/flags/pointer/frame authority grew", err) + } + }) + } + } +} + +func TestX86CMOVFamilyFixedLocalSharedAUTOExistingFalseCertificate(t *testing.T) { + // This is never executed: Go's named AUTO displacement is 160+16, + // while the old private proof mistakenly certified virtual offset 16. + pkg, _, file, options := fixedLocalFrameFixture(t, "PADDQ tmp+16(SP),X0") + goAssemblerOracleOutput(t, "amd64", fixedLocalVoidAssembly(file), "", true) + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("named AUTO native out-of-frame bytes acquired an offset-zero virtual certificate", err) + } +} + +func TestX86CMOVFamilyFixedLocalSharedAUTOGPAndSIMD(t *testing.T) { + for _, operation := range []string{"MOVQ tmp-8(SP),BX", "MOVQ AX,tmp-8(SP)\nMOVQ tmp-8(SP),BX", "MOVOU tmp-16(SP),X0\nMOVQ X0,BX"} { + t.Run(operation, func(t *testing.T) { + assembly := "TEXT ·Value(SB),4,$16-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ AX,0(SP)\nMOVQ BX,8(SP)\n" + operation + "\nMOVQ BX,ret+16(FP)\nRET\n" + pkg, request, file, options := x86CMOVFixedLocalFixture(t, assembly, "func Value(a,b uint64) uint64", "linux/amd64", "x86_64-unknown-linux-gnu") + observer := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := observer.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("the shared original AUTO role needs its actual native displacement", err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "fixed_local_native") { + t.Fatal("shared scalar/SIMD native offset consumer", err) + } + }) + }) + } +} diff --git a/amd64_cmov_forms.go b/amd64_cmov_forms.go new file mode 100644 index 000000000..91bad1976 --- /dev/null +++ b/amd64_cmov_forms.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ConditionalMoveSpec struct { + bits int + condition string +} + +// Each entry is a Go yml_rl row, not a conditional memory-access permission. +// W preserves the other GP bytes. L clears the upper half even on a false +// predicate. Both the source and the old destination are potential inputs. +var x86ConditionalMoveSpecs = map[Op]x86ConditionalMoveSpec{ + "CMOVWCC": {16, "CC"}, "CMOVWCS": {16, "CS"}, "CMOVWEQ": {16, "EQ"}, "CMOVWGE": {16, "GE"}, + "CMOVWGT": {16, "GT"}, "CMOVWHI": {16, "HI"}, "CMOVWLE": {16, "LE"}, "CMOVWLS": {16, "LS"}, + "CMOVWLT": {16, "LT"}, "CMOVWMI": {16, "MI"}, "CMOVWNE": {16, "NE"}, "CMOVWOC": {16, "OC"}, + "CMOVWOS": {16, "OS"}, "CMOVWPC": {16, "PC"}, "CMOVWPL": {16, "PL"}, "CMOVWPS": {16, "PS"}, + "CMOVLCC": {32, "CC"}, "CMOVLCS": {32, "CS"}, "CMOVLEQ": {32, "EQ"}, "CMOVLGE": {32, "GE"}, + "CMOVLGT": {32, "GT"}, "CMOVLHI": {32, "HI"}, "CMOVLLE": {32, "LE"}, "CMOVLLS": {32, "LS"}, + "CMOVLLT": {32, "LT"}, "CMOVLMI": {32, "MI"}, "CMOVLNE": {32, "NE"}, "CMOVLOC": {32, "OC"}, + "CMOVLOS": {32, "OS"}, "CMOVLPC": {32, "PC"}, "CMOVLPL": {32, "PL"}, "CMOVLPS": {32, "PS"}, + "CMOVQCC": {64, "CC"}, "CMOVQCS": {64, "CS"}, "CMOVQEQ": {64, "EQ"}, "CMOVQGE": {64, "GE"}, + "CMOVQGT": {64, "GT"}, "CMOVQHI": {64, "HI"}, "CMOVQLE": {64, "LE"}, "CMOVQLS": {64, "LS"}, + "CMOVQLT": {64, "LT"}, "CMOVQMI": {64, "MI"}, "CMOVQNE": {64, "NE"}, "CMOVQOC": {64, "OC"}, + "CMOVQOS": {64, "OS"}, "CMOVQPC": {64, "PC"}, "CMOVQPL": {64, "PL"}, "CMOVQPS": {64, "PS"}, +} + +type x86ConditionalMoveForm struct { + x86ConditionalMoveSpec + source, destination Operand +} + +func x86ConditionalMoveProperties(op Op, ins Instr, arch string) (x86ConditionalMoveForm, bool, error) { + name := strings.ToUpper(string(op)) + base, _, suffixed := strings.Cut(name, ".") + spec, known := x86ConditionalMoveSpecs[Op(base)] + if !known { + return x86ConditionalMoveForm{}, false, nil + } + fail := func(reason string) (x86ConditionalMoveForm, bool, error) { + return x86ConditionalMoveForm{}, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + if arch != "amd64" && arch != "386" || arch == "386" && spec.bits == 64 || suffixed || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) || len(ins.Args) != 2 { + return fail("has an unbound mode, suffix, dispatch identity or operand count") + } + source, destination := ins.Args[0], ins.Args[1] + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, arch) { + return fail("expects a Yrl destination") + } + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, arch) { + return fail("register source is outside the Go Yml class") + } + } else if source.Kind == OpMem { + if !x86ConditionalMoveOffsetKnown(source.Mem) || source.Mem.IndexExt != "" || + source.Mem.Segment != "" && source.Mem.Segment != FS && source.Mem.Segment != GS { + return fail("has an unresolved or unencodable memory source") + } + if err := x86SymbolIndexedMemoryError(source.Mem, arch); err != nil { + return x86ConditionalMoveForm{}, true, err + } + validation := source + if arch == "386" && strings.HasSuffix(source.Mem.Sym, "(SB)") { + validation.Mem.Base = "" // Go ignores Reg, not Index, for named memory. + } + if !x86PackedShiftMemoryValid(validation, arch) { + return fail("has an unencodable source address") + } + } else if !x86PackedShiftMemoryValid(source, arch) { + return fail("expects a Yml register or memory source") + } + if source.Kind == OpSym { + if address, err := parseInt(strings.TrimSpace(source.Sym)); err == nil && + address != int64(int32(address)) && !(arch == "386" && address == int64(uint32(address))) { + return fail("absolute source address is outside Go's physical disp32 extent") + } + } + return x86ConditionalMoveForm{spec, source, destination}, true, nil +} + +func x86TypedConditionalMoveByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86ConditionalMoveProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.bits / 8) + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: form.source, width: width, read: true}, + {index: 1, operand: form.destination, width: width, read: true, write: true}, + }, flagsRead: x86ByteConditionFlags(form.condition), requiresResultProof: true}, true, nil +} + +func x86CallerConditionalMoveStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (bool, bool) { + form, known, err := x86ConditionalMoveProperties(ins.Op, *ins, "amd64") + if !known || err != nil || !x86SourceStackNumericFlagsRead(state, x86ByteConditionFlags(form.condition)) { + return known, false + } + width := int64(form.bits / 8) + if !x86CallerNumericRead(form.destination, width, state) { + return true, false // Neither possible old pointer bits nor missing bytes are washed away. + } + if form.source.Kind == OpMem { + if !x86CallerOrdinaryMemoryStep(ins, 0, state, proof, width, false) { + return true, false + } + } else if !x86CallerNumericRead(form.source, width, state) { + return true, false // FP pointer slots are imports only, never CMOV inputs. + } + return true, x86CallerNumericWrite(form.destination.Reg, width, state) +} diff --git a/amd64_cmov_forms_test.go b/amd64_cmov_forms_test.go new file mode 100644 index 000000000..1f8b0ace1 --- /dev/null +++ b/amd64_cmov_forms_test.go @@ -0,0 +1,447 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +// Independent Intel condition/opcode/flag inventory, not the production map. +var x86CMOVOracleConditions = []struct { + name string + opcode byte + flags uint8 +}{ + {"CC", 0x43, x86ByteFlagCF}, {"CS", 0x42, x86ByteFlagCF}, + {"EQ", 0x44, x86ByteFlagZF}, {"GE", 0x4d, x86ByteFlagSF | x86ByteFlagOF}, + {"GT", 0x4f, x86ByteFlagZF | x86ByteFlagSF | x86ByteFlagOF}, {"HI", 0x47, x86ByteFlagCF | x86ByteFlagZF}, + {"LE", 0x4e, x86ByteFlagZF | x86ByteFlagSF | x86ByteFlagOF}, {"LS", 0x46, x86ByteFlagCF | x86ByteFlagZF}, + {"LT", 0x4c, x86ByteFlagSF | x86ByteFlagOF}, {"MI", 0x48, x86ByteFlagSF}, + {"NE", 0x45, x86ByteFlagZF}, {"OC", 0x41, x86ByteFlagOF}, + {"OS", 0x40, x86ByteFlagOF}, {"PC", 0x4b, x86ByteFlagPF}, + {"PL", 0x49, x86ByteFlagSF}, {"PS", 0x4a, x86ByteFlagPF}, +} + +func TestX86CMOVFamilyActualGoCompletePhysicalBytesAndCaller(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, width := range []string{"W", "L", "Q"} { + if arch == "386" && width == "Q" { + continue + } + for _, condition := range x86CMOVOracleConditions { + name := "CMOV" + width + condition.name + for _, row := range []struct { + source, destination string + modrm, rex byte + }{ + {"BX", "AX", 0xc3, 0}, {"AX", "AX", 0xc0, 0}, {"(DI)", "AX", 0x07, 0}, {"R8", "R9", 0xc8, 0x45}, + } { + if arch == "386" && row.rex != 0 { + continue + } + t.Run(arch+"/"+name+"/"+row.source, func(t *testing.T) { + instruction := name + " " + row.source + "," + row.destination + actual := x86OrderingHintGoBytes(t, arch, "TEXT original(SB),4,$0-0\n"+instruction+"\nRET\n") + var want []byte + if width == "W" { + want = append(want, 0x66) + } + rex := row.rex + if width == "Q" { + if rex == 0 { + rex = 0x48 + } else { + rex |= 8 + } + } + if rex != 0 { + want = append(want, rex) + } + want = append(want, 0x0f, condition.opcode, row.modrm, 0xc3) + if !bytes.Equal(actual, want) { + t.Fatalf("independent exact Go prefix/opcode/ModRM bytes %s got %x want %x", instruction, actual, want) + } + file, err := Parse(ArchAMD64, "TEXT original(SB),4,$0-0\n"+x86DwordProductDirectives(actual[:len(actual)-1])+"\nRET\n") + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal(err) + } + if decoded.Instrs[1].Op != Op(name) { + t.Fatal("physical condition alias/width normalization", decoded.Instrs[1].Op, name) + } + if arch != "amd64" || row.rex != 0 { + return + } + prefix := "MOVQ $9,BX\nCMPQ AX,BX\n" + body := x86DwordProductDirectives(actual[:len(actual)-1]) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", prefix+body+"\nMOVL AX,(DX)", 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("actual Go physical source/caller effect", err) + } + }) + } + } + } + } +} + +func x86CMOVOracleNames(arch string) []string { + var names []string + for _, width := range []string{"W", "L", "Q"} { + if arch == "386" && width == "Q" { + continue + } + for _, condition := range x86CMOVOracleConditions { + names = append(names, "CMOV"+width+condition.name) + } + } + return names +} + +func TestX86CMOVFamilyCompleteIndependentEncoderRows(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + table := string(data) + if !regexp.MustCompile(`(?s)var yml_rl = \[\]ytab\{\s*\{Zm_r, 1, argList\{Yml, Yrl\}\},\s*\}`).MatchString(table) { + t.Fatal("complete Go conditional-move zcase/zoffset/classes changed") + } + rows := regexp.MustCompile(`\{A(CMOV[A-Z]+), ([^,]+), ([^,]+), opBytes\{([^}]+)\}\}`).FindAllStringSubmatch(table, -1) + if len(rows) != 48 || len(x86ConditionalMoveSpecs) != 48 { + t.Fatal("complete family inventory changed", len(rows), len(x86ConditionalMoveSpecs)) + } + seen := map[string]bool{} + for _, width := range []string{"W", "L", "Q"} { + for _, condition := range x86CMOVOracleConditions { + name, prefix := "CMOV"+width+condition.name, map[string]string{"W": "Pq", "L": "Pm", "Q": "Pw"}[width] + bytes := fmt.Sprintf("0x%02x", condition.opcode) + if width == "Q" { + bytes = "0x0f, " + bytes + } + want := fmt.Sprintf("{A%s, yml_rl, %s, opBytes{%s}}", name, prefix, bytes) + if !strings.Contains(table, want) { + t.Fatal("independent complete encoder attributes changed", want) + } + if _, found := x86ConditionalMoveSpecs[Op(name)]; !found { + t.Fatal("missing actual Go member", name) + } + seen[name] = true + } + } + for _, row := range rows { + if !seen[row[1]] { + t.Fatal("unaccounted actual Go family row", row) + } + } + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(aliases), `instructions["CMOV`) { + t.Fatal("actual Go source alias inventory changed") + } +} + +func TestX86CMOVFamilyIndependentAbsoluteAddressExtent(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86CMOVOracleNames(arch) { + for _, test := range []struct { + address string + valid bool + }{ + {"0", true}, {"-1", true}, {"2147483647", true}, {"-2147483648", true}, + {"2147483648", arch == "386"}, {"4294967295", arch == "386"}, {"4294967296", false}, {"-2147483649", false}, + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+name+" "+test.address+",AX\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + _, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch) + if !known || (err == nil) != test.valid { + t.Errorf("%s %s absolute address %s valid=%t err=%v", arch, name, test.address, test.valid, err) + } + } + } + for _, address := range []string{"0", "-1", "2147483647", "-2147483648", "2147483648", "4294967295", "4294967296", "-2147483649"} { + valid := address != "4294967296" && address != "-2147483649" && (arch == "386" || address != "2147483648" && address != "4294967295") + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\nCMOVLLE "+address+",AX\nRET\n", "", valid) + } + } +} + +func TestX86CMOVFamilyWholeFormatsEffectsAndShapes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + registers := strings.Fields("AX BX CX DX SP BP SI DI") + if arch == "amd64" { + registers = append(registers, strings.Fields("R8 R9 R10 R11 R12 R13 R14 R15")...) + } + sources := append(append([]string(nil), registers...), "(AX)", "-1(AX)(CX*2)", "127(AX)(DX*8)", "value+0(FP)", "local-8(SP)", "·data(SB)", "(FS)", "(GS)") + var assembly strings.Builder + count := 0 + for _, name := range x86CMOVOracleNames(arch) { + for _, source := range sources { + row := name + " " + source + ",AX" + fmt.Fprintf(&assembly, "TEXT f%d(SB),4,$16-8\n%s\nRET\n", count, row) + count++ + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$16-8\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch) + if !known || err != nil { + t.Fatal(arch, row, err) + } + width := int64(form.bits / 8) + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + flags := uint8(0) + for _, condition := range x86CMOVOracleConditions { + if strings.HasSuffix(name, condition.name) { + flags = condition.flags + } + } + if !known || err != nil || effect.flagsRead != flags || effect.flagsDefined != 0 || effect.flagsKilled != 0 || effect.dynamicMemory || + len(effect.operands) != 2 || effect.operands[0].index != 0 || effect.operands[1].index != 1 || + !effect.operands[0].read || effect.operands[0].write || !effect.operands[1].read || !effect.operands[1].write || + effect.operands[0].width != width || effect.operands[1].width != width { + t.Fatal(row, effect, err) + } + if !x86SourceStackInstructionKnown(ins, "MOVQ", "Q", arch) { + t.Fatal("source-stack omitted typed member", row) + } + for index, arg := range ins.Args { + bytes, write, valid := x86InstructionMemoryShape(ins, index, arch) + memory := index == 0 && isAMD64MemoryOperand(arg) + if valid != memory || valid && (bytes != width || write || x86ScalarMoveMemoryBytes(ins, index, arch) != width) { + t.Fatal("source extent/direction", row, bytes, write, valid) + } + if arg.Kind == OpFP { + if bytes, write, valid := X86FPAccessBytes(ins, index, arch); !valid || write || bytes != width { + t.Fatal("FP extent disagreement", row) + } + } + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("source identity mutated", row) + } + } + // Destination Yrl is independent of the source class, including self aliases. + for _, destination := range registers { + fmt.Fprintf(&assembly, "TEXT f%d(SB),4,$0-0\n%s %s,%s\nRET\n", count, name, destination, destination) + count++ + } + } + goAssemblerOracleOutput(t, arch, assembly.String(), "", true) + t.Logf("%s independent actual-Go forms %d", arch, count) + } +} + +func TestX86CMOVFamilyDispatchAndGoRejectedForms(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86CMOVOracleNames("amd64") { + if arch == "386" && name[4] == 'Q' { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+name+" AX,BX\nRET\n", "", false) + continue + } + operands := []string{"$1,AX", "AX,(BX)", "AL,AX", "AH,AX", "AX,AL", "X1,AX", "K1,AX", "M0,AX", "(AX)(SP*2),BX"} + if arch == "386" { + operands = append(operands, "R8,AX", "AX,R8") + } + for _, operands := range operands { + source := "TEXT f(SB),4,$0-0\n" + name + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if _, known, err := x86ConditionalMoveProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], arch); !known || err == nil { + t.Fatal("Go rejection obtained an effect", arch, source, err) + } + } + } + } + for _, identity := range []Op{"", "CMOVLLE", "cmovlle", "CMOVLGE", "CMOVLLE.Z"} { + ins := Instr{Op: identity, Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}} + before := append([]Operand(nil), ins.Args...) + _, known, err := x86ConditionalMoveProperties("CMOVLLE", ins, "amd64") + valid := identity == "" || strings.EqualFold(string(identity), "CMOVLLE") + if !known || (err == nil) != valid || !reflect.DeepEqual(ins.Args, before) { + t.Fatal("dispatch identity", identity, err) + } + if !valid { + var out strings.Builder + ctx := amd64Ctx{b: &out, goarch: "amd64"} + if _, _, err := ctx.lowerConditionalMove("CMOVLLE", ins); err == nil || out.Len() != 0 { + t.Fatal("contradictory identity emitted IR") + } + } + } +} + +func TestX86CMOVFamilyThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, target := range x86OrderingHintTargets { + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/raw=%t", target.triple, raw), func(t *testing.T) { + var body strings.Builder + for _, name := range x86CMOVOracleNames(target.arch) { + for _, source := range []string{"AX", "BX", "(DI)", "-1(DI)(SI*2)"} { + fmt.Fprintf(&body, "%s %s,BX\n", name, source) + } + } + if raw { + bytes := x86OrderingHintGoBytes(t, target.arch, "TEXT original(SB),4,$0-0\n"+body.String()+"RET\n") + body.Reset() + if bytes[len(bytes)-1] != 0xc3 { + t.Fatal("actual Go body lacks its exact trailing RET") + } + for _, value := range bytes[:len(bytes)-1] { + fmt.Fprintf(&body, "BYTE $%d\n", value) + } + } else { + for _, name := range x86CMOVOracleNames(target.arch) { + fmt.Fprintf(&body, "%s value+0(FP),BX\n%s ·data(SB),BX\n%s (FS),BX\n%s (GS),BX\n", name, name, name, name) + } + } + source := "TEXT ·Forms(SB),4,$0-8\n" + body.String() + "RET\nGLOBL ·data(SB),8,$8\nDATA ·data+0(SB)/8,$0x1020304050607080\n" + goAssemblerOracleOutput(t, target.arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, "package ordering\nfunc Forms(value uint64)\n", target.arch, target.triple) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original formats invented predicate, data or native input bytes", api, err) + } + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + file, err = normalizeX86RawFile(file, target.arch) + if err != nil { + t.Fatal(err) + } + signatures := map[string]FuncSig{"Forms": {Name: "Forms", Args: []LLVMType{I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}}}} + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.arch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerConditionalMove(ins.Op, ins) }) + for _, input := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, input, level) + } + } + }) + } + } +} + +func TestX86CMOVFamilyBoundFrameNumericalExtent(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for _, width := range []string{"W", "L", "Q"} { + if target.arch == "386" && width == "Q" { + continue + } + for _, test := range []struct { + declaration, field string + valid bool + }{ + {"func Value(x uint64)", "x+0(FP)", true}, + {"func Value(x uint64)", "x+1(FP)", width != "Q"}, + {"func Value(x uint8,y uint8)", "x+0(FP)", width == "W"}, + {"func Value(x uint8,y uint32)", "x+0(FP)", false}, + {"func Value(x float64)", "x+0(FP)", true}, + {"func Value(x *uint64)", "x+0(FP)", false}, + {"func Value()", "ret+0(FP)", false}, + } { + for _, api := range []string{"text", "module", "Go"} { + size := 8 + if test.declaration == "func Value(x uint8,y uint8)" { + size = 2 + } + if test.declaration == "func Value(x *uint64)" && target.arch == "386" { + size = 4 + } + if test.declaration == "func Value()" { + size = 0 + } + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\nCMOV%sLE %s,AX\nRET\n", size, width, test.field) + if test.valid { + ir, err := x86OrderingHintTranslateAPI(t, api, source, "package ordering\n"+test.declaration+"\n", target.arch, target.triple) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original FP format invented old destination or predicate bytes", api, err) + } + } + // Only the source storage varies: give every row independent + // old-destination data and FLAGS before testing its exact span. + source = strings.Replace(source, fmt.Sprintf("$0-%d\n", size), + fmt.Sprintf("$0-%d\nMOVL $7,AX\nCMPL AX,$0\n", size), 1) + _, err := x86OrderingHintTranslateAPI(t, api, source, "package ordering\n"+test.declaration+"\n", target.arch, target.triple) + if test.valid && err != nil || !test.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("numerical FP bit extent/hole/type permission", target.triple, width, test.declaration, api, err) + } + } + } + } + } +} + +func TestX86CMOVFamilyNativeStackAndFPAuthorityUnchanged(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for _, operand := range []string{"(SP)", "(BP)", "(SI)(BP*1)", "value+7(FP)", "missing+8(FP)", "p+0(FP)"} { + t.Run(target.triple+"/"+operand, func(t *testing.T) { + declaration := "package ordering\nfunc Value(value uint64)\n" + size, width, setup := 8, "Q", "" + if target.arch == "386" { + width = "L" + } + if operand == "p+0(FP)" { + declaration = "package ordering\nfunc Value(p *uint64)\n" + if target.arch == "386" { + size = 4 + } + } + ordinary := target.arch == "386" && strings.Contains(operand, "BP") + if ordinary { + size, declaration = 4, "package ordering\nfunc Value(p *byte)\n" + setup = "MOVL p+0(FP),BP\n" + if strings.Contains(operand, "BP*") { + setup = "MOVL p+0(FP),SI\nMOVL $0,BP\n" + } + } + // Every row has an independent defined numerical old destination + // and FLAGS, so a Context result must still be about source storage. + setup += fmt.Sprintf("MOV%s $7,AX\nCMP%s AX,$0\n", width, width) + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\n%sCMOV%sLE %s,AX\nRET\n", size, setup, width, operand) + goAssemblerOracleOutput(t, target.arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, declaration, target.arch, target.triple) + if ordinary { + if err != nil { + t.Fatal("ordinary386 BP native CMOV", api, err) + } + for _, input := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + x86RequireCMOVVolatileWidth(t, input, 4, 1) + } + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("new native-stack, pointer or missing-FP input authority", api, err) + } + } + }) + } + } +} diff --git a/amd64_cmov_fp_bitview_runtime_test.go b/amd64_cmov_fp_bitview_runtime_test.go new file mode 100644 index 000000000..3c1b5db64 --- /dev/null +++ b/amd64_cmov_fp_bitview_runtime_test.go @@ -0,0 +1,150 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The native oracle contains actual Go bytes and receives raw IEEE bits by +// MOVSS/MOVSD only. Its predicates and byte reference are independent of the +// typed grammar; signaling NaNs never undergo arithmetic or conversion. +func TestCrossLinuxRuntimeMatrixX86CMOVFPBitViewsWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, prototypes, originals, rows, goRows strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, typ := range []string{"float32", "float64"} { + for _, width := range []string{"W", "L", "Q"} { + if typ == "float32" && width == "Q" { + continue + } + for condition, row := range x86CMOVOracleConditions { + fn := fmt.Sprintf("F%d", count) + count++ + name := "CMOV" + width + row.name + fmt.Fprintf(&declarations, "func %s(out *[16]byte,f %s,old,seed uint64)\n", fn, typ) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-32\nMOVQ out+0(FP),DI\nMOVQ old+16(FP),BX\nPUSHQ seed+24(FP)\nPOPFQ\n%s f+8(FP),BX\nMOVQ BX,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,8(DI)\nRET\n", fn, name) + move, ctype, size := "MOVSS", "float", 32 + if typ == "float64" { + move, ctype, size = "MOVSD", "double", 64 + } + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\nPUSHQ BX\nSUBQ $8,SP\n"+move+" X0,(SP)\nMOVQ SI,BX\nPUSHQ DX\nPOPFQ\n"+name+" (SP),BX\nMOVQ BX,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,8(DI)\nADDQ $8,SP\nPOPQ BX\nRET\n") + var bytes []string + for _, value := range actual { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + original, symbol := "Original"+fn, "Original"+fn + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,%s,uint64_t,uint64_t);\n", fn, ctype) + prototypes.WriteString(x86OriginalSysVDeclaration("void", original, "uint8_t*,"+ctype+",uint64_t,uint64_t")) + if size == 32 { + fmt.Fprintf(&rows, "{%s,%s,0,0,%d,%d,32},\n", fn, original, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + fmt.Fprintf(&goRows, "{%s,nil,%d,%d,32},\n", fn, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + } else { + fmt.Fprintf(&rows, "{0,0,%s,%s,%d,%d,64},\n", fn, original, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + fmt.Fprintf(&goRows, "{nil,%s,%d,%d,64},\n", fn, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + } + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + if x86CMOVFPHasNumericConversion(ir) { + t.Fatal("CMOV floating bit views numerically converted") + } + reference := fmt.Sprintf(x86CMOVFPWholeBytesC, prototypes.String(), originals.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-fp-whole-bytes", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("cmov FP kernels %d translated/original whole-byte outcomes %d NaN/zero/flags/mxcsr/canary failures 0\n", count, count*64*8*2) + if string(output) != want { + t.Fatal("independent FP payload/predicate mismatch", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), fmt.Sprintf(x86CMOVFPOriginalGo, goRows.String())) + t.Logf("%d actual original Go ABI0 floating-source kernels ×64 flag seeds ×8 IEEE edge payloads; native host execution is not strict native Linux evidence", count) +} + +var x86CMOVFPWholeBytesC = ` +#include +#include +#include +#include +` + x86CMOVConditionC + + x86OriginalSysVPointer("void", "OriginalProgram32", "uint8_t*,float,uint64_t,uint64_t") + + x86OriginalSysVPointer("void", "OriginalProgram64", "uint8_t*,double,uint64_t,uint64_t") + ` +%s +%s +typedef void(*Program32)(uint8_t*,float,uint64_t,uint64_t); +typedef void(*Program64)(uint8_t*,double,uint64_t,uint64_t); +typedef struct {Program32 f32;OriginalProgram32 o32;Program64 f64;OriginalProgram64 o64;unsigned width,condition,bits;} Kernel; +static Kernel kernels[]={%s}; +int main(void) { + const uint32_t bits32[]={0,0x80000000,0x7fc00001,0x7f800001,0xffc12345,0x7f800000,0xffffffff,0x00000001}; + const uint64_t bits64[]={0,UINT64_C(0x8000000000000000),UINT64_C(0x7ff8000000000001),UINT64_C(0x7ff0000000000001),UINT64_C(0xfff8123456789abc),UINT64_C(0x7ff0000000000000),UINT64_MAX,1}; + unsigned count=0; + for(unsigned k=0;k>1&1)<<2)|((f>>2&1)<<4)|((f>>3&1)<<6)|((f>>4&1)<<7)|((f>>5&1)<<11) + cf,pf,zf,sf,of:=flags&1!=0,flags&4!=0,flags&64!=0,flags&128!=0,flags&2048!=0 + predicates:=[]bool{!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf} + source:=bits64[v];if kernel.bits==32 {source=uint64(bits32[v])} + old:=uint64(0x87654321fedcba98)^(uint64(v)<<40);selected:=old;if predicates[kernel.condition] {selected=source} + mask:=uint64(0xffffffffffffffff);if kernel.width<8 {mask=(uint64(1)<>16"}, + {"f float64,old uint64", "W", "f+6(FP)", "double,uint64_t", "d,old", "math.Float64frombits(q),old", "q>>48"}, + {"f float64,old uint64", "L", "f+4(FP)", "double,uint64_t", "d,old", "math.Float64frombits(q),old", "q>>32"}, + {"f,g float32,old uint64", "Q", "f+0(FP)", "float,float,uint64_t", "f,g,old", "math.Float32frombits(a),math.Float32frombits(b),old", "uint64_t(a)|(uint64_t(b)<<32)"}, + {"f,g float32,old uint64", "L", "f+2(FP)", "float,float,uint64_t", "f,g,old", "math.Float32frombits(a),math.Float32frombits(b),old", "(uint64_t(a)|(uint64_t(b)<<32))>>16"}, + {"f float32,g uint32,old uint64", "Q", "f+0(FP)", "float,uint32_t,uint64_t", "f,b,old", "math.Float32frombits(a),b,old", "uint64_t(a)|(uint64_t(b)<<32)"}, + } { + fn := fmt.Sprintf("S%d", index) + fmt.Fprintf(&declarations, "func %s(%s) uint64\n", fn, row.declaration) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-24\nMOVQ old+8(FP),AX\nCLC\nCMOV%sCC %s,AX\nMOVQ AX,ret+16(FP)\nRET\n", fn, row.width, row.source) + fmt.Fprintf(&prototypes, "extern uint64_t %s(%s);\n", fn, row.ctype) + raw := strings.ReplaceAll(row.raw, "uint64_t(", "uint64(") + craw := strings.ReplaceAll(row.raw, "uint64_t(", "(uint64_t)(") + want, goWant := "("+craw+")", "("+raw+")" + if row.width == "W" { + want, goWant = "((old&~UINT64_C(0xffff))|(("+craw+")&0xffff))", "((old&^uint64(0xffff))|(("+raw+")&0xffff))" + } else if row.width == "L" { + want, goWant = "(("+craw+")&UINT64_C(0xffffffff))", "(("+raw+")&uint64(0xffffffff))" + } + fmt.Fprintf(&checks, "if(%s(%s)!=%s) {fprintf(stderr,\"span%d mismatch\\n\");return 1;} count++;\n", fn, row.call, want, index) + fmt.Fprintf(&goChecks, "if got,want:=%s(%s),%s;got!=want {t.Fatalf(\"span%d got=%%x want=%%x\",got,want)};count++\n", fn, row.goCall, goWant, index) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + if x86CMOVFPHasNumericConversion(ir) { + t.Fatal("partial/cross-slot bits numerically converted") + } + reference := fmt.Sprintf(x86CMOVFPSpanC, prototypes.String(), checks.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-fp-spans", pipeline.ir, reference, pipeline.backend) + if string(output) != "cmov FP partial/cross-slot whole-byte outcomes 384 failures 0\n" { + t.Fatal("independent exact FP bytes", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), fmt.Sprintf(x86CMOVFPSpanGo, goChecks.String())) +} + +const x86CMOVFPSpanC = ` +#include +#include +#include +%s +int main(void) { + const uint32_t bits[]={0,0x80000000,0x7fc00001,0x7f800001,0xffc12345,0x7f800000,0xffffffff,1}; + const uint64_t wide[]={0,UINT64_C(0x8000000000000000),UINT64_C(0x7ff8000000000001),UINT64_C(0x7ff0000000000001),UINT64_C(0xfff8123456789abc),UINT64_C(0x7ff0000000000000),UINT64_MAX,1}; + unsigned count=0; + for(unsigned x=0;x<8;x++) for(unsigned y=0;y<8;y++) { + uint32_t a=bits[x],b=bits[y];uint64_t q=wide[x],old=UINT64_C(0x87654321fedcba98)^((uint64_t)y<<40); + float f,g;double d;memcpy(&f,&a,4);memcpy(&g,&b,4);memcpy(&d,&q,8); + %s + } + printf("cmov FP partial/cross-slot whole-byte outcomes %%u failures 0\n",count);return 0; +} +` + +const x86CMOVFPSpanGo = `package oracle +import("math";"testing") +func TestOriginal(t *testing.T) { + bits:=[]uint32{0,0x80000000,0x7fc00001,0x7f800001,0xffc12345,0x7f800000,0xffffffff,1} + wide:=[]uint64{0,0x8000000000000000,0x7ff8000000000001,0x7ff0000000000001,0xfff8123456789abc,0x7ff0000000000000,0xffffffffffffffff,1} + count:=0 + for x,a:=range bits {for y,b:=range bits {q:=wide[x];old:=uint64(0x87654321fedcba98)^(uint64(y)<<40) + %s + }} + if count!=384 {t.Fatal(count)};t.Logf("actual Go partial/cross-slot outcomes %%d failures 0",count) +} +` diff --git a/amd64_cmov_runtime_test.go b/amd64_cmov_runtime_test.go new file mode 100644 index 000000000..423467102 --- /dev/null +++ b/amd64_cmov_runtime_test.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Actual Go-emitted bytes are embedded in independent native wrappers; none +// of the C arithmetic or condition reference is derived from the typed forms. +func TestCrossLinuxRuntimeMatrixX86CMOVIndependentWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, originals, prototypes, rows strings.Builder + declarations.WriteString("package narroworacle\n") + count := 0 + for _, width := range []string{"W", "L", "Q"} { + for condition, row := range x86CMOVOracleConditions { + name := "CMOV" + width + row.name + for mode, source := range []string{"BX", "(SI)", "AX"} { + fn := fmt.Sprintf("C%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[16]byte,src *[8]byte,old,seed uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-32\nMOVQ out+0(FP),DI\nMOVQ src+8(FP),SI\nMOVQ old+16(FP),AX\nMOVQ (SI),BX\nPUSHQ seed+24(FP)\nPOPFQ\n%s %s,AX\nMOVQ AX,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,8(DI)\nRET\n", fn, name, source) + original := "Original" + fn + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\nPUSHQ BX\nMOVQ DX,AX\nMOVQ (SI),BX\nPUSHQ CX\nPOPFQ\n"+name+" "+source+",AX\nMOVQ AX,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,8(DI)\nPOPQ BX\nRET\n") + var bytes []string + for _, value := range actual { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,uint64_t,uint64_t);\n", fn) + prototypes.WriteString(x86OriginalSysVDeclaration("void", original, "uint8_t*,const uint8_t*,uint64_t,uint64_t")) + fmt.Fprintf(&rows, "{%s,%s,%d,%d,%d},\n", fn, original, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition, mode) + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + reference := fmt.Sprintf(x86CMOVWholeBytesC, prototypes.String(), originals.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-whole-bytes", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("cmov kernels %d translated/original whole-byte outcomes %d flags/mxcsr/input/canary failures 0\n", count, count*64*3*2) + if string(output) != want { + t.Fatal("independent full bytes, flags and native predicate mismatch", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } +} + +const x86CMOVConditionC = ` +static int predicate(unsigned c,uint64_t f) { + int cf=(f>>0)&1,pf=(f>>2)&1,zf=(f>>6)&1,sf=(f>>7)&1,of=(f>>11)&1; + switch(c) { + case 0:return !cf;case 1:return cf;case 2:return zf;case 3:return sf==of; + case 4:return !zf&&sf==of;case 5:return !cf&&!zf;case 6:return zf||sf!=of; + case 7:return cf||zf;case 8:return sf!=of;case 9:return sf;case 10:return !zf; + case 11:return !of;case 12:return of;case 13:return !pf;case 14:return !sf;case 15:return pf; + }return -1; +} +static uint64_t seed(unsigned n) { + return ((uint64_t)(n&1))|((uint64_t)((n>>1)&1)<<2)|((uint64_t)((n>>2)&1)<<4)| + ((uint64_t)((n>>3)&1)<<6)|((uint64_t)((n>>4)&1)<<7)|((uint64_t)((n>>5)&1)<<11); +} +` + +var x86CMOVWholeBytesC = ` +#include +#include +#include +#include +` + x86CMOVConditionC + + x86OriginalSysVPointer("void", "OriginalProgram", "uint8_t*,const uint8_t*,uint64_t,uint64_t") + ` +%s +%s +typedef void(*Program)(uint8_t*,const uint8_t*,uint64_t,uint64_t); +typedef struct {Program translated;OriginalProgram original;unsigned width,condition,mode;} Kernel; +static Kernel kernels[]={%s}; +struct Guard {uint8_t before[32],value[16],after[32];}; +int main(void) { + unsigned count=0; + const uint64_t edges[]={UINT64_C(0x87654321fedcba98),UINT64_C(0xffff000080000001),UINT64_C(0x0123456789abcdef)}; + for(unsigned k=0;k+0(SB)/4,$0x1f80 +GLOBL cmpxchgCSRseed<>(SB),8,$4 +TEXT ·resetCSR(SB),4,$0-0 +LDMXCSR cmpxchgCSRseed<>(SB) +RET +TEXT ·restoreCSR(SB),4,$0-4 +LDMXCSR value+0(FP) +RET +` + +func x86CMPXCHGCSRControlObject(t *testing.T) { + t.Helper() + directory := t.TempDir() + source, object := filepath.Join(directory, "control.s"), filepath.Join(directory, "control.o") + if err := os.WriteFile(source, []byte(x86CMPXCHGCSRControlAssembly), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "asm", "-S", + "-p", "ordering", "-o", object, source) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64", "GOFLAGS=", "GOENV=off", "GOWORK=off") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(source, object, output, err, ctx.Err(), true, "amd64"); failure != nil { + t.Fatalf("actual CSR control object: %v\n%s", failure, output) + } + if !strings.Contains(string(output), "0f ae 15 00 00 00 00 c3") || + !strings.Contains(string(output), "0f ae 54 24 08 c3") || + !strings.Contains(string(output), "80 1f 00 00") || strings.Count(string(output), "LDMXCSR") != 2 { + t.Fatalf("actual DATA source/load/restore bytes changed:\n%s", output) + } + bytes, err := os.ReadFile(object) + if err != nil { + t.Fatal(err) + } + t.Logf("actual source/Go CSR object SHA256 %x/%x; complete listing:\n%s", sha256.Sum256([]byte(x86CMPXCHGCSRControlAssembly)), sha256.Sum256(bytes), output) +} + +func x86CMPXCHGObserverReplace(t *testing.T, source, from, to string) string { + t.Helper() + if strings.Count(source, from) != 1 { + t.Fatalf("independent observer seam disappeared or became ambiguous: %q", from) + } + return strings.Replace(source, from, to, 1) +} + +// The exact assembly and strict mathematical drivers are not changed. These +// diagnostic drivers only emit every actual observation and separate canary/ +// CSR transport failure from the already-preserved mathematical deviation. +func x86CMPXCHGObserveGoDriver(t *testing.T, source string) string { + t.Helper() + source = x86CMPXCHGObserverReplace(t, source, "observations, failures := 0, 0", "observations, failures, transportFailures := 0, 0, 0") + source = x86CMPXCHGObserverReplace(t, source, "if failures < 24 {", "if false && failures < 24 {") + source = x86CMPXCHGObserverReplace(t, source, "if kernelFailures != 0 {", "if false && kernelFailures != 0 {") + source = x86CMPXCHGObserverReplace(t, source, " before := csr()", " originalCSR := csr()\n resetCSR()\n before := csr()") + source = x86CMPXCHGObserverReplace(t, source, " after := csr()", " after := csr()\n restoreCSR(originalCSR)\n restoredCSR := csr()") + source = x86CMPXCHGObserverReplace(t, source, " observations++", ` transportBad := out[6]>>32 != canary>>32 || csr != before || after != before || restoredCSR != originalCSR || raw != expectedRaw + for i := 7; i < len(out); i++ { transportBad = transportBad || out[i] != canary } + if transportBad { transportFailures++ } + fmt.Printf("OBS %d %d %d %d", index, observations%48/4, map[bool]int{false:0, true:1}[equal], observations%2) + for i, value := range out { + if i == 4 { value &= 0x8d5 } + fmt.Printf(" %x", value) + } + fmt.Printf(" %x %x %x %x %x\n", before, after, originalCSR, restoredCSR, raw) + observations++`) + source = x86CMPXCHGObserverReplace(t, source, + ` fmt.Printf("original CMPXCHG numerical observations %d failures %d\n", observations, failures) + if failures != 0 { os.Exit(1) }`, + ` fmt.Printf("END %d %d %d\n", observations, failures, transportFailures) + if transportFailures != 0 { os.Exit(1) }`) + return source +} + +func x86CMPXCHGObserveCDriver(t *testing.T, source string) string { + t.Helper() + source = x86CMPXCHGObserverReplace(t, source, "unsigned observations = 0, failures = 0;", "unsigned observations = 0, failures = 0, transport_failures = 0;") + source = x86CMPXCHGObserverReplace(t, source, "if (kernel_failures <= 2) {", "if (0 && kernel_failures <= 2) {") + source = x86CMPXCHGObserverReplace(t, source, "if (kernel_failures) {", "if (0 && kernel_failures) {") + source = x86CMPXCHGObserverReplace(t, source, ` __asm__ volatile("stmxcsr %0" : "=m"(before));`, + ` unsigned original_csr, controlled_csr = 0x1f80; + __asm__ volatile("stmxcsr %0" : "=m"(original_csr)); + __asm__ volatile("ldmxcsr %0" : : "m"(controlled_csr)); + __asm__ volatile("stmxcsr %0" : "=m"(before));`) + source = x86CMPXCHGObserverReplace(t, source, ` __asm__ volatile("stmxcsr %0" : "=m"(after));`, + ` __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(original_csr)); + unsigned restored_csr; + __asm__ volatile("stmxcsr %0" : "=m"(restored_csr));`) + source = x86CMPXCHGObserverReplace(t, source, " observations++;", ` int transport_bad = (out[6]>>32) != (canary>>32) || + (uint32_t)out[6] != before || after != before || restored_csr != original_csr || memcmp(raw, expected, sizeof(raw)); + for (unsigned i = 7; i < 16; i++) transport_bad |= out[i] != canary; + transport_failures += transport_bad != 0; + printf("OBS %u %u %u %u", index, edge, equal, seed); + for (unsigned i = 0; i < 16; i++) { + uint64_t value = i == 4 ? out[i]&0x8d5 : out[i]; + printf(" %llx", (unsigned long long)value); + } + printf(" %x %x %x %x ", before, after, original_csr, restored_csr); + for (unsigned i = 0; i < sizeof(raw); i++) printf("%02x", raw[i]); + printf("\n"); + observations++;`) + source = x86CMPXCHGObserverReplace(t, source, + ` printf("actual C opcode CMPXCHG numerical observations %u failures %u\n", observations, failures); + return failures ? 1 : 0;`, + ` printf("END %u %u %u\n", observations, failures, transport_failures); + return transport_failures ? 1 : 0;`) + return source +} + +func x86CMPXCHGParseActualPacket(output []byte) (x86CMPXCHGActualPacket, error) { + var packet x86CMPXCHGActualPacket + lines := strings.Split(string(output), "\n") + if len(output) > 1<<20 || len(lines) != 578 || lines[577] != "" { + return packet, fmt.Errorf("incomplete, extra or over-bound actual observation output") + } + // Windows C stdio emits CRLF. Accept that text transport without + // trimming field whitespace or discarding malformed embedded CR bytes. + for index := range lines { + lines[index] = strings.TrimSuffix(lines[index], "\r") + } + parse := func(text string, base, bits int) (uint64, error) { + value, err := strconv.ParseUint(text, base, bits) + if err != nil || strconv.FormatUint(value, base) != text { + return 0, fmt.Errorf("noncanonical actual value %q", text) + } + return value, nil + } + for ordinal, line := range lines[:576] { + fields := strings.Split(line, " ") + if len(fields) != 26 || fields[0] != "OBS" { + return packet, fmt.Errorf("actual observation %d has incomplete/unknown fields", ordinal) + } + want := []int{ordinal / 48, ordinal % 48 / 4, ordinal % 4 / 2, ordinal % 2} + for index, value := range want { + if fields[index+1] != strconv.Itoa(value) { + return packet, fmt.Errorf("duplicate, missing or reordered actual observation %d", ordinal) + } + } + row := &packet.rows[ordinal] + for index := range row.output { + value, err := parse(fields[index+5], 16, 64) + if err != nil { + return packet, err + } + row.output[index] = value + } + before, err := parse(fields[21], 16, 32) + if err != nil { + return packet, err + } + after, err := parse(fields[22], 16, 32) + if err != nil || before != 0x1f80 || before != after || uint32(row.output[6]) != uint32(before) { + return packet, fmt.Errorf("actual MXCSR transport changed at observation %d", ordinal) + } + row.csr = uint32(before) + // Ambient input differs between independent Go/C processes. Preserve + // both complete values in the raw trace and require exact restoration; + // opcode equality concerns the identical controlled MXCSR input. + originalCSR, err := parse(fields[23], 16, 32) + if err != nil { + return packet, err + } + restoredCSR, err := parse(fields[24], 16, 32) + if err != nil || originalCSR != restoredCSR { + return packet, fmt.Errorf("actual ambient MXCSR restoration failed at observation %d", ordinal) + } + if row.output[4]&^uint64(0x8d5) != 0 || row.output[5] != row.output[4]&0xd5|2 || row.output[6]>>32 != 0xa7a7a7a7 { + return packet, fmt.Errorf("actual status/LAHF or output canary was incomplete at observation %d", ordinal) + } + for index := 7; index < 16; index++ { + if row.output[index] != 0xa7a7a7a7a7a7a7a7 { + return packet, fmt.Errorf("actual output canary changed at observation %d", ordinal) + } + } + if len(fields[25]) != 192 || strings.ToLower(fields[25]) != fields[25] { + return packet, fmt.Errorf("actual complete byte observation missing at %d", ordinal) + } + bytes, err := hex.DecodeString(fields[25]) + if err != nil { + return packet, err + } + copy(row.bytes[:], bytes) + family := ordinal / 96 + start := 17 + if family == 5 { + start = 16 + } + width := []int{1, 2, 4, 8, 8, 16}[family] + for index, value := range row.bytes { + if (index < start || index >= start+width) && value != 0xa7 { + return packet, fmt.Errorf("actual input canary changed at observation %d", ordinal) + } + } + } + footer := strings.Split(lines[576], " ") + if len(footer) != 4 || footer[0] != "END" || footer[1] != "576" || footer[3] != "0" { + return packet, fmt.Errorf("actual observer has missing/unknown/nonzero transport terminal") + } + deviations, err := parse(footer[2], 10, 16) + if err != nil || deviations > 576 { + return packet, fmt.Errorf("actual observer has invalid mathematical deviation count") + } + packet.deviations = int(deviations) + return packet, nil +} + +func TestX86CMPXCHGActualObservationPacketRejectsIncompleteTransport(t *testing.T) { + // Synthetic output-protocol tripwires, not actual opcode execution evidence. + var source strings.Builder + for ordinal := 0; ordinal < 576; ordinal++ { + fmt.Fprintf(&source, "OBS %d %d %d %d", ordinal/48, ordinal%48/4, ordinal%4/2, ordinal%2) + for index := 0; index < 16; index++ { + value := uint64(0xa7a7a7a7a7a7a7a7) + switch index { + case 4: + value = 64 + case 5: + value = 66 + case 6: + value = 0xa7a7a7a700001f80 + } + fmt.Fprintf(&source, " %x", value) + } + fmt.Fprintf(&source, " 1f80 1f80 1f80 1f80 %s\n", strings.Repeat("a7", 96)) + } + source.WriteString("END 576 0 0\n") + valid := source.String() + packet, err := x86CMPXCHGParseActualPacket([]byte(valid)) + if err != nil || packet.deviations != 0 { + t.Fatalf("complete synthetic protocol control: %v", err) + } + for name, output := range map[string]string{ + "CRLF": strings.ReplaceAll(valid, "\n", "\r\n"), + "mixed": strings.Replace(valid, "\n", "\r\n", 1), + } { + t.Run(name, func(t *testing.T) { + actual, err := x86CMPXCHGParseActualPacket([]byte(output)) + if err != nil || actual != packet { + t.Fatalf("text-mode line endings changed the complete observation: %v", err) + } + }) + } + for name, mutation := range map[string]string{ + "bare-CR": strings.Replace(valid, "\n", "\r", 1), + "double-CR": strings.Replace(valid, "\n", "\r\r\n", 1), + "embedded-CR": strings.Replace(valid, "OBS 0", "OBS\r 0", 1), + "missing": strings.TrimSuffix(valid, "END 576 0 0\n"), + "unknown": valid + "unknown\n", + "duplicate": strings.Replace(valid, "OBS 0 0 0 1", "OBS 0 0 0 0", 1), + "truncated": strings.Replace(valid, strings.Repeat("a7", 96), "a7", 1), + "canary": strings.Replace(valid, strings.Repeat("a7", 96), "a6"+strings.Repeat("a7", 95), 1), + "changed-CSR": strings.Replace(valid, "1f80 1f80", "1f80 1f81", 1), + "uncontrolled": strings.Replace(valid, "1f80 1f80", "1fa0 1fa0", 1), + "unrestored": strings.Replace(valid, "1f80 1f80 1f80 1f80", "1f80 1f80 1fa0 1f80", 1), + "unknown-LAHF": strings.Replace(valid, " 40 42 ", " 40 43 ", 1), + "transport-exit": strings.Replace(valid, "END 576 0 0", "END 576 0 1", 1), + "wrong-count": strings.Replace(valid, "END 576", "END 575", 1), + "overflow": strings.Replace(valid, "END 576 0 0", "END 576 65536 0", 1), + "diagnostic-only": "END 576 240 0\n", + } { + t.Run(name, func(t *testing.T) { + if _, err := x86CMPXCHGParseActualPacket([]byte(mutation)); err == nil { + t.Fatal("incomplete/unknown actual transport got equivalence credit") + } + }) + } + mutated := packet + mutated.rows[575].output[0] ^= 1 + if reflect.DeepEqual(packet, mutated) { + t.Fatal("full observation comparison did not retain a late register byte") + } +} diff --git a/amd64_cmpxchg_boundaries_test.go b/amd64_cmpxchg_boundaries_test.go new file mode 100644 index 000000000..2b6f86c64 --- /dev/null +++ b/amd64_cmpxchg_boundaries_test.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86CMPXCHGDirectHelpersCannotBypassOrderedGrammar(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + op Op + identity Op + bits int + pair bool + args []Operand + context bool + }{ + {"CMPXCHG8B", "CMPXCHG8B", 64, true, []Operand{{Kind: OpReg, Reg: BX}}, true}, + {"CMPXCHGL", "MOVL", 32, false, []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: BX}}, false}, + {"CMPXCHGL", "CMPXCHGL", 16, false, []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: BX}}, false}, + {"UNKNOWN", "UNKNOWN", 32, false, []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: BX}}, false}, + } { + t.Run(fmt.Sprintf("%s/%s/%s/width=%d", arch, row.op, row.identity, row.bits), func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + ins := Instr{Op: row.identity, Args: row.args} + var known, terminated bool + var err error + if row.pair { + known, terminated, err = ctx.lowerPairCompareExchange(string(row.op), row.bits, ins) + } else { + known, terminated, err = ctx.lowerScalarCompareExchange(string(row.op), row.bits, ins) + } + if !known || terminated || err == nil || errors.Is(err, ErrProbeNeedsContext) != row.context || + before != output.String() || temporary != ctx.tmp { + t.Fatalf("direct helper bypassed complete grammar or emitted partial IR: %t/%t/%v", known, terminated, err) + } + }) + } + } +} + +func TestX86CMPXCHGFPRequiresCompleteNativeExtentAndIdentity(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + triple := testTargetTriple("linux", arch) + for _, row := range []struct { + body, declaration string + valid bool + }{ + {"CMPXCHGB CL,value+0(FP)", "func locked(value uint8)", true}, + {"CMPXCHGW CX,value+0(FP)", "func locked(value uint8)", false}, + {"CMPXCHGL CX,value+0(FP)", "func locked(value uint16)", false}, + {"CMPXCHG8B value+0(FP)", "func locked(value uint32)", false}, + {"CMPXCHGL CX,value+2(FP)", "func locked(value uint32)", false}, + {"CMPXCHGL CX,value+8(FP)", "func locked(value uint32)", false}, + } { + t.Run(arch+"/"+row.body+"/"+row.declaration, func(t *testing.T) { + source := "TEXT ·locked(SB),4,$0-16\nMOVL $1,AX\nMOVL $2,BX\nMOVL $3,CX\nMOVL $4,DX\n" + row.body + "\nRET\n" + actual := x86OrderingHintGoBytes(t, arch, source) + t.Logf("actual Go accepts the source spelling: %x; this does not prove its typed FP extent", actual) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\n"+row.declaration+"\n", arch, triple) + if row.valid { + if err != nil || !strings.Contains(ir, "cmpxchgb") { + t.Fatalf("complete pre-existing one-byte FP extent was rejected in %s: %v", api, err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "fp-control.ll", "fp-control.o", ir) + continue + } + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("incomplete FP native extent acquired an atomic write in %s: %v", api, err) + } + } + }) + } + } + for _, api := range []string{"text", "module", "Go"} { + t.Run("amd64/16B/shadow/"+api, func(t *testing.T) { + source := "TEXT ·locked(SB),4,$0-16\nMOVL $1,AX\nMOVL $2,BX\nMOVL $3,CX\nMOVL $4,DX\nCMPXCHG16B value+0(FP)\nRET\n" + actual := x86OrderingHintGoBytes(t, "amd64", source) + t.Logf("actual original FP16 opcode: %x; no native alignment identity supplied by a logical shadow", actual) + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(value [2]uint64)\n", "amd64", "x86_64-unknown-linux-gnu") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("virtual FP shadow fabricated CMPXCHG16B native address/alignment: %v", err) + } + }) + } +} + +func TestX86CMPXCHGFPBackendRequiresExplicitNativeTarget(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + otherMode := "i386-unknown-linux-gnu" + if arch == "386" { + otherMode = "x86_64-unknown-linux-gnu" + } + // Use the complete byte extent from the positive owner. A missing + // implicit register or FP byte must not substitute for backend rejection. + source := "TEXT ·locked(SB),4,$0-16\nMOVL $1,AX\nMOVL $2,BX\nMOVL $3,CX\nMOVL $4,DX\nCMPXCHGB CL,value+0(FP)\nRET\n" + x86OrderingHintGoBytes(t, arch, source) + for _, triple := range []string{"", otherMode, "aarch64-unknown-linux-gnu", "wasm32-unknown-unknown"} { + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(value uint8)\n", arch, triple) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "inline assembly") { + t.Fatal("complete native FP CAS escaped an absent or foreign backend", ir != "", err) + } + }) + } + } + } +} diff --git a/amd64_cmpxchg_fault_observer_test.go b/amd64_cmpxchg_fault_observer_test.go new file mode 100644 index 000000000..d4eddf50b --- /dev/null +++ b/amd64_cmpxchg_fault_observer_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os/exec" + "runtime" + "strconv" + "syscall" + "testing" + "time" +) + +func x86CMPXCHGObserveFault(t *testing.T, execution x86FloatCompareExecution, executable string, index, address, original int) (string, error) { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + return x86CMPXCHGObserveFaultContext(ctx, execution, executable, index, address, original) +} + +func x86CMPXCHGObserveFaultContext(ctx context.Context, execution x86FloatCompareExecution, executable string, index, address, original int) (string, error) { + // Each leaf executable is the owned child: no fork, descendant, exception + // handler or native unwinder can hide the actual instruction's termination. + child, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + args := execution.command(executable, strconv.Itoa(index), strconv.Itoa(address), strconv.Itoa(original)) + stdout, stderr, err := runtimeOracleMachineOutput(child, exec.CommandContext(child, args[0], args[1:]...)) + if child.Err() != nil { + return "", child.Err() + } + witness := fmt.Sprintf("CMPXCHG enter %d %d %d\n", index, address, original) + result, cause := x86CMPXCHGFaultResult(runtime.GOOS, string(stdout), witness, err) + if cause != nil { + return "", fmt.Errorf("kernel %d address %d original %d: %w\nstdout: %q\nstderr: %q", index, address, original, cause, stdout, stderr) + } + return result, nil +} + +func x86CMPXCHGFaultResult(goos, stdout, witness string, err error) (string, error) { + result, cause := x86NativeFaultResult(goos, stdout, witness, "CMPXCHG complete\n", err) + if cause == nil && result == "fpe" { + return "", fmt.Errorf("floating-point fault is not a CMPXCHG outcome") + } + return result, cause +} + +func x86NativeFaultResult(goos, stdout, witness, completed string, err error) (string, error) { + if err == nil { + if stdout == witness+completed { + return "return", nil + } + return "", fmt.Errorf("missing exact completed state/canary receipt") + } + // Do not unwrap joined errors: cancellation, pipes and ownership cleanup + // failures must not become accepted native exceptions. + exit, ok := err.(*exec.ExitError) + if !ok || exit.ProcessState == nil || stdout != witness { + return "", fmt.Errorf("not an isolated native fault after the exact entry receipt: %w", err) + } + status, ok := exit.ProcessState.Sys().(syscall.WaitStatus) + if !ok { + return "", fmt.Errorf("missing actual process termination status") + } + if goos == "windows" { + return x86NativeWindowsFault(uint32(exit.ExitCode())) + } + if status.Signaled() { + switch status.Signal() { + case syscall.SIGSEGV: + return "segv", nil + case syscall.SIGBUS: + return "bus", nil + case syscall.SIGILL: + return "ill", nil + case syscall.SIGFPE: + return "fpe", nil + } + } + return "", fmt.Errorf("setup, state or unexpected process failure: %w", err) +} + +func x86NativeWindowsFault(code uint32) (string, error) { + if code == 0xc000008f { // EXCEPTION_FLT_INEXACT_RESULT, not other FP exceptions. + return "fpe", nil + } + return x86CMPXCHGWindowsFault(code) +} + +func TestX86NativeWindowsPrecisionFaultCode(t *testing.T) { + if got, err := x86NativeWindowsFault(0xc000008f); err != nil || got != "fpe" { + t.Fatalf("native precision exception: %s, %v", got, err) + } + for _, code := range []uint32{136, 0xc000008e, 0xc0000090, 0xc0000091, 0xc0000094} { + if _, err := x86NativeWindowsFault(code); err == nil { + t.Fatalf("different exception/ordinary exit became precision evidence: %#x", code) + } + } + if _, err := x86CMPXCHGWindowsFault(0xc000008f); err == nil { + t.Fatal("precision exception became a registered CMPXCHG fault") + } +} + +func x86CMPXCHGWindowsFault(code uint32) (string, error) { + // GetExitCodeProcess returns the unhandled exception value, not 128+signal. + // Keep only these three native exception classes; ordinary exits never + // count as faults, even when original and translated setup fail identically. + switch code { + case 0xc0000005: // EXCEPTION_ACCESS_VIOLATION + return "segv", nil + case 0x80000002: // EXCEPTION_DATATYPE_MISALIGNMENT + return "bus", nil + case 0xc000001d: // EXCEPTION_ILLEGAL_INSTRUCTION + return "ill", nil + } + return "", fmt.Errorf("not a registered native Windows exception: %#x", code) +} + +func TestX86CMPXCHGWindowsFaultCodes(t *testing.T) { + for code, want := range map[uint32]string{0xc0000005: "segv", 0x80000002: "bus", 0xc000001d: "ill"} { + if got, err := x86CMPXCHGWindowsFault(code); err != nil || got != want { + t.Fatalf("exception %#x: %s, %v", code, got, err) + } + } + for _, code := range []uint32{0, 1, 80, 82, 83, 84, 85, 86, 87, 88, 120, 125, 132, 135, 137, 139, 259, 0xc00000fd, 0xffffffff} { + if _, err := x86CMPXCHGWindowsFault(code); err == nil { + t.Fatalf("ordinary/setup/stack-overflow exit %#x became native opcode evidence", code) + } + } +} + +func TestX86CMPXCHGFaultObserverRejectsFalseEvidence(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + const driver = ` +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif +int main(int argc, char **argv) { +#ifdef _WIN32 + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); +#else + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) return 120; +#endif + if (argc != 4) return 80; + int mode = atoi(argv[1]); + if (mode == 2) return 82; + printf("CMPXCHG enter %d 0 0\n", mode == 7 ? 99 : mode); + if (fflush(stdout)) return 87; + if (mode == 1 || mode == 7) __asm__ volatile("movl $0,(%0)" :: "r"((uintptr_t)0) : "memory"); + if (mode == 3) return 139; + if (mode == 4) return 0; + if (mode == 5) __asm__ volatile("ud2"); + if (mode == 6) return 85; + if (mode == 8) for (;;) __asm__ volatile("" ::: "memory"); + puts("CMPXCHG complete"); + return 0; +} +` + executable := execution.compileLLVM(t, "fault-controls", "", driver, "-O0") + for _, test := range []struct { + mode int + want string + }{{0, "return"}, {1, "segv"}, {2, ""}, {3, ""}, {4, ""}, {5, "ill"}, {6, ""}, {7, ""}} { + t.Run(strconv.Itoa(test.mode), func(t *testing.T) { + got, err := x86CMPXCHGObserveFault(t, execution, executable, test.mode, 0, 0) + if got != test.want || (err == nil) != (test.want != "") { + t.Fatalf("got %s, %v; want %s", got, err, test.want) + } + }) + } + t.Run("cancellation", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + if _, err := x86CMPXCHGObserveFaultContext(ctx, execution, executable, 8, 0, 0); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("cancellation became a native opcode outcome: %v", err) + } + }) + t.Run("joined-cleanup", func(t *testing.T) { + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + args := execution.command(executable, "5", "0", "0") + stdout, _, err := runtimeOracleMachineOutput(ctx, exec.CommandContext(ctx, args[0], args[1:]...)) + const witness = "CMPXCHG enter 5 0 0\n" + if got, cause := x86CMPXCHGFaultResult(runtime.GOOS, string(stdout), witness, err); cause != nil || got != "ill" { + t.Fatal("missing actual illegal-instruction control", got, cause) + } + if _, cause := x86CMPXCHGFaultResult(runtime.GOOS, string(stdout), witness, errors.Join(err, fmt.Errorf("owned cleanup failure"))); cause == nil { + t.Fatal("cleanup error was erased by the native exception") + } + }) +} diff --git a/amd64_cmpxchg_fault_portability_test.go b/amd64_cmpxchg_fault_portability_test.go new file mode 100644 index 000000000..fc8d729b9 --- /dev/null +++ b/amd64_cmpxchg_fault_portability_test.go @@ -0,0 +1,72 @@ +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestX86CMPXCHGFaultWindowsHeaderBoundary(t *testing.T) { + x86FaultWindowsHeaderBoundary(t, x86CMPXCHGFaultReference, + []string{"VirtualAlloc", "VirtualProtect", "VirtualFree", "kernel.original(p)", "kernel.translated(p)"}) +} + +func TestX86MemoryFaultWindowsHeaderBoundary(t *testing.T) { + driver := x86MemoryFaultDriver("static int sample(const uint8_t *p) { return *p; }\n", "{sample, 4, 3},") + x86FaultWindowsHeaderBoundary(t, driver, + []string{"VirtualAlloc", "VirtualProtect", "VirtualFree", "kernel.run(input)", "source[i]"}) +} + +func TestX86SignedConversionFaultWindowsHeaderBoundary(t *testing.T) { + driver := x86MemoryFaultSupportC + ` +static void sample(const void *p) { (void)p; } +static const struct { void (*run)(const void *); unsigned width, bits; } + cases[] = {{sample, 4, 32}}; +` + x86SignedIntegerFloatFaultMainC + x86FaultWindowsHeaderBoundary(t, driver, + []string{"VirtualAlloc", "VirtualProtect", "VirtualFree", "cases[row].run(input)", "cvtsi2ssl", "cvtsi2sdq"}) +} + +func x86FaultWindowsHeaderBoundary(t *testing.T, driver string, required []string) { + t.Helper() + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("LLVM 22 clang required") + } + directory := t.TempDir() + if err := os.Mkdir(filepath.Join(directory, "sys"), 0700); err != nil { + t.Fatal(err) + } + // This observes actual preprocessing of the complete driver, not a + // Windows SDK, object, link or runtime claim. Missing POSIX dependencies + // are poisoned so the host cannot accidentally supply them. + for _, name := range []string{"stdint.h", "stdio.h", "string.h", "stdlib.h", "signal.h", "windows.h"} { + if err := os.WriteFile(filepath.Join(directory, name), nil, 0600); err != nil { + t.Fatal(err) + } + } + for _, name := range []string{"unistd.h", "sys/mman.h", "sys/resource.h", "sys/wait.h"} { + if err := os.WriteFile(filepath.Join(directory, filepath.FromSlash(name)), []byte("#error unexpected_POSIX_fault_dependency\n"), 0600); err != nil { + t.Fatal(err) + } + } + input := filepath.Join(directory, "fault.c") + if err := os.WriteFile(input, []byte(driver), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, clang, "-target", "x86_64-w64-windows-gnu", "-nostdinc", + "-I", directory, "-E", "-P", input) + output, err := runtimeOracleCombinedOutput(ctx, command) + if err != nil { + t.Fatalf("Windows fault driver still consumes POSIX headers: %v\n%s", err, output) + } + for _, operation := range required { + if !strings.Contains(string(output), operation) { + t.Fatal("Windows fault driver lost its real guard or operation", operation) + } + } +} diff --git a/amd64_cmpxchg_fault_runtime_test.go b/amd64_cmpxchg_fault_runtime_test.go new file mode 100644 index 000000000..b69af4f86 --- /dev/null +++ b/amd64_cmpxchg_fault_runtime_test.go @@ -0,0 +1,214 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +// Native opcode outcomes, not CPUID advertisements, establish availability and +// fault behavior. This is an instruction-level C observer tied to original Go +// opcode bytes, not execution of the stock Go ABI0 caller or a native CI claim. +func TestX86CMPXCHGOriginalOpcodeOutcomeEquivalence(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + x86CMPXCHGRunFaultCollector(t, execution, false) +} + +func x86CMPXCHGRunFaultCollector(t *testing.T, execution x86FloatCompareExecution, native bool) { + t.Helper() + var assembly, declarations, prototypes, originals, cases strings.Builder + declarations.WriteString("package ordering\n") + for index, row := range x86CMPXCHGOriginalRows { + for _, locked := range []bool{false, true} { + name := fmt.Sprintf("cas%d_%t", index, locked) + body := x86CMPXCHGOriginalBody(index, "(SI)") + if locked { + body = "LOCK\n\t" + body + } + // Use the source register initialized by the reference: desired low + // is in CX for scalar forms, and BX/CX for pair forms. + source := strings.Replace(x86NamedLockOracleSource("amd64", body), "·locked", "·"+name, 1) + actual := x86OrderingHintGoBytes(t, "amd64", source) + opcode := []byte{0x0f, 0xb1, 0x0e} + switch row.bits { + case 8: + if !row.pair { + opcode = []byte{0x0f, 0xb0, 0x0e} + } + case 16: + opcode = append([]byte{0x66}, opcode...) + case 64: + if row.pair { + opcode = []byte{0x0f, 0xc7, 0x0e} + } else { + opcode = append([]byte{0x48}, opcode...) + } + case 128: + opcode = []byte{0x48, 0x0f, 0xc7, 0x0e} + } + if locked { + opcode = append([]byte{0xf0}, opcode...) + } + if !bytes.HasSuffix(actual, append(append([]byte(nil), opcode...), 0xc3)) { + t.Fatalf("complete original opcode/prefix binding changed: %s %x", name, actual) + } + t.Logf("actual Go %s terminal machine bytes=%x; object identity only", name, opcode) + assembly.WriteString(source) + fmt.Fprintf(&declarations, "func %s(p *[32]byte)\n", name) + fmt.Fprintf(&prototypes, "extern void %s(unsigned char *);\n", name) + fmt.Fprintf(&cases, " {%s, original_%s, %d, %d},\n", name, name, row.bits/8, index) + prefix := "" + if locked { + prefix = "lock; " + } + if row.pair { + part, mnemonic := "uint32_t", "cmpxchg8b" + if row.bits == 128 { + part, mnemonic = "uint64_t", "cmpxchg16b" + } + fmt.Fprintf(&originals, `static void original_%s(unsigned char *p) { + %s low = 2, high = 5; + __asm__ volatile("%s%s (%%4)" + : "+a"(low), "+d"(high) + : "b"((%s)3), "c"((%s)4), "r"(p) + : "cc", "memory"); +} +`, name, part, prefix, mnemonic, part, part) + } else { + typ := fmt.Sprintf("uint%d_t", row.bits) + width := map[int]string{8: "b", 16: "w", 32: "l", 64: "q"}[row.bits] + fmt.Fprintf(&originals, `static void original_%s(unsigned char *p) { + %s accumulator = 2; + __asm__ volatile("%scmpxchg%s %%1, (%%2)" + : "+a"(accumulator) + : "c"((%s)4), "r"(p) + : "cc", "memory"); +} +`, name, typ, prefix, width, typ) + } + } + } + ir, err := x86OrderingHintTranslateAPI(t, "text", assembly.String(), declarations.String(), "amd64", execution.triple) + if err != nil { + t.Fatal(err) + } + reference := fmt.Sprintf(x86CMPXCHGFaultReference, prototypes.String(), originals.String(), cases.String()) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := execution.compileLLVM(t, "cmpxchg-fault", ir, reference, level) + observations, deviations := 0, 0 + for index := 0; index < len(x86CMPXCHGOriginalRows)*2; index++ { + for address := 0; address < 4; address++ { + original, err := x86CMPXCHGObserveFault(t, execution, executable, index, address, 1) + if err != nil { + t.Fatal(err) + } + translated, err := x86CMPXCHGObserveFault(t, execution, executable, index, address, 0) + if err != nil { + t.Fatal(err) + } + if original != translated { + t.Fatalf("kernel %d address %d: original %s, translated %s", index, address, original, translated) + } + misalignedPair := x86CMPXCHGOriginalRows[index/2].bits == 128 && address == 1 + fault := address >= 2 || misalignedPair + if fault && original != "segv" && original != "bus" || !fault && original != "return" { + deviations++ + } + if native && misalignedPair && original != "segv" { + t.Fatal("native Linux misaligned CMPXCHG16B did not report #GP as SIGSEGV") + } + observations += 2 + } + } + if observations != 96 || native && deviations != 0 { + t.Fatalf("CMPXCHG opcode outcomes %d, ISA deviations %d", observations, deviations) + } + t.Logf("CMPXCHG opcode outcomes %d; ISA deviations %d; outcome failures 0", observations, deviations) + }) + } +} + +const x86CMPXCHGFaultReference = ` +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif +` + x86TestGuardPagesC + ` + +%s +%s +typedef struct { + void (*translated)(unsigned char *); + void (*original)(unsigned char *); + unsigned width, family; +} Kernel; + +static int exercise(Kernel kernel, unsigned index, unsigned address, unsigned original) { + _Alignas(16) unsigned char raw[64]; + memset(raw, 0xa7, sizeof(raw)); + unsigned char *p = raw + 16; + if (address == 1) p++; + if (address == 2) p = NULL; + if (address == 3) { + if (setup_guard()) return 82; + memset(guard, 0xa7, page); + p = guard + page - (kernel.width - 1); + } + unsigned char expected[16] = {0}; + uint64_t low = 2, high = 5, desired_low = 3, desired_high = 4; + if (kernel.family == 4) { + uint32_t pair[2] = {2, 5}, desired[2] = {3, 4}; + memcpy(expected, desired, 8); + if (address < 2) memcpy(p, pair, 8); + } else if (kernel.family == 5) { + memcpy(expected, &desired_low, 8); + memcpy(expected + 8, &desired_high, 8); + if (address < 2) { + memcpy(p, &low, 8); + memcpy(p + 8, &high, 8); + } + } else { + memcpy(expected, &desired_high, kernel.width); + if (address < 2) memcpy(p, &low, kernel.width); + } + if (printf("CMPXCHG enter %%u %%u %%u\n", index, address, original) < 0 || fflush(stdout)) return 87; + if (original) kernel.original(p); + else kernel.translated(p); + if (address == 3 && free_guard()) return 83; + if (address >= 2) return 84; + if (memcmp(p, expected, kernel.width)) return 85; + size_t begin = (size_t)(p - raw); + for (size_t i = 0; i < sizeof(raw); i++) { + if ((i < begin || i >= begin + kernel.width) && raw[i] != 0xa7) return 86; + } + if (puts("CMPXCHG complete") < 0 || fflush(stdout)) return 88; + return 0; +} + +int main(int argc, char **argv) { +#ifdef _WIN32 + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); +#else + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) return 120; +#endif + Kernel kernels[] = { +%s + }; + if (argc != 4 || strlen(argv[1]) > 2 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1) return 80; + unsigned long values[3]; + for (unsigned i = 0; i < 3; i++) { + char *end; + if (argv[i+1][0] < '0' || argv[i+1][0] > '9') return 80; + values[i] = strtoul(argv[i+1], &end, 10); + if (*end) return 80; + } + if (values[0] >= sizeof(kernels)/sizeof(kernels[0]) || values[1] >= 4 || values[2] >= 2) return 80; + return exercise(kernels[values[0]], (unsigned)values[0], (unsigned)values[1], (unsigned)values[2]); +} +` diff --git a/amd64_cmpxchg_forms.go b/amd64_cmpxchg_forms.go new file mode 100644 index 000000000..eb1469135 --- /dev/null +++ b/amd64_cmpxchg_forms.go @@ -0,0 +1,137 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86CompareExchangeSpec struct { + bits int + pair bool +} + +// Complete original yrb_mb, yrl_ml and yscond rows. Syntax acceptance is +// distinct from a physical encoder-view or native-memory permission. +var x86CompareExchangeSpecs = map[Op]x86CompareExchangeSpec{ + "CMPXCHGB": {bits: 8}, + "CMPXCHGW": {bits: 16}, + "CMPXCHGL": {bits: 32}, + "CMPXCHGQ": {bits: 64}, + "CMPXCHG8B": {bits: 64, pair: true}, + "CMPXCHG16B": {bits: 128, pair: true}, +} + +type x86CompareExchangeForm struct { + spec x86CompareExchangeSpec + source Operand + destination Operand +} + +func x86CompareExchangeProperties(op Op, ins Instr, arch string) (x86CompareExchangeForm, bool, error) { + name := strings.ToUpper(string(op)) + base := strings.SplitN(name, ".", 2)[0] + spec, recognized := x86CompareExchangeSpecs[Op(base)] + if !recognized { + return x86CompareExchangeForm{}, false, nil + } + invalid := func(reason string) (x86CompareExchangeForm, bool, error) { + return x86CompareExchangeForm{}, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + context := func(reason string) (x86CompareExchangeForm, bool, error) { + return x86CompareExchangeForm{}, true, fmt.Errorf("%w: %s %s %s: %q", ErrProbeNeedsContext, arch, op, reason, ins.Raw) + } + if name != base || ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return invalid("does not accept suffixes or a contradictory source identity") + } + if arch != "amd64" && arch != "386" { + return invalid("requires a registered x86 mode") + } + if arch == "386" && (spec.bits == 128 || spec.bits == 64 && !spec.pair) { + return invalid("is illegal in 32-bit mode") + } + count := 2 + if spec.pair { + count = 1 + } + if len(ins.Args) != count { + return invalid("does not have its complete ordered operand count") + } + form := x86CompareExchangeForm{spec: spec, destination: ins.Args[count-1]} + if !spec.pair { + if projected, segment, err := x86NamedSegmentMemorySource(ins, form.destination, arch); segment { + if err != nil { + return form, true, err + } + form.destination = projected + } + } + registerAllowed := func(reg Reg) bool { + if spec.pair || spec.bits == 8 { + return isGoYmbRegisterForArch(reg, arch) + } + return isX86YrlRegisterForArch(reg, arch) + } + if !spec.pair { + form.source = ins.Args[0] + if form.source.Kind != OpReg || !registerAllowed(form.source.Reg) { + return invalid("source is outside its original Yrb/Yrl class") + } + } + destination := form.destination + if destination.Kind == OpReg { + if !registerAllowed(destination.Reg) { + return invalid("destination is outside its original Ymb/Yml class") + } + } else if !isAMD64MemoryOperand(destination) || + destination.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(destination.Sym), "$") { + return invalid("destination is not its original register/memory operand") + } else if destination.Kind == OpMem && !x86MemoryRegistersValidForArch(destination.Mem, arch) { + return invalid("memory uses an out-of-range address register") + } + if spec.pair && destination.Kind == OpReg { + return context("Go emits a non-memory pair operand with native #UD, not a numerical CAS") + } + if spec.bits == 8 { + if arch == "386" { + for _, operand := range []Operand{form.source, destination} { + if operand.Kind == OpReg && (operand.Reg == BP || operand.Reg == SI || operand.Reg == DI) { + return context("Go's byte-register expansion changes the implicit accumulator before CMPXCHG") + } + } + } else if x86CompareExchangeHasREXByteView(form) { + for _, operand := range []Operand{form.source, destination} { + if operand.Kind == OpReg && (operand.Reg == AH || operand.Reg == BH || operand.Reg == CH || operand.Reg == DH) { + return context("Go's REX encoding changes a high-byte spelling to an unbound low-byte physical view") + } + } + } + } + return form, true, nil +} + +func x86CompareExchangeHasREXByteView(form x86CompareExchangeForm) bool { + requiresREX := func(reg Reg) bool { + switch reg { + case SP, BP, SI, DI, BPB, SIB, DIB: + return true + } + base, _, alias := amd64ByteAlias(reg) + if alias { + reg = base + } + return strings.HasPrefix(string(reg), "R") + } + for _, operand := range []Operand{form.source, form.destination} { + if operand.Kind == OpReg && requiresREX(operand.Reg) { + return true + } + if operand.Kind == OpMem { + for _, reg := range []Reg{operand.Mem.Base, operand.Mem.Index} { + if strings.HasPrefix(string(reg), "R") { + return true + } + } + } + } + return false +} diff --git a/amd64_cmpxchg_linux_amd64_test.go b/amd64_cmpxchg_linux_amd64_test.go new file mode 100644 index 000000000..373ce1e75 --- /dev/null +++ b/amd64_cmpxchg_linux_amd64_test.go @@ -0,0 +1,44 @@ +package plan9asm + +import "testing" + +// These strict mathematical oracles require real Linux/AMD64 execution. +// The unchanged shared fixtures preserve all six flags, complete aliases, +// MXCSR and canaries. Local Go/C equivalence never satisfies this ISA axis. +func TestCrossLinuxRuntimeMatrixX86CMPXCHGOriginalGoCompleteNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + x86CMPXCHGOriginalNumerical(t, x86FloatCompareExecutionTools(t)) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixX86CMPXCHGIndependentCOriginalOpcodeNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + x86CMPXCHGOriginalCNumerical(t, x86FloatCompareExecutionTools(t)) + x87IA32RequireNativeHost(t) +} + +// This required native axis is selected by the existing CrossLinuxRuntimeMatrix +// CI prefix. Rosetta/QEMU equivalence does not satisfy its actual #GP assertion. +func TestCrossLinuxRuntimeMatrixX86CMPXCHGNativeFaultAndAccessWidth(t *testing.T) { + x87IA32RequireNativeHost(t) + x86CMPXCHGRunFaultCollector(t, x86FloatCompareExecutionTools(t), true) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixX86CMPXCHGNativeNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + execution := x86FloatCompareExecutionTools(t) + t.Run("original-Go", func(t *testing.T) { + x86CMPXCHGOriginalNumerical(t, execution) + }) + t.Run("original-C-opcode", func(t *testing.T) { + x86CMPXCHGOriginalCNumerical(t, execution) + }) + t.Run("LLVM", func(t *testing.T) { + x86CMPXCHGLLVMNumerical(t, execution) + }) + t.Run("Go-C-actual-equivalence", func(t *testing.T) { + x86CMPXCHGActualEquivalence(t, execution) + }) + x87IA32RequireNativeHost(t) +} diff --git a/amd64_cmpxchg_native.go b/amd64_cmpxchg_native.go new file mode 100644 index 000000000..0e3fd88ff --- /dev/null +++ b/amd64_cmpxchg_native.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) compareExchangeNativePrefix(memory Operand) (lock, segment string) { + if c.namedAtomicLock { + lock = "lock; " + } + if memory.Kind == OpMem && memory.Mem.Segment != "" { + segment = "%" + strings.ToLower(string(memory.Mem.Segment)) + ":" + } + return lock, segment +} + +func (c *amd64Ctx) emitScalarCompareExchangeNative(bits int, memory Operand, pointer, pointerType, expected, desired string) (old, success string) { + lock, segment := c.compareExchangeNativePrefix(memory) + typ := amd64IntegerTypeForBits(bits) + width := map[int]string{8: "b", 16: "w", 32: "l", 64: "q"}[bits] + register := "r" + if bits == 8 { + register = "q" + } + result, oldName, successByte, successName := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + // The complete native opcode supplies the read/write cycle on both paths, + // including failure. A plain original opcode does not acquire LOCK. No LLVM + // alignment precondition is asserted, and explicit FS/GS remains attached. + fmt.Fprintf(c.b, " %%%s = call { %s, i8 } asm sideeffect \"%scmpxchg%s $3, %s$2; sete $1\", \"={ax},=q,=*m,%s,0,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %s, %s %s, %s %s, %s elementtype(%s) %s)\n", + result, typ, lock, width, segment, register, pointerType, typ, pointer, typ, desired, typ, expected, pointerType, typ, pointer) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i8 } %%%s, 0\n", oldName, typ, result) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i8 } %%%s, 1\n", successByte, typ, result) + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %%%s, 0\n", successName, successByte) + return "%" + oldName, "%" + successName +} + +func (c *amd64Ctx) emitPairCompareExchangeNative(bits int, memory Operand, pointer, pointerType, ax, dx, bx, cx string) (old, success string) { + lock, segment := c.compareExchangeNativePrefix(memory) + part, pair, mnemonic := I32, LLVMType("i64"), "cmpxchg8b" + if bits == 128 { + part, pair, mnemonic = I64, LLVMType("i128"), "cmpxchg16b" + } else { + ax, dx = c.truncI64(ax, I32), c.truncI64(dx, I32) + bx, cx = c.truncI64(bx, I32), c.truncI64(cx, I32) + } + result := c.newTmp() + // 8B supports unaligned memory without an invented natural alignment. + // 16B retains the actual opcode's native #GP when its address is unaligned; + // LLVM align=16 would instead give undefined behavior, not that exception. + fmt.Fprintf(c.b, " %%%s = call { %s, %s, i8 } asm sideeffect \"%s%s %s$3; sete $2\", \"={ax},={dx},=q,=*m,{bx},{cx},0,1,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %s, %s %s, %s %s, %s %s, %s %s, %s elementtype(%s) %s)\n", + result, part, part, lock, mnemonic, segment, pointerType, pair, pointer, part, bx, part, cx, part, ax, part, dx, pointerType, pair, pointer) + low, high, byteName, successName := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, %s, i8 } %%%s, 0\n", low, part, part, result) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, %s, i8 } %%%s, 1\n", high, part, part, result) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, %s, i8 } %%%s, 2\n", byteName, part, part, result) + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %%%s, 0\n", successName, byteName) + low64, high64 := "%"+low, "%"+high + if part == I32 { + lowWide, highWide := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", lowWide, low64) + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", highWide, high64) + low64, high64 = "%"+lowWide, "%"+highWide + } + return c.compareExchangeJoinPair(low64, high64, bits), "%" + successName +} diff --git a/amd64_cmpxchg_native_family_test.go b/amd64_cmpxchg_native_family_test.go new file mode 100644 index 000000000..f93002c89 --- /dev/null +++ b/amd64_cmpxchg_native_family_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Independent complete original Go registrations, including ordered encoder +// class, prefix and bytes. Go's yscond admits a register spelling even though +// Intel's pair instructions require memory and raise #UD for ModRM.mod=3. +var x86CMPXCHGOriginalRows = []struct { + op, registration string + bits int + pair bool +}{ + {"CMPXCHGB", "{ACMPXCHGB, yrb_mb, Pb, opBytes{0x0f, 0xb0}}", 8, false}, + {"CMPXCHGW", "{ACMPXCHGW, yrl_ml, Pe, opBytes{0x0f, 0xb1}}", 16, false}, + {"CMPXCHGL", "{ACMPXCHGL, yrl_ml, Px, opBytes{0x0f, 0xb1}}", 32, false}, + {"CMPXCHGQ", "{ACMPXCHGQ, yrl_ml, Pw, opBytes{0x0f, 0xb1}}", 64, false}, + {"CMPXCHG8B", "{ACMPXCHG8B, yscond, Pm, opBytes{0xc7, 01}}", 64, true}, + {"CMPXCHG16B", "{ACMPXCHG16B, yscond, Pw, opBytes{0x0f, 0xc7, 01}}", 128, true}, +} + +func TestX86CMPXCHGByteEncoderViewsCannotInventSourceAliases(t *testing.T) { + for _, row := range []struct { + arch, body string + encoded []byte + }{ + {"amd64", "CMPXCHGB AH,R11", []byte{0x41, 0x0f, 0xb0, 0xe3}}, + {"amd64", "CMPXCHGB AH,(R12)", []byte{0x41, 0x0f, 0xb0, 0x24, 0x24}}, + {"amd64", "CMPXCHGB R11,AH", []byte{0x44, 0x0f, 0xb0, 0xdc}}, + {"386", "CMPXCHGB BP,(BX)", []byte{0x95, 0x0f, 0xb0, 0x03, 0x95}}, + {"386", "CMPXCHGB SI,DI", []byte{0x96, 0x87, 0xdf, 0x0f, 0xb0, 0xc3, 0x87, 0xdf, 0x96}}, + } { + t.Run(row.arch+"/"+row.body, func(t *testing.T) { + source := x86NamedLockOracleSource(row.arch, row.body) + if row.arch == "amd64" { + source = strings.Replace(source, "\t"+row.body, "\tMOVQ $9,R11\n\tMOVQ p+0(FP),R12\n\t"+row.body, 1) + } + actual := x86OrderingHintGoBytes(t, row.arch, source) + if !bytes.HasSuffix(actual, append(append([]byte(nil), row.encoded...), 0xc3)) { + t.Fatalf("original whole encoder effect changed: %x", actual) + } + mode := 64 + if row.arch == "386" { + mode = 32 + } + physical, err := x86asm.Decode(row.encoded, mode) + if err != nil { + t.Fatal(err) + } + t.Logf("original first physical opcode=%s; full sequence=%x", x86asm.GoSyntax(physical, 0, nil), row.encoded) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[32]byte)\n", row.arch, testTargetTriple("linux", row.arch)) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("Go encoder view/implicit accumulator rewrite acquired a false source effect: %s/%v", api, err) + } + } + }) + } +} + +func TestX86CMPXCHGCompleteOriginalOrderedRows(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + table := string(data) + for _, row := range x86CMPXCHGOriginalRows { + if !strings.Contains(table, row.registration) { + t.Fatal("original opcode-specific encoding changed", row.registration) + } + } + for _, grammar := range []string{ + "var yrb_mb = []ytab{\n\t{Zr_m, 1, argList{Yrb, Ymb}},\n}", + "var yrl_ml = []ytab{\n\t{Zr_m, 1, argList{Yrl, Yml}},\n}", + "var yscond = []ytab{\n\t{Zo_m, 2, argList{Ymb}},\n}", + } { + if !strings.Contains(table, grammar) { + t.Fatal("complete ordered operand class changed", grammar) + } + } +} + +func x86CMPXCHGOriginalBody(row int, destination string) string { + op := x86CMPXCHGOriginalRows[row] + if op.pair { + return op.op + " " + destination + } + source := "CX" + if op.bits == 8 { + source = "CL" + } + return op.op + " " + source + "," + destination +} + +func TestX86CMPXCHGNativeMemoryHasNoInventedLLVMAlignment(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for index, row := range x86CMPXCHGOriginalRows { + if target.arch == "386" && (row.op == "CMPXCHGQ" || row.bits == 128) { + continue + } + for _, destination := range []string{"1(SI)", "0(FS)", "0(GS)"} { + for _, locked := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/%s/locked=%t", target.triple, row.op, destination, locked), func(t *testing.T) { + body := x86CMPXCHGOriginalBody(index, destination) + if locked { + body = "LOCK\n\t" + body + } + source := x86NamedLockOracleSource(target.arch, body) + actual := x86OrderingHintGoBytes(t, target.arch, source) + t.Logf("actual original %s bytes=%x (object only, no unsafe execution)", row.op, actual) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[32]byte)\n", target.arch, target.triple) + if err != nil { + t.Fatal("Go-accepted complete native memory operation", err) + } + if regexp.MustCompile(`(?m)^\s*%[^\n]+ = cmpxchg `).MatchString(ir) { + t.Fatal("native memory operand acquired an unproved LLVM atomic alignment") + } + want := strings.ToLower(row.op) + if locked { + want = "lock; " + want + } else if strings.Contains(ir, "lock; cmpxchg") { + t.Fatal("unprefixed original instruction acquired a LOCK effect") + } + if !strings.Contains(ir, "asm sideeffect \""+want) || !strings.Contains(ir, "=*m") || !strings.Contains(ir, "*m") { + t.Fatal("full native read/write opcode was not retained", want) + } + if destination == "0(FS)" && !strings.Contains(ir, "%fs:") || destination == "0(GS)" && !strings.Contains(ir, "%gs:") { + t.Fatal("actual original segment override disappeared") + } + compileLLVMToObject(t, findLLVM22Tool("llc"), target.triple, "cmpxchg-native.ll", "cmpxchg-native.o", ir) + }) + } + }) + } + } + } + } +} + +func TestX86CMPXCHGPairRegisterDoesNotInventSoftwareCAS(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86CMPXCHGOriginalRows[4:] { + if arch == "386" && row.bits == 128 { + continue + } + for _, register := range []string{"AX", "BX", "SI"} { + t.Run(arch+"/"+row.op+"/"+register, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, row.op+" "+register) + actual := x86OrderingHintGoBytes(t, arch, source) + t.Logf("Go-accepted pair-register #UD object bytes=%x; not a numerical oracle", actual) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[32]byte)\n", arch, testTargetTriple("linux", arch)) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("pair-register #UD became a normal software CAS in %s: %v", api, err) + } + } + }) + } + } + } +} + +func TestX86CMPXCHGDispatchIdentityFailsBeforeEmission(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + file, err := Parse(ArchAMD64, "TEXT locked(SB),4,$0-0\nCMPXCHGL CX,BX\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, identity := range []Op{"CMPXCHGQ", "MOVL", "CMPXCHGL.BAD"} { + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + ins := file.Funcs[0].Instrs[1] + ins.Op = identity + before, temporary := out.String(), ctx.tmp + known, terminated, err := ctx.lowerCompareExchange("CMPXCHGL", ins) + if !known || terminated || err == nil || before != out.String() || temporary != ctx.tmp { + t.Fatalf("contradictory dispatched/source opcode changed IR: %s %v", identity, err) + } + } + } +} diff --git a/amd64_cmpxchg_numerical_runtime_test.go b/amd64_cmpxchg_numerical_runtime_test.go new file mode 100644 index 000000000..c3d718cde --- /dev/null +++ b/amd64_cmpxchg_numerical_runtime_test.go @@ -0,0 +1,347 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86CMPXCHGOriginalNumerical(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + declarations, source, names := x86CMPXCHGNumericalFixture() + driver := strings.Replace(x86CMPXCHGGoNumerical, "KERNEL_NAMES", names, 1) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), runtime.GOOS, "amd64", declarations, source, driver) + output := execution.run(t, original) + if string(output) != "original CMPXCHG numerical observations 576 failures 0\n" { + t.Fatalf("actual unchanged Go ABI0 numerical/status/canary oracle: %s", output) + } + t.Log(string(output)) +} + +func TestX86CMPXCHGLLVMCompleteNumerical(t *testing.T) { + x86CMPXCHGLLVMNumerical(t, x86FloatCompareExecutionTools(t)) +} + +func x86CMPXCHGLLVMNumerical(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + declarations, source, names := x86CMPXCHGNumericalFixture() + ir, err := x86OrderingHintTranslateAPI(t, "Go", source, declarations, "amd64", execution.triple) + if err != nil { + t.Fatal(err) + } + var prototypes strings.Builder + for _, name := range strings.Split(strings.TrimSpace(names), ",") { + name = strings.TrimSpace(name) + if name != "" { + fmt.Fprintf(&prototypes, "extern void %s(uint64_t *, unsigned char *, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t);\n", name) + } + } + reference := strings.Replace(x86CMPXCHGCNumerical, "KERNEL_PROTOTYPES", prototypes.String(), 1) + reference = strings.Replace(reference, "KERNEL_NAMES", names, 1) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmpxchg-numerical", ir, reference, level) + if string(output) != "LLVM CMPXCHG numerical observations 576 failures 0\n" { + t.Fatalf("independent full byte/status/canary oracle: %s", output) + } + t.Log(string(output)) + }) + } +} + +func x86CMPXCHGNumericalFixture() (declarations, source, names string) { + var decl, assembly, functions strings.Builder + decl.WriteString("package main\n") + // The independent family order binds the oracle's widths and pair policy. + for index, op := range []string{"CMPXCHGB", "CMPXCHGW", "CMPXCHGL", "CMPXCHGQ", "CMPXCHG8B", "CMPXCHG16B"} { + for _, locked := range []bool{false, true} { + name := fmt.Sprintf("numeric_%d_%t", index, locked) + fmt.Fprintf(&decl, "func %s(out *[16]uint64, p *[32]byte, expectedLo, expectedHi, desiredLo, desiredHi, seed uint64)\n", name) + fmt.Fprintf(&functions, "%s,\n", name) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$0-56 +MOVQ out+0(FP),R10 +MOVQ p+8(FP),SI +MOVQ expectedLo+16(FP),AX +MOVQ expectedHi+24(FP),DX +MOVQ desiredLo+32(FP),BX +MOVQ desiredHi+40(FP),CX +MOVQ seed+48(FP),R8 +PUSHQ R8 +POPFQ +`, name) + if locked { + assembly.WriteString("LOCK\n") + } + switch index { + case 0: + assembly.WriteString(op + " BL,(SI)\n") + case 4, 5: + assembly.WriteString(op + " (SI)\n") + default: + assembly.WriteString(op + " BX,(SI)\n") + } + assembly.WriteString(`MOVQ AX,0(R10) +MOVQ DX,8(R10) +MOVQ BX,16(R10) +MOVQ CX,24(R10) +PUSHFQ +POPQ R9 +MOVQ R9,32(R10) +LAHF +MOVBLZX AH,BX +MOVQ BX,40(R10) +STMXCSR 48(R10) +RET +`) + } + } + decl.WriteString("func csr() uint32\n") + assembly.WriteString("TEXT ·csr(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\n") + return decl.String(), assembly.String(), functions.String() +} + +const x86CMPXCHGGoNumerical = `package main + +import ( + "encoding/binary" + "fmt" + "os" + "runtime" + "unsafe" +) + +func widthMask(bits uint) uint64 { + if bits == 64 { return ^uint64(0) } + return uint64(1)<>4 + parity ^= parity>>2 + parity ^= parity>>1 + if parity&1 == 0 { flags |= 4 } + if a&15 < b&15 { flags |= 16 } + if result == 0 { flags |= 64 } + if result>>(bits-1) != 0 { flags |= 128 } + if ((a^b)&(a^result))>>(bits-1) != 0 { flags |= 2048 } + return flags +} + +func main() { + runtime.LockOSThread() + kernels := []func(*[16]uint64, *[32]byte, uint64, uint64, uint64, uint64, uint64){ + KERNEL_NAMES + } + edges := []uint64{0, 1, 15, 16, 127, 128, 255, 256, + 0x7fffffff, 0x80000000, 0x8000000000000000, ^uint64(0)} + widths := []uint{8, 16, 32, 64, 32, 64} + const canary uint64 = 0xa7a7a7a7a7a7a7a7 + observations, failures := 0, 0 + for index, kernel := range kernels { + kernelFailures, registerFailures, statusFailures := 0, 0, 0 + family := index/2 + bits := widths[family] + mask := widthMask(bits) + size := int(bits/8) + if family >= 4 { size *= 2 } + for _, edge := range edges { + for _, equal := range []bool{false, true} { + for _, seed := range []uint64{0x8c7, 0x12} { + raw := [96]byte{} + for i := range raw { raw[i] = 0xa7 } + aligned := 16 + int((-uintptr(unsafe.Pointer(&raw[0])))&15) + if family != 5 { aligned++ } + p := (*[32]byte)(unsafe.Pointer(&raw[aligned])) + low, high := edge&mask, (^edge)&mask + for byteIndex := 0; byteIndex < size; byteIndex++ { + value := low + if byteIndex >= int(bits/8) { value = high } + p[byteIndex] = byte(value >> (8*uint(byteIndex%int(bits/8)))) + } + expectedLow, expectedHigh := low, high + if !equal { expectedLow = (low+1)&mask } + expectedLow |= 0x13579bdf2468ace0 & ^mask + expectedHigh |= 0x2468ace013579bdf & ^mask + desiredLow := edge ^ 0xd3c2b1a097867564 + desiredHigh := ^edge ^ 0x789abcdef0123456 + expectedRaw := raw + if equal { + for byteIndex := 0; byteIndex < size; byteIndex++ { + value := desiredLow + if byteIndex >= int(bits/8) { value = desiredHigh } + expectedRaw[aligned+byteIndex] = byte(value >> (8*uint(byteIndex%int(bits/8)))) + } + } + out := [16]uint64{} + for i := range out { out[i] = canary } + before := csr() + kernel(&out, p, expectedLow, expectedHigh, desiredLow, desiredHigh, seed) + after := csr() + wantLow, wantHigh := expectedLow, expectedHigh + if !equal { + if bits == 32 { wantLow = low } else { wantLow = expectedLow&^mask | low } + if family >= 4 { + if bits == 32 { wantHigh = high } else { wantHigh = high } + } + } + flags := subtractFlags(expectedLow, low, bits) + if family >= 4 { + flags = seed & 0x8d5 &^ 64 + if equal { flags |= 64 } + } + csr := binary.LittleEndian.Uint32((*[4]byte)(unsafe.Pointer(&out[6]))[:]) + bad := out[0] != wantLow || out[1] != wantHigh || + out[2] != desiredLow || out[3] != desiredHigh || + out[4]&0x8d5 != flags || out[5] != (flags&0xd5 | 2) || + out[6]>>32 != canary>>32 || csr != before || after != before || raw != expectedRaw + for i := 7; i < len(out); i++ { bad = bad || out[i] != canary } + if bad { + kernelFailures++ + if out[0] != wantLow || out[1] != wantHigh || out[2] != desiredLow || out[3] != desiredHigh { + registerFailures++ + } + if out[4]&0x8d5 != flags || out[5] != (flags&0xd5 | 2) { + statusFailures++ + } + if failures < 24 { + fmt.Printf("original failure kernel=%d bits=%d edge=%x equal=%t seed=%x out=%x wantAX=%x wantDX=%x flags=%x\n", + index, bits, edge, equal, seed, out, wantLow, wantHigh, flags) + } + failures++ + } + observations++ + } + } + } + if kernelFailures != 0 { + fmt.Printf("original kernel=%d observations=48 failures=%d register=%d status=%d\n", + index, kernelFailures, registerFailures, statusFailures) + } + } + fmt.Printf("original CMPXCHG numerical observations %d failures %d\n", observations, failures) + if failures != 0 { os.Exit(1) } +} +` + +const x86CMPXCHGCNumerical = ` +#include +#include +#include + +KERNEL_PROTOTYPES +typedef void (*Kernel)(uint64_t *, unsigned char *, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); + +static uint64_t width_mask(unsigned bits) { + return bits == 64 ? UINT64_MAX : ((UINT64_C(1)<>bit)&1; + if (!(ones&1)) flags |= 4; + if ((a&15) < (b&15)) flags |= 16; + if (!difference) flags |= 64; + if (difference>>(bits-1)) flags |= 128; + if (((a^b)&(a^difference))>>(bits-1)) flags |= 2048; + return flags; +} + +int main(void) { + Kernel kernels[] = { KERNEL_NAMES }; + const uint64_t edges[] = {0, 1, 15, 16, 127, 128, 255, 256, + 0x7fffffff, 0x80000000, UINT64_C(0x8000000000000000), UINT64_MAX}; + const unsigned widths[] = {8, 16, 32, 64, 32, 64}; + const uint64_t seeds[] = {0x8c7, 0x12}; + const uint64_t canary = UINT64_C(0xa7a7a7a7a7a7a7a7); + unsigned observations = 0, failures = 0; + for (unsigned index = 0; index < 12; index++) { + unsigned kernel_failures = 0, register_failures = 0, status_failures = 0; + unsigned family = index/2, bits = widths[family]; + uint64_t mask = width_mask(bits); + unsigned size = bits/8 * (family >= 4 ? 2 : 1); + for (unsigned edge = 0; edge < 12; edge++) { + for (unsigned equal = 0; equal < 2; equal++) { + for (unsigned seed = 0; seed < 2; seed++) { + _Alignas(16) unsigned char raw[96], expected[96]; + memset(raw, 0xa7, sizeof(raw)); + unsigned start = family == 5 ? 16 : 17; + unsigned char *p = raw+start; + uint64_t low = edges[edge]&mask, high = ~edges[edge]&mask; + for (unsigned byte = 0; byte < size; byte++) { + uint64_t value = byte >= bits/8 ? high : low; + p[byte] = value>>(8*(byte%(bits/8))); + } + uint64_t expected_low = equal ? low : (low+1)&mask; + expected_low |= UINT64_C(0x13579bdf2468ace0)&~mask; + uint64_t expected_high = high | (UINT64_C(0x2468ace013579bdf)&~mask); + uint64_t desired_low = edges[edge]^UINT64_C(0xd3c2b1a097867564); + uint64_t desired_high = ~edges[edge]^UINT64_C(0x789abcdef0123456); + memcpy(expected, raw, sizeof(raw)); + if (equal) { + for (unsigned byte = 0; byte < size; byte++) { + uint64_t value = byte >= bits/8 ? desired_high : desired_low; + expected[start+byte] = value>>(8*(byte%(bits/8))); + } + } + uint64_t out[16]; + for (unsigned i = 0; i < 16; i++) out[i] = canary; + unsigned before, after; + __asm__ volatile("stmxcsr %0" : "=m"(before)); + kernels[index](out, p, expected_low, expected_high, desired_low, desired_high, seeds[seed]); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + uint64_t want_low = expected_low, want_high = expected_high; + if (!equal) { + want_low = bits == 32 ? low : (expected_low&~mask)|low; + if (family >= 4) want_high = high; + } + uint64_t flags = subtraction(expected_low, low, bits); + if (family >= 4) { + flags = (seeds[seed]&0x8d5)&~UINT64_C(64); + if (equal) flags |= 64; + } + int bad = out[0] != want_low || out[1] != want_high || + out[2] != desired_low || out[3] != desired_high || + (out[4]&0x8d5) != flags || out[5] != ((flags&0xd5)|2) || + (out[6]>>32) != (canary>>32) || (uint32_t)out[6] != before || + after != before || memcmp(raw, expected, sizeof(raw)); + for (unsigned i = 7; i < 16; i++) bad |= out[i] != canary; + if (bad) { + kernel_failures++; + register_failures += out[0] != want_low || out[1] != want_high || + out[2] != desired_low || out[3] != desired_high; + status_failures += (out[4]&0x8d5) != flags || out[5] != ((flags&0xd5)|2); + if (kernel_failures <= 2) { + fprintf(stdout, "LLVM failure kernel=%u bits=%u edge=%llx equal=%u seed=%llx AX=%llx/%llx DX=%llx/%llx flags=%llx/%llx BX=%llx CX=%llx\n", + index, bits, (unsigned long long)edges[edge], equal, + (unsigned long long)seeds[seed], (unsigned long long)out[0], + (unsigned long long)want_low, (unsigned long long)out[1], + (unsigned long long)want_high, (unsigned long long)out[4], + (unsigned long long)flags, (unsigned long long)out[2], (unsigned long long)out[3]); + } + failures++; + } + observations++; + } + } + } + if (kernel_failures) { + printf("LLVM kernel=%u observations=48 failures=%u register=%u status=%u\n", + index, kernel_failures, register_failures, status_failures); + } + } + printf("LLVM CMPXCHG numerical observations %u failures %u\n", observations, failures); + return failures ? 1 : 0; +} +` diff --git a/amd64_cmpxchg_object_test.go b/amd64_cmpxchg_object_test.go new file mode 100644 index 000000000..3015dce06 --- /dev/null +++ b/amd64_cmpxchg_object_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "bytes" + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86CMPXCHGThreeAPIsFiveTargetsExactNativeO0O2Objects(t *testing.T) { + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 compiler and disassembler are required") + } + for _, target := range x86OrderingHintTargets { + var source, declarations strings.Builder + declarations.WriteString("package ordering\n") + want := make(map[string]struct { + op string + bits int + locked bool + }) + for index, row := range x86CMPXCHGOriginalRows { + if target.arch == "386" && (row.op == "CMPXCHGQ" || row.bits == 128) { + continue + } + for _, locked := range []bool{false, true} { + name := fmt.Sprintf("cas%d_%t", index, locked) + body := x86CMPXCHGOriginalBody(index, "1(SI)") + if locked { + body = "LOCK\n\t" + body + } + original := strings.Replace(x86NamedLockOracleSource(target.arch, body), "·locked", "·"+name, 1) + // Original objects are accepted without executing unsafe forms. + x86OrderingHintGoBytes(t, target.arch, original) + source.WriteString(original) + fmt.Fprintf(&declarations, "func %s(p *[32]byte)\n", name) + opcode := "CMPXCHG" + if row.pair { + opcode = row.op + } + want[name] = struct { + op string + bits int + locked bool + }{opcode, row.bits, locked} + } + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + dir := t.TempDir() + input, object := filepath.Join(dir, "native.ll"), filepath.Join(dir, "native.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+target.triple, "-filetype=obj", input, "-o", object)) + listing := x86FloatCompareCaptureCommand(t, exec.Command(disassembler, "--disassemble", object)) + sections := regexp.MustCompile(`(?m)^[0-9a-f]+ <_?(cas[0-9]+_(?:false|true))>:\n`).FindAllSubmatchIndex(listing, -1) + seen := make(map[string]bool) + for index, section := range sections { + name := string(listing[section[2]:section[3]]) + expected, known := want[name] + if !known || seen[name] { + t.Fatalf("unexpected or duplicate original function identity %s", name) + } + end := len(listing) + if index+1 < len(sections) { + end = sections[index+1][0] + } + var code []byte + for _, line := range strings.Split(string(listing[section[1]:end]), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasSuffix(fields[0], ":") { + continue + } + var machine []byte + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := hex.DecodeString(field) + if err != nil { + break + } + machine = append(machine, value[0]) + } + code = append(code, machine...) + } + // LLVM's disassembler displays a standalone F0 byte as a + // separate "lock" line. Decode complete contiguous bytes, + // not that presentation boundary, as the CPU does. + mode := 64 + if target.arch == "386" { + mode = 32 + } + count := 0 + for len(code) != 0 { + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len <= 0 || physical.Len > len(code) { + t.Fatalf("incomplete physical function decode %s: %x %v", name, code, err) + } + machine := code[:physical.Len] + code = code[physical.Len:] + if !strings.HasPrefix(physical.Op.String(), "CMPXCHG") { + continue + } + count++ + locked := false + for _, prefix := range physical.Prefix { + if byte(prefix) == 0xf0 { + locked = true + } + } + width := physical.DataSize + if bytes.Contains(machine, []byte{0x0f, 0xb0}) { + // x/arch DataSize retains the operand-size state for + // B0, whose opcode independently specifies byte width. + width = 8 + } + if physical.Op.String() != expected.op || locked != expected.locked || + expected.op == "CMPXCHG" && width != expected.bits { + t.Fatalf("%s/%s lost native width/prefix/opcode: %x %s\n%s", level, name, machine, + x86asm.GoSyntax(physical, 0, nil), listing) + } + } + if count != 1 { + t.Fatalf("%s/%s has %d complete native operations\n%s", level, name, count, listing) + } + seen[name] = true + } + if len(seen) != len(want) { + t.Fatalf("%s emitted %d/%d complete functions\n%s", level, len(seen), len(want), listing) + } + t.Logf("%s independently decoded %d native opcodes with exact width and LOCK identity", level, len(seen)) + } + }) + } + } +} diff --git a/amd64_cmpxchg_original_c_runtime_test.go b/amd64_cmpxchg_original_c_runtime_test.go new file mode 100644 index 000000000..422e51533 --- /dev/null +++ b/amd64_cmpxchg_original_c_runtime_test.go @@ -0,0 +1,76 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +func x86CMPXCHGOriginalCNumerical(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + reference := x86CMPXCHGOriginalCSource() + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := x86CMPXCHGCompileOriginalC(t, execution, reference, level) + if string(output) != "actual C opcode CMPXCHG numerical observations 576 failures 0\n" { + t.Fatalf("actual original C opcode vs independent math: %s", output) + } + t.Log(string(output)) + }) + } +} + +func x86CMPXCHGOriginalCSource() string { + _, _, names := x86CMPXCHGNumericalFixture() + var definitions strings.Builder + for index, op := range []string{"cmpxchgb %bl,", "cmpxchgw %bx,", "cmpxchgl %ebx,", "cmpxchgq %rbx,", "cmpxchg8b ", "cmpxchg16b "} { + for _, locked := range []bool{false, true} { + name := fmt.Sprintf("numeric_%d_%t", index, locked) + prefix := "" + if locked { + prefix = "lock; " + } + fmt.Fprintf(&definitions, `static void %s(uint64_t *out, unsigned char *p, + uint64_t expected_low, uint64_t expected_high, uint64_t desired_low, + uint64_t desired_high, uint64_t seed) { + __asm__ volatile( + "pushq %%[seed]; popfq; %s%s(%%%%rsi);" +`, name, prefix, strings.ReplaceAll(op, "%", "%%")) + definitions.WriteString(` "movq %%rax,0(%%rdi); movq %%rdx,8(%%rdi);" + "movq %%rbx,16(%%rdi); movq %%rcx,24(%%rdi);" + "pushfq; popq %%r9; movq %%r9,32(%%rdi);" + "lahf; movzbl %%ah,%%ebx; movq %%rbx,40(%%rdi);" + "stmxcsr 48(%%rdi);" + : "+a"(expected_low), "+d"(expected_high), "+b"(desired_low), "+c"(desired_high) + : "D"(out), "S"(p), [seed] "r"(seed) + : "r9", "cc", "memory"); +} +`) + } + } + reference := strings.Replace(x86CMPXCHGCNumerical, "KERNEL_PROTOTYPES", definitions.String(), 1) + reference = strings.Replace(reference, "KERNEL_NAMES", names, 1) + reference = strings.ReplaceAll(reference, "LLVM failure", "actual C opcode failure") + reference = strings.ReplaceAll(reference, "LLVM kernel", "actual C opcode kernel") + reference = strings.ReplaceAll(reference, "LLVM CMPXCHG numerical", "actual C opcode CMPXCHG numerical") + return reference +} + +func x86CMPXCHGCompileOriginalC(t *testing.T, execution x86FloatCompareExecution, sourceText, level string) []byte { + t.Helper() + directory := t.TempDir() + source := filepath.Join(directory, "original.c") + executable := filepath.Join(directory, gotoolchain.HostExecutable("original")) + if err := os.WriteFile(source, []byte(sourceText), 0600); err != nil { + t.Fatal(err) + } + command := append([]string(nil), execution.compiler...) + command = append(command, level, source, "-o", executable) + x86FloatCompareCaptureCommand(t, exec.Command(command[0], command[1:]...)) + return execution.run(t, executable) +} diff --git a/amd64_cmpxchg_original_catalogue_test.go b/amd64_cmpxchg_original_catalogue_test.go new file mode 100644 index 000000000..480b20152 --- /dev/null +++ b/amd64_cmpxchg_original_catalogue_test.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// Keep the complete historical source bodies and all 33/24 accepted operands. +// The official include changes no instruction: it supplies their original +// NOSPLIT/NOPTR spellings from the selected actual SDK, not a invented ABI. +func x86CMPXCHGMixedOriginalCatalogue(t *testing.T, arch, source string) { + t.Helper() + directory := t.TempDir() + input, object := filepath.Join(directory, "complete.s"), filepath.Join(directory, "complete.o") + if err := os.WriteFile(input, []byte("#include \"textflag.h\"\n"+source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "tool", "asm", "-S", + "-I", filepath.Join(runtime.GOROOT(), "pkg", "include"), "-p", "ordering", "-o", object, input) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH="+arch, + "GOFLAGS=", "GOENV=off", "GOWORK=off", "GOEXPERIMENT=") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), true, arch); failure != nil { + t.Fatalf("actual complete original syntax object: %v\n%s", failure, output) + } + t.Logf("actual original complete accepted catalogue, not unsafe execution:\n%s", output) + file, err := Parse(ArchAMD64, source) + if err != nil || len(file.Funcs) != 1 { + t.Fatalf("complete original source parse: %v", err) + } + count, physicalContext := 0, 0 + for _, instruction := range file.Funcs[0].Instrs { + if !strings.HasPrefix(string(instruction.Op), "CMPXCHG") { + continue + } + _, recognized, err := x86CompareExchangeProperties(instruction.Op, instruction, arch) + if !recognized || err != nil && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("actual original accepted row disappeared from grammar: %+v/%v", instruction, err) + } + count++ + if err != nil { + physicalContext++ + t.Logf("source/physical bank or native #UD Context: %s: %v", instruction.Raw, err) + } + } + want := 33 + if arch == "386" { + want = 24 + } + if count != want || physicalContext != 5 { + t.Fatalf("complete original syntax/physical-context denominator changed: %d/%d, want %d/5", count, physicalContext, want) + } +} diff --git a/amd64_comi_boundaries_test.go b/amd64_comi_boundaries_test.go new file mode 100644 index 000000000..7fbbe628d --- /dev/null +++ b/amd64_comi_boundaries_test.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86COMIWholeFileNativeTargetAndDefinedFlags(t *testing.T) { + for _, name := range x86COMIOracleNames() { + for _, api := range []string{"text", "module", "Go"} { + for _, dead := range []bool{false, true} { + before := "" + if dead { + before = "RET\n" + } + for _, triple := range []string{"", "aarch64-unknown-linux-gnu", "wasm32-unknown-unknown", "i386-unknown-linux-gnu", "x86_64madeup-unknown-linux-gnu"} { + source := "TEXT ·view(SB),4,$0-0\n" + before + name + " X0,X1\nRET\n" + if _, err := x86ScalarFPBitViewAPI(t, source, "func view()\n", "amd64", triple, api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing native target accepted", name, api, dead, triple, err) + } + malformed := strings.Replace(source, "X0,X1", "X0,Y1", 1) + if _, err := x86ScalarFPBitViewAPI(t, malformed, "func view()\n", "amd64", triple, api); err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native-target context hid malformed/dead source", name, api, dead, triple, err) + } + } + } + for _, missing := range []int{-1, 0, 1} { + var setup strings.Builder + for input := 0; input < 2; input++ { + if input != missing { + fmt.Fprintf(&setup, "XORPS X%d,X%d\n", input, input) + } + } + source := "TEXT ·view(SB),4,$0-4\n" + setup.String() + "XORL AX,AX\n" + name + " X0,X1\nSETEQ AL\nMOVL AX,X2\nPEXTRD $0,X2,AX\nMOVL AX,ret+0(FP)\nRET\n" + ir, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32\n", "amd64", "x86_64-unknown-linux-gnu", api) + if missing != -1 { + x86VectorScratchCodegenControl(t, source, "func view() uint32\n", "amd64", "x86_64-unknown-linux-gnu", ir, err) + continue + } + if err != nil { + t.Fatal("undefined input acquired public result permission", name, api, missing, err) + } + } + } + } +} + +func TestX86COMIRuntimePartitionCompleteAndOriginalInventory(t *testing.T) { + remaining := make(map[string]bool) + for _, row := range x86ScalarCompareAFCases("amd64", true) { + remaining[row.name] = true + } + var counts [3]int + for _, axis := range []x86FloatCompareRuntimeISA{x86FloatCompareSSE, x86FloatCompareVEX, x86FloatCompareEVEX} { + cases := x86ScalarCompareAFRuntimeCases(axis) + for _, row := range cases { + if !remaining[row.name] { + t.Fatal("duplicate runtime kernel", row) + } + delete(remaining, row.name) + counts[axis-1]++ + } + goAssemblerOracleOutput(t, "amd64", x86ScalarCompareAFAssembly("amd64", cases), "", true) + } + if len(remaining) != 0 || counts != [3]int{48, 48, 48} { + t.Fatal("complete runtime partition changed", remaining, counts) + } +} diff --git a/amd64_comi_fault_runtime_test.go b/amd64_comi_fault_runtime_test.go new file mode 100644 index 000000000..60cac94c5 --- /dev/null +++ b/amd64_comi_fault_runtime_test.go @@ -0,0 +1,169 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86COMIDeadMemoryExtent(t *testing.T) { + x86COMIDeadFaults(t, false) +} + +func x86COMIDeadFaults(t *testing.T, native bool) { + t.Helper() + axis := x86FloatCompareSSE + if native { + axis = x86FloatCompareEVEX + } + execution := x86FloatCompareRuntimeTools(t, axis) + if native && (runtime.GOOS != "linux" || runtime.GOARCH != "amd64" || len(execution.prefix) != 0) { + t.Fatal("COMI exception delivery requires actual native Linux AMD64") + } + assembly, declarations, driver, _, trapChecks := x86COMIDeadFaultFixture(t, native, execution.triple) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, assembly, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, pipeline := range []struct{ name, ir, level string }{ + {"backend_O0", ir, "-O0"}, {"IR_O2/backend_O2", x86MulDivRoundOptimize(t, ir), "-O2"}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "comi-fault", pipeline.ir, driver, pipeline.level) + want := fmt.Sprintf("COMI original Go/LLVM dead reads matched; native IE/DE/SAE checks=%d\n", trapChecks) + if string(output) != want { + t.Fatal("missing complete fault receipt", string(output)) + } + }) + } + }) + } +} + +func x86COMIDeadFaultFixture(t *testing.T, native bool, triple string) (string, string, string, int, int) { + t.Helper() + var assembly, declarations, original, prototypes, pairs strings.Builder + count := 0 + for _, name := range x86COMIOracleNames() { + if strings.HasPrefix(name, "V") && !native { + continue + } + suffixes := []string{""} + if strings.HasPrefix(name, "V") { + suffixes = append(suffixes, ".SAE") + } + for _, suffix := range suffixes { + width, load := 4, "MOVSS" + if strings.HasSuffix(name, "D") { + width, load = 8, "MOVSD" + } + body := "XORPS X1,X1\nLDMXCSR (SI)\n" + name + " (DI),X1\n" + if suffix != "" { + body = "XORPS X1,X1\n" + load + " (DI),X0\nLDMXCSR (SI)\n" + name + suffix + " X0,X1\n" + } + body += "XORL AX,AX\nRET\n" // The comparison FLAGS are deliberately dead. + function, nativeFunction := fmt.Sprintf("fault%d", count), fmt.Sprintf("original%d", count) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),DI\nMOVQ csr+8(FP),SI\n%s", function, body) + fmt.Fprintf(&declarations, "func %s(p *uint64,csr *uint32)\n", function) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+body) + bytes := make([]string, len(code)) + for index, value := range code { + bytes[index] = fmt.Sprintf("0x%02x", value) + } + symbol := nativeFunction + if strings.Contains(triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&original, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint64_t*,uint32_t*),%s(uint64_t*,uint32_t*);\n", function, nativeFunction) + quiet, sae := 0, 0 + if strings.Contains(name, "UCOMI") { + quiet = 1 + } + if suffix != "" { + sae = 1 + } + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d},\n", nativeFunction, function, width, quiet, sae) + count++ + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + trapChecks := 0 + if native { + trapChecks = 1 + } + driver := fmt.Sprintf("#define NATIVE_TRAPS %d\n", trapChecks) + ` +#include +#include +#include +#include +#include +#include +#include +#include +` + original.String() + prototypes.String() + ` +typedef void (*kernel)(uint64_t*,uint32_t*); +static const struct {kernel original,translated;int width,quiet,sae;} pairs[]={ +` + pairs.String() + `}; +static int outcome(kernel fn,uint8_t *p,uint32_t csr) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + fn((uint64_t*)p,&csr); + _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status)) { + int signal=WTERMSIG(status); + if(signal==SIGSEGV || signal==SIGBUS) return 1; + if(signal==SIGFPE) return 2; + } + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + uint8_t *p=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(p==MAP_FAILED || mprotect(p+page,page,PROT_NONE)) return 2; + memset(p,0,page); + for(unsigned i=0;i= 16 || arch == "386" && register >= 8) { + return fail() + } + if arch == "386" && register >= 8 { + contextErr = fmt.Errorf("%w: %s lacks physical mode32 high-X identity", ErrProbeNeedsContext, ins.Op) + } + form.evex = form.evex || register >= 16 + continue + } + if index != 0 || !isAMD64MemoryOperand(operand) { + return fail() + } + if operand.Kind == OpMem { + if operand.Mem.OffRaw != "" || operand.Mem.IndexExt != "" { + return fail() + } + if err := x86SymbolIndexedMemoryError(operand.Mem, arch); err != nil { + if !errors.Is(err, ErrProbeNeedsContext) { + return form, true, err + } + contextErr = err + continue + } + if arch == "386" && strings.HasSuffix(operand.Mem.Sym, "(SB)") { + operand.Mem.Base = "" + } + } + if !x86PackedShiftMemoryValid(operand, arch) { + return fail() + } + } + return form, true, contextErr +} + +func validateX86COMIFile(file *File, options Options) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + needed := false + var contextErr error + for _, function := range file.Funcs { + for _, instruction := range function.Instrs { + _, known, err := x86COMIProperties(instruction, options.Goarch) + if known && err != nil { + if !errors.Is(err, ErrProbeNeedsContext) { + return fmt.Errorf("%s: %w", function.Sym, err) + } + if contextErr == nil { + contextErr = fmt.Errorf("%s: %w", function.Sym, err) + } + } + needed = needed || known + } + } + if contextErr != nil { + return contextErr + } + if needed && !x86ScalarFloatIntegerNativeTarget(options.Goarch, options.TargetTriple) { + return fmt.Errorf("%w: scalar flag comparison requires its native x86 MXCSR target", ErrProbeNeedsContext) + } + return nil +} + +func x86TypedCOMIByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + name, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86COMISpecs[Op(name)]; !known { + return x86TypedByteEffect{}, false, nil + } + if !strings.EqualFold(string(op), string(ins.Op)) { + return x86TypedByteEffect{}, true, fmt.Errorf("scalar flag comparison effect/source mismatch") + } + form, known, err := x86COMIProperties(ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.spec.bits / 8) + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: width, read: true}, + {index: 1, operand: ins.Args[1], width: width, read: true}, + }, flagsDefined: x86ByteFlagsAll, requiresResultProof: true}, true, nil +} + +func x86ResultCOMIStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + return x86ResultCOMIStepWithRead(ins, state, owned, arch, false) +} + +// Only the paired native comparison supplies conditional numerical inputs. +// Its complete opcode preserves the scalar read, MXCSR and exception effects; +// no memory backing or reusable input fact reaches the declaration observer. +func x86ResultCOMIStepWithRead(ins Instr, state *x86ResultState, owned []bool, arch string, nativeRead bool) (bool, bool) { + form, known, err := x86COMIProperties(ins, arch) + if !known || err != nil { + return false, known + } + width := int64(form.spec.bits / 8) + for _, operand := range ins.Args { + if operand.Kind == OpReg { + index, _ := amd64ParseXReg(operand.Reg) + mask := x86CallerVectorByteMask(width) + if state.vectorBytes(index)&mask != mask { + return false, true + } + } else if !x86ResultReadWithNativeContinuation(ins, operand, width, state, owned, arch, nativeRead) { + return false, true + } + } + state.flags = x86ByteFlagsAll + return true, true +} diff --git a/amd64_comi_grammar_test.go b/amd64_comi_grammar_test.go new file mode 100644 index 000000000..c12a824a7 --- /dev/null +++ b/amd64_comi_grammar_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func x86COMIOracleNames() []string { + return []string{"COMISS", "COMISD", "UCOMISS", "UCOMISD", "VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} +} + +func TestX86COMIIndependentCompleteEncoderRows(t *testing.T) { + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(legacy), "var yxm = []ytab{\n\t{Zm_r_xm, 1, argList{Yxm, Yxr}},\n}") { + t.Fatal("complete legacy yxm operand table changed") + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, name := range x86COMIOracleNames() { + double, vector := strings.HasSuffix(name, "D"), strings.HasPrefix(name, "V") + opcode := "0x2F" + if strings.Contains(name, "UCOMI") { + opcode = "0x2E" + } + if !vector { + prefix := "Pm" + if double { + prefix = "Pe" + } + entry := fmt.Sprintf("{A%s, yxm, %s, opBytes{%s}}", name, prefix, strings.ToLower(opcode)) + if !strings.Contains(string(legacy), entry) { + t.Fatal("legacy scalar compare opcode attributes changed", entry) + } + count++ + continue + } + vex, evex, tuple := "avxEscape | vex128 | vex0F | vexW0", "avxEscape | evex128 | evex0F | evexW0", "evexN4 | evexSaeEnabled" + if double { + vex, evex, tuple = "avxEscape | vex128 | vex66 | vex0F | vexW0", "avxEscape | evex128 | evex66 | evex0F | evexW1", "evexN8 | evexSaeEnabled" + } + opbytes := []string{vex, opcode, evex, tuple, opcode} + index := 0 + for _, row := range rules { + if row.Opcode != name { + continue + } + class, zcase, advance, offset := "Yxm", "Zvex_rm_v_r", 2, 0 + if index == 1 { + class, zcase, advance, offset = "YxmEvex", "Zevex_rm_v_r", 3, 2 + } + destination := "Yxr" + if index == 1 { + destination = "YxrEvex" + } + if row.Ytab != "_yvcomisd" || row.Prefix != "Pavx" || row.RowIndex != index || row.EncodingOffset != offset || + row.ZCase != zcase || row.ZOffset != advance || !reflect.DeepEqual(row.Classes, []string{class, destination}) || !reflect.DeepEqual(row.OpBytes, opbytes) { + t.Fatalf("%s independent complete encoder row changed: %+v", name, row) + } + index++ + count++ + } + if index != 2 { + t.Fatal("missing scalar compare encoder rows", name, index) + } + } + if count != 12 || len(x86COMISpecs) != 8 { + t.Fatal("complete eight-name/twelve-row family changed", count, len(x86COMISpecs)) + } +} + +func TestX86COMITypedEveryByteAndEffects(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86COMIOracleNames() { + width := 4 + if strings.HasSuffix(name, "D") { + width = 8 + } + instruction := x86ScalarLaneInstruction(t, name+" X1,X2") + effect, known, err := x86TypedInstructionByteEffect(instruction.Op, instruction, arch) + want := x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: instruction.Args[0], width: int64(width), read: true}, + {index: 1, operand: instruction.Args[1], width: int64(width), read: true}, + }, flagsDefined: x86ByteFlagsAll, requiresResultProof: true} + if !known || err != nil || !reflect.DeepEqual(effect, want) { + t.Fatal("comparison gained a write or lost exact input bytes/FLAGS", arch, name, effect, err) + } + for _, input := range []int{1, 2} { + for missing := -1; missing < 64; missing++ { + state := &x86ResultState{} + state.writeVector(1, 64, ^uint64(0), false) + state.writeVector(2, 64, ^uint64(0), false) + bytes := ^uint64(0) + if missing >= 0 { + bytes &^= 1 << uint(missing) + } + state.writeVector(input, 64, bytes, false) + accepted := x86ResultTransfer(instruction, state, nil, 0, arch) + want := missing < 0 || missing >= width + if accepted != want || state.vectorBytes(input) != bytes || accepted && state.flags != x86ByteFlagsAll || !accepted && state.flags != 0 { + t.Fatal("comparison consumed undefined bytes or changed inputs/FLAGS on failure", arch, name, input, missing) + } + } + } + for _, kind := range []string{"FP", "AUTO", "SB", "pointer"} { + for missing := -1; missing < width; missing++ { + owned := make([]bool, 16) + state := &x86ResultState{fp: make([]bool, 16), auto: make([]bool, 16)} + state.writeVector(2, 64, ^uint64(0), false) + for i := range owned { + owned[i], state.fp[i], state.auto[i] = i != missing, i != missing, i != missing + } + operand := "value+0(FP)" + switch kind { + case "AUTO": + operand = "local-16(SP)" + case "SB": + operand = "external(SB)" + case "pointer": + operand = "(BX)" + state.pointerGP[3] = true + } + accepted := x86ResultTransfer(x86ScalarLaneInstruction(t, name+" "+operand+",X2"), state, owned, 0, arch) + want := (kind == "FP" || kind == "AUTO") && missing == -1 + if accepted != want { + t.Fatal("comparison acquired unowned/undefined memory", arch, name, kind, missing) + } + } + } + } + } +} + +func TestX86COMIHelperCannotBorrowForeignDispatch(t *testing.T) { + for _, operation := range []Op{"ADDQ", "MOVQ", "UNKNOWN.COMISS", "COMISS"} { + var ir strings.Builder + ctx := amd64Ctx{b: &ir, goarch: "amd64", targetTriple: "x86_64-unknown-linux-gnu"} + instruction := x86ScalarLaneInstruction(t, "MOVQ AX,BX") + known, terminated, err := ctx.lowerVectorScalarFlagCompare(operation, instruction) + _, effectKnown, effectErr := x86TypedCOMIByteEffect(operation, instruction, "amd64") + want := operation == "COMISS" + if known != want || effectKnown != want || (err != nil) != want || (effectErr != nil) != want || terminated || ir.Len() != 0 { + t.Fatal("COMI helper borrowed foreign source/dispatch identity", operation, known, effectKnown, err, effectErr) + } + } +} diff --git a/amd64_comi_native_avx512_linux_amd64_test.go b/amd64_comi_native_avx512_linux_amd64_test.go new file mode 100644 index 000000000..53318021b --- /dev/null +++ b/amd64_comi_native_avx512_linux_amd64_test.go @@ -0,0 +1,26 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86COMIRawNativeCompare(t *testing.T) { + x86COMIRawNativeCompare(t) +} + +func TestCrossLinuxRuntimeMatrixX86COMINativeDeadExceptions(t *testing.T) { + x86COMIDeadFaults(t, true) +} + +func TestCrossLinuxRuntimeMatrixX86COMINativeVectorFlagsEVEX(t *testing.T) { + t.Run("EVEX", func(t *testing.T) { + x86ScalarCompareAFOriginalGo(t, x86FloatCompareEVEX) + x86ScalarCompareAFLLVM(t, x86FloatCompareEVEX) + }) +} + +func TestCrossLinuxRuntimeMatrixX86COMINativeVectorMXCSREVEX(t *testing.T) { + t.Run("EVEX", func(t *testing.T) { + x86COMIOriginalConstantsAndMXCSR(t, x86FloatCompareEVEX) + }) +} diff --git a/amd64_comi_native_input_test.go b/amd64_comi_native_input_test.go new file mode 100644 index 000000000..fe3156880 --- /dev/null +++ b/amd64_comi_native_input_test.go @@ -0,0 +1,174 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func x86COMINativeInputSource(arch, name, op, setup, address, tail string) string { + word, size := "Q", 8 + if arch == "386" { + word, size = "L", 4 + } + return fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\nMOV%s p+0(FP),DX\n"+ + "MOVL $0,AX\nMOVL AX,X1\nMOVL $2,CX\n%s%s %s,X1\n%s"+ + "PUSHF%s\nPOP%s AX\nMOV%s AX,ret+%d(FP)\nRET\n", + name, 2*size, word, setup, op, address, tail, word, word, word, size) +} + +func x86COMINativeInputStatusProjection(arch, name, op, setup, address, tail string) string { + source := x86COMINativeInputSource(arch, name, op, setup, address, tail) + word := "Q" + if arch == "386" { + word = "L" + } + // COMI defines only the six status flags, not incoming DF or ID. Keep + // the original full-word observation below as an explicit negative. + pop := "POP" + word + " AX\n" + return strings.Replace(source, pop, pop+"AND"+word+" $0x8d5,AX\n", 1) +} + +func TestX86COMINativeInputCompleteFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + for _, op := range x86COMIOracleNames() { + source.WriteString(x86COMINativeInputStatusProjection(target.arch, op, op, "", "-7(DX)(CX*4)", "")) + fmt.Fprintf(&declarations, "func %s(p *byte) uintptr\n", op) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("initialized native compare input rejected", api, err) + } + if strings.Count(ir, "asm sideeffect") != 8 || + strings.Count(ir, "elementtype([4 x i8])") != 4 || + strings.Count(ir, "elementtype([8 x i8])") != 4 { + t.Fatal("native comparison lost its exact scalar access or MXCSR effect", api) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + } + }) + } +} + +func TestX86COMINativeInputCannotSupplyOtherRoles(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + word, triple := "L", "i386-unknown-linux-gnu" + if arch == "amd64" { + word, triple = "Q", "x86_64-unknown-linux-gnu" + } + for _, op := range x86COMIOracleNames() { + for _, row := range []struct{ name, remove, setup, address, tail string }{ + {"vector-input", "MOVL AX,X1\n", "", "(DX)", ""}, + {"index-input", "MOVL $2,CX\n", "", "(DX)(CX*4)", ""}, + {"pointer-index", "", "MOV" + word + " p+0(FP),CX\n", "(DX)(CX*4)", ""}, + {"numeric-base", "", "MOVL $4,DX\n", "(DX)", ""}, + {"unknown-base", "MOV" + word + " p+0(FP),DX\n", "", "(DX)", ""}, + {"unbound-symbol", "", "", "external(SB)", ""}, + {"unbound-segment", "", "", "(FS)", ""}, + {"unknown-store", "", "", "(DX)", "MOVL BX,(DX)\n"}, + {"no-future-pointer", "", "", "(DX)", "MOVL (DX),DX\n" + op + " (DX),X1\n"}, + } { + t.Run(arch+"/"+op+"/"+row.name, func(t *testing.T) { + source := x86COMINativeInputStatusProjection(arch, "view", op, row.setup, row.address, row.tail) + if row.remove != "" { + source = strings.Replace(source, row.remove, "", 1) + } + paired := row.name == "numeric-base" || row.name == "no-future-pointer" + if paired { + x86RequireNumericNativeSourceContext(t, source, "var external [8]byte\nfunc view(p *byte) uintptr\n", arch) + x86LEAActualTargetGoObject(t, arch, triple, source) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "var external [8]byte\nfunc view(p *byte) uintptr\n", arch, triple, api) + if paired { + width, count := 4, 1 + if strings.HasSuffix(op, "D") { + width = 8 + } + if row.name == "no-future-pointer" { + count = 2 + } + if err != nil || strings.Count(ir, fmt.Sprintf("elementtype([%d x i8])", width)) != count || + !strings.Contains(ir, strings.ToLower(op)+" ") || !strings.Contains(ir, "~{fpsr}") || + row.name == "no-future-pointer" && !strings.Contains(ir, "load volatile i32,") { + t.Fatal("independently paired COMI reads lost width, opcode or FP effects", api, err) + } + x86NumericNativeAddressObject(t, ir, triple) + continue + } + if arch == "amd64" && row.name == "vector-input" { + x86VectorScratchCodegenControl(t, source, "var external [8]byte\nfunc view(p *byte) uintptr\n", arch, triple, ir, err) + continue + } + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native comparison supplied missing input or storage authority", api, err) + } + } + }) + } + } + } +} + +func TestX86COMINativeStatusDoesNotInventDFOrID(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, op := range x86COMIOracleNames() { + source := x86COMINativeInputSource(target.arch, "view", op, "", "-7(DX)(CX*4)", "") + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *byte) uintptr\n", target.arch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native status query invented full FLAGS including unknown DF/ID", target, op, api, err) + } + } + } + } +} + +func TestX86COMINativeReadPreservesUnrelatedState(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86COMIOracleNames() { + for missing := -1; missing < 64; missing++ { + ins := x86ScalarLaneInstruction(t, op+" -7(DX)(CX*4),X1") + state := x86ResultState{direction: x86ResultDirectionBackward, + valueStack: &x86ResultValueStack{bytes: []uint8{1, 2}, id: true}} + state.gp[1], state.pointerGP[2], state.masks[7] = 0xff, true, true + state.callerAddressGP[2], state.flags = true, x86ByteFlagCF + bytes := ^uint64(0) + if missing >= 0 { + bytes &^= 1 << uint(missing) + } + state.writeVector(1, 64, bytes, false) + before := state.clone() + if accepted, known := x86ResultCOMIStep(ins, &state, nil, arch); !known || accepted || !reflect.DeepEqual(state, before) { + t.Fatal("standalone observer borrowed a conditional native read", arch, op, missing) + } + width := 4 + if strings.HasSuffix(op, "D") { + width = 8 + } + want := missing < 0 || missing >= width + accepted, known := x86ResultCOMIStepWithRead(ins, &state, nil, arch, true) + if want { + before.flags = x86ByteFlagsAll + } + if !known || accepted != want || !reflect.DeepEqual(state, before) { + t.Fatal("native continuation changed unrelated state or ignored a missing scalar byte", arch, op, missing) + } + } + } + } +} diff --git a/amd64_comi_review_test.go b/amd64_comi_review_test.go new file mode 100644 index 000000000..e53b0ab1f --- /dev/null +++ b/amd64_comi_review_test.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86COMIModeContextCannotHideMalformedSource(t *testing.T) { + for _, name := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, body := range []string{ + name + " X8,Y1\n", + name + " slot+0(FP)(AX*1),Y1\n", + name + " X8,X1\nRET\n" + name + " X0,Y1\n", + name + " X8,X1\nRET\nTEXT ·later(SB),4,$0-0\n" + name + " X0,Y1\n", + } { + for _, api := range []string{"text", "module", "Go"} { + t.Run(name+"/"+api+"/"+body, func(t *testing.T) { + _, err := x86ScalarFPBitViewAPI(t, "TEXT ·view(SB),4,$0-0\n"+body+"RET\n", + "func view()\nfunc later()\n", "386", "i386-unknown-linux-gnu", api) + if err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mode32 context hid malformed original source: %v", err) + } + }) + } + } + } +} + +func TestX86COMIDirectLowererRequiresNativeTarget(t *testing.T) { + for _, triple := range []string{"", "aarch64-unknown-linux-gnu", "i386-unknown-linux-gnu"} { + var output strings.Builder + ctx := amd64Ctx{b: &output, goarch: "amd64", targetTriple: triple} + instruction := x86ScalarLaneInstruction(t, "COMISS X0,X1") + known, terminated, err := ctx.lowerVectorScalarFlagCompare(instruction.Op, instruction) + if !known || terminated || !errors.Is(err, ErrProbeNeedsContext) || output.Len() != 0 { + t.Fatal("direct helper emitted without native target", triple, known, terminated, err, output.String()) + } + } +} + +func TestX86COMIRawVEXLengthIsNotPortableIgnoredBit(t *testing.T) { + for _, name := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+name+" X0,X1\nRET\n") + if len(code) != 5 || code[0] != 0xc5 || code[1]&4 != 0 { + t.Fatalf("actual Go must retain its VEX.L0 encoding: %s %x", name, code) + } + code[1] |= 4 + for _, mode := range []int{32, 64} { + if _, _, matched, err := decodedX86ScalarFlagCompareInstruction(code[:4], mode); !matched || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("raw VEX.L1 normalized to L0 without physical contract: %s mode%d %x %v", name, mode, code, err) + } + var source strings.Builder + source.WriteString("TEXT ·view(SB),4,$0-0\n") + for _, value := range code[:4] { + fmt.Fprintf(&source, "BYTE $%#x\n", value) + } + source.WriteString("RET\n") + arch, triple := "amd64", "x86_64-unknown-linux-gnu" + if mode == 32 { + arch, triple = "386", "i386-unknown-linux-gnu" + } + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source.String(), "func view()\n", arch, triple, api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("raw L1 lost context through public API", name, arch, api, err) + } + } + } + } +} + +func TestX86COMIRawEVEXGenerationSurvivesLowRegisters(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT ·view(SB),4,$0-0\n") + for _, opcode := range []int{0x2e, 0x2f} { + for _, double := range []bool{false, true} { + code := []byte{0x62, 0xf1, 0x7c, 0x08, byte(opcode), 0xc8} + if double { + code[2] |= 0x81 + } + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#x\n", value) + } + } + } + source.WriteString("RET\n") + for _, target := range x86RawReturnTargets { + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + if ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func view()\n", target.arch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original low-bank EVEX format invented its X inputs", err) + } + ir := x86COMIOriginalFormatObject(t, source.String(), target.arch, target.triple) + if strings.Count(ir, "{evex} v") != 4 || !strings.Contains(ir, "+avx512f") { + t.Fatal("low-register raw EVEX private format lost generation/feature", ir) + } + x86COMICompileOriginalFormatObjects(t, llc, target.triple, "raw-evex-comi", ir) + }) + } + } +} diff --git a/amd64_comi_runtime_linux_amd64_test.go b/amd64_comi_runtime_linux_amd64_test.go new file mode 100644 index 000000000..952da9004 --- /dev/null +++ b/amd64_comi_runtime_linux_amd64_test.go @@ -0,0 +1,16 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86COMINativeVectorFlags(t *testing.T) { + t.Run("VEX", func(t *testing.T) { + x86ScalarCompareAFOriginalGo(t, x86FloatCompareVEX) + x86ScalarCompareAFLLVM(t, x86FloatCompareVEX) + }) +} + +func TestCrossLinuxRuntimeMatrixX86COMINativeVectorMXCSR(t *testing.T) { + t.Run("VEX", func(t *testing.T) { + x86COMIOriginalConstantsAndMXCSR(t, x86FloatCompareVEX) + }) +} diff --git a/amd64_comi_runtime_owner_test.go b/amd64_comi_runtime_owner_test.go new file mode 100644 index 000000000..f44ff7ffa --- /dev/null +++ b/amd64_comi_runtime_owner_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "go/ast" + "go/build" + "go/parser" + "go/token" + "os" + "reflect" + "strconv" + "testing" +) + +func TestX86COMIRequiredNativeRuntimeOwners(t *testing.T) { + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil { + t.Fatal(err) + } + selector, err := x86FloatCompareLinuxCISelector(string(workflow)) + if err != nil { + t.Fatal(err) + } + for _, source := range []struct { + path string + tagged bool + owners map[string]struct { + calls map[string]string + axis string + } + }{ + {"amd64_comi_runtime_linux_amd64_test.go", false, map[string]struct { + calls map[string]string + axis string + }{ + "TestCrossLinuxRuntimeMatrixX86COMINativeVectorFlags": {map[string]string{ + "x86ScalarCompareAFOriginalGo": "x86FloatCompareVEX", "x86ScalarCompareAFLLVM": "x86FloatCompareVEX", + }, "VEX"}, + "TestCrossLinuxRuntimeMatrixX86COMINativeVectorMXCSR": {map[string]string{ + "x86COMIOriginalConstantsAndMXCSR": "x86FloatCompareVEX", + }, "VEX"}, + }}, + {"amd64_comi_native_avx512_linux_amd64_test.go", true, map[string]struct { + calls map[string]string + axis string + }{ + "TestCrossLinuxRuntimeMatrixX86COMIRawNativeCompare": {map[string]string{"x86COMIRawNativeCompare": ""}, ""}, + "TestCrossLinuxRuntimeMatrixX86COMINativeDeadExceptions": {map[string]string{"x86COMIDeadFaults": "true"}, ""}, + "TestCrossLinuxRuntimeMatrixX86COMINativeVectorFlagsEVEX": {map[string]string{ + "x86ScalarCompareAFOriginalGo": "x86FloatCompareEVEX", "x86ScalarCompareAFLLVM": "x86FloatCompareEVEX", + }, "EVEX"}, + "TestCrossLinuxRuntimeMatrixX86COMINativeVectorMXCSREVEX": {map[string]string{ + "x86COMIOriginalConstantsAndMXCSR": "x86FloatCompareEVEX", + }, "EVEX"}, + }}, + } { + for _, target := range []struct{ os, arch string }{ + {"linux", "amd64"}, {"linux", "386"}, {"linux", "arm64"}, + {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, + } { + for _, qualified := range []bool{false, true} { + ctx := build.Default + ctx.GOOS, ctx.GOARCH = target.os, target.arch + ctx.BuildTags = nil + if qualified { + ctx.BuildTags = []string{"plan9asm_native_avx512"} + } + matched, err := ctx.MatchFile(".", source.path) + want := target.os == "linux" && target.arch == "amd64" && (!source.tagged || qualified) + if err != nil || matched != want { + t.Fatal("COMI ISA ownership changed", source.path, target, qualified, matched, err) + } + } + } + file, err := parser.ParseFile(token.NewFileSet(), source.path, nil, 0) + if err != nil { + t.Fatal(err) + } + for _, declaration := range file.Decls { + fn, ok := declaration.(*ast.FuncDecl) + if !ok { + continue + } + expected, known := source.owners[fn.Name.Name] + if !known || !selector.MatchString(fn.Name.Name) { + t.Fatal("unexpected or unselected native owner", fn.Name.Name) + } + delete(source.owners, fn.Name.Name) + actual, runs := make(map[string]string), 0 + ast.Inspect(fn.Body, func(node ast.Node) bool { + switch node := node.(type) { + case *ast.IfStmt, *ast.ReturnStmt, *ast.ForStmt, *ast.RangeStmt, *ast.BranchStmt: + t.Fatal("native owner cannot conditionally skip or truncate its fixture", fn.Name.Name) + case *ast.CallExpr: + if method, ok := node.Fun.(*ast.SelectorExpr); ok { + receiver, owned := method.X.(*ast.Ident) + if !owned || receiver.Name != "t" || method.Sel.Name != "Run" || len(node.Args) != 2 { + t.Fatal("native owner cannot replace execution with skip/log helpers", fn.Name.Name) + } + literal, ok := node.Args[0].(*ast.BasicLit) + if !ok { + t.Fatal("native ISA child must be explicit", fn.Name.Name) + } + axis, err := strconv.Unquote(literal.Value) + if err != nil || axis != expected.axis { + t.Fatal("native ISA child changed", fn.Name.Name, axis, err) + } + runs++ + return true + } + callee, ok := node.Fun.(*ast.Ident) + if !ok { + return true + } + argument, known := expected.calls[callee.Name] + if !known || len(node.Args) != 1 && len(node.Args) != 2 { + t.Fatal("unknown native fixture call", callee.Name) + } + owner, ok := node.Args[0].(*ast.Ident) + if !ok || owner.Name != "t" { + t.Fatal("native fixture lost its test owner") + } + got := "" + if len(node.Args) == 2 { + value, ok := node.Args[1].(*ast.Ident) + if !ok { + t.Fatal("native fixture lost explicit native/ISA argument") + } + got = value.Name + } + if _, duplicate := actual[callee.Name]; duplicate || got != argument { + t.Fatal("duplicated or weakened native fixture", callee.Name, got) + } + actual[callee.Name] = got + } + return true + }) + if !reflect.DeepEqual(actual, expected.calls) || (runs == 1) != (expected.axis != "") || runs > 1 { + t.Fatal("incomplete native fixture dispatch", fn.Name.Name, actual, runs) + } + } + if len(source.owners) != 0 { + t.Fatal("native COMI execution owners missing", source.owners) + } + } +} diff --git a/amd64_comi_status_runtime_test.go b/amd64_comi_status_runtime_test.go new file mode 100644 index 000000000..e94f633bf --- /dev/null +++ b/amd64_comi_status_runtime_test.go @@ -0,0 +1,145 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86COMIConstantOracle(axis x86FloatCompareRuntimeISA) (assembly, declarations, original, driver string, functions int) { + var asm, decl, goCalls, cDecl, cCalls strings.Builder + index := 0 + for _, name := range []string{"COMISS", "UCOMISS", "COMISD", "UCOMISD"} { + if axis != x86FloatCompareSSE { + name = "V" + name + } + values, one := []uint64{0x7fc12345, 0x7f812345, 1}, uint64(0x3f800000) + if strings.HasSuffix(name, "D") { + values, one = []uint64{0x7ff8123456789abc, 0x7ff0123456789abc, 1}, 0x3ff0000000000000 + } + forms := []struct{ suffix, source, destination string }{{"", "X0", "X1"}} + if axis == x86FloatCompareEVEX { + forms = []struct{ suffix, source, destination string }{ + {"", "X16", "X17"}, {"", "X30", "X31"}, + {".SAE", "X0", "X1"}, {".SAE", "X16", "X17"}, + } + } + for _, form := range forms { + for _, bits := range values { + for _, dead := range []bool{false, true} { + function := fmt.Sprintf("comi%d", index) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$8-12\nSTMXCSR saved-4(SP)\nMOVQ out+0(FP),DI\n", function) + fmt.Fprintf(&asm, "MOVQ $%#x,AX\nMOVQ AX,X0\nMOVQ $%#x,AX\nMOVQ AX,X1\n", bits, one) + if form.source != "X0" { + fmt.Fprintf(&asm, "VMOVAPS X0,%s\nVMOVAPS X1,%s\n", form.source, form.destination) + } + fmt.Fprintf(&asm, "LDMXCSR csr+8(FP)\n%s%s %s,%s\n", name, form.suffix, form.source, form.destination) + if dead { + asm.WriteString("MOVQ $0x77,BX\n") + } else { + asm.WriteString("PUSHFQ\nPOPQ BX\nANDQ $0x8d5,BX\n") + } + asm.WriteString("MOVQ BX,(DI)\nSTMXCSR 8(DI)\nLDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&decl, "func %s(out *[4]uint64, csr uint32)\n", function) + fmt.Fprintf(&goCalls, "check(%d,%s)\n", index, function) + fmt.Fprintf(&cDecl, "extern void %s(uint64_t*,uint32_t);\n", function) + fmt.Fprintf(&cCalls, "check(%d,%s);\n", index, function) + index++ + } + } + } + } + original = `package main +import "fmt" +func check(index int, fn func(*[4]uint64,uint32)) { + for mode:=0; mode<8; mode++ { + out:=[4]uint64{0x1111111111111111,0x2222222222222222,0x3333333333333333,0x4444444444444444} + csr:=uint32(0x1f80)|uint32(mode&1)<<6|uint32(mode>>1&1)<<15|uint32(mode>>2)<<5 + fn(&out,csr) + fmt.Printf("%d %d %016x %016x %016x %016x\n",index,mode,out[0],out[1],out[2],out[3]) + } +} +` + decl.String() + "func main(){\n" + goCalls.String() + "}\n" + driver = `#include +#include +static void check(int index,void (*fn)(uint64_t*,uint32_t)) { + for(unsigned mode=0; mode<8; mode++) { + uint64_t out[]={UINT64_C(0x1111111111111111),UINT64_C(0x2222222222222222),UINT64_C(0x3333333333333333),UINT64_C(0x4444444444444444)}; + uint32_t csr=0x1f80|((mode&1)<<6)|((mode>>1&1)<<15)|((mode>>2)<<5); + fn(out,csr); + printf("%d %u %016llx %016llx %016llx %016llx\n",index,mode,(unsigned long long)out[0],(unsigned long long)out[1],(unsigned long long)out[2],(unsigned long long)out[3]); + } +} +` + cDecl.String() + "int main(void){\n" + cCalls.String() + "return 0;}\n" + return asm.String(), decl.String(), original, driver, index +} + +func TestCrossLinuxRuntimeMatrixX86COMIOriginalConstantsAndMXCSR(t *testing.T) { + x86COMIOriginalConstantsAndMXCSR(t, x86FloatCompareSSE) +} + +func TestX86COMICompleteStatusOriginalInventory(t *testing.T) { + for _, axis := range []x86FloatCompareRuntimeISA{x86FloatCompareSSE, x86FloatCompareVEX, x86FloatCompareEVEX} { + assembly, _, _, _, functions := x86COMIConstantOracle(axis) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + want := 24 + if axis == x86FloatCompareEVEX { + want = 96 + } + if functions != want { + t.Fatal("changed constant status denominator", axis, functions) + } + } +} + +func x86COMIOriginalConstantsAndMXCSR(t *testing.T, axis x86FloatCompareRuntimeISA) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, axis) + assembly, declarations, main, driver, count := x86COMIConstantOracle(axis) + original := x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, main) + want := strings.Split(strings.TrimSpace(string(original)), "\n") + expected := 24 + if axis == x86FloatCompareEVEX { + expected = 96 + } + if count != expected || len(want) != expected*8 { + t.Fatalf("original Go produced %d functions/%d rows, want %d functions x 8 MXCSR controls", count, len(want), expected) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, assembly, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, ir) + } + for _, backend := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR_O2=%v/backend%s", optimized, backend), func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "comi-status", input, driver, backend) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM rows=%d original=%d", len(got), len(want)) + } + mismatches := 0 + for index := range want { + if got[index] != want[index] { + if mismatches == 0 { + t.Logf("first mismatch: original %s; LLVM %s", want[index], got[index]) + } + mismatches++ + } + } + if mismatches != 0 { + t.Errorf("original Go FLAGS/MXCSR/canary mismatches=%d/%d", mismatches, len(want)) + } + }) + } + } + }) + } +} diff --git a/amd64_compact_native_memory.go b/amd64_compact_native_memory.go new file mode 100644 index 000000000..89c4039ee --- /dev/null +++ b/amd64_compact_native_memory.go @@ -0,0 +1,90 @@ +package plan9asm + +import "fmt" + +// Native compact arrays advance only for enabled lanes. Ordinary masked LLVM +// intrinsics may discard a dead read; guarded volatile accesses retain the +// source instruction's enabled access/fault, even when its vector is unused. +// No inbounds, alignment, extent or allocation assumption follows from a GEP. +func (c *amd64Ctx) lowerNativeCompactMemory(ins Instr, form x86PackedCompactForm) error { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return fmt.Errorf("%w: compact memory needs its original native address size", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(ins.Args[form.memory]) + if err != nil { + return err + } + width, bits := form.byteWidth, form.laneBits + if form.mask < 0 { + if form.expand { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, %s %s, align 1\n", value, width, pointerType, pointer) + return c.storePackedMoveOperand(ins.Args[form.destination], width, "%"+value) + } + value, err := c.loadPackedCompareBytes(ins.Args[0], width) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store volatile <%d x i8> %s, %s %s, align 1\n", width, value, pointerType, pointer) + return nil + } + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + lanes := width * 8 / bits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, bits) + value := "zeroinitializer" + if !form.expand || !form.zeroing { + operand := ins.Args[0] + if form.expand { + operand = ins.Args[form.destination] + } + value, err = c.loadPackedCompareLanes(operand, width, bits, false) + if err != nil { + return err + } + } + offset := "0" + for lane := 0; lane < lanes; lane++ { + active := amd64MaskBitI1(c, mask, lane) + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %s\n", address, pointerType, pointer, offset) + if form.expand { + loaded, err := c.loadX86ScalarIf(bits, active, func() (string, error) { + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %%%s, align 1\n", loaded, bits, pointerType, address) + return "%" + loaded, nil + }) + if err != nil { + return err + } + if !form.zeroing { + old, selected := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", old, vectorType, value, lane) + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %s, i%d %%%s\n", selected, active, bits, loaded, bits, old) + loaded = "%" + selected + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, value, bits, loaded, lane) + value = "%" + inserted + } else { + element := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", element, vectorType, value, lane) + storeLabel, joinLabel := c.newTmp()+"_compact_store", c.newTmp()+"_compact_join" + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n%s:\n", active, storeLabel, joinLabel, storeLabel) + fmt.Fprintf(c.b, " store volatile i%d %%%s, %s %%%s, align 1\n", bits, element, pointerType, address) + fmt.Fprintf(c.b, " br label %%%s\n%s:\n", joinLabel, joinLabel) + } + step, next := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i64 %d, i64 0\n", step, active, bits/8) + fmt.Fprintf(c.b, " %%%s = add i64 %s, %%%s\n", next, offset, step) + offset = "%" + next + } + if form.expand { + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytes, vectorType, value, width) + return c.storePackedMoveOperand(ins.Args[form.destination], width, "%"+bytes) + } + return nil +} diff --git a/amd64_conditional_set_effects_test.go b/amd64_conditional_set_effects_test.go new file mode 100644 index 000000000..37a55626e --- /dev/null +++ b/amd64_conditional_set_effects_test.go @@ -0,0 +1,271 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86ConditionalSetTestNames = strings.Fields("SETCC SETCS SETEQ SETGE SETGT SETHI SETLE SETLS SETLT SETMI SETNE SETOC SETOS SETPC SETPL SETPS") + +func TestX86ConditionalSetActualGoTableAndOrderedForms(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`\{A(SET[A-Z]+), yscond, Pb, opBytes\{0x0f, (0x[0-9a-f]+), 00\}\}`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 16 || !strings.Contains(string(data), "var yscond = []ytab{\n\t{Zo_m, 2, argList{Ymb}},\n}") { + t.Fatalf("actual SDK SETcc row/class inventory changed: %d", len(rows)) + } + for _, row := range rows { + if _, registered := x86ConditionalSetConditions[Op(row[1])]; !registered { + t.Fatalf("actual Go row has no shared conditional byte form: %v", row) + } + } + for _, arch := range []string{"amd64", "386"} { + registers := "AX BX CX DX AH AL BH BL CH CL DH DL BP SI DI" + if arch == "amd64" { + registers += " SP BPB SIB DIB R8 R9 R10 R11 R12 R13 R14 R15 R8B R9B R10B R11B R12B R13B R14B R15B" + } + var source strings.Builder + source.WriteString("TEXT ·Value(SB),4,$0-1\n") + for _, op := range x86ConditionalSetTestNames { + for _, destination := range append(strings.Fields(registers), "(AX)", "7(AX)(CX*4)", "state(SB)", "ret0+0(FP)") { + line := op + " " + destination + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-1\n"+line+"\nRET") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := fmt.Sprintf("%#v", ins) + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); !recognized || err != nil { + t.Fatalf("%s %s shared row: %v", arch, line, err) + } + if !x86SourceStackInstructionKnown(ins, "MOVQ", "Q", arch) { + t.Fatalf("valid lowerer row is missing source-stack registration: %s", line) + } + if isAMD64MemoryOperand(ins.Args[0]) { + width, write, known := x86InstructionMemoryShape(ins, 0, arch) + if width != 1 || !write || !known || x86ScalarMoveMemoryBytes(ins, 0, arch) != 1 { + t.Fatalf("conditional byte memory shape: %s %d/%t/%t", line, width, write, known) + } + if ins.Args[0].Kind == OpFP { + if width, write, known := X86FPAccessBytes(ins, 0, arch); width != 1 || !write || !known { + t.Fatal("FP observation disagrees with the ordered form") + } + } + } + if fmt.Sprintf("%#v", ins) != before { + t.Fatal("typed form changed source identity") + } + source.WriteString(line + "\n") + } + } + source.WriteString("RET\n") + goAssemblerOracleOutput(t, arch, source.String(), "", true) + } +} + +func TestX86ConditionalSetRejectsMalformedAndConflictingIdentity(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []string{"SETNE", "SETNE $1", "SETNE X0", "SETNE M0", "SETNE K1", "SETNE AX,BX", "SETNE.Z AX", "SETNE (AX)(SP*1)"} { + source := "TEXT ·Value(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + ins := file.Funcs[0].Instrs[1] + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); !recognized || err == nil { + t.Fatalf("malformed source obtained a conditional byte effect: %s/%s", arch, row) + } + } + } + for _, identity := range []Op{"", "SETNE", "setne", "SETEQ", "SETNE.BAD"} { + var output strings.Builder + ctx := amd64Ctx{b: &output, goarch: "amd64"} + ins := Instr{Op: identity, Args: []Operand{{Kind: OpReg, Reg: AX}}} + // Success is tested at the grammar seam without relying on default + // LLVM flags. Contradictions must fail before even loading flags. + _, recognized, err := x86ConditionalSetProperties("SETNE", ins, "amd64") + valid := identity == "" || strings.EqualFold(string(identity), "SETNE") + if !recognized || (err == nil) != valid { + t.Fatalf("dispatch identity %q: %v", identity, err) + } + if !valid { + if _, _, err := ctx.lowerConditionalSet("SETNE", ins); err == nil || output.Len() != 0 { + t.Fatal("contradictory dispatch emitted IR") + } + } + } +} + +func TestX86ConditionalSetCallerConsumesOnlyDefinedFlagsAndExactByte(t *testing.T) { + for _, op := range x86ConditionalSetTestNames { + for _, destination := range []string{"AX", "AH", "(DX)"} { + body := "CMPL AX,$0\n" + op + " " + destination + if destination != "(DX)" { + body += "\nMOVB " + destination + ",(DX)" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + source := strings.NewReplacer("var res float32", "var res uint8", ") float32 {", ") uint8 {").Replace(x86CallerFixtureSource) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("%s byte admission: %v", body, err) + } + } + } + for name, body := range map[string]string{ + "unknown flags": "SETNE AX\nMOVB AX,(DX)", + "pointer flags": "CMPQ DX,$0\nSETNE AX\nMOVB AX,(DX)", + "partial pointer": "CMPL AX,$0\nSETNE DX\nMOVB AX,(DX)", + "input write": "CMPL AX,$0\nSETNE (DI)", + "unknown address": "CMPL AX,$0\nSETNE (BX)", + "wide result": "CMPL AX,$0\nSETNE 4(DX)", + "undefined upper": "CMPL AX,$0\nSETNE BX\nMOVL BX,(DX)", + "join flags": "CMPL AX,$0\nJE other\nADDQ $0,DX\nother:\nSETNE AX\nMOVB AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved flags/byte/address acquired continuation: %v", err) + } + }) + } + // A private-region pointer is never washed into a number by an eight-bit + // output. The exact AH bit mask preserves unrelated known bytes. + state := x86SourceStackState{numericFlags: true, known: map[Reg]uint8{AX: 1}, callers: map[Reg]x86CallerPointerValue{DX: {argument: 0}}} + ins := Instr{Op: "SETNE", Args: []Operand{{Kind: OpReg, Reg: AH}}} + if _, valid := x86CallerConditionalSetStep(&ins, &state, nil); !valid || state.known[AX] != 3 || !state.numericFlags { + t.Fatal("SETcc does not preserve exact byte knowledge and flags") + } + ins.Args[0].Reg = DX + if _, valid := x86CallerConditionalSetStep(&ins, &state, nil); valid || len(state.callers) != 1 { + t.Fatal("SETcc washed surviving pointer bytes") + } +} + +func TestX86ConditionalSetThreeAPIFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + word, move := 8, "MOVQ" + if target.arch == "386" { + word, move = 4, "MOVL" + } + var body strings.Builder + fmt.Fprintf(&body, "%s x+0(FP),SI\nMOVL $7,AX\nCMPL AX,$3\n", move) + for _, op := range x86ConditionalSetTestNames { + fmt.Fprintf(&body, "%s AX\n%s AH\n%s (SI)\n%s ret0+%d(FP)\n", op, op, op, op, word) + } + body.WriteString("RET") + pkg, file, sig, source := x86ResultInitializationFixture(t, target.arch, "func Value(x *uint8) uint8", body.String()) + goAssemblerOracleOutput(t, target.arch, source, "", true) + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Value": sig}, ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }} + text, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, text, "text") + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, module.String(), "module") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: options.ResolveSym}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, translated.Module.String(), "typed-go") + }) + } +} + +func TestX86ABI0OccupiedPointerAndSetRetainUnavailableInputs(t *testing.T) { + for name, body := range map[string]string{ + "pointer bytes": "MOVQ x+0(FP),AX\nMOVL AX,ret0+24(FP)\nRET", + "partial pointer bytes": "MOVB x+0(FP),AX\nMOVL AX,ret0+24(FP)\nRET", + "pointer dereference": "MOVQ x+0(FP),AX\nMOVL (AX),BX\nMOVL BX,ret0+24(FP)\nRET", + "unknown flags": "SETNE AX\nMOVB AX,ret0+24(FP)\nRET", + "unknown upper": "MOVL $1,BX\nCMPL BX,$0\nSETNE AX\nMOVL AX,ret0+24(FP)\nRET", + "missing result bytes": "MOVL $1,AX\nCMPL AX,$0\nSETNE ret0+24(FP)\nRET", + "flags join": "MOVL $1,AX\nCMPL AX,$0\nJE other\nSETNE ret0+24(FP)\nother:\nRET", + "call": "MOVL $1,ret0+24(FP)\nCALL ·other(SB)\nRET", + } { + t.Run(name, func(t *testing.T) { + _, file, sig, source := x86ResultInitializationFixture(t, "amd64", "func Value(x []uint32) uint32", body) + goAssemblerOracleOutput(t, "amd64", source, "", true) + before := fmt.Sprintf("%#v %#v", file, sig) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("occupied pointer or byte output manufactured input/result: %v", err) + } + if !reflect.DeepEqual(before, fmt.Sprintf("%#v %#v", file, sig)) { + t.Fatal("observer mutated source/layout") + } + }) + } + // Result initialization observes reachable execution, not unused source + // bytes. A complete result followed by an unconditional jump does not read + // the dead pointer input. Transporting an unused opaque pointer is also + // legal; consuming its bytes as a numeric result must still fail. + const dead = "MOVL $1,ret0+24(FP)\nJMP end\nMOVQ x+0(FP),AX\nend:\nRET" + for _, fixture := range []struct { + name, body string + context bool + }{ + {"dead pointer bytes", dead, false}, + {"reachable opaque pointer unused", strings.Replace(dead, "JMP end", "JMP input\ninput:", 1), false}, + {"reachable pointer result", strings.Replace(strings.Replace(dead, "JMP end", "JMP input\ninput:", 1), + "MOVQ x+0(FP),AX", "MOVQ x+0(FP),AX\nMOVL AX,ret0+24(FP)", 1), true}, + } { + t.Run(fixture.name, func(t *testing.T) { + _, file, sig, source := x86ResultInitializationFixture(t, "amd64", "func Value(x []uint32) uint32", fixture.body) + goAssemblerOracleOutput(t, "amd64", source, "", true) + before := fmt.Sprintf("%#v %#v", file, sig) + err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64") + if fixture.context && !errors.Is(err, ErrProbeNeedsContext) || !fixture.context && err != nil { + t.Fatalf("reachable result/input boundary changed: %v", err) + } + if before != fmt.Sprintf("%#v %#v", file, sig) { + t.Fatal("result observer changed source or manufactured pointer layout") + } + }) + } + // A derived pointer layout still cannot become a pointer result contract. + pkg := mustGoPackage(t, "example.invalid/pointer", "package pointer\nfunc Value(x *uint32) *uint32\n") + sig, err := DeriveGoFuncABI0Signature("Value", pkg.Types.Scope().Lookup("Value").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-16\nMOVQ x+0(FP),AX\nMOVQ AX,ret0+8(FP)\nRET") + if err != nil { + t.Fatal(err) + } + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("pointer result gained numeric result authority") + } +} diff --git a/amd64_conditional_set_forms.go b/amd64_conditional_set_forms.go new file mode 100644 index 000000000..1cff89b8b --- /dev/null +++ b/amd64_conditional_set_forms.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Both reviewed Go encoders use yscond: one Ymb destination, one byte +// written, no input data operand and no flags output. This grammar establishes +// neither initialized flags nor any authority for the destination address. +var x86ConditionalSetConditions = map[Op]string{ + "SETCC": "CC", "SETCS": "CS", "SETEQ": "EQ", "SETGE": "GE", + "SETGT": "GT", "SETHI": "HI", "SETLE": "LE", "SETLS": "LS", + "SETLT": "LT", "SETMI": "MI", "SETNE": "NE", "SETOC": "OC", + "SETOS": "OS", "SETPC": "PC", "SETPL": "PL", "SETPS": "PS", +} + +type x86ConditionalSetForm struct { + condition string + destination Operand +} + +func x86ConditionalSetProperties(op Op, ins Instr, arch string) (x86ConditionalSetForm, bool, error) { + var form x86ConditionalSetForm + name := strings.ToUpper(string(op)) + base := strings.SplitN(name, ".", 2)[0] + condition, recognized := x86ConditionalSetConditions[Op(base)] + if !recognized { + return form, false, nil + } + fail := func(reason string) (x86ConditionalSetForm, bool, error) { + return form, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + if arch != "amd64" && arch != "386" || name != base || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail("has an unbound mode, suffix or dispatch identity") + } + if len(ins.Args) != 1 { + return fail("expects one Ymb destination") + } + dst := ins.Args[0] + if dst.Kind == OpReg { + if !isGoYmbRegisterForArch(dst.Reg, arch) { + return fail("register is outside the Go Ymb class") + } + } else if !isAMD64MemoryOperand(dst) { + return fail("expects a Ymb register or memory destination") + } + if dst.Kind == OpMem { + memory := dst.Mem + if err := x86SymbolIndexedMemoryError(memory, arch); err != nil { + return form, true, err + } + if arch == "386" && strings.HasSuffix(memory.Sym, "(SB)") { + memory.Base = "" // The named-memory encoder discards this annotation. + } + if !x86MemoryRegistersValidForArch(memory, arch) || memory.Index == SP || + memory.Index != "" && memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8 || + memory.Off != int64(int32(memory.Off)) && !(arch == "386" && memory.Off == int64(uint32(memory.Off))) { + return fail("has an unencodable memory address") + } + } + form.condition, form.destination = condition, dst + return form, true, nil +} + +func x86CallerConditionalSetStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (bool, bool) { + form, recognized, err := x86ConditionalSetProperties(ins.Op, *ins, "amd64") + if !recognized || err != nil || !x86SourceStackNumericFlagsRead(state, x86ByteConditionFlags(form.condition)) { + return recognized, false + } + if form.destination.Kind == OpReg { + return true, x86CallerNumericWrite(form.destination.Reg, 1, state) + } + return true, x86CallerOrdinaryMemoryStep(ins, 0, state, proof, 1, true) +} + +// Only the paired lowering may continue an actual volatile native-byte store. +// FLAGS and incoming address identity must already exist; no logical memory, +// canonical result, caller envelope or pointer bound is produced here. +func x86LoweredNativeConditionalSet(ins Instr, state *x86ResultState, arch string) (bool, bool) { + form, known, err := x86ConditionalSetProperties(ins.Op, ins, arch) + if !known || err != nil || form.destination.Kind != OpMem || !x86OrdinaryNativeMemoryOperand(form.destination, arch) { + return false, false + } + required := x86ByteConditionFlags(form.condition) + return x86ResultNamedPredicateInstruction(ins) && ins.x86NumericalRead == nil && + required != 0 && state.flags&required == required && + x86ResultIncomingPointerAddressIdentity(form.destination, state, arch), true +} diff --git a/amd64_conditional_set_runtime_test.go b/amd64_conditional_set_runtime_test.go new file mode 100644 index 000000000..3a9c4ce6e --- /dev/null +++ b/amd64_conditional_set_runtime_test.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// This independent boolean reference uses architectural flags, not the +// translator's condition builder or the form map's condition strings. +const x86ConditionalSetIndependentC = ` +#include +#include +#include +static unsigned truth(unsigned condition,uint64_t flags) { + unsigned c=flags&1,p=flags>>2&1,z=flags>>6&1,s=flags>>7&1,o=flags>>11&1; + switch(condition) { + case 0:return !c;case 1:return c;case 2:return z;case 3:return s==o; + case 4:return !z&&s==o;case 5:return !c&&!z;case 6:return z||s!=o; + case 7:return c||z;case 8:return s!=o;case 9:return s;case 10:return !z; + case 11:return !o;case 12:return o;case 13:return !p;case 14:return !s;case 15:return p; + } return 2; +} +` + +func TestCrossLinuxRuntimeMatrixX86ConditionalSetAllFlagsAliasesAndCanaries(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, tests, c strings.Builder + declarations.WriteString("package conditional\n") + tests.WriteString(`package conditional +import("testing";"encoding/binary") +func TestOriginal(t *testing.T) { + for sample:=0;sample<64;sample++ { + flags:=uint64(2) + for bit,index:=range []uint{0,2,4,6,7,11} {if sample&(1< +#include +#include +#include +#include +#include +#include +#include +` + prototypes.String() + ` +typedef void (*Kernel)(const uint8_t *,uint64_t); +typedef struct { Kernel run; int bytes,lane,nf; } Row; +static const Row rows[] = { +` + metadata.String() + ` +}; +static int fault(Kernel run,const uint8_t *p,uint64_t mask) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + run(p,mask); + _exit(0); + } + int status=0; + if(waitpid(child,&status,0)!=child) return -2; + if(WIFEXITED(status)) return WEXITSTATUS(status)==0?0:-3; + if(WIFSIGNALED(status)) return WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS?1:-4; + return -5; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<64) return 1; + uint8_t *p=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(p==MAP_FAILED || mprotect(p+page,(size_t)page,PROT_NONE)) return 1; + for(long i=0;inf}, + {0,UINT64_C(1)<<63,row->nf}, + {p+page-row->bytes,UINT64_MAX,0}, + {p+page-row->bytes+1,UINT64_MAX,1}, + {p+page-row->lane,1,row->nf && row->bytes>row->lane}, + {p+page-row->lane+1,1,1}, + }; + for(unsigned c=0;crun,cases[c].p,cases[c].mask); + if(actual!=cases[c].want) { + fprintf(stderr,"dead access row=%u case=%u bytes=%d actual=%d want=%d\n",r,c,row->bytes,actual,cases[c].want); + return 1; + } + } + } + return munmap(p,(size_t)page*2)?1:0; +} +` + x86ConflictFunnelPortableRuntime(t, llc, clang, "conflict_funnel_dead_faults", triple, ir, mainC) +} diff --git a/amd64_conflict_funnel_forms.go b/amd64_conflict_funnel_forms.go new file mode 100644 index 000000000..9c418bfe3 --- /dev/null +++ b/amd64_conflict_funnel_forms.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedConflictForm struct { + x86PackedBinaryForm + spec amd64PackedConflictSpec +} + +// Unary _yvexpandpd has the same width/mask/address axes as the binary +// grammar. The local extra operand exists only while validating that shape; +// consumers retain the original source indexes and instruction identity. +func x86PackedConflictProperties(op Op, ins Instr, arch string) (x86PackedConflictForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64PackedConflictSpecs[Op(base)] + result := x86PackedConflictForm{spec: spec} + if !known { + return result, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || len(ins.Args) != 2 && len(ins.Args) != 3 { + return result, true, fmt.Errorf("%s %s has an unbound dispatch or unary operand count", arch, op) + } + local := ins + local.Op = op + destination := ins.Args[len(ins.Args)-1] + local.Args = []Operand{ins.Args[0], destination, destination} + if len(ins.Args) == 3 { + local.Args = []Operand{ins.Args[0], destination, ins.Args[1], destination} + } + var err error + result.x86PackedBinaryForm, err = x86ConflictFunnelOperandGrammar(local, arch, + x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, broadcastBytes: spec.laneBits / 8, zeroing: true}, true) + result.second, result.destination = -1, len(ins.Args)-1 + if result.mask >= 0 { + result.mask = 1 + } + return result, true, err +} + +type x86FunnelShiftForm struct { + x86PackedBinaryForm + spec amd64FunnelShiftSpec +} + +func x86FunnelShiftProperties(op Op, ins Instr, arch string) (x86FunnelShiftForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64FunnelShiftSpecs[Op(base)] + result := x86FunnelShiftForm{spec: spec} + if !known { + return result, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return result, true, fmt.Errorf("funnel-shift dispatch/source opcode mismatch") + } + local := ins + local.Op = op + offset := 0 + if !spec.variable { + if len(ins.Args) != 4 && len(ins.Args) != 5 || !amd64UnsignedImmediate(ins.Args[0], 8) || arch == "386" { + return result, true, fmt.Errorf("%s %s has an invalid imm8 or named frontend arity", arch, op) + } + offset = 1 + local.Args = ins.Args[1:] + } + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, zeroing: true} + if spec.laneBits != 16 { + grammar.broadcastBytes = spec.laneBits / 8 + } + var err error + result.x86PackedBinaryForm, err = x86ConflictFunnelOperandGrammar(local, arch, grammar, false) + result.first += offset + result.second += offset + result.destination += offset + if result.mask >= 0 { + result.mask += offset + } + if result.memory >= 0 { + result.memory += offset + } + return result, true, err +} + +func x86ConflictFunnelOperandGrammar(ins Instr, arch string, spec x86PackedBinarySpec, unary bool) (x86PackedBinaryForm, error) { + local, contextual := x86VectorValueMemoryInput(ins, arch) + form, err := x86PackedBinaryOperandGrammar(local, arch, spec, unary) + if err != nil { + return form, err + } + if err := x86Mode32VectorRegisterContext(ins, arch); err != nil { + return form, err + } + if form.memory >= 0 && !contextual && !x86PackedShiftMemoryValid(ins.Args[form.memory], arch) { + return form, fmt.Errorf("%s %s has an unresolved or unbound memory operand", arch, ins.Op) + } + if form.memory >= 0 { + source := ins.Args[form.memory] + if source.Kind == OpMem { + if !x86ConditionalMoveOffsetKnown(source.Mem) || source.Mem.IndexExt != "" || + source.Mem.Segment != "" && source.Mem.Segment != FS && source.Mem.Segment != GS { + return form, fmt.Errorf("%s %s has an unresolved or foreign address", arch, ins.Op) + } + if err := x86SymbolIndexedMemoryError(source.Mem, arch); err != nil { + return form, err + } + } + if source.Kind == OpSym { + if address, parseErr := parseInt(strings.TrimSpace(source.Sym)); parseErr == nil && + address != int64(int32(address)) && !(arch == "386" && address == int64(uint32(address))) { + return form, fmt.Errorf("%s %s absolute source exceeds its disp32 extent", arch, ins.Op) + } + } + } + if contextual { + return form, x86VectorValueMemoryContext(ins) + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return form, fmt.Errorf("%w: %s needs its original default address mode", ErrProbeNeedsContext, ins.Op) + } + return form, nil +} + +func (c *amd64Ctx) validateConflictFunnelMemory(ins Instr, form x86PackedBinaryForm) error { + if form.memory < 0 { + return nil + } + source := ins.Args[form.memory] + if source.Kind == OpMem && !c.nativeMemoryOperand(ins, form.memory) && + !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + return fmt.Errorf("%w: %s needs its original native storage binding", ErrProbeNeedsContext, ins.Op) + } + if source.Kind == OpFP { + return c.validateX86PackedShiftFPSpan(source.FPOffset, form.memoryBytes) + } + return nil +} + +// The typed operand description is shared by byte consumers, not a caller, +// native backing or raw-entry permission. Variable funnel destinations are +// numerical inputs even with EVEX.z; immediate results only merge old bytes. +func x86TypedConflictFunnelByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + var form x86PackedBinaryForm + var old, known bool + var err error + if parsed, recognized, failure := x86PackedConflictProperties(op, ins, arch); recognized { + form, known, err = parsed.x86PackedBinaryForm, true, failure + } else if parsed, recognized, failure := x86FunnelShiftProperties(op, ins, arch); recognized { + form, old, known, err = parsed.x86PackedBinaryForm, parsed.spec.variable, true, failure + } + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + return x86TypedFixedVectorInputs(ins, form, old), true, nil +} + +func x86TypedFixedVectorInputs(ins Instr, form x86PackedBinaryForm, old bool) x86TypedByteEffect { + width, firstWidth := int64(form.byteWidth), int64(form.byteWidth) + if form.memory >= 0 { + firstWidth = int64(form.memoryBytes) + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: firstWidth, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: width, + read: old || form.mask >= 0 && !form.zeroing, write: true}, + }} + if form.second >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.second, operand: ins.Args[form.second], width: width, read: true}) + } + if form.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return effect +} diff --git a/amd64_conflict_funnel_forms_test.go b/amd64_conflict_funnel_forms_test.go new file mode 100644 index 000000000..3f9f8c8b3 --- /dev/null +++ b/amd64_conflict_funnel_forms_test.go @@ -0,0 +1,303 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86ConflictFunnelCompleteIndependentGoRows(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + counts, total := make(map[string]int), 0 + for _, row := range x86ConflictFunnelRows() { + for _, rule := range rules { + if rule.Opcode != row.op { + continue + } + index := counts[row.op] + counts[row.op]++ + total++ + ytab, zcase := "_yvblendmpd", "Zevex_rm_v_r" + classes := []string{"YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", + "YymEvex,YyrEvex,YyrEvex", "YymEvex,YyrEvex,Yknot0,YyrEvex", "Yzm,Yzr,Yzr", "Yzm,Yzr,Yknot0,Yzr"} + if index%2 != 0 { + zcase = "Zevex_rm_v_k_r" + } + if row.unary { + ytab = "_yvexpandpd" + classes = []string{"YxmEvex,YxrEvex", "YxmEvex,Yknot0,YxrEvex", + "YymEvex,YyrEvex", "YymEvex,Yknot0,YyrEvex", "Yzm,Yzr", "Yzm,Yknot0,Yzr"} + if index%2 != 0 { + zcase = "Zevex_rm_k_r" + } + } else if !row.variable { + ytab, zcase = "_yvalignd", "Zevex_i_rm_v_r" + if index%2 != 0 { + zcase = "Zevex_i_rm_v_k_r" + } + for i := range classes { + classes[i] = "Yu8," + classes[i] + } + } + mapName, opcode, w := "evex0F38", "0xC4", 0 + if row.bits != 32 { + w = 1 + } + if !row.unary { + number := 0x70 + if row.bits != 16 { + number++ + } + if strings.HasPrefix(row.op, "VPSHR") { + number += 2 + } + opcode = fmt.Sprintf("0x%X", number) + if !row.variable { + mapName = "evex0F3A" + } + } + var attributes []string + for _, width := range []int{128, 256, 512} { + memory := fmt.Sprintf("evexN%d", width/8) + if row.bits != 16 { + memory += fmt.Sprintf(" | evexBcstN%d", row.bits/8) + } + memory += " | evexZeroingEnabled" + attributes = append(attributes, + fmt.Sprintf("avxEscape | evex%d | evex66 | %s | evexW%d", width, mapName, w), memory, opcode) + } + if index >= 6 || rule.RowIndex != index || rule.Ytab != ytab || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != index%2*3 || rule.EncodingOffset != index/2*3 || + strings.Join(rule.Classes, ",") != classes[index] || !reflect.DeepEqual(rule.OpBytes, attributes) { + t.Fatal("complete independent encoder row or opcode attributes changed", row.op, rule) + } + } + if counts[row.op] != 6 { + t.Fatal("incomplete independent family row inventory", row.op, counts[row.op]) + } + } + if len(counts) != 14 || total != 84 { + t.Fatal("complete conflict/funnel family changed", len(counts), total) + } +} + +func TestX86ConflictFunnelMaskAndOldDestinationRoles(t *testing.T) { + for _, row := range x86ConflictFunnelRows() { + for _, bank := range []string{"X", "Y", "Z"} { + for _, mode := range []string{"plain", "merge", "zero"} { + op, mask := row.op, "" + if mode != "plain" { + mask = "K1," + } + if mode == "zero" { + op += ".Z" + } + ins := x86ScalarLaneInstruction(t, x86ConflictFunnelInstruction(op, bank, bank+"0", mask, row.unary, row.variable)) + for _, missing := range []string{"none", "mask", "old"} { + state := x86ResultState{flags: x86ByteFlagsAll, direction: x86ResultDirectionForward} + state.writeVector(0, 64, ^uint64(0), false) + state.writeVector(1, 64, ^uint64(0), false) + state.masks[1] = missing != "mask" + if missing != "old" { + state.writeVector(2, 64, ^uint64(0), false) + } + before := state.clone() + want := !(missing == "mask" && mode != "plain" || missing == "old" && (row.variable || mode == "merge")) + if accepted := x86ResultTransfer(ins, &state, nil, 0, "amd64"); accepted != want { + t.Fatal("mask merge and old data inputs were conflated", row.op, bank, mode, missing, accepted, want) + } + if !want && !reflect.DeepEqual(state, before) { + t.Fatal("rejected input mutated reaching numerical state") + } + } + effect, known, err := x86TypedConflictFunnelByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || !effect.requiresResultProof { + t.Fatal("complete typed byte shape omitted an input obligation", row.op, err) + } + writes := 0 + for _, operand := range effect.operands { + if operand.write { + writes++ + if operand.read != (row.variable || mode == "merge") || operand.width != int64(amd64VectorByteWidth(Reg(bank+"0"))) { + t.Fatal("typed numerical destination role changed", row.op, mode, operand) + } + } + } + if writes != 1 { + t.Fatal("unexpected typed destination inventory", writes) + } + } + } + } +} + +func TestX86ConflictFunnelRejectsUnresolvedShapesBeforeIR(t *testing.T) { + for _, row := range x86ConflictFunnelRows() { + for _, mutation := range []string{"suffix", "offset", "extension", "segment", "index", "absolute", "dispatch", "float", "unresolved"} { + if row.unary || row.variable { + if mutation == "float" || mutation == "unresolved" { + continue + } + } + t.Run(row.op+"/"+mutation, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, x86ConflictFunnelInstruction(row.op, "X", "(AX)", "", row.unary, row.variable)) + source := 0 + if !row.unary && !row.variable { + source = 1 + } + op := ins.Op + switch mutation { + case "suffix": + op += ".SAE" + ins.Op = op + case "offset": + ins.Args[source].Mem.OffRaw = "UNKNOWN" + case "extension": + ins.Args[source].Mem.IndexExt = "UXTW" + case "segment": + ins.Args[source].Mem.Segment = "DS" + case "index": + ins.Args[source].Mem.Index, ins.Args[source].Mem.Scale = SP, 2 + case "absolute": + ins.Args[source] = Operand{Kind: OpSym, Sym: "4294967296"} + case "dispatch": + ins.Op = "VPXORD" + case "float": + ins.Args[0].ImmIsFloat = true + case "unresolved": + ins.Args[0].ImmRaw = "UNKNOWN" + } + var ir strings.Builder + consumer := &amd64Ctx{b: &ir, goarch: "amd64"} + var known bool + var err error + if row.unary { + known, _, err = consumer.lowerPackedConflict(op, ins) + } else { + known, _, err = consumer.lowerFunnelShift(op, ins) + } + if !known || err == nil || ir.Len() != 0 { + t.Fatal("invalid original shape reached IR or an unknown dispatch", known, err, ir.String()) + } + }) + } + } +} + +func TestX86ConflictFunnelNativeContinuationKeepsAddressAndOriginBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86ConflictFunnelRows() { + if arch == "386" && !row.unary && !row.variable { + continue + } + t.Run(arch+"/"+row.op, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, x86ConflictFunnelInstruction(row.op, "X", "-1(SI)(BX*2)", "", row.unary, row.variable)) + state := x86ResultState{flags: x86ByteFlagsAll, direction: x86ResultDirectionBackward} + state.writeVector(1, 64, ^uint64(0), false) + state.writeVector(2, 64, ^uint64(0), false) + state.pointerGP[6], state.gp[3] = true, 0xff + before := state.clone() + if accepted := x86ResultTransfer(ins, &state, nil, 0, arch); accepted || !reflect.DeepEqual(state, before) { + t.Fatal("standalone observer acquired a paired native read") + } + want := before.clone() + want.writeVector(2, 16, 0xffff, false) + accepted, known := x86LoweredFixedVectorNativeRead(ins, &state, nil, arch) + if !accepted || !known || !reflect.DeepEqual(state, want) { + t.Fatal("paired native read lost its exact output or changed independent state", accepted, known) + } + for _, mutation := range []string{"pointer", "index", "pointer-index", "raw", "encoded", "address", "segment", "raw-return"} { + input, changed := before.clone(), ins + switch mutation { + case "pointer": + input.pointerGP[6] = false + case "index": + input.gp[3] = 1 + case "pointer-index": + input.pointerGP[3], input.gp[3] = true, 0 + case "raw": + changed.x86SourceRaw = new(x86SourceRawInstruction) + case "encoded": + changed.x86Encoded = true + case "address": + changed.x86AddressBits = 32 + case "segment": + changed.x86SegmentPrefix = 0x64 + case "raw-return": + changed.x86RawNearReturn = new(x86RawNearReturnForm) + } + original := input.clone() + if accepted, _ := x86LoweredFixedVectorNativeRead(changed, &input, nil, arch); accepted || !reflect.DeepEqual(input, original) { + t.Fatal("unproved source or address acquired native result bytes", mutation) + } + } + }) + } + } +} + +func TestX86ConflictFunnelZeroMasksRetainDistinctNativeReads(t *testing.T) { + for _, target := range x86RawReturnTargets { + word, move := goWordSize(target.arch), "MOVQ" + if word == 4 { + move = "MOVL" + } + for _, row := range x86ConflictFunnelRows() { + if target.arch == "386" && !row.unary && !row.variable { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, broadcast := range []bool{false, true} { + if broadcast && row.bits == 16 { + continue + } + op, mask := row.op, "K1," + if broadcast { + op += ".BCST" + } + masked := target.arch != "386" || row.unary + if masked { + op += ".Z" + } else { + mask = "" + } + instruction := x86ConflictFunnelInstruction(op, bank, "(AX)", mask, row.unary, row.variable) + width := amd64VectorByteWidth(Reg(bank + "0")) + t.Run(target.triple+"/"+strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\n%s p+0(FP),AX\n"+ + "MOVL $0,BX\nKMOVD BX,K1\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\n%s\nRET\n", word, move, instruction) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[64]byte)", target.arch, target.triple, api) + if err != nil { + t.Fatal("complete dead native source rejected", api, err) + } + optimized := x86DwordProductIROptimize(t, ir) + if !row.unary && masked { + if !strings.Contains(ir, fmt.Sprintf("load volatile i%d", row.bits)) || strings.Contains(optimized, "load volatile") { + t.Fatal("E4 zero mask failed to suppress only its native lane reads", api) + } + } else { + bytes := width + if broadcast { + bytes = row.bits / 8 + } + load := fmt.Sprintf("load volatile <%d x i8>", bytes) + if strings.Count(ir, load) != 1 || strings.Count(optimized, load) != 1 { + t.Fatal("E4NF or unmasked source lost its exact complete tuple", api, load) + } + } + } + }) + } + } + } + } +} diff --git a/amd64_conflict_funnel_runtime_test.go b/amd64_conflict_funnel_runtime_test.go new file mode 100644 index 000000000..cd1bdfaec --- /dev/null +++ b/amd64_conflict_funnel_runtime_test.go @@ -0,0 +1,172 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// These are the unchanged original C numerical oracles, compiled for the +// actual host. They do not claim original AVX512 hardware execution. +func x86ConflictFunnelPortableRuntime(t *testing.T, llc, clang, name, triple, ir, mainC string) { + t.Helper() + for _, pipeline := range []struct { + name string + optimized bool + backend int + }{ + {name: "backend_O0"}, + {name: "backend_O2", backend: 2}, + {name: "IR_O2/backend_O2", optimized: true, backend: 2}, + } { + t.Run(pipeline.name, func(t *testing.T) { + text := ir + if pipeline.optimized { + text = x86DwordProductIROptimize(t, ir) + } + compileAndRunRuntimeTestWithCompiler(t, llc, []string{clang}, name, triple, + text, mainC, nil, pipeline.backend) + }) + } +} + +func x86ConflictFunnelWholeByteRuntime(t *testing.T, bank string) { + t.Helper() + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable whole-byte oracle requires its native 64-bit host counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 llc and clang required") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var source, declarations, prototypes, metadata strings.Builder + count := 0 + for _, row := range x86ConflictFunnelRows() { + for mode, suffix := range []string{"", "", ".Z"} { + name := fmt.Sprint("Whole", count) + mask := "" + if mode != 0 { + mask = "K1," + } + selector := 255 + if bank == "X" { + selector = 0 + } else if bank == "Y" { + selector = row.bits // The immediate wraps modulo the lane width. + } + instruction := x86ConflictFunnelInstruction(row.op+suffix, bank, bank+"0", mask, row.unary, row.variable) + instruction = strings.Replace(instruction, "$255,", fmt.Sprintf("$%d,", selector), 1) + fmt.Fprintf(&declarations, "func %s(out *[257]byte,a,b,old *[64]byte,mask uint64)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-40\n"+ + "MOVQ out+0(FP),DI\nMOVQ a+8(FP),AX\nMOVQ b+16(FP),BX\nMOVQ old+24(FP),DX\n"+ + "VMOVUPS (AX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (DX),Z2\nVMOVUPS (DX),Z3\n"+ + "KMOVQ mask+32(FP),K1\nSTC\n%s\n"+ + "VMOVUPS Z2,0(DI)\nVMOVUPS Z0,64(DI)\nVMOVUPS Z1,128(DI)\nVMOVUPS Z3,192(DI)\nSETCS 256(DI)\nRET\n", name, instruction) + fmt.Fprintf(&prototypes, "extern void %s(uint8_t *,const uint8_t *,const uint8_t *,const uint8_t *,uint64_t);\n", name) + kind, left := 0, 0 + if !row.unary { + kind = 1 + if row.variable { + kind = 2 + } + if strings.HasPrefix(row.op, "VPSHL") { + left = 1 + } + } + fmt.Fprintf(&metadata, "{%s,%d,%d,%d,%d,%d,%d},\n", name, kind, row.bits, + amd64VectorByteWidth(Reg(bank+"0")), left, mode, selector) + count++ + } + } + if count != 42 { + t.Fatal("incomplete independent whole-byte numerical union", count) + } + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source.String()) + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", triple, "text") + if err != nil { + t.Fatal("complete original whole-byte source rejected", err) + } + mainC := ` +#include +#include +#include +` + prototypes.String() + ` +typedef void (*Kernel)(uint8_t *,const uint8_t *,const uint8_t *,const uint8_t *,uint64_t); +typedef struct { Kernel run; int kind,bits,width,left,mode,selector; } Row; +static const Row rows[] = { +` + metadata.String() + ` +}; +static uint64_t read_lane(const uint8_t *p,int bytes) { + uint64_t value=0; + for(int i=0;i>(8*i)); +} +int main(void) { + static const uint64_t masks[]={0,UINT64_MAX,UINT64_C(0xaaaaaaaa55555555),UINT64_C(0x8000000000000000),1}; + for(unsigned r=0;rbits/8,lanes=row->width/bytes; + if(row->kind==0) { + for(int lane=0;lanekind==2) { + const uint64_t counts[]={0,1,(uint64_t)(row->bits-1),(uint64_t)row->bits,(uint64_t)(row->bits+1),UINT64_MAX}; + for(int lane=0;lanekind==0) { + for(int prior=0;priorkind==2?(unsigned)first:(unsigned)row->selector)&(row->bits-1); + uint64_t base=row->kind==2?previous:second,other=row->kind==2?second:first; + if(!count) result=base; + else if(row->left) result=(base<>(row->bits-count)); + else result=(base>>count)|(other<<(row->bits-count)); + } + if(row->mode && !((mask>>lane)&1)) result=row->mode==2?0:previous; + write_lane(want+lane*bytes,bytes,result); + } + rows[r].run(out+1,a,b,old,mask); + if(out[0]!=0xa7 || out[258]!=0xa7 || memcmp(out+1,want,sizeof(want))) { + fprintf(stderr,"whole-byte row=%u trial=%d bits=%d width=%d mode=%d\n",r,trial,row->bits,row->width,row->mode); + return 1; + } + } + } + return 0; +} +` + x86ConflictFunnelPortableRuntime(t, llc, clang, "conflict_funnel_whole_"+bank, triple, ir, mainC) +} + +func TestX86ConflictFunnelWholeXByteSemantics(t *testing.T) { + x86ConflictFunnelWholeByteRuntime(t, "X") +} + +func TestX86ConflictFunnelWholeYByteSemantics(t *testing.T) { + x86ConflictFunnelWholeByteRuntime(t, "Y") +} + +func TestX86ConflictFunnelWholeZByteSemantics(t *testing.T) { + x86ConflictFunnelWholeByteRuntime(t, "Z") +} diff --git a/amd64_conformance_axis_test.go b/amd64_conformance_axis_test.go new file mode 100644 index 000000000..b1c42af28 --- /dev/null +++ b/amd64_conformance_axis_test.go @@ -0,0 +1,36 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +func TestAMD64ConformanceNativeMoveAssertionsRetainRequiredAxis(t *testing.T) { + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("LLVM 22 clang is required") + } + for _, native := range []bool{false, true} { + t.Run(fmt.Sprint(native), func(t *testing.T) { + command := exec.Command(clang, "-E", "-P", "-x", "c", "-") + command.Stdin = strings.NewReader(amd64ConformanceRuntimeC(native)) + out, err := command.CombinedOutput() + if err != nil { + t.Fatalf("actual driver preprocessing: %v\n%s", err, out) + } + for _, call := range []string{ + "packedFloatMoveMaskSemantics(move_mask_out, move_src, move_old)", + "sameWidthPackedConversionSemantics(same_width_out,", + } { + if strings.Contains(string(out), call) != native { + t.Errorf("native=%v has wrong actual call selection for %s", native, call) + } + } + if !strings.Contains(string(out), "packedFloatMoveVEXSemantics(move_vex_out, move_src)") { + t.Error("portable register-move assertions were lost") + } + }) + } +} diff --git a/amd64_conformance_entry_closure_test.go b/amd64_conformance_entry_closure_test.go new file mode 100644 index 000000000..74ba2455c --- /dev/null +++ b/amd64_conformance_entry_closure_test.go @@ -0,0 +1,82 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "os" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// This directory checks each original public entry independently, not complete +// program execution. KeepFunc receives the complete original ASM and the same +// complete Go declarations on every request; no producer, DATA or signature is +// rewritten to bypass the first failure in the combined runtime translation. +func TestAMD64ConformanceEveryOriginalEntrySourceClosure(t *testing.T) { + source, err := os.ReadFile("testdata/conformance/amd64/conformance_amd64.s") + if err != nil { + t.Fatal(err) + } + declarations, err := os.ReadFile("testdata/conformance/amd64/conformance.go") + if err != nil { + t.Fatal(err) + } + if fmt.Sprintf("%x", sha256.Sum256(source)) != "3c105a49b705062c1ced2ecb492e5319e8d6836a8ca35a1cb276b9969f6ebc0e" || + fmt.Sprintf("%x", sha256.Sum256(declarations)) != "00bc95eb6d5cd5d1c976c8e077fa3f48f3c65bfd47815d7614b2866d2b7c41d2" { + t.Fatal("original complete conformance ASM or Go declarations changed") + } + file, err := Parse(ArchAMD64, string(source)) + if err != nil { + t.Fatal(err) + } + if len(file.Funcs) != 45 { + t.Fatal("complete original public entry directory changed", len(file.Funcs)) + } + pkg := mustGoPackage(t, "amd64conformance", string(declarations)) + resolve := func(symbol string) string { + return strings.TrimPrefix(goStripABISuffix(symbol), "·") + } + signatures, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil || len(signatures) != len(file.Funcs) { + t.Fatal("complete original Go signature directory differs", len(signatures), err) + } + // This existing stock-Go compiler prior includes both entire original + // files and runs before any translator request, without native execution. + if !t.Run("OriginalGoObjects", TestAMD64ConformanceOriginalWideSourceCompiles) { + t.Fatal("original complete Go program must compile before source-closure observations") + } + seen := make(map[string]bool) + for _, fn := range file.Funcs { + name := resolve(fn.Sym) + if seen[name] { + t.Fatal("duplicate original public entry", name) + } + seen[name] = true + t.Run(name, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + translated, err := translateGoModuleInContext(ctx, pkg, source, GoModuleOptions{ + FileName: "conformance_amd64.s", GOOS: "linux", GOARCH: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, + KeepFunc: func(_ string, resolved string) bool { return resolved == name }, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil || translated == nil || translated.Module.IsNil() { + t.Fatal("original initialized public entry lacks its own complete source closure", err) + } + if len(translated.Functions) != 1 || translated.Functions[0].TextSymbol != fn.Sym || + translated.Functions[0].ResolvedSymbol != name || + !reflect.DeepEqual(translated.Signatures[name], signatures[name]) { + t.Fatal("isolated entry changed its original public identity or Go signature") + } + if translated.Module.String() == "" { + t.Fatal("source closure returned no original public function") + } + }) + } +} diff --git a/amd64_conformance_native_test.go b/amd64_conformance_native_test.go new file mode 100644 index 000000000..1c32b5823 --- /dev/null +++ b/amd64_conformance_native_test.go @@ -0,0 +1,161 @@ +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" +) + +// The same complete first-party original Go/ASM bytes are compiled under the +// actual selected driver. This program is executed only after the mandatory +// native AVX512 preflight; the historical NativeGo test never called this TEXT. +func amd64ConformanceOriginalWideGo(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" || len(execution.prefix) != 0 || execution.preserveArgv0 { + t.Fatal("original wide conformance requires its actual native Linux AMD64 counterpart") + } + dir := amd64ConformanceOriginalWidePackage(t) + command := exec.Command("go", "test", "-p=2", "-count=1", "-run", "^TestOriginal(SameWidthPackedConversion|PackedFloatMoveMasks)$", "-v", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + t.Logf("actual original Go full-source wide conformance: %s", x86FloatCompareCaptureCommand(t, command)) +} + +func amd64ConformanceOriginalWidePackage(t *testing.T) string { + t.Helper() + dir := t.TempDir() + for _, name := range []string{"conformance.go", "conformance_amd64.s"} { + contents, err := os.ReadFile(filepath.Join("testdata", "conformance", "amd64", name)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), contents, 0600); err != nil { + t.Fatal(err) + } + } + for name, source := range map[string]string{ + "go.mod": "module nativewideconformance\n\ngo 1.20\n", + "wide_test.go": amd64ConformanceOriginalWideTest, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + return dir +} + +func TestAMD64ConformanceOriginalWideSourceCompiles(t *testing.T) { + dir := amd64ConformanceOriginalWidePackage(t) + command := exec.Command("go", "test", "-p=2", "-c", "-o", filepath.Join(dir, "wide"), ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + x86FloatCompareCaptureCommand(t, command) + t.Log("complete original Go/ASM and new original wide oracle compile for Linux AMD64; object is not executed on this portable axis") +} + +const amd64ConformanceOriginalWideTest = `package amd64conformance + +import ( + "encoding/binary" + "math" + "testing" + "unsafe" +) + +func TestOriginalPackedFloatMoveMasks(t *testing.T) { + aligned := func(size, phase int) ([]byte, []byte, int) { + storage := make([]byte, size+128) + for i := range storage { + storage[i] = 0xa7 + } + base := uintptr(unsafe.Pointer(&storage[0])) + offset := int((64-base%64)%64) + phase + return storage, storage[offset:offset+size], offset + } + _, src, _ := aligned(64, 0) + _, old, _ := aligned(64, 0) + storage, out, offset := aligned(688, 16) + for i := range src { + src[i] = byte(3 + 17*i) + old[i] = byte(0xf0 - 3*i) + } + packedFloatMoveMaskSemantics((*[688]byte)(out), (*[64]byte)(src), (*[64]byte)(old)) + + forms := []struct { offset, width, lane int; zero bool }{ + {0,16,4,false}, {16,16,4,true}, {32,16,4,false}, {48,16,4,false}, + {64,16,8,false}, {80,16,8,true}, {96,16,8,false}, + {112,32,4,false}, {144,32,4,true}, {176,32,4,false}, + {208,32,8,false}, {240,32,8,true}, {272,32,8,false}, + {304,64,4,false}, {368,64,4,true}, {432,64,4,false}, + {496,64,8,false}, {560,64,8,true}, {624,64,8,false}, + } + for _, form := range forms { + for i := 0; i < form.width; i++ { + want := old[i] + if (0xa55a >> (i / form.lane)) & 1 != 0 { + want = src[i] + } else if form.zero { + want = 0 + } + if got := out[form.offset+i]; got != want { + t.Fatalf("original move offset=%d byte=%d: got=%02x want=%02x", form.offset, i, got, want) + } + } + } + for i, value := range storage { + if (i < offset || i >= offset+len(out)) && value != 0xa7 { + t.Fatalf("original move output canary changed at %d", i) + } + } +} + +func TestOriginalSameWidthPackedConversion(t *testing.T) { + var ints [16]int32 + floats := [16]float32{ + 1.5,-1.5,2.75,-2.75,0.5,-0.5,3.25,-3.25, + 4.5,-4.5,5.75,-5.75,6.5,-6.5,7.25,-7.25, + } + var squares32 [16]float32 + var squares64 [8]float64 + for i:=0;i<16;i++ { + ints[i]=int32(i-8) + squares32[i]=float32((i+1)*(i+1)) + } + for i:=0;i<8;i++ { + squares64[i]=float64((i+1)*(i+1)) + } + nearest := [16]int32{2,-2,3,-3,0,0,3,-3,4,-4,6,-6,6,-6,7,-7} + truncated := [16]int32{1,-1,2,-2,0,0,3,-3,4,-4,5,-5,6,-6,7,-7} + down := [16]int32{1,-2,2,-3,0,-1,3,-4,4,-5,5,-6,6,-7,7,-8} + up := [16]int32{2,-1,3,-2,1,0,4,-3,5,-4,6,-5,7,-6,8,-7} + var guarded [578]byte + guarded[0],guarded[577]=0xa7,0xb9 + output:=guarded[1:577] + sameWidthPackedConversionSemantics((*[576]byte)(output),&ints,&floats,&squares32,&squares64) + if guarded[0]!=0xa7 || guarded[577]!=0xb9 { + t.Fatal("original wide conversion output canary changed") + } + single := func(offset int) float32 { + return math.Float32frombits(binary.LittleEndian.Uint32(output[offset:])) + } + integer := func(offset int) int32 { + return int32(binary.LittleEndian.Uint32(output[offset:])) + } + for i:=0;i<16;i++ { + if single(4*i)!=float32(ints[i]) || integer(64+4*i)!=nearest[i] || + integer(128+4*i)!=truncated[i] || single(192+4*i)!=float32(i+1) || + integer(320+4*i)!=nearest[i] || integer(384+4*i)!=down[i] || + integer(448+4*i)!=up[i] || integer(512+4*i)!=truncated[i] { + t.Fatalf("original wide conversion lane %d output=%x",i,output) + } + } + for i:=0;i<8;i++ { + value:=math.Float64frombits(binary.LittleEndian.Uint64(output[256+8*i:])) + if value!=float64(i+1) { + t.Fatalf("original wide sqrt lane %d value=%v",i,value) + } + } +} +` diff --git a/amd64_conformance_test.go b/amd64_conformance_test.go index f481eb5d6..13cad5a4c 100644 --- a/amd64_conformance_test.go +++ b/amd64_conformance_test.go @@ -1,8 +1,7 @@ -//go:build !llgo - package plan9asm import ( + "fmt" "os" "os/exec" "path/filepath" @@ -15,9 +14,9 @@ func TestAMD64ConformanceNativeGo(t *testing.T) { if runtime.GOARCH != "amd64" && !(runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable()) { t.Skip("native Go assembler oracle requires an amd64 host") } - cmd := exec.Command("go", "test", "./testdata/conformance/amd64") + cmd := exec.Command("go", "test", "-count=1", "./testdata/conformance/amd64") if runtime.GOARCH != "amd64" { - cmd.Env = append(os.Environ(), "GOARCH=amd64") + cmd.Env = append(os.Environ(), "GOARCH=amd64", "PLAN9ASM_ROSETTA=1") } if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("native Go conformance failed: %v\n%s", err, out) @@ -25,14 +24,11 @@ func TestAMD64ConformanceNativeGo(t *testing.T) { } func TestAMD64ConformanceLLVMRuntime(t *testing.T) { - crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() - if runtime.GOARCH != "amd64" && !crossRosetta { - t.Skip("runtime execution test only runs on amd64 host") - } - llc, clang, ok := findLlcAndClang(t) - if !ok { - t.Skip("llc/clang not found") - } + amd64ConformanceLLVMRuntime(t, x86FloatCompareRuntimeTools(t, x86FloatCompareSSE), false) +} + +func amd64ConformanceLLVMRuntime(t *testing.T, execution x86FloatCompareExecution, nativeWide bool) { + t.Helper() src, err := os.ReadFile(filepath.Join("testdata", "conformance", "amd64", "conformance_amd64.s")) if err != nil { @@ -47,12 +43,7 @@ func TestAMD64ConformanceLLVMRuntime(t *testing.T) { {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, }} - triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) - var runPrefix []string - if crossRosetta { - triple = "x86_64-apple-macosx" - runPrefix = []string{"/usr/bin/arch", "-x86_64"} - } + triple := execution.triple ll, err := Translate(file, Options{ TargetTriple: triple, ResolveSym: resolve, @@ -133,6 +124,238 @@ func TestAMD64ConformanceLLVMRuntime(t *testing.T) { {Offset: 24, Type: I64, Index: 3, Field: -1}, }}, }, + "scalarShiftRotateSemantics": { + Name: "scalarShiftRotateSemantics", + Args: []LLVMType{Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + "incDecSemantics": { + Name: "incDecSemantics", + Args: []LLVMType{Ptr, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + }}, + }, + "scalarAddSubSemantics": { + Name: "scalarAddSubSemantics", + Args: []LLVMType{Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + "compareExchangeScalarSemantics": { + Name: "compareExchangeScalarSemantics", + Args: []LLVMType{Ptr, Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }, + "compareExchangePairSemantics": { + Name: "compareExchangePairSemantics", + Args: []LLVMType{Ptr, Ptr, I64, I64, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }}, + }, + "parallelBitSemantics": { + Name: "parallelBitSemantics", + Args: []LLVMType{Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + "bitTestRegisterSemantics": { + Name: "bitTestRegisterSemantics", + Args: []LLVMType{Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + "bitTestMemorySemantics": { + Name: "bitTestMemorySemantics", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "packedWordMultiplyAddSemantics": { + Name: "packedWordMultiplyAddSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedUnsignedSignedByteMultiplyAddSemantics": { + Name: "packedUnsignedSignedByteMultiplyAddSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedDotProductSemantics": { + Name: "packedDotProductSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + }}, + }, + "packedIntegerMinMaxSemantics": { + Name: "packedIntegerMinMaxSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedWordMultiplySemantics": { + Name: "packedWordMultiplySemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedFloatShuffleSemantics": { + Name: "packedFloatShuffleSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "inLaneFloatingPermuteSemantics": { + Name: "inLaneFloatingPermuteSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "immediatePackedBlendSemantics": { + Name: "immediatePackedBlendSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedUnpackSemantics": { + Name: "packedUnpackSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "qwordPermuteSemantics": { + Name: "qwordPermuteSemantics", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "qwordVariablePermuteSemantics": { + Name: "qwordVariablePermuteSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedFloatMoveVEXSemantics": { + Name: "packedFloatMoveVEXSemantics", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "packedFloatMoveMaskSemantics": { + Name: "packedFloatMoveMaskSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedSignExtendMoveVEXSemantics": { + Name: "packedSignExtendMoveVEXSemantics", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "packedSignExtendMoveMaskSemantics": { + Name: "packedSignExtendMoveMaskSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, "goHexVectorOps": { Name: "goHexVectorOps", Args: []LLVMType{Ptr, Ptr, Ptr}, @@ -161,12 +384,141 @@ func TestAMD64ConformanceLLVMRuntime(t *testing.T) { Ret: Void, Frame: ptrParams, }, + "packedSubtractSemantics": { + Name: "packedSubtractSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "expandedEcosystemVectors": { + Name: "expandedEcosystemVectors", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "parityBranches": { + Name: "parityBranches", + Args: []LLVMType{I8}, + Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I8, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }, + }, + "unorderedBranch": { + Name: "unorderedBranch", + Args: []LLVMType{LLVMType("double")}, + Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: LLVMType("double"), Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }, + }, + "packedFloatToDwordModes": { + Name: "packedFloatToDwordModes", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "sameWidthPackedConversionSemantics": { + Name: "sameWidthPackedConversionSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }}, + }, + "packedSingleToDouble": { + Name: "packedSingleToDouble", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "conditionalMoveCodes": { + Name: "conditionalMoveCodes", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "fma3Semantics": { + Name: "fma3Semantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + }}, + }, + "binaryFloatingSemantics": { + Name: "binaryFloatingSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }}, + }, + "horizontalFloatingSemantics": { + Name: "horizontalFloatingSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }}, + }, }, }) if err != nil { t.Fatal(err) } - mainC := ` + // Numeric fixtures must return normally. Go-accepted EVEX zeroing memory + // stores raise #UD and belong in the separate exhaustive fault oracle, + // not in a fixture whose C driver expects an arithmetic result. + if strings.Contains(ll, `asm sideeffect "ud2"`) { + t.Fatal("numeric conformance fixture contains an architectural trap; preserve it in the separate expected-fault oracle") + } + // Every source function reaches the LLVM object on both execution axes. + // Only native aligned moves and wide conversions require AVX512 execution; + // their complete assertions also run against the original Go object there. + execution.compileAndRunLLVM(t, "amd64_conformance", ll, amd64ConformanceRuntimeC(nativeWide), "-O2") + t.Logf("complete original assembly object; native-wide runtime=%v, actual transport=%v", nativeWide, execution.prefix) +} + +func amd64ConformanceRuntimeC(nativeWide bool) string { + return fmt.Sprintf("#define PLAN9ASM_CONFORMANCE_NATIVE_WIDE %d\n", x86FloatAddSubBoolInt(nativeWide)) + ` #include extern void byteMemory(uint8_t *p, uint8_t value); extern void byteFlags(uint8_t *p, uint8_t *flags); @@ -176,9 +528,61 @@ extern uint32_t shiftLegacyThreeOperand(uint32_t src, uint32_t dst, uint32_t amo extern uint64_t clearTopBit(uint64_t value); extern void doubleShift32(uint32_t *out, uint32_t src, uint32_t dst, uint32_t amount); extern void doubleShift64(uint64_t *out, uint64_t src, uint64_t dst, uint64_t amount); +extern void scalarShiftRotateSemantics(uint64_t *out, uint64_t value, uint64_t count); +extern void incDecSemantics(uint64_t *out, uint64_t value); +extern void scalarAddSubSemantics(uint64_t *out, uint64_t value, uint64_t source); +extern void compareExchangeScalarSemantics(uint64_t *out, uint64_t *memory, uint64_t expected, uint64_t desired); +extern void compareExchangePairSemantics(uint64_t *out, uint64_t *memory, uint64_t expected_low, uint64_t expected_high, uint64_t desired_low, uint64_t desired_high); +extern void parallelBitSemantics(uint64_t *out, uint64_t mask, uint64_t source); +extern void bitTestRegisterSemantics(uint64_t *out, uint64_t value, uint64_t index); +extern void bitTestMemorySemantics(uint8_t *out, uint64_t *data); +extern void packedWordMultiplyAddSemantics(int32_t *out, int16_t *a, int16_t *b); +extern void packedUnsignedSignedByteMultiplyAddSemantics(int16_t *out, int8_t *signed_input, uint8_t *unsigned_input); +extern void packedDotProductSemantics(int32_t *out, int8_t *signed_bytes, uint8_t *unsigned_bytes, int16_t *words_a, int16_t *words_b, int32_t *accumulator); +extern void packedIntegerMinMaxSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void packedWordMultiplySemantics(uint16_t *out, int16_t *a, int16_t *b); +extern void packedFloatShuffleSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void inLaneFloatingPermuteSemantics(uint8_t *out, uint8_t *data, uint8_t *control); +extern void immediatePackedBlendSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void packedUnpackSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void qwordPermuteSemantics(uint64_t *out, uint64_t *data); +extern void qwordVariablePermuteSemantics(uint64_t *out, uint64_t *data, uint64_t *control); +extern void packedFloatMoveVEXSemantics(uint8_t *out, uint8_t *src); +extern void packedFloatMoveMaskSemantics(uint8_t *out, uint8_t *src, uint8_t *old); +extern void packedSignExtendMoveVEXSemantics(uint8_t *out, uint8_t *src); +extern void packedSignExtendMoveMaskSemantics(uint8_t *out, uint8_t *src, uint8_t *old); extern void goHexVectorOps(uint8_t *out, uint8_t *a, uint8_t *b); extern uint64_t goHexWordOps(uint64_t value, uint64_t count); extern void packedArithmeticShift32(int32_t *out, int32_t *src); +extern void packedSubtractSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void expandedEcosystemVectors(uint8_t *out, uint8_t *a, uint8_t *b); +extern uint64_t parityBranches(uint8_t value); +extern uint64_t unorderedBranch(double value); +extern void packedFloatToDwordModes(int32_t *out, float *src); +extern void sameWidthPackedConversionSemantics(uint8_t *out, int32_t *ints, float *floats, float *squares32, double *squares64); +extern void packedSingleToDouble(double *out, float *src); +extern void conditionalMoveCodes(uint64_t *out, uint64_t *src); +extern void fma3Semantics(uint8_t *out, double *a64, double *b64, double *c64, float *a32, float *b32, float *c32); +extern void binaryFloatingSemantics(uint8_t *out, double *a64, double *b64, float *a32, float *b32); +extern void horizontalFloatingSemantics(uint8_t *out, float *a32, float *b32, double *a64, double *b64); + +static uint16_t load16(const uint8_t *p) { + return (uint16_t)p[0] | (uint16_t)p[1] << 8; +} +static uint32_t load32(const uint8_t *p) { + return (uint32_t)load16(p) | (uint32_t)load16(p + 2) << 16; +} +static uint64_t load64(const uint8_t *p) { + return (uint64_t)load32(p) | (uint64_t)load32(p + 4) << 32; +} +static float loadf32(const uint8_t *p) { + union { uint32_t bits; float value; } converted = {load32(p)}; + return converted.value; +} +static double loadf64(const uint8_t *p) { + union { uint64_t bits; double value; } converted = {load64(p)}; + return converted.value; +} static uint32_t shld32(uint32_t src, uint32_t dst, uint32_t count) { count &= 31; @@ -196,6 +600,77 @@ static uint64_t shrd64(uint64_t src, uint64_t dst, uint64_t count) { count &= 63; return count == 0 ? dst : (dst >> count) | (src << (64 - count)); } +static uint64_t rol(uint64_t value, unsigned bits, uint64_t count) { + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + count %= bits; + value &= mask; + return count == 0 ? value : ((value << count) | (value >> (bits - count))) & mask; +} +static uint64_t ror(uint64_t value, unsigned bits, uint64_t count) { + return rol(value, bits, bits - count % bits); +} +static uint64_t replace_low(uint64_t value, uint64_t low, unsigned bits) { + uint64_t mask = (UINT64_C(1) << bits) - 1; + return (value & ~mask) | (low & mask); +} + +static uint64_t reference_scalar_add_sub(int add, unsigned bits, uint64_t value, uint64_t source, + int register_destination, uint64_t *flags) { + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + uint64_t destination_low = value & mask, source_low = source & mask; + uint64_t result_low = add ? (destination_low + source_low) & mask : (destination_low - source_low) & mask; + uint64_t carry = add ? result_low < destination_low : destination_low < source_low; + uint64_t sign_bit = UINT64_C(1) << (bits - 1); + uint64_t overflow = add + ? ((~(destination_low ^ source_low) & (destination_low ^ result_low) & sign_bit) != 0) + : (((destination_low ^ source_low) & (destination_low ^ result_low) & sign_bit) != 0); + uint8_t low_byte = (uint8_t)result_low; + uint64_t parity = 1; + while (low_byte != 0) { + parity ^= 1; + low_byte &= low_byte - 1; + } + *flags = carry | overflow << 8 | (result_low == 0) << 16 | + ((result_low & sign_bit) != 0) << 24 | parity << 32; + if (bits == 64 || (register_destination && bits == 32)) return result_low; + return (value & ~mask) | result_low; +} + +static uint64_t reference_pdep(uint64_t mask, uint64_t source) { + uint64_t result = 0; + while (mask != 0) { + uint64_t lowest = mask & (0 - mask); + if (source & 1) result |= lowest; + source >>= 1; + mask &= mask - 1; + } + return result; +} + +static uint64_t reference_pext(uint64_t mask, uint64_t source) { + uint64_t result = 0, output_bit = 1; + while (mask != 0) { + uint64_t lowest = mask & (0 - mask); + if (source & lowest) result |= output_bit; + output_bit <<= 1; + mask &= mask - 1; + } + return result; +} + +static uint64_t reference_bit_test_register(int operation, unsigned bits, uint64_t value, uint64_t index, uint64_t *carry) { + uint64_t bit = index & (bits - 1); + *carry = (value >> bit) & 1; + if (operation == 0) return value; + uint64_t mask = UINT64_C(1) << bit; + uint64_t low = value; + if (operation == 1) low ^= mask; + if (operation == 2) low &= ~mask; + if (operation == 3) low |= mask; + if (bits == 16) return (value & ~UINT64_C(0xffff)) | (low & UINT64_C(0xffff)); + if (bits == 32) return (uint32_t)low; + return low; +} int main(void) { uint8_t bytes[4] = {0x10, 0xf0, 0x5a, 0xa0}; @@ -265,6 +740,538 @@ int main(void) { return 50 + i; } } + const uint64_t scalar_value = UINT64_C(0x8123456789abcdef); + const uint64_t word_dst = UINT64_C(0xfedcba9876543210); + for (uint64_t count = 0; count < 16; count++) { + uint64_t out[14] = {0}, want[14] = {0}; + scalarShiftRotateSemantics(out, scalar_value, count); + uint64_t hardware_count = count & 31; + uint64_t byte_logical = hardware_count < 8 ? (uint8_t)scalar_value >> hardware_count : 0; + uint64_t byte_arithmetic = hardware_count < 8 ? (uint8_t)((int8_t)scalar_value >> hardware_count) : UINT8_MAX; + uint64_t word_count = count & 15; + uint16_t shldw = (uint16_t)word_dst, shrdw = (uint16_t)word_dst; + if (word_count != 0) { + shldw = (uint16_t)(((uint16_t)word_dst << word_count) | ((uint16_t)scalar_value >> (16 - word_count))); + shrdw = (uint16_t)(((uint16_t)word_dst >> word_count) | ((uint16_t)scalar_value << (16 - word_count))); + } + want[0] = replace_low(scalar_value, rol(scalar_value, 8, count), 8); + want[1] = replace_low(scalar_value, ror(scalar_value, 8, count), 8); + want[2] = replace_low(scalar_value, byte_arithmetic, 8); + want[3] = replace_low(scalar_value, byte_logical, 8); + want[4] = replace_low(scalar_value, rol(scalar_value, 16, count), 16); + want[5] = replace_low(scalar_value, (uint16_t)((int16_t)scalar_value >> hardware_count), 16); + want[6] = (uint32_t)scalar_value << hardware_count; + want[7] = ror(scalar_value, 64, count); + want[8] = replace_low(scalar_value, byte_logical, 8); + want[9] = replace_low(scalar_value, rol(scalar_value, 16, count), 16); + want[10] = replace_low(word_dst, shldw, 16); + want[11] = replace_low(word_dst, shrdw, 16); + want[12] = replace_low(scalar_value, (uint8_t)scalar_value << 3, 8); + want[13] = (uint32_t)((int32_t)scalar_value >> 3); + for (int i = 0; i < 14; i++) + if (out[i] != want[i]) return 196 + i; + } + const uint64_t inc_values[] = {0, 1, 0x7f, 0xff, 0xffff, 0xffffffff, UINT64_C(0x8123456789abcdef)}; + for (unsigned v = 0; v < sizeof(inc_values) / sizeof(inc_values[0]); v++) { + uint64_t value = inc_values[v], out[12] = {0}; + incDecSemantics(out, value); + uint64_t want[12] = { + replace_low(value, (uint8_t)value + 1, 8), + replace_low(value, (uint8_t)value - 1, 8), + replace_low(value, (uint16_t)value + 1, 16), + replace_low(value, (uint16_t)value - 1, 16), + (uint32_t)value + 1, (uint32_t)value - 1, value + 1, value - 1, + replace_low(value, (uint8_t)value + 1, 8), + replace_low(value, (uint16_t)value - 1, 16), + UINT64_C(0x0000000000010101), UINT64_C(0x0000000101000001), + }; + for (int i = 0; i < 12; i++) + if (out[i] != want[i]) return 220 + i; + } + const uint64_t scalar_add_sub_cases[][2] = { + {0, 0}, {UINT64_MAX, 1}, {0x7f, 1}, {0x7fff, 1}, + {0x7fffffff, 1}, {UINT64_C(0x7fffffffffffffff), 1}, + {UINT64_C(0x8123456789abcdef), UINT64_C(0xfedcba9876543210)}, + }; + const unsigned scalar_add_sub_widths[] = {8, 16, 32, 64}; + for (unsigned v = 0; v < sizeof(scalar_add_sub_cases) / sizeof(scalar_add_sub_cases[0]); v++) { + uint64_t value = scalar_add_sub_cases[v][0], source = scalar_add_sub_cases[v][1]; + uint64_t out[16] = {0}, want[16] = {0}; + scalarAddSubSemantics(out, value, source); + for (unsigned width_index = 0; width_index < 4; width_index++) { + unsigned bits = scalar_add_sub_widths[width_index]; + want[width_index * 4] = reference_scalar_add_sub(1, bits, value, source, 1, &want[width_index * 4 + 1]); + want[width_index * 4 + 2] = reference_scalar_add_sub(0, bits, value, UINT64_MAX, 0, &want[width_index * 4 + 3]); + } + for (int i = 0; i < 16; i++) + if (out[i] != want[i]) return 232 + i; + } + const uint64_t cmpxchg_expected_values[] = { + 0, UINT64_MAX, 0x7f, 0x7fff, 0x7fffffff, + UINT64_C(0x7fffffffffffffff), UINT64_C(0x8123456789abcdef), + }; + const uint64_t cmpxchg_desired = UINT64_C(0xfedcba9876543210); + for (unsigned v = 0; v < sizeof(cmpxchg_expected_values) / sizeof(cmpxchg_expected_values[0]); v++) { + uint64_t expected = cmpxchg_expected_values[v], memory[8] = {0}, initial[8] = {0}; + uint64_t out[24] = {0}, want[24] = {0}; + for (unsigned width_index = 0; width_index < 4; width_index++) { + unsigned bits = scalar_add_sub_widths[width_index]; + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + uint64_t base = UINT64_C(0xa5a5a5a5a5a5a5a5) & ~mask; + memory[width_index * 2] = base | (expected & mask); + memory[width_index * 2 + 1] = base | ((expected ^ (UINT64_C(1) << (bits - 1))) & mask); + initial[width_index * 2] = memory[width_index * 2]; + initial[width_index * 2 + 1] = memory[width_index * 2 + 1]; + } + compareExchangeScalarSemantics(out, memory, expected, cmpxchg_desired); + for (unsigned width_index = 0; width_index < 4; width_index++) { + unsigned bits = scalar_add_sub_widths[width_index]; + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + uint64_t success_initial = initial[width_index * 2]; + uint64_t failure_initial = initial[width_index * 2 + 1]; + unsigned output_index = width_index * 6; + want[output_index] = (success_initial & ~mask) | (cmpxchg_desired & mask); + want[output_index + 1] = expected; + reference_scalar_add_sub(0, bits, success_initial, expected, 1, &want[output_index + 2]); + want[output_index + 3] = failure_initial; + if (bits == 8 || bits == 16) want[output_index + 4] = (expected & ~mask) | (failure_initial & mask); + if (bits == 32) want[output_index + 4] = (uint32_t)failure_initial; + if (bits == 64) want[output_index + 4] = failure_initial; + reference_scalar_add_sub(0, bits, expected, failure_initial, 1, &want[output_index + 5]); + } + for (int i = 0; i < 24; i++) + if (out[i] != want[i]) return 248 + i; + } + const uint64_t cmpxchg_expected_low = UINT64_C(0xaaaaaaaa11223344); + const uint64_t cmpxchg_expected_high = UINT64_C(0xbbbbbbbb55667788); + const uint64_t cmpxchg_desired_low = UINT64_C(0x0123456789abcdef); + const uint64_t cmpxchg_desired_high = UINT64_C(0xfedcba9876543210); + for (int success = 0; success <= 1; success++) { + _Alignas(16) uint64_t memory[4] = {0}, out[9] = {0}, want[9] = {0}; + if (success) { + memory[0] = cmpxchg_expected_low; + memory[1] = cmpxchg_expected_high; + memory[2] = (cmpxchg_expected_low & UINT32_MAX) | (cmpxchg_expected_high & UINT32_MAX) << 32; + } else { + memory[0] = UINT64_C(0x1020304050607080); + memory[1] = UINT64_C(0x90a0b0c0d0e0f001); + memory[2] = UINT64_C(0x8877665544332211); + } + uint64_t old16_low = memory[0], old16_high = memory[1], old8 = memory[2]; + compareExchangePairSemantics(out, memory, cmpxchg_expected_low, cmpxchg_expected_high, + cmpxchg_desired_low, cmpxchg_desired_high); + if (success) { + want[0] = (cmpxchg_desired_low & UINT32_MAX) | (cmpxchg_desired_high & UINT32_MAX) << 32; + want[1] = cmpxchg_expected_low; + want[2] = cmpxchg_expected_high; + want[3] = 1; + want[4] = cmpxchg_desired_low; + want[5] = cmpxchg_desired_high; + want[6] = cmpxchg_expected_low; + want[7] = cmpxchg_expected_high; + want[8] = 1; + } else { + want[0] = old8; + want[1] = (uint32_t)old8; + want[2] = (uint32_t)(old8 >> 32); + want[4] = old16_low; + want[5] = old16_high; + want[6] = old16_low; + want[7] = old16_high; + } + for (int i = 0; i < 9; i++) + if (out[i] != want[i]) return 272 + i; + } + const uint64_t parallel_cases[][2] = { + {0, UINT64_C(0x0123456789abcdef)}, + {UINT64_MAX, UINT64_C(0x0123456789abcdef)}, + {UINT64_C(0xaaaaaaaaaaaaaaaa), UINT64_C(0x0123456789abcdef)}, + {UINT64_C(0x8040201008040201), UINT64_C(0xfedcba9876543210)}, + {UINT64_C(0x8000000100000081), UINT64_MAX}, + }; + for (unsigned v = 0; v < sizeof(parallel_cases) / sizeof(parallel_cases[0]); v++) { + uint64_t mask = parallel_cases[v][0], source = parallel_cases[v][1], out[8] = {0}; + parallelBitSemantics(out, mask, source); + uint64_t mask32 = (uint32_t)mask, source32 = (uint32_t)source; + uint64_t want[8] = { + reference_pdep(mask, source), reference_pdep(mask, source), + reference_pext(mask, source), reference_pext(mask, source), + reference_pdep(mask32, source32), reference_pdep(mask32, source32), + reference_pext(mask32, source32), reference_pext(mask32, source32), + }; + for (int i = 0; i < 8; i++) + if (out[i] != want[i]) return 124; + } + const uint64_t bit_test_value = UINT64_C(0xfedcba9876543210); + const uint64_t bit_test_index = 21; + uint64_t bit_test_out[24] = {0}; + bitTestRegisterSemantics(bit_test_out, bit_test_value, bit_test_index); + const unsigned bit_test_widths[3] = {16, 32, 64}; + for (int width_index = 0; width_index < 3; width_index++) { + unsigned bits = bit_test_widths[width_index]; + for (int operation = 0; operation < 4; operation++) { + uint64_t operand_index = bit_test_index; + if (operation == 0) operand_index = bits - 1; + if (operation == 2) operand_index = 4; + uint64_t carry = 0; + uint64_t result = reference_bit_test_register(operation, bits, bit_test_value, operand_index, &carry); + int output_index = 2 * (width_index * 4 + operation); + if (bit_test_out[output_index] != result || bit_test_out[output_index + 1] != carry) return 125; + } + } + uint64_t bit_test_data[9] = { + UINT64_MAX, UINT64_C(1) << 17, 0, + UINT64_MAX, UINT64_C(1) << 33, 0, + UINT64_MAX, 0, UINT64_C(1) << 1, + }; + uint8_t bit_test_carry[12] = {0}; + bitTestMemorySemantics(bit_test_carry, bit_test_data); + const uint8_t bit_test_want_carry[12] = {1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0}; + const uint64_t bit_test_want_data[9] = { + UINT64_C(0x7fffffffffffffff), 1, 0, + UINT64_C(0x7fffffffffffffff), 1, 0, + UINT64_C(0x7fffffffffffffff), 1, 0, + }; + for (int i = 0; i < 12; i++) + if (bit_test_carry[i] != bit_test_want_carry[i]) return 126; + for (int i = 0; i < 9; i++) + if (bit_test_data[i] != bit_test_want_data[i]) return 127; + int16_t word_madd_a[16] = { + INT16_MIN, INT16_MIN, -30000, -1, 0, 1, 12345, INT16_MAX, + 7, -11, 1000, -2000, 22222, -12345, INT16_MAX, INT16_MAX, + }; + int16_t word_madd_b[16] = { + INT16_MIN, INT16_MIN, -23456, INT16_MAX, -1, INT16_MAX, -2345, INT16_MAX, + -9, 13, -3000, 4000, -11111, 23456, INT16_MAX, INT16_MAX, + }; + int32_t word_madd_out[18] = {0}, word_madd_reference[8] = {0}, word_madd_want[18] = {0}; + for (int i = 0; i < 8; i++) { + int64_t sum = (int64_t)word_madd_a[2*i] * word_madd_b[2*i] + + (int64_t)word_madd_a[2*i+1] * word_madd_b[2*i+1]; + word_madd_reference[i] = (int32_t)(uint32_t)sum; + } + for (int i = 0; i < 2; i++) word_madd_want[i] = word_madd_reference[i]; + for (int i = 0; i < 4; i++) { + word_madd_want[2+i] = word_madd_reference[i]; + word_madd_want[6+i] = word_madd_reference[i]; + } + for (int i = 0; i < 8; i++) word_madd_want[10+i] = word_madd_reference[i]; + packedWordMultiplyAddSemantics(word_madd_out, word_madd_a, word_madd_b); + for (int i = 0; i < 18; i++) + if (word_madd_out[i] != word_madd_want[i]) return 128; + int8_t byte_madd_signed[32] = { + 127, 127, -128, -128, 100, -100, 1, -1, + 64, -64, 11, 12, -13, 14, 15, -16, + 31, -32, 33, -34, 35, -36, 37, -38, + 39, -40, 41, -42, 43, -44, 45, -46, + }; + uint8_t byte_madd_unsigned[32] = { + 255, 255, 255, 255, 200, 3, 255, 128, + 250, 249, 17, 19, 23, 29, 31, 37, + 41, 43, 47, 53, 59, 61, 67, 71, + 73, 79, 83, 89, 97, 101, 103, 107, + }; + int16_t byte_madd_out[32] = {0}, byte_madd_reference[16] = {0}, byte_madd_want[32] = {0}; + for (int i = 0; i < 16; i++) { + int32_t sum = (int32_t)byte_madd_signed[2*i] * byte_madd_unsigned[2*i] + + (int32_t)byte_madd_signed[2*i+1] * byte_madd_unsigned[2*i+1]; + if (sum > INT16_MAX) sum = INT16_MAX; + if (sum < INT16_MIN) sum = INT16_MIN; + byte_madd_reference[i] = (int16_t)sum; + } + for (int i = 0; i < 8; i++) { + byte_madd_want[i] = byte_madd_reference[i]; + byte_madd_want[8+i] = byte_madd_reference[i]; + } + for (int i = 0; i < 16; i++) byte_madd_want[16+i] = byte_madd_reference[i]; + packedUnsignedSignedByteMultiplyAddSemantics(byte_madd_out, byte_madd_signed, byte_madd_unsigned); + for (int i = 0; i < 32; i++) + if (byte_madd_out[i] != byte_madd_want[i]) return 129; + int8_t dot_signed_bytes[64]; + uint8_t dot_unsigned_bytes[64]; + int16_t dot_words_a[32], dot_words_b[32]; + int32_t dot_accumulator[16], dot_out[64] = {0}; + for (int i = 0; i < 64; i++) { + dot_signed_bytes[i] = (i / 4) & 1 ? -128 : 127; + dot_unsigned_bytes[i] = (uint8_t)(255 - (i & 3)); + } + for (int i = 0; i < 32; i++) { + dot_words_a[i] = (i / 2) & 1 ? INT16_MIN : INT16_MAX; + dot_words_b[i] = (i & 1) ? INT16_MAX : INT16_MIN; + } + for (int i = 0; i < 16; i++) { + switch (i & 3) { + case 0: dot_accumulator[i] = INT32_MAX - 10; break; + case 1: dot_accumulator[i] = INT32_MIN + 10; break; + case 2: dot_accumulator[i] = 123456789; break; + default: dot_accumulator[i] = -987654321; break; + } + } + packedDotProductSemantics(dot_out, dot_signed_bytes, dot_unsigned_bytes, dot_words_a, dot_words_b, dot_accumulator); + for (int lane = 0; lane < 16; lane++) { + int64_t byte_sum = dot_accumulator[lane]; + for (int item = 0; item < 4; item++) byte_sum += (int64_t)dot_signed_bytes[4*lane+item] * dot_unsigned_bytes[4*lane+item]; + int64_t byte_sat = byte_sum; + if (byte_sat > INT32_MAX) byte_sat = INT32_MAX; + if (byte_sat < INT32_MIN) byte_sat = INT32_MIN; + int64_t word_sum = dot_accumulator[lane]; + for (int item = 0; item < 2; item++) word_sum += (int64_t)dot_words_a[2*lane+item] * dot_words_b[2*lane+item]; + int64_t word_sat = word_sum; + if (word_sat > INT32_MAX) word_sat = INT32_MAX; + if (word_sat < INT32_MIN) word_sat = INT32_MIN; + if ((uint32_t)dot_out[lane] != (uint32_t)byte_sum) return 930 + lane; + if (dot_out[16+lane] != (int32_t)byte_sat) return 950 + lane; + if ((uint32_t)dot_out[32+lane] != (uint32_t)word_sum) return 970 + lane; + if (dot_out[48+lane] != (int32_t)word_sat) return 990 + lane; + } + uint8_t minmax_a[64], minmax_b[64], minmax_out[1024] = {0}; + for (int i = 0; i < 64; i++) { + minmax_a[i] = (uint8_t)(0x80 + 73*i); + minmax_b[i] = (uint8_t)(0xff - 29*i); + } + packedIntegerMinMaxSemantics(minmax_out, minmax_a, minmax_b); + for (int lane = 0; lane < 64; lane++) { + int8_t a = (int8_t)minmax_a[lane], b = (int8_t)minmax_b[lane]; + if ((int8_t)minmax_out[lane] != (a < b ? a : b)) return 1010 + lane; + if (minmax_out[64+lane] != (minmax_a[lane] < minmax_b[lane] ? minmax_a[lane] : minmax_b[lane])) return 1080 + lane; + if ((int8_t)minmax_out[128+lane] != (a > b ? a : b)) return 1150 + lane; + if (minmax_out[192+lane] != (minmax_a[lane] > minmax_b[lane] ? minmax_a[lane] : minmax_b[lane])) return 1220 + lane; + } + for (int lane = 0; lane < 32; lane++) { + uint16_t au = load16(minmax_a + 2*lane), bu = load16(minmax_b + 2*lane); + int16_t as = (int16_t)au, bs = (int16_t)bu; + if ((int16_t)load16(minmax_out + 256 + 2*lane) != (as < bs ? as : bs)) return 1290 + lane; + if (load16(minmax_out + 320 + 2*lane) != (au < bu ? au : bu)) return 1330 + lane; + if ((int16_t)load16(minmax_out + 384 + 2*lane) != (as > bs ? as : bs)) return 1370 + lane; + if (load16(minmax_out + 448 + 2*lane) != (au > bu ? au : bu)) return 1410 + lane; + } + for (int lane = 0; lane < 16; lane++) { + uint32_t au = load32(minmax_a + 4*lane), bu = load32(minmax_b + 4*lane); + int32_t as = (int32_t)au, bs = (int32_t)bu; + if ((int32_t)load32(minmax_out + 512 + 4*lane) != (as < bs ? as : bs)) return 1450 + lane; + if (load32(minmax_out + 576 + 4*lane) != (au < bu ? au : bu)) return 1470 + lane; + if ((int32_t)load32(minmax_out + 640 + 4*lane) != (as > bs ? as : bs)) return 1490 + lane; + if (load32(minmax_out + 704 + 4*lane) != (au > bu ? au : bu)) return 1510 + lane; + } + for (int lane = 0; lane < 8; lane++) { + uint64_t au = load64(minmax_a + 8*lane), bu = load64(minmax_b + 8*lane); + int64_t as = (int64_t)au, bs = (int64_t)bu; + if ((int64_t)load64(minmax_out + 768 + 8*lane) != (as < bs ? as : bs)) return 1530 + lane; + if (load64(minmax_out + 832 + 8*lane) != (au < bu ? au : bu)) return 1540 + lane; + if ((int64_t)load64(minmax_out + 896 + 8*lane) != (as > bs ? as : bs)) return 1550 + lane; + if (load64(minmax_out + 960 + 8*lane) != (au > bu ? au : bu)) return 1560 + lane; + } + int16_t word_a[8] = {INT16_MIN, INT16_MIN, -30000, -1, 0, 1, 12345, INT16_MAX}; + int16_t word_b[8] = {INT16_MIN, INT16_MAX, -23456, INT16_MAX, -1, INT16_MAX, -2345, INT16_MAX}; + uint16_t word_mul_out[64] = {0}, word_mul_want[64] = {0}; + packedWordMultiplySemantics(word_mul_out, word_a, word_b); + for (int i = 0; i < 8; i++) { + int32_t product = (int32_t)word_a[i] * (int32_t)word_b[i]; + uint32_t unsigned_product = (uint32_t)(uint16_t)word_a[i] * (uint32_t)(uint16_t)word_b[i]; + int32_t rounded = (product + 0x4000) >> 15; + uint16_t values[4] = {(uint16_t)product, (uint16_t)(product >> 16), (uint16_t)(unsigned_product >> 16), (uint16_t)rounded}; + for (int mode = 0; mode < 4; mode++) { + word_mul_want[mode*8+i] = values[mode]; + word_mul_want[(mode+4)*8+i] = values[mode]; + } + } + for (int i = 0; i < 64; i++) + if (word_mul_out[i] != word_mul_want[i]) return 240 + i; + uint32_t shuffle_a[8], shuffle_b[8]; + uint8_t shuffle_out[96] = {0}; + for (int i = 0; i < 8; i++) { + shuffle_a[i] = 0xa0 + i; + shuffle_b[i] = 0xb0 + i; + } + packedFloatShuffleSemantics(shuffle_out, (uint8_t *)shuffle_a, (uint8_t *)shuffle_b); + const uint32_t shuffle_ps[12] = { + 0xb3, 0xb2, 0xa1, 0xa0, + 0xb3, 0xb2, 0xa1, 0xa0, + 0xb7, 0xb6, 0xa5, 0xa4, + }; + for (int i = 0; i < 4; i++) + if (load32(shuffle_out + 4*i) != shuffle_ps[i]) return 310 + i; + if (load64(shuffle_out + 16) != UINT64_C(0x000000b1000000b0) || + load64(shuffle_out + 24) != UINT64_C(0x000000a3000000a2)) return 314; + for (int i = 0; i < 8; i++) + if (load32(shuffle_out + 32 + 4*i) != shuffle_ps[4+i]) return 315 + i; + const uint64_t shuffle_pd[4] = { + UINT64_C(0x000000b1000000b0), UINT64_C(0x000000a3000000a2), + UINT64_C(0x000000b5000000b4), UINT64_C(0x000000a7000000a6), + }; + for (int i = 0; i < 4; i++) + if (load64(shuffle_out + 64 + 8*i) != shuffle_pd[i]) return 323 + i; + uint32_t in_lane_data[8] = {0x100, 0x101, 0x102, 0x103, 0x104, 0x105, 0x106, 0x107}; + uint32_t in_lane_control[8] = {3, 2, 1, 0, 4, 5, 6, 7}; + uint8_t in_lane_out[144] = {0}; + inLaneFloatingPermuteSemantics(in_lane_out, (uint8_t *)in_lane_data, (uint8_t *)in_lane_control); + for (int lane = 0; lane < 4; lane++) + if (load32(in_lane_out + 4*lane) != in_lane_data[3-lane]) return 324; + for (int lane = 0; lane < 8; lane++) { + uint32_t immediate_value = in_lane_data[(lane/4)*4 + 3-(lane%4)]; + uint32_t variable_value = in_lane_data[(lane/4)*4 + (in_lane_control[lane]&3)]; + if (load32(in_lane_out + 16 + 4*lane) != immediate_value) return 325; + if (load32(in_lane_out + 48 + 4*lane) != variable_value) return 326; + } + for (int lane = 0; lane < 4; lane++) { + uint64_t data_q[4], control_q[4]; + for (int i = 0; i < 4; i++) { + data_q[i] = (uint64_t)in_lane_data[2*i] | (uint64_t)in_lane_data[2*i+1] << 32; + control_q[i] = (uint64_t)in_lane_control[2*i] | (uint64_t)in_lane_control[2*i+1] << 32; + } + uint64_t immediate_value = data_q[(lane/2)*2 + ((0x05 >> lane)&1)]; + uint64_t variable_value = data_q[(lane/2)*2 + ((control_q[lane] >> 1)&1)]; + if (load64(in_lane_out + 80 + 8*lane) != immediate_value) return 327; + if (load64(in_lane_out + 112 + 8*lane) != variable_value) return 328; + } + uint8_t blend_a[32], blend_b[32], blend_out[240] = {0}; + for (int i = 0; i < 32; i++) { + blend_a[i] = (uint8_t)(0x20 + i); + blend_b[i] = (uint8_t)(0xc0 + i); + } + immediatePackedBlendSemantics(blend_out, blend_a, blend_b); + const int blend_offsets[11] = {0, 16, 32, 48, 64, 96, 112, 144, 160, 192, 208}; + const int blend_widths[11] = {16, 16, 16, 16, 32, 16, 32, 16, 32, 16, 32}; + const int blend_lane_bytes[11] = {2, 4, 8, 2, 2, 4, 4, 4, 4, 8, 8}; + for (int block = 0; block < 11; block++) { + for (int i = 0; i < blend_widths[block]; i++) { + int bit = (i / blend_lane_bytes[block]) % 8; + uint8_t want = (0xa5 & (1 << bit)) ? blend_a[i] : blend_b[i]; + if (blend_out[blend_offsets[block] + i] != want) return 327 + block; + } + } + uint8_t unpack_out[384] = {0}; + packedUnpackSemantics(unpack_out, blend_a, blend_b); + const int unpack_offsets[16] = {0,16,32,48,64,80,96,112,128,160,192,224,256,288,320,352}; + const int unpack_widths[16] = {16,16,16,16,16,16,16,16,32,32,32,32,32,32,32,32}; + const int unpack_lane_bytes[16] = {1,1,2,2,4,4,8,8,1,1,2,2,4,4,8,8}; + for (int block = 0; block < 16; block++) { + int output = unpack_offsets[block]; + int high = block & 1; + for (int group = 0; group < unpack_widths[block]; group += 16) { + int lanes_per_group = 16 / unpack_lane_bytes[block]; + int start = high ? lanes_per_group / 2 : 0; + for (int lane = start; lane < start + lanes_per_group / 2; lane++) { + int input = group + lane * unpack_lane_bytes[block]; + for (int byte = 0; byte < unpack_lane_bytes[block]; byte++) { + if (unpack_out[output++] != blend_b[input + byte]) return 338 + block; + } + for (int byte = 0; byte < unpack_lane_bytes[block]; byte++) { + if (unpack_out[output++] != blend_a[input + byte]) return 354 + block; + } + } + } + } + uint64_t permute_data[4] = {0x10, 0x21, 0x32, 0x43}; + uint64_t permute_control[4] = {3, 0, 5, 2}; + uint64_t permute_out[8] = {0}, variable_permute_out[8] = {0}; + const uint64_t permute_immediate[4] = {0x10, 0x32, 0x21, 0x43}; + const uint64_t permute_variable[4] = {0x43, 0x10, 0x21, 0x32}; + qwordPermuteSemantics(permute_out, permute_data); + qwordVariablePermuteSemantics(variable_permute_out, permute_data, permute_control); + for (int block = 0; block < 2; block++) { + for (int lane = 0; lane < 4; lane++) + if (permute_out[4*block+lane] != permute_immediate[lane]) return 370 + block; + } + for (int block = 0; block < 2; block++) { + for (int lane = 0; lane < 4; lane++) + if (variable_permute_out[4*block+lane] != permute_variable[lane]) return 372 + block; + } + _Alignas(64) uint8_t move_src[64], move_old[64]; + uint8_t move_vex_out[128] = {0}; + for (int i = 0; i < 64; i++) { + move_src[i] = (uint8_t)(3 + 17*i); + move_old[i] = (uint8_t)(0xf0 - 3*i); + } + packedFloatMoveVEXSemantics(move_vex_out, move_src); + for (int block = 0; block < 4; block++) + for (int i = 0; i < 32; i++) + if (move_vex_out[32*block+i] != move_src[i]) return 374 + block; +#if PLAN9ASM_CONFORMANCE_NATIVE_WIDE + // X stores need 16-byte alignment. Z stores at offsets 432 and 624 + // need a base congruent to 16 modulo 64, not a 64-aligned base itself. + _Alignas(64) uint8_t move_mask_storage[768]; + for (int i = 0; i < 768; i++) move_mask_storage[i] = 0xa7; + uint8_t *move_mask_out = move_mask_storage + 16; + packedFloatMoveMaskSemantics(move_mask_out, move_src, move_old); + for (int i = 0; i < 16; i++) + if (move_mask_storage[i] != 0xa7) return 950; + for (int i = 704; i < 768; i++) + if (move_mask_storage[i] != 0xa7) return 951; + const int move_offsets[14] = {0,16,32,48,64,80,96,112,144,176,208,240,272,304}; + const int move_widths[14] = {16,16,16,16,16,16,16,32,32,32,32,32,32,64}; + const int move_lane_bytes[14] = {4,4,4,4,8,8,8,4,4,4,8,8,8,4}; + const int move_zero[14] = {0,1,0,0,0,1,0,0,1,0,0,1,0,0}; + for (int block = 0; block < 14; block++) { + for (int i = 0; i < move_widths[block]; i++) { + int active = (0xa55a >> (i / move_lane_bytes[block])) & 1; + uint8_t want = active ? move_src[i] : (move_zero[block] ? 0 : move_old[i]); + if (move_mask_out[move_offsets[block]+i] != want) return 378 + block; + } + } + const int move_z_offsets[5] = {368,432,496,560,624}; + const int move_z_lane_bytes[5] = {4,4,8,8,8}; + const int move_z_zero[5] = {1,0,0,1,0}; + for (int block = 0; block < 5; block++) { + for (int i = 0; i < 64; i++) { + int active = (0xa55a >> (i / move_z_lane_bytes[block])) & 1; + uint8_t want = active ? move_src[i] : (move_z_zero[block] ? 0 : move_old[i]); + if (move_mask_out[move_z_offsets[block]+i] != want) return 392 + block; + } + } +#endif + uint8_t extend_src[32] = { + 0x00,0x01,0x7f,0x80,0xff,0x55,0xaa,0x40, + 0xc0,0x11,0xee,0x33,0xcc,0x66,0x99,0x22, + 0xdd,0x44,0xbb,0x77,0x88,0x10,0xf0,0x20, + 0xe0,0x30,0xd0,0x50,0xb0,0x60,0xa0,0x70, + }; + uint8_t extend_vex_out[288] = {0}; + const int extend_input_bits[6] = {8,8,8,16,16,32}; + const int extend_output_bits[6] = {16,32,64,32,64,64}; + packedSignExtendMoveVEXSemantics(extend_vex_out, extend_src); + for (int op = 0; op < 6; op++) { + for (int form = 0; form < 2; form++) { + int width = form ? 32 : 16; + int offset = form ? 96 + 32*op : 16*op; + int lanes = width * 8 / extend_output_bits[op]; + for (int lane = 0; lane < lanes; lane++) { + const uint8_t *p = extend_src + lane * extend_input_bits[op] / 8; + int64_t expected = extend_input_bits[op] == 8 ? (int64_t)(int8_t)p[0] : + extend_input_bits[op] == 16 ? (int64_t)(int16_t)load16(p) : (int64_t)(int32_t)load32(p); + const uint8_t *actual = extend_vex_out + offset + lane * extend_output_bits[op] / 8; + uint64_t actual_bits = extend_output_bits[op] == 16 ? load16(actual) : + extend_output_bits[op] == 32 ? load32(actual) : load64(actual); + uint64_t mask = extend_output_bits[op] == 16 ? UINT64_C(0xffff) : + extend_output_bits[op] == 32 ? UINT64_C(0xffffffff) : UINT64_MAX; + if (actual_bits != ((uint64_t)expected & mask)) return 397 + op; + } + } + } + uint8_t extend_old[64], extend_mask_out[768] = {0}; + for (int i = 0; i < 64; i++) extend_old[i] = (uint8_t)(0xe7 - 5*i); + packedSignExtendMoveMaskSemantics(extend_mask_out, extend_src, extend_old); + for (int op = 0; op < 6; op++) { + int lanes = 512 / extend_output_bits[op]; + for (int zero = 0; zero < 2; zero++) { + int offset = 128*op + 64*zero; + for (int lane = 0; lane < lanes; lane++) { + const uint8_t *input = extend_src + lane * extend_input_bits[op] / 8; + int64_t extended = extend_input_bits[op] == 8 ? (int64_t)(int8_t)input[0] : + extend_input_bits[op] == 16 ? (int64_t)(int16_t)load16(input) : (int64_t)(int32_t)load32(input); + const uint8_t *actual = extend_mask_out + offset + lane * extend_output_bits[op] / 8; + uint64_t actual_bits = extend_output_bits[op] == 16 ? load16(actual) : + extend_output_bits[op] == 32 ? load32(actual) : load64(actual); + const uint8_t *old = extend_old + lane * extend_output_bits[op] / 8; + uint64_t old_bits = extend_output_bits[op] == 16 ? load16(old) : + extend_output_bits[op] == 32 ? load32(old) : load64(old); + uint64_t width_mask = extend_output_bits[op] == 16 ? UINT64_C(0xffff) : + extend_output_bits[op] == 32 ? UINT64_C(0xffffffff) : UINT64_MAX; + uint64_t expected = (UINT64_C(0xa55aa55a) >> lane) & 1 ? (uint64_t)extended & width_mask : (zero ? 0 : old_bits); + if (actual_bits != expected) return 403 + op; + } + } + } uint8_t a[16], b[16], vector_out[160] = {0}; const uint8_t or_mask[16] = { 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, @@ -310,10 +1317,321 @@ int main(void) { packedArithmeticShift32(packed_out, packed_src); for (int i = 0; i < 8; i++) if (packed_out[i] != packed_want[i]) return 80 + i; + + uint8_t subtract_a[16] = {0x7f, 0x80, 0x64, 0x9c, 0x32, 0xce, 0x00, 0x01, 0xff, 0x00, 0x34, 0x12, 0x00, 0x80, 0xff, 0x7f}; + uint8_t subtract_b[16] = {0xff, 0x01, 0x9c, 0x64, 0xce, 0x32, 0x01, 0x00, 0x01, 0xff, 0x78, 0x56, 0xff, 0x7f, 0x01, 0x80}; + uint8_t subtract_out[80] = {0}; + packedSubtractSemantics(subtract_out, subtract_a, subtract_b); + for (int i = 0; i < 16; i++) { + int signed_value = (int)(int8_t)subtract_a[i] - (int)(int8_t)subtract_b[i]; + if (signed_value < -128) signed_value = -128; + if (signed_value > 127) signed_value = 127; + if (subtract_out[i] != (uint8_t)(int8_t)signed_value) return 190; + int unsigned_value = (int)subtract_a[i] - (int)subtract_b[i]; + if (unsigned_value < 0) unsigned_value = 0; + if (subtract_out[32+i] != (uint8_t)unsigned_value) return 191; + } + for (int i = 0; i < 8; i++) { + int signed_value = (int)(int16_t)load16(subtract_a + 2*i) - (int)(int16_t)load16(subtract_b + 2*i); + if (signed_value < -32768) signed_value = -32768; + if (signed_value > 32767) signed_value = 32767; + if (load16(subtract_out + 16 + 2*i) != (uint16_t)(int16_t)signed_value) return 192; + int unsigned_value = (int)load16(subtract_a + 2*i) - (int)load16(subtract_b + 2*i); + if (unsigned_value < 0) unsigned_value = 0; + if (load16(subtract_out + 48 + 2*i) != (uint16_t)unsigned_value) return 193; + } + for (int i = 0; i < 2; i++) + if (load64(subtract_out + 64 + 8*i) != load64(subtract_a + 8*i) - load64(subtract_b + 8*i)) return 194; + + uint8_t ecosystem_a[16] = { + 0x00, 0x7f, 0x80, 0xff, 0x34, 0x12, 0xfe, 0xff, + 0x78, 0x56, 0x34, 0x12, 0x00, 0x00, 0x00, 0x80, + }; + uint8_t ecosystem_b[16] = { + 0xff, 0x80, 0x7f, 0x00, 0x78, 0x56, 0x02, 0x00, + 0xef, 0xcd, 0xab, 0x90, 0xff, 0xff, 0xff, 0x7f, + }; + uint8_t ecosystem_out[816] = {0}; + expandedEcosystemVectors(ecosystem_out, ecosystem_a, ecosystem_b); + for (int i = 0; i < 16; i++) { + uint8_t min_u = ecosystem_a[i] < ecosystem_b[i] ? ecosystem_a[i] : ecosystem_b[i]; + uint8_t max_u = ecosystem_a[i] > ecosystem_b[i] ? ecosystem_a[i] : ecosystem_b[i]; + int8_t as = (int8_t)ecosystem_a[i], bs = (int8_t)ecosystem_b[i]; + if (ecosystem_out[i] != min_u || ecosystem_out[16+i] != (uint8_t)(as < bs ? as : bs)) return 100; + if (ecosystem_out[96+i] != max_u || ecosystem_out[112+i] != (uint8_t)(as > bs ? as : bs)) return 101; + } + for (int i = 0; i < 8; i++) { + uint16_t au = load16(ecosystem_a + 2*i), bu = load16(ecosystem_b + 2*i); + int16_t as = (int16_t)au, bs = (int16_t)bu; + if (load16(ecosystem_out + 32 + 2*i) != (au < bu ? au : bu)) return 102; + if ((int16_t)load16(ecosystem_out + 48 + 2*i) != (as < bs ? as : bs)) return 103; + if (load16(ecosystem_out + 128 + 2*i) != (au > bu ? au : bu)) return 104; + if ((int16_t)load16(ecosystem_out + 144 + 2*i) != (as > bs ? as : bs)) return 105; + } + for (int i = 0; i < 4; i++) { + uint32_t au = load32(ecosystem_a + 4*i), bu = load32(ecosystem_b + 4*i); + int32_t as = (int32_t)au, bs = (int32_t)bu; + if (load32(ecosystem_out + 64 + 4*i) != (au < bu ? au : bu)) return 106; + if ((int32_t)load32(ecosystem_out + 80 + 4*i) != (as < bs ? as : bs)) return 107; + if (load32(ecosystem_out + 160 + 4*i) != (au > bu ? au : bu)) return 108; + if ((int32_t)load32(ecosystem_out + 176 + 4*i) != (as > bs ? as : bs)) return 109; + } + for (int i = 0; i < 2; i++) + if (load64(ecosystem_out + 192 + 8*i) != (load64(ecosystem_a + 8*i) << 13)) return 110; + for (int i = 208; i < 224; i++) + if (ecosystem_out[i] != 0) return 111; + if (load64(ecosystem_out + 224) != 0x0123456789abcdefULL) return 112; + if (load64(ecosystem_out + 232) != 0x0000000089abcdefULL) return 113; + if (load32(ecosystem_out + 240) != 0x89abcdefU) return 119; + for (int i = 244; i < 256; i++) if (ecosystem_out[i] != 0) return 121; + if (load64(ecosystem_out + 256) != load64(ecosystem_a)) return 122; + if (load64(ecosystem_out + 264) != 0) return 123; + for (int i = 0; i < 16; i++) + if (ecosystem_out[272+i] != (uint8_t)(((uint16_t)ecosystem_a[i] + ecosystem_b[i] + 1) >> 1)) return 124; + for (int i = 0; i < 8; i++) + if (load16(ecosystem_out + 288 + 2*i) != (uint16_t)(((uint32_t)load16(ecosystem_a + 2*i) + load16(ecosystem_b + 2*i) + 1) >> 1)) return 125; + for (int i = 0; i < 16; i++) + if (ecosystem_out[304+i] != ((int8_t)ecosystem_a[i] > (int8_t)ecosystem_b[i] ? 0xff : 0)) return 126; + for (int i = 0; i < 8; i++) + if (load16(ecosystem_out + 320 + 2*i) != ((int16_t)load16(ecosystem_a + 2*i) > (int16_t)load16(ecosystem_b + 2*i) ? 0xffff : 0)) return 127; + for (int i = 0; i < 4; i++) + if (load32(ecosystem_out + 336 + 4*i) != ((int32_t)load32(ecosystem_a + 4*i) > (int32_t)load32(ecosystem_b + 4*i) ? 0xffffffffU : 0)) return 128; + for (int i = 0; i < 2; i++) + if (load64(ecosystem_out + 352 + 8*i) != ((int64_t)load64(ecosystem_a + 8*i) > (int64_t)load64(ecosystem_b + 8*i) ? ~0ULL : 0)) return 129; + for (int i = 0; i < 8; i++) { + int16_t value = (int16_t)load16(ecosystem_a + 2*i); + if ((int16_t)load16(ecosystem_out + 368 + 2*i) != (int16_t)(value >> 3)) return 150; + if ((int16_t)load16(ecosystem_out + 384 + 2*i) != (int16_t)(value >> 15)) return 151; + } + for (int i = 0; i < 2; i++) { + if (load32(ecosystem_out + 400 + 8*i) != load32(ecosystem_a + 4*i)) return 152; + if (load32(ecosystem_out + 404 + 8*i) != load32(ecosystem_b + 4*i)) return 153; + if (load32(ecosystem_out + 416 + 8*i) != load32(ecosystem_a + 8 + 4*i)) return 154; + if (load32(ecosystem_out + 420 + 8*i) != load32(ecosystem_b + 8 + 4*i)) return 155; + } + if (load64(ecosystem_out + 432) != load64(ecosystem_a) || load64(ecosystem_out + 440) != load64(ecosystem_b)) return 156; + if (load64(ecosystem_out + 448) != load64(ecosystem_a + 8) || load64(ecosystem_out + 456) != load64(ecosystem_b + 8)) return 157; + for (int i = 0; i < 8; i++) if (load16(ecosystem_out + 464 + 2*i) != ecosystem_a[i]) return 158; + for (int i = 0; i < 4; i++) if (load32(ecosystem_out + 480 + 4*i) != ecosystem_a[i]) return 159; + for (int i = 0; i < 2; i++) if (load64(ecosystem_out + 496 + 8*i) != ecosystem_a[i]) return 160; + for (int i = 0; i < 4; i++) if (load32(ecosystem_out + 512 + 4*i) != load16(ecosystem_a + 2*i)) return 161; + for (int i = 0; i < 2; i++) if (load64(ecosystem_out + 528 + 8*i) != load16(ecosystem_a + 2*i)) return 162; + for (int i = 0; i < 2; i++) if (load64(ecosystem_out + 544 + 8*i) != load32(ecosystem_a + 4*i)) return 163; + for (int i = 0; i < 2; i++) { + if (load32(ecosystem_out + 560 + 4*i) != load32(ecosystem_a + 8*i) + load32(ecosystem_a + 8*i + 4)) return 164; + if (load32(ecosystem_out + 568 + 4*i) != load32(ecosystem_b + 8*i) + load32(ecosystem_b + 8*i + 4)) return 165; + if (load32(ecosystem_out + 608 + 4*i) != load32(ecosystem_a + 8*i) - load32(ecosystem_a + 8*i + 4)) return 166; + if (load32(ecosystem_out + 616 + 4*i) != load32(ecosystem_b + 8*i) - load32(ecosystem_b + 8*i + 4)) return 167; + } + for (int i = 0; i < 4; i++) { + int32_t aa = (int16_t)load16(ecosystem_a + 4*i) + (int16_t)load16(ecosystem_a + 4*i + 2); + int32_t ba = (int16_t)load16(ecosystem_b + 4*i) + (int16_t)load16(ecosystem_b + 4*i + 2); + int32_t as = (int16_t)load16(ecosystem_a + 4*i) - (int16_t)load16(ecosystem_a + 4*i + 2); + int32_t bs = (int16_t)load16(ecosystem_b + 4*i) - (int16_t)load16(ecosystem_b + 4*i + 2); + if (aa < -32768) aa = -32768; if (aa > 32767) aa = 32767; + if (ba < -32768) ba = -32768; if (ba > 32767) ba = 32767; + if (as < -32768) as = -32768; if (as > 32767) as = 32767; + if (bs < -32768) bs = -32768; if (bs > 32767) bs = 32767; + if ((int16_t)load16(ecosystem_out + 576 + 2*i) != (int16_t)aa) return 168; + if ((int16_t)load16(ecosystem_out + 584 + 2*i) != (int16_t)ba) return 169; + if (load16(ecosystem_out + 592 + 2*i) != (uint16_t)(load16(ecosystem_a + 4*i) + load16(ecosystem_a + 4*i + 2))) return 170; + if (load16(ecosystem_out + 600 + 2*i) != (uint16_t)(load16(ecosystem_b + 4*i) + load16(ecosystem_b + 4*i + 2))) return 171; + if ((int16_t)load16(ecosystem_out + 624 + 2*i) != (int16_t)as) return 172; + if ((int16_t)load16(ecosystem_out + 632 + 2*i) != (int16_t)bs) return 173; + if (load16(ecosystem_out + 640 + 2*i) != (uint16_t)(load16(ecosystem_a + 4*i) - load16(ecosystem_a + 4*i + 2))) return 174; + if (load16(ecosystem_out + 648 + 2*i) != (uint16_t)(load16(ecosystem_b + 4*i) - load16(ecosystem_b + 4*i + 2))) return 175; + } + uint16_t minimum = 0xffff, minimum_index = 0; + for (int i = 0; i < 8; i++) { + uint16_t value = load16(ecosystem_a + 2*i); + if (value < minimum) { minimum = value; minimum_index = (uint16_t)i; } + } + if (load16(ecosystem_out + 656) != minimum || load16(ecosystem_out + 658) != minimum_index) return 176; + for (int i = 660; i < 672; i++) if (ecosystem_out[i] != 0) return 177; + for (int block = 0; block < 2; block++) + for (int i = 0; i < 16; i++) { + uint8_t want_masked = (int8_t)ecosystem_b[i] < 0 ? ecosystem_a[i] : ecosystem_b[i]; + if (ecosystem_out[672 + 16*block + i] != want_masked) return 178 + block; + } + for (int i = 0; i < 8; i++) { + if (load16(ecosystem_out + 704 + 2*i) != (uint16_t)(load16(ecosystem_a + 2*i) << 3)) return 180; + if (load16(ecosystem_out + 752 + 2*i) != (uint16_t)(load16(ecosystem_a + 2*i) >> 3)) return 181; + } + for (int i = 0; i < 4; i++) { + if (load32(ecosystem_out + 720 + 4*i) != load32(ecosystem_a + 4*i) << 3) return 182; + if (load32(ecosystem_out + 768 + 4*i) != load32(ecosystem_a + 4*i) >> 3) return 183; + } + for (int i = 0; i < 2; i++) { + if (load64(ecosystem_out + 736 + 8*i) != load64(ecosystem_a + 8*i) << 3) return 184; + if (load64(ecosystem_out + 784 + 8*i) != load64(ecosystem_a + 8*i) >> 3) return 185; + } + if (load64(ecosystem_out + 800) != 3 || load64(ecosystem_out + 808) != 0) return 186; + uint64_t parity0 = parityBranches(0x00); + if (parity0 != 1) return 120 + (int)parity0; + if (parityBranches(0x03) != 1) return 114; + if (parityBranches(0x01) != 2) return 115; + if (parityBranches(0x07) != 2) return 116; + volatile double zero = 0.0; + if (unorderedBranch(1.25) != 0) return 117; + if (unorderedBranch(zero / zero) != 1) return 118; + float conversion_src[4] = {1.5f, -1.5f, 2.75f, -2.75f}; + int32_t conversion_out[20] = {0}; + int32_t conversion_want[20] = { + 2, -2, 3, -3, + 1, -2, 2, -3, + 2, -1, 3, -2, + 1, -1, 2, -2, + 1, -1, 2, -2, + }; + packedFloatToDwordModes(conversion_out, conversion_src); + for (int i = 0; i < 20; i++) + if (conversion_out[i] != conversion_want[i]) return 130 + i; +#if PLAN9ASM_CONFORMANCE_NATIVE_WIDE + int32_t same_width_ints[16]; + float same_width_floats[16] = { + 1.5f, -1.5f, 2.75f, -2.75f, 0.5f, -0.5f, 3.25f, -3.25f, + 4.5f, -4.5f, 5.75f, -5.75f, 6.5f, -6.5f, 7.25f, -7.25f, + }; + float same_width_squares32[16]; + double same_width_squares64[8]; + for (int i = 0; i < 16; i++) { + same_width_ints[i] = i - 8; + same_width_squares32[i] = (float)((i + 1) * (i + 1)); + } + for (int i = 0; i < 8; i++) same_width_squares64[i] = (double)((i + 1) * (i + 1)); + int32_t same_width_nearest[16] = {2,-2,3,-3,0,0,3,-3,4,-4,6,-6,6,-6,7,-7}; + int32_t same_width_trunc[16] = {1,-1,2,-2,0,0,3,-3,4,-4,5,-5,6,-6,7,-7}; + int32_t same_width_down[16] = {1,-2,2,-3,0,-1,3,-4,4,-5,5,-6,6,-7,7,-8}; + int32_t same_width_up[16] = {2,-1,3,-2,1,0,4,-3,5,-4,6,-5,7,-6,8,-7}; + uint8_t same_width_out[576] = {0}; + sameWidthPackedConversionSemantics(same_width_out, same_width_ints, same_width_floats, same_width_squares32, same_width_squares64); + for (int i = 0; i < 16; i++) { + if (loadf32(same_width_out + 4*i) != (float)same_width_ints[i]) return 760 + i; + if ((int32_t)load32(same_width_out + 64 + 4*i) != same_width_nearest[i]) return 780 + i; + if ((int32_t)load32(same_width_out + 128 + 4*i) != same_width_trunc[i]) return 800 + i; + if (loadf32(same_width_out + 192 + 4*i) != (float)(i + 1)) return 820 + i; + if ((int32_t)load32(same_width_out + 320 + 4*i) != same_width_nearest[i]) return 840 + i; + if ((int32_t)load32(same_width_out + 384 + 4*i) != same_width_down[i]) return 860 + i; + if ((int32_t)load32(same_width_out + 448 + 4*i) != same_width_up[i]) return 880 + i; + if ((int32_t)load32(same_width_out + 512 + 4*i) != same_width_trunc[i]) return 900 + i; + } + for (int i = 0; i < 8; i++) + if (loadf64(same_width_out + 256 + 8*i) != (double)(i + 1)) return 920 + i; +#endif + double widening_out[4] = {0}; + double widening_want[4] = {1.5, -1.5, 2.75, -2.75}; + packedSingleToDouble(widening_out, conversion_src); + for (int i = 0; i < 4; i++) + if (widening_out[i] != widening_want[i]) return 180 + i; + uint64_t cmov_src[16]; + uint64_t cmov_out[16] = {0}; + for (int i = 0; i < 16; i++) cmov_src[i] = 101 + (uint64_t)i; + conditionalMoveCodes(cmov_out, cmov_src); + int cmov_taken[16] = {1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1}; + for (int i = 0; i < 16; i++) { + uint64_t want = cmov_taken[i] ? cmov_src[i] : UINT64_MAX; + if (cmov_out[i] != want) return 160 + i; + } + double fma_a64[2] = {2, -3}, fma_b64[2] = {5, 7}, fma_c64[2] = {11, -13}; + float fma_a32[4] = {2, -3, 4, -5}; + float fma_b32[4] = {5, 7, -11, -13}; + float fma_c32[4] = {17, -19, 23, -29}; + uint8_t fma_out[288] = {0}; + fma3Semantics(fma_out, fma_a64, fma_b64, fma_c64, fma_a32, fma_b32, fma_c32); + for (int lane = 0; lane < 2; lane++) { + double a = fma_a64[lane], b = fma_b64[lane], c = fma_c64[lane]; + double want[8] = { + c*a+b, b*c+a, b*a+c, c*a-b, -(b*c)+a, -(b*a)-c, + lane % 2 == 0 ? c*a-b : c*a+b, + lane % 2 == 0 ? b*a+c : b*a-c, + }; + for (int block = 0; block < 8; block++) + if (loadf64(fma_out + 16*block + 8*lane) != want[block]) return 500 + 2*block + lane; + } + for (int lane = 0; lane < 4; lane++) { + float a = fma_a32[lane], b = fma_b32[lane], c = fma_c32[lane]; + float want[6] = { + c*a+b, b*c-a, -(b*a)+c, -(c*a)-b, + lane % 2 == 0 ? b*c-a : b*c+a, + lane % 2 == 0 ? b*a+c : b*a-c, + }; + for (int block = 0; block < 6; block++) + if (loadf32(fma_out + 128 + 16*block + 4*lane) != want[block]) return 520 + 4*block + lane; + } + if (loadf64(fma_out + 224) != fma_c64[0]*fma_a64[0]+fma_b64[0] || + loadf64(fma_out + 232) != fma_c64[1]) return 550; + if (loadf64(fma_out + 240) != fma_b64[0]*fma_c64[0]-fma_a64[0] || + loadf64(fma_out + 248) != fma_c64[1]) return 551; + if (loadf32(fma_out + 256) != -(fma_b32[0]*fma_a32[0])+fma_c32[0] || + loadf32(fma_out + 260) != fma_c32[1] || + loadf32(fma_out + 264) != fma_c32[2] || + loadf32(fma_out + 268) != fma_c32[3]) return 552; + if (loadf32(fma_out + 272) != -(fma_c32[0]*fma_a32[0])-fma_b32[0] || + loadf32(fma_out + 276) != fma_c32[1] || + loadf32(fma_out + 280) != fma_c32[2] || + loadf32(fma_out + 284) != fma_c32[3]) return 553; + double binary_a64[2] = {2, -3}, binary_b64[2] = {20, 30}; + float binary_a32[4] = {2, -3, 4, -5}, binary_b32[4] = {20, 30, -40, -50}; + double binary_want64[6][2] = { + {22, 27}, {18, 33}, {40, -90}, {10, -10}, {20, 30}, {2, -3}, + }; + float binary_want32[6][4] = { + {22, 27, -36, -55}, + {18, 33, -44, -45}, + {40, -90, -160, 250}, + {10, -10, -10, 10}, + {20, 30, 4, -5}, + {2, -3, -40, -50}, + }; + uint8_t binary_out[384] = {0}; + binaryFloatingSemantics(binary_out, binary_a64, binary_b64, binary_a32, binary_b32); + for (int operation = 0; operation < 6; operation++) { + for (int lane = 0; lane < 2; lane++) + if (loadf64(binary_out + 16*operation + 8*lane) != binary_want64[operation][lane]) return 600 + 2*operation + lane; + for (int lane = 0; lane < 4; lane++) + if (loadf32(binary_out + 96 + 16*operation + 4*lane) != binary_want32[operation][lane]) return 620 + 4*operation + lane; + for (int lane = 0; lane < 2; lane++) { + double want = lane == 0 ? binary_want64[operation][0] : binary_b64[lane]; + if (loadf64(binary_out + 192 + 16*operation + 8*lane) != want) return 650 + 2*operation + lane; + } + for (int lane = 0; lane < 4; lane++) { + float want = lane == 0 ? binary_want32[operation][0] : binary_b32[lane]; + if (loadf32(binary_out + 288 + 16*operation + 4*lane) != want) return 670 + 4*operation + lane; + } + } + float horizontal_a32[8] = {1, 2, 3, 4, 5, 6, 7, 8}; + float horizontal_b32[8] = {10, 20, 30, 40, 50, 60, 70, 80}; + double horizontal_a64[4] = {1, 2, 3, 4}; + double horizontal_b64[4] = {10, 20, 30, 40}; + float horizontal_want32[4][8] = { + {30, 70, 3, 7, 110, 150, 11, 15}, + {-10, -10, -1, -1, -10, -10, -1, -1}, + {30, 70, 3, 7, 0, 0, 0, 0}, + {-10, -10, -1, -1, 0, 0, 0, 0}, + }; + double horizontal_want64[4][4] = { + {30, 3, 70, 7}, + {-10, -1, -10, -1}, + {30, 3, 0, 0}, + {-10, -1, 0, 0}, + }; + uint8_t horizontal_out[192] = {0}; + horizontalFloatingSemantics(horizontal_out, horizontal_a32, horizontal_b32, horizontal_a64, horizontal_b64); + for (int block = 0; block < 2; block++) + for (int lane = 0; lane < 8; lane++) + if (loadf32(horizontal_out + 32*block + 4*lane) != horizontal_want32[block][lane]) return 700 + 8*block + lane; + for (int block = 0; block < 2; block++) + for (int lane = 0; lane < 4; lane++) + if (loadf64(horizontal_out + 64 + 32*block + 8*lane) != horizontal_want64[block][lane]) return 720 + 4*block + lane; + for (int block = 0; block < 2; block++) + for (int lane = 0; lane < 4; lane++) + if (loadf32(horizontal_out + 128 + 16*block + 4*lane) != horizontal_want32[2+block][lane]) return 740 + 4*block + lane; + for (int block = 0; block < 2; block++) + for (int lane = 0; lane < 2; lane++) + if (loadf64(horizontal_out + 160 + 16*block + 8*lane) != horizontal_want64[2+block][lane]) return 750 + 2*block + lane; return 0; } ` - compileAndRunRuntimeTestForTarget(t, llc, clang, "amd64_conformance", triple, ll, mainC, runPrefix) } func rosettaAvailable() bool { diff --git a/amd64_continuation.go b/amd64_continuation.go new file mode 100644 index 000000000..7c928220a --- /dev/null +++ b/amd64_continuation.go @@ -0,0 +1,192 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// X86TailGroup names one ordinary entry and its private continuation TEXTs. +// Names use source spelling, before Options.ResolveSym. See X86TailGroups. +type X86TailGroup struct { + Root string + Helpers []string +} + +type x86Continuation struct { + root string + label string +} + +func (f *File) isX86Continuation(name string, resolve func(string) string) bool { + for symbol := range f.x86Continuations { + if resolve(symbol) == name { + return true + } + } + return false +} + +func coalesceX86Continuations(file *File, opt Options) (*File, error) { + if file == nil { + return nil, fmt.Errorf("nil file") + } + if file.x86ContinuationGraph != nil { + if err := file.x86ContinuationGraph.verify(file, opt); err != nil { + return nil, err + } + return file, nil + } + if len(file.x86Continuations) != 0 { + return nil, x86ContinuationContext("unbound prepared continuation inventory") + } + if len(opt.X86TailGroups) == 0 { + return file, nil + } + if file.Arch != ArchAMD64 || (opt.Goarch != "amd64" && opt.Goarch != "386") { + return nil, fmt.Errorf("x86 continuation groups require an x86 target") + } + resolve := opt.ResolveSym + if resolve == nil { + resolve = func(s string) string { return s } + } + functions := make(map[string]Func, len(file.Funcs)) + for _, fn := range file.Funcs { + if _, duplicate := functions[fn.Sym]; duplicate { + return nil, fmt.Errorf("duplicate continuation TEXT %q", fn.Sym) + } + functions[fn.Sym] = fn + } + addresses := map[string]x86Continuation{} + owners := map[string]string{} + for gi, group := range opt.X86TailGroups { + if len(group.Helpers) == 0 { + return nil, fmt.Errorf("empty continuation group %q", group.Root) + } + members := append([]string{group.Root}, group.Helpers...) + for mi, name := range members { + fn, exists := functions[name] + if !exists || owners[name] != "" { + return nil, fmt.Errorf("missing or repeated continuation TEXT %q", name) + } + owners[name] = group.Root + if fn.X86RawText != nil || fn.FrameSize != 0 { + return nil, fmt.Errorf("continuation TEXT %q requires a zero frame and no native layout dependency", name) + } + if len(fn.Instrs) == 0 { + return nil, fmt.Errorf("empty continuation TEXT %q", name) + } + last := fn.Instrs[len(fn.Instrs)-1] + if (last.Op != "RET" || len(last.Args) != 0) && last.Op != "JMP" { + return nil, fmt.Errorf("continuation TEXT %q has no explicit terminating transfer", name) + } + if mi == 0 { + continue + } + sig, ok := opt.Sigs[resolve(name)] + if !ok || sig.Ret != Void || len(sig.Args) != 0 || fn.ArgSize != 0 || + len(sig.Frame.Params)+len(sig.Frame.Results) != 0 { + return nil, fmt.Errorf("continuation %q must be a zero-frame void helper", name) + } + addresses[name] = x86Continuation{ + root: group.Root, label: fmt.Sprintf("p9_cont_%d_%d", gi, mi), + } + } + } + + // These addresses are local control-flow labels, not callable functions or + // byte-addressable code. Refuse a group before emitting any changed ABI. + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + for ai, arg := range ins.Args { + if arg.Kind != OpSym { + continue + } + base, off, ok := parseSBRef(arg.Sym) + target, continuation := addresses[strings.TrimPrefix(base, "$")] + if !ok || !continuation { + continue + } + if off != 0 || ins.Op == "CALL" || (ins.Op == "JMP" && owners[fn.Sym] != target.root) { + return nil, fmt.Errorf("invalid continuation reference in %q: %s", fn.Sym, ins.Raw) + } + if ins.Op == "JMP" && len(ins.Args) == 1 { + continue + } + form := x86SourceMoveInstruction(ins, opt.Goarch) + address := form.Op == "LEAQ" || form.Op == "LEAL" || + ((form.Op == "MOVQ" || form.Op == "MOVL") && strings.HasPrefix(arg.Sym, "$")) + if !address || ai != 0 || len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return nil, fmt.Errorf("continuation %q is not readable or callable code: %s", base, ins.Raw) + } + } + } + } + for _, d := range file.Data { + if d.Addr != "" { + base, _, _ := parseSBRef(d.Addr) + if _, ok := addresses[strings.TrimPrefix(base, "$")]; ok { + return nil, fmt.Errorf("continuation DATA relocations require an explicit initializer") + } + } + } + + fused := map[string]Func{} + for gi, group := range opt.X86TailGroups { + root := functions[group.Root] + root.Instrs = nil + root.x86IndirectLabels = nil + for mi, name := range append([]string{group.Root}, group.Helpers...) { + fn := functions[name] + prefix := fmt.Sprintf("p9_body_%d_%d_", gi, mi) + if mi != 0 { + label := addresses[name].label + root.Instrs = append(root.Instrs, Instr{Op: OpLABEL, Args: []Operand{{Kind: OpLabel, Sym: label}}}) + root.x86IndirectLabels = append(root.x86IndirectLabels, label) + } + labels := map[string]string{} + for i, ins := range fn.Instrs { + if ins.Op == OpLABEL && len(ins.Args) == 1 { + labels[ins.Args[0].Sym] = fmt.Sprintf("%s%d", prefix, i) + } + } + for _, ins := range fn.Instrs { + ins.Args = append([]Operand(nil), ins.Args...) + for ai, arg := range ins.Args { + switch arg.Kind { + case OpLabel: + ins.Args[ai].Sym = labels[arg.Sym] + case OpIdent: + if label, ok := labels[arg.Ident]; ok { + ins.Args[ai].Ident = label + } + case OpReg: + if label, ok := labels[string(arg.Reg)]; ok && (ins.Op == "JMP" || isAMD64ConditionalBranch(ins.Op)) { + ins.Args[ai] = Operand{Kind: OpIdent, Ident: label} + } + case OpSym: + base, off, ok := parseSBRef(arg.Sym) + if target, exists := addresses[base]; ok && exists && off == 0 && ins.Op == "JMP" { + ins.Args[ai] = Operand{Kind: OpIdent, Ident: target.label} + } + } + } + root.Instrs = append(root.Instrs, ins) + } + } + fused[group.Root] = root + } + result := *file + result.Funcs = nil + result.x86Continuations = addresses + for _, fn := range file.Funcs { + if _, helper := addresses[fn.Sym]; helper { + continue + } + if root, ok := fused[fn.Sym]; ok { + fn = root + } + fn.x86ContinuationAddresses = addresses + result.Funcs = append(result.Funcs, fn) + } + return sealX86Continuations(file, &result, opt) +} diff --git a/amd64_continuation_test.go b/amd64_continuation_test.go new file mode 100644 index 000000000..25d66051b --- /dev/null +++ b/amd64_continuation_test.go @@ -0,0 +1,274 @@ +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// The empty helper declarations describe Go entries, not the register/frame +// contract of a JMP from another TEXT. This fixture borrows the root frame. +const x86ContinuationSource = ` +TEXT ·initTable(SB),$0-0 + LEAQ ·step(SB),AX + MOVQ AX,table<>(SB) + RET +GLOBL table<>(SB),16,$8 +TEXT ·run(SB),$0-24 + MOVQ out+0(FP),DI + MOVQ n+8(FP),BX + MOVQ $7,R15 + MOVQ R15,X13 + LEAQ table<>(SB),R14 + XORQ R15,R15 + TESTQ BX,BX + JEQ done +dispatch: + MOVQ (R14),AX + JMP AX +done: + JMP ·exit(SB) +TEXT ·step(SB),$0-0 + MOVQ X13,DX + ADDQ DX,R15 + SUBQ $1,BX + JMP ·check(SB) +TEXT ·check(SB),$0-0 + JEQ done + MOVQ (R14),AX + JMP AX +done: + JMP ·exit(SB) +TEXT ·exit(SB),$0-0 + MOVQ R15,(DI) + MOVQ R15,result+16(FP) + RET +` + +func x86ContinuationOptions(triple string) Options { + return Options{ + Goarch: "amd64", TargetTriple: triple, + X86TailGroups: []X86TailGroup{{Root: "·run", Helpers: []string{"·step", "·check", "·exit"}}}, + ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }, + Sigs: map[string]FuncSig{ + "initTable": {Ret: Void}, + "run": { + Args: []LLVMType{Ptr, I64}, Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + }, + Results: []FrameSlot{{Offset: 16, Type: I64, Index: 0, Field: -1}}, + }, + }, + "step": {Ret: Void}, "check": {Ret: Void}, "exit": {Ret: Void}, + }, + } +} + +func TestX86ContinuationBorrowedFrame(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", x86ContinuationSource, true) + ir := x86ContinuationIR(t, "x86_64-unknown-linux-gnu", x86ContinuationSource) + if !strings.Contains(ir, "indirectbr") || !strings.Contains(ir, "blockaddress") { + t.Fatal("continuations must branch within the root CFG") + } + if strings.Contains(ir, "define void @step") || strings.Contains(ir, "call i64 %") { + t.Fatal("a continuation must not be emitted as a fresh function invocation") + } +} + +func x86ContinuationIR(t *testing.T, triple, source string) string { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86ContinuationOptions(triple)) + if err != nil { + t.Fatal(err) + } + return ir +} + +func TestX86ContinuationLLVM(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH == "amd64" || crossRosetta { + runX86ContinuationGo(t) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + for _, memoryJump := range []bool{false, true} { + source := x86ContinuationSource + if memoryJump { + source = strings.ReplaceAll(source, " MOVQ (R14),AX\n JMP AX", " JMP (R14)") + } + ir := x86ContinuationIR(t, triple, source) + compileLLVMToObject(t, llc, triple, "continuation.ll", "continuation.o", ir) + if (runtime.GOARCH == "amd64" && runtime.GOOS == "linux" && triple == "x86_64-unknown-linux-gnu") || + ((runtime.GOARCH == "amd64" || crossRosetta) && runtime.GOOS == "darwin" && triple == "x86_64-apple-darwin") { + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("LLVM 22 clang not found") + } + runX86Continuation(t, llc, clang, triple, ir) + } + } + }) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir := x86Continuation386IR(t, triple) + compileLLVMToObject(t, llc, triple, "continuation386.ll", "continuation386.o", ir) + }) + } +} + +func x86Continuation386IR(t *testing.T, triple string) string { + t.Helper() + source := strings.NewReplacer( + " MOVQ R15,X13\n", "", " MOVQ X13,DX\n ADDQ DX,R15", " ADDL $7,DX", + "R15", "DX", "R14", "SI", "MOVQ", "MOVL", "LEAQ", "LEAL", + "XORQ", "XORL", "TESTQ", "TESTL", "SUBQ", "SUBL", + "n+8(FP)", "n+4(FP)", "result+16(FP)", "result+8(FP)", + "$0-24", "$0-12", "GLOBL table<>(SB),16,$8", "GLOBL table<>(SB),16,$4", + ).Replace(x86ContinuationSource) + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86ContinuationOptions(triple) + opt.Goarch = "386" + sig := opt.Sigs["run"] + sig.Args[1], sig.Ret = I32, I32 + sig.Frame.Params[1].Offset, sig.Frame.Params[1].Type = 4, I32 + sig.Frame.Results[0].Offset, sig.Frame.Results[0].Type = 8, I32 + opt.Sigs["run"] = sig + ir, err := Translate(file, opt) + if err != nil { + t.Fatal(err) + } + return ir +} + +func runX86Continuation(t *testing.T, llc, clang, triple, ir string) { + t.Helper() + if dir := os.Getenv("PLAN9ASM_RUNTIME_ARTIFACT_DIR"); dir != "" { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + for name, data := range map[string]string{"continuation.ll": ir, "main.c": x86ContinuationMain} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0644); err != nil { + t.Fatal(err) + } + } + } + compiler := []string{clang} + var runner []string + if runtime.GOARCH != "amd64" && strings.Contains(triple, "darwin") { + compiler = append(compiler, "-target", triple) + runner = []string{"/usr/bin/arch", "-x86_64"} + } + if strings.Contains(triple, "linux") { + // The shared harness invokes llc with its default static relocation + // model. Match that object when linking a native executable. + compiler = append(compiler, "-no-pie") + } + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, "continuation", triple, ir, x86ContinuationMain, runner) +} + +func runX86ContinuationGo(t *testing.T) { + t.Helper() + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module example.com/continuation\n\ngo 1.20\n", + "continuation_amd64.s": strings.ReplaceAll(x86ContinuationSource, "(SB),$0-", "(SB),4,$0-"), + "continuation.go": "package p\nfunc initTable()\nfunc run(out *uint64, n uint64) (result uint64)\nfunc step()\nfunc check()\nfunc exit()\n", + "continuation_test.go": `package p +import "testing" +func TestState(t *testing.T) { + initTable() + for _, n := range []uint64{0, 1, 2, 19, 100000} { + out := [3]uint64{0x1234, ^uint64(0), 0x5678} + result := run(&out[1], n) + if result != n*7 || out[1] != result || out[0] != 0x1234 || out[2] != 0x5678 { + t.Fatalf("n=%d result=%d out=%v", n, result, out) + } + } +} +`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + // asmdecl describes callable entries, not the helpers' borrowed frame. + // This oracle tests Go's actual assembler/linker/runtime semantics. + cmd := exec.Command("go", "test", "-vet=off", "-count=1", ".") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("native Go continuation oracle: %v\n%s", err, out) + } +} + +func TestX86ContinuationRejectsInvalidGroups(t *testing.T) { + for _, test := range []struct { + name string + change func(*File, *Options) + }{ + {"other_arch", func(f *File, o *Options) { f.Arch = ArchWASM; o.Goarch = "wasm" }}, + {"missing_root", func(f *File, o *Options) { o.X86TailGroups[0].Root = "absent" }}, + {"missing_helper", func(f *File, o *Options) { o.X86TailGroups[0].Helpers[0] = "absent" }}, + {"duplicate", func(f *File, o *Options) { o.X86TailGroups = append(o.X86TailGroups, o.X86TailGroups[0]) }}, + {"empty", func(f *File, o *Options) { o.X86TailGroups[0].Helpers = nil }}, + {"local_frame", func(f *File, o *Options) { f.Funcs[2].FrameSize = 8 }}, + {"root_frame", func(f *File, o *Options) { f.Funcs[1].FrameSize = 8 }}, + {"fallthrough", func(f *File, o *Options) { f.Funcs[2].Instrs = append(f.Funcs[2].Instrs, Instr{Op: "NOP"}) }}, + {"code_read", func(f *File, o *Options) { f.Funcs[0].Instrs[1].Op = "MOVQ" }}, + {"arg_frame", func(f *File, o *Options) { f.Funcs[2].ArgSize = 8 }}, + {"typed_entry", func(f *File, o *Options) { o.Sigs["step"] = FuncSig{Args: []LLVMType{I64}, Ret: Void} }}, + {"callable", func(f *File, o *Options) { + f.Funcs[0].Instrs[0] = Instr{Op: "CALL", Args: []Operand{{Kind: OpSym, Sym: "·step(SB)"}}} + }}, + {"offset", func(f *File, o *Options) { f.Funcs[0].Instrs[1].Args[0].Sym = "·step+1(SB)" }}, + {"data", func(f *File, o *Options) { f.Data = []DataStmt{{Sym: "table<>", Width: 8, Addr: "·step(SB)"}} }}, + } { + t.Run(test.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, x86ContinuationSource) + if err != nil { + t.Fatal(err) + } + opt := x86ContinuationOptions("x86_64-unknown-linux-gnu") + test.change(file, &opt) + if _, err := coalesceX86Continuations(file, opt); err == nil { + t.Fatal("invalid continuation proof accepted") + } + }) + } +} + +const x86ContinuationMain = ` +#include +extern void initTable(void); +extern uint64_t run(uint64_t *, uint64_t); +int main(void) { + initTable(); + const uint64_t counts[] = {0, 1, 2, 19, 100000}; + for (unsigned i = 0; i < sizeof(counts) / sizeof(counts[0]); i++) { + uint64_t out[3] = {0x1234, UINT64_MAX, 0x5678}; + uint64_t result = run(out + 1, counts[i]); + if (result != counts[i] * 7 || out[1] != result || + out[0] != 0x1234 || out[2] != 0x5678) return 1; + } + return 0; +} +` diff --git a/amd64_cpu_query_consumers_test.go b/amd64_cpu_query_consumers_test.go new file mode 100644 index 000000000..4a7b50679 --- /dev/null +++ b/amd64_cpu_query_consumers_test.go @@ -0,0 +1,226 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86CPUQueryTypedGrammarAndDispatchBoundaries(t *testing.T) { + if len(x86CPUQuerySpecs) != 2 { + t.Fatal("complete official CPU-query family changed") + } + for _, op := range []Op{OpCPUID, OpXGETBV} { + spec := x86CPUQuerySpecs[op] + inputs, outputs, encoding := []Reg{AX, CX}, []Reg{AX, BX, CX, DX}, []byte{0xa2} + if op == OpXGETBV { + inputs, outputs, encoding = []Reg{CX}, []Reg{AX, DX}, []byte{0x01, 0xd0} + } + if spec.native != strings.ToLower(string(op)) || !reflect.DeepEqual(spec.inputs, inputs) || + !reflect.DeepEqual(spec.outputs, outputs) || !bytes.Equal(spec.opBytes, encoding) { + t.Fatal("CPU-query typed syntax or implicit effect changed", op) + } + for _, arch := range []string{"386", "amd64", "", "arm", "arm64", "wasm"} { + for _, suffix := range []string{"", ".Z", ".SAE", ".BCST"} { + for _, identity := range []Op{"", op, Op(strings.ToLower(string(op))), "MOVQ", Op(string(op) + ".BAD")} { + t.Run(arch+"/"+string(op)+suffix+"/identity="+string(identity), func(t *testing.T) { + instruction := Instr{Op: identity, Raw: string(op) + suffix} + before := fmt.Sprintf("%#v", instruction) + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + ir, temporary := output.String(), ctx.tmp + known, terminated, err := ctx.lowerX86CPUQuery(Op(string(op)+suffix), instruction) + invalid := arch != "386" && arch != "amd64" || suffix != "" || + identity != "" && !strings.EqualFold(string(identity), string(op)) + if !known || terminated || (err != nil) != invalid || + invalid && (ir != output.String() || temporary != ctx.tmp) || before != fmt.Sprintf("%#v", instruction) { + t.Fatal("CPU-query dispatch mutated input or emitted before rejection", err) + } + }) + } + } + } + } + if _, known, err := x86CPUQueryProperties("RDTSC", Instr{}, "amd64"); known || err != nil { + t.Fatal("timestamps must not acquire the CPU-query grammar") + } +} + +func TestX86CPUQueryConstructedFileConsumersRetainParserGrammar(t *testing.T) { + pkg := mustGoPackage(t, "cpuquery", "package cpuquery\nfunc query()\n") + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + for _, arch := range []string{"386", "amd64"} { + for _, op := range []Op{OpCPUID, OpXGETBV} { + for _, operand := range []string{"AX", "(AX)", "$1", "AX,DX"} { + t.Run(arch+"/"+string(op)+"/"+operand, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·query(SB),4,$0-0\n"+string(op)+"\nRET\n") + if err != nil { + t.Fatal(err) + } + signatures, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + file.Funcs[0].Instrs[1].Args, err = parseOperandsCSV(ArchAMD64, op, operand) + if err != nil { + t.Fatal(err) + } + before := fmt.Sprintf("%#v", file.Funcs[0]) + options := Options{Goarch: arch, ResolveSym: resolve, Sigs: signatures} + if ir, err := Translate(file, options); err == nil || ir != "" { + t.Fatal("constructed text consumer bypassed Go's zero-operand grammar", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + t.Fatal("constructed module consumer returned an invalid query module", err) + } + if err == nil || before != fmt.Sprintf("%#v", file.Funcs[0]) { + t.Fatal("constructed module grammar changed its source or silently accepted it", err) + } + }) + } + } + } +} + +func TestX86CPUQueryDescriptorCannotSupplyCallerOrStackPermission(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + move, suffix := OpMOVQ, "Q" + if arch == "386" { + move, suffix = OpMOVL, "L" + } + for _, op := range []Op{OpCPUID, OpXGETBV} { + instruction := Instr{Op: op, Raw: string(op)} + if _, known, err := x86CPUQueryProperties(op, instruction, arch); !known || err != nil { + t.Fatal("valid CPU-query grammar is not bound", op, err) + } + if x86SourceStackInstructionKnown(instruction, move, suffix, arch) { + t.Fatal("implicit-register descriptor granted a native caller/stack contract", op) + } + } + } +} + +func x86CPUQueryIR(t *testing.T, source, arch, triple, api string) string { + t.Helper() + return x86CPUQueryIRFromDeclaration(t, source, "func query()\n", arch, triple, api) +} + +func x86CPUQueryIRFromDeclaration(t *testing.T, source, declarations, arch, triple, api string) string { + t.Helper() + pkg := mustGoPackage(t, "cpuquery", "package cpuquery\n"+declarations) + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + ctx := llvm.NewContext() + defer ctx.Dispose() + if api == "Go" { + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: resolve}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil || translated == nil || translated.Module.IsNil() { + t.Fatalf("Go module CPU-query consumer: %v", err) + } + return translated.Module.String() + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + signatures, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, TargetTriple: triple, ResolveSym: resolve, Sigs: signatures} + if api == "text" { + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + return ir + } + if api != "module" { + t.Fatal("unknown CPU-query consumer", api) + } + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil || module.IsNil() { + t.Fatalf("LLVM module CPU-query consumer: %v", err) + } + return module.String() +} + +func TestX86CPUQueryNamedRawThreeAPIsFiveTargetsProduceActualObjects(t *testing.T) { + llc, objdump := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || objdump == "" { + t.Fatal("LLVM 22 llc and llvm-objdump required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + for _, op := range []Op{OpCPUID, OpXGETBV} { + for _, raw := range []bool{false, true} { + body := string(op) + "\n" + if raw { + body = "BYTE $0x0f\n" + encoding := []byte{0xa2} + if op == OpXGETBV { + encoding = []byte{0x01, 0xd0} + } + for _, value := range encoding { + body += fmt.Sprintf("BYTE $%#x\n", value) + } + } + source := "TEXT ·query(SB),4,$0-0\n" + body + "RET\n" + goAssemblerOracleOutput(t, target.arch, source, "", true) + initialized := "TEXT ·query(SB),4,$0-0\nMOVL $0,AX\nMOVL $0,CX\n" + body + "RET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, initialized) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+string(op)+fmt.Sprint("/raw=", raw)+"/"+api, func(t *testing.T) { + unbound, err := x86ScalarFPBitViewAPI(t, source, "func query()", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || unbound != "" { + t.Fatal("original uninitialized query acquired an entry-register contract", err) + } + ir := x86CPUQueryIR(t, initialized, target.arch, target.triple, api) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprint("O2=", optimized), func(t *testing.T) { + input := ir + if optimized { + input = x86UndefinedTrapOptimizedIR(t, input) + } + if strings.Count(input, "asm sideeffect \""+strings.ToLower(string(op))+"\"") != 1 { + t.Fatal("whole CPU query was removed or duplicated") + } + dir := t.TempDir() + ll, object := filepath.Join(dir, "query.ll"), filepath.Join(dir, "query.o") + if err := os.WriteFile(ll, []byte(input), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, "-filetype=obj", "-mtriple="+target.triple, ll, "-o", object)) + actual := x86FloatCompareCaptureCommand(t, exec.Command(objdump, "--disassemble", object)) + physical := regexp.MustCompile(`(?m)\b` + strings.ToLower(string(op)) + `\b`) + if len(physical.FindAll(actual, -1)) != 1 { + t.Fatalf("actual LLVM object lost native CPU-query identity:\n%s", actual) + } + }) + } + }) + } + } + } + } +} diff --git a/amd64_cpu_query_family_test.go b/amd64_cpu_query_family_test.go new file mode 100644 index 000000000..2f95d89e2 --- /dev/null +++ b/amd64_cpu_query_family_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86CPUQueryCompleteOfficialRowsAndImplicitRegisters(t *testing.T) { + table, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(table), "var ynone = []ytab{\n\t{Zlit, 1, argList{}},\n}") { + t.Fatal("complete Go zero-operand table changed") + } + for _, form := range []struct { + op Op + row string + encoding []byte + registers []Reg + }{ + {OpCPUID, "{ACPUID, ynone, Pm, opBytes{0xa2}}", []byte{0x0f, 0xa2, 0xc3}, []Reg{AX, BX, CX, DX}}, + {OpXGETBV, "{AXGETBV, ynone, Pm, opBytes{01, 0xd0}}", []byte{0x0f, 0x01, 0xd0, 0xc3}, []Reg{AX, CX, DX}}, + } { + if !strings.Contains(string(table), form.row) { + t.Fatal("complete Go CPU-query encoding changed", form.op) + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch+"/"+string(form.op), func(t *testing.T) { + source := "TEXT query(SB),4,$0-0\n" + string(form.op) + "\nRET\n" + if actual := assembleX87ControlBytes(t, arch, source); !bytes.Equal(actual, form.encoding) { + t.Fatalf("original Go CPU-query identity: %x", actual) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, _ := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + for _, register := range form.registers { + if ctx.regSlot[register] == "" { + t.Errorf("implicit CPU-query register %s has no allocated slot", register) + } + } + }) + } + } +} + +func TestX86CPUQueryBareAcceptedSourceAcrossThreeAPIs(t *testing.T) { + pkg := mustGoPackage(t, "cpuquery", "package cpuquery\nfunc query()\n") + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + for _, arch := range []string{"386", "amd64"} { + for _, op := range []Op{OpCPUID, OpXGETBV} { + source := "TEXT ·query(SB),4,$0-0\n" + string(op) + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+string(op)+"/"+api, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + signatures, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, ResolveSym: resolve, Sigs: signatures} + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + module, translateErr := TranslateModuleInContext(ctx, file, options) + err = translateErr + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "Go": + translated, translateErr := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: arch, ResolveSym: resolve}) + err = translateErr + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + ir = translated.Module.String() + } + } + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("Go-accepted syntax supplied missing implicit inputs to %s: %v\n%s", api, err, ir) + } + }) + } + } + } +} + +func TestX86CPUQueryConstructedInvalidOperandsRejectBeforeIR(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []Op{OpCPUID, OpXGETBV} { + for _, operand := range []string{"AX", "(AX)", "$1", "AX,DX"} { + t.Run(arch+"/"+string(op)+"/"+operand, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT query(SB),4,$0-0\n"+string(op)+" "+operand+"\nRET\n", "", false) + // Parse already rejects these source forms. Constructed File/helper + // callers must not bypass that grammar and silently emit a query. + args, err := parseOperandsCSV(ArchAMD64, op, operand) + if err != nil { + t.Fatal(err) + } + fn := Func{Instrs: []Instr{{Op: OpMOVL, Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: CX}}}, {Op: OpMOVL, Args: []Operand{{Kind: OpReg, Reg: DX}, {Kind: OpReg, Reg: AX}}}}} + ctx, output := newAMD64CtxWithFuncForTest(t, fn, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + known, terminated, err := ctx.lowerArith(op, Instr{Op: op, Args: args, Raw: string(op) + " " + operand}) + if !known || terminated || err == nil || before != output.String() || temporary != ctx.tmp { + t.Fatalf("invalid CPU-query form emitted IR: %t/%t/%v", known, terminated, err) + } + }) + } + } + } +} diff --git a/amd64_cpu_query_forms.go b/amd64_cpu_query_forms.go new file mode 100644 index 000000000..304d0ff1b --- /dev/null +++ b/amd64_cpu_query_forms.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86CPUQuerySpec struct { + native string + inputs []Reg + outputs []Reg + opBytes []byte // Complete Go ynone/Pm opcode row, after its 0x0f prefix. +} + +var x86CPUQuerySpecs = map[Op]x86CPUQuerySpec{ + OpCPUID: {"cpuid", []Reg{AX, CX}, []Reg{AX, BX, CX, DX}, []byte{0xa2}}, + OpXGETBV: {"xgetbv", []Reg{CX}, []Reg{AX, DX}, []byte{0x01, 0xd0}}, +} + +func x86CPUQueryProperties(op Op, ins Instr, arch string) (x86CPUQuerySpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86CPUQuerySpecs[Op(name)] + if !known { + return x86CPUQuerySpec{}, false, nil + } + if len(ins.Args) != 0 || dotted || arch != "amd64" && arch != "386" || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return spec, true, fmt.Errorf("%s %s requires the exact zero-operand Go CPU-query form: %q", arch, op, ins.Raw) + } + return spec, true, nil +} + +// The native emitter consumes these same low dword inputs before overwriting +// its zero-extended outputs. Continued execution supplies numerical bytes, +// not a pointer, caller region, raw entry or promise that XGETBV cannot fault. +func x86TypedCPUQueryByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + spec, known, err := x86CPUQueryProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + if !x86RegisterQueryPreservesSourceState(ins) { + return x86TypedByteEffect{}, true, fmt.Errorf("%w: CPU query original/normalized source identity differs", ErrProbeNeedsContext) + } + effect := x86TypedByteEffect{requiresResultProof: true} + for _, register := range spec.inputs { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: -1, operand: Operand{Kind: OpReg, Reg: register}, width: 4, read: true, + }) + } + for _, register := range spec.outputs { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: -1, operand: Operand{Kind: OpReg, Reg: register}, width: 4, write: true, + }) + } + return effect, true, nil +} + +func x86TypedRegisterQueryByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + if effect, known, err := x86TypedHardwareRandomByteEffect(op, ins, arch); known { + return effect, known, err + } + if effect, known, err := x86TypedCPUQueryByteEffect(op, ins, arch); known { + return effect, known, err + } + if effect, known, err := x86TypedTimestampByteEffect(op, ins, arch); known { + return effect, known, err + } + return x86TypedRTMStatusByteEffect(op, ins, arch) +} + +// Register queries keep DF and the private numerical stack. Their separate +// descriptors retain all GP and FLAGS writes: an RTM status query, unlike a CPU +// query or timestamp, replaces FLAGS. Random producers also replace their one +// destination and FLAGS. Other opcodes cannot inherit this exact source boundary. +func x86RegisterQueryPreservesSourceState(ins Instr) bool { + if x86HardwareRandomPreservesSourceState(ins) { + return true + } + name := Op(strings.ToUpper(string(ins.Op))) + _, cpu := x86CPUQuerySpecs[name] + _, timestamp := x86TimestampSpecs[name] + rtm, transaction := amd64RTMSpecs[name] + status := transaction && rtm.kind == amd64RTMTest + if !cpu && !timestamp && !status || len(ins.Args) != 0 || ins.x86Encoded || ins.x86RawNearReturn != nil || + ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 { + return false + } + origin := ins.x86SourceRaw + return origin == nil || len(origin.args) == 0 && strings.EqualFold(string(origin.op), string(ins.Op)) +} + +func (spec x86CPUQuerySpec) resultType() LLVMType { + return LLVMType("{ " + strings.TrimSuffix(strings.Repeat("i32, ", len(spec.outputs)), ", ") + " }") +} + +func (spec x86CPUQuerySpec) constraints() string { + var fields []string + for _, register := range spec.outputs { + fields = append(fields, "={"+strings.ToLower(string(register))+"}") + } + for _, register := range spec.inputs { + fields = append(fields, "{"+strings.ToLower(string(register))+"}") + } + // Preserve source ordering at the compiler boundary. These conservative + // clobbers do not assert that XGETBV serializes hardware or changes flags. + fields = append(fields, "~{dirflag}", "~{fpsr}", "~{flags}", "~{memory}") + return strings.Join(fields, ",") +} + +func emitX86CPUQuery(b *strings.Builder, spec x86CPUQuerySpec, inputs []string, newTmp func() string) []string { + arguments := make([]string, len(inputs)) + for index, input := range inputs { + arguments[index] = "i32 " + input + } + call := newTmp() + fmt.Fprintf(b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", + call, spec.resultType(), spec.native, spec.constraints(), strings.Join(arguments, ", ")) + outputs := make([]string, len(spec.outputs)) + for index := range outputs { + part := newTmp() + fmt.Fprintf(b, " %%%s = extractvalue %s %%%s, %d\n", part, spec.resultType(), call, index) + outputs[index] = "%" + part + } + return outputs +} + +func (c *amd64Ctx) lowerX86CPUQuery(op Op, ins Instr) (bool, bool, error) { + spec, known, err := x86CPUQueryProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + inputs := make([]string, len(spec.inputs)) + for index, register := range spec.inputs { + value, err := c.loadReg(register) + if err != nil { + return true, false, err + } + inputs[index] = c.truncI64(value, I32) + } + outputs := emitX86CPUQuery(c.b, spec, inputs, c.newTmp) + for index, register := range spec.outputs { + if err := c.storeRegSized(register, I32, outputs[index]); err != nil { + return true, false, err + } + } + return true, false, nil +} diff --git a/amd64_cpu_query_linux_amd64_test.go b/amd64_cpu_query_linux_amd64_test.go new file mode 100644 index 000000000..447a75f5e --- /dev/null +++ b/amd64_cpu_query_linux_amd64_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CPUQuery386IndependentC() string { + return strings.NewReplacer( + "uint64_t", "uint32_t", "UINT64_C(0xabcdef0123456789)", "UINT32_C(0x23456789)", + "UINT64_C(0x13579bdf2468ace0)", "UINT32_C(0x2468ace0)", + "UINT64_C(0x7e6d5c4b3a291807)", "UINT32_C(0x3a291807)", + "UINT64_C(0xa1b2c3d400000000)", "UINT32_C(0)", + ).Replace(x86CPUQueryIndependentC) +} + +// Real Linux IA32 execution is required separately from Rosetta, emulators +// and portable object compilation. Missing LLVM 22/cross tools fail this root. +func TestCrossLinuxRuntimeMatrixX86CPUQueryNative386(t *testing.T) { + x87IA32RequireNativeHost(t) + llc := findLLVM22Tool("llc") + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if llc == "" || err != nil { + t.Fatalf("required LLVM 22/native IA32 compiler: %v", err) + } + dir := t.TempDir() + referenceSource, referenceExecutable := filepath.Join(dir, "reference.c"), filepath.Join(dir, "reference") + if err := os.WriteFile(referenceSource, []byte("#define REFERENCE_ONLY 1\n"+x86CPUQuery386IndependentC()), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(compiler, "-static", "-no-pie", referenceSource, "-O2", "-o", referenceExecutable)) + referenceProgram, err := x87IA32StaticExecutable(referenceExecutable) + if err != nil { + t.Fatal(err) + } + reference := string(x87IA32RunDirect(t, referenceProgram, nil, os.Environ())) + lines := strings.Split(strings.TrimSpace(reference), "\n") + if len(lines) != 2 || !strings.HasPrefix(lines[0], "cpu query ") || !strings.HasPrefix(lines[1], "xgetbv ") { + t.Fatalf("actual native IA32 query reference: %q", reference) + } + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + original := x86CPUQueryBuildOriginal386(t, raw) + for _, suffix := range []string{"", "_af"} { + if err := original.verify(); err != nil { + t.Fatal(err) + } + output, err := x87IA32Capture(t, original.path, []string{suffix}, "", append(os.Environ(), "GOTRACEBACK=system")) + if guard := original.verify(); guard != nil { + t.Fatal(guard) + } + if lines[1] == "xgetbv SIGILL" { + if !x86UndefinedTrapOriginalSignal(output, err) || !strings.HasPrefix(string(output), lines[0]+"\n") { + t.Fatalf("original native IA32 query fault: %v\n%s", err, output) + } + } else if err != nil || string(output) != reference { + t.Fatalf("original native IA32 query state: %v\n%s", err, output) + } + } + declarations, assembly := x86CPUQueryRuntimeFixtures(raw, "386") + pkg := mustGoPackage(t, "cpuquery386", "package cpuquery386\n"+declarations) + ctx := llvm.NewContext() + defer ctx.Dispose() + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + GOARCH: "386", TargetTriple: "i386-unknown-linux-gnu", + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if translated != nil && !translated.Module.IsNil() { + if err != nil { + translated.Module.Dispose() + } + } + if err != nil || translated == nil || translated.Module.IsNil() { + t.Fatalf("native IA32 query module: %v", err) + } + ir := translated.Module.String() + verification := llvm.VerifyModule(translated.Module, llvm.ReturnStatusAction) + translated.Module.Dispose() + if verification != nil { + t.Fatal(verification) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + dir := t.TempDir() + ll, object := filepath.Join(dir, "query.ll"), filepath.Join(dir, "query.o") + main, executable := filepath.Join(dir, "main.c"), filepath.Join(dir, "query") + for name, contents := range map[string]string{ll: input, main: x86CPUQuery386IndependentC()} { + if err := os.WriteFile(name, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple=i386-unknown-linux-gnu", "-filetype=obj", ll, "-o", object)) + x86FloatCompareCaptureCommand(t, exec.Command(compiler, "-static", "-no-pie", object, main, "-O2", "-o", executable)) + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != reference { + t.Fatalf("actual native IA32 LLVM query state: %q / %q", output, reference) + } + }) + } + }) + } + x87IA32RequireNativeHost(t) +} diff --git a/amd64_cpu_query_native_reference_test.go b/amd64_cpu_query_native_reference_test.go new file mode 100644 index 000000000..097f83abf --- /dev/null +++ b/amd64_cpu_query_native_reference_test.go @@ -0,0 +1,82 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func x86CPUQueryBuildOriginal386(t *testing.T, raw bool) x87IA32Executable { + t.Helper() + declarations, assembly := x86CPUQueryRuntimeFixtures(raw, "386") + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module cpuquery386\n\ngo 1.20\n", "main.go": x86CPUQuery386OriginalDriver(), + "decl.go": "package main\n" + declarations, "query_386.s": assembly, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-go") + command := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-p=1", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", "GOARCH=386", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOEXPERIMENT=") + x86FloatCompareCaptureCommand(t, command) + sealed, err := x87IA32StaticExecutable(executable) + if err != nil || sealed.verify() != nil { + t.Fatalf("unchanged IA32 query reference did not compile as stable ELF32: %v", err) + } + return sealed +} + +// This portable axis compiles unchanged original Go objects, not a native +// execution pass. The required Linux counterpart runs these same programs. +func TestX86CPUQuery386OriginalNativeReferenceCompiles(t *testing.T) { + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + _ = x86CPUQueryBuildOriginal386(t, raw) + }) + } +} + +func TestX86CPUQuery386RuntimeSourceObjects(t *testing.T) { + x86CPUQueryRuntimeSourceObjects(t, "386", []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"}) +} + +func TestX86CPUQueryAMD64RuntimeSourceObjects(t *testing.T) { + x86CPUQueryRuntimeSourceObjects(t, "amd64", []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + }) +} + +func x86CPUQueryRuntimeSourceObjects(t *testing.T, arch string, triples []string) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, raw := range []bool{false, true} { + declarations, source := x86CPUQueryRuntimeFixtures(raw, arch) + for _, triple := range triples { + x86LEAActualTargetGoObject(t, arch, triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(fmt.Sprintf("%s/raw=%t/%s", triple, raw, api), func(t *testing.T) { + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, arch, triple, api) + for _, op := range []string{"cpuid", "xgetbv"} { + if strings.Count(ir, "asm sideeffect \""+op+"\"") != 2 { + t.Fatal("original native query reference lost a complete opcode", op) + } + } + compileLLVMToObject(t, llc, triple, "query.ll", "query.o", ir) + compileLLVMToObject(t, llc, triple, "query-opt.ll", "query-opt.o", x86UndefinedTrapOptimizedIR(t, ir)) + }) + } + } + } +} diff --git a/amd64_cpu_query_runtime_fixtures_test.go b/amd64_cpu_query_runtime_fixtures_test.go new file mode 100644 index 000000000..70c4508bc --- /dev/null +++ b/amd64_cpu_query_runtime_fixtures_test.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func x86CPUQueryRuntimeFixtures(raw bool, arch string) (declarations, assembly string) { + for _, op := range []Op{OpCPUID, OpXGETBV} { + name := "query" + if op == OpXGETBV { + name = "fault" + } + body := string(op) + "\n" + if raw { + body = "BYTE $0x0f\nBYTE $0xa2\n" + if op == OpXGETBV { + body = "BYTE $0x0f\nBYTE $0x01\nBYTE $0xd0\n" + } + } + for index, seed := range []uint64{0x8c7, 0x12} { + entry := name + if index != 0 { + entry += "_af" + } + declarations += fmt.Sprintf("func %s(out *[8]uint64)\n", entry) + assembly += fmt.Sprintf(`TEXT ·%s(SB),4,$0-8 + MOVQ out+0(FP),DI + MOVQ $0xa1b2c3d400000000,AX + MOVQ $0xa1b2c3d400000000,CX + MOVQ $0x7e6d5c4b3a291807,BX + MOVQ $-1,DX + PUSHQ $%#x + POPFQ + %s + MOVQ AX,0(DI) + MOVQ BX,8(DI) + MOVQ CX,16(DI) + MOVQ DX,24(DI) + PUSHFQ + POPQ R8 + MOVQ R8,32(DI) + STMXCSR 40(DI) + RET +`, entry, seed, body) + } + } + if arch == "386" { + declarations = strings.ReplaceAll(declarations, "uint64", "uint32") + assembly = strings.NewReplacer( + "$0-8", "$0-4", "MOVQ", "MOVL", "PUSHQ", "PUSHL", "POPFQ", "POPFL", + "PUSHFQ", "PUSHFL", "POPQ R8", "POPL SI", "R8,32(DI)", "SI,16(DI)", + "$0xa1b2c3d400000000", "$0", "$0x7e6d5c4b3a291807", "$0x3a291807", + "BX,8(DI)", "BX,4(DI)", "CX,16(DI)", "CX,8(DI)", "DX,24(DI)", "DX,12(DI)", + "40(DI)", "20(DI)", + ).Replace(assembly) + } else if arch != "amd64" { + panic("CPU-query fixture requires an x86 architecture") + } + return +} + +const x86CPUQueryOriginalDriver = `package main + +import ( + "fmt" + "os" + "syscall" +) + +func main() { + if err := syscall.Setrlimit(syscall.RLIMIT_CORE, &syscall.Rlimit{}); err != nil { + panic(err) + } + cpu, state, flags := query, fault, uint64(0x8c5) + if len(os.Args) == 2 && os.Args[1] == "_af" { + cpu, state, flags = query_af, fault_af, 0x10 + } + sample := func(call func(*[8]uint64)) [8]uint64 { + out := [8]uint64{0,0,0,0,0,0,0xabcdef0123456789,0x13579bdf2468ace0} + call(&out) + if out[4]&0x8d5 != flags || out[6] != 0xabcdef0123456789 || out[7] != 0x13579bdf2468ace0 { + panic(fmt.Sprintf("original query flags/canary: %x", out)) + } + return out + } + out := sample(cpu) + for _, value := range out[:4] { + if value>>32 != 0 { panic("query output did not zero-extend") } + } + fmt.Printf("cpu query %x %x %x %x\n", out[0],out[1],out[2],out[3]) + out = sample(state) + if out[0]>>32 != 0 || out[3]>>32 != 0 || out[1] != 0x7e6d5c4b3a291807 || out[2] != 0xa1b2c3d400000000 { + panic(fmt.Sprintf("original XGETBV implicit registers: %x", out)) + } + fmt.Printf("xgetbv %x %x\n", out[0],out[3]) +} +` + +func x86CPUQuery386OriginalDriver() string { + return strings.NewReplacer( + "uint64", "uint32", "0xabcdef0123456789", "0x23456789", "0x13579bdf2468ace0", "0x2468ace0", + "0x7e6d5c4b3a291807", "0x3a291807", "0xa1b2c3d400000000", "0", + ).Replace(x86CPUQueryOriginalDriver) +} diff --git a/amd64_cpu_query_runtime_unix_test.go b/amd64_cpu_query_runtime_unix_test.go new file mode 100644 index 000000000..3df4dde82 --- /dev/null +++ b/amd64_cpu_query_runtime_unix_test.go @@ -0,0 +1,186 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86CPUQueryOriginalAndLLVM(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + referenceSource, referenceExecutable := filepath.Join(dir, "reference.c"), filepath.Join(dir, "reference") + if err := os.WriteFile(referenceSource, []byte("#define REFERENCE_ONLY 1\n"+x86CPUQueryIndependentC), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, referenceSource, referenceExecutable) + reference := string(execution.run(t, referenceExecutable)) + lines := strings.Split(strings.TrimSpace(reference), "\n") + if len(lines) != 2 || !strings.HasPrefix(lines[0], "cpu query ") || !strings.HasPrefix(lines[1], "xgetbv ") { + t.Fatalf("independent native CPU/OS-state reference: %q", reference) + } + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + declarations, assembly := x86CPUQueryRuntimeFixtures(raw, "amd64") + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + original := x86CPUQueryBuildOriginal(t, execution, declarations, assembly) + for _, suffix := range []string{"", "_af"} { + arguments := execution.command(original) + arguments = append(arguments, suffix) + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, arguments[0], arguments[1:]...) + command.Env = append(os.Environ(), "GOTRACEBACK=system") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if lines[1] == "xgetbv SIGILL" { + if !x86UndefinedTrapOriginalSignal(output, err) || !strings.HasPrefix(string(output), lines[0]+"\n") { + t.Fatalf("unchanged Go XGETBV did not retain its actual unavailable-ISA fault: %v\n%s", err, output) + } + } else if err != nil || string(output) != reference { + t.Fatalf("original Go query values/zero extension/flags/canaries differ from native reference: %v\n%s\nreference=%s", err, output, reference) + } + } + ir := x86NarrowOracleIR(t, assembly, "package cpuquery\n"+declarations, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + output := execution.compileAndRunLLVM(t, "cpuquery", input, x86CPUQueryIndependentC, level) + if string(output) != reference { + t.Fatalf("LLVM query result/fault differs from independent native reference: %q / %q", output, reference) + } + }) + } + }) + } +} + +func x86CPUQueryBuildOriginal(t *testing.T, execution x86FloatCompareExecution, declarations, assembly string) string { + t.Helper() + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module cpuquery\n\ngo 1.20\n", "main.go": x86CPUQueryOriginalDriver, + "decl.go": "package main\n" + declarations, "query_amd64.s": assembly, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-go") + command := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-p=1", "-o", executable, ".") + command.Dir = dir + goos := runtime.GOOS + if strings.Contains(execution.triple, "linux") { + goos = "linux" + } + command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+goos, "GOARCH=amd64", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOEXPERIMENT=") + x86FloatCompareCaptureCommand(t, command) + return executable +} + +const x86CPUQueryIndependentC = ` +#include +#include +#include +#include +#include +#include +#include +#include + +// Observe the actual instruction, not an advertised CPUID capability. An +// emulator may mask OSXSAVE while still implementing XGETBV successfully. +static int native_state(unsigned *low, unsigned *high) { + int channel[2]; + if (pipe(channel)) return -1; + pid_t child=fork(); + if (child<0) { + close(channel[0]); + close(channel[1]); + return -1; + } + if (child==0) { + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) _exit(120); + if (close(channel[0])) _exit(121); + unsigned values[2]; + __asm__ volatile("xgetbv" : "=a"(values[0]),"=d"(values[1]) : "c"(0)); + if (write(channel[1],values,sizeof(values))!=sizeof(values) || close(channel[1])) _exit(122); + _exit(0); + } + if (close(channel[1])) return -1; + int status=0; + pid_t waited; + do { waited=waitpid(child,&status,0); } while (waited<0 && errno==EINTR); + unsigned values[2]={0,0}; + ssize_t count=read(channel[0],values,sizeof(values)); + if (close(channel[0]) || waited!=child) return -1; + if (WIFSIGNALED(status) && WTERMSIG(status)==SIGILL && count==0) return 0; + if (!WIFEXITED(status) || WEXITSTATUS(status)!=0 || count!=sizeof(values)) return -1; + *low=values[0]; + *high=values[1]; + return 1; +} + +#if !REFERENCE_ONLY +extern void query(uint64_t *); +extern void query_af(uint64_t *); +extern void fault(uint64_t *); +extern void fault_af(uint64_t *); + +static int check(uint64_t *out, uint64_t flags, unsigned csr) { + return (out[4]&0x8d5)==flags && out[5]==csr && + out[6]==UINT64_C(0xabcdef0123456789) && out[7]==UINT64_C(0x13579bdf2468ace0); +} +#endif + +int main(void) { + unsigned a,b,c,d,low=0,high=0; + int available=native_state(&low,&high); + if (available<0) return 10; + if (!__get_cpuid(0,&a,&b,&c,&d)) return 11; +#if !REFERENCE_ONLY + void (*cpu[])(uint64_t *)={query,query_af}; + void (*state[])(uint64_t *)={fault,fault_af}; + const uint64_t flags[]={0x8c5,0x10}; + for (unsigned index=0;index<2;index++) { + uint64_t out[8]={0,0,0,0,0,0,UINT64_C(0xabcdef0123456789),UINT64_C(0x13579bdf2468ace0)}; + unsigned before,after; + __asm__ volatile("stmxcsr %0" : "=m"(before)); + cpu[index](out); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + if (!check(out,flags[index],before) || after!=before || out[0]!=a || out[1]!=b || out[2]!=c || out[3]!=d) return 12; + if (available) { + state[index](out); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + if (!check(out,flags[index],before) || after!=before || out[0]!=low || out[3]!=high || + out[1]!=UINT64_C(0x7e6d5c4b3a291807) || out[2]!=UINT64_C(0xa1b2c3d400000000)) return 13; + } else { + pid_t child=fork(); + if (child<0) return 14; + if (child==0) { + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) _exit(120); + state[index](out); + _exit(91); + } + int status=0; + if (waitpid(child,&status,0)!=child || !WIFSIGNALED(status) || WTERMSIG(status)!=SIGILL) return 15; + } + } +#endif + printf("cpu query %x %x %x %x\n",a,b,c,d); + if (available) printf("xgetbv %x %x\n",low,high); + else puts("xgetbv SIGILL"); + return 0; +} +` diff --git a/amd64_crc32_forms.go b/amd64_crc32_forms.go new file mode 100644 index 000000000..b394afb73 --- /dev/null +++ b/amd64_crc32_forms.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86CRC32Form struct { + bits int + source, destination Operand +} + +// ycrc32b is Zlitm_r/Ymb,Yrl; W/L/Q use Zlitm_r/Yml,Yrl. +// Px/Pe/Pw select the source width, not the accumulator width: every row +// reads only the old low32 and writes a zero-extended CRC32 result. +var x86CRC32Specs = map[Op]int{"CRC32B": 8, "CRC32W": 16, "CRC32L": 32, "CRC32Q": 64} + +func x86CRC32Properties(op Op, ins Instr, arch string) (x86CRC32Form, bool, error) { + name, _, suffix := strings.Cut(strings.ToUpper(string(op)), ".") + bits, known := x86CRC32Specs[Op(name)] + if !known { + return x86CRC32Form{}, false, nil + } + fail := func(reason string) (x86CRC32Form, bool, error) { + return x86CRC32Form{}, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + if arch != "386" && arch != "amd64" || arch == "386" && bits == 64 || suffix || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) || len(ins.Args) != 2 { + return fail("requires its complete Go mode and source/destination row") + } + source, destination := ins.Args[0], ins.Args[1] + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, arch) { + return fail("requires a Go Yrl accumulator destination") + } + if source.Kind == OpReg { + valid := isX86YrlRegisterForArch(source.Reg, arch) + if bits == 8 { + valid = isGoYmbRegisterForArch(source.Reg, arch) + } + if !valid { + return fail("source is outside its Go Ymb/Yml register class") + } + } else { + if x86SourceTLSMemoryRole(source, arch) { + return x86CRC32Form{}, true, fmt.Errorf("%w: CRC32 TLS source needs its actual native address contract", ErrProbeNeedsContext) + } + if !x86ScalarCompareMemoryValid(source, arch) { + return fail("requires an encodable Go Ymb/Yml memory source") + } + } + if arch == "amd64" && bits == 8 && source.Kind == OpReg && !ins.x86Encoded && ins.x86SourceRaw == nil { + source.Reg = x86CRC32GoByteRegister(source.Reg, destination.Reg) + } + return x86CRC32Form{bits: bits, source: source, destination: destination}, true, nil +} + +// Unlike Pb, Go's Px/Zlitm_r CRC row never calls bytereg. Full SP/BP/SI/DI +// therefore encode AH/CH/DH/BH without REX. An extended accumulator or an +// explicit low-byte/extended source selects the REX low-byte view instead. +// This local operand copy does not rewrite original source/certificate identity. +func x86CRC32GoByteRegister(source, destination Reg) Reg { + rex := strings.HasPrefix(string(destination), "R") || strings.HasPrefix(string(source), "R") || + source == BPB || source == SIB || source == DIB + if rex { + switch source { + case AH, CH, DH, BH: + return amd64YmbEffectiveRegister(source, 16) + } + return source + } + switch source { + case SP: + return AH + case BP: + return CH + case SI: + return DH + case DI: + return BH + } + return source +} + +// NUM-only dependency registration is deliberately not admission in the raw, +// source-stack or caller grammars. All inputs precede any destination write. +func x86TypedCRC32ByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + if !x86ResultNamedPredicateInstruction(ins) { + return x86TypedByteEffect{}, false, nil + } + form, known, err := x86CRC32Properties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + return x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: 0, operand: form.source, width: int64(form.bits / 8), read: true}, + {index: 1, operand: form.destination, width: 4, read: true, write: true}, + }}, true, nil // CRC32 preserves all six status bits and DF. +} + +func x86LoweredCRC32NativeRead(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) (bool, bool) { + form, known, err := x86CRC32Properties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + if !x86ResultNamedPredicateInstruction(ins) || + !x86ResultRead(form.destination, 4, state, owned, arch) || + !x86ResultReadWithNativeContinuation(ins, form.source, int64(form.bits/8), state, owned, arch, true) { + return false, true + } + return x86ResultStore(form.destination, 4, state, owned, resultOffset, arch), true +} diff --git a/amd64_ctx.go b/amd64_ctx.go index 3eeef7edb..498b5b2b4 100644 --- a/amd64_ctx.go +++ b/amd64_ctx.go @@ -16,13 +16,17 @@ type amd64Ctx struct { goarch string targetTriple string - resolve func(string) string - sigs map[string]FuncSig - annotate bool + resolve func(string) string + sigs map[string]FuncSig + annotate bool + continuations map[string]x86Continuation + indirectLabels []string + indirectABI0 *FuncSig // proven straight-line stack-argument callback adapter tmp int - blocks []amd64Block + blocks []amd64Block + directTail *x86DirectTailRoot // Mapping between linear instruction index and block index. Used to // resolve n(PC) branches which are instruction-relative. blockBase []int @@ -31,9 +35,8 @@ type amd64Ctx struct { usedRegs map[Reg]bool regSlot map[Reg]string // gp reg -> alloca name - zeroUpperVector bool // VEX/EVEX writes clear the upper register bits - usedXRegs map[int]bool - xRegSlot map[int]string // xmm reg index -> alloca name (<16 x i8>) + usedXRegs map[int]bool + xRegSlot map[int]string // xmm reg index -> alloca name (<16 x i8>) usedYRegs map[int]bool yRegSlot map[int]string // ymm reg index -> alloca name (<32 x i8>) @@ -41,38 +44,70 @@ type amd64Ctx struct { usedZRegs map[int]bool zRegSlot map[int]string // zmm reg index -> alloca name (<64 x i8>) - usedKRegs map[int]bool - kRegSlot map[int]string // avx512 mask reg index -> alloca name (i64) - usedX87 bool - usesX87Convert bool - x87Slot [8]string // x87 stack registers, represented as f64 values - x87ControlSlot string - x87StatusSlot string // condition-code status populated by FTST for FSTSW - x87IntegerSlot string // hardware FISTP result for explicit 386 x87 lowering - x87Mode X87Mode - - flagsZSlot string - flagsSltSlot string // signed negative-style bit for J{L,LE,G,GE}-like checks - flagsCFSlot string // carry/borrow style bit for J{B,BE,A,AE,NC,C}-like checks - flagsOFSlot string // overflow-style bit used by ADOX carry chain modeling - flagsIDSlot string // CPUID availability bit preserved by PUSHFL/POPFL - flagsWritten bool - allowSPWrite bool // current instruction has an explicitly modeled 386 SP destination - directionSlot string // x86 DF, used by MOVS/STOS/SCAS instructions - repeatPrefix string // pending REP/REPN prefix for the following instruction - vstackSlot string // [64 x i64] virtual stack for PUSHQ/POPQ - vspSlot string // i64 virtual stack pointer (next free slot) - localStackSlot string // byte-addressable backing storage for x86 SP references - classicFrame string // contiguous classic Go ABI frame used by 386 FP addressing - classicSize int64 - classicBias int64 - - fpParams map[int64]FrameSlot // off(FP) -> slot - fpResults []FrameSlot - fpResAllocaOff map[int64]string // off(FP) -> alloca - fpResAllocaIdx map[int]string // result index -> alloca - fpResWritten map[int]bool // result index -> whether written via +off(FP) - fpResAddrTaken map[int]bool // result index -> address of fp_ret_* escaped + usedKRegs map[int]bool + kRegSlot map[int]string // avx512 mask reg index -> alloca name (i64) + usedX87 bool + usesX87Convert bool + x87Slot [8]string // x87 stack registers, represented as f64 values + x87IntegerValueSlot [8]string // Exact signed-integer payload for each logical value. + x87ExactSlot [8]string // Runtime tag; never source availability or entry authority. + x87ControlSlot string + x87StatusSlot string // condition-code status populated by FTST for FSTSW + x87IntegerSlot string // hardware FISTP result for explicit 386 x87 lowering + x87Mode X87Mode + + flagsZSlot string + flagsSltSlot string // architectural SF; signed conditions combine it with OF + flagsCFSlot string // carry/borrow style bit for J{B,BE,A,AE,NC,C}-like checks + flagsPFSlot string // parity bit for J{P,PE,PS,NP,PO,PC}-like checks + flagsAFSlot string // auxiliary carry/borrow from bit 3, exposed by LAHF/PUSHF + flagsOFSlot string // overflow-style bit used by ADOX carry chain modeling + flagsIDSlot string // CPUID ID bit preserved by native-width flags stack forms + flagsWritten bool + allowSPWrite bool // current instruction has an explicitly modeled 386 SP destination + directionSlot string // x86 DF, used by MOVS/STOS/SCAS instructions + repeatPrefix string // pending REP/REPN prefix for the following instruction + namedAtomicLock bool // actual same-block source-adjacent LOCK for this instruction only + lockScalarNS string // compiler name allocation, not a source capability + bitScanNS string // compiler name allocation, not a source capability + vstackSlot string // [64 x i64] virtual stack for PUSHQ/POPQ + vspSlot string // i64 virtual stack pointer (next free slot) + localStackSlot string // byte-addressable backing storage for x86 SP references + logicalAUTOBase string // immutable entry-SP SSA address, not a source pointer fact + logicalAUTOExtent int // proved ordinary logical frame bytes above entry SP + privateFlagsStack bool // complete paired mode32 proof owns bare nonnegative SP offsets + sourceStack *x86SourceStackProof + fixedLocal *x86FixedLocalProof + fixedMemRoles map[MemRef]MemRef // exact certified current-instruction native role projection + frameSize int64 // local bytes below the Go assembler's implicit amd64 BP + classicFrame string // contiguous classic Go ABI frame used by 386 FP addressing + classicSize int64 + classicBias int64 + + fpParams map[int64]FrameSlot // off(FP) -> slot + fpParamAlloca map[int64]string // off(FP) -> mutable parameter shadow + fpResults []FrameSlot + fpResAllocaOff map[int64]string // off(FP) -> alloca + fpResAllocaIdx map[int]string // result index -> alloca + fpResWritten map[int]bool // result index -> whether written via +off(FP) + fpResAddrTaken map[int]bool // result index -> address of fp_ret_* escaped + fpScalarSource Func // source for all-path checks of new scalar result views + fpSourceSig FuncSig // original request-local declaration before feature rendering + fpFeatureAttrs string // exact compiler-rendered feature reference, never supplied attributes + fixedLocalResult *x86FixedLocalResultSnapshot // pure per-call source index; live module guards remain separate + fpBoolCallSource *x86ABI0BoolCallSource // complete custom-helper graph, not a standalone register entry + fpScalarData map[x86StaticSBAddress]int64 // exact effective SB address -> available emitted numerical bytes + fpShiftLiterals map[x86StaticSBAddress]x86ShiftLiteral + goReadOnlyStorage *goReadOnlyStorage // declaration-only numerical reads, never emitted/writable DATA + goReadOnlyOperand *goReadOnlyRead // exact current instruction; cleared before the next instruction + fpScalarChecked bool + fpScalarCheckErr error + fpScalarTrapOnly bool // Complete private logical-shadow proof: every path ends in an exact trap. + fpMixedBoolBytes bool // Ordinary tuple inputs/canonical bool passed; result audit is separate. + dataBPSource bool // Complete named leaf's private opaque-pointer holder, never physical BP. + + fpStaticAddressValues *x86StaticAddressValues // No dereference capability. + nativeMMXRegion *x86NativeMMXRegion // Whole source/NUM proof paired to the actual emitted MMX region. } func newAMD64Ctx(b *strings.Builder, fn Func, sig FuncSig, resolve func(string) string, sigs map[string]FuncSig, annotate bool) *amd64Ctx { @@ -88,6 +123,10 @@ func newX86Ctx(b *strings.Builder, fn Func, sig FuncSig, resolve func(string) st resolve: resolve, sigs: sigs, annotate: annotate, + frameSize: fn.FrameSize, + sourceStack: fn.x86SourceStack, + fixedLocal: fn.x86FixedLocal, + indirectABI0: inferX86ABI0Forwarder(fn, sig, goarch), blocks: amd64SplitBlocks(fn), usedRegs: map[Reg]bool{}, regSlot: map[Reg]string{}, @@ -100,15 +139,29 @@ func newX86Ctx(b *strings.Builder, fn Func, sig FuncSig, resolve func(string) st usedKRegs: map[int]bool{}, kRegSlot: map[int]string{}, fpParams: map[int64]FrameSlot{}, + fpParamAlloca: map[int64]string{}, fpResAllocaOff: map[int64]string{}, fpResAllocaIdx: map[int]string{}, fpResWritten: map[int]bool{}, fpResAddrTaken: map[int]bool{}, + fpScalarSource: fn, + fpSourceSig: sig, + dataBPSource: x86DataBPSourceScope(fn, sig, goarch), blockByIdx: map[int]int{}, } + if fn.x86DirectTailRoot != nil { + c.directTail = fn.x86DirectTailRoot + c.blocks = c.directTail.blocks() + } for _, s := range sig.Frame.Params { c.fpParams[s.Offset] = s } + c.continuations = make(map[string]x86Continuation, len(fn.x86ContinuationAddresses)) + for symbol, target := range fn.x86ContinuationAddresses { + target.root = resolve(target.root) + c.continuations[resolve(symbol)] = target + } + c.indirectLabels = fn.x86IndirectLabels c.fpResults = append([]FrameSlot(nil), sig.Frame.Results...) base := 0 for i, blk := range c.blocks { @@ -120,7 +173,7 @@ func newX86Ctx(b *strings.Builder, fn Func, sig FuncSig, resolve func(string) st } func (c *amd64Ctx) useHardwareX87() bool { - return c.goarch == "386" && c.x87Mode != X87Software + return c.x87Mode != X87Software } func (c *amd64Ctx) emitSourceComment(ins Instr) { @@ -136,7 +189,7 @@ func (c *amd64Ctx) newTmp() string { } func (c *amd64Ctx) slotName(r Reg) string { - return "%" + amd64LLVMBlockName("reg_"+string(r)) + return "%reg_" + amd64LLVMSafeName(string(r)) } func (c *amd64Ctx) xSlotName(i int) string { @@ -208,6 +261,10 @@ func (c *amd64Ctx) scanUsedRegs() { if r == "" { return } + // Byte aliases are read and written through their containing GP slot. + if base, _, ok := amd64ByteAlias(r); ok { + r = base + } if idx, ok := amd64ParseXReg(r); ok { c.usedXRegs[idx] = true return @@ -224,7 +281,7 @@ func (c *amd64Ctx) scanUsedRegs() { c.usedKRegs[idx] = true return } - if _, ok := amd64ParseX87Reg(r); ok && c.goarch == "386" { + if _, ok := amd64ParseX87Reg(r); ok { c.usedX87 = true return } @@ -235,9 +292,13 @@ func (c *amd64Ctx) scanUsedRegs() { case OpReg: markReg(op.Reg) case OpMem: - markReg(op.Mem.Base) - if op.Mem.Index != "" { - markReg(op.Mem.Index) + memory := op.Mem + if named, valid := x86SBMemoryAddressRole(op, c.goarch); valid { + memory = named // An ignored Reg must not allocate a virtual stack. + } + markReg(memory.Base) + if memory.Index != "" { + markReg(memory.Index) } case OpRegList: for _, r := range op.RegList { @@ -248,27 +309,128 @@ func (c *amd64Ctx) scanUsedRegs() { for _, blk := range c.blocks { for _, ins := range blk.instrs { + if form, known, err := x86ScalarNativeLoadProperties(ins.Op, ins, c.goarch); known && err == nil { + markReg(form.destination) + } + if form, known, err := x86CRC32Properties(ins.Op, ins, c.goarch); known && err == nil { + markOp(form.source) // Px/Zlitm_r can select a different byte's GP root. + markOp(form.destination) + } + if effect, known, err := x86TypedPackedIntegerCompareByteEffect(ins.Op, ins, c.goarch); known && err == nil { + for _, operand := range effect.operands { + markOp(operand.operand) + } + } op := strings.ToUpper(string(ins.Op)) - if c.goarch == "386" && isX87Op(Op(op)) { + if dot := strings.IndexByte(op, '.'); dot >= 0 { + op = op[:dot] + } + if isX87Op(Op(op)) { c.usedX87 = true if op == "FMOVVP" { c.usesX87Convert = true } } + if op == "LAHF" || op == "SAHF" { + markReg(AX) + } + if spec, ok := amd64ImplicitSystemSpecs[Op(op)]; ok { + registers := spec.inputs | spec.outputs + if registers&amd64ImplicitAX != 0 { + markReg(AX) + } + if registers&amd64ImplicitCX != 0 { + markReg(CX) + } + if registers&amd64ImplicitDX != 0 { + markReg(DX) + } + } + if spec, known, err := x86TimestampProperties(Op(op), ins, c.goarch); known && err == nil { + for _, register := range x86TimestampRegisters(spec) { + markReg(register) + } + } + if spec, known, err := x86CPUQueryProperties(Op(op), ins, c.goarch); known && err == nil { + for _, register := range spec.inputs { + markReg(register) + } + for _, register := range spec.outputs { + markReg(register) + } + } + if spec, known, err := amd64PackedStringCompareProperties(Op(op), ins, c.goarch); known && err == nil { + output, _, _ := spec.resultEffect() + markReg(output) + if spec.explicitLength { + markReg(AX) + markReg(DX) + } + } + if spec, ok := amd64RTMSpecs[Op(op)]; ok && spec.outputAX { + markReg(AX) + } + if spec, ok := amd64SystemTransferSpecs[Op(op)]; ok { + if spec.inputs&amd64SystemTransferCX != 0 { + markReg(CX) + } + if spec.inputs&amd64SystemTransferDX != 0 { + markReg(DX) + } + if c.goarch != "386" && spec.inputs&amd64SystemTransferR11 != 0 { + markReg(Reg("R11")) + } + } + if _, ok := amd64LeaveSpecs[Op(op)]; ok { + markReg(BP) + markReg(SP) + } + if op == "XLAT" { + markReg(AX) + markReg(BX) + } if c.goarch == "386" { switch op { - case "PUSHL", "POPL", "PUSHFL", "POPFL", "PUSHAL", "POPAL", "ADJSP": + case "PUSHW", "POPW", "PUSHFW", "POPFW", "PUSHL", "POPL", "PUSHFL", "POPFL", "PUSHAL", "POPAL", "ADJSP": markReg(SP) } } for _, arg := range ins.Args { markOp(arg) } + if form, known, err := x86PackedAverageProperties(ins.Op, ins, c.goarch); known && err == nil { + markOp(form.source) + } + if form, recognized, err := x86ScalarExtensionProperties(Op(op), ins, c.goarch); recognized && err == nil && form.source.Kind == OpReg { + markReg(form.source.Reg) + } + if _, fourSource := amd64FourSourceSpecs[Op(op)]; fourSource && len(ins.Args) >= 2 && ins.Args[1].Kind == OpRegList && len(ins.Args[1].RegList) != 0 { + first := ins.Args[1].RegList[0] + if index, ok := amd64ParseXReg(first); ok { + base := index &^ 3 + for register := base; register < base+4; register++ { + markReg(Reg(fmt.Sprintf("X%d", register))) + } + } else if index, ok := amd64ParseZReg(first); ok { + base := index &^ 3 + for register := base; register < base+4; register++ { + markReg(Reg(fmt.Sprintf("Z%d", register))) + } + } + } + if len(ins.Args) == 1 && ins.Args[0].Kind == OpReg { + if spec, ok := x86UnaryYmbSpecs[Op(op)]; ok { + markReg(amd64YmbEffectiveRegister(ins.Args[0].Reg, spec.bits)) + } + } } } // Ensure a few common regs exist even if only used implicitly by helpers. markReg(AX) + if c.goarch != "386" && c.frameSize > 0 && c.usedRegs[BP] { + markReg(SP) + } // Ensure arg regs exist for ABIInternal-style stdlib asm. This matters for: // - functions like runtime·cmpstring that tail-call helpers @@ -295,8 +457,45 @@ func (c *amd64Ctx) scanUsedRegs() { } } +func (c *amd64Ctx) vectorAllocationWidths() map[int]int { + widths := make(map[int]int) + for i := range c.usedXRegs { + widths[i] = 16 + } + for i := range c.usedYRegs { + widths[i] = 32 + } + for i := range c.usedZRegs { + widths[i] = 64 + } + return widths +} + func (c *amd64Ctx) emitEntryAllocas() error { + if err := c.prepareLogicalAUTOStorage(); err != nil { + return err + } + resultRegion, regionalResults := x86DirectTailResultRegion(c.fpScalarSource, c.sig, c.goarch) + if regionalResults && (!c.fpScalarChecked || c.fpScalarCheckErr != nil) { + return fmt.Errorf("%w: contiguous result backing lacks its complete source-byte proof", ErrProbeNeedsContext) + } + if c.fixedLocal != nil && (c.fixedLocal.binding == nil || c.fixedLocal.binding != c.sig.x86FixedLocal || + c.goarch != "amd64" || c.targetTriple != c.fixedLocal.binding.target || c.frameSize != c.fixedLocal.frame) { + return goAMD64FrameContext("fixed local storage consumer lost its sealed source identity") + } c.scanUsedRegs() + if c.sourceStack != nil || c.fixedLocal != nil { + c.usedRegs[SP] = true + } + var framedCalls []int + if c.directTail != nil { + for pc, ins := range c.directTail.instructions { + if call, framed := c.directTail.privateFramedCall(pc); framed && ins.Op == OpCALL { + framedCalls = append(framedCalls, call) + c.usedRegs[BP] = true + } + } + } regs := make([]string, 0, len(c.usedRegs)) for r := range c.usedRegs { @@ -304,7 +503,7 @@ func (c *amd64Ctx) emitEntryAllocas() error { } sort.Strings(regs) - c.b.WriteString(amd64LLVMBlockName("entry") + ":\n") + c.b.WriteString(amd64LLVMBlockName(c.blocks[0].name) + ":\n") for _, rs := range regs { r := Reg(rs) name := c.slotName(r) @@ -312,41 +511,80 @@ func (c *amd64Ctx) emitEntryAllocas() error { fmt.Fprintf(c.b, " %s = alloca i64\n", name) fmt.Fprintf(c.b, " store i64 0, ptr %s\n", name) } - if spSlot, ok := c.regSlot[SP]; ok && c.goarch == "386" { - minOff, maxOff := c.stackOffsetRange() - movement, err := c.stackMovementBudget() - if err != nil { - return err + for _, call := range framedCalls { + fmt.Fprintf(c.b, " %%private_call_bp_%d = alloca i64\n", call) + } + if spSlot, ok := c.regSlot[SP]; ok { + if c.fixedLocal != nil { + // The complete proof excludes saved BP/return/incoming FP bytes, + // proves every observed word-aligned SP residue, and prevents local + // addresses escaping or being compared numerically. A word-aligned + // physical buffer preserves that residue domain, not a stronger + // SIMD-aligned shadow capable of repairing an uncertain source. + size := c.fixedLocal.frame + 64 + c.localStackSlot = "%fixed_local_native" + fmt.Fprintf(c.b, " %s = alloca [%d x i8], align 8\n", c.localStackSlot, size) + base := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr inbounds [%d x i8], ptr %s, i32 0, i64 32\n", base, size, c.localStackSlot) + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %%%s to i64\n", address, base) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s\n", address, spSlot) + } else { + minOff, maxOff := c.stackOffsetRange() + if c.sourceStack != nil && c.sourceStack.minimum < minOff { + minOff = c.sourceStack.minimum + } + var movement int64 + if c.goarch == "386" { + var err error + movement, err = c.stackMovementBudget() + if err != nil { + return err + } + } + minOff -= movement + maxOff += movement + const guard = int64(64) + bias := guard - minOff + if c.sourceStack != nil { + // A native Go entry SP is word-aligned. Preserve that same + // congruence so proved AND alignment has identical byte offsets. + word := c.sourceStack.word + bias = (bias + word - 1) & -word + } + size := bias + maxOff + guard + if size < 256 { + size = 256 + } + c.localStackSlot = "%local_stack" + if c.sourceStack != nil { + fmt.Fprintf(c.b, " %s = alloca [%d x i8], align %d\n", c.localStackSlot, size, c.sourceStack.word) + } else { + fmt.Fprintf(c.b, " %s = alloca [%d x i8]\n", c.localStackSlot, size) + } + base := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr inbounds [%d x i8], ptr %s, i32 0, i64 %d\n", base, size, c.localStackSlot, bias) + addr := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %%%s to i64\n", addr, base) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s\n", addr, spSlot) + if c.logicalAUTOExtent != 0 { + c.logicalAUTOBase = "%" + addr + } + if bpSlot, ok := c.regSlot[BP]; ok && c.goarch != "386" && c.frameSize > 0 { + // Go saves BP before reserving TEXT's local frame. Thus a source + // -8(BP) aliases FrameSize-8(SP), including across outgoing calls. + bp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %d\n", bp, addr, c.frameSize) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s\n", bp, bpSlot) + } } - minOff -= movement - maxOff += movement - const guard = int64(64) - bias := guard - minOff - size := bias + maxOff + guard - if size < 256 { - size = 256 - } - c.localStackSlot = "%local_stack" - fmt.Fprintf(c.b, " %s = alloca [%d x i8]\n", c.localStackSlot, size) - base := c.newTmp() - fmt.Fprintf(c.b, " %%%s = getelementptr inbounds [%d x i8], ptr %s, i32 0, i64 %d\n", base, size, c.localStackSlot, bias) - addr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %%%s to i64\n", addr, base) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s\n", addr, spSlot) } - // XMM, YMM and ZMM are overlapping views of the same physical register. - // Allocate the widest view used by the function and share its low bytes. - widths := make(map[int]int) - for i := range c.usedXRegs { - widths[i] = 16 - } - for i := range c.usedYRegs { - widths[i] = 32 - } - for i := range c.usedZRegs { - widths[i] = 64 - } + // Keep all live X/Y/Z views on one physical byte bank. Encoding-specific + // writers still decide whether to preserve or clear the upper bytes; + // source mnemonic prefixes cannot select that effect (Go rewrites some + // V-prefixed scalar literals into legacy SSE instructions). + widths := c.vectorAllocationWidths() indices := make([]int, 0, len(widths)) for i := range widths { indices = append(indices, i) @@ -357,8 +595,7 @@ func (c *amd64Ctx) emitEntryAllocas() error { name := c.xSlotName(i) if width == 32 { name = fmt.Sprintf("%%y%d", i) - } - if width == 64 { + } else if width == 64 { name = fmt.Sprintf("%%z%d", i) } c.xRegSlot[i] = name @@ -389,6 +626,12 @@ func (c *amd64Ctx) emitEntryAllocas() error { c.x87Slot[i] = name fmt.Fprintf(c.b, " %s = alloca double\n", name) fmt.Fprintf(c.b, " store double 0.000000e+00, ptr %s\n", name) + c.x87IntegerValueSlot[i] = fmt.Sprintf("%%x87_integer%d", i) + c.x87ExactSlot[i] = fmt.Sprintf("%%x87_exact%d", i) + fmt.Fprintf(c.b, " %s = alloca i64\n", c.x87IntegerValueSlot[i]) + fmt.Fprintf(c.b, " store i64 0, ptr %s\n", c.x87IntegerValueSlot[i]) + fmt.Fprintf(c.b, " %s = alloca i1\n", c.x87ExactSlot[i]) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.x87ExactSlot[i]) } c.x87ControlSlot = "%x87_control" c.x87StatusSlot = "%x87_status" @@ -401,21 +644,24 @@ func (c *amd64Ctx) emitEntryAllocas() error { fmt.Fprintf(c.b, " %s = alloca i64\n", c.x87IntegerSlot) } } - c.flagsZSlot = "%flags_z" c.flagsSltSlot = "%flags_slt" c.flagsCFSlot = "%flags_cf" + c.flagsPFSlot = "%flags_pf" + c.flagsAFSlot = "%flags_af" c.flagsOFSlot = "%flags_of" - if c.goarch == "386" { - c.directionSlot = "%direction_backward" - c.flagsIDSlot = "%flags_id" - } + c.directionSlot = "%direction_backward" + c.flagsIDSlot = "%flags_id" fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsZSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsZSlot) fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsSltSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsSltSlot) fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsCFSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) + fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsPFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsPFSlot) + fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsAFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsAFSlot) fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsOFSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) if c.flagsIDSlot != "" { @@ -445,17 +691,51 @@ func (c *amd64Ctx) emitEntryAllocas() error { return err } } else { - for _, r := range c.fpResults { - name := fmt.Sprintf("%%fp_ret_%d", r.Index) - c.fpResAllocaIdx[r.Index] = name - c.fpResAllocaOff[r.Offset] = name - fmt.Fprintf(c.b, " %s = alloca %s\n", name, r.Type) - fmt.Fprintf(c.b, " store %s %s, ptr %s\n", r.Type, llvmZeroValue(r.Type), name) + // Named FP parameters have mutable frame-slot semantics in Go assembly. + // Keep shadow storage even though LLVM function arguments are SSA values; + // assembly kernels commonly advance slice pointers by writing base+off(FP) + // and reading the updated value in a later loop iteration. + for _, slot := range c.sig.Frame.Params { + if slot.Index < 0 || slot.Index >= len(c.sig.Args) { + return fmt.Errorf("FP frame slot: invalid arg index %d at +%d(FP)", slot.Index, slot.Offset) + } + name := amd64FPParamSlotName(slot.Offset) + c.fpParamAlloca[slot.Offset] = name + value := fmt.Sprintf("%%arg%d", slot.Index) + if fields := frameSlotFields(slot); len(fields) != 0 { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s%s\n", extracted, c.sig.Args[slot.Index], value, frameSlotExtractSuffix(slot)) + value = "%" + extracted + } + fmt.Fprintf(c.b, " %s = alloca %s\n", name, x86FPStorageType(slot.Type)) + c.storeFPStorage(slot.Type, value, name, "", "") + } + if regionalResults { + c.emitDirectTailResultRegion(resultRegion) + } else { + for _, r := range c.fpResults { + name := fmt.Sprintf("%%fp_ret_%d", r.Index) + c.fpResAllocaIdx[r.Index] = name + c.fpResAllocaOff[r.Offset] = name + fmt.Fprintf(c.b, " %s = alloca %s\n", name, x86FPStorageType(r.Type)) + c.storeFPStorage(r.Type, llvmZeroValue(r.Type), name, "", "") + } + } + } + if regionalResults { + // One region address may reach any of its fields. RET must load each + // member's actual storage, never a return-register fallback. This is + // addressability, not a numerical initialization fact. + for _, slot := range c.fpResults { + c.markFPResultAddrTaken(slot.Offset) } } // Map LLVM args -> simulated registers for ABIInternal-ish entrypoints and // for helper<> bodies with explicit ArgRegs. + if c.sig.amd64GoArguments != nil { + return c.emitAMD64GoArguments() + } if len(c.sig.ArgRegs) > 0 { for i := 0; i < len(c.sig.Args) && i < len(c.sig.ArgRegs); i++ { r := c.sig.ArgRegs[i] @@ -525,6 +805,13 @@ func (c *amd64Ctx) emitEntryAllocas() error { return nil } +func amd64FPParamSlotName(off int64) string { + if off < 0 { + return fmt.Sprintf("%%fp_arg_n%d", -off) + } + return fmt.Sprintf("%%fp_arg_%d", off) +} + func x86FrameTypeSize(ty LLVMType) int64 { switch ty { case I1, I8: @@ -591,29 +878,33 @@ func (c *amd64Ctx) emit386ClassicFrame() error { return fmt.Errorf("FP frame slot: invalid arg index %d at +%d(FP)", slot.Index, slot.Offset) } value := fmt.Sprintf("%%arg%d", slot.Index) - if slot.Field >= 0 { + if fields := frameSlotFields(slot); len(fields) != 0 { extracted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", extracted, c.sig.Args[slot.Index], value, slot.Field) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s%s\n", extracted, c.sig.Args[slot.Index], value, frameSlotExtractSuffix(slot)) value = "%" + extracted } - fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", slot.Type, value, c.classicFramePtr(slot.Offset)) + c.storeFPStorage(slot.Type, value, c.classicFramePtr(slot.Offset), ", align 1", "") } for _, slot := range c.fpResults { ptr := c.classicFramePtr(slot.Offset) c.fpResAllocaIdx[slot.Index] = ptr c.fpResAllocaOff[slot.Offset] = ptr - fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", slot.Type, llvmZeroValue(slot.Type), ptr) + c.storeFPStorage(slot.Type, llvmZeroValue(slot.Type), ptr, ", align 1", "") } return nil } func (c *amd64Ctx) stackOffsetRange() (minOff, maxOff int64) { + maxOff = c.frameSize for _, block := range c.blocks { for _, ins := range block.instrs { for _, arg := range ins.Args { if arg.Kind != OpMem || arg.Mem.Base != SP { continue } + if _, static := x86SBMemoryAddressRole(arg, c.goarch); static { + continue + } if arg.Mem.Off < minOff { minOff = arg.Mem.Off } @@ -648,6 +939,10 @@ func (c *amd64Ctx) stackMovementBudget() (int64, error) { if err := add(4, ins); err != nil { return 0, err } + case "PUSHW", "POPW", "PUSHFW", "POPFW": + if err := add(2, ins); err != nil { + return 0, err + } case "PUSHAL", "POPAL": if err := add(32, ins); err != nil { return 0, err @@ -664,6 +959,26 @@ func (c *amd64Ctx) stackMovementBudget() (int64, error) { return 0, err } } + case "INCL", "DECL": + if is386SPDestination(ins) { + if err := add(1, ins); err != nil { + return 0, err + } + } + case "INCW", "DECW": + if is386SPDestination(ins) { + // A low-word wrap can move the modeled SP by as much as 65535. + if err := add(65535, ins); err != nil { + return 0, err + } + } + case "ADDW", "SUBW": + if is386SPDestination(ins) { + // A low-word update can move the modeled SP by up to 65535. + if err := add(65535, ins); err != nil { + return 0, err + } + } case "ANDL": if movement, ok := bounded386SPAndMovement(ins); ok { if err := add(movement, ins); err != nil { @@ -709,15 +1024,67 @@ func bounded386SPAndMovement(ins Instr) (int64, bool) { } func models386SPWrite(ins Instr) bool { + op := strings.ToUpper(string(ins.Op)) + if dot := strings.IndexByte(op, '.'); dot >= 0 { + op = op[:dot] + } + if op == "MOVQ" && len(ins.Args) == 2 && ins.Args[1].Kind == OpReg && ins.Args[1].Reg == SP && ins.Args[0].Kind == OpReg { + class, _, special := x86MachineRegister(ins.Args[0].Reg) + return special && (class == "cr" || class == "dr") + } + if len(ins.Args) == 1 && ins.Args[0].Kind == OpReg { + if spec, ok := x86UnaryYmbSpecs[Op(op)]; ok && spec.bits != 64 { + return amd64YmbEffectiveRegister(ins.Args[0].Reg, spec.bits) == SP + } + } if !is386SPDestination(ins) { return false } - switch strings.ToUpper(string(ins.Op)) { - case "MOVL", "ADDL", "SUBL", "LEAL", "POPL": + if _, ok := amd64BMI2ShiftSpecs[Op(op)]; ok { + return true + } + if _, known, err := x86ScalarFloatIntegerProperties(ins, "386"); known { + return err == nil + } + switch op { + case "MOVL", "ADDW", "SUBW", "ADCW", "SBBW", "ADDL", "SUBL", "ADCL", "SBBL", "ADCXL", "ADOXL", "LEAW", "LEAL", "POPW", "POPL", + "CRC32B", "CRC32W", "CRC32L", + "CMPXCHGW", "CMPXCHGL", + "POPCNTW", "POPCNTL", "TZCNTW", "TZCNTL", "LZCNTW", "LZCNTL", "PMOVMSKB", "VPMOVMSKB", + "BEXTRL", "BEXTRQ", "BZHIL", "BZHIQ", "BLSIL", "BLSIQ", "BLSMSKL", "BLSMSKQ", "BLSRL", "BLSRQ", + "ANDNL", "ANDNQ", + "PDEPL", "PDEPQ", "PEXTL", "PEXTQ": // These are the direct SP forms used by the official 386 corpus. MOVL // and non-SP LEAL rebase to an explicitly supplied stack context; // arithmetic and SP-relative LEAL retain the current stack model. return true + case "RDPID": + return len(ins.Args) == 1 && ins.Args[0].Kind == OpReg && ins.Args[0].Reg == SP + case "RCLW", "RCLL", "RCRW", "RCRL", "ROLW", "ROLL", "RORW", "RORL", "SARW", "SARL", + "SALW", "SALL", "SHLW", "SHLL", "SHRW", "SHRL": + return true + case "INCW", "DECW", "INCL", "DECL": + return true + case "BTCW", "BTCL", "BTRW", "BTRL", "BTSW", "BTSL": + return true + case "SLDTW", "SLDTL", "SMSWW", "SMSWL", "STRW", "STRL": + // Go's Yml destination class includes SP in 386 mode. These system + // instructions write their result directly to that destination. + return true + case "LARW", "LARL", "LSLW", "LSLL": + // LAR/LSL use the same Yrl destination class, including SP. + return true + case "LFSW", "LFSL", "LGSW", "LGSL", "LSSW", "LSSL": + // Far-pointer loads also use Yrl, including SP, for their offset. + return true + case "RDFSBASEL", "RDGSBASEL": + // The 386 FSGSBASE read forms use Yrl destinations, including SP. + return true + case "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", + "MOVWLSX", "MOVWLZX", "MOVLQZX", "MOVSWW", "MOVZWW": + return true + case "KMOVB", "KMOVW", "KMOVD", "KMOVQ": + return len(ins.Args) == 2 && amd64IsKOperand(ins.Args[0]) case "ANDL": _, ok := bounded386SPAndMovement(ins) return ok @@ -760,6 +1127,37 @@ func (c *amd64Ctx) popI64() string { return "%" + val } +func (c *amd64Ctx) pushI16(v string) error { + sp, err := c.loadReg(SP) + if err != nil { + return err + } + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i64 %s, 2\n", next, sp) + if err := c.storeRegUnchecked(SP, "%"+next); err != nil { + return err + } + p := c.ptrFromAddrI64("%" + next) + fmt.Fprintf(c.b, " store i16 %s, ptr %s, align 1\n", v, p) + return nil +} + +func (c *amd64Ctx) popI16() (string, error) { + sp, err := c.loadReg(SP) + if err != nil { + return "", err + } + p := c.ptrFromAddrI64(sp) + v := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s, align 1\n", v, p) + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, 2\n", next, sp) + if err := c.storeRegUnchecked(SP, "%"+next); err != nil { + return "", err + } + return "%" + v, nil +} + func (c *amd64Ctx) pushI32(v string) error { sp, err := c.loadReg(SP) if err != nil { @@ -838,6 +1236,28 @@ func amd64ByteAlias(rr Reg) (base Reg, shift uint, ok bool) { return DX, 0, true case DH: return DX, 8, true + case BPB: + return BP, 0, true + case SIB: + return SI, 0, true + case DIB: + return DI, 0, true + case R8B: + return Reg("R8"), 0, true + case R9B: + return Reg("R9"), 0, true + case R10B: + return Reg("R10"), 0, true + case R11B: + return Reg("R11"), 0, true + case R12B: + return Reg("R12"), 0, true + case R13B: + return Reg("R13"), 0, true + case R14B: + return Reg("R14"), 0, true + case R15B: + return Reg("R15"), 0, true default: return "", 0, false } @@ -877,6 +1297,9 @@ func amd64ByteRegBase(r Reg) (base Reg, shift uint, ok bool) { } func (c *amd64Ctx) loadReg(r Reg) (string, error) { + if r == TLS { + return "", fmt.Errorf("%w: TLS pseudo-register needs its native Go TLS source contract", ErrProbeNeedsContext) + } // Model byte aliases used by stdlib asm. Reads return the selected byte as a // zero-extended i64. if base, shift, ok := amd64ByteAlias(r); ok { @@ -1000,21 +1423,6 @@ func (c *amd64Ctx) storeRegSized(r Reg, ty LLVMType, v string) error { } } -// clearVectorUpper clears bytes above the written width while leaving the -// low view unchanged. Legacy SSE writes deliberately preserve those bytes. -func (c *amd64Ctx) clearVectorUpper(idx, lowBytes int) { - slot, width := c.zRegSlot[idx], 64 - if slot == "" { - slot, width = c.yRegSlot[idx], 32 - } - if slot == "" || width <= lowBytes { - return - } - upper := c.newTmp() - fmt.Fprintf(c.b, " %%%s = getelementptr i8, ptr %s, i64 %d\n", upper, slot, lowBytes) - fmt.Fprintf(c.b, " store <%d x i8> zeroinitializer, ptr %%%s, align 1\n", width-lowBytes, upper) -} - func (c *amd64Ctx) loadX(r Reg) (string, error) { idx, ok := amd64ParseXReg(r) if !ok { @@ -1022,7 +1430,7 @@ func (c *amd64Ctx) loadX(r Reg) (string, error) { } slot, ok := c.xRegSlot[idx] if !ok { - return "<16 x i8> zeroinitializer", nil + return "zeroinitializer", nil } t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s\n", t, slot) @@ -1038,9 +1446,6 @@ func (c *amd64Ctx) storeX(r Reg, v string) error { if !ok { return nil } - if c.zeroUpperVector { - c.clearVectorUpper(idx, 16) - } fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s\n", v, slot) return nil } @@ -1052,7 +1457,7 @@ func (c *amd64Ctx) loadY(r Reg) (string, error) { } slot, ok := c.yRegSlot[idx] if !ok { - return "<32 x i8> zeroinitializer", nil + return "zeroinitializer", nil } t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load <32 x i8>, ptr %s\n", t, slot) @@ -1060,19 +1465,12 @@ func (c *amd64Ctx) loadY(r Reg) (string, error) { } func (c *amd64Ctx) storeY(r Reg, v string) error { - idx, ok := amd64ParseYReg(r) - if !ok { + if _, ok := amd64ParseYReg(r); !ok { return fmt.Errorf("not a Y reg: %s", r) } - slot, ok := c.yRegSlot[idx] - if !ok { - return nil - } - if c.zeroUpperVector { - c.clearVectorUpper(idx, 32) - } - fmt.Fprintf(c.b, " store <32 x i8> %s, ptr %s\n", v, slot) - return nil + // A 256-bit VEX write also replaces X and clears the upper Z view. + // Raw and named instructions must observe the same physical register. + return c.storePackedMoveOperand(Operand{Kind: OpReg, Reg: r}, 32, v) } func (c *amd64Ctx) loadZ(r Reg) (string, error) { @@ -1082,7 +1480,7 @@ func (c *amd64Ctx) loadZ(r Reg) (string, error) { } slot, ok := c.zRegSlot[idx] if !ok { - return "<64 x i8> zeroinitializer", nil + return "zeroinitializer", nil } t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load <64 x i8>, ptr %s\n", t, slot) @@ -1142,6 +1540,7 @@ func (c *amd64Ctx) setZSFlagsFromI64(v string) { slt := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp slt i64 %s, 0\n", slt, v) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", slt, c.flagsSltSlot) + c.setParityFlagSized(I64, v) } func (c *amd64Ctx) setZSFlagsFromI32(v string) { @@ -1151,6 +1550,33 @@ func (c *amd64Ctx) setZSFlagsFromI32(v string) { slt := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp slt i32 %s, 0\n", slt, v) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", slt, c.flagsSltSlot) + c.setParityFlagSized(I32, v) +} + +func (c *amd64Ctx) setParityFlagSized(ty LLVMType, v string) { + low := v + if ty != I8 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to i8\n", truncated, ty, v) + low = "%" + truncated + } + shift4 := c.newTmp() + fold4 := c.newTmp() + shift2 := c.newTmp() + fold2 := c.newTmp() + shift1 := c.newTmp() + fold1 := c.newTmp() + bit := c.newTmp() + parity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i8 %s, 4\n", shift4, low) + fmt.Fprintf(c.b, " %%%s = xor i8 %s, %%%s\n", fold4, low, shift4) + fmt.Fprintf(c.b, " %%%s = lshr i8 %%%s, 2\n", shift2, fold4) + fmt.Fprintf(c.b, " %%%s = xor i8 %%%s, %%%s\n", fold2, fold4, shift2) + fmt.Fprintf(c.b, " %%%s = lshr i8 %%%s, 1\n", shift1, fold2) + fmt.Fprintf(c.b, " %%%s = xor i8 %%%s, %%%s\n", fold1, fold2, shift1) + fmt.Fprintf(c.b, " %%%s = and i8 %%%s, 1\n", bit, fold1) + fmt.Fprintf(c.b, " %%%s = icmp eq i8 %%%s, 0\n", parity, bit) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", parity, c.flagsPFSlot) } func (c *amd64Ctx) setCmpFlags(a, b string) { @@ -1165,6 +1591,9 @@ func (c *amd64Ctx) setCmpFlags(a, b string) { ult := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, %s\n", ult, a, b) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", ult, c.flagsCFSlot) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i64 %s, %s\n", result, a, b) + c.setParityFlagSized(I64, "%"+result) } func (c *amd64Ctx) loadFlag(slot string) string { @@ -1181,6 +1610,25 @@ func (c *amd64Ctx) fpParam(off int64) (slot FrameSlot, ok bool) { return s, true } +func (c *amd64Ctx) loadFPParamValue(slot FrameSlot) (string, error) { + if slot.Index < 0 || slot.Index >= len(c.sig.Args) { + return "", fmt.Errorf("FP read slot: invalid arg index %d at +%d(FP)", slot.Index, slot.Offset) + } + if c.classicFrame != "" { + return c.loadFPStorage(slot.Type, c.classicFramePtr(slot.Offset), ", align 1"), nil + } + if shadow := c.fpParamAlloca[slot.Offset]; shadow != "" { + return c.loadFPStorage(slot.Type, shadow, ""), nil + } + value := fmt.Sprintf("%%arg%d", slot.Index) + if fields := frameSlotFields(slot); len(fields) != 0 { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s%s\n", extracted, c.sig.Args[slot.Index], value, frameSlotExtractSuffix(slot)) + value = "%" + extracted + } + return value, nil +} + func (c *amd64Ctx) fpResultAlloca(off int64) (string, LLVMType, bool) { s, ok := c.fpParams[off] _ = s @@ -1233,13 +1681,14 @@ func (c *amd64Ctx) evalFPToI64(off int64) (string, error) { } } if alloca, ty, rok := c.fpResultAlloca(off); rok && ty != "" { - ld := c.newTmp() align := "" if c.classicFrame != "" { align = ", align 1" } - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s%s\n", ld, ty, alloca, align) - v := "%" + ld + if ty == I1 && c.abi0BoolResultByteStorage() { + ty = I8 // An intermediate numerical read is not a logical bool conversion. + } + v := c.loadFPStorage(ty, alloca, align) switch ty { case I64: return v, nil @@ -1283,24 +1732,23 @@ func (c *amd64Ctx) evalFPToI64(off int64) (string, error) { // inference (common in low-level runtime assembly). return "0", nil } - idx := slot.Index - if idx < 0 || idx >= len(c.sig.Args) { - return "", fmt.Errorf("FP read slot: invalid arg index %d at +%d(FP)", idx, off) - } - arg := fmt.Sprintf("%%arg%d", idx) - - // If this FP slot refers to a field within an aggregate argument (string/slice), - // extract that field first. ty := slot.Type - if c.classicFrame != "" { - loaded := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", loaded, ty, c.classicFramePtr(off)) - arg = "%" + loaded - } else if slot.Field >= 0 { - aggTy := c.sig.Args[idx] - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", t, aggTy, arg, slot.Field) - arg = "%" + t + if ty == I1 && c.abi0BoolResultByteStorage() { + pointer := c.fpParamAlloca[slot.Offset] + if c.classicFrame != "" { + pointer = c.classicFramePtr(slot.Offset) + } + if pointer == "" { + return "", fmt.Errorf("missing physical bool parameter copy at +%d(FP)", slot.Offset) + } + byteValue := c.loadFPStorage(I8, pointer, ", align 1") + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i8 %s to i64\n", wide, byteValue) + return "%" + wide, nil + } + arg, err := c.loadFPParamValue(slot) + if err != nil { + return "", err } switch ty { case Ptr: @@ -1342,33 +1790,143 @@ func (c *amd64Ctx) evalFPToI64(off int64) (string, error) { } } +func amd64IntegerTypeBits(ty LLVMType) (int, bool) { + switch ty { + case I1: + return 1, true + case I8: + return 8, true + case I16: + return 16, true + case I32: + return 32, true + case I64: + return 64, true + default: + return 0, false + } +} + +func amd64IntegerTypeForBits(bits int) LLVMType { + switch bits { + case 1: + return I1 + case 8: + return I8 + case 16: + return I16 + case 32: + return I32 + case 64: + return I64 + default: + return "" + } +} + +func (c *amd64Ctx) coerceFPStoreValue(from, to LLVMType, value string) (string, error) { + if from == to { + return value, nil + } + pointerBits := 64 + if c.goarch == "386" { + pointerBits = 32 + } + + rawType := from + rawValue := value + switch from { + case Ptr: + rawType = amd64IntegerTypeForBits(pointerBits) + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to %s\n", cast, value, rawType) + rawValue = "%" + cast + case LLVMType("float"): + rawType = I32 + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", cast, value) + rawValue = "%" + cast + case LLVMType("double"): + rawType = I64 + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", cast, value) + rawValue = "%" + cast + default: + if _, ok := amd64IntegerTypeBits(from); !ok { + return "", fmt.Errorf("unsupported FP store source type %s", from) + } + } + + targetBits := 0 + switch to { + case Ptr: + targetBits = pointerBits + case LLVMType("float"): + targetBits = 32 + case LLVMType("double"): + targetBits = 64 + default: + var ok bool + targetBits, ok = amd64IntegerTypeBits(to) + if !ok { + return "", fmt.Errorf("unsupported FP store destination type %s", to) + } + } + + fromBits, _ := amd64IntegerTypeBits(rawType) + targetIntType := amd64IntegerTypeForBits(targetBits) + if fromBits > targetBits { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", cast, rawType, rawValue, targetIntType) + rawType, rawValue = targetIntType, "%"+cast + } else if fromBits < targetBits { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", cast, rawType, rawValue, targetIntType) + rawType, rawValue = targetIntType, "%"+cast + } + + switch to { + case Ptr: + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr %s %s to ptr\n", cast, rawType, rawValue) + return "%" + cast, nil + case LLVMType("float"), LLVMType("double"): + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", cast, rawType, rawValue, to) + return "%" + cast, nil + default: + return rawValue, nil + } +} + func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { + return c.storeFPResultWithMetadata(off, ty, v, "") +} + +func (c *amd64Ctx) storeFPResultWithMetadata(off int64, ty LLVMType, v, metadata string) error { align := "" if c.classicFrame != "" { // The 386 ABI frame is a byte array, and FP offsets need not provide // the natural alignment required by the value stored in a slot. align = ", align 1" } - if slot, ok := c.fpParams[off]; ok && c.classicFrame != "" { - // 386 ABIInternal stubs write register arguments back into the - // caller-provided FP argument area before tail-calling an ABI0 - // handler (for example runtime.panicIndex). The classic frame is - // deliberately mutable, unlike an LLVM SSA argument. - value := v - switch { - case ty == slot.Type: - case ty == I32 && slot.Type == Ptr: - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = inttoptr i32 %s to ptr\n", t, v) - value = "%" + t - case ty == Ptr && slot.Type == I32: - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i32\n", t, v) - value = "%" + t - default: - return fmt.Errorf("FP parameter write type mismatch: have %s want %s at +%d(FP)", ty, slot.Type, off) + if slot, ok := c.fpParams[off]; ok { + ptr := c.fpParamAlloca[off] + if c.classicFrame != "" { + ptr = c.classicFramePtr(off) + } + if ptr == "" { + return fmt.Errorf("missing mutable FP parameter slot at +%d(FP)", off) } - fmt.Fprintf(c.b, " store %s %s, ptr %s%s\n", slot.Type, value, c.classicFramePtr(off), align) + storageType := slot.Type + if storageType == I1 && c.abi0BoolResultByteStorage() { + storageType = I8 + } + value, err := c.coerceFPStoreValue(ty, storageType, v) + if err != nil { + return fmt.Errorf("FP parameter write type mismatch: have %s want %s at +%d(FP): %w", ty, slot.Type, off, err) + } + c.storeFPStorage(storageType, value, ptr, align, metadata) return nil } if c.goarch == "386" && ty == I32 { @@ -1403,9 +1961,9 @@ func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { if slot.Type == LLVMType("double") { cast := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast i64 %%%s to double\n", cast, merged) - fmt.Fprintf(c.b, " store double %%%s, ptr %s%s\n", cast, alloca, align) + fmt.Fprintf(c.b, " store double %%%s, ptr %s%s%s\n", cast, alloca, align, metadata) } else { - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s\n", merged, alloca, align) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s%s\n", merged, alloca, align, metadata) } c.fpResWritten[slot.Index] = true return nil @@ -1415,11 +1973,43 @@ func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { if !ok { return fmt.Errorf("unsupported FP write slot: +%d(FP)", off) } + if slotTy == I1 { + if c.abi0BoolResultByteStorage() { + value, err := c.coerceFPStoreValue(ty, I8, v) + if err != nil { + return fmt.Errorf("FP bool result byte write at +%d(FP): %w", off, err) + } + c.storeFPStorage(I8, value, alloca, align, metadata) + c.markFPResultWritten(off) + return nil + } + value, err := c.coerceFPStoreValue(ty, I1, v) + if err != nil { + return fmt.Errorf("FP bool result write at +%d(FP): %w", off, err) + } + c.storeFPStorage(I1, value, alloca, align, metadata) + c.markFPResultWritten(off) + return nil + } if slotTy != "" && slotTy != ty { if c.goarch == "386" && ty == I32 && slotTy == Ptr { p := c.newTmp() fmt.Fprintf(c.b, " %%%s = inttoptr i32 %s to ptr\n", p, v) - fmt.Fprintf(c.b, " store ptr %%%s, ptr %s%s\n", p, alloca, align) + fmt.Fprintf(c.b, " store ptr %%%s, ptr %s%s%s\n", p, alloca, align, metadata) + c.markFPResultWritten(off) + return nil + } + if ty == I32 && slotTy == LLVMType("float") { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %s to float\n", cast, v) + fmt.Fprintf(c.b, " store float %%%s, ptr %s%s%s\n", cast, alloca, align, metadata) + c.markFPResultWritten(off) + return nil + } + if ty == LLVMType("float") && slotTy == I32 { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", cast, v) + fmt.Fprintf(c.b, " store i32 %%%s, ptr %s%s%s\n", cast, alloca, align, metadata) c.markFPResultWritten(off) return nil } @@ -1451,41 +2041,68 @@ func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", t, ty, v, slotTy) cast = "%" + t } - fmt.Fprintf(c.b, " store %s %s, ptr %s%s\n", slotTy, cast, alloca, align) + fmt.Fprintf(c.b, " store %s %s, ptr %s%s%s\n", slotTy, cast, alloca, align, metadata) c.markFPResultWritten(off) return nil } } + if (ty == I64 || ty == LLVMType("double")) && slotTy == LLVMType("float") { + secondAlloca, secondTy, secondOK := c.fpResultAlloca(off + 4) + if !secondOK || secondTy != LLVMType("float") { + return fmt.Errorf("FP write at +%d(FP) cannot split a 64-bit memory operand", off) + } + bits := v + if ty == LLVMType("double") { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", cast, v) + bits = "%" + cast + } + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", low, bits) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %s, 32\n", shifted, bits) + high := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", high, shifted) + lowFloat := c.newTmp() + highFloat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", lowFloat, low) + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", highFloat, high) + fmt.Fprintf(c.b, " store float %%%s, ptr %s%s%s\n", lowFloat, alloca, align, metadata) + fmt.Fprintf(c.b, " store float %%%s, ptr %s%s%s\n", highFloat, secondAlloca, align, metadata) + c.markFPResultWritten(off) + c.markFPResultWritten(off + 4) + return nil + } // Cast integer sizes when needed (common: i64 reg -> i32 return slot). switch { case ty == I64 && slotTy == I32: t := c.newTmp() fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", t, v) - fmt.Fprintf(c.b, " store i32 %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store i32 %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == I64 && slotTy == LLVMType("double"): t := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to double\n", t, v) - fmt.Fprintf(c.b, " store double %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store double %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == LLVMType("double") && slotTy == I64: t := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", t, v) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == I64 && slotTy == Ptr: t := c.newTmp() fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", t, v) - fmt.Fprintf(c.b, " store ptr %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store ptr %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == Ptr && slotTy == I64: t := c.newTmp() fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", t, v) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == I64 && slotTy == I64: @@ -1494,7 +2111,7 @@ func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { return fmt.Errorf("FP write type mismatch: have %s want %s at +%d(FP)", ty, slotTy, off) } } - fmt.Fprintf(c.b, " store %s %s, ptr %s%s\n", ty, v, alloca, align) + fmt.Fprintf(c.b, " store %s %s, ptr %s%s%s\n", ty, v, alloca, align, metadata) c.markFPResultWritten(off) return nil } @@ -1508,13 +2125,11 @@ func (c *amd64Ctx) loadFPResult(slot FrameSlot) (string, error) { if !ok { return "", fmt.Errorf("missing fp result alloca for index %d", slot.Index) } - t := c.newTmp() align := "" if c.classicFrame != "" { align = ", align 1" } - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s%s\n", t, slot.Type, alloca, align) - return "%" + t, nil + return c.loadFPStorage(slot.Type, alloca, align), nil } func isAMD64FloatRetTy(ty LLVMType) bool { @@ -1639,8 +2254,7 @@ func parseSBRef(sym string) (base string, off int64, ok bool) { if s == "" { return "", 0, false } - base, off = splitSymPlusOff(s) - return base, off, true + return parseSymbolOffset(s) } // Also accept bare symbol names used by a few platform asm stubs // (e.g. windows arm64 shared-user-data aliases). @@ -1650,8 +2264,8 @@ func parseSBRef(sym string) (base string, off int64, ok bool) { if _, ok := parseMem(sym); ok { return "", 0, false } - base, off = splitSymPlusOff(sym) - if strings.TrimSpace(base) == "" { + base, off, ok = parseSymbolOffset(sym) + if !ok || strings.TrimSpace(base) == "" { return "", 0, false } return base, off, true @@ -1665,6 +2279,17 @@ func (c *amd64Ctx) addrFromMem(mem MemRef) (addrI64 string, err error) { } func (c *amd64Ctx) addrFromPlainMem(mem MemRef) (addrI64 string, err error) { + if mem.Base == TLS || mem.Index == TLS { + return "", fmt.Errorf("%w: TLS memory needs its native Go progedit, relocation and TLS address contract", ErrProbeNeedsContext) + } + if physical, certified := c.fixedMemRoles[mem]; certified { + mem = physical + } else if address, logical, err := c.logicalAUTOAddress(mem); logical { + return address, err + } + if err := x86SymbolIndexedMemoryError(mem, c.goarch); err != nil { + return "", err + } var cur string if mem.Sym != "" { base, err := c.ptrFromSB(mem.Sym) @@ -1735,18 +2360,24 @@ func (c *amd64Ctx) ptrFromAddrI64(addrI64 string) string { } func (c *amd64Ctx) ptrFromSB(sym string) (ptr string, err error) { + // This entry point consumes memory, not Go's TYPE_ADDR immediate. Address + // evaluators must first select that role explicitly; silently removing '$' + // here lets direct OpSym load/store dispatch bypass the memory grammar. + if strings.HasPrefix(strings.TrimSpace(sym), "$") { + return "", fmt.Errorf("address immediate is not symbolic memory: %q", sym) + } base, off, ok := parseSBRef(sym) if !ok { return "", fmt.Errorf("invalid (SB) sym ref: %q", sym) } - base = strings.TrimPrefix(base, "$") - res := base - if strings.Contains(base, "·") || strings.Contains(base, "/") || strings.Contains(base, ".") { - res = c.resolve(base) - } else { - res = c.resolve("·" + base) - } + res := resolveAssemblyDataSymbol(base, c.resolve) p := llvmGlobal(res) + if continuation, ok := c.continuations[res]; ok { + if off != 0 { + return "", fmt.Errorf("continuation address cannot have a byte offset: %q", sym) + } + return fmt.Sprintf("blockaddress(%s, %%%s)", llvmGlobal(continuation.root), amd64LLVMBlockName(continuation.label)), nil + } if off == 0 { return p, nil } diff --git a/amd64_data_bp.go b/amd64_data_bp.go new file mode 100644 index 000000000..a34e768cf --- /dev/null +++ b/amd64_data_bp.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "reflect" + "strings" +) + +// This is an ordinary source's data-register view, not physical BP entry or +// frame backing. Go's nonempty NOSPLIT frame preserves its caller BP. The +// complete named source must independently exclude every observation of that +// frame, native entry, call and escaping frame address. Reaching input facts +// are still proved by the ordinary whole-source CFG below. +func x86DataBPSourceScope(fn Func, sig FuncSig, arch string) bool { + if arch != "amd64" || fn.FrameSize <= 0 || fn.FrameSize > 4096 || fn.FrameSize%8 != 0 || + strings.Contains(fn.Sym, "<") || len(sig.ArgRegs) != 0 || len(fn.X86RawText) != 0 || + fn.x86SourceStack != nil || fn.x86FixedLocal != nil || fn.x86DirectTailRoot != nil || + fn.x86ContinuationFlow != nil || fn.x86BoolCalls != nil || sig.x86CallerRegions != nil || + len(fn.x86ContinuationAddresses) != 0 || len(fn.x86IndirectLabels) != 0 || fn.x86BoolReadonlyInputs || + len(fn.Instrs) == 0 || len(fn.Instrs) > 4096 || fn.Instrs[0].Op != OpTEXT { + return false + } + _, rest := splitOpcode(fn.Instrs[0].Raw) + parts := strings.Split(rest, ",") + if len(parts) != 3 { + return false + } + frame, args, err := parseTEXTFrame(parts) + if err != nil || frame != fn.FrameSize || args != fn.ArgSize { + return false + } + expression := globlFlagName.ReplaceAllStringFunc(parts[1], func(name string) string { + if value, known := goTextFlagValues[name]; known { + return value + } + return name + }) + flags, known := parseImmExpr(strings.TrimSpace(expression)) + if !known || flags != 4 { // NOSPLIT, never NOFRAME, NEEDCTXT or another prologue role. + return false + } + found := false + var source strings.Builder + for _, ins := range fn.Instrs { + if ins.Raw == "" || len(ins.Raw) > (1<<20)-source.Len()-1 { + return false + } + source.WriteString(ins.Raw) + source.WriteByte('\n') + if !x86ResultNamedPredicateInstruction(ins) { + return false + } + switch ins.Op { + case OpCALL, "BYTE", "WORD", "LONG", "QUAD": + return false + } + if _, stack, _ := x86StackWidthProperties(ins, arch); stack { + return false // Private PUSH/FLAGS storage keeps its own source owner. + } + if x86LEAWidth(ins.Op, arch) != 0 && len(ins.Args) != 0 && ins.Args[0].Kind == OpFP { + return false // LEA's Ym operand takes the FP address without OpFPAddr. + } + for _, operand := range ins.Args { + if operand.Kind == OpFPAddr || operand.Kind == OpSym { + return false + } + if operand.Kind == OpReg { + root, gp := x86CallerGP(operand.Reg) + if gp && root == SP { + return false + } + found = found || operand.Reg == BP + } + if operand.Kind == OpMem { + memory := operand.Mem + if memory.Base == SP || memory.Index == SP || memory.Sym != "" || memory.Segment != "" { + return false + } + found = found || memory.Base == BP || memory.Index == BP + } + } + } + if !found { + return false + } + // One bounded source replay, not one parse per instruction. Mutable fields + // cannot reinterpret a physical-frame use or a different source spelling. + parsed, err := Parse(ArchAMD64, source.String()) + return err == nil && len(parsed.Funcs) == 1 && len(parsed.UnlinkedPrelude) == 0 && + len(parsed.Data) == 0 && len(parsed.Globl) == 0 && parsed.Funcs[0].Sym == fn.Sym && + parsed.Funcs[0].FrameSize == fn.FrameSize && parsed.Funcs[0].ArgSize == fn.ArgSize && + reflect.DeepEqual(parsed.Funcs[0].Instrs, fn.Instrs) +} + +// Project only the grammar query, never the original source or emitted address. +// AX is an ordinary encoding-equivalent base; the complete displacement/index +// checks remain with the existing scalar move descriptor. BP as an index is +// not a numerical input and cannot borrow this opaque-pointer holder role. +func x86DataBPScalarMove(ins Instr) (x86ScalarNativeLoadForm, x86ScalarNativeStoreForm, bool, bool) { + if len(ins.Args) != 2 { + return x86ScalarNativeLoadForm{}, x86ScalarNativeStoreForm{}, false, false + } + projected := ins + projected.Args = append([]Operand(nil), ins.Args...) + for index, operand := range ins.Args { + if operand.Kind != OpMem || operand.Mem.Base != BP || operand.Mem.Index == BP { + continue + } + projected.Args[index].Mem.Base = AX + if index == 0 { + form, known, err := x86ScalarNativeLoadProperties(ins.Op, projected, "amd64") + if known && err == nil && !x86LogicalSourceFrameRegister(form.destination, "amd64") { + form.source = operand + return form, x86ScalarNativeStoreForm{}, true, false + } + } else { + form, known, err := x86ScalarNativeStoreProperties(ins.Op, projected, "amd64") + if known && err == nil && (form.source.Kind != OpReg || !x86LogicalSourceFrameRegister(form.source.Reg, "amd64")) { + form.destination = operand + return x86ScalarNativeLoadForm{}, form, false, true + } + } + } + return x86ScalarNativeLoadForm{}, x86ScalarNativeStoreForm{}, false, false +} + +func x86ResultDataBPStep(ins Instr, state *x86ResultState, owned []bool, resultOffset int64) (bool, bool) { + if !x86ResultNamedPredicateInstruction(ins) || len(ins.Args) != 2 { + return false, false + } + local := x86SourceMoveInstruction(ins, "amd64") + first, last := ins.Args[0], ins.Args[1] + if local.Op == "MOVQ" && (first.Kind == OpReg && first.Reg == BP || last.Kind == OpReg && last.Reg == BP) { + pointer := false + if first.Kind == OpFP { + pointer = state.pointerFP[first.FPOffset] + } else if first.Kind == OpReg && first.Reg == BP { + pointer = state.shapeOnly || state.pointerGP[5] + } else if first.Kind == OpReg { + if index, _, valid := x86ResultGP(first.Reg, 8, "amd64"); valid { + pointer = state.shapeOnly || state.pointerGP[index] + } + } + index, _, valid := x86ResultGP(last.Reg, 8, "amd64") + if last.Kind == OpReg && last.Reg == BP { + index, valid = 5, true + } + if last.Kind != OpReg || !valid || !pointer { + return false, true + } + state.pointerGP[index], state.gp[index], state.canonicalGP[index] = true, 0, 0 + state.callerAddressGP[index], state.callerAddressOrigin[index] = false, 0 + state.resultAddressGP[index] = x86ResultSlotAddress{} + return true, true // No pointer bits become NUM or a caller/frame address. + } + load, store, read, write := x86DataBPScalarMove(ins) + if !read && !write { + return false, false + } + memory := load.source + if write { + memory = store.destination + } + if !state.shapeOnly && !state.pointerGP[5] { + return false, true + } + if memory.Mem.Index != "" && !x86ResultRead(Operand{Kind: OpReg, Reg: memory.Mem.Index}, 8, state, owned, "amd64") { + return false, true + } + if write { + return x86ResultRead(store.source, store.bytes, state, owned, "amd64"), true + } + return x86ResultStore(Operand{Kind: OpReg, Reg: load.destination}, load.bytes, state, owned, resultOffset, "amd64"), true +} + +func (c *amd64Ctx) lowerDataBPScalarMove(ins Instr) (bool, error) { + if !c.dataBPSource { + return false, nil + } + load, store, read, write := x86DataBPScalarMove(ins) + if !read && !write { + return false, nil + } + if err := c.scalarFPResultInitialized(); err != nil { + return true, err + } + if read { + return true, c.lowerScalarNativeLoad(load, ins) + } + return true, c.lowerScalarNativeStore(store, ins) +} diff --git a/amd64_data_bp_overwrite_test.go b/amd64_data_bp_overwrite_test.go new file mode 100644 index 000000000..04afc9417 --- /dev/null +++ b/amd64_data_bp_overwrite_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +type amd64DataBPOverwriteCase struct { + name, before, overwrite, after string + extraParameter bool +} + +// Every negative keeps the same initialized pointer producer and numerical +// shift as its positive control. Removing the later native BP access must not +// turn an unsupported overwrite into an accepted dead-value transfer. +func amd64CheckDataBPOverwriteCases(t *testing.T, cases []amd64DataBPOverwriteCase) { + t.Helper() + type program struct { + name, source, declaration string + overwrite bool + } + var programs []program + var originalObjects strings.Builder + for index, test := range cases { + for _, useBP := range []bool{false, true} { + for _, overwrite := range []bool{false, true} { + symbol := fmt.Sprintf("BPOverwrite%d_%t_%t", index, useBP, overwrite) + arguments, extra := 8, "" + if test.extraParameter { + arguments, extra = 16, ", adjacent uint64" + } + var body strings.Builder + body.WriteString("MOVQ p+0(FP),BP\nMOVL $0x12345678,AX\n") + body.WriteString(test.before + "\n") + if overwrite { + body.WriteString(test.overwrite + "\n") + } + body.WriteString(test.after + "\n") + if useBP { + body.WriteString("MOVL (BP),AX\n") + } + body.WriteString("ROLL $7,AX\n") + if useBP { + body.WriteString("MOVL AX,12(BP)\n") + } + source := fmt.Sprintf("TEXT ·%s(SB),4,$8-%d\n%sRET\n", symbol, arguments, body.String()) + originalObjects.WriteString(source) + programs = append(programs, program{ + name: fmt.Sprintf("%s/use-BP=%t/overwrite=%t", test.name, useBP, overwrite), + source: source, declaration: fmt.Sprintf("func %s(p *[16]byte%s)\n", symbol, extra), + overwrite: overwrite, + }) + } + } + } + // These are all original, legal stock-Go sources, including overlapping + // unaligned FP writes. A parser/encoder/tool failure is not Context evidence. + if !t.Run("OriginalGoObjects", func(t *testing.T) { + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", originalObjects.String()) + }) { + t.Fatal("original Go object prerequisite failed; no overwrite rejection is certified") + } + for _, test := range programs { + t.Run(test.name, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, test.source, test.declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if test.overwrite { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("an overwritten register or FP pointer slot retained its old data-BP authority", err) + } + } else if err != nil || ir == "" { + t.Fatal("the identical initialized pointer source without the overwrite must remain accepted", err) + } + }) + } + }) + } +} + +func TestAMD64DataBPRejectsPartialAndNumericRegisterOverwrites(t *testing.T) { + var cases []amd64DataBPOverwriteCase + for _, width := range []struct{ suffix, source, destination string }{ + {"B", "AL", "BPB"}, {"W", "AX", "BP"}, {"L", "AX", "BP"}, {"Q", "AX", "BP"}, + } { + cases = append(cases, + amd64DataBPOverwriteCase{name: "immediate/" + width.suffix, + overwrite: fmt.Sprintf("MOV%s $7,%s", width.suffix, width.destination)}, + amd64DataBPOverwriteCase{name: "numeric-register/" + width.suffix, + overwrite: fmt.Sprintf("MOV%s %s,%s", width.suffix, width.source, width.destination)}, + ) + } + amd64CheckDataBPOverwriteCases(t, cases) +} + +func TestAMD64DataBPRejectsXToBPOverwrites(t *testing.T) { + var cases []amd64DataBPOverwriteCase + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "MOVDQ2Q", "VMOVD", "VMOVQ"} { + cases = append(cases, amd64DataBPOverwriteCase{ + name: op, before: "PXOR X0,X0", overwrite: op + " X0,BP", + }) + } + amd64CheckDataBPOverwriteCases(t, cases) +} + +func TestAMD64DataBPRejectsOverwrittenAliasesAndCFGJoins(t *testing.T) { + var cases []amd64DataBPOverwriteCase + for _, width := range []struct{ suffix, source, destination string }{ + {"B", "AL", "DL"}, {"W", "AX", "DX"}, {"L", "AX", "DX"}, {"Q", "AX", "DX"}, + } { + overwrite := fmt.Sprintf("MOV%s %s,%s", width.suffix, width.source, width.destination) + cases = append(cases, + amd64DataBPOverwriteCase{name: "alias/" + width.suffix, before: "MOVQ BP,DX", + overwrite: overwrite, after: "MOVQ DX,BP"}, + amd64DataBPOverwriteCase{name: "reaching-join/" + width.suffix, extraParameter: true, + before: "MOVQ BP,DX\nMOVQ adjacent+8(FP),CX\nTESTQ CX,CX\nJE joined", + overwrite: overwrite, after: "joined:\nMOVQ DX,BP"}, + ) + } + amd64CheckDataBPOverwriteCases(t, cases) +} + +func amd64CheckDataBPFPSlotOverwrites(t *testing.T, suffix string) { + t.Helper() + var cases []amd64DataBPOverwriteCase + for offset := 0; offset < 8; offset++ { + cases = append(cases, amd64DataBPOverwriteCase{ + name: fmt.Sprintf("MOV%s/offset=%d", suffix, offset), extraParameter: true, + overwrite: fmt.Sprintf("MOV%s $7,p+%d(FP)", suffix, offset), after: "MOVQ p+0(FP),BP", + }) + } + amd64CheckDataBPOverwriteCases(t, cases) +} + +func TestAMD64DataBPRejectsByteFPSlotOverwriteBeforeReimport(t *testing.T) { + amd64CheckDataBPFPSlotOverwrites(t, "B") +} + +func TestAMD64DataBPRejectsWordFPSlotOverwriteBeforeReimport(t *testing.T) { + amd64CheckDataBPFPSlotOverwrites(t, "W") +} + +func TestAMD64DataBPRejectsDwordFPSlotOverwriteBeforeReimport(t *testing.T) { + amd64CheckDataBPFPSlotOverwrites(t, "L") +} + +func TestAMD64DataBPRejectsQwordFPSlotOverwriteBeforeReimport(t *testing.T) { + amd64CheckDataBPFPSlotOverwrites(t, "Q") +} diff --git a/amd64_data_bp_prologue_test.go b/amd64_data_bp_prologue_test.go new file mode 100644 index 000000000..d471826de --- /dev/null +++ b/amd64_data_bp_prologue_test.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "fmt" + "regexp" + "strings" + "testing" +) + +// Observe either complete stock-Go encoding, never choose by the SDK version. +// This recognizes only the exact first-party fixture, including its original +// body, adjusted FP input, total frame and both halves of the saved BP cell. +// It remains a test observation, not a source-frame permission producer. +func amd64DataBPSavedFrameListing(listing string, frame int) (string, error) { + if frame != 8 && frame != 16 { + return "", fmt.Errorf("unregistered first-party frame %d", frame) + } + line := regexp.MustCompile(`(?m)^[ \t]*0x[0-9a-f]+[ \t]+[0-9]+[ \t]+\([^\r\n]*:[0-9]+\)[ \t]+([A-Z][A-Z0-9]*)(?:[ \t]+([^\r\n]*))?\r?$`) + var instructions []string + texts := 0 + for _, match := range line.FindAllStringSubmatch(listing, -1) { + op, args := match[1], strings.Join(strings.Fields(match[2]), " ") + if op == "TEXT" { + parts := strings.Split(args, ",") + if texts != 0 || len(instructions) != 0 || len(parts) != 3 || + !strings.HasSuffix(strings.TrimSpace(parts[0]), ".Value(SB)") || + strings.TrimSpace(parts[1]) != "NOSPLIT" || strings.TrimSpace(parts[2]) != fmt.Sprintf("$%d-8", frame+8) { + return "", fmt.Errorf("original Go TEXT lost its exact saved-word frame: %s", args) + } + texts++ + continue + } + if op == "FUNCDATA" || op == "PCDATA" { + continue // Zero-width compiler metadata is not an executed instruction. + } + instructions = append(instructions, strings.TrimSpace(op+" "+args)) + } + if texts != 1 || len(instructions) != 10 { + return "", fmt.Errorf("incomplete or extended fixture: TEXTs=%d instructions=%v", texts, instructions) + } + body := []string{ + fmt.Sprintf("MOVQ p+%d(FP), BP", frame+16), + "MOVL (BP), AX", "ROLL $7, AX", "MOVL AX, 12(BP)", + } + var style string + var expected []string + switch instructions[0] { + case "PUSHQ BP": + style = "PUSH/MOV/SUB + ADD/POP/RET" + expected = []string{"PUSHQ BP", "MOVQ SP, BP", fmt.Sprintf("SUBQ $%d, SP", frame)} + expected = append(expected, body...) + expected = append(expected, fmt.Sprintf("ADDQ $%d, SP", frame), "POPQ BP", "RET") + case fmt.Sprintf("SUBQ $%d, SP", frame+8): + style = "SUB/store/LEA + load/ADD/RET" + expected = []string{ + fmt.Sprintf("SUBQ $%d, SP", frame+8), fmt.Sprintf("MOVQ BP, %d(SP)", frame), + fmt.Sprintf("LEAQ %d(SP), BP", frame), + } + expected = append(expected, body...) + expected = append(expected, fmt.Sprintf("MOVQ %d(SP), BP", frame), fmt.Sprintf("ADDQ $%d, SP", frame+8), "RET") + default: + return "", fmt.Errorf("unknown original Go frame entry: %s", instructions[0]) + } + for index := range expected { + if instructions[index] != expected[index] { + return "", fmt.Errorf("%s instruction %d=%q, want %q", style, index, instructions[index], expected[index]) + } + } + return style, nil +} + +func TestAMD64DataBPSavedFrameListingRequiresCompleteOrderedEnvelope(t *testing.T) { + for _, frame := range []int{8, 16} { + for _, style := range []string{"PUSH", "SP-cell"} { + name := fmt.Sprintf("%s/%d", style, frame) + var instructions []string + if style == "PUSH" { + instructions = []string{ + "PUSHQ BP", "MOVQ SP, BP", fmt.Sprintf("SUBQ $%d, SP", frame), + fmt.Sprintf("MOVQ p+%d(FP), BP", frame+16), "MOVL (BP), AX", "ROLL $7, AX", "MOVL AX, 12(BP)", + fmt.Sprintf("ADDQ $%d, SP", frame), "POPQ BP", "RET", + } + } else { + instructions = []string{ + fmt.Sprintf("SUBQ $%d, SP", frame+8), fmt.Sprintf("MOVQ BP, %d(SP)", frame), fmt.Sprintf("LEAQ %d(SP), BP", frame), + fmt.Sprintf("MOVQ p+%d(FP), BP", frame+16), "MOVL (BP), AX", "ROLL $7, AX", "MOVL AX, 12(BP)", + fmt.Sprintf("MOVQ %d(SP), BP", frame), fmt.Sprintf("ADDQ $%d, SP", frame+8), "RET", + } + } + listing := func(rows []string, extent int) string { + var text strings.Builder + fmt.Fprintf(&text, "\t0x0000 00000 (fixture.s:1)\tTEXT\tfixture.Value(SB), NOSPLIT, $%d-8\n", extent) + for index, row := range rows { + op, args, _ := strings.Cut(row, " ") + fmt.Fprintf(&text, "\t0x%04x %05d (fixture.s:%d)\t%s\t%s\n", index, index, index+1, op, args) + } + return text.String() + } + t.Run(name+"/complete", func(t *testing.T) { + if _, err := amd64DataBPSavedFrameListing(listing(instructions, frame+8), frame); err != nil { + t.Fatal("independent complete stock-Go fixture was rejected", err) + } + }) + for _, mutation := range []string{ + "delete-save", "delete-restore", "wrong-save", "wrong-restore", "wrong-frame-setup", + "wrong-allocation", "wrong-release", "wrong-FP-input", "swap-save-and-setup", "duplicate-save", "delete-return", "wrong-TEXT", + } { + t.Run(name+"/"+mutation, func(t *testing.T) { + rows := append([]string(nil), instructions...) + save, setup, allocation, restore, release := 0, 1, 2, 8, 7 + if style == "SP-cell" { + save, setup, allocation, restore, release = 1, 2, 0, 7, 8 + } + extent := frame + 8 + switch mutation { + case "delete-save": + rows = append(rows[:save], rows[save+1:]...) + case "delete-restore": + rows = append(rows[:restore], rows[restore+1:]...) + case "wrong-save": + rows[save] = fmt.Sprintf("MOVQ BP, %d(SP)", frame+8) + case "wrong-restore": + rows[restore] = fmt.Sprintf("MOVQ %d(SP), BP", frame+8) + case "wrong-frame-setup": + rows[setup] = fmt.Sprintf("LEAQ %d(SP), BP", frame+8) + case "wrong-allocation": + rows[allocation] = fmt.Sprintf("SUBQ $%d, SP", frame+16) + case "wrong-release": + rows[release] = fmt.Sprintf("ADDQ $%d, SP", frame+16) + case "wrong-FP-input": + rows[3] = fmt.Sprintf("MOVQ p+%d(FP), BP", frame+8) + case "swap-save-and-setup": + rows[save], rows[setup] = rows[setup], rows[save] + case "duplicate-save": + rows = append(rows, rows[save]) + case "delete-return": + rows = rows[:len(rows)-1] + case "wrong-TEXT": + extent += 8 + } + if _, err := amd64DataBPSavedFrameListing(listing(rows, extent), frame); err == nil { + t.Fatal("an incomplete, reordered or wrong-width saved-BP frame passed observation") + } + }) + } + } + } +} diff --git a/amd64_data_bp_source_test.go b/amd64_data_bp_source_test.go new file mode 100644 index 000000000..a8046ea11 --- /dev/null +++ b/amd64_data_bp_source_test.go @@ -0,0 +1,140 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// These fixtures need only a source-defined data register. They do not read +// physical BP, expose a frame address or use their allocated local storage. +// The real Go prologue/epilogue is observed separately below; a frame-size +// number by itself is not a reusable native-frame permission. +func amd64DataBPSource(frame int, setup, body string) string { + return fmt.Sprintf("TEXT ·Value(SB),4,$%d-8\n%s\n%s\nRET\n", frame, setup, body) +} + +func TestAMD64DataBPActualGoSavedFramePrologue(t *testing.T) { + for _, frame := range []int{8, 16} { + t.Run(fmt.Sprint(frame), func(t *testing.T) { + source := amd64DataBPSource(frame, "MOVQ p+0(FP),BP", + "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)") + listing, err := x86ScalarLiteralGoListing(t, "amd64", source) + if err != nil { + t.Fatal(err) + } + style, err := amd64DataBPSavedFrameListing(listing, frame) + if err != nil { + t.Fatalf("original Go did not preserve the complete ordered BP/frame envelope: %v\n%s", err, listing) + } + t.Logf("original Go preserves its physical BP using %s; this observation grants no translator frame capability", style) + }) + } +} + +func TestAMD64SourceDefinedDataBPInitializedProgramsThreeAPIs(t *testing.T) { + for _, row := range []struct { + name, setup, body string + frame int + }{ + {"direct", "MOVQ p+0(FP),BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"copied-pointer", "MOVQ p+0(FP),SI\nMOVQ SI,BP", "MOVL 4(BP),AX\nROLL $7,AX\nMOVL AX,8(BP)", 8}, + {"copied-back", "MOVQ p+0(FP),BP\nMOVQ BP,BX", "MOVL 8(BX),AX\nROLL $7,AX\nMOVL AX,12(BX)", 16}, + {"same-origin-join", "MOVL $1,CX\nCMPL CX,$0\nJE other\nMOVQ p+0(FP),BP\nJMP join\nother:\nMOVQ p+0(FP),BP\njoin:", + "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 16}, + } { + t.Run(row.name, func(t *testing.T) { + source := amd64DataBPSource(row.frame, row.setup, row.body) + for _, triple := range []string{ + "x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, "amd64", triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[4]uint32)\n", "amd64", triple, api) + if err != nil || ir == "" { + t.Fatalf("complete source-defined data BP lost its initialized input: %v", err) + } + if !strings.Contains(ir, "load volatile i32") || !strings.Contains(ir, "store volatile i32") { + t.Fatal("source-defined data BP must retain the original exact native accesses") + } + }) + } + }) + } + }) + } +} + +func TestAMD64SourceDefinedDataBPDoesNotGrantMissingInputsOrFrameRoles(t *testing.T) { + for _, row := range []struct { + name, setup, body string + frame int + }{ + {"unknown-entry", "NOP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"partial-pointer", "MOVW p+0(FP),BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"truncated-pointer", "MOVL p+0(FP),BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"numeric-address", "MOVQ $7,BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"unknown-store-data", "MOVQ p+0(FP),BP", "MOVL AX,12(BP)\nROLL $7,AX", 8}, + {"lost-pointer", "MOVQ p+0(FP),BP\nMOVL $7,BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"missing-path", "MOVL $1,CX\nCMPL CX,$0\nJE join\nMOVQ p+0(FP),BP\njoin:", + "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"entry-physical-BP", "MOVQ BP,SI\nMOVQ p+0(FP),BP", "MOVL (SI),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"physical-SP", "MOVQ SP,BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"FP-address", "LEAQ p+0(FP),BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"unbound-AUTO", "MOVQ p+0(FP),BP\nMOVQ $7,slot-8(SP)", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 8}, + {"frameless-clobber", "MOVQ p+0(FP),BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)", 0}, + {"raw-RET", "MOVQ p+0(FP),BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)\nBYTE $0xc3", 8}, + } { + t.Run(row.name, func(t *testing.T) { + source := amd64DataBPSource(row.frame, row.setup, row.body) + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[4]uint32)\n", "amd64", "x86_64-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("a data-register source view manufactured missing input or physical-frame authority: %v", err) + } + }) + } + }) + } +} + +func TestAMD64SourceDefinedDataBPRejectsUnusedFPAddressesThreeAPIs(t *testing.T) { + for _, address := range []string{ + "LEAW n+8(FP),AX", "LEAL n+8(FP),AX", "LEAQ n+8(FP),AX", + "MOVQ $n+8(FP),AX", "MOVD $n+8(FP),AX", "MOVDQ2Q $n+8(FP),AX", + } { + t.Run(address, func(t *testing.T) { + for _, takeAddress := range []bool{false, true} { + t.Run(fmt.Sprintf("take-address=%t", takeAddress), func(t *testing.T) { + body := "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)" + if takeAddress { + // n is a real initialized numerical parameter, not a missing + // slot or pointer. AX is overwritten before observation, so + // unused-address transfer cannot conceal a missing input. + body = address + "\n" + body + } + source := strings.Replace(amd64DataBPSource(8, "MOVQ p+0(FP),BP", body), "$8-8", "$8-16", 1) + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[4]uint32,n uint64)\n", + "amd64", "x86_64-unknown-linux-gnu", api) + if takeAddress { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("an unused but actual FP address borrowed the data BP source scope", err) + } + } else if err != nil || ir == "" { + t.Fatal("the same complete initialized program without FP-address observation must remain supported", err) + } + }) + } + }) + } + }) + } +} diff --git a/amd64_data_bp_state_test.go b/amd64_data_bp_state_test.go new file mode 100644 index 000000000..7fa22fe2e --- /dev/null +++ b/amd64_data_bp_state_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestAMD64SourceDefinedDataBPExactNativeWidthsAndAliases(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc is required") + } + var source, declarations strings.Builder + for index, row := range []struct{ move, rotate, register string }{ + {"MOVB", "ROLB", "AL"}, {"MOVW", "ROLW", "AX"}, + {"MOVL", "ROLL", "AX"}, {"MOVQ", "ROLQ", "AX"}, + {"MOVD", "ROLQ", "AX"}, {"MOVDQ2Q", "ROLQ", "AX"}, + } { + name := fmt.Sprintf("Value%d", index) + body := fmt.Sprintf("%s (BP),%s\n%s $7,%s\n%s %s,8(BP)", + row.move, row.register, row.rotate, row.register, row.move, row.register) + source.WriteString(strings.Replace(amd64DataBPSource(8, "MOVQ p+0(FP),BP", body), "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p *[16]byte)\n", name) + } + for _, triple := range []string{ + "x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, "amd64", triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", triple, api) + if err != nil || ir == "" { + t.Fatal("complete data BP move widths/source aliases lost their typed input", err) + } + for _, width := range []int{8, 16, 32, 64} { + want := 1 + if width == 64 { + want = 3 + } + if strings.Count(ir, fmt.Sprintf("load volatile i%d,", width)) != want || + strings.Count(ir, fmt.Sprintf("store volatile i%d ", width)) != want { + t.Fatal("native BP input/output must retain every exact-width original access", width) + } + } + for _, body := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + compileLLVMToObject(t, llc, triple, "data-bp.ll", "data-bp.o", body) + } + }) + } + }) + } +} + +func TestAMD64SourceDefinedDataBPUsesExistingPointerCloneJoinAndAtomicity(t *testing.T) { + var initial x86ResultState + initial.pointerFP = map[int64]bool{0: true} + initial.gp[1], initial.canonicalGP[1] = 0xff, 1 + initial.flags, initial.direction = x86ByteFlagCF, x86ResultDirectionBackward + state := initial.clone() + imported := x86ScalarLaneInstruction(t, "MOVQ p+0(FP),BP") + if accepted, known := x86ResultDataBPStep(imported, &state, nil, 0); !accepted || !known { + t.Fatal("complete typed import did not define the existing BP opaque-pointer slot") + } + want := initial.clone() + want.pointerGP[5] = true + if !reflect.DeepEqual(state, want) || state.gp[5] != 0 || state.callerAddressGP[5] { + t.Fatal("a data BP import fabricated NUM, caller-address or unrelated state") + } + + clone := state.clone() + clone.pointerGP[5] = false + if !state.pointerGP[5] || state.intersect(clone).pointerGP[5] { + t.Fatal("a clone/join retained an absent incoming BP pointer identity") + } + if !state.intersect(state.clone()).pointerGP[5] { + t.Fatal("equal complete pointer paths lost their existing must-fact") + } + for _, spelling := range []string{ + "MOVQ BP,BX", "MOVQ BX,BP", "MOVL (BP),AX", "MOVL AX,8(BP)", + } { + t.Run("initialized/"+spelling, func(t *testing.T) { + before := state.clone() + before.pointerGP[3], before.gp[0] = true, 0xff + if accepted, known := x86ResultDataBPStep(x86ScalarLaneInstruction(t, spelling), &before, nil, 0); !accepted || !known { + t.Fatal("an actual initialized pointer/data input was lost", accepted, known) + } + if before.flags != state.flags || before.direction != state.direction || before.gp[5] != 0 || before.callerAddressGP[5] { + t.Fatal("data BP transfer created pointer bits or changed FLAGS/DF") + } + }) + } + for _, spelling := range []string{ + "MOVQ BP,BX", "MOVQ BX,BP", "MOVL (BP),AX", "MOVL AX,8(BP)", "MOVL (BP)(CX*1),AX", + } { + t.Run("missing/"+spelling, func(t *testing.T) { + before := initial.clone() + before.gp[1] = 0 + frozen := before.clone() + if accepted, known := x86ResultDataBPStep(x86ScalarLaneInstruction(t, spelling), &before, nil, 0); accepted || !known || !reflect.DeepEqual(before, frozen) { + t.Fatal("a rejected transfer manufactured input facts or partially mutated state", accepted, known) + } + }) + } + for byteIndex := 0; byteIndex < 8; byteIndex++ { + t.Run(fmt.Sprintf("index-byte/%d", byteIndex), func(t *testing.T) { + before := state.clone() + before.gp[1] = 0xff &^ (1 << byteIndex) + frozen := before.clone() + instruction := x86ScalarLaneInstruction(t, "MOVL (BP)(CX*1),AX") + if accepted, known := x86ResultDataBPStep(instruction, &before, nil, 0); accepted || !known || !reflect.DeepEqual(before, frozen) { + t.Fatal("an actual incoming BP pointer substituted for a missing numerical index byte") + } + }) + } +} + +func TestAMD64SourceDefinedDataBPScopeDoesNotBorrowOtherOwners(t *testing.T) { + source := amd64DataBPSource(8, "MOVQ p+0(FP),BP", "MOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)") + _, file, signature, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[4]uint32)", + "MOVQ p+0(FP),BP\nMOVL (BP),AX\nROLL $7,AX\nMOVL AX,12(BP)\nRET") + parsed, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + file.Funcs[0] = parsed.Funcs[0] + if !x86DataBPSourceScope(file.Funcs[0], signature, "amd64") { + t.Fatal("the independent initialized complete named source lost its data-register view") + } + for _, mutation := range []string{ + "mode32", "frameless", "changed-frame", "changed-args", "NOFRAME", "NEEDCTXT", "CALL", "raw-origin", + "raw-entry", "source-stack", "fixed-local", "continuation", "register-entry", "FP-address", "SP", "private-PUSH", + "changed-source-raw", "changed-source-operands", + } { + t.Run(mutation, func(t *testing.T) { + fn, sig, arch := file.Funcs[0], signature, "amd64" + fn.Instrs = append([]Instr(nil), fn.Instrs...) + switch mutation { + case "mode32": + arch = "386" + case "frameless": + fn.FrameSize = 0 + case "changed-frame": + fn.FrameSize += 8 + case "changed-args": + fn.ArgSize += 8 + case "NOFRAME": + fn.Instrs[0].Raw = "TEXT ·Value(SB),516,$8-8" + case "NEEDCTXT": + fn.Instrs[0].Raw = "TEXT ·Value(SB),68,$8-8" + case "CALL": + fn.Instrs[2] = x86ScalarLaneInstruction(t, "CALL foreign(SB)") + case "raw-origin": + fn.Instrs[2].x86Encoded = true + case "raw-entry": + fn.X86RawText = []byte{0xc3} + case "source-stack": + fn.x86SourceStack = new(x86SourceStackProof) + case "fixed-local": + fn.x86FixedLocal = new(x86FixedLocalProof) + case "continuation": + fn.x86ContinuationFlow = new(x86ContinuationFlow) + case "register-entry": + sig.ArgRegs = []Reg{AX} + case "FP-address": + fn.Instrs[2] = x86ScalarLaneInstruction(t, "LEAQ p+0(FP),AX") + case "SP": + fn.Instrs[2] = x86ScalarLaneInstruction(t, "MOVQ SP,AX") + case "private-PUSH": + fn.Instrs[2] = x86ScalarLaneInstruction(t, "PUSHQ AX") + case "changed-source-raw": + fn.Instrs[1].Raw = "MOVQ unknown+0(FP),AX" + case "changed-source-operands": + fn.Instrs[1].Args = append([]Operand(nil), fn.Instrs[1].Args...) + fn.Instrs[1].Args[1].Reg = BX + } + if x86DataBPSourceScope(fn, sig, arch) { + t.Fatal("the bounded data-register source view borrowed another physical/source owner") + } + }) + } + for _, address := range []string{ + "LEAW p+0(FP),AX", "LEAL p+0(FP),AX", "LEAQ p+0(FP),AX", + "MOVQ $p+0(FP),AX", "MOVD $p+0(FP),AX", "MOVDQ2Q $p+0(FP),AX", + } { + t.Run("complete-FP-address-family/"+address, func(t *testing.T) { + fn := file.Funcs[0] + fn.Instrs = append([]Instr(nil), fn.Instrs...) + fn.Instrs[2] = x86ScalarLaneInstruction(t, address) + if x86DataBPSourceScope(fn, signature, "amd64") { + t.Fatal("the complete LEA width/address-MOV alias family acquired a data BP source role") + } + }) + } +} diff --git a/amd64_defined_status_consumers_test.go b/amd64_defined_status_consumers_test.go new file mode 100644 index 000000000..89716399a --- /dev/null +++ b/amd64_defined_status_consumers_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86StatusFlagMaskClearsExactlyItsPhysicalSubset(t *testing.T) { + // Independent architectural bit positions, not the production slot list. + flags := []struct { + bit uint + slot string + }{ + {0, "%flags_cf"}, {2, "%flags_pf"}, {4, "%flags_af"}, + {6, "%flags_z"}, {7, "%flags_slt"}, {11, "%flags_of"}, + } + for _, arch := range []string{"386", "amd64"} { + for subset := 0; subset < 64; subset++ { + t.Run(fmt.Sprintf("%s/subset=%02x", arch, subset), func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + output.Reset() + temporary := ctx.tmp + var mask x86StatusFlagMask + var want strings.Builder + for index, flag := range flags { + if subset&(1<= la && i >= lb || i < la && i < lb", "if int64(i) >= la && int64(i) >= lb || int64(i) < la && int64(i) < lb", + ).Replace(x86DefinedStatusGoDriver) + formatted, err := format.Source([]byte(source)) + if err != nil { + t.Fatal(err) + } + return string(formatted) +} + +func x86DefinedStatusIA32CDriver() string { + var prototypes, kernels strings.Builder + for _, kernel := range x86DefinedStatusIA32Kernels() { + fmt.Fprintf(&prototypes, "extern void %s(uint32_t *, uint32_t, uint32_t, uint32_t);\n", kernel.name) + fmt.Fprintf(&kernels, "{%s,%d,%d,\"%s\"},\n", kernel.name, kernel.kind, kernel.bits, kernel.instruction) + } + source := strings.NewReplacer( + "KERNEL_PROTOTYPES", prototypes.String(), "KERNELS", kernels.String(), + "uint64_t", "uint32_t", "UINT64_MAX", "UINT32_MAX", "UINT64_C", "UINT32_C", + "0xa7a7a7a7a7a7a7a7", "0xa7a7a7a7", + "bits == 64 ?", "bits == 32 ?", + "a[i] = right>>(8*i);", "a[i] = ((uint64_t)right)>>(8*i);", + "b[i] = left>>(8*i);", "b[i] = ((uint64_t)left)>>(8*i);", + ).Replace(x86DefinedStatusCDriver) + return source +} + +func x86DefinedStatusIA32BuildOriginal(t *testing.T) x87IA32Executable { + t.Helper() + dir := t.TempDir() + declarations, assembly := x86DefinedStatusIA32Fixture("386") + inputs := map[string]string{ + "go.mod": "module definedstatus386\n\ngo 1.20\n", + "main.go": x86DefinedStatusIA32GoDriver(t), + "decl.go": "package main\n" + declarations, + "status_386.s": assembly, + } + for name, contents := range inputs { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + t.Logf("original IA32 input=%s bytes=%d sha256=%x", name, len(contents), sha256.Sum256([]byte(contents))) + } + goBinary := filepath.Join(runtime.GOROOT(), "bin", "go") + env := append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", + "GOROOT="+runtime.GOROOT(), "GOFLAGS=", "GOEXPERIMENT=", "GOOS=linux", + "GOARCH=386", "CGO_ENABLED=0", "GOMAXPROCS=2") + version, err := x87IA32Capture(t, goBinary, []string{"version"}, dir, env) + if err != nil || !strings.HasPrefix(string(version), "go version "+runtime.Version()+" ") { + t.Fatalf("original IA32 SDK identity mismatch: %v\n%s", err, version) + } + executable := filepath.Join(dir, "original-go") + output, err := x87IA32Capture(t, goBinary, + []string{"build", "-p=2", "-o", executable, "."}, dir, env) + if err != nil || len(output) != 0 { + t.Fatalf("original IA32 Go build failed: %v\n%s", err, output) + } + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + t.Logf("original Go %s static IA32 ELF bytes=%d sha256=%x; compile-only receipt", + runtime.Version(), sealed.size, sealed.hash) + return sealed +} + +func TestX86DefinedStatusIA32OriginalGoStaticELFActualCompile(t *testing.T) { + executable := x86DefinedStatusIA32BuildOriginal(t) + if err := executable.verify(); err != nil { + t.Fatal(err) + } + t.Log("seven kernels and 896 mathematical observations built; native execution is a separate required root") +} + +func TestX86DefinedStatusIA32RequiredNativeCounterpartSource(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "amd64_defined_status_linux_amd64_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + var counterpart *ast.FuncDecl + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == "TestCrossLinuxRuntimeMatrixX86DefinedStatusNative386" { + counterpart = function + } + } + if counterpart == nil { + t.Fatal("required original-Go/LLVM O0/O2 native IA32 defined-status counterpart is missing") + } + want := map[string]bool{ + "x87IA32RequireNativeHost": false, + "x86DefinedStatusIA32BuildOriginal": false, + "x87IA32RunDirect": false, + "x86DefinedStatusIA32CompileAndRun": false, + } + levels := map[string]bool{"-O0": false, "-O2": false} + ast.Inspect(counterpart.Body, func(node ast.Node) bool { + if literal, ok := node.(*ast.BasicLit); ok && literal.Kind == token.STRING { + value, err := strconv.Unquote(literal.Value) + if err != nil { + t.Fatal(err) + } + if _, required := levels[value]; required { + levels[value] = true + } + } + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if function, ok := call.Fun.(*ast.Ident); ok { + if _, required := want[function.Name]; required { + want[function.Name] = true + } + } + if selector, ok := call.Fun.(*ast.SelectorExpr); ok && strings.HasPrefix(selector.Sel.Name, "Skip") { + t.Error("required native IA32 counterpart cannot skip failed tools or execution") + } + return true + }) + for function, found := range want { + if !found { + t.Errorf("required native IA32 counterpart lacks %s", function) + } + } + for level, found := range levels { + if !found { + t.Errorf("required native IA32 counterpart lacks %s", level) + } + } + ast.Inspect(file, func(node ast.Node) bool { + literal, ok := node.(*ast.BasicLit) + if ok && literal.Kind == token.STRING { + value, err := strconv.Unquote(literal.Value) + if err != nil { + t.Fatal(err) + } + if value == "qemu-i386" || value == "-exec" { + t.Error("required native IA32 counterpart cannot substitute a guest executor") + } + } + return true + }) +} + +func x86DefinedStatusIA32Capabilities(cpuinfo string) error { + flags := make(map[string]bool) + for _, line := range strings.Split(cpuinfo, "\n") { + key, value, ok := strings.Cut(line, ":") + if ok && strings.TrimSpace(key) == "flags" { + for _, flag := range strings.Fields(value) { + flags[flag] = true + } + } + } + for _, required := range []string{"sse4_1", "sse4_2", "popcnt"} { + if !flags[required] { + return fmt.Errorf("required native IA32 capability missing: %s", required) + } + } + return nil +} + +func x86DefinedStatusIA32RequireCapabilities(t *testing.T) { + t.Helper() + file, err := os.Open("/proc/cpuinfo") + if err != nil { + t.Fatal(err) + } + defer file.Close() + contents, err := io.ReadAll(io.LimitReader(file, (4<<20)+1)) + if err != nil || len(contents) > 4<<20 { + t.Fatal("unbound native IA32 capability observation", err) + } + if err := x86DefinedStatusIA32Capabilities(string(contents)); err != nil { + t.Fatal(err) + } +} + +func TestX86DefinedStatusIA32CapabilitiesAreRequired(t *testing.T) { + if err := x86DefinedStatusIA32Capabilities("flags: fpu cmov popcnt sse4_1 sse4_2\n"); err != nil { + t.Fatal(err) + } + for _, observation := range []string{ + "flags: fpu cmov sse4_1 sse4_2\n", "flags: fpu cmov popcnt sse4_2\n", + "flags: fpu cmov popcnt sse4_1\n", "notflags: popcnt sse4_1 sse4_2\n", + "flags: popcnt_extended sse4_1_extended sse4_2_extended\n", "", + } { + if x86DefinedStatusIA32Capabilities(observation) == nil { + t.Errorf("missing/foreign CPU capability was accepted: %q", observation) + } + } +} + +func TestX86DefinedStatusIA32ReferenceTransportWidth(t *testing.T) { + kernels := x86DefinedStatusIA32Kernels() + if len(kernels) != 7 || kernels[0].bits != 16 || kernels[1].bits != 32 { + t.Fatal("native IA32 kernel denominator changed", kernels) + } + declarations, source := x86DefinedStatusIA32Fixture("386") + if strings.Count(declarations, "*[8]uint32") != 7 || + strings.Count(source, "$0-16") != 7 || strings.Count(source, "MOVB AH,4(DI)") != 7 || + strings.Contains(source, "Q ") || strings.Contains(source, "PUSHFQ") { + t.Fatal("IA32 fixture lost actual four-byte transport or physical instruction width") + } + goSource := x86DefinedStatusIA32GoDriver(t) + if !strings.Contains(goSource, "int64(int32(left))") || !strings.Contains(goSource, "int64(int32(right))") { + t.Fatal("minimum signed IA32 length must be widened before negation") + } + cSource := x86DefinedStatusIA32CDriver() + if strings.Contains(cSource, "= right>>(8*i)") || strings.Contains(cSource, "= left>>(8*i)") || + !strings.Contains(cSource, "((uint64_t)right)>>(8*i)") || !strings.Contains(cSource, "((uint64_t)left)>>(8*i)") { + t.Fatal("independent C vector filling must not shift a uint32 by 32 or more") + } + if !strings.Contains(cSource, "bits == 32 ? UINT32_MAX") || !strings.Contains(goSource, "if bits == 32") { + t.Fatal("full IA32 mask must not shift a uint32 by its width") + } +} + +func TestX86DefinedStatusIA32IndependentCActualCompile(t *testing.T) { + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("required LLVM 22 clang missing") + } + dir := t.TempDir() + input, object := filepath.Join(dir, "model.c"), filepath.Join(dir, "model.o") + // Freestanding syntax/object checking needs no host libc headers or fake + // sysroot. The required native counterpart uses the unchanged stdio driver. + source := strings.Replace(x86DefinedStatusIA32CDriver(), "#include ", + "extern int printf(const char *, ...);", 1) + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output, err := x87IA32Capture(t, clang, []string{"-target", "i386-unknown-linux-gnu", + "-ffreestanding", "-Wall", "-Wextra", "-Werror", level, "-c", input, "-o", object}, dir, os.Environ()) + if err != nil || len(output) != 0 { + t.Fatalf("independent IA32 C model object failed: %v\n%s", err, output) + } + parsed, err := elf.Open(object) + if err != nil { + t.Fatal(err) + } + defer parsed.Close() + if parsed.Class != elf.ELFCLASS32 || parsed.Machine != elf.EM_386 || parsed.Type != elf.ET_REL { + t.Fatal("independent C model did not produce an IA32 object") + } + contents, err := os.ReadFile(object) + if err != nil || len(contents) == 0 { + t.Fatal("missing IA32 C model object", err) + } + t.Logf("independent IA32 C model %s bytes=%d sha256=%x; compile-only, not executed", + level, len(contents), sha256.Sum256(contents)) + }) + } +} + +// These objects cover every affected LLVM target/API independently of the +// required direct Linux/386 executable. No cross-object is a runtime receipt. +func TestX86DefinedStatusIA32ThreeAPIsFiveTargetsObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("required LLVM 22 llc missing") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + declarations, source := x86DefinedStatusIA32Fixture(target.arch) + goAssemblerOracleOutput(t, target.arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, target.arch, target.triple, api) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprint("O2=", optimized), func(t *testing.T) { + input := ir + if optimized { + input = x86UndefinedTrapOptimizedIR(t, input) + } + compileLLVMToObject(t, llc, target.triple, "status.ll", "status.o", input) + }) + } + }) + } + } +} + +func x86DefinedStatusIA32CompileAndRun(t *testing.T, llc, ir, level string) { + t.Helper() + dir := t.TempDir() + llPath, object := filepath.Join(dir, "status.ll"), filepath.Join(dir, "status.o") + mainPath, executable := filepath.Join(dir, "main.c"), filepath.Join(dir, "translated") + for name, contents := range map[string]string{llPath: ir, mainPath: x86DefinedStatusIA32CDriver()} { + if err := os.WriteFile(name, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + output, err := x87IA32Capture(t, llc, + []string{level, "-mtriple=i386-unknown-linux-gnu", "-filetype=obj", llPath, "-o", object}, dir, os.Environ()) + if err != nil || len(output) != 0 { + t.Fatalf("LLVM22 IA32 %s object failed: %v\n%s", level, err, output) + } + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if err != nil { + t.Fatal(err) + } + output, err = x87IA32Capture(t, compiler, + []string{"-static", "-no-pie", level, "-Wall", "-Wextra", "-Werror=shift-count-overflow", + object, mainPath, "-o", executable}, dir, os.Environ()) + if err != nil || len(output) != 0 { + t.Fatalf("static IA32 %s independent C link failed: %v\n%s", level, err, output) + } + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != "LLVM defined-status observations 896 failures 0\n" { + t.Fatalf("LLVM IA32 %s six-flag/LAHF/full canary oracle: %q", level, output) + } +} diff --git a/amd64_defined_status_linux_amd64_test.go b/amd64_defined_status_linux_amd64_test.go new file mode 100644 index 000000000..4459675f6 --- /dev/null +++ b/amd64_defined_status_linux_amd64_test.go @@ -0,0 +1,51 @@ +//go:build linux && amd64 + +package plan9asm + +import ( + "os" + "testing" +) + +// Unlike emulated-host observation agreement, these required native roots +// retain the full independent ISA expectations, including PTEST's cleared AF. +// Missing tools, instructions or actual native backing are failures. +func TestCrossLinuxRuntimeMatrixX86DefinedStatusOriginalGoNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + x86DefinedStatusOriginalGoNumerical(t) +} + +func TestCrossLinuxRuntimeMatrixX86DefinedStatusIndependentOriginalCNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + x86DefinedStatusIndependentOriginalCNumerical(t) +} + +// The native kernel, original ABI0 executable and both translated ELF32 +// executables are required. No Rosetta/QEMU or cross-object substitutes them. +func TestCrossLinuxRuntimeMatrixX86DefinedStatusNative386(t *testing.T) { + x87IA32RequireNativeHost(t) + x86DefinedStatusIA32RequireCapabilities(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("required LLVM 22 llc missing") + } + t.Run("OriginalGo", func(t *testing.T) { + original := x86DefinedStatusIA32BuildOriginal(t) + output := x87IA32RunDirect(t, original, nil, os.Environ()) + if string(output) != "original defined-status observations 896 failures 0\n" { + t.Fatalf("original Go IA32 ABI0 six-flag/LAHF/full canary oracle: %q", output) + } + }) + declarations, source := x86DefinedStatusIA32Fixture("386") + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, "386", "i386-unknown-linux-gnu", "Go") + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + x86DefinedStatusIA32CompileAndRun(t, llc, input, level) + }) + } + x87IA32RequireNativeHost(t) +} diff --git a/amd64_defined_status_numerical_test.go b/amd64_defined_status_numerical_test.go new file mode 100644 index 000000000..5f31cd839 --- /dev/null +++ b/amd64_defined_status_numerical_test.go @@ -0,0 +1,541 @@ +//go:build unix + +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +type x86DefinedStatusKernel struct { + name, instruction string + kind, bits int +} + +func x86DefinedStatusKernels() []x86DefinedStatusKernel { + var kernels []x86DefinedStatusKernel + add := func(instruction string, kind, bits int) { + kernels = append(kernels, x86DefinedStatusKernel{ + fmt.Sprintf("status%d", len(kernels)), instruction, kind, bits, + }) + } + for _, width := range []struct { + suffix string + bits int + }{{"W", 16}, {"L", 32}, {"Q", 64}} { + add("POPCNT"+width.suffix+" AX,BX", 0, width.bits) + } + add("PTEST X0,X1", 1, 128) + for _, op := range []string{"PCMPESTRI", "PCMPESTRM", "PCMPISTRI", "PCMPISTRM"} { + kind := 2 + if strings.HasPrefix(op, "PCMPI") { + kind = 3 + } + add(op+" $8,X0,X1", kind, 128) + } + for _, op := range []string{"VPCMPESTRI", "VPCMPESTRM", "VPCMPISTRI", "VPCMPISTRM"} { + kind := 2 + if strings.HasPrefix(op, "VPCMPI") { + kind = 3 + } + add(op+" $8,X0,X1", kind, 128) + } + for _, stem := range []string{"KTEST", "KORTEST"} { + for _, width := range []struct { + suffix string + bits int + }{{"B", 8}, {"W", 16}, {"D", 32}, {"Q", 64}} { + kind := 4 + if stem == "KORTEST" { + kind = 5 + } + add(stem+width.suffix+" K1,K2", kind, width.bits) + } + } + return kernels +} + +func x86DefinedStatusNumericalFixture() (declarations, source string) { + var decl, assembly strings.Builder + for _, kernel := range x86DefinedStatusKernels() { + fmt.Fprintf(&decl, "func %s(out *[8]uint64, left, right, seed uint64)\n", kernel.name) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$0-32 +MOVQ out+0(FP),DI +MOVQ left+8(FP),AX +MOVQ right+16(FP),BX +MOVQ seed+24(FP),SI +MOVQ BX,DX +MOVQ AX,X0 +MOVQ BX,X1 +`, kernel.name) + if kernel.kind >= 4 { + assembly.WriteString("KMOVQ AX,K1\nKMOVQ BX,K2\n") + } + fmt.Fprintf(&assembly, ` +PUSHQ SI +POPFQ +%s +PUSHFQ +POPQ SI +MOVQ SI,0(DI) +LAHF +MOVB AH,8(DI) +RET +`, kernel.instruction) + } + return decl.String(), assembly.String() +} + +// The baseline original-Go root executes SSE4.2/POPCNT, not AVX512 masks. +// Its exact source includes the mask objects, but no mask body is called here. +func x86DefinedStatusOriginalGoNumerical(t *testing.T) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + declarations, source := x86DefinedStatusNumericalFixture() + driver := x86DefinedStatusOriginalGoDriver(false) + original := x86DefinedStatusBuildOriginal(t, execution, declarations, source, driver) + output := execution.run(t, original) + if string(output) != "original defined-status observations 1024 failures 0\n" { + t.Fatalf("original Go six-bit status/LAHF/canary model: %s", output) + } +} + +func x86DefinedStatusOriginalGoDriver(observe bool) string { + kernels := x86DefinedStatusKernels()[:8] + var names, kinds, widths strings.Builder + for _, kernel := range kernels { + fmt.Fprintf(&names, "%s,\n", kernel.name) + fmt.Fprintf(&kinds, "%d,", kernel.kind) + fmt.Fprintf(&widths, "%d,", kernel.bits) + } + return strings.NewReplacer( + "KERNEL_NAMES", names.String(), "KERNEL_KINDS", kinds.String(), + "KERNEL_WIDTHS", widths.String(), "ACTUAL_OBSERVATIONS", fmt.Sprint(observe), + ).Replace(x86DefinedStatusGoDriver) +} + +func x86DefinedStatusBuildOriginal(t *testing.T, execution x86FloatCompareExecution, declarations, assembly, driver string) string { + t.Helper() + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module definedstatus\n\ngo 1.20\n", + "main.go": driver, + "decl.go": "package main\n" + declarations, + "status_amd64.s": assembly, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-go") + command := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-p=1", "-o", executable, ".") + command.Dir = dir + goos := runtime.GOOS + if strings.Contains(execution.triple, "linux") { + goos = "linux" + } + command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+goos, "GOARCH=amd64", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOEXPERIMENT=") + x86FloatCompareCaptureCommand(t, command) + return executable +} + +func TestX86DefinedStatusLLVMNumerical(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, source := x86DefinedStatusNumericalFixture() + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, "amd64", execution.triple, "Go") + var prototypes, kernels strings.Builder + for _, kernel := range x86DefinedStatusKernels() { + fmt.Fprintf(&prototypes, "extern void %s(uint64_t *, uint64_t, uint64_t, uint64_t);\n", kernel.name) + fmt.Fprintf(&kernels, "{%s,%d,%d,\"%s\"},\n", kernel.name, kernel.kind, kernel.bits, kernel.instruction) + } + driver := strings.NewReplacer("KERNEL_PROTOTYPES", prototypes.String(), + "KERNELS", kernels.String()).Replace(x86DefinedStatusCDriver) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + output := execution.compileAndRunLLVM(t, "defined-status", input, driver, level) + want := fmt.Sprintf("LLVM defined-status observations %d failures 0\n", len(x86DefinedStatusKernels())*128) + if string(output) != want { + t.Fatalf("independent six-bit status/LAHF/canary model: %s", output) + } + }) + } +} + +// The C producer executes entire physical opcodes and captures flags in the +// same inline block, independently of Go's ABI0 wrapper and LLVM lowering. +func x86DefinedStatusOriginalCSource() string { + instructions := []string{ + "popcntw %%ax,%%bx", "popcntl %%eax,%%ebx", "popcntq %%rax,%%rbx", + "ptest %%xmm0,%%xmm1", + "pcmpestri $8,%%xmm0,%%xmm1", "pcmpestrm $8,%%xmm0,%%xmm1", + "pcmpistri $8,%%xmm0,%%xmm1", "pcmpistrm $8,%%xmm0,%%xmm1", + } + var functions, registrations strings.Builder + for index, kernel := range x86DefinedStatusKernels()[:8] { + fmt.Fprintf(&functions, `static void %s(uint64_t *out, uint64_t left, uint64_t right, uint64_t seed) { + uint64_t length = right; + __asm__ volatile( + "movq %%%%rax,%%%%xmm0\n\t" + "movq %%%%rbx,%%%%xmm1\n\t" + "pushq %%%%rsi\n\t" + "popfq\n\t" + "%s\n\t" + "pushfq\n\t" + "popq %%%%rsi\n\t" + "movq %%%%rsi,0(%%%%rdi)\n\t" + "lahf\n\t" + "movb %%%%ah,8(%%%%rdi)\n\t" + : "+a"(left), "+b"(right), "+d"(length), "+S"(seed) + : "D"(out) + : "cc", "memory", "xmm0", "xmm1", "rcx"); +} + +`, kernel.name, instructions[index]) + fmt.Fprintf(®istrations, "{%s,%d,%d,%q},\n", kernel.name, kernel.kind, kernel.bits, kernel.instruction) + } + source := strings.NewReplacer("KERNEL_PROTOTYPES", functions.String(), + "KERNELS", registrations.String()).Replace(x86DefinedStatusCDriver) + return strings.ReplaceAll(source, "LLVM", "original-C") +} + +func x86DefinedStatusIndependentOriginalCNumerical(t *testing.T) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source := filepath.Join(dir, "original.c") + if err := os.WriteFile(source, []byte(x86DefinedStatusOriginalCSource()), 0600); err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := filepath.Join(dir, "original-c"+level) + arguments := append([]string(nil), execution.compiler...) + arguments = append(arguments, level, source, "-o", executable) + x86FloatCompareCaptureCommand(t, exec.Command(arguments[0], arguments[1:]...)) + output := execution.run(t, executable) + if string(output) != "original-C defined-status observations 1024 failures 0\n" { + t.Fatalf("actual C opcodes versus independent six-bit status model: %s", output) + } + }) + } +} + +// Common emulated-host evidence is exact original-Go/C agreement, not ISA +// conformance. Every original result is retained; native Linux separately +// runs the unchanged strict mathematical oracle. No deviation is a skip. +func TestX86DefinedStatusOriginalGoAndCObservationAgreement(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, assembly := x86DefinedStatusNumericalFixture() + original := x86DefinedStatusBuildOriginal(t, execution, declarations, assembly, x86DefinedStatusOriginalGoDriver(true)) + observations := execution.run(t, original) + lines := strings.Split(strings.TrimSuffix(string(observations), "\n"), "\n") + summary := regexp.MustCompile(`^observed-status observations 1024 model-deviations ([0-9]+) invalid-observations 0$`) + if len(lines) != 1025 || !summary.MatchString(lines[len(lines)-1]) { + t.Fatalf("incomplete or invalid actual original-Go observation transcript: %s", observations) + } + dir := t.TempDir() + source := filepath.Join(dir, "original.c") + if err := os.WriteFile(source, []byte("#define OBSERVE_ACTUAL 1\n"+x86DefinedStatusOriginalCSource()), 0600); err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := filepath.Join(dir, "original-c"+level) + arguments := append([]string(nil), execution.compiler...) + arguments = append(arguments, level, source, "-o", executable) + x86FloatCompareCaptureCommand(t, exec.Command(arguments[0], arguments[1:]...)) + actual := execution.run(t, executable) + if string(actual) != string(observations) { + t.Fatalf("original Go/C full status/LAHF/canary transcript differs: Go=%x C=%x\n%s\n%s", + sha256.Sum256(observations), sha256.Sum256(actual), observations, actual) + } + t.Logf("original observation agreement sha=%x; %s; not native ISA conformance", + sha256.Sum256(actual), lines[len(lines)-1]) + }) + } +} + +const x86DefinedStatusGoDriver = `package main + +import ( + "fmt" + "os" +) + +func mask(bits uint) uint64 { + if bits == 64 { + return ^uint64(0) + } + return uint64(1)<> (8*uint(i))) + b[i] = byte(left >> (8*uint(i))) + } + la, lb := 16, 16 + if kind == 2 { + la, lb = int(int32(left)), int(int32(right)) + if la < 0 { + la = -la + } + if lb < 0 { + lb = -lb + } + if la > 16 { + la = 16 + } + if lb > 16 { + lb = 16 + } + } else { + la, lb = 0, 0 + for la < 16 && a[la] != 0 { + la++ + } + for lb < 16 && b[lb] != 0 { + lb++ + } + } + var matches uint64 + for i := 0; i < 16; i++ { + if i >= la && i >= lb || i < la && i < lb && a[i] == b[i] { + matches |= 1 << uint(i) + } + } + var result uint64 + if matches != 0 { + result |= 1 + } + if lb < 16 { + result |= 64 + } + if la < 16 { + result |= 128 + } + if matches&1 != 0 { + result |= 2048 + } + return result +} + +func main() { + const observeActual = ACTUAL_OBSERVATIONS + kernels := []func(*[8]uint64,uint64,uint64,uint64){KERNEL_NAMES} + kinds := []int{KERNEL_KINDS} + widths := []uint{KERNEL_WIDTHS} + values := []uint64{0,1,0x55,0xaa,0xff,0xffff,0x80000000,^uint64(0)} + const canary = uint64(0xa7a7a7a7a7a7a7a7) + observations, failures, invalid := 0, 0, 0 + for index, kernel := range kernels { + kernelFailures := 0 + for _, left := range values { + for _, right := range values { + for _, seed := range []uint64{0x8d7,0x12} { + var out [8]uint64 + for i := range out { + out[i] = canary + } + kernel(&out,left,right,seed) + want := flags(kinds[index],widths[index],left,right) + invalidObservation := byte(out[1]) != byte(out[0]&0xd5|2) || + out[1]>>8 != canary>>8 + for i := 2; i < len(out); i++ { + invalidObservation = invalidObservation || out[i] != canary + } + if invalidObservation { + invalid++ + } + if observeActual { + fmt.Printf("observed-status %d %016x", index, out[0]&0x8d5) + for i := 1; i < len(out); i++ { + fmt.Printf(" %016x", out[i]) + } + fmt.Println() + } + bad := out[0]&0x8d5 != want || invalidObservation + if bad { + if !observeActual && failures < 12 { + fmt.Printf("original status kernel=%d left=%x right=%x flags=%x want=%x\n",index,left,right,out[0]&0x8d5,want) + } + failures++ + kernelFailures++ + } + observations++ + } + } + } + if !observeActual && kernelFailures != 0 { + fmt.Printf("original status kernel=%d failures=%d\n",index,kernelFailures) + } + } + if observeActual { + fmt.Printf("observed-status observations %d model-deviations %d invalid-observations %d\n",observations,failures,invalid) + if invalid != 0 { + os.Exit(1) + } + return + } + fmt.Printf("original defined-status observations %d failures %d\n",observations,failures) + if failures != 0 { + os.Exit(1) + } +} +` + +const x86DefinedStatusCDriver = ` +#include +#include +#ifndef OBSERVE_ACTUAL +#define OBSERVE_ACTUAL 0 +#endif +KERNEL_PROTOTYPES + +typedef struct { + void (*run)(uint64_t *, uint64_t, uint64_t, uint64_t); + unsigned kind, bits; + const char *name; +} Kernel; + +static uint64_t mask(unsigned bits) { + return bits == 64 ? UINT64_MAX : (UINT64_C(1)<= 4) { + uint64_t m = mask(bits); + left &= m; + right &= m; + if (kind == 4) { + return (left&right ? 0 : 64) | (left&~right ? 0 : 1); + } + uint64_t combined = left|right; + return (combined ? 0 : 64) | (combined == m ? 1 : 0); + } + unsigned char a[16] = {0}, b[16] = {0}; + for (unsigned i = 0; i < 8; i++) { + a[i] = right>>(8*i); + b[i] = left>>(8*i); + } + unsigned la = 16, lb = 16; + if (kind == 2) { + int64_t na = (int32_t)left, nb = (int32_t)right; + if (na < 0) { + na = -na; + } + if (nb < 0) { + nb = -nb; + } + la = na > 16 ? 16 : (unsigned)na; + lb = nb > 16 ? 16 : (unsigned)nb; + } else { + la = lb = 0; + while (la < 16 && a[la]) { + la++; + } + while (lb < 16 && b[lb]) { + lb++; + } + } + unsigned matches = 0; + for (unsigned i = 0; i < 16; i++) { + if ((i >= la && i >= lb) || (i < la && i < lb && a[i] == b[i])) { + matches |= 1u<>8) != (canary>>8); + for (unsigned i = 2; i < 8; i++) { + invalid_observation |= out[i] != canary; + } + invalid += !!invalid_observation; + if (OBSERVE_ACTUAL) { + printf("observed-status %u %016llx",index,(unsigned long long)(out[0]&0x8d5)); + for (unsigned i = 1; i < 8; i++) { + printf(" %016llx",(unsigned long long)out[i]); + } + printf("\n"); + } + int bad = (out[0]&0x8d5) != want || invalid_observation; + if (bad) { + if (!OBSERVE_ACTUAL && failures < 12) { + printf("LLVM status kernel=%u left=%llx right=%llx flags=%llx want=%llx\n", + index,(unsigned long long)values[l],(unsigned long long)values[r], + (unsigned long long)(out[0]&0x8d5),(unsigned long long)want); + } + failures++; + kernel_failures++; + } + observations++; + } + } + } + if (!OBSERVE_ACTUAL && kernel_failures) { + printf("LLVM status family=%s failures=%u\n",kernels[index].name,kernel_failures); + } + } + if (OBSERVE_ACTUAL) { + printf("observed-status observations %u model-deviations %u invalid-observations %u\n",observations,failures,invalid); + return invalid ? 1 : 0; + } + printf("LLVM defined-status observations %u failures %u\n",observations,failures); + return failures ? 1 : 0; +} +` diff --git a/amd64_defined_zero_af_test.go b/amd64_defined_zero_af_test.go new file mode 100644 index 000000000..1ee9c5404 --- /dev/null +++ b/amd64_defined_zero_af_test.go @@ -0,0 +1,166 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86DefinedZeroAFCase struct { + op, instruction string + amd64Only bool +} + +// These independent ISA witnesses cover every current non-arithmetic writer +// whose AF result is defined to be zero. Go encoding tables establish syntax; +// they do not describe the status effects. ENQCMD/S currently use raw source. +func x86DefinedZeroAFWriters() []x86DefinedZeroAFCase { + cases := []x86DefinedZeroAFCase{ + {"UMWAIT", "UMWAIT BX", false}, + {"TPAUSE", "TPAUSE BX", false}, + {"PTEST", "PTEST X0,X1", false}, + {"VPTEST", "VPTEST X0,X1", false}, + {"VTESTPD", "VTESTPD X0,X1", false}, + {"VTESTPS", "VTESTPS X0,X1", false}, + {"XTEST", "XTEST", false}, + {"ENQCMD", "BYTE $0xf2\nBYTE $0x0f\nBYTE $0x38\nBYTE $0xf8\nBYTE $0x38", false}, + {"ENQCMDS", "BYTE $0xf3\nBYTE $0x0f\nBYTE $0x38\nBYTE $0xf8\nBYTE $0x38", false}, + } + for _, stem := range []string{"RDRAND", "RDSEED", "POPCNT"} { + for _, width := range []string{"W", "L", "Q"} { + op := stem + width + operand := "BX" + if stem == "POPCNT" { + operand = "AX,BX" + } + cases = append(cases, x86DefinedZeroAFCase{op, op + " " + operand, width == "Q"}) + } + } + for _, stem := range []string{"KTEST", "KORTEST"} { + for _, width := range []string{"B", "W", "D", "Q"} { + op := stem + width + cases = append(cases, x86DefinedZeroAFCase{op, op + " K1,K2", false}) + } + } + for _, stem := range []string{"PCMPESTRI", "PCMPESTRM", "PCMPISTRI", "PCMPISTRM"} { + for _, prefix := range []string{"", "V"} { + op := prefix + stem + cases = append(cases, x86DefinedZeroAFCase{op, op + " $0,X0,X1", false}) + } + } + return cases +} + +func x86DefinedZeroAFSource(arch, name, instruction string) string { + word := "Q" + if arch == "386" { + word = "L" + } + return fmt.Sprintf("TEXT %s(SB),4,$0-0\nMOVL $0,AX\nMOVL $0,DX\nMOVL $0,BX\n"+ + "PUSH%s $0x8d7\nPOPF%s\n%s\nPUSHF%s\nPOP%s CX\nRET\n", + name, word, word, instruction, word, word) +} + +func TestX86DefinedZeroAFWritersClearAfterOperation(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86DefinedZeroAFWriters() { + if arch == "386" && row.amd64Only { + continue // Go's explicit 64-bit operand row is not a 386 form. + } + t.Run(arch+"/"+row.op, func(t *testing.T) { + source := x86DefinedZeroAFSource(arch, "status", row.instruction) + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + var ir string + if strings.Contains(row.instruction, "X0") || strings.Contains(row.instruction, "K1") || strings.HasPrefix(row.op, "ENQCMD") { + // The unchanged source does not initialize vectors/masks or + // either device address. Keep its original format and Context; + // the complete combined status test owns initialized programs. + ir = x86UnboundVoidRawScalarFormatObject(t, source, arch, triple) + } else { + requireX86GoAssemblerResult(t, arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err = Translate(file, Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"status": {Name: "status", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + } + // One false store initializes the slot before the explicit AF=1 + // seed. A defined-zero writer needs its own subsequent store. + if strings.Count(ir, "store i1 false, ptr %flags_af") != 2 { + t.Fatalf("%s retained seeded AF instead of its defined-zero status result", row.op) + } + compileLLVMToObject(t, llc, triple, "af.ll", "af.o", ir) + }) + } + } +} + +func TestX86DefinedZeroAFExistingCompareWriterControls(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + for _, op := range []string{"COMISS", "COMISD", "UCOMISS", "UCOMISD", "VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + t.Run(arch+"/"+op, func(t *testing.T) { + source := x86DefinedZeroAFSource(arch, "status", op+" X0,X1") + // The original source has no X0/X1 inputs. Keep its actual Go + // object and explicit public Context before inspecting emission. + ir := x86UnboundVoidScalarFormatObject(t, source, "status", arch, triple) + if strings.Count(ir, "store i1 false, ptr %flags_af") != 2 { + t.Fatal("original compare emitter lost AF clearing") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + compileLLVMToObject(t, llc, triple, "compare-status.ll", "compare-status.o", ir) + }) + } + } +} + +func TestX86DefinedCompareStatusInitializedFiveTargetsObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + word, value, size := "L", "uint32", 4 + if target.arch == "amd64" { + word, value, size = "Q", "uint64", 8 + } + var source, declarations strings.Builder + for _, op := range []string{"COMISS", "COMISD", "UCOMISS", "UCOMISD", "VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + fmt.Fprintf(&declarations, "func %s(seed %s) %s\n", op, value, value) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\nMOVL $0,AX\nMOVL AX,X0\nMOVL AX,X1\nPUSH%s seed+0(FP)\nPOPF%s\n%s X0,X1\nPUSHF%s\nPOP%s AX\nMOV%s AX,ret+%d(FP)\nRET\n", + op, 2*size, word, word, op, word, word, word, size) + } + t.Run(target.triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil || strings.Count(ir, "store i1 false, ptr %flags_af") != 16 { + t.Fatal("initialized compare family lost its public status effect", api, err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + } + }) + } +} diff --git a/amd64_direction.go b/amd64_direction.go new file mode 100644 index 000000000..adecda81a --- /dev/null +++ b/amd64_direction.go @@ -0,0 +1,50 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86DirectionControlSpec struct { + backward bool +} + +// These are Go's complete two ynone/Px rows, in both x86 source modes. +// DF is separate from the six arithmetic status flags and from pointer facts. +var x86DirectionControlSpecs = map[Op]x86DirectionControlSpec{ + "CLD": {}, + "STD": {backward: true}, +} + +func x86DirectionControlProperties(op Op, ins Instr, arch string) (x86DirectionControlSpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86DirectionControlSpecs[Op(name)] + if !known { + return spec, false, nil + } + if len(ins.Args) != 0 { + return spec, true, fmt.Errorf("%s takes no operands; %s direction-control ynone grammar: %q", name, arch, ins.Raw) + } + if arch != "386" && arch != "amd64" || dotted || len(ins.Args) != 0 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return spec, true, fmt.Errorf("%s %s outside Go direction-control ynone grammar: %q", arch, op, ins.Raw) + } + return spec, true, nil +} + +func (c *amd64Ctx) lowerDirectionControl(ins Instr) (bool, error) { + spec, known, err := x86DirectionControlProperties(ins.Op, ins, c.goarch) + if !known || err != nil { + return known, err + } + if c.directionSlot != "" { + fmt.Fprintf(c.b, " store i1 %t, ptr %s\n", spec.backward, c.directionSlot) + } + // Every target consumes virtual DF in the exact string helpers. Only a + // matching x86 backend also owns the physical direction-flag side effect; + // embedding CLD/STD in an ARM/WASM module is not portable emulation. + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + fmt.Fprintf(c.b, " call void asm sideeffect %q, \"~{dirflag}\"()\n", strings.ToLower(string(ins.Op))) + } + return true, nil +} diff --git a/amd64_directive_effects.go b/amd64_directive_effects.go new file mode 100644 index 000000000..214d1f463 --- /dev/null +++ b/amd64_directive_effects.go @@ -0,0 +1,97 @@ +package plan9asm + +import ( + "fmt" + "math" + "strings" +) + +type x86DirectiveEffect uint8 + +const ( + x86DirectiveLayout x86DirectiveEffect = iota + 1 + x86DirectiveMetadata +) + +type x86DirectiveForm struct { + effect x86DirectiveEffect + alignment int64 +} + +// These are the existing metadata directives and the complete PCALIGN family, +// not a blanket pseudo-op exemption. They preserve numerical availability but +// still have native layout/GC effects. This grammar grants no caller, raw-PC, +// stack, metadata ownership or physical function-alignment permission. +func x86DirectiveProperties(ins Instr) (x86DirectiveForm, bool, error) { + var form x86DirectiveForm + base, _, _ := strings.Cut(string(ins.Op), ".") + count := 2 + switch base { + case "PCALIGN": + form.effect, count = x86DirectiveLayout, 1 + case "PCALIGNMAX": + form.effect = x86DirectiveLayout + case "PCDATA", "FUNCDATA": + form.effect = x86DirectiveMetadata + default: + return form, false, nil + } + fail := func() (x86DirectiveForm, bool, error) { + return form, true, fmt.Errorf("invalid x86 %s directive operands: %q", base, ins.Raw) + } + if string(ins.Op) != base || len(ins.Args) != count { + return fail() + } + constant := func(arg Operand) bool { + return arg.Kind == OpImm && arg.ImmRaw == "" && !arg.ImmIsFloat + } + if !constant(ins.Args[0]) { + return fail() + } + if form.effect == x86DirectiveLayout { + // obj.AlignmentPaddingLength: both spellings retain the function's + // required alignment, even when PCALIGNMAX emits no padding here. + form.alignment = ins.Args[0].Imm + if form.alignment < 8 || form.alignment > 2048 || form.alignment&(form.alignment-1) != 0 { + return fail() + } + if count == 2 && (!constant(ins.Args[1]) || ins.Args[1].Imm < 0 || ins.Args[1].Imm >= form.alignment) { + return fail() + } + return form, true, nil + } + if base == "PCDATA" { + // The backend's ypcdata row is Yi32,Yi32 and linkpcln retains an + // int32 value. Frontend constant syntax alone is insufficient. + if ins.Args[0].Imm < 0 || ins.Args[0].Imm > math.MaxInt32 || !constant(ins.Args[1]) || + ins.Args[1].Imm < math.MinInt32 || ins.Args[1].Imm > math.MaxInt32 { + return fail() + } + return form, true, nil + } + // cmd/asm.validSymbol alone is not the full contract: x86's yfuncdata + // row requires memory and linkpcln requires its offset to be zero. An + // accepted frontend offset otherwise panics during object production. + arg := ins.Args[1] + symbol := strings.TrimSpace(arg.Sym) + if ins.Args[0].Imm < 0 || ins.Args[0].Imm > math.MaxInt32 || + arg.Kind != OpSym || !strings.HasSuffix(symbol, "(SB)") { + return fail() + } + name, offset := splitSymPlusOff(strings.TrimSpace(strings.TrimSuffix(symbol, "(SB)"))) + if offset != 0 || !validPPIdentifier(strings.TrimSuffix(name, "<>")) { + return fail() + } + return form, true, nil +} + +func validateX86DirectiveFile(file *File) error { + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + if _, known, err := x86DirectiveProperties(ins); known && err != nil { + return fmt.Errorf("%s: %w", fn.Sym, err) + } + } + } + return nil +} diff --git a/amd64_directive_effects_go127_test.go b/amd64_directive_effects_go127_test.go new file mode 100644 index 000000000..1a7e1e5c1 --- /dev/null +++ b/amd64_directive_effects_go127_test.go @@ -0,0 +1,41 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "testing" +) + +// PCALIGNMAX was added after the oldest supported SDK. Current Go checks its +// real encoding; untagged tests still check every consumer on historical Go. +func TestX86DirectiveCurrentGoCompleteFormats(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86DirectiveValidForms() { + t.Run(arch+"/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT value(SB),4,$0-0\n"+row+"\nRET\n", "", true) + }) + } + for _, row := range []string{ + "PCALIGN", "PCALIGN $16,$1", "PCALIGN AX", "PCALIGN $16.0", + "PCDATA $0", "PCDATA AX,$1", "PCDATA $0,$1.0", + "FUNCDATA $0", "FUNCDATA AX,metadata(SB)", "FUNCDATA $0,$1", + "FUNCDATA $0,metadata(SB)(AX)", "FUNCDATA $0,metadata(SB)(AX*1)", + } { + t.Run(arch+"/invalid/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT value(SB),4,$0-0\n"+row+"\nRET\n", "", false) + }) + } + } +} + +func TestX86DirectiveCurrentGoFiveTargetObjects(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, row := range []string{"PCALIGN $16", "PCALIGNMAX $16,$7", "PCDATA $0,$-1", "FUNCDATA $0,metadata(SB)"} { + t.Run(target.triple+"/"+row, func(t *testing.T) { + source := fmt.Sprintf("TEXT value(SB),4,$0-4\nMOVL $1,AX\n%s\nMOVL AX,ret+0(FP)\nRET\n", row) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + }) + } + } +} diff --git a/amd64_directive_effects_test.go b/amd64_directive_effects_test.go new file mode 100644 index 000000000..f474fa92b --- /dev/null +++ b/amd64_directive_effects_test.go @@ -0,0 +1,218 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86DirectiveResultInitialization(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, directive := range []string{ + "PCALIGN $16", + "PCALIGNMAX $16,$7", + "PCDATA $0,$-1", + "FUNCDATA $0,metadata(SB)", + } { + t.Run(arch+"/"+directive, func(t *testing.T) { + body := fmt.Sprintf("MOVL $1,AX\n%s\nMOVL AX,ret0+0(FP)\nRET", directive) + _, file, sig, _ := x86ResultInitializationFixture(t, arch, "func Value() uint32", body) + for name, check := range map[string]func(Func, FuncSig, string) error{ + "public": CheckX86ABI0ResultInitialization, + "private": checkX86ScalarFPResultInitialization, + } { + if err := check(file.Funcs[0], sig, arch); err != nil { + t.Errorf("%s: valid directive changed numerical availability: %v", name, err) + } + } + }) + } + } +} + +func x86DirectiveValidForms() []string { + forms := []string{ + "PCDATA $0,$-1", "PCDATA $1,$2", + "FUNCDATA $0,metadata(SB)", "FUNCDATA $1,metadata<>+0(SB)", + "FUNCDATA $2,metadata+(4-4)(SB)", + } + for _, alignment := range []int{8, 16, 32, 64, 128, 256, 512, 1024, 2048} { + forms = append(forms, fmt.Sprintf("PCALIGN $%d", alignment)) + for _, limit := range []int{0, alignment - 1} { + forms = append(forms, fmt.Sprintf("PCALIGNMAX $%d,$%d", alignment, limit)) + } + } + return forms +} + +func TestX86DirectiveNumericalStateUnchanged(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86DirectiveValidForms() { + t.Run(arch+"/"+row, func(t *testing.T) { + ins := x86PureGPUnknownInstruction(t, row) + for _, transfer := range []func(Instr, *x86ResultState, []bool, int64, string) bool{ + x86ResultTransfer, x86ScalarFPResultTransfer, + } { + for _, shape := range []bool{false, true} { + state := x86ResultState{fp: []bool{true, false}, auto: []bool{false, true}, + pointerFP: map[int64]bool{0: true}, shapeOnly: shape, flags: x86ByteFlagCF} + for index := range state.gp { + state.gp[index] = uint8(index * 7) + state.pointerGP[index] = index%2 == 0 + } + for index := range state.xmm { + state.xmm[index] = uint16(index * 31) + state.vectorUpper[index] = uint64(index) << 40 + } + state.masks[3] = true + before := state.clone() + if !transfer(ins, &state, nil, 0, arch) || !reflect.DeepEqual(state, before) { + t.Fatal("directive changed definedness or pointer/flag identity") + } + } + } + }) + } + } +} + +func TestX86DirectiveMalformedAndUnknownEffects(t *testing.T) { + rows := []string{ + "PCALIGN", "PCALIGN $16,$1", "PCALIGN $0", "PCALIGN $4", "PCALIGN $24", "PCALIGN $4096", + "PCALIGN $-16", "PCALIGN $16.0", "PCALIGN $(missing)", "PCALIGN AX", "PCALIGN 16(AX)", "PCALIGN.Z $16", + "PCALIGNMAX", "PCALIGNMAX $16", "PCALIGNMAX $16,$0,$1", "PCALIGNMAX $16,$16", "PCALIGNMAX $16,$-1", + "PCALIGNMAX $24,$1", "PCALIGNMAX $16,$1.0", "PCALIGNMAX $16,AX", "PCALIGNMAX AX,$1", "PCALIGNMAX $16,$missing", + "PCDATA $0", "PCDATA $0,$1,$2", "PCDATA AX,$1", "PCDATA $0,$1.0", "PCDATA $0,$missing", "PCDATA.Z $0,$1", + "FUNCDATA $0", "FUNCDATA $0,$1", "FUNCDATA AX,metadata(SB)", "FUNCDATA $0.0,metadata(SB)", + "FUNCDATA $0,metadata", "FUNCDATA $0,metadata(FP)", "FUNCDATA $0,metadata(SB)(AX)", + "FUNCDATA $0,metadata(SB)(AX*1)", "FUNCDATA $0,metadata+missing(SB)", "FUNCDATA.Z $0,metadata(SB)", + "FUNCDATA $0,$metadata(SB)", "FUNCDATA $0,metadata+4(SB)", "FUNCDATA $-1,metadata(SB)", + "PCDATA $-1,$0", "PCDATA $0,$2147483648", "PCDATA $0,$-2147483649", + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + ins := x86PureGPUnknownInstruction(t, row) + if _, known, err := x86DirectiveProperties(ins); !known || err == nil { + t.Fatal("invalid directive accepted", known, err) + } + for _, target := range x86RawReturnTargets { + for _, dead := range []bool{false, true} { + body := row + "\nMOVL $1,ret+0(FP)\nRET" + if dead { + body = "MOVL $1,ret+0(FP)\nRET\n" + row + } + source := "TEXT ·view(SB),4,$0-4\n" + body + "\n" + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32", target.arch, target.triple, api); err == nil { + t.Fatal("malformed source bypassed full source grammar", target.triple, dead, api) + } + } + } + } + }) + } + for _, row := range []string{"BYTE $0x90", "WORD $0x9090", "LONG $0", "QUAD $0", "NO_LOCAL_POINTERS", "UNDEF", "NOT_A_DIRECTIVE"} { + if _, known, _ := x86DirectiveProperties(x86PureGPUnknownInstruction(t, row)); known { + t.Fatal("unregistered pseudo/native operation acquired numerical noop", row) + } + } +} + +func TestX86DirectiveDoesNotInitializeOrAuthorizeEffects(t *testing.T) { + for _, row := range []string{"PCALIGN $16", "PCALIGNMAX $16,$7", "PCDATA $0,$-1", "FUNCDATA $0,metadata(SB)"} { + for name, body := range map[string]string{ + "missing result": row + "\nRET", + "unknown GP": row + "\nMOVL AX,ret0+0(FP)\nRET", + "unknown vector": row + "\nMOVL X0,ret0+0(FP)\nRET", + "unknown flags": row + "\nJE missing\nMOVL $1,ret0+0(FP)\nmissing:\nRET", + "native pointer": "MOVL $1,AX\n" + row + "\nMOVL (AX),BX\nMOVL BX,ret0+0(FP)\nRET", + "raw bytes": row + "\nBYTE $0x90\nMOVL $1,ret0+0(FP)\nRET", + "unknown dead": "MOVL $1,ret0+0(FP)\nRET\n" + row + "\nNOT_A_DIRECTIVE\nRET", + } { + t.Run(row+"/"+name, func(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + _, file, sig, _ := x86ResultInitializationFixture(t, arch, "func Value() uint32", body) + for _, check := range []func(Func, FuncSig, string) error{ + CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization, + } { + if err := check(file.Funcs[0], sig, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("directive supplied missing numerical/native authority", err) + } + } + } + }) + } + } +} + +func TestX86DirectiveThreeAPIsFiveLLVMTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + body := strings.Join(x86DirectiveValidForms(), "\n") + source := "TEXT ·view(SB),4,$0-4\nMOVL $305419896,AX\n" + body + "\nMOVL AX,X0\nMOVL X0,ret+0(FP)\nRET\n" + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32", target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, api) + } + }) + } +} + +func TestX86DirectiveCommonGoFormats(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + // Old x86 backends predate alignment directives. The shared grammar + // and LLVM tests above still run there, while these original-Go forms + // are genuinely common to every supported compiler. + for _, row := range []string{"PCDATA $0,$-1", "FUNCDATA $0,metadata<>+0(SB)"} { + goAssemblerOracleOutput(t, arch, "TEXT value(SB),4,$0-0\n"+row+"\nRET\n", "", true) + } + } +} + +func TestX86DirectiveLayoutIsNotNumericalAuthority(t *testing.T) { + for _, row := range []string{"PCALIGN $32", "PCALIGNMAX $32,$0", "PCALIGNMAX $32,$31"} { + t.Run(row, func(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := "TEXT refer(SB),4,$0-0\nMOVL data(SB),AX\nRET\nTEXT data(SB),4,$0-0\n" + row + "\nLONG $0xdeadbeef\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for pass := 0; pass < 2; pass++ { + file, err = normalizeX86RawFile(file, arch) + if err != nil { + t.Fatal(err) + } + if file.Funcs[1].X86RawAlign != 32 || string(file.Funcs[1].X86RawText) != string([]byte{0xef, 0xbe, 0xad, 0xde}) { + t.Fatal("layout-only directive lost exact raw TEXT alignment/bytes") + } + } + framed, err := Parse(ArchAMD64, strings.Replace(source, "TEXT data(SB),4,$0-0", "TEXT data(SB),4,$16-0", 1)) + if err != nil { + t.Fatal(err) + } + if _, err := normalizeX86RawFile(framed, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("layout-only directive authorized a native source frame", err) + } + // A native byte jump cannot infer its displacement across an + // alignment boundary from ordinary numerical no-op semantics. + crossing, err := Parse(ArchAMD64, "TEXT value(SB),4,$0-0\nBYTE $0xeb\nBYTE $2\n"+row+"\nBYTE $0x90\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := normalizeX86RawFile(crossing, arch); err == nil { + t.Fatal("unknown native byte jump crossed layout boundary") + } + } + }) + } +} diff --git a/amd64_duplicate_move_forms.go b/amd64_duplicate_move_forms.go new file mode 100644 index 000000000..86f9b7d98 --- /dev/null +++ b/amd64_duplicate_move_forms.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86DuplicateMoveSpec struct { + laneBytes int + odd bool + vector bool +} + +// All three legacy yxm and all three shared _yvmovddup rows. The selector +// describes numerical bytes; it never reduces the native memory access tuple. +var x86DuplicateMoveSpecs = map[Op]x86DuplicateMoveSpec{ + "MOVDDUP": {8, false, false}, "MOVSHDUP": {4, true, false}, "MOVSLDUP": {4, false, false}, + "VMOVDDUP": {8, false, true}, "VMOVSHDUP": {4, true, true}, "VMOVSLDUP": {4, false, true}, +} + +type x86DuplicateMoveForm struct { + x86PackedBinaryForm + spec x86DuplicateMoveSpec + source Operand +} + +func x86DuplicateMoveProperties(op Op, ins Instr, arch string) (x86DuplicateMoveForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86DuplicateMoveSpecs[Op(base)] + form := x86DuplicateMoveForm{spec: spec} + if !known { + return form, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || len(ins.Args) != 2 && len(ins.Args) != 3 { + return form, true, fmt.Errorf("%s %s has an unbound dispatch or unary operand count", arch, op) + } + local, grammar := ins, x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP} + local.Op = op + if spec.vector { + grammar.encoding, grammar.zeroing = x86PackedLogicalVEXOrEVEX, true + destination := ins.Args[len(ins.Args)-1] + local.Args = []Operand{ins.Args[0], destination, destination} + if len(ins.Args) == 3 { + local.Args = []Operand{ins.Args[0], destination, ins.Args[1], destination} + } + } + var err error + form.x86PackedBinaryForm, err = x86ConflictFunnelOperandGrammar(local, arch, grammar, spec.vector) + form.second, form.destination = -1, len(ins.Args)-1 + if form.mask >= 0 { + form.mask = 1 + } + if form.memory >= 0 && spec.laneBytes == 8 && form.byteWidth == 16 { + form.memoryBytes = 8 // Only X-width DDUP reads one scalar qword. + } + form.source = ins.Args[0] + return form, true, err +} + +func (form x86DuplicateMoveForm) sourceByte(output int) int { + lane := output / form.spec.laneBytes + if form.spec.odd { + lane |= 1 + } else { + lane &^= 1 + } + return lane*form.spec.laneBytes + output%form.spec.laneBytes +} + +func (form x86DuplicateMoveForm) sourceMask() uint64 { + var mask uint64 + for output := 0; output < form.byteWidth; output++ { + mask |= uint64(1) << uint(form.sourceByte(output)) + } + return mask +} + +func x86ResultDuplicateMoveStep(ins Instr, state *x86ResultState, owned []bool, arch string, nativeRead bool) (bool, bool) { + form, known, err := x86DuplicateMoveProperties(ins.Op, ins, arch) + if !known || err != nil || !x86ResultNamedPredicateInstruction(ins) { + return false, known + } + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + widthMask := x86CallerVectorByteMask(int64(form.byteWidth)) + if form.mask >= 0 { + mask, _ := amd64ParseKReg(ins.Args[form.mask].Reg) + if !state.masks[mask] || !form.zeroing && state.vectorBytes(destination)&widthMask != widthMask { + return false, true + } + } + if form.memory >= 0 { + if form.legacy && form.spec.laneBytes == 4 { + // Virtual FP/AUTO bytes are not the original aligned m128. + if _, static := x86StaticSBReadAddress(form.source, arch); !static && + (!nativeRead || !x86OrdinaryNativeMemoryOperand(form.source, arch)) { + return false, true + } + } + if !x86ResultReadWithNativeContinuation(ins, form.source, int64(form.memoryBytes), state, owned, arch, nativeRead) { + return false, true + } + } else { + source, _ := amd64VectorRegisterIndex(form.source.Reg, form.byteWidth) + required := form.sourceMask() + if state.vectorBytes(source)&required != required { + return false, true + } + } + // Every potentially selected numerical byte is now known. All reads, + // including an aliased source and masked merge, precede any output write. + state.writeVector(destination, form.byteWidth, widthMask, form.legacy) + return true, true +} diff --git a/amd64_dword_product_displacement_test.go b/amd64_dword_product_displacement_test.go new file mode 100644 index 000000000..08301e7f4 --- /dev/null +++ b/amd64_dword_product_displacement_test.go @@ -0,0 +1,274 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86DwordProductDisplacementOracle struct { + row string + width int + tuple int + units int + mask bool + high bool + mem MemRef + code []byte +} + +// These tuples and offsets are independently prescribed from the actual +// Go encoder's Full/N16,N32,N64 and BcstN8 rows, not production properties. +func x86DwordProductDisplacementOracles(t *testing.T) []x86DwordProductDisplacementOracle { + t.Helper() + var result []x86DwordProductDisplacementOracle + addresses := []MemRef{ + {Base: AX}, {Base: BP}, {Base: SP, Index: CX, Scale: 2}, + {Base: AX, Index: CX, Scale: 4}, {Base: "R13", Index: "R12", Scale: 8}, + } + for _, name := range []string{"VPMULDQ", "VPMULUDQ"} { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + for masking := 0; masking < 3; masking++ { + for _, units := range []int{-129, -128, -1, 1, 127, 128} { + for _, memory := range addresses { + tuple, op := width, name + if broadcast { + tuple, op = 8, name+".BCST" + } + if masking == 2 { + op += ".Z" + } + memory.Off = int64(units * tuple) + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + second, destination := bank+"1", bank+"2" + high := masking == 0 && !broadcast && width != 64 + if high { + second, destination = bank+"17", bank+"18" // Force actual EVEX. + } + args := (Operand{Kind: OpMem, Mem: memory}).String() + "," + second + if masking == 1 { + args += ",K1" + } else if masking == 2 { + args += ",K7" + } + row := op + " " + args + "," + destination + code := x86OrderingHintGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n") + if len(code) < 8 || code[0] != 0x62 || code[2]&0x80 == 0 || code[len(code)-1] != 0xc3 { + t.Fatalf("independent original EVEX.W1 object changed: %s %x", row, code) + } + mod, end := code[5]>>6, 6 + if code[5]&7 == 4 { + end++ // Independently observe SIB, not decoded source. + } + if units >= -128 && units <= 127 { + if mod != 1 || int(int8(code[end])) != units || len(code) != end+2 { + t.Fatalf("original compressed disp8/%d changed: %s %x", tuple, row, code) + } + } else if mod != 2 || len(code) != end+5 { + t.Fatalf("original unscaled disp32 boundary changed: %s %x", row, code) + } + result = append(result, x86DwordProductDisplacementOracle{row, width, tuple, units, masking != 0, high, memory, code[:len(code)-1]}) + } + } + } + } + } + } + return result +} + +func TestX86DwordProductCompressedDisp8IndependentPhysicalSource(t *testing.T) { + fixtures := x86DwordProductDisplacementOracles(t) + for _, fixture := range fixtures { + for _, mode := range []int{32, 64} { + for _, prefix := range []struct { + byte byte + segment Reg + }{{0, ""}, {0x64, FS}, {0x65, GS}} { + t.Run(fmt.Sprintf("%s/mode%d/prefix%x", fixture.row, mode, prefix.byte), func(t *testing.T) { + code := append([]byte(nil), fixture.code...) + if prefix.byte != 0 { + code = append([]byte{prefix.byte}, code...) + } + ins, length, known, err := decodedX86DwordToQwordMultiplyInstruction(code, mode) + if mode == 32 && (fixture.high || fixture.mem.Base == "R13") { + if !known || err == nil { + t.Fatal("extended physical mode32 bank/address acquired permission", ins, err) + } + return + } + want := fixture.mem + want.Segment = prefix.segment + if !known || err != nil || length != len(code) || !ins.x86Encoded || len(ins.Args) == 0 || ins.Args[0].Kind != OpMem || !reflect.DeepEqual(ins.Args[0].Mem, want) { + t.Fatalf("native physical tuple source changed: %x got=%+v length%d known%t err%v want=%+v", code, ins, length, known, err, want) + } + }) + } + for _, bad := range [][]byte{ + append([]byte{0x67}, fixture.code...), + append([]byte{0x64, 0x67}, fixture.code...), + append([]byte{0x65, 0x67}, fixture.code...), + } { + if _, _, known, err := decodedX86DwordToQwordMultiplyInstruction(bad, mode); !known || err == nil { + t.Fatal("address override acquired unbound source permission", fixture.row, mode, err) + } + } + } + wrongW := append([]byte(nil), fixture.code...) + wrongW[2] &^= 0x80 + if _, _, known, err := decodedX86DwordToQwordMultiplyInstruction(wrongW, 64); !known || err == nil { + t.Fatal("full/broadcast EVEX.W0 accepted", fixture.row, err) + } + } + t.Logf("%d independently assembled full/broadcast,unmasked/merge/zero,disp8/disp32,SIB/bank witnesses ×two modes ×three segment prefixes; address/W rejection retained", len(fixtures)) +} + +func x86DwordProductDirectives(code []byte) string { + var out strings.Builder + for _, value := range code { + fmt.Fprintf(&out, "BYTE $%d\n", value) + } + return out.String() +} + +func TestX86DwordProductTupleOffsetsThreeAPIsFiveTargetObjects(t *testing.T) { + fixtures := x86DwordProductDisplacementOracles(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/raw%t", triple, raw), func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + var source, declarations strings.Builder + declarations.WriteString("package tuples\n") + count := 0 + for _, fixture := range fixtures { + // SP/BP decoder identity is proved above, without using it + // to confer an unbound native frame in this object fixture. + if fixture.mem.Base == SP || fixture.mem.Base == BP || arch == "386" && (fixture.high || fixture.mem.Base == "R13" || !raw && fixture.mask) { + continue + } + row := fixture.row + if raw { + row = x86DwordProductDirectives(fixture.code) + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", count, row) + fmt.Fprintf(&declarations, "func F%d()\n", count) + count++ + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + ir := x86DwordProductOriginalFormatObject(t, source.String(), declarations.String(), arch, triple, 0) + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + t.Logf("%d original tuple/disp32 forms: three public Context checks plus emitter objects", count) + }) + } + } +} + +func TestX86DwordProductTupleRawRejectedPrefixesThreeAPIs(t *testing.T) { + for _, name := range []string{"VPMULDQ", "VPMULUDQ"} { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + op, tuple := name, width + if broadcast { + op, tuple = name+".BCST", 8 + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + row := fmt.Sprintf("%s %d(AX),%s1,K1,%s2", op, tuple, bank, bank) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n") + wrongW := append([]byte(nil), code[:len(code)-1]...) + wrongW[2] &^= 0x80 + for _, bad := range [][]byte{append([]byte{0x67}, code[:len(code)-1]...), append([]byte{0x64, 0x67}, code[:len(code)-1]...), wrongW} { + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(fmt.Sprintf("%s/%x/%s", row, bad, triple), func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + source := "TEXT ·F(SB),4,$0-0\n" + x86DwordProductDirectives(bad) + "RET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"F": {Name: "F", Ret: Void}}, ResolveSym: func(name string) string { return strings.TrimPrefix(name, "·") }} + if ir, err := Translate(file, options); err == nil || ir != "" { + t.Fatal("reserved tuple/address text accepted", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err == nil || !module.IsNil() { + t.Fatal("reserved tuple/address module accepted", err) + } + pkg := mustGoPackage(t, "tuples", "package tuples\nfunc F()\n") + result, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: options.ResolveSym}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err == nil || result != nil { + t.Fatal("reserved tuple/address Go module accepted", err) + } + }) + } + } + } + } + } +} + +func TestX86DwordProductVEXDisplacementsRemainUnscaled(t *testing.T) { + for _, name := range []string{"VPMULDQ", "VPMULUDQ"} { + for _, bank := range []string{"X", "Y"} { + for _, units := range []int{-129, -128, -1, 1, 127, 128} { + for _, memory := range []MemRef{{Base: AX}, {Base: BP}, {Base: AX, Index: CX, Scale: 4}, {Base: SP, Index: CX, Scale: 2}, {Base: "R13", Index: "R12", Scale: 8}} { + memory.Off = int64(units) + row := fmt.Sprintf("%s %s,%s1,%s2", name, (Operand{Kind: OpMem, Mem: memory}).String(), bank, bank) + original := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+row+"\nRET\n") + if original[0] != 0xc4 && original[0] != 0xc5 { + t.Fatal("actual Go no longer emitted VEX", row, original) + } + for _, mode := range []int{32, 64} { + for _, segment := range []Reg{"", FS, GS} { + t.Run(fmt.Sprintf("%s/mode%d/segment%s", row, mode, segment), func(t *testing.T) { + code := append([]byte(nil), original[:len(original)-1]...) + if segment != "" { + prefix := byte(0x64) + if segment == GS { + prefix = 0x65 + } + code = append([]byte{prefix}, code...) + } + ins, length, known, err := decodedX86DwordToQwordMultiplyInstruction(code, mode) + if mode == 32 && memory.Base == "R13" { + if !known || err == nil { + t.Fatal("VEX extended address acquired mode32 binding") + } + return + } + want := memory + want.Segment = segment + if !known || err != nil || length != len(code) || !ins.x86Encoded || ins.Args[0].Kind != OpMem || !reflect.DeepEqual(ins.Args[0].Mem, want) { + t.Fatal("VEX source displacement incorrectly scaled", row, ins, err, want) + } + }) + } + } + } + } + } + } +} diff --git a/amd64_dword_product_fault_runtime_unix_test.go b/amd64_dword_product_fault_runtime_unix_test.go new file mode 100644 index 000000000..935188954 --- /dev/null +++ b/amd64_dword_product_fault_runtime_unix_test.go @@ -0,0 +1,179 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Independent fault envelopes include all eight bytes of each potentially +// active qword, even though only its low dword affects the numerical product. +// Original SSE instructions are the bytes emitted by Go, not mnemonic aliases +// assembled by the C compiler. No original AVX512 execution is claimed here. +func TestCrossLinuxRuntimeMatrixX86DwordProductDeadResultFaultEnvelopes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, prototypes, originals, cases strings.Builder + declarations.WriteString("package faultoracle\n") + count := 0 + for _, name := range x86DwordProductOracleNames { + legacy := !strings.HasPrefix(name, "V") + widths := []int{16} + if !legacy { + widths = append(widths, 32, 64) + } + for _, width := range widths { + for broadcast := 0; broadcast < 2; broadcast++ { + if legacy && broadcast != 0 { + continue + } + for masking := 0; masking < 3; masking++ { + if legacy && masking != 0 { + continue + } + tuple := width + if broadcast != 0 { + tuple = 8 + } + for _, displacement := range []int{0, tuple, -tuple} { + if legacy && displacement != 0 { + continue + } + fn := fmt.Sprintf("F%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + op, args := name, fmt.Sprintf("%d(BX),%s1", displacement, bank) + if broadcast != 0 { + op += ".BCST" + } + if masking == 2 { + op += ".Z" + } + if !legacy { + if masking != 0 { + args += ",K1" + } + args += "," + bank + "2" + } + instruction := op + " " + args + if displacement != 0 { + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+instruction+"\nRET\n") + instruction = x86DwordProductDirectives(actual[:len(actual)-1]) + } + fmt.Fprintf(&declarations, "func %s(p *[64]byte,mask uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),BX\nMOVQ mask+8(FP),AX\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\nKMOVQ AX,K1\n%s\nRET\n", fn, instruction) + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint64_t);\n", fn) + original := "NULL" + if legacy { + original = "original_" + fn + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\nPXOR X1,X1\n"+name+" (DI),X1\nRET\n") + var bytes []string + for _, value := range actual { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "extern void %s(const uint8_t*,uint64_t);\n__asm__(%q);\n", original, ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + } + fmt.Fprintf(&cases, "{%s,%s,%d,%d,%d,%t,%d},\n", fn, original, width, broadcast, masking, legacy, displacement) + } + } + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + strict, deviations := 1, 0 + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + strict, deviations = 0, 8 + } + reference := fmt.Sprintf(x86DwordProductFaultC, prototypes.String(), originals.String(), cases.String(), strict) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, + {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "dword-product-fault", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("dword-product fault outcomes %d original SSE outcomes 80 alignment deviations %d failures 0\n", count*10*4, deviations) + if string(output) != want { + t.Fatalf("native width/alignment, dead result or qword masked fault suppression: %s", output) + } + t.Log(strings.TrimSpace(string(output))) + }) + } +} + +const x86DwordProductFaultC = ` +#include +#include +#include +#include +#include +#include +#include +#include +#include +%s +%s +typedef struct {void(*translated)(const uint8_t*,uint64_t);void(*original)(const uint8_t*,uint64_t);unsigned width,broadcast,masking,legacy;int displacement;} Kernel; +static Kernel kernels[]={%s}; +static const uint64_t masks[]={0,1,0x80,UINT64_MAX}; +static unsigned need(Kernel k,uint64_t mask) { + if(!k.masking) return k.broadcast?8:k.width; + unsigned high=0; + for(unsigned i=0;i>i)&1) high=k.broadcast?8:(i+1)*8; + return high; +} +static int observe(Kernel k,unsigned address,uint64_t mask,int original) { + pid_t child=fork(); + if(child<0) return -1; + if(child==0) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + _Alignas(64) uint8_t data[256];memset(data,0xa7,sizeof(data)); + const uint8_t *p=data+64; + if(address==1) p++; + if(address==2) p=NULL; + if(address>=3) { + long page=sysconf(_SC_PAGESIZE);if(page<=0) _exit(121); + uint8_t *mapping=mmap(NULL,2*(size_t)page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(mapping==MAP_FAILED) _exit(122); + memset(mapping,0xa7,(size_t)page); + if(mprotect(mapping+page,(size_t)page,PROT_NONE)) _exit(123); + unsigned available[]={0,4,7,8,k.width-1,k.width,64}; + p=mapping+page-available[address-3]; + } + if(original) k.original(p,mask);else k.translated(p,mask); + _exit(0); + } + int status=0;if(waitpid(child,&status,0)!=child) return -2; + if(WIFEXITED(status)) return WEXITSTATUS(status)?-3:0; + if(WIFSIGNALED(status)&&(WTERMSIG(status)==SIGSEGV||WTERMSIG(status)==SIGBUS)) return 1; + return -4; +} +static int expected(Kernel k,unsigned address,uint64_t mask) { + unsigned bytes=need(k,mask),available[]={0,4,7,8,k.width-1,k.width,64}; + if(k.legacy&&(address==1||(address>=3&&(available[address-3]%%16)!=0))) return 1; + return (address==2&&bytes)||(address>=3&&bytes&&(int)bytes+k.displacement>(int)available[address-3]); +} +int main(void) { + unsigned count=0,native=0,deviations=0,failures=0;const int strict=%d; + for(unsigned i=0;i= 8 || rule.RowIndex != index || rule.Ytab != "_yvandnpd" || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != []int{2, 2, 0, 3, 0, 3, 0, 3}[index] || + rule.EncodingOffset != []int{0, 2, 4, 4, 7, 7, 10, 10}[index] || + strings.Join(rule.Classes, ",") != classes[index] || !reflect.DeepEqual(rule.OpBytes, opbytes) { + t.Fatal("independent whole Go encoder row changed", rule) + } + counts[rule.Opcode]++ + } + if len(x86DwordProductSpecs) != 4 || counts["VPMULDQ"] != 8 || counts["VPMULUDQ"] != 8 { + t.Fatal("entire signed/unsigned family is not closed", counts) + } + for _, name := range x86DwordProductOracleNames { + if _, exists := x86DwordProductSpecs[Op(name)]; !exists { + t.Fatal("production omitted independent family member", name) + } + } + // PMULUDQ is a decoder alias, never another accepted Go source mnemonic. + arch, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil || strings.Contains(string(arch), "instructions[\"PMULUDQ\"]") { + t.Fatal("actual Go alias inventory changed", err) + } +} + +func TestX86DwordProductWholeFormatsEffectsAndMemoryShapes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + rows := x86DwordProductOracleRows(t, arch) + var source strings.Builder + for i, row := range rows { + fmt.Fprintf(&source, "TEXT f%d(SB),4,$64-64\n%s\nRET\n", i, row) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for index, fn := range file.Funcs { + ins := fn.Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, known, err := x86DwordProductProperties(ins.Op, ins, arch) + high := arch == "386" && strings.Contains(rows[index], "20") + if !known || high && !errors.Is(err, ErrProbeNeedsContext) || !high && err != nil { + t.Fatal("Go-accepted source lost syntax/context classification", arch, rows[index], err) + } + if high { + continue + } + for i, operand := range ins.Args { + width, write, valid := x86InstructionMemoryShape(ins, i, arch) + if valid != (i == form.memory && !form.mmx) || valid && (write || width != int64(form.memoryBytes)) { + t.Fatal("native tuple shape changed", rows[index], width, write, valid) + } + if operand.Kind == OpFP { + width, write, valid = X86FPAccessBytes(ins, i, arch) + if !valid || write || width != int64(form.memoryBytes) { + t.Fatal("FP observation lost native memory span", rows[index], width, write, valid) + } + } + } + if form.legacy { + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 2 || effect.flagsDefined != 0 || effect.flagsKilled != 0 || effect.flagsRead != 0 || + !effect.operands[0].read || effect.operands[0].write || !effect.operands[1].read || !effect.operands[1].write || + effect.operands[0].width != int64(form.byteWidth) || effect.operands[1].width != int64(form.byteWidth) || !reflect.DeepEqual(effect.operands[0].operand, form.source) { + t.Fatal("legacy typed read/modify/write/flags/bank mismatch", effect, err) + } + } + if before != fmt.Sprintf("%#v", ins) || x86CallerVectorMoveKnown(ins) != !form.mmx { + t.Fatal("form modified source or borrowed MMX input permission", ins) + } + } + t.Logf("%s: %d independently Go-accepted complete-family forms; syntax/context/effects distinct", arch, len(rows)) + } +} + +func TestX86DwordProductThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/raw=%t", triple, raw), func(t *testing.T) { + rows := x86DwordProductOracleRows(t, arch) + if raw { + rows = nil + for _, name := range x86DwordProductOracleNames[2:] { + for _, row := range x86OfficialPackedBinaryWitnesses(t)[name] { + code := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + var directives strings.Builder + for _, value := range code[:len(code)-1] { + fmt.Fprintf(&directives, "BYTE $%d\n", value) + } + rows = append(rows, directives.String()) + } + } + } + var source, declarations strings.Builder + declarations.WriteString("package product\n") + count := 0 + for _, row := range rows { + if !raw { + parsed, err := Parse(ArchAMD64, "TEXT f(SB),4,$64-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := parsed.Funcs[0].Instrs[1] + form, _, err := x86DwordProductProperties(ins.Op, ins, arch) + if errors.Is(err, ErrProbeNeedsContext) || form.mmx || form.legacy && form.memory >= 0 && (form.source.Kind == OpFP || form.source.Kind == OpMem && form.source.Mem.Base == SP) { + continue + } + if err != nil { + t.Fatal(err) + } + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$64-64\n%s\nRET\n", count, row) + fmt.Fprintf(&declarations, "func F%d(value [64]byte)\n", count) + count++ + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + ir := x86DwordProductOriginalFormatObject(t, source.String(), declarations.String(), arch, triple, 64) + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + t.Logf("%d original formats: three public Context checks and emitter objects, not initialized public entries", count) + }) + } + } +} + +func x86DwordProductReservedZeroObservation(t *testing.T, name string) { + t.Helper() + source := "TEXT f(SB),4,$0-0\n" + name + ".Z (AX),X1,X2\nRET\n" + dir := t.TempDir() + asm, object := filepath.Join(dir, "reserved.s"), filepath.Join(dir, "reserved.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-o", object, asm) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), err == nil, "amd64"); failure != nil { + t.Fatal(failure) + } + if err != nil { + if !bytes.Contains(output, []byte("mask register must be specified for .Z instructions")) { + t.Fatal("unknown Go rejection", string(output)) + } + return + } + // Historical Go1.20 emits reserved EVEX.z=1,aaa=0. It is not a legal + // format and remains rejected by every translation API and raw decoder. + mapByte, opcode := byte(0xf1), byte(0xf4) + if name == "VPMULDQ" { + mapByte, opcode = 0xf2, 0x28 + } + code := []byte{0x62, mapByte, 0xf5, 0x88, opcode, 0x10, 0xc3} + var listing []string + for _, value := range code { + listing = append(listing, fmt.Sprintf("%02x", value)) + } + if !bytes.Contains(output, []byte(strings.Join(listing, " "))) { + t.Fatal("historical reserved encoding changed", string(output)) + } + if _, _, known, err := decodedX86DwordToQwordMultiplyInstruction(code[:len(code)-1], 64); !known || err == nil { + t.Fatal("historical maskless zero encoding gained physical permission", known, err) + } +} + +func TestX86DwordProductDispatchIdentityBeforeEmission(t *testing.T) { + for _, name := range x86DwordProductOracleNames { + opcode := Op(name) + row := name + " X0,X1" + if strings.HasPrefix(name, "V") { + row += ",X2" + } + for _, sourceOp := range []Op{"", opcode, Op(strings.ToLower(name)), "UNKNOWN", "MOVQ"} { + file, err := Parse(ArchAMD64, "TEXT one(SB),$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + ins.Op = sourceOp + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + known, term, err := ctx.lowerDwordToQwordMultiply(opcode, ins) + if sourceOp == "UNKNOWN" || sourceOp == "MOVQ" { + if !known || term || err == nil || before != out.String() { + t.Fatal("conflicting dispatch/source identity emitted IR", opcode, sourceOp, err) + } + } else if !known || term || err != nil { + t.Fatal("compatible helper identity rejected", opcode, sourceOp, err) + } + if ins.Op != sourceOp { + t.Fatal("helper mutated input identity") + } + } + } +} + +func TestX86DwordProductMalformedAndContextThreeAPIsBeforeIR(t *testing.T) { + for _, name := range x86DwordProductOracleNames { + vector := strings.HasPrefix(name, "V") + rows := []string{name + " $1,X1", name + " Y0,Y1", name + " X16,X0", name + " X0,(AX)", name + " (AX)(SP*2),X1"} + if vector { + rows = []string{name + " $1,X1,X2", name + " X0,Y1,X2", name + " X0,X1,K0,X2", name + " X0,X1,(AX)", name + ".BCST X0,X1,X2", name + ".SAE X0,X1,X2", name + ".RN_SAE X0,X1,X2", name + ".Z.BCST (AX),X1,K1,X2", name + " (AX)(SP*2),X1,X2"} + x86DwordProductReservedZeroObservation(t, name) + rows = append(rows, name+".Z (AX),X1,X2") + } + for _, row := range rows { + if !strings.Contains(row, ".Z (") { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(row+"/"+triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + file, err := Parse(ArchAMD64, "TEXT ·F(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + return + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"F": {Name: "F", Ret: Void}}, ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + if ir, err := Translate(file, options); err == nil || ir != "" { + t.Fatal("malformed text emitted IR", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err == nil || !module.IsNil() { + t.Fatal("malformed direct module accepted", err) + } + pkg := mustGoPackage(t, "product", "package product\nfunc F()\n") + result, err := translateGoModuleInContext(ctx, pkg, []byte("TEXT ·F(SB),4,$0-0\n"+row+"\nRET\n"), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: options.ResolveSym}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err == nil || result != nil { + t.Fatal("malformed Go module accepted", err) + } + }) + } + } + } +} + +func TestX86DwordProductMode32PhysicalMarkerAndMMXBank(t *testing.T) { + for _, name := range x86DwordProductOracleNames[2:] { + for _, row := range x86OfficialPackedBinaryWitnesses(t)[name] { + if strings.Count(row, ",") != 3 { + continue + } + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins, length, known, err := decodedX86DwordToQwordMultiplyInstruction(code[:len(code)-1], 32) + if !known || err != nil || length != len(code)-1 || !ins.x86Encoded { + t.Fatal("actual physical low-bank mode32 decode missing", row, ins, err) + } + if _, _, err := x86DwordProductProperties(ins.Op, ins, "386"); err != nil { + t.Fatal("physical mask inherited named frontend limit", row, err) + } + data, err := json.Marshal(ins) + if err != nil { + t.Fatal(err) + } + var restored Instr + if err := json.Unmarshal(data, &restored); err != nil { + t.Fatal(err) + } + restored.Raw = "BYTE genuine-looking decoder text" + if _, _, err := x86DwordProductProperties(restored.Op, restored, "386"); err == nil { + t.Fatal("JSON/raw text fabricated private physical marker") + } + } + } + for _, source := range []string{"AX", "SP", "BP", "R14", "R15"} { + row := "PMULULQ " + source + ",M1" + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + if known, _, err := ctx.lowerDwordToQwordMultiply(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]); !known || !errors.Is(err, ErrProbeNeedsContext) || out.String() != before { + t.Fatal("GP spelling became initialized MMX transport", row, err) + } + } +} diff --git a/amd64_dword_product_runtime_test.go b/amd64_dword_product_runtime_test.go new file mode 100644 index 000000000..55bda4703 --- /dev/null +++ b/amd64_dword_product_runtime_test.go @@ -0,0 +1,243 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86DwordProductIROptimize(t *testing.T, ir string) string { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := parseIRModuleInContext(ctx, ir) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + if err := module.RunPasses("default", llvm.TargetMachine{}, options); err != nil { + t.Fatalf("LLVM22 dword-product IR default: %v", err) + } + return module.String() +} + +// This scalar reference is independent of the production lane model, typed +// specifications and LLVM operations. All products fit widened 64-bit math. +const x86DwordProductCReference = ` +#include +#include +#include +#include +#include +static uint32_t dword(const uint8_t *p) { + return (uint32_t)p[0]|(uint32_t)p[1]<<8|(uint32_t)p[2]<<16|(uint32_t)p[3]<<24; +} +static uint64_t product(uint32_t x,uint32_t y,int sign) { + if(!sign) return (uint64_t)x*y; + int64_t a=x81) {x=(uint32_t)(sample*7919+i*104729);y=(uint32_t)(sample*65537+i*37);} + for(int j=0;j<4;j++) {a.value[i+j]=(uint8_t)(x>>(j*8));b.value[i+j]=(uint8_t)(y>>(j*8));} + } + struct Aligned beforeA=a,beforeB=b,beforeOld=old; + uint64_t seed=state?UINT64_C(0x8d5):0; + uint32_t profile=(uint32_t)(0x1f80|((sample%4)<<13)|(sample%64)),saved=_mm_getcsr(); + _mm_setcsr(profile); + program(out.value,a.value,b.value,old.value,masks[m],seed,profile); + uint32_t after=_mm_getcsr();_mm_setcsr(saved); + uint64_t flags=0;uint32_t csr=0;memcpy(&flags,out.value+64,8);memcpy(&csr,out.value+72,4); + if((flags&0x8d5)!=seed||csr!=profile||after!=profile||memcmp(&a,&beforeA,sizeof(a))||memcmp(&b,&beforeB,sizeof(b))||memcmp(&old,&beforeOld,sizeof(old))) return 1; + const uint8_t *retained=alias==0?a.value:alias==1?b.value:old.value; + for(int i=0;i<64;i++) { + int active=!masking||((masks[m]>>(i/8))&1); + int broadcast=source==2||source>=5; + int displacement=source==3?width:source==4?-width:source==5?8:source==6?-8:0; + const uint8_t *sourceBytes=(const uint8_t *)&a+offsetof(struct Aligned,value); + uint64_t value=product(dword(sourceBytes+displacement+(broadcast?0:i/8*8)),dword(b.value+i/8*8),sign); + uint8_t want=i>((i%8)*8)):(masking==2?0:retained[i])):(legacy?retained[i]:0); + if(out.value[i]!=want) {fprintf(stderr,"%s sample%d mask%llx byte%d got%x want%x\n",name,sample,(unsigned long long)masks[m],i,out.value[i],want);return 2;} + } + for(int i=0;i<64;i++) if(out.before[i]!=0xc3||out.after[i]!=0xc3) return 3; + for(int i=76;i<80;i++) if(out.value[i]!=0xc3) return 4; + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86DwordProductIndependentWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, reference, checks strings.Builder + declarations.WriteString("package narroworacle\n") + reference.WriteString(x86DwordProductCReference) + count := 0 + for _, name := range x86DwordProductOracleNames { + legacy, sign := 0, 0 + if !strings.HasPrefix(name, "V") { + legacy = 1 + } + if strings.Contains(name, "MULDQ") { + sign = 1 + } + widths := []int{16} + if legacy == 0 { + widths = append(widths, 32, 64) + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for source := 0; source < 7; source++ { + if legacy != 0 && source >= 2 { + continue + } + for alias := 0; alias < 3; alias++ { + if legacy != 0 && alias != 1 { + continue + } + for masking := 0; masking < 3; masking++ { + if legacy != 0 && masking != 0 { + continue + } + fn := fmt.Sprintf("P%d", count) + count++ + first := bank + "0" + op := name + if source != 0 { + first = "(CX)" + } + if source == 2 || source >= 5 { + op += ".BCST" + } + if source == 3 || source == 4 { + displacement := width + if source == 4 { + displacement = -width + } + first = fmt.Sprintf("%d(CX)", displacement) + } else if source >= 5 { + displacement := 8 + if source == 6 { + displacement = -8 + } + first = fmt.Sprintf("%d(CX)", displacement) + } + if masking == 2 { + op += ".Z" + } + args := first + "," + bank + "1" + if legacy == 0 { + if masking != 0 { + args += ",K1" + } + args += fmt.Sprintf(",%s%d", bank, alias) + } + instruction := op + " " + args + if source >= 3 { + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+instruction+"\nRET\n") + instruction = x86DwordProductDirectives(actual[:len(actual)-1]) + } + fmt.Fprintf(&declarations, "func %s(out *[80]byte,a,b,old *[64]byte,mask,flags uint64,csr uint32)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-52\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVQ mask+32(FP),R9\nVMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nKMOVQ R9,K1\nLDMXCSR csr+48(FP)\nPUSHQ flags+40(FP)\nPOPFQ\n%s\nVMOVUPS Z%d,(AX)\nPUSHFQ\nPOPQ R9\nMOVQ R9,64(AX)\nSTMXCSR 72(AX)\nRET\n", fn, instruction, alias) + fmt.Fprintf(&reference, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint64_t,uint32_t);\n", fn) + fmt.Fprintf(&checks, "if(check(%s,%q,%d,%d,%d,%d,%d,%d))return 1;\n", fn, op+" "+args, width, sign, source, alias, masking, legacy) + } + } + } + } + } + reference.WriteString("int main(void){\n" + checks.String() + "return 0;}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "dword-product-full", ir, reference.String(), level) + }) + } + t.Run("IR_O2/backend_O2", func(t *testing.T) { + execution.compileAndRunLLVM(t, "dword-product-ir-o2", x86DwordProductIROptimize(t, ir), reference.String(), "-O2") + }) + t.Logf("%d LLVM kernels ×97 initialized samples ×2 six-flag seeds ×4 masks; all64 output bytes, both operand/destination aliases, actual-original-Go raw ±Full/±m64bcst offsets, odd dwords, full MXCSR/input/canaries; explicit IR default distinct from llc backend O0/O2. Portable LLVM execution does not execute original AVX512.", count) +} + +func TestCrossLinuxRuntimeMatrixX86DwordProductOriginalGoSSE(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\nfunc ReadCSR() uint32\nfunc SetCSR(value uint32)\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\nTEXT ·SetCSR(SB),4,$0-4\nLDMXCSR value+0(FP)\nRET\n") + tests.WriteString(`package oracle +import("testing";"unsafe";"runtime") +func TestOriginal(t *testing.T) { + runtime.LockOSThread();defer runtime.UnlockOSThread() + // Exercise the runtime's floating-point heap-sampling path between kernels. + // LockOSThread alone does not preserve MXCSR across ordinary Go allocations. + oldRate:=runtime.MemProfileRate;runtime.MemProfileRate=2 + defer func(){runtime.MemProfileRate=oldRate}() + saved:=ReadCSR();defer SetCSR(saved) + edges:=[...]uint32{0,1,2,0x7fffffff,0x80000000,0x80000001,0xfffffffe,0xffffffff} + for sample:=0;sample<97;sample++ {for _,seed:=range []uint64{0,0x8d5} { + var a,b,out [160]byte + align:=func(value *[160]byte)(*[64]byte,int){at:=16+int((-uintptr(unsafe.Pointer(&value[16])))&15);return (*[64]byte)(unsafe.Pointer(&value[at])),at} + ap,_:=align(&a);bp,_:=align(&b);op,offset:=align(&out) + for i:=range a {a[i]=0xa7;b[i]=0xb9;out[i]=0xc3} + for i:=0;i<64;i+=4 {x,y:=edges[(sample+i/4)%8],edges[(sample/8+i*3)%8];for j:=0;j<4;j++ {ap[i+j]=byte(x>>uint(j*8));bp[i+j]=byte(y>>uint(j*8))}} + originalA,originalB:=a,b + profile:=uint32(0x1f80|((sample%4)<<13)|(sample%64)) +`) + count := 0 + for _, name := range x86DwordProductOracleNames[:2] { + for _, source := range []string{"X0", "X1", "(DI)"} { + fn := fmt.Sprintf("S%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed uint64,profile uint32)\n", fn) + // Own the complete state window in assembly. A Go allocation (even + // on a locked thread) can set MXCSR.PE in runtime heap sampling. + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-52\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\nMOVOU (DI),X0\nMOVOU (SI),X1\nLDMXCSR profile+48(FP)\nPUSHQ seed+40(FP)\nPOPFQ\n%s %s,X1\nMOVOU X1,(DX)\nPUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n", fn, name, source) + alias, signed := 0, 0 + if source == "X1" { + alias = 1 + } + if name == "PMULDQ" { + signed = 1 + } + fmt.Fprintf(&tests, `{ + flags:=struct{before,value,after uint64}{0x11223344,0,0x44332211};csr:=struct{before,value,after uint32}{0x55667788,0,0x88776655} + // This deliberate prior-state difference makes loss of per-kernel setup + // deterministic, without relying on when the runtime happens to sample. + SetCSR(profile^0x20) + %s(op,ap,bp,&flags.value,&csr.value,seed,profile) + after:=ReadCSR() + if flags.value&0x8d5!=seed||csr.value!=profile||after!=profile||flags.before!=0x11223344||flags.after!=0x44332211||csr.before!=0x55667788||csr.after!=0x88776655||a!=originalA||b!=originalB { + t.Fatalf("%s sample%%d flags=%%x want=%%x csr=%%x after=%%x want=%%x flagCanaries=%%x/%%x csrCanaries=%%x/%%x inputChanges=%%t/%%t", + sample,flags.value,seed,csr.value,after,profile,flags.before,flags.after,csr.before,csr.after,a!=originalA,b!=originalB) + } + for i,value:=range out {want:=byte(0xc3);if i>=offset&&i>uint((index%%8)*8)) + };if value!=want {t.Fatalf("%s sample%%d byte%%d got%%x want%%x",sample,i,value,want)}} +} +`, fn, fn, alias, signed, fn) + } + } + tests.WriteString("}}\n}\n") + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), tests.String()) + t.Logf("%d actual original Go SSE kernels ×97 samples ×2 seeds, complete bytes, signed/unsigned extreme products, ignored odd dwords, flags/MXCSR/canaries. Rosetta is diagnostic; native Linux axis remains required.", count) +} diff --git a/amd64_ecosystem_test.go b/amd64_ecosystem_test.go new file mode 100644 index 000000000..db6b9390b --- /dev/null +++ b/amd64_ecosystem_test.go @@ -0,0 +1,7804 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strconv" + "strings" + "testing" +) + +func TestTranslateAMD64LegacyDoubleShiftRegisterPair(t *testing.T) { + src := ` +TEXT shiftpair(SB),NOSPLIT,$0-0 + MOVQ $1, SI + MOVQ $2, CX + SHLQ $13, CX:SI + RET +` + translateAMD64EcosystemCase(t, src, "shiftpair") +} + +func TestTranslateX86UD2CompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns UD2 to ynone on both 386 and amd64: its complete + // accepted surface is exactly the no-operand form. + file, err := Parse(ArchAMD64, "TEXT ud2forms(SB),NOSPLIT,$0-0\n\tUD2\n") + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"ud2forms": {Name: "ud2forms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "unreachable") { + t.Fatalf("%s UD2 translation is not terminating:\n%s", target.goarch, ll) + } + compileLLVMToObject(t, llc, target.triple, "ud2-"+target.goarch+".ll", "ud2-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86UD2RejectsFormsOutsideGoAssemblerTables(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\tUD2 AX\n") + if err != nil { + return + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("%s UD2 with an operand unexpectedly translated", target.goarch) + } + } +} + +func TestTranslateAMD64ScalarShiftRotateCompleteGoAssemblerForms(t *testing.T) { + // ROL/ROR/SAR/SAL/SHL/SHR share yshb/yshl. Each width accepts $1, + // unsigned imm8, CL, and CX counts with a same-width GP-or-memory target. + var src strings.Builder + src.WriteString("TEXT scalarshiftrotateforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"ROL", "ROR", "SAR", "SAL", "SHL", "SHR"} { + for _, width := range []string{"B", "W", "L", "Q"} { + op := stem + width + if width == "B" { + fmt.Fprintf(&src, "\t%s $1, AH\n", op) + fmt.Fprintf(&src, "\t%s $255, R11\n", op) + fmt.Fprintf(&src, "\t%s CL, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, AL\n", op) + } else { + fmt.Fprintf(&src, "\t%s $1, AX\n", op) + fmt.Fprintf(&src, "\t%s $255, R11\n", op) + fmt.Fprintf(&src, "\t%s CL, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, AX\n", op) + } + } + } + // Go's special double-shift table adds signed imm8/CL/CX, GP source, + // GP-or-memory destination rows to SHL/SHR W/L/Q only. + for _, op := range []string{"SHLW", "SHLL", "SHLQ", "SHRW", "SHRL", "SHRQ"} { + fmt.Fprintf(&src, "\t%s $-128, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s $127, R11, (DI)\n", op) + fmt.Fprintf(&src, "\t%s CL, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s CX, R11, (DI)\n", op) + } + src.WriteString("\tRET\n") + // The complete original encoder inventory has no input declaration and + // includes undefined word double-shift counts. It is a format object, + // not an initialized numerical program; keep its public Context result. + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "scalarshiftrotateforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "scalar-shift-rotate-forms.ll", "scalar-shift-rotate-forms.o", ll) +} + +func TestTranslate386ScalarShiftRotateCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT scalarshiftrotateforms386(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"ROL", "ROR", "SAR", "SAL", "SHL", "SHR"} { + for _, width := range []string{"B", "W", "L"} { + op := stem + width + if width == "B" { + fmt.Fprintf(&src, "\t%s $1, AH\n", op) + fmt.Fprintf(&src, "\t%s $255, DX\n", op) + fmt.Fprintf(&src, "\t%s CL, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, AL\n", op) + fmt.Fprintf(&src, "\t%s $1, BP\n", op) + fmt.Fprintf(&src, "\t%s $255, SI\n", op) + fmt.Fprintf(&src, "\t%s CL, DI\n", op) + } else { + fmt.Fprintf(&src, "\t%s $1, AX\n", op) + fmt.Fprintf(&src, "\t%s $255, SI\n", op) + fmt.Fprintf(&src, "\t%s CL, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, DI\n", op) + } + } + } + for _, op := range []string{"SHLW", "SHLL", "SHRW", "SHRL"} { + fmt.Fprintf(&src, "\t%s $-128, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s $127, SI, (DI)\n", op) + fmt.Fprintf(&src, "\t%s CL, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s CX, SI, (DI)\n", op) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "scalarshiftrotateforms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "scalar-shift-rotate-386-forms.ll", "scalar-shift-rotate-386-forms.o", ll) +} + +func TestTranslateX86PortIOCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT portioforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"INB", "INW", "INL", "OUTB", "OUTW", "OUTL"} { + fmt.Fprintf(&src, "\t%s\n", op) + fmt.Fprintf(&src, "\t%s $7\n", op) + } + for _, op := range []string{"INSB", "INSW", "INSL", "OUTSB", "OUTSW", "OUTSL"} { + fmt.Fprintf(&src, "\t%s\n", op) + fmt.Fprintf(&src, "\tCLD; REP; %s\n", op) + fmt.Fprintf(&src, "\tSTD; REPN; %s\n", op) + fmt.Fprintln(&src, "\tCLD") + } + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "portioforms": {Name: "portioforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `asm sideeffect "inb`, `asm sideeffect "inw`, `asm sideeffect "inl`, + `asm sideeffect "outb`, `asm sideeffect "outw`, `asm sideeffect "outl`, + `asm sideeffect "insb`, `asm sideeffect "insw`, `asm sideeffect "insl`, + `asm sideeffect "outsb`, `asm sideeffect "outsw`, `asm sideeffect "outsl`, + `asm sideeffect "rep; insb`, `asm sideeffect "rep; outsb`, + `asm sideeffect "repne; insb`, `asm sideeffect "repne; outsb`, + } { + if !strings.Contains(ll, want) { + t.Fatalf("%s port-I/O translation missing %q:\n%s", target.goarch, want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "port-io-"+target.goarch+".ll", "port-io-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86PortIORejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + for _, instruction := range []string{ + "INB AX", + "INW $1, AX", + "OUTL AX", + "OUTB $1, AX", + "INSB AX", + "OUTSL (SI)", + "REP; INL", + } { + name := target.goarch + "/" + strings.NewReplacer(" ", "_", ";", "_").Replace(instruction) + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("%s %q unexpectedly translated", target.goarch, instruction) + } + }) + } + } +} + +func TestTranslateX86StringInstructionsCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + widths []string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", widths: []string{"B", "W", "L"}}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", widths: []string{"B", "W", "L", "Q"}}, + } { + t.Run(target.goarch, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT stringforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"MOVS", "STOS", "SCAS"} { + for _, width := range target.widths { + op := stem + width + fmt.Fprintf(&src, "\t%s\n", op) + fmt.Fprintf(&src, "\tCLD; REP; %s\n", op) + fmt.Fprintf(&src, "\tSTD; REPN; %s\n", op) + fmt.Fprintln(&src, "\tCLD") + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "stringforms", target.goarch, target.triple) + for _, stem := range []string{"movs", "rep_stos", "rep_scas"} { + for _, width := range target.widths { + want := "@__plan9asm_" + stem + strings.ToLower(width) + if !strings.Contains(ll, want) { + t.Fatalf("%s string translation missing %q:\n%s", target.goarch, want, ll) + } + } + } + compileLLVMToObject(t, llc, target.triple, "string-forms-"+target.goarch+".ll", "string-forms-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslate386RejectsQwordStringInstructions(t *testing.T) { + for _, op := range []string{"MOVSQ", "STOSQ", "SCASQ"} { + t.Run(op, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+op+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("386 %s unexpectedly translated", op) + } + }) + } +} + +func TestTranslate386RejectsStaleNamedFPResultOffsets(t *testing.T) { + src := `TEXT legacyresults(SB),NOSPLIT,$0-56 + MOVL AX, retax+28(FP) + MOVL BX, retbx+32(FP) + MOVL CX, retcx+40(FP) + MOVL DX, retdx+44(FP) + MOVL SI, retsi+48(FP) + MOVL DI, retdi+52(FP) + MOVL BP, retbp+56(FP) + RET +` + requireStaleNamedFPGoDiagnostic(t, src) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + results := make([]FrameSlot, 0, 7) + names := []string{"retax", "retbx", "retcx", "retdx", "retsi", "retdi", "retbp"} + for i, name := range names { + results = append(results, FrameSlot{Offset: int64(28 + 4*i), Type: I32, Index: i, Field: -1, Name: name}) + } + _, err = Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "legacyresults": { + Name: "legacyresults", + Args: []LLVMType{I32, I32, I32, I32, I32, I32, I32}, + Ret: LLVMType("{ i32, i32, i32, i32, i32, i32, i32 }"), + Frame: FrameLayout{Results: results}, + }, + }, + }) + if err == nil { + t.Fatal("stale explicit result offsets were silently rewritten by name") + } +} + +func TestTranslateX86MOVLScalarResultMemoryForms(t *testing.T) { + const source = `TEXT movlresults(SB),NOSPLIT,$0-8 + MOVL $-1, first+0(FP) + MOVL AX, second+4(FP) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + options := Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"movlresults": { + Name: "movlresults", + Ret: LLVMType("{ i32, i32 }"), + Frame: FrameLayout{Results: []FrameSlot{ + {Offset: 0, Type: I32, Index: 0, Field: -1, Name: "first"}, + {Offset: 4, Type: I32, Index: 1, Field: -1, Name: "second"}, + }}, + }}, + } + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], options.Sigs["movlresults"], target.goarch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("the original undeclared AX input became a complete source proof", err) + } + if ir, err := Translate(file, options); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("the original fragment acquired an incoming AX argument", err) + } + // Preserve both original MOVL result forms. A separate complete + // producer initializes the GP input instead of borrowing entry zeroes. + initialized := strings.Replace(source, "\tMOVL AX, second+4(FP)", "\tMOVL $7,AX\n\tMOVL AX, second+4(FP)", 1) + x86LEAActualTargetGoObject(t, target.goarch, target.triple, "#include \"textflag.h\"\n"+source+strings.Replace(initialized, "movlresults", "initialized", 1)) + for _, api := range []string{"text", "module", "Go"} { + t.Run("initialized-"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, initialized, "func movlresults() (first,second uint32)", target.goarch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("initialized MOVL result forms rejected", err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + }) + } + }) + } +} + +func TestTranslateX86MOVLRejectsMemoryToResultMemory(t *testing.T) { + for _, source := range []string{"arg+0(FP)", "(AX)", "global(SB)"} { + t.Run(strings.NewReplacer("(", "_", ")", "_", "+", "_").Replace(source), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-4\n\tMOVL "+source+", result+0(FP)\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": { + Name: "bad", + Ret: I32, + Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1, Name: "result"}}}, + }}, + }); err == nil { + t.Fatalf("MOVL %s, result(FP) unexpectedly translated", source) + } + }) + } +} + +func TestTranslateX86ImplicitMultiplyDivideCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's ydivb/ydivl tables give MUL, IMUL, DIV, and IDIV one + // GP-or-memory operand. IMUL W/L/Q additionally uses yimul/yimul3 for + // the two- and three-operand signed multiply forms. + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + widths []string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", widths: []string{"B", "W", "L"}}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", widths: []string{"B", "W", "L", "Q"}}, + } { + t.Run(target.goarch, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT muldivforms(SB),NOSPLIT,$0-0\n") + for _, width := range target.widths { + reg := "BX" + for _, stem := range []string{"MUL", "IMUL", "DIV", "IDIV"} { + fmt.Fprintf(&src, "\t%s%s %s\n", stem, width, reg) + fmt.Fprintf(&src, "\t%s%s (SI)\n", stem, width) + } + if width != "B" { + fmt.Fprintf(&src, "\tIMUL%s $-128, CX\n", width) + fmt.Fprintf(&src, "\tIMUL%s $4294967295, DI\n", width) + fmt.Fprintf(&src, "\tIMUL%s BX, CX\n", width) + fmt.Fprintf(&src, "\tIMUL%s (SI), DI\n", width) + fmt.Fprintf(&src, "\tIMUL3%s $-128, BX, CX\n", width) + fmt.Fprintf(&src, "\tIMUL3%s $4294967295, (SI), DI\n", width) + } + } + src.WriteString("\tRET\n") + // Retain every original Go row and its missing-input Context. + // Emitter-only objects are not a Void-entry exemption; initialized + // product objects have their independent five-target owner. + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "muldivforms", target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "mul-div-forms-"+target.goarch+".ll", "mul-div-forms-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86ImplicitMultiplyDivideRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + common := []string{ + "MULB $1", + "DIVW AX, BX", + "IDIVL AX, BX", + "IMULB AX, BX", + "IMUL3B $1, AX, BX", + "IMULW (AX), (BX)", + "IMUL3W AX, BX, CX", + "IMUL3L $1, AX, (BX)", + "IMULL X0", + } + for _, target := range []struct { + goarch string + triple string + instructions []string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", instructions: append(append([]string{}, common...), "MULQ AX", "IMULQ AX", "DIVQ AX", "IDIVQ AX", "IMUL3Q $1, AX, BX")}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instructions: append(append([]string{}, common...), "IMULQ $4294967296, AX", "IMUL3Q $-2147483649, AX, BX")}, + } { + for _, instruction := range target.instructions { + name := target.goarch + "/" + strings.NewReplacer(" ", "_", ",", "_", "(", "_", ")", "_").Replace(instruction) + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("%s accepted %q outside Go 1.27's multiply/divide tables", target.goarch, instruction) + } + }) + } + } +} + +func TestTranslateAMD64ScalarShiftRotateRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "SHRB $-1, AL", + "SHLB $256, AL", + "SARW AX, BX", + "SALL (AX), BX", + "ROLL $1, X1", + "RORB $1, X1", + "RORQ.Z $1, AX, AX", + "SHLB $1, AL, BL", + "SARL $1, AX, BX", + "SALW $1, AX, BX", + "ROLQ $1, AX, BX", + "SHLL $128, AX, BX", + "SHRL DX, AX, BX", + "SHLW $1, (AX), BX", + "SHRQ $1, AX, $2", + "SHLB $1, AX, BX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidscalarshiftrotate(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidscalarshiftrotate": {Name: "invalidscalarshiftrotate", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's shift/rotate tables", instruction) + } + }) + } +} + +func TestTranslate386ScalarShiftRotateRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "SHRQ $1, AX", + "ROLQ CL, (BX)", + "SHRB $1, SP", + "SARL $1, R8", + "SHLL $1, R8, AX", + "SHLQ $1, AX, BX", + "SHRQ CL, AX, (BX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidscalarshiftrotate386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidscalarshiftrotate386": {Name: "invalidscalarshiftrotate386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64IncDecCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT incdecforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"INCB", "DECB"} { + fmt.Fprintf(&src, "\t%s AL\n", op) + fmt.Fprintf(&src, "\t%s AH\n", op) + fmt.Fprintf(&src, "\t%s BPB\n", op) + fmt.Fprintf(&src, "\t%s R15B\n", op) + fmt.Fprintf(&src, "\t%s R11\n", op) + fmt.Fprintf(&src, "\t%s (BX)\n", op) + } + for _, op := range []string{"INCW", "DECW", "INCL", "DECL", "INCQ", "DECQ"} { + fmt.Fprintf(&src, "\t%s AX\n", op) + fmt.Fprintf(&src, "\t%s R11\n", op) + fmt.Fprintf(&src, "\t%s (BX)\n", op) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "incdecforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "inc-dec-forms.ll", "inc-dec-forms.o", ll) +} + +func TestTranslate386IncDecCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT incdecforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"INCB", "DECB"} { + fmt.Fprintf(&src, "\t%s AL\n", op) + fmt.Fprintf(&src, "\t%s AH\n", op) + fmt.Fprintf(&src, "\t%s DX\n", op) + fmt.Fprintf(&src, "\t%s BP\n", op) + fmt.Fprintf(&src, "\t%s SI\n", op) + fmt.Fprintf(&src, "\t%s DI\n", op) + fmt.Fprintf(&src, "\t%s (BX)\n", op) + } + for _, op := range []string{"INCW", "DECW", "INCL", "DECL"} { + fmt.Fprintf(&src, "\t%s AX\n", op) + fmt.Fprintf(&src, "\t%s SI\n", op) + fmt.Fprintf(&src, "\t%s (BX)\n", op) + fmt.Fprintf(&src, "\t%s SP\n", op) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "incdecforms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "inc-dec-386-forms.ll", "inc-dec-386-forms.o", ll) +} + +func TestTranslateAMD64IncDecRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "INCB", + "INCW AX, BX", + "DECL $1", + "DECQ X0", + "INCQ.Z AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidincdec(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidincdec": {Name: "invalidincdec", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's INC/DEC tables", instruction) + } + }) + } +} + +func TestTranslate386IncDecRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "INCB SP", + "INCW R8", + "DECL R11", + "INCQ AX", + "DECQ (BX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidincdec386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidincdec386": {Name: "invalidincdec386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedWordMultiplyCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedwordmulforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PMULLW", "PMULHW", "PMULHUW"} { + fmt.Fprintf(&src, "\t%s M0, M1\n", op) + fmt.Fprintf(&src, "\t%s (AX), M1\n", op) + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1\n", op) + } + src.WriteString("\tPMULHRSW X0, X1\n\tPMULHRSW (AX), X1\n") + for _, op := range []string{"VPMULLW", "VPMULHW", "VPMULHUW", "VPMULHRSW"} { + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1, X2\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s X20, X21, X22\n", op) + fmt.Fprintf(&src, "\t%s X0, X1, K1, X2\n", op) + fmt.Fprintf(&src, "\t%s.Z Y0, Y1, K2, Y2\n", op) + fmt.Fprintf(&src, "\t%s.Z Z0, Z1, K3, Z2\n", op) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedwordmulforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-word-multiply-forms.ll", "packed-word-multiply-forms.o", ll) +} + +func TestTranslate386PackedWordMultiplyCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedwordmulforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PMULLW", "PMULHW", "PMULHUW", "PMULHRSW"} { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1\n", op) + } + for _, op := range []string{"VPMULLW", "VPMULHW", "VPMULHUW", "VPMULHRSW"} { + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1, X2\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z1, Z2\n", op) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedwordmulforms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-word-multiply-386-forms.ll", "packed-word-multiply-386-forms.o", ll) +} + +func TestTranslateAMD64PackedWordMultiplyRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMULLW Y0, Y1", + "PMULHRSW M0, M1", + "PMULHW.Z X0, X1", + "VPMULLW.BCST (AX), X1, X2", + "VPMULHW.SAE X0, X1, X2", + "VPMULHUW.Z X0, X1, X2", + "VPMULLW X0, Y1, Y2", + "VPMULHW X0, X1, K0, X2", + "VPMULHUW X0, X1, AX, X2", + "VPMULHRSW X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedwordmul(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedwordmul": {Name: "invalidpackedwordmul", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-word multiply tables", instruction) + } + }) + } +} + +func TestTranslate386PackedWordMultiplyRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PMULLW M0, M1", + "PMULHW (AX), M1", + "PMULHUW X8, X0", + "PMULHRSW X0, X8", + "VPMULLW Z8, Z1, Z2", + "VPMULHW X0, X1, K1, X2", + "VPMULHUW.Z X0, X1, K1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedwordmul386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedwordmul386": {Name: "invalidpackedwordmul386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedWordMultiplyAddCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses ymm for PMADDWL and _yvandnpd for VPMADDWD. + src := `TEXT packedwordmaddforms(SB),NOSPLIT,$0-0 + PMADDWL M0, M1 + PMADDWL (AX), M2 + PMADDWL X0, X1 + PMADDWL (AX), X15 + VPMADDWD X0, X1, X2 + VPMADDWD (AX), X1, X2 + VPMADDWD Y0, Y1, Y2 + VPMADDWD (AX), Y1, Y2 + VPMADDWD Z0, Z1, Z2 + VPMADDWD (AX), Z1, Z2 + VPMADDWD X20, X21, X22 + VPMADDWD Y20, Y21, Y22 + VPMADDWD Z20, Z21, Z22 + VPMADDWD X20, X21, K1, X22 + VPMADDWD.Z (AX), X21, K2, X22 + VPMADDWD Y20, Y21, K3, Y22 + VPMADDWD.Z (AX), Y21, K4, Y22 + VPMADDWD Z20, Z21, K5, Z22 + VPMADDWD.Z (AX), Z21, K6, Z22 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, src, "packedwordmaddforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-word-multiply-add-forms.ll", "packed-word-multiply-add-forms.o", ll) +} + +func TestTranslate386PackedWordMultiplyAddCompleteGoAssemblerForms(t *testing.T) { + src := `TEXT packedwordmaddforms386(SB),NOSPLIT,$0-0 + PMADDWL X0, X1 + PMADDWL (AX), X7 + VPMADDWD X0, X1, X2 + VPMADDWD (AX), X1, X2 + VPMADDWD Y0, Y1, Y2 + VPMADDWD (AX), Y1, Y2 + VPMADDWD Z0, Z1, Z7 + VPMADDWD (AX), Z6, Z7 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, src, "packedwordmaddforms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-word-multiply-add-386-forms.ll", "packed-word-multiply-add-386-forms.o", ll) +} + +func TestTranslateAMD64PackedWordMultiplyAddRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMADDWL.Z X0, X1", + "PMADDWL X0", + "PMADDWL Y0, Y1", + "PMADDWL X0, M1", + "PMADDWL X16, X1", + "PMADDWL X0, (AX)", + "VPMADDWD X0, X1", + "VPMADDWD M0, M1, M2", + "VPMADDWD X0, Y1, Y2", + "VPMADDWD X0, X1, (AX)", + "VPMADDWD.BCST (AX), X1, X2", + "VPMADDWD.Z X0, X1, X2", + "VPMADDWD X0, X1, K0, X2", + "VPMADDWD X0, X1, AX, X2", + "VPMADDWD X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidpackedwordmadd(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedwordmadd": {Name: "invalidpackedwordmadd", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PMADDWL/VPMADDWD tables", instruction) + } + }) + } +} + +func TestTranslate386PackedWordMultiplyAddRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PMADDWL M0, M1", + "PMADDWL (AX), M1", + "PMADDWL X8, X0", + "PMADDWL X0, X8", + "VPMADDWD Z8, Z1, Z2", + "VPMADDWD Z0, Z1, Z8", + "VPMADDWD X0, X1, K1, X2", + "VPMADDWD.Z X0, X1, K1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidpackedwordmadd386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedwordmadd386": {Name: "invalidpackedwordmadd386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedFloatShuffleCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedfloatshuffleforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"SHUFPS", "SHUFPD"} { + fmt.Fprintf(&src, "\t%s $0, X0, X1\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), X15\n", op) + } + for _, op := range []string{"VSHUFPS", "VSHUFPD"} { + fmt.Fprintf(&src, "\t%s $0, X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), X1, X2\n", op) + fmt.Fprintf(&src, "\t%s $1, Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s $2, (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s $3, Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s $4, (AX), Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s $5, X20, X21, X22\n", op) + fmt.Fprintf(&src, "\t%s $6, X0, X1, K1, X2\n", op) + fmt.Fprintf(&src, "\t%s.Z $7, Y0, Y1, K2, Y2\n", op) + fmt.Fprintf(&src, "\t%s.BCST $8, (AX), Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s.BCST.Z $9, (AX), Z1, K3, Z2\n", op) + } + src.WriteString("\tRET\n") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir := x86PackedFloatShuffleUnboundFormatObject(t, src.String(), "amd64", triple, 26) + for _, pipeline := range []struct{ text, level string }{ + {ir, "-O0"}, {ir, "-O2"}, {x86DwordProductIROptimize(t, ir), "-O2"}, + } { + x86PackedStringCompileObject(t, llc, triple, pipeline.text, pipeline.level) + } + }) + } +} + +func TestTranslate386PackedFloatShuffleCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT packedfloatshuffleforms386(SB),NOSPLIT,$0-0 + SHUFPS $0, X0, X1 + SHUFPS $255, (AX), X7 + SHUFPD $0, X0, X1 + SHUFPD $255, (AX), X7 + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir := x86PackedFloatShuffleUnboundFormatObject(t, src, "386", triple, 4) + for _, pipeline := range []struct{ text, level string }{ + {ir, "-O0"}, {ir, "-O2"}, {x86DwordProductIROptimize(t, ir), "-O2"}, + } { + x86PackedStringCompileObject(t, llc, triple, pipeline.text, pipeline.level) + } + }) + } +} + +func TestTranslateAMD64PackedFloatShuffleRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "SHUFPS $-1, X0, X1", + "SHUFPD $256, X0, X1", + "SHUFPS.Z $0, X0, X1", + "VSHUFPS $0, X0, X1", + "VSHUFPD.SAE $0, X0, X1, X2", + "VSHUFPS.Z $0, X0, X1, X2", + "VSHUFPD $0, X0, Y1, Y2", + "VSHUFPS $0, X0, X1, K0, X2", + "VSHUFPD $0, X0, X1, AX, X2", + "VSHUFPS.BCST $0, X0, X1, X2", + "VSHUFPD.BCST $0, (AX), X1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedfloatshuffle(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedfloatshuffle": {Name: "invalidpackedfloatshuffle", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-float shuffle tables", instruction) + } + }) + } +} + +func TestTranslate386PackedFloatShuffleRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "SHUFPS $0, X8, X0", + "SHUFPD $0, X0, X8", + "VSHUFPS $0, X0, X1, X2", + "VSHUFPD $0, Y0, Y1, Y2", + "VSHUFPS.BCST $0, (AX), X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedfloatshuffle386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedfloatshuffle386": {Name: "invalidpackedfloatshuffle386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64ImmediatePackedBlendCompleteGoAssemblerForms(t *testing.T) { + x86ImmediateBlendOriginalBareFormatObjects(t, "amd64", "x86_64-unknown-linux-gnu") +} + +func TestTranslate386ImmediatePackedBlendCompleteGoAssemblerForms(t *testing.T) { + x86ImmediateBlendOriginalBareFormatObjects(t, "386", "i386-unknown-linux-gnu") +} + +func TestTranslateAMD64ImmediatePackedBlendRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PBLENDW $-1, X0, X1", + "BLENDPS $256, X0, X1", + "BLENDPD.Z $0, X0, X1", + "PBLENDW $0, X20, X1", + "BLENDPS $0, X0, X20", + "VPBLENDW $-1, X0, X1, X2", + "VPBLENDD $256, X0, X1, X2", + "VBLENDPS $0, X0, X1", + "VBLENDPD $0, X0, Y1, Y2", + "VPBLENDW $0, X20, X1, X2", + "VPBLENDD $0, Y0, Y20, Y2", + "VBLENDPS $0, Z0, Z1, Z2", + "VBLENDPD.Z $0, X0, X1, X2", + "VPBLENDW.BCST $0, (AX), X1, X2", + "VPBLENDD $0, X0, X1, K1, X2", + "VBLENDPS $0, X0, (AX), X2", + "VBLENDPD $0, X0, X1, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidimmediatepackedblend(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidimmediatepackedblend": {Name: "invalidimmediatepackedblend", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's immediate packed-blend tables", instruction) + } + }) + } +} + +func TestTranslate386ImmediatePackedBlendRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PBLENDW $0, X8, X0", + "BLENDPS $0, X0, X8", + "BLENDPD.Z $0, X0, X1", + "VPBLENDW $0, X0, X1, X2", + "VPBLENDD $0, Y0, Y1, Y2", + "VBLENDPS $0, X0, X1, X2", + "VBLENDPD $0, Y0, Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidimmediatepackedblend386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidimmediatepackedblend386": {Name: "invalidimmediatepackedblend386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedUnpackCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedunpackforms(SB),NOSPLIT,$0-0\n") + legacyMMX := []string{"PUNPCKLBW", "PUNPCKHBW", "PUNPCKLWL", "PUNPCKHWL", "PUNPCKLLQ", "PUNPCKHLQ"} + for _, op := range legacyMMX { + fmt.Fprintf(&src, "\t%s M0, M1\n", op) + fmt.Fprintf(&src, "\t%s (AX), M7\n", op) + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15\n", op) + } + for _, op := range []string{"PUNPCKLQDQ", "PUNPCKHQDQ"} { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15\n", op) + } + vectorOps := []string{ + "VPUNPCKLBW", "VPUNPCKHBW", "VPUNPCKLWD", "VPUNPCKHWD", + "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ", + } + for _, op := range vectorOps { + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1, X15\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y15\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z1, Z31\n", op) + fmt.Fprintf(&src, "\t%s X20, X21, X22\n", op) + fmt.Fprintf(&src, "\t%s Y20, Y21, K1, Y22\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), Z21, K2, Z22\n", op) + if strings.Contains(op, "DQ") { + fmt.Fprintf(&src, "\t%s.BCST (AX), X21, X22\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Y21, Y22\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Z21, Z22\n", op) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), Z21, K3, Z22\n", op) + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedunpackforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-unpack-forms.ll", "packed-unpack-forms.o", ll) +} + +func TestTranslate386PackedUnpackCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedunpackforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{ + "PUNPCKLBW", "PUNPCKHBW", "PUNPCKLWL", "PUNPCKHWL", + "PUNPCKLLQ", "PUNPCKHLQ", "PUNPCKLQDQ", "PUNPCKHQDQ", + } { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + } + for _, op := range []string{ + "VPUNPCKLBW", "VPUNPCKHBW", "VPUNPCKLWD", "VPUNPCKHWD", + "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ", + } { + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + // Named high-register mode32 acceptance is retained by the separate + // actual-Go/physical-Context oracle, not fabricated LLVM registers. + fmt.Fprintf(&src, "\t%s X5, X6, X7\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + if strings.Contains(op, "DQ") { + fmt.Fprintf(&src, "\t%s.BCST (AX), Y1, Y2\n", op) + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedunpackforms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-unpack-386-forms.ll", "packed-unpack-386-forms.o", ll) +} + +func TestTranslateAMD64PackedUnpackRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PUNPCKLBW.Z X0, X1", + "PUNPCKLWL X20, X1", + "PUNPCKHWL X0, X20", + "PUNPCKLLQ M0, X1", + "PUNPCKHLQ X0, M1", + "PUNPCKLQDQ M0, M1", + "PUNPCKHQDQ Y0, X1", + "VPUNPCKLBW X0, X1", + "VPUNPCKHWD X0, Y1, Y2", + "VPUNPCKLDQ X0, (AX), X2", + "VPUNPCKHQDQ X0, X1, (AX)", + "VPUNPCKLBW X32, X1, X2", + "VPUNPCKHWD Z0, Z1, K0, Z2", + "VPUNPCKLWD.Z Z0, Z1, Z2", + "VPUNPCKHBW.BCST (AX), X1, X2", + "VPUNPCKLWD.BCST (AX), Y1, Y2", + "VPUNPCKLDQ.BCST X0, X1, X2", + "VPUNPCKHQDQ.BCST (AX), X1, Y2", + "VPUNPCKHDQ.SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedunpack(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedunpack": {Name: "invalidpackedunpack", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-unpack tables", instruction) + } + }) + } +} + +func TestTranslate386PackedUnpackRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PUNPCKLBW M0, M1", + "PUNPCKHWL (AX), M1", + "PUNPCKLQDQ M0, M1", + "PUNPCKHBW X8, X0", + "PUNPCKHLQ X0, X8", + "VPUNPCKLWD Z8, Z1, Z2", + "VPUNPCKHWD Z0, Z8, Z2", + "VPUNPCKLDQ Z0, Z1, Z8", + "VPUNPCKLBW X0, X1, K1, X2", + "VPUNPCKLDQ.Z Z0, Z1, K1, Z2", + "VPUNPCKHQDQ.BCST.Z (AX), Y1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedunpack386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedunpack386": {Name: "invalidpackedunpack386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64QwordPermuteCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT qwordpermuteforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMQ", "VPERMPD"} { + fmt.Fprintf(&src, "\t%s $-128, Y0, Y1\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), Y15\n", op) + fmt.Fprintf(&src, "\t%s $0, Y20, Y21\n", op) + fmt.Fprintf(&src, "\t%s $1, Z20, Z21\n", op) + fmt.Fprintf(&src, "\t%s $2, Y20, K1, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z $3, Z20, K2, Z21\n", op) + fmt.Fprintf(&src, "\t%s.BCST $4, (AX), Y21\n", op) + fmt.Fprintf(&src, "\t%s.BCST.Z $5, (AX), K3, Z21\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s Y20, Y21, Y22\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y21, K1, Y22\n", op) + fmt.Fprintf(&src, "\t%s.Z Z20, Z21, K2, Z22\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Y21, Y22\n", op) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), Z21, K3, Z22\n", op) + } + src.WriteString("\tRET\n") + // All original 32 rows remain Go/public-Context inputs. These private + // format objects are distinct from the initialized public family matrix. + ll := x86ElementPermuteUnboundFormatObject(t, src.String(), "amd64", "x86_64-unknown-linux-gnu", 32, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + x86ElementPermuteCompileFormatObjects(t, llc, "x86_64-unknown-linux-gnu", ll) +} + +func TestTranslate386QwordPermuteCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT qwordpermuteforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMQ", "VPERMPD"} { + fmt.Fprintf(&src, "\t%s $-128, Y0, Y1\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), Y7\n", op) + fmt.Fprintf(&src, "\t%s $0, Y20, Y21\n", op) + fmt.Fprintf(&src, "\t%s $1, Z0, Z1\n", op) + fmt.Fprintf(&src, "\t%s.BCST $2, (AX), Y21\n", op) + fmt.Fprintf(&src, "\t%s.BCST $3, (AX), Z7\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s Y20, Y21, Y22\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Z1, Z7\n", op) + } + src.WriteString("\tRET\n") + // Go accepts six extended-Y rows, but they do not authorize physical + // mode32 registers. Preserve all 22 original Go/Context rows before + // compiling the other 16 physical rows through the private emitter. + ll := x86ElementPermuteUnboundFormatObject(t, src.String(), "386", "i386-unknown-linux-gnu", 16, 6) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + x86ElementPermuteCompileFormatObjects(t, llc, "i386-unknown-linux-gnu", ll) +} + +func TestTranslateAMD64QwordPermuteRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VPERMQ $-129, Y0, Y1", + "VPERMPD $256, Y0, Y1", + "VPERMQ $-1, Y20, Y21", + "VPERMPD $-1, Z0, Z1", + "VPERMQ.BCST $-1, (AX), Y1", + "VPERMQ $0, X0, X1", + "VPERMPD X0, X1, X2", + "VPERMQ $0, Y0, Z1", + "VPERMPD Y0, Z1, Z2", + "VPERMQ Y0, (AX), Y2", + "VPERMPD Y0, Y1, (AX)", + "VPERMQ $0, Y0, K0, Y1", + "VPERMPD Z0, Z1, K0, Z2", + "VPERMQ.Z $0, Y0, Y1", + "VPERMPD.Z Y0, Y1, Y2", + "VPERMQ.BCST $0, Y0, Y1", + "VPERMPD.BCST Y0, Y1, Y2", + "VPERMQ.SAE $0, Y0, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidqwordpermute(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidqwordpermute": {Name: "invalidqwordpermute", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvpermq table", instruction) + } + }) + } +} + +func TestTranslate386QwordPermuteRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "VPERMQ $-1, Y20, Y21", + "VPERMPD $0, Z8, Z1", + "VPERMQ $0, Z0, Z8", + "VPERMPD Z8, Z1, Z2", + "VPERMQ Z0, Z8, Z2", + "VPERMPD Z0, Z1, Z8", + "VPERMQ $0, Y0, K1, Y1", + "VPERMPD Z0, Z1, K1, Z2", + "VPERMQ.BCST.Z (AX), Z1, K1, Z2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidqwordpermute386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidqwordpermute386": {Name: "invalidqwordpermute386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedFloatMoveCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns VMOVAPD/APS/UPD/UPS the same 16-row _yvmovapd + // table: bidirectional X/Y/Z register-or-memory moves, EVEX high + // registers, and K1-K7 merge/zero masking for every width and direction. + var src strings.Builder + src.WriteString("TEXT packedfloatmoveforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VMOVAPD", "VMOVAPS", "VMOVUPD", "VMOVUPS"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\t%s %s0, %s1\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s1, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), %s2\n", op, width) + fmt.Fprintf(&src, "\t%s %s20, %s21\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s20, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), %s21\n", op, width) + } + fmt.Fprintf(&src, "\t%s Z0, Z1\n", op) + fmt.Fprintf(&src, "\t%s Z1, (AX)\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z2\n", op) + fmt.Fprintf(&src, "\t%s Z20, Z21\n", op) + fmt.Fprintf(&src, "\t%s Z20, (AX)\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z21\n", op) + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s %s1, K1, %s2\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s1, K2, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), K3, %s2\n", op, width) + fmt.Fprintf(&src, "\t%s.Z %s1, K4, %s2\n", op, width, width) + // Go accepts .Z memory stores, but their native EVEX.z encoding + // raises #UD. Independent-entry and fault oracles cover that effect. + fmt.Fprintf(&src, "\t%s.Z %s1, K5, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s.Z (AX), K6, %s2\n", op, width) + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidVectorMoveFormatObject(t, src.String(), "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-float-move-forms.ll", "packed-float-move-forms.o", ll) +} + +func TestTranslate386PackedFloatMoveCompleteGoAssemblerForms(t *testing.T) { + // Go accepts high X/Y spellings and all three-operand mask rows. Observe + // their Context separately; actual mode32 objects use the physical low bank. + source := x86EVEXPackedFloat386OriginalSource() + physical := x86PhysicalMode32ObjectSource(t, source, "386") + ll := x86UnboundVoidVectorMoveFormatObject(t, physical, "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-float-move-386-forms.ll", "packed-float-move-386-forms.o", ll) +} + +func TestTranslateAMD64PackedFloatMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VMOVAPD X1, Y2", + "VMOVAPS Y1, Z2", + "VMOVUPD Z1, X2", + "VMOVUPS (AX), (BX)", + "VMOVAPD K1, X2", + "VMOVAPS X1, K0, X2", + "VMOVUPD (AX), K0, X2", + "VMOVUPS.Z X1, X2", + "VMOVAPD.BCST (AX), X2", + "VMOVAPS.SAE X1, X2", + "VMOVUPD.RN_SAE X1, X2", + "VMOVUPS X1, K1, Y2", + "VMOVAPD X1, X2, K1", + "VMOVAPS X1", + "VMOVUPD X1, K1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedfloatmove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedfloatmove": {Name: "invalidpackedfloatmove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvmovapd table", instruction) + } + }) + } +} + +func TestTranslate386PackedFloatMoveRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "VMOVAPD Z8, Z1", + "VMOVAPS Z0, Z8", + "VMOVUPD Z8, (AX)", + "VMOVUPS (AX), Z8", + "VMOVAPD Z0, K1, Z8", + "VMOVAPS.Z Z8, K1, (AX)", + "VMOVUPD.Z (AX), K1, Z8", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedfloatmove386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedfloatmove386": {Name: "invalidpackedfloatmove386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateX86EVEXPackedIntegerMoveCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns this complete family the same 12-row _yvmovdqa32 + // table. Each opcode accepts bidirectional X/Y/Z register-or-memory + // moves plus K1-K7 merge/zero masking for every width and direction. + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + source := x86EVEXPackedIntegerMoveOriginalSource(target.goarch) + physical := x86PhysicalMode32ObjectSource(t, source, target.goarch) + ll := x86UnboundVoidVectorMoveFormatObject(t, physical, target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "evex-packed-integer-move-"+target.goarch+".ll", "evex-packed-integer-move-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86EVEXPackedIntegerMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VMOVDQA32 X1, Y2", + "VMOVDQA64 Y1, Z2", + "VMOVDQU8 (AX), (BX)", + "VMOVDQU16 X1, K0, X2", + "VMOVDQU32.Z X1, X2", + "VMOVDQU64.BCST (AX), Z2", + "VMOVDQU8.SAE X1, X2", + "VMOVDQU16 X1, K1, Y2", + "VMOVDQU32 X1", + "VMOVDQA64 X1, K1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvmovdqa32 table", instruction) + } + }) + } + for _, instruction := range []string{ + "VMOVDQA32 Z8, Z0", + "VMOVDQA64 Z0, Z8", + "VMOVDQU8 Z8, (AX)", + "VMOVDQU16 (AX), Z8", + "VMOVDQU32 Z0, K1, Z8", + "VMOVDQU64.Z (AX), K1, Z8", + } { + file, err := Parse(ArchAMD64, "TEXT bad386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad386": {Name: "bad386", Ret: Void}}, + }); err == nil { + t.Fatalf("386 Translate accepted architecture-restricted %q", instruction) + } + } +} + +func TestTranslateAMD64PackedSignExtendMoveCompleteGoAssemblerForms(t *testing.T) { + // The complete signed widening family consists of six legacy yxm_q4 + // instructions plus six V instructions. BW/WD/DQ use _yvcvtdq2pd + // (X source for X/Y destinations and Y source for Z); BD/BQ/WQ use + // _yvbroadcastss (X source for every destination width). + var src strings.Builder + src.WriteString("TEXT packedsignextendmoveforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PMOVSXBW", "PMOVSXBD", "PMOVSXBQ", "PMOVSXWD", "PMOVSXWQ", "PMOVSXDQ"} { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X2\n", op) + fmt.Fprintf(&src, "\t%s X15, X14\n", op) + } + for _, op := range []string{"VPMOVSXBW", "VPMOVSXWD", "VPMOVSXDQ"} { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X2\n", op) + fmt.Fprintf(&src, "\t%s X0, Y1\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y2\n", op) + fmt.Fprintf(&src, "\t%s Y0, Z1\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z2\n", op) + fmt.Fprintf(&src, "\t%s X20, X21\n", op) + fmt.Fprintf(&src, "\t%s X20, Y21\n", op) + fmt.Fprintf(&src, "\t%s Y20, Z21\n", op) + fmt.Fprintf(&src, "\t%s X20, K1, X21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s X20, K3, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K4, Y21\n", op) + fmt.Fprintf(&src, "\t%s Y20, K5, Z21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K6, Z21\n", op) + } + for _, op := range []string{"VPMOVSXBD", "VPMOVSXBQ", "VPMOVSXWQ"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s X0, %s1\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), %s2\n", op, width) + fmt.Fprintf(&src, "\t%s X20, %s21\n", op, width) + fmt.Fprintf(&src, "\t%s X20, K1, %s21\n", op, width) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, %s21\n", op, width) + } + } + src.WriteString("\tRET\n") + ll := x86SignExtendOriginalFormatObject(t, src.String(), "amd64", "x86_64-unknown-linux-gnu", 108, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-sign-extend-move-forms.ll", "packed-sign-extend-move-forms.o", ll) +} + +func TestTranslate386PackedSignExtendMoveCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedsignextendmoveforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PMOVSXBW", "PMOVSXBD", "PMOVSXBQ", "PMOVSXWD", "PMOVSXWQ", "PMOVSXDQ"} { + fmt.Fprintf(&src, "\t%s X0, X7\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + } + for _, op := range []string{"VPMOVSXBW", "VPMOVSXWD", "VPMOVSXDQ"} { + fmt.Fprintf(&src, "\t%s X20, X21\n", op) + fmt.Fprintf(&src, "\t%s X20, Y21\n", op) + fmt.Fprintf(&src, "\t%s Y20, Z7\n", op) + fmt.Fprintf(&src, "\t%s X20, K1, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, Z7\n", op) + } + for _, op := range []string{"VPMOVSXBD", "VPMOVSXBQ", "VPMOVSXWQ"} { + fmt.Fprintf(&src, "\t%s X20, X21\n", op) + fmt.Fprintf(&src, "\t%s X20, Y21\n", op) + fmt.Fprintf(&src, "\t%s X20, Z7\n", op) + fmt.Fprintf(&src, "\t%s X20, K1, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, Z7\n", op) + } + src.WriteString("\tRET\n") + ll := x86SignExtendOriginalFormatObject(t, src.String(), "386", "i386-unknown-linux-gnu", 18, 24) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-sign-extend-move-386-forms.ll", "packed-sign-extend-move-386-forms.o", ll) +} + +func TestTranslateAMD64PackedSignExtendMoveRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMOVSXBW X16, X1", + "PMOVSXBD X1, X16", + "PMOVSXBQ Y1, X2", + "PMOVSXWD X1, Y2", + "PMOVSXWQ X1, (AX)", + "PMOVSXDQ.Z X1, X2", + "PMOVSXBW X1, K1, X2", + "VPMOVSXBW Y1, Y2", + "VPMOVSXWD X1, Z2", + "VPMOVSXDQ Z1, Z2", + "VPMOVSXBD Y1, Z2", + "VPMOVSXBQ Y1, Y2", + "VPMOVSXWQ Z1, X2", + "VPMOVSXBW X1, (AX)", + "VPMOVSXBD (AX), (BX)", + "VPMOVSXBQ X1, K0, Y2", + "VPMOVSXWD.Z X1, Y2", + "VPMOVSXDQ.BCST (AX), Y2", + "VPMOVSXWQ.SAE X1, Z2", + "VPMOVSXBW.RN_SAE X1, Y2", + "VPMOVSXBD X1", + "VPMOVSXBQ X1, K1, Y2, Y3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedsignextendmove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedsignextendmove": {Name: "invalidpackedsignextendmove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's signed-extension move tables", instruction) + } + }) + } +} + +func TestTranslate386PackedSignExtendMoveRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PMOVSXBW X8, X1", + "PMOVSXBD X1, X8", + "PMOVSXBQ X15, X7", + "VPMOVSXBW Y1, Z8", + "VPMOVSXWD (AX), Z8", + "VPMOVSXDQ Y20, K1, Z8", + "VPMOVSXBD X20, Z8", + "VPMOVSXBQ.Z (AX), K1, Z8", + "VPMOVSXWQ X20, K1, Z8", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedsignextendmove386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedsignextendmove386": {Name: "invalidpackedsignextendmove386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64MOVDFromXRegister(t *testing.T) { + src := ` +TEXT movdfromx(SB),NOSPLIT,$0-0 + PXOR X12, X12 + MOVD X12, DX + RET +` + translateAMD64EcosystemCase(t, src, "movdfromx") +} + +func TestTranslateAMD64MOVDAliasCompleteGo127Forms(t *testing.T) { + // cmd/asm aliases MOVD to AMOVQ on x86. These cover every ymovq operand + // class that the alias can select in 64-bit mode: GP/immediate/memory, + // MMX/memory/GP/XMM, and XMM/memory/GP/XMM. + src := ` +TEXT movdaliasforms(SB),NOSPLIT,$0-0 + MOVD AX, BX + MOVD $1, AX + MOVD AX, 8(BX) + MOVD 8(BX), AX + MOVD (BX), M0 + MOVD M0, 8(BX) + MOVD AX, M1 + MOVD M1, AX + MOVD X0, M2 + MOVD M2, M3 + MOVD AX, X1 + MOVD 24(SP), X9 + MOVD X1, AX + MOVD X9, 32(SP) + MOVD X1, X2 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, src, "movdaliasforms", "amd64", "x86_64-unknown-linux-gnu") + if !strings.Contains(ll, "load i64") || !strings.Contains(ll, "store i64") { + t.Fatalf("MOVD alias did not preserve MOVQ-width memory semantics:\n%s", ll) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "movd-alias-forms.ll", "movd-alias-forms.o", ll) +} + +func TestTranslateAMD64MOVDAliasRejectsFormsOutsideGo127Ymovq(t *testing.T) { + for _, instruction := range []string{ + "MOVD $1, X0", + "MOVD M0, X0", + "MOVD X0, X1, X2", + "MOVD.Z X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badmovd(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badmovd": {Name: "badmovd", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's MOVD/ymovq forms", instruction) + } + }) + } +} + +func TestTranslateAMD64LegacyPrefetchFamilyCompleteGo127Forms(t *testing.T) { + src := ` +TEXT prefetchforms(SB),NOSPLIT,$0-0 + PREFETCHNTA (AX) + PREFETCHT0 8(BX) + PREFETCHT1 source(SB) + PREFETCHT2 16(SP) + RET +` + ll := translateAMD64EcosystemCase(t, src, "prefetchforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "prefetch-forms.ll", "prefetch-forms.o", ll) +} + +func TestTranslateAMD64LegacyPrefetchFamilyRejectsNonMemoryForms(t *testing.T) { + for _, instruction := range []string{ + "PREFETCHNTA", + "PREFETCHT0 AX", + "PREFETCHT1 $1", + "PREFETCHT2 (AX), (BX)", + "PREFETCHT0.Z (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badprefetch(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badprefetch": {Name: "badprefetch", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yprefetch table", instruction) + } + }) + } +} + +func TestTranslateAMD64ArithmeticDisplacementExpression(t *testing.T) { + const source = ` +TEXT arithmeticdisplacement(SB),NOSPLIT,$32-0 + ADDQ 0+(1*16)(BP), R10 + MOVQ (-16+8)(SP), AX + RET +` + goAssemblerOracleOutput(t, "amd64", "#define NOSPLIT 4\n"+source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if got := file.Funcs[0].Instrs[1].Args[0]; got.Kind != OpMem || got.Mem.Base != BP || got.Mem.Off != 16 { + t.Fatalf("BP arithmetic displacement parsed as %+v, want offset 16", got) + } + if got := file.Funcs[0].Instrs[2].Args[0]; got.Kind != OpMem || got.Mem.Base != SP || got.Mem.Off != -8 { + t.Fatalf("SP arithmetic displacement parsed as %+v, want offset -8", got) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run("unbound/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func arithmeticdisplacement()", + "amd64", "x86_64-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original displacement grammar supplied BP/R10 or physical stack input", err) + } + }) + } + + // The exact original memory expressions now address initialized local + // bytes: BP is an explicit private-frame pointer and SP is restored before + // returning. Neither expression grants a physical entry register value. + const initialized = `TEXT ·arithmeticdisplacement(SB),4,$32-24 + MOVQ SP, BP + MOVQ a+0(FP), AX + MOVQ b+8(FP), R10 + MOVQ AX, 16(SP) + MOVQ AX, 8(SP) + MOVQ SP, R11 + ADDQ $16, SP + ADDQ 0+(1*16)(BP), R10 + MOVQ (-16+8)(SP), AX + MOVQ R11, SP + ADDQ R10, AX + MOVQ AX, ret+16(FP) + RET +` + x86StackExpressionBoundObjects(t, initialized, "func arithmeticdisplacement(a,b uint64) uint64") +} + +func TestTranslateAMD64GoTLSPseudoRegisterForms(t *testing.T) { + const source = ` +TEXT ·tlsforms(SB),4,$0-0 + MOVQ TLS, CX + MOVQ 0(CX)(TLS*1), BX + MOVQ 8(TLS), DX + RET + ` + x86ScalarCompareTLSGoObject(t, source, "amd64", "linux", false) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func tlsforms()", "amd64", "x86_64-unknown-linux-gnu", api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("%s: TLS source without native contract returned error=%v, IR=%s", api, err, ir) + } + } +} + +func TestTranslateAMD64LegacyIntegerToFloatConversionFamilyCompleteGo127Forms(t *testing.T) { + ll := x86UnboundVoidScalarFormatObject(t, ` +TEXT integerfloatconversions(SB),NOSPLIT,$0-0 + CVTSL2SS AX, X0 + CVTSL2SS 4(BX), X1 + CVTSL2SD CX, X2 + CVTSL2SD 8(BX), X3 + CVTSQ2SS SI, X4 + CVTSQ2SS 16(BX), X5 + CVTSQ2SD DI, X6 + CVTSQ2SD 24(BX), X7 + CVTPL2PS X0, X1 + CVTPL2PS 32(BX), X2 + CVTPL2PD X4, X5 + CVTPL2PD 48(BX), X6 + RET +`, "integerfloatconversions", "amd64", "x86_64-unknown-linux-gnu") + for _, want := range []string{"@llvm.experimental.constrained.sitofp.f32.i32", "@llvm.experimental.constrained.sitofp.f64.i64", "cvtdq2ps", "cvtdq2pd"} { + if !strings.Contains(ll, want) { + t.Fatalf("missing %q in integer-to-float conversion IR:\n%s", want, ll) + } + } + if strings.Count(ll, "asm sideeffect") != 4 || strings.Contains(ll, " = sitofp <") { + t.Fatal("packed native conversions lost whole-opcode side effects") + } + ll = x86SignedIntegerFloatOptimize(t, ll) + if strings.Count(ll, "asm sideeffect") != 4 { + t.Fatal("O2 dropped a native packed conversion") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "integer-float-conversions.ll", "integer-float-conversions.o", ll) +} + +func TestTranslateAMD64LegacyIntegerToFloatConversionFamilyRejectsNonGoForms(t *testing.T) { + for _, instruction := range []string{ + "CVTSL2SS $1, X0", + "CVTSQ2SS X0, X1", + "CVTSQ2SD AX, Y0", + "CVTPL2PS AX, X0", + "CVTPL2PD X0, Y0", + "CVTPL2PS.Z X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badconversion(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badconversion": {Name: "badconversion", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's legacy integer-to-float tables", instruction) + } + }) + } +} + +func TestTranslateAMD64ScalarFloatBroadcastFamilyCompleteGo127Forms(t *testing.T) { + ll := x86UnboundVoidScalarFormatObject(t, ` +TEXT scalarfloatbroadcasts(SB),NOSPLIT,$0-0 + VBROADCASTSS X0, X1 + VBROADCASTSS 4(AX), Y1 + VBROADCASTSS X16, Z17 + VBROADCASTSS X16, K1, X17 + VBROADCASTSS.Z 8(AX), K2, Y18 + VBROADCASTSS 12(AX), K3, Z19 + VBROADCASTSD X0, Y1 + VBROADCASTSD 8(AX), Z1 + VBROADCASTSD X16, K1, Y17 + VBROADCASTSD.Z 16(AX), K2, Z18 + RET +`, "scalarfloatbroadcasts", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "scalar-float-broadcasts.ll", "scalar-float-broadcasts.o", ll) +} + +func TestTranslateAMD64ScalarFloatBroadcastFamilyRejectsNonGoForms(t *testing.T) { + for _, instruction := range []string{ + "VBROADCASTSS AX, X0", + "VBROADCASTSD X0, X1", + "VBROADCASTSS.Z X0, Y1", + "VBROADCASTSD X0, K0, Y1", + "VBROADCASTSS.BCST 4(AX), Y1", + "VBROADCASTSD X0, K1, K2, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badbroadcast(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badbroadcast": {Name: "badbroadcast", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar broadcast tables", instruction) + } + }) + } +} + +func TestTranslateAMD64DwordToQwordMultiplyFamilyCompleteGo127Forms(t *testing.T) { + ll := x86UnboundVoidScalarFormatObject(t, ` +TEXT dwordqwordmultiply(SB),NOSPLIT,$0-0 + PMULDQ X0, X1 + PMULDQ 16(AX), X2 + PMULULQ M0, M1 + PMULULQ 24(AX), M2 + PMULULQ X0, X1 + PMULULQ 32(AX), X2 + VPMULDQ X0, X1, X2 + VPMULDQ Y0, Y1, Y2 + VPMULDQ Z0, Z1, Z2 + VPMULDQ Z0, Z1, K1, Z2 + VPMULDQ.Z Z0, Z1, K2, Z2 + VPMULDQ.BCST 40(AX), Z1, Z2 + VPMULDQ.BCST.Z 48(AX), Z1, K3, Z2 + VPMULUDQ X0, X1, X2 + VPMULUDQ Y0, Y1, Y2 + VPMULUDQ Z0, Z1, Z2 + VPMULUDQ Z0, Z1, K1, Z2 + VPMULUDQ.Z Z0, Z1, K2, Z2 + VPMULUDQ.BCST 56(AX), Z1, Z2 + VPMULUDQ.BCST.Z 64(AX), Z1, K3, Z2 + RET +`, "dwordqwordmultiply", "amd64", "x86_64-unknown-linux-gnu") + for _, want := range []string{"sext <", "zext <", "mul <2 x i64>", "mul <4 x i64>", "mul <8 x i64>"} { + if !strings.Contains(ll, want) { + t.Fatalf("missing %q in dword-to-qword multiply IR:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "dword-qword-multiply.ll", "dword-qword-multiply.o", ll) +} + +func TestTranslateAMD64DwordToQwordMultiplyFamilyRejectsNonGoForms(t *testing.T) { + for _, instruction := range []string{ + "PMULDQ M0, M1", + "PMULULQ Y0, Y1", + "VPMULUDQ X0, Y1, X2", + "VPMULDQ Z0, Z1, K0, Z2", + "VPMULUDQ.Z Z0, Z1, Z2", + "VPMULDQ.BCST Z0, Z1, Z2", + "VPMULUDQ.Z.BCST 8(AX), Z1, K1, Z2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badmultiply(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badmultiply": {Name: "badmultiply", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's dword-to-qword multiply tables", instruction) + } + }) + } +} + +func TestTranslate386NewlyDiscoveredX86FamiliesCompleteGo127Forms(t *testing.T) { + // Keep the 386 matrix explicit: x86 opcode names are shared with amd64, but + // cmd/asm's operand tables and available registers differ in 32-bit mode. + src := ` +TEXT discoveredx86forms386(SB),NOSPLIT,$0-0 + MOVD (AX), X0 + MOVD X0, 8(AX) + MOVD X0, X1 + MOVD 16(AX), M0 + MOVD M0, 24(AX) + MOVD X0, M1 + CVTSL2SS AX, X0 + CVTSL2SS 4(BX), X1 + CVTSL2SD CX, X2 + CVTSL2SD 8(BX), X3 + CVTPL2PS X0, X1 + CVTPL2PS 16(BX), X2 + CVTPL2PD X4, X5 + CVTPL2PD 24(BX), X6 + VBROADCASTSS X0, X1 + VBROADCASTSS 4(AX), Y1 + VBROADCASTSS 8(AX), Z1 + VBROADCASTSD X0, Y2 + VBROADCASTSD 16(AX), Z2 + PMULDQ X0, X1 + PMULDQ 16(AX), X2 + PMULULQ X0, X1 + PMULULQ 24(AX), X2 + VPMULDQ X0, X1, X2 + VPMULDQ Y0, Y1, Y2 + VPMULDQ Z0, Z1, Z2 + VPMULUDQ X0, X1, X2 + VPMULUDQ Y0, Y1, Y2 + VPMULUDQ Z0, Z1, Z2 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, src, "discoveredx86forms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "discovered-x86-386-forms.ll", "discovered-x86-386-forms.o", ll) +} + +func TestTranslate386NewlyDiscoveredX86FamiliesRejectArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "MOVD AX, BX", + "MOVD M0, M1", + "MOVD M0, X0", + "CVTSQ2SS AX, X0", + "CVTSQ2SD 8(AX), X0", + "PMULULQ M0, M1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad386discovered(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "bad386discovered": {Name: "bad386discovered", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for GOARCH=386", instruction) + } + }) + } +} + +func TestTranslateAMD64MOVQFromXRegisterToFrameResult(t *testing.T) { + src := ` +TEXT vectorret(SB),NOSPLIT,$0-8 + PXOR X0, X0 + MOVQ X0, vectorGlobal<>(SB) + MOVQ X0, ret+0(FP) + RET +` + opts := Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"vectorret": { + Name: "vectorret", + Ret: I64, + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: I64, + Index: 0, + Field: -1, + }}}, + }}, + } + t.Run("defined-storage", func(t *testing.T) { + bound := "GLOBL vectorGlobal<>(SB),16,$8\n" + strings.Replace(src, ",NOSPLIT,", ",4,", 1) + x86LEAActualTargetGoObject(t, "amd64", opts.TargetTriple, bound) + file, err := Parse(ArchAMD64, bound) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, opts) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, backend := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, opts.TargetTriple, ir, backend) + } + }) + t.Run("unbound-storage", func(t *testing.T) { + // The original source has no storage definition. A signature and + // initialized X0 cannot establish the external symbol's extent. + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if ir, err := Translate(file, opts); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("unbound static storage acquired an extent", err) + } + }) +} + +func TestTranslateAMD64X87Arithmetic(t *testing.T) { + src := ` +TEXT x87amd64(SB),NOSPLIT,$0-0 + FLDZ + FLD1 + FADDD F1, F0 + FSUBD F1, F0 + FLDCW 0(SP) + RET +` + translateAMD64EcosystemCase(t, src, "x87amd64") +} + +func TestTranslateAMD64FuzzerBitTestAndVectorSubtract(t *testing.T) { + src := ` +TEXT fuzzops(SB),NOSPLIT,$0-0 + BTL $16, CX + JCS done + VPSUBB Y0, Y1, Y2 +done: + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + // Keep the exact original fuzzer source as Go and public Context coverage. + // CX and both vector inputs are absent from its Void entry contract. + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + x86UnboundVoidFormatSource(t, src, "fuzzops", target.arch, target.triple) + inputs := "MOVL $0,CX\nVPXOR Y0,Y0,Y0\nVPXOR Y1,Y1,Y1\n" + initialized := strings.Replace(src, "BTL $16, CX", inputs+"BTL $16, CX", 1) + live := strings.Replace(initialized, "$0-0", "$0-32", 1) + live = strings.Replace(live, inputs, "VPXOR Y2,Y2,Y2\n"+inputs, 1) + live = strings.Replace(live, "done:", "done:\nVMOVDQU Y2,ret+0(FP)", 1) + const liveDeclaration = "func fuzzops() (ret, b, c, d uint64)" + for _, program := range []struct{ source, declaration string }{ + {initialized, "func fuzzops()"}, + {live, liveDeclaration}, + } { + x86LEAActualTargetGoObject(t, target.arch, target.triple, "#include \"textflag.h\"\n"+program.source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, program.source, program.declaration, target.arch, target.triple, api) + if err != nil { + t.Fatal("initialized bit-test/branch/subtract program rejected", api, err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + } + } + for _, missing := range strings.Split(strings.TrimSpace(inputs), "\n") { + t.Run("missing-"+missing, func(t *testing.T) { + // Pure dead integer arithmetic may retain unknown bytes. + // Returning the full Y register makes every byte observable. + incomplete := strings.Replace(live, missing+"\n", "", 1) + x86LEAActualTargetGoObject(t, target.arch, target.triple, "#include \"textflag.h\"\n"+incomplete) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, incomplete, liveDeclaration, target.arch, target.triple, api) + if target.arch == "amd64" && strings.HasPrefix(missing, "VPXOR ") { + x86VectorScratchCodegenControl(t, incomplete, liveDeclaration, target.arch, target.triple, ir, err) + continue + } + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("live result acquired missing control or vector inputs", api, err) + } + } + }) + } + }) + } +} + +func TestTranslateX86DivisionAndFrameArithmetic(t *testing.T) { + amd64Source := ` +TEXT divmod(SB),NOSPLIT,$0-32 + MOVQ $0, DX + MOVQ a+0(FP), AX + DIVQ b+8(FP) + MOVQ AX, quo+16(FP) + MOVQ DX, rem+24(FP) + RET +` + translateX86FrameCase(t, amd64Source, "amd64", "x86_64-unknown-linux-gnu", "divmod") + + x86Source := ` +TEXT mulparts(SB),NOSPLIT,$0-32 + MOVL a+0(FP), AX + MULL b+8(FP) + MOVL AX, hi+16(FP) + ADDL DX, hi+16(FP) + RET +` + // Retain the complete historical body and its old manual two-I64 result + // layout as a missing-byte audit: it writes only the low four bytes of + // the first result and none of the second, not two initialized I64s. + file, err := Parse(ArchAMD64, x86Source) + if err != nil { + t.Fatal(err) + } + old := FuncSig{Name: "mulparts", Args: []LLVMType{I64, I64}, Ret: LLVMType("{ i64, i64 }"), Frame: FrameLayout{ + Params: []FrameSlot{{Type: I64, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}, + Results: []FrameSlot{{Offset: 16, Type: I64, Field: -1}, {Offset: 24, Type: I64, Index: 1, Field: -1}}, + }} + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: map[string]FuncSig{"mulparts": old}}) + x86PartialTupleCodegenControl(t, x86Source, "func mulparts(a,b uint64) (hi,lo uint64)", "386", "i386-unknown-linux-gnu", ir, err) + // Only the low written word is observed; neither the high word nor the + // completely unwritten second tuple element is a deterministic output. + defined := "define i32 @DefinedLow() {\n %r = call { i64, i64 } @mulparts(i64 4294967295,i64 2)\n" + + " %first = extractvalue { i64, i64 } %r, 0\n %low = trunc i64 %first to i32\n ret i32 %low\n}\n" + x86ScalarFPNamedConstantResult(t, ir+defined, "DefinedLow", []uint64{0xffffffff}) + // The same complete operations have one genuine I32 result. Bind the + // actual declaration and exact argument/result extent, not borrowed slots. + complete := strings.Replace(x86Source, "$0-32", "$0-20", 1) + for _, api := range []string{"text", "module", "Go"} { + t.Run("complete-mulparts/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, complete, "func mulparts(a,b uint64) uint32", "386", "i386-unknown-linux-gnu", api) + if err != nil || ir == "" { + t.Fatal("complete typed multiply/frame operations rejected", err) + } + probe := "define i32 @Probe() {\n %r = call i32 @mulparts(i64 4294967295,i64 2)\n ret i32 %r\n}\n" + x86ScalarFPNamedConstantResult(t, ir+probe, "Probe", []uint64{0xffffffff}) + }) + } +} + +func TestTranslateAMD64LegacyBranchHintAndSignedDivision(t *testing.T) { + src := ` +TEXT signeddivmod(SB),NOSPLIT,$0-32 + MOVQ a+0(FP), AX + MOVQ b+8(FP), CX + CMPQ CX, $-1 + JEQ $1, special + CQO + IDIVQ CX + JMP done +special: + NEGQ AX + MOVQ $0, DX +done: + MOVQ AX, quo+16(FP) + MOVQ DX, rem+24(FP) + RET +` + translateX86FrameCase(t, src, "amd64", "x86_64-unknown-linux-gnu", "signeddivmod") +} + +func TestTranslateAMD64LoopBranch(t *testing.T) { + src := ` +TEXT countedloop(SB),NOSPLIT,$0-0 + MOVQ $2, CX +loop: + LOOP loop + RET +` + translateAMD64EcosystemCase(t, src, "countedloop") +} + +func TestTranslateAMD64RawDataDirectives(t *testing.T) { + src := ` +TEXT rawencoding(SB),NOSPLIT,$0-0 + LONG $0xe77da1c4 + QUAD $0x4109048d47d18941 + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawencoding": {Name: "rawencoding", Ret: Void}}}); err == nil || !strings.Contains(err.Error(), "decode raw x86 directive group") { + t.Fatalf("reachable opaque data directives must fail closed, got %v", err) + } +} + +func TestTranslateAMD64HighQuadwordInterleave(t *testing.T) { + src := ` +TEXT highquadword(SB),NOSPLIT,$0-0 + PUNPCKHQDQ X1, X0 + VPUNPCKHQDQ Y0, Y1, Y2 + RET +` + translateAMD64EcosystemCase(t, src, "highquadword") +} + +func TestTranslateAMD64AdditionalAVXUnpackAndBroadcastFamilies(t *testing.T) { + src := ` +TEXT avxunpackbroadcast(SB),NOSPLIT,$0-0 + VPUNPCKLQDQ X0, X1, X2 + VPUNPCKLQDQ Y0, Y1, Y2 + VPUNPCKLDQ X0, X1, X2 + VPUNPCKLDQ Y0, Y1, Y2 + VPUNPCKHDQ X0, X1, X2 + VPUNPCKHDQ Y0, Y1, Y2 + VPBROADCASTD (AX), X3 + VPBROADCASTD (AX), Y3 + VPSHUFD $0x4e, (AX), X4 + VPSHUFD $0x4e, X0, X4 + VPSHUFD $0x4e, (AX), Z4 + VPSLLD $12, X5, X13 + VPSRLD $7, (AX), X13 + VPSLLD $12, Y5, Y13 + VPSLLD $12, Z5, Z13 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, src, "avxunpackbroadcast", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "avx-unpack-broadcast-formats.ll", "avx-unpack-broadcast-formats.o", ll) +} + +func TestTranslateAMD64DwordShuffleCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT dwordshuffleforms(SB),NOSPLIT,$0-0 + PSHUFD $1, X1, X2 + PSHUFD $1, (AX), X2 + PSHUFL $1, X1, X2 + VPSHUFD $1, X1, X2 + VPSHUFD $1, (AX), X2 + VPSHUFD $1, Y1, Y2 + VPSHUFD $1, (AX), Y2 + VPSHUFD $1, Z1, Z2 + VPSHUFD $1, (AX), Z2 + VPSHUFD $1, X1, K1, X2 + VPSHUFD.Z $1, (AX), K1, X2 + VPSHUFD $1, Y1, K1, Y2 + VPSHUFD.Z $1, (AX), K1, Y2 + VPSHUFD $1, Z1, K1, Z2 + VPSHUFD.Z $1, (AX), K1, Z2 + RET +` + translateAMD64EcosystemCase(t, src, "dwordshuffleforms") +} + +func TestTranslateAMD64CanonicalPackedByteShiftAliases(t *testing.T) { + src := ` +TEXT packedbyteshifts(SB),NOSPLIT,$0-0 + PSLLO $4, X0 + PSRLO $4, X0 + RET +` + translateAMD64EcosystemCase(t, src, "packedbyteshifts") +} + +func TestTranslateAMD64PackedUniformShiftCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's x86 assembler assigns all of these instructions the same + // _yvpslld operand-format table. Keep this test in lockstep with that + // family instead of adding only the form first observed in a dependency. + for _, op := range []string{ + "VPSLLD", "VPSLLQ", "VPSLLW", + "VPSRAD", "VPSRAW", + "VPSRLD", "VPSRLQ", "VPSRLW", + } { + var src strings.Builder + fmt.Fprintf(&src, "TEXT uniformshift%s(SB),NOSPLIT,$0-0\n", op) + for _, operands := range []string{ + "$3, X1, X2", "$-1, X1, X2", "$3, (AX), X2", + "$3, Y1, Y2", "$-1, Y1, Y2", "$3, (AX), Y2", + "$3, Z1, Z2", "$3, (AX), Z2", + "X0, X1, X2", "(AX), X1, X2", + "X0, Y1, Y2", "(AX), Y1, Y2", + "X0, Z1, Z2", "(AX), Z1, Z2", + "$3, X1, K1, X2", "$3, (AX), K1, X2", + "$3, Y1, K1, Y2", "$3, (AX), K1, Y2", + "$3, Z1, K1, Z2", "$3, (AX), K1, Z2", + "X0, X1, K1, X2", "(AX), X1, K1, X2", + "X0, Y1, K1, Y2", "(AX), Y1, K1, Y2", + "X0, Z1, K1, Z2", "(AX), Z1, K1, Z2", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, operands) + } + for _, operands := range []string{ + "$3, X1, K1, X2", "$3, (AX), K1, X2", + "$3, Y1, K1, Y2", "$3, (AX), K1, Y2", + "$3, Z1, K1, Z2", "$3, (AX), K1, Z2", + "X0, X1, K1, X2", "(AX), X1, K1, X2", + "X0, Y1, K1, Y2", "(AX), Y1, K1, Y2", + "X0, Z1, K1, Z2", "(AX), Z1, K1, Z2", + } { + fmt.Fprintf(&src, "\t%s.Z %s\n", op, operands) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "uniformshift"+op, "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "uniform-shift.ll", "uniform-shift.o", ll) + } +} + +func TestTranslateAMD64PackedIntegerCompareCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns all eight equality/greater-than variants the shared + // _yvpcmpeqb table. VEX writes X/Y vectors; EVEX compares X/Y/Z lanes into + // K registers, optionally masked by K1-K7. D/Q also permit memory broadcast. + ops := []string{ + "VPCMPEQB", "VPCMPEQW", "VPCMPEQD", "VPCMPEQQ", + "VPCMPGTB", "VPCMPGTW", "VPCMPGTD", "VPCMPGTQ", + } + forms := []string{ + "X1, X2, X3", "(AX), X2, X3", + "Y1, Y2, Y3", "(AX), Y2, Y3", + "X1, X2, K3", "(AX), X2, K3", + "X1, X2, K1, K3", "(AX), X2, K1, K3", + "Y1, Y2, K3", "(AX), Y2, K3", + "Y1, Y2, K1, K3", "(AX), Y2, K1, K3", + "Z1, Z2, K3", "(AX), Z2, K3", + "Z1, Z2, K1, K3", "(AX), Z2, K1, K3", + } + var src strings.Builder + src.WriteString("TEXT packedintegercompareforms(SB),NOSPLIT,$0-0\n") + for _, op := range ops { + for _, form := range forms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + for _, form := range []string{ + "(AX), X2, K3", "(AX), X2, K1, K3", + "(AX), Y2, K3", "(AX), Y2, K1, K3", + "(AX), Z2, K3", "(AX), Z2, K1, K3", + } { + fmt.Fprintf(&src, "\t%s.BCST %s\n", op, form) + } + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedintegercompareforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-integer-compare-forms.ll", "packed-integer-compare-forms.o", ll) +} + +func TestTranslateAMD64PackedIntegerCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPCMPEQB Z1, Z2, Z3", + "VPCMPEQW X1, Y2, Y3", + "VPCMPEQD X1, X2, K0, K3", + "VPCMPEQQ.Z X1, X2, K1, K3", + "VPCMPGTB.BCST (AX), X2, K3", + "VPCMPGTW.SAE X1, X2, K3", + "VPCMPGTD.BCST X1, X2, K3", + "VPCMPGTQ.BCST (AX), X2, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedintegercompare(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedintegercompare": {Name: "invalidpackedintegercompare", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvpcmpeqb table", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedAndNotCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 defines the whole AND-NOT family across three operand tables: + // PANDN's ymm table (MMX/XMM), VPANDN's _yvaddsubpd table (X/Y VEX), + // and VPANDND/Q's _yvblendmpd table (X/Y/Z EVEX with optional K1-K7 + // masking and zeroing). Only the D/Q EVEX variants permit memory broadcast. + src := ` +TEXT packedandnotforms(SB),NOSPLIT,$0-0 + PANDN M1, M2 + PANDN (AX), M2 + PANDN X1, X2 + PANDN (AX), X2 + VPANDN X1, X2, X3 + VPANDN (AX), X2, X3 + VPANDN Y1, Y2, Y3 + VPANDN (AX), Y2, Y3 + VPANDND X1, X2, X3 + VPANDND (AX), X2, X3 + VPANDND X1, X2, K1, X3 + VPANDND.Z (AX), X2, K1, X3 + VPANDND Y1, Y2, Y3 + VPANDND (AX), Y2, Y3 + VPANDND Y1, Y2, K1, Y3 + VPANDND.Z (AX), Y2, K1, Y3 + VPANDND Z1, Z2, Z3 + VPANDND (AX), Z2, Z3 + VPANDND Z1, Z2, K1, Z3 + VPANDND.Z (AX), Z2, K1, Z3 + VPANDND.BCST (AX), X2, X3 + VPANDND.BCST (AX), X2, K1, X3 + VPANDND.BCST.Z (AX), X2, K1, X3 + VPANDND.BCST (AX), Y2, Y3 + VPANDND.BCST (AX), Y2, K1, Y3 + VPANDND.BCST.Z (AX), Y2, K1, Y3 + VPANDND.BCST (AX), Z2, Z3 + VPANDND.BCST (AX), Z2, K1, Z3 + VPANDND.BCST.Z (AX), Z2, K1, Z3 + VPANDNQ X1, X2, X3 + VPANDNQ (AX), X2, X3 + VPANDNQ X1, X2, K1, X3 + VPANDNQ.Z (AX), X2, K1, X3 + VPANDNQ Y1, Y2, Y3 + VPANDNQ (AX), Y2, Y3 + VPANDNQ Y1, Y2, K1, Y3 + VPANDNQ.Z (AX), Y2, K1, Y3 + VPANDNQ Z1, Z2, Z3 + VPANDNQ (AX), Z2, Z3 + VPANDNQ Z1, Z2, K1, Z3 + VPANDNQ.Z (AX), Z2, K1, Z3 + VPANDNQ.BCST (AX), X2, X3 + VPANDNQ.BCST (AX), X2, K1, X3 + VPANDNQ.BCST.Z (AX), X2, K1, X3 + VPANDNQ.BCST (AX), Y2, Y3 + VPANDNQ.BCST (AX), Y2, K1, Y3 + VPANDNQ.BCST.Z (AX), Y2, K1, Y3 + VPANDNQ.BCST (AX), Z2, Z3 + VPANDNQ.BCST (AX), Z2, K1, Z3 + VPANDNQ.BCST.Z (AX), Z2, K1, Z3 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, src, "packedandnotforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-and-not-forms.ll", "packed-and-not-forms.o", ll) +} + +func TestTranslateAMD64PackedAndNotRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PANDN Y1, Y2", + "VPANDN Z1, Z2, Z3", + "VPANDN.X Z1, Z2, Z3", + "VPANDND X1, Y2, Y3", + "VPANDND X1, X2, K0, X3", + "VPANDND.Z X1, X2, X3", + "VPANDND.SAE X1, X2, K1, X3", + "VPANDND.BCST X1, X2, X3", + "VPANDNQ.BCST (AX), X2, Y3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedandnot(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedandnot": {Name: "invalidpackedandnot", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PANDN/VPANDN operand tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedScalarBroadcastCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns VPBROADCASTB/W/D/Q the same _yvpbroadcastb table. + // VEX broadcasts X/m to X/Y; EVEX adds GP and X/m sources, X/Y/Z + // destinations, K1-K7 merge masking, and masked .Z zeroing. + forms := []string{ + "X1, X2", "(AX), X2", "table<>(SB), X2", "0, X2", + "X1, Y2", "(AX), Y2", "table<>(SB), Y2", "0x1234, Y2", + "AX, X2", "AX, Y2", "AX, Z2", + "X1, Z2", "(AX), Z2", + "AX, K1, X2", "X1, K1, X2", "(AX), K1, X2", + "AX, K1, Y2", "X1, K1, Y2", "(AX), K1, Y2", + "AX, K1, Z2", "X1, K1, Z2", "(AX), K1, Z2", + } + zeroForms := []string{ + "AX, K1, X2", "X1, K1, X2", "(AX), K1, X2", + "AX, K1, Y2", "X1, K1, Y2", "(AX), K1, Y2", + "AX, K1, Z2", "X1, K1, Z2", "(AX), K1, Z2", + } + var src strings.Builder + src.WriteString("TEXT packedscalarbroadcastforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPBROADCASTB", "VPBROADCASTW", "VPBROADCASTD", "VPBROADCASTQ"} { + for _, form := range forms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + for _, form := range zeroForms { + fmt.Fprintf(&src, "\t%s.Z %s\n", op, form) + } + } + src.WriteString("\tRET\n") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedscalarbroadcastforms", "amd64", target.triple) + compileLLVMToObject(t, llc, target.triple, "packed-scalar-broadcast-forms-"+target.name+".ll", "packed-scalar-broadcast-forms-"+target.name+".o", ll) + }) + } +} + +func TestTranslateAMD64PackedScalarBroadcastRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPBROADCASTB $1, X2", + "VPBROADCASTW Y1, Y2", + "VPBROADCASTD X1, K0, X2", + "VPBROADCASTQ X1, AX", + "VPBROADCASTB.Z X1, X2", + "VPBROADCASTW.BCST (AX), X2", + "VPBROADCASTD.SAE X1, K1, X2", + "VPBROADCASTQ X1, K1, Y2, Z3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedscalarbroadcast(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedscalarbroadcast": {Name: "invalidpackedscalarbroadcast", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvpbroadcastb table", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedIntegerAddCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Go 1.27's legacy ymm table gives these instructions MMX and XMM forms. + for _, op := range []string{"PADDB", "PADDL", "PADDSB", "PADDSW", "PADDUSB", "PADDUSW", "PADDW"} { + src := fmt.Sprintf(` +TEXT legacy%sforms(SB),NOSPLIT,$0-0 + %s M1, M2 + %s (AX), M2 + %s X1, X2 + %s (AX), X2 + RET +`, op, op, op, op, op) + ir := x86UnboundVoidScalarFormatObject(t, src, "legacy"+op+"forms", "amd64", "x86_64-unknown-linux-gnu") + x86PackedStringCompileObject(t, llc, "x86_64-unknown-linux-gnu", ir, "-O0") + } + // PADDQ is deliberately narrower in Go's legacy yxm table: XMM only. + legacyQ := ` +TEXT legacyPADDQforms(SB),NOSPLIT,$0-0 + PADDQ X1, X2 + PADDQ (AX), X2 + RET +` + legacyIR := x86UnboundVoidScalarFormatObject(t, legacyQ, "legacyPADDQforms", "amd64", "x86_64-unknown-linux-gnu") + x86PackedStringCompileObject(t, llc, "x86_64-unknown-linux-gnu", legacyIR, "-O0") + + // Every AVX variant shares _yvandnpd: X/Y/Z register-or-memory inputs, + // optional K1-K7 merge/zero masks. D/Q additionally enable broadcast. + commonForms := []string{ + "X1, X2, X3", "(AX), X2, X3", + "X1, X2, K1, X3", "(AX), X2, K1, X3", + "Y1, Y2, Y3", "(AX), Y2, Y3", + "Y1, Y2, K1, Y3", "(AX), Y2, K1, Y3", + "Z1, Z2, Z3", "(AX), Z2, Z3", + "Z1, Z2, K1, Z3", "(AX), Z2, K1, Z3", + } + maskedForms := []string{ + "X1, X2, K1, X3", "(AX), X2, K1, X3", + "Y1, Y2, K1, Y3", "(AX), Y2, K1, Y3", + "Z1, Z2, K1, Z3", "(AX), Z2, K1, Z3", + } + var src strings.Builder + src.WriteString("TEXT packedintegeraddforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{ + "VPADDB", "VPADDW", "VPADDD", "VPADDQ", + "VPADDSB", "VPADDSW", "VPADDUSB", "VPADDUSW", + } { + for _, form := range commonForms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + for _, form := range maskedForms { + fmt.Fprintf(&src, "\t%s.Z %s\n", op, form) + } + if op == "VPADDD" || op == "VPADDQ" { + for _, form := range []string{ + "(AX), X2, X3", "(AX), X2, K1, X3", + "(AX), Y2, Y3", "(AX), Y2, K1, Y3", + "(AX), Z2, Z3", "(AX), Z2, K1, Z3", + } { + fmt.Fprintf(&src, "\t%s.BCST %s\n", op, form) + } + for _, form := range []string{ + "(AX), X2, K1, X3", "(AX), Y2, K1, Y3", "(AX), Z2, K1, Z3", + } { + fmt.Fprintf(&src, "\t%s.BCST.Z %s\n", op, form) + } + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedintegeraddforms", "amd64", "x86_64-unknown-linux-gnu") + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-integer-add-forms.ll", "packed-integer-add-forms.o", ll) +} + +func TestTranslateAMD64PackedIntegerAddRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PADDQ M1, M2", + "PADDB Y1, Y2", + "VPADDB X1, Y2, Y3", + "VPADDW X1, X2, K0, X3", + "VPADDD.Z X1, X2, X3", + "VPADDQ.BCST X1, X2, X3", + "VPADDSB.BCST (AX), X2, X3", + "VPADDSW.SAE X1, X2, K1, X3", + "VPADDUSB X1, X2, K1, X3, X4", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedintegeradd(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedintegeradd": {Name: "invalidpackedintegeradd", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-add tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedIntegerSubtractCompleteGoAssemblerForms(t *testing.T) { + // All eight legacy instructions use yxm: X/m128, X. Unlike the packed-add + // family, none of these legacy subtract spellings has an MMX row. + var src strings.Builder + src.WriteString("TEXT packedintegersubtractforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSUBB", "PSUBL", "PSUBQ", "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW", "PSUBW"} { + fmt.Fprintf(&src, "\t%s X1, X15\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15\n", op) + } + + // All eight V spellings share _yvandnpd. Exercise every VEX/EVEX width, + // both register and memory members of each rm class, high EVEX registers, + // K merge/zero masks, and the D/Q-only broadcast rows. + commonForms := []string{ + "X1, X2, X3", "(AX), X2, X3", + "Y1, Y2, Y3", "(AX), Y2, Y3", + "Z1, Z2, Z3", "(AX), Z2, Z3", + "X20, X21, X22", "Y20, Y21, Y22", "Z20, Z21, Z22", + } + maskedForms := []string{ + "X20, X21, K1, X22", "(AX), X21, K1, X22", + "Y20, Y21, K1, Y22", "(AX), Y21, K1, Y22", + "Z20, Z21, K1, Z22", "(AX), Z21, K1, Z22", + } + for _, op := range []string{ + "VPSUBB", "VPSUBD", "VPSUBQ", "VPSUBSB", + "VPSUBSW", "VPSUBUSB", "VPSUBUSW", "VPSUBW", + } { + for _, form := range commonForms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + for _, form := range maskedForms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + fmt.Fprintf(&src, "\t%s.Z %s\n", op, form) + } + if op == "VPSUBD" || op == "VPSUBQ" { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, K1, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), %s21, K1, %s22\n", op, width, width) + } + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedintegersubtractforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-integer-subtract-forms.ll", "packed-integer-subtract-forms.o", ll) +} + +func TestTranslate386PackedIntegerSubtractCompleteGoAssemblerForms(t *testing.T) { + // In 386 mode only the low eight registers have proved physical bindings. + // Go-accepted high-bank EVEX spellings require separate Context tests: + // frontend acceptance alone is not a mode32 physical register identity. + var src strings.Builder + src.WriteString("TEXT packedintegersubtractforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSUBB", "PSUBL", "PSUBQ", "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW", "PSUBW"} { + fmt.Fprintf(&src, "\t%s X1, X7\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + } + for _, op := range []string{ + "VPSUBB", "VPSUBD", "VPSUBQ", "VPSUBSB", + "VPSUBSW", "VPSUBUSB", "VPSUBUSW", "VPSUBW", + } { + for _, form := range []string{ + "X1, X2, X7", "(AX), X2, X7", + "Y1, Y2, Y7", "(AX), Y2, Y7", + "Z1, Z2, Z7", "(AX), Z2, Z7", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + if op == "VPSUBD" || op == "VPSUBQ" { + fmt.Fprintf(&src, "\t%s.BCST (AX), X2, X7\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Y2, Y7\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Z2, Z7\n", op) + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedintegersubtractforms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-integer-subtract-386-forms.ll", "packed-integer-subtract-386-forms.o", ll) +} + +func TestTranslateAMD64PackedIntegerSubtractRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PSUBB M1, M2", + "PSUBQ Y1, Y2", + "VPSUBB X1, Y2, Y3", + "VPSUBW X1, X2, K0, X3", + "VPSUBD.Z X1, X2, X3", + "VPSUBQ.BCST X1, X2, X3", + "VPSUBSB.BCST (AX), X2, X3", + "VPSUBSW.SAE X1, X2, K1, X3", + "VPSUBUSB X1, X2, K1, X3, X4", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedintegersubtract(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedintegersubtract": {Name: "invalidpackedintegersubtract", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-subtract tables", instruction) + } + }) + } +} + +func TestTranslate386PackedIntegerSubtractRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PSUBB X1, X8", + "VPSUBQ Z8, Z2, Z3", + "VPSUBD X1, X2, K1, X3", + "VPSUBQ.Z X1, X2, K1, X3", + "VPSUBD.BCST.Z (AX), X2, K1, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedintegersubtract386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedintegersubtract386": {Name: "invalidpackedintegersubtract386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedSumAbsoluteDifferencesCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT packedsadforms(SB),NOSPLIT,$0-0 + PSADBW X1, X2 + PSADBW (AX), X2 + VPSADBW X1, X2, X3 + VPSADBW (AX), X2, X3 + VPSADBW Y1, Y2, Y3 + VPSADBW (AX), Y2, Y3 + VPSADBW Z1, Z2, Z3 + VPSADBW (AX), Z2, Z3 + RET +` + // These original formats have neither vector inputs nor a pointer entry. + // Retain their Go objects and public Context checks; initialized public + // objects for the complete SAD family live in x86_result_sad_test.go. + ll := x86UnboundVoidScalarFormatObject(t, src, "packedsadforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-sad-forms.ll", "packed-sad-forms.o", ll) +} + +func TestTranslateAMD64PackedSumAbsoluteDifferencesRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PSADBW Y1, Y2", + "VPSADBW X1, Y2, Y3", + "VPSADBW X1, X2, K1, X3", + "VPSADBW.Z X1, X2, X3", + "VPSADBW.BCST (AX), X2, X3", + "VPSADBW $1, X2, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedsad(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedsad": {Name: "invalidpackedsad", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PSADBW/VPSADBW tables", instruction) + } + }) + } +} + +func TestTranslateAMD64VectorLaneExtractCompleteGoAssemblerForms(t *testing.T) { + // VEXTRACTF128/I128 share Go 1.27's _yvextractf128 table. + var src strings.Builder + src.WriteString("TEXT vectorlaneextractforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VEXTRACTF128", "VEXTRACTI128"} { + fmt.Fprintf(&src, "\t%s $1, Y1, X2\n", op) + fmt.Fprintf(&src, "\t%s $-1, Y1, (AX)\n", op) + } + // These four produce 128 bits from either Y or Z and share + // _yvextractf32x4, including masked register/memory destinations. + for _, op := range []string{"VEXTRACTF32X4", "VEXTRACTF64X2", "VEXTRACTI32X4", "VEXTRACTI64X2"} { + for _, form := range []string{ + "$1, Y1, X2", "$1, Y1, (AX)", + "$1, Y1, K1, X2", "$1, Y1, K1, (AX)", + "$3, Z1, X2", "$3, Z1, (AX)", + "$3, Z1, K1, X2", "$3, Z1, K1, (AX)", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + fmt.Fprintf(&src, "\t%s.Z $1, Y1, K1, X2\n", op) + fmt.Fprintf(&src, "\t%s.Z $3, Z1, K1, X2\n", op) + } + // These four produce 256 bits from Z and share _yvextractf32x8. + for _, op := range []string{"VEXTRACTF32X8", "VEXTRACTF64X4", "VEXTRACTI32X8", "VEXTRACTI64X4"} { + for _, form := range []string{ + "$1, Z1, Y2", "$1, Z1, (AX)", + "$1, Z1, K1, Y2", "$1, Z1, K1, (AX)", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + fmt.Fprintf(&src, "\t%s.Z $1, Z1, K1, Y2\n", op) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "vectorlaneextractforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "vector-lane-extract-forms.ll", "vector-lane-extract-forms.o", ll) +} + +func TestTranslateAMD64VectorLaneExtractRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VEXTRACTI128 $1, Z1, X2", + "VEXTRACTF128 $1, Y1, K1, X2", + "VEXTRACTI32X4 $1, X1, X2", + "VEXTRACTF64X2 $1, Y1, Y2", + "VEXTRACTI32X8 $1, Y1, Y2", + "VEXTRACTF64X4 $1, Z1, K0, Y2", + "VEXTRACTF32X4.Z $1, Y1, X2", + "VEXTRACTI64X2.Z $1, Z1, K1, (AX)", + "VEXTRACTI64X4.SAE $1, Z1, Y2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidvectorlaneextract(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + // Go encodes EVEX.z memory extracts, although Intel specifies #UD. + // Retain this control alongside the eight translator refusals. + terminal := instruction == "VEXTRACTI64X2.Z $1, Z1, K1, (AX)" + goSource := strings.Replace(src, "NOSPLIT", "4", 1) + if instruction == "VEXTRACTF32X4.Z $1, Y1, X2" { + // Old Go could encode z=1/aaa=0; retain that observation + // without granting the reserved form translation semantics. + x86InsertExtractReservedZeroMaskGoObservation(t, goSource, + 0x62, 0xf3, 0x7d, 0xa8, 0x19, 0xca, 0x01) + } else { + x86ScalarLiteralExpectedGoListing(t, "amd64", goSource, terminal) + } + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidvectorlaneextract": {Name: "invalidvectorlaneextract", Ret: Void}, + }, + }) + if terminal { + if err != nil { + t.Fatal("Go-encodable memory zeroing must terminate before reading its inputs", err) + } + if _, err := x86OfficialMoveFaultBody(ll, "invalidvectorlaneextract"); err != nil { + t.Fatal(err) + } + } else if err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's vector-lane extract tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedScalarExtractCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yextr/yextrw tables accept an unsigned imm8, an + // X0-X15 source, and a GP-or-memory destination. The yextr table used by + // B/D/Q also (unusually) classifies M0-M7 as destinations; those names + // encode the correspondingly numbered GP registers. + var src strings.Builder + src.WriteString("TEXT packedscalarextractforms(SB),NOSPLIT,$0-0\n") + for _, spec := range []struct { + op string + maxIndex int + mmxDst bool + }{ + {op: "PEXTRB", maxIndex: 15, mmxDst: true}, + {op: "PEXTRW", maxIndex: 7}, + {op: "PEXTRD", maxIndex: 3, mmxDst: true}, + {op: "PEXTRQ", maxIndex: 1, mmxDst: true}, + } { + fmt.Fprintf(&src, "\t%s $%d, X15, AX\n", spec.op, spec.maxIndex) + fmt.Fprintf(&src, "\t%s $255, X15, (AX)\n", spec.op) + if spec.mmxDst { + fmt.Fprintf(&src, "\t%s $%d, X15, M2\n", spec.op, spec.maxIndex) + } + } + + // Go 1.27's _yvextractps/_yvpextrw tables add signed-imm8 VEX forms + // for X0-X15 and unsigned-imm8 EVEX forms for X16-X31. They do not add + // mask or instruction-suffix forms. + for _, spec := range []struct { + op string + maxIndex int + }{ + {op: "VPEXTRB", maxIndex: 15}, + {op: "VPEXTRW", maxIndex: 7}, + {op: "VPEXTRD", maxIndex: 3}, + {op: "VPEXTRQ", maxIndex: 1}, + } { + fmt.Fprintf(&src, "\t%s $%d, X1, R8\n", spec.op, spec.maxIndex) + fmt.Fprintf(&src, "\t%s $255, X1, (AX)\n", spec.op) + fmt.Fprintf(&src, "\t%s $-128, X1, R8\n", spec.op) + fmt.Fprintf(&src, "\t%s $%d, X31, R15\n", spec.op, spec.maxIndex) + fmt.Fprintf(&src, "\t%s $255, X31, (AX)\n", spec.op) + } + src.WriteString("\tRET\n") + + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedscalarextractforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-scalar-extract-forms.ll", "packed-scalar-extract-forms.o", ll) +} + +func TestTranslateAMD64PackedScalarExtractRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PEXTRB $-1, X1, AX", + "PEXTRD $256, X1, AX", + "PEXTRW $1, X1, M2", + "PEXTRQ $1, X16, AX", + "PEXTRB $1, Y1, AX", + "VPEXTRB $-129, X1, AX", + "VPEXTRW $256, X1, AX", + "VPEXTRD $-1, X20, AX", + "VPEXTRQ $1, X1, M2", + "VPEXTRB $1, X1, K1, AX", + "VPEXTRW.Z $1, X1, AX", + "VPEXTRD.BCST $1, X1, (AX)", + "VPEXTRQ.SAE $1, X1, AX", + "VPEXTRB $1, X1, X2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedscalarextract(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedscalarextract": {Name: "invalidpackedscalarextract", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PEXTR/VPEXTR tables", instruction) + } + }) + } +} + +func TestTranslateAMD64MoveHighLowPackedSingleCompleteGoAssemblerForms(t *testing.T) { + // MOVHLPS/MOVLHPS use Go 1.27's yxr table (X0-X15, two operands). + // VMOVHLPS/VMOVLHPS share _yvmovhlps: three X operands, with both the + // X0-X15 VEX entry and the X0-X31 EVEX entry. + src := ` +TEXT movehighlowpackedsingleforms(SB),NOSPLIT,$0-0 + MOVHLPS X1, X2 + MOVLHPS X15, X0 + VMOVHLPS X1, X2, X3 + VMOVLHPS X15, X14, X13 + VMOVHLPS X20, X2, X3 + VMOVHLPS X1, X20, X3 + VMOVLHPS X1, X2, X31 + RET +` + ll := translateAMD64EcosystemCase(t, src, "movehighlowpackedsingleforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "move-high-low-packed-single-forms.ll", "move-high-low-packed-single-forms.o", ll) +} + +func TestTranslateAMD64MoveHighLowPackedSingleRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "MOVHLPS X16, X1", + "MOVLHPS X1, X16", + "MOVHLPS (AX), X1", + "MOVLHPS X1, X2, X3", + "VMOVHLPS X1, X2", + "VMOVLHPS (AX), X2, X3", + "VMOVHLPS Y1, Y2, Y3", + "VMOVLHPS X1, X2, K1, X3", + "VMOVHLPS.Z X1, X2, X3", + "VMOVLHPS.BCST X1, X2, X3", + "VMOVHLPS.SAE X1, X2, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidmovehighlowpackedsingle(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidmovehighlowpackedsingle": {Name: "invalidmovehighlowpackedsingle", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's MOVHLPS/MOVLHPS tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PopulationCountCompleteGoAssemblerForms(t *testing.T) { + // POPCNTW/L/Q all use Go 1.27's yml_rl table: a Yml GP-or-memory + // source followed by a Yrl GP destination. + src := ` +TEXT populationcountforms(SB),NOSPLIT,$0-0 + POPCNTW AX, BX + POPCNTW (AX), R8 + POPCNTW R15, R14 + POPCNTL AX, BX + POPCNTL (AX), R8 + POPCNTL R15, R14 + POPCNTQ AX, BX + POPCNTQ (AX), R8 + POPCNTQ R15, R14 + RET +` + ll := x86ScalarCountUnboundFormatObject(t, src, "populationcountforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "population-count-forms.ll", "population-count-forms.o", ll) +} + +func TestTranslate386PopulationCountCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT populationcount386forms(SB),NOSPLIT,$0-0 + POPCNTW AX, BX + POPCNTW (AX), CX + POPCNTL AX, BX + POPCNTL (AX), CX + RET +` + ll := x86ScalarCountUnboundFormatObject(t, src, "populationcount386forms", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "population-count-386-forms.ll", "population-count-386-forms.o", ll) + + invalid, err := Parse(ArchAMD64, "TEXT invalidpopulationcount386(SB),NOSPLIT,$0-0\n\tPOPCNTQ AX, BX\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(invalid, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpopulationcount386": {Name: "invalidpopulationcount386", Ret: Void}, + }, + }); err == nil { + t.Fatal("Translate accepted POPCNTQ for GOARCH=386") + } +} + +func TestTranslateAMD64PopulationCountRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "POPCNTW AL, BX", + "POPCNTL AX, AH", + "POPCNTQ M1, AX", + "POPCNTW X1, AX", + "POPCNTL AX, (BX)", + "POPCNTQ $1, AX", + "POPCNTW.Z AX, BX", + "POPCNTL AX, BX, CX", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpopulationcount(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpopulationcount": {Name: "invalidpopulationcount", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's POPCNT tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedMoveMaskCompleteGoAssemblerForms(t *testing.T) { + // PMOVMSKB's ymskb table has X and M source rows. VPMOVMSKB's + // _yvmovmskpd table has X and Y source rows. Every row writes Yrl and + // every vector register is limited to 0-15; neither table has EVEX rows. + src := ` +TEXT packedmovemaskforms(SB),NOSPLIT,$0-0 + PMOVMSKB X0, AX + PMOVMSKB X15, R8 + PMOVMSKB M0, BX + PMOVMSKB M7, R15 + VPMOVMSKB X0, AX + VPMOVMSKB X15, R8 + VPMOVMSKB Y0, BX + VPMOVMSKB Y15, R15 + RET +` + // Keep the original uninitialized, unterminated MMX source as a public + // context boundary. Legal Go operand forms do not supply physical inputs. + goAssemblerOracleOutput(t, "amd64", "#include \"textflag.h\"\n"+src, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run("original-context/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, src, "func packedmovemaskforms()", "amd64", "x86_64-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original source acquired initialized vectors or MMX tag cleanup", err) + } + }) + } + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + var initialized, declarations strings.Builder + widths := map[string]int{} + widthCounts := map[int]int{} + for _, instruction := range file.Funcs[0].Instrs { + form, known, err := x86MoveMaskProperties(instruction.Op, instruction, "amd64") + if !known { + continue + } + if err != nil { + t.Fatal(err) + } + name := fmt.Sprintf("PackedMoveMaskForm%d", len(widths)) + widths[name] = form.byteWidth + widthCounts[form.byteWidth]++ + original := strings.TrimSpace(instruction.Raw) + load, cleanup := "MOVOU", "" + if form.byteWidth == 8 { + load, cleanup = "MOVQ", "EMMS\n" + } else if form.byteWidth == 32 { + load = "VMOVDQU" + } + body := fmt.Sprintf("MOVQ p+0(FP),SI\n%s (SI),%s\n%s\n%sMOVQ %s,ret+8(FP)\nRET\n", + load, instruction.Args[0].Reg, original, cleanup, instruction.Args[1].Reg) + fmt.Fprintf(&initialized, "TEXT ·%s(SB),4,$0-16\n%s", name, body) + fmt.Fprintf(&declarations, "func %s(p *[32]byte) uint64\n", name) + + // Observe each original operation's result, rather than allowing an + // unused Void destination to hide an undefined source register. + undefined := fmt.Sprintf("TEXT ·%s(SB),4,$0-8\n%s\n%sMOVQ %s,ret+0(FP)\nRET\n", + name, original, cleanup, instruction.Args[1].Reg) + goAssemblerOracleOutput(t, "amd64", undefined, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run("undefined-input/"+name+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, undefined, "func "+name+"() uint64", "amd64", "x86_64-unknown-linux-gnu", api) + if form.byteWidth != 8 { + x86VectorScratchCodegenControl(t, undefined, "func "+name+"() uint64", "amd64", "x86_64-unknown-linux-gnu", ir, err) + return + } + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("mask result acquired an undefined physical source", original, err) + } + }) + if form.byteWidth == 8 { + t.Run("missing-EMMS/"+name+"/"+api, func(t *testing.T) { + unterminated := "TEXT ·" + name + "(SB),4,$0-16\n" + strings.ReplaceAll(body, cleanup, "") + ir, err := x86ScalarFPBitViewAPI(t, unterminated, "func "+name+"(p *[32]byte) uint64", "amd64", "x86_64-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("initialized MMX source dispensed with real tag cleanup", original, err) + } + }) + } + } + } + if len(widths) != 8 || widthCounts[8] != 2 || widthCounts[16] != 4 || widthCounts[32] != 2 { + t.Fatal("original eight move-mask forms changed", widths, widthCounts) + } + goAssemblerOracleOutput(t, "amd64", initialized.String(), "", true) + llc, objdump := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || objdump == "" { + t.Fatal("LLVM 22 llc and llvm-objdump required") + } + for _, api := range []string{"text", "module", "Go"} { + t.Run("initialized/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, initialized.String(), declarations.String(), "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil || ir == "" { + t.Fatal("complete initialized move-mask forms rejected", err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + listing := x86ExchangeObjectDisassembly(t, llc, objdump, "x86_64-unknown-linux-gnu", ir, level) + assertEcosystemPackedMoveMaskObjectResults(t, listing, widths) + }) + } + }) + } +} + +func assertEcosystemPackedMoveMaskObjectResults(t *testing.T, listing string, widths map[string]int) { + t.Helper() + owner := "" + masks, cleanups := map[string]int{}, map[string]int{} + for _, line := range strings.Split(listing, "\n") { + if _, name, found := strings.Cut(line, " <"); found && strings.HasSuffix(name, ">:") { + owner = strings.TrimSuffix(name, ">:") + continue + } + if _, known := widths[owner]; !known { + continue + } + for _, atom := range strings.Fields(line) { + if atom == "pmovmskb" || atom == "vpmovmskb" { + masks[owner]++ + } else if atom == "emms" { + cleanups[owner]++ + } + } + } + for name, width := range widths { + // A Y source may lower to one AVX2 mask or two SSE2 halves. Every + // independently returned mask must survive both optimization levels. + if masks[name] != 1 && (width != 32 || masks[name] != 2) { + t.Fatal("observable move-mask output missing from its target function", name, masks[name]) + } + if width == 8 && cleanups[name] != 1 { + t.Fatal("MMX target region lost its tag cleanup", name, cleanups[name]) + } + } +} + +func TestTranslateAMD64PackedMoveMaskRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PMOVMSKB Y1, AX", + "PMOVMSKB X16, AX", + "PMOVMSKB (AX), BX", + "PMOVMSKB X1, M2", + "VPMOVMSKB M1, AX", + "VPMOVMSKB X16, AX", + "VPMOVMSKB Y16, AX", + "VPMOVMSKB X1, AL", + "VPMOVMSKB X1, X2", + "VPMOVMSKB X1, K1, AX", + "PMOVMSKB.Z X1, AX", + "VPMOVMSKB.BCST Y1, AX", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedmovemask(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedmovemask": {Name: "invalidpackedmovemask", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PMOVMSKB/VPMOVMSKB tables", instruction) + } + }) + } +} + +func TestTranslateAMD64BitwiseNotCompleteGoAssemblerForms(t *testing.T) { + // NOTB/W/L/Q all use Go 1.27's yscond table, whose sole row is Ymb. + // Besides memory and full GP spellings, Ymb admits low/high byte register + // spellings; for wider operations Go encodes those spellings by ModRM + // number (for example, NOTW AH encodes NOTW SP). + var src strings.Builder + src.WriteString("TEXT bitwisenotforms(SB),NOSPLIT,$0-0\n") + operands := []string{ + "AX", "BX", "CX", "DX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15", + "AL", "AH", "BL", "BH", "CL", "CH", "DL", "DH", + "BPB", "SIB", "DIB", "R8B", "R9B", "R10B", "R11B", "R12B", "R13B", "R14B", "R15B", + "(BX)", + } + for _, op := range []string{"NOTB", "NOTW", "NOTL", "NOTQ"} { + for _, operand := range operands { + fmt.Fprintf(&src, "\t%s %s\n", op, operand) + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "bitwisenotforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "bitwise-not-forms.ll", "bitwise-not-forms.o", ll) +} + +func TestTranslate386BitwiseNotCompleteGoAssemblerForms(t *testing.T) { + // In 386 mode Go classifies SP as Yrl32, which does not cover Ymb, but + // accepts BP/SI/DI by synthesizing an equivalent instruction sequence. + // Exercise every accepted GP spelling, both byte aliases, and memory. + var src strings.Builder + src.WriteString("TEXT bitwisenot386forms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"NOTB", "NOTW", "NOTL"} { + for _, operand := range []string{ + "AX", "BX", "CX", "DX", "BP", "SI", "DI", + "AL", "AH", "BL", "BH", "CL", "CH", "DL", "DH", + "(BX)", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, operand) + } + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "bitwisenot386forms", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "bitwise-not-386-forms.ll", "bitwise-not-386-forms.o", ll) + + for _, instruction := range []string{"NOTB SP", "NOTW SP", "NOTL SP", "NOTQ AX", "NOTB R8"} { + t.Run("reject_"+strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + invalid, err := Parse(ArchAMD64, "TEXT invalidbitwisenot386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(invalid, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidbitwisenot386": {Name: "invalidbitwisenot386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for GOARCH=386", instruction) + } + }) + } +} + +func TestTranslateAMD64BitwiseNotRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "NOTB M1", + "NOTW X1", + "NOTL K1", + "NOTQ $1", + "NOTB AX, BX", + "NOTW.Z AX", + "NOTL.BCST (AX)", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidbitwisenot(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidbitwisenot": {Name: "invalidbitwisenot", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's NOT tables", instruction) + } + }) + } +} + +func TestTranslateAMD64ParallelBitDepositExtractCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses one _yandnl row for the whole family: + // Yml source/mask, Yrl source value, and Yrl destination. This gives each + // L/Q instruction both register and memory first-operand forms. + var src strings.Builder + src.WriteString("TEXT parallelbitforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + fmt.Fprintf(&src, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&src, "\t%s (SI), R8, R15\n", op) + fmt.Fprintf(&src, "\t%s parallelBitMask(SB), R9, R10\n", op) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "parallelbitforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "parallel-bit-forms.ll", "parallel-bit-forms.o", ll) +} + +func TestTranslate386ParallelBitDepositExtractCompleteGoAssemblerForms(t *testing.T) { + // The Go 1.27 assembler accepts the same four instruction names in 386 + // mode. Yml/Yrl admit all eight 386 GP registers, including SP in any + // register position; only the first operand can instead be memory. + var src strings.Builder + src.WriteString("TEXT parallelbitforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + fmt.Fprintf(&src, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&src, "\t%s (SI), BP, DI\n", op) + fmt.Fprintf(&src, "\t%s SP, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s AX, SP, BX\n", op) + fmt.Fprintf(&src, "\t%s AX, BX, SP\n", op) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "parallelbitforms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "parallel-bit-386-forms.ll", "parallel-bit-386-forms.o", ll) +} + +func TestTranslateAMD64ParallelBitDepositExtractRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PDEPL AX, BX", + "PDEPQ AX, (BX), CX", + "PEXTL AX, BX, (CX)", + "PEXTQ $1, AX, BX", + "PDEPL AL, BX, CX", + "PDEPQ AX, X0, BX", + "PEXTL AX, BX, X0", + "PEXTQ.Z AX, BX, CX", + "PDEPL.BCST (AX), BX, CX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidparallelbit(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidparallelbit": {Name: "invalidparallelbit", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PDEP/PEXT table", instruction) + } + }) + } +} + +func TestTranslate386ParallelBitDepositExtractRejectsHighRegisters(t *testing.T) { + for _, instruction := range []string{ + "PDEPL R8, AX, BX", + "PDEPQ AX, R8, BX", + "PEXTL AX, BX, R8", + } { + file, err := Parse(ArchAMD64, "TEXT invalidparallelbit386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidparallelbit386": {Name: "invalidparallelbit386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for GOARCH=386", instruction) + } + } +} + +func TestTranslateAMD64BitTestFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 gives all BT/BTC/BTR/BTS W/L/Q instructions exactly two ybtl + // rows: signed imm8 + Yml and Yrl + Yml. + var src strings.Builder + src.WriteString("TEXT bittestforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"BT", "BTC", "BTR", "BTS"} { + for _, width := range []string{"W", "L", "Q"} { + op := stem + width + fmt.Fprintf(&src, "\t%s $-128, AX\n", op) + fmt.Fprintf(&src, "\t%s $127, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, R15\n", op) + fmt.Fprintf(&src, "\t%s R8, 8(SI)\n", op) + fmt.Fprintf(&src, "\t%s R9, bitTestGlobal(SB)\n", op) + } + } + src.WriteString("\tRET\n") + x86BitTestCompositeFormatObjects(t, src.String(), "bittestforms", "amd64", "x86_64-unknown-linux-gnu") +} + +func TestTranslate386BitTestFamilyCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT bittestforms386(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"BT", "BTC", "BTR", "BTS"} { + for _, width := range []string{"W", "L"} { + op := stem + width + fmt.Fprintf(&src, "\t%s $-128, AX\n", op) + fmt.Fprintf(&src, "\t%s $127, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, DI\n", op) + fmt.Fprintf(&src, "\t%s SP, BP\n", op) + fmt.Fprintf(&src, "\t%s AX, SP\n", op) + } + } + src.WriteString("\tRET\n") + x86BitTestCompositeFormatObjects(t, src.String(), "bittestforms386", "386", "i386-unknown-linux-gnu") +} + +func TestTranslateAMD64BitTestFamilyRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "BTB $1, AX", + "BTRL $128, AX", + "BTSQ $-129, AX", + "BTCW AX", + "BTL (AX), BX", + "BTRQ AL, AX", + "BTSW AX, $1", + "BTQ AX, X0", + "BTCL.Z AX, BX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidbittest(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidbittest": {Name: "invalidbittest", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's ybtl table", instruction) + } + }) + } +} + +func TestTranslate386BitTestFamilyRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "BTQ $1, AX", + "BTCQ AX, BX", + "BTRQ $1, (BX)", + "BTSQ AX, SP", + "BTL R8, AX", + "BTRW AX, R8", + } { + file, err := Parse(ArchAMD64, "TEXT invalidbittest386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidbittest386": {Name: "invalidbittest386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for GOARCH=386", instruction) + } + } +} + +func TestTranslateAMD64ScalarAddSubCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT scalaraddsubforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"ADD", "SUB"} { + fmt.Fprintf(&src, "\t%sB $-2147483648, AL\n", stem) + fmt.Fprintf(&src, "\t%sB $4294967295, bytearg+0(FP)\n", stem) + fmt.Fprintf(&src, "\t%sB BP, (BX)\n", stem) + fmt.Fprintf(&src, "\t%sB (BX), R15\n", stem) + fmt.Fprintf(&src, "\t%sB bytearg+0(FP), R14\n", stem) + fmt.Fprintf(&src, "\t%sB CL, bytearg+0(FP)\n", stem) + for _, form := range []struct { + width string + fpName string + offset int + }{ + {width: "W", fpName: "wordarg", offset: 8}, + {width: "L", fpName: "longarg", offset: 16}, + {width: "Q", fpName: "quadarg", offset: 24}, + } { + op := stem + form.width + fmt.Fprintf(&src, "\t%s $-2147483648, AX\n", op) + fmt.Fprintf(&src, "\t%s $4294967295, %s+%d(FP)\n", op, form.fpName, form.offset) + fmt.Fprintf(&src, "\t%s R8, (BX)\n", op) + fmt.Fprintf(&src, "\t%s (BX), R15\n", op) + fmt.Fprintf(&src, "\t%s %s+%d(FP), R14\n", op, form.fpName, form.offset) + fmt.Fprintf(&src, "\t%s CX, %s+%d(FP)\n", op, form.fpName, form.offset) + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + options := Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "scalaraddsubforms": { + Name: "scalaraddsubforms", + Args: []LLVMType{I8, I16, I32, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I8, Index: 0, Field: -1}, + {Offset: 8, Type: I16, Index: 1, Field: -1}, + {Offset: 16, Type: I32, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }, + }, + } + // Keep the original Go-accepted source and all public Context controls. + // Its synthetic FP offsets and incoming native registers are format + // witnesses, not an ordinary initialized ABI0 entry. + x86UnboundVoidFormatSource(t, src.String(), "scalaraddsubforms", "amd64", options.TargetTriple) + ll := x86ScalarFormatObjectUnit(t, file, options.Sigs, "amd64", options.TargetTriple, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "scalar-add-sub-forms.ll", "scalar-add-sub-forms.o", ll) +} + +func TestTranslate386ScalarAddSubCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT scalaraddsubforms386(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"ADD", "SUB"} { + fmt.Fprintf(&src, "\t%sB $4294967296, BP\n", stem) + fmt.Fprintf(&src, "\t%sB $-2147483649, bytearg+0(FP)\n", stem) + fmt.Fprintf(&src, "\t%sB SI, (BX)\n", stem) + fmt.Fprintf(&src, "\t%sB (BX), DI\n", stem) + for _, form := range []struct { + width string + fpName string + offset int + }{ + {width: "W", fpName: "wordarg", offset: 4}, + {width: "L", fpName: "longarg", offset: 8}, + } { + op := stem + form.width + fmt.Fprintf(&src, "\t%s $4294967296, AX\n", op) + fmt.Fprintf(&src, "\t%s $1, SP\n", op) + fmt.Fprintf(&src, "\t%s $-2147483649, %s+%d(FP)\n", op, form.fpName, form.offset) + fmt.Fprintf(&src, "\t%s SI, (BX)\n", op) + fmt.Fprintf(&src, "\t%s (BX), DI\n", op) + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + options := Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "scalaraddsubforms386": { + Name: "scalaraddsubforms386", + Args: []LLVMType{I8, I16, I32}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I8, Index: 0, Field: -1}, + {Offset: 4, Type: I16, Index: 1, Field: -1}, + {Offset: 8, Type: I32, Index: 2, Field: -1}, + }}, + }, + }, + } + x86UnboundVoidFormatSource(t, src.String(), "scalaraddsubforms386", "386", options.TargetTriple) + ll := x86ScalarFormatObjectUnit(t, file, options.Sigs, "386", options.TargetTriple, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "scalar-add-sub-386-forms.ll", "scalar-add-sub-386-forms.o", ll) +} + +func TestTranslateAMD64ScalarAddSubRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "ADDB (AX), (BX)", + "SUBW (AX), (BX)", + "ADDL $-2147483649, AX", + "SUBQ $4294967296, AX", + "ADDQ.Z AX, BX", + "SUBQ X0, AX", + "ADDQ AX, X0", + "SUBQ AX, $1", + "ADDB SP, (AX)", + "SUBB (AX), SP", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidscalaraddsub(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidscalaraddsub": {Name: "invalidscalaraddsub", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's ADD/SUB tables", instruction) + } + }) + } +} + +func TestTranslate386ScalarAddSubRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "ADDQ AX, BX", + "SUBQ $1, (BX)", + "ADDL R8, AX", + "SUBW AX, R8", + "ADDB R8, (AX)", + "SUBB (AX), R8", + "ADDB SP, (AX)", + } { + file, err := Parse(ArchAMD64, "TEXT invalidscalaraddsub386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidscalaraddsub386": {Name: "invalidscalaraddsub386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + } +} + +func TestTranslateAMD64CompareExchangeOriginalSyntaxContextCatalogue(t *testing.T) { + src := ` +DATA cmpxchgdata+0(SB)/8, $0 +DATA cmpxchgdata+8(SB)/8, $0 +GLOBL cmpxchgdata(SB), NOPTR, $16 +TEXT cmpxchgforms(SB),NOSPLIT,$0-16 + CMPXCHGB AH, R11 + CMPXCHGB R11, SP + CMPXCHGB DL, (R12) + CMPXCHGB DL, 0(FS) + CMPXCHGB DL, cmpxchgdata(SB) + CMPXCHGB DL, value+0(FP) + CMPXCHGW DX, R11 + CMPXCHGW R11, (R12) + CMPXCHGW DX, 0(GS) + CMPXCHGW DX, cmpxchgdata(SB) + CMPXCHGW DX, value+0(FP) + CMPXCHGL DX, R11 + CMPXCHGL R11, (R12) + CMPXCHGL DX, 0(FS) + CMPXCHGL DX, cmpxchgdata(SB) + CMPXCHGL DX, value+0(FP) + CMPXCHGQ DX, R11 + CMPXCHGQ R11, (R12) + CMPXCHGQ DX, 0(GS) + CMPXCHGQ DX, cmpxchgdata(SB) + CMPXCHGQ DX, value+0(FP) + CMPXCHG8B R11 + CMPXCHG8B SP + CMPXCHG8B (R12) + CMPXCHG8B 0(FS) + CMPXCHG8B cmpxchgdata(SB) + CMPXCHG8B value+0(FP) + CMPXCHG16B R11 + CMPXCHG16B SP + CMPXCHG16B (R12) + CMPXCHG16B 0(GS) + CMPXCHG16B cmpxchgdata(SB) + CMPXCHG16B value+0(FP) + RET +` + x86CMPXCHGMixedOriginalCatalogue(t, "amd64", src) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "cmpxchgforms": { + Name: "cmpxchgforms", + Args: []LLVMType{LLVMType("i128")}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: LLVMType("i128"), Index: 0, Field: -1}, + }}, + }, + }, + }) + // This unchanged syntax catalogue contains Go-accepted physical high-byte + // rewrites, non-memory pair #UD and an unbound 16B FP shadow. It is not a + // numerical native-positive. Closed memory forms have independent objects. + if !errors.Is(err, ErrProbeNeedsContext) || ll != "" { + t.Fatalf("mixed original native-identity gaps acquired a numerical object: %v", err) + } +} + +func TestTranslate386CompareExchangeOriginalSyntaxContextCatalogue(t *testing.T) { + src := ` +DATA cmpxchgdata386+0(SB)/8, $0 +GLOBL cmpxchgdata386(SB), NOPTR, $8 +TEXT cmpxchgforms386(SB),NOSPLIT,$0-8 + CMPXCHGB BP, (BX) + CMPXCHGB SI, DI + CMPXCHGB DL, 0(FS) + CMPXCHGB DL, cmpxchgdata386(SB) + CMPXCHGB DL, value+0(FP) + CMPXCHGW SI, DI + CMPXCHGW DX, SP + CMPXCHGW DX, (BX) + CMPXCHGW DX, 0(FS) + CMPXCHGW DX, cmpxchgdata386(SB) + CMPXCHGW DX, value+0(FP) + CMPXCHGL SI, DI + CMPXCHGL DX, SP + CMPXCHGL DX, (BX) + CMPXCHGL DX, 0(FS) + CMPXCHGL DX, cmpxchgdata386(SB) + CMPXCHGL DX, value+0(FP) + CMPXCHG8B AX + CMPXCHG8B BP + CMPXCHG8B SI + CMPXCHG8B (BX) + CMPXCHG8B 0(FS) + CMPXCHG8B cmpxchgdata386(SB) + CMPXCHG8B value+0(FP) + RET +` + x86CMPXCHGMixedOriginalCatalogue(t, "386", src) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "cmpxchgforms386": { + Name: "cmpxchgforms386", + Args: []LLVMType{I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, + }}, + }, + }, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ll != "" { + t.Fatalf("mixed original byte expansion/pair #UD acquired a numerical object: %v", err) + } +} + +func TestTranslateAMD64CompareExchangeRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "CMPXCHGB (BX), DL", + "CMPXCHGW (BX), DX", + "CMPXCHGL $1, DX", + "CMPXCHGQ DX, $1", + "CMPXCHGB X0, DL", + "CMPXCHGL DX, X0", + "CMPXCHG8B", + "CMPXCHG8B (BX), AX", + "CMPXCHG8B $1", + "CMPXCHG16B", + "CMPXCHG16B (BX), AX", + "CMPXCHG16B.Z (BX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidcmpxchg(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidcmpxchg": {Name: "invalidcmpxchg", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's CMPXCHG tables", instruction) + } + }) + } +} + +func TestTranslate386CompareExchangeRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "CMPXCHGQ DX, (BX)", + "CMPXCHG16B (BX)", + "CMPXCHGB R8, (BX)", + "CMPXCHGW R8, (BX)", + "CMPXCHGL R8, (BX)", + "CMPXCHGB SP, (BX)", + "CMPXCHGB DL, SP", + "CMPXCHG8B SP", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidcmpxchg386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidcmpxchg386": {Name: "invalidcmpxchg386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64TrailingZeroCountCompleteGoAssemblerForms(t *testing.T) { + // TZCNTW/L/Q all use Go 1.27's ycrc32l table: Yml source, Yrl + // destination. The width is selected only by the opcode prefix. + src := ` +TEXT trailingzerocountforms(SB),NOSPLIT,$0-0 + TZCNTW AX, BX + TZCNTW (AX), R8 + TZCNTW R15, R14 + TZCNTL AX, BX + TZCNTL (AX), R8 + TZCNTL R15, R14 + TZCNTQ AX, BX + TZCNTQ (AX), R8 + TZCNTQ R15, R14 + RET +` + ll := x86ScalarCountUnboundFormatObject(t, src, "trailingzerocountforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "trailing-zero-count-forms.ll", "trailing-zero-count-forms.o", ll) +} + +func TestTranslate386TrailingZeroCountCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT trailingzerocount386forms(SB),NOSPLIT,$0-0 + TZCNTW AX, BX + TZCNTW (AX), CX + TZCNTL AX, BX + TZCNTL (AX), CX + RET +` + ll := x86ScalarCountUnboundFormatObject(t, src, "trailingzerocount386forms", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "trailing-zero-count-386-forms.ll", "trailing-zero-count-386-forms.o", ll) + + invalid, err := Parse(ArchAMD64, "TEXT invalidtrailingzerocount386(SB),NOSPLIT,$0-0\n\tTZCNTQ AX, BX\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(invalid, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidtrailingzerocount386": {Name: "invalidtrailingzerocount386", Ret: Void}, + }, + }); err == nil { + t.Fatal("Translate accepted TZCNTQ for GOARCH=386") + } +} + +func TestTranslateAMD64TrailingZeroCountRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "TZCNTW AL, BX", + "TZCNTL AX, AH", + "TZCNTQ M1, AX", + "TZCNTW X1, AX", + "TZCNTL AX, (BX)", + "TZCNTQ $1, AX", + "TZCNTW.Z AX, BX", + "TZCNTL AX, BX, CX", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidtrailingzerocount(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidtrailingzerocount": {Name: "invalidtrailingzerocount", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's TZCNT tables", instruction) + } + }) + } +} + +func TestTranslateAMD64FloatingCompareCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yxcmpi table accepts X/m, X0-X15, signed imm8. + // The packed AVX _yvcmppd table adds X/Y vector-result VEX forms and + // X/Y/Z K-result EVEX forms. EVEX memory sources may use .BCST; only + // the Z-register source form may use .SAE. The scalar _yvcmpsd table + // has X vector-result VEX forms and X K-result EVEX forms; only the + // EVEX register-source form may use .SAE. + var src strings.Builder + src.WriteString("TEXT floatingcompareforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"CMPPD", "CMPPS", "CMPSD", "CMPSS"} { + fmt.Fprintf(&src, "\t%s X1, X15, $-128\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15, $127\n", op) + } + for _, op := range []string{"VCMPPD", "VCMPPS"} { + fmt.Fprintf(&src, "\t%s $255, X1, X2, X3\n", op) + fmt.Fprintf(&src, "\t%s $0, (AX), X2, X3\n", op) + fmt.Fprintf(&src, "\t%s $255, Y1, Y2, Y3\n", op) + fmt.Fprintf(&src, "\t%s $0, (AX), Y2, Y3\n", op) + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s $255, %s20, %s21, K0\n", op, width, width) + fmt.Fprintf(&src, "\t%s $0, (AX), %s21, K1, K7\n", op, width) + fmt.Fprintf(&src, "\t%s.BCST $255, (AX), %s21, K7\n", op, width) + } + fmt.Fprintf(&src, "\t%s.SAE $255, Z20, Z21, K1, K7\n", op) + } + for _, op := range []string{"VCMPSD", "VCMPSS"} { + fmt.Fprintf(&src, "\t%s $255, X1, X2, X3\n", op) + fmt.Fprintf(&src, "\t%s $0, (AX), X2, X3\n", op) + fmt.Fprintf(&src, "\t%s $255, X20, X21, K0\n", op) + fmt.Fprintf(&src, "\t%s $0, (AX), X21, K1, K7\n", op) + fmt.Fprintf(&src, "\t%s.SAE $255, X20, X21, K1, K7\n", op) + } + src.WriteString("\tRET\n") + + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "floatingcompareforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "floating-compare-forms.ll", "floating-compare-forms.o", ll) +} + +func TestTranslate386FloatingCompareCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT floatingcompareforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"CMPPD", "CMPPS", "CMPSD", "CMPSS"} { + fmt.Fprintf(&src, "\t%s X1, X7, $-128\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7, $127\n", op) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "floatingcompareforms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "floating-compare-forms-386.ll", "floating-compare-forms-386.o", ll) +} + +func TestTranslateAMD64FloatingCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "CMPPD X1, X2, $-129", + "CMPPS X1, X2, $128", + "CMPSD X16, X2, $0", + "CMPSS X1, X16, $0", + "CMPPD Y1, X2, $0", + "CMPPS.BCST (AX), X2, $0", + "VCMPPD $-1, X1, X2, X3", + "VCMPPS $256, X1, X2, X3", + "VCMPSD $0, Y1, X2, X3", + "VCMPSS $0, X1, Y2, X3", + "VCMPPD $0, X1, X2, Y3", + "VCMPPS $0, Y1, Y2, X3", + "VCMPPD $0, X20, X21, X22", + "VCMPSD $0, X20, X21, X22", + "VCMPSS $0, X1, X2, K0, K1", + "VCMPPD $0, X1, X2, K0, K1", + "VCMPPS.Z $0, Z1, Z2, K1, K2", + "VCMPPD.BCST $0, Z1, Z2, K1", + "VCMPPS.BCST $0, (AX), X2, X3", + "VCMPSD.BCST $0, (AX), X2, K1", + "VCMPSS.BCST $0, (AX), X2, K1", + "VCMPPD.SAE $0, X1, X2, K1", + "VCMPPS.SAE $0, Y1, Y2, K1", + "VCMPPD.SAE $0, (AX), Z2, K1", + "VCMPSD.SAE $0, (AX), X2, K1", + "VCMPSS.SAE $0, X1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidfloatingcompare(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidfloatingcompare": {Name: "invalidfloatingcompare", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's CMP/VCMP floating-point tables", instruction) + } + }) + } +} + +func TestTranslate386FloatingCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "CMPPD X8, X1, $0", + "CMPSD X1, X8, $0", + "VCMPPD $0, X1, X2, X3", + "VCMPSD $0, X1, X2, X3", + } { + src := "TEXT invalidfloatingcompare386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidfloatingcompare386": {Name: "invalidfloatingcompare386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for GOARCH=386", instruction) + } + } +} + +func TestTranslateAMD64PackedSaturatingNarrowCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 names the legacy signed dword-to-word instruction PACKSSLW. + // PACKSSLW/PACKSSWB/PACKUSWB use ymm (MMX/m64 and X/m128 forms), while + // PACKUSDW uses yxm_q4 (X/m128 only). All four V instructions share + // _yvandnpd: X/Y VEX plus X/Y/Z EVEX, optional K1-K7 and .Z. Only the + // dword-to-word forms enable EVEX 32-bit memory broadcast. + var src strings.Builder + src.WriteString("TEXT packedsaturatingnarrowforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PACKSSLW", "PACKSSWB", "PACKUSWB"} { + fmt.Fprintf(&src, "\t%s M1, M7\n", op) + fmt.Fprintf(&src, "\t%s (AX), M7\n", op) + fmt.Fprintf(&src, "\t%s X1, X15\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15\n", op) + } + src.WriteString("\tPACKUSDW X1, X15\n\tPACKUSDW (AX), X15\n") + for _, op := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + fmt.Fprintf(&src, "\t%s X1, X2, X3\n", op) + fmt.Fprintf(&src, "\t%s (AX), X2, X3\n", op) + fmt.Fprintf(&src, "\t%s Y1, Y2, Y3\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y2, Y3\n", op) + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s %s20, %s21, %s22\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s (AX), %s21, K1, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.Z %s20, %s21, K7, %s22\n", op, width, width, width) + } + } + for _, op := range []string{"VPACKSSDW", "VPACKUSDW"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, K1, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), %s21, K7, %s22\n", op, width, width) + } + } + src.WriteString("\tRET\n") + + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedsaturatingnarrowforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-saturating-narrow-forms.ll", "packed-saturating-narrow-forms.o", ll) +} + +func TestTranslate386PackedSaturatingNarrowCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedsaturatingnarrowforms386(SB),4,$0-0\n") + for _, op := range []string{"PACKSSLW", "PACKSSWB", "PACKUSWB"} { + fmt.Fprintf(&src, "\t%s X1, X7\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + } + src.WriteString("\tPACKUSDW X1, X7\n\tPACKUSDW (AX), X7\n") + for _, op := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for _, bank := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s %s1,%s2,%s3\n", op, bank, bank, bank) + fmt.Fprintf(&src, "\t%s (AX),%s2,%s3\n", op, bank, bank) + if op == "VPACKSSDW" || op == "VPACKUSDW" { + fmt.Fprintf(&src, "\t%s.BCST (AX),%s2,%s3\n", op, bank, bank) + } + } + } + src.WriteString("\tRET\n") + goAssemblerOracleOutput(t, "386", src.String(), "", true) + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "packedsaturatingnarrowforms386", "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-saturating-narrow-forms-386.ll", "packed-saturating-narrow-forms-386.o", ll) +} + +func TestTranslateAMD64PackedSaturatingNarrowRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PACKSSLW M1, X2", + "PACKSSWB X1, M2", + "PACKUSDW M1, M2", + "PACKUSWB X16, X1", + "PACKSSLW X1, X16", + "PACKSSWB.Z X1, X2", + "PACKUSWB X1, X2, X3", + "VPACKSSDW X1, X2", + "VPACKSSWB X1, Y2, Y3", + "VPACKUSDW Y1, Y2, X3", + "VPACKUSWB Z1, Z2, Y3", + "VPACKSSDW X1, X2, K0, X3", + "VPACKSSWB.Z X1, X2, X3", + "VPACKUSDW.BCST X1, X2, X3", + "VPACKUSWB.BCST (AX), X2, X3", + "VPACKSSWB.BCST.Z (AX), X2, K1, X3", + "VPACKSSDW.SAE X1, X2, X3", + "VPACKUSWB X1, X2, K1", + "VPACKUSDW X1, (AX), X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedsaturatingnarrow(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedsaturatingnarrow": {Name: "invalidpackedsaturatingnarrow", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PACK/VPACK tables", instruction) + } + }) + } +} + +func TestTranslate386PackedSaturatingNarrowRejectsVectorForms(t *testing.T) { + for _, op := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for _, row := range []string{op + " X1,X2,K1,X3", op + ".Z Z1,Z2,K1,Z3", op + " Z1,Z2,Z8"} { + t.Run(row, func(t *testing.T) { + source := "TEXT invalidpackedsaturatingnarrow386(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{TargetTriple: "i386-unknown-linux-gnu", Goarch: "386", + Sigs: map[string]FuncSig{"invalidpackedsaturatingnarrow386": {Name: "invalidpackedsaturatingnarrow386", Ret: Void}}, + }); err == nil { + t.Fatal("Go-rejected mode32 pack syntax acquired successful lowering") + } + }) + } + } +} + +func TestTranslateAMD64VariableDwordPermuteCompleteGoAssemblerForms(t *testing.T) { + // VPERMD and VPERMPS share Go 1.27's _yvpermd table: Y-only VEX, + // Y/Z EVEX, optional K1-K7 and .Z, plus 32-bit memory broadcast. + var src strings.Builder + src.WriteString("TEXT variabledwordpermuteforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMD", "VPERMPS"} { + fmt.Fprintf(&src, "\t%s Y1, Y2, Y3\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y2, Y3\n", op) + for _, width := range []string{"Y", "Z"} { + fmt.Fprintf(&src, "\t%s %s20, %s21, %s22\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s (AX), %s21, K1, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.Z %s20, %s21, K7, %s22\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), %s21, K7, %s22\n", op, width, width) + } + } + src.WriteString("\tRET\n") + + ll := x86ElementPermuteUnboundFormatObject(t, src.String(), "amd64", "x86_64-unknown-linux-gnu", 24, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "variable-dword-permute-forms.ll", "variable-dword-permute-forms.o", ll) +} + +func TestTranslateAMD64VariableDwordPermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERMD X1, X2, X3", + "VPERMPS Y1, Z2, Z3", + "VPERMD Z1, Z2, Y3", + "VPERMPS Y1, (AX), Y3", + "VPERMD Y1, Y2, K0, Y3", + "VPERMPS.Z Y1, Y2, Y3", + "VPERMD.BCST Y1, Y2, Y3", + "VPERMPS.BCST.Z (AX), Z2, Z3", + "VPERMD.SAE Y1, Y2, Y3", + "VPERMPS Y1, Y2, K1", + "VPERMD Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidvariabledwordpermute(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidvariabledwordpermute": {Name: "invalidvariabledwordpermute", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's VPERMD/VPERMPS table", instruction) + } + }) + } +} + +func TestTranslate386VariableDwordPermuteCompleteGoAssemblerForms(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT variabledwordpermuteforms386(SB),$0-0\n") + for _, op := range []string{"VPERMD", "VPERMPS"} { + fmt.Fprintf(&source, "\t%s Y1, Y2, Y3\n", op) + fmt.Fprintf(&source, "\t%s (AX), Y2, Y3\n", op) + fmt.Fprintf(&source, "\t%s Y31, Y30, Y29\n", op) + fmt.Fprintf(&source, "\t%s Z7, Z6, Z5\n", op) + fmt.Fprintf(&source, "\t%s.BCST (AX), Y30, Y29\n", op) + fmt.Fprintf(&source, "\t%s.BCST (AX), Z6, Z5\n", op) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "linux", triple: "i386-unknown-linux-gnu"}, + {name: "windows", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ll := x86ElementPermuteUnboundFormatObject(t, source.String(), "386", target.triple, 8, 4) + compileLLVMToObject(t, llc, target.triple, "variable-dword-permute-forms-386-"+target.name+".ll", "variable-dword-permute-forms-386-"+target.name+".o", ll) + }) + } +} + +func TestTranslate386VariableDwordPermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERMD Z8, Z0, Z1", + "VPERMPS Z0, Z8, Z1", + "VPERMD Z0, Z1, Z8", + "VPERMD Y1, Y2, K1, Y3", + "VPERMPS.Z Y1, Y2, K1, Y3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's 386 VPERMD/VPERMPS table", instruction) + } + }) + } +} + +func TestTranslateAMD64ConditionalSetCompleteGoAssemblerForms(t *testing.T) { + conditions := []string{"CC", "CS", "EQ", "GE", "GT", "HI", "LE", "LS", "LT", "MI", "NE", "OC", "OS", "PC", "PL", "PS"} + var src strings.Builder + src.WriteString("TEXT conditionalsetforms(SB),NOSPLIT,$0-1\n") + for _, condition := range conditions { + op := "SET" + condition + fmt.Fprintf(&src, "\t%s R15\n", op) + fmt.Fprintf(&src, "\t%s AH\n", op) + fmt.Fprintf(&src, "\t%s BPB\n", op) + fmt.Fprintf(&src, "\t%s R15B\n", op) + fmt.Fprintf(&src, "\t%s (AX)\n", op) + fmt.Fprintf(&src, "\t%s conditionalSetGlobal(SB)\n", op) + fmt.Fprintf(&src, "\t%s ret+0(FP)\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "conditionalsetforms": { + Name: "conditionalsetforms", + Ret: I8, + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: I8, + Index: 0, + Field: -1, + }}}, + }, + }, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ll != "" { + t.Fatal("original format-only SETcc source invented FLAGS or a native address", err) + } + goAssemblerOracleOutput(t, "amd64", "#include \"textflag.h\"\n"+src.String(), "", true) + ll = x86ConditionalSetOriginalEmitterUnit(t, file, "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "conditional-set-forms.ll", "conditional-set-forms.o", ll) +} + +func TestTranslate386ConditionalSetCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT conditionalsetforms386(SB),NOSPLIT,$0-1\n") + for _, condition := range []string{"CC", "CS", "EQ", "GE", "GT", "HI", "LE", "LS", "LT", "MI", "NE", "OC", "OS", "PC", "PL", "PS"} { + op := "SET" + condition + fmt.Fprintf(&src, "\t%s DX\n", op) + fmt.Fprintf(&src, "\t%s AH\n", op) + fmt.Fprintf(&src, "\t%s BP\n", op) + fmt.Fprintf(&src, "\t%s SI\n", op) + fmt.Fprintf(&src, "\t%s DI\n", op) + fmt.Fprintf(&src, "\t%s (AX)\n", op) + fmt.Fprintf(&src, "\t%s ret+0(FP)\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "conditionalsetforms386": { + Name: "conditionalsetforms386", + Ret: I8, + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: I8, + Index: 0, + Field: -1, + }}}, + }, + }, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ll != "" { + t.Fatal("original format-only SETcc source invented FLAGS or a native address", err) + } + goAssemblerOracleOutput(t, "386", "#include \"textflag.h\"\n"+src.String(), "", true) + ll = x86ConditionalSetOriginalEmitterUnit(t, file, "386", "i386-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "conditional-set-forms-386.ll", "conditional-set-forms-386.o", ll) +} + +func TestTranslateAMD64ConditionalSetRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "SETNE X1", + "SETLE M1", + "SETEQ K1", + "SETCC $1", + "SETGT", + "SETGE AX, BX", + "SETHI.Z AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidconditionalset(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidconditionalset": {Name: "invalidconditionalset", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's SETcc tables", instruction) + } + }) + } +} + +func TestTranslate386ConditionalSetRejectsArchitectureRestrictedRegisters(t *testing.T) { + for _, register := range []string{"SP", "R8"} { + file, err := Parse(ArchAMD64, "TEXT invalidconditionalset386(SB),NOSPLIT,$0-0\n\tSETNE "+register+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidconditionalset386": {Name: "invalidconditionalset386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted SETNE %s for GOARCH=386", register) + } + } +} + +func TestTranslateAMD64VectorScalarMoveCompleteGoAssemblerForms(t *testing.T) { + // VMOVSD and VMOVSS share Go 1.27's _yvmovsd table. It contains VEX + // scalar register-to-memory, memory-to-register, and three-register merge + // forms plus their EVEX high-register and optional K-mask counterparts. + // Zeroing is valid only for masked register destinations. + var src strings.Builder + src.WriteString("TEXT vectorscalarmoveforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VMOVSD", "VMOVSS"} { + fmt.Fprintf(&src, "\t%s X1, (AX)\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1\n", op) + fmt.Fprintf(&src, "\t%s X1, X2, X3\n", op) + fmt.Fprintf(&src, "\t%s X20, (AX)\n", op) + fmt.Fprintf(&src, "\t%s X20, K1, (AX)\n", op) + fmt.Fprintf(&src, "\t%s (AX), X20\n", op) + fmt.Fprintf(&src, "\t%s (AX), K1, X20\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K7, X20\n", op) + fmt.Fprintf(&src, "\t%s X20, X21, X22\n", op) + fmt.Fprintf(&src, "\t%s X20, X21, K1, X22\n", op) + fmt.Fprintf(&src, "\t%s.Z X20, X21, K7, X22\n", op) + } + src.WriteString("\tRET\n") + + ll := x86UnboundVoidVectorMoveFormatObject(t, src.String(), "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "vector-scalar-move-forms.ll", "vector-scalar-move-forms.o", ll) +} + +func TestTranslateAMD64VectorScalarMoveFrameSources(t *testing.T) { + src := ` +TEXT vmovsdframe(SB),NOSPLIT,$0-8 + VMOVSD value+0(FP), X0 + RET +TEXT vmovssframe(SB),NOSPLIT,$0-4 + VMOVSS value+0(FP), X0 + RET +TEXT vmovsdframeresult(SB),NOSPLIT,$0-8 + VMOVSD X0, value+0(FP) + RET +TEXT vmovssframeresult(SB),NOSPLIT,$0-4 + VMOVSS X0, value+0(FP) + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "vmovsdframe": { + Name: "vmovsdframe", + Args: []LLVMType{LLVMType("double")}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{ + Offset: 0, + Type: LLVMType("double"), + Index: 0, + Field: -1, + }}}, + }, + "vmovssframe": { + Name: "vmovssframe", + Args: []LLVMType{LLVMType("float")}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{ + Offset: 0, + Type: LLVMType("float"), + Index: 0, + Field: -1, + }}}, + }, + "vmovsdframeresult": { + Name: "vmovsdframeresult", + Ret: LLVMType("double"), + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: LLVMType("double"), + Index: 0, + Field: -1, + }}}, + }, + "vmovssframeresult": { + Name: "vmovssframeresult", + Ret: LLVMType("float"), + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: LLVMType("float"), + Index: 0, + Field: -1, + }}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "vector-scalar-move-frame.ll", "vector-scalar-move-frame.o", ll) +} + +func TestTranslateAMD64VectorScalarMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VMOVSD X1, X2", + "VMOVSS (AX), (BX)", + "VMOVSD Y1, X2, X3", + "VMOVSS X1, Y2, X3", + "VMOVSD X1, X2, Y3", + "VMOVSS X1, X2, (AX)", + "VMOVSD (AX), K0, X1", + "VMOVSS X1, K0, (AX)", + "VMOVSD X1, X2, K0, X3", + "VMOVSS.Z (AX), X1", + "VMOVSD.Z X1, K1, (AX)", + "VMOVSS.Z X1, X2, X3", + "VMOVSD.BCST (AX), X1", + "VMOVSS.SAE X1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidvectorscalarmove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", strings.ReplaceAll(src, "NOSPLIT", "4"), false) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidvectorscalarmove": {Name: "invalidvectorscalarmove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's VMOVSD/VMOVSS table", instruction) + } + }) + } +} + +func TestTranslate386VectorScalarMoveRejectsForms(t *testing.T) { + for _, op := range []string{"VMOVSD", "VMOVSS"} { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),4,$0-0\n"+op+" X0, X1, K1, X2\nRET\n", false) + file, err := Parse(ArchAMD64, "TEXT invalidvectorscalarmove386(SB),NOSPLIT,$0-0\n\t"+op+" X0, X1, K1, X2\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidvectorscalarmove386": {Name: "invalidvectorscalarmove386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %s for GOARCH=386", op) + } + } +} + +func TestTranslateX86VectorScalarFlagCompareCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 gives VCOMISD/VCOMISS/VUCOMISD/VUCOMISS the shared + // _yvcomisd table. It has exactly two rows: VEX X/m,X and EVEX X/m,X. + // EVEX permits X0-X31 and .SAE for register sources; it does not permit + // masks, zeroing, broadcast, or vector widths other than X. Go's 386 + // assembler exposes the same two rows, including the EVEX register class. + var src strings.Builder + src.WriteString("TEXT vectorscalarflagcompareforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VCOMISD", "VCOMISS", "VUCOMISD", "VUCOMISS"} { + fmt.Fprintf(&src, "\t%s X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), X2\n", op) + fmt.Fprintf(&src, "\t%s X20, X2\n", op) + fmt.Fprintf(&src, "\t%s X1, X21\n", op) + fmt.Fprintf(&src, "\t%s (AX), X21\n", op) + fmt.Fprintf(&src, "\t%s.SAE X1, X2\n", op) + fmt.Fprintf(&src, "\t%s.SAE X20, X21\n", op) + } + src.WriteString("\tRET\n") + // Named Go frontend compatibility includes high X registers even on + // 386; raw architectural 32-bit decoding has stricter register limits. + for _, arch := range []string{"386", "amd64"} { + assembleX87ControlBytes(t, arch, strings.ReplaceAll(src.String(), "NOSPLIT", "4")) + } + + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-macosx"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "vectorscalarflagcompareforms": {Name: "vectorscalarflagcompareforms", Ret: Void}, + }, + }) + if target.goarch == "386" { + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "physical mode32 high-X") { + t.Fatalf("source-accepted high banks lack a physical 386 register binding: %v", err) + } + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original named COMI formats invented their missing inputs", err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + ir := x86COMIOriginalFormatObject(t, src.String(), target.goarch, target.triple) + x86COMICompileOriginalFormatObjects(t, llc, target.triple, "vector-scalar-flag-compare-"+target.goarch, ir) + }) + } +} + +func TestTranslateX86VectorScalarFlagCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + for _, instruction := range []string{ + "VCOMISD", + "VCOMISS X1", + "VUCOMISD X1, X2, X3", + "VUCOMISS Y1, X2", + "VCOMISD X1, Y2", + "VCOMISS Z1, X2", + "VUCOMISD K1, X2", + "VUCOMISS X1, (AX)", + "VCOMISD $1, X2", + "VCOMISS.SAE (AX), X2", + "VUCOMISD.BCST (AX), X2", + "VUCOMISS.Z X1, X2", + "VCOMISD X1, K1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidvectorscalarflagcompare(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalidvectorscalarflagcompare": {Name: "invalidvectorscalarflagcompare", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s, which is absent from Go 1.27's _yvcomisd table", instruction, target.goarch) + } + }) + } + }) + } +} + +func TestTranslateAMD64AdditionalPackedIntegerFamilies(t *testing.T) { + src := ` +TEXT packedintegerfamilies(SB),NOSPLIT,$0-0 + PADDB X0, X1 + PCMPEQW X0, X1 + PMULLW X0, X1 + PABSD X0, X1 + PSUBUSB X0, X1 + PTEST X0, X1 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, src, "packedintegerfamilies", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-integer-formats.ll", "packed-integer-formats.o", ll) +} + +func TestTranslateAMD64AdditionalConditionalMoveWidths(t *testing.T) { + src := ` +TEXT conditionalmoves(SB),NOSPLIT,$0-0 + CMPQ AX, BX + CMOVQLE CX, DX + CMOVLGT CX, DX + CMOVWCC CX, DX + RET +` + ir := x86UnboundVoidScalarFormatObject(t, src, "conditionalmoves", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "conditional-move-formats.ll", "conditional-move-formats.o", ir) +} + +func TestTranslateAMD64LegacyScalarAndPackedFloatingPointFamilies(t *testing.T) { + src := ` +TEXT floatfamilies(SB),NOSPLIT,$0-0 + MOVSS (AX), X0 + SQRTSS X0, X1 + ADDSS X0, X1 + SUBSS X0, X1 + MULSS X0, X1 + DIVSS X0, X1 + MINSS X0, X1 + MAXSS X0, X1 + MOVSS X1, (BX) + MOVUPD (AX), X2 + MOVAPD X2, (BX) + MOVUPD X2, 16(BX) + XORPD X2, X3 + ANDPS X2, X3 + ADDPS X2, X3 + SUBPS X2, X3 + MULPS X2, X3 + DIVPS X2, X3 + MINPS X2, X3 + MAXPS X2, X3 + SQRTPS X2, X3 + ADDPD X2, X3 + SUBPD X2, X3 + MULPD X2, X3 + DIVPD X2, X3 + MINPD X2, X3 + MAXPD X2, X3 + SQRTPD X2, X3 + MOVLHPS X2, X3 + MOVHLPS X2, X3 + MOVLPD (AX), X3 + MOVLPD X3, (BX) + MOVHPD (AX), X3 + MOVHPD X3, (BX) + SHUFPD $1, X2, X3 + UNPCKLPS X2, X3 + UNPCKHPD X2, X3 + MOVDDUP X2, X3 + HADDPD X2, X3 + HADDPD (AX), X3 + HADDPS X2, X3 + HADDPS (AX), X3 + CVTSS2SD X2, X3 + CVTSS2SD (AX), X3 + CVTSD2SS X2, X3 + CVTSD2SS (AX), X3 + UCOMISS X2, X3 + UCOMISD X2, X3 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, src, "floatfamilies", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "legacy-floating-formats.ll", "legacy-floating-formats.o", ll) +} + +func TestTranslateAMD64DuplicateMoveFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yxm table accepts X/m -> X for MOVDDUP, + // MOVSHDUP, and MOVSLDUP. Its shared _yvmovddup table accepts + // X/m -> X, Y/m -> Y, and Z/m -> Z for all three V-prefixed forms, + // with optional K masking (and .Z zero masking) on EVEX forms. + src := ` +TEXT duplicatemoveforms(SB),NOSPLIT,$0-8 + MOVDDUP X1, X2 + MOVDDUP (AX), X2 + MOVDDUP source(SB), X2 + MOVDDUP alpha+0(FP), X2 + MOVSHDUP X1, X2 + MOVSHDUP (AX), X2 + MOVSHDUP source(SB), X2 + MOVSLDUP X1, X2 + MOVSLDUP (AX), X2 + MOVSLDUP source(SB), X2 + VMOVDDUP X1, X2 + VMOVDDUP (AX), X2 + VMOVDDUP Y1, Y2 + VMOVDDUP (AX), Y2 + VMOVDDUP X16, X17 + VMOVDDUP X16, K1, X17 + VMOVDDUP.Z (AX), K1, X17 + VMOVDDUP Y16, Y17 + VMOVDDUP Y16, K1, Y17 + VMOVDDUP.Z (AX), K1, Y17 + VMOVDDUP Z1, Z2 + VMOVDDUP Z1, K1, Z2 + VMOVDDUP.Z source(SB), K1, Z2 + VMOVSHDUP X1, X2 + VMOVSHDUP (AX), X2 + VMOVSHDUP Y1, Y2 + VMOVSHDUP (AX), Y2 + VMOVSHDUP X16, X17 + VMOVSHDUP X16, K1, X17 + VMOVSHDUP.Z (AX), K1, X17 + VMOVSHDUP Y16, Y17 + VMOVSHDUP Y16, K1, Y17 + VMOVSHDUP.Z (AX), K1, Y17 + VMOVSHDUP Z1, Z2 + VMOVSHDUP Z1, K1, Z2 + VMOVSHDUP.Z source(SB), K1, Z2 + VMOVSLDUP X1, X2 + VMOVSLDUP (AX), X2 + VMOVSLDUP Y1, Y2 + VMOVSLDUP (AX), Y2 + VMOVSLDUP X16, X17 + VMOVSLDUP X16, K1, X17 + VMOVSLDUP.Z (AX), K1, X17 + VMOVSLDUP Y16, Y17 + VMOVSLDUP Y16, K1, Y17 + VMOVSLDUP.Z (AX), K1, Y17 + VMOVSLDUP Z1, Z2 + VMOVSLDUP Z1, K1, Z2 + VMOVSLDUP.Z source(SB), K1, Z2 + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + signatures := map[string]FuncSig{"duplicatemoveforms": { + Name: "duplicatemoveforms", + Args: []LLVMType{LLVMType("double")}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{ + Offset: 0, + Type: LLVMType("double"), + Index: 0, + Field: -1, + }}}, + }} + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", "#include \"textflag.h\"\n"+src) + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, src, "func duplicatemoveforms(alpha float64)", "amd64", "x86_64-unknown-linux-gnu", api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original format table invented its missing register/address/mask inputs", api, err) + } + } + ir := x86DuplicateMoveFormatObjectUnit(t, file, signatures, "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "duplicate-formats.ll", "duplicate-formats.o", ir) +} + +func TestTranslateAMD64DuplicateMoveFamilyRejectsFormsOutsideGoOptabs(t *testing.T) { + for _, instruction := range []string{ + "MOVSHDUP Y1, Y2", + "MOVSLDUP.Z X1, X2", + "VMOVSHDUP X1, Y2", + "VMOVSLDUP.Z X1, X2", + "VMOVDDUP.Z.Z X1, K1, X2", + "VMOVDDUP X1, K0, X2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidduplicatemove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidduplicatemove": {Name: "invalidduplicatemove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's yxm/_yvmovddup tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedDwordMultiplyFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses yxm_q4 for PMULLD (X/m, X) and _yvandnpd for + // VPMULLD. The latter has VEX X/Y forms plus EVEX X/Y/Z forms with + // optional K1-K7 masking, .Z zero masking, and memory-only .BCST. + src := ` +TEXT packeddwordmultiplyforms(SB),NOSPLIT,$0-0 + PMULLD X1, X2 + PMULLD (AX), X2 + PMULLD source(SB), X2 + VPMULLD X1, X2, X3 + VPMULLD (AX), X2, X3 + VPMULLD Y1, Y2, Y3 + VPMULLD (AX), Y2, Y3 + VPMULLD X16, X17, X18 + VPMULLD X16, X17, K1, X18 + VPMULLD.Z (AX), X17, K1, X18 + VPMULLD Y16, Y17, Y18 + VPMULLD Y16, Y17, K1, Y18 + VPMULLD.Z (AX), Y17, K1, Y18 + VPMULLD Z1, Z2, Z3 + VPMULLD Z1, Z2, K1, Z3 + VPMULLD.Z source(SB), Z2, K1, Z3 + VPMULLD.BCST (AX), X17, X18 + VPMULLD.BCST (AX), Y17, K1, Y18 + VPMULLD.BCST.Z source(SB), Z2, K1, Z3 + RET +` + translateAMD64EcosystemCase(t, src, "packeddwordmultiplyforms") +} + +func TestTranslateAMD64PackedDwordMultiplyFamilyRejectsFormsOutsideGoOptabs(t *testing.T) { + for _, instruction := range []string{ + "PMULLD Y1, Y2", + "VPMULLD X1, Y2, Y3", + "VPMULLD X1, X2, K0, X3", + "VPMULLD.Z X1, X2, X3", + "VPMULLD.BCST X1, X2, K1, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackeddwordmultiply(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackeddwordmultiply": {Name: "invalidpackeddwordmultiply", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PMULLD/VPMULLD tables", instruction) + } + }) + } +} + +func TestTranslateAMD64MXCSRStateFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's ysvrs_mo/ysvrs_om and _yvldmxcsr tables give all four + // instructions exactly one m32 operand. Syntax is not storage authority: + // exercise bounded initialized AUTO, actual typed FP and emitted DATA. + // Native pointers and static stores retain separate negative controls. + for _, target := range x86RawReturnTargets { + for _, row := range x86MXCSRBoundMemoryForms(t, target.arch) { + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+row.name+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, row.source, row.declaration, target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatalf("complete initialized MXCSR storage: %v", err) + } + if row.local && !strings.Contains(ir, "%local_stack = alloca") { + t.Fatal("bounded AUTO operand lost byte-addressable local storage") + } + if !strings.Contains(ir, row.intrinsic) { + t.Fatalf("MXCSR lowering omitted %s", row.intrinsic) + } + }) + } + } + } +} + +func TestTranslateAMD64MXCSRStateFamilyRejectsNonMemoryForms(t *testing.T) { + for _, instruction := range []string{ + "LDMXCSR AX", + "VLDMXCSR $1", + "STMXCSR X1", + "VSTMXCSR.Z (AX)", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidmxcsrstate(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidmxcsrstate": {Name: "invalidmxcsrstate", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, but Go's MXCSR tables require one memory operand", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedFloatToDwordFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's yxcvm1 table gives both instructions X/m -> X and X/m -> M. + // An X destination consumes four float32 lanes; an MMX destination consumes + // two lanes (and therefore only 64 bits for a memory source). + src := ` +TEXT packedfloattodwordforms(SB),NOSPLIT,$0-0 + CVTPS2PL X1, X2 + CVTPS2PL (AX), X2 + CVTPS2PL source(SB), X2 + CVTTPS2PL X1, X2 + CVTTPS2PL (AX), X2 + CVTTPS2PL source(SB), X2 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, src, "packedfloattodwordforms", "amd64", "x86_64-unknown-linux-gnu") + for _, intrinsic := range []string{"asm sideeffect \"cvtps2dq", "asm sideeffect \"cvttps2dq"} { + if !strings.Contains(ll, intrinsic) { + t.Fatalf("packed float-to-dword lowering omitted %s", intrinsic) + } + } +} + +func TestTranslateAMD64PackedFloatToDwordFamilyRejectsFormsOutsideGoOptab(t *testing.T) { + for _, instruction := range []string{ + "CVTPS2PL Y1, X2", + "CVTPS2PL X1, Y2", + "CVTPS2PL $1, X2", + "CVTTPS2PL.Z X1, X2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedfloattodword(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedfloattodword": {Name: "invalidpackedfloattodword", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's yxcvm1 table", instruction) + } + }) + } +} + +func TestTranslateAMD64ConditionalMoveFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 has 48 entries sharing yml_rl: W/L/Q width crossed with all + // 16 x86 condition codes. Yml accepts a same-width GP register or memory; + // Yrl requires a GP register destination. Exercise both source categories + // and both ordinary and extended registers for every instruction. + conditions := []string{"CC", "CS", "EQ", "GE", "GT", "HI", "LE", "LS", "LT", "MI", "NE", "OC", "OS", "PC", "PL", "PS"} + var src strings.Builder + src.WriteString("TEXT conditionalmoveforms(SB),NOSPLIT,$0-8\n") + for _, width := range []string{"W", "L", "Q"} { + for _, condition := range conditions { + op := "CMOV" + width + condition + fmt.Fprintf(&src, "\t%s CX, DX\n", op) + fmt.Fprintf(&src, "\t%s (R10), R11\n", op) + } + } + src.WriteString("\tCMOVQPS source(SB), R10\n") + src.WriteString("\tCMOVQPS input+0(FP), R9\n") + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + options := Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"conditionalmoveforms": { + Name: "conditionalmoveforms", + Args: []LLVMType{I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{ + Offset: 0, Type: I64, Index: 0, Field: -1, + }}}, + }}, + } + // All original register and native memory rows remain format witnesses, + // not initialized public programs. Keep the actual FP input declaration. + original := strings.Replace(src.String(), "TEXT conditionalmoveforms", "TEXT ·conditionalmoveforms", 1) + x86LEAActualTargetGoObject(t, "amd64", options.TargetTriple, "#include \"textflag.h\"\n"+original) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, original, + "func conditionalmoveforms(input uint64)\n", "amd64", options.TargetTriple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized CMOV data/address/predicate became public input", api, err) + } + } + ir := x86ScalarFormatObjectUnit(t, file, options.Sigs, "amd64", options.TargetTriple, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + compileLLVMToObject(t, llc, options.TargetTriple, "cmov-original-formats.ll", "cmov-original-formats.o", ir) +} + +func TestTranslateAMD64ConditionalMoveFamilyRejectsFormsOutsideGoOptab(t *testing.T) { + for _, instruction := range []string{ + "CMOVQPS $1, AX", + "CMOVQPS AX, (BX)", + "CMOVQPS X1, AX", + "CMOVQPS.Z AX, BX", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidconditionalmove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidconditionalmove": {Name: "invalidconditionalmove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, but Go 1.27's yml_rl requires register/memory source and register destination", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedSingleToDoubleFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yxm entry is X/m64 -> X. The VCVTPS2PD + // _yvcvtph2ps table adds X/m64 -> X, X/m128 -> Y, and Y/m256 -> Z, + // with EVEX masks/zeroing, memory broadcast, and SAE on the Z form. + src := ` +TEXT packedsingletodoubleforms(SB),NOSPLIT,$0-0 + CVTPS2PD X1, X2 + CVTPS2PD (AX), X2 + CVTPS2PD source(SB), X2 + VCVTPS2PD X1, X2 + VCVTPS2PD (AX), X2 + VCVTPS2PD X16, X17 + VCVTPS2PD X16, K1, X17 + VCVTPS2PD.Z (AX), K7, X17 + VCVTPS2PD.BCST (AX), X17 + VCVTPS2PD.BCST.Z source(SB), K1, X17 + VCVTPS2PD X1, Y2 + VCVTPS2PD (AX), Y2 + VCVTPS2PD X16, Y17 + VCVTPS2PD X16, K1, Y17 + VCVTPS2PD.Z (AX), K7, Y17 + VCVTPS2PD.BCST (AX), Y17 + VCVTPS2PD.BCST.Z source(SB), K1, Y17 + VCVTPS2PD Y1, Z2 + VCVTPS2PD (AX), Z2 + VCVTPS2PD Y16, Z17 + VCVTPS2PD Y16, K1, Z17 + VCVTPS2PD.Z (AX), K7, Z17 + VCVTPS2PD.BCST (AX), Z17 + VCVTPS2PD.BCST.Z source(SB), K1, Z17 + VCVTPS2PD.SAE Y16, Z17 + VCVTPS2PD.SAE.Z Y16, K1, Z17 + RET +` + // This original complete format inventory has no vector/address/mask + // producers. Keep its actual Go object and all three public Context results; + // private format objects do not replace the initialized 53-row public matrix. + ll := x86PackedPrecisionOriginalFormatObject(t, src, "packedsingletodoubleforms", + "amd64", "x86_64-unknown-linux-gnu", false, 26, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-single-to-double.ll", "packed-single-to-double.o", ll) +} + +func TestTranslateAMD64PackedSingleToDoubleFamilyRejectsFormsOutsideGoOptabs(t *testing.T) { + for _, instruction := range []string{ + "CVTPS2PD Y1, X2", + "CVTPS2PD.Z X1, X2", + "VCVTPS2PD Y1, Y2", + "VCVTPS2PD X1, Z2", + "VCVTPS2PD Z1, Z2", + "VCVTPS2PD X1, K0, X2", + "VCVTPS2PD.Z X1, X2", + "VCVTPS2PD.BCST X1, X2", + "VCVTPS2PD.SAE (AX), Z2", + "VCVTPS2PD.SAE X1, X2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedsingletodouble(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedsingletodouble": {Name: "invalidpackedsingletodouble", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's CVTPS2PD/VCVTPS2PD tables", instruction) + } + }) + } +} + +func TestTranslateAMD64ComplexFrameBitwiseMoveForms(t *testing.T) { + // MOVSD and MOVUPS retain their ordinary yxmov X/m<->X formats when the + // memory operand names a complex64 or complex128 value in the Go FP frame. + src := ` +TEXT complexframemoves(SB),NOSPLIT,$0-48 + MOVSD input64+0(FP), X0 + MOVSD X0, result64+24(FP) + MOVUPS input128+8(FP), X1 + MOVUPS X1, result128+32(FP) + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "complexframemoves": { + Name: "complexframemoves", + Args: []LLVMType{"{ float, float }", "{ double, double }"}, + Ret: LLVMType("{ float, float, double, double }"), + Frame: FrameLayout{ + Params: []FrameSlot{ + {Offset: 0, Type: LLVMType("float"), Index: 0, Field: 0}, + {Offset: 4, Type: LLVMType("float"), Index: 0, Field: 1}, + {Offset: 8, Type: LLVMType("double"), Index: 1, Field: 0}, + {Offset: 16, Type: LLVMType("double"), Index: 1, Field: 1}, + }, + Results: []FrameSlot{ + {Offset: 24, Type: LLVMType("float"), Index: 0, Field: -1}, + {Offset: 28, Type: LLVMType("float"), Index: 1, Field: -1}, + {Offset: 32, Type: LLVMType("double"), Index: 2, Field: -1}, + {Offset: 40, Type: LLVMType("double"), Index: 3, Field: -1}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "complex-frame-moves.ll", "complex-frame-moves.o", ll) +} + +func TestTranslateAMD64WritableFPParameterSlots(t *testing.T) { + // Go's assembler gives named FP parameter slots mutable frame semantics. + // Real assembly such as Gonum's GEMV kernels advances a slice base pointer + // by writing it back to base+off(FP), then reads the updated value later. + src := ` +TEXT writablefpparam(SB),NOSPLIT,$0-16 +body: + MOVQ input+0(FP), AX + ADDQ $8, AX + MOVQ AX, input+0(FP) + MOVQ input+0(FP), BX + MOVQ BX, result+8(FP) + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "writablefpparam": { + Name: "writablefpparam", + Args: []LLVMType{Ptr}, + Ret: Ptr, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: Ptr, Index: 0, Field: -1}}, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "%fp_arg_0 = alloca ptr") { + t.Fatalf("writable FP parameter did not receive mutable shadow storage:\n%s", ll) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "writable-fp-param.ll", "writable-fp-param.o", ll) +} + +func TestTranslateAMD64ConditionalBranchAliasFamilyCompleteGoAssemblerForms(t *testing.T) { + // Keep this list aligned with the x86 aliases installed by + // cmd/asm/internal/arch.archX86 in Go 1.27. Every conditional branch uses + // yjcond: target, $0+target, and $1+target. + aliases := []string{ + "JA", "JAE", "JB", "JBE", "JC", "JCC", "JCS", + "JE", "JEQ", "JG", "JGE", "JGT", "JHI", "JHS", + "JL", "JLE", "JLO", "JLS", "JLT", "JMI", "JNA", + "JNAE", "JNB", "JNBE", "JNC", "JNE", "JNG", "JNGE", + "JNL", "JNLE", "JNO", "JNP", "JNS", "JNZ", "JO", + "JOC", "JOS", "JP", "JPC", "JPE", "JPL", "JPO", + "JPS", "JS", "JZ", + } + var src strings.Builder + src.WriteString("TEXT conditionalbranchaliases(SB),NOSPLIT,$0-0\n") + src.WriteString("\tUCOMISD X0, X1\n") + for i, alias := range aliases { + for hint := -1; hint <= 1; hint++ { + label := fmt.Sprintf("branch_alias_%d_%d", i, hint+1) + if hint < 0 { + fmt.Fprintf(&src, "\t%s %s\n", alias, label) + } else { + fmt.Fprintf(&src, "\t%s $%d, %s\n", alias, hint, label) + } + fmt.Fprintf(&src, "%s:\n", label) + } + } + src.WriteString("\tRET\n") + + // Preserve the complete original Go object and missing-input Context. + // The independent initialized counterpart supplies both COMI inputs. + ll := x86ConditionalBranchAliasOriginalFormatObject(t, src.String(), "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "conditional-branch-alias-formats.ll", "conditional-branch-alias-formats.o", ll) +} + +func TestTranslateX86MSRAccessFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns both RDMSR and WRMSR the operand-free ynone table on + // amd64 and 386. Their inputs/outputs are the architectural CX and DX:AX + // register tuple. + src := ` +TEXT msraccessforms(SB),NOSPLIT,$0-0 + MOVL $0x10, CX + RDMSR + WRMSR + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "msraccessforms": {Name: "msraccessforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`asm sideeffect "rdmsr"`, `asm sideeffect "wrmsr"`} { + if !strings.Contains(ll, want) { + t.Fatalf("MSR access lowering omitted %q:\n%s", want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "msr-access.ll", "msr-access.o", ll) + }) + } +} + +func TestTranslateX86MSRAccessFamilyRejectsFormsOutsideGoOptab(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + for _, instruction := range []string{"RDMSR AX", "WRMSR CX", "RDMSR.P", "WRMSR.Z"} { + name := target.goarch + "_" + strings.NewReplacer(" ", "_", ".", "_").Replace(instruction) + t.Run(name, func(t *testing.T) { + src := "TEXT invalidmsraccess(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalidmsraccess": {Name: "invalidmsraccess", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s, but Go 1.27 uses ynone", instruction, target.goarch) + } + }) + } + } +} + +func TestTranslateX86DescriptorTableFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 groups the system descriptor instructions into these complete + // operand tables: LGDT/LIDT and SGDT/SIDT take memory; LLDT/LTR/LMSW take + // GP-register or memory inputs; SLDT*/SMSW*/STR* write GP registers or + // memory. Q forms exist only on amd64. + common := ` +TEXT descriptorforms(SB),NOSPLIT,$32-0 + LGDT (BX) + LIDT 8(SP) + SGDT (CX) + SIDT 16(SP) + LLDT AX + LLDT (BX) + LTR DI + LTR 8(SP) + LMSW CX + LMSW (DX) + SLDTW AX + SLDTW (BX) + SLDTL DI + SLDTL 8(SP) + SMSWW CX + SMSWW (DX) + SMSWL BP + SMSWL 16(SP) + STRW SI + STRW (DI) + STRL BX + STRL 24(SP) + STRL SP +` + amd64Only := ` + SLDTQ R11 + SLDTQ (R12) + SMSWQ R13 + SMSWQ (R14) + STRQ R15 + STRQ (R10) +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-macosx"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + src := common + if target.goarch == "amd64" { + src += amd64Only + } + src += "\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "descriptorforms": {Name: "descriptorforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `asm sideeffect "lgdt`, `asm sideeffect "lidt`, + `asm sideeffect "sgdt`, `asm sideeffect "sidt`, + `asm sideeffect "lldt`, `asm sideeffect "ltr`, `asm sideeffect "lmsw`, + `asm sideeffect "sldt`, `asm sideeffect "smsw`, `asm sideeffect "str`, + } { + if !strings.Contains(ll, want) { + t.Fatalf("descriptor-table lowering omitted %q:\n%s", want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "descriptor-table.ll", "descriptor-table.o", ll) + }) + } +} + +func TestTranslateX86DescriptorTableFamilyRejectsFormsOutsideGoOptab(t *testing.T) { + tests := []struct { + name string + goarch string + triple string + instruction string + }{ + {name: "lgdt-register", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LGDT AX"}, + {name: "sgdt-register", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SGDT R11"}, + {name: "lidt-immediate", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LIDT $1"}, + {name: "sidt-two-operands", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SIDT (AX), (BX)"}, + {name: "lldt-immediate", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LLDT $1"}, + {name: "ltr-vector", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LTR X0"}, + {name: "lmsw-two-operands", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LMSW AX, BX"}, + {name: "sldtw-immediate", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SLDTW $1"}, + {name: "smswl-vector", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SMSWL X0"}, + {name: "strq-two-operands", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "STRQ AX, BX"}, + {name: "lgdt-suffix", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LGDT.P (AX)"}, + {name: "sldtw-suffix", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SLDTW.Z AX"}, + {name: "386-sldtq", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "SLDTQ AX"}, + {name: "386-smswq", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "SMSWQ AX"}, + {name: "386-strq", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "STRQ AX"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + src := "TEXT invaliddescriptor(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{ + "invaliddescriptor": {Name: "invaliddescriptor", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside the Go 1.27 optab", test.instruction, test.goarch) + } + }) + } +} + +func TestTranslateX86MachineRegisterMoveFamilyCompleteGoAssemblerForms(t *testing.T) { + // This is the complete special-register portion of Go 1.27's ymovtab, + // excluding the independently handled segment and TLS moves. MOVQ special + // forms are accepted in 386 mode too; CR8 is the sole 64-bit-only register. + common := ` +TEXT machineregistermoves(SB),NOSPLIT,$32-0 + MOVL CR0, AX + MOVL BX, CR0 + MOVL CR2, CX + MOVL DX, CR2 + MOVL CR3, BP + MOVL SI, CR3 + MOVL CR4, DI + MOVL AX, CR4 + MOVQ CR0, BX + MOVQ CX, CR0 + MOVQ CR2, DX + MOVQ BP, CR2 + MOVQ CR3, SI + MOVQ DI, CR3 + MOVQ CR4, AX + MOVQ BX, CR4 + MOVL DR0, CX + MOVL DX, DR0 + MOVL DR6, BP + MOVL SI, DR6 + MOVL DR7, DI + MOVL AX, DR7 + MOVQ DR0, BX + MOVQ CX, DR0 + MOVQ DR2, DX + MOVQ BP, DR2 + MOVQ DR3, SI + MOVQ DI, DR3 + MOVQ DR6, AX + MOVQ BX, DR6 + MOVQ DR7, CX + MOVQ DX, DR7 + MOVL TR6, AX + MOVL TR6, 0(SP) + MOVL BX, TR6 + MOVL 4(SP), TR6 + MOVL TR7, CX + MOVL TR7, 8(SP) + MOVL DX, TR7 + MOVL 12(SP), TR7 + MOVL 0(SP), GDTR + MOVL GDTR, 16(SP) + MOVL 0(SP), IDTR + MOVL IDTR, 16(SP) + MOVQ 0(SP), GDTR + MOVQ GDTR, 16(SP) + MOVQ 0(SP), IDTR + MOVQ IDTR, 16(SP) + MOVW AX, LDTR + MOVW 0(SP), LDTR + MOVW LDTR, BX + MOVW LDTR, 0(SP) + MOVW CX, MSW + MOVW 4(SP), MSW + MOVW MSW, DX + MOVW MSW, 4(SP) + MOVW BP, TASK + MOVW 8(SP), TASK + MOVW TASK, SI + MOVW TASK, 8(SP) + MOVL CR0, SP + MOVQ CR0, SP +` + amd64Only := ` + MOVL CR8, R11 + MOVL R12, CR8 + MOVQ CR8, R13 + MOVQ R14, CR8 +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-macosx"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + src := common + if target.goarch == "amd64" { + src += amd64Only + } + src += "\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "machineregistermoves": {Name: "machineregistermoves", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"%cr0", "%dr0", ".byte 0x0f, 0x24", ".byte 0x0f, 0x26"} { + if !strings.Contains(ll, want) { + t.Fatalf("machine-register lowering omitted %q:\n%s", want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "machine-register.ll", "machine-register.o", ll) + }) + } +} + +func TestTranslateX86MachineRegisterMoveFamilyRejectsFormsOutsideGoMovtab(t *testing.T) { + tests := []struct { + name string + goarch string + triple string + instruction string + }{ + {name: "cr1", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ CR1, AX"}, + {name: "cr5", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVL AX, CR5"}, + {name: "cr-memory", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ CR0, (AX)"}, + {name: "386-cr8-l", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVL CR8, AX"}, + {name: "386-cr8-q", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVQ AX, CR8"}, + {name: "movl-dr2", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVL DR2, AX"}, + {name: "movq-dr1", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ AX, DR1"}, + {name: "dr-memory", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ (AX), DR0"}, + {name: "tr5", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVL TR5, AX"}, + {name: "tr-q", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ TR6, AX"}, + {name: "gdtr-register", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ AX, GDTR"}, + {name: "gdtr-width", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVW (AX), GDTR"}, + {name: "ldtr-width", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVL AX, LDTR"}, + {name: "machine-suffix", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ.P CR0, AX"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + src := "TEXT invalidmachineregistermove(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{ + "invalidmachineregistermove": {Name: "invalidmachineregistermove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's ymovtab", test.instruction, test.goarch) + } + }) + } +} + +func TestTranslateX86ScalarExtensionMoveCompleteGo127Forms(t *testing.T) { + amd64Ops := []string{ + "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", "MOVBQSX", "MOVBQZX", + "MOVWLSX", "MOVWLZX", "MOVWQSX", "MOVWQZX", "MOVLQSX", "MOVLQZX", + "MOVSWW", "MOVZWW", + } + ops386 := []string{ + "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", + "MOVWLSX", "MOVWLZX", "MOVLQZX", "MOVSWW", "MOVZWW", + } + for _, target := range []struct { + name string + goarch string + triple string + ops []string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", ops: amd64Ops}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", ops: amd64Ops}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", ops: amd64Ops}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", ops: ops386}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", ops: ops386}, + } { + t.Run(target.name, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT scalar_extension_forms(SB),NOSPLIT,$0-1\n") + for _, op := range target.ops { + source := "AX" + if strings.HasPrefix(op, "MOVB") { + source = "AL" + if target.goarch == "386" { + source = "BP" // Go rewrites this Yrl32 byte source through BX. + } + } + fmt.Fprintf(&src, "\t%s %s, CX\n", op, source) + fmt.Fprintf(&src, "\t%s 8(BX), DX\n", op) + } + src.WriteString("\tMOVBWZX x+0(FP), SP\n\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + // Keep the original arbitrary BX reads and physical SP write as + // Go/emitter formats. Neither syntax grants a source/frame role. + x86LEAActualTargetGoObject(t, target.goarch, target.triple, "#include \"textflag.h\"\n"+src.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, src.String(), "func scalar_extension_forms(x uint8)", target.goarch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original extension formats acquired unbound source/physical SP inputs", api, err) + } + } + ll := x86ScalarFormatObjectUnit(t, file, map[string]FuncSig{ + "scalar_extension_forms": { + Name: "scalar_extension_forms", Args: []LLVMType{I8}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I8, Index: 0, Field: -1}}}, + }, + }, target.goarch, target.triple, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ll, level) + } + word, move, typ := 8, "MOVQ", "uint64" + if target.goarch == "386" { + word, move, typ = 4, "MOVL", "uint32" + } + var initialized, declarations, scratch, scratchDeclarations strings.Builder + for index, op := range target.ops { + register := "AX" + if strings.HasPrefix(op, "MOVB") { + register = "AL" + if target.goarch == "386" { + register = "BP" + } + } + for _, source := range []string{register, "8(BX)"} { + name := fmt.Sprintf("Extension%d", initialized.Len()) + fmt.Fprintf(&declarations, "func %s(x %s,p *[16]byte) %s\n", name, typ, typ) + fmt.Fprintf(&initialized, "TEXT ·%s(SB),4,$0-%d\n%s x+0(FP),AX\n%s p+%d(FP),BX\n%s $0,CX\n", name, 3*word, move, move, word, move) + if register == "BP" { + initialized.WriteString("MOVL AX,BP\n") + } + fmt.Fprintf(&initialized, "%s %s,CX\n%s CX,ret+%d(FP)\nRET\n", op, source, move, 2*word) + } + fmt.Fprintf(&scratchDeclarations, "func Scratch%d()\n", index) + fmt.Fprintf(&scratch, "TEXT ·Scratch%d(SB),4,$0-0\n%s %s,CX\nRET\n", index, op, register) + } + for _, fixture := range []struct{ name, source, declarations string }{ + {"initialized", initialized.String(), declarations.String()}, + {"dead-register", scratch.String(), scratchDeclarations.String()}, + } { + x86LEAActualTargetGoObject(t, target.goarch, target.triple, fixture.source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, fixture.source, fixture.declarations, target.goarch, target.triple, api) + if err != nil { + t.Fatal("complete extension counterpart", fixture.name, api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + } + } + }) + } +} + +func TestTranslateX86ScalarExtensionMoveRejectsFormsOutsideGo127Tables(t *testing.T) { + tests := []struct { + name string + goarch string + triple string + instruction string + }{ + {name: "missing-source", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBWZX AX"}, + {name: "too-many", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBLZX AL, AX, BX"}, + {name: "immediate-source", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBQZX $1, AX"}, + {name: "vector-source", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBWZX X1, AX"}, + {name: "byte-source-for-word-class", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVWLZX AL, AX"}, + {name: "memory-destination", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBLSX AL, (AX)"}, + {name: "byte-destination", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVWQZX AX, AL"}, + {name: "suffix", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVLQZX.Z AX, BX"}, + {name: "386-byte-to-qword-sign", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVBQSX AL, AX"}, + {name: "386-byte-to-qword-zero", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVBQZX AL, AX"}, + {name: "386-word-to-qword-sign", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVWQSX AX, BX"}, + {name: "386-word-to-qword-zero", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVWQZX AX, BX"}, + {name: "386-long-to-qword-sign", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVLQSX AX, BX"}, + {name: "386-r8-source", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVWLZX R8, AX"}, + {name: "386-r8-destination", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVBLZX AL, R8"}, + {name: "386-sp-byte-source", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVBWZX SP, AX"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + src := "TEXT invalid_scalar_extension(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{ + "invalid_scalar_extension": {Name: "invalid_scalar_extension", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's ymb_rl/yml_rl tables", test.instruction, test.goarch) + } + }) + } +} + +func TestTranslateX86MaskLogicalCompleteGo127Forms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT mask_logical_forms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"KAND", "KANDN", "KOR", "KXNOR", "KXOR"} { + for _, width := range []string{"B", "W", "D", "Q"} { + fmt.Fprintf(&src, "\t%s%s K1, K2, K3\n", stem, width) + } + } + for _, stem := range []string{"KNOT", "KTEST", "KORTEST"} { + for _, width := range []string{"B", "W", "D", "Q"} { + fmt.Fprintf(&src, "\t%s%s K1, K2\n", stem, width) + } + } + src.WriteString("\tRET\n") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + // Preserve the original uninitialized format inventory. Complete + // initialized programs have independent public API/runtime owners. + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "mask_logical_forms", target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "mask-logical-"+target.name+".ll", "mask-logical-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86MaskLogicalRejectsFormsOutsideGo127Tables(t *testing.T) { + for _, instruction := range []string{ + "KXNORW K1, K2", + "KXNORW K1, K2, K3, K4", + "KANDW K1, K2, AX", + "KANDNW K1, (AX), K3", + "KORQ $1, K2, K3", + "KXORB K1, K2, K8", + "KNOTD K1", + "KNOTQ K1, K2, K3", + "KNOTW AX, K2", + "KTESTB K1, (AX)", + "KTESTW K1, K2, K3", + "KORTESTD K1", + "KORTESTQ.Z K1, K2", + "KXORQ.Z K1, K2, K3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalid_mask_logical(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalid_mask_logical": {Name: "invalid_mask_logical", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's _ykaddb/_yknotb tables", instruction, target.goarch) + } + } + }) + } +} + +func TestTranslateX86MaskMoveCompleteGo127Forms(t *testing.T) { + // Go 1.27 assigns KMOVB/W/D/Q the same _ykmovb table. Its four rows are: + // K -> memory, K -> GP, K-or-memory -> K, and GP -> K. Ykm therefore also + // includes the K -> K form. Exercise every row plus each Plan 9 memory + // spelling accepted as Ym, on both x86 architectures and all CI object + // formats. + var src strings.Builder + src.WriteString("TEXT mask_move_forms(SB),NOSPLIT,$0-16\n") + for _, op := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + fmt.Fprintf(&src, "\t%s K1, 16(BX)\n", op) + fmt.Fprintf(&src, "\t%s K1, AX\n", op) + fmt.Fprintf(&src, "\t%s K1, K2\n", op) + fmt.Fprintf(&src, "\t%s K0, K7\n", op) + fmt.Fprintf(&src, "\t%s 16(BX), K2\n", op) + fmt.Fprintf(&src, "\t%s AX, K2\n", op) + fmt.Fprintf(&src, "\t%s K5, SP\n", op) + fmt.Fprintf(&src, "\t%s SP, K5\n", op) + fmt.Fprintf(&src, "\t%s x+0(FP), K3\n", op) + fmt.Fprintf(&src, "\t%s K3, ret+8(FP)\n", op) + fmt.Fprintf(&src, "\t%s mask_move_source(SB), K4\n", op) + fmt.Fprintf(&src, "\t%s K4, mask_move_destination(SB)\n", op) + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + targetSource := src.String() + if target.goarch == "amd64" { + var highRegisters strings.Builder + for _, op := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + fmt.Fprintf(&highRegisters, "\t%s K6, R15\n", op) + fmt.Fprintf(&highRegisters, "\t%s R15, K6\n", op) + } + targetSource += highRegisters.String() + } + targetSource += "\tRET\n" + file, err := Parse(ArchAMD64, targetSource) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "mask_move_forms": { + Name: "mask_move_forms", Args: []LLVMType{I64}, Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }, + }, + }, + }) + if ll != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("format-only KMOV source invented initialized masks or native storage", err) + } + goAssemblerOracleOutput(t, target.goarch, "#include \"textflag.h\"\n"+targetSource, "", true) + for _, instruction := range file.Funcs[0].Instrs[1 : len(file.Funcs[0].Instrs)-1] { + if !amd64MaskMoveOperandsValid(instruction, target.goarch) { + t.Fatal("complete original Go row lost its typed grammar", instruction.Raw) + } + } + t.Log("original complete Go forms retain Context; initialized GP, FP, native and caller counterparts own LLVM objects") + }) + } +} + +func TestTranslate386MaskMoveRejectsAMD64OnlyYrlRegisters(t *testing.T) { + for _, instruction := range []string{"KMOVB K1, R8", "KMOVW R8, K1", "KMOVD K1, R15", "KMOVQ R15, K1"} { + src := "TEXT invalid_mask_move_386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalid_mask_move_386": {Name: "invalid_mask_move_386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for 386 outside Go 1.27's Yrl class", instruction) + } + } +} + +func TestTranslateX86MaskMoveRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "KMOVB K1", + "KMOVW K1, K2, K3", + "KMOVD $1, K1", + "KMOVQ K1, $1", + "KMOVB 8(AX), BX", + "KMOVW AX, 8(BX)", + "KMOVD AX, BX", + "KMOVQ 8(AX), 16(BX)", + "KMOVB X1, K2", + "KMOVW K1, X2", + "KMOVD K8, AX", + "KMOVQ AX, K8", + "KMOVQ.Z K1, K2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalid_mask_move(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalid_mask_move": {Name: "invalid_mask_move", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's _ykmovb table", instruction, target.goarch) + } + } + }) + } +} + +func TestTranslateX86GatherCompleteGo127Forms(t *testing.T) { + // Go 1.27 has eight data-gather instructions over three operand tables, + // plus eight EVEX gather-prefetch instructions over two tables. Generate + // every row of all five tables for each opcode. The 386 assembler accepts + // EVEX gather with X/Y/Z0..7 VSIB indices; amd64 additionally accepts the + // high EVEX register classes. + dataGroups := []struct { + ops []string + table string + }{ + {ops: []string{"VGATHERDPS", "VGATHERQPD", "VPGATHERDD", "VPGATHERQQ"}, table: "dps"}, + {ops: []string{"VGATHERDPD", "VPGATHERDQ"}, table: "dpd"}, + {ops: []string{"VGATHERQPS", "VPGATHERQD"}, table: "qps"}, + } + prefetchGroups := []struct { + ops []string + indexWidth string + }{ + {ops: []string{"VGATHERPF0DPD", "VGATHERPF1DPD"}, indexWidth: "Y"}, + {ops: []string{ + "VGATHERPF0DPS", "VGATHERPF0QPD", "VGATHERPF0QPS", + "VGATHERPF1DPS", "VGATHERPF1QPD", "VGATHERPF1QPS", + }, indexWidth: "Z"}, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT gather_forms(SB),NOSPLIT,$0-0\n") + for _, group := range dataGroups { + for _, op := range group.ops { + switch group.table { + case "dps": + fmt.Fprintf(&src, "\t%s X1, (BX)(X2*1), X3\n", op) + fmt.Fprintf(&src, "\t%s Y1, 4(BX)(Y2*2), Y3\n", op) + fmt.Fprintf(&src, "\t%s 8(X2*4), K1, X3\n", op) + fmt.Fprintf(&src, "\t%s 12(BX)(Y2*8), K1, Y3\n", op) + fmt.Fprintf(&src, "\t%s 16(BX)(Z2*4), K1, Z3\n", op) + case "dpd": + fmt.Fprintf(&src, "\t%s X1, (BX)(X2*1), X3\n", op) + fmt.Fprintf(&src, "\t%s Y1, 8(BX)(X2*2), Y3\n", op) + fmt.Fprintf(&src, "\t%s 16(X2*4), K1, X3\n", op) + fmt.Fprintf(&src, "\t%s 24(BX)(X2*8), K1, Y3\n", op) + fmt.Fprintf(&src, "\t%s 32(BX)(Y2*4), K1, Z3\n", op) + case "qps": + fmt.Fprintf(&src, "\t%s X1, (BX)(X2*1), X3\n", op) + fmt.Fprintf(&src, "\t%s X1, 4(BX)(Y2*2), X3\n", op) + fmt.Fprintf(&src, "\t%s 8(X2*4), K1, X3\n", op) + fmt.Fprintf(&src, "\t%s 12(BX)(Y2*8), K1, X3\n", op) + fmt.Fprintf(&src, "\t%s 16(BX)(Z2*4), K1, Y3\n", op) + } + // Unlike VSIB indices, Go 1.27 permits high EVEX X/Y + // destination registers even in 386 mode. + switch group.table { + case "dps": + fmt.Fprintf(&src, "\t%s (BX)(X2*4), K2, X20\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Y2*4), K2, Y20\n", op) + case "dpd": + fmt.Fprintf(&src, "\t%s (BX)(X2*4), K2, X20\n", op) + fmt.Fprintf(&src, "\t%s (BX)(X2*4), K2, Y20\n", op) + case "qps": + fmt.Fprintf(&src, "\t%s (BX)(X2*8), K2, X20\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Z2*8), K2, Y20\n", op) + } + if target.goarch == "amd64" { + switch group.table { + case "dps": + fmt.Fprintf(&src, "\t%s (BX)(X20*4), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Y20*4), K2, Y21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Z20*4), K2, Z21\n", op) + case "dpd": + fmt.Fprintf(&src, "\t%s (BX)(X20*4), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(X20*4), K2, Y21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Y20*4), K2, Z21\n", op) + case "qps": + fmt.Fprintf(&src, "\t%s (BX)(X20*8), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Y20*8), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Z20*8), K2, Y21\n", op) + } + } + } + } + for _, group := range prefetchGroups { + for _, op := range group.ops { + fmt.Fprintf(&src, "\t%s K1, 40(BX)(%s2*4)\n", op, group.indexWidth) + if target.goarch == "amd64" { + fmt.Fprintf(&src, "\t%s K7, 48(%s20*8)\n", op, group.indexWidth) + } + } + } + src.WriteString("\tRET\n") + + // Original operand formats have neither incoming values nor a + // pointer base. Preserve Go bytes and objects with public Context; + // the separate official-row owner supplies real initialized inputs. + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "gather_forms", target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "gather-"+target.name+".ll", "gather-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86GatherRejectsFormsOutsideGo127Tables(t *testing.T) { + tests := []struct { + name string + instruction string + }{ + {name: "missing-mask", instruction: "VPGATHERDD (AX)(X2*4), X3"}, + {name: "too-many", instruction: "VGATHERDPS (AX)(X2*4), K1, K2, X3"}, + {name: "plain-memory", instruction: "VPGATHERDD (AX), K1, X3"}, + {name: "gp-index", instruction: "VPGATHERDD (AX)(CX*4), K1, X3"}, + {name: "bad-scale", instruction: "VPGATHERDD (AX)(X2*3), K1, X3"}, + {name: "k0-mask", instruction: "VPGATHERDD (AX)(X2*4), K0, X3"}, + {name: "vector-mask-in-evex", instruction: "VPGATHERDD (AX)(X2*4), X1, X3"}, + {name: "k-mask-in-vex", instruction: "VPGATHERDD K1, (AX)(X2*4), X3"}, + {name: "dps-wrong-index", instruction: "VPGATHERDD (AX)(Y2*4), K1, X3"}, + {name: "dps-wrong-destination", instruction: "VPGATHERDD (AX)(X2*4), K1, Y3"}, + {name: "dpd-wrong-index", instruction: "VPGATHERDQ (AX)(Y2*4), K1, Y3"}, + {name: "dpd-wrong-destination", instruction: "VPGATHERDQ (AX)(Y2*4), K1, X3"}, + {name: "qps-wrong-index", instruction: "VPGATHERQD (AX)(Y2*8), K1, Y3"}, + {name: "qps-wrong-destination", instruction: "VPGATHERQD (AX)(Z2*8), K1, Z3"}, + {name: "vex-mask-index-alias", instruction: "VPGATHERDD X2, (AX)(X2*4), X3"}, + {name: "vex-mask-dest-alias", instruction: "VPGATHERDD X3, (AX)(X2*4), X3"}, + {name: "vex-index-dest-alias", instruction: "VPGATHERDD X1, (AX)(X3*4), X3"}, + {name: "evex-index-dest-alias", instruction: "VPGATHERDD (AX)(X3*4), K1, X3"}, + {name: "zero-suffix", instruction: "VPGATHERDD.Z (AX)(X2*4), K1, X3"}, + {name: "broadcast-suffix", instruction: "VPGATHERQQ.BCST (AX)(X2*8), K1, X3"}, + {name: "prefetch-k0", instruction: "VGATHERPF0DPD K0, (AX)(Y2*8)"}, + {name: "prefetch-wrong-index", instruction: "VGATHERPF0DPD K1, (AX)(Z2*8)"}, + {name: "prefetch-extra-dest", instruction: "VGATHERPF1DPS K1, (AX)(Z2*4), Z3"}, + {name: "prefetch-suffix", instruction: "VGATHERPF1QPS.Z K1, (AX)(Z2*8)"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + src := "TEXT invalid_gather(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalid_gather": {Name: "invalid_gather", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's gather tables", test.instruction, target.goarch) + } + } + }) + } +} + +func TestTranslate386GatherRejectsHighVSIBAndZRegisters(t *testing.T) { + for _, instruction := range []string{ + "VPGATHERDD (AX)(X8*4), K1, X3", + "VPGATHERDQ (AX)(Y8*4), K1, Z3", + "VPGATHERQD (AX)(Z8*8), K1, Y3", + "VPGATHERDD (AX)(Z2*4), K1, Z8", + "VGATHERPF0DPD K1, (AX)(Y8*8)", + "VGATHERPF1DPS K1, (AX)(Z8*4)", + } { + src := "TEXT invalid_gather_386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalid_gather_386": {Name: "invalid_gather_386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's 386 VSIB/register classes", instruction) + } + } +} + +func TestTranslateAMD64TernaryLogicCompleteGo127Forms(t *testing.T) { + // VPTERNLOGD/Q share Go 1.27's six-row _yvalignd table: X/Y/Z, + // each with an unmasked and K1-K7 masked row. Their EVEX encodings add + // scalar D/Q broadcast and zeroing. Exercise every row/suffix combination, + // high EVEX registers, and every value in the unsigned-imm8 truth table. + var src strings.Builder + src.WriteString("TEXT ternary_logic_forms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPTERNLOGD", "VPTERNLOGQ"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s $0, %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s $255, 8(BX), %s3, %s4\n", op, width, width) + fmt.Fprintf(&src, "\t%s $0xd8, %s5, %s6, K1, %s7\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s.Z $0xaa, 16(BX), %s8, K2, %s9\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST $0xcc, 24(BX), %s10, %s11\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z $0xf0, 32(BX), %s12, K3, %s13\n", op, width, width) + } + fmt.Fprintf(&src, "\t%s $0x96, X29, X30, X31\n", op) + fmt.Fprintf(&src, "\t%s $0x96, Y29, Y30, Y31\n", op) + fmt.Fprintf(&src, "\t%s $0x96, Z29, Z30, Z31\n", op) + } + for immediate := 0; immediate <= 255; immediate++ { + fmt.Fprintf(&src, "\tVPTERNLOGD $%d, X0, X1, X2\n", immediate) + fmt.Fprintf(&src, "\tVPTERNLOGQ $%d, X3, X4, X5\n", immediate) + } + src.WriteString("\tRET\n") + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + // Keep all554 original rows, including high banks and every + // selector. Missing BX/K1-K3 inputs make this an original Go + // and emitter format object, not a public initialized program. + file, signatures := x86UnboundVoidFormatSources(t, src.String(), "amd64", target.triple) + ll := x86ScalarFormatObjectUnit(t, file, signatures, "amd64", target.triple, 0) + compileLLVMToObject(t, llc, target.triple, "ternary-logic-"+target.name+".ll", "ternary-logic-"+target.name+".o", ll) + }) + } +} + +func TestAMD64TernaryLogicTruthTableOrderingExhaustive(t *testing.T) { + for immediate := 0; immediate <= 255; immediate++ { + for truthIndex := 0; truthIndex < 8; truthIndex++ { + a := truthIndex&4 != 0 + b := truthIndex&2 != 0 + third := truthIndex&1 != 0 + want := immediate&(1<= 8 + } + if target.goarch != "386" || !highRegister || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original arithmetic format became malformed", instruction.Raw, err) + } + contextRows++ + continue // Only the unbound physical bank lacks an emitter object. + } + local.Funcs[0].Instrs = append(local.Funcs[0].Instrs, instruction) + } + if target.goarch == "386" && contextRows != 44 || target.goarch == "amd64" && contextRows != 0 { + t.Fatal("original arithmetic physical-bank partition changed", target.goarch, contextRows) + } + ll := x86ScalarFormatObjectUnit(t, &local, signatures, target.goarch, target.triple, 0) + compileLLVMToObject(t, llc, target.triple, "binary-floating-complete-"+target.name+".ll", "binary-floating-complete-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86BinaryFloatingRejectsFormsOutsideGo127Tables(t *testing.T) { + reject := func(t *testing.T, instruction string) { + t.Helper() + src := "TEXT invalid_binary_floating(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalid_binary_floating": {Name: "invalid_binary_floating", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvaddpd/_yvaddsd tables", instruction) + } + } + for _, op := range go127BinaryFloatingOps(false) { + forms := []string{ + "X0, X1", + "X0, X1, X2, X3, X4", + "X0, Y1, Y2", + "X0, 8(BX), X2", + "X0, X1, K0, X2", + "X0, X1, AX", + ".Z X0, X1, X2", + ".BCST X0, X1, X2", + ".BCST.Z 8(BX), X1, X2", + ".Z.BCST 8(BX), X1, K1, X2", + } + if go127BinaryFloatingUsesSAE(op) { + forms = append(forms, + ".RN_SAE Z0, Z1, Z2", + ".SAE X0, X1, X2", + ".SAE 8(BX), Z1, Z2", + ".BCST.SAE 8(BX), Z1, Z2", + ) + } else { + forms = append(forms, + ".SAE Z0, Z1, Z2", + ".RN_SAE X0, X1, X2", + ".RN_SAE 8(BX), Z1, Z2", + ".BCST.RN_SAE 8(BX), Z1, Z2", + ) + } + for _, operands := range forms { + instruction := op + " " + operands + if strings.HasPrefix(operands, ".") { + parts := strings.SplitN(operands, " ", 2) + instruction = op + parts[0] + " " + parts[1] + } + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + reject(t, instruction) + }) + } + } + for _, op := range go127BinaryFloatingOps(true) { + forms := []string{ + "X0, X1", + "X0, X1, X2, X3, X4", + "Y0, X1, X2", + "X0, 8(BX), X2", + "X0, X1, K0, X2", + ".Z X0, X1, X2", + ".BCST 8(BX), X1, X2", + } + if go127BinaryFloatingUsesSAE(op) { + forms = append(forms, ".RN_SAE X0, X1, X2", ".SAE 8(BX), X1, X2") + } else { + forms = append(forms, ".SAE X0, X1, X2", ".RN_SAE 8(BX), X1, X2") + } + for _, operands := range forms { + instruction := op + " " + operands + if strings.HasPrefix(operands, ".") { + parts := strings.SplitN(operands, " ", 2) + instruction = op + parts[0] + " " + parts[1] + } + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + reject(t, instruction) + }) + } + } +} + +func TestTranslate386BinaryFloatingRejectsArchitectureRestrictedForms(t *testing.T) { + var instructions []string + for _, op := range append(go127BinaryFloatingOps(false), go127BinaryFloatingOps(true)...) { + if strings.HasSuffix(op, "PS") || strings.HasSuffix(op, "PD") { + instructions = append(instructions, op+" X0, X1, K1, X2", op+" Z6, Z7, Z8", op+" Z8, Z7, Z6") + } else { + instructions = append(instructions, op+" X0, X1, K1, X2") + } + } + for _, instruction := range instructions { + src := "TEXT invalid_binary_floating_386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + continue + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalid_binary_floating_386": {Name: "invalid_binary_floating_386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's 386 register/operand limits", instruction) + } + } +} + +func TestTranslateX86HorizontalFloatingCompleteGo127Forms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + second, destination := 14, 15 + if target.goarch == "386" { + second, destination = 6, 7 + } + var src strings.Builder + src.WriteString("TEXT horizontal_floating_complete_forms(SB),4,$0-0\n") + for _, operation := range []string{"HADD", "HSUB"} { + for _, element := range []string{"PS", "PD"} { + op := operation + element + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s 8(BX), X%d\n", op, destination) + vop := "V" + op + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", vop) + fmt.Fprintf(&src, "\t%s 16(BX), X%d, X%d\n", vop, second, destination) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", vop) + fmt.Fprintf(&src, "\t%s 32(BX), Y%d, Y%d\n", vop, second, destination) + } + } + src.WriteString("\tRET\n") + goAssemblerOracleOutput(t, target.goarch, src.String(), "", true) + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "horizontal_floating_complete_forms", target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "horizontal-floating-complete-"+target.name+".ll", "horizontal-floating-complete-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86HorizontalFloatingRejectsFormsOutsideGo127Tables(t *testing.T) { + for _, op := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD", "VHADDPS", "VHADDPD", "VHSUBPS", "VHSUBPD"} { + forms := []string{"X0", "X0, X1, X2, X3", "Y0, X1", "X16, X1"} + validOperands := "X0, X1" + if strings.HasPrefix(op, "V") { + forms = []string{ + "X0, X1", "X0, X1, X2, X3", "X0, Y1, Y2", "X16, X1, X2", + "X0, X1, X16", "Z0, Z1, Z2", "X0, X1, K1, X2", + } + validOperands = "X0, X1, X2" + } + var instructions []string + for _, operands := range forms { + instructions = append(instructions, op+" "+operands) + } + for _, suffix := range []string{".Z", ".BCST", ".SAE", ".RN_SAE"} { + instructions = append(instructions, op+suffix+" "+validOperands) + } + for _, instruction := range instructions { + t.Run(strings.NewReplacer(" ", "_", ".", "_", ",", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalid_horizontal_floating(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalid_horizontal_floating": {Name: "invalid_horizontal_floating", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yxm/_yvaddsubpd tables", instruction) + } + }) + } + } +} + +func translateAMD64EcosystemCase(t *testing.T, src, symbol string) string { + t.Helper() + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{symbol: {Name: symbol, Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + return ll +} + +func translateX86FrameCase(t *testing.T, src, goarch, triple, symbol string) { + t.Helper() + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{ + Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + }, + Results: []FrameSlot{ + {Offset: 16, Type: I64, Index: 0, Field: -1}, + {Offset: 24, Type: I64, Index: 1, Field: -1}, + }, + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{symbol: { + Name: symbol, + Args: []LLVMType{I64, I64}, + Ret: LLVMType("{ i64, i64 }"), + Frame: frame, + }}, + }); err != nil { + t.Fatal(err) + } +} diff --git a/amd64_encoding_writeback_arm64_test.go b/amd64_encoding_writeback_arm64_test.go new file mode 100644 index 000000000..ae7eecdf0 --- /dev/null +++ b/amd64_encoding_writeback_arm64_test.go @@ -0,0 +1,10 @@ +package plan9asm + +import "testing" + +// These source instructions use their actual portable ARM64 backend paths. +// Native x86-only reciprocal/ER and MXCSR-reading paths remain in the object +// inventory; no intrinsic or status computation is stripped for this host. +func TestX86EncodingWritebackPortableARM64(t *testing.T) { + x86EncodingWritebackRuntime(t, x86ScalarCountHostExecution(t), "portable") +} diff --git a/amd64_encoding_writeback_linux_amd64_test.go b/amd64_encoding_writeback_linux_amd64_test.go new file mode 100644 index 000000000..dba1ba02b --- /dev/null +++ b/amd64_encoding_writeback_linux_amd64_test.go @@ -0,0 +1,9 @@ +package plan9asm + +import "testing" + +// This counterpart observes the actual Y bank and requires native AVX2. +// It does not substitute a Y observation for AVX512's complete Z bank. +func TestCrossLinuxRuntimeMatrixX86EncodingWritebackAVX2(t *testing.T) { + x86EncodingWritebackRuntime(t, fixedLocalVEXNativeTools(t), "avx2") +} diff --git a/amd64_encoding_writeback_test.go b/amd64_encoding_writeback_test.go new file mode 100644 index 000000000..84a2c2473 --- /dev/null +++ b/amd64_encoding_writeback_test.go @@ -0,0 +1,222 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86EncodingWritebackCase struct { + instruction string + width, scalarBytes int + legacy bool + portable, avx2 bool + gather int // 1: VEX vector mask; 2: EVEX predicate mask. +} + +// One observation per writer path, plus independent legacy and wider controls. +// Arithmetic and fault oracles stay with each family; this matrix observes +// physical upper bytes, scalar passthrough, aliases and unrelated state only. +var x86EncodingWritebackCases = []x86EncodingWritebackCase{ + {"VRANGEPS $0,X1,X2,X@", 16, 0, false, true, false, 0}, + {"VRANGESS $0,X1,X2,X@", 16, 4, false, true, false, 0}, + {"VGETMANTPS $0,X1,X@", 16, 0, false, false, false, 0}, + {"VGETMANTSD $0,X1,X2,X@", 16, 8, false, false, false, 0}, + {"VADDSUBPS X1,X2,X@", 16, 0, false, true, true, 0}, + {"ADDSUBPS X1,X@", 16, 0, true, true, true, 0}, + {"VADDPS X1,X2,X@", 16, 0, false, true, true, 0}, + {"VADDSD X1,X2,X@", 16, 8, false, true, true, 0}, + {"VADDPS X1,X2,K1,X@", 16, 0, false, true, false, 0}, + {"VADDPS.Z X1,X2,K1,X@", 16, 0, false, true, false, 0}, + {"VADDSD X1,X2,K1,X@", 16, 8, false, true, false, 0}, + {"VADDSD.Z X1,X2,K1,X@", 16, 8, false, true, false, 0}, + {"VGATHERDPS X2,(AX)(X1*4),X@", 16, 0, false, true, true, 1}, + {"VGATHERDPS (AX)(X1*4),K1,X@", 16, 0, false, true, false, 2}, + {"VPMOVDB Z1,X@", 16, 0, false, true, false, 0}, + {"VPMOVDW Z1,Y@", 32, 0, false, true, false, 0}, + {"VGETEXPPS X1,X@", 16, 0, false, false, false, 0}, + {"VGETEXPSS X1,X2,X@", 16, 4, false, false, false, 0}, + {"VRCP14PS X1,X@", 16, 0, false, false, false, 0}, + {"VRCP14SS X1,X2,X@", 16, 4, false, false, false, 0}, + {"VRCPPS X1,X@", 16, 0, false, false, true, 0}, + {"RCPPS X1,X@", 16, 0, true, false, true, 0}, + {"VRCP28SS X1,X2,X@", 16, 4, false, false, false, 0}, + {"VEXP2PS Z1,Z@", 64, 0, false, false, false, 0}, + {"VRNDSCALEPS $0,X1,X@", 16, 0, false, false, false, 0}, + {"VREDUCESD $0,X1,X2,X@", 16, 8, false, false, false, 0}, + {"VROUNDPS $0,X1,X@", 16, 0, false, true, true, 0}, + {"ROUNDPS $0,X1,X@", 16, 0, true, true, true, 0}, + {"VROUNDSS $0,X1,X2,X@", 16, 4, false, true, true, 0}, + {"ROUNDSS $0,X1,X@", 16, 4, true, true, true, 0}, + {"VSCALEFPS X1,X2,X@", 16, 0, false, false, false, 0}, + {"VSCALEFSS X1,X2,X@", 16, 4, false, false, false, 0}, + {"VFIXUPIMMPS $0,X1,X2,X@", 16, 0, false, false, false, 0}, + {"VFIXUPIMMSD $0,X1,X2,X@", 16, 8, false, false, false, 0}, + {"VSHUFI32X4 $0,Y1,Y2,Y@", 32, 0, false, true, false, 0}, + {"VPERMD Y1,Y2,Y@", 32, 0, false, true, true, 0}, + {"VRANGEPS $0,X1,X2,K1,X@", 16, 0, false, true, false, 0}, + {"VRANGEPS.Z $0,X1,X2,K1,X@", 16, 0, false, true, false, 0}, + {"VFIXUPIMMSD $0,X1,X2,K1,X@", 16, 8, false, false, false, 0}, + {"VFIXUPIMMSD.Z $0,X1,X2,K1,X@", 16, 8, false, false, false, 0}, +} + +func x86EncodingWritebackProgram(t *testing.T, mode string) (string, string, string) { + t.Helper() + physical, bank := 64, "Z" + if mode == "avx2" { + physical, bank = 32, "Y" + } + var source, declarations, prototypes, rows strings.Builder + count := 0 + for _, row := range x86EncodingWritebackCases { + if mode == "portable" && !row.portable || mode == "avx2" && !row.avx2 { + continue + } + aliases := []int{0, 1} + if row.gather != 0 { + aliases = []int{0, 3} // A gather destination must not alias its VSIB index or mask. + } + for _, destination := range aliases { + instruction := strings.ReplaceAll(row.instruction, "@", fmt.Sprint(destination)) + name := fmt.Sprint("Writer", count) + fmt.Fprintf(&declarations, "func %s(out *[260]byte,state *[256]byte,p *[64]byte,mask uint64)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-32\nMOVQ out+0(FP),DI\nMOVQ state+8(FP),SI\nMOVQ p+16(FP),AX\n", name) + for register := 0; register < 4; register++ { + fmt.Fprintf(&source, "VMOVUPS %d(SI),%s%d\n", register*64, bank, register) + } + if strings.Contains(instruction, "K1") { + source.WriteString("KMOVQ mask+24(FP),K1\n") + } + source.WriteString("MOVL $5,R8\nCMPL R8,$3\n") + source.WriteString(instruction + "\n") + for register := 0; register < 4; register++ { + fmt.Fprintf(&source, "VMOVUPS %s%d,%d(DI)\n", bank, register, register*64) + } + source.WriteString("SETCS 256(DI)\nSETOS 257(DI)\nRET\n") + fmt.Fprintf(&prototypes, "extern void InstructionFormat%d(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", count) + fmt.Fprintf(&rows, "{InstructionFormat%d,%q,%d,%d,%d,%d,%t,%d},\n", + count, instruction, physical, row.width, row.scalarBytes, destination, row.legacy, row.gather) + count++ + } + } + if len(x86EncodingWritebackCases) != 40 || count == 0 { + t.Fatal("encoding writer path inventory changed", len(x86EncodingWritebackCases), mode, count) + } + want := map[string]int{"all": 80, "portable": 44, "avx2": 24}[mode] + if count != want { + t.Fatal("execution counterpart lost its exact original row partition", mode, count, want) + } + t.Logf("%s: %d original functions; native-only/MXCSR/ER rows remain in the full 80-function object inventory", mode, count) + driver := strings.NewReplacer("PROTOTYPES", prototypes.String(), "ROWS", rows.String()).Replace(x86EncodingWritebackDriver) + return source.String(), declarations.String(), driver +} + +func x86EncodingWritebackIR(t *testing.T, mode, triple string) (string, string) { + t.Helper() + source, declarations, driver := x86EncodingWritebackProgram(t, mode) + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "writer", "package writer\n"+declarations) + resolve := func(name string) string { return strings.TrimPrefix(name, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + // The complete initialized instruction emitter is isolated from public + // source-effect registration. Do not count this as public API acceptance. + return x86ScalarFormatObjectUnit(t, file, sigs, "amd64", triple, 0), driver +} + +func TestX86EncodingWritebackCompleteEmitterObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, _ := x86EncodingWritebackIR(t, "all", triple) + for _, level := range []string{"-O0", "-O2"} { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + x86PackedStringCompileObject(t, llc, triple, input, level) + } + }) + } +} + +func x86EncodingWritebackRuntime(t *testing.T, execution x86FloatCompareExecution, mode string) { + t.Helper() + ir, driver := x86EncodingWritebackIR(t, mode, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + execution.compileAndRunLLVM(t, "encoding-writeback", input, driver, level) + }) + } +} + +const x86EncodingWritebackDriver = ` +#include +#include +#include +#include +PROTOTYPES +struct row { + void (*run)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t); + const char *name; + unsigned physical,width,scalar,destination; + bool legacy; + unsigned gather; +}; +static const struct row rows[]={ROWS}; +int main(void) { + const uint64_t masks[]={0,1,5,UINT64_MAX}; + unsigned failures=0; + for(unsigned r=0;rgather) memset(state+64,0,64); + if(row->gather==1) memset(state+128,255,64); + memcpy(before,state,256); + memset(memory,0x37,64); + memset(output,0xa7,sizeof(output)); + row->run(output+1,state,memory,masks[sample]); + if(memcmp(state,before,256)||output[0]!=0xa7||output[259]!=0xa7|| + output[260]!=0xa7||output[261]!=0xa7||output[257]||output[258])return 1; + for(unsigned byte=0;byte<64;byte++)if(memory[byte]!=0x37)return 1; + for(unsigned byte=0;byte<256;byte++) { + unsigned bank=byte/64,lane=byte%64; + unsigned expected=before[byte]; + if(lane>=row->physical)expected=0xa7; + else if(bank==row->destination) { + if(lane>=row->width)expected=row->legacy?before[byte]:0; + else if(row->scalar && lane>=row->scalar && lane<16) + expected=before[(row->legacy?row->destination:2)*64+lane]; + else continue; /* Per-family oracles own the arithmetic result. */ + } else if(row->gather==1 && bank==2)expected=0; + if(output[byte+1]!=expected) { + fprintf(stderr,"%s sample=%u bank=%u byte=%u got=%x want=%x\n", + row->name,sample,bank,lane,output[byte+1],expected); + failures++; + break; + } + } + } + } + fprintf(stderr,"encoding writeback rows=%zu failed samples=%u\n",sizeof(rows)/sizeof(rows[0]),failures); + return failures?1:0; +} +` diff --git a/amd64_enqueue_forms.go b/amd64_enqueue_forms.go new file mode 100644 index 000000000..772721ca8 --- /dev/null +++ b/amd64_enqueue_forms.go @@ -0,0 +1,134 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" +) + +type x86EnqueueForm struct { + native string + source Operand + destination Operand + addressBits int + wordBits int +} + +// Both instructions use m512 and a read-only GP address. REX.W and 66 do +// not select that register's width; the address-size attribute does. +func x86EnqueueProperties(op Op, ins Instr, arch string) (x86EnqueueForm, bool, error) { + name := strings.ToUpper(string(op)) + base := strings.SplitN(name, ".", 2)[0] + if _, known := x86EnqueueOps[base]; !known { + return x86EnqueueForm{}, false, nil + } + fail := func() (x86EnqueueForm, bool, error) { + return x86EnqueueForm{}, true, fmt.Errorf("%s %s requires an exact memory/GP enqueue address form: %q", arch, op, ins.Raw) + } + if name != base || arch != "amd64" && arch != "386" || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) || len(ins.Args) != 2 { + return fail() + } + form := x86EnqueueForm{native: strings.ToLower(base), source: ins.Args[0], destination: ins.Args[1], wordBits: 64} + if arch == "386" { + form.wordBits = 32 + } + form.addressBits = ins.x86AddressBits + if form.addressBits == 0 { + form.addressBits = form.wordBits + } + if form.addressBits != form.wordBits && form.addressBits != form.wordBits/2 || + !reflect.DeepEqual(form.destination, Operand{Kind: OpReg, Reg: form.destination.Reg}) || + !isX86YrlRegisterForArch(form.destination.Reg, arch) { + return fail() + } + source := form.source + switch source.Kind { + case OpMem: + memory := source.Mem + if !reflect.DeepEqual(source, Operand{Kind: OpMem, Mem: memory}) || + memory.OffRaw != "" || memory.IndexExt != "" || !x86MemoryRegistersValidForArch(memory, arch) || + !x86EnqueueAddressEncoding(memory, form.addressBits) || + memory.Segment != "" && memory.Segment != FS && memory.Segment != GS || + memory.Index == SP || memory.Index == "" && memory.Scale != 0 || + memory.Index != "" && memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8 { + return fail() + } + case OpSym: + if source.Sym == "" || !reflect.DeepEqual(source, Operand{Kind: OpSym, Sym: source.Sym}) { + return fail() + } + case OpFP: + if !reflect.DeepEqual(source, Operand{Kind: OpFP, FPName: source.FPName, FPOffset: source.FPOffset}) { + return fail() + } + default: + return fail() + } + return form, true, nil +} + +func x86EnqueueAddressEncoding(memory MemRef, bits int) bool { + if bits == 16 { + if memory.Off < -1<<15 || memory.Off > 1<<16-1 { + return false + } + if memory.Index != "" { + return (memory.Base == BX || memory.Base == BP) && + (memory.Index == SI || memory.Index == DI) && memory.Scale == 1 + } + return memory.Base == "" || memory.Base == BX || memory.Base == BP || memory.Base == SI || memory.Base == DI + } + maximum := int64(1<<31 - 1) + if bits == 32 { + maximum = 1<<32 - 1 + } + return memory.Off >= -1<<31 && memory.Off <= maximum +} + +// This exact ordinary-address source boundary is shared by the paired native +// continuation and its storage post-consumers, never by declaration queries. +func x86EnqueueSourceForm(ins Instr, arch string) (x86EnqueueForm, bool) { + form, known, err := x86EnqueueProperties(ins.Op, ins, arch) + if !known || err != nil || form.addressBits != form.wordBits || form.source.Kind != OpMem || + form.source.Mem.Segment != "" || ins.x86Encoded || ins.x86RawNearReturn != nil || + ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 || ins.x86NumericalRead != nil { + return form, false + } + origin := ins.x86SourceRaw + if origin == nil { + return form, ins.x86AddressBits == 0 + } + return form, ins.x86AddressBits == form.wordBits && + strings.EqualFold(string(origin.op), string(ins.Op)) && reflect.DeepEqual(origin.args, ins.Args) +} + +// Only the accepted transfer establishes inputs. This predicate preserves +// existing DF/ID/private storage; it creates no memory or entry permission. +func x86EnqueuePreservesStorage(ins Instr) bool { + arch := "amd64" + if ins.x86AddressBits == 32 { + arch = "386" + } + _, valid := x86EnqueueSourceForm(ins, arch) + return valid +} + +func x86LoweredEnqueueResultStep(ins Instr, state *x86ResultState, arch string) (bool, bool) { + _, known, err := x86EnqueueProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + form, valid := x86EnqueueSourceForm(ins, arch) + if !valid || !x86ResultIncomingPointerAddressIdentity(form.source, state, arch) { + return false, true + } + device := Operand{Kind: OpMem, Mem: MemRef{Base: form.destination.Reg}} + if !x86ResultIncomingPointerAddressIdentity(device, state, arch) { + return false, true + } + // The original hardware opcode either faults or returns retry status. No + // successful submission, PASID, privilege, pointee or future read follows. + state.flags = x86ByteFlagsAll + return true, true +} diff --git a/amd64_eval.go b/amd64_eval.go index 16a0454d8..c155d3569 100644 --- a/amd64_eval.go +++ b/amd64_eval.go @@ -6,6 +6,12 @@ import ( ) func (c *amd64Ctx) evalI64(op Operand) (string, error) { + if c.goReadOnlyOperand.allows(op, 8) { + return c.evalIntSized(op, I64) + } + if err := unresolvedSymbolicImmediateErrorForArch(op, ArchAMD64); err != nil { + return "", err + } switch op.Kind { case OpImm: return fmt.Sprintf("%d", op.Imm), nil @@ -26,18 +32,18 @@ func (c *amd64Ctx) evalI64(op Operand) (string, error) { addrOnly := strings.HasPrefix(s, "$") if addrOnly { s = strings.TrimSpace(strings.TrimPrefix(s, "$")) + if mem, matched, err := parseX86SymbolIndexedMemory(s); matched { + if err != nil { + return "", err + } + return c.addrFromSymbolIndexedAddress(mem) + } if mem, ok := parseMem(s); ok { return c.addrFromMem(mem) } } p, err := c.ptrFromSB(s) if err != nil { - // Some runtime asm constants (e.g. $const_stackGuard) come from - // includes/macros that we don't fully materialize. Treat unresolved - // bare symbols as immediate zero to keep translation progressing. - if !strings.Contains(s, "(SB)") { - return "0", nil - } return "", err } if addrOnly { diff --git a/amd64_explicit_mask_move_forms.go b/amd64_explicit_mask_move_forms.go new file mode 100644 index 000000000..9a4ecc0ca --- /dev/null +++ b/amd64_explicit_mask_move_forms.go @@ -0,0 +1,99 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ExplicitMaskMoveSpec struct { + laneBits int +} + +// Go's complete _yvmaskmovpd table: X/Y load/store with vector sign-bit +// masks, not K writemasks. VMASKMOVPS/PD are W0, not WIG (W1 raises #UD). +var amd64ExplicitMaskMoveSpecs = map[Op]amd64ExplicitMaskMoveSpec{ + "VMASKMOVPS": {laneBits: 32}, + "VMASKMOVPD": {laneBits: 64}, + "VPMASKMOVD": {laneBits: 32}, + "VPMASKMOVQ": {laneBits: 64}, +} + +type x86ExplicitMaskMoveForm struct { + spec amd64ExplicitMaskMoveSpec + byteWidth int + memory int + load bool +} + +// Shape supplies no numerical mask bits, native address or frame backing. +func x86ExplicitMaskMoveProperties(op Op, ins Instr, arch string) (x86ExplicitMaskMoveForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64ExplicitMaskMoveSpecs[Op(name)] + form := x86ExplicitMaskMoveForm{spec: spec} + if !known { + return form, false, nil + } + fail := func() (x86ExplicitMaskMoveForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go explicit vector-mask move grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != 3 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return fail() + } + form.load = isAMD64MemoryOperand(ins.Args[0]) + if form.load == isAMD64MemoryOperand(ins.Args[2]) { + return fail() + } + form.memory = 2 + data := ins.Args[0] + if form.load { + form.memory, data = 0, ins.Args[2] + } + form.byteWidth = amd64VectorByteWidth(ins.Args[1].Reg) + if form.byteWidth != 16 && form.byteWidth != 32 || + !amd64VEXVectorRegister(ins.Args[1], form.byteWidth) || !amd64VEXVectorRegister(data, form.byteWidth) { + return fail() + } + memory := ins.Args[form.memory] + if memory.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(memory.Mem, arch); err != nil { + return form, true, err + } + if memory.Mem.OffRaw != "" || memory.Mem.IndexExt != "" { + return fail() + } + if arch == "386" && strings.HasSuffix(memory.Mem.Sym, "(SB)") { + memory.Mem.Base = "" // Go ignores this annotation, not the index input. + } + } + if !x86PackedShiftMemoryValid(memory, arch) { + return fail() + } + return form, true, nil +} + +// Only the byte containing each lane's most significant bit is an input. +func (f x86ExplicitMaskMoveForm) signByteInputs() uint64 { + lane := f.spec.laneBits / 8 + var required uint64 + for offset := lane - 1; offset < f.byteWidth; offset += lane { + required |= uint64(1) << uint(offset) + } + return required +} + +func x86TypedExplicitMaskMoveByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + f, known, err := x86ExplicitMaskMoveProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + if err := x86Mode32VectorRegisterContext(ins, arch); err != nil { + return x86TypedByteEffect{}, true, err + } + width := int64(f.byteWidth) + return x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: width, read: true}, + {index: 1, operand: ins.Args[1], width: width, read: true}, + {index: 2, operand: ins.Args[2], width: width, write: true}, + }}, true, nil +} diff --git a/amd64_extension_native_result_test.go b/amd64_extension_native_result_test.go new file mode 100644 index 000000000..56d1ae1f1 --- /dev/null +++ b/amd64_extension_native_result_test.go @@ -0,0 +1,233 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// Independent Go encoder rows, not the lowerer's extension specification. +var x86ExtensionNativeOracle = []struct { + op, table, prefix, bytes string + source, destination int + signed, amd64Only bool +}{ + {"MOVBWSX", "ymb_rl", "Pq", "0xbe", 1, 2, true, false}, + {"MOVBWZX", "ymb_rl", "Pq", "0xb6", 1, 2, false, false}, + {"MOVBLSX", "ymb_rl", "Pm", "0xbe", 1, 4, true, false}, + {"MOVBLZX", "ymb_rl", "Pm", "0xb6", 1, 4, false, false}, + {"MOVBQSX", "ymb_rl", "Pw", "0x0f, 0xbe", 1, 8, true, true}, + {"MOVBQZX", "ymb_rl", "Pw", "0x0f, 0xb6", 1, 8, false, true}, + {"MOVWLSX", "yml_rl", "Pm", "0xbf", 2, 4, true, false}, + {"MOVWLZX", "yml_rl", "Pm", "0xb7", 2, 4, false, false}, + {"MOVWQSX", "yml_rl", "Pw", "0x0f, 0xbf", 2, 8, true, true}, + {"MOVWQZX", "yml_rl", "Pw", "0x0f, 0xb7", 2, 8, false, true}, + {"MOVLQSX", "yml_rl", "Pw", "0x63", 4, 8, true, true}, + {"MOVLQZX", "yml_rl", "Px", "0x8b", 4, 8, false, false}, + {"MOVSWW", "ymb_rl", "Pe", "0x0f, 0xbf", 2, 2, true, false}, + {"MOVZWW", "ymb_rl", "Pe", "0x0f, 0xb7", 2, 2, false, false}, +} + +func TestX86ExtensionCompleteEncoderAttributes(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range x86ExtensionNativeOracle { + want := fmt.Sprintf("{A%s, %s, %s, opBytes{%s}}", row.op, row.table, row.prefix, row.bytes) + if !strings.Contains(string(data), want) { + t.Fatal("original Go encoder attributes changed:", want) + } + } +} + +func x86ExtensionNativeFixture(name, arch, body string, dead bool) (string, string) { + word, move, lane, typ, seed := 8, "MOVQ", "Q", "uint64", "0x87654321fedcba98" + if arch == "386" { + word, move, lane, typ, seed = 4, "MOVL", "D", "uint32", "0xfedcba98" + } + tail := fmt.Sprintf("PINSR%s $0,CX,X1\nPEXTR%s $0,X1,AX\n%s AX,ret+%d(FP)\n", lane, lane, move, word) + decl := fmt.Sprintf("func %s(p *[4]byte) %s\n", name, typ) + args := word * 2 + if dead { + tail = "" + decl = fmt.Sprintf("func %s(p *[4]byte)\n", name) + args = word + } + source := fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),DX\nMOVL $1,BX\n%s $%s,CX\n%s\n%sRET\n", + name, args, move, move, seed, body, tail) + return source, decl +} + +func TestX86ExtensionNativeResultsThreeAPIFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, row := range x86ExtensionNativeOracle { + if target.arch == "386" && row.amd64Only { + continue // Independently Go-rejected mode; covered by the existing rejection oracle. + } + for _, address := range []string{"(DX)", "-1(DX)(BX*1)"} { + asm, decl := x86ExtensionNativeFixture(fmt.Sprintf("F%d", count), target.arch, row.op+" "+address+",CX", false) + source.WriteString(asm) + declarations.WriteString(decl) + count++ + } + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("native extension must supply only its continued numerical result:", err) + } + if strings.Count(ir, "load volatile i") != count { + t.Fatal("conditional results lack paired exact native reads") + } + compileLLVMToObject(t, llc, target.triple, "extension-native.ll", "extension-native.o", ir) + }) + } + }) + } +} + +func TestX86ExtensionDeadNativeReadSurvivesOptimization(t *testing.T) { + for _, row := range x86ExtensionNativeOracle { + t.Run(row.op, func(t *testing.T) { + source, declaration := x86ExtensionNativeFixture("F", "amd64", row.op+" (DX),CX", true) + goAssemblerOracleOutput(t, "amd64", source, "", true) + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, "amd64", "x86_64-unknown-linux-gnu", "Go") + if err != nil { + t.Fatal(err) + } + want := fmt.Sprintf("load volatile i%d,", row.source*8) + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + if strings.Count(text, want) != 1 { + t.Fatalf("dead native %s lost its exact %d-byte read", row.op, row.source) + } + } + }) + } +} + +func TestX86ExtensionNativeResultsDoNotGrantStandaloneMemory(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + for _, row := range x86ExtensionNativeOracle { + if arch == "386" && row.amd64Only { + continue + } + body := fmt.Sprintf("%s p+0(FP),DX\nMOVL $0,CX\n%s (DX),CX\nMOVL CX,ret0+%d(FP)\nRET", move, row.op, word) + _, file, sig, _ := x86ResultInitializationFixture(t, arch, "func Value(p *[4]byte) uint32", body) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s/%s acquired standalone memory: %v", arch, row.op, err) + } + } + } + } +} + +func TestX86ExtensionNativeResultBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + move, triple := "MOVQ", "x86_64-unknown-linux-gnu" + if arch == "386" { + move, triple = "MOVL", "i386-unknown-linux-gnu" + } + for name, body := range map[string]string{ + "unknown base": "MOVBLZX (DI),CX", + "numeric base": "MOVL $42,DX\nMOVBLZX (DX),CX", + "unknown index": "MOVBLZX (DX)(DI*1),CX", + "pointer index": "MOVBLZX (DX)(DX*1),CX", + "unbound data": "MOVBLZX ·external(SB),CX", + "word high bytes": move + " DX,CX\nMOVBWSX (DX),CX", + "pointer clobbered by partial result": "MOVBWSX (DX),DX\nMOVBLZX (DX),CX", + "mixed pointer join": "CMPL BX,$0\nJE other\n" + move + " DX,DI\nJMP end\n" + + "other:\nMOVL $42,DI\nend:\nMOVBLZX (DI),CX", + } { + t.Run(arch+"/"+name, func(t *testing.T) { + source, decl := x86ExtensionNativeFixture("F", arch, body, false) + goAssemblerOracleOutput(t, arch, source, "", true) + if name == "numeric base" { + x86RequireNumericNativeSourceContext(t, source, decl, arch) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, decl, arch, triple, api) + if name == "numeric base" { + if err != nil || !strings.Contains(ir, "load volatile i8,") { + t.Fatal("complete NUM address lost its byte extension read", api, err) + } + x86NumericNativeAddressObject(t, ir, triple) + continue + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s manufactured numerical or pointer bytes: %v", api, err) + } + } + }) + } + } +} + +func TestX86ExtensionNativeResultEffectsAndOriginBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86ExtensionNativeOracle { + if arch == "386" && row.amd64Only { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row.op+" -1(DX)(BX*1),CX\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + base, _, _ := x86ResultGP(DX, 4, arch) + index, _, _ := x86ResultGP(BX, 4, arch) + destination, _, _ := x86ResultGP(CX, 4, arch) + state := x86ResultState{flags: x86ByteFlagsAll} + state.pointerGP[base] = true + state.gp[index] = 0xff + before := state.clone() + accepted, known := x86LoweredScalarExtensionRead(ins, &state, nil, 0, arch) + want := uint8(0xff) + if row.destination == 2 { + want = 3 // W leaves every other byte unknown. + } + if !accepted || !known || state.gp[destination] != want || state.flags != before.flags || + !state.pointerGP[base] || state.gp[base] != 0 || state.pointerGP[destination] { + t.Fatalf("%s/%s widened its effect: %+v", arch, row.op, state) + } + for _, mutation := range []string{"decoded", "raw", "raw return", "address", "unbound address", "segment"} { + changed, rejected := ins, before.clone() + switch mutation { + case "decoded": + changed.x86Encoded = true + case "raw": + changed.x86SourceRaw = new(x86SourceRawInstruction) + case "raw return": + changed.x86RawNearReturn = new(x86RawNearReturnForm) + case "address": + changed.x86AddressBits = 32 + case "unbound address": + changed.x86UnboundAddressOverride = true + case "segment": + changed.x86SegmentPrefix = 0x64 + } + if accepted, _ := x86LoweredScalarExtensionRead(changed, &rejected, nil, 0, arch); accepted || rejected.gp != before.gp || rejected.pointerGP != before.pointerGP { + t.Fatalf("%s/%s granted an unbound %s effect", arch, row.op, mutation) + } + } + } + } +} diff --git a/amd64_extension_native_runtime_unix_test.go b/amd64_extension_native_runtime_unix_test.go new file mode 100644 index 000000000..675312185 --- /dev/null +++ b/amd64_extension_native_runtime_unix_test.go @@ -0,0 +1,164 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ExtensionNativeResultsAndFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, originals, rows strings.Builder + count := 0 + for _, row := range x86ExtensionNativeOracle { + for _, dead := range []bool{false, true} { + name, original := fmt.Sprintf("F%d", count), fmt.Sprintf("Original%d", count) + body := row.op + " -1(DX)(BX*1),CX" + native := "MOVQ $0x87654321fedcba98,AX\n" + row.op + " (DI),AX\n" + if dead { + body += "\nMOVQ $0x87654321fedcba98,CX" + native += "MOVQ $0x87654321fedcba98,AX\n" + } + asm, decl := x86ExtensionNativeFixture(name, "amd64", body, false) + source.WriteString(asm) + declarations.WriteString(decl) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+native+"RET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\nextern uint64_t %s(const uint8_t*);\n", name, original) + signed, discarded := 0, 0 + if row.signed { + signed = 1 + } + if dead { + discarded = 1 + } + fmt.Fprintf(&rows, "{%s,%s,%d,%d,%d,%d},\n", name, original, row.source, row.destination, signed, discarded) + count++ + } + } + if count != 28 { + t.Fatal("incomplete live/dead extension family", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf(x86ExtensionNativeRuntimeC, prototypes.String(), originals.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text, level := ir, "-O0" + if optimized { + text, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + output := execution.compileAndRunLLVM(t, "extension-native", text, driver, level) + if string(output) != "28 extension kernels: numerical 3584, fault 336, tripwires 2; failures 0\n" { + t.Fatal("original/LLVM extension semantics or read envelope differs:", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } +} + +const x86ExtensionNativeRuntimeC = ` +#include +#include +#include +#include +#include +#include +#include +#include +%s +%s +typedef uint64_t (*Program)(const uint8_t*); +typedef struct { + Program translated, original; + unsigned width, destination, sign, dead; +} Kernel; +static const Kernel kernels[]={%s}; +static const uint64_t seed=UINT64_C(0x87654321fedcba98); + +static uint64_t reference(Kernel k, const uint8_t *p) { + if(k.dead) return seed; + uint64_t value=0; + for(unsigned b=0;b(SB)", "1234"} { + source := fmt.Sprintf("TEXT f(SB),4,$0-0\n%s %s,AX\nRET\n", row.op, address) + goAssemblerOracleOutput(t, arch, source, "", true) + control := ins + control.Args = []Operand{{Kind: OpSym, Sym: address}, {Kind: OpReg, Reg: AX}} + control.Raw = row.op + " " + address + ",AX" + if _, known, err := x86ScalarExtensionProperties(control.Op, control, arch); !known || err != nil { + t.Fatal("valid SB/absolute source lost its existing grammar", address, err) + } + } + }) + } + } +} diff --git a/amd64_fixed_local_address_flags_runtime_go127_test.go b/amd64_fixed_local_address_flags_runtime_go127_test.go new file mode 100644 index 000000000..2e3d26a0a --- /dev/null +++ b/amd64_fixed_local_address_flags_runtime_go127_test.go @@ -0,0 +1,187 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Each backend checks FLAGS against its own actual global addresses. Different +// linkers need not place the two objects in the same relative order. Address +// excursions are integer operations and are never dereferenced here. +func fixedLocalAddressFlagsProgram(triple string) (assembly, declarations, original, driver string) { + operations := []string{ + "ADDQ CX,SI\nSETCS DL", + "SUBQ CX,SI\nSETCS DL", + "ADDQ CX,SI\nINCQ SI\nSETEQ DL", + "ADDQ CX,SI\nDECQ SI\nSETEQ DL", + "ADDQ CX,SI\nMOVQ $·Other(SB),DI\nCMPQ SI,DI\nSETCS DL", + } + declarations = "var Table, Other [4]uint64\n" + var goFunctions, cFunctions, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, operation := range operations { + name := fmt.Sprintf("Flags%d", index) + assembly += fixedLocalDataAddressFunction(name, "MOVQ $·Table(SB),SI\n"+operation) + declarations += "func " + name + "(a uint64) uint64\n" + fmt.Fprintf(&goFunctions, "%s,", name) + fmt.Fprintf(&cFunctions, "flags%d,", index) + fmt.Fprintf(&cDeclarations, "extern uint64_t flags%d(uint64_t) __asm__(%s);\n", index, + runtimeCAsmLabel(prefix+"example.invalid/framefixture."+name)) + } + original = `package main +import ("fmt"; "unsafe") +` + declarations + ` +func expected(index int, base, other, delta uint64) uint64 { + var flag bool + switch index { + case 0: flag = base+delta < base + case 1: flag = base < delta + case 2: flag = base+delta+1 == 0 + case 3: flag = base+delta-1 == 0 + case 4: flag = base+delta < other + default: panic("unknown reference operation") + } + if flag { + return 1 + } + return 0 +} +func main() { + base := uint64(uintptr(unsafe.Pointer(&Table[0]))) + other := uint64(uintptr(unsafe.Pointer(&Other[0]))) + inputs := []uint64{0,1,8,^uint64(0),-base,-base-1,1-base,other-base-1,other-base,other-base+1} + for i := range Table { + Table[i] = uint64(i+1)*0x0102030405060708 + Other[i] = ^Table[i] + } + before, beforeOther := Table, Other + functions := []func(uint64)uint64{` + goFunctions.String() + `} + for index, function := range functions { + for sample, delta := range inputs { + want, got := expected(index,base,other,delta), function(delta) + if got != want || Table != before || Other != beforeOther { + panic(fmt.Sprintf("case=%d sample=%d got=%d want=%d",index,sample,got,want)) + } + fmt.Printf("%d %d PASS\n",index,sample) + } + } +} +` + driver = "#include \n#include \n" + runtimeCAsmLabelPreamble + cDeclarations.String() + + fmt.Sprintf("uint64_t table[4] __asm__(%s);\nuint64_t other_table[4] __asm__(%s);\n", + runtimeCAsmLabel(prefix+"example.invalid/framefixture.Table"), + runtimeCAsmLabel(prefix+"example.invalid/framefixture.Other")) + ` +static uint64_t expected(unsigned index, uint64_t base, uint64_t other, uint64_t delta) { + switch (index) { + case 0: return base+delta < base; + case 1: return base < delta; + case 2: return base+delta+1 == 0; + case 3: return base+delta-1 == 0; + case 4: return base+delta < other; + default: return UINT64_MAX; + } +} +int main(void) { + uint64_t base=(uintptr_t)table, other=(uintptr_t)other_table; + uint64_t inputs[]={0,1,8,UINT64_MAX,-base,-base-1,1-base,other-base-1,other-base,other-base+1}; + uint64_t (*functions[])(uint64_t)={` + cFunctions.String() + `}; + for (unsigned i=0;i<4;i++) { + table[i]=(uint64_t)(i+1)*UINT64_C(0x0102030405060708); + other_table[i]=~table[i]; + } + for (unsigned index=0;index<5;index++) { + for (unsigned sample=0;sample<10;sample++) { + uint64_t want=expected(index,base,other,inputs[sample]); + uint64_t got=functions[index](inputs[sample]); + if (got!=want) { + fprintf(stderr,"case=%u sample=%u got=%llu want=%llu\n", + index,sample,(unsigned long long)got,(unsigned long long)want); + return 1; + } + for (unsigned i=0;i<4;i++) { + uint64_t value=(uint64_t)(i+1)*UINT64_C(0x0102030405060708); + if (table[i]!=value || other_table[i]!=~value) { + return 2; + } + } + printf("%u %u PASS\n",index,sample); + } + } + return 0; +} +` + return +} + +func fixedLocalAddressFlagsRuntime(t *testing.T, execution x86FloatCompareExecution, execute bool) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + assembly, declarations, original, driver := fixedLocalAddressFlagsProgram(execution.triple) + var want []string + for index := 0; index < 5; index++ { + for sample := 0; sample < 10; sample++ { + want = append(want, fmt.Sprintf("%d %d PASS", index, sample)) + } + } + check := func(output []byte) { + t.Helper() + if strings.TrimSpace(string(output)) != strings.Join(want, "\n") { + t.Fatalf("native address FLAGS reference did not complete every case:\n%s", output) + } + } + if execute { + check(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original)) + } else { + x86COMICompileOriginalGo(t, map[string]string{ + "go.mod": "module addressflags\ngo 1.20\n", "main.go": original, "original_amd64.s": assembly, + }, false) + } + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, declarations, + runtime.GOOS+"/amd64", execution.triple) + fixedLocalResultsRuntimeAPIs(t, pkg, request, file, options, assembly, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + if execute { + check(execution.compileAndRunLLVM(t, "address-flags", input, driver, level)) + } else { + execution.compileLLVM(t, "address-flags", input, driver, level) + } + }) + } + }) + t.Logf("5 complete functions, 10 deltas from each backend's own addresses, independent arithmetic references and 64-byte global canaries; execute=%v", execute) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalAddressFlags(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + t.Run("address-flags", func(t *testing.T) { + fixedLocalAddressFlagsRuntime(t, execution, true) + }) + t.Run("machine-bits", func(t *testing.T) { + fixedLocalMachineBitsRuntime(t, execution, true) + }) + for _, scope := range []string{"auxiliary", "vector spill", "multiply"} { + t.Run(scope, func(t *testing.T) { + fixedLocalMachineRuntime(t, execution, true, scope) + }) + } +} + +func TestGoAMD64FixedLocalAddressFlagsRuntimeObjects(t *testing.T) { + fixedLocalAddressFlagsRuntime(t, fixedLocalPackedCompileOnlyTools(t), false) +} diff --git a/amd64_fixed_local_aes.go b/amd64_fixed_local_aes.go new file mode 100644 index 000000000..b80f02012 --- /dev/null +++ b/amd64_fixed_local_aes.go @@ -0,0 +1,31 @@ +package plan9asm + +func fixedLocalAESByteStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + f, known, err := x86AESProperties(ins.Op, *ins, "amd64") + if !known { + return false, nil + } + if err != nil || f.byteWidth != 16 { + return true, goAMD64FrameContext("fixed local AES shape is not closed") + } + width := uint64(65535) + source := state.vectors[x86CallerVectorRoot(f.sourceOperand.Reg)] & width + if f.memory >= 0 { + if _, _, err := fixedLocalReadMemory(ins, f.memory, state, proof, residue, 16, 1); err != nil { + return true, err + } + source = width + } + input := uint64(0) + if f.state >= 0 { + input = state.vectors[x86CallerVectorRoot(ins.Args[f.state].Reg)] & width + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + defined := x86AESDefinedBytes(f, source, input) + if !f.spec.vector { + state.vectors[destination] = state.vectors[destination]&^width | defined + } else { + state.vectors[destination] = ^width | defined + } + return true, nil +} diff --git a/amd64_fixed_local_affine.go b/amd64_fixed_local_affine.go new file mode 100644 index 000000000..07bf446db --- /dev/null +++ b/amd64_fixed_local_affine.go @@ -0,0 +1,271 @@ +package plan9asm + +import "math" + +// A full-word LEA keeps the same private-frame identity with a checked offset. +// It does not read memory or initialize numerical bytes; the later access must +// still establish its width, bounds, alignment and reaching initialized bytes. +func fixedLocalPrivateAddressStep(ins Instr, state *x86FixedLocalState, proof *x86FixedLocalProof) (bool, error) { + view := ins + if derived, known, err := x86NamedFrameMoveAddressInstruction(ins, "amd64"); known { + if err != nil { + return true, goAMD64FrameContext("fixed local original MOV address grammar is not closed") + } + view = derived + } + if x86LEAWidth(view.Op, "amd64") == 0 || len(view.Args) != 2 || view.Args[0].Kind != OpMem { + return false, nil + } + if _, private := state.pointers[view.Args[0].Mem.Base]; !private { + return false, nil + } + if proof == nil || ins.x86AddressBits != 0 { + return true, goAMD64FrameContext("fixed local LEA requires the original native address width and frame") + } + width, err := x86LEAOperandWidth(view, "amd64") + address, addressErr := fixedLocalMemoryAddress(ins, 0, proof.frame) + offset, known := fixedLocalPrivateOffset(address, state) + destination := ins.Args[1].Reg + if err != nil || addressErr != nil || !known || destination == SP { + return true, goAMD64FrameContext("fixed local LEA requires an original private address and checked numerical displacement") + } + delete(state.opaque, destination) + delete(state.constants, destination) + delete(state.booleans, destination) + state.revokeRange(destination) + if width == 8 { + state.known[destination] = 0 + state.pointers[destination] = offset + } else { + // Truncated address bytes remain unknown. They cannot be reused as + // NUM or as a pointer; W preserves old upper bytes and L zeroes them. + delete(state.pointers, destination) + state.known[destination] &^= x86CallerByteMask(destination, width) + if width == 4 { + state.known[destination] = 0xf0 + } + } + return true, nil +} + +// Full-word affine operations preserve an actual typed input's identity, not +// numerical pointer bytes, pointee bounds or private-frame ownership. Grammar, +// widths and FLAGS come from the same typed forms as ordinary GP lowering. +func fixedLocalInputAddressStep(ins *Instr, state *x86FixedLocalState, sig FuncSig, proof *x86FixedLocalProof, residue int) (bool, error) { + // A typed FP source can introduce the first native address. Otherwise no + // address operation is possible before a register or spill carries one. + if len(state.opaque) == 0 && len(state.opaqueCells) == 0 && + (len(ins.Args) == 0 || ins.Args[0].Kind != OpFP) { + return false, nil + } + fail := func() (bool, error) { + return true, goAMD64FrameContext("fixed local affine input lacks a unique pointer and numerical offset: %s", ins.Op) + } + write := func(destination Operand, origin x86FixedLocalOrigins, killed, defined uint8) (bool, error) { + root, valid := x86CallerGP(destination.Reg) + if destination.Kind != OpReg || !valid || root != destination.Reg || root == SP { + return fail() + } + writeFixedLocalOrigin(state, root, origin) + state.flags = state.flags&^killed | defined + if killed|defined != 0 { + state.comparison = x86FixedLocalComparison{} + } + return true, nil + } + shift := func(origins x86FixedLocalOrigins, operand Operand, subtract bool) (x86FixedLocalOrigins, bool) { + if !fixedLocalOriginOwners(origins, proof) { + return origins, false + } + if origins.address.symbol == "" { + return origins, true // Actual native arithmetic supplies no pointee bound. + } + delta, known := operand.Imm, operand.Kind == OpImm && operand.ImmRaw == "" && !operand.ImmIsFloat + if operand.Kind == OpReg { + delta, known = state.constants[operand.Reg] + } + if !known || subtract && delta == math.MinInt64 { + origins.address = x86StaticSBAddress{} + return origins, true + } + if subtract { + delta = -delta + } + origins.address.offset += delta // Match the emitted wrapping i64 address arithmetic. + return origins, true + } + if !x86ResultNamedPredicateInstruction(*ins) { + return false, nil + } + if form, known, err := x86LockScalarRMWProperties(ins.Op, *ins, "amd64"); known && err == nil && form.spec.bits == 64 { + origin, pointer := fixedLocalOpaqueRegister(form.destination, state) + switch form.spec.stem { + case "ADD", "SUB", "ADC", "SBB": + if form.destination.Kind != OpReg { + return false, nil + } + carry := form.spec.carryInput + if carry && state.flags&x86ByteFlagCF == 0 { + return fail() + } + subtract := form.spec.stem == "SUB" || form.spec.stem == "SBB" + value, err := fixedLocalReadValue(ins, 0, state, sig, proof, residue, 8, false) + if err != nil || value.pointer { + return false, nil // The shared machine reader reports invalid inputs. + } + if pointer && !value.opaque { + var valid bool + origin, valid = shift(origin, form.source, subtract) + if !valid { + return fail() + } + if carry { + origin.address = x86StaticSBAddress{} // Defined CF need not be constant. + } + return write(form.destination, origin, form.spec.flagsKilled, form.spec.flagsDefined) + } + if !subtract && value.opaque && !pointer && fixedLocalNumericRead(form.destination, 8, state) { + sourceOrigin, valid := shift(value.identity, form.destination, false) + if !valid { + return fail() + } + if carry { + sourceOrigin.address = x86StaticSBAddress{} + } + return write(form.destination, sourceOrigin, form.spec.flagsKilled, form.spec.flagsDefined) + } + return false, nil // Other arithmetic results are bits, not addresses. + case "INC", "DEC": + if pointer { + var valid bool + origin, valid = shift(origin, Operand{Kind: OpImm, Imm: 1}, form.spec.stem == "DEC") + if !valid { + return fail() + } + return write(form.destination, origin, form.spec.flagsKilled, form.spec.flagsDefined) + } + } + } + if effect, known, err := x86NamedLEAByteEffect(ins.Op, *ins, "amd64"); known && err == nil && x86LEAWidth(ins.Op, "amd64") == 8 { + memory := ins.Args[0].Mem + origin, pointer := state.opaque[memory.Base] + if pointer && origin.valid() { + for _, input := range effect.operands { + if input.read && input.operand.Reg != memory.Base && !fixedLocalNumericRead(input.operand, input.width, state) { + return fail() + } + } + // An aliased pointer index is still a second pointer, even if it + // happens to share the base's spelling or original entry slot. + if _, secondPointer := state.opaque[memory.Index]; secondPointer { + return fail() + } + displacement, known, valid := fixedLocalNativeDisplacement(memory, state) + if !valid || !fixedLocalOriginOwners(origin, proof) { + return fail() + } + if origin.address.symbol != "" { + if known { + origin.address.offset += displacement + } else { + origin.address = x86StaticSBAddress{} + } + } + return write(ins.Args[1], origin, effect.flagsKilled, effect.flagsDefined) + } + } + form, known, err := x86ScalarCompareProperties(ins.Op, *ins, "amd64") + if !known || err != nil || form.test || form.bits != 64 { + return false, nil + } + values := [2]x86FixedLocalOrigins{} + opaque := func(index int) bool { + operand := form.args[index] + if origin, found := fixedLocalOpaqueRegister(operand, state); found { + values[index] = origin + return true + } + if operand.Kind != OpMem || proof == nil { + return false + } + address, err := fixedLocalMemoryAddress(*ins, index, proof.frame) + if err != nil { + return false + } + offset, found := fixedLocalPrivateOffset(address, state) + if !found { + return false + } + origin, found := state.opaqueCells[offset] + values[index] = origin + return found && origin.valid() + } + pointers := [2]bool{opaque(0), opaque(1)} + if !pointers[0] || !pointers[1] { + return false, nil + } + for index, operand := range form.args { + if !fixedLocalOriginOwners(values[index], proof) { + return fail() + } + if operand.Kind == OpMem { + if _, err := fixedLocalMemory(ins, index, state, proof, residue, 8, 1, false, true); err != nil { + return true, err + } + } + } + flags := uint8(x86ByteFlagsAll) + comparison := x86FixedLocalComparison{} + if values[0].hasLiteral() || values[1].hasLiteral() { + if !pointers[0] || !pointers[1] || !fixedLocalOriginOwners(values[0], proof) || !fixedLocalOriginOwners(values[1], proof) { + return fail() + } + // The actual CMP preserves every architectural flag. Its optional + // literal equality fact is narrower than that emitted operation. + for index, value := range values { + if literal, single := value.literal(); single && form.args[1-index].Kind == OpReg { + comparison = x86FixedLocalComparison{register: form.args[1-index].Reg, literal: literal} + break + } + } + } + state.flags, state.comparison = flags, comparison + return true, nil +} + +func fixedLocalOpaqueRegister(operand Operand, state *x86FixedLocalState) (x86FixedLocalOrigins, bool) { + root, valid := x86CallerGP(operand.Reg) + if operand.Kind != OpReg || !valid || root != operand.Reg { + return x86FixedLocalOrigins{}, false + } + origin, found := state.opaque[root] + return origin, found && origin.valid() +} + +// Native effective addresses use wrapping machine arithmetic, unlike relocated +// private-frame offsets. A complete NUM index needs no caller bounds proof. +func fixedLocalNativeDisplacement(memory MemRef, state *x86FixedLocalState) (offset int64, known, valid bool) { + if memory.OffRaw != "" || memory.Sym != "" || memory.Segment != "" || memory.IndexExt != "" { + return 0, false, false + } + if memory.Index == "" { + return memory.Off, true, memory.Scale == 0 + } + if !fixedLocalNumericRead(Operand{Kind: OpReg, Reg: memory.Index}, 8, state) || + memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8 { + return 0, false, false + } + index, known := state.constants[memory.Index] + return memory.Off + index*memory.Scale, known, true +} + +func intersectFixedLocalOrigins[K, V comparable](previous, current map[K]V) bool { + changed := false + for key, value := range previous { + if incoming, found := current[key]; !found || incoming != value { + delete(previous, key) + changed = true + } + } + return changed +} diff --git a/amd64_fixed_local_affine_runtime_test.go b/amd64_fixed_local_affine_runtime_test.go new file mode 100644 index 000000000..8c650b0ca --- /dev/null +++ b/amd64_fixed_local_affine_runtime_test.go @@ -0,0 +1,141 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func fixedLocalAffineRuntimeProgram(t *testing.T, triple string) (string, string, string, string) { + t.Helper() + bodies := []string{ + "ADDQ AX,SI\nMOVQ (SI),AX", + "ADDQ SI,AX\nMOVQ (AX),AX", + "SUBQ AX,SI\nMOVQ (SI),AX", + "LEAQ -8(SI)(AX*2),CX\nMOVQ (CX),AX", + "INCQ SI\nDECQ SI\nMOVQ (SI),AX", + "MOVQ $0,AX\nCMPQ SI,DI\nSETCS AL", + "MOVQ $0,AX\nMOVQ SI,8(SP)\nCMPQ DI,8(SP)\nSETCS AL", + "MOVQ $0,AX\nMOVQ SI,8(SP)\nCMPQ 8(SP),DI\nSETCS AL", + "CMPQ AX,$0\nJEQ left\nLEAQ 8(SI),CX\nJMP join\nleft:\nLEAQ 16(SI),CX\njoin:\nMOVQ (CX),AX", + "CMPQ AX,$0\nJEQ join\nMOVQ SI,8(SP)\njoin:", + } + for _, register := range []string{"BP", "R14"} { + bodies = append(bodies, + fmt.Sprintf("MOVQ SI,%s\nADDQ AX,%s\nMOVQ (%s),AX", register, register, register), + fmt.Sprintf("LEAQ 8(SI),%s\nADDQ AX,%s\nMOVQ (%s),AX", register, register, register)) + } + var assembly, declarations, goRows, cRows, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, body := range bodies { + _, _, source := fixedLocalInputFixture(t, body) + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(source, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p, q *uint64, a uint64) uint64\n", name) + fmt.Fprintf(&goRows, "%s,\n", name) + fmt.Fprintf(&cRows, "value%d,\n", index) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t *, uint64_t *, uint64_t) __asm__(%s);\n", + index, runtimeCAsmLabel(prefix+"example.invalid/framefixture."+name)) + } + original := `package main +import "fmt" +` + declarations.String() + ` +func main() { + rows := []func(*uint64, *uint64, uint64) uint64{ +` + goRows.String() + ` + } + var data [16]uint64 + for i := range data { data[i] = uint64(i)*0x0102030405060708 ^ 0xfedcba9876543210 } + for n, call := range rows { + for v := 0; v < 4; v++ { + got := call(&data[8], &data[7+v%3], uint64(v*8)) + fmt.Printf("%d %d %016x\n", n, v, got) + } + } + for i, got := range data { + if got != uint64(i)*0x0102030405060708 ^ 0xfedcba9876543210 { panic("input modified") } + } +} +` + driver := `#include +#include +` + runtimeCAsmLabelPreamble + cDeclarations.String() + ` +int main(void) { + uint64_t (*const rows[])(uint64_t *, uint64_t *, uint64_t) = { +` + cRows.String() + ` + }; + uint64_t data[16]; + for (unsigned i = 0; i < 16; i++) data[i] = i*UINT64_C(0x0102030405060708) ^ UINT64_C(0xfedcba9876543210); + for (unsigned n = 0; n < sizeof(rows)/sizeof(rows[0]); n++) { + for (unsigned v = 0; v < 4; v++) { + printf("%u %u %016llx\n", n, v, (unsigned long long)rows[n](&data[8], &data[7+v%3], v*8)); + } + } + for (unsigned i = 0; i < 16; i++) { + if (data[i] != (i*UINT64_C(0x0102030405060708) ^ UINT64_C(0xfedcba9876543210))) return 2; + } + return 0; +} +` + return assembly.String(), declarations.String(), original, driver +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalAffine(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + t.Setenv("GOCACHE", t.TempDir()) + assembly, declarations, original, driver := fixedLocalAffineRuntimeProgram(t, execution.triple) + want := strings.Split(strings.TrimSpace(string(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original))), "\n") + if len(want) != 14*4 { + t.Fatalf("original Go produced %d rows, want 14 affine kernels x 4 offsets", len(want)) + } + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, runtime.GOOS+"/amd64", execution.triple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + got := strings.Split(strings.TrimSpace(string(execution.compileAndRunLLVM(t, "private-affine", input, driver, level))), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM rows = %d, original = %d", len(got), len(want)) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("affine value/compare row %d: original %s; LLVM %s", index, want[index], got[index]) + } + } + }) + } + }) +} + +func TestGoAMD64FixedLocalInputAffineMemoryFaults(t *testing.T) { + cases := []fixedLocalInputFaultCase{ + {"ADDQ $8,SI\nMOVQ -8(SI),CX", 8}, + {"SUBQ $8,SI\nMOVQ 8(SI),CX", 8}, + {"ADDQ SI,AX\nMOVQ (AX),CX", 8}, + {"LEAQ 8(SI)(AX*8),CX\nMOVQ -8(CX),AX", 8}, + {"MOVQ SI,BP\nADDQ AX,BP\nMOVQ (BP),CX", 8}, + {"LEAQ 8(SI),BP\nADDQ AX,BP\nMOVQ -8(BP),CX", 8}, + } + for _, register := range []string{"BP", "R14"} { + for index, suffix := range []string{"B", "W", "L", "Q"} { + for _, operands := range []string{"(%s),CX", "AX,(%s)"} { + operation := "MOVQ SI," + register + "\nMOV" + suffix + " " + fmt.Sprintf(operands, register) + cases = append(cases, fixedLocalInputFaultCase{operation, 1 << uint(index)}) + } + } + } + fixedLocalInputMemoryFaults(t, cases) +} diff --git a/amd64_fixed_local_affine_test.go b/amd64_fixed_local_affine_test.go new file mode 100644 index 000000000..552cb1b4c --- /dev/null +++ b/amd64_fixed_local_affine_test.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalInputAffineAddresses(t *testing.T) { + var original strings.Builder + for index, row := range []struct { + name, body string + valid bool + }{ + {"pointer plus number", "ADDQ AX,SI\nMOVQ (SI),AX", true}, + {"number plus pointer", "ADDQ SI,AX\nMOVQ (AX),AX", true}, + {"pointer minus number", "SUBQ AX,SI\nMOVQ (SI),AX", true}, + {"pointer immediate", "ADDQ $8,SI\nSUBQ $16,SI\nMOVQ (SI),AX", true}, + {"indexed LEA scale1", "LEAQ -64(SI)(AX*1),CX\nMOVQ (CX),AX", true}, + {"indexed LEA scale2", "LEAQ -64(SI)(AX*2),CX\nMOVQ (CX),AX", true}, + {"indexed LEA scale4", "LEAQ -64(SI)(AX*4),CX\nMOVQ (CX),AX", true}, + {"indexed LEA scale8", "LEAQ -64(SI)(AX*8),CX\nMOVQ (CX),AX", true}, + {"pointer increment", "INCQ SI\nDECQ SI\nMOVQ (SI),AX", true}, + {"pointer compare", "CMPQ SI,DI\nSETCS AL", true}, + {"pointer immediate compare", "CMPQ SI,$0\nSETCS AL", true}, + {"pointer spill compare", "MOVQ SI,8(SP)\nCMPQ DI,8(SP)\nSETCS AL", true}, + {"reverse spill compare", "MOVQ SI,8(SP)\nCMPQ 8(SP),DI\nSETCS AL", true}, + {"dead differing origin", "CMPQ AX,$0\nJEQ left\nMOVQ SI,CX\nJMP join\nleft:\nMOVQ DI,CX\njoin:", true}, + {"dead conditional spill", "CMPQ AX,$0\nJEQ join\nMOVQ SI,8(SP)\njoin:", true}, + {"same origin derived join", "CMPQ AX,$0\nJEQ left\nLEAQ 8(SI),CX\nJMP join\nleft:\nLEAQ 16(SI),CX\njoin:\nMOVQ (CX),AX", true}, + {"pointer plus pointer", "ADDQ DI,SI\nMOVQ (SI),AX", true}, + {"number minus pointer", "SUBQ SI,AX\nMOVQ (AX),AX", true}, + {"pointer bits escape", "ADDQ SI,AX", false}, + {"partial pointer arithmetic", "ADDL $8,SI\nMOVQ (SI),AX", true}, + {"pointer index", "LEAQ (SI)(DI*2),CX\nMOVQ (CX),AX", false}, + {"aliased pointer index", "LEAQ (SI)(SI*1),CX\nMOVQ (CX),AX", false}, + {"unknown index", "LEAQ (SI)(R8*2),CX\nMOVQ (CX),AX", false}, + {"native and numerical comparison", "CMPQ SI,AX\nSETCS AL", true}, + {"uninitialized spill compare", "CMPQ SI,8(SP)\nSETCS AL", false}, + {"different origin consumed", "CMPQ AX,$0\nJEQ left\nMOVQ SI,CX\nJMP join\nleft:\nMOVQ DI,CX\njoin:\nMOVQ (CX),AX", true}, + {"pointer and number join", "CMPQ AX,$0\nJEQ left\nMOVQ SI,CX\nJMP join\nleft:\nMOVQ AX,CX\njoin:\nMOVQ (CX),AX", false}, + {"conditional spill consumed", "CMPQ AX,$0\nJEQ join\nMOVQ SI,8(SP)\njoin:\nMOVQ 8(SP),CX\nMOVQ (CX),AX", false}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, source := fixedLocalInputFixture(t, row.body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + proved, err := proveX86FixedLocal(fn, sig) + if row.valid && err != nil { + t.Fatal("typed input address lost its original identity", err) + } + if !row.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved pointer acquired numerical or native access", err) + } + if row.name == "pointer plus pointer" || row.name == "partial pointer arithmetic" { + fixedLocalRequireNumericNativeRole(t, proved, SI) + } + if row.name == "number minus pointer" { + fixedLocalRequireNumericNativeRole(t, proved, AX) + } + }) + } + // Positive and negative proof rows alike remain valid original Go syntax. + // A refusal here must not masquerade as a translator provenance check. + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} diff --git a/amd64_fixed_local_alias_candidate_go127_test.go b/amd64_fixed_local_alias_candidate_go127_test.go new file mode 100644 index 000000000..897144879 --- /dev/null +++ b/amd64_fixed_local_alias_candidate_go127_test.go @@ -0,0 +1,62 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalAliasedFrameProgram() (string, string, []string) { + var assembly, declarations strings.Builder + var expected []string + for _, register := range []Reg{AX, BX, CX, DX, SI, DI, BP, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + for _, address := range []string{"MOVQ SP,%s", "LEAQ 16(SP),%s"} { + name := fmt.Sprintf("Value%d", len(expected)) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$160-16\n", name) + fmt.Fprintf(&assembly, address+"\n", register) + fmt.Fprintf(&assembly, "MOVQ $7,8(%s)\nMOVQ 8(%s),AX\nMOVQ AX,ret0+8(FP)\nRET\n", register, register) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + expected = append(expected, "0000000000000007") + } + } + return assembly.String(), declarations.String(), expected +} + +func TestGoAMD64FixedLocalAliasedFrameObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + assembly, declarations, expected := fixedLocalAliasedFrameProgram() + if len(expected) != 30 { + t.Fatal("complete GP/address family inventory changed", len(expected)) + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") { + t.Fatal("complete aliased scratch source bypassed actual frame capture") + } + var names []string + for index := range expected { + names = append(names, fmt.Sprintf("%s.Value%d", pkg.Path, index)) + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("complete aliased scratch source rejected", err) + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + compileLLVMToObject(t, llc, target.triple, "alias-frame.ll", "alias-frame.o", ir) + compileLLVMToObject(t, llc, target.triple, "alias-frame-opt.ll", "alias-frame-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} diff --git a/amd64_fixed_local_alias_candidate_runtime_go127_test.go b/amd64_fixed_local_alias_candidate_runtime_go127_test.go new file mode 100644 index 000000000..010d13a68 --- /dev/null +++ b/amd64_fixed_local_alias_candidate_runtime_go127_test.go @@ -0,0 +1,13 @@ +//go:build unix && go1.27 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalAliasedFrame(t *testing.T) { + assembly, declarations, expected := fixedLocalAliasedFrameProgram() + if len(expected) != 30 { + t.Fatal("complete aliased frame runtime inventory changed", len(expected)) + } + fixedLocalConstantResultsRuntime(t, assembly, declarations, expected) +} diff --git a/amd64_fixed_local_alias_candidate_test.go b/amd64_fixed_local_alias_candidate_test.go new file mode 100644 index 000000000..f89157b87 --- /dev/null +++ b/amd64_fixed_local_alias_candidate_test.go @@ -0,0 +1,126 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalPrivateAddressRoles(t *testing.T) { + for _, row := range []struct { + name, body string + want bool + }{ + {"private base", "LEAQ 8(SP),SI\nMOVQ AX,(SI)\nMOVQ (SI),AX", true}, + {"private alias", "MOVQ SP,SI\nLEAQ 8(SI),SI\nMOVQ AX,(SI)\nMOVQ (SI),AX", true}, + {"constant index", "MOVQ $2,CX\nLEAQ (SP)(CX*8),SI\nMOVQ AX,(SI)\nMOVQ (SI),AX", true}, + {"logical displacement", "LEAQ tmp-8(SP),SI\nMOVQ AX,(SI)\nMOVQ (SI),AX", true}, + {"uninitialized memory", "LEAQ 8(SP),SI\nMOVQ (SI),AX", false}, + {"narrow pointer", "LEAL 8(SP),SI\nMOVQ AX,(SI)", false}, + {"stack mutation", "LEAQ 8(SP),SP", false}, + {"unknown index", "LEAQ (SP)(CX*8),SI\nMOVQ AX,(SI)", false}, + {"pointer index", "MOVQ SP,CX\nLEAQ (SP)(CX*1),SI\nMOVQ AX,(SI)", false}, + {"index overflow", "MOVQ $0x7fffffffffffffff,CX\nLEAQ (SP)(CX*8),SI", false}, + {"outside frame", "LEAQ 4096(SP),SI\nMOVQ AX,(SI)", false}, + {"before frame", "LEAQ -8(SP),SI\nMOVQ AX,(SI)", false}, + {"pointer is not NUM", "LEAQ 8(SP),AX\nADDQ AX,0(SP)", false}, + {"unknown base", "LEAQ 8(SI),SI\nMOVQ AX,(SI)", false}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalGPFixture(t, row.body) + _, err := proveX86FixedLocal(fn, sig) + if row.want && err != nil || !row.want && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("private address role = %v: %v", row.want, err) + } + }) + } +} + +func TestGoAMD64FixedLocalPrivateAddressWidthAndFrame(t *testing.T) { + fn, _, _ := fixedLocalGPFixture(t, "LEAQ 8(SP),SI") + var address Instr + for _, ins := range fn.Instrs { + if ins.Op == "LEAQ" { + address = ins + } + } + for _, bits := range []int{0, 16, 32} { + for _, proof := range []*x86FixedLocalProof{nil, {frame: 64}} { + state := x86FixedLocalState{pointers: map[Reg]int64{SP: 0}, known: make(map[Reg]uint8)} + ins := address + ins.x86AddressBits = bits + known, err := fixedLocalPrivateAddressStep(ins, &state, proof) + want := bits == 0 && proof != nil + if !known || want && err != nil || !want && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("address bits %d, frame %v: known=%v error=%v", bits, proof != nil, known, err) + } + } + } +} + +func TestGoAMD64FixedLocalAliasedFrameCandidates(t *testing.T) { + var originals strings.Builder + type example struct{ name, source string } + var examples []example + for _, register := range []Reg{AX, BX, CX, DX, SI, DI, BP, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + for _, address := range []string{"MOVQ SP,%s", "LEAQ 16(SP),%s"} { + for _, access := range []string{"MOVQ 8(%s),AX", "MOVOU X0,8(%s)"} { + name := fmt.Sprintf("Value%d", len(examples)) + source := fmt.Sprintf("TEXT ·%s(SB),4,$160-0\n", name) + + fmt.Sprintf(address, register) + "\n" + fmt.Sprintf(access, register) + "\nRET\n" + originals.WriteString(source) + examples = append(examples, example{name, source}) + } + } + } + requireX86GoAssemblerResult(t, "amd64", originals.String(), true) + for _, row := range examples { + t.Run(row.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, row.source) + if err != nil { + t.Fatal(err) + } + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") { + t.Fatal("physical frame address through a GP register bypassed source observation") + } + if GoAMD64FixedLocalFrameBindingNeeded(file, "386") { + t.Fatal("amd64 query selected another source mode") + } + }) + } +} + +func TestGoAMD64FixedLocalAliasedCandidateSourceRoles(t *testing.T) { + for _, row := range []struct { + name, body string + want bool + }{ + {"copy chain", "MOVQ SP,R14\nMOVQ R14,R15\nMOVQ 8(R15),AX", true}, + {"inverse register chain", "MOVQ SP,R15\nMOVQ R15,R14\nMOVQ R14,AX\nMOVQ 8(AX),BX", true}, + {"MOVD source alias", "MOVD SP,R14\nMOVQ 8(R14),AX", true}, + {"MOVDQ2Q source alias", "MOVDQ2Q SP,R14\nMOVQ 8(R14),AX", true}, + {"overwritten possible alias", "MOVQ SP,R14\nMOVQ SI,R14\nMOVQ 8(R14),AX", true}, + {"address chain", "MOVQ SP,R14\nLEAQ 16(R14),R15\nMOVQ 8(R15),AX", true}, + {"CFG order", "JMP define\nuse:\nMOVQ 8(R15),AX\nRET\ndefine:\nMOVQ SP,R14\nMOVQ R14,R15\nJMP use", true}, + {"index", "MOVQ SP,R14\nMOVQ (AX)(R14*1),BX", true}, + {"unrelated base", "LEAQ 16(SP),R14\nMOVQ 8(R15),AX", false}, + {"no memory", "LEAQ 16(SP),R14", false}, + {"address only", "MOVQ SP,R14\nLEAQ 16(R14),AX", false}, + {"sized NOP", "MOVQ SP,R14\nNOPL 16(R14)", false}, + {"logical AUTO", "LEAQ tmp-16(SP),R14\nMOVQ (R14),AX", false}, + {"partial copy", "MOVL SP,R14\nMOVQ (R14),AX", false}, + {"partial address", "LEAL 16(SP),R14\nMOVQ (R14),AX", false}, + {"different source", "MOVQ SI,R14\nMOVQ (R14),AX", false}, + } { + t.Run(row.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$160-0\n"+row.body+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if got := GoAMD64FixedLocalFrameBindingNeeded(file, "amd64"); got != row.want { + t.Fatalf("source observation candidate = %v, want %v", got, row.want) + } + }) + } +} diff --git a/amd64_fixed_local_api_lease_inventory_unix_test.go b/amd64_fixed_local_api_lease_inventory_unix_test.go new file mode 100644 index 000000000..194972caf --- /dev/null +++ b/amd64_fixed_local_api_lease_inventory_unix_test.go @@ -0,0 +1,149 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Inventory is independent of the future scheduling change. These are the +// original full payloads, not new reduced numerical examples or source proofs. +func TestFixedLocalWideAPILeasePreservesCompletePayloadInventory(t *testing.T) { + for _, row := range []struct { + all bool + count int + }{{false, 28}, {true, 340}} { + assembly, declarations, cases := fixedLocalPackedRuntimeProgram(row.all) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal("original complete numerical source no longer parses", err) + } + if len(cases) != row.count || len(file.Funcs) != row.count { + t.Fatal("original complete numerical source changed", row.all, len(cases), err) + } + var assembled, declared strings.Builder + for batch := 0; batch < (row.count+63)/64; batch++ { + start, end, ok := fixedLocalObjectBatchRange(row.count, batch) + if !ok || end-start > 64 || end <= start { + t.Fatal("lost bounded complete-source batch", row.all, batch, start, end) + } + // Split at original TEXT boundaries without changing a single raw + // byte or renaming ValueN; the future real producer must agree. + for index := start; index < end; index++ { + begin := strings.Index(assembly, fmt.Sprintf("TEXT ·Value%d(SB)", index)) + finish := len(assembly) + if index+1 < row.count { + finish = strings.Index(assembly, fmt.Sprintf("TEXT ·Value%d(SB)", index+1)) + } + if begin < 0 || finish <= begin || file.Funcs[index].Sym != fmt.Sprintf("·Value%d", index) { + t.Fatal("missing or substituted original source member", index) + } + assembled.WriteString(assembly[begin:finish]) + fmt.Fprintf(&declared, "func Value%d(a, mask uint64) uint64\n", index) + } + } + if assembled.String() != assembly || declared.String() != declarations { + t.Fatal("source batches changed, cropped or duplicated original full ASM/declarations") + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + goOracle, cOracle := fixedLocalPackedRuntimeOracles(cases, triple) + if strings.Count(goOracle, "{Value") != row.count || strings.Count(cOracle, "{value") != row.count || + strings.Count(cOracle, "extern uint64_t value") != row.count { + t.Fatal("original Go/C oracle lost a complete source member", triple, row.count) + } + for index, entry := range cases { + goRow := fmt.Sprintf("{Value%d,%d,%d,%d,%d},\n", index, entry.width, entry.bits, entry.word, entry.mode) + cRow := fmt.Sprintf("{value%d,%d,%d,%d,%d},\n", index, entry.width, entry.bits, entry.word, entry.mode) + if strings.Count(goOracle, goRow) != 1 || strings.Count(cOracle, cRow) != 1 { + t.Fatal("original oracle source width/lane/merge/canary axis changed", triple, index) + } + } + for _, required := range []string{ + "const uint64_t values[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000),UINT64_C(0x123456789abcdef0),UINT64_C(0x7ff81234567890ab)};", + "const uint64_t masks[]={0,1,UINT64_MAX,UINT64_C(0x5555555555555555),UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_C(0x8000000000000000)};", + "for(unsigned v=0;v<6;v++) for(unsigned m=0;m<6;m++)", + "for(unsigned bit=0;bit<64;bit++)", + } { + if !strings.Contains(cOracle, required) { + t.Fatal("lease repair cannot reduce original C inputs/masks/whole-byte observations", required) + } + } + batches := fixedLocalPackedLeaseBatches(t, row.all, triple) + if len(batches) != (row.count+63)/64 { + t.Fatal("source batches omitted a member", triple, len(batches)) + } + var batchAssembly, batchDeclarations, batchRows, batchPrototypes strings.Builder + for index, batch := range batches { + start, end, ok := fixedLocalObjectBatchRange(row.count, index) + if !ok || batch.start != start || batch.end != end { + t.Fatal("source batch changed its exact original range", index, batch.start, batch.end) + } + observed, err := Parse(ArchAMD64, batch.assembly) + if err != nil || observed == nil || len(observed.Funcs) != end-start { + t.Fatal("source batch is not an independently complete source", index, err) + } + batchAssembly.WriteString(batch.assembly) + batchDeclarations.WriteString(batch.declarations) + batchRows.WriteString(batch.rows) + batchPrototypes.WriteString(batch.prototypes) + if strings.Count(batch.oracle, "{value") != end-start { + t.Fatal("C batch contains an omitted or foreign active oracle row", index) + } + } + if batchAssembly.String() != assembly || batchDeclarations.String() != declarations || + !strings.Contains(cOracle, batchRows.String()) || !strings.Contains(cOracle, batchPrototypes.String()) { + t.Fatal("the bounded source/C union differs from the complete original payload", triple) + } + } + } +} + +type fixedLocalPackedLeaseBatch struct { + start, end int + assembly, declarations, oracle, rows, prototypes string +} + +// This fixture partitions only original input text, before any source owner +// exists. Each future consumer must create a separate real package/capture for +// a batch; it cannot crop a File and borrow the full 340-function capability. +func fixedLocalPackedLeaseBatches(t *testing.T, all bool, triple string) []fixedLocalPackedLeaseBatch { + t.Helper() + assembly, declarations, cases := fixedLocalPackedRuntimeProgram(all) + _, oracle := fixedLocalPackedRuntimeOracles(cases, triple) + const table = "static const struct entry cases[]={\n" + prototypeStart := strings.Index(oracle, "extern uint64_t value0(") + prototypeEnd := strings.Index(oracle, "\nstruct entry {") + rowStart := strings.Index(oracle, table) + len(table) + rowEnd := strings.Index(oracle, "\n};\nint main(void)") + if prototypeStart < 0 || prototypeEnd <= prototypeStart || rowStart < len(table) || rowEnd <= rowStart { + t.Fatal("original C oracle lost its independent declaration/case boundaries") + } + text := strings.SplitAfter(assembly, "RET\n") + decls := strings.SplitAfter(declarations, "\n") + prototypes := strings.SplitAfter(oracle[prototypeStart:prototypeEnd], "\n") + rows := strings.SplitAfter(oracle[rowStart:rowEnd], "\n") + for name, entries := range map[string][]string{"assembly": text, "declarations": decls, "C prototypes": prototypes, "C rows": rows} { + if len(entries) != len(cases)+1 || entries[len(cases)] != "" { + t.Fatal("original fixture has ambiguous source boundaries", name, len(entries), len(cases)) + } + } + var batches []fixedLocalPackedLeaseBatch + for index := 0; index < (len(cases)+63)/64; index++ { + start, end, ok := fixedLocalObjectBatchRange(len(cases), index) + if !ok { + t.Fatal("missing bounded original source range", index) + } + batch := fixedLocalPackedLeaseBatch{ + start: start, end: end, + assembly: strings.Join(text[start:end], ""), + declarations: strings.Join(decls[start:end], ""), + rows: strings.Join(rows[start:end], ""), + prototypes: strings.Join(prototypes[start:end], ""), + } + batch.oracle = oracle[:prototypeStart] + batch.prototypes + oracle[prototypeEnd:rowStart] + batch.rows + oracle[rowEnd:] + batches = append(batches, batch) + } + return batches +} diff --git a/amd64_fixed_local_api_lease_lifecycle_test.go b/amd64_fixed_local_api_lease_lifecycle_test.go new file mode 100644 index 000000000..1c5a01e87 --- /dev/null +++ b/amd64_fixed_local_api_lease_lifecycle_test.go @@ -0,0 +1,61 @@ +package plan9asm + +import ( + "errors" + "testing" + "time" +) + +func TestFixedLocalFreshCaptureNeverRevivesPreviousAPILease(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + var previous Options + var previousSource *GoAMD64FrameSource + for _, capture := range []string{"first", "second", "third"} { + t.Run(capture, func(t *testing.T) { + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("fresh independent original-Go capture", err) + } + defer func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }() + if source == previousSource { + t.Fatal("a new API reused the closed previous observation") + } + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("fresh observation lost the original complete source binding", err) + } + if ir, err := Translate(file, bound); err != nil || ir == "" { + t.Fatal("fresh source cannot independently consume the same original payload", err) + } + if previousSource != nil { + if ir, err := Translate(file, previous); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("new capture revived a previous API's closed/cancelled binding", err) + } + } + previous, previousSource = bound, source + }) + } +} + +func TestFixedLocalFreshAPILeaseRetainsOriginalStageBudget(t *testing.T) { + now := time.Date(2026, 10, 21, 0, 0, 0, 0, time.UTC) + for _, delay := range []time.Duration{0, 4 * time.Minute, 8 * time.Minute} { + start := now.Add(delay) + if got := runtimeOracleStageDeadline(start, time.Time{}, false); got != start.Add(5*time.Minute) { + t.Fatal("API lease changed the original five-minute bound", delay, got) + } + } + deadline := now.Add(4 * time.Minute) + if got := runtimeOracleStageDeadline(now, deadline, true); got != deadline.Add(-3*time.Second) { + t.Fatal("fresh API lease escaped the real test deadline/cleanup reserve", got) + } + if got := runtimeOracleStageDeadline(deadline, deadline, true); !got.Before(deadline) { + t.Fatal("new API revived an exhausted test lifetime", got) + } +} diff --git a/amd64_fixed_local_api_lease_test.go b/amd64_fixed_local_api_lease_test.go new file mode 100644 index 000000000..0dc47c4ef --- /dev/null +++ b/amd64_fixed_local_api_lease_test.go @@ -0,0 +1,304 @@ +package plan9asm + +import ( + "go/ast" + "go/parser" + "go/token" + "strconv" + "strings" + "testing" +) + +// These are scheduling checks, not fabricated source observations. Every API +// still has to capture and consume its complete actual Go package independently. +func TestFixedLocalAPIsHaveIndependentBoundedCaptureLeases(t *testing.T) { + functions := fixedLocalLeaseFunctions(t, "amd64_fixed_local_frame_test.go") + helper := functions["fixedLocalFreshAPIs"] + if helper == nil { + t.Fatal("three sequential APIs still share one source/binding lease; missing per-API capture owner") + } + var scopes []*ast.FuncLit + ast.Inspect(helper.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok || fixedLocalLeaseCallName(call) != "t.Run" || len(call.Args) != 2 { + return true + } + if scope, ok := call.Args[1].(*ast.FuncLit); ok { + scopes = append(scopes, scope) + } + return true + }) + if len(scopes) != 1 { + t.Fatalf("each selected API needs one independent synchronous owner, got %d", len(scopes)) + } + scope := scopes[0] + counts := fixedLocalLeaseCalls(scope.Body) + for _, name := range []string{"runtimeOracleStageContext", "CaptureGoAMD64FrameSource", "source.preprocess", "Parse", "source.BindFixedLocal", "fixedLocalAPI"} { + if counts[name] != 1 { + t.Errorf("per-API owner must retain exactly one %s, got %d", name, counts[name]) + } + } + for _, name := range []string{"runtimeOracleProofContext", "requireGoAMD64FrameCapture", "fixedLocalAPIs", "t.Parallel"} { + if counts[name] != 0 { + t.Errorf("per-API owner cannot extend, prematurely cancel, share or overlap its lease: %s", name) + } + } + all := fixedLocalLeaseCalls(helper.Body) + for _, name := range []string{"runtimeOracleStageContext", "CaptureGoAMD64FrameSource", "source.BindFixedLocal"} { + if all[name] != counts[name] { + t.Errorf("%s escaped the API lifetime", name) + } + } + if all["runtimeOracleProofContext"] != 0 { + t.Error("an outer proof context cannot extend the independent five-minute API stage") + } + + var stage, capture, bind, consume token.Pos + var cancel, close bool + ast.Inspect(scope.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + switch fixedLocalLeaseCallName(call) { + case "runtimeOracleStageContext": + stage = call.Pos() + fixedLocalLeaseRequireArguments(t, call, "t") + case "CaptureGoAMD64FrameSource": + capture = call.Pos() + fixedLocalLeaseRequireArguments(t, call, "ctx", "pkg", "request") + case "source.BindFixedLocal": + bind = call.Pos() + fixedLocalLeaseRequireArguments(t, call, "ctx", "pkg", "file", "options") + case "fixedLocalAPI": + consume = call.Pos() + fixedLocalLeaseRequireArguments(t, call, "t", "pkg", "file", "bound", "api", "check", "expanded") + } + return true + }) + for _, statement := range scope.Body.List { + deferred, ok := statement.(*ast.DeferStmt) + if !ok { + continue + } + if fixedLocalLeaseCallName(deferred.Call) == "cancel" { + cancel = true + } + if closure, ok := deferred.Call.Fun.(*ast.FuncLit); ok { + close = close || fixedLocalLeaseCalls(closure.Body)["source.Close"] == 1 + } + } + if !cancel || !close { + t.Error("each API must cancel and close its own source before the next API starts, including fatal paths") + } + if !(stage < capture && capture < bind && bind < consume) { + t.Error("each complete source capture and binding must precede only that API's consumption") + } +} + +func TestFixedLocalCompleteConsumersDoNotShareAPILease(t *testing.T) { + for _, owner := range []struct{ filename, helper string }{ + {"amd64_fixed_local_conversion_test.go", "fixedLocalConversionActualObjects"}, + {"amd64_fixed_local_packed_matrix_test.go", "fixedLocalPackedActualObjects"}, + {"amd64_cmov_fixed_local_test.go", "x86CMOVFixedLocalActualObjects"}, + {"amd64_fixed_local_conversion_runtime_unix_test.go", "fixedLocalConversionRuntimeBatch"}, + {"amd64_fixed_local_packed_runtime_test.go", "fixedLocalPackedRuntimeAPIs"}, + {"amd64_fixed_local_data_matrix_go127_test.go", "fixedLocalDataTargetConsumers"}, + {"amd64_fixed_local_data_go127_test.go", "TestGoAMD64FixedLocalReadonlyDataThreeTargetObjects"}, + {"amd64_fixed_local_data_runtime_go127_test.go", "TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalReadonlyData"}, + {"amd64_fixed_local_transport_runtime_test.go", "TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalTransportByteValues"}, + {"amd64_fixed_local_transport_test.go", "TestGoAMD64FixedLocalTransportActualObjects"}, + {"amd64_fixed_local_nontemporal_test.go", "fixedLocalNonTemporalActualObjects"}, + {"amd64_aes_family_fixed_local_test.go", "TestX86AESFamilyActualBoundFixedLocalConsumers"}, + } { + t.Run(owner.helper, func(t *testing.T) { + function := fixedLocalLeaseFunctions(t, owner.filename)[owner.helper] + if function == nil { + t.Fatal("missing complete independently scheduled source owner", owner.helper) + } + calls := fixedLocalLeaseCalls(function.Body) + if calls["fixedLocalFreshAPIs"] != 1 { + t.Error("complete source must use the shared fresh-capture API owner exactly once") + } + for _, name := range []string{"fixedLocalAPIs", "requireGoAMD64FrameCapture", "runtimeOracleStageContext", "runtimeOracleProofContext"} { + if calls[name] != 0 { + t.Errorf("outer %s would start or share a lease before the current API", name) + } + } + ast.Inspect(function.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if ok { + if name := fixedLocalLeaseCallName(call); name == "source.BindFixedLocal" || name == "observer.BindFixedLocal" { + t.Error("complete source must not transport an outer API's bound options") + } + } + return true + }) + }) + } + + // CMOV runtime currently owns one public API. Do not silently multiply its + // expensive execution matrix while repairing the three-API object owners. + function := fixedLocalLeaseFunctions(t, "amd64_cmov_fixed_local_runtime_test.go")["x86CMOVFixedLocalRuntimeBatch"] + if function == nil || fixedLocalLeaseCalls(function.Body)["Translate"] != 1 || fixedLocalLeaseCalls(function.Body)["fixedLocalFreshAPIs"] != 0 { + t.Fatal("single-API CMOV runtime ownership changed") + } +} + +func TestFixedLocalWideCompileOnlyHasThreeCompleteAPIOwners(t *testing.T) { + functions := fixedLocalLeaseFunctions(t, "amd64_fixed_local_packed_runtime_test.go") + owners := []struct{ root, api string }{ + {"TestGoAMD64FixedLocalPackedWideRuntimeOracleCompileOnly", "Translate"}, + {"TestGoAMD64FixedLocalPackedWideRuntimeOracleModuleCompileOnly", "Module"}, + {"TestGoAMD64FixedLocalPackedWideRuntimeOracleGoModuleCompileOnly", "GoModule"}, + } + want := make(map[string]bool) + for _, owner := range owners { + want[owner.root] = true + } + for name := range functions { + if strings.HasPrefix(name, "TestGoAMD64FixedLocalPackedWideRuntimeOracle") && !want[name] { + t.Error("unknown or duplicated complete-wide API owner", name) + } + } + for _, owner := range owners { + t.Run(owner.api, func(t *testing.T) { + function := functions[owner.root] + if function == nil { + t.Fatal("missing independent complete 340-kernel API owner", owner.root) + } + var calls []*ast.CallExpr + ast.Inspect(function.Body, func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok && fixedLocalLeaseCallName(call) == "fixedLocalPackedRuntimeAPIs" { + calls = append(calls, call) + } + return true + }) + if len(calls) != 1 || len(calls[0].Args) != 5 { + t.Fatal("API owner must call the unchanged full wide program once, without three-API accumulation") + } + call := calls[0] + for index, name := range []string{"t", "execution", "true", "false"} { + identifier, ok := call.Args[index].(*ast.Ident) + if !ok || identifier.Name != name { + t.Fatal("wide compile-only owner lost its original complete-source/execute=false role") + } + } + apis, ok := call.Args[4].(*ast.CompositeLit) + if !ok || len(apis.Elts) != 1 { + t.Fatal("one top-level owner must execute exactly one API, not crop the source") + } + literal, ok := apis.Elts[0].(*ast.BasicLit) + if !ok || literal.Kind != token.STRING { + t.Fatal("API owner must retain an explicit API identity") + } + api, err := strconv.Unquote(literal.Value) + if err != nil || api != owner.api { + t.Fatal("substituted or duplicated complete-source API", api, err) + } + }) + } + runtime := functions["fixedLocalPackedRuntime"] + if runtime == nil || fixedLocalLeaseCalls(runtime.Body)["fixedLocalPackedRuntimeAPIs"] != 1 { + t.Error("original native runtime must still execute its complete three-API plan") + } else { + apis := make(map[string]int) + ast.Inspect(runtime.Body, func(node ast.Node) bool { + literal, ok := node.(*ast.BasicLit) + if ok && literal.Kind == token.STRING { + value, err := strconv.Unquote(literal.Value) + if err == nil { + apis[value]++ + } + } + return true + }) + for _, api := range []string{"Translate", "Module", "GoModule"} { + if apis[api] != 1 { + t.Error("original runtime lost one complete public API", api) + } + } + } + helper := functions["fixedLocalPackedRuntimeAPIs"] + if helper != nil { + calls := fixedLocalLeaseCalls(helper.Body) + for _, name := range []string{"fixedLocalPackedLeaseBatches", "fixedLocalFreshAPIs"} { + if calls[name] != 1 { + t.Errorf("each API must own complete <=64-function source batches: %s count %d", name, calls[name]) + } + } + if calls["fixedLocalPackedRunOriginal"] != 1 { + t.Error("the complete original Go oracle must remain independently compiled/executed once") + } + pipelines := make(map[string]int) + ast.Inspect(helper.Body, func(node ast.Node) bool { + literal, ok := node.(*ast.BasicLit) + if ok && literal.Kind == token.STRING { + value, err := strconv.Unquote(literal.Value) + if err == nil { + pipelines[value]++ + } + } + return true + }) + for _, name := range []string{"backend_O0", "backend_O2", "IR_O2/backend_O2"} { + if pipelines[name] != 1 { + t.Error("every complete source batch must retain the three independent LLVM pipelines", name) + } + } + } + + fixedLocalPackedRequireNativeBatchOwners(t) +} + +func fixedLocalLeaseFunctions(t *testing.T, filename string) map[string]*ast.FuncDecl { + t.Helper() + syntax, err := parser.ParseFile(token.NewFileSet(), filename, nil, 0) + if err != nil { + t.Fatal(err) + } + functions := make(map[string]*ast.FuncDecl) + for _, declaration := range syntax.Decls { + if function, ok := declaration.(*ast.FuncDecl); ok { + functions[function.Name.Name] = function + } + } + return functions +} + +func fixedLocalLeaseCalls(node ast.Node) map[string]int { + counts := make(map[string]int) + ast.Inspect(node, func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok { + counts[fixedLocalLeaseCallName(call)]++ + } + return true + }) + return counts +} + +func fixedLocalLeaseCallName(call *ast.CallExpr) string { + if name, ok := call.Fun.(*ast.Ident); ok { + return name.Name + } + if selector, ok := call.Fun.(*ast.SelectorExpr); ok { + if receiver, ok := selector.X.(*ast.Ident); ok { + return receiver.Name + "." + selector.Sel.Name + } + } + return "" +} + +func fixedLocalLeaseRequireArguments(t *testing.T, call *ast.CallExpr, names ...string) { + t.Helper() + if len(call.Args) != len(names) { + t.Errorf("%s changed complete source/binding arguments", fixedLocalLeaseCallName(call)) + return + } + for index, name := range names { + identifier, ok := call.Args[index].(*ast.Ident) + if !ok || identifier.Name != name { + t.Errorf("%s substituted argument %d: want %s", fixedLocalLeaseCallName(call), index, name) + } + } +} diff --git a/amd64_fixed_local_auto.go b/amd64_fixed_local_auto.go new file mode 100644 index 000000000..19ab49bc4 --- /dev/null +++ b/amd64_fixed_local_auto.go @@ -0,0 +1,183 @@ +package plan9asm + +import ( + "math" + "reflect" + "strings" +) + +// NAME_AUTO is a source address role, not an input/storage permission. Go +// obj6 adds deltasp-bpsize to its explicit offset. This bounded observer has +// no CALL/PUSH/POP/ADJSP producer, so the lexical bias is exactly TEXT's local +// size; separately proved SP relocation still contributes its native offset. +// Replay the original operand because parsing constant expressions alone +// loses the difference between tmp-8(SP) and physical -8(SP). +func fixedLocalMemoryAddress(ins Instr, index int, frame int64) (MemRef, error) { + fail := func() (MemRef, error) { + return MemRef{}, goAMD64FrameContext("fixed local original memory address role is not closed") + } + if index < 0 || index >= len(ins.Args) { + return fail() + } + operand := ins.Args[index] + addressMove := false + if index == 0 { + if view, known, err := x86NamedFrameMoveAddressInstruction(ins, "amd64"); known { + if err != nil { + return fail() + } + operand, addressMove = view.Args[0], true + } + } + if operand.Kind != OpMem { + return fail() + } + memory := operand.Mem + if memory.Base != SP { + if memory.OffRaw != "" { + return fail() + } + return memory, nil + } + op, rest := splitOpcode(ins.Raw) + literals := splitTopLevelCSV(rest) + if !strings.EqualFold(op, string(ins.Op)) || len(literals) != len(ins.Args) { + return fail() + } + parsed, err := parseOperandForArch(ArchAMD64, literals[index]) + if err != nil || !reflect.DeepEqual(parsed, ins.Args[index]) { + return fail() + } + spelling := strings.TrimSpace(literals[index]) + if addressMove { + spelling = strings.TrimSpace(strings.TrimPrefix(spelling, "$")) + } + literal := x86ResultAUTOLiteral.FindStringSubmatch(spelling) + if literal == nil { + // Indexing/segment/symbol roles are independently validated by the + // native observer; none may be relabeled as a named local offset. + if memory.OffRaw != "" { + return fail() + } + return memory, nil + } + prefix := strings.TrimSpace(literal[1]) + offset, named := parseNamedStackConstantOffset(prefix) + if validPPIdentifier(prefix) { + offset, named = 0, true + } + if !named { + if memory.OffRaw != "" { + return fail() + } + return memory, nil // Numeric, unnamed (SP) remains post-prologue SP. + } + if offset != memory.Off || frame <= 0 || offset > 0 && frame > math.MaxInt64-offset { + return fail() + } + memory.Off += frame + memory.OffRaw = "" + return memory, nil +} + +// This query resolves only a complete numerical constant index. It neither +// changes the emitted address nor supplies a memory origin, extent or access. +// All arithmetic is checked before an original private/literal bound is used. +func fixedLocalConstantDisplacement(memory MemRef, state *x86FixedLocalState) (int64, bool) { + if memory.OffRaw != "" || memory.Sym != "" || memory.Segment != "" || memory.IndexExt != "" { + return 0, false + } + if memory.Index == "" { + return memory.Off, memory.Scale == 0 + } + root, valid := x86CallerGP(memory.Index) + value, known := state.constants[root] + scale := memory.Scale + if !valid || root != memory.Index || !known || + !fixedLocalNumericRead(Operand{Kind: OpReg, Reg: root}, 8, state) || + scale != 1 && scale != 2 && scale != 4 && scale != 8 { + return 0, false + } + if value > math.MaxInt64/scale || value < math.MinInt64/scale { + return 0, false + } + delta := value * scale + if delta > 0 && memory.Off > math.MaxInt64-delta || delta < 0 && memory.Off < math.MinInt64-delta { + return 0, false + } + return memory.Off + delta, true +} + +// Reads, writes and complete pointer-spill lookups use the same resolved +// byte address. An indexed read must not borrow the unindexed slot's role. +func fixedLocalPrivateOffset(memory MemRef, state *x86FixedLocalState) (int64, bool) { + base, private := state.pointers[memory.Base] + displacement, known := fixedLocalConstantDisplacement(memory, state) + if !private || !known || displacement > 0 && base > math.MaxInt64-displacement || + displacement < 0 && base < math.MinInt64-displacement { + return 0, false + } + return base + displacement, true +} + +// Descriptive bare NAME_AUTO is offset zero, not an undefined preprocessor +// expression. This syntax query does not grant a native memory certificate. +func x86ConditionalMoveOffsetKnown(memory MemRef) bool { + if memory.OffRaw == "" { + return true + } + if memory.Base != SP || memory.Sym != "" { + return false + } + if offset, named := parseNamedStackConstantOffset(memory.OffRaw); named { + return offset == memory.Off + } + return validPPIdentifier(memory.OffRaw) && memory.Off == 0 +} + +func fixedLocalConditionalMoveStep(ins *Instr, state *x86FixedLocalState, sig FuncSig, proof *x86FixedLocalProof, residue int) (bool, error) { + form, known, err := x86ConditionalMoveProperties(ins.Op, *ins, "amd64") + if !known { + return false, nil + } + fail := func() (bool, error) { + return true, goAMD64FrameContext("fixed local CMOV flags or conditional value roles are not defined") + } + required := x86ByteConditionFlags(form.condition) + if err != nil || required == 0 || state.flags&required != required { + return fail() + } + if ins.Args[0].Kind == OpFP { + // Declared numerical FP inputs use the shared GP effect. This value + // transport path retains its existing pointer and private-spill roles. + value, err := fixedLocalReadValue(ins, 0, state, sig, proof, residue, int64(form.bits/8), false) + if err != nil || value.pointer || value.opaque { + return fail() + } + old, err := fixedLocalReadValue(ins, 1, state, sig, proof, residue, int64(form.bits/8), false) + if err != nil || old.pointer || old.opaque { + return fail() + } + return false, nil + } + width := int64(form.bits / 8) + old, err := fixedLocalReadValue(ins, 1, state, FuncSig{}, proof, residue, width, false) + if err != nil { + return true, err + } + // Both potential values remain mandatory. In particular memory is read + // unconditionally; selecting it never grants logical FP pointer imports. + value, err := fixedLocalReadValue(ins, 0, state, FuncSig{}, proof, residue, width, false) + if err != nil { + return true, err + } + if value.opaque && old.opaque && !value.identity.equal(old.identity) && + (!fixedLocalOriginOwners(value.identity, proof) || !fixedLocalOriginOwners(old.identity, proof)) { + return fail() + } + value, valid := mergeFixedLocalScalarValues(value, old) + if !valid { + return fail() + } + return true, fixedLocalWriteValue(ins, 1, 0, value, state, FuncSig{}, proof, residue, width) +} diff --git a/amd64_fixed_local_binary_effect_test.go b/amd64_fixed_local_binary_effect_test.go new file mode 100644 index 000000000..7727851dc --- /dev/null +++ b/amd64_fixed_local_binary_effect_test.go @@ -0,0 +1,336 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +// This independent inventory includes PADDD's source alias separately from +// the 27 canonical legacy XMM instructions. No production spec defines it. +func fixedLocalBinaryEffectNames() []string { + names := append([]string(nil), x86LegacyIntegerArithmeticTestNames...) + return append(names, "PMULLD", "PMULLW", "PMULHW", "PMULHUW", "PMULHRSW", + "PMULDQ", "PMULULQ", "PAVGB", "PAVGW", "PMADDWL", "PMADDUBSW") +} + +func TestGoAMD64FixedLocalTypedBinaryEffects(t *testing.T) { + names := fixedLocalBinaryEffectNames() + if len(names) != 28 { + t.Fatalf("independent legacy family inventory: %d, want 27 + one alias", len(names)) + } + var original strings.Builder + ordinal := 0 + for _, name := range names { + for _, input := range []string{"16(SP)", "X1", "X0"} { + t.Run(name+"/"+input, func(t *testing.T) { + operation := name + " " + input + ",X0" + source, sig, _ := x86PrivateAlignedFrameAuditFixture(t, "MOVOU 16(SP),X1\n"+operation) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", ordinal), 1)) + ordinal++ + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + proved, err := proveX86FixedLocal(file.Funcs[0], sig) + if err != nil { + t.Fatal("initialized complete legacy family source rejected", err) + } + observed := 0 + for _, ins := range proved.Instrs { + if string(ins.Op) != name { + continue + } + observed++ + access, found := ins.x86FixedLocalMemory[0] + if found != (input == "16(SP)") || found && + (access.width != 16 || access.write || access.nativeInput || access.owner != proved.x86FixedLocal) { + t.Fatalf("wrong exact private memory role: %+v", access) + } + } + if observed != 1 { + t.Fatalf("proved source retained %d target instructions, want one", observed) + } + candidate, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$160-40\n"+operation+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if GoAMD64FixedLocalFrameBindingNeeded(candidate, "amd64") != (input == "16(SP)") || + GoAMD64FixedLocalFrameBindingNeeded(candidate, "386") { + t.Fatal("candidate query differs from the typed memory role") + } + }) + } + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalTypedBinaryAliasBytes(t *testing.T) { + for _, name := range fixedLocalBinaryEffectNames() { + for _, old := range []uint64{0xffff, 0x804020100000ffff, ^uint64(0)} { + t.Run(fmt.Sprintf("%s/%x", name, old), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-0\n"+name+" X1,X0\nRET\n") + if err != nil { + t.Fatal(err) + } + state := x86FixedLocalState{ + vectors: map[Reg]uint64{x86CallerVectorRoot("X0"): old, x86CallerVectorRoot("X1"): 0xffff}, + flags: x86ByteFlagCF, + } + if err := fixedLocalVectorStep(&file.Funcs[0].Instrs[1], &state, nil, 0); err != nil { + t.Fatal(err) + } + if state.vectors[x86CallerVectorRoot("X0")] != old || state.flags != x86ByteFlagCF { + t.Fatal("legacy arithmetic changed upper vector definedness or FLAGS", state) + } + }) + } + } +} + +func TestGoAMD64FixedLocalTypedBinaryMetadata(t *testing.T) { + for _, name := range fixedLocalBinaryEffectNames() { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-0\n"+name+" X1,X0\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || effect.vector.upperWrite != x86TypedVectorUpperPreserve || + effect.vector.memoryAlignment != 16 || len(effect.operands) != 2 { + t.Fatal("shared descriptor lost the complete validated alias/storage role", name, effect, err) + } + lowercase := ins + lowercase.Op = Op(strings.ToLower(name)) + lowerEffect, lowerKnown, lowerErr := x86TypedInstructionByteEffect(lowercase.Op, lowercase, "amd64") + if !lowerKnown || lowerErr != nil || !reflect.DeepEqual(lowerEffect, effect) { + t.Fatal("negative prefix filter changed the case-insensitive family contract", name, lowerErr) + } + effect.vector = x86TypedVectorEffect{} + state := x86FixedLocalState{vectors: map[Reg]uint64{ + x86CallerVectorRoot("X0"): ^uint64(0), x86CallerVectorRoot("X1"): ^uint64(0), + }} + if err := fixedLocalTypedByteStep(&ins, &state, FuncSig{}, nil, 0, effect); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unregistered default metadata authorized a vector effect", name, err) + } + } + for _, name := range []string{"PADDW", "PMULLW", "PMULULQ", "PAVGB", "PMADDWL"} { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-0\n"+name+" AX,M1\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || len(effect.operands) != 2 || effect.operands[0].operand.Reg != "M0" || + effect.operands[0].width != 8 || effect.vector != (x86TypedVectorEffect{}) { + t.Fatal("GP spelling changed the physical MMX bank or granted XMM metadata", name, effect, err) + } + } +} + +func TestGoAMD64FixedLocalTypedBinaryBoundaries(t *testing.T) { + for _, name := range []string{"PSUBSW", "PMULLD", "PMULHW", "PMULDQ", "PAVGB", "PMADDUBSW"} { + source, sig, _ := x86PrivateAlignedFrameAuditFixture(t, name+" 16(SP),X0") + for _, row := range []struct{ name, from, to string }{ + {"missing memory byte", "MOVQ CX,24(SP)", "MOVB CX,24(SP)"}, + {"unknown destination", "MOVOU 48(SP),X0", "MOVOU 48(SP),X1"}, + {"unaligned memory", name + " 16(SP)", name + " 17(SP)"}, + {"outside frame", name + " 16(SP)", name + " 160(SP)"}, + {"unknown source", name + " 16(SP)", name + " X2"}, + {"MMX source bank", name + " 16(SP),X0", name + " 16(SP),M0"}, + {"unbound FP", name + " 16(SP)", name + " a+0(FP)"}, + {"raw source", name + " 16(SP),X0", "BYTE $0x90"}, + {"private pointer spill", "MOVQ AX,16(SP)", "MOVQ SP,16(SP)"}, + } { + t.Run(name+"/"+row.name, func(t *testing.T) { + changed := strings.Replace(source, row.from, row.to, 1) + if changed == source { + t.Fatal("negative fixture did not change the complete source") + } + file, err := Parse(ArchAMD64, changed) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved native bytes gained fixed-local authority", err) + } + }) + } + t.Run(name+"/opaque pointer spill", func(t *testing.T) { + source, sig, _ := x86PrivateAlignedFrameAuditTypedFixture(t, name+" 16(SP),X0", + "package frameaudit\nfunc Value(a *uint64, b, c, d uint64) uint64\n") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); err != nil { + t.Fatal("complete native spill bits were rejected as vector input", err) + } + }) + } +} + +func TestGoAMD64FixedLocalTypedBinaryObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + assembly, declarations, _, _ := fixedLocalBinaryRuntimeProgram(t, "x86_64-unknown-linux-gnu") + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + var names []string + for index := range fixedLocalBinaryEffectNames() { + names = append(names, fmt.Sprintf("%s.Value%d", pkg.Path, index)) + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + compileLLVMToObject(t, llc, target.triple, "legacy-binary.ll", "legacy-binary.o", ir) + compileLLVMToObject(t, llc, target.triple, "legacy-binary-opt.ll", "legacy-binary-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +// One complete original source serves both the object matrix and native +// differential test. Observe both result halves and a separate input canary. +func fixedLocalBinaryRuntimeProgram(t *testing.T, triple string) (string, string, string, string) { + t.Helper() + return fixedLocalBinaryRuntimeProgramForNames(t, triple, fixedLocalBinaryEffectNames()) +} + +func fixedLocalBinaryRuntimeProgramForNames(t *testing.T, triple string, names []string) (string, string, string, string) { + t.Helper() + var assembly, declarations, goRows, cRows, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, name := range names { + source, _, _ := x86PrivateAlignedFrameAuditFixture(t, name+" 16(SP),X0") + source = strings.Replace(source, "$160-40", "$160-48", 1) + source = strings.Replace(source, "MOVQ SP,R11", "MOVQ mode+32(FP),R8\nMOVQ SP,R11", 1) + source = strings.Replace(source, "MOVQ X0,AX", `MOVOU X0,48(SP) +CMPQ R8,$1 +JEQ high +CMPQ R8,$2 +JEQ canary +MOVQ 48(SP),AX +JMP done +high: +MOVQ 56(SP),AX +JMP done +canary: +MOVQ 16(SP),AX +done:`, 1) + source = strings.Replace(source, "ret+32(FP)", "ret+40(FP)", 1) + function := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(source, "·Value", "·"+function, 1)) + fmt.Fprintf(&declarations, "func %s(a,b,c,d,mode uint64) uint64\n", function) + fmt.Fprintf(&goRows, "%s,\n", function) + fmt.Fprintf(&cRows, "value%d,\n", index) + label := runtimeCAsmLabel(prefix + "example.invalid/framefixture." + function) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t,uint64_t,uint64_t,uint64_t,uint64_t) __asm__(%s);\n", index, label) + } + const values = `0,1,0x7f7f7f7f7f7f7f7f,0x8080808080808080, +0x7fff7fff7fff7fff,0x8000800080008000,0x7fffffff7fffffff, +0x8000000080000000,0xffffffffffffffff,0x0123456789abcdef` + original := `package main +import "fmt" +` + declarations.String() + ` +func main() { + rows := []func(uint64,uint64,uint64,uint64,uint64) uint64{ +` + goRows.String() + ` + } + values := []uint64{` + values + `} + for n, call := range rows { + for v, value := range values { + for mode := uint64(0); mode < 3; mode++ { + got := call(value,values[(v+1)%len(values)],^value,values[(v+len(values)-1)%len(values)],mode) + fmt.Printf("%d %d %d %016x\n",n,v,mode,got) + } + } + } +} +` + driver := `#include +#include +` + runtimeCAsmLabelPreamble + cDeclarations.String() + ` +int main(void) { + uint64_t (*rows[])(uint64_t,uint64_t,uint64_t,uint64_t,uint64_t) = { +` + cRows.String() + ` + }; + const uint64_t values[] = {` + values + `}; + const size_t count = sizeof(values)/sizeof(values[0]); + for (size_t n = 0; n < sizeof(rows)/sizeof(rows[0]); n++) { + for (size_t v = 0; v < count; v++) { + for (unsigned mode = 0; mode < 3; mode++) { + uint64_t got = rows[n](values[v],values[(v+1)%count],~values[v],values[(v+count-1)%count],mode); + printf("%zu %zu %u %016llx\n",n,v,mode,(unsigned long long)got); + } + } + } + return 0; +} +` + return assembly.String(), declarations.String(), original, driver +} + +func BenchmarkFixedLocalLegacyVectorStep(b *testing.B) { + for _, operation := range []string{"PADDQ X1,X0", "PMADDWL X1,X0", "CMPPS X1,X0,$0"} { + b.Run(strings.Fields(operation)[0], func(b *testing.B) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-0\n"+operation+"\nRET\n") + if err != nil { + b.Fatal(err) + } + state := x86FixedLocalState{vectors: map[Reg]uint64{ + x86CallerVectorRoot("X0"): ^uint64(0), x86CallerVectorRoot("X1"): ^uint64(0), + }} + instruction := &file.Funcs[0].Instrs[1] + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := fixedLocalVectorStep(instruction, &state, nil, 0); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkFixedLocalLegacyVectorProof(b *testing.B) { + for _, operation := range []string{"PADDQ X1,X0", "PMADDWL X1,X0", "CMPPS X1,X0,$0"} { + b.Run(strings.Fields(operation)[0], func(b *testing.B) { + source := "TEXT ·Value(SB),4,$64-16\nMOVQ a+0(FP),AX\n" + + "MOVQ AX,0(SP)\nMOVQ AX,8(SP)\nMOVOU 0(SP),X0\nMOVOU X0,X1\n" + + strings.Repeat(operation+"\n", 8) + "MOVQ X0,AX\nMOVQ AX,ret+8(FP)\nRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + b.Fatal(err) + } + sig := FuncSig{Name: "Value", Args: []LLVMType{I64}, Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := proveX86FixedLocal(file.Funcs[0], sig); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/amd64_fixed_local_binary_runtime_test.go b/amd64_fixed_local_binary_runtime_test.go new file mode 100644 index 000000000..bb04824a7 --- /dev/null +++ b/amd64_fixed_local_binary_runtime_test.go @@ -0,0 +1,61 @@ +//go:build unix + +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalTypedBinary(t *testing.T) { + fixedLocalBinaryRuntime(t, fixedLocalBinaryEffectNames(), 28) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalTypedLogical(t *testing.T) { + fixedLocalBinaryRuntime(t, x86LegacyLogicalMemoryNames, 12) +} + +func fixedLocalBinaryRuntime(t *testing.T, names []string, count int) { + t.Helper() + if len(names) != count { + t.Fatalf("independent runtime kernel inventory: %d, want %d", len(names), count) + } + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + t.Setenv("GOCACHE", t.TempDir()) + assembly, declarations, original, driver := fixedLocalBinaryRuntimeProgramForNames(t, execution.triple, names) + want := strings.Split(strings.TrimSpace(string(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original))), "\n") + if len(want) != count*10*3 { + t.Fatalf("original Go produced %d rows, want %d kernels x 10 values x 3 observations", len(want), count) + } + pkg, request, _, options := fixedLocalTransportFixture(t, + assembly, declarations, runtime.GOOS+"/amd64", execution.triple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + output := execution.compileAndRunLLVM(t, "private-binary", input, driver, level) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM row count = %d, original = %d", len(got), len(want)) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("private packed value/canary row %d: original %s; LLVM %s", index, want[index], got[index]) + } + } + }) + } + }) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("original Go and LLVM amd64 execution via Rosetta, not native x86 hardware") + } +} diff --git a/amd64_fixed_local_binding.go b/amd64_fixed_local_binding.go new file mode 100644 index 000000000..a96d576b4 --- /dev/null +++ b/amd64_fixed_local_binding.go @@ -0,0 +1,548 @@ +package plan9asm + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "reflect" + "strings" +) + +// The public value transports only a private live observer binding. Neither +// a declaration, a frame size nor a JSON round trip can construct this proof. +type x86FixedLocalBinding struct { + identity *x86FixedLocalBinding + source *GoAMD64FrameSource + context context.Context + file [32]byte + assembly string + target string + inputs map[string][32]byte + signatures map[string]FuncSig + frames map[string]bool // Only entries whose complete local proof succeeded. + leaves map[string]string // Original source symbol to observed native TEXT, never a signature. + symbols map[string]string + declarations map[string]goAMD64FrameDeclaration + callees map[string]goAMD64FrameDeclaration // Separate typed Go/SDK callees, not private source TEXTs. + numericalReads *GoX86NumericalReadBinding + literalData map[string]x86FixedLocalData + globals map[string]goReadOnlyObject +} + +// BindFixedLocal binds a complete original typed ABI0 file to its actual Go +// object observation. It does not support dynamic incoming frames, pointees, +// unauthenticated imported declarations, native entries or arbitrary local-memory +// effects. Foreign SDK calls require their captured declaration and ABI0 entry. +// The returned Options are independent copies; their private signatures may +// also be transported by GoModuleOptions.ManualSig. The capture context must +// remain live through translation. Close or any failed Verify revokes them. +func (source *GoAMD64FrameSource) BindFixedLocal(ctx context.Context, pkg GoPackage, file *File, options Options) (Options, error) { + if err := source.Verify(ctx); err != nil { + return Options{}, err + } + if file == nil || options.ResolveSym == nil { + return Options{}, goAMD64FrameContext("fixed local binding needs complete source and symbol resolution") + } + originalOptions := options + numerical := options.NumericalReads + if numerical != nil { + if err := verifyNumericalReadInputs(file, options); err != nil { + return Options{}, err + } + source.mu.Lock() + joined := source.numericalBinding == numerical + source.mu.Unlock() + if !joined || ctx != numerical.context || pkg.Types != numerical.originalTypes || + !numericalSameImports(pkg.Imports, numerical.originalImports) { + return Options{}, goAMD64FrameContext("fixed local source has a foreign numerical read owner") + } + } + // Never execute a caller callback while holding the observer lock. Freeze + // its answers before the locked full-source/declaration comparison; that + // comparison rejects any mutation made while resolving these names. + resolved := make(map[string]string, len(file.Funcs)) + for _, fn := range file.Funcs { + resolved[fn.Sym] = options.ResolveSym(fn.Sym) + for _, ins := range fn.Instrs { + if ins.Op == OpCALL && len(ins.Args) == 1 && ins.Args[0].Kind == OpSym { + if name, offset, valid := parseSBRef(ins.Args[0].Sym); valid && offset == 0 { + resolved[name] = options.ResolveSym(name) + } + } + } + } + for _, global := range file.Globl { + if name, valid := fixedLocalDataSymbol(global.Sym); valid { + resolved[name] = options.ResolveSym(name) + } + } + for symbol := range fixedLocalGlobalSymbols(file) { + resolved[symbol] = options.ResolveSym(symbol) + } + options.ResolveSym = func(symbol string) string { return resolved[symbol] } + if numerical != nil { + // The caller has been checked; the locked observation consumes only + // sealed answers, without re-entering numerical/owner callbacks. + options.ResolveSym = numerical.resolve + } + // ResolveSym may change the caller's lifetime. Sample its context only + // outside the observer lock and retain the original, not this temporary + // standard command context, in every returned private binding. + private, contextErr := captureGoAMD64FramePrivateContext(ctx) + if contextErr != nil { + source.mu.Lock() + source.ready = false + source.mu.Unlock() + return Options{}, errors.Join(goAMD64FrameContext("fixed local caller context expired"), contextErr) + } + defer private.stop() + source.mu.Lock() + bound, err := source.bindFixedLocalLocked(private, ctx, pkg, file, options) + source.mu.Unlock() + if contextErr := private.verify(); contextErr != nil { + source.mu.Lock() + source.ready = false + source.mu.Unlock() + return Options{}, errors.Join(err, goAMD64FrameContext("fixed local caller context expired"), contextErr) + } + if err != nil { + return Options{}, err + } + if err := source.Verify(ctx); err != nil { + return Options{}, err + } + if numerical != nil { + if err := verifyNumericalReadInputs(file, originalOptions); err != nil { + return Options{}, err + } + // Bind a fresh numerical token to the independently established local + // capability. The frame keeps its original token; no token is mutated. + bound.NumericalReads = nil + return numerical.source.BindNumericalReads(ctx, pkg, file, bound) + } + return bound, nil +} + +func (source *GoAMD64FrameSource) bindFixedLocalLocked(ctx context.Context, lifetime context.Context, pkg GoPackage, file *File, options Options) (Options, error) { + if !source.ready || ctx == nil || ctx.Err() != nil || file == nil || file.Arch != ArchAMD64 || + options.Goarch != "amd64" || pkg.Types == nil || pkg.Path != source.selection.ImportPath || pkg.Types.Path() != pkg.Path { + return Options{}, goAMD64FrameContext("fixed local binding has no matching live package/target") + } + targets := map[string]string{ + "x86_64-unknown-linux-gnu": "linux/amd64", + "x86_64-apple-darwin": "darwin/amd64", + "x86_64-apple-macosx": "darwin/amd64", + "x86_64-pc-windows-msvc": "windows/amd64", + "x86_64-w64-windows-gnu": "windows/amd64", + } + if targets[options.TargetTriple] != source.request.Target || options.WASMABI != WASMABIDirect || + len(file.UnlinkedPrelude)+len(options.X86TailGroups) != 0 { + return Options{}, goAMD64FrameContext("fixed local binding target or complete source roles disagree") + } + original, err := goAMD64FrameRead(filepath.Join(source.request.Directory, source.request.AssemblyFile)) + if err != nil { + return Options{}, err + } + expanded, err := source.preprocess(source.request.AssemblyFile, original) + if err != nil { + return Options{}, err + } + actual, err := ParseWithDefines(ArchAMD64, expanded, nil) + if err != nil { + return Options{}, err + } + fingerprint, err := x86CallerSourceFingerprint(actual) + if err != nil { + return Options{}, err + } + provided, err := x86CallerSourceFingerprint(file) + if err != nil || provided != fingerprint { + return Options{}, goAMD64FrameContext("fixed local binding requires the exact full original parsed file") + } + resolve := options.ResolveSym + if resolve == nil { + return Options{}, goAMD64FrameContext("fixed local binding needs actual package symbol resolution") + } + globals, err := fixedLocalGlobalRequests(pkg, actual, resolve) + if err != nil { + return Options{}, err + } + declarations, err := source.typedDeclarationsForSourceRole(pkg, false, "amd64", false, nil, false, globals) + if err != nil { + return Options{}, err + } + callees, err := source.ordinaryCallDeclarations(pkg, actual) + if err != nil || !reflect.DeepEqual(callees, source.callees) { + return Options{}, errors.Join(goAMD64FrameContext("ordinary Go call declarations changed"), err) + } + binding := &x86FixedLocalBinding{ + source: source, + context: lifetime, + file: fingerprint, + assembly: expanded, + target: options.TargetTriple, + inputs: make(map[string][32]byte, len(source.inputs)), + signatures: make(map[string]FuncSig), + frames: make(map[string]bool), + leaves: make(map[string]string), + symbols: make(map[string]string), + declarations: make(map[string]goAMD64FrameDeclaration), + callees: callees, + numericalReads: options.NumericalReads, + globals: globals, + } + binding.identity = binding + literalData, dataSymbols, err := source.fixedLocalData(actual, expanded, resolve) + if err != nil { + return Options{}, err + } + binding.literalData = literalData + for symbol, name := range dataSymbols { + binding.symbols[symbol] = name + } + for symbol, object := range globals { + binding.symbols[symbol] = object.native + } + if options.NumericalReads != nil { + if err := joinNumericalReadCertificates(actual, options); err != nil { + return Options{}, err + } + } + for name, sum := range source.inputs { + binding.inputs[name] = sum + } + // Only the authenticated foreign SDK owner can supply a missing callee. + // Keep explicit consumer signatures strict and copy before exposing slices. + joined := make(map[string]FuncSig, len(options.Sigs)+len(callees)) + for name, sig := range options.Sigs { + joined[name] = sig + } + for name, declaration := range callees { + path, _ := goReadOnlyImportedName(name) + if _, supplied := joined[name]; !supplied && path != pkg.Path { + selected, captured := source.imports[path] + if !captured || !selected.Standard || selected.Module != nil { + return Options{}, goAMD64FrameContext("ordinary callee defaults require their actual SDK owner") + } + joined[name] = declaration.signature + } + } + options.Sigs, err = cloneFixedLocalSigs(joined) + if err != nil { + return Options{}, err + } + actualSignatures, err := cloneFixedLocalSigs(options.Sigs) + if err != nil { + return Options{}, err + } + for name, declaration := range callees { + sig, supplied := options.Sigs[name] + if !supplied || !reflect.DeepEqual(sig, declaration.signature) { + return Options{}, goAMD64FrameContext("ordinary Go callee signature or resolution changed: %s", name) + } + binding.signatures[name] = actualSignatures[name] + binding.declarations[name] = declaration + binding.symbols[name] = name + } + leafSources := make(map[string]string) + for name, observed := range source.functions { + if observed.privateLeaf { + leafSources[observed.sourceSymbol] = name + } + } + for _, fn := range actual.Funcs { + name := resolve(fn.Sym) + for _, ins := range fn.Instrs { + if ins.Op != OpCALL || len(ins.Args) != 1 || ins.Args[0].Kind != OpSym { + continue + } + symbol, _, _ := parseSBRef(ins.Args[0].Sym) + target := goAMD64FrameCallName(pkg.Path, symbol) + if _, ordinary := callees[target]; ordinary && resolve(symbol) != target { + return Options{}, goAMD64FrameContext("ordinary Go callsite changed its exact target resolution") + } + if _, ordinary := callees[target]; ordinary { + binding.symbols[symbol] = target + } + } + if native, leaf := leafSources[fn.Sym]; leaf { + if _, supplied := options.Sigs[name]; supplied || name == "" { + return Options{}, goAMD64FrameContext("private leaf cannot acquire a supplied callable signature") + } + binding.leaves[fn.Sym], binding.symbols[fn.Sym] = native, name + continue + } + observed, found := source.functions[name] + declaration, declared := declarations[name] + expectedName := pkg.Path + "." + strings.TrimPrefix(fn.Sym, "·") + sig, supplied := options.Sigs[name] + if !found || !supplied || name != expectedName || len(observed.code) == 0 || + observed.locals != goAMD64FrameNativeLocals(fn.FrameSize) || !goAMD64FrameArgumentsMatch(fn, observed.args) || + !reflect.DeepEqual(sig, observed.signature) || !declared || + !source.matchesFixedLocalDeclaration(name, declaration) { + return Options{}, goAMD64FrameContext("fixed local actual declaration/object/symbol mismatch: %s", fn.Sym) + } + if fn.FrameSize > 0 { + binding.frames[name] = true + sig.x86FixedLocal = binding + } + binding.signatures[name] = actualSignatures[name] + // Preserve the independently re-typechecked language identity, not + // merely the collapsed LLVM signature. Neither observation-only + // composite roles nor stale private metadata can replace this witness. + declaration.signature = actualSignatures[name] + binding.declarations[name] = declaration + binding.symbols[fn.Sym] = name + options.Sigs[name] = sig + } + if len(options.Sigs)+len(binding.leaves) != len(actual.Funcs)+len(callees) { + return Options{}, goAMD64FrameContext("fixed local complete entry/leaf signature inventory differs") + } + if len(binding.frames) == 0 { + return Options{}, goAMD64FrameContext("complete source has no private fixed frame to bind") + } + graph, err := newX86FixedLocalLeafGraph(actual, options, binding) + if err != nil { + return Options{}, err + } + for _, fn := range actual.Funcs { + name := binding.symbols[fn.Sym] + if !binding.frames[name] { + continue + } + if err := ctx.Err(); err != nil { + return Options{}, err + } + if _, err := proveX86FixedLocalRoot(fn, options.Sigs[name], graph); err != nil { + return Options{}, err + } + } + return options, nil +} + +// After validating mutable public input, consume only independent immutable +// copies. A stateful resolver or a mutation of the caller's Options map cannot +// change the declaration or function name between normalization and emission. +func freezeX86FixedLocalOptions(options Options, bindings []*x86FixedLocalBinding) (Options, error) { + if len(bindings) == 0 { + return options, nil + } + symbols := make(map[string]string) + options.Sigs = make(map[string]FuncSig) + for _, binding := range bindings { + for symbol, name := range binding.symbols { + symbols[symbol] = name + } + signatures, err := cloneFixedLocalSigs(binding.signatures) + if err != nil { + return Options{}, err + } + for name, sig := range signatures { + if binding.frames[name] { + sig.x86FixedLocal = binding + } + options.Sigs[name] = sig + } + } + options.ResolveSym = func(symbol string) string { + if name, found := symbols[symbol]; found { + return name + } + if options.NumericalReads != nil { + return options.NumericalReads.resolve(symbol) + } + return symbol // No source contract admits external calls or addresses. + } + return options, nil +} + +func freezeX86FixedLocalFile(file *File, bindings []*x86FixedLocalBinding) (*File, error) { + if len(bindings) == 0 { + return file, nil + } + if len(bindings) != 1 { + return nil, goAMD64FrameContext("fixed local complete file has multiple source owners") + } + binding := bindings[0] + fingerprint, err := x86CallerSourceFingerprint(file) + if err != nil || fingerprint != binding.file { + return nil, goAMD64FrameContext("fixed local resolver changed the original source during validation") + } + // Parsing the guarded observer's immutable expanded bytes gives the + // consumer independent Func/Instr/Operand slices with no caller aliases + // or pre-existing private certificates. + actual, err := ParseWithDefines(ArchAMD64, binding.assembly, nil) + if err != nil { + return nil, err + } + fingerprint, err = x86CallerSourceFingerprint(actual) + if err != nil || fingerprint != binding.file { + return nil, goAMD64FrameContext("fixed local sealed source snapshot changed") + } + if graph := file.x86DirectTails; graph != nil && graph.fixedLocal == binding { + actual.x86DirectTails = graph // Same verified complete source, fresh instruction slices. + } + return actual, nil +} + +func cloneFixedLocalSigs(input map[string]FuncSig) (map[string]FuncSig, error) { + // The source signatures are JSON-encodable public ABI metadata. Reusing + // neither caller slices nor nested FrameSlot fields prevents mutation leaks. + encoded, err := json.Marshal(input) + if err != nil { + return nil, goAMD64FrameContext("fixed local declaration cannot be independently copied: %v", err) + } + var result map[string]FuncSig + if err := json.Unmarshal(encoded, &result); err != nil { + return nil, goAMD64FrameContext("fixed local declaration copy cannot be decoded: %v", err) + } + return result, nil +} + +func verifyX86FixedLocalBindings(file *File, options Options) ([]*x86FixedLocalBinding, error) { + var bindings []*x86FixedLocalBinding + seen := make(map[*x86FixedLocalBinding]bool) + for _, sig := range options.Sigs { + binding := sig.x86FixedLocal + if binding == nil || seen[binding] { + continue + } + seen[binding] = true + if file == nil || file.Arch != ArchAMD64 || options.Goarch != "amd64" || options.TargetTriple != binding.target || + options.ResolveSym == nil || options.WASMABI != WASMABIDirect || len(options.X86TailGroups) != 0 { + return nil, goAMD64FrameContext("fixed local consumer target or resolver changed") + } + if err := binding.verify(); err != nil { + return nil, err + } + if binding.numericalReads != nil && (options.NumericalReads == nil || + options.NumericalReads.source != binding.numericalReads.source || + options.NumericalReads.context != binding.context) { + return nil, goAMD64FrameContext("fixed local numerical consumer has a foreign source/context") + } + fingerprint, err := x86CallerSourceFingerprint(file) + if err != nil || fingerprint != binding.file || len(options.Sigs) != len(binding.signatures) { + return nil, goAMD64FrameContext("fixed local consumer original/normalized source changed") + } + for _, fn := range file.Funcs { + name := options.ResolveSym(fn.Sym) + for _, ins := range fn.Instrs { + if _, ordinary := fixedLocalOrdinaryCall(ins, binding); ordinary { + symbol, _, _ := parseSBRef(ins.Args[0].Sym) + if options.ResolveSym(symbol) != binding.symbols[symbol] { + return nil, goAMD64FrameContext("ordinary Go callsite consumer resolution changed") + } + } + } + if _, leaf := binding.leaves[fn.Sym]; leaf { + if _, supplied := options.Sigs[name]; supplied || name != binding.symbols[fn.Sym] { + return nil, goAMD64FrameContext("private leaf consumer changed its non-callable identity") + } + continue + } + actual, found := options.Sigs[name] + expected, declared := binding.signatures[name] + var frame *x86FixedLocalBinding + if binding.frames[name] { + frame = binding + } + actual.x86FixedLocal = nil + if !found || !declared || name != binding.symbols[fn.Sym] || !reflect.DeepEqual(actual, expected) || + options.Sigs[name].x86FixedLocal != frame { + return nil, goAMD64FrameContext("fixed local consumer declaration or unique symbol binding changed") + } + } + for name := range binding.callees { + if actual, found := options.Sigs[name]; !found || !reflect.DeepEqual(actual, binding.signatures[name]) { + return nil, goAMD64FrameContext("ordinary Go callee consumer signature or identity changed") + } + } + for _, global := range file.Globl { + name, private := fixedLocalDataSymbol(global.Sym) + if !private || options.ResolveSym(name) != binding.symbols[name] { + return nil, goAMD64FrameContext("fixed local consumer literal symbol resolution changed") + } + } + for symbol, object := range binding.globals { + if options.ResolveSym(symbol) != object.native { + return nil, goAMD64FrameContext("fixed local consumer Go variable resolution changed") + } + } + bindings = append(bindings, binding) + } + return bindings, nil +} + +func (binding *x86FixedLocalBinding) verify() error { + if binding == nil || binding.identity != binding || binding.source == nil { + return goAMD64FrameContext("missing or expired fixed local observer binding") + } + if err := binding.source.Verify(binding.context); err != nil { + return err + } + binding.source.mu.Lock() + defer binding.source.mu.Unlock() + if !binding.source.ready || !reflect.DeepEqual(binding.inputs, binding.source.inputs) { + return goAMD64FrameContext("fixed local observer identity changed or was revoked") + } + if len(binding.declarations) != len(binding.signatures) || len(binding.frames) == 0 { + return goAMD64FrameContext("fixed local typed language witnesses are incomplete") + } + for name, signature := range binding.signatures { + declaration, found := binding.declarations[name] + if callee, ordinary := binding.callees[name]; ordinary { + if !found || binding.frames[name] || !reflect.DeepEqual(callee, declaration) || + !reflect.DeepEqual(callee, binding.source.callees[name]) || !reflect.DeepEqual(signature, callee.signature) { + return goAMD64FrameContext("ordinary Go callee source identity changed") + } + continue + } + if !found || !reflect.DeepEqual(signature, declaration.signature) || + !binding.source.matchesFixedLocalDeclaration(name, declaration) || + binding.frames[name] != (binding.source.functions[name].locals > 0) { + return goAMD64FrameContext("fixed local typed language/object identity changed: %s", name) + } + } + for symbol, native := range binding.leaves { + fn, found := binding.source.functions[native] + if !found || !fn.privateLeaf || fn.sourceSymbol != symbol || fn.signature.Name != "" { + return goAMD64FrameContext("private leaf original machine identity changed") + } + } + return nil +} + +// Called only while source.mu is held. The expected declaration comes from +// an independent original-source typecheck at Bind, then a private deep copy +// retained by that binding. A factory observation alone grants no permission. +func (source *GoAMD64FrameSource) matchesFixedLocalDeclaration(name string, expected goAMD64FrameDeclaration) bool { + observed, found := source.functions[name] + original, declared := source.declarations[name] + return found && declared && fixedLocalTransportRole(expected.transport) && + observed.transport == expected.transport && original.transport == expected.transport && + expected.languageType != "" && observed.languageType == expected.languageType && + original.languageType == expected.languageType && + reflect.DeepEqual(observed.signature, expected.signature) && reflect.DeepEqual(original.signature, expected.signature) +} + +func finishX86FixedLocalBindings(bindings []*x86FixedLocalBinding) error { + for _, binding := range bindings { + if err := binding.verify(); err != nil { + return fmt.Errorf("fixed local post-consumption verification: %w", err) + } + } + return nil +} + +// Use a digest of the complete public instruction shape; private certificates +// are not serialized and cannot self-authenticate a modified instruction. +func fixedLocalInstructionDigest(ins Instr) ([32]byte, error) { + encoded, err := json.Marshal(ins) + if err != nil { + return [32]byte{}, goAMD64FrameContext("fixed local instruction cannot be bound: %v", err) + } + return sha256.Sum256(encoded), nil +} diff --git a/amd64_fixed_local_block_native_linux_amd64_test.go b/amd64_fixed_local_block_native_linux_amd64_test.go new file mode 100644 index 000000000..240949aea --- /dev/null +++ b/amd64_fixed_local_block_native_linux_amd64_test.go @@ -0,0 +1,7 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalBlockTransfer(t *testing.T) { + fixedLocalVectorRuntime(t, fixedLocalVEXNativeTools(t), true, fixedLocalBlockRuntimeKernels()) +} diff --git a/amd64_fixed_local_block_runtime_test.go b/amd64_fixed_local_block_runtime_test.go new file mode 100644 index 000000000..e2a891ae5 --- /dev/null +++ b/amd64_fixed_local_block_runtime_test.go @@ -0,0 +1,39 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalBlockRuntimeKernels() []fixedLocalVEXRuntimeKernel { + var kernels []fixedLocalVEXRuntimeKernel + for _, name := range []string{"VINSERTF128", "VINSERTI128", "VEXTRACTF128", "VEXTRACTI128"} { + for _, selector := range []int{0, 1, 254, 255} { + prefix := fmt.Sprintf("%s $%d,", name, selector) + var operations []string + if strings.HasPrefix(name, "VINSERT") { + for _, source := range []string{"X0", "64(BP)", "65(BP)", "0(SI)", "1(SI)"} { + operations = append(operations, prefix+source+",Y1,Y2") + } + operations = append(operations, prefix+"X0,Y1,Y0", prefix+"X0,Y1,Y1", prefix+"X0,Y0,Y0") + } else { + for _, destination := range []string{"X2", "X1", "64(BP)", "65(BP)", "0(SI)", "1(SI)"} { + operations = append(operations, prefix+"Y1,"+destination) + } + } + kernels = append(kernels, fixedLocalVEXRuntimeKernel{shape: operations[0], operations: operations}) + } + } + return kernels +} + +func TestGoAMD64FixedLocalBlockTransferRuntimeObjects(t *testing.T) { + kernels := fixedLocalBlockRuntimeKernels() + if len(kernels) != 16 { + t.Fatal("block-transfer runtime inventory changed", len(kernels)) + } + fixedLocalVectorRuntime(t, fixedLocalPackedCompileOnlyTools(t), false, kernels) +} diff --git a/amd64_fixed_local_block_transfer_test.go b/amd64_fixed_local_block_transfer_test.go new file mode 100644 index 000000000..14f6fed12 --- /dev/null +++ b/amd64_fixed_local_block_transfer_test.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalBlockWitnesses(t *testing.T) []fixedLocalVEXWitness { + t.Helper() + // Original Go has one unsigned insertion row and two overlapping + // signed/unsigned extraction rows for each of the F/I spellings. + return fixedLocalVEXFamilyWitnesses(t, `^V(INSERT|EXTRACT)[FI]128$`, 6, 18) +} + +func TestGoAMD64FixedLocalBlockTransferCompleteFamilies(t *testing.T) { + witnesses := fixedLocalBlockWitnesses(t) + fixedLocalCheckVEXEncodings(t, witnesses) + fixedLocalCheckVEXPhysicalBoundaries(t, witnesses) + fixedLocalCheckVEXFamilies(t, witnesses) +} + +func TestGoAMD64FixedLocalBlockTransferSelectedBytes(t *testing.T) { + for _, name := range []string{"VINSERTF128", "VINSERTI128", "VEXTRACTF128", "VEXTRACTI128"} { + insert := strings.HasPrefix(name, "VINSERT") + for _, alias := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/alias=%t", name, alias), func(t *testing.T) { + minimum := -128 + if insert { + minimum = 0 + } + for selector := minimum; selector <= 255; selector++ { + selected := uint64(0xffff) << (uint(selector&1) * 16) + input, destination := selected, "X2" + args := "Y1," + destination + if alias { + destination, args = "X1", "Y1,X1" + } + if insert { + input, destination = uint64(0xffffffff)&^selected, "Y2" + if alias { + destination = "Y1" + } + args = "X0,Y1," + destination + } + ins := x86YMMReadSource(t, fmt.Sprintf("%s $%d,%s", name, selector, args)) + state := x86FixedLocalState{vectors: map[Reg]uint64{ + "Z0": 0xffff, "Z1": input, "Z2": 0, "Z7": 123, + }, flags: x86ByteFlagsAll} + if err := fixedLocalVectorStep(&ins, &state, nil, 0); err != nil { + t.Fatal("unselected source bytes must not be inputs", selector, err) + } + if state.vectors[x86CallerVectorRoot(Reg(destination))] != ^uint64(0) || + !alias && state.vectors["Z1"] != input || state.vectors["Z7"] != 123 || state.flags != x86ByteFlagsAll { + t.Fatal("block transfer lost an alias, upper zero or unrelated state", selector, state) + } + for bit := uint(0); bit < 32; bit++ { + if input&(1<= 0 { + known &^= 1 << uint(missing) + } + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z0": known}} + err := fixedLocalVectorStep(&ins, &state, nil, 0) + if known == input { + if err != nil || state.vectors["Z0"] != ^uint64(0) { + t.Fatal("aliased inputs lost their common selected bytes", selector, missing, err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("insertion invented a source/retained byte", selector, missing, err) + } + } + } + }) + } +} + +func TestGoAMD64FixedLocalBlockTransferMemoryBounds(t *testing.T) { + for _, name := range []string{"VINSERTF128", "VINSERTI128", "VEXTRACTF128", "VEXTRACTI128"} { + for _, offset := range []int{0, 65, 176, 177, 256} { + t.Run(fmt.Sprintf("%s/%d", name, offset), func(t *testing.T) { + args := fmt.Sprintf("$1,Y1,%d(BP)", offset) + if strings.HasPrefix(name, "VINSERT") { + args = fmt.Sprintf("$1,%d(BP),Y1,Y2", offset) + } + fn, sig, _ := fixedLocalVEXFixture(t, name+" "+args) + _, err := proveX86FixedLocal(fn, sig) + // The fixture initializes the private read extent through 191. + // Writes may initialize a new in-frame span, but never escape it. + want := offset >= 64 && offset <= 176 || !strings.HasPrefix(name, "VINSERT") && offset == 0 + if (err == nil) != want { + t.Fatal("complete unaligned m128 access has wrong bounds", want, err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalBlockTransferObjects(t *testing.T) { + var assembly, declarations strings.Builder + witnesses := fixedLocalBlockWitnesses(t) + for index, row := range witnesses { + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(fixedLocalVEXAssembly(row.source), "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p *uint64,a uint64) uint64\n", name) + } + fixedLocalCompleteVectorObjects(t, assembly.String(), declarations.String(), len(witnesses)) +} diff --git a/amd64_fixed_local_broadcast_test.go b/amd64_fixed_local_broadcast_test.go new file mode 100644 index 000000000..b759ca44b --- /dev/null +++ b/amd64_fixed_local_broadcast_test.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +type fixedLocalBroadcastCase struct { + name, source, destination string + bytes int +} + +// The independent complete Go table directory includes EVEX rows too. Only +// these four original ytabs contain VEX rows; GP, mask and newer block sources +// remain with their existing separate consumers. +func fixedLocalBroadcastCases(t *testing.T) []fixedLocalBroadcastCase { + t.Helper() + var rows []fixedLocalBroadcastCase + for _, spec := range x86BroadcastTestDirectory { + switch spec.table { + case "_yvpbroadcastb", "_yvbroadcastss", "_yvbroadcastsd", "_yvbroadcastf128": + default: + continue + } + for _, bank := range spec.destinations { + if bank == 'Z' { + continue + } + sources := []string{"65(BP)"} + if strings.Contains(spec.source, "X") { + sources = append(sources, "X0") + } + for _, source := range sources { + rows = append(rows, fixedLocalBroadcastCase{spec.name, source, string(bank) + "2", spec.bytes}) + } + } + } + if len(rows) != 24 { + t.Fatal("complete VEX broadcast memory/register and width inventory changed", len(rows)) + } + return rows +} + +func (row fixedLocalBroadcastCase) instruction() string { + return row.name + " " + row.source + "," + row.destination +} + +func TestX86FixedLocalBroadcastOriginalFormsAndEffects(t *testing.T) { + rows := fixedLocalBroadcastCases(t) + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + for _, row := range rows { + source.WriteString(row.instruction() + "\n") + } + code, pcs := x86NumericalBoundaryGoBytes(t, arch, source.String()+"RET") + if len(pcs) != len(rows)+1 { + t.Fatal("original Go instruction inventory changed", len(pcs)) + } + for index, row := range rows { + t.Run(arch+"/"+row.instruction(), func(t *testing.T) { + encoded := code[pcs[index]:pcs[index+1]] + if len(encoded) < 3 || encoded[0] != 0xc4 && encoded[0] != 0xc5 { + t.Fatalf("original row did not select VEX: %x", encoded) + } + ins := x86ScalarLaneInstruction(t, row.instruction()) + effect, known, err := x86UnmaskedVEXByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 2 { + t.Fatal("complete broadcast family lacks its typed dependency", known, err) + } + input, output := effect.operands[0], effect.operands[1] + width := int64(row.bytes) + var required uint64 + if row.source == "X0" { + width, required = 16, uint64(1)<= 0 { + input &^= uint64(1) << uint(missing) + } + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z0": input, "Z2": 0, "Z7": 0xa55a}, flags: x86ByteFlagsAll} + before := cloneFixedLocalState(state) + err := fixedLocalVectorStep(&ins, &state, nil, 0) + if missing >= 0 { + if !errors.Is(err, ErrProbeNeedsContext) || !reflect.DeepEqual(state.vectors, before.vectors) || state.flags != before.flags { + t.Fatal("missing scalar byte invented or partially published", missing, err, state) + } + continue + } + before.vectors[x86CallerVectorRoot(Reg(destination))] = ^uint64(0) + if err != nil || !reflect.DeepEqual(state.vectors, before.vectors) || state.flags != before.flags { + t.Fatal("broadcast lost a source alias, upper zero or FLAGS", err, state) + } + } + }) + } + } +} + +func fixedLocalBroadcastBody(row fixedLocalBroadcastCase) string { + // Consume every byte of the Y alias, including VEX.128's zeroed upper + // half, through private memory and the real ABI0 result. + return row.instruction() + "\nVMOVDQU Y2,128(BP)\n" + + "MOVQ 128(BP),AX\nXORQ 136(BP),AX\nORQ 144(BP),AX\nADDQ 152(BP),AX\nMOVQ AX,128(BP)" +} + +func TestGoAMD64FixedLocalBroadcastCompleteFunctions(t *testing.T) { + var original strings.Builder + for index, row := range fixedLocalBroadcastCases(t) { + original.WriteString(strings.Replace(fixedLocalVEXAssembly(fixedLocalBroadcastBody(row)), "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) + for _, row := range fixedLocalBroadcastCases(t) { + t.Run(row.instruction(), func(t *testing.T) { + fn, sig, _ := fixedLocalVEXFixture(t, fixedLocalBroadcastBody(row)) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("complete initialized broadcast frame/result rejected", err) + } + }) + } +} + +func TestGoAMD64FixedLocalBroadcastMemoryAndEncodingBoundaries(t *testing.T) { + for _, row := range fixedLocalBroadcastCases(t) { + if row.source == "X0" { + continue + } + for _, offset := range []int{0, 65, 192 - row.bytes, 193 - row.bytes, 256} { + t.Run(fmt.Sprintf("%s/%s/%d", row.name, row.destination, offset), func(t *testing.T) { + operation := fmt.Sprintf("%s %d(BP),%s", row.name, offset, row.destination) + fn, sig, source := fixedLocalVEXFixture(t, operation) + requireX86GoAssemblerResult(t, "amd64", source, true) + _, err := proveX86FixedLocal(fn, sig) + want := offset >= 64 && offset+row.bytes <= 192 + if want && err != nil || !want && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("broadcast lost its exact initialized source extent", want, err) + } + }) + } + } + for _, source := range []string{ + "VPBROADCASTQ AX,Y2", "VPBROADCASTD X16,Y2", "VBROADCASTSS X0,Z2", + "VPBROADCASTW X0,K1,Y2", "VPBROADCASTB.Z X0,K1,Y2", + "VBROADCASTI32X2 X0,Y2", "VBROADCASTF32X4 64(BP),Y2", "VPBROADCASTMB2Q K1,Y2", + "VBROADCASTI128 (AX),Y2", "VBROADCASTF128 256(BP),Y2", + } { + t.Run(source, func(t *testing.T) { + fn, sig, original := fixedLocalVEXFixture(t, source) + requireX86GoAssemblerResult(t, "amd64", original, true) + if source == "VBROADCASTI128 (AX),Y2" { + fixedLocalRequireNumericControl(t, fn, sig, original, "func Value(p *uint64,a uint64) uint64", + source, 0, 16, false, "load volatile <16 x i8>") + return + } + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("broadcast invented an EVEX, mask, pointer or memory role", err) + } + }) + } +} + +func TestGoAMD64FixedLocalBroadcastObjects(t *testing.T) { + var assembly, declarations strings.Builder + rows := fixedLocalBroadcastCases(t) + for index, row := range rows { + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(fixedLocalVEXAssembly(fixedLocalBroadcastBody(row)), "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p *uint64,a uint64) uint64\n", name) + } + fixedLocalCompleteVectorObjects(t, assembly.String(), declarations.String(), len(rows)) +} diff --git a/amd64_fixed_local_byte_state_test.go b/amd64_fixed_local_byte_state_test.go new file mode 100644 index 000000000..06d60ac87 --- /dev/null +++ b/amd64_fixed_local_byte_state_test.go @@ -0,0 +1,212 @@ +package plan9asm + +import ( + "errors" + "fmt" + "testing" +) + +// Keep the oracle byte-addressed. The production representation may change, +// but a partially overwritten byte and an adjacent byte remain independent. +func fixedLocalByteStateFixture(size int) (x86FixedLocalState, []bool, []bool) { + state := fixedLocalExchangeState() + state.bytes = newFixedLocalByteSet(int64(size)) + state.booleanBytes = newFixedLocalByteSet(int64(size)) + known, booleans := make([]bool, size), make([]bool, size) + for index := range known { + known[index] = index%5 != 0 + booleans[index] = known[index] && index%3 == 0 + fixedLocalByteStateSet(&state, index, known[index], booleans[index]) + } + return state, known, booleans +} + +func fixedLocalByteStateSet(state *x86FixedLocalState, index int, known, boolean bool) { + state.bytes.set(int64(index), known) + state.booleanBytes.set(int64(index), boolean) +} + +func checkFixedLocalByteState(t *testing.T, state x86FixedLocalState, known, booleans []bool) { + t.Helper() + missing := make([]int, len(known)+1) + for index := range known { + missing[index+1] = missing[index] + if !known[index] { + missing[index+1]++ + } + if state.bytes.has(int64(index)) != known[index] || state.booleanBytes.has(int64(index)) != booleans[index] { + t.Fatalf("byte %d: numerical=%v boolean=%v, want %v/%v", index, + state.bytes.has(int64(index)), state.booleanBytes.has(int64(index)), known[index], booleans[index]) + } + } + for _, offset := range []int{0, 1, 7, 31, 63, 64, 65, len(known)} { + for _, width := range []int{0, 1, 2, 8, 16, 63, 64, 65, len(known) - offset} { + if offset > len(known) || width < 0 || width > len(known)-offset { + continue + } + want := missing[offset+width] == missing[offset] + if got := state.bytes.all(int64(offset), int64(width)); got != want { + t.Fatalf("numerical read [%d,%d): got %v, want %v", offset, offset+width, got, want) + } + } + } + for index := len(known); index < len(state.bytes)*64; index++ { + if state.bytes.has(int64(index)) || state.booleanBytes.has(int64(index)) { + t.Fatalf("padding byte %d acquired facts outside the original frame", index) + } + } + for offset := 0; offset < len(known); offset++ { + width := len(known) - offset + if width > 8 { + width = 8 + } + var expected uint8 + for index := 0; index < width; index++ { + if booleans[offset+index] { + expected |= 1 << uint(index) + } + } + if got := fixedLocalMemoryBooleans(&state, int64(offset), int64(width)); got != expected { + t.Fatalf("boolean read [%d,%d): got %#x, want %#x", offset, offset+width, got, expected) + } + } +} + +func TestGoAMD64FixedLocalByteStatePartialWrites(t *testing.T) { + for _, size := range []int{1, 7, 8, 63, 64, 65, 127, 128, 129, 1408} { + t.Run(fmt.Sprint(size), func(t *testing.T) { + state, known, booleans := fixedLocalByteStateFixture(size) + for _, offset := range []int{0, 1, 7, 8, 31, 60, 63, 64, 65, 127, size - 1} { + for _, width := range []int{1, 2, 4, 8, 16, 32, 64} { + if offset < 0 || width > size-offset { + continue + } + for _, complete := range []bool{false, true} { + fixedLocalNumericalStore(&state, int64(offset), int64(width), complete) + for index := offset; index < offset+width; index++ { + known[index] = known[index] || complete + booleans[index] = false + } + checkFixedLocalByteState(t, state, known, booleans) + } + } + } + }) + } +} + +func TestGoAMD64FixedLocalByteStateScalarTransport(t *testing.T) { + const size = 129 + state, known, booleans := fixedLocalByteStateFixture(size) + proof := &x86FixedLocalProof{frame: size} + for _, offset := range []int{0, 7, 60, 63, 64, 65, 120} { + for _, form := range []struct { + op string + width int64 + }{{"MOVB", 1}, {"MOVW", 2}, {"MOVL", 4}, {"MOVQ", 8}} { + width := form.width + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s $1,%d(SP)", form.op, offset)) + value := fixedLocalScalarValue{booleans: 0xa5} + if err := fixedLocalWriteValue(&ins, 1, 0, value, &state, FuncSig{}, proof, 0, width); err != nil { + t.Fatal(err) + } + for index := int64(0); index < width; index++ { + known[offset+int(index)] = true + booleans[offset+int(index)] = 0xa5&(1<>6]&(uint64(1)<>6] |= mask + } else { + bytes[index>>6] &^= mask + } +} + +func fixedLocalByteWordMask(offset, width int64) (uint64, int64) { + count := int64(64) - (offset & 63) + if width < count { + count = width + } + return (^uint64(0) >> uint(64-count)) << uint(offset&63), count +} + +func (bytes x86FixedLocalByteSet) setRange(offset, width int64, value bool) { + for width > 0 { + mask, count := fixedLocalByteWordMask(offset, width) + if value { + bytes[offset>>6] |= mask + } else { + bytes[offset>>6] &^= mask + } + offset, width = offset+count, width-count + } +} + +func (bytes x86FixedLocalByteSet) all(offset, width int64) bool { + for width > 0 { + mask, count := fixedLocalByteWordMask(offset, width) + if bytes[offset>>6]&mask != mask { + return false + } + offset, width = offset+count, width-count + } + return true +} + +func (bytes x86FixedLocalByteSet) intersect(other x86FixedLocalByteSet) bool { + changed := false + for index, previous := range bytes { + current := previous & other[index] + changed = changed || current != previous + bytes[index] = current + } + return changed +} diff --git a/amd64_fixed_local_candidate.go b/amd64_fixed_local_candidate.go new file mode 100644 index 000000000..c400f45e3 --- /dev/null +++ b/amd64_fixed_local_candidate.go @@ -0,0 +1,184 @@ +package plan9asm + +// GoAMD64FixedLocalFrameBindingNeeded is a source-role query, not a frame or +// memory certificate. The complete actual-object factory and fixed-local +// proof must independently succeed before any instruction gains backing. +// One optional typed declaration set keeps complete ABI0 forwarding adapters +// on their existing call route. Missing or ambiguous options retain the query's +// original conservative behavior and grant no source or translation permission. +func GoAMD64FixedLocalFrameBindingNeeded(file *File, goarch string, options ...Options) bool { + if file == nil || file.Arch != ArchAMD64 || goarch != "amd64" { + return false + } + for _, fn := range file.Funcs { + if fn.FrameSize <= 0 { + continue + } + if len(options) == 1 && x86ABI0OutgoingForwarder(fn, options[0]) { + continue + } + bases := fixedLocalCandidateBases(fn) + for _, ins := range fn.Instrs { + // The final candidate role requires possible physical-frame memory. Avoid + // parsing allocating effects for register-only or unrelated inputs; + // this negative filter grants neither grammar nor storage permission. + localMemory := false + for _, operand := range ins.Args { + if operand.Kind == OpMem && bases.has(operand.Mem) { + localMemory = true + break + } + } + if !localMemory { + continue + } + memory, eligible := -1, false + physicalOnly := false + if _, _, known, err := x86SwapProperties(ins, goarch); known { + eligible, physicalOnly = err == nil, true + for index, operand := range ins.Args { + if operand.Kind == OpMem { + memory = index + } + } + } else if _, known, err := x86ScalarRegisterLoadProperties(ins.Op, ins, goarch); known { + memory, eligible, physicalOnly = 0, err == nil, true + } else if _, known, err := x86ScalarRegisterStoreProperties(ins.Op, ins, goarch); known { + memory, eligible, physicalOnly = 1, err == nil, true + } else if form, known, err := fixedLocalConversionProperties(ins); known { + memory, eligible = form.memory, err == nil && !form.mmx + } else if form, known, err := fixedLocalScalarLaneProperties(ins); known { + memory, eligible = form.memory, err == nil + } else if form, known, err := amd64PackedVectorMoveProperties(ins, goarch); known { + memory, eligible = form.memory, err == nil && !form.undefinedOpcode() + physicalOnly = !form.spec.aligned + } else if form, known, err := x86NonTemporalVectorProperties(ins.Op, ins, goarch); known { + memory, eligible = form.move.memory, err == nil && !form.trap + } else if form, known, err := x86LegacyPackedMoveProperties(ins.Op, ins, goarch); known { + memory, eligible = form.memory, err == nil + physicalOnly = !form.aligned + } else if form, known, err := x86SHAProperties(ins.Op, ins, goarch); known { + memory, eligible = form.memory, err == nil + } else if form, known, err := x86AESProperties(ins.Op, ins, goarch); known { + memory, eligible = form.memory, err == nil && form.byteWidth == 16 + } else if effect, known, err := fixedLocalVectorByteEffect(ins); known { + eligible = err == nil + for _, operand := range effect.operands { + if operand.operand.Kind == OpMem { + memory = operand.index + } + } + } else if form, known, err := x86FloatCompareProperties(ins, goarch); known { + memory, eligible = form.memory, err == nil && form.legacy && !form.scalar && form.byteWidth == 16 + } else if form, known, err := x86ConditionalMoveProperties(ins.Op, ins, goarch); known { + memory, eligible = 0, err == nil && form.source.Kind == OpMem + } else if effect, known, err := fixedLocalGPByteEffect(ins); known { + eligible, physicalOnly = err == nil && !effect.dynamicMemory, true + for _, operand := range effect.operands { + if operand.operand.Kind == OpMem && (operand.read || operand.write) { + memory = operand.index + } + } + } + if eligible && memory >= 0 && memory < len(ins.Args) { + source := ins.Args[memory] + // Static SB and incoming native GP addresses keep their existing + // contracts. Virtual FP is not original local native storage. + if source.Kind == OpMem && bases.has(source.Mem) { + if physicalOnly { + // Scalar and unaligned transport can use logical AUTO. Only + // physical SP/BP needs this additional source observer; the + // existing parser retains the distinct named displacement. + address, err := fixedLocalMemoryAddress(ins, memory, fn.FrameSize) + if err != nil || address != source.Mem { + continue + } + } + return true + } + } + } + } + return false +} + +type fixedLocalCandidateBaseSet uint16 + +func fixedLocalCandidateRegister(reg Reg) (int, bool) { + if reg == SP { + return 4, true + } + if reg == BP { + return 5, true + } + index, _, valid := x86ResultGP(reg, 8, "amd64") + return index, valid +} + +func (bases fixedLocalCandidateBaseSet) has(memory MemRef) bool { + for _, reg := range []Reg{memory.Base, memory.Index} { + if index, valid := fixedLocalCandidateRegister(reg); valid && bases&(1<(SB),X0\nRET\n", + "TEXT ·Value(SB),4,$160-0\nPADDL (AX),X0\nRET\n", + "TEXT ·Value(SB),4,$160-0\nPADDL (AX)(CX*2),X0\nRET\n", + "TEXT ·Value(SB),4,$160-16\nPADDL input+0(FP),X0\nRET\n", + } { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") { + t.Fatalf("query widened unrelated source grammar: %s", source) + } + } +} diff --git a/amd64_fixed_local_clmul_native_linux_amd64_test.go b/amd64_fixed_local_clmul_native_linux_amd64_test.go new file mode 100644 index 000000000..3411c1b61 --- /dev/null +++ b/amd64_fixed_local_clmul_native_linux_amd64_test.go @@ -0,0 +1,13 @@ +package plan9asm + +import "testing" + +// Original Go executes every legacy and VEX kernel before comparison with +// the three LLVM APIs. Missing ISA or a failed preflight is a failure, not a +// skipped execution receipt or an object-only substitute. +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalCLMUL(t *testing.T) { + execution := fixedLocalVEXNativeTools(t) + t.Logf("%s", execution.run(t, fixedLocalCLMULRuntimePreflight(t, execution))) + fixedLocalVectorRuntime(t, execution, true, fixedLocalCLMULRuntimeKernels(t)) + x87IA32RequireNativeHost(t) +} diff --git a/amd64_fixed_local_clmul_test.go b/amd64_fixed_local_clmul_test.go new file mode 100644 index 000000000..6ea694042 --- /dev/null +++ b/amd64_fixed_local_clmul_test.go @@ -0,0 +1,229 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalCLMULCases() []string { + var operations []string + for _, selector := range []int{0, 1, 16, 17} { + for _, source := range []string{"X1", "X2", "64(BP)", "(SI)"} { + operations = append(operations, fmt.Sprintf("PCLMULQDQ $%d,%s,X2", selector, source)) + } + for _, bank := range []string{"X", "Y"} { + for _, source := range []string{bank + "1", "64(BP)", "(SI)", "65(BP)", "1(SI)"} { + operations = append(operations, fmt.Sprintf("VPCLMULQDQ $%d,%s,%s2,%s3", selector, source, bank, bank)) + } + for _, destination := range []string{bank + "1", bank + "2"} { + operations = append(operations, fmt.Sprintf("VPCLMULQDQ $%d,%s1,%s2,%s", selector, bank, bank, destination)) + } + } + } + return operations +} + +func TestGoAMD64FixedLocalCLMULOfficialVEXBytes(t *testing.T) { + // The existing complete CLMUL audit owns all six original rows. These + // two VEX rows supply source witnesses, not exact encoder-row selection. + witnesses := fixedLocalVEXFamilyWitnesses(t, `^VPCLMULQDQ$`, 2, 6) + fixedLocalCheckVEXEncodings(t, witnesses) + fixedLocalCheckVEXPhysicalBoundaries(t, witnesses) +} + +func TestGoAMD64FixedLocalCLMULCompleteFamilies(t *testing.T) { + operations := fixedLocalCLMULCases() + if len(operations) != 72 { + t.Fatal("CLMUL selector/width/alias/memory inventory changed", len(operations)) + } + var assembly strings.Builder + for index, operation := range operations { + assembly.WriteString(strings.Replace(fixedLocalVEXAssembly(operation), "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + for _, operation := range operations { + t.Run(operation, func(t *testing.T) { + fn, sig, _ := fixedLocalVEXFixture(t, operation) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("original Go accepts complete carryless-multiply source but private-frame effect is missing", err) + } + }) + } +} + +func TestGoAMD64FixedLocalCLMULSelectedBytesAndAliases(t *testing.T) { + for _, shape := range []struct { + name, bank string + width int + }{ + {"PCLMULQDQ", "X", 16}, {"VPCLMULQDQ", "X", 16}, {"VPCLMULQDQ", "Y", 32}, + } { + for _, destination := range []int{1, 2, 3} { + if shape.name == "PCLMULQDQ" && destination == 3 { + continue + } + t.Run(fmt.Sprintf("%s/%s/destination%d", shape.name, shape.bank, destination), func(t *testing.T) { + for immediate := 0; immediate < 256; immediate++ { + second := 2 + operation := fmt.Sprintf("VPCLMULQDQ $%d,%s1,%s2,%s%d", immediate, shape.bank, shape.bank, shape.bank, destination) + if shape.name == "PCLMULQDQ" { + second = destination + operation = fmt.Sprintf("PCLMULQDQ $%d,X1,X%d", immediate, destination) + } + ins := x86YMMReadSource(t, operation) + secondRoot := Reg(fmt.Sprintf("Z%d", second)) + output := Reg(fmt.Sprintf("Z%d", destination)) + // Independent byte positions: bit 4 selects the first source; + // bit 0 selects the second (old destination for legacy). + needed := map[Reg]uint64{} + for lane := 0; lane < shape.width; lane += 16 { + for byteIndex := 0; byteIndex < 8; byteIndex++ { + needed["Z1"] |= uint64(1) << uint(lane+(immediate>>4&1)*8+byteIndex) + needed[secondRoot] |= uint64(1) << uint(lane+(immediate&1)*8+byteIndex) + } + } + for missing := -1; missing < shape.width; missing++ { + for _, input := range []Reg{"Z1", secondRoot} { + state := x86FixedLocalState{ + vectors: map[Reg]uint64{"Z1": needed["Z1"], "Z2": needed["Z2"], "Z3": 0, "Z7": 0x1234}, + flags: x86ByteFlagsAll, + } + state.vectors[output] |= uint64(1) << 60 + if missing >= 0 { + state.vectors[input] &^= uint64(1) << uint(missing) + } + beforeOutput := state.vectors[output] + wantFailure := missing >= 0 && needed[input]&(uint64(1)<", 16}}}, + {"global", x86FixedLocalOrigin{global: "example.global"}}, + } { + for _, source := range []string{"AX", "BX", "8(SP)"} { + t.Run(condition.name+"/"+row.name+"/"+source, func(t *testing.T) { + state := fixedLocalExchangeState() + if row.name == "private" { + state.pointers[AX], state.pointers[BX], state.addresses[8] = 24, 24, 24 + } else { + origin := fixedLocalOrigin(row.origin) + state.opaque[AX], state.opaque[BX], state.opaqueCells[8] = origin, origin, origin + } + before := cloneFixedLocalState(state) + ins := x86ScalarLaneInstruction(t, "CMOVQ"+condition.name+" "+source+",BX") + if err := fixedLocalStep(&ins, &state, FuncSig{}, &x86FixedLocalProof{frame: 64}, 0); err != nil { + t.Fatal("same complete pointer identity was treated as numerical bytes", err) + } + if state.known[BX] != 0 || state.flags != before.flags || + !reflect.DeepEqual(state.pointers, before.pointers) || !reflect.DeepEqual(state.opaque, before.opaque) { + t.Fatal("conditional pointer acquired NUM, lost identity or changed FLAGS", state, before) + } + }) + } + } + } +} + +func TestGoAMD64FixedLocalConditionalRangeAndAuthority(t *testing.T) { + t.Run("different inputs", func(t *testing.T) { + state := fixedLocalExchangeState() + state.opaque[SI] = fixedLocalOrigin(x86FixedLocalOrigin{parameter: 0}) + state.opaque[DI] = fixedLocalOrigin(x86FixedLocalOrigin{parameter: 8}) + ins := x86ScalarLaneInstruction(t, "CMOVQEQ SI,DI") + proof := &x86FixedLocalProof{frame: 64} + if err := fixedLocalStep(&ins, &state, FuncSig{}, proof, 0); err != nil { + t.Fatal("two bounded native alternatives rejected", err) + } + if !state.opaque[DI].valid() || state.opaque[DI].readonly() || fixedLocalNumericRead(Operand{Kind: OpReg, Reg: DI}, 8, &state) { + t.Fatal("conditional input address was lost or acquired an unrelated role") + } + read := x86ScalarLaneInstruction(t, "MOVQ (DI),AX") + if err := fixedLocalStep(&read, &state, FuncSig{}, proof, 0); err != nil { + t.Fatal("native alternatives lost their existing faulting read", err) + } + }) + for _, bounds := range []struct { + name string + left, right, want x86UnsignedRange + both bool + }{ + {"same constant", x86UnsignedRange{8, 8}, x86UnsignedRange{8, 8}, x86UnsignedRange{8, 8}, true}, + {"same", x86UnsignedRange{8, 16}, x86UnsignedRange{8, 16}, x86UnsignedRange{8, 16}, true}, + {"overlap", x86UnsignedRange{8, 16}, x86UnsignedRange{12, 24}, x86UnsignedRange{8, 24}, true}, + {"reverse overlap", x86UnsignedRange{12, 24}, x86UnsignedRange{8, 16}, x86UnsignedRange{8, 24}, true}, + {"disjoint", x86UnsignedRange{8, 8}, x86UnsignedRange{24, 24}, x86UnsignedRange{8, 24}, true}, + {"reverse disjoint", x86UnsignedRange{24, 24}, x86UnsignedRange{8, 8}, x86UnsignedRange{8, 24}, true}, + {"one unbounded", x86UnsignedRange{8, 16}, x86UnsignedRange{}, x86UnsignedRange{}, false}, + } { + t.Run(bounds.name, func(t *testing.T) { + state := fixedLocalExchangeState() + state.known[AX], state.known[BX] = 255, 255 + state.setRange(AX, bounds.left) + if bounds.both { + state.setRange(BX, bounds.right) + } + if bounds.left.minimum == bounds.left.maximum { + state.constants[AX] = int64(bounds.left.minimum) + } + if bounds.both && bounds.right.minimum == bounds.right.maximum { + state.constants[BX] = int64(bounds.right.minimum) + } + state.comparison = x86FixedLocalComparison{register: BX, constant: 31} + ins := x86ScalarLaneInstruction(t, "CMOVQEQ AX,BX") + if err := fixedLocalStep(&ins, &state, FuncSig{}, &x86FixedLocalProof{frame: 64}, 0); err != nil { + t.Fatal(err) + } + value, bounded := state.ranges[BX] + if bounded != bounds.both || bounded && value != bounds.want || state.comparison.register != "" { + t.Fatal("selection intersected ranges or retained its old comparison value", value, bounded, state.comparison) + } + constant, known := state.constants[BX] + if known != (bounds.name == "same constant") || known && constant != 8 { + t.Fatal("equal constants were lost or unequal alternatives invented a constant", constant, known) + } + }) + } + for _, row := range []struct{ name, body string }{ + {"different private offsets", "LEAQ 8(SP),CX\nLEAQ 24(SP),DX\nCMOVQEQ CX,DX"}, + {"input and NUM", "CMOVQEQ SI,AX"}, + {"NUM and input", "CMOVQEQ AX,SI"}, + {"private and input", "LEAQ 8(SP),CX\nCMOVQEQ CX,SI"}, + {"partial input", "MOVQ SI,CX\nCMOVLEQ SI,CX"}, + {"logical pointer slot", "MOVQ SI,CX\nCMOVQEQ p+0(FP),CX"}, + {"uninitialized potential memory", "CMOVQEQ 8(SP),AX"}, + {"memory extent", "CMOVQEQ 60(SP),AX"}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalInputFixture(t, "CMPQ AX,AX\n"+row.body) + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("conditional selection invented a pointer or access authority", err) + } + }) + } + for _, operands := range []string{"AX,BX", "BX,AX"} { + t.Run("literal-input/"+operands, func(t *testing.T) { + state := fixedLocalExchangeState() + state.opaque[AX] = fixedLocalOrigin(x86FixedLocalOrigin{parameter: 0}) + state.opaque[BX] = fixedLocalOrigin(x86FixedLocalOrigin{literal: x86StaticSBAddress{"table<>", 16}}) + before := cloneFixedLocalState(state) + ins := x86ScalarLaneInstruction(t, "CMOVQEQ "+operands) + if err := fixedLocalStep(&ins, &state, FuncSig{}, &x86FixedLocalProof{frame: 64}, 0); !errors.Is(err, ErrProbeNeedsContext) || + !reflect.DeepEqual(state, before) { + t.Fatal("readonly/native selection widened authority or changed rejected state", err) + } + }) + t.Run("bound-literal-input/"+operands, func(t *testing.T) { + state := fixedLocalExchangeState() + state.opaque[AX] = fixedLocalOrigin(x86FixedLocalOrigin{parameter: 0}) + state.opaque[BX] = fixedLocalOrigin(x86FixedLocalOrigin{literal: x86StaticSBAddress{"table<>", 16}}) + proof := &x86FixedLocalProof{frame: 64, binding: &x86FixedLocalBinding{ + literalData: map[string]x86FixedLocalData{"table<>": {size: 32, alignment: 16}}, + }} + ins := x86ScalarLaneInstruction(t, "CMOVQEQ "+operands) + if err := fixedLocalStep(&ins, &state, FuncSig{}, proof, 0); err != nil { + t.Fatal("bounded readonly/native alternatives rejected", err) + } + destination := ins.Args[1].Reg + read := x86ScalarLaneInstruction(t, fmt.Sprintf("MOVQ (%s),CX", destination)) + if err := fixedLocalStep(&read, &state, FuncSig{}, proof, 0); err != nil { + t.Fatal("complete readonly/native read rejected", err) + } + write := x86ScalarLaneInstruction(t, fmt.Sprintf("MOVQ $1,(%s)", destination)) + if err := fixedLocalStep(&write, &state, FuncSig{}, proof, 0); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("readonly alternative acquired write permission", err) + } + }) + } +} + +func fixedLocalConditionalBodies() []string { + var bodies []string + for _, condition := range x86CMOVOracleConditions { + for _, body := range []string{ + "MOVQ $8,CX\nMOVQ $8,DX\nCMPQ AX,AX\nCMOVQ%s CX,DX\nMOVQ AX,(SP)(DX*1)\nMOVQ 8(SP),AX", + "LEAQ 0(SP),CX\nMOVQ CX,DX\nCMPQ AX,AX\nCMOVQ%s CX,DX\nMOVQ (DX),AX", + "MOVQ SI,CX\nCMPQ AX,AX\nCMOVQ%s SI,CX\nMOVQ (CX),AX", + } { + bodies = append(bodies, fmt.Sprintf(body, condition.name)) + } + } + return bodies +} + +func TestGoAMD64FixedLocalConditionalCompleteSources(t *testing.T) { + var source strings.Builder + bodies := fixedLocalConditionalBodies() + for index, body := range bodies { + fn, sig, assembly := fixedLocalInputFixture(t, body) + fmt.Fprint(&source, strings.Replace(assembly, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(fmt.Sprint(index+1), func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("whole conditional value transport source rejected", err) + } + }) + } + if len(bodies) != 48 { + t.Fatal("complete condition/value inventory changed", len(bodies)) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) +} + +func TestGoAMD64FixedLocalConditionalObjects(t *testing.T) { + var assembly, declarations strings.Builder + bodies := fixedLocalConditionalBodies() + for index, body := range bodies { + _, _, source := fixedLocalInputFixture(t, body) + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(source, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p,q *uint64,a uint64) uint64\n", name) + } + fixedLocalCompleteVectorObjects(t, assembly.String(), declarations.String(), len(bodies)) +} diff --git a/amd64_fixed_local_constant_arithmetic_runtime_go127_test.go b/amd64_fixed_local_constant_arithmetic_runtime_go127_test.go new file mode 100644 index 000000000..60e1edf26 --- /dev/null +++ b/amd64_fixed_local_constant_arithmetic_runtime_go127_test.go @@ -0,0 +1,40 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalConstantArithmeticProgram(t *testing.T) (string, string, []string) { + t.Helper() + cases := fixedLocalConstantArithmeticCases() + if len(cases) != 20 { + t.Fatal("constant arithmetic runtime inventory changed", len(cases)) + } + var assembly, declarations strings.Builder + var expected []string + for _, input := range []int{0, -1} { + for _, row := range cases { + name := fmt.Sprintf("Value%d", len(expected)) + // Exercise both successors of the same-value join independently. + body := fmt.Sprintf("MOVQ $77,24(SP)\nMOVQ $%d,AX\n%s\nMOVQ (SP)(SI*8),AX", input, row.body) + assembly.WriteString(fixedLocalConstantIndexFunction(name, body)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + expected = append(expected, "000000000000004d") + } + } + return assembly.String(), declarations.String(), expected +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConstantArithmetic(t *testing.T) { + assembly, declarations, expected := fixedLocalConstantArithmeticProgram(t) + fixedLocalConstantResultsRuntime(t, assembly, declarations, expected) +} + +func TestGoAMD64FixedLocalConstantArithmeticObjects(t *testing.T) { + assembly, declarations, expected := fixedLocalConstantArithmeticProgram(t) + fixedLocalCompleteVectorObjects(t, assembly, declarations, len(expected)) +} diff --git a/amd64_fixed_local_constant_arithmetic_test.go b/amd64_fixed_local_constant_arithmetic_test.go new file mode 100644 index 000000000..40933a8f4 --- /dev/null +++ b/amd64_fixed_local_constant_arithmetic_test.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalConstantArithmeticCases() []struct{ name, body string } { + return []struct{ name, body string }{ + {"zero xor", "XORQ SI,SI\nADDQ $3,SI"}, + {"add immediate", "MOVQ $1,SI\nADDQ $2,SI"}, + {"subtract immediate", "MOVQ $5,SI\nSUBQ $2,SI"}, + {"and immediate", "MOVQ $7,SI\nANDQ $3,SI"}, + {"or immediate", "MOVQ $2,SI\nORQ $1,SI"}, + {"xor immediate", "MOVQ $4,SI\nXORQ $7,SI"}, + {"add register", "MOVQ $1,SI\nMOVQ $2,DI\nADDQ DI,SI"}, + {"subtract register", "MOVQ $5,SI\nMOVQ $2,DI\nSUBQ DI,SI"}, + {"and register", "MOVQ $7,SI\nMOVQ $3,DI\nANDQ DI,SI"}, + {"or register", "MOVQ $2,SI\nMOVQ $1,DI\nORQ DI,SI"}, + {"xor register", "MOVQ $4,SI\nMOVQ $7,DI\nXORQ DI,SI"}, + {"increment", "MOVQ $2,SI\nINCQ SI"}, + {"decrement", "MOVQ $4,SI\nDECQ SI"}, + {"negate", "MOVQ $-3,SI\nNEGQ SI"}, + {"complement", "MOVQ $-4,SI\nNOTQ SI"}, + {"sign extended immediate", "MOVQ $4,SI\nADDQ $0xffffffff,SI"}, + {"negative operand", "MOVQ $-1,SI\nADDQ $4,SI"}, + {"modular arithmetic", "MOVQ $0x7fffffffffffffff,SI\nADDQ $1,SI\nSUBQ $1,SI\nANDQ $3,SI"}, + {"value copied before overwrite", "MOVQ $3,DI\nMOVQ DI,SI\nMOVQ $5,DI\nADDQ $0,SI"}, + {"same value at join", "MOVQ $3,SI\nCMPQ AX,$0\nJEQ joined\nADDQ $0,SI\njoined:"}, + } +} + +func TestGoAMD64FixedLocalConstantArithmeticIndex(t *testing.T) { + var original strings.Builder + cases := fixedLocalConstantArithmeticCases() + if len(cases) != 20 { + t.Fatal("constant arithmetic operand and boundary inventory changed", len(cases)) + } + for index, row := range cases { + body := "MOVQ $77,24(SP)\n" + row.body + "\nMOVQ (SP)(SI*8),AX" + fn, sig, source := fixedLocalGPFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + proved, err := proveX86FixedLocal(fn, sig) + if err != nil { + t.Fatal("exact numerical index from typed arithmetic was lost", err) + } + found := false + for _, instruction := range proved.Instrs { + if instruction.Op != "MOVQ" || instruction.Args[0].Kind != OpMem || instruction.Args[0].Mem.Index != SI { + continue + } + access, present := instruction.x86FixedLocalMemory[0] + if !present || access.nativeInput || access.write || access.width != 8 || + access.offset != [8]int64{24, 24, 24, 24, 24, 24, 24, 24} { + t.Fatalf("constant arithmetic acquired different bytes: %+v", access) + } + found = true + } + if !found { + t.Fatal("original indexed instruction disappeared") + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalConstantArithmeticKeepsUnknownRoles(t *testing.T) { + var original strings.Builder + for index, row := range []struct{ name, body string }{ + {"unknown initial value", "ADDQ $3,SI"}, + {"unknown source", "MOVQ $3,SI\nADDQ CX,SI"}, + {"partial source", "MOVQ $3,SI\nMOVW $0,CX\nADDQ CX,SI"}, + {"partial overwrite", "MOVQ $3,SI\nADDB $0,SI"}, + {"pointer source", "MOVQ $3,SI\nMOVQ SP,DI\nADDQ DI,SI"}, + {"numeric is not private", "MOVQ $0,DI\nADDQ $0,DI\nMOVQ (DI),SI"}, + {"carry value unavailable", "MOVQ $3,SI\nSTC\nADCQ $0,SI"}, + {"borrow value unavailable", "MOVQ $3,SI\nSTC\nSBBQ $0,SI"}, + {"different values at join", "MOVQ $3,SI\nCMPQ AX,$0\nJEQ joined\nADDQ $1,SI\njoined:"}, + {"zero xor word is partial", "XORW SI,SI\nADDQ $3,SI"}, + {"overflow cannot grant extent", "MOVQ $0x7fffffffffffffff,SI\nADDQ $1,SI"}, + {"arithmetic cannot grant extent", "MOVQ $7,SI\nADDQ $1,SI"}, + } { + body := "MOVQ $77,24(SP)\nMOVQ $77,32(SP)\n" + row.body + "\nMOVQ (SP)(SI*8),AX" + fn, sig, source := fixedLocalGPFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err == nil { + t.Fatal("unknown numerical value or memory origin became an exact index") + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} diff --git a/amd64_fixed_local_conversion.go b/amd64_fixed_local_conversion.go new file mode 100644 index 000000000..8e2293eea --- /dev/null +++ b/amd64_fixed_local_conversion.go @@ -0,0 +1,73 @@ +package plan9asm + +// This normalizes the existing complete conversion grammars for the private +// frame observer. It grants neither caller storage nor a physical MMX role. +type x86FixedLocalConversion struct { + inputBytes, outputBytes, activeBytes int + memoryBytes, memory, destination int + mask int + legacy, zeroing, mmx bool +} + +func fixedLocalConversionProperties(ins Instr) (x86FixedLocalConversion, bool, error) { + if form, known, err := x86SignedPackedFloatProperties(ins, "amd64"); known { + return x86FixedLocalConversion{ + inputBytes: form.inputBytes, outputBytes: form.byteWidth, + activeBytes: form.byteWidth, memoryBytes: form.memoryBytes, + memory: form.memory, destination: form.destination, mask: form.mask, + legacy: form.spec.legacy, zeroing: form.zeroing, mmx: form.mmx, + }, true, err + } + form, known, err := x86PackedConversionProperties(ins, "amd64") + return x86FixedLocalConversion{ + inputBytes: form.inputBytes, outputBytes: form.shape.destinationBytes, + activeBytes: form.shape.lanes * form.spec.destinationBits / 8, + memoryBytes: form.memoryBytes, memory: form.memory, + destination: form.destination, mask: form.mask, + legacy: form.spec.legacy, zeroing: form.properties.zeroing, mmx: form.mmx, + }, known, err +} + +func fixedLocalConversionStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + form, known, err := fixedLocalConversionProperties(*ins) + if !known { + return false, nil + } + if err != nil || form.mmx { + return true, goAMD64FrameContext("fixed local conversion lacks an ordinary physical vector shape") + } + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return true, goAMD64FrameContext("fixed local conversion has an undefined mask") + } + if form.memory >= 0 { + // Legacy m128 retains its alignment requirement. CVTDQ2PD m64 + // and VEX/EVEX have no corresponding 16-byte #GP requirement. + alignment := int64(1) + if form.legacy && form.memoryBytes == 16 { + alignment = 16 + } + if _, _, err := fixedLocalReadMemory(ins, form.memory, state, proof, residue, + int64(form.memoryBytes), alignment); err != nil { + return true, err + } + } else { + input := state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] + required := x86CallerVectorByteMask(int64(form.inputBytes)) + if input&required != required { + return true, goAMD64FrameContext("fixed local conversion reads undefined numerical bytes") + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + active := x86CallerVectorByteMask(int64(form.activeBytes)) + if form.mask >= 0 && !form.zeroing && state.vectors[destination]&active != active { + return true, goAMD64FrameContext("fixed local conversion merges undefined destination bytes") + } + // Narrowing explicitly clears its carrier padding. Legacy preserves + // wider aliases; VEX/EVEX clears them. Integer FLAGS remain unchanged. + if form.legacy { + state.vectors[destination] |= x86CallerVectorByteMask(int64(form.outputBytes)) + } else { + state.vectors[destination] = ^uint64(0) + } + return true, nil +} diff --git a/amd64_fixed_local_conversion_runtime_linux_amd64_test.go b/amd64_fixed_local_conversion_runtime_linux_amd64_test.go new file mode 100644 index 000000000..7ccf8f07e --- /dev/null +++ b/amd64_fixed_local_conversion_runtime_linux_amd64_test.go @@ -0,0 +1,33 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNativeBatch00(t *testing.T) { + fixedLocalConversionRuntimeBatch(t, x86PackedDoubleDwordNativeTools(t), true, 0) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNativeBatch01(t *testing.T) { + fixedLocalConversionRuntimeBatch(t, x86PackedDoubleDwordNativeTools(t), true, 1) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNativeBatch02(t *testing.T) { + fixedLocalConversionRuntimeBatch(t, x86PackedDoubleDwordNativeTools(t), true, 2) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNativeBatch03(t *testing.T) { + fixedLocalConversionRuntimeBatch(t, x86PackedDoubleDwordNativeTools(t), true, 3) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNativeBatch04(t *testing.T) { + fixedLocalConversionRuntimeBatch(t, x86PackedDoubleDwordNativeTools(t), true, 4) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNativeBatch05(t *testing.T) { + fixedLocalConversionRuntimeBatch(t, x86PackedDoubleDwordNativeTools(t), true, 5) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNativeBatch06(t *testing.T) { + fixedLocalConversionRuntimeBatch(t, x86PackedDoubleDwordNativeTools(t), true, 6) +} diff --git a/amd64_fixed_local_conversion_runtime_unix_test.go b/amd64_fixed_local_conversion_runtime_unix_test.go new file mode 100644 index 000000000..a378f2171 --- /dev/null +++ b/amd64_fixed_local_conversion_runtime_unix_test.go @@ -0,0 +1,307 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "runtime" + "strings" + "testing" +) + +type fixedLocalConversionRuntimeCase struct { + name, destination string + width, offset int + dead bool +} + +func fixedLocalConversionRuntimeCases(all bool) []fixedLocalConversionRuntimeCase { + rows := x86PackedNativeAddressRows() + rows = append(rows, + struct{ name, destination string }{"CVTPL2PS", "X1"}, + struct{ name, destination string }{"CVTPL2PD", "X1"}, + struct{ name, destination string }{"VCVTDQ2PS", "Y1"}, + struct{ name, destination string }{"VCVTDQ2PD", "Y1"}) + var cases []fixedLocalConversionRuntimeCase + for _, row := range rows { + if !all && strings.HasPrefix(row.name, "V") { + continue + } + width := 16 + if strings.HasPrefix(row.destination, "Y") { + width = 32 + } + for _, dead := range []bool{false, true} { + for _, offset := range []int{120, 128, 128 + width - 8, 128 + width} { + cases = append(cases, fixedLocalConversionRuntimeCase{ + name: row.name, destination: row.destination, + width: width, offset: offset, dead: dead, + }) + } + } + } + return cases +} + +func fixedLocalConversionRuntimeProgram(cases []fixedLocalConversionRuntimeCase, triple string) (string, string, string, string) { + var assembly, declarations, goRows, cRows, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, row := range cases { + name := fmt.Sprintf("Value%d", index) + operation := row.name + " 64(BP)," + row.destination + if !row.dead { + move := "VMOVDQU" + if !strings.HasPrefix(row.name, "V") { + move = "MOVOU" + } + operation += "\n" + move + " " + row.destination + ",128(BP)" + } + body := fixedLocalPackedAssembly(operation) + body = strings.Replace(body, "$256-16", "$512-16", 1) + body = strings.Replace(body, "MOVQ 128(BP),AX", fmt.Sprintf("MOVQ %d(BP),AX", row.offset), 1) + assembly.WriteString(strings.Replace(body, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + canary := row.dead || row.offset < 128 || row.offset >= 128+row.width + fmt.Fprintf(&goRows, "{%s, %v},\n", name, canary) + canaryValue := 0 + if canary { + canaryValue = 1 + } + fmt.Fprintf(&cRows, "{value%d, %d},\n", index, canaryValue) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t) __asm__(\"%sexample.invalid/framefixture.%s\");\n", index, prefix, name) + } + // The caller controls and snapshots MXCSR outside the observed frame. + // These helpers belong only to the independently compiled Go executable. + original := `package main +import "fmt" +func readCSR() uint32 +func writeCSR(csr uint32) +` + declarations.String() + ` +func main() { + rows := []struct { call func(uint64) uint64; canary bool }{ +` + goRows.String() + ` + } + values := []uint64{0,1,^uint64(0),0x8000000080000000,0x3fc000003fc00000,0x3ff8000000000000,0x7fc123457fc12345,0x7ff8123456789abc} + for n,row := range rows { + for v,value := range values { + for mode:=0; mode<32; mode++ { + csr := uint32(0x1f80) | uint32(mode&3)<<13 | uint32((mode>>2)&1)<<6 | uint32((mode>>3)&1)<<15 + if mode&16 != 0 { csr |= 63 } + writeCSR(csr) + got := row.call(value) + status := readCSR() + writeCSR(0x1f80) + if row.canary && got != value { panic("original private-frame canary changed") } + fmt.Printf("%d %d %d %016x %08x\n", n,v,mode,got,status) + } + } + } +} +` + driver := `#include +#include +#include +` + cDeclarations.String() + ` +int main(void) { + const struct { uint64_t (*call)(uint64_t); int canary; } rows[] = { +` + cRows.String() + ` + }; + const uint64_t values[] = {0,1,UINT64_MAX,UINT64_C(0x8000000080000000),UINT64_C(0x3fc000003fc00000),UINT64_C(0x3ff8000000000000),UINT64_C(0x7fc123457fc12345),UINT64_C(0x7ff8123456789abc)}; + for (unsigned n=0; n>2)&1)<<6) | (((mode>>3)&1)<<15); + if (mode&16) csr |= 63; + _mm_setcsr(csr); + uint64_t got = rows[n].call(values[v]); + unsigned status = _mm_getcsr(); + _mm_setcsr(0x1f80); + if (rows[n].canary && got != values[v]) return 2; + printf("%u %u %u %016llx %08x\n",n,v,mode,(unsigned long long)got,status); + } + } + } + return 0; +} +` + return assembly.String(), declarations.String(), original, driver +} + +func fixedLocalConversionRuntime(t *testing.T, execution x86FloatCompareExecution, all bool) { + t.Helper() + cases := fixedLocalConversionRuntimeCases(all) + for batch := 0; batch*48 < len(cases); batch++ { + fixedLocalConversionRuntimeBatch(t, execution, all, batch) + } +} + +func fixedLocalConversionRuntimeBatchRange(total, batch int) (start, end int, ok bool) { + if total <= 0 || batch < 0 || batch > (total-1)/48 { + return 0, 0, false + } + start, end = batch*48, total + if total-start > 48 { + end = start + 48 + } + return start, end, true +} + +func fixedLocalConversionRuntimeBatch(t *testing.T, execution x86FloatCompareExecution, all bool, batch int) { + t.Helper() + cases := fixedLocalConversionRuntimeCases(all) + start, end, ok := fixedLocalConversionRuntimeBatchRange(len(cases), batch) + if !ok { + t.Fatal("invalid complete conversion runtime batch", batch, len(cases)) + } + // Reuse only this batch's disposable compiler artifacts. Each API still + // captures, binds, consumes and closes its own complete source observation. + t.Setenv("GOCACHE", t.TempDir()) + t.Run(fmt.Sprintf("batch%d", batch), func(t *testing.T) { + assembly, declarations, original, driver := fixedLocalConversionRuntimeProgram(cases[start:end], execution.triple) + helpers := "TEXT ·readCSR(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\nTEXT ·writeCSR(SB),4,$0-4\nLDMXCSR csr+0(FP)\nRET\n" + want := string(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly+helpers, original)) + if strings.Count(want, "\n") != (end-start)*8*32 { + t.Fatal("original Go numerical/status denominator changed") + } + pkg, request, _, options := x86CMOVFixedLocalFixture(t, + assembly, declarations, runtime.GOOS+"/amd64", execution.triple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, optimize := range []bool{false, true} { + input, level := ir, "-O0" + if optimize { + input, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + got := string(execution.compileAndRunLLVM(t, "private-conversion", input, driver, level)) + if got != want { + t.Fatal("original Go and LLVM private-frame value/MXCSR/canary results differ") + } + } + }) + }) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionSSE(t *testing.T) { + fixedLocalConversionRuntime(t, x86FloatCompareRuntimeTools(t, x86FloatCompareSSE), false) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("Rosetta SSE diagnostic, not native AVX512 execution") + } +} + +func TestGoAMD64FixedLocalConversionRuntimeOwnership(t *testing.T) { + for _, input := range [][2]int{{0, 0}, {48, -1}, {48, 1}, {336, 7}} { + if _, _, ok := fixedLocalConversionRuntimeBatchRange(input[0], input[1]); ok { + t.Fatal("invalid conversion runtime batch accepted", input) + } + } + for _, row := range []struct { + all bool + count int + }{{false, 48}, {true, 336}} { + cases := fixedLocalConversionRuntimeCases(row.all) + if len(cases) != row.count { + t.Fatalf("runtime cases all=%v: %d, want %d", row.all, len(cases), row.count) + } + seen := make([]int, len(cases)) + for batch := 0; batch*48 < len(cases); batch++ { + start, end, ok := fixedLocalConversionRuntimeBatchRange(len(cases), batch) + if !ok || end-start != 48 { + t.Fatal("incomplete conversion runtime batch", batch, start, end) + } + for index := start; index < end; index++ { + seen[index]++ + } + } + for index, count := range seen { + if count != 1 { + t.Fatal("missing or duplicated original runtime case", index, count) + } + } + names := make(map[string]int) + for _, entry := range cases { + names[entry.name]++ + } + if len(names)*8 != row.count { + t.Fatal("runtime inventory lost distinct family members", names) + } + for name, count := range names { + if count != 8 { + t.Fatalf("%s lost a live/dead or canary axis: %d", name, count) + } + } + assembly, _, _, _ := fixedLocalConversionRuntimeProgram(cases, "x86_64-unknown-linux-gnu") + requireX86GoAssemblerResult(t, "amd64", assembly, true) + } + const filename = "amd64_fixed_local_conversion_runtime_linux_amd64_test.go" + for _, qualified := range []bool{false, true} { + for _, target := range []struct{ os, arch string }{ + {"linux", "amd64"}, {"linux", "386"}, {"linux", "arm"}, {"linux", "arm64"}, + {"js", "wasm"}, {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, + } { + t.Run(fmt.Sprintf("selection/qualified=%t/%s-%s", qualified, target.os, target.arch), func(t *testing.T) { + context := ciNativeAVX512BuildContext(target.os, target.arch, qualified) + selected, err := context.MatchFile(".", filename) + want := qualified && target.os == "linux" && target.arch == "amd64" + if err != nil || selected != want { + t.Fatal("native conversion batch lost its explicit selection", target, qualified, selected, err) + } + }) + } + } + syntax, err := parser.ParseFile(token.NewFileSet(), filename, nil, 0) + if err != nil { + t.Fatal(err) + } + owners := 0 + for _, declaration := range syntax.Decls { + function, ok := declaration.(*ast.FuncDecl) + if !ok { + continue + } + owners++ + name := fmt.Sprintf("TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNativeBatch%02d", owners-1) + if function.Name.Name != name || function.Body == nil || len(function.Body.List) != 1 { + t.Fatal("native owner missing or conditional") + } + statement, ok := function.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("native runtime must execute directly") + } + call, ok := statement.X.(*ast.CallExpr) + if !ok || len(call.Args) != 4 || !fixedLocalConversionIdent(call.Fun, "fixedLocalConversionRuntimeBatch") || + !fixedLocalConversionIdent(call.Args[0], "t") || !fixedLocalConversionIdent(call.Args[2], "true") { + t.Fatal("native runtime lost its complete-family axis") + } + batch, ok := call.Args[3].(*ast.BasicLit) + if !ok || batch.Kind != token.INT || batch.Value != fmt.Sprint(owners-1) { + t.Fatal("native runtime lost an original batch") + } + tools, ok := call.Args[1].(*ast.CallExpr) + if !ok || len(tools.Args) != 1 || !fixedLocalConversionIdent(tools.Fun, "x86PackedDoubleDwordNativeTools") || + !fixedLocalConversionIdent(tools.Args[0], "t") { + t.Fatal("native runtime bypassed actual CPU/OS/opcode checks") + } + } + if owners != 7 { + t.Fatal("missing or ambiguous native owner", owners) + } + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil || !strings.Contains(string(workflow), "-ci-selector '^(TestCrossLinuxRuntimeMatrix|") { + t.Fatal("required native runtime root not selected by CI", err) + } +} + +func fixedLocalConversionIdent(expression ast.Expr, name string) bool { + identifier, ok := expression.(*ast.Ident) + return ok && identifier.Name == name +} diff --git a/amd64_fixed_local_conversion_test.go b/amd64_fixed_local_conversion_test.go new file mode 100644 index 000000000..17e0c7c4a --- /dev/null +++ b/amd64_fixed_local_conversion_test.go @@ -0,0 +1,534 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalConversionOperations() []string { + var operations []string + seen := make(map[string]bool) + for _, row := range append(x86PackedConversionOracleRows("amd64"), x86SignedPackedFloatRows("amd64")...) { + // Keep the complete independent register/mask/rounding inventory and + // one private address representative. Physical MMX stays unbound. + if strings.Contains(row, "M0") || strings.Contains(row, "M7") { + continue + } + _, operands, _ := strings.Cut(row, " ") + source, _, _ := strings.Cut(operands, ",") + if source == "(AX)" || source == "3(AX)" { + row = strings.Replace(row, source, "64(BP)", 1) + } else if strings.Contains(row, "(") || strings.Contains(row, " 256,") { + continue + } + if !seen[row] { + seen[row] = true + operations = append(operations, row) + } + } + return operations +} + +func fixedLocalConversionOperation(row string) string { + // Every source and merge byte is independently numerical, including + // legacy upper aliases. Conversion output is then observed by a store. + prefix := "KMOVQ AX,K1\nKMOVQ AX,K3\nKMOVQ AX,K7\n" + for _, index := range []int{0, 7, 15, 20, 21, 31} { + prefix += fmt.Sprintf("VMOVDQU64 64(BP),Z%d\n", index) + } + parts := strings.Split(row, ",") + destination := strings.TrimSpace(parts[len(parts)-1]) + return prefix + row + "\nVMOVDQU64 Z" + destination[1:] + ",128(BP)" +} + +func TestGoAMD64FixedLocalConversionCompleteFamilyProof(t *testing.T) { + var original strings.Builder + names := make(map[string]bool) + for index, row := range fixedLocalConversionOperations() { + t.Run(row, func(t *testing.T) { + op := strings.Fields(row)[0] + name, _, _ := strings.Cut(op, ".") + names[name] = true + operation := fixedLocalConversionOperation(row) + pkg, _, file, options := fixedLocalPackedFixture(t, operation) + assembly := fixedLocalPackedAssembly(operation) + original.WriteString(strings.Replace(assembly, "·Value", fmt.Sprintf("·Value%d", index), 1)) + if strings.Contains(row, "(BP)") && !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") { + t.Error("native conversion memory did not request a complete original frame binding") + } + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); err != nil { + t.Fatalf("initialized private conversion rejected: %v", err) + } + }) + } + if len(names) != 42 { + t.Fatalf("complete independent conversion namespace = %d, want 42", len(names)) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalConversionWholeSourceNegatives(t *testing.T) { + operation := "VCVTDQ2PD 64(BP),Y0\nVMOVDQU Y0,128(BP)" + pkg, _, _, options := fixedLocalPackedFixture(t, operation) + original := fixedLocalPackedAssembly(operation) + for name, source := range map[string]string{ + "input hole": strings.Replace(original, "MOVQ AX,64(BP)", "MOVB AX,64(BP)", 1), + "outside frame": strings.Replace(original, "VCVTDQ2PD 64(BP)", "VCVTDQ2PD 256(BP)", 1), + "incoming pointer": strings.Replace(original, "MOVQ SP,BP", "MOVQ AX,BP", 1), + "unknown mask": strings.Replace(original, "64(BP),Y0", "64(BP),K1,Y0", 1), + "unknown merge": strings.Replace(original, operation, "KMOVQ AX,K1\nVCVTDQ2PD 64(BP),K1,Y0\nVMOVDQU Y0,128(BP)", 1), + "zero mask missing storage": strings.Replace(original, operation, "MOVQ $0,DX\nKMOVQ DX,K1\nVCVTDQ2PD.Z 256(BP),K1,Y0\nVMOVDQU Y0,128(BP)", 1), + "unknown source vector": strings.Replace(original, "VCVTDQ2PD 64(BP)", "VCVTDQ2PD X0", 1), + "legacy m128 alignment": strings.Replace(original, operation, "CVTPL2PS 65(BP),X0\nMOVOU X0,128(BP)", 1), + "MMX input": strings.Replace(original, operation, "CVTPL2PD M0,X0\nMOVOU X0,128(BP)", 1), + "MMX output": strings.Replace(original, operation, "CVTPS2PL X0,M0", 1), + "private escape": strings.Replace(original, "MOVQ AX,ret+8(FP)", "MOVQ BP,ret+8(FP)", 1), + } { + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if name == "incoming pointer" { + fixedLocalRequireNumericControl(t, file.Funcs[0], options.Sigs[pkg.Path+".Value"], source, + "func Value(a uint64) uint64", "VCVTDQ2PD 64(BP),Y0", 0, 16, false, "vcvtdq2pd") + return + } + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unclosed native conversion accepted: %v", err) + } + }) + } +} + +func TestGoAMD64FixedLocalConversionSparseByteEffects(t *testing.T) { + // Independent representatives of every width, alias and mask rule. The + // complete family test separately binds all encoder forms to these rules. + for _, row := range []struct { + instruction string + input, active int + legacy, mask, zero bool + }{ + {"CVTPL2PD X0,X1", 8, 16, true, false, false}, + {"CVTPS2PL X0,X1", 16, 16, true, false, false}, + {"CVTPD2PL X0,X1", 16, 8, true, false, false}, + {"VCVTDQ2PD X0,Y1", 16, 32, false, false, false}, + {"VCVTPD2DQY Y0,X1", 32, 16, false, false, false}, + {"VCVTPD2DQ Z0,Y1", 64, 32, false, false, false}, + {"VCVTPS2QQ X0,Y1", 16, 32, false, false, false}, + {"VCVTQQ2PSY Y0,X1", 32, 16, false, false, false}, + {"VCVTPS2DQ.Z X0,K1,X1", 16, 16, false, true, true}, + {"VCVTPS2DQ X0,K1,X1", 16, 16, false, true, false}, + {"VCVTPD2DQX X0,K1,X1", 16, 8, false, true, false}, + } { + t.Run(row.instruction, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT value(SB),4,$0-0\n"+row.instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + source := x86CallerVectorRoot(ins.Args[0].Reg) + destination := x86CallerVectorRoot(ins.Args[len(ins.Args)-1].Reg) + fresh := func() x86FixedLocalState { + return x86FixedLocalState{ + vectors: map[Reg]uint64{source: ^uint64(0), destination: ^uint64(0)}, + masks: map[Reg]bool{"K1": true}, + flags: x86ByteFlagsAll, + } + } + check := func(state *x86FixedLocalState, accepted bool) { + t.Helper() + known, err := fixedLocalConversionStep(&ins, state, &x86FixedLocalProof{}, 0) + if !known || accepted && err != nil || !accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("conversion effect accepted=%t: known=%t, %v", accepted, known, err) + } + } + for missing := 0; missing < 64; missing++ { + state := fresh() + state.vectors[source] &^= uint64(1) << uint(missing) + check(&state, missing >= row.input) + } + for missing := 0; missing < 64; missing++ { + state := fresh() + state.vectors[destination] &^= uint64(1) << uint(missing) + accepted := !row.mask || row.zero || missing >= row.active + check(&state, accepted) + if !accepted { + continue + } + want := ^uint64(0) + if row.legacy && missing >= 16 { + want &^= uint64(1) << uint(missing) + } + if state.vectors[destination] != want || state.flags != x86ByteFlagsAll { + t.Fatalf("byte %d: alias/zero padding/FLAGS changed: %+v", missing, state) + } + } + if row.mask { + state := fresh() + state.masks["K1"] = false + check(&state, false) + } + }) + } +} + +func TestGoAMD64FixedLocalConversionExactMemoryTuples(t *testing.T) { + for _, row := range []struct { + instruction string + width int + aligned bool + }{ + {"CVTPL2PD (BP),X1", 8, false}, + {"CVTPL2PS (BP),X1", 16, true}, + {"CVTPS2PL (BP),X1", 16, true}, + {"CVTPD2PL (BP),X1", 16, true}, + {"VCVTDQ2PD (BP),X1", 8, false}, + {"VCVTDQ2PD (BP),Y1", 16, false}, + {"VCVTDQ2PD (BP),Z1", 32, false}, + {"VCVTDQ2PS (BP),Z1", 64, false}, + {"VCVTDQ2PD.BCST.Z (BP),K1,Z1", 4, false}, + {"VCVTPD2DQX (BP),X1", 16, false}, + {"VCVTPD2DQY (BP),X1", 32, false}, + {"VCVTPD2DQ (BP),Y1", 64, false}, + {"VCVTPS2QQ.BCST.Z (BP),K1,Z1", 4, false}, + {"VCVTUQQ2PS.BCST.Z (BP),K1,Y1", 8, false}, + } { + t.Run(row.instruction, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT value(SB),4,$128-0\n"+row.instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + destination := x86CallerVectorRoot(original.Args[len(original.Args)-1].Reg) + check := func(missing, offset int, accepted bool) { + t.Helper() + ins := original + state := x86FixedLocalState{ + pointers: map[Reg]int64{BP: int64(offset)}, + vectors: map[Reg]uint64{destination: ^uint64(0)}, + masks: map[Reg]bool{"K1": true}, + bytes: newFixedLocalByteSet(128), + } + for index := 0; index < 128; index++ { + state.bytes.set(int64(index), index != missing) + } + proof := &x86FixedLocalProof{frame: 128} + known, err := fixedLocalConversionStep(&ins, &state, proof, 0) + if !known || accepted && err != nil || !accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("byte=%d offset=%d accepted=%t: known=%t, %v", missing, offset, accepted, known, err) + } + if accepted { + access := ins.x86FixedLocalMemory[0] + if access.owner != proof || access.width != int64(row.width) || access.write || state.flags != 0 { + t.Fatal("wrong native memory tuple, write permission or invented FLAGS", access) + } + } + if missing >= 0 && state.bytes.has(int64(missing)) { + t.Fatal("conversion manufactured initialized memory") + } + } + for missing := 0; missing < 128; missing++ { + check(missing, 0, missing >= row.width) + } + check(-1, 1, !row.aligned) + check(-1, 129-row.width, false) + }) + } +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch00(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 0) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch01(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 1) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch02(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 2) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch03(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 3) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch04(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 4) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch05(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 5) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch06(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 6) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch07(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 7) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch08(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 8) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch09(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 9) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch10(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 10) +} + +func TestGoAMD64FixedLocalConversionActualObjectsLinuxBatch11(t *testing.T) { + fixedLocalConversionActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 11) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch00(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 0) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch01(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 1) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch02(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 2) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch03(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 3) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch04(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 4) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch05(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 5) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch06(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 6) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch07(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 7) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch08(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 8) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch09(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 9) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch10(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 10) +} + +func TestGoAMD64FixedLocalConversionActualObjectsDarwinBatch11(t *testing.T) { + fixedLocalConversionActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 11) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch00(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 0) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch01(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 1) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch02(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 2) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch03(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 3) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch04(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 4) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch05(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 5) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch06(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 6) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch07(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 7) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch08(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 8) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch09(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 9) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch10(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 10) +} + +func TestGoAMD64FixedLocalConversionActualObjectsWindowsBatch11(t *testing.T) { + fixedLocalConversionActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 11) +} + +func fixedLocalObjectBatchRange(total, batch int) (start, end int, ok bool) { + const size = 64 + if total <= 0 || batch < 0 || batch >= (total+size-1)/size { + return 0, 0, false + } + start, end = batch*size, (batch+1)*size + if end > total { + end = total + } + return start, end, true +} + +func TestGoAMD64FixedLocalConversionObjectBatchPlan(t *testing.T) { + rows := fixedLocalConversionOperations() + if len(rows) != 741 { + t.Fatal("complete independent conversion inventory changed", len(rows)) + } + seen := make([]int, len(rows)) + for batch := 0; batch < 12; batch++ { + start, end, ok := fixedLocalObjectBatchRange(len(rows), batch) + if !ok || end-start > 64 || end <= start { + t.Fatal("invalid independent conversion batch", batch, start, end, ok) + } + for index := start; index < end; index++ { + seen[index]++ + } + } + // Scheduling remains twelve complete batches. Independent source captures + // cover those same 741 functions exactly once in bounded 32-function units. + chunks, sourceSeen := 0, make([]int, len(rows)) + for batch := 0; batch < 12; batch++ { + start, end, _ := fixedLocalObjectBatchRange(len(rows), batch) + for chunk := start; chunk < end; chunk += 32 { + limit := fixedLocalConversionSourceEnd(chunk, end) + if limit <= chunk || limit > end || limit-chunk > 32 { + t.Fatal("invalid independently captured conversion source", chunk, limit, end) + } + for index := chunk; index < limit; index++ { + sourceSeen[index]++ + } + chunks++ + } + } + if chunks != 24 { + t.Fatal("complete conversion source plan changed", chunks) + } + for index, count := range seen { + if count != 1 || sourceSeen[index] != 1 { + t.Fatal("missing or duplicated original conversion form", index, rows[index], count) + } + } + for _, input := range [][2]int{{0, 0}, {741, -1}, {741, 12}} { + if _, _, ok := fixedLocalObjectBatchRange(input[0], input[1]); ok { + t.Fatal("invalid conversion batch accepted", input) + } + } +} + +func fixedLocalConversionSourceEnd(start, end int) int { + if end-start > 32 { + return start + 32 + } + return end +} + +func fixedLocalConversionActualObjects(t *testing.T, target, triple string, batch int) { + t.Helper() + // Go 1.27 builds objdump into GOCACHE. Reuse this batch's test-owned + // compiler cache, not its independent source, object or lifetime records. + t.Setenv("GOCACHE", t.TempDir()) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + rows := fixedLocalConversionOperations() + start, end, ok := fixedLocalObjectBatchRange(len(rows), batch) + if !ok { + t.Fatal("invalid complete conversion object batch", batch, len(rows)) + } + // Keep the twelve complete scheduling owners, but give each independent + // source proof at most 32 functions within its unchanged five-minute lease. + for chunk := start; chunk < end; chunk += 32 { + limit := fixedLocalConversionSourceEnd(chunk, end) + t.Run(fmt.Sprintf("batch%d/source%d", batch, (chunk-start)/32), func(t *testing.T) { + var assembly, declarations strings.Builder + var names []string + for index := chunk; index < limit; index++ { + name := fmt.Sprintf("Value%d", index) + body := fixedLocalPackedAssembly(fixedLocalConversionOperation(rows[index])) + assembly.WriteString(strings.Replace(body, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + names = append(names, "example.invalid/framefixture."+name) + } + pkg, request, _, options := x86CMOVFixedLocalFixture(t, + assembly.String(), declarations.String(), target, triple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "fixed_local_native") || !strings.Contains(ir, "asm sideeffect") { + t.Fatal("conversion did not consume native private backing") + } + for _, optimize := range []bool{false, true} { + input := ir + if optimize { + input = x86MulDivRoundOptimize(t, ir) + } + x86CompactExpandRequireAllDefinitions(t, input, names) + compileLLVMToObject(t, llc, triple, "frame-conversion.ll", "frame-conversion.o", input) + } + }) + }) + } + t.Logf("%s batch%d: %d/%d forms, three LLVM APIs, O0/O2", target, batch, end-start, len(rows)) +} + +func TestGoAMD64FixedLocalConversionSharedCacheKeepsSourcesIndependent(t *testing.T) { + cache := t.TempDir() + t.Setenv("GOCACHE", cache) + for _, key := range []string{"GOPATH", "GOMODCACHE", "TMPDIR", "GOTMPDIR"} { + t.Setenv(key, "") + } + const assembly = "TEXT ·Value(SB),4,$0-16\nMOVQ a+0(FP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + first, a, _, _ := x86CMOVFixedLocalFixture(t, assembly, "func Value(a uint64) uint64", + "linux/amd64", "x86_64-unknown-linux-gnu") + second, b, _, _ := x86CMOVFixedLocalFixture(t, assembly, "func Value(a uint64) uint64", + "linux/amd64", "x86_64-unknown-linux-gnu") + values := func(request GoAMD64FrameCaptureRequest) map[string]string { + environment := make(map[string]string) + for _, entry := range request.Environment { + key, value, _ := strings.Cut(entry, "=") + environment[key] = value + } + return environment + } + left, right := values(a), values(b) + if left["GOCACHE"] != cache || right["GOCACHE"] != cache { + t.Fatal("batch fixtures did not inherit their parent's owned build cache") + } + for _, key := range []string{"GOPATH", "GOMODCACHE", "TMPDIR", "GOTMPDIR"} { + if left[key] == "" || left[key] == right[key] || left[key] == cache || right[key] == cache { + t.Fatal("independent source observations shared private state", key) + } + } + if a.Directory == b.Directory || first.Types == second.Types { + t.Fatal("sharing compiler artifacts reused an original source or typed package") + } +} diff --git a/amd64_fixed_local_conversion_tool_cache_test.go b/amd64_fixed_local_conversion_tool_cache_test.go new file mode 100644 index 000000000..23494800d --- /dev/null +++ b/amd64_fixed_local_conversion_tool_cache_test.go @@ -0,0 +1,46 @@ +package plan9asm + +import ( + "os/exec" + "strings" + "testing" +) + +// Share only this test process's disposable compiler artifacts. Each source +// lease and command workspace remains independently owned by its fixture. +func goAMD64FramePrepareOwnedToolCache(t *testing.T) { + t.Helper() + cache, err := fixedLocalProcessToolCache.directory() + if err != nil { + t.Fatal(err) + } + t.Setenv("GOCACHE", cache) + _, request := goAMD64FrameSourceFixture(t) + fixedLocalConversionPrepareToolCache(t, request) +} + +// Tool construction is setup, not a source/object observation. In particular, +// this output cannot replace Capture's independent bounded tool query or seals. +func fixedLocalConversionPrepareToolCache(t *testing.T, request GoAMD64FrameCaptureRequest) { + t.Helper() + cache := "" + for _, entry := range request.Environment { + key, value, _ := strings.Cut(entry, "=") + if key == "GOCACHE" { + if cache != "" || value == "" { + t.Fatal("conversion setup requires one explicit owned compiler cache") + } + cache = value + } + } + if cache == "" || request.GoBinary == "" || request.Directory == "" { + t.Fatal("conversion setup lost its actual tool, directory or compiler cache") + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, request.GoBinary, "tool", "-n", "objdump") + command.Dir, command.Env = request.Directory, request.Environment + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("prepare actual Go objdump before source capture: %v\n%s", err, output) + } +} diff --git a/amd64_fixed_local_counted_loop.go b/amd64_fixed_local_counted_loop.go new file mode 100644 index 000000000..99dfe4b49 --- /dev/null +++ b/amd64_fixed_local_counted_loop.go @@ -0,0 +1,137 @@ +package plan9asm + +import "math" + +// A syntax-only descriptor for an innermost, single-entry straight-line loop. +// Counter values, storage and all per-iteration effects are checked by the +// ordinary replay. Unknown or more general loops retain the existing CFG meet. +type x86FixedLocalCountedLoop struct { + header, update, latch int + counter Reg + delta int64 + bound uint64 +} + +func fixedLocalCountedLoops(fn Func, edges map[int][]int, instructions []int) map[int]x86FixedLocalCountedLoop { + linear := fixedLocalLinearEdges(fn, edges, instructions) + end := make([]int, len(instructions)) + for pc := len(instructions) - 1; pc >= 0; pc-- { + end[pc] = pc + if linear[pc] { + end[pc] = end[pc+1] + } + } + var loops map[int]x86FixedLocalCountedLoop + for latch := 2; latch+1 < len(instructions); latch++ { + targets := edges[latch] + if len(targets) != 2 || targets[0] != latch+1 { + continue + } + header := targets[1] + if header <= 0 || header > latch-2 || end[header] != latch { + continue // Full predecessor counts exclude side entries and nested branches. + } + condition := x86BranchCondition(fn.Instrs[instructions[latch]].Op) + if condition != "CS" && condition != "HI" { + continue + } + compare := fn.Instrs[instructions[latch-1]] + cmp, recognized, err := x86ScalarCompareProperties(compare.Op, compare, "amd64") + if !recognized || err != nil || cmp.test || cmp.bits != 64 || + cmp.args[0].Kind != OpReg || cmp.args[1].Kind != OpImm { + continue + } + counter, valid := x86CallerGP(cmp.args[0].Reg) + if !valid || counter != cmp.args[0].Reg || counter == SP { + continue + } + // These straight-line bodies cannot overlap: another latch would + // terminate end[header]. Their combined backwards scan stays O(N). + // Replay checks that every other typed transfer preserves the counter. + for update := latch - 2; update >= header; update-- { + delta := fixedLocalCountedDelta(fn.Instrs[instructions[update]], counter) + if delta == 0 || delta > 0 && condition != "CS" || delta < 0 && condition != "HI" { + continue + } + if loops == nil { + loops = make(map[int]x86FixedLocalCountedLoop) + } + loops[header] = x86FixedLocalCountedLoop{header, update, latch, counter, delta, uint64(cmp.args[1].Imm)} + break + } + } + return loops +} + +func fixedLocalCountedDelta(ins Instr, counter Reg) int64 { + form, recognized, err := x86LockScalarRMWProperties(ins.Op, ins, "amd64") + if !recognized || err != nil || form.spec.bits != 64 || + form.destination.Kind != OpReg || form.destination.Reg != counter { + return 0 + } + switch form.spec.stem { + case "ADD", "SUB": + if form.source.Kind == OpImm { + delta := form.source.Imm // The shared grammar applies encoded imm32 sign extension. + if form.spec.stem == "SUB" { + delta = -delta + } + return delta + } + case "INC": + return 1 + case "DEC": + return -1 + } + return 0 +} + +func (loop x86FixedLocalCountedLoop) number(state *x86FixedLocalState) (uint64, bool) { + value, known := state.constants[loop.counter] + return uint64(value), known && fixedLocalNumericRead(Operand{Kind: OpReg, Reg: loop.counter}, 8, state) +} + +func (loop x86FixedLocalCountedLoop) next(value uint64) (uint64, error) { + if loop.delta > 0 { + step := uint64(loop.delta) + if value > math.MaxUint64-step { + return 0, goAMD64FrameContext("fixed local counted-loop counter wraps") + } + return value + step, nil + } + step := uint64(-loop.delta) + if value < step { + return 0, goAMD64FrameContext("fixed local counted-loop counter wraps") + } + return value - step, nil +} + +func (loop x86FixedLocalCountedLoop) continues(value uint64) bool { + if loop.delta > 0 { + return value < loop.bound + } + return value > loop.bound +} + +// Keep every iteration, including early aliases. A fresh visit from an outer +// fixed point replaces the provisional series; it never appends stale visits. +func fixedLocalCountedAccesses(previous, current map[int]x86FixedLocalAccess, iteration int) error { + if iteration != 0 && len(previous) != len(current) { + return goAMD64FrameContext("fixed local counted-loop access inventory changed") + } + for operand, access := range current { + if iteration == 0 { + access.iterationOffsets = [][8]int64{access.offset} + } else { + prior, found := previous[operand] + if !found || !fixedLocalAccessShapeEqual(prior, access) || + len(prior.iterationOffsets) != iteration { + return goAMD64FrameContext("fixed local counted-loop access role changed") + } + prior.iterationOffsets = append(prior.iterationOffsets, access.offset) + access = prior + } + current[operand] = access + } + return nil +} diff --git a/amd64_fixed_local_counted_loop_alias_go127_test.go b/amd64_fixed_local_counted_loop_alias_go127_test.go new file mode 100644 index 000000000..d532bcf78 --- /dev/null +++ b/amd64_fixed_local_counted_loop_alias_go127_test.go @@ -0,0 +1,57 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type fixedLocalCountedLoopAliasCase struct { + offsets []int +} + +func fixedLocalCountedLoopAliasProgram(t *testing.T) (string, string, []fixedLocalCountedLoopAliasCase) { + t.Helper() + var assembly, declarations strings.Builder + var cases []fixedLocalCountedLoopAliasCase + for _, step := range []int{1, 4} { + for _, descending := range []bool{false, true} { + first, limit, update, branch := 0, 4*step, "ADDQ", "JB" + if descending { + first, limit, update, branch = 4*step, 0, "SUBQ", "JA" + } + var body strings.Builder + body.WriteString("MOVQ $77,248(SP)\nMOVQ a+0(FP),DI\nANDQ $1,DI\nADDQ $1,DI\nMOVQ $0,DX\nouter:\n") + fmt.Fprintf(&body, "MOVQ $%d,SI\nloop:\nMOVQ AX,BX\nADDQ SI,BX\n"+ + "MOVQ BX,16(SP)(SI*1)\nADDQ $2,16(SP)(SI*1)\n"+ + "%s $%d,SI\nCMPQ SI,$%d\n%s loop\n"+ + "ADDQ $1,DX\nCMPQ DX,DI\nJB outer\nMOVQ $0,AX\n", first, update, step, limit, branch) + var offsets []int + for iteration := 0; iteration < 4; iteration++ { + offset := iteration * step + if descending { + offset = (4 - iteration) * step + } + offsets = append(offsets, offset) + fmt.Fprintf(&body, "ADDQ %d(SP),AX\n", 16+offset) + } + body.WriteString("ADDQ 248(SP),AX") + _, _, source := fixedLocalCountedLoopFixture(t, body.String()) + name := fmt.Sprintf("Value%d", len(cases)) + assembly.WriteString(strings.Replace(source, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + cases = append(cases, fixedLocalCountedLoopAliasCase{offsets}) + } + } + if len(cases) != 4 { + t.Fatal("overlapping-iteration inventory changed", len(cases)) + } + return assembly.String(), declarations.String(), cases +} + +func TestGoAMD64FixedLocalCountedLoopAliasObjects(t *testing.T) { + assembly, declarations, cases := fixedLocalCountedLoopAliasProgram(t) + fixedLocalCountedLoopObjects(t, assembly, declarations, len(cases)) +} diff --git a/amd64_fixed_local_counted_loop_contract_test.go b/amd64_fixed_local_counted_loop_contract_test.go new file mode 100644 index 000000000..c2e2ae0ad --- /dev/null +++ b/amd64_fixed_local_counted_loop_contract_test.go @@ -0,0 +1,207 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "sort" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalCountedLoopAliasesAndUpdates(t *testing.T) { + var original strings.Builder + count := 0 + aliases := x86ConditionalBranchAliasOracle() + var names []string + for alias := range aliases { + names = append(names, alias) + } + sort.Strings(names) + for _, alias := range names { + canonical := aliases[alias] + if got := x86BranchCondition(Op(alias)); got != strings.TrimPrefix(canonical, "J") { + t.Fatalf("complete branch condition %s = %s, want %s", alias, got, canonical) + } + if canonical != "JCS" && canonical != "JHI" { + continue + } + for _, register := range strings.Fields("AX BX CX DX DI SI BP R8 R9 R10 R11 R12 R13 R14 R15") { + for variant := 0; variant < 3; variant++ { + first, bound, delta, operation := 0, 24, 8, "ADDQ" + if canonical == "JHI" { + first, bound, operation = 24, 0, "SUBQ" + } + if variant == 1 { + delta = -delta + if operation == "ADDQ" { + operation = "SUBQ" + } else { + operation = "ADDQ" + } + } + update, scale := fmt.Sprintf("%s $%d,%s", operation, delta, register), 1 + if variant == 2 { + first, bound, operation, scale = 0, 3, "INCQ", 8 + if canonical == "JHI" { + first, bound, operation = 3, 0, "DECQ" + } + update = operation + " " + register + } + body := fmt.Sprintf("MOVQ $%d,%s\nloop:\nMOVQ AX,16(SP)(%s*%d)\n%s\nCMPQ %s,$%d\n%s loop\nMOVQ 24(SP),AX", + first, register, register, scale, update, register, bound, alias) + fn, sig, source := fixedLocalCountedLoopFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", count), 1)) + count++ + t.Run(alias+"/"+register+"/"+operation, func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("canonical unsigned condition or signed update spelling was lost", err) + } + }) + } + } + } + if count != 360 { + t.Fatal("eight unsigned aliases by 15 GP registers and three update spellings changed", count) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalCountedLoopCertificates(t *testing.T) { + const body = "MOVQ $0,SI\nloop:\nMOVQ AX,16(SP)(SI*8)\nADDQ $1,SI\nCMPQ SI,$3\nJB loop\nMOVQ 32(SP),AX" + fn, sig, source := fixedLocalCountedLoopFixture(t, body) + requireX86GoAssemblerResult(t, "amd64", source, true) + for _, mutation := range []string{"none", "early offset", "missing iteration", "inline offset", "snapshot copy"} { + t.Run(mutation, func(t *testing.T) { + proved, err := proveX86FixedLocal(fn, sig) + if err != nil { + t.Fatal(err) + } + lower := &amd64Ctx{fixedLocal: proved.x86FixedLocal} + found := false + for _, instruction := range proved.Instrs { + for operand, access := range instruction.x86FixedLocalMemory { + if len(access.iterationOffsets) == 0 { + continue + } + found = true + if len(access.iterationOffsets) != 3 || access.offset != access.iterationOffsets[0] { + t.Fatal("loop did not preserve its complete ordered access inventory", access) + } + for iteration, offsets := range access.iterationOffsets { + for _, offset := range offsets { + if offset != int64(16+8*iteration) { + t.Fatal("one original SP residue or iteration changed bytes", offsets) + } + } + } + switch mutation { + case "early offset": + // Mutate the original instruction's backing slice directly. + // Its separately frozen expected access must not change. + access.iterationOffsets[0][0]++ + case "missing iteration": + access.iterationOffsets = access.iterationOffsets[1:] + case "inline offset": + access.offset[0]++ + case "snapshot copy": + copy := cloneX86ReplayFixedLocalMemory(instruction.x86FixedLocalMemory) + copy[operand].iterationOffsets[0][0]++ + if !lower.fixedLocalMemoryCertified(instruction, operand) { + t.Fatal("changing a copied snapshot changed its source") + } + access = copy[operand] + } + instruction.x86FixedLocalMemory[operand] = access + if lower.fixedLocalMemoryCertified(instruction, operand) != (mutation == "none") { + t.Fatal("incomplete or changed iteration series retained its authority") + } + } + } + if !found { + t.Fatal("no loop access received a complete series") + } + }) + } +} + +func TestGoAMD64FixedLocalCountedLoopBudgetIsAtomic(t *testing.T) { + const body = "MOVQ $0,SI\nloop:\nMOVQ AX,16(SP)(SI*8)\nADDQ $1,SI\nCMPQ SI,$3\nJB loop\nMOVQ 32(SP),AX" + fn, sig, _ := fixedLocalCountedLoopFixture(t, body) + for _, budget := range []x86FixedLocalBudget{{steps: 9, bytes: 64 << 20}, {steps: 1 << 18, bytes: 9 * fn.FrameSize}} { + proof := &x86FixedLocalProof{frame: fn.FrameSize} + if err := replayX86FixedLocal(&fn, sig, proof, 0, &budget, nil, nil); err == nil { + t.Fatal("partial counted-loop execution exhausted its budget but succeeded") + } + for _, instruction := range fn.Instrs { + if len(instruction.x86FixedLocalMemory) != 0 { + t.Fatal("unfinished counted loop published provisional access history") + } + } + } +} + +func TestGoAMD64FixedLocalCountedLoopPreservesGenericCFG(t *testing.T) { + var original strings.Builder + for index, row := range []struct{ name, body string }{ + {"wrapping counter with fixed access", "MOVQ $-1,SI\nloop:\nMOVQ AX,16(SP)\nADDQ $1,SI\nCMPQ SI,$3\nJB loop\nMOVQ 16(SP),AX"}, + {"counter replaced with fixed access", "MOVQ $0,SI\nloop:\nMOVQ $3,SI\nMOVQ AX,16(SP)\nADDQ $1,SI\nCMPQ SI,$3\nJB loop\nMOVQ 16(SP),AX"}, + {"counter replaced before indexed access", "MOVQ $0,SI\nloop:\nMOVQ $3,SI\nMOVQ AX,16(SP)(SI*8)\nADDQ $1,SI\nCMPQ SI,$3\nJB loop\nMOVQ 40(SP),AX"}, + } { + fn, sig, source := fixedLocalCountedLoopFixture(t, row.body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + plain := fn + plain.Instrs = append([]Instr(nil), fn.Instrs...) + edges, instructions, err := fixedLocalEdges(plain) + if err != nil { + t.Fatal(err) + } + proof := &x86FixedLocalProof{frame: fn.FrameSize} + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + for residue := 0; residue < 8; residue++ { + if err := replayX86FixedLocalCFG(&plain, sig, proof, residue, &budget, nil, edges, instructions, nil, nil, nil); err != nil { + t.Fatal("the original generic CFG must independently establish this fixture", err) + } + } + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("new loop closure must preserve ordinary CFG acceptance", err) + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalReplayLateResidueFailureIsAtomic(t *testing.T) { + fn, sig, source := fixedLocalGPFixture(t, "MOVQ AX,16(SP)\nMOVQ AX,24(SP)\nMOVQ 24(SP),AX") + requireX86GoAssemblerResult(t, "amd64", source, true) + proof := &x86FixedLocalProof{frame: fn.FrameSize} + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + if err := replayX86FixedLocal(&fn, sig, proof, 0, &budget, nil, nil); err != nil { + t.Fatal(err) + } + last := -1 + for index, instruction := range fn.Instrs { + if len(instruction.x86FixedLocalMemory) != 0 { + last = index + } + } + if last < 0 { + t.Fatal("fixture has no late memory annotation") + } + for operand, access := range fn.Instrs[last].x86FixedLocalMemory { + access.width++ + fn.Instrs[last].x86FixedLocalMemory[operand] = access + } + before := make([]map[int]x86FixedLocalAccess, len(fn.Instrs)) + for index, instruction := range fn.Instrs { + before[index] = cloneX86ReplayFixedLocalMemory(instruction.x86FixedLocalMemory) + } + if err := replayX86FixedLocal(&fn, sig, proof, 1, &budget, nil, nil); err == nil { + t.Fatal("late cross-residue shape mismatch was accepted") + } + for index, instruction := range fn.Instrs { + if !reflect.DeepEqual(before[index], instruction.x86FixedLocalMemory) { + t.Fatal("failed final validation published an earlier partial annotation", index) + } + } +} diff --git a/amd64_fixed_local_counted_loop_go127_test.go b/amd64_fixed_local_counted_loop_go127_test.go new file mode 100644 index 000000000..91ff2d098 --- /dev/null +++ b/amd64_fixed_local_counted_loop_go127_test.go @@ -0,0 +1,105 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalCountedLoopProgram(t *testing.T) (string, string, []string) { + t.Helper() + var assembly, declarations strings.Builder + assembly.WriteString(fixedLocalAddressData) + var expected []string + appendFunction := func(body, want string) { + name := fmt.Sprintf("Value%d", len(expected)) + _, _, source := fixedLocalCountedLoopFixture(t, body) + assembly.WriteString(strings.Replace(source, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + expected = append(expected, want) + } + for _, row := range fixedLocalCountedLoopCases() { + // Every stored qword is input + 2, read once after the complete + // loop; a separate canary contributes 77. Native callers use -1. + appendFunction(row.body, fmt.Sprintf("%016x", row.iterations+77)) + } + for _, descending := range []bool{false, true} { + appendFunction(fixedLocalCountedLiteralBody(descending), "64686c7074787c80") + } + return assembly.String(), declarations.String(), expected +} + +func fixedLocalCountedLiteralBody(descending bool) string { + first, offset, limit, update, branch := 0, 0, 4, "ADDQ", "JB" + if descending { + first, offset, limit, update, branch = 4, -8, 0, "SUBQ", "JA" + } + return fmt.Sprintf("LEAQ literal<>(SB),BP\nMOVQ $0,8(SP)\nMOVQ $%d,SI\nloop:\n"+ + "MOVQ %d(BP)(SI*8),AX\nMOVQ AX,16(SP)(SI*8)\nADDQ AX,8(SP)\n"+ + "%s $1,SI\nCMPQ SI,$%d\n%s loop\nMOVQ 8(SP),AX", first, offset, update, limit, branch) +} + +func TestGoAMD64FixedLocalCountedLoopObjects(t *testing.T) { + assembly, declarations, expected := fixedLocalCountedLoopProgram(t) + if len(expected) != 38 { + t.Fatal("counted-loop object inventory changed", len(expected)) + } + fixedLocalCountedLoopObjects(t, assembly, declarations, len(expected)) +} + +func fixedLocalCountedLoopObjects(t *testing.T, assembly, declarations string, count int) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + var names []string + for index := 0; index < count; index++ { + names = append(names, fmt.Sprintf("%s.Value%d", pkg.Path, index)) + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("original bounded-loop source rejected", err) + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + compileLLVMToObject(t, llc, target.triple, "counted-loop.ll", "counted-loop.o", ir) + compileLLVMToObject(t, llc, target.triple, "counted-loop-opt.ll", "counted-loop-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +func TestGoAMD64FixedLocalCountedLiteralLoopGuards(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + valid := fixedLocalCountedLiteralBody(false) + for _, row := range []struct{ name, body string }{ + {"last read exceeds DATA", strings.Replace(valid, "CMPQ SI,$4", "CMPQ SI,$5", 1)}, + {"negative first read", strings.Replace(valid, "MOVQ 0(BP)", "MOVQ -8(BP)", 1)}, + {"readonly indexed store", strings.Replace(valid, "MOVQ 0(BP)(SI*8),AX", "MOVQ AX,0(BP)(SI*8)", 1)}, + {"readonly indexed RMW", strings.Replace(valid, "MOVQ 0(BP)(SI*8),AX", "ADDQ AX,0(BP)(SI*8)", 1)}, + } { + t.Run(row.name, func(t *testing.T) { + _, _, source := fixedLocalCountedLoopFixture(t, row.body) + pkg, request, file, options := fixedLocalTransportFixture(t, fixedLocalAddressData+source, + "func Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + owner := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := owner.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("counted loop acquired an invalid DATA extent or write", err) + } + }) + } +} diff --git a/amd64_fixed_local_counted_loop_runtime_go127_test.go b/amd64_fixed_local_counted_loop_runtime_go127_test.go new file mode 100644 index 000000000..5ff1df87c --- /dev/null +++ b/amd64_fixed_local_counted_loop_runtime_go127_test.go @@ -0,0 +1,57 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "encoding/binary" + "fmt" + "testing" +) + +func fixedLocalCountedLoopRuntimeInputs() []uint64 { + // Both low-bit values exercise one and two outer passes. The remaining + // values distinguish wrapped arithmetic and overlapping byte stores. + return []uint64{0, 1, ^uint64(0), ^uint64(0) - 1, 0x0123456789abcdef} +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalCountedLoop(t *testing.T) { + assembly, declarations, inventory := fixedLocalCountedLoopProgram(t) + if len(inventory) != 38 { + t.Fatal("counted-loop runtime inventory changed", len(inventory)) + } + inputs := fixedLocalCountedLoopRuntimeInputs() + var expected [][]string + for _, input := range inputs { + var values []string + for _, row := range fixedLocalCountedLoopCases() { + values = append(values, fmt.Sprintf("%016x", uint64(row.iterations)*(input+2)+77)) + } + expected = append(expected, append(values, "64686c7074787c80", "64686c7074787c80")) + } + fixedLocalResultsRuntime(t, assembly, declarations, inputs, expected) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalCountedLoopAliases(t *testing.T) { + assembly, declarations, cases := fixedLocalCountedLoopAliasProgram(t) + inputs := fixedLocalCountedLoopRuntimeInputs() + var expected [][]string + for _, input := range inputs { + var values []string + for _, row := range cases { + // This byte-array oracle preserves earlier stores that overlap + // later iterations; retaining only the last access gives a + // different result for these nonuniform input bytes. + var memory [32]byte + for _, offset := range row.offsets { + binary.LittleEndian.PutUint64(memory[offset:], input+uint64(offset)+2) + } + sum := uint64(77) + for _, offset := range row.offsets { + sum += binary.LittleEndian.Uint64(memory[offset:]) + } + values = append(values, fmt.Sprintf("%016x", sum)) + } + expected = append(expected, values) + } + fixedLocalResultsRuntime(t, assembly, declarations, inputs, expected) +} diff --git a/amd64_fixed_local_counted_loop_test.go b/amd64_fixed_local_counted_loop_test.go new file mode 100644 index 000000000..eb0a9fe24 --- /dev/null +++ b/amd64_fixed_local_counted_loop_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type fixedLocalCountedLoopCase struct { + name, body string + iterations int +} + +func fixedLocalCountedLoopCases() []fixedLocalCountedLoopCase { + var cases []fixedLocalCountedLoopCase + for _, count := range []int{2, 3, 8} { + for _, step := range []int{1, 8, 16} { + for _, descending := range []bool{false, true} { + for _, outer := range []bool{false, true} { + scale := 1 + if step == 1 { + scale = 8 + } + first, limit, update, branch := 0, count*step, "ADDQ", "JB" + if descending { + first, limit, update, branch = count*step, 0, "SUBQ", "JA" + } + var body strings.Builder + body.WriteString("MOVQ $77,248(SP)\n") + if outer { + // The input-dependent outer loop remains an ordinary CFG + // fixed point. Its one or two passes must each re-establish + // the inner loop's complete private initialization. + body.WriteString("MOVQ a+0(FP),DI\nANDQ $1,DI\nADDQ $1,DI\nMOVQ $0,DX\nouter:\n") + } + fmt.Fprintf(&body, "MOVQ $%d,SI\nloop:\nMOVQ AX,16(SP)(SI*%d)\nADDQ $2,16(SP)(SI*%d)\n%s $%d,SI\nCMPQ SI,$%d\n%s loop\n", + first, scale, scale, update, step, limit, branch) + if outer { + body.WriteString("ADDQ $1,DX\nCMPQ DX,DI\nJB outer\n") + } + // An independent fixed-address read of every iteration's + // store detects partial initialization and skipped visits. + for iteration := 0; iteration < count; iteration++ { + offset := 16 + iteration*step*scale + if descending { + offset = 16 + (count-iteration)*step*scale + } + op := "ADDQ" + if iteration == 0 { + op = "MOVQ" + } + fmt.Fprintf(&body, "%s %d(SP),AX\n", op, offset) + } + body.WriteString("ADDQ 248(SP),AX") + name := fmt.Sprintf("count%d/step%d/descending%t/outer%t", count, step, descending, outer) + cases = append(cases, fixedLocalCountedLoopCase{name, body.String(), count}) + } + } + } + } + return cases +} + +func fixedLocalCountedLoopFixture(t *testing.T, body string) (Func, FuncSig, string) { + t.Helper() + _, sig, source := fixedLocalGPFixture(t, body) + source = strings.Replace(source, "$64-16", "$256-16", 1) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + return file.Funcs[0], sig, source +} + +func TestGoAMD64FixedLocalCountedLoopCompleteInitialization(t *testing.T) { + var original strings.Builder + cases := fixedLocalCountedLoopCases() + if len(cases) != 36 { + t.Fatal("count/stride/direction/outer-loop inventory changed", len(cases)) + } + for index, row := range cases { + fn, sig, source := fixedLocalCountedLoopFixture(t, row.body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + proved, err := proveX86FixedLocal(fn, sig) + if err != nil { + t.Fatal("bounded iterations lost initialized private bytes", err) + } + before, beforeErr := x86CallerSourceFingerprint(fn) + after, afterErr := x86CallerSourceFingerprint(proved) + if beforeErr != nil || afterErr != nil || before != after || len(fn.Instrs) != len(proved.Instrs) { + t.Fatal("proof must retain the complete original source and loop body", beforeErr, afterErr) + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalCountedLoopRejectsUnprovedIterations(t *testing.T) { + const prefix = "MOVQ $0,SI\nloop:\n" + const store = "MOVQ AX,16(SP)(SI*8)\n" + const suffix = "ADDQ $1,SI\nCMPQ SI,$3\nJB loop\nMOVQ 32(SP),AX" + var original strings.Builder + for index, row := range []struct{ name, body string }{ + {"read before first write", prefix + "MOVQ 16(SP)(SI*8),BX\n" + store + suffix}, + {"read next iteration before write", prefix + store + "MOVQ 24(SP)(SI*8),BX\n" + suffix}, + {"partial initialization", prefix + "MOVB AX,16(SP)(SI*8)\n" + suffix}, + {"partial counter overwrite", prefix + "ADDB $0,SI\n" + store + suffix}, + {"counter overwritten in body", prefix + "ADDQ $1,SI\n" + store + suffix}, + {"unknown initial counter", "MOVQ a+0(FP),SI\nloop:\n" + store + suffix}, + {"zero update cannot initialize later bytes", prefix + store + "ADDQ $0,SI\nCMPQ SI,$3\nJB loop\nMOVQ 32(SP),AX"}, + {"unbounded limit", prefix + store + "ADDQ $1,SI\nCMPQ SI,AX\nJB loop\nMOVQ AX,AX"}, + {"latch flags replaced", prefix + store + "ADDQ $1,SI\nCMPQ SI,$3\nTESTQ AX,AX\nJB loop\nMOVQ 32(SP),AX"}, + {"latch compares another counter", prefix + store + "ADDQ $1,SI\nCMPQ SI,$3\nCMPQ AX,$3\nJB loop\nMOVQ AX,AX"}, + {"counter wraps", "MOVQ $-1,SI\nloop:\n" + store + "ADDQ $1,SI\nCMPQ SI,$3\nJB loop"}, + {"entry bypasses counter", "CMPQ AX,$0\nJEQ inside\nMOVQ $0,SI\nloop:\ninside:\n" + store + suffix}, + {"conditional write is incomplete", prefix + "CMPQ AX,$0\nJEQ missing\n" + store + "missing:\n" + suffix}, + {"unwritten gap after loop", prefix + store + "ADDQ $2,SI\nCMPQ SI,$4\nJB loop\nMOVQ 24(SP),AX"}, + {"last iteration exceeds frame", "MOVQ $29,SI\nloop:\n" + store + "ADDQ $1,SI\nCMPQ SI,$32\nJB loop"}, + {"early iteration changes residue alias", "MOVQ $77,16(SP)\nMOVQ SP,R8\nADDQ $16,R8\nANDQ $-16,R8\nMOVQ $1,SI\nloop:\nMOVQ AX,(R8)(SI*8)\nADDQ $1,SI\nCMPQ SI,$3\nJB loop\nMOVQ 16(SP),AX"}, + } { + fn, sig, source := fixedLocalCountedLoopFixture(t, row.body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err == nil { + t.Fatal("incomplete iterations or private-byte aliases acquired a proof") + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} diff --git a/amd64_fixed_local_counted_loop_update_gap_go127_test.go b/amd64_fixed_local_counted_loop_update_gap_go127_test.go new file mode 100644 index 000000000..eebb66c4c --- /dev/null +++ b/amd64_fixed_local_counted_loop_update_gap_go127_test.go @@ -0,0 +1,39 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalCountedLoopUpdateGapProgram(t *testing.T) (string, string, int) { + t.Helper() + var assembly, declarations strings.Builder + count := 0 + for _, row := range fixedLocalCountedLoopUpdateGapCases() { + if !strings.HasPrefix(row.name, "SI/") || + !strings.HasSuffix(row.name, "/GP and flags") && !strings.HasSuffix(row.name, "/two vector moves") { + continue + } + // The scratch GP in the gap changes AX. Keep the input separately + // and make every initialized slot equal input+2 modulo the word size. + body := "MOVQ a+0(FP),R8\n" + strings.Replace(row.body, + "MOVQ $7,16(SP)(SI*8)", "MOVQ R8,16(SP)(SI*8)\nADDQ $2,16(SP)(SI*8)", 1) + _, _, source := fixedLocalCountedLoopFixture(t, body) + name := fmt.Sprintf("Value%d", count) + assembly.WriteString(strings.Replace(source, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + count++ + } + if count != 8 { + t.Fatal("four counter updates by GP/vector gap inventory changed", count) + } + return assembly.String(), declarations.String(), count +} + +func TestGoAMD64FixedLocalCountedLoopUpdateGapObjects(t *testing.T) { + assembly, declarations, count := fixedLocalCountedLoopUpdateGapProgram(t) + fixedLocalCountedLoopObjects(t, assembly, declarations, count) +} diff --git a/amd64_fixed_local_counted_loop_update_gap_runtime_linux_go127_test.go b/amd64_fixed_local_counted_loop_update_gap_runtime_linux_go127_test.go new file mode 100644 index 000000000..46c82e05d --- /dev/null +++ b/amd64_fixed_local_counted_loop_update_gap_runtime_linux_go127_test.go @@ -0,0 +1,24 @@ +//go:build linux && (amd64 || arm64) && go1.27 + +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalCountedLoopUpdateGap(t *testing.T) { + assembly, declarations, count := fixedLocalCountedLoopUpdateGapProgram(t) + inputs := fixedLocalCountedLoopRuntimeInputs() + var expected [][]string + for _, input := range inputs { + values := make([]string, count) + for index := range values { + values[index] = fmt.Sprintf("%016x", input+2) + } + expected = append(expected, values) + } + // Original Go executes the Y-register moves. Require actual AVX and + // enabled OS register state before either source program runs. + fixedLocalResultsRuntimeISA(t, assembly, declarations, inputs, expected, x86FloatCompareVEX) +} diff --git a/amd64_fixed_local_counted_loop_update_gap_test.go b/amd64_fixed_local_counted_loop_update_gap_test.go new file mode 100644 index 000000000..ea36c8364 --- /dev/null +++ b/amd64_fixed_local_counted_loop_update_gap_test.go @@ -0,0 +1,103 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalCountedLoopUpdateGapCases() []fixedLocalCountedLoopCase { + var cases []fixedLocalCountedLoopCase + for _, register := range strings.Fields("AX BX CX DX SI DI BP R8 R9 R10 R11 R12 R13 R14 R15") { + scratch := "AX" + if register == scratch { + scratch = "BX" + } + for _, operation := range []string{"ADDQ", "SUBQ", "INCQ", "DECQ"} { + first, bound, branch := 0, 3, "JB" + if operation == "SUBQ" || operation == "DECQ" { + first, bound, branch = 3, 0, "JA" + } + update := operation + " " + register + if operation == "ADDQ" || operation == "SUBQ" { + update = operation + " $1," + register + } + for _, gap := range []struct{ name, initialize, body string }{ + {"NOP", "", "NOP"}, + {"GP and flags", "", fmt.Sprintf("MOVQ %s,%s\nADDQ $3,%s", register, scratch, scratch)}, + {"two vector moves", "MOVQ $0,64(SP)\nMOVQ $0,72(SP)\nMOVQ $0,80(SP)\nMOVQ $0,88(SP)\n" + + "VMOVDQU 64(SP),Y6\nVMOVDQU Y6,Y7\n", "VMOVDQU Y6,Y4\nVMOVDQU Y7,Y5"}, + {"private word", "", "MOVQ $7,64(SP)\nMOVQ 64(SP)," + scratch}, + } { + body := gap.initialize + fmt.Sprintf("MOVQ $%d,%s\nloop:\n"+ + "MOVQ $7,16(SP)(%s*8)\n%s\n%s\nCMPQ %s,$%d\n%s loop\nMOVQ 24(SP),AX", + first, register, register, update, gap.body, register, bound, branch) + name := register + "/" + operation + "/" + gap.name + cases = append(cases, fixedLocalCountedLoopCase{name, body, 3}) + } + } + } + return cases +} + +func TestGoAMD64FixedLocalCountedLoopUpdateGap(t *testing.T) { + cases := fixedLocalCountedLoopUpdateGapCases() + if len(cases) != 240 { + t.Fatal("15 counters by four updates and four typed gaps changed", len(cases)) + } + var original strings.Builder + for index, row := range cases { + _, _, source := fixedLocalCountedLoopFixture(t, row.body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) + for _, row := range cases { + t.Run(row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalCountedLoopFixture(t, row.body) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("typed instructions preserving the counter must not hide its unique update", err) + } + }) + } +} + +func TestGoAMD64FixedLocalCountedLoopUpdateGapRejectsCounterChanges(t *testing.T) { + const prefix = "MOVQ $0,SI\nloop:\nMOVQ $7,16(SP)(SI*8)\n" + const suffix = "CMPQ SI,$3\nJB loop\nMOVQ 32(SP),AX" + type example struct{ name, body string } + var cases []example + for _, row := range []struct{ name, middle string }{ + {"changed before update", "MOVQ $7,SI\nADDQ $1,SI\nNOP\n"}, + {"changed after update", "ADDQ $1,SI\nMOVQ $7,SI\n"}, + {"partial before update", "ADDB $1,SI\nADDQ $1,SI\nNOP\n"}, + {"partial after update", "ADDQ $1,SI\nADDB $1,SI\n"}, + {"zero-extended counter", "ADDQ $1,SI\nMOVL SI,SI\n"}, + {"two updates", "ADDQ $1,SI\nNOP\nINCQ SI\nNOP\n"}, + {"canceling updates", "ADDQ $1,SI\nNOP\nSUBQ $1,SI\nNOP\n"}, + {"partial counter alias", "MOVQ SI,AX\nADDQ $1,SI\nMOVB AL,SI\n"}, + {"unknown counter replacement", "ADDQ $1,SI\nMOVQ a+0(FP),SI\n"}, + } { + cases = append(cases, example{row.name, prefix + row.middle + suffix}) + } + // LAHF has a typed implicit AH write; FLAGS are initialized by CMP. + // An otherwise unsupported string instruction would fail too early to + // establish that the counter guard sees implicit register writes. + implicit := prefix + "ADDQ $1,SI\nCMPQ SI,SI\nLAHF\n" + suffix + cases = append(cases, example{"implicit counter write", strings.ReplaceAll(implicit, "SI", "AX")}) + var original strings.Builder + for index, row := range cases { + fn, sig, source := fixedLocalCountedLoopFixture(t, row.body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + for _, instruction := range fn.Instrs { + if !fixedLocalInstructionShape(instruction) { + t.Fatal("counter negative needs registered typed instructions", instruction.Raw) + } + } + if _, err := proveX86FixedLocal(fn, sig); err == nil { + t.Fatal("a changed counter or latch comparison acquired complete iteration authority") + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} diff --git a/amd64_fixed_local_data.go b/amd64_fixed_local_data.go new file mode 100644 index 000000000..650f571e5 --- /dev/null +++ b/amd64_fixed_local_data.go @@ -0,0 +1,218 @@ +package plan9asm + +import ( + "bytes" + "strings" +) + +type x86FixedLocalData struct { + size, alignment int64 + literals map[int64]x86ShiftLiteral +} + +type x86FixedLocalMemoryRole uint8 + +const ( + x86FixedLocalPrivateMemory x86FixedLocalMemoryRole = iota + x86FixedLocalLiteralMemory + x86FixedLocalNativeInput +) + +func fixedLocalDataSymbol(symbol string) (string, bool) { + name := strings.TrimPrefix(symbol, "·") + if !strings.HasSuffix(name, "<>") || len(name) <= 2 || strings.ContainsAny(name, "·/.") { + return "", false + } + return "·" + name, true +} + +// Library callers preserve the private <> spelling; the ordinary CLIs render +// it as $local. Both describe the same source-local LLVM namespace, not the +// original Go object's name/ABI. Never accept an arbitrary caller renaming. +func fixedLocalDataEmissionMatches(symbol, packagePath, emitted string) bool { + canonical := numericalNativeName(symbol, packagePath) + return emitted == canonical || emitted == strings.TrimSuffix(canonical, "<>")+"$local" +} + +// Called under the original source lock after its whole parsed file has been +// matched. Private literal definitions cannot be replaced by a Go variable, +// linker winner, mutable global, address initializer or caller-supplied extent. +func (source *GoAMD64FrameSource) fixedLocalData(file *File, expanded string, resolve func(string) string) (map[string]x86FixedLocalData, map[string]string, error) { + if len(file.Data)+len(file.Globl) == 0 { + return nil, nil, nil + } + observed, err := source.fixedLocalDataObservation(source.request.AssemblyFile, expanded) + if err != nil { + return nil, nil, err + } + if len(observed.Layouts) != len(file.Globl) { + return nil, nil, goAMD64FrameContext("fixed local DATA lacks its whole original definition inventory") + } + data := make(map[string]x86FixedLocalData) + requests := x86PackedLogicalLiteralRequests(file, "amd64") + symbols := make(map[string]string) + // BindFixedLocal has already frozen the resolver outside the source lock. + // Keep every emitted definition injective, including the complete TEXT set. + used := make(map[string]bool, len(file.Funcs)+len(file.Globl)) + for _, fn := range file.Funcs { + name := resolve(fn.Sym) + if name == "" || used[name] { + return nil, nil, goAMD64FrameContext("fixed local emitted function identities are empty or collapsed") + } + used[name] = true + } + var total int64 + for index, global := range file.Globl { + layout := observed.Layouts[index] + definition := layout.Definition + name, private := fixedLocalDataSymbol(global.Sym) + emitted := resolve(name) + if _, duplicate := symbols[name]; duplicate { + return nil, nil, goAMD64FrameContext("fixed local DATA definitions collapse to the same translated identity") + } + if !private || !fixedLocalDataEmissionMatches(name, source.selection.ImportPath, emitted) || used[emitted] { + return nil, nil, goAMD64FrameContext("fixed local DATA resolver changed or collapsed its private emitted identity") + } + used[emitted] = true + total += int64(definition.Size) + if !layout.Static || layout.ABI != 65535 || + layout.Flags != 8 && layout.Flags != 24 || definition.Kind != 3 && definition.Kind != 4 || definition.Flags != 0 || + definition.Flags2 != 0 || len(definition.Relocations) != 0 || global.SizeRaw != "" || + definition.Size == 0 || definition.Size > 64<<10 || total > 1<<20 || len(definition.Data) > int(definition.Size) { + return nil, nil, goAMD64FrameContext("fixed local DATA requires bounded private relocation-free readonly storage") + } + // Compare the translator's complete initializer with independent actual + // Go bytes, including sparse zero gaps and trailing allocation. A GLOBL + // that shortens preceding DATA must not acquire a larger LLVM extent. + image := make([]byte, definition.Size) + size := global.Size + for _, fragment := range file.Data { + if fragment.Sym != global.Sym { + continue + } + payload, err := dataStmtPayload(fragment) + if err != nil || fragment.Addr != "" || fragment.Off < 0 || fragment.Off > int64(len(image))-int64(len(payload)) { + return nil, nil, goAMD64FrameContext("fixed local DATA initializer has an unbound source byte extent") + } + copy(image[fragment.Off:], payload) + if end := fragment.Off + int64(len(payload)); end > size { + size = end + } + } + if size != int64(definition.Size) || !bytes.Equal(image[:len(definition.Data)], definition.Data) || + !bytes.Equal(image[len(definition.Data):], make([]byte, len(image)-len(definition.Data))) { + return nil, nil, goAMD64FrameContext("fixed local translated DATA differs from the original Go initialized extent") + } + literal := x86FixedLocalData{size: size, alignment: goAssemblyDataAlignment(ArchAMD64, size)} + for offset, width := range requests[global.Sym] { + if offset > size-int64(width) { + continue // The ordinary full access check owns this rejection. + } + if literal.literals == nil { + literal.literals = make(map[int64]x86ShiftLiteral) + } + tuple := x86ShiftLiteral{width: uint8(width)} + copy(tuple.bytes[:], image[offset:offset+int64(width)]) + literal.literals[offset] = tuple + } + data[global.Sym] = literal + symbols[name] = emitted + } + return data, symbols, nil +} + +// Project only immutable original DATA facts while constructing the local +// effect plan. The full memory read and its bounds/alignment remain unchanged. +func fixedLocalLogicalLiteralEffect(ins Instr, effect x86TypedByteEffect, binding *x86FixedLocalBinding) x86TypedByteEffect { + if binding == nil || len(ins.Args) == 0 || ins.Args[0].Kind != OpSym { + return effect + } + form, known, err := x86PackedLogicalProperties(ins, "amd64") + if !known || err != nil || form.memory != form.first || form.memoryBytes <= 0 { + return effect + } + address, valid := x86StaticSBReadAddress(form.source, "amd64") + literal := binding.literalData[address.symbol].literals[address.offset] + if !valid || int(literal.width) < form.memoryBytes { + return effect + } + var required uint64 + for index := 0; index < form.byteWidth; index++ { + if x86PackedLogicalSecondBits(form.spec.mode, literal.bytes[index%form.memoryBytes]) != 0 { + required |= uint64(1) << uint(index) + } + } + if required == x86CallerVectorByteMask(int64(form.byteWidth)) { + return effect + } + effect.operands = append([]x86TypedByteOperand(nil), effect.operands...) + for index := range effect.operands { + operand := &effect.operands[index] + if operand.index == form.second && operand.operand.Kind == OpReg { + // requiredBytes zero otherwise means the ordinary full-width read. + operand.read, operand.requiredBytes = required != 0, required + } + } + return effect +} + +// Readonly DATA never receives a frame offset or a native-frame certificate. +// The only consumer is an already registered numerical memory operand. The +// complete original file and immutable symbol resolution own its LLVM global. +func fixedLocalReadMemory(ins *Instr, index int, state *x86FixedLocalState, proof *x86FixedLocalProof, + residue int, width, alignment int64) (offset int64, role x86FixedLocalMemoryRole, err error) { + if index >= 0 && index < len(ins.Args) && ins.Args[index].Kind == OpSym { + if proof.binding == nil || width <= 0 || alignment <= 0 { + return 0, x86FixedLocalLiteralMemory, goAMD64FrameContext("fixed local DATA read has no original literal owner") + } + name, offset, valid := parseSBRef(ins.Args[index].Sym) + if !valid || !fixedLocalLiteralRange(proof.binding, x86StaticSBAddress{name, offset}, width, alignment) { + return 0, x86FixedLocalLiteralMemory, goAMD64FrameContext("fixed local DATA read exceeds original identity, extent or alignment") + } + return offset, x86FixedLocalLiteralMemory, nil + } + return fixedLocalMemoryAccess(ins, index, state, proof, residue, width, alignment, false, false) +} + +func fixedLocalLiteralRange(binding *x86FixedLocalBinding, address x86StaticSBAddress, width, alignment int64) bool { + if binding == nil || width <= 0 || alignment <= 0 { + return false + } + data, found := binding.literalData[address.symbol] + return found && address.offset >= 0 && width <= data.size && address.offset <= data.size-width && + alignment <= data.alignment && address.offset%alignment == 0 +} + +func fixedLocalStaticAddressStep(ins Instr, state *x86FixedLocalState, proof *x86FixedLocalProof) (bool, error) { + address, valid := fixedLocalStaticAddress(ins) + if !valid { + return false, nil + } + fail := func() (bool, error) { + return true, goAMD64FrameContext("fixed local address has no matching original storage: %s", ins.Op) + } + if proof == nil || proof.binding == nil { + return fail() + } + var origin x86FixedLocalOrigin + if _, literal := proof.binding.literalData[address.symbol]; literal { + // Address computation may leave an object before returning. A later + // exact access owns its bounds diagnostic, never adjacent storage. + origin.literal = address + } else { + object, found := proof.binding.globals[fixedLocalGlobalSymbol(address.symbol)] + if ins.Op == "LEAQ" || !found || object.variable == nil || object.native == "" { + return fail() + } + origin.global = object.native + } + writeFixedLocalOrigin(state, ins.Args[1].Reg, fixedLocalOrigin(origin)) + return true, nil +} + +func writeFixedLocalOrigin(state *x86FixedLocalState, destination Reg, origin x86FixedLocalOrigins) { + state.revokeRange(destination) + delete(state.pointers, destination) + delete(state.constants, destination) + state.opaque[destination], state.known[destination], state.booleans[destination] = origin, 0, 0 +} diff --git a/amd64_fixed_local_data_address_go127_test.go b/amd64_fixed_local_data_address_go127_test.go new file mode 100644 index 000000000..586cb8fbf --- /dev/null +++ b/amd64_fixed_local_data_address_go127_test.go @@ -0,0 +1,294 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +const fixedLocalAddressData = "DATA literal<>+0(SB)/8,$0x0102030405060708\n" + + "DATA literal<>+8(SB)/8,$0x1112131415161718\n" + + "DATA literal<>+16(SB)/8,$0x2122232425262728\n" + + "DATA literal<>+24(SB)/8,$0x3132333435363738\nGLOBL literal<>(SB),8,$32\n" + +func fixedLocalDataAddressFunction(name, body string) string { + return fmt.Sprintf("TEXT ·%s(SB),4,$32-16\nMOVQ a+0(FP),CX\nMOVQ $0,DX\n%s\nMOVQ DX,ret+8(FP)\nRET\n", name, body) +} + +func fixedLocalDataAddressKernels() (string, string, int) { + var assembly, declarations strings.Builder + assembly.WriteString(fixedLocalAddressData) + count := 0 + // The register inventory is independent of the translator's bank helper. + // SP is the private frame, not an ordinary destination for a DATA address. + for _, register := range strings.Fields("AX BX CX DX DI SI BP R8 R9 R10 R11 R12 R13 R14 R15") { + for _, address := range []string{"MOVQ $literal<>(SB),", "LEAQ literal<>(SB),"} { + name := fmt.Sprintf("Value%d", count) + body := fmt.Sprintf(`%s%s +MOVQ %s,8(SP) +MOVQ $0,%s +MOVQ 8(SP),%s +ADDQ $16,%s +SUBQ $8,%s +LEAQ 8(%s),%s +INCQ %s +DECQ %s +MOVOU (%s),X0 +PADDQ X0,X0 +MOVQ X0,DX`, address, register, register, register, register, + register, register, register, register, register, register, register) + assembly.WriteString(fixedLocalDataAddressFunction(name, body)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + count++ + } + } + return assembly.String(), declarations.String(), count +} + +func TestGoAMD64FixedLocalReadonlyDataAddressObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + assembly, declarations, count := fixedLocalDataAddressKernels() + if count != 30 { + t.Fatal("complete GP bank/address grammar denominator changed", count) + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("bounded readonly DATA addresses must compose with the complete private frame", err) + } + if !strings.Contains(ir, "internal constant [32 x i8]") { + t.Fatal("original private readonly DATA definition missing") + } + compileLLVMToObject(t, llc, target.triple, "data-address.ll", "data-address.o", ir) + compileLLVMToObject(t, llc, target.triple, "data-address-opt.ll", "data-address-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + t.Log("30 original-Go kernels: two address forms, full non-SP GP bank, complete spills and constant affine offsets; three APIs at O0/O2") +} + +func TestGoAMD64FixedLocalReadonlyDataAddressGuards(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + const acquire = "LEAQ literal<>(SB),SI\n" + // These complete machine addresses need their original storage, but not a + // universal bound for every possible invocation. This is translation only. + dynamic := map[string]bool{ + "unknown affine offset": true, + "unknown index": true, + "joined offset past end": true, + "numeric pointer": true, + } + for _, row := range []struct{ name, body string }{ + {"past end", acquire + "MOVQ 25(SI),DX"}, + {"negative", acquire + "MOVQ -1(SI),DX"}, + {"affine past end", acquire + "ADDQ $25,SI\nMOVQ (SI),DX"}, + {"affine negative", acquire + "SUBQ $1,SI\nMOVQ (SI),DX"}, + {"initial displacement", "LEAQ literal<>+25(SB),SI\nMOVQ (SI),DX"}, + {"unaligned", acquire + "INCQ SI\nMOVO (SI),X0"}, + {"unknown affine offset", acquire + "ADDQ CX,SI\nMOVQ (SI),DX"}, + {"unknown index", acquire + "MOVQ (SI)(CX*1),DX"}, + {"offset overflow", acquire + "MOVQ $9223372036854775807,R8\nADDQ R8,SI\nINCQ SI\nMOVQ (SI),DX"}, + {"joined offset past end", acquire + "CMPQ CX,$0\nJEQ joined\nADDQ $32,SI\njoined:\nMOVQ (SI),DX"}, + {"joined readonly write", acquire + "CMPQ CX,$0\nJEQ joined\nADDQ $8,SI\njoined:\nMOVQ DX,(SI)"}, + {"partial spill", acquire + "MOVL SI,8(SP)\nMOVQ 8(SP),SI\nMOVQ (SI),DX"}, + {"write", acquire + "MOVQ DX,(SI)"}, + {"write after spill", acquire + "MOVQ SI,8(SP)\nMOVQ 8(SP),BP\nMOVQ DX,(BP)"}, + {"numeric result", acquire + "MOVQ SI,DX"}, + {"numeric pointer", "MOVQ $1234,SI\nMOVQ (SI),DX"}, + {"unowned symbol", "MOVQ $foreign(SB),SI\nMOVQ (SI),DX"}, + } { + t.Run(row.name, func(t *testing.T) { + assembly := fixedLocalAddressData + fixedLocalDataAddressFunction("Value", row.body) + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + // Capture must succeed first: a producer/tool failure does not + // establish that the translator rejects an unsafe memory role. + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if dynamic[row.name] { + if err != nil { + t.Fatal("dynamic native address incorrectly required a caller bounds proof", err) + } + if row.name == "numeric pointer" { + proved, err := proveX86FixedLocal(file.Funcs[0], bound.Sigs[pkg.Path+".Value"]) + if err != nil { + t.Fatal(err) + } + fixedLocalRequireNumericNativeRole(t, proved, SI) + } + return + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("readonly address acquired unknown bytes, alignment, mutable or numerical permission", err) + } + }) + } + t.Run("different bounded offsets at join", func(t *testing.T) { + assembly := fixedLocalAddressData + fixedLocalDataAddressFunction("Value", + acquire+"CMPQ CX,$0\nJEQ joined\nADDQ $8,SI\njoined:\nMOVQ (SI),DX") + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("both readonly alternatives must retain their bounded read", err) + } + compileLLVMToObject(t, llc, options.TargetTriple, "joined-data.ll", "joined-data.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "joined-data-opt.ll", "joined-data-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + t.Run("pointer result cannot escape its bounds", func(t *testing.T) { + assembly := fixedLocalAddressData + "TEXT ·Value(SB),4,$32-8\n" + acquire + "MOVQ SI,ret+0(FP)\nRET\n" + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value() *uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("private literal escaped as an unbounded externally mutable pointer", err) + } + }) +} + +func TestGoAMD64FixedLocalReadonlyDataAddressBinding(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + assembly := fixedLocalAddressData + fixedLocalDataAddressFunction("Value", + "MOVQ SP,R12\nADDQ $16,R12\nANDQ $-16,R12\n"+ + "MOVQ $literal<>+8(SB),BP\nMOVQ BP,(R12)\nMOVQ (R12),R14\nMOVQ 16(R14),DX") + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + fixedLocalDataAPIs(t, pkg, file, assembly, options, fixedLocalRequireContext) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("actual original DATA address failed binding", err) + } + for _, mutation := range []string{"resolver", "JSON", "byte", "extent"} { + t.Run(mutation, func(t *testing.T) { + changed, input := bound, assembly + switch mutation { + case "resolver": + changed.ResolveSym = func(symbol string) string { + if symbol == "·literal<>" { + return "foreign.literal<>" + } + return options.ResolveSym(symbol) + } + case "JSON": + changed.Sigs, err = cloneFixedLocalSigs(bound.Sigs) + if err != nil { + t.Fatal(err) + } + case "byte": + input = strings.Replace(input, "0x3132333435363738", "0x3132333435363739", 1) + case "extent": + input = strings.Replace(input, ",8,$32", ",8,$64", 1) + } + parsed, err := Parse(ArchAMD64, input) + if err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, parsed, input, changed, fixedLocalRequireContext) + }) + } + path := filepath.Join(request.Directory, request.AssemblyFile) + changed := strings.Replace(assembly, "0x3132333435363738", "0x3132333435363739", 1) + if err := os.WriteFile(path, []byte(changed), 0600); err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, file, assembly, bound, fixedLocalRequireContext) + if err := os.WriteFile(path, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, file, assembly, bound, fixedLocalRequireContext) +} + +func TestGoAMD64FixedLocalReadonlyDataAddressLayoutFlags(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + const acquire = "LEAQ literal<>(SB),SI\n" + const other = "DATA other<>+0(SB)/8,$77\nGLOBL other<>(SB),8,$32\n" + for _, row := range []struct { + name, body string + emit bool + }{ + {"ADD carry", acquire + "ADDQ $8,SI\nSETCS DL", true}, + {"SUB carry", acquire + "ADDQ $8,SI\nSUBQ $8,SI\nSETCS DL", true}, + {"INC zero", acquire + "INCQ SI\nSETEQ DL", true}, + {"DEC zero", acquire + "ADDQ $8,SI\nDECQ SI\nSETEQ DL", true}, + {"cross literal comparison", acquire + "LEAQ other<>(SB),DI\nCMPQ SI,DI\nSETCS DL", true}, + {"literal numerical comparison", acquire + "CMPQ SI,$1234\nSETCS DL", true}, + {"spilled literal comparison", acquire + "MOVQ SI,8(SP)\nCMPQ 8(SP),$0\nSETEQ DL", true}, + } { + t.Run(row.name, func(t *testing.T) { + assembly := fixedLocalAddressData + other + fixedLocalDataAddressFunction("Value", row.body) + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + if row.emit { + // Preserve the actual machine operation. Compiling independent + // globals does not promise Go's final linked addresses, relative + // ordering or cross-object adjacency; this is object evidence. + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("complete native address FLAGS source rejected", err) + } + x86CompactExpandRequireAllDefinitions(t, ir, []string{pkg.Path + ".Value"}) + compileLLVMToObject(t, llc, options.TargetTriple, "address-flags.ll", "address-flags.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "address-flags-opt.ll", "address-flags-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + return + } + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unsupported literal/immediate comparator acquired an address role", err) + } + }) + } + for _, row := range []struct{ name, body string }{ + {"LEA preserves flags", "CMPQ CX,$1\n" + acquire + "LEAQ 8(SI),SI\nSETEQ DL"}, + {"INC preserves carry", "CMPQ CX,$1\n" + acquire + "INCQ SI\nSETCS DL"}, + {"DATA bytes are numeric", acquire + "CMPQ (SI),$0\nSETNE DL"}, + } { + t.Run(row.name, func(t *testing.T) { + assembly := fixedLocalAddressData + fixedLocalDataAddressFunction("Value", row.body) + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil || ir == "" { + t.Fatal("unaffected numerical FLAGS were lost across a DATA address operation", err) + } + }) + }) + } +} diff --git a/amd64_fixed_local_data_address_runtime_go127_test.go b/amd64_fixed_local_data_address_runtime_go127_test.go new file mode 100644 index 000000000..e9f437992 --- /dev/null +++ b/amd64_fixed_local_data_address_runtime_go127_test.go @@ -0,0 +1,162 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalReadonlyDataAddress(t *testing.T) { + assembly, declarations, count := fixedLocalDataAddressKernels() + if count != 30 { + t.Fatal("complete DATA address runtime inventory changed", count) + } + expected := make([]string, count) + for index := range expected { + // The source DATA's third qword is loaded and added to itself. + expected[index] = "424446484a4c4e50" + } + fixedLocalConstantResultsRuntime(t, assembly, declarations, expected) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConstantIndex(t *testing.T) { + cases := fixedLocalConstantIndexCases() + if len(cases) != 30 { + t.Fatal("complete constant index runtime inventory changed", len(cases)) + } + var assembly, declarations strings.Builder + assembly.WriteString(fixedLocalAddressData) + var expected []string + for index, row := range cases { + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(fixedLocalConstantIndexFunction(name, row.body)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + // Indexed and fixed-address reads each sum to 19+20+22+26, plus + // the adjacent private canary 77. The fixed tail is reset each time. + value := "00000000000000fb" + if row.literal { + value = "64686c7074787c80" // Sum of the four independent source DATA qwords. + } + expected = append(expected, value) + } + fixedLocalConstantResultsRuntime(t, assembly.String(), declarations.String(), expected) +} + +func fixedLocalConstantResultsRuntime(t *testing.T, assembly, declarations string, expected []string) { + t.Helper() + fixedLocalResultsRuntime(t, assembly, declarations, []uint64{^uint64(0)}, [][]string{expected}) +} + +func fixedLocalResultsRuntime(t *testing.T, assembly, declarations string, inputs []uint64, expected [][]string) { + t.Helper() + fixedLocalResultsRuntimeISA(t, assembly, declarations, inputs, expected, x86FloatCompareSSE) +} + +func fixedLocalResultsRuntimeISA(t *testing.T, assembly, declarations string, inputs []uint64, expected [][]string, axis x86FloatCompareRuntimeISA) { + t.Helper() + if len(inputs) == 0 || len(expected) != len(inputs) || len(expected[0]) == 0 { + t.Fatal("invalid fixed-local runtime input/result inventory") + } + count := len(expected[0]) + var values []string + for index, row := range expected { + if len(row) != count { + t.Fatal("inconsistent fixed-local runtime result count", index, len(row), count) + } + values = append(values, row...) + } + execution := x86FloatCompareRuntimeTools(t, axis) + goAMD64FramePrepareOwnedToolCache(t) + prefix := "" + if strings.Contains(execution.triple, "apple") { + prefix = "_" + } + var goCalls, cDeclarations, cCalls strings.Builder + for index := 0; index < count; index++ { + label := runtimeCAsmLabel(prefix + fmt.Sprintf("example.invalid/framefixture.Value%d", index)) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t) __asm__(%s);\n", index, label) + } + for _, input := range inputs { + for index := 0; index < count; index++ { + fmt.Fprintf(&goCalls, "fmt.Printf(\"%%016x\\n\",Value%d(uint64(0x%x)))\n", index, input) + fmt.Fprintf(&cCalls, "printf(\"%%016llx\\n\",(unsigned long long)value%d(UINT64_C(0x%x)));\n", index, input) + } + } + original := "package main\nimport \"fmt\"\n" + declarations + "func main() {\n" + goCalls.String() + "}\n" + want := strings.Split(strings.TrimSpace(string(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original))), "\n") + if len(want) != len(values) { + t.Fatal("original Go fixed-local output inventory changed", len(want), len(values)) + } + for index, value := range want { + if value != values[index] { + t.Fatalf("original Go kernel %d input %#x returned %s, want %s", index%count, inputs[index/count], value, values[index]) + } + } + driver := "#include \n#include \n" + runtimeCAsmLabelPreamble + + cDeclarations.String() + "int main(void) {\n" + cCalls.String() + "return 0;\n}\n" + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, declarations, runtime.GOOS+"/amd64", execution.triple) + check := func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + output := execution.compileAndRunLLVM(t, "fixed-local-results", input, driver, level) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatal("LLVM fixed-local output inventory changed", len(got), len(want)) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("fixed-local kernel %d input %#x: original %s, LLVM %s", index%count, inputs[index/count], want[index], got[index]) + } + } + }) + } + } + fixedLocalResultsRuntimeAPIs(t, pkg, request, file, options, assembly, check) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("original Go and LLVM amd64 execute through Rosetta, not native x86 hardware") + } +} + +func fixedLocalResultsRuntimeAPIs(t *testing.T, pkg GoPackage, request GoAMD64FrameCaptureRequest, + file *File, options Options, assembly string, check func(*testing.T, string, error)) { + t.Helper() + apis := []string{"Translate", "Module", "GoModule"} + if fixedLocalResultsRuntimeNeedsBinding(file, options) { + fixedLocalFreshAPIs(t, pkg, request, options, apis, check) + } else { + // Ordinary sources require no private-frame observation or permission. + // Reuse the independent original Go and C drivers with their own APIs. + for _, api := range apis { + t.Run(api, func(t *testing.T) { + fixedLocalAPI(t, pkg, file, options, api, check, assembly) + }) + } + } +} + +func fixedLocalResultsRuntimeNeedsBinding(file *File, options Options) bool { + if file == nil || file.Arch != ArchAMD64 || len(file.Funcs) == 0 || options.NumericalReads != nil { + return true + } + for _, fn := range file.Funcs { + if fn.FrameSize != 0 || fn.x86FixedLocal != nil || fn.x86SourceStack != nil { + return true + } + } + for _, sig := range options.Sigs { + if sig.x86FixedLocal != nil { + return true + } + } + return false +} diff --git a/amd64_fixed_local_data_go127_test.go b/amd64_fixed_local_data_go127_test.go new file mode 100644 index 000000000..8e0e53e58 --- /dev/null +++ b/amd64_fixed_local_data_go127_test.go @@ -0,0 +1,89 @@ +//go:build go1.27 + +package plan9asm + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const fixedLocalLiteralData = "DATA literal<>+0(SB)/8,$0x0102030405060708\n" + + "DATA literal<>+8(SB)/8,$0x1112131415161718\nGLOBL literal<>(SB),8,$32\n" + +func fixedLocalDataFixture(t *testing.T, operation, data string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options, string) { + t.Helper() + pkg, request, _, options := fixedLocalFrameFixture(t, operation) + name := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + assembly := data + string(original) + if err := os.WriteFile(name, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + return pkg, request, file, options, assembly +} + +func fixedLocalDataCLIOptions(options Options) Options { + resolve := options.ResolveSym + options.ResolveSym = func(symbol string) string { + name := resolve(symbol) + if strings.HasSuffix(symbol, "<>") { + return strings.TrimSuffix(name, "<>") + "$local" + } + return name + } + return options +} + +func TestGoAMD64FixedLocalReadonlyDataThreeTargetObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + for _, spelling := range []string{"source", "CLI"} { + t.Run(spelling, func(t *testing.T) { + pkg, request, _, options, assembly := fixedLocalDataFixture(t, + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", fixedLocalLiteralData) + request.Target, options.TargetTriple = target.name, target.triple + if spelling == "CLI" { + options = fixedLocalDataCLIOptions(options) + } + x86LEAActualTargetGoObject(t, "amd64", target.triple, assembly) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "internal constant [32 x i8]") || !strings.Contains(ir, "fixed_local_native") { + t.Fatal("missing joined private DATA/frame translation", err) + } + compileLLVMToObject(t, llc, target.triple, "data-frame.ll", "data-frame.o", ir) + compileLLVMToObject(t, llc, target.triple, "data-frame-opt.ll", "data-frame-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + }) + } +} + +func fixedLocalDataAPIs(t *testing.T, pkg GoPackage, file *File, assembly string, options Options, check func(*testing.T, string, error)) { + t.Helper() + // Negative/mutation callers deliberately keep their supplied binding. Never + // renew an expired or revoked observation while testing its rejection. + for _, api := range []string{"Translate", "Module", "GoModule"} { + t.Run(api, func(t *testing.T) { + fixedLocalAPI(t, pkg, file, options, api, check, assembly) + }) + } +} diff --git a/amd64_fixed_local_data_guards_go127_test.go b/amd64_fixed_local_data_guards_go127_test.go new file mode 100644 index 000000000..a201f737c --- /dev/null +++ b/amd64_fixed_local_data_guards_go127_test.go @@ -0,0 +1,168 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalReadonlyDataConsumerIdentity(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + pkg, request, file, options, assembly := fixedLocalDataFixture(t, + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", fixedLocalLiteralData) + fixedLocalDataAPIs(t, pkg, file, assembly, options, fixedLocalRequireContext) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + for _, mutation := range []string{"resolver", "JSON", "byte", "extent", "flags", "missing DATA", "new DATA"} { + t.Run(mutation, func(t *testing.T) { + changed, input := bound, assembly + switch mutation { + case "resolver": + changed.ResolveSym = func(symbol string) string { + if symbol == "·literal<>" { + return "foreign.literal<>" + } + return options.ResolveSym(symbol) + } + case "JSON": + changed.Sigs, err = cloneFixedLocalSigs(bound.Sigs) + if err != nil { + t.Fatal(err) + } + case "byte": + input = strings.Replace(input, "0x0102030405060708", "0x0102030405060709", 1) + case "extent": + input = strings.Replace(input, ",8,$32", ",8,$64", 1) + case "flags": + input = strings.Replace(input, ",8,$32", ",16,$32", 1) + case "missing DATA": + input = strings.TrimPrefix(input, fixedLocalLiteralData) + case "new DATA": + input = "GLOBL extra<>(SB),8,$8\n" + input + } + parsed, err := Parse(ArchAMD64, input) + if err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, parsed, input, changed, fixedLocalRequireContext) + }) + } + // A callback's later answer cannot replace the already verified symbol. + t.Run("stateful", func(t *testing.T) { + calls := 0 + changed := bound + changed.ResolveSym = func(symbol string) string { + if symbol == "·literal<>" { + calls++ + if calls > 1 { + return "foreign.literal<>" + } + } + return options.ResolveSym(symbol) + } + // Each API gets a fresh callback counter. + fixedLocalDataAPIs(t, pkg, file, assembly, changed, func(t *testing.T, ir string, err error) { + if err != nil || strings.Contains(ir, "foreign.literal") || calls != 1 { + t.Fatal("mutable resolver escaped sealed literal identity", calls, err) + } + calls = 0 + }) + }) + path := filepath.Join(request.Directory, request.AssemblyFile) + changed := strings.Replace(assembly, "0x0102030405060708", "0x0102030405060709", 1) + if err := os.WriteFile(path, []byte(changed), 0600); err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, file, assembly, bound, fixedLocalRequireContext) + if err := os.WriteFile(path, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, file, assembly, bound, fixedLocalRequireContext) +} + +func TestGoAMD64FixedLocalReadonlyDataRejectsCoalescedDefinitions(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + // Go's bare and qualified static names are distinct definitions. The + // ordinary LLVM resolver cannot fold them into one global in a binding. + data := fixedLocalLiteralData + "DATA ·literal<>+0(SB)/8,$99\nGLOBL ·literal<>(SB),8,$32\n" + pkg, request, file, options, assembly := fixedLocalDataFixture(t, + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", data) + x86LEAActualTargetGoObject(t, "amd64", options.TargetTriple, assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("distinct original globals acquired one LLVM identity", err) + } +} + +func TestGoAMD64FixedLocalReadonlyDataCLIEmissionIdentity(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + pkg, request, file, options, assembly := fixedLocalDataFixture(t, + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", fixedLocalLiteralData) + options = fixedLocalDataCLIOptions(options) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("original private DATA cannot retain the CLI's existing local spelling", err) + } + for _, direction := range []string{"function to DATA", "DATA to function"} { + t.Run(direction, func(t *testing.T) { + changed := options + changed.ResolveSym = func(symbol string) string { + if direction == "function to DATA" && symbol == file.Funcs[0].Sym { + return options.ResolveSym("·literal<>") + } + if direction == "DATA to function" && symbol == "·literal<>" { + return options.ResolveSym(file.Funcs[0].Sym) + } + return options.ResolveSym(symbol) + } + if _, err := source.BindFixedLocal(ctx, pkg, file, changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original TEXT and DATA acquired one emitted definition", err) + } + }) + } + for _, name := range []string{"", "foreign.literal$local", pkg.Path + ".literal", pkg.Path + ".literal<>", options.ResolveSym(file.Funcs[0].Sym)} { + t.Run("changed-emission="+name, func(t *testing.T) { + changed := bound + changed.ResolveSym = func(symbol string) string { + if symbol == "·literal<>" { + return name + } + return options.ResolveSym(symbol) + } + fixedLocalDataAPIs(t, pkg, file, assembly, changed, fixedLocalRequireContext) + }) + } + t.Run("stateful", func(t *testing.T) { + calls := 0 + changed := bound + changed.ResolveSym = func(symbol string) string { + if symbol == "·literal<>" { + calls++ + if calls > 1 { + return "foreign.literal$local" + } + } + return options.ResolveSym(symbol) + } + fixedLocalDataAPIs(t, pkg, file, assembly, changed, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "literal$local") || strings.Contains(ir, "foreign.literal") || calls != 1 { + t.Fatal("mutable resolver escaped the original DATA's sealed emission", calls, err) + } + calls = 0 + }) + }) +} diff --git a/amd64_fixed_local_data_matrix_go127_test.go b/amd64_fixed_local_data_matrix_go127_test.go new file mode 100644 index 000000000..f6abf9b78 --- /dev/null +++ b/amd64_fixed_local_data_matrix_go127_test.go @@ -0,0 +1,144 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalDataOperations() []string { + var operations []string + for _, name := range []string{"MOVB", "MOVW", "MOVL", "MOVQ"} { + operations = append(operations, name+" literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0") + } + for _, name := range []string{"MOVO", "MOVOA", "MOVOU", "MOVAPS", "MOVAPD", "MOVUPS", "MOVUPD"} { + operations = append(operations, name+" literal<>(SB),X0") + } + for _, row := range fixedLocalPackedMoveOracle { + for _, bank := range []string{"X", "Y", "Z"} { + aligned := row.name == "VMOVDQA" || row.name == "VMOVAPD" || row.name == "VMOVAPS" || + row.name == "VMOVDQA32" || row.name == "VMOVDQA64" + // Original Go's default assembly DATA alignment is at most 32. + // A 64-byte aligned memory instruction needs a separate witness. + if bank == "Z" && (!row.evex || aligned) { + continue + } + operations = append(operations, row.name+" literal<>(SB),"+bank+"0") + if row.evex { + operations = append(operations, + "KMOVQ AX,K1\n"+row.name+" literal<>(SB),K1,"+bank+"0", + "KMOVQ AX,K1\n"+row.name+".Z literal<>(SB),K1,"+bank+"0") + } + } + } + for _, name := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + operations = append(operations, name+" literal<>+1(SB),K1\nKMOVQ K1,DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0") + } + for _, operation := range []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", + } { + operations = append(operations, strings.Replace(operation, "16(SP)", "literal<>(SB)", 1)) + } + for _, name := range []string{"AESENC", "AESENCLAST", "AESDEC", "AESDECLAST", "AESIMC"} { + operations = append(operations, name+" literal<>(SB),X0") + } + operations = append(operations, "AESKEYGENASSIST $255,literal<>(SB),X0") + for _, name := range []string{"VAESENC", "VAESENCLAST", "VAESDEC", "VAESDECLAST"} { + operations = append(operations, name+" literal<>(SB),X0,X0") + } + operations = append(operations, "VAESIMC literal<>(SB),X0", "VAESKEYGENASSIST $255,literal<>(SB),X0") + for _, name := range []string{"CMOVWEQ", "CMOVLEQ", "CMOVQEQ"} { + operations = append(operations, "CMPQ AX,AX\n"+name+" literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0") + } + return operations +} + +func TestGoAMD64FixedLocalReadonlyDataLinuxRegisteredConsumers(t *testing.T) { + fixedLocalDataTargetConsumers(t, "linux/amd64", "x86_64-unknown-linux-gnu") +} + +func TestGoAMD64FixedLocalReadonlyDataDarwinRegisteredConsumers(t *testing.T) { + fixedLocalDataTargetConsumers(t, "darwin/amd64", "x86_64-apple-darwin") +} + +func TestGoAMD64FixedLocalReadonlyDataWindowsRegisteredConsumers(t *testing.T) { + fixedLocalDataTargetConsumers(t, "windows/amd64", "x86_64-pc-windows-msvc") +} + +func fixedLocalDataTargetConsumers(t *testing.T, target, triple string) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + operations := fixedLocalDataOperations() + if len(operations) != 119 { + t.Fatal("complete literal consumer denominator changed", len(operations)) + } + for start := 0; start < len(operations); start += 32 { + end := start + 32 + if end > len(operations) { + end = len(operations) + } + t.Run(fmt.Sprintf("%s/batch%d", target, start/32), func(t *testing.T) { + var assembly, declarations strings.Builder + assembly.WriteString(strings.Replace(fixedLocalLiteralData, "$32", "$64", 1)) + for index := start; index < end; index++ { + name := fmt.Sprintf("Value%d", index) + body, _, _ := x86PrivateAlignedFrameAuditFixture(t, operations[index]+"\nPADDQ 16(SP),X0") + assembly.WriteString(strings.Replace(body, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a,b,c,d uint64) uint64\n", name) + } + pkg, request, _, options := x86CMOVFixedLocalFixture(t, assembly.String(), declarations.String(), target, triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "literal-consumers.ll", "literal-consumers.o", ir) + compileLLVMToObject(t, llc, triple, "literal-consumers-opt.ll", "literal-consumers-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + t.Logf("independent literal consumer operations=%d; actual Go target=%s, three APIs, O0/O2; at most 32 functions/module", len(operations), target) +} + +func TestGoAMD64FixedLocalReadonlyDataRejectsUnownedAccesses(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, row := range []struct{ name, operation, data string }{ + {"past end", "MOVOU literal<>+24(SB),X0", fixedLocalLiteralData}, + {"negative", "MOVQ literal<>-1(SB),DX", fixedLocalLiteralData}, + {"unaligned", "MOVO literal<>+1(SB),X0", fixedLocalLiteralData}, + {"wide alignment", "VMOVDQA64 literal<>(SB),Z0", strings.Replace(fixedLocalLiteralData, "$32", "$64", 1)}, + {"zero mask cannot hide extent", "MOVQ $0,AX\nKMOVQ AX,K1\nVMOVDQU64.Z literal<>(SB),K1,Z0", fixedLocalLiteralData}, + {"write", "MOVOU X0,literal<>(SB)", fixedLocalLiteralData}, + {"address past end", "LEAQ literal<>(SB),DX\nADDQ $40,DX\nMOVQ (DX),DX", fixedLocalLiteralData}, + {"mutable", "MOVOU literal<>(SB),X0", strings.Replace(fixedLocalLiteralData, ",8,$32", ",16,$32", 1)}, + {"nonprivate", "MOVOU literal(SB),X0", strings.ReplaceAll(fixedLocalLiteralData, "literal<>", "literal")}, + {"address initializer", "MOVOU literal<>(SB),X0", fixedLocalLiteralData + "DATA pointer<>(SB)/8,$literal<>(SB)\nGLOBL pointer<>(SB),8,$8\n"}, + {"shortened extent", "MOVQ literal<>(SB),DX", strings.Replace(fixedLocalLiteralData, "$32", "$8", 1)}, + } { + t.Run(row.name, func(t *testing.T) { + pkg, request, file, options, assembly := fixedLocalDataFixture(t, row.operation+"\nPADDQ 16(SP),X0", row.data) + x86LEAActualTargetGoObject(t, "amd64", options.TargetTriple, assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + expanded, err := source.preprocess(request.AssemblyFile, []byte(assembly)) + if err != nil { + t.Fatal(err) + } + file, err = Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unowned literal operation obtained fixed-local permission", err) + } + }) + } +} diff --git a/amd64_fixed_local_data_runtime_go127_test.go b/amd64_fixed_local_data_runtime_go127_test.go new file mode 100644 index 000000000..990f8ba2c --- /dev/null +++ b/amd64_fixed_local_data_runtime_go127_test.go @@ -0,0 +1,135 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalReadonlyData(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + operations := []string{ + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", + "PADDQ literal<>(SB),X0", + "MOVQ literal<>+8(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + "MOVOU literal<>+16(SB),X0\nPADDQ 16(SP),X0", + "MOVQ literal<>+3(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + "MOVB literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + "MOVW literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + "MOVL literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + } + var assembly, declarations, calls, cDeclarations, cCalls strings.Builder + assembly.WriteString(fixedLocalLiteralData) + prefix := "" + if runtime.GOOS == "darwin" { + prefix = "_" + } + for index, operation := range operations { + name := fmt.Sprintf("Value%d", index) + body, _, _ := x86PrivateAlignedFrameAuditFixture(t, operation) + assembly.WriteString(strings.Replace(body, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a,b,c,d uint64) uint64\n", name) + fmt.Fprintf(&calls, ` + if got := %s(a, b, c, d); got != expected(%d, a, c) { + t.Fatalf("case %d seed=%%d got=%%x", seed, got) + } +`, name, index, index) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t,uint64_t,uint64_t,uint64_t) __asm__(\"%sexample.invalid/framefixture.%s\");\n", index, prefix, name) + fmt.Fprintf(&cCalls, ` + if (value%d(a, b, c, d) != expected(%d, a, c)) { + fprintf(stderr, "case %d seed=%%u\n", seed); + return 1; + } +`, index, index, index) + } + goOracle := `package framefixture +import "testing" +func expected(op int, a, c uint64) uint64 { + const lo = uint64(0x0102030405060708) + const hi = uint64(0x1112131415161718) + switch op { + case 0: + return lo+a + case 1: + return lo+c + case 2: + return hi + case 3: + return a + case 4: + return lo>>24 | (hi&0xffffff)<<40 + case 5: + return c&^255 | 7 + case 6: + return c&^65535 | 0x0607 + case 7: + return 0x04050607 + } + panic("unknown case") +} +func TestOriginal(t *testing.T) { + for seed := uint64(0); seed < 256; seed++ { + a := seed*0x102030405060708 + b, c, d := ^a, a^0xfedcba9876543210, a+0x1234 +` + calls.String() + ` + } +} +` + cOracle := `#include +#include +` + cDeclarations.String() + ` +static uint64_t expected(unsigned op, uint64_t a, uint64_t c) { + const uint64_t lo = UINT64_C(0x0102030405060708); + const uint64_t hi = UINT64_C(0x1112131415161718); + switch (op) { + case 0: + return lo+a; + case 1: + return lo+c; + case 2: + return hi; + case 3: + return a; + case 4: + return lo>>24 | (hi&0xffffff)<<40; + case 5: + return (c&~UINT64_C(255)) | 7; + case 6: + return (c&~UINT64_C(65535)) | 0x0607; + case 7: + return 0x04050607; + } + return 0; +} +int main(void) { + for (unsigned seed=0; seed<256; seed++) { + uint64_t a = seed*UINT64_C(0x102030405060708); + uint64_t b = ~a, c = a^UINT64_C(0xfedcba9876543210), d = a+0x1234; +` + cCalls.String() + ` + } + return 0; +} +` + pkg, request, _, options := x86CMOVFixedLocalFixture(t, assembly.String(), declarations.String(), runtime.GOOS+"/amd64", execution.triple) + fixedLocalPackedRunOriginal(t, execution, assembly.String(), declarations.String(), goOracle, true) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + execution.compileAndRunLLVM(t, "readonly-frame", input, cOracle, level) + } + }) + t.Log("2048 independent numeric outcomes: original Go and three LLVM APIs at O0/O2; includes sparse zero tails and unaligned/partial scalar reads") + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("Rosetta legacy SSE diagnostic, not native Linux or AVX/AVX512 execution") + } +} diff --git a/amd64_fixed_local_dead_blocks_test.go b/amd64_fixed_local_dead_blocks_test.go new file mode 100644 index 000000000..7b40f5920 --- /dev/null +++ b/amd64_fixed_local_dead_blocks_test.go @@ -0,0 +1,238 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// These are source/CFG regressions, not execution witnesses for the dead MOVs. +func fixedLocalDeadBlockSource(operation, prefix, branch string) string { + return "TEXT ·Value(SB),4,$24-16\nMOVQ a+0(FP),BX\n" + + "MOVQ $7,AX\nMOVQ AX,0(SP)\n" + prefix + branch + + "gap:\n" + operation + "\ndone:\n" + + "MOVQ 0(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n" +} + +func TestGoAMD64FixedLocalDeadMemoryBlocks(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, width := range []string{"B", "W", "L", "Q"} { + for _, direction := range []string{"read", "write"} { + t.Run(width+"/"+direction, func(t *testing.T) { + operation := "MOV" + width + " (CX),AX" + if direction == "write" { + operation = "MOV" + width + " AX,(CX)" + } + assembly := fixedLocalDeadBlockSource(operation, "", "JMP done\n") + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64", "linux/amd64", "x86_64-unknown-linux-gnu") + // Preserve the complete original instruction, including its + // uninitialized address. Assembly acceptance grants no access. + frameRelocationOriginal(t, pkg, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("unconditional source jump did not exclude its dead memory block", err) + } + compileLLVMToObject(t, llc, options.TargetTriple, "dead.ll", "dead.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "dead-o2.ll", "dead-o2.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + } +} + +func TestGoAMD64FixedLocalLiveMemoryStillRequiresProof(t *testing.T) { + for _, width := range []string{"B", "W", "L", "Q"} { + for _, direction := range []string{"read", "write"} { + for _, address := range []string{"unknown", "partial"} { + t.Run(width+"/"+direction+"/"+address, func(t *testing.T) { + operation := "MOV" + width + " (CX),AX" + if direction == "write" { + operation = "MOV" + width + " AX,(CX)" + } + prefix := "" + if address == "partial" { + prefix = "MOVB $1,CX\n" + } + // Both successors are possible for the declared input; + // AX is known even in the write cases, isolating address duty. + assembly := fixedLocalDeadBlockSource(operation, prefix, "CMPQ BX,$0\nJEQ gap\nJMP done\n") + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64", "linux/amd64", "x86_64-unknown-linux-gnu") + frameRelocationOriginal(t, pkg, request) + for _, api := range []string{"Translate", "Module", "GoModule"} { + t.Run(api, func(t *testing.T) { + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source := requireGoAMD64FrameCapture(t, pkg, request) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "unknown/escaping address provenance") { + t.Fatal("live address failed outside its expected binding obligation", err) + } + t.Log("live address rejected at Bind before codegen", err) + return + } + fixedLocalAPI(t, pkg, file, bound, api, fixedLocalRequireContext, assembly) + }) + } + }) + } + } + } +} + +func TestGoAMD64FixedLocalDeadBlocksKeepCompleteGrammar(t *testing.T) { + const operation = "MOVQ (CX),AX" + fn, sig := fixedLocalReplayFixture(t, "JMP done\n"+operation+"\ndone:") + proven, err := proveX86FixedLocal(fn, sig) + if err != nil { + t.Fatal("legal complete dead source failed the independent proof baseline", err) + } + matched := 0 + for _, ins := range proven.Instrs { + if len(ins.Args) == 2 && ins.Args[0].Kind == OpMem && ins.Args[0].Mem.Base == CX { + matched++ + if len(ins.x86FixedLocalMemory) != 0 { + t.Fatal("an unreachable unknown address acquired a memory certificate") + } + } + } + if matched != 1 { + t.Fatal("independent dead-address control is missing or duplicated", matched) + } + for _, invalid := range []string{"MOVQ AX", "VADDPS X0,X1,AX", "MOVQ AX,(CX)\nSYSCALL"} { + t.Run(fmt.Sprint(invalid), func(t *testing.T) { + assembly := strings.Replace(fixedLocalReplaySource("JMP done\n"+operation+"\ndone:"), operation, invalid, 1) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); err == nil { + t.Fatal("dead source bypassed complete typed-format validation", err) + } + }) + } +} + +func TestGoAMD64FixedLocalDeadBlockLabelAliases(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, label := range []string{"gap", "done"} { + t.Run(label, func(t *testing.T) { + assembly := fixedLocalDeadBlockSource("MOVQ (CX),AX", "", "JMP done\n") + assembly = strings.Replace(assembly, label+":\n", label+":\n"+label+"_alias:\n", 1) + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64", "linux/amd64", "x86_64-unknown-linux-gnu") + frameRelocationOriginal(t, pkg, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("adjacent labels lost their common successor", err) + } + if label == "done" { + found := false + for _, line := range strings.Split(ir, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), amd64LLVMBlockName("done")+":") { + found = true + continue + } + if found && strings.TrimSpace(line) != "" && !strings.HasPrefix(strings.TrimSpace(line), ";") { + if !strings.HasPrefix(strings.TrimSpace(line), "br label %"+amd64LLVMBlockName("done_alias")) { + t.Fatal("reachable empty label did not continue to its live successor", line) + } + break + } + } + if !found { + t.Fatal("reachable source label disappeared") + } + } + compileLLVMToObject(t, llc, options.TargetTriple, "aliases.ll", "aliases.o", ir) + }) + }) + } +} + +func TestGoAMD64FixedLocalDeadBlockProjectionGuards(t *testing.T) { + assembly := fixedLocalDeadBlockSource("MOVQ (CX),AX", "", "JMP done\n") + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64", "linux/amd64", "x86_64-unknown-linux-gnu") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", bound) + if err != nil { + t.Fatal(err) + } + fn := normalized.Funcs[0] + makeConsumer := func() *amd64Ctx { + return newX86Ctx(new(strings.Builder), fn, bound.Sigs[pkg.Path+".Value"], bound.ResolveSym, + bound.Sigs, "amd64", options.TargetTriple, false) + } + baseline, err := makeConsumer().fixedLocalBlockReachability() + if err != nil || len(baseline) == 0 { + t.Fatal("actual source did not establish a valid emission projection", err) + } + for _, name := range []string{"owner", "source", "emitted instruction", "block mapping"} { + t.Run(name, func(t *testing.T) { + consumer := makeConsumer() + switch name { + case "owner": + consumer.fixedLocal = &x86FixedLocalProof{} + case "source": + consumer.fpScalarSource.Instrs = append([]Instr(nil), fn.Instrs...) + consumer.fpScalarSource.Instrs[0].Raw += " // changed source" + case "emitted instruction": + for index, live := range baseline { + if !live && len(consumer.blocks[index].instrs) != 0 { + consumer.blocks[index].instrs[0].Op = "SYSCALL" + break + } + } + case "block mapping": + consumer.blockBase[1]++ + } + if projection, err := consumer.fixedLocalBlockReachability(); !errors.Is(err, ErrProbeNeedsContext) || projection != nil { + t.Fatal("changed emission acquired an unreachable-source exception", err) + } + }) + } + for _, name := range []string{"private graph", "indirect", "address-taken"} { + t.Run(name, func(t *testing.T) { + consumer := makeConsumer() + switch name { + case "private graph": + consumer.directTail = &x86DirectTailRoot{} + case "indirect": + consumer.indirectABI0 = &FuncSig{} + case "address-taken": + consumer.indirectLabels = []string{"gap"} + } + if projection, err := consumer.fixedLocalBlockReachability(); err != nil || projection != nil { + t.Fatal("a separate entry graph was pruned", err) + } + }) + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + if projection, err := makeConsumer().fixedLocalBlockReachability(); !errors.Is(err, ErrProbeNeedsContext) || projection != nil { + t.Fatal("closed actual source retained an emission exception", err) + } +} diff --git a/amd64_fixed_local_exchange.go b/amd64_fixed_local_exchange.go new file mode 100644 index 000000000..b7ed96b13 --- /dev/null +++ b/amd64_fixed_local_exchange.go @@ -0,0 +1,52 @@ +package plan9asm + +func fixedLocalExchangeStep(ins *Instr, state *x86FixedLocalState, sig FuncSig, + proof *x86FixedLocalProof, residue int) (bool, error) { + form, width, known, err := x86SwapProperties(*ins, "amd64") + if !known { + return false, nil + } + if err != nil { + return true, err + } + if form.nop { + return true, nil // Go's accumulator 90 encoding preserves even unknown bytes. + } + for _, operand := range ins.Args { + root, _ := x86CallerGP(operand.Reg) + if operand.Kind != OpReg && operand.Kind != OpMem || operand.Kind == OpReg && root == SP { + return true, goAMD64FrameContext("fixed local exchange requires registers or independently owned memory") + } + } + first, err := fixedLocalReadValue(ins, 0, state, sig, proof, residue, width, true) + if err != nil { + return true, err + } + last, err := fixedLocalReadValue(ins, 1, state, sig, proof, residue, width, true) + if err != nil { + return true, err + } + comparison := state.comparison + order := [2]int{0, 1} + if ins.Args[1].Kind == OpMem { + order = [2]int{1, 0} + } + // Snapshot reads precede both writes. Write memory first so a base/index + // that is also exchanged still addresses its original location. + values := [2]fixedLocalScalarValue{last, first} + for _, index := range order { + if err := fixedLocalWriteValue(ins, index, 1-index, values[index], state, sig, proof, residue, width); err != nil { + return true, err + } + } + if width == 8 && ins.Args[0].Kind == OpReg && ins.Args[1].Kind == OpReg { + for index, operand := range ins.Args { + if comparison.register == operand.Reg { + comparison.register = ins.Args[1-index].Reg + break + } + } + state.comparison = comparison + } + return true, nil // XCHG preserves FLAGS; only their value correlation moves. +} diff --git a/amd64_fixed_local_exchange_go127_test.go b/amd64_fixed_local_exchange_go127_test.go new file mode 100644 index 000000000..1fae3f60b --- /dev/null +++ b/amd64_fixed_local_exchange_go127_test.go @@ -0,0 +1,105 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "math/bits" + "strings" + "testing" +) + +type fixedLocalExchangeCase struct { + body string + want func(uint64) uint64 +} + +func fixedLocalExchangeCases() []fixedLocalExchangeCase { + const other = uint64(0x1122334455667788) + var rows []fixedLocalExchangeCase + for _, width := range []struct { + suffix string + bits uint + }{{"B", 8}, {"W", 16}, {"L", 32}, {"Q", 64}} { + for _, location := range []string{"register", "memory first", "memory last"} { + width, location := width, location + memory := "8(SP)" + if location == "memory first" && width.bits < 64 { + // Exercise unaligned native exchange without crossing a cacheline + // on machines that trap split locks. Keep qwords word-aligned. + memory = "3(SP)" + } + body := "MOVQ $0x1122334455667788,CX\nMOVQ CX," + memory + "\n" + switch location { + case "register": + body += "XCHG" + width.suffix + " AX,CX\n" + case "memory first": + body += "XCHG" + width.suffix + " " + memory + ",AX\nMOVQ " + memory + ",CX\n" + case "memory last": + body += "XCHG" + width.suffix + " AX,8(SP)\nMOVQ 8(SP),CX\n" + } + // Observe both outputs asymmetrically: XOR or addition alone would + // hide an incorrect implementation that did not exchange anything. + body += "RORQ $13,CX\nXORQ CX,AX" + rows = append(rows, fixedLocalExchangeCase{body, func(input uint64) uint64 { + mask := uint64(1)<>8&255 + }}) + } + rows = append(rows, + fixedLocalExchangeCase{ + "MOVQ AX,8(SP)\nMOVQ $0x1122334455667788,CX\nMOVQ CX,24(SP)\nLEAQ 8(SP),CX\nLEAQ 24(SP),DX\nXCHGQ CX,DX\nMOVQ (CX),AX", + func(uint64) uint64 { return other }, + }, + fixedLocalExchangeCase{ + "LEAQ 8(SP),CX\nMOVQ AX,DX\nXCHGQ CX,DX\nMOVQ CX,(DX)\nMOVQ 8(SP),AX", + func(input uint64) uint64 { return input }, + }, + fixedLocalExchangeCase{ + "MOVQ AX,8(SP)\nLEAQ 8(SP),CX\nXCHGQ CX,(CX)\nMOVQ 8(SP),DX\nMOVQ CX,(DX)\nMOVQ 8(SP),AX", + func(input uint64) uint64 { return input }, + }, + fixedLocalExchangeCase{ + "MOVQ AX,8(SP)\nMOVQ $0x1122334455667788,CX\nMOVQ CX,24(SP)\nMOVQ $8,CX\nMOVQ $24,DX\nXCHGQ CX,DX\nMOVQ (SP)(CX*1),AX", + func(uint64) uint64 { return other }, + }, + fixedLocalExchangeCase{"XCHGL AX,AX", func(input uint64) uint64 { return input }}, + fixedLocalExchangeCase{"MOVQ AX,BX\nXCHGL BX,BX\nMOVQ BX,AX", func(input uint64) uint64 { return uint64(uint32(input)) }}, + fixedLocalExchangeCase{"MOVQ $0x1122334455667788,CX\nSTC\nXCHGQ AX,CX\nADCQ $0,AX", func(uint64) uint64 { return other + 1 }}, + ) + return rows +} + +func fixedLocalExchangeProgram(t *testing.T) (string, string, []fixedLocalExchangeCase) { + t.Helper() + rows := fixedLocalExchangeCases() + if len(rows) != 21 { + t.Fatal("exchange width, memory, alias and typed-value inventory changed", len(rows)) + } + var assembly, declarations strings.Builder + for index, row := range rows { + _, _, source := fixedLocalGPFixture(t, row.body) + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(source, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + } + return assembly.String(), declarations.String(), rows +} + +func TestGoAMD64FixedLocalExchangeObjects(t *testing.T) { + assembly, declarations, rows := fixedLocalExchangeProgram(t) + fixedLocalCompleteVectorObjects(t, assembly, declarations, len(rows)) +} diff --git a/amd64_fixed_local_exchange_runtime_go127_test.go b/amd64_fixed_local_exchange_runtime_go127_test.go new file mode 100644 index 000000000..720034159 --- /dev/null +++ b/amd64_fixed_local_exchange_runtime_go127_test.go @@ -0,0 +1,22 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalExchange(t *testing.T) { + assembly, declarations, rows := fixedLocalExchangeProgram(t) + inputs := []uint64{0, 1, ^uint64(0), 0x0123456789abcdef} + var expected [][]string + for _, input := range inputs { + var values []string + for _, row := range rows { + values = append(values, fmt.Sprintf("%016x", row.want(input))) + } + expected = append(expected, values) + } + fixedLocalResultsRuntime(t, assembly, declarations, inputs, expected) +} diff --git a/amd64_fixed_local_exchange_test.go b/amd64_fixed_local_exchange_test.go new file mode 100644 index 000000000..0f8e7bf63 --- /dev/null +++ b/amd64_fixed_local_exchange_test.go @@ -0,0 +1,360 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func fixedLocalExchangeState() x86FixedLocalState { + return x86FixedLocalState{ + pointers: map[Reg]int64{SP: 0}, opaque: make(map[Reg]x86FixedLocalOrigins), + opaqueCells: make(map[int64]x86FixedLocalOrigins), addresses: make(map[int64]int64), + known: make(map[Reg]uint8), constants: make(map[Reg]int64), + booleans: make(map[Reg]uint8), vectors: make(map[Reg]uint64), + masks: make(map[Reg]bool), results: make(map[int]bool), + bytes: newFixedLocalByteSet(64), booleanBytes: newFixedLocalByteSet(64), + flags: x86ByteFlagsAll, + } +} + +func TestGoAMD64FixedLocalExchangeRegisters(t *testing.T) { + registers := append(x86CommutedPointerAddRegisters("amd64"), BP) + var source strings.Builder + source.WriteString("TEXT ·Value(SB),4,$0-0\n") + for _, width := range []struct { + suffix string + bytes int64 + }{{"B", 1}, {"W", 2}, {"L", 4}, {"Q", 8}} { + for _, first := range registers { + for _, last := range registers { + body := fmt.Sprintf("XCHG%s %s,%s", width.suffix, first, last) + fmt.Fprintln(&source, body) + t.Run(body, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, body) + if !fixedLocalInstructionShape(ins) { + t.Fatal("existing typed exchange family is absent from fixed-frame shapes") + } + state := fixedLocalExchangeState() + state.known[first], state.known[last] = 255, 255 + state.booleans[first], state.booleans[last] = 0x55, 0xaa + oldFirst, oldLast := state.booleans[first], state.booleans[last] + if err := fixedLocalStep(&ins, &state, FuncSig{}, &x86FixedLocalProof{frame: 64}, 0); err != nil { + t.Fatal("initialized exchange rejected", err) + } + mask := uint8((1 << uint(width.bytes)) - 1) + wantFirst, wantLast := oldFirst&^mask|oldLast&mask, oldLast&^mask|oldFirst&mask + if width.bytes == 4 && !(first == AX && last == AX) { + wantFirst, wantLast = wantFirst|240, wantLast|240 + } + if state.booleans[first] != wantFirst || state.booleans[last] != wantLast || + state.known[first] != 255 || state.known[last] != 255 || state.flags != x86ByteFlagsAll { + t.Fatal("swap lost byte identity, dword zero extension or FLAGS", state) + } + }) + } + } + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) +} + +func TestGoAMD64FixedLocalExchangeByteAliasesAndValues(t *testing.T) { + for _, width := range []struct { + suffix string + bytes int64 + }{{"B", 1}, {"W", 2}, {"L", 4}, {"Q", 8}} { + for _, reg := range []Reg{AX, BX} { + for missing := int64(0); missing < width.bytes; missing++ { + t.Run(fmt.Sprintf("missing/%s/%s/%d", width.suffix, reg, missing), func(t *testing.T) { + state := fixedLocalExchangeState() + state.known[AX], state.known[BX] = 255, 255 + state.known[reg] &^= 1 << uint(missing) + before := cloneFixedLocalState(state) + ins := x86ScalarLaneInstruction(t, "XCHG"+width.suffix+" AX,BX") + if err := fixedLocalStep(&ins, &state, FuncSig{}, &x86FixedLocalProof{frame: 64}, 0); !errors.Is(err, ErrProbeNeedsContext) || !reflect.DeepEqual(state, before) { + t.Fatal("incomplete input mutated state or acquired initialized bytes", err) + } + }) + } + } + } + for _, operands := range []string{"AH,AL", "AL,AH", "CH,CL", "CL,CH", "AH,BH", "AL,BL", "R11,R12", "SI,DI"} { + t.Run(operands, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, "XCHGB "+operands) + state := fixedLocalExchangeState() + for _, reg := range []Reg{AX, BX, CX, SI, DI, "R11", "R12"} { + state.known[reg], state.booleans[reg] = 255, 0x55 + state.constants[reg] = 0x1234 + state.setRange(reg, x86UnsignedRange{1, 2}) + } + before := cloneFixedLocalState(state) + first, last := ins.Args[0], ins.Args[1] + firstRoot, _ := x86CallerGP(first.Reg) + lastRoot, _ := x86CallerGP(last.Reg) + firstBits := fixedLocalBooleanRead(first, 1, &before) + lastBits := fixedLocalBooleanRead(last, 1, &before) + if err := fixedLocalStep(&ins, &state, FuncSig{}, &x86FixedLocalProof{frame: 64}, 0); err != nil { + t.Fatal(err) + } + fixedLocalWriteBooleans(&before, first.Reg, 1, lastBits) + fixedLocalWriteBooleans(&before, last.Reg, 1, firstBits) + delete(before.constants, firstRoot) + delete(before.constants, lastRoot) + if !reflect.DeepEqual(state, before) { + t.Fatal("overlapping aliases did not snapshot both inputs or revoke fullword facts", state, before) + } + }) + } + for _, width := range []string{"W", "L", "Q"} { + t.Run("accumulator NOP/"+width, func(t *testing.T) { + state := fixedLocalExchangeState() + state.opaque[AX] = fixedLocalOrigin(x86FixedLocalOrigin{parameter: 8}) + state.comparison = x86FixedLocalComparison{register: AX, constant: 7} + before := cloneFixedLocalState(state) + ins := x86ScalarLaneInstruction(t, "XCHG"+width+" AX,AX") + if err := fixedLocalStep(&ins, &state, FuncSig{}, &x86FixedLocalProof{frame: 64}, 0); err != nil || !reflect.DeepEqual(state, before) { + t.Fatal("Go opcode 90 is a NOP even for opaque or undefined bytes", err) + } + }) + } +} + +func TestGoAMD64FixedLocalExchangeValueDomains(t *testing.T) { + roles := []string{"number", "private", "incoming", "readonly"} + for _, first := range roles { + for _, last := range roles { + t.Run(first+"/"+last, func(t *testing.T) { + state := fixedLocalExchangeState() + seed := func(reg Reg, role string, value int64) { + switch role { + case "number": + state.known[reg], state.constants[reg], state.booleans[reg] = 255, value, 0xfe + state.setRange(reg, x86UnsignedRange{uint64(value), uint64(value)}) + case "private": + state.pointers[reg] = value + case "incoming": + state.opaque[reg] = fixedLocalOrigin(x86FixedLocalOrigin{parameter: value}) + case "readonly": + state.opaque[reg] = fixedLocalOrigin(x86FixedLocalOrigin{literal: x86StaticSBAddress{symbol: "table<>(SB)", offset: value}}) + } + } + seed(AX, first, 8) + seed(BX, last, 24) + state.comparison = x86FixedLocalComparison{register: AX, constant: 31} + before := cloneFixedLocalState(state) + ins := x86ScalarLaneInstruction(t, "XCHGQ AX,BX") + if err := fixedLocalStep(&ins, &state, FuncSig{}, &x86FixedLocalProof{frame: 64}, 0); err != nil { + t.Fatal(err) + } + for _, pair := range [][2]Reg{{AX, BX}, {BX, AX}} { + to, from := pair[0], pair[1] + if state.known[to] != before.known[from] || state.booleans[to] != before.booleans[from] || + state.pointers[to] != before.pointers[from] || state.opaque[to] != before.opaque[from] || + state.constants[to] != before.constants[from] || state.ranges[to] != before.ranges[from] { + t.Fatal("exchange lost a complete typed value or retained an old value", to, state, before) + } + } + if state.comparison != (x86FixedLocalComparison{register: BX, constant: 31}) || state.flags != before.flags { + t.Fatal("FLAGS must follow the same compared value after a full-word exchange", state.comparison) + } + }) + } + } +} + +func TestGoAMD64FixedLocalExchangeCompleteRoles(t *testing.T) { + rows := []struct { + name, body string + input, ok bool + }{ + {"private pointers", "LEAQ 8(SP),R14\nLEAQ 24(SP),R15\nXCHGQ R15,R14\nMOVQ AX,(R14)\nMOVQ 24(SP),AX", false, true}, + {"private and NUM", "LEAQ 8(SP),CX\nMOVQ AX,DX\nXCHGQ CX,DX\nMOVQ CX,(DX)\nMOVQ 8(SP),AX", false, true}, + {"NUM and private", "LEAQ 8(SP),CX\nMOVQ AX,DX\nXCHGQ DX,CX\nMOVQ CX,(DX)\nMOVQ 8(SP),AX", false, true}, + {"old roles cannot survive swap", "LEAQ 8(SP),CX\nMOVQ AX,DX\nXCHGQ DX,CX\nMOVQ DX,(CX)\nMOVQ 8(SP),AX", false, false}, + {"private spill and NUM", "LEAQ 24(SP),CX\nMOVQ CX,8(SP)\nXCHGQ AX,8(SP)\nMOVQ 8(SP),DX\nMOVQ DX,(AX)\nMOVQ 24(SP),AX", false, true}, + {"NUM spill and private", "LEAQ 24(SP),CX\nXCHGQ 0(SP),CX\nMOVQ 0(SP),DX\nMOVQ CX,(DX)\nMOVQ 24(SP),AX", false, true}, + {"memory address uses old register", "MOVQ AX,8(SP)\nLEAQ 8(SP),CX\nXCHGQ CX,(CX)\nMOVQ 8(SP),DX\nMOVQ CX,(DX)\nMOVQ 8(SP),AX", false, true}, + {"native pointers", "XCHGQ SI,DI\nMOVQ (DI),AX", true, true}, + {"native and NUM", "XCHGQ SI,AX\nMOVQ (AX),AX", true, true}, + {"native spill", "MOVQ SI,8(SP)\nXCHGQ 8(SP),AX\nMOVQ (AX),AX", true, true}, + {"native private swap", "LEAQ 16(SP),CX\nXCHGQ SI,CX\nMOVQ (CX),AX\nMOVQ AX,(SI)\nMOVQ 16(SP),AX", true, true}, + {"constant index", "MOVQ $8,CX\nMOVQ $16,DX\nXCHGQ CX,DX\nMOVQ AX,(SP)(CX*1)\nMOVQ 16(SP),AX", false, true}, + {"flags preserved", "MOVQ AX,CX\nSTC\nXCHGQ AX,CX\nADCQ $0,AX", false, true}, + {"unknown register", "XCHGQ AX,CX", false, false}, + {"unknown memory", "XCHGQ AX,8(SP)", false, false}, + {"crossing frame", "XCHGQ AX,60(SP)", false, false}, + {"partial pointer", "MOVQ SP,CX\nXCHGW AX,CX", false, false}, + {"partial opaque", "XCHGL SI,AX", true, false}, + {"partial spill", "MOVQ SI,8(SP)\nXCHGL AX,8(SP)", true, false}, + {"pointer cannot escape", "LEAQ 8(SP),CX\nXCHGQ CX,(SI)", true, false}, + {"NUM not address", "XCHGQ AX,0(SP)\nMOVQ (AX),AX", false, false}, + {"pointer not NUM", "LEAQ 8(SP),CX\nXCHGQ AX,CX", false, false}, + {"stack register", "XCHGQ AX,SP", false, false}, + {"flags not invented", "MOVQ AX,CX\nXCHGQ AX,CX\nADCQ $0,AX", false, false}, + } + for _, width := range []string{"B", "W", "L", "Q"} { + for _, memory := range []string{"0(SP)", "3(SI)"} { + for _, reverse := range []bool{false, true} { + first, last := "AX", memory + if reverse { + first, last = last, first + } + rows = append(rows, struct { + name, body string + input, ok bool + }{"XCHG" + width + " " + first + "," + last, "XCHG" + width + " " + first + "," + last, memory == "3(SI)", true}) + } + } + } + var source strings.Builder + for index, row := range rows { + fixture := fixedLocalGPFixture + if row.input { + fixture = fixedLocalInputFixture + } + fn, sig, original := fixture(t, row.body) + source.WriteString(strings.Replace(original, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + if row.name == "NUM not address" { + fixedLocalRequireNumericControl(t, fn, sig, original, "func Value(a uint64) uint64", + "MOVQ (AX),AX", 0, 8, false, "load volatile i64") + return + } + _, err := proveX86FixedLocal(fn, sig) + if row.ok && err != nil || !row.ok && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("complete typed exchange valid=%t: %v", row.ok, err) + } + }) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) +} + +func TestGoAMD64FixedLocalExchangeCandidates(t *testing.T) { + for _, row := range []struct { + body string + want bool + }{ + {"MOVQ SP,CX\nXCHGQ DX,CX\nMOVQ (DX),AX", true}, + {"MOVQ SP,CX\nXCHGQ CX,DX\nMOVQ (DX),AX", true}, + {"MOVQ SP,CX\nXCHGQ CX,DX\nXCHGQ DX,R15\nMOVQ (R15),AX", true}, + {"XCHGQ CX,0(SP)", true}, + {"XCHGQ 0(SP),CX", true}, + {"XCHGB CX,0(SP)", true}, + {"XCHGW 0(SP),CX", true}, + {"XCHGL CX,0(SP)", true}, + {"MOVQ SP,CX\nXCHGL CX,DX\nMOVQ (DX),AX", false}, + {"MOVQ SI,CX\nXCHGQ CX,DX\nMOVQ (DX),AX", false}, + {"MOVQ SP,CX\nXCHGQ CX,DX", false}, + {"XCHGQ CX,scratch-8(SP)", false}, + } { + t.Run(row.body, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$64-0\n"+row.body+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if got := GoAMD64FixedLocalFrameBindingNeeded(file, "amd64"); got != row.want { + t.Fatalf("candidate=%t, want %t", got, row.want) + } + }) + } +} + +func TestX86ExchangeCallerOrigins(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word, suffix := x86ResultPointerWord(arch), "Q" + if arch == "386" { + suffix = "L" + } + t.Run(arch+"/complete same-origin difference", func(t *testing.T) { + body := fmt.Sprintf("MOV%s p+0(FP),AX\nMOV%s n+%d(FP),BX\nXCHG%s AX,BX\nADD%s AX,BX\nSUB%s p+0(FP),BX\nMOV%s BX,ret+%d(FP)\nRET", suffix, suffix, word, suffix, suffix, suffix, suffix, 2*word) + _, file, sig, source := x86ResultInitializationFixture(t, arch, "func Value(p *byte,n uintptr) uintptr", body) + requireX86GoAssemblerResult(t, arch, source, true) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, arch); err != nil { + t.Fatal("swapped incoming pointer lost its complete original-source origin", err) + } + } + }) + for _, operands := range []string{"AX,BX", "BX,AX", "AX,AX"} { + t.Run(arch+"/"+operands, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, "XCHG"+suffix+" "+operands) + state := x86ResultState{callerAddressFP: map[int64]bool{0: true, word: true}} + first, _, _ := x86ResultGP(AX, word, arch) + last, _, _ := x86ResultGP(BX, word, arch) + state.pointerGP[first], state.pointerGP[last] = true, true + state.callerAddressGP[first], state.callerAddressGP[last] = true, true + state.callerAddressOrigin[first], state.callerAddressOrigin[last] = 1, uint16(word+1) + before := state.clone() + if !x86ScalarFPResultTransfer(ins, &state, nil, 0, arch) { + t.Fatal("existing typed whole-word swap rejected") + } + state.callerAddressGP = x86ResultCallerAddressesAfterTransfer(ins, before.callerAddressGP, state.callerAddressGP, state.callerAddressFP, arch) + state.callerAddressOrigin = x86ResultCallerOriginsAfter(ins, before.callerAddressOrigin, state.callerAddressGP, state.callerAddressFP, arch) + if operands != "AX,AX" { + before.callerAddressOrigin[first], before.callerAddressOrigin[last] = before.callerAddressOrigin[last], before.callerAddressOrigin[first] + } + if state.callerAddressGP != before.callerAddressGP || state.callerAddressOrigin != before.callerAddressOrigin { + t.Fatal("whole-word swap lost or invented incoming-pointer identity", state, before) + } + }) + } + } +} + +func TestX86ExchangeCanonicalBytes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + body string + ax, bx uint8 + }{ + {"XCHGB AH,AL", 0x56, 0xaa}, + {"XCHGB AL,AH", 0x56, 0xaa}, + {"XCHGB AX,BX", 0x54, 0xab}, + {"XCHGW AX,BX", 0x56, 0xa9}, + {"XCHGL AX,BX", 0x5a, 0xa5}, + {"XCHGL AX,AX", 0x55, 0xaa}, + {"XCHGL BX,BX", 0x55, 0xaa}, + {"XCHGB AX,(SI)", 0x54, 0xaa}, + {"XCHGB (SI),AX", 0x54, 0xaa}, + } { + t.Run(arch+"/"+row.body, func(t *testing.T) { + before := [16]uint8{0: 0x55, 3: 0xaa, 6: 0x81} + want := before + want[0], want[3] = row.ax, row.bx + if arch == "amd64" && strings.HasPrefix(row.body, "XCHGL") && row.body != "XCHGL AX,AX" { + want[3] |= 240 + if row.body != "XCHGL BX,BX" { + want[0] |= 240 + } + } + if got := x86RegisterCanonicalBytesAfter(x86ScalarLaneInstruction(t, row.body), before, arch); got != want { + t.Fatal("independent canonical-byte domain did not follow the same typed swap", got, want) + } + }) + } + } +} + +func TestGoAMD64FixedLocalExchangeReadonlyAndRange(t *testing.T) { + for _, row := range []struct { + name, body string + valid bool + }{ + {"readonly origin moves", "LEAQ literal<>(SB),CX\nMOVQ AX,DX\nXCHGQ CX,DX\nMOVQ (DX),AX", true}, + {"readonly remains readonly", "LEAQ literal<>(SB),CX\nMOVQ AX,DX\nXCHGQ CX,DX\nXCHGQ AX,(DX)", false}, + {"readonly extent unchanged", "LEAQ literal<>+240(SB),CX\nMOVQ AX,DX\nXCHGQ CX,DX\nMOVQ (DX),AX", false}, + {"range moves", fixedLocalRangePrefix + "MOVQ $0,CX\nXCHGQ R9,CX\n" + strings.ReplaceAll(fixedLocalRangeLoad, "R9", "CX"), true}, + {"comparison follows value", "MOVQ AX,R9\nMOVQ AX,CX\nCMPQ R9,$16\nXCHGQ R9,CX\nJAE done\nTESTQ CX,CX\nJE done\n" + strings.ReplaceAll(fixedLocalRangeLoad, "R9", "CX"), true}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalRangeFixture(t, row.body) + _, err := proveX86FixedLocal(fn, sig) + if row.valid && err != nil || !row.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("readonly/range value valid=%t: %v", row.valid, err) + } + }) + } +} diff --git a/amd64_fixed_local_frame_test.go b/amd64_fixed_local_frame_test.go new file mode 100644 index 000000000..cd1d91410 --- /dev/null +++ b/amd64_fixed_local_frame_test.go @@ -0,0 +1,785 @@ +package plan9asm + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" +) + +func fixedLocalFrameFixture(t *testing.T, operation string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options) { + t.Helper() + _, request := goAMD64FrameSourceFixture(t) + source, _, _ := x86PrivateAlignedFrameAuditFixture(t, operation) + declaration := []byte("package framefixture\nfunc Value(a, b, c, d uint64) uint64\n") + for name, data := range map[string][]byte{ + "value.go": declaration, "value_amd64.s": []byte(source), + } { + if err := os.WriteFile(filepath.Join(request.Directory, name), data, 0600); err != nil { + t.Fatal(err) + } + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "value.go", declaration, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + typed, err := (&types.Config{Sizes: sizes}).Check("example.invalid/framefixture", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + sig, err := DeriveGoFuncABI0Signature(typed.Path()+".Value", typed.Scope().Lookup("Value").(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", typed.Path()+".", 1) } + pkg := GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{syntax}} + return pkg, request, file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, + Sigs: map[string]FuncSig{sig.Name: sig}, + } +} + +func TestGoAMD64FixedLocalFrameSevenOriginalNativeOperations(t *testing.T) { + for _, operation := range []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", + } { + t.Run(strings.Fields(operation)[0], func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, operation) + source := requireGoAMD64FrameCapture(t, pkg, request) + observed := source.functions[pkg.Path+".Value"] + if observed.locals != 168 || observed.args != 40 || len(observed.code) == 0 { + t.Fatalf("missing actual original Go frame/object: %+v", observed) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + options, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, options, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatalf("original numeric frame needs sealed native backing: %v", err) + } + for _, required := range []string{"fixed_local_native", "align 8", "asm sideeffect"} { + if !strings.Contains(ir, required) { + t.Fatalf("missing actual native storage/opcode %q", required) + } + } + }) + }) + } +} + +func TestGoAMD64FixedLocalFrameActualTargetObjectsPreserveProducerBoundary(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64", "linux/386", "windows/386"} { + t.Run(target, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + request.Target = target + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if strings.HasSuffix(target, "/386") || target == "windows/amd64" && err != nil { + if !errors.Is(err, ErrProbeNeedsContext) || source != nil || + target == "windows/amd64" && !strings.Contains(err.Error(), "SSEHUNWINDINFO") { + t.Fatalf("unclosed physical target unexpectedly obtained authority: %v", err) + } + t.Logf("existing actual observer boundary remains Context, not a positive fixed-local object: %v", err) + return + } + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + if target == "darwin/amd64" { + options.TargetTriple = "x86_64-apple-darwin" + } + if target == "windows/amd64" { + options.TargetTriple = "x86_64-pc-windows-msvc" + } + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, options.TargetTriple, "fixed-local.ll", "fixed-local.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "fixed-local-opt.ll", "fixed-local-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +func TestGoAMD64FixedLocalFrameScalarMemoryWidthsHaveExactCertificates(t *testing.T) { + for _, form := range []struct { + opcode string + width int64 + }{{"MOVB", 1}, {"MOVW", 2}, {"MOVL", 4}, {"MOVQ", 8}} { + t.Run(form.opcode, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + var source, stores strings.Builder + for _, ins := range file.Funcs[0].Instrs { + source.WriteString(ins.Raw + "\n") + } + for offset := int64(16); offset < 32; offset += form.width { + fmt.Fprintf(&stores, "%s $1,%d(SP)\n", form.opcode, offset) + } + fmt.Fprintf(&stores, "%s 16(SP),DX\n", form.opcode) + original := strings.Replace(source.String(), "MOVQ AX,16(SP)\nMOVQ CX,24(SP)\n", stores.String(), 1) + requireX86GoAssemblerResult(t, "amd64", original, true) + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(original), 0600); err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, original) + if err != nil { + t.Fatal(err) + } + observed := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := observed.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", bound) + if err != nil { + t.Fatal(err) + } + read, write := false, false + for _, ins := range normalized.Funcs[0].Instrs { + if string(ins.Op) != form.opcode { + continue + } + for _, access := range ins.x86FixedLocalMemory { + if access.width != form.width { + t.Fatalf("scalar byte envelope changed: %+v", access) + } + read = read || !access.write + write = write || access.write + } + } + if !read || !write { + t.Fatal("missing independently proved scalar read/write") + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil || ir == "" { + t.Fatalf("actual scalar memory width failed consumer: %v", err) + } + }) + }) + } +} + +func fixedLocalAPIs(t *testing.T, pkg GoPackage, file *File, options Options, check func(*testing.T, string, error)) { + t.Helper() + for _, api := range []string{"Translate", "Module", "GoModule"} { + t.Run(api, func(t *testing.T) { + fixedLocalAPI(t, pkg, file, options, api, check) + }) + } +} + +// Each API consumes its own complete actual source observation. Neither the +// compiler cache nor a previous API's options can extend or revive that lease. +func fixedLocalFreshAPIs(t *testing.T, pkg GoPackage, request GoAMD64FrameCaptureRequest, options Options, apis []string, check func(*testing.T, string, error)) { + t.Helper() + seen := make(map[string]bool) + if len(apis) == 0 { + t.Fatal("missing fixed-local API inventory") + } + for _, api := range apis { + if (api != "Translate" && api != "Module" && api != "GoModule") || seen[api] { + t.Fatal("unknown or duplicated fixed-local API", api) + } + seen[api] = true + } + for _, api := range apis { + t.Run(api, func(t *testing.T) { + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }() + original, err := os.ReadFile(filepath.Join(request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + // Bind consumes the exact producer's complete CPP output, including + // suffix formatting. The original file remains an immutable input. + expanded, err := source.preprocess(request.AssemblyFile, original) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + if len(source.functions) != len(file.Funcs) { + t.Fatal("incomplete actual original Go object inventory") + } + fixedLocalAPI(t, pkg, file, bound, api, check, expanded) + }) + } +} + +func fixedLocalAPI(t *testing.T, pkg GoPackage, file *File, options Options, api string, check func(*testing.T, string, error), assembly ...string) { + t.Helper() + if len(assembly) > 1 || len(assembly) == 1 && assembly[0] == "" { + t.Fatal("expected at most one complete nonempty assembly source") + } + switch api { + case "Translate": + ir, err := Translate(file, options) + check(t, ir, err) + case "Module": + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + ir := "" + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + check(t, ir, err) + case "GoModule": + var source strings.Builder + // The fresh owner supplies its exact preprocessed source, retaining + // DATA/GLOBL as well as every TEXT body in this independent API. + if len(assembly) == 1 { + source.WriteString(assembly[0]) + } else { + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + source.WriteString(ins.Raw + "\n") + } + } + } + goos := "linux" + if strings.Contains(options.TargetTriple, "apple") { + goos = "darwin" + } + if strings.Contains(options.TargetTriple, "windows") { + goos = "windows" + } + translation, err := TranslateGoModule(pkg, []byte(source.String()), GoModuleOptions{ + GOOS: goos, GOARCH: options.Goarch, + TargetTriple: options.TargetTriple, Sizes: types.SizesFor("gc", options.Goarch), WASMABI: options.WASMABI, + ResolveSym: options.ResolveSym, + ManualSig: func(name string) (FuncSig, bool) { sig, found := options.Sigs[name]; return sig, found }, + }) + ir := "" + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + ir = translation.Module.String() + } + check(t, ir, err) + default: + t.Fatal("unknown fixed-local API", api) + } +} + +func fixedLocalRequireContext(t *testing.T, ir string, err error) { + t.Helper() + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("unbound frame/effect produced IR: %v %s", err, ir) + } +} + +func TestGoAMD64FixedLocalFrameThreeAPIsCannotForgeOrMutateBinding(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + fixedLocalAPIs(t, pkg, file, options, fixedLocalRequireContext) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + for _, mutation := range []string{"JSON", "signature", "slot", "attrs", "target", "arch", "resolver", "source", "missing function", "ABI"} { + t.Run(mutation, func(t *testing.T) { + changed := bound + changed.Sigs, err = cloneFixedLocalSigs(bound.Sigs) + if err != nil { + t.Fatal(err) + } + for name, sig := range changed.Sigs { + sig.x86FixedLocal = bound.Sigs[name].x86FixedLocal + changed.Sigs[name] = sig + } + changedFile := *file + changedFile.Funcs = append([]Func(nil), file.Funcs...) + sig := changed.Sigs[pkg.Path+".Value"] + switch mutation { + case "JSON": + changed.Sigs, err = cloneFixedLocalSigs(bound.Sigs) + if err != nil { + t.Fatal(err) + } + case "signature": + sig.Args[0] = I32 + case "slot": + sig.Frame.Params[0].Offset++ + case "attrs": + sig.Attrs = "nounwind" + case "target": + changed.TargetTriple = "x86_64-apple-darwin" + case "arch": + changed.Goarch = "386" + case "resolver": + changed.ResolveSym = func(string) string { return "foreign.Value" } + case "ABI": + changed.WASMABI = WASMABIGo + case "source": + changedFile.Funcs[0].FrameSize += 8 + changedFile.Funcs[0].Instrs = append([]Instr(nil), file.Funcs[0].Instrs...) + changedFile.Funcs[0].Instrs[0].Raw = strings.Replace(changedFile.Funcs[0].Instrs[0].Raw, "$160-40", "$168-40", 1) + case "missing function": + changedFile.Funcs = nil + } + if mutation != "JSON" { + changed.Sigs[pkg.Path+".Value"] = sig + } + check := fixedLocalRequireContext + if mutation == "missing function" { + check = func(t *testing.T, ir string, err error) { + if err == nil || ir != "" { + t.Fatalf("missing entire source produced a module: %v", err) + } + } + } + fixedLocalAPIs(t, pkg, &changedFile, changed, check) + }) + } + // Mutating the original input does not mutate the returned signature nor + // the separately cloned sealed expected declaration. + input := options.Sigs[pkg.Path+".Value"] + input.Args[0] = I32 + options.Sigs[pkg.Path+".Value"] = input + if _, err := Translate(file, bound); err != nil { + t.Fatalf("caller-owned Options changed the immutable binding: %v", err) + } + // Tail groups exist only in Options, not in GoModuleOptions. Exercise the + // actual two consumers that can receive this request, not a fake transport. + changed := bound + changed.X86TailGroups = []X86TailGroup{{Root: "·Value", Helpers: []string{"·Value"}}} + ir, err := Translate(file, changed) + fixedLocalRequireContext(t, ir, err) + llvmContext := llvm.NewContext() + defer llvmContext.Dispose() + module, err := TranslateModuleInContext(llvmContext, file, changed) + if !module.IsNil() { + defer module.Dispose() + t.Fatal("unbound continuation request produced a module") + } + fixedLocalRequireContext(t, "", err) +} + +func TestGoAMD64FixedLocalFrameBindResolverMayRevokeWithoutLockingDeadlock(t *testing.T) { + if os.Getenv("PLAN9ASM_FIXED_LOCAL_REENTRANCY_CHILD") != "1" { + stage, cancelStage := runtimeOracleStageContext(t) + defer cancelStage() + ctx, cancel := context.WithTimeout(stage, 30*time.Second) + defer cancel() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestGoAMD64FixedLocalFrameBindResolverMayRevokeWithoutLockingDeadlock$", "-test.timeout=1m") + command.Env = append(os.Environ(), "PLAN9ASM_FIXED_LOCAL_REENTRANCY_CHILD=1") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("bounded actual bind resolver/Close child failed or deadlocked: %v\n%s", err, output) + } + return + } + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + resolve := options.ResolveSym + options.ResolveSym = func(symbol string) string { + if err := source.Close(); err != nil { + t.Fatal(err) + } + return resolve(symbol) + } + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("callback-revoked observation obtained a binding: %v", err) + } +} + +func TestGoAMD64FixedLocalFrameTruncatedConstantCannotAuthorizePointerArithmetic(t *testing.T) { + for _, move := range []string{"MOVL", "MOVW", "MOVB"} { + t.Run(move, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + var original strings.Builder + for _, ins := range file.Funcs[0].Instrs { + original.WriteString(ins.Raw + "\n") + } + changed := strings.Replace(original.String(), "MOVQ $31,R9\nNOTQ R9\nANDQ R9,SP", "MOVQ $-16,AX\n"+move+" AX,BX\nANDQ BX,SP", 1) + requireX86GoAssemblerResult(t, "amd64", changed, true) + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(changed), 0600); err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, changed) + if err != nil { + t.Fatal(err) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("actual Go accepted %s narrow constant acquired full-width pointer authority: %v", move, err) + } + }) + } +} + +func TestGoAMD64FixedLocalFrameClosedExpiredAndChangedSourceRevokes(t *testing.T) { + for _, mutation := range []string{"close", "cancel", "file", "object", "listing", "export", "post-consumer"} { + t.Run(mutation, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + stage, finish := runtimeOracleStageContext(t) + defer finish() + ctx, cancel := context.WithCancel(stage) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + var path string + switch mutation { + case "close": + if err := source.Close(); err != nil { + t.Fatal(err) + } + case "cancel": + cancel() + case "file": + path = filepath.Join(request.Directory, "value.go") + case "object": + path = filepath.Join(source.owned, request.AssemblyFile+".o") + case "listing": + path = filepath.Join(source.owned, request.AssemblyFile+".o.listing") + case "export": + path = source.selection.Export + case "post-consumer": + calls := 0 + bound.ResolveSym = func(symbol string) string { + calls++ + if calls == 1 { + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package changed\n"), 0600); err != nil { + t.Fatal(err) + } + } + return options.ResolveSym(symbol) + } + } + if path != "" { + if err := os.WriteFile(path, []byte("mutation"), 0600); err != nil { + t.Fatal(err) + } + } + fixedLocalAPIs(t, pkg, file, bound, fixedLocalRequireContext) + if mutation != "post-consumer" && source.ready { + t.Fatal("failed observer verification did not permanently revoke the binding") + } + }) + } +} + +func TestGoAMD64FixedLocalFrameResolverCannotChangeParsedSourceAfterValidation(t *testing.T) { + for _, api := range []string{"Translate", "Module", "GoModule"} { + t.Run(api, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + resolve := bound.ResolveSym + bound.ResolveSym = func(symbol string) string { + for index := range file.Funcs[0].Instrs { + ins := &file.Funcs[0].Instrs[index] + if ins.Op == "PADDQ" { + ins.Op = "PADDB" + ins.Raw = strings.Replace(ins.Raw, "PADDQ", "PADDB", 1) + } + } + return resolve(symbol) + } + if api == "Translate" { + ir, err := Translate(file, bound) + fixedLocalRequireContext(t, ir, err) + return + } + if api == "Module" { + llvmContext := llvm.NewContext() + defer llvmContext.Dispose() + module, err := TranslateModuleInContext(llvmContext, file, bound) + if !module.IsNil() { + defer module.Dispose() + t.Fatal("resolver-mutated original source produced a module") + } + fixedLocalRequireContext(t, "", err) + return + } + // GoModule reparses the original bytes into its own private file; + // this public-file mutation cannot change that source. Its resolver + // is frozen before consumer normalization, so the original opcode + // must remain independently bound rather than silently substituted. + original := strings.Builder{} + for _, ins := range file.Funcs[0].Instrs { + original.WriteString(ins.Raw + "\n") + } + translation, err := TranslateGoModule(pkg, []byte(original.String()), GoModuleOptions{ + GOOS: "linux", GOARCH: "amd64", TargetTriple: bound.TargetTriple, + Sizes: types.SizesFor("gc", "amd64"), ResolveSym: bound.ResolveSym, + ManualSig: func(name string) (FuncSig, bool) { sig, found := bound.Sigs[name]; return sig, found }, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + if err != nil || translation == nil || !strings.Contains(translation.Module.String(), "paddq") { + t.Fatalf("GoModule did not preserve its independent original source: %v", err) + } + }) + } +} + +func TestGoAMD64FixedLocalFrameProofFullByteWidthAddressAndControlNegatives(t *testing.T) { + base, sig, _ := x86PrivateAlignedFrameAuditFixture(t, "PADDQ 16(SP),X0") + mutations := map[string]string{ + "alignment": strings.Replace(base, "ADDQ $32,SP", "ADDQ $33,SP", 1), + "read extent": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 152(SP)", 1), + "write extent": strings.Replace(base, "MOVQ AX,16(SP)", "MOVQ AX,160(SP)", 1), + "unknown pointer": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 16(SI)", 1), + "indexed pointer": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 16(SP)(AX*1)", 1), + "segment": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 16(FS)", 1), + "pointer result": strings.Replace(base, "MOVQ AX,ret+32(FP)", "MOVQ R11,ret+32(FP)", 1), + "pointer flags": strings.Replace(base, "MOVQ AX,ret+32(FP)", "CMPQ R11,AX\nMOVQ AX,ret+32(FP)", 1), + "pointer truncated": strings.Replace(base, "MOVQ SP,R11", "MOVL SP,R11", 1), + "missing restore": strings.Replace(base, "MOVQ R11,SP", "MOVQ $0,R10", 1), + "pointer cell clobber": strings.Replace(base, "MOVQ 96(SP),R11", "MOVB $1,96(SP)\nMOVQ 96(SP),R11", 1), + "pointer bytes as numeric": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 96(SP)", 1), + "unknown call": strings.Replace(base, "PADDQ", "CALL ·Other(SB)\nPADDQ", 1), + "unknown dead effect": base + "SYSCALL\n", + "result width": strings.Replace(base, "MOVQ AX,ret+32(FP)", "MOVL AX,ret+32(FP)", 1), + "local spill escape": strings.Replace(base, "MOVQ AX,ret+32(FP)", "MOVQ R11,·global(SB)\nMOVQ AX,ret+32(FP)", 1), + "unknown flags branch": strings.Replace(base, "MOVQ AX,16(SP)", "JEQ done\nMOVQ AX,16(SP)", 1) + "done:\nRET\n", + "FP after SP change": strings.Replace(base, "MOVQ AX,16(SP)", "MOVQ a+0(FP),AX\nMOVQ AX,16(SP)", 1), + "high legacy vector bank": strings.ReplaceAll(base, "X0", "X16"), + } + for byteIndex := 0; byteIndex < 16; byteIndex++ { + var stores strings.Builder + for index := 0; index < 16; index++ { + if index != byteIndex { + fmt.Fprintf(&stores, "MOVB $1,%d(SP)\n", 16+index) + } + } + mutated := strings.Replace(base, "MOVQ AX,16(SP)\nMOVQ CX,24(SP)\n", stores.String(), 1) + mutations[fmt.Sprintf("missing byte %02d", byteIndex)] = mutated + } + for name, source := range mutations { + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source admitted: %v", err) + } + }) + } +} + +func TestGoAMD64FixedLocalFrameAllPathInitializationAndResourceBounds(t *testing.T) { + base, sig, _ := x86PrivateAlignedFrameAuditFixture(t, "PADDQ 16(SP),X0") + branch := strings.Replace(base, "MOVQ CX,24(SP)", "MOVQ CX,24(SP)\nCMPQ AX,CX\nJEQ joined\nMOVQ CX,24(SP)\njoined:", 1) + requireX86GoAssemblerResult(t, "amd64", branch, true) + file, err := Parse(ArchAMD64, branch) + if err != nil { + t.Fatal(err) + } + proved, err := proveX86FixedLocal(file.Funcs[0], sig) + if err != nil { + t.Fatal(err) + } + for _, ins := range proved.Instrs { + if ins.Op != "PADDQ" { + continue + } + access := ins.x86FixedLocalMemory[0] + if access.offset != [8]int64{48, 40, 32, 24, 48, 40, 32, 24} { + t.Fatalf("missing all original SP residue accesses: %v", access.offset) + } + } + for name, source := range map[string]string{ + "one predecessor uninitialized": strings.Replace(branch, "MOVQ CX,24(SP)\nCMPQ", "CMPQ", 1), + "stale pointer flags": strings.Replace(branch, "CMPQ AX,CX", "ADDQ $0,SP", 1), + "loop loses flags": strings.Replace(branch, "JEQ joined", "before:\nJEQ joined\nIMULQ $3,AX\nJMP before", 1), + "different address join": strings.Replace(branch, "JEQ joined\nMOVQ CX,24(SP)", "JEQ joined\nADDQ $8,SP\nMOVQ CX,24(SP)", 1), + "fallthrough": strings.TrimSuffix(base, "RET\n"), + } { + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved CFG gained backing: %v", err) + } + }) + } + tooLarge := file.Funcs[0] + tooLarge.FrameSize = 1 << 20 + if _, err := proveX86FixedLocal(tooLarge, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("oversized physical frame was admitted") + } + // A long straight block no longer retains one state per instruction. + // Exercise the unchanged storage cap with actual control-flow boundaries. + tooLarge, largeSig := fixedLocalRetainedStateBudgetFixture(t) + if _, err := proveX86FixedLocal(tooLarge, largeSig); !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "state storage budget") { + t.Fatal("retained frame-state storage budget was ignored", err) + } +} + +func TestGoAMD64FixedLocalFrameResidueDependentAliasesCannotRelocate(t *testing.T) { + base, sig, _ := x86PrivateAlignedFrameAuditFixture(t, "PADDQ 16(SP),X0") + // Both address origins belong to the same local frame, and every byte is + // initialized for every residue. Nevertheless this write overlaps the + // source's low word, high word or neither depending on original SP mod 32. + // A new physical alloca cannot silently choose one of these observations. + source := strings.Replace(base, "MOVQ BX,56(SP)", "MOVQ BX,56(SP)\nMOVQ $1,48(R11)", 1) + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("residue-dependent initialized aliases were relocated: %v", err) + } + // Independent little-endian byte-memory mathematics, not the production + // address/state/certificate implementation, exposes the actual data change. + want := [4][2]uint64{{6, 10}, {7, 8}, {7, 10}, {7, 10}} + for residue := 0; residue < 4; residue++ { + var memory [256]byte + sp := 32 + residue*8 + aligned := sp&^31 + 32 + for index, value := range []uint64{2, 3} { + binary.LittleEndian.PutUint64(memory[aligned+16+index*8:], value) + } + for index, value := range []uint64{5, 7} { + binary.LittleEndian.PutUint64(memory[aligned+48+index*8:], value) + } + binary.LittleEndian.PutUint64(memory[sp+48:], 1) + got := [2]uint64{} + for index := range got { + got[index] = binary.LittleEndian.Uint64(memory[aligned+16+index*8:]) + + binary.LittleEndian.Uint64(memory[aligned+48+index*8:]) + } + if got != want[residue] { + t.Fatalf("independent original residue %d data: got %v want %v", residue, got, want[residue]) + } + } +} + +func TestGoAMD64FixedLocalFrameCertificatesArePrivateExactAndRevocable(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", bound) + if err != nil { + t.Fatal(err) + } + fn := normalized.Funcs[0] + lower := &amd64Ctx{fixedLocal: fn.x86FixedLocal} + for _, ins := range fn.Instrs { + for index, access := range ins.x86FixedLocalMemory { + if !lower.fixedLocalMemoryCertified(ins, index) { + t.Fatal("missing independently replayed original access") + } + for residue, offset := range access.offset { + if offset < 0 || offset+access.width > 160 { + t.Fatalf("invalid residue %d certified extent", residue) + } + } + for _, mutation := range []string{"op", "args", "width", "owner", "JSON", "address size"} { + changed := ins + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86FixedLocalMemory = make(map[int]x86FixedLocalAccess) + for key, value := range ins.x86FixedLocalMemory { + changed.x86FixedLocalMemory[key] = value + } + switch mutation { + case "op": + changed.Op = "UNDEF" + case "args": + changed.Args[index].Mem.Off++ + case "width": + value := changed.x86FixedLocalMemory[index] + value.width++ + changed.x86FixedLocalMemory[index] = value + case "owner": + value := changed.x86FixedLocalMemory[index] + value.owner = &x86FixedLocalProof{} + changed.x86FixedLocalMemory[index] = value + case "JSON": + data, _ := json.Marshal(ins) + changed = Instr{} + if err := json.Unmarshal(data, &changed); err != nil { + t.Fatal(err) + } + case "address size": + changed.x86AddressBits = 32 + } + if lower.fixedLocalMemoryCertified(changed, index) { + t.Fatalf("%s preserved a stale native access certificate", mutation) + } + } + } + } +} diff --git a/amd64_fixed_local_framed_leaf.go b/amd64_fixed_local_framed_leaf.go new file mode 100644 index 000000000..4b769d72f --- /dev/null +++ b/amd64_fixed_local_framed_leaf.go @@ -0,0 +1,129 @@ +package plan9asm + +import "fmt" + +// The existing expansion gives every helper RET one original callsite resume. +// Reuse that identity; no independent call graph or stack-coordinate map exists. +func (view *x86DirectTailRoot) privateFramedCall(pc int) (int, bool) { + if !view.privateCallTransition(pc) || view.graph == nil || view.graph.fixedLocal == nil { + return 0, false + } + call := pc + if view.instructions[pc].Op == OpRET { + call = view.edges[pc][0] - 1 + } + if call < 0 || call >= len(view.instructions) || view.instructions[call].Op != OpCALL || len(view.edges[call]) != 1 { + return 0, false + } + target := view.edges[call][0] + if target < 0 || target >= len(view.points) { + return 0, false + } + fn := view.graph.original.Funcs[view.points[target].text] + return call, fn.FrameSize > 0 && view.graph.fixedLocal.leaves[fn.Sym] != "" +} + +// A source leaf cannot call another helper. One immutable saved value therefore +// suffices, including partially known bytes and addresses; it grants no read. +type x86FixedLocalSavedBP struct { + call int + value fixedLocalScalarValue + known uint8 +} + +func fixedLocalFrameTransition(view *x86DirectTailRoot, pc int, state *x86FixedLocalState) error { + call, framed := view.privateFramedCall(pc) + if !framed { + return nil // An observed zero-frame leaf preserves BP and FLAGS. + } + var saved *x86FixedLocalSavedBP + if view.instructions[pc].Op == OpCALL { + if state.savedBP != nil { + return goAMD64FrameContext("framed source leaf acquired a nested call") + } + saved = &x86FixedLocalSavedBP{call: call, known: state.known[BP]} + saved.value.address, saved.value.pointer = state.pointers[BP] + saved.value.identity, saved.value.opaque = state.opaque[BP] + saved.value.constant, saved.value.constantKnown = state.constants[BP] + saved.value.bounds, saved.value.bounded = state.ranges[BP] + saved.value.booleans = state.booleans[BP] + state.savedBP = saved + } else { + saved = state.savedBP + if saved == nil || saved.call != call { + return goAMD64FrameContext("framed leaf RET lost its matching saved BP") + } + state.savedBP = nil + } + delete(state.pointers, BP) + delete(state.opaque, BP) + delete(state.constants, BP) + delete(state.ranges, BP) + delete(state.booleans, BP) + state.known[BP] = 0 + state.flags = 0 + state.comparison = x86FixedLocalComparison{} + if view.instructions[pc].Op == OpRET { + value := saved.value + state.known[BP], state.booleans[BP] = saved.known, value.booleans + if value.pointer { + state.pointers[BP] = value.address + } + if value.opaque { + state.opaque[BP] = value.identity + } + if value.constantKnown { + state.constants[BP] = value.constant + } + if value.bounded { + state.setRange(BP, value.bounds) + } + } + return nil +} + +func mergeFixedLocalSavedBP(previous *x86FixedLocalState, current x86FixedLocalState) (bool, error) { + first, last := previous.savedBP, current.savedBP + if first == last { + return false, nil + } + if first == nil || last == nil || first.call != last.call { + return false, goAMD64FrameContext("fixed local CFG crosses distinct automatic BP saves") + } + left, right := first.value, last.value + if left.opaque != right.opaque { + // Match the ordinary register join: an address on only one incoming + // path becomes unknown, never numerical knowledge or private storage. + left.opaque, right.opaque = false, false + left.identity, right.identity = x86FixedLocalOrigins{}, x86FixedLocalOrigins{} + } + value, compatible := mergeFixedLocalScalarValues(left, right) + if !compatible { + return false, goAMD64FrameContext("saved BP has inconsistent private address identities") + } + merged := x86FixedLocalSavedBP{call: first.call, value: value, known: first.known & last.known} + if merged == *first { + return false, nil + } + previous.savedBP = &merged + return true, nil +} + +func (c *amd64Ctx) lowerPrivateFrameTransition(pc int) error { + call, framed := c.directTail.privateFramedCall(pc) + if !framed { + return nil + } + slot := fmt.Sprintf("%%private_call_bp_%d", call) + if c.directTail.instructions[pc].Op == OpCALL { + value, err := c.loadReg(BP) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i64 %s, ptr %s\n", value, slot) + return nil + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i64, ptr %s\n", value, slot) + return c.storeReg(BP, "%"+value) +} diff --git a/amd64_fixed_local_framed_leaf_go127_test.go b/amd64_fixed_local_framed_leaf_go127_test.go new file mode 100644 index 000000000..2e985ae72 --- /dev/null +++ b/amd64_fixed_local_framed_leaf_go127_test.go @@ -0,0 +1,299 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +type fixedLocalFramedLeafCase struct { + name, caller, helper string + frame int +} + +func fixedLocalFramedLeafCases() []fixedLocalFramedLeafCase { + helper := "MOVQ $7,BP\n ADDQ BP,AX" + return []fixedLocalFramedLeafCase{ + {"numeric_bp", "MOVQ $11,BP\n CALL leaf<>(SB)\n ADDQ BP,AX", helper, 8}, + {"private_address_bp", "LEAQ 16(SP),BP\n CALL leaf<>(SB)\n ADDQ (BP),AX", helper, 16}, + {"unknown_bp", "CALL leaf<>(SB)", helper, 24}, + {"untouched_bp", "CALL leaf<>(SB)", "ADDQ $7,AX", 56}, + {"partial_bp", "MOVB $11,BP\n CALL leaf<>(SB)\n MOVBLZX BP,BX\n ADDQ BX,AX", helper, 32}, + {"two_call_sites", "MOVQ $3,BP\n CALL leaf<>(SB)\n ADDQ BP,AX\n CALL leaf<>(SB)\n ADDQ BP,AX", helper, 40}, + {"loop_reentry", "MOVQ $11,BP\n MOVQ $3,SI\nagain:\n CALL leaf<>(SB)\n ADDQ BP,AX\n DECQ SI\n JNZ again", helper, 64}, + {"two_returns", "LEAQ 16(SP),BP\n CALL leaf<>(SB)\n ADDQ (BP),AX", `MOVQ $7,BP + CMPQ AX,BP + JCS small + ADDQ BP,AX + RET +small: + SUBQ BP,AX`, 128}, + } +} + +func fixedLocalFramedLeafSource(row fixedLocalFramedLeafCase) string { + return fmt.Sprintf(`TEXT ·Value(SB),4,$80-16 + MOVQ a+0(FP),AX + MOVQ AX,16(SP) + MOVQ $7,BX + %s + MOVQ AX,ret+8(FP) + RET +TEXT leaf<>(SB),4,$%d + %s + RET +`, row.caller, row.frame, row.helper) +} + +func TestGoAMD64FixedLocalFramedLeafCompleteSources(t *testing.T) { + for _, row := range fixedLocalFramedLeafCases() { + t.Run(row.name, func(t *testing.T) { + assembly := fixedLocalFramedLeafSource(row) + x86InsertExtractGoListing(t, "amd64", assembly) + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil || strings.Count(ir, "define ") != 1 { + t.Fatal("observed framed leaf lost its caller or acquired an independent ABI", err) + } + fixedLocalPrivateLeafObjects(t, ir) + }) + }) + } +} + +const fixedLocalFramedLeafIncomingAddress = `TEXT ·Value(SB),4,$80-16 + MOVQ a+0(FP),BP + MOVQ (BP),AX + CALL leaf<>(SB) + ADDQ (BP),AX + MOVQ AX,ret+8(FP) + RET +TEXT leaf<>(SB),4,$48 + MOVQ $7,BP + ADDQ BP,AX + RET +` + +func TestGoAMD64FixedLocalFramedLeafIncomingAddress(t *testing.T) { + x86InsertExtractGoListing(t, "amd64", fixedLocalFramedLeafIncomingAddress) + pkg, request, options := fixedLocalPrivateLeafFixture(t, fixedLocalFramedLeafIncomingAddress, + "func Value(a *uint64) uint64") + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("caller incoming address did not survive the observed helper frame", err) + } + fixedLocalPrivateLeafObjects(t, ir) + }) +} + +func TestGoAMD64WindowsFramedLeafCompleteObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, spelling := range []string{"leaf<>", "leaf"} { + t.Run(spelling, func(t *testing.T) { + assembly := strings.ReplaceAll(fixedLocalFramedLeafSource(fixedLocalFramedLeafCases()[1]), "leaf<>", spelling) + dir := t.TempDir() + input, object := filepath.Join(dir, "original.s"), filepath.Join(dir, "original.o") + if err := os.WriteFile(input, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "oracle", "-o", object, input) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=windows", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), true, "amd64"); failure != nil { + t.Fatalf("original Windows framed-helper object: %v\n%s", failure, output) + } + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly) + request.Target = "windows/amd64" + options.TargetTriple = "x86_64-pc-windows-msvc" + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("Windows helper lost its complete frame and SEH observation", err) + } + compileLLVMToObject(t, llc, options.TargetTriple, "framed-leaf.ll", "framed-leaf.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "framed-leaf-opt.ll", "framed-leaf-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +func TestGoAMD64WindowsFramedLeafRequiresSEH(t *testing.T) { + assembly := fixedLocalFramedLeafSource(fixedLocalFramedLeafCases()[1]) + pkg, request, _ := fixedLocalPrivateLeafFixture(t, assembly) + request.Target = "windows/amd64" + source := requireGoAMD64FrameCapture(t, pkg, request) + object, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o")) + if err != nil { + t.Fatal(err) + } + listing, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + data, err := source.frameDataObject(assembly, object) + if err != nil { + t.Fatal(err) + } + leaves, err := goAMD64WindowsSourceLeafTexts(file, pkg, listing, data.Member.Texts, data.TextOrder) + if err != nil || len(leaves) != 1 { + t.Fatal("original source lost its single private framed helper", err) + } + observed, err := source.observeWindowsUnwind(object, leaves) + if err != nil { + t.Fatal("original framed helper must carry actual Go unwind metadata", err) + } + for index, text := range observed.Texts { + if text.Index != leaves[0].Index { + continue + } + filtered := text.Auxiliary[:0:0] + links := 0 + for _, aux := range text.Auxiliary { + if aux.Type == observed.AuxType { + links++ + } else { + filtered = append(filtered, aux) + } + } + if links != 1 { + t.Fatal("original private frame lacks exactly one SEH owner", links) + } + observed.Texts[index].Auxiliary = filtered + if err := goAMD64FrameWindowsUnwindFunctions(source.functions, source.origins, request.Directory, observed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("framed helper borrowed the zero-frame SEH exemption", err) + } + return + } + t.Fatal("original complete object lost its helper TEXT") +} + +func TestGoAMD64FixedLocalFramedLeafBoundaries(t *testing.T) { + for _, row := range []fixedLocalFramedLeafCase{ + {"incoming_bp", "MOVQ $11,BP\n CALL leaf<>(SB)", "MOVQ BP,AX", 8}, + {"incoming_bp_memory", "LEAQ 16(SP),BP\n CALL leaf<>(SB)", "MOVQ (BP),AX", 8}, + {"partial_callee_bp", "CALL leaf<>(SB)", "MOVB $11,BP\n MOVQ BP,AX", 16}, + {"partial_caller_bp", "MOVB $11,BP\n CALL leaf<>(SB)\n MOVQ BP,AX", "MOVQ $7,BP", 24}, + {"incoming_flags", "CMPQ AX,BX\n CALL leaf<>(SB)", "SETCS CL\n MOVBLZX CL,AX", 32}, + {"returned_flags", "CALL leaf<>(SB)\n SETCS CL\n MOVBLZX CL,AX", "CMPQ AX,BX", 128}, + {"different_saved_private_bp", `LEAQ 16(SP),BP + LEAQ 24(SP),R8 + MOVQ $3,SI +again: + CALL leaf<>(SB) + XCHGQ BP,R8 + DECQ SI + JNZ again`, "MOVQ $7,BP\n ADDQ BP,AX", 8}, + {"no_automatic_bp", "MOVQ $11,BP\n CALL leaf<>(SB)\n ADDQ BP,AX", "MOVQ $7,BP\n ADDQ BP,AX", 8}, + } { + t.Run(row.name, func(t *testing.T) { + assembly := fixedLocalFramedLeafSource(row) + if row.name == "no_automatic_bp" { + // NOSPLIT|NOFRAME is legal Go source with explicit locals, + // but it lacks the automatic BP save that this class requires. + assembly = strings.Replace(assembly, "TEXT leaf<>(SB),4,$8", "TEXT leaf<>(SB),516,$8", 1) + } + x86InsertExtractGoListing(t, "amd64", assembly) + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if owner != nil { + defer owner.Close() + } + if err != nil && row.name != "no_automatic_bp" { + t.Fatal("semantic boundary requires successful original source/frame capture", err) + } + if err == nil { + file, parseErr := Parse(ArchAMD64, assembly) + if parseErr != nil { + t.Fatal(parseErr) + } + _, err = owner.BindFixedLocal(ctx, pkg, file, options) + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("automatic frame supplied caller BP or arithmetic FLAGS to source", err) + } + t.Logf("original Go accepted; observed frame or typed state refused: %v", err) + }) + } +} + +func TestGoAMD64SDKFramedLeafWholeOriginalSource(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "crypto/internal/fips140/nistec") + original, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal("complete original SDK source/object precondition", err) + } + defer original.Close() + pkg := sdkAMD64FrameActualPackage(t, original.source) + owner, err := CaptureGoAMD64SDKFrameSource(ctx, pkg, GoAMD64FrameCaptureRequest{ + GoBinary: request.GoBinary, Directory: request.Directory, Target: request.Target, + AssemblyFile: "p256_asm_amd64.s", Environment: request.Environment, + }) + if err != nil { + t.Fatal("complete SDK source lost its independently observed helper frames", err) + } + defer owner.Close() + // This supersedes the former whole-source rejection: automatic private + // frames now have an observed contract. Keep every original TEXT and the + // native-identity negatives below; cropping the helper is not acceptance. + expanded := owner.sdkFrame.owner.source.observation.CPP[0].Output + file, err := Parse(ArchAMD64, expanded) + if err != nil || len(file.Funcs) != len(owner.functions) { + t.Fatal("complete original SDK TEXT inventory changed", err) + } + for _, name := range []string{"p256MulInternal", "p256SqrInternal"} { + fn, found := owner.functions[name] + if !found || !fn.privateLeaf || fn.locals != 16 || fn.signature.Name != "" || len(fn.code) == 0 { + t.Fatal("original nonzero helper lost its native frame or acquired a callable ABI", name) + } + } + if err := owner.Verify(ctx); err != nil { + t.Fatal("complete SDK frame observation changed during consumption", err) + } + data := owner.sdkFrame.owner.source + data.operation.Lock() + defer data.operation.Unlock() + controls := 0 + for frame := range data.observation.Frames { + for index := range data.observation.Frames[frame].Records { + record := &data.observation.Frames[frame].Records[index] + if record.Symbol != "p256MulInternal" && record.Symbol != "p256SqrInternal" { + continue + } + for _, locals := range []uint64{0, 8, 24} { + t.Run(fmt.Sprintf("%s/native_locals_%d", record.Symbol, locals), func(t *testing.T) { + original := record.Locals + record.Locals = locals + _, _, err := owner.sdkFrameRecords() + record.Locals = original + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "SDK-primary private leaf/native identity differs") { + t.Fatal("mismatched native locals acquired the original helper frame", err) + } + }) + controls++ + } + } + } + if controls != 6 { + t.Fatal("complete original SDK helper mismatch inventory changed", controls) + } +} diff --git a/amd64_fixed_local_framed_leaf_runtime_go127_test.go b/amd64_fixed_local_framed_leaf_runtime_go127_test.go new file mode 100644 index 000000000..32ec428fa --- /dev/null +++ b/amd64_fixed_local_framed_leaf_runtime_go127_test.go @@ -0,0 +1,148 @@ +//go:build linux && amd64 && go1.27 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalFramedLeaf(t *testing.T) { + cases := fixedLocalFramedLeafCases() + var assembly, declarations strings.Builder + for index, row := range cases { + name := fmt.Sprintf("Value%d", index) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + source := strings.NewReplacer("·Value(", "·"+name+"(", "leaf<>", fmt.Sprintf("leaf%d<>", index)).Replace( + fixedLocalFramedLeafSource(row)) + assembly.WriteString(source) + } + inputs := []uint64{0, 1, 6, 7, 8, 0xffffffff, 0x100000000, ^uint64(0)} + expected := make([][]string, len(inputs)) + for index, input := range inputs { + for _, row := range cases { + var result uint64 + switch row.name { + case "numeric_bp", "partial_bp": + result = input + 18 + case "private_address_bp": + result = input*2 + 7 + case "unknown_bp", "untouched_bp": + result = input + 7 + case "two_call_sites": + result = input + 20 + case "loop_reentry": + result = input + 54 + case "two_returns": + result = input*2 + 7 + if input < 7 { + result = input*2 - 7 + } + default: + t.Fatal("missing independent original-frame oracle", row.name) + } + expected[index] = append(expected[index], fmt.Sprintf("%016x", result)) + } + } + fixedLocalResultsRuntime(t, assembly.String(), declarations.String(), inputs, expected) + if !t.Failed() { + t.Logf("FRAMED_LEAF_NATIVE_COMPLETE functions=%d inputs_per_function=%d APIs=3 LLVM_levels=O0,O2 original_observations=%d LLVM_comparisons=%d", + len(cases), len(inputs), len(cases)*len(inputs), len(cases)*len(inputs)*3*2) + } +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FramedLeafIncomingAddress(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + inputs := []uint64{0, 1, 0xffffffff, 0x100000000, ^uint64(0)} + var goCalls, cCalls strings.Builder + var expected []string + for _, input := range inputs { + fmt.Fprintf(&goCalls, "{ a := uint64(0x%x); fmt.Printf(\"%%016x\\n\",Value(&a)) }\n", input) + fmt.Fprintf(&cCalls, "{ uint64_t a = UINT64_C(0x%x); printf(\"%%016llx\\n\",(unsigned long long)value(&a)); }\n", input) + expected = append(expected, fmt.Sprintf("%016x", input*2+7)) + } + const declaration = "func Value(a *uint64) uint64" + original := "package main\nimport \"fmt\"\n" + declaration + "\nfunc main() {\n" + goCalls.String() + "}\n" + want := strings.Fields(string(x86ScalarFloatIntegerOriginalOutput(t, execution, fixedLocalFramedLeafIncomingAddress, original))) + if strings.Join(want, " ") != strings.Join(expected, " ") { + t.Fatal("original helper did not restore its caller's input address", want, expected) + } + label := runtimeCAsmLabel("example.invalid/framefixture.Value") + driver := "#include \n#include \n" + runtimeCAsmLabelPreamble + + "extern uint64_t value(uint64_t *) __asm__(" + label + ");\nint main(void) {\n" + cCalls.String() + "return 0;\n}\n" + pkg, request, options := fixedLocalPrivateLeafFixture(t, fixedLocalFramedLeafIncomingAddress, declaration) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + got := strings.Fields(string(execution.compileAndRunLLVM(t, "framed-leaf-input-address", input, driver, level))) + if strings.Join(got, " ") != strings.Join(want, " ") { + t.Fatal("translated helper changed caller input-address semantics", got, want) + } + }) + } + }) + if !t.Failed() { + t.Logf("FRAMED_LEAF_NATIVE_ADDRESS functions=1 inputs_per_function=%d APIs=3 LLVM_levels=O0,O2 original_observations=%d LLVM_comparisons=%d", + len(inputs), len(inputs), len(inputs)*3*2) + } +} + +// These accepted original Go programs observe the automatic physical frame. +// Their framed variants deliberately remain translation counterexamples: +// arithmetic FLAGS do not retain their caller/body value across ADD/SUB SP. +func TestCrossLinuxRuntimeMatrixGoAMD64FramedLeafPhysicalFlags(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + var assembly, declarations, calls strings.Builder + var frames []int + for _, frame := range []int{0, 8, 128} { + for _, incoming := range []bool{true, false} { + row := fixedLocalFramedLeafCase{ + caller: "CALL leaf<>(SB)\n SETCS CL\n MOVBLZX CL,AX", + helper: "CMPQ AX,BX", + frame: frame, + } + if incoming { + row.caller = "CMPQ AX,BX\n CALL leaf<>(SB)" + row.helper = "SETCS CL\n MOVBLZX CL,AX" + } + index := len(frames) + name := fmt.Sprintf("Value%d", index) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + fmt.Fprintf(&calls, "fmt.Println(%s(0))\nfmt.Println(%s(8))\n", name, name) + assembly.WriteString(strings.NewReplacer("·Value(", "·"+name+"(", + "leaf<>", fmt.Sprintf("leaf%d<>", index)).Replace(fixedLocalFramedLeafSource(row))) + frames = append(frames, frame) + } + } + original := "package main\nimport \"fmt\"\n" + declarations.String() + "func main() {\n" + calls.String() + "}\n" + got := strings.Fields(string(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly.String(), original))) + if len(got) != 2*len(frames) { + t.Fatal("original physical-FLAGS observation inventory changed", len(got)) + } + for index, frame := range frames { + first, second := got[2*index], got[2*index+1] + if frame == 0 { + if first != "1" || second != "0" { + t.Fatal("original zero-frame CALL/RET did not preserve both carry values", first, second) + } + continue + } + // Go can encode a 128-byte adjustment using the opposite operation + // and a negative imm8. Do not assume every physical frame clears CF; + // neither encoding preserves the preceding arithmetic input's CF. + if first != second || first != "0" && first != "1" { + t.Fatal("original automatic frame unexpectedly preserved arithmetic CF", frame, first, second) + } + } + t.Logf("FRAMED_LEAF_PHYSICAL_FLAGS original_only=true functions=%d inputs_per_function=2 original_observations=%d LLVM_comparisons=0", + len(frames), len(frames)*2) +} diff --git a/amd64_fixed_local_global.go b/amd64_fixed_local_global.go new file mode 100644 index 000000000..04bc6d833 --- /dev/null +++ b/amd64_fixed_local_global.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "go/ast" + "go/types" + "reflect" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gosym" +) + +// Keep native addresses opaque through the existing CFG, affine and spill +// transfers. A Go variable is not an FP slot, numeric bits or private storage. +type x86FixedLocalOrigin struct { + parameter int64 + global string + literal x86StaticSBAddress +} + +func fixedLocalGlobalSymbol(symbol string) string { + if !strings.ContainsAny(symbol, "·./") { + return "·" + symbol + } + return symbol +} + +func fixedLocalStaticAddress(ins Instr) (x86StaticSBAddress, bool) { + ins = x86SourceMoveInstruction(ins, "amd64") + if ins.Op != "LEAQ" && x86RawReturnScalarMoveBytes(ins.Op) != 8 || + !x86ResultNamedPredicateInstruction(ins) || len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return x86StaticSBAddress{}, false + } + root, valid := x86CallerGP(ins.Args[1].Reg) + if !valid || root != ins.Args[1].Reg || root == SP { + return x86StaticSBAddress{}, false + } + if ins.Op == "LEAQ" { + width, err := x86LEAOperandWidth(ins, "amd64") + address, valid := x86StaticSBSourceAddress(ins.Args[0], "amd64") + return address, err == nil && width == 8 && valid + } + address, immediate, valid := x86StaticAddressValueSource(ins.Args[0], "amd64") + return address, valid && immediate +} + +func fixedLocalGlobalSymbols(file *File) map[string]bool { + var symbols map[string]bool + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + address, valid := fixedLocalStaticAddress(ins) + if !valid || ins.Op == "LEAQ" { + continue + } + if symbols == nil { + symbols = make(map[string]bool) + } + symbols[fixedLocalGlobalSymbol(address.symbol)] = true + } + } + return symbols +} + +// Resolve only actual same-package variable candidates. The caller freezes +// callbacks outside its source lock; the same original typecheck below joins +// their complete numerical layout. No process, new source owner or initializer +// evaluation is needed, and no initializer bytes become LLVM constants. +func fixedLocalGlobalRequests(pkg GoPackage, file *File, resolve func(string) string) (map[string]goReadOnlyObject, error) { + var objects map[string]goReadOnlyObject + for symbol := range fixedLocalGlobalSymbols(file) { + name := goReadOnlyLocalName(pkg, symbol) + variable, exists := pkg.Types.Scope().Lookup(name).(*types.Var) + if !exists || variable.IsField() || variable.Pkg() != pkg.Types { + continue + } + native := gosym.PackagePrefix(pkg.Path) + "." + name + if resolve(symbol) != native { + return nil, goAMD64FrameContext("ordinary Go variable has a foreign native identity: %s", symbol) + } + if objects == nil { + objects = make(map[string]goReadOnlyObject) + } + objects[symbol] = goReadOnlyObject{native: native} + } + return objects, nil +} + +// Called inside the frame observer's one complete independent typecheck. +// Reuse the ordinary numerical-variable layout grammar, never just a name, +// supplied type, guessed extent, symbol address or the variable's initial value. +func joinFixedLocalGlobalDeclarations(actual, supplied *types.Package, syntax []*ast.File, sizes types.Sizes, objects map[string]goReadOnlyObject) error { + if len(objects) == 0 { + return nil + } + declarations := make(map[string]goReadOnlyObject, len(objects)) + for symbol, object := range objects { + name := goReadOnlyName(actual.Path(), symbol) + variable, found := actual.Scope().Lookup(name).(*types.Var) + previous, provided := supplied.Scope().Lookup(name).(*types.Var) + if !found || !provided || variable.IsField() || previous.IsField() || variable.Pkg() != actual { + return goAMD64FrameContext("ordinary address lacks its original Go variable: %s", symbol) + } + size, spans, shape, err := goReadOnlyLayout(variable.Type(), sizes) + priorSize, priorSpans, priorShape, priorErr := goReadOnlyLayout(previous.Type(), sizes) + if err != nil || priorErr != nil || size != priorSize || shape != priorShape || !reflect.DeepEqual(spans, priorSpans) { + return goAMD64FrameContext("ordinary Go variable differs from its complete original numerical type: %s", symbol) + } + object.variable, object.size, object.spans, object.shape = variable, size, spans, shape + objects[symbol], declarations[name] = object, object + } + return goReadOnlyCheckAliases(syntax, actual, declarations) +} diff --git a/amd64_fixed_local_global_runtime_test.go b/amd64_fixed_local_global_runtime_test.go new file mode 100644 index 000000000..1500c6579 --- /dev/null +++ b/amd64_fixed_local_global_runtime_test.go @@ -0,0 +1,80 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalMutableGlobal(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + goAMD64FramePrepareOwnedToolCache(t) + const original = `package main +import "fmt" +var Table [4]uint64 +func Value(a uint64) uint64 +func main() { + for i := uint64(0); i < 17; i++ { + for j := range Table { Table[j] = (i + uint64(j)) * 0x1020304050607081 } + fmt.Printf("%016x\n", Value(i)) + Table[0] ^= ^uint64(0) + fmt.Printf("%016x\n", Value(i)) + } +} +` + want := strings.Split(strings.TrimSpace(string(x86ScalarFloatIntegerOriginalOutput(t, execution, fixedLocalGlobalAssembly, original))), "\n") + if len(want) != 34 { + t.Fatalf("original mutable Go variable produced %d rows, want 34", len(want)) + } + prefix := "" + if strings.Contains(execution.triple, "apple") { + prefix = "_" + } + driver := runtimeCAsmLabelPreamble + fmt.Sprintf(`#include +#include +uint64_t table[4] __asm__(%s); +extern uint64_t value(uint64_t) __asm__(%s); +int main(void) { + for (uint64_t i = 0; i < 17; i++) { + for (unsigned j = 0; j < 4; j++) table[j] = (i + j) * UINT64_C(0x1020304050607081); + printf("%%016llx\n", (unsigned long long)value(i)); + table[0] ^= UINT64_MAX; + printf("%%016llx\n", (unsigned long long)value(i)); + } + return 0; +} +`, runtimeCAsmLabel(prefix+"example.invalid/framefixture.Table"), runtimeCAsmLabel(prefix+"example.invalid/framefixture.Value")) + pkg, request, _, options := fixedLocalTransportFixture(t, fixedLocalGlobalAssembly, + fixedLocalGlobalDeclarations, runtime.GOOS+"/amd64", execution.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + fixedLocalGlobalCheckExternal(t, ir) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + output := execution.compileAndRunLLVM(t, "mutable-global", input, driver, level) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM row count = %d, original = %d", len(got), len(want)) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("mutable global row %d: original %s; LLVM %s", index, want[index], got[index]) + } + } + }) + } + }) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("original Go and LLVM amd64 execution via Rosetta, not native x86 hardware") + } +} diff --git a/amd64_fixed_local_global_test.go b/amd64_fixed_local_global_test.go new file mode 100644 index 000000000..0ee1cf068 --- /dev/null +++ b/amd64_fixed_local_global_test.go @@ -0,0 +1,191 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" +) + +// A complete original Go definition supplies the variable's typed identity, +// not a constant value, an ASM DATA object or a private-frame byte extent. +func TestGoAMD64FixedLocalGlobalAddressObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + for _, register := range []string{"SI", "BP", "R14"} { + t.Run(register, func(t *testing.T) { + assembly := strings.NewReplacer("SI", register, "DX", register).Replace(fixedLocalGlobalAssembly) + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, + fixedLocalGlobalDeclarations, target.name, target.triple) + // Each API first compiles/captures this complete original Go + // source independently; only then does it bind and translate. + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("ordinary mutable global address and private frame must compose", err) + } + fixedLocalGlobalCheckExternal(t, ir) + compileLLVMToObject(t, llc, target.triple, "global.ll", "global.o", ir) + compileLLVMToObject(t, llc, target.triple, "global-opt.ll", "global-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + }) + } +} + +func fixedLocalGlobalCheckExternal(t *testing.T, ir string) { + t.Helper() + for _, line := range strings.Split(ir, "\n") { + if strings.HasPrefix(line, "@") && strings.Contains(line, "framefixture.Table") { + if !strings.Contains(line, "external global [32 x i8]") { + t.Fatal("ordinary mutable Go variable was not retained as external storage", line) + } + return + } + } + t.Fatal("ordinary mutable Go variable declaration missing") +} + +func TestGoAMD64FixedLocalGlobalAddressGuards(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, row := range []struct{ name, old, replacement string }{ + {"partial address", "MOVQ SI,8(SP)", "MOVL SI,8(SP)"}, + {"pointer arithmetic", "ADDQ $16,DX", "ADDQ SI,DX"}, + {"numeric address", "MOVQ $·Table(SB),SI", "MOVQ $1234,SI"}, + {"numeric result", "MOVQ AX,ret+8(FP)", "MOVQ SI,ret+8(FP)"}, + {"write role", "MOVOU (DX),X1", "MOVOU X0,(DX)"}, + } { + t.Run(row.name, func(t *testing.T) { + assembly := strings.Replace(fixedLocalGlobalAssembly, row.old, row.replacement, 1) + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + fixedLocalGlobalDeclarations, "linux/amd64", "x86_64-unknown-linux-gnu") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if row.name == "pointer arithmetic" || row.name == "numeric address" { + if err != nil { + t.Fatal("defined numerical address lost its ordinary native read", err) + } + proved, err := proveX86FixedLocal(file.Funcs[0], bound.Sigs[pkg.Path+".Value"]) + if err != nil { + t.Fatal(err) + } + fixedLocalRequireNumericNativeRole(t, proved, DX) + return + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("static address acquired unrelated numeric, write or private-frame permission", err) + } + }) + } + for _, declaration := range []string{"var Table [4]*uint64", "var Table [4]float64", "var Table [8]uint64"} { + t.Run("supplied_type/"+declaration, func(t *testing.T) { + pkg, request, file, options := fixedLocalTransportFixture(t, fixedLocalGlobalAssembly, + fixedLocalGlobalDeclarations, "linux/amd64", "x86_64-unknown-linux-gnu") + source := requireGoAMD64FrameCapture(t, pkg, request) + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "fixture.go", "package framefixture\n"+declaration+"\nfunc Value(a uint64) uint64\n", 0) + if err != nil { + t.Fatal(err) + } + config := types.Config{Sizes: types.SizesFor("gc", "amd64")} + typed, err := config.Check(pkg.Path, fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + pkg.Types, pkg.Syntax = typed, []*ast.File{syntax} + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("supplied variable type replaced the original complete Go source", err) + } + }) + } + t.Run("global and FP slot zero remain typed alternatives", func(t *testing.T) { + assembly := strings.Replace(fixedLocalGlobalAssembly, "$32-16", "$32-24", 1) + assembly = strings.Replace(assembly, "a+0(FP)", "a+8(FP)", 1) + assembly = strings.Replace(assembly, "ret+8(FP)", "ret+16(FP)", 1) + assembly = strings.Replace(assembly, "MOVQ SI,8(SP)", + "MOVQ p+0(FP),DI\nCMPQ AX,$0\nJEQ joined\nMOVQ DI,SI\njoined:\nMOVQ SI,8(SP)", 1) + declarations := "var Table [4]uint64\nfunc Value(p *uint64, a uint64) uint64\n" + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, + declarations, "linux/amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("distinct global/input alternatives lost their native read", err) + } + fixedLocalGlobalCheckExternal(t, ir) + compileLLVMToObject(t, llc, options.TargetTriple, "joined-global.ll", "joined-global.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "joined-global-opt.ll", "joined-global-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + + changed := strings.Replace(assembly, "joined:\nMOVQ SI,8(SP)", "joined:\nXORQ SI,AX\nMOVQ SI,8(SP)", 1) + if changed == assembly { + t.Fatal("machine-bit fixture lost its joined address operand") + } + assembly = changed + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "var Table [4]uint64\nfunc Value(p *uint64, a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); err != nil { + t.Fatal("original global/input machine-bit arithmetic rejected", err) + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, options.TargetTriple, "global-bits.ll", "global-bits.o", ir) + }) + }) + t.Run("bound symbol and lifetime", func(t *testing.T) { + pkg, request, file, options := fixedLocalTransportFixture(t, fixedLocalGlobalAssembly, + fixedLocalGlobalDeclarations, "linux/amd64", "x86_64-unknown-linux-gnu") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + changed := bound + changed.ResolveSym = func(symbol string) string { + if symbol == "·Table" { + return "foreign.Table" + } + return bound.ResolveSym(symbol) + } + if ir, err := Translate(file, changed); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("different variable inherited the original declaration", err) + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + if ir, err := Translate(file, bound); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("closed original source retained a static address role", err) + } + }) +} diff --git a/amd64_fixed_local_gp.go b/amd64_fixed_local_gp.go new file mode 100644 index 000000000..41216b076 --- /dev/null +++ b/amd64_fixed_local_gp.go @@ -0,0 +1,289 @@ +package plan9asm + +// Reuse the ordinary typed GP grammar. The descriptor only names byte and +// FLAGS dependencies; this consumer must still establish every private read, +// address origin and write. CMP/TEST retain their opposite immediate roles. +func fixedLocalGPByteEffect(ins Instr) (x86TypedByteEffect, bool, error) { + if effect, known, err := x86NamedLEAByteEffect(ins.Op, ins, "amd64"); known { + return effect, true, err + } + if effect, known, err := x86TypedGPByteEffect(ins.Op, ins, "amd64"); known { + return effect, true, err + } + if effect, known, err := x86TypedImplicitMultiplyByteEffect(ins.Op, ins, "amd64"); known { + return effect, true, err + } + form, known, err := x86ScalarCompareProperties(ins.Op, ins, "amd64") + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + effect := x86TypedByteEffect{flagsDefined: x86ByteFlagsAll, flagsKilled: x86ByteFlagsAll} + if form.test { + effect.flagsDefined &^= x86ByteFlagAF + } + for index, operand := range form.args { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: index, operand: operand, width: int64(form.bits / 8), read: true, + }) + } + return effect, true, nil +} + +// Preserve a full numerical word only after the ordinary typed transfer has +// succeeded. These are machine-word values with wraparound, not checked pointer +// offsets or termination facts. The existing family owns grammar and immediates; +// partial writes and numerical values depending on unknown carry stay unknown. +func fixedLocalConstantArithmetic(ins Instr, effect x86TypedByteEffect, state *x86FixedLocalState) (Reg, int64, bool) { + spec, recognized := x86LockScalarRMWSpecs[ins.Op] + if !recognized || spec.bits != 64 || len(effect.operands) == 0 { + return "", 0, false + } + destination := effect.operands[len(effect.operands)-1] + root, valid := x86CallerGP(destination.operand.Reg) + old, known := state.constants[root] + if destination.operand.Kind != OpReg || destination.width != 8 || !destination.write || + !valid || root != destination.operand.Reg || root == SP || + !known && spec.stem != "XOR" { + return "", 0, false + } + // The shared typed family already resolved aliases and encoded immediates. + // Its input-free XOR descriptor is the existing self-XOR zero rule. + if spec.stem == "XOR" && !effect.operands[0].read && !destination.read { + return root, 0, true + } + if !known || !fixedLocalNumericRead(destination.operand, 8, state) { + return "", 0, false + } + value, input := uint64(old), uint64(0) + if !spec.unary { + operand := effect.operands[0].operand + source, constant := operand.Imm, operand.Kind == OpImm + if operand.Kind == OpReg { + source, constant = state.constants[operand.Reg] + } + if !constant || !fixedLocalNumericRead(operand, 8, state) { + return "", 0, false + } + input = uint64(source) + } + switch spec.stem { + case "ADD": + value += input + case "SUB": + value -= input + case "AND": + value &= input + case "OR": + value |= input + case "XOR": + value ^= input + case "INC": + value++ + case "DEC": + value-- + case "NEG": + value = -value + case "NOT": + value = ^value + default: + return "", 0, false + } + return root, int64(value), true +} + +// A parsed address is not necessarily a memory access. Numerical LEA and +// sized NOP use the same syntax without loading/storing it. Query the validated +// effect rather than granting them a fabricated private-storage certificate. +func fixedLocalGPAddressOnly(ins Instr, index int) bool { + if index < 0 || index >= len(ins.Args) || ins.Args[index].Kind != OpMem { + return false + } + effect, known, err := fixedLocalGPByteEffect(ins) + if !known || err != nil || effect.dynamicMemory { + return false + } + for _, operand := range effect.operands { + if operand.index == index && (operand.operand.Kind == OpMem || operand.operand.Kind == OpSym) && + (operand.read || operand.write) { + return false + } + } + return true +} + +func fixedLocalTypedByteStep(ins *Instr, state *x86FixedLocalState, sig FuncSig, proof *x86FixedLocalProof, residue int, + effect x86TypedByteEffect) error { + fail := func() error { + return goAMD64FrameContext("fixed local typed effect lacks numerical bytes, FLAGS or private storage: %s", ins.Op) + } + if effect.dynamicMemory || state.flags&effect.flagsRead != effect.flagsRead { + return fail() + } + alignment := int64(1) + if effect.vector.upperWrite != x86TypedVectorUpperUnknown { + if effect.vector.upperWrite != x86TypedVectorUpperPreserve && effect.vector.upperWrite != x86TypedVectorUpperZero || + effect.vector.memoryAlignment <= 0 { + return fail() + } + alignment = effect.vector.memoryAlignment + } + // Read all inputs before any write, including aliased outputs. RMW receives + // one write-capable access certificate only after its old bytes are proved. + for _, operand := range effect.operands { + if operand.requiredBytes != 0 && (!operand.read || operand.operand.Kind != OpReg || + amd64VectorByteWidth(operand.operand.Reg) == 0 || + operand.requiredBytes&^x86CallerVectorByteMask(operand.width) != 0) { + return fail() + } + if !operand.read { + continue + } + if operand.operand.Kind != OpMem && operand.operand.Kind != OpSym { + if operand.operand.Kind == OpReg && amd64VectorByteWidth(operand.operand.Reg) != 0 { + mask := x86CallerVectorByteMask(operand.width) + if operand.requiredBytes != 0 { + mask = operand.requiredBytes + } + if effect.vector.upperWrite == x86TypedVectorUpperUnknown || mask == 0 || + state.vectors[x86CallerVectorRoot(operand.operand.Reg)]&mask != mask { + return fail() + } + continue + } + if !fixedLocalMachineRead(ins, operand, state, sig, proof, residue) { + return fail() + } + continue + } + if err := fixedLocalMachineMemory(ins, operand, state, proof, residue, alignment); err != nil { + return err + } + } + for _, operand := range effect.operands { + if !operand.write { + continue + } + switch operand.operand.Kind { + case OpMem: + offset, role, err := fixedLocalMemoryAccess(ins, operand.index, state, proof, residue, operand.width, alignment, true, false) + if err != nil { + return err + } + if role == x86FixedLocalPrivateMemory { + // An integer RMW drops an overlapping native-address role, + // but its untouched actual machine bytes remain initialized. + for start := offset - 7; start < offset+operand.width; start++ { + origin, found := state.opaqueCells[start] + if found && fixedLocalOriginOwners(origin, proof) { + if start < 0 || start > proof.frame-8 { + return fail() + } + state.bytes.setRange(start, 8, true) + } + } + fixedLocalNumericalStore(state, offset, operand.width, true) + } + case OpReg: + if amd64VectorByteWidth(operand.operand.Reg) != 0 { + mask := x86CallerVectorByteMask(operand.width) + if effect.vector.upperWrite == x86TypedVectorUpperUnknown || mask == 0 { + return fail() + } + root := x86CallerVectorRoot(operand.operand.Reg) + if effect.vector.upperWrite == x86TypedVectorUpperZero { + state.vectors[root] = ^uint64(0) + } else { + state.vectors[root] |= mask + } + continue + } + root, valid := x86CallerGP(operand.operand.Reg) + mask := x86CallerByteMask(operand.operand.Reg, operand.width) + _, private := state.pointers[root] + origin, opaque := state.opaque[root] + if !valid || root == SP || mask == 0 || + private && operand.width != 8 { + return fail() + } + if opaque && fixedLocalOriginOwners(origin, proof) { + state.known[root] = 255 // Narrow writes preserve defined native upper bits. + } + if operand.width == 4 { + mask = 255 // Architectural dword writes define the upper half. + } + delete(state.pointers, root) + delete(state.opaque, root) + delete(state.constants, root) + fixedLocalWriteBooleans(state, operand.operand.Reg, operand.width, 0) + state.known[root] |= mask + default: + return fail() // FP transport and external stores have separate owners. + } + } + state.flags = state.flags&^effect.flagsKilled | effect.flagsDefined + if effect.flagsKilled|effect.flagsDefined != 0 { + state.comparison = x86FixedLocalComparison{} + } + return nil +} + +// Integer effects consume machine bits. Native addresses have all eight bytes; +// relocated private-frame identities do not. This does not change the stricter +// address-index, scalar transport or result-ABI predicates. +func fixedLocalMachineRead(ins *Instr, operand x86TypedByteOperand, state *x86FixedLocalState, + sig FuncSig, proof *x86FixedLocalProof, residue int) bool { + if amd64IsKOperand(operand.operand) { + return operand.width == 8 && state.masks[operand.operand.Reg] + } + if operand.operand.Kind == OpFP { + value, err := fixedLocalReadValue(ins, operand.index, state, sig, proof, residue, operand.width, false) + return err == nil && !value.pointer && (!value.opaque || fixedLocalOriginOwners(value.identity, proof)) + } + if fixedLocalNumericRead(operand.operand, operand.width, state) { + return true + } + root, valid := x86CallerGP(operand.operand.Reg) + origin, native := state.opaque[root] + _, private := state.pointers[root] + return operand.operand.Kind == OpReg && valid && root != SP && !private && native && + x86CallerByteMask(operand.operand.Reg, operand.width) != 0 && fixedLocalOriginOwners(origin, proof) +} + +func fixedLocalMachineMemory(ins *Instr, operand x86TypedByteOperand, state *x86FixedLocalState, + proof *x86FixedLocalProof, residue int, alignment int64) error { + if operand.operand.Kind == OpSym { + _, _, err := fixedLocalReadMemory(ins, operand.index, state, proof, residue, operand.width, alignment) + return err + } + address, err := fixedLocalMemoryAddress(*ins, operand.index, proof.frame) + if err != nil { + return err + } + offset, private := fixedLocalPrivateOffset(address, state) + if !private { + _, _, err := fixedLocalMemoryAccess(ins, operand.index, state, proof, residue, operand.width, alignment, operand.write, false) + return err + } + if _, err := fixedLocalMemory(ins, operand.index, state, proof, residue, operand.width, alignment, operand.write, true); err != nil { + return err + } + for index := offset; index < offset+operand.width; index++ { + if state.bytes.has(index) { + continue + } + defined := false + // Full pointer spills need not be aligned. Exactly eight possible + // starting offsets can contain a byte, independent of frame size. + for start := index - 7; start <= index; start++ { + origin, found := state.opaqueCells[start] + if found && start >= 0 && start <= proof.frame-8 && fixedLocalOriginOwners(origin, proof) { + defined = true + break + } + } + if !defined { + return goAMD64FrameContext("fixed local machine read contains undefined or relocated private bytes") + } + } + return nil +} diff --git a/amd64_fixed_local_gp_bank_test.go b/amd64_fixed_local_gp_bank_test.go new file mode 100644 index 000000000..a09c1a275 --- /dev/null +++ b/amd64_fixed_local_gp_bank_test.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// ABI0 does not give R14 an initialized g value. Nonzero original Go frames +// independently save/restore BP outside the body. Both registers can therefore +// hold explicitly defined virtual values, without exposing their entry state. +func TestGoAMD64FixedLocalDefinedGPBank(t *testing.T) { + var original strings.Builder + ordinal := 0 + for _, register := range []string{"AX", "BX", "CX", "DX", "SI", "DI", "BP", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + for _, width := range []string{"B", "W", "L", "Q"} { + body := fmt.Sprintf("MOVQ AX,%s\nADD%s $1,%s\nMOVQ %s,AX", register, width, register, register) + fn, sig, source := fixedLocalGPFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", ordinal), 1)) + ordinal++ + t.Run(register+"/"+width, func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("explicitly defined ABI0 register rejected", err) + } + }) + } + } + if ordinal != 60 { + t.Fatal("incomplete independent GP bank/width matrix", ordinal) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) + for _, register := range []string{"BP", "R14"} { + for index, body := range []string{ + "MOVQ " + register + ",AX", + "MOVB $0," + register + "\nMOVQ " + register + ",AX", + "MOVQ (" + register + "),AX", + "MOVQ SP," + register + "\nMOVQ " + register + ",AX", + "MOVQ $1," + register + "\nMOVQ (" + register + "),AX", + } { + t.Run(fmt.Sprintf("unbound/%s/%d", register, index), func(t *testing.T) { + fn, sig, _ := fixedLocalGPFixture(t, body) + proved, err := proveX86FixedLocal(fn, sig) + if index == 4 { + if err != nil { + t.Fatal("defined numerical bits lost their faulting native access", err) + } + fixedLocalRequireNumericNativeRole(t, proved, Reg(register)) + return + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("entry register or numerical bits gained pointer authority", err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalDefinedGPBankRoles(t *testing.T) { + var original strings.Builder + ordinal := 0 + for _, register := range []string{"BP", "R14"} { + for index, operation := range []string{ + "MOVQ AX,%s\nCMPQ AX,AX\nSETEQ %s\nMOVQ %s,AX", + "MOVQ AX,%s\nCMPQ AX,AX\nCMOVQEQ AX,%s\nMOVQ %s,AX", + "KMOVQ AX,K1\nKMOVQ K1,%s\nADDQ $1,%s\nMOVQ %s,AX", + } { + body := fmt.Sprintf(operation, register, register, register) + fn, sig, source := fixedLocalGPFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", ordinal), 1)) + ordinal++ + t.Run(fmt.Sprintf("%s/numeric/%d", register, index), func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("defined register rejected by an independent effect consumer", err) + } + }) + } + for index, operation := range []string{ + "MOVQ SI,%s\nADDQ AX,%s\nMOVQ (%s),AX", + "LEAQ 8(SI),%s\nADDQ AX,%s\nMOVQ (%s),AX", + } { + body := fmt.Sprintf(operation, register, register, register) + fn, sig, source := fixedLocalInputFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", ordinal), 1)) + ordinal++ + t.Run(fmt.Sprintf("%s/input/%d", register, index), func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("explicit typed input origin rejected", err) + } + }) + } + pkg, _, file, options := fixedLocalVoidFixture(t, "PADDQ 16(SP),X0") + source := strings.Replace(fixedLocalVoidAssembly(file), "MOVQ AX,16(SP)", "MOVQ AX,"+register+"\nMOVQ AX,16(SP)", 1) + parsed, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + t.Run(register+"/void", func(t *testing.T) { + if _, err := proveX86FixedLocal(parsed.Funcs[0], options.Sigs[pkg.Path+".Value"]); err != nil { + t.Fatal("void source rejected a defined register", err) + } + }) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", ordinal), 1)) + ordinal++ + } + if ordinal != 12 { + t.Fatal("incomplete independent role matrix", ordinal) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} diff --git a/amd64_fixed_local_gp_runtime_test.go b/amd64_fixed_local_gp_runtime_test.go new file mode 100644 index 000000000..be355c2be --- /dev/null +++ b/amd64_fixed_local_gp_runtime_test.go @@ -0,0 +1,231 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// These numerical kernels exercise the same original amd64 program on both +// sides. In particular, a host-target rewrite cannot stand in for x86 FLAGS, +// partial memory writes or the original Go-generated frame. +func fixedLocalGPRuntimeProgram(triple string) (string, string, string, string) { + var bodies []string + for _, width := range []string{"B", "W", "L", "Q"} { + for _, stem := range []string{"ADD", "SUB", "AND", "OR", "XOR", "ADC", "SBB"} { + bodies = append(bodies, "STC\n"+stem+width+" $1,0(SP)\n") + } + } + for _, stem := range []string{"ADD", "SUB", "AND", "OR", "XOR", "ADC", "SBB"} { + bodies = append(bodies, "STC\n"+stem+"Q a+0(FP),AX\nMOVQ AX,0(SP)\n") + } + for _, operation := range []string{ + "CMPQ a+0(FP),AX", "CMPQ AX,a+0(FP)", "CMPQ a+0(FP),$1", + "TESTQ a+0(FP),AX", "TESTQ AX,a+0(FP)", "TESTQ $1,a+0(FP)", + } { + bodies = append(bodies, operation+"\n") + } + for _, condition := range []string{ + "CC", "CS", "EQ", "GE", "GT", "HI", "LE", "LS", + "LT", "MI", "NE", "OC", "OS", "PC", "PL", "PS", + } { + bodies = append(bodies, "MOVQ $42,CX\nCMPQ a+0(FP),$1\nCMOVQ"+condition+ + " a+0(FP),CX\nMOVQ CX,0(SP)\n") + } + for _, body := range []string{ + "MOVQ $3,CX\nagain:\nADDQ CX,AX\nSUBQ $1,CX\nJNE again\n", + "MOVQ $3,16(SP)\nagain:\nADDQ $1,AX\nSUBQ $1,16(SP)\nJNE again\n", + "CMPQ AX,AX\nMOVQ AX,CX\nLEAQ 7(AX)(CX*2),AX\nNOPL 24(SP)\n", + "CMPQ AX,$1\nJEQ left\nMOVQ $2,CX\nJMP join\nleft:\nMOVQ $3,CX\njoin:\nADDQ CX,AX\n", + } { + bodies = append(bodies, body+"MOVQ AX,0(SP)\n") + } + for _, register := range []string{"AX", "BX", "CX", "DX", "SI", "DI", "BP", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + for _, width := range []string{"B", "W", "L", "Q"} { + bodies = append(bodies, fmt.Sprintf("MOVQ AX,%s\nADD%s $1,%s\nMOVQ %s,AX\nMOVQ AX,0(SP)\n", register, width, register, register)) + } + } + + var assembly, declarations, goRows, cRows, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, body := range bodies { + name := fmt.Sprintf("Value%d", index) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$64-24 +MOVQ a+0(FP),AX +MOVQ AX,0(SP) +MOVQ AX,8(SP) +%s`, name, body) + // MOV and SETcc preserve FLAGS. Capture every defined arithmetic flag + // before the shifts/ORs used to pack them; AF is undefined for logic. + assembly.WriteString(`MOVQ $0,CX +MOVQ $0,DX +MOVQ $0,BX +MOVQ $0,R8 +MOVQ $0,R9 +SETCS CL +SETPS DL +SETEQ BL +SETMI R8 +SETOS R9 +SHLQ $1,DX +SHLQ $2,BX +SHLQ $3,R8 +SHLQ $4,R9 +ORQ DX,CX +ORQ BX,CX +ORQ R8,CX +ORQ R9,CX +MOVQ mode+8(FP),DX +CMPQ DX,$1 +JEQ flags +CMPQ DX,$2 +JEQ canary +MOVQ 0(SP),AX +JMP done +flags: +MOVQ CX,AX +JMP done +canary: +MOVQ 8(SP),AX +done: +MOVQ AX,ret+16(FP) +RET +`) + fmt.Fprintf(&declarations, "func %s(a, mode uint64) uint64\n", name) + fmt.Fprintf(&goRows, "%s,\n", name) + fmt.Fprintf(&cRows, "value%d,\n", index) + label := runtimeCAsmLabel(prefix + "example.invalid/framefixture." + name) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t, uint64_t) __asm__(%s);\n", index, label) + } + original := `package main +import "fmt" +` + declarations.String() + ` +func main() { + rows := []func(uint64, uint64) uint64{ +` + goRows.String() + ` + } + values := []uint64{ + 0, 1, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, + 0x7fffffff, 0x80000000, 0xffffffff, + 0x7fffffffffffffff, 0x8000000000000000, ^uint64(0), 0x8080808080808080, + } + for n, call := range rows { + for v, value := range values { + for mode := uint64(0); mode < 3; mode++ { + got := call(value, mode) + if mode == 2 && got != value { + panic("original private-frame canary changed") + } + fmt.Printf("%d %d %d %016x\n", n, v, mode, got) + } + } + } +} +` + driver := `#include +#include +` + runtimeCAsmLabelPreamble + cDeclarations.String() + ` +int main(void) { + uint64_t (*const rows[])(uint64_t, uint64_t) = { +` + cRows.String() + ` + }; + const uint64_t values[] = { + 0, 1, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, + 0x7fffffff, UINT64_C(0x80000000), UINT64_C(0xffffffff), + UINT64_C(0x7fffffffffffffff), UINT64_C(0x8000000000000000), + UINT64_MAX, UINT64_C(0x8080808080808080), + }; + for (unsigned n = 0; n < sizeof(rows)/sizeof(rows[0]); n++) { + for (unsigned v = 0; v < sizeof(values)/sizeof(values[0]); v++) { + for (unsigned mode = 0; mode < 3; mode++) { + uint64_t got = rows[n](values[v], mode); + if (mode == 2 && got != values[v]) { + return 2; + } + printf("%u %u %u %016llx\n", n, v, mode, (unsigned long long)got); + } + } + } + return 0; +} +` + return assembly.String(), declarations.String(), original, driver +} + +func TestGoAMD64FixedLocalTypedGPRuntimeObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + const triple = "x86_64-unknown-linux-gnu" + assembly, declarations, _, _ := fixedLocalGPRuntimeProgram(triple) + pkg, request, _, options := fixedLocalTransportFixture(t, + assembly, declarations, "linux/amd64", triple) + var names []string + for index := 0; index < 121; index++ { + names = append(names, fmt.Sprintf("%s.Value%d", pkg.Path, index)) + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + x86CompactExpandRequireAllDefinitions(t, input, names) + x86PackedStringCompileObject(t, llc, triple, input, level) + } + }) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalTypedGP(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + // Each API owns a fresh source observation, not a reused proof. Compile + // original Go and prepare objdump before opening any observation lease. + t.Setenv("GOCACHE", t.TempDir()) + assembly, declarations, original, driver := fixedLocalGPRuntimeProgram(execution.triple) + want := strings.Split(strings.TrimSpace(string(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original))), "\n") + if len(want) != 121*15*3 { + t.Fatalf("original Go produced %d rows, want 121 kernels x 15 values x 3 observations", len(want)) + } + pkg, request, _, options := fixedLocalTransportFixture(t, + assembly, declarations, runtime.GOOS+"/amd64", execution.triple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + output := execution.compileAndRunLLVM(t, "private-gp", input, driver, level) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM row count = %d, original = %d", len(got), len(want)) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("private GP value/FLAGS/canary row %d: original %s; LLVM %s", index, want[index], got[index]) + } + } + }) + } + }) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("original Go and LLVM amd64 execution via Rosetta, not native x86 hardware") + } +} diff --git a/amd64_fixed_local_gp_simd.go b/amd64_fixed_local_gp_simd.go new file mode 100644 index 000000000..cd6acecca --- /dev/null +++ b/amd64_fixed_local_gp_simd.go @@ -0,0 +1,59 @@ +package plan9asm + +// The complete GP/X grammar owns widths and aliases. This consumer supplies +// only the existing numerical register, private/native memory and FP roles; +// it does not turn relocated or opaque addresses into numerical vector bytes. +func fixedLocalGPSIMDTransferStep(ins *Instr, state *x86FixedLocalState, sig FuncSig, + proof *x86FixedLocalProof, residue int) (bool, error) { + form, recognized, err := x86GPSIMDTransferProperties(*ins, "amd64") + if !recognized || err != nil { + return recognized, err + } + fail := func() (bool, error) { + return true, goAMD64FrameContext("fixed local GP/X transfer lacks numerical bytes or a bound storage role") + } + if form.mmx || form.spec.rawOnly { + return fail() + } + width := int64(form.bits / 8) + low := x86CallerVectorByteMask(width) + var known uint64 + if form.sourceX { + known = state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] & low + } else if form.memory < 0 { + root, valid := x86CallerGP(ins.Args[0].Reg) + _, private := state.pointers[root] + _, opaque := state.opaque[root] + if ins.Args[0].Kind != OpReg || !valid || root == SP || private || opaque { + return fail() + } + known = uint64(state.known[root]) & low + } else { + value, err := fixedLocalReadValue(ins, 0, state, sig, proof, residue, width, false) + if err != nil { + return true, err + } + if value.pointer || value.opaque { + return fail() + } + known = low // Memory and FP inputs require their complete numerical role. + } + if form.destinationX { + root := x86CallerVectorRoot(ins.Args[1].Reg) + state.vectors[root] = form.definedVectorBytes(known, state.vectors[root]) + return true, nil + } + if ins.Args[1].Kind != OpReg && known != low { + return fail() // An unknown stored byte cannot initialize memory or FP. + } + if err := fixedLocalWriteValue(ins, 1, 0, fixedLocalScalarValue{}, state, sig, proof, residue, width); err != nil { + return true, err + } + if ins.Args[1].Kind == OpReg { + root, _ := x86CallerGP(ins.Args[1].Reg) + // Replace old facts even for unknown inputs; dword GP writes define + // the upper dword as zero. The shared writer revoked address/value facts. + state.known[root] = uint8(known) | ^uint8(low) + } + return true, nil +} diff --git a/amd64_fixed_local_gp_simd_native_linux_amd64_test.go b/amd64_fixed_local_gp_simd_native_linux_amd64_test.go new file mode 100644 index 000000000..d67b606ef --- /dev/null +++ b/amd64_fixed_local_gp_simd_native_linux_amd64_test.go @@ -0,0 +1,37 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalGPSIMD(t *testing.T) { + var kernels []fixedLocalVEXRuntimeKernel + for _, family := range fixedLocalGPSIMDNames { + operations := []string{ + family.name + " AX,X1", + family.name + " X0,BX\nMOVQ BX,72(BP)", + family.name + " 65(BP),X1", + family.name + " X0,65(BP)", + family.name + " 1(SI),X1", + family.name + " X0,1(SI)", + } + if family.bits == 64 { + operations = append(operations, family.name+" X0,X1", family.name+" X1,X1") + } + for index, operation := range operations { + // Observe the whole Y destination, all private canaries, native + // input stores and dword GP zero-extension. SETCS observes the + // original carry without an instruction which changes FLAGS. + operations[index] = "MOVQ (SI),AX\nMOVQ $-1,BX\nSTC\n" + operation + + "\nSETCS R8\nMOVBQZX R8,R8\nMOVQ R8,88(BP)" + } + kernels = append(kernels, fixedLocalVEXRuntimeKernel{family.name, operations}) + } + if len(kernels) != 6 { + t.Fatal("complete low-bank native transfer inventory changed", len(kernels)) + } + // AVX2 is required for the independent driver to observe preserved upper + // Y aliases. X31/EVEX objects have a separate hardware acceptance boundary. + fixedLocalVectorRuntime(t, fixedLocalVEXNativeTools(t), true, kernels) + if !t.Failed() { + t.Log("GP_SIMD_NATIVE_COMPLETE families=6 inputs_per_family=16 APIs=3 LLVM_levels=O0,O2 original_observations=96 LLVM_comparisons=576") + } +} diff --git a/amd64_fixed_local_gp_simd_test.go b/amd64_fixed_local_gp_simd_test.go new file mode 100644 index 000000000..81276f7be --- /dev/null +++ b/amd64_fixed_local_gp_simd_test.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// Independent Go spellings: MOVD and MOVDQ2Q are MOVQ aliases. The MMX row +// and decoder-only VMOVW retain their separate physical-state contracts. +var fixedLocalGPSIMDNames = []struct { + name string + bits int + legacy bool +}{ + {"MOVL", 32, true}, {"MOVQ", 64, true}, + {"MOVD", 64, true}, {"MOVDQ2Q", 64, true}, + {"VMOVD", 32, false}, {"VMOVQ", 64, false}, +} + +type fixedLocalGPSIMDCase struct { + name, body string + result string +} + +func fixedLocalGPSIMDCases() []fixedLocalGPSIMDCase { + var rows []fixedLocalGPSIMDCase + for _, family := range fixedLocalGPSIMDNames { + banks := []string{"X1"} + if !family.legacy { + banks = append(banks, "X31") + } + for _, bank := range banks { + seed, extract := "MOVOU 16(SP),"+bank, "MOVQ "+bank+",AX" + if bank == "X31" { + seed, extract = "VMOVQ AX,"+bank, "VMOVQ "+bank+",AX" + } + add := func(label, body, result string) { + rows = append(rows, fixedLocalGPSIMDCase{family.name + "/" + bank + "/" + label, body, result}) + } + add("GP to X", family.name+" AX,"+bank+"\n"+extract, "uint64") + add("X to GP", seed+"\n"+family.name+" "+bank+",AX", "uint64") + for _, address := range []string{"16(SP)", "17(SP)", "1(SI)"} { + add("load "+address, family.name+" "+address+","+bank+"\n"+extract, "uint64") + add("store "+address, seed+"\n"+family.name+" "+bank+","+address, "uint64") + } + input := "a+8(FP)" + if family.bits == 32 { + input = "b+16(FP)" + } + add("FP input", family.name+" "+input+","+bank+"\n"+extract, "uint64") + add("FP result", seed+"\n"+family.name+" "+bank+",ret+24(FP)", fmt.Sprintf("uint%d", family.bits)) + if family.bits == 64 { + add("X copy", family.name+" X0,"+bank+"\n"+extract, "uint64") + add("X self", seed+"\n"+family.name+" "+bank+","+bank+"\n"+extract, "uint64") + } + } + } + return rows +} + +func fixedLocalGPSIMDFixture(t *testing.T, row fixedLocalGPSIMDCase) (Func, FuncSig, string) { + t.Helper() + body := "MOVQ p+0(FP),SI\nMOVQ a+8(FP),AX\nMOVL b+16(FP),BX\n" + + "MOVQ AX,16(SP)\nMOVQ AX,24(SP)\nMOVOU 16(SP),X0\n" + row.body + "\n" + if !strings.HasSuffix(row.name, "/FP result") { + body += "MOVQ AX,ret+24(FP)\n" + } + _, _, sig, source := x86ResultInitializationFixture(t, "amd64", + "func Value(p *uint64, a uint64, b uint32) "+row.result, body+"RET") + source = strings.Replace(source, "$0-", "$80-", 1) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + return file.Funcs[0], sig, source +} + +func TestGoAMD64FixedLocalGPSIMDCompleteSources(t *testing.T) { + rows := fixedLocalGPSIMDCases() + if len(rows) != 90 { + t.Fatal("complete GP/X source inventory changed", len(rows)) + } + var original strings.Builder + for index, row := range rows { + _, _, source := fixedLocalGPSIMDFixture(t, row) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + // An original assembler/fixture failure is not a translator rejection. + requireX86GoAssemblerResult(t, "amd64", original.String(), true) + for _, row := range rows { + t.Run(row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalGPSIMDFixture(t, row) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("original complete GP/X transfer lacks fixed-local support", err) + } + }) + } +} + +func TestGoAMD64FixedLocalGPSIMDCompleteObjects(t *testing.T) { + var assembly, declarations strings.Builder + rows := fixedLocalGPSIMDCases() + for index, row := range rows { + _, _, source := fixedLocalGPSIMDFixture(t, row) + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(source, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p *uint64, a uint64, b uint32) %s\n", name, row.result) + } + fixedLocalCompleteVectorObjects(t, assembly.String(), declarations.String(), len(rows)) +} + +func TestGoAMD64FixedLocalGPSIMDDefinedBytes(t *testing.T) { + for _, family := range fixedLocalGPSIMDNames { + for _, direction := range []string{"GP to X", "X to GP", "X self"} { + if direction == "X self" && family.bits == 32 { + continue + } + t.Run(family.name+"/"+direction, func(t *testing.T) { + width := family.bits / 8 + low := uint64(1)<(SB)"}, + {"MMX needs physical owner", "MOVQ X0,M0\nEMMS"}, + } { + t.Run(family.name+"/"+row.name, func(t *testing.T) { + fn, sig, source := fixedLocalGPSIMDFixture(t, fixedLocalGPSIMDCase{body: row.body, result: "uint64"}) + requireX86GoAssemblerResult(t, "amd64", source, true) + if row.name == "numeric bits are not address" { + fixedLocalRequireNumericControl(t, fn, sig, source, "func Value(p *uint64,a uint64,b uint32) uint64", + "MOVQ (CX),AX", 0, 8, false, "load volatile i64") + return + } + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("numeric transfer acquired missing bytes or another binding role", err) + } + }) + } + } +} diff --git a/amd64_fixed_local_gp_test.go b/amd64_fixed_local_gp_test.go new file mode 100644 index 000000000..b3c1bb393 --- /dev/null +++ b/amd64_fixed_local_gp_test.go @@ -0,0 +1,260 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalGPFixture(t *testing.T, body string) (Func, FuncSig, string) { + t.Helper() + _, _, sig, source := x86ResultInitializationFixture(t, "amd64", "func Value(a uint64) uint64", + "MOVQ a+0(FP),AX\nMOVQ AX,0(SP)\n"+body+"\nMOVQ AX,ret+8(FP)\nRET") + source = strings.Replace(source, "$0-16", "$64-16", 1) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + return file.Funcs[0], sig, source +} + +func TestGoAMD64FixedLocalTypedGPWidths(t *testing.T) { + var original strings.Builder + count := 0 + // Independent family/width axes. Every memory RMW must read initialized + // numerical bytes before writing; the syntax oracle is Go's assembler. + for _, width := range []string{"B", "W", "L", "Q"} { + for _, stem := range []string{"ADD", "SUB", "AND", "OR", "XOR", "ADC", "SBB"} { + for _, destination := range []string{"AX", "0(SP)"} { + body := "CMPQ AX,AX\n" + stem + width + " $1," + destination + fn, sig, source := fixedLocalGPFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", count), 1)) + count++ + t.Run(stem+width+"/"+destination, func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal(err) + } + }) + } + } + for _, stem := range []string{"NEG", "NOT", "INC", "DEC"} { + for _, destination := range []string{"AX", "0(SP)"} { + body := stem + width + " " + destination + fn, sig, source := fixedLocalGPFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", count), 1)) + count++ + t.Run(stem+width+"/"+destination, func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal(err) + } + }) + } + } + } + if count != 88 { + t.Fatalf("family/width fixture count changed: %d", count) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalTypedGPFlagsAndProvenance(t *testing.T) { + for _, row := range []struct { + name, body string + valid bool + }{ + {"carry only", "STC\nADCQ $1,AX", true}, + {"NOP preserves carry", "STC\nNOP\nADCQ $1,AX", true}, + {"NOP does not define carry", "NOP\nADCQ $1,AX", false}, + {"NOP does not define registers", "NOP\nADDQ CX,AX", false}, + {"carry branch", "STC\nJCS done\ndone:", true}, + {"carry is not zero", "STC\nJEQ done\ndone:", false}, + {"carry is not set zero", "STC\nSETEQ CL", false}, + {"carry is not conditional zero", "STC\nMOVQ AX,CX\nCMOVQEQ CX,AX", false}, + {"undefined carry", "ADCQ $1,AX", false}, + {"compare defines AH flags", "CMPQ AX,AX\nLAHF", true}, + {"test does not define AF", "TESTQ AX,AX\nLAHF", false}, + {"ADX does not define zero", "STC\nADCXQ AX,AX\nJEQ done\ndone:", false}, + {"multiply defines carry", "IMULQ $3,AX\nJCS done\ndone:", true}, + {"multiply kills old zero", "CMPQ AX,AX\nIMULQ $3,AX\nJEQ done\ndone:", false}, + {"test memory", "TESTQ $1,0(SP)\nJEQ done\ndone:", true}, + {"compare memory", "CMPQ 0(SP),$1\nJEQ done\ndone:", true}, + {"self xor initializes", "XORQ CX,CX\nADDQ CX,AX", true}, + {"dword zero extension", "XORL CX,CX\nADDQ CX,AX", true}, + {"word not zero extension", "XORW CX,CX\nADDQ CX,AX", false}, + {"uninitialized register", "ADDQ CX,AX", false}, + {"uninitialized memory", "SUBQ $1,8(SP)", false}, + {"uninitialized self memory", "XORQ AX,8(SP)", false}, + {"crossing frame", "ADDQ $1,60(SP)", false}, + {"numeric not a pointer", "ADDQ $1,(AX)", true}, + {"dynamic memory bit index", "BTQ AX,0(SP)", false}, + {"spilled pointer is not numeric", "MOVQ SP,CX\nMOVQ CX,8(SP)\nADDQ $1,8(SP)", false}, + {"private not numerical", "MOVQ SP,CX\nXORQ CX,AX", false}, + {"partial private overwrite", "MOVQ SP,CX\nXORW CX,CX", false}, + {"stack overwrite", "XORQ SP,SP", false}, + {"unknown suffix", "ADDQ.Z $1,AX", false}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalGPFixture(t, row.body) + proved, err := proveX86FixedLocal(fn, sig) + if (err == nil) != row.valid { + t.Fatalf("valid=%v: %v", row.valid, err) + } + if row.name == "numeric not a pointer" { + fixedLocalRequireNumericNativeRole(t, proved, AX) + } + }) + } +} + +func TestGoAMD64FixedLocalTypedGPActualObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var assembly, declarations strings.Builder + for index, body := range []string{ + "STC\nNOP\nADCQ $1,AX", + "ADDQ $1,AX\nSUBW $1,0(SP)\nMOVQ 0(SP),AX", + "STC\nADCQ $1,AX\nSBBL $1,0(SP)\nMOVQ 0(SP),AX", + "NOTQ AX\nNEGB 0(SP)\nINCW 0(SP)\nDECL 0(SP)\nMOVQ 0(SP),AX", + "TESTQ $1,0(SP)\nCMOVQEQ 0(SP),AX\nSETCS CL", + "MOVQ AX,CX\nIMULQ $3,CX\nADDQ CX,AX", + "CMPQ AX,AX\nLAHF\nSAHF\nSHLQ $1,0(SP)\nMOVQ 0(SP),AX", + "MOVQ AX,CX\nLEAQ 7(AX)(CX*2),AX\nNOPL 8(SP)", + "MOVQ $3,CX\nagain:\nADDQ CX,AX\nSUBQ $1,CX\nJNE again", + "MOVQ $3,8(SP)\nagain:\nADDQ $1,AX\nSUBQ $1,8(SP)\nJNE again", + "CMPQ AX,$1\nJEQ left\nMOVQ $2,CX\nJMP join\nleft:\nMOVQ $3,CX\njoin:\nADDQ CX,AX", + } { + _, _, source := fixedLocalGPFixture(t, body) + assembly.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + fmt.Fprintf(&declarations, "func Value%d(a uint64) uint64\n", index) + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly.String(), declarations.String(), target.name, target.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "typed-gp.ll", "typed-gp.o", ir) + }) + }) + } +} + +func TestGoAMD64FrameSourceLocalBranches(t *testing.T) { + for _, row := range []struct { + name, body string + valid bool + }{ + {"forward", "JMP done\ndone:\nRET", true}, + {"backward", "again:\nSUBQ $1,AX\nJNE done\nJMP again\ndone:\nRET", true}, + {"missing", "JMP absent\nRET", false}, + {"past end", "JMP done\nRET\ndone:", false}, + {"external", "JMP ·Other(SB)\nRET", false}, + {"indirect", "JMP AX\nRET", false}, + {"relative", "JMP 2(PC)\nRET", false}, + {"call", "CALL ·Other(SB)\nRET", false}, + {"tail return", "RET ·Other(SB)", false}, + {"raw", "BYTE $0xc3\nRET", false}, + {"alignment", "PCALIGN $16\nRET", false}, + } { + t.Run(row.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$16-0\n"+row.body+"\n") + if err != nil || len(file.Funcs) != 1 { + t.Fatalf("parse original source: %v", err) + } + err = goAMD64FrameSourceInstructions(file.Funcs[0]) + if (err == nil) != row.valid { + t.Fatalf("valid=%v: %v", row.valid, err) + } + }) + } +} + +func TestGoAMD64FixedLocalTypedGPLoopAndJoin(t *testing.T) { + for _, row := range []struct { + name, body string + valid bool + }{ + {"numeric counter", "MOVQ $3,CX\nagain:\nADDQ CX,AX\nSUBQ $1,CX\nJNE again", true}, + {"private counter", "MOVQ $3,8(SP)\nagain:\nADDQ $1,AX\nSUBQ $1,8(SP)\nJNE again", true}, + {"numeric constants join", "CMPQ AX,$1\nJEQ left\nMOVQ $2,CX\nJMP join\nleft:\nMOVQ $3,CX\njoin:\nADDQ CX,AX", true}, + {"missing first iteration", "again:\nADDQ CX,AX\nMOVQ $3,CX\nJNE again", false}, + {"flags lost on backedge", "CMPQ AX,AX\nagain:\nSETNE CL\nIMULQ $3,AX\nJCS again", false}, + {"drifting private address", "MOVQ SP,CX\nagain:\nADDQ $8,CX\nCMPQ AX,AX\nJNE again", false}, + {"numeric becomes pointer", "MOVQ AX,CX\nagain:\nADDQ CX,AX\nMOVQ SP,CX\nCMPQ AX,AX\nJNE again", false}, + {"missing byte on one path", "CMPQ AX,$1\nJEQ done\nMOVQ AX,8(SP)\ndone:\nMOVQ 8(SP),AX", false}, + {"no reachable return", "again:\nJMP again", false}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalGPFixture(t, row.body) + _, err := proveX86FixedLocal(fn, sig) + if (err == nil) != row.valid { + t.Fatalf("valid=%v: %v", row.valid, err) + } + }) + } +} + +func TestGoAMD64FixedLocalTypedGPAddressIsNotAccess(t *testing.T) { + for _, row := range []struct { + body string + want bool + }{ + {"LEAQ 7(AX)(CX*2),AX", true}, + {"LEAL 7(AX),CX", true}, + {"NOPL 8(SP)", true}, + {"NOPW (AX)", true}, + {"LEAQ 8(SP),AX", true}, // Address-only grammar does not grant private backing or NUM. + {"LEAQ slot(SP),AX", false}, // A source owner must resolve the descriptive AUTO role. + {"LEAQ δοκιμή(SP),AX", false}, + {"LEAQ UNKNOWN(RSP),AX", false}, + {"LEAQ UNKNOWN(AX),AX", false}, + {"LEAQ.Z (AX),CX", false}, + {"MOVQ (AX),CX", false}, + {"CMPQ (AX),CX", false}, + {"ADDQ (AX),CX", false}, + {"INCQ (AX)", false}, + {"BTQ CX,(AX)", false}, + {"UNKNOWN (AX)", false}, + } { + t.Run(row.body, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-0\n"+row.body+"\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, ins := range file.Funcs[0].Instrs { + if ins.Raw != row.body { + continue + } + if got := fixedLocalGPAddressOnly(ins, 0); got != row.want { + t.Fatalf("address-only = %v, want %v", got, row.want) + } + } + }) + } +} + +func TestGoAMD64FixedLocalTypedGPCFGBudgetIsAtomic(t *testing.T) { + fn, sig, _ := fixedLocalGPFixture(t, "MOVQ $3,CX\nagain:\nADDQ CX,AX\nSUBQ $1,CX\nJNE again") + for _, budget := range []x86FixedLocalBudget{ + {steps: 4, bytes: 64 << 20}, // Includes the initial private store. + {steps: 1 << 18, bytes: fn.FrameSize - 1}, + } { + proof := &x86FixedLocalProof{frame: fn.FrameSize} + if err := replayX86FixedLocal(&fn, sig, proof, 0, &budget, nil, nil); err == nil { + t.Fatal("exhausted analysis budget was accepted") + } + for _, ins := range fn.Instrs { + if len(ins.x86FixedLocalMemory) != 0 { + t.Fatal("incomplete analysis published an access") + } + } + } +} diff --git a/amd64_fixed_local_gp_work_test.go b/amd64_fixed_local_gp_work_test.go new file mode 100644 index 000000000..6ac2643f4 --- /dev/null +++ b/amd64_fixed_local_gp_work_test.go @@ -0,0 +1,176 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +const fixedLocalGPWorkBlock = "ADDQ $3,AX\nRORQ $7,AX\nIMULQ $3,AX\nMULQ BX\nADCQ $1,DX\nSBBQ $1,AX\nCMPQ AX,DX\n" + +func fixedLocalGPWorkSource(block string, repeats int, frame int64, partitioned bool) string { + body := strings.Repeat(block, repeats) + if partitioned { + body = strings.Replace(fixedLocalLayoutPingPong, "loop:\n", "loop:\n"+body, 1) + body = strings.Replace(body, "ret+16(FP)", "ret+8(FP)", 1) + } else { + body += "MOVQ AX,ret+8(FP)\nRET\n" + } + return fmt.Sprintf("TEXT ·Value(SB),4,$%d-16\nMOVQ a+0(FP),CX\nMOVQ $1,AX\nMOVQ $3,BX\n", frame) + body +} + +func TestGoAMD64FixedLocalPartitionGPWork(t *testing.T) { + source := fixedLocalGPWorkSource(fixedLocalGPWorkBlock, 128, 4096, true) + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil || len(file.Funcs) != 1 { + t.Fatal(err) + } + fn := file.Funcs[0] + _, sig := fixedLocalReplayFixture(t, "") + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + proven, err := proveX86FixedLocalAttempt(fn, sig, &budget, true) + t.Logf("partition GP: steps=%d bytes=%d error=%v", budget.steps, budget.bytes, err) + if err != nil || proven.x86FixedLocal == nil { + t.Fatal("complete initialized GP register work must retain bounded frame costs", err) + } + used := (1 << 18) - budget.steps + if used < 16*2*128*7 || budget.bytes <= 0 { + t.Fatal("both layouts and sixteen phases must retain every real step", used, budget) + } + budget = x86FixedLocalBudget{steps: used - 1, bytes: 64 << 20} + proven, err = proveX86FixedLocalAttempt(fn, sig, &budget, true) + if !errors.Is(err, ErrProbeNeedsContext) || budget.steps != 0 || !reflect.DeepEqual(proven, fn) { + t.Fatal("one missing GP step must fail atomically", budget, err) + } +} + +func TestGoAMD64FixedLocalPartitionGPRouting(t *testing.T) { + rows := []struct { + name, body string + valid bool + }{ + {"multiply", "MOVQ $2,AX\nMOVQ $3,BX\nMULQ BX\nADDQ DX,AX", true}, + {"mulx", "MOVQ $2,DX\nMOVQ $3,BX\nMULXQ BX,AX,SI\nADDQ SI,AX", true}, + {"carry", "MOVQ $2,AX\nCMPQ AX,AX\nADCQ $1,AX\nSBBQ $1,AX", true}, + {"private add and mask", "MOVQ SP,AX\nADDQ $16,AX\nANDQ $-16,AX\nMOVQ $7,(AX)\nMOVQ (AX),AX", true}, + {"private constant", "MOVQ $16,BX\nMOVQ SP,AX\nADDQ BX,AX\nMOVQ $7,(AX)\nMOVQ (AX),AX", true}, + {"missing AX", "MOVQ $3,BX\nMULQ BX", false}, + {"missing DX", "MOVQ $3,BX\nMULXQ BX,AX,SI", false}, + {"missing carry", "MOVQ $2,AX\nADCQ $1,AX", false}, + {"private partial write", "MOVQ SP,AX\nADDL $8,AX", false}, + {"private escape", "MOVQ SP,AX\nADDQ $8,AX", false}, + {"private offset", "MOVQ SP,AX\nADDQ $72,AX\nMOVQ (AX),AX", false}, + {"private flags", "MOVQ $1,BX\nMOVQ SP,AX\nADDQ $16,AX\nADCQ $1,BX\nMOVQ BX,AX", false}, + } + var assembly strings.Builder + for index, row := range rows { + fmt.Fprintf(&assembly, "TEXT ·Value%d(SB),4,$64-16\n%s\nMOVQ AX,ret+8(FP)\nRET\n", index, row.body) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil || len(file.Funcs) != len(rows) { + t.Fatal("complete routing inventory changed", err) + } + _, sig := fixedLocalReplayFixture(t, "") + for index, row := range rows { + t.Run(row.name, func(t *testing.T) { + fn := file.Funcs[index] + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + proven, err := proveX86FixedLocalAttempt(fn, sig, &budget, true) + if (err == nil) != row.valid { + t.Fatal("cost classification changed the original semantic route", err) + } + if err != nil && (!errors.Is(err, ErrProbeNeedsContext) || !reflect.DeepEqual(proven, fn)) { + t.Fatal("rejected GP input changed its error class or published a proof", err) + } + }) + } +} + +func TestGoAMD64FixedLocalPartitionGPWorkProjection(t *testing.T) { + rows := []struct { + source string + cheap bool + }{ + {"ADDB $1,AX", true}, {"ADDW $1,AX", true}, {"ADDL $1,AX", true}, {"ADDQ $1,AX", true}, + {"ANDQ $-16,AX", true}, {"ADCQ BX,AX", true}, {"SBBQ BX,AX", true}, {"CMPQ AX,$0", true}, + {"MULB BX", true}, {"MULW BX", true}, {"MULL BX", true}, {"MULQ BX", true}, + {"IMULQ BX", true}, {"MULXQ BX,AX,SI", true}, {"CMOVQEQ BX,AX", true}, + {"ADDQ (SP),AX", false}, {"MULQ (SP)", false}, {"ADDQ a+0(FP),AX", false}, + {"ADDQ value(SB),AX", false}, {"LEAQ (AX)(BX*8),CX", false}, {"BTQ AX,(BX)", false}, + {"PUSHQ AX", false}, {"POPQ AX", false}, {"CALL helper(SB)", false}, {"RET", false}, + {"REP", false}, {"JMP done", false}, {"ADDQ AX", false}, {"UNKNOWN AX", false}, + } + for _, row := range rows { + t.Run(row.source, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$64-16\n"+row.source+"\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[len(file.Funcs[0].Instrs)-1] + effect, got := fixedLocalGPRegisterWork(ins) + if got != row.cheap { + t.Fatal("conservative source/effect work projection changed", got) + } + if !got && !reflect.DeepEqual(effect, x86TypedByteEffect{}) { + t.Fatal("failed cost query retained a partial descriptor") + } + if !row.cheap { + return + } + for _, metadata := range []string{"encoded", "raw", "address", "segment", "numerical"} { + changed := ins + switch metadata { + case "encoded": + changed.x86Encoded = true + case "raw": + changed.x86SourceRaw = &x86SourceRawInstruction{} + case "address": + changed.x86AddressBits = 32 + case "segment": + changed.x86SegmentPrefix = 0x64 + case "numerical": + changed.x86NumericalRead = &x86NumericalReadCertificate{} + } + if effect, cheap := fixedLocalGPRegisterWork(changed); cheap || !reflect.DeepEqual(effect, x86TypedByteEffect{}) { + t.Fatal("source/native or numerical-read metadata received a register discount", metadata) + } + } + }) + } +} + +func BenchmarkFixedLocalGPWorkProjection(b *testing.B) { + for _, partitioned := range []bool{false, true} { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("partition=%t/memory=%t", partitioned, memory) + b.Run(name, func(b *testing.B) { + block := fixedLocalGPWorkBlock + if memory { + block += "MOVQ AX,0(SP)\nMOVQ 0(SP),AX\n" + } + file, err := Parse(ArchAMD64, fixedLocalGPWorkSource(block, 8, 64, partitioned)) + if err != nil { + b.Fatal(err) + } + fn := file.Funcs[0] + _, sig := fixedLocalReplayFixture(b, "") + run := func() { + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + if _, err := proveX86FixedLocalAttempt(fn, sig, &budget, partitioned); err != nil { + b.Fatal("benchmark must be a complete accepted proof", err) + } + } + run() + b.ReportAllocs() + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + run() + } + }) + } + } +} diff --git a/amd64_fixed_local_implicit_multiply_legacy_test.go b/amd64_fixed_local_implicit_multiply_legacy_test.go new file mode 100644 index 000000000..10e09dc4a --- /dev/null +++ b/amd64_fixed_local_implicit_multiply_legacy_test.go @@ -0,0 +1,24 @@ +//go:build !go1.27 + +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalImplicitMultiplyPrivateLeafNeedsAuditedProducer(t *testing.T) { + assembly, declarations, _, _ := fixedLocalImplicitMultiplyProgram("x86_64-unknown-linux-gnu", true) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, request, _, _ := fixedLocalTransportFixture(t, assembly, declarations, "linux/amd64", "x86_64-unknown-linux-gnu") + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil { + defer source.Close() + } + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "mixed DATA needs its independent actual Go object producer") { + t.Fatal("private CALL borrowed an unaudited original object association", err) + } +} diff --git a/amd64_fixed_local_implicit_multiply_objects_test.go b/amd64_fixed_local_implicit_multiply_objects_test.go new file mode 100644 index 000000000..e0b8b1c6a --- /dev/null +++ b/amd64_fixed_local_implicit_multiply_objects_test.go @@ -0,0 +1,173 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalImplicitMultiplyProgram(triple string, privateLeaf bool) (string, string, string, string) { + var assembly, declarations, goRows, cRows, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + count := 0 + for _, stem := range []string{"MUL", "IMUL"} { + for _, width := range []string{"B", "W", "L", "Q"} { + inputs := []string{"CX", "AX", "DX", "8(SP)"} + if width == "B" { + inputs = append(inputs, "AH") + } + if width == "Q" { + inputs = append(inputs, "(DX)") + } + for _, input := range inputs { + name := fmt.Sprintf("Value%d", count) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$64-32 +MOVQ a+0(FP),AX +MOVQ b+8(FP),DX +MOVQ AX,0(SP) +MOVQ DX,8(SP) +MOVQ DX,CX +`, name) + if input == "(DX)" { + assembly.WriteString("LEAQ 8(SP),DX\n") + } + if privateLeaf && input == "(DX)" { + // The source pointer addresses this caller's private frame. + // The observed leaf never reads its own SP or a fabricated ABI. + fmt.Fprintf(&assembly, "CALL multiply%d<>(SB)\n", count) + } else { + fmt.Fprintf(&assembly, "%s%s %s\n", stem, width, input) + } + assembly.WriteString(`MOVQ $0,R8 +MOVQ $0,R9 +SETCS R8 +SETOS R9 +MOVQ mode+16(FP),CX +CMPQ CX,$1 +JEQ high +CMPQ CX,$2 +JEQ carry +CMPQ CX,$3 +JEQ overflow +CMPQ CX,$4 +JEQ canary +JMP done +high: +MOVQ DX,AX +JMP done +carry: +MOVQ R8,AX +JMP done +overflow: +MOVQ R9,AX +JMP done +canary: +MOVQ 0(SP),AX +done: +MOVQ AX,ret+24(FP) +RET +`) + if privateLeaf && input == "(DX)" { + fmt.Fprintf(&assembly, "TEXT multiply%d<>(SB),4,$0-0\n%sQ (DX)\nRET\n", count, stem) + } + fmt.Fprintf(&declarations, "func %s(a,b,mode uint64) uint64\n", name) + fmt.Fprintf(&goRows, "%s,\n", name) + fmt.Fprintf(&cRows, "value%d,\n", count) + label := runtimeCAsmLabel(prefix + "example.invalid/framefixture." + name) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t,uint64_t,uint64_t) __asm__(%s);\n", count, label) + count++ + } + } + } + const values = `0,1,0x7f,0x80,0xff,0x7fff,0x8000,0xffff, +0x7fffffff,0x80000000,0xffffffff,0x7fffffffffffffff, +0x8000000000000000,0xffffffffffffffff,0x8080808080808080,0x0123456789abcdef` + original := `package main +import "fmt" +` + declarations.String() + ` +func main() { + rows := []func(uint64,uint64,uint64) uint64{ +` + goRows.String() + ` + } + values := []uint64{` + values + `} + for n, call := range rows { + for v, a := range values { + b := values[(v+1)%len(values)] + for mode := uint64(0); mode < 5; mode++ { + got := call(a,b,mode) + if mode == 4 && got != a { panic("private input canary changed") } + fmt.Printf("%d %d %d %016x\n",n,v,mode,got) + } + } + } +} +` + driver := `#include +#include +` + runtimeCAsmLabelPreamble + cDeclarations.String() + ` +int main(void) { + uint64_t (*const rows[])(uint64_t,uint64_t,uint64_t) = { +` + cRows.String() + ` + }; + const uint64_t values[] = {` + values + `}; + const unsigned count = sizeof(values)/sizeof(values[0]); + for (unsigned n = 0; n < sizeof(rows)/sizeof(rows[0]); n++) { + for (unsigned v = 0; v < count; v++) { + for (unsigned mode = 0; mode < 5; mode++) { + uint64_t got = rows[n](values[v],values[(v+1)%count],mode); + if (mode == 4 && got != values[v]) { return 2; } + printf("%u %u %u %016llx\n",n,v,mode,(unsigned long long)got); + } + } + } + return 0; +} +` + return assembly.String(), declarations.String(), original, driver +} + +func TestGoAMD64FixedLocalImplicitMultiplyObjects(t *testing.T) { + fixedLocalImplicitMultiplyObjects(t, false) +} + +func fixedLocalImplicitMultiplyObjects(t *testing.T, privateLeaf bool) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + assembly, declarations, _, _ := fixedLocalImplicitMultiplyProgram("x86_64-unknown-linux-gnu", privateLeaf) + helpers := 0 + if privateLeaf { + helpers = 2 + } + if strings.Count(assembly, "TEXT ·Value") != 36 || strings.Count(assembly, "TEXT multiply") != helpers { + t.Fatal("incomplete independent multiply kernel/private-leaf inventory") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + var names []string + for index := 0; index < 36; index++ { + names = append(names, fmt.Sprintf("%s.Value%d", pkg.Path, index)) + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + compileLLVMToObject(t, llc, target.triple, "implicit-multiply.ll", "implicit-multiply.o", ir) + compileLLVMToObject(t, llc, target.triple, "implicit-multiply-opt.ll", "implicit-multiply-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} diff --git a/amd64_fixed_local_implicit_multiply_runtime_go127_test.go b/amd64_fixed_local_implicit_multiply_runtime_go127_test.go new file mode 100644 index 000000000..bf16792cd --- /dev/null +++ b/amd64_fixed_local_implicit_multiply_runtime_go127_test.go @@ -0,0 +1,52 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalImplicitMultiplyPrivateLeafObjects(t *testing.T) { + fixedLocalImplicitMultiplyObjects(t, true) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalImplicitMultiply(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + t.Setenv("GOCACHE", t.TempDir()) + assembly, declarations, original, driver := fixedLocalImplicitMultiplyProgram(execution.triple, true) + want := strings.Split(strings.TrimSpace(string(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original))), "\n") + if len(want) != 36*16*5 { + t.Fatalf("original Go produced %d rows, want 36 kernels x 16 inputs x 5 observations", len(want)) + } + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, runtime.GOOS+"/amd64", execution.triple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + output := execution.compileAndRunLLVM(t, "private-implicit-multiply", input, driver, level) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM row count = %d, original = %d", len(got), len(want)) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("private multiply halves/FLAGS/canary row %d: original %s; LLVM %s", index, want[index], got[index]) + } + } + }) + } + }) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("original Go and LLVM amd64 execution via Rosetta, not native x86 hardware") + } +} diff --git a/amd64_fixed_local_implicit_multiply_test.go b/amd64_fixed_local_implicit_multiply_test.go new file mode 100644 index 000000000..9167abd7f --- /dev/null +++ b/amd64_fixed_local_implicit_multiply_test.go @@ -0,0 +1,192 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +// The family and physical widths are an independent inventory; the existing +// official-table owner checks these names against Go's complete multiply rows. +var fixedLocalImplicitMultiplyWidths = []struct { + suffix string + width int64 + input uint8 + output uint8 +}{ + {"B", 1, 1, 3}, + {"W", 2, 3, 3}, + {"L", 4, 15, 255}, + {"Q", 8, 255, 255}, +} + +func TestGoAMD64FixedLocalImplicitMultiplyForms(t *testing.T) { + var original strings.Builder + count := 0 + for _, stem := range []string{"MUL", "IMUL"} { + for _, width := range fixedLocalImplicitMultiplyWidths { + name := stem + width.suffix + inputs := []string{"AX", "CX", "DX", "R8", "8(SP)", "(DX)"} + if width.width == 1 { + inputs = append(inputs, "AL", "AH") + } + for _, input := range inputs { + t.Run(name+"/"+input, func(t *testing.T) { + body := "MOVQ AX,CX\nMOVQ AX,R8\nMOVQ AX,DX\nMOVQ AX,8(SP)\n" + if input == "(DX)" { + body += "LEAQ 8(SP),DX\n" + } + body += name + " " + input + // A narrow output cannot overwrite only part of a private + // pointer. Byte multiply leaves DX unchanged; qword replaces it. + valid := input != "(DX)" || width.width == 1 || width.width == 8 + fn, sig, source := fixedLocalGPFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", count), 1)) + count++ + proved, err := proveX86FixedLocal(fn, sig) + if !valid { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("partial output overwrote a private pointer", err) + } + return + } + if err != nil { + t.Fatal("complete initialized implicit multiply rejected", err) + } + observed := 0 + for _, ins := range proved.Instrs { + if string(ins.Op) != name { + continue + } + observed++ + access, found := ins.x86FixedLocalMemory[0] + memory := strings.Contains(input, "(") + if found != memory || found && (access.width != width.width || access.write || + access.nativeInput || access.owner != proved.x86FixedLocal) { + t.Fatal("multiply did not preserve its exact private read", access) + } + } + if observed != 1 { + t.Fatalf("proved source retained %d multiply instructions, want one", observed) + } + }) + } + } + } + if count != 52 { + t.Fatalf("independent multiply inventory changed: %d", count) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalImplicitMultiplyByteEffects(t *testing.T) { + for _, stem := range []string{"MUL", "IMUL"} { + for _, width := range fixedLocalImplicitMultiplyWidths { + sources := []struct { + name, root Reg + mask uint8 + }{{AX, AX, width.input}, {CX, CX, width.input}, {DX, DX, width.input}} + if width.width == 1 { + sources = append(sources, struct { + name, root Reg + mask uint8 + }{"AH", AX, 2}) + } + for _, source := range sources { + name := stem + width.suffix + t.Run(name+"/"+string(source.name), func(t *testing.T) { + ins := x86YMMReadSource(t, name+" "+string(source.name)) + effect, known, err := fixedLocalGPByteEffect(ins) + if !known || err != nil || !effect.requiresResultProof { + t.Fatal("closed implicit family lacks its existing numerical effect", known, err) + } + state := x86FixedLocalState{ + known: map[Reg]uint8{AX: width.input, DX: 0, CX: 0}, + booleans: make(map[Reg]uint8), + flags: x86ByteFlagsAll, + } + state.known[source.root] |= source.mask + before := cloneFixedLocalState(state) + want := cloneFixedLocalState(before) + want.known[AX] |= width.output + if width.width != 1 { + want.known[DX] |= width.output + } + want.flags = x86ByteFlagCF | x86ByteFlagOF + if err := fixedLocalTypedByteStep(&ins, &state, FuncSig{}, nil, 0, effect); err != nil || + !reflect.DeepEqual(state.known, want.known) || state.flags != want.flags { + t.Fatal("implicit accumulator/output width or FLAGS changed", state, want, err) + } + for _, input := range []struct { + root Reg + mask uint8 + }{{AX, width.input}, {source.root, source.mask}} { + for bit := uint8(1); bit != 0; bit <<= 1 { + if input.mask&bit == 0 { + continue + } + missing := cloneFixedLocalState(before) + missing.known[input.root] &^= bit + if err := fixedLocalTypedByteStep(&ins, &missing, FuncSig{}, nil, 0, effect); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unknown input byte acquired a numerical result", input, bit, err) + } + } + } + }) + } + } + } +} + +func TestGoAMD64FixedLocalImplicitMultiplyBoundaries(t *testing.T) { + for _, stem := range []string{"MUL", "IMUL"} { + for _, width := range fixedLocalImplicitMultiplyWidths { + name := stem + width.suffix + for _, row := range []struct { + name, before, source, after string + valid bool + }{ + {"carry", "", "0(SP)", "ADCQ $0,AX", true}, + {"overflow", "", "0(SP)", "JOS done\ndone:", true}, + {"old zero revoked", "CMPQ AX,AX\n", "0(SP)", "JEQ done\ndone:", false}, + {"old parity revoked", "CMPQ AX,AX\n", "0(SP)", "JPS done\ndone:", false}, + {"old sign revoked", "CMPQ AX,AX\n", "0(SP)", "JMI done\ndone:", false}, + {"old auxiliary revoked", "CMPQ AX,AX\n", "0(SP)", "LAHF", false}, + {"missing source", "", "CX", "", false}, + {"private accumulator", "MOVQ SP,AX\n", "0(SP)", "", false}, + {"private source", "MOVQ SP,CX\n", "CX", "", false}, + {"unwritten memory", "", "8(SP)", "", false}, + {"one missing memory byte", "MOVQ AX,8(SP)\n", fmt.Sprintf("%d(SP)", 17-width.width), "", false}, + {"outside frame", "", "64(SP)", "", false}, + {"one byte outside frame", "MOVQ AX,56(SP)\n", fmt.Sprintf("%d(SP)", 65-width.width), "", false}, + {"numerical address", "", "(AX)", "", true}, + {"private spill", "MOVQ SP,8(SP)\n", "8(SP)", "", false}, + } { + t.Run(name+"/"+row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalGPFixture(t, row.before+name+" "+row.source+"\n"+row.after) + proved, err := proveX86FixedLocal(fn, sig) + if (err == nil) != row.valid { + t.Fatalf("valid=%v: %v", row.valid, err) + } + if row.name == "numerical address" { + fixedLocalRequireNumericNativeRole(t, proved, AX) + } + }) + } + for _, role := range []string{"accumulator", "source"} { + t.Run(name+"/opaque "+role, func(t *testing.T) { + body := "MOVQ SI,CX\n" + name + " CX" + if role == "accumulator" { + body = "MOVQ SI,AX\n" + name + " 0(SP)" + } + fn, sig, _ := fixedLocalInputFixture(t, body) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("complete native machine bits were rejected as multiply input", err) + } + }) + } + } + } +} diff --git a/amd64_fixed_local_index_go127_test.go b/amd64_fixed_local_index_go127_test.go new file mode 100644 index 000000000..ee57851f7 --- /dev/null +++ b/amd64_fixed_local_index_go127_test.go @@ -0,0 +1,143 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalConstantIndexFunction(name, body string) string { + return fmt.Sprintf("TEXT ·%s(SB),4,$64-16\nMOVQ a+0(FP),AX\nMOVQ AX,0(SP)\n%s\nMOVQ AX,ret+8(FP)\nRET\n", name, body) +} + +func TestGoAMD64FixedLocalConstantIndexObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var assembly, declarations strings.Builder + assembly.WriteString(fixedLocalAddressData) + cases := fixedLocalConstantIndexCases() + if len(cases) != 30 { + t.Fatal("two origins by complete non-SP GP bank changed", len(cases)) + } + for index, row := range cases { + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(fixedLocalConstantIndexFunction(name, row.body)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly.String(), declarations.String(), target.name, target.triple) + var names []string + for index := range cases { + names = append(names, fmt.Sprintf("%s.Value%d", pkg.Path, index)) + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("original constant-index private/DATA source rejected", err) + } + if !strings.Contains(ir, "internal constant [32 x i8]") { + t.Fatal("private readonly DATA was not retained") + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + compileLLVMToObject(t, llc, target.triple, "constant-index.ll", "constant-index.o", ir) + compileLLVMToObject(t, llc, target.triple, "constant-index-opt.ll", "constant-index-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + t.Log("30 original-Go functions cover 15 non-SP index registers, four scales, private store/RMW/load and readonly DATA reads; three APIs at O0/O2") +} + +func TestGoAMD64FixedLocalConstantIndexGuards(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, literal := range []bool{false, true} { + name, acquire, size := "private", "MOVQ SP,DI\n", 64 + if literal { + name, acquire, size = "literal", "LEAQ literal<>(SB),DI\n", 32 + } + for _, row := range []struct { + name, body string + valid bool + }{ + {"first byte via negative index", "MOVQ $-1,SI\nMOVQ 8(DI)(SI*8),AX", true}, + {"last byte", fmt.Sprintf("MOVQ $%d,SI\nMOVQ $0,AX\nMOVB (DI)(SI*1),AX", size-1), true}, + {"last word", fmt.Sprintf("MOVQ $%d,SI\nMOVQ $0,AX\nMOVW (DI)(SI*1),AX", size-2), true}, + {"last dword", fmt.Sprintf("MOVQ $%d,SI\nMOVL (DI)(SI*1),AX", size-4), true}, + {"last qword", fmt.Sprintf("MOVQ $%d,SI\nMOVQ (DI)(SI*8),AX", size/8-1), true}, + {"last vector", fmt.Sprintf("MOVQ $%d,SI\nMOVOU (DI)(SI*1),X0\nMOVQ X0,AX", size-16), true}, + {"indexed full spill round trip", "MOVQ DI,16(SP)\nMOVQ $2,SI\nMOVQ (SP)(SI*8),R8\nMOVQ (R8),AX", true}, + {"before allocation", "MOVQ $-1,SI\nMOVQ (DI)(SI*1),AX", false}, + {"past allocation", fmt.Sprintf("MOVQ $%d,SI\nMOVQ 1(DI)(SI*8),AX", size/8-1), false}, + {"positive multiply wraps to base", "MOVQ $9223372036854775807,SI\nMOVQ 2(DI)(SI*2),AX", literal}, + {"negative multiply wraps to base", "MOVQ $-9223372036854775808,SI\nMOVQ (DI)(SI*2),AX", literal}, + {"displacement overflow", "MOVQ $9223372036854775807,SI\nMOVQ 1(DI)(SI*1),AX", false}, + {"unaligned vector", "MOVQ $1,SI\nMOVO (DI)(SI*1),X0", false}, + {"dynamic numerical index", "MOVQ a+0(FP),SI\nMOVQ (DI)(SI*1),AX", literal}, + {"unknown upper index bytes", "MOVW $0,SI\nMOVQ (DI)(SI*1),AX", false}, + {"defined partial index overwrite", "MOVQ $0,SI\nMOVB $1,SI\nMOVQ (DI)(SI*1),AX", literal}, + {"pointer used as index", "MOVQ DI,SI\nMOVQ (DI)(SI*1),AX", false}, + {"numeric base loses origin", "MOVQ $0,DI\nMOVQ $0,SI\nMOVQ (DI)(SI*1),AX", true}, + {"different constants at join", "MOVQ $77,8(SP)\nMOVQ $0,SI\nCMPQ AX,$0\nJEQ joined\nMOVQ $1,SI\njoined:\nMOVQ (DI)(SI*8),AX", literal}, + {"loaded numerical index", "MOVQ $0,SI\nMOVQ (DI)(SI*1),SI\nMOVQ (DI)(SI*1),AX", literal}, + } { + t.Run(name+"/"+row.name, func(t *testing.T) { + body := "MOVQ $77,48(SP)\nMOVQ $77,56(SP)\n" + acquire + row.body + assembly := fixedLocalAddressData + fixedLocalConstantIndexFunction("Value", body) + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + // The actual Go capture is a prerequisite, not evidence that an + // invalid extent or provenance was rejected by our translator. + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if row.valid && err != nil { + t.Fatal("valid constant-index boundary rejected", err) + } + if !row.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved constant-index bytes or origin accepted", err) + } + if row.name == "numeric base loses origin" { + proved, err := proveX86FixedLocal(file.Funcs[0], bound.Sigs[pkg.Path+".Value"]) + if err != nil { + t.Fatal(err) + } + fixedLocalRequireNumericNativeRole(t, proved, DI) + } + }) + } + } + for _, row := range []struct{ name, body string }{ + {"readonly store", "LEAQ literal<>(SB),DI\nMOVQ $1,SI\nMOVQ AX,(DI)(SI*8)"}, + {"readonly RMW", "LEAQ literal<>(SB),DI\nMOVQ $1,SI\nADDQ AX,(DI)(SI*8)"}, + {"uninitialized private bytes", "MOVQ $1,SI\nMOVQ (SP)(SI*8),AX"}, + {"partial private initialization", "MOVQ $1,SI\nMOVL AX,(SP)(SI*8)\nMOVQ (SP)(SI*8),AX"}, + {"indexed write revokes spilled origin", "MOVQ SP,R8\nMOVQ R8,16(SP)\nMOVQ $2,SI\nMOVL $0,(SP)(SI*8)\nMOVQ 16(SP),DI\nMOVQ (DI),AX"}, + {"index cannot borrow unindexed private spill", "MOVQ SP,R8\nMOVQ R8,0(SP)\nMOVQ $1,SI\nMOVQ (SP)(SI*8),DI\nMOVQ $77,(DI)\nMOVQ $0,AX"}, + {"index cannot borrow unindexed literal spill", "LEAQ literal<>(SB),DI\nMOVQ DI,0(SP)\nMOVQ $1,SI\nMOVQ (SP)(SI*8),DI\nMOVQ (DI),AX"}, + {"mixed private and literal index", "LEAQ literal<>(SB),SI\nMOVQ (SP)(SI*1),AX"}, + {"mixed literal and private index", "LEAQ literal<>(SB),DI\nMOVQ SP,SI\nMOVQ (DI)(SI*1),AX"}, + } { + t.Run(row.name, func(t *testing.T) { + assembly := fixedLocalAddressData + fixedLocalConstantIndexFunction("Value", row.body) + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("indexed access acquired uninitialized/mutable/address permission", err) + } + }) + } +} diff --git a/amd64_fixed_local_index_test.go b/amd64_fixed_local_index_test.go new file mode 100644 index 000000000..885a95b84 --- /dev/null +++ b/amd64_fixed_local_index_test.go @@ -0,0 +1,98 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type fixedLocalConstantIndexCase struct { + name, body string + literal bool +} + +func fixedLocalConstantIndexCases() []fixedLocalConstantIndexCase { + var cases []fixedLocalConstantIndexCase + // Independent physical GP inventory: SP is a base, never an index. + for _, register := range strings.Fields("AX BX CX DX DI SI BP R8 R9 R10 R11 R12 R13 R14 R15") { + for _, literal := range []bool{false, true} { + var body strings.Builder + body.WriteString("MOVQ $0,8(SP)\n") + base, name := "SP", "private/"+register + if literal { + base, name = "SI", "literal/"+register + if register == base { + base = "DI" + } + fmt.Fprintf(&body, "LEAQ literal<>(SB),%s\n", base) + } else { + body.WriteString("MOVQ $77,48(SP)\n") + } + for ordinal, scale := range []int{1, 2, 4, 8} { + offset := 56 // The final qword of this private 64-byte frame. + if literal { + offset = ordinal * 8 + } + address := fmt.Sprintf("%d(%s)(%s*%d)", offset-3*scale, base, register, scale) + if !literal { + body.WriteString("MOVQ $0,56(SP)\n") + } + fmt.Fprintf(&body, "MOVQ $3,%s\n", register) + if !literal { + fmt.Fprintf(&body, "MOVQ $%d,%s\nADDQ $1,%s\n", 17+scale, address, address) + } + fmt.Fprintf(&body, "MOVQ %s,AX\nADDQ AX,8(SP)\n", address) + if !literal { + body.WriteString("MOVQ 56(SP),AX\nADDQ AX,8(SP)\n") + } + } + if !literal { + body.WriteString("MOVQ 48(SP),AX\nADDQ AX,8(SP)\n") + } + body.WriteString("MOVQ 8(SP),AX") + cases = append(cases, fixedLocalConstantIndexCase{name, body.String(), literal}) + } + } + return cases +} + +func TestGoAMD64FixedLocalConstantIndexPrivate(t *testing.T) { + var original strings.Builder + count := 0 + for _, row := range fixedLocalConstantIndexCases() { + if row.literal { + continue + } + fn, sig, source := fixedLocalGPFixture(t, row.body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", count), 1)) + count++ + t.Run(row.name, func(t *testing.T) { + proved, err := proveX86FixedLocal(fn, sig) + if err != nil { + t.Fatal("constant index must retain exact private bytes", err) + } + accesses := 0 + for _, instruction := range proved.Instrs { + for index, operand := range instruction.Args { + if operand.Kind != OpMem || operand.Mem.Index == "" { + continue + } + accesses++ + access, found := instruction.x86FixedLocalMemory[index] + write := instruction.Op == "ADDQ" || index == 1 + if !found || access.owner != proved.x86FixedLocal || access.width != 8 || + access.nativeInput || access.write != write || access.offset != [8]int64{56, 56, 56, 56, 56, 56, 56, 56} { + t.Fatalf("indexed access lost its exact byte/role certificate: %+v", access) + } + } + } + if accesses != 12 { + t.Fatal("expected one store/RMW/load for each scale", accesses) + } + }) + } + if count != 15 { + t.Fatal("independent non-SP GP inventory changed", count) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} diff --git a/amd64_fixed_local_input_runtime_test.go b/amd64_fixed_local_input_runtime_test.go new file mode 100644 index 000000000..34f2f7332 --- /dev/null +++ b/amd64_fixed_local_input_runtime_test.go @@ -0,0 +1,91 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// The independent volatile-byte reference checks an observable access, not +// original SIMD execution or any claimed pointee extent from its Go type. +func TestGoAMD64FixedLocalInputMemoryDeadReadFaults(t *testing.T) { + t.Run("native-numeric-address", fixedLocalNativeNumericAddressRuntime) + fixedLocalInputMemoryFaults(t, []fixedLocalInputFaultCase{{"MOVQ (SI),CX", 8}, {"CMPQ (SI),$0", 8}}) +} + +func TestGoAMD64FixedLocalInputMemoryDeadMultiplyFaults(t *testing.T) { + var cases []fixedLocalInputFaultCase + for index, suffix := range []string{"W", "L", "Q"} { + for _, instruction := range []string{"IMUL" + suffix + " (SI),AX", "IMUL3" + suffix + " $0,(SI),AX"} { + cases = append(cases, fixedLocalInputFaultCase{instruction, 2 << uint(index)}) + } + } + fixedLocalInputMemoryFaults(t, cases) +} + +func TestGoAMD64FixedLocalInputMemoryDeadLogicalFaults(t *testing.T) { + var cases []fixedLocalInputFaultCase + for index, suffix := range []string{"B", "W", "L", "Q"} { + for _, stem := range []string{"AND", "OR", "XOR"} { + cases = append(cases, fixedLocalInputFaultCase{stem + suffix + " (SI),AX", 1 << uint(index)}) + } + } + fixedLocalInputMemoryFaults(t, cases) +} + +type fixedLocalInputFaultCase struct { + instruction string + width int +} + +func fixedLocalInputMemoryFaults(t *testing.T, cases []fixedLocalInputFaultCase) { + t.Helper() + execution := x86ScalarCountHostExecution(t) + originalTriple := testTargetTriple(runtime.GOOS, "amd64") + var assembly, declarations, prototypes, rows strings.Builder + prefix := "" + if strings.Contains(execution.triple, "apple") { + prefix = "_" + } + for index, row := range cases { + _, _, source := fixedLocalInputFixture(t, row.instruction+"\nMOVQ $0,AX") + assembly.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + fmt.Fprintf(&declarations, "func Value%d(p, q *uint64, a uint64) uint64\n", index) + fmt.Fprintf(&prototypes, "extern uint64_t value%d(const uint8_t *,uint64_t *,uint64_t) __asm__(%q);\n", + index, prefix+"example.invalid/framefixture.Value"+fmt.Sprint(index)) + fmt.Fprintf(&prototypes, "static int run%d(const uint8_t *p) {\n uint64_t output = 0;\n return value%d(p, &output, 0) != 0;\n}\n", index, index) + fmt.Fprintf(&rows, "{run%d,%d,%d},\n", index, row.width, row.width-1) + } + driver := x86MemoryFaultDriver(prototypes.String(), rows.String()) + pkg, request, _, options := fixedLocalTransportFixture(t, assembly.String(), declarations.String(), + runtime.GOOS+"/amd64", originalTriple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(ir, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "declare ") { + continue // The common prelude includes unused x86 declarations. + } + if strings.Contains(line, " asm ") || strings.Contains(line, "@llvm.x86.") { + t.Fatal("host scalar oracle cannot execute x86-specific operations") + } + } + for _, pipeline := range []struct{ name, level, ir string }{ + {"backend_O0", "-O0", ir}, + {"IR_O2_backend_O2", "-O2", x86MulDivRoundOptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + compileLLVMToObject(t, execution.llc, originalTriple, "input-fault.ll", "input-fault.o", pipeline.ir) + executable := execution.compileLLVM(t, "fixed-local-input-fault", pipeline.ir, driver, pipeline.level) + x86CheckMemoryFaultMatrix(t, execution, executable, len(cases)) + }) + } + }) + t.Log("original amd64 three-API objects; actual-host pure scalar LLVM fault execution, not original x86/Go execution") +} diff --git a/amd64_fixed_local_input_test.go b/amd64_fixed_local_input_test.go new file mode 100644 index 000000000..104d92567 --- /dev/null +++ b/amd64_fixed_local_input_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalInputFixture(t *testing.T, body string) (Func, FuncSig, string) { + t.Helper() + _, _, sig, source := x86ResultInitializationFixture(t, "amd64", + "func Value(p, q *uint64, a uint64) uint64", + "MOVQ p+0(FP),SI\nMOVQ q+8(FP),DI\nMOVQ a+16(FP),AX\nMOVQ AX,0(SP)\n"+ + body+"\nMOVQ AX,ret+24(FP)\nRET") + source = strings.Replace(source, "$0-32", "$64-32", 1) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + return file.Funcs[0], sig, source +} + +func TestGoAMD64FixedLocalInputMemoryWidths(t *testing.T) { + var original strings.Builder + count := 0 + for _, width := range []string{"B", "W", "L", "Q"} { + for _, operation := range []string{ + "MOV%s 3(SI),CX\nMOVQ CX,8(SP)\nMOVQ 8(SP),AX", + "MOV%s AX,5(DI)", + "ADD%s 3(SI),AX", + "ADD%s $1,5(DI)", + "NEG%s 5(DI)", + "CMP%s 3(SI),$1\nSETCS CL", + } { + body := "MOVQ AX,CX\n" + fmt.Sprintf(operation, width) + fn, sig, source := fixedLocalInputFixture(t, body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", count), 1)) + count++ + t.Run(fmt.Sprintf("%s/%d", width, count), func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal(err) + } + }) + } + } + if count != 24 { + t.Fatal("changed independent scalar memory width inventory", count) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalInputMemoryProvenance(t *testing.T) { + for _, row := range []struct { + name, body string + valid bool + }{ + {"copy", "MOVQ SI,CX\nMOVQ 3(CX),AX", true}, + {"private pointer spill", "MOVQ SI,8(SP)\nMOVQ 8(SP),CX\nMOVQ 3(CX),AX", true}, + {"private numerical spill", "MOVQ (SI),CX\nMOVQ CX,8(SP)\nMOVQ 8(SP),AX", true}, + {"unaligned vector", "MOVOU (SI),X0\nMOVOU X0,16(SP)\nMOVQ 16(SP),AX", true}, + {"aligned native vector", "MOVO (SI),X0\nMOVOU X0,16(SP)\nMOVQ 16(SP),AX", true}, + {"native vector output", "MOVOU (SI),X0\nMOVOU X0,(DI)", true}, + {"mask zeroing native input", "KMOVQ AX,K1\nVMOVDQU8.Z (SI),K1,Z0\nVMOVDQU8 Z0,0(SP)\nMOVQ 0(SP),AX", true}, + {"mask native output", "KMOVQ AX,K1\nVMOVDQU8 (SI),Z0\nVMOVDQU8 Z0,K1,(DI)", true}, + {"numerical loop", "MOVQ $3,CX\nagain:\nADDQ (SI),AX\nSUBQ $1,CX\nJNE again", true}, + {"same input join", "CMPQ AX,$1\nJEQ left\nMOVQ SI,CX\nJMP join\nleft:\nMOVQ SI,CX\njoin:\nMOVQ (CX),AX", true}, + {"numerical address", "MOVQ (AX),CX", true}, + {"loaded integer is not pointer", "MOVQ (SI),CX\nMOVQ (CX),AX", true}, + {"private pointer escape", "MOVQ SP,CX\nMOVQ CX,(DI)", false}, + {"opaque pointer store", "MOVQ SI,(DI)", false}, + {"private pointer arithmetic", "ADDQ SI,AX", false}, + {"input pointer arithmetic", "ADDQ $8,SI\nMOVQ (SI),AX", true}, + {"input pointer LEA", "LEAQ 8(SI),CX\nMOVQ (CX),AX", true}, + {"input pointer partial write", "MOVB $0,SI\nMOVQ (SI),AX", false}, + {"input pointer partial spill", "MOVQ SI,8(SP)\nMOVB $0,8(SP)\nMOVQ 8(SP),CX\nMOVQ (CX),AX", false}, + {"numerical index", "MOVQ (SI)(AX*2),CX", true}, + {"private index", "MOVQ (SI)(SP*1),CX", false}, + {"segment", "MOVQ (SI)(FS*1),CX", false}, + {"different input join", "CMPQ AX,$1\nJEQ left\nMOVQ SI,CX\nJMP join\nleft:\nMOVQ DI,CX\njoin:\nMOVQ (CX),AX", true}, + {"pointer and number join", "CMPQ AX,$1\nJEQ left\nMOVQ SI,CX\nJMP join\nleft:\nMOVQ AX,CX\njoin:\nMOVQ (CX),AX", false}, + {"private missing byte", "MOVQ (SI),AX\nADDQ $1,8(SP)", false}, + {"private crossing frame", "MOVQ (SI),AX\nMOVQ AX,60(SP)", false}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalInputFixture(t, row.body) + proved, err := proveX86FixedLocal(fn, sig) + if (err == nil) != row.valid { + t.Fatalf("valid=%v: %v", row.valid, err) + } + if row.name == "numerical address" { + fixedLocalRequireNumericNativeRole(t, proved, AX) + } + if row.name == "loaded integer is not pointer" { + fixedLocalRequireNumericNativeRole(t, proved, CX) + } + }) + } +} + +func TestGoAMD64FixedLocalInputMemoryActualObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var assembly, declarations strings.Builder + for index, body := range []string{ + "MOVQ (SI),CX\nMOVQ CX,8(SP)\nMOVQ 8(SP),AX\nMOVQ AX,(DI)", + "ADDQ (SI),AX\nSUBW $1,(DI)\nNEGB 1(DI)", + "CMPQ (SI),$1\nCMOVQCS 8(SI),AX\nSETCS 8(DI)", + "MOVO (SI),X0\nMOVOU X0,16(SP)\nMOVOU X0,(DI)\nMOVQ 16(SP),AX", + "KMOVQ AX,K1\nVMOVDQU8.Z (SI),K1,Z0\nVMOVDQU8 Z0,0(SP)\nVMOVDQU8 Z0,K1,(DI)\nMOVQ 0(SP),AX", + "MOVQ SI,8(SP)\nMOVQ 8(SP),CX\nMOVQ (CX),AX\nMOVQ $0,AX", + "ADDQ AX,SI\nMOVQ (SI),AX", + "ADDQ SI,AX\nMOVQ (AX),AX", + "SUBQ AX,SI\nMOVQ (SI),AX", + "LEAQ -64(SI)(AX*8),CX\nMOVQ (CX),AX", + "INCQ SI\nDECQ SI\nMOVQ (SI),AX", + "MOVQ SI,8(SP)\nCMPQ DI,8(SP)\nSETCS AL", + "MOVQ SI,8(SP)\nCMPQ 8(SP),DI\nSETCS AL", + "CMPQ AX,$0\nJEQ join\nMOVQ SI,8(SP)\njoin:", + } { + _, _, source := fixedLocalInputFixture(t, body) + assembly.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + fmt.Fprintf(&declarations, "func Value%d(p, q *uint64, a uint64) uint64\n", index) + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly.String(), declarations.String(), target.name, target.triple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, required := range []string{"load volatile i64", "store volatile i64", "movdqa"} { + if !strings.Contains(ir, required) { + t.Fatal("native memory instruction lost observable access", required) + } + } + masked := x86StringMemoryFunction(t, ir, pkg.Path+".Value4") + if strings.Count(masked, "load volatile i8,") != 64 || + strings.Count(masked, "store volatile i8 ") != 64 || strings.Count(masked, "br i1 ") != 128 { + t.Fatal("masked input/output lost its active-lane volatile accesses") + } + optimized := x86MulDivRoundOptimize(t, ir) + for _, version := range []string{ir, optimized} { + dead := x86StringMemoryFunction(t, version, pkg.Path+".Value5") + if !strings.Contains(dead, "load volatile i64") { + t.Fatal("dead native input read disappeared from its own function") + } + } + compileLLVMToObject(t, llc, target.triple, "input-memory.ll", "input-memory.o", ir) + compileLLVMToObject(t, llc, target.triple, "input-memory-opt.ll", "input-memory-opt.o", optimized) + }) + }) + } +} + +func TestGoAMD64FixedLocalInputMemoryRoleCertificate(t *testing.T) { + fn, sig, _ := fixedLocalInputFixture(t, "MOVQ (SI),CX\nMOVQ CX,8(SP)\nMOVQ 8(SP),AX\nMOVQ AX,(DI)") + proved, err := proveX86FixedLocal(fn, sig) + if err != nil { + t.Fatal(err) + } + ctx := amd64Ctx{fixedLocal: proved.x86FixedLocal} + native, private := 0, 0 + for _, instruction := range proved.Instrs { + for index, access := range instruction.x86FixedLocalMemory { + if _, valid := ctx.fixedLocalMemoryAccess(instruction, index); !valid { + t.Fatal("original access lost its whole-function binding") + } + if ctx.fixedLocalMemoryCertified(instruction, index) == access.nativeInput { + t.Fatal("native input acquired a private extent/alignment certificate") + } + if access.nativeInput { + native++ + } else { + private++ + } + changed := instruction + changed.x86FixedLocalMemory = map[int]x86FixedLocalAccess{index: access} + access.nativeInput = !access.nativeInput + changed.x86FixedLocalMemory[index] = access + if _, valid := ctx.fixedLocalMemoryAccess(changed, index); valid { + t.Fatal("mutated address role retained authority") + } + } + } + if native != 2 || private != 3 { + t.Fatal("incomplete independent memory role inventory", native, private) + } +} diff --git a/amd64_fixed_local_integer_compare_objects_test.go b/amd64_fixed_local_integer_compare_objects_test.go new file mode 100644 index 000000000..7bef3b91b --- /dev/null +++ b/amd64_fixed_local_integer_compare_objects_test.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestGoAMD64FixedLocalIntegerCompareCompleteObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + rows := fixedLocalIntegerCompareCases() + if len(rows) != 168 { + t.Fatal("complete comparison object inventory changed", len(rows)) + } + var assembly, declarations strings.Builder + for index, row := range rows { + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(fixedLocalIntegerCompareSource(row), "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p *uint64, a uint64) uint64\n", name) + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, + assembly.String(), declarations.String(), target.name, target.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("complete fixed-frame integer comparisons failed translation", err) + } + optimized := fixedLocalIntegerCompareConstantResults(t, ir, pkg.Path, rows) + compileLLVMToObject(t, llc, target.triple, "compare.ll", "compare.o", ir) + compileLLVMToObject(t, llc, target.triple, "compare-opt.ll", "compare-opt.o", optimized) + }) + }) + } +} + +// Parse and optimize each complete module once. Check the actual return value +// of every self relation, including unknown inputs, rather than IR substrings. +func fixedLocalIntegerCompareConstantResults(t *testing.T, ir, path string, rows []fixedLocalIntegerCompareCase) string { + t.Helper() + context := llvm.NewContext() + defer context.Dispose() + module, err := parseIRModuleInContext(context, ir) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + for index := range rows { + function := module.NamedFunction(fmt.Sprintf("%s.Value%d", path, index)) + if function.IsNil() || function.FirstBasicBlock().IsNil() { + t.Fatal("complete source omitted original function", index) + } + } + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + if err := module.RunPasses("default", llvm.TargetMachine{}, options); err != nil { + t.Fatal(err) + } + checked := 0 + for index, row := range rows { + if !row.self { + continue + } + checked++ + want := ^uint64(0) + if strings.Contains(row.operation, "PCMPGT") { + want = 0 + } + function := module.NamedFunction(fmt.Sprintf("%s.Value%d", path, index)) + returns := 0 + for block := function.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.IsAReturnInst().IsNil() { + continue + } + returns++ + if instruction.OperandsCount() != 1 || instruction.Operand(0).IsAConstantInt().IsNil() || + instruction.Operand(0).ZExtValue() != want { + t.Fatalf("self relation %s did not return %016x: %s", row.name, want, function.String()) + } + } + } + if returns != 1 { + t.Fatal("self relation lost its sole return", row.name, returns) + } + } + if checked != 48 { + t.Fatal("self-comparison constant inventory changed", checked) + } + return module.String() +} diff --git a/amd64_fixed_local_integer_compare_runtime_linux_amd64_test.go b/amd64_fixed_local_integer_compare_runtime_linux_amd64_test.go new file mode 100644 index 000000000..44323a5d9 --- /dev/null +++ b/amd64_fixed_local_integer_compare_runtime_linux_amd64_test.go @@ -0,0 +1,7 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalIntegerCompare(t *testing.T) { + fixedLocalVectorRuntime(t, fixedLocalVEXNativeTools(t), true, fixedLocalIntegerCompareRuntimeKernels(t)) +} diff --git a/amd64_fixed_local_integer_compare_runtime_test.go b/amd64_fixed_local_integer_compare_runtime_test.go new file mode 100644 index 000000000..8b3d1e84b --- /dev/null +++ b/amd64_fixed_local_integer_compare_runtime_test.go @@ -0,0 +1,46 @@ +//go:build unix + +package plan9asm + +import ( + "strings" + "testing" +) + +func fixedLocalIntegerCompareRuntimeKernels(t *testing.T) []fixedLocalVEXRuntimeKernel { + t.Helper() + var kernels []fixedLocalVEXRuntimeKernel + owners := make(map[string]int) + rows := fixedLocalIntegerCompareCases() + for _, row := range rows { + parts := strings.Split(row.name, "/") + shape := parts[0] + "/" + parts[1] + index, exists := owners[shape] + if !exists { + index, owners[shape] = len(kernels), len(kernels) + kernels = append(kernels, fixedLocalVEXRuntimeKernel{shape: shape}) + } + // Observe defined CMP FLAGS after each comparison, including AF via + // LAHF, in scratch bytes outside all comparison input ranges. + operation := "MOVQ (SI),AX\nCMPQ AX,$1\n" + row.operation + + "\nLAHF\nMOVB AH,120(BP)\nSETOS 121(BP)" + if strings.Contains(row.name, "unknown self") { + // The common oracle observes Y0..Y3. Expose each high-bank result + // without reading undefined legacy upper bytes; VEX defines them. + if row.legacy { + operation += "\nMOVOU " + row.destination + ",X2" + } else { + operation += "\nVMOVDQU " + strings.Replace(row.destination, "X", "Y", 1) + ",Y2" + } + } + kernels[index].operations = append(kernels[index].operations, operation) + } + if len(kernels) != 24 || len(rows) != 168 { + t.Fatal("comparison runtime family/width/operand inventory changed", len(kernels), len(rows)) + } + return kernels +} + +func TestGoAMD64FixedLocalIntegerCompareRuntimeObjects(t *testing.T) { + fixedLocalVectorRuntime(t, fixedLocalPackedCompileOnlyTools(t), false, fixedLocalIntegerCompareRuntimeKernels(t)) +} diff --git a/amd64_fixed_local_integer_compare_test.go b/amd64_fixed_local_integer_compare_test.go new file mode 100644 index 000000000..2a4e696d6 --- /dev/null +++ b/amd64_fixed_local_integer_compare_test.go @@ -0,0 +1,202 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +type fixedLocalIntegerCompareCase struct { + name, operation, destination string + width int + legacy, self bool +} + +func fixedLocalIntegerCompareCases() []fixedLocalIntegerCompareCase { + var rows []fixedLocalIntegerCompareCase + for index, name := range x86PackedIntegerCompareTestNames { + legacy := index < 8 + banks := []string{"X", "Y"} + if legacy { + banks = banks[:1] + } + for _, bank := range banks { + width := 16 + if bank == "Y" { + width = 32 + } + add := func(label, operands, destination string, self bool) { + rows = append(rows, fixedLocalIntegerCompareCase{ + name + "/" + bank + "/" + label, name + " " + operands, + destination, width, legacy, self, + }) + } + if legacy { + add("register", "X1,X0", "X0", false) + add("alias", "X0,X0", "X0", true) + add("private", "64(BP),X0", "X0", false) + add("native", "(SI),X0", "X0", false) + add("unknown self", "X15,X15", "X15", true) + continue + } + for _, destination := range []string{bank + "2", bank + "0", bank + "1"} { + add("register "+destination, bank+"0,"+bank+"1,"+destination, destination, false) + } + for _, memory := range []string{"64(BP)", "65(BP)", "(SI)"} { + add("memory "+memory, memory+","+bank+"1,"+bank+"2", bank+"2", false) + } + for _, destination := range []string{bank + "6", bank + "15"} { + add("unknown self "+destination, bank+"15,"+bank+"15,"+destination, destination, true) + } + } + } + return rows +} + +func fixedLocalIntegerCompareSource(row fixedLocalIntegerCompareCase) string { + // Keep the complete original aligned scratch function. Store the compared + // low qword into its actual result slot, so the comparison is observable. + low := strings.Replace(row.destination, "Y", "X", 1) + return fixedLocalVEXAssembly(row.operation + "\nMOVQ " + low + ",AX\nMOVQ AX,128(BP)") +} + +func TestGoAMD64FixedLocalIntegerCompareCompleteSources(t *testing.T) { + rows := fixedLocalIntegerCompareCases() + if len(rows) != 168 { + t.Fatal("fixed-frame source inventory changed", len(rows)) + } + var original strings.Builder + for index, row := range rows { + source := fixedLocalIntegerCompareSource(row) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, _, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *uint64, a uint64) uint64", "RET") + if _, err := proveX86FixedLocal(file.Funcs[0], sig); err != nil { + t.Fatal("complete integer-comparison source rejected", err) + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalIntegerCompareAliasesAndFlags(t *testing.T) { + for index, name := range x86PackedIntegerCompareTestNames { + legacy := index < 8 + for _, width := range []int{16, 32} { + if legacy && width != 16 { + continue + } + bank := "X" + if width == 32 { + bank = "Y" + } + for _, self := range []bool{false, true} { + for _, upper := range []uint64{0, 1 << 50, 0xffffffffffff0000} { + t.Run(fmt.Sprintf("%s/%s/self=%v/upper=%x", name, bank, self, upper), func(t *testing.T) { + operands := bank + "1," + bank + "0" + if self { + operands = bank + "0," + bank + "0" + } + if !legacy { + operands += "," + bank + "0" + } + ins := x86ScalarLaneInstruction(t, name+" "+operands) + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z0": upper, "Z1": 0}, flags: x86ByteFlagsAll} + mask := x86CallerVectorByteMask(int64(width)) + if !self { + state.vectors["Z0"] |= mask + state.vectors["Z1"] = mask + } + if err := fixedLocalVectorStep(&ins, &state, nil, 0); err != nil { + t.Fatal("shared compare effect was not consumed", err) + } + want := ^uint64(0) + if legacy { + want = upper | mask + } + if state.vectors["Z0"] != want || state.flags != x86ByteFlagsAll { + t.Fatal("comparison changed upper aliases or architectural FLAGS", state) + } + }) + } + } + } + } +} + +func TestGoAMD64FixedLocalIntegerCompareConsumerBoundaries(t *testing.T) { + for index, name := range x86PackedIntegerCompareTestNames { + legacy := index < 8 + banks := []string{"X", "Y"} + if legacy { + banks = banks[:1] + } + for _, bank := range banks { + operands := "64(BP)," + bank + "1" + if !legacy { + operands += "," + bank + "2" + } + _, sig, source := fixedLocalVEXFixture(t, name+" "+operands) + changes := []struct{ name, from, to string }{ + {"missing byte", "MOVQ AX,64(BP)", "MOVB AX,64(BP)"}, + {"private pointer", "MOVQ AX,64(BP)", "MOVQ SP,64(BP)"}, + {"missing second input", "VMOVDQU 64(BP),Y1", "NOP"}, + {"outside frame", name + " 64(BP)", name + " 512(BP)"}, + } + if legacy { + changes = append(changes, struct{ name, from, to string }{"unaligned legacy", name + " 64(BP)", name + " 65(BP)"}) + } + for _, change := range changes { + t.Run(name+"/"+bank+"/"+change.name, func(t *testing.T) { + changed := strings.Replace(source, change.from, change.to, 1) + if changed == source { + t.Fatal("complete-source mutation did not change its intended input") + } + file, err := Parse(ArchAMD64, changed) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("compare supplied missing private storage or input authority", err) + } + }) + } + } + } + // These valid ISA forms have separate MMX or EVEX mask-result consumers. + // Descriptor metadata must not silently grant fixed-frame vector storage. + var separate []string + for index, name := range x86PackedIntegerCompareTestNames { + if index < 8 { + if index%4 != 3 { + separate = append(separate, name+" M0,M1") + } + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + separate = append(separate, name+" "+bank+"0,"+bank+"1,K0", name+" "+bank+"0,"+bank+"1,K1,K2") + } + } + for _, name := range x86ImmediateCompareResultMembers { + for _, bank := range []string{"X", "Y", "Z"} { + separate = append(separate, name+" $255,"+bank+"0,"+bank+"1,K0", name+" $7,"+bank+"0,"+bank+"1,K1,K2") + } + } + for _, operation := range separate { + t.Run(operation, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, operation) + effect, known, err := x86TypedPackedIntegerCompareByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || effect.vector != (x86TypedVectorEffect{}) { + t.Fatal("separate valid ISA form changed grammar or gained vector storage", err, effect) + } + if _, known, err := fixedLocalVectorByteEffect(ins); known && err == nil { + t.Fatal("MMX or EVEX mask result acquired this fixed-frame vector consumer") + } + }) + } +} diff --git a/amd64_fixed_local_layout.go b/amd64_fixed_local_layout.go new file mode 100644 index 000000000..0cbd68662 --- /dev/null +++ b/amd64_fixed_local_layout.go @@ -0,0 +1,185 @@ +package plan9asm + +import "reflect" + +// Nodes are source CFG boundaries with distinct private layouts. Edge order is +// the source CFG order; -1 is infeasible and -2 is an ordinary final return. +type x86FixedLocalLayoutNode struct { + pc, end int + targets [2]int +} + +type x86FixedLocalLayoutGraph struct { + nodes []x86FixedLocalLayoutNode + retained int64 // First-phase graph and new layout series during final replay. + series int64 // Newly allocated layout series in the previous residue. +} + +type x86FixedLocalLayouts struct { + reference []x86FixedLocalLayoutNode + nodes []x86FixedLocalLayoutNode + at map[int][]int +} + +func newFixedLocalLayouts(reference []x86FixedLocalLayoutNode) x86FixedLocalLayouts { + count := len(reference) + if count == 0 { + count = 1 + } + result := x86FixedLocalLayouts{reference: reference, nodes: make([]x86FixedLocalLayoutNode, count), at: map[int][]int{0: {0}}} + for index := range result.nodes { + result.nodes[index].targets = [2]int{-1, -1} + if len(reference) != 0 { + result.nodes[index].pc = reference[index].pc + if index != 0 { + pc := reference[index].pc + result.at[pc] = append(result.at[pc], index) + } + } + } + return result +} + +func (layouts *x86FixedLocalLayouts) target(states map[int]x86FixedLocalState, from, edge, pc int, next x86FixedLocalState, + budget *x86FixedLocalBudget) (int, error) { + match := -1 + for _, index := range layouts.at[pc] { + prior, found := states[index] + if err := budget.consumeOrigins(64 + 2*fixedLocalStateWork(prior) + fixedLocalStateWork(next)); err != nil { + return 0, err + } + if !found { + continue + } + if err := fixedLocalLayoutRoles(prior, next); err != nil { + return 0, err + } + if reflect.DeepEqual(prior.pointers, next.pointers) && reflect.DeepEqual(prior.addresses, next.addresses) { + match = index + } + } + if len(layouts.reference) != 0 { + match = layouts.reference[from].targets[edge] + if match < 0 || match >= len(layouts.nodes) || layouts.nodes[match].pc != pc { + return 0, goAMD64FrameContext("fixed local layout edge depends on original SP residue") + } + } else if match < 0 { + match = len(layouts.nodes) + layouts.nodes = append(layouts.nodes, x86FixedLocalLayoutNode{pc: pc, targets: [2]int{-1, -1}}) + } + if _, found := states[match]; !found && len(layouts.reference) == 0 { + layouts.at[pc] = append(layouts.at[pc], match) + } + layouts.nodes[from].targets[edge] = match + return match, nil +} + +func fixedLocalLayoutRoles(first, last x86FixedLocalState) error { + if len(first.pointers) != len(last.pointers) || len(first.addresses) != len(last.addresses) || first.pointers[SP] != last.pointers[SP] { + return goAMD64FrameContext("fixed local CFG private roles disagree") + } + for reg := range first.pointers { + if _, found := last.pointers[reg]; !found { + return goAMD64FrameContext("fixed local CFG private roles disagree") + } + } + for offset := range first.addresses { + if _, found := last.addresses[offset]; !found { + return goAMD64FrameContext("fixed local CFG private spill roles disagree") + } + } + _, err := mergeFixedLocalSavedBP(&first, last) + return err +} + +func (layouts *x86FixedLocalLayouts) finish() error { + seen, work := map[int]bool{0: true}, []int{0} + for head := 0; head < len(work); head++ { + for _, target := range layouts.nodes[work[head]].targets { + if target >= 0 && !seen[target] { + seen[target] = true + work = append(work, target) + } + } + } + if len(seen) != len(layouts.nodes) || len(layouts.reference) != 0 && !reflect.DeepEqual(layouts.reference, layouts.nodes) { + return goAMD64FrameContext("fixed local final layout graph has new, missing or provisional edges") + } + return nil +} + +// These are bounded logical payload charges, not an estimate of Go heap RSS. +// The node charge includes current/reference arrays with spare capacity, source +// indexes and the observation index. A completed first phase is reserved apart. +// Map bucket/allocator overhead remains outside this logical payload bound. +const fixedLocalLayoutNodeBytes int64 = 192 + +func fixedLocalLayoutAccessBytes(accesses map[int]x86FixedLocalAccess) int64 { + bytes := int64(len(accesses)) * 192 + for _, access := range accesses { + bytes += int64(cap(access.iterationOffsets)) * 64 + } + return bytes +} + +func fixedLocalAccessShapeEqual(first, last x86FixedLocalAccess) bool { + return first.owner == last.owner && first.shape == last.shape && first.width == last.width && + first.write == last.write && first.nativeInput == last.nativeInput && first.ordinal == last.ordinal +} + +func collectFixedLocalLayoutAccesses(nodes int, observations map[int]map[int]map[int]x86FixedLocalAccess, + available int64) (map[int]map[int]x86FixedLocalAccess, int64, error) { + result := make(map[int]map[int]x86FixedLocalAccess) + groups := make(map[int][]map[int]x86FixedLocalAccess) + for node := 0; node < nodes; node++ { + for pc, current := range observations[node] { + if first, found := result[pc]; found { + if len(groups[pc]) == 0 { + groups[pc] = append(groups[pc], first) + } + groups[pc] = append(groups[pc], current) + } else { + result[pc] = current + } + } + } + available -= int64(len(result)) * 16 // Temporary source-PC index, not old certificates. + for _, group := range groups { + available -= 40 + int64(cap(group))*8 + } + var allocated int64 + for pc, group := range groups { + for operand, first := range result[pc] { + count := 0 + for _, current := range group { + access, found := current[operand] + if len(current) != len(result[pc]) || !found || !fixedLocalAccessShapeEqual(first, access) { + return nil, 0, goAMD64FrameContext("fixed local layout access role changed") + } + count += len(access.iterationOffsets) + if len(access.iterationOffsets) == 0 { + count++ + } + } + allocated += int64(count) * 64 + if allocated > available { + return nil, 0, goAMD64FrameContext("fixed local retained state storage budget exceeded") + } + offsets := make([][8]int64, 0, count) + for _, current := range group { + access := current[operand] + if len(access.iterationOffsets) == 0 { + offsets = append(offsets, access.offset) + } else { + offsets = append(offsets, access.iterationOffsets...) + } + } + first.iterationOffsets = offsets + result[pc][operand] = first + } + } + if allocated > available { + return nil, 0, goAMD64FrameContext("fixed local retained state storage budget exceeded") + } + return result, allocated, nil +} diff --git a/amd64_fixed_local_layout_boundary_test.go b/amd64_fixed_local_layout_boundary_test.go new file mode 100644 index 000000000..57ee20822 --- /dev/null +++ b/amd64_fixed_local_layout_boundary_test.go @@ -0,0 +1,19 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalOrdinaryRetainedBoundary(t *testing.T) { + var body strings.Builder + for index := 0; index < 15; index++ { + fmt.Fprintf(&body, "JMP next%d\nnext%d:\n", index, index) + } + fn, sig := fixedLocalReplayFixture(t, body.String()) + fn.FrameSize = 64 << 10 + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("sixteen ordinary states must retain the original one-MiB allowance", err) + } +} diff --git a/amd64_fixed_local_layout_join_test.go b/amd64_fixed_local_layout_join_test.go new file mode 100644 index 000000000..8de42b091 --- /dev/null +++ b/amd64_fixed_local_layout_join_test.go @@ -0,0 +1,127 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +const fixedLocalLayoutPingPong = ` +LEAQ 16(SP),R14 +LEAQ 24(SP),R15 +MOVQ $1,(R14) +loop: +CMPQ CX,$0 +JEQ done +MOVQ (R14),AX +ADDQ $1,AX +MOVQ AX,(R15) +XCHGQ R14,R15 +SUBQ $1,CX +JMP loop +done: +MOVQ (R14),AX +MOVQ AX,ret+16(FP) +RET +` + +const fixedLocalLayoutDeadReturn = ` +MOVQ $7,ret+16(FP) +CMPQ CX,$0 +JEQ done +LEAQ 16(SP),AX +done: +RET +` + +func fixedLocalTerminalReturnBodies() []string { + return []string{fixedLocalLayoutDeadReturn, strings.Replace(fixedLocalLayoutDeadReturn, + "LEAQ 16(SP),AX", "LEAQ 24(SP),AX\nMOVQ AX,16(SP)", 1)} +} + +func TestGoAMD64FixedLocalTerminalJoinPrograms(t *testing.T) { + rows := []struct { + name, body string + accept bool + }{ + {"dead private register at return", fixedLocalTerminalReturnBodies()[0], true}, + {"dead private spill at return", fixedLocalTerminalReturnBodies()[1], true}, + {"undefined joined register is used", strings.Replace(fixedLocalLayoutDeadReturn, + "done:\nRET", "done:\nMOVQ (AX),BX\nRET", 1), false}, + {"result is missing", strings.Replace(fixedLocalLayoutDeadReturn, + "MOVQ $7,ret+16(FP)\n", "", 1), false}, + {"only one predecessor writes the result", ` +CMPQ CX,$0 +JEQ missing +MOVQ $7,ret+16(FP) +JMP done +missing: +NOP +done: +RET +`, false}, + {"private SP is not restored", strings.Replace(fixedLocalLayoutDeadReturn, + "LEAQ 16(SP),AX", "LEAQ 8(SP),AX\nMOVQ AX,SP", 1), false}, + {"private and native roles conflict", ` +CMPQ CX,$0 +JEQ native +LEAQ 16(SP),AX +MOVQ $1,(AX) +JMP joined +native: +MOVQ p+0(FP),AX +joined: +MOVQ (AX),BX +MOVQ BX,ret+16(FP) +RET +`, false}, + {"initial buffer is undefined", strings.Replace(fixedLocalLayoutPingPong, + "MOVQ $1,(R14)\n", "", 1), false}, + {"next buffer is not written", strings.Replace(fixedLocalLayoutPingPong, + "MOVQ AX,(R15)\n", "", 1), false}, + } + functions := make([]Func, len(rows)) + signatures := make([]FuncSig, len(rows)) + var original strings.Builder + for index, row := range rows { + _, _, sig, source := x86ResultInitializationFixture(t, "amd64", + "func Value(p *uint64, n uint64) uint64", "MOVQ n+8(FP),CX\n"+row.body) + source = strings.Replace(source, "$0-24", "$64-24", 1) + file, err := Parse(ArchAMD64, source) + if err != nil || len(file.Funcs) != 1 || file.Funcs[0].FrameSize != 64 { + t.Fatal("complete source fixture has no expected private frame", err) + } + functions[index], signatures[index] = file.Funcs[0], sig + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + // All complete source functions must assemble before their distinct + // translation outcomes are checked. Source legality is not a memory proof. + goAssemblerOracleOutput(t, "amd64", original.String(), "", true) + for index, row := range rows { + t.Run(row.name, func(t *testing.T) { + _, err := proveX86FixedLocal(functions[index], signatures[index]) + if row.accept && err != nil { + t.Fatal("complete initialized source lost a valid private layout", err) + } + if !row.accept && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved source gained a private frame", err) + } + }) + } +} + +func fixedLocalTerminalReturnProgram() (assembly, declarations string) { + var asm, decl strings.Builder + for index, body := range fixedLocalTerminalReturnBodies() { + body = strings.ReplaceAll(body, "ret+16(FP)", "ret+8(FP)") + fmt.Fprintf(&asm, "TEXT ·Value%d(SB),4,$64-16\nMOVQ a+0(FP),CX\n%s", index, body) + fmt.Fprintf(&decl, "func Value%d(a uint64) uint64\n", index) + } + return asm.String(), decl.String() +} + +func TestGoAMD64FixedLocalTerminalJoinObjects(t *testing.T) { + assembly, declarations := fixedLocalTerminalReturnProgram() + fixedLocalCompleteVectorObjects(t, assembly, declarations, 2) +} diff --git a/amd64_fixed_local_layout_partition_test.go b/amd64_fixed_local_layout_partition_test.go new file mode 100644 index 000000000..16dfe98e2 --- /dev/null +++ b/amd64_fixed_local_layout_partition_test.go @@ -0,0 +1,251 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func fixedLocalLayoutPartitionBodies() []string { + three := strings.Replace(fixedLocalLayoutPingPong, + "MOVQ $1,(R14)", "LEAQ 32(SP),R13\nMOVQ R14,R12\nMOVQ $1,(R14)", 1) + three = strings.Replace(three, "XCHGQ R14,R15", + "MOVQ R14,R12\nMOVQ R15,R14\nMOVQ R13,R15\nMOVQ R12,R13", 1) + spill := strings.Replace(fixedLocalLayoutPingPong, "MOVQ $1,(R14)", + "MOVQ R14,32(SP)\nMOVQ R15,40(SP)\nMOVQ $1,(R14)", 1) + spill = strings.Replace(spill, "loop:\n", "loop:\nMOVQ 32(SP),R14\nMOVQ 40(SP),R15\n", 1) + spill = strings.Replace(spill, "XCHGQ R14,R15", + "MOVQ R15,32(SP)\nMOVQ R14,40(SP)", 1) + return []string{fixedLocalLayoutPingPong, three, spill} +} + +func fixedLocalLayoutPartitionProgram() (assembly, declarations string) { + var asm, decl strings.Builder + for index, body := range fixedLocalLayoutPartitionBodies() { + body = strings.ReplaceAll(body, "ret+16(FP)", "ret+8(FP)") + fmt.Fprintf(&asm, "TEXT ·Value%d(SB),4,$64-16\nMOVQ a+0(FP),CX\n%s", index, body) + fmt.Fprintf(&decl, "func Value%d(a uint64) uint64\n", index) + } + return asm.String(), decl.String() +} + +func TestGoAMD64FixedLocalLayoutPartitionPrograms(t *testing.T) { + assembly, _ := fixedLocalLayoutPartitionProgram() + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil || len(file.Funcs) != 3 { + t.Fatal("complete private-layout source inventory changed", err) + } + _, sig := fixedLocalReplayFixture(t, "") + for index, fn := range file.Funcs { + t.Run(fmt.Sprint(index), func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("current initialized buffer lost its matching private layout", err) + } + }) + } +} + +func TestGoAMD64FixedLocalLayoutPartitionObjects(t *testing.T) { + assembly, declarations := fixedLocalLayoutPartitionProgram() + fixedLocalCompleteVectorObjects(t, assembly, declarations, 3) +} + +func TestGoAMD64FixedLocalLayoutPartitionResidueAliases(t *testing.T) { + body := strings.Replace(fixedLocalLayoutPingPong, "LEAQ 16(SP),R14", + "MOVQ $77,16(SP)\nLEAQ 16(SP),R14\nANDQ $-16,R14", 1) + body = strings.Replace(body, "MOVQ AX,ret+16(FP)", + "ADDQ 16(SP),AX\nMOVQ AX,ret+16(FP)", 1) + _, _, sig, source := x86ResultInitializationFixture(t, "amd64", + "func Value(p *uint64, n uint64) uint64", "MOVQ n+8(FP),CX\n"+body) + source = strings.Replace(source, "$0-24", "$64-24", 1) + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "byte identity depends") { + t.Fatal("layout alternatives lost the canary's residue-dependent alias", err) + } +} + +func TestGoAMD64FixedLocalLayoutPartitionStorageBound(t *testing.T) { + var body strings.Builder + for index := 0; index < 8; index++ { + fmt.Fprintf(&body, "LEAQ %d(SP),R%d\n", index*8, index+8) + } + body.WriteString("loop:\nTESTQ $1,AX\nJZ rotate\nXCHGQ R8,R9\nJMP count\nrotate:\n") + for index := 8; index < 15; index++ { + fmt.Fprintf(&body, "XCHGQ R%d,R%d\n", index, index+1) + } + body.WriteString("count:\nSUBQ $1,AX\nJNZ loop\n") + for _, frame := range []int64{64, 32 << 10} { + t.Run(fmt.Sprint(frame), func(t *testing.T) { + fn, sig := fixedLocalReplayFixture(t, body.String()) + fn.FrameSize = frame + proof := &x86FixedLocalProof{frame: frame} + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + var layouts x86FixedLocalLayoutGraph + err := replayX86FixedLocal(&fn, sig, proof, 0, &budget, nil, &layouts) + t.Logf("frame=%d remaining_steps=%d remaining_bytes=%d error=%v", frame, budget.steps, budget.bytes, err) + failure := "state storage budget" + if frame == 64 { + // The original small-frame stress now pays for its candidate + // comparisons first. Larger frames isolate the retained cap. + failure = "state byte bound" + } + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), failure) { + t.Fatal("independent work/retained layout limit was not enforced", err) + } + if budget.steps <= 0 || budget.bytes <= 0 { + t.Fatal("layout bounds must reject before instruction/work counters underflow") + } + for _, ins := range fn.Instrs { + if len(ins.x86FixedLocalMemory) != 0 { + t.Fatal("layout storage failure published provisional accesses") + } + } + }) + } +} + +func TestGoAMD64FixedLocalLayoutPartitionFrozenEdges(t *testing.T) { + assembly, _ := fixedLocalLayoutPartitionProgram() + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + _, sig := fixedLocalReplayFixture(t, "") + fn := file.Funcs[0] + proof := &x86FixedLocalProof{frame: fn.FrameSize} + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + var layouts x86FixedLocalLayoutGraph + if err := replayX86FixedLocal(&fn, sig, proof, 0, &budget, nil, &layouts); err != nil { + t.Fatal(err) + } + for _, mutation := range []string{"missing edge", "different block", "extra node", "other layout"} { + t.Run(mutation, func(t *testing.T) { + changed := layouts + changed.nodes = append([]x86FixedLocalLayoutNode(nil), layouts.nodes...) + switch mutation { + case "missing edge": + changed.nodes[0].targets = [2]int{-1, -1} + case "different block": + changed.nodes[0].end++ + case "extra node": + changed.nodes = append(changed.nodes, x86FixedLocalLayoutNode{targets: [2]int{-1, -1}}) + case "other layout": + changedEdge := false + for index, node := range changed.nodes { + for edge, target := range node.targets { + for other, candidate := range changed.nodes { + if !changedEdge && target >= 0 && target != other && candidate.pc == changed.nodes[target].pc { + changed.nodes[index].targets[edge], changedEdge = other, true + } + } + } + } + if !changedEdge { + t.Fatal("fixture has no separate private layouts at the same source PC") + } + } + before := append([]Instr(nil), fn.Instrs...) + for index := range before { + before[index].x86FixedLocalMemory = cloneX86ReplayFixedLocalMemory(before[index].x86FixedLocalMemory) + } + current := fn + current.Instrs = append([]Instr(nil), fn.Instrs...) + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + err := replayX86FixedLocal(¤t, sig, proof, 1, &budget, nil, &changed) + if mutation == "other layout" { + if err != nil { + t.Fatal("one residue can establish the relabeled cycle locally", err) + } + // Relabeling a cycle is locally consistent. Its complete byte + // correspondence and independent final graph are the consumers + // that must reject a different layout relative to the prefix. + for residue := 2; residue < 8; residue++ { + if err := replayX86FixedLocal(¤t, sig, proof, residue, &budget, nil, &changed); err != nil { + t.Fatal("complete-residue replay", err) + } + } + accesses := make(map[int]map[int]x86FixedLocalAccess) + anchors, mismatches := 0, 0 + for index, instruction := range current.Instrs { + if len(instruction.x86FixedLocalMemory) != 0 { + accesses[index] = instruction.x86FixedLocalMemory + for _, access := range instruction.x86FixedLocalMemory { + if instruction.Raw == "MOVQ $1,(R14)" && access.offset[0] == 16 && access.offset[1] == 16 { + anchors++ + } + if instruction.Raw == "MOVQ (R14),AX" && len(access.iterationOffsets) == 2 && + access.iterationOffsets[0][0] == 16 && access.iterationOffsets[0][1] == 24 { + mismatches++ + } + } + } + } + if anchors != 1 || mismatches != 2 || current.x86FixedLocal != nil { + t.Fatal("fixture lost its 16-to-16 prefix versus 16-to-24 reads, or published an unfinished proof") + } + if err := fixedLocalRelocationAliases(accesses); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("one original byte acquired two relocated identities", err) + } + freshFile, parseErr := Parse(ArchAMD64, assembly) + if parseErr != nil { + t.Fatal(parseErr) + } + fresh := freshFile.Funcs[0] + if _, err := proveX86FixedLocal(fresh, sig); err != nil { + t.Fatal("the unchanged complete source must still succeed", err) + } + freshProof := &x86FixedLocalProof{frame: fresh.FrameSize} + freshBudget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + var freshGraph x86FixedLocalLayoutGraph + for residue := 0; residue < 8; residue++ { + if err := replayX86FixedLocal(&fresh, sig, freshProof, residue, &freshBudget, nil, &freshGraph); err != nil { + t.Fatal("independent final graph rebuild", err) + } + } + if !reflect.DeepEqual(freshGraph.nodes, layouts.nodes) || reflect.DeepEqual(freshGraph.nodes, changed.nodes) { + t.Fatal("independent final replay accepted the changed prefix/layout correspondence") + } + return + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a different residue acquired changed layout correspondence", err) + } + if !reflect.DeepEqual(before, current.Instrs) { + t.Fatal("failed layout replay published a provisional observation") + } + }) + } +} + +func TestGoAMD64FixedLocalLayoutPartitionLateResidue(t *testing.T) { + fn, sig := fixedLocalReplayFixture(t, `LEAQ 16(SP),BX +CMPQ AX,$0 +JEQ unchanged +ANDQ $-16,BX +unchanged: +MOVQ AX,(BX)`) + original := fn + original.Instrs = append([]Instr(nil), fn.Instrs...) + proof := &x86FixedLocalProof{frame: fn.FrameSize} + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + if err := replayX86FixedLocal(&fn, sig, proof, 0, &budget, nil, nil); err != nil { + t.Fatal("residue zero has a single private layout", err) + } + if err := replayX86FixedLocal(&fn, sig, proof, 1, &budget, nil, nil); !errors.Is(err, errFixedLocalLayoutJoin) { + t.Fatal("a later residue must expose the distinct private layouts", err) + } + // A fresh partition attempt still cannot match this source's differing + // residue graphs. It must not mix earlier flat and later partition records. + got, err := proveX86FixedLocal(original, sig) + if !errors.Is(err, ErrProbeNeedsContext) || !reflect.DeepEqual(got, original) { + t.Fatal("late partition failure published an incomplete source proof", err) + } +} diff --git a/amd64_fixed_local_logical_literal_go127_test.go b/amd64_fixed_local_logical_literal_go127_test.go new file mode 100644 index 000000000..322eb3609 --- /dev/null +++ b/amd64_fixed_local_logical_literal_go127_test.go @@ -0,0 +1,150 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func fixedLocalLogicalLiteralRequireBinding(t *testing.T, assembly string) { + t.Helper() + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value(input, output *byte)\n", "linux/amd64", "x86_64-unknown-linux-gnu") + // A real successful original-Go observation precedes the translator + // assertion. Assembler/tool failures cannot stand in for this RED. + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); err != nil { + t.Fatal("complete original source lost constant-mask byte dependencies", err) + } +} + +func TestGoAMD64FixedLocalLogicalLiteralPartialInsert(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + fixedLocalLogicalLiteralRequireBinding(t, + fixedLocalLogicalLiteralSource(fixedLocalLogicalLiteralCase{"PAND", 0, 16, false}, false)) +} + +func TestGoAMD64FixedLocalLogicalLiteralCompleteFormats(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + rows := fixedLocalLogicalLiteralCases() + if len(rows) != 36 { + t.Fatal("independent logical family/legacy-X/VEX-X-Y inventory changed", len(rows)) + } + for _, row := range rows { + t.Run(fmt.Sprintf("%s/%d", row.opcode, row.width), func(t *testing.T) { + fixedLocalLogicalLiteralRequireBinding(t, fixedLocalLogicalLiteralSource(row, false)) + }) + } +} + +func TestGoAMD64FixedLocalLogicalLiteralNoRegisterRead(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + count := 0 + for _, row := range fixedLocalLogicalLiteralCases() { + // Select the integer spelling once per operation and vector width. + name := strings.TrimPrefix(row.opcode, "V") + if name != "PAND" && name != "PANDN" && name != "POR" { + continue + } + count++ + t.Run(fmt.Sprintf("%s/%d", row.opcode, row.width), func(t *testing.T) { + fixedLocalLogicalLiteralRequireBinding(t, fixedLocalLogicalLiteralSource(row, true)) + }) + } + if count != 9 { + t.Fatal("complete constant annihilator inventory changed", count) + } +} + +func TestGoAMD64FixedLocalLogicalLiteralAccessGuards(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + and := fixedLocalLogicalLiteralSource(fixedLocalLogicalLiteralCase{"PAND", 0, 16, false}, false) + andNot := fixedLocalLogicalLiteralSource(fixedLocalLogicalLiteralCase{"PANDN", 1, 16, false}, false) + or := fixedLocalLogicalLiteralSource(fixedLocalLogicalLiteralCase{"POR", 2, 16, false}, false) + xor := fixedLocalLogicalLiteralSource(fixedLocalLogicalLiteralCase{"PXOR", 3, 16, false}, false) + zero := fixedLocalLogicalLiteralSource(fixedLocalLogicalLiteralCase{"PAND", 0, 16, false}, true) + for _, row := range []struct{ name, source string }{ + {"AND retains one unknown bit", strings.Replace(and, "/8,$0x0\n", "/8,$0x1\n", 1)}, + {"ANDNOT retains one unknown bit", strings.Replace(andNot, "/8,$0x0\n", "/8,$0x1\n", 1)}, + {"OR retains one unknown bit", strings.Replace(or, "$0xffffffffffffffff", "$0xfffffffffffffffe", 1)}, + {"XOR keeps every input", strings.Replace(xor, "MOVOU (DI),X5", "PINSRD $3,(DI),X5", 1)}, + {"missing selected lane", strings.Replace(and, "PINSRD $3,(DI),X5\n", "", 1)}, + {"zero still needs extent", strings.Replace(zero, "PAND mask<>(SB)", "PAND mask<>+1(SB)", 1)}, + {"zero still needs alignment", strings.Replace(strings.Replace(zero, + "GLOBL mask<>(SB),8,$16", "GLOBL mask<>(SB),8,$32", 1), "PAND mask<>(SB)", "PAND mask<>+1(SB)", 1)}, + {"mutable bytes are not constants", strings.Replace(zero, "GLOBL mask<>(SB),8,", "GLOBL mask<>(SB),0,", 1)}, + {"native memory supplies no constant", strings.Replace(and, "PAND mask<>(SB)", "PAND (DI)", 1)}, + {"register value is not tracked constant", strings.Replace(and, + "PAND mask<>(SB)", "MOVOU mask<>(SB),X7\nPAND X7", 1)}, + {"readonly write remains forbidden", strings.Replace(zero, + "MOVOU X5,0(SP)", "MOVOU X5,mask<>(SB)\nMOVOU X5,0(SP)", 1)}, + {"numeric result supplies no pointer", strings.Replace(zero, + "MOVOU X5,0(SP)", "MOVQ X5,BX\nMOVQ (BX),AX\nMOVOU X5,0(SP)", 1)}, + } { + t.Run(row.name, func(t *testing.T) { + pkg, request, file, options := fixedLocalTransportFixture(t, row.source, + "func Value(input, output *byte)\n", "linux/amd64", "x86_64-unknown-linux-gnu") + if row.name == "numeric result supplies no pointer" { + fixedLocalRequireNumericControl(t, file.Funcs[0], options.Sigs[pkg.Path+".Value"], row.source, + "func Value(input,output *byte)", "MOVQ (BX),AX", 0, 8, false, "load volatile i64") + return + } + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("constant mask removed an independent input or storage requirement", err) + } + }) + } +} + +func TestGoAMD64FixedLocalLogicalLiteralOwnerLifetime(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + assembly := fixedLocalLogicalLiteralSource(fixedLocalLogicalLiteralCase{"PAND", 0, 16, false}, false) + for _, mutation := range []string{"unbound", "public DATA", "original DATA", "closed", "cancelled"} { + t.Run(mutation, func(t *testing.T) { + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value(input, output *byte)\n", "linux/amd64", "x86_64-unknown-linux-gnu") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("valid owner control failed before lifetime mutation", err) + } + input := assembly + switch mutation { + case "unbound": + bound = options + case "public DATA": + input = strings.Replace(assembly, "/8,$0x0\n", "/8,$0x1\n", 1) + file, err = Parse(ArchAMD64, input) + if err != nil { + t.Fatal(err) + } + case "original DATA": + path := filepath.Join(request.Directory, request.AssemblyFile) + changed := strings.Replace(assembly, "/8,$0x0\n", "/8,$0x1\n", 1) + if err := os.WriteFile(path, []byte(changed), 0600); err != nil { + t.Fatal(err) + } + case "closed": + if err := source.Close(); err != nil { + t.Fatal(err) + } + case "cancelled": + cancel() + } + // Retain the original binding, including its revoked lifetime. + fixedLocalDataAPIs(t, pkg, file, input, bound, fixedLocalRequireContext) + }) + } +} diff --git a/amd64_fixed_local_logical_literal_objects_go127_test.go b/amd64_fixed_local_logical_literal_objects_go127_test.go new file mode 100644 index 000000000..b944538f0 --- /dev/null +++ b/amd64_fixed_local_logical_literal_objects_go127_test.go @@ -0,0 +1,87 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type fixedLocalLogicalLiteralKernel struct { + name string + row fixedLocalLogicalLiteralCase + annihilate bool +} + +func fixedLocalLogicalLiteralKernels(t *testing.T) []fixedLocalLogicalLiteralKernel { + t.Helper() + var kernels []fixedLocalLogicalLiteralKernel + for _, row := range fixedLocalLogicalLiteralCases() { + kernels = append(kernels, fixedLocalLogicalLiteralKernel{row: row}) + name := strings.TrimPrefix(row.opcode, "V") + if name == "PAND" || name == "PANDN" || name == "POR" { + kernels = append(kernels, fixedLocalLogicalLiteralKernel{row: row, annihilate: true}) + } + } + if len(kernels) != 45 { + t.Fatal("complete logical literal source denominator changed", len(kernels)) + } + for index := range kernels { + kernels[index].name = fmt.Sprintf("Value%d", index) + } + return kernels +} + +func fixedLocalLogicalLiteralProgram(kernels []fixedLocalLogicalLiteralKernel) (string, string) { + var assembly, declarations strings.Builder + for _, kernel := range kernels { + source := fixedLocalLogicalLiteralSource(kernel.row, kernel.annihilate) + source = strings.ReplaceAll(source, "mask<>", kernel.name+"Mask<>") + source = strings.Replace(source, "·Value", "·"+kernel.name, 1) + assembly.WriteString(source) + fmt.Fprintf(&declarations, "func %s(input, output *byte)\n", kernel.name) + } + return assembly.String(), declarations.String() +} + +func TestGoAMD64FixedLocalLogicalLiteralThreeTargetObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + kernels := fixedLocalLogicalLiteralKernels(t) + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + for start := 0; start < len(kernels); start += 24 { + end := start + 24 + if end > len(kernels) { + end = len(kernels) + } + t.Run(fmt.Sprint(start/24), func(t *testing.T) { + assembly, declarations := fixedLocalLogicalLiteralProgram(kernels[start:end]) + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("complete constant-mask logical program did not translate", err) + } + var names []string + for _, kernel := range kernels[start:end] { + names = append(names, pkg.Path+"."+kernel.name) + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + compileLLVMToObject(t, llc, target.triple, "logical-literal.ll", "logical-literal.o", ir) + compileLLVMToObject(t, llc, target.triple, "logical-literal-opt.ll", "logical-literal-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + }) + } + t.Log("45 complete kernels: 36 formats and 9 annihilators; 3 targets, 3 independent API owners, O0/O2; no execution claim") +} diff --git a/amd64_fixed_local_logical_literal_runtime_go127_test.go b/amd64_fixed_local_logical_literal_runtime_go127_test.go new file mode 100644 index 000000000..0a898929d --- /dev/null +++ b/amd64_fixed_local_logical_literal_runtime_go127_test.go @@ -0,0 +1,200 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func fixedLocalLogicalLiteralOracles(kernels []fixedLocalLogicalLiteralKernel, triple string) (string, string) { + var goRows, declarations, cRows strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for _, kernel := range kernels { + annihilate := 0 + if kernel.annihilate { + annihilate = 1 + } + fmt.Fprintf(&goRows, "{%s,%d,%d,%t},\n", kernel.name, kernel.row.width, kernel.row.mode, kernel.annihilate) + fmt.Fprintf(&declarations, "extern void %s(const uint8_t *,uint8_t *) __asm__(%q);\n", + kernel.name, prefix+"example.invalid/framefixture."+kernel.name) + fmt.Fprintf(&cRows, "{%s,%d,%d,%d},\n", kernel.name, kernel.row.width, kernel.row.mode, annihilate) + } + original := `package framefixture + +import "testing" + +func TestOriginal(t *testing.T) { + rows := []struct { + call func(*byte, *byte) + width, mode int + annihilate bool + }{ +` + goRows.String() + ` + } + count := 0 + for row, test := range rows { + for sample := 0; sample < 256; sample++ { + for alias := 0; alias < 2; alias++ { + var input, output [96]byte + for at := range input { + input[at] = byte(at*29 + sample) + output[at] = byte(at*7 + sample + 151) + } + before := input + wantInput, wantOutput := input, output + destination := &output[17] + if alias != 0 { + destination = &input[17] + } + for at := 0; at < test.width; at++ { + selected := !test.annihilate && at >= 12 && at < 16 + var first, second byte + if selected { + second = before[24+at-12] + } + switch test.mode { + case 0, 1: + if selected { first = 255 } + case 2: + if !selected { first = 255 } + case 3: + first, second = byte(at*13+7), before[24+at] + default: + t.Fatal("unknown operation") + } + var want byte + switch test.mode { + case 0: want = first & second + case 1: want = first & ^second + case 2: want = first | second + case 3: want = first ^ second + } + if alias != 0 { + wantInput[17+at] = want + } else { + wantOutput[17+at] = want + } + } + test.call(&input[24], destination) + if input != wantInput || output != wantOutput { + t.Fatalf("row=%d sample=%d alias=%d input=%x/%x output=%x/%x", row, sample, alias, + input, wantInput, output, wantOutput) + } + count++ + } + } + } + if count != len(rows)*256*2 { t.Fatal("incomplete source execution", count) } +} +` + driver := `#include +#include +#include +` + declarations.String() + ` +static const struct { + void (*call)(const uint8_t *, uint8_t *); + unsigned width, mode, annihilate; +} cases[] = { +` + cRows.String() + ` +}; + +int main(void) { + unsigned count = 0; + for (unsigned row = 0; row < sizeof(cases)/sizeof(cases[0]); row++) { + for (unsigned sample = 0; sample < 256; sample++) { + for (unsigned alias = 0; alias < 2; alias++) { + uint8_t input[96], output[96], before[96], want_input[96], want_output[96]; + for (unsigned at = 0; at < sizeof(input); at++) { + input[at] = (uint8_t)(at*29 + sample); + output[at] = (uint8_t)(at*7 + sample + 151); + } + memcpy(before, input, sizeof(input)); + memcpy(want_input, input, sizeof(input)); + memcpy(want_output, output, sizeof(output)); + for (unsigned at = 0; at < cases[row].width; at++) { + unsigned selected = !cases[row].annihilate && at >= 12 && at < 16; + uint8_t first = 0, second = selected ? before[24+at-12] : 0, want; + switch (cases[row].mode) { + case 0: case 1: first = selected ? 255 : 0; break; + case 2: first = selected ? 0 : 255; break; + case 3: first = (uint8_t)(at*13+7); second = before[24+at]; break; + default: return 2; + } + switch (cases[row].mode) { + case 0: want = first & second; break; + case 1: want = first & (uint8_t)~second; break; + case 2: want = first | second; break; + case 3: want = first ^ second; break; + default: return 3; + } + if (alias) want_input[17+at] = want; + else want_output[17+at] = want; + } + cases[row].call(input+24, alias ? input+17 : output+17); + if (memcmp(input, want_input, sizeof(input)) || memcmp(output, want_output, sizeof(output))) { + fprintf(stderr, "row=%u sample=%u alias=%u full buffer/canary mismatch\n", row, sample, alias); + return 1; + } + count++; + } + } + } + if (count != sizeof(cases)/sizeof(cases[0])*256*2) return 4; + puts("logical literal values and complete canaries PASS"); + return 0; +} +` + return original, driver +} + +func fixedLocalLogicalLiteralRuntime(t *testing.T, execution x86FloatCompareExecution, execute bool) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + kernels := fixedLocalLogicalLiteralKernels(t) + for start := 0; start < len(kernels); start += 24 { + end := start + 24 + if end > len(kernels) { + end = len(kernels) + } + t.Run(fmt.Sprint(start/24), func(t *testing.T) { + assembly, declarations := fixedLocalLogicalLiteralProgram(kernels[start:end]) + original, driver := fixedLocalLogicalLiteralOracles(kernels[start:end], execution.triple) + fixedLocalPackedRunOriginal(t, execution, assembly, declarations, original, execute) + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, + runtime.GOOS+"/amd64", execution.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + if !execute { + execution.compileLLVM(t, "logical-literal-oracle", input, driver, level) + return + } + output := execution.compileAndRunLLVM(t, "logical-literal-oracle", input, driver, level) + if strings.TrimSpace(string(output)) != "logical literal values and complete canaries PASS" { + t.Fatal("incomplete original byte oracle execution", string(output)) + } + }) + } + }) + }) + } + t.Logf("45 original kernels, 256 samples, disjoint/overlapping inputs and outputs, both complete 96-byte buffers; execute=%t", execute) +} + +func TestGoAMD64FixedLocalLogicalLiteralRuntimeObjects(t *testing.T) { + fixedLocalLogicalLiteralRuntime(t, fixedLocalPackedCompileOnlyTools(t), false) +} diff --git a/amd64_fixed_local_logical_literal_runtime_linux_amd64_test.go b/amd64_fixed_local_logical_literal_runtime_linux_amd64_test.go new file mode 100644 index 000000000..1e3ad4eef --- /dev/null +++ b/amd64_fixed_local_logical_literal_runtime_linux_amd64_test.go @@ -0,0 +1,9 @@ +//go:build go1.27 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalLogicalLiterals(t *testing.T) { + fixedLocalLogicalLiteralRuntime(t, fixedLocalVEXNativeTools(t), true) +} diff --git a/amd64_fixed_local_logical_literal_test.go b/amd64_fixed_local_logical_literal_test.go new file mode 100644 index 000000000..2daab41e6 --- /dev/null +++ b/amd64_fixed_local_logical_literal_test.go @@ -0,0 +1,212 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "reflect" + "strings" + "testing" +) + +type fixedLocalLogicalLiteralCase struct { + opcode string + mode int // Independent oracle: AND, ANDNOT, OR, XOR. + width int + vex bool +} + +func fixedLocalLogicalLiteralCases() []fixedLocalLogicalLiteralCase { + var rows []fixedLocalLogicalLiteralCase + for mode, names := range [][3]string{ + {"PAND", "ANDPS", "ANDPD"}, + {"PANDN", "ANDNPS", "ANDNPD"}, + {"POR", "ORPS", "ORPD"}, + {"PXOR", "XORPS", "XORPD"}, + } { + for _, name := range names { + rows = append(rows, fixedLocalLogicalLiteralCase{name, mode, 16, false}) + for _, width := range []int{16, 32} { + rows = append(rows, fixedLocalLogicalLiteralCase{"V" + name, mode, width, true}) + } + } + } + return rows +} + +func fixedLocalLogicalLiteralSource(row fixedLocalLogicalLiteralCase, annihilate bool) string { + mask := make([]byte, row.width) + for index := range mask { + switch row.mode { + case 0, 1: + if !annihilate && index >= 12 && index < 16 { + mask[index] = 255 + } + case 2: + if annihilate || index < 12 || index >= 16 { + mask[index] = 255 + } + case 3: + mask[index] = byte(index*13 + 7) + } + } + var assembly strings.Builder + for offset := 0; offset < row.width; offset += 8 { + fmt.Fprintf(&assembly, "DATA mask<>+%d(SB)/8,$%#x\n", offset, binary.LittleEndian.Uint64(mask[offset:])) + } + fmt.Fprintf(&assembly, "GLOBL mask<>(SB),8,$%d\n", row.width) + assembly.WriteString("TEXT ·Value(SB),4,$32-16\nMOVQ input+0(FP),DI\nMOVQ output+8(FP),SI\n") + bank, move := "X", "MOVOU" + if row.vex { + move = "VMOVDQU" + if row.width == 32 { + bank = "Y" + } + } + if row.mode == 3 { + fmt.Fprintf(&assembly, "%s (DI),%s5\n", move, bank) + } else if !annihilate { + // The other physical vector bytes deliberately remain undefined in + // the source proof. No full-register zero idiom precedes this insert. + assembly.WriteString("PINSRD $3,(DI),X5\n") + } + destination := bank + "5" + if row.vex { + destination = bank + "6" + fmt.Fprintf(&assembly, "%s mask<>(SB),%s5,%s\n", row.opcode, bank, destination) + } else { + fmt.Fprintf(&assembly, "%s mask<>(SB),%s\n", row.opcode, destination) + } + fmt.Fprintf(&assembly, "%s %s,0(SP)\n%s 0(SP),%s\n%s %s,(SI)\nRET\n", + move, destination, move, destination, move, destination) + return assembly.String() +} + +// This is an independent byte oracle, not a call to a production operation. +func fixedLocalLogicalLiteralByte(mode int, first, second byte) byte { + switch mode { + case 0: + return first & second + case 1: + return first & ^second + case 2: + return first | second + case 3: + return first ^ second + default: + panic("unknown test operation") + } +} + +func TestX86PackedLogicalConstantByteDependencies(t *testing.T) { + for mode, operation := range []amd64PackedLogicalMode{ + amd64PackedLogicalAnd, amd64PackedLogicalAndNot, amd64PackedLogicalOr, amd64PackedLogicalXor, + } { + for first := 0; first < 256; first++ { + actual := x86PackedLogicalSecondBits(operation, byte(first)) + for second := 0; second < 256; second++ { + base := fixedLocalLogicalLiteralByte(mode, byte(first), byte(second)) + for bit := 0; bit < 8; bit++ { + changed := fixedLocalLogicalLiteralByte(mode, byte(first), byte(second)^(1<": {size: int64(row.width), alignment: 16, + literals: map[int64]x86ShiftLiteral{0: literal}}, + }} + var logical Instr + for _, ins := range file.Funcs[0].Instrs { + if string(ins.Op) == row.opcode { + logical = ins + } + } + effect, known, err := fixedLocalVectorByteEffect(logical) + if !known || err != nil { + t.Fatal("existing complete typed form rejected", known, err) + } + original := effect + original.operands = append([]x86TypedByteOperand(nil), effect.operands...) + projected := fixedLocalLogicalLiteralEffect(logical, effect, binding) + if !reflect.DeepEqual(effect, original) { + t.Fatal("projection mutated its caller's shared operand slice") + } + want := uint64(0xf000) + if annihilate { + want = 0 + } + if row.mode == 3 { + want = x86CallerVectorByteMask(int64(row.width)) + } + if len(projected.operands) != len(effect.operands) || + !reflect.DeepEqual(projected.operands[0], original.operands[0]) || !projected.operands[0].read { + t.Fatal("constant dependency removed or changed the complete memory read") + } + register := projected.operands[1] + required := register.requiredBytes + if register.read && required == 0 { + required = x86CallerVectorByteMask(register.width) + } + if required != want || register.read != (want != 0) || + register.write != original.operands[1].write || projected.vector != original.vector || + projected.flagsRead != original.flagsRead || projected.flagsDefined != original.flagsDefined || + projected.flagsKilled != original.flagsKilled { + t.Fatal("projection changed required bytes, aliases or FLAGS", required, want, projected) + } + state := x86FixedLocalState{flags: x86ByteFlagsAll, vectors: map[Reg]uint64{ + "Z5": want | 1<<40, "Z6": 1 << 50, + }} + proof := &x86FixedLocalProof{binding: binding, frame: 32} + if err := fixedLocalTypedByteStep(&logical, &state, FuncSig{}, proof, 0, projected); err != nil { + t.Fatal("exact required input bytes failed the unchanged typed evaluator", err) + } + if state.flags != x86ByteFlagsAll { + t.Fatal("logical operation changed FLAGS") + } + if row.vex { + if state.vectors["Z6"] != ^uint64(0) || state.vectors["Z5"] != want|1<<40 { + t.Fatal("VEX output failed to zero upper aliases or changed the source") + } + } else if state.vectors["Z5"] != 0xffff|1<<40 || state.vectors["Z6"] != 1<<50 { + t.Fatal("legacy output changed upper aliases or unrelated registers") + } + for _, invalid := range []*x86FixedLocalBinding{ + nil, {}, {literalData: map[string]x86FixedLocalData{ + "mask<>": {size: int64(row.width), alignment: 16}, + }}, + } { + if got := fixedLocalLogicalLiteralEffect(logical, effect, invalid); !reflect.DeepEqual(got, original) { + t.Fatal("missing actual literal tuple acquired a constant dependency") + } + } + }) + } + } +} diff --git a/amd64_fixed_local_logical_test.go b/amd64_fixed_local_logical_test.go new file mode 100644 index 000000000..5b0ad5a17 --- /dev/null +++ b/amd64_fixed_local_logical_test.go @@ -0,0 +1,209 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestGoAMD64FixedLocalLogicalOfficialRows(t *testing.T) { + catalog, err := goasmrules.LoadX86LegacyRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + wanted := make(map[string]bool) + for _, name := range x86LegacyLogicalMemoryNames { + wanted[name] = true + } + xmm, mmx := 0, 0 + for _, opcode := range catalog.Optab { + if !wanted[opcode.Opcode] { + continue + } + for _, row := range opcode.Rows { + if row.ZCase != "Zm_r_xm" { + t.Fatalf("unmodeled original logical row: %+v", row) + } + switch strings.Join(row.Classes, ",") { + case "Yxm,Yxr": + xmm++ + case "Ymm,Ymr": + mmx++ // Inventoried, not admitted to the private XMM consumer. + default: + t.Fatalf("unmodeled original logical operands: %+v", row) + } + } + } + if len(wanted) != 12 || xmm != 12 || mmx != 4 { + t.Fatalf("official logical family changed: names=%d XMM=%d MMX=%d", len(wanted), xmm, mmx) + } +} + +func fixedLocalLogicalZeroName(name string) bool { + switch name { + case "PXOR", "XORPS", "XORPD", "PANDN", "ANDNPS", "ANDNPD": + return true + } + return false +} + +func fixedLocalLogicalFixture(t *testing.T, name, input string, uninitialized bool) (string, FuncSig) { + t.Helper() + source, sig, _ := x86PrivateAlignedFrameAuditFixture(t, "MOVOU 16(SP),X1\n"+name+" "+input+",X0") + if uninitialized { + source = strings.Replace(source, "MOVOU 48(SP),X0\n", "", 1) + } + return source, sig +} + +func fixedLocalLogicalProgram(t *testing.T) (string, string, int) { + t.Helper() + var assembly, declarations strings.Builder + count := 0 + for _, name := range x86LegacyLogicalMemoryNames { + for _, input := range []string{"16(SP)", "X1", "X0", "zero"} { + zero := input == "zero" + if zero && !fixedLocalLogicalZeroName(name) { + continue + } + if zero { + input = "X0" + } + source, _ := fixedLocalLogicalFixture(t, name, input, zero) + function := fmt.Sprintf("Value%d", count) + assembly.WriteString(strings.Replace(source, "·Value", "·"+function, 1)) + fmt.Fprintf(&declarations, "func %s(a,b,c,d uint64) uint64\n", function) + count++ + } + } + return assembly.String(), declarations.String(), count +} + +func TestGoAMD64FixedLocalLogicalCompleteFamilies(t *testing.T) { + assembly, _, count := fixedLocalLogicalProgram(t) + if count != 42 { + t.Fatal("complete 12-name register/memory/alias and six zero identities changed", count) + } + requireX86GoAssemblerResult(t, "amd64", assembly, true) + for _, name := range x86LegacyLogicalMemoryNames { + for _, input := range []string{"16(SP)", "X1", "X0"} { + t.Run(name+"/"+input, func(t *testing.T) { + source, sig := fixedLocalLogicalFixture(t, name, input, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); err != nil { + t.Fatal("initialized complete logical family source rejected", err) + } + }) + } + if fixedLocalLogicalZeroName(name) { + t.Run(name+"/zero identity", func(t *testing.T) { + source, sig := fixedLocalLogicalFixture(t, name, "X0", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); err != nil { + t.Fatal("same-register zero identity incorrectly needs old numerical bytes", err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalLogicalUpperAliasesAndFlags(t *testing.T) { + for _, name := range x86LegacyLogicalMemoryNames { + for _, old := range []uint64{0xffff, 0x804020100000ffff, ^uint64(0)} { + t.Run(fmt.Sprintf("%s/%x", name, old), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-0\n"+name+" X1,X0\nRET\n") + if err != nil { + t.Fatal(err) + } + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z0": old, "Z1": 0xffff}, flags: x86ByteFlagsAll} + if err := fixedLocalVectorStep(&file.Funcs[0].Instrs[1], &state, nil, 0); err != nil { + t.Fatal(err) + } + if state.vectors["Z0"] != old || state.flags != x86ByteFlagsAll { + t.Fatal("legacy logical operation changed upper bytes or FLAGS", state) + } + }) + } + } +} + +func TestGoAMD64FixedLocalLogicalBoundaries(t *testing.T) { + for _, name := range x86LegacyLogicalMemoryNames { + for _, row := range []struct{ name, input, from, to string }{ + {"unknown destination", "X1", "MOVOU 48(SP),X0", "MOVOU 48(SP),X2"}, + {"unknown source", "X2", "", ""}, + {"missing memory byte", "16(SP)", "MOVQ CX,24(SP)", "MOVB CX,24(SP)"}, + {"unaligned memory", "17(SP)", "", ""}, + {"outside frame", "160(SP)", "", ""}, + {"MMX source", "M0", "", ""}, + {"high bank", "X16", "", ""}, + } { + t.Run(name+"/"+row.name, func(t *testing.T) { + source, sig := fixedLocalLogicalFixture(t, name, row.input, false) + if row.from != "" { + changed := strings.Replace(source, row.from, row.to, 1) + if changed == source { + t.Fatal("negative fixture did not change the source") + } + source = changed + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved bytes, alignment or physical bank acquired a logical effect", err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalLogicalObjects(t *testing.T) { + assembly, declarations, count := fixedLocalLogicalProgram(t) + if count != 42 { + t.Fatal("complete-source logical denominator changed", count) + } + fixedLocalCompleteVectorObjects(t, assembly, declarations, count) +} + +func fixedLocalCompleteVectorObjects(t *testing.T, assembly, declarations string, count int) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("ordinary vector/private-frame source failed complete translation", err) + } + var names []string + for index := 0; index < count; index++ { + names = append(names, fmt.Sprintf("%s.Value%d", pkg.Path, index)) + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + compileLLVMToObject(t, llc, target.triple, "private-vector.ll", "private-vector.o", ir) + compileLLVMToObject(t, llc, target.triple, "private-vector-opt.ll", "private-vector-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} diff --git a/amd64_fixed_local_machine_bits_policy_test.go b/amd64_fixed_local_machine_bits_policy_test.go new file mode 100644 index 000000000..e5c3d7dc8 --- /dev/null +++ b/amd64_fixed_local_machine_bits_policy_test.go @@ -0,0 +1,55 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Retain the complete formerly rejected sources, including their original +// private frame setup and ABI. A successful internal effect probe is not an +// object certificate for these newly accepted machine-bit consumers. +func TestGoAMD64FixedLocalMachineBitsRetainedPolicyObjects(t *testing.T) { + t.Run("native vector spill", func(t *testing.T) { + var assembly, declarations strings.Builder + for index, name := range []string{"PSUBSW", "PMULLD", "PMULHW", "PMULDQ", "PAVGB", "PMADDUBSW"} { + source, _, _ := x86PrivateAlignedFrameAuditTypedFixture(t, name+" 16(SP),X0", + "package frameaudit\nfunc Value(a *uint64, b, c, d uint64) uint64\n") + function := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(source, "·Value", "·"+function, 1)) + fmt.Fprintf(&declarations, "func %s(a *uint64, b, c, d uint64) uint64\n", function) + } + fixedLocalCompleteVectorObjects(t, assembly.String(), declarations.String(), 6) + }) + t.Run("native multiply", func(t *testing.T) { + var assembly, declarations strings.Builder + count := 0 + for _, stem := range []string{"MUL", "IMUL"} { + for _, width := range fixedLocalImplicitMultiplyWidths { + name := stem + width.suffix + for _, role := range []string{"accumulator", "source"} { + body := "MOVQ SI,CX\n" + name + " CX" + if role == "accumulator" { + body = "MOVQ SI,AX\n" + name + " 0(SP)" + } + _, _, source := fixedLocalInputFixture(t, body) + function := fmt.Sprintf("Value%d", count) + assembly.WriteString(strings.Replace(source, "·Value", "·"+function, 1)) + fmt.Fprintf(&declarations, "func %s(p, q *uint64, a uint64) uint64\n", function) + count++ + } + } + } + if count != 16 { + t.Fatal("original native multiply source inventory changed", count) + } + fixedLocalCompleteVectorObjects(t, assembly.String(), declarations.String(), count) + }) + t.Run("unused native arithmetic", func(t *testing.T) { + base := fixedLocalTransportAssembly("Value0", 49, fixedLocalTransportBoolBody) + body := strings.Replace(fixedLocalTransportBoolBody, "MOVQ p+24(FP),R8", "ADDQ CX,DX", 1) + assembly := strings.Replace(base, fixedLocalTransportBoolBody, body, 1) + declaration := "func Value0(" + fixedLocalTransportParams + ") bool" + fixedLocalCompleteVectorObjects(t, assembly, declaration, 1) + }) +} diff --git a/amd64_fixed_local_machine_bits_runtime_go127_test.go b/amd64_fixed_local_machine_bits_runtime_go127_test.go new file mode 100644 index 000000000..98636fc5c --- /dev/null +++ b/amd64_fixed_local_machine_bits_runtime_go127_test.go @@ -0,0 +1,513 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +type fixedLocalMachineRuntimeCase struct { + operation, bits, left, right, carry int + destinationShift, sourceShift int + body string +} + +// Native pointers contribute their actual initialized bits. The independent +// Go and C references each use their own buffers' addresses, never assume the +// same linker/stack layout, and check every canary byte after the assembly call. +func fixedLocalMachineRuntimeCases() []fixedLocalMachineRuntimeCase { + flags := "SETCS 0(R8)\nSETEQ 1(R8)\nSETMI 2(R8)\nSETOS 3(R8)\nSETPS 4(R8)\n" + var rows []fixedLocalMachineRuntimeCase + for wi, suffix := range []string{"B", "W", "L", "Q"} { + for op, stem := range []string{"ADD", "ADC", "SUB", "SBB", "AND", "OR", "XOR"} { + for _, left := range []int{0, 1} { + for _, right := range []int{0, 2} { + for carry := 0; carry < 2; carry++ { + register := []string{"AX", "SI", "DI"} + body := "MOVQ SI,R8\nMOVQ " + register[left] + ",BX\nMOVQ " + register[right] + ",CX\nCMPQ AX,AX\n" + if carry != 0 { + body += "STC\n" + } + body += stem + suffix + " CX,BX\n" + flags + "XORQ $0,BX\nMOVQ BX,AX" + rows = append(rows, fixedLocalMachineRuntimeCase{op, []int{8, 16, 32, 64}[wi], left, right, carry, 0, 0, body}) + } + } + } + } + } + for _, operand := range []struct { + source string + role int + }{{"SI", 1}, {"p+0(FP)", 1}} { + body := "MOVQ SI,R8\nMULQ " + operand.source + "\nSETCS 0(R8)\nMOVB $0,1(R8)\nMOVB $0,2(R8)\nSETOS 3(R8)\nMOVB $0,4(R8)" + rows = append(rows, fixedLocalMachineRuntimeCase{7, 64, 0, operand.role, 0, 0, 0, body}) + } + for _, shifts := range [][2]int{{0, 8}, {8, 0}} { + registers := map[int]string{0: "AL", 8: "AH"} + body := "MOVQ SI,R8\nMOVQ SI,AX\nSUBB " + registers[shifts[1]] + "," + registers[shifts[0]] + "\n" + flags + "XORQ $0,AX" + rows = append(rows, fixedLocalMachineRuntimeCase{2, 8, 1, 1, 0, shifts[0], shifts[1], body}) + } + return rows +} + +func fixedLocalMachineRuntimeScope(scope string) []fixedLocalMachineRuntimeCase { + rows := fixedLocalMachineRuntimeCases() + if scope == "machine" { + return rows + } + if scope == "auxiliary" { + var selected []fixedLocalMachineRuntimeCase + for _, row := range rows { + if row.operation > 3 || row.bits != 64 || row.left != 1 || row.right != 0 { + continue + } + // Preserve BX and the five SETcc observations. AH cannot be + // encoded directly with the REX prefix required by an R8 address. + row.body = strings.Replace(row.body, "XORQ $0,BX", + "LAHF\nMOVB AH,CL\nANDB $16,CL\nMOVB CL,5(R8)\nXORQ $0,BX", 1) + selected = append(selected, row) + } + return selected + } + if scope == "vector spill" { + return []fixedLocalMachineRuntimeCase{{operation: 8, bits: 64, left: 1, + body: "MOVQ SI,R8\nMOVQ SP,R11\nANDQ $-16,SP\nADDQ $16,SP\n" + + "MOVQ SI,16(SP)\nMOVQ AX,24(SP)\nCMPQ AX,AX\n" + + "PXOR X0,X0\nPADDQ 16(SP),X0\nMOVQ R11,SP\n" + + "SETCS 0(R8)\nSETEQ 1(R8)\nSETMI 2(R8)\nSETOS 3(R8)\nSETPS 4(R8)\nMOVQ X0,AX", + }} + } + if scope == "multiply" { + var selected []fixedLocalMachineRuntimeCase + for _, stem := range []string{"MUL", "IMUL"} { + operation := 7 + if stem == "IMUL" { + operation = 9 + } + for _, width := range fixedLocalImplicitMultiplyWidths { + for _, accumulator := range []bool{false, true} { + left, right := 0, 1 + body := "MOVQ SI,R8\nMOVQ SI,CX\n" + stem + width.suffix + " CX\n" + if accumulator { + left, right = 1, 0 + body = "MOVQ SI,R8\nMOVQ SI,AX\n" + stem + width.suffix + " 0(SP)\n" + } + body += "SETCS 0(R8)\nMOVB $0,1(R8)\nMOVB $0,2(R8)\nSETOS 3(R8)\nMOVB $0,4(R8)" + selected = append(selected, fixedLocalMachineRuntimeCase{ + operation: operation, bits: int(width.width * 8), left: left, right: right, body: body, + }) + } + } + } + return selected + } + return nil +} + +func fixedLocalMachineBitsRuntimeProgram(t *testing.T, triple, scope string) (assembly, declarations, original, driver string) { + t.Helper() + rows := fixedLocalMachineRuntimeScope(scope) + want := map[string]int{"machine": 228, "auxiliary": 8, "vector spill": 1, "multiply": 16}[scope] + if want == 0 || len(rows) != want { + t.Fatal("independent machine-bit runtime inventory changed", len(rows)) + } + checkedFlags := 5 + if scope == "auxiliary" { + checkedFlags = 6 + } + var functions []fixedLocalPointerChoiceCase + var goRows, cRows, goFunctions, cFunctions, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, row := range rows { + functions = append(functions, fixedLocalPointerChoiceCase{fmt.Sprintf("machine%d", index), row.body}) + fields := fmt.Sprintf("{%d,%d,%d,%d,%d,%d,%d},", row.operation, row.bits, row.left, row.right, row.carry, row.destinationShift, row.sourceShift) + goRows.WriteString(fields) + cRows.WriteString(fields) + fmt.Fprintf(&goFunctions, "Value%d,", index) + fmt.Fprintf(&cFunctions, "value%d,", index) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t*,uint64_t*,uint64_t) __asm__(%s);\n", index, + runtimeCAsmLabel(prefix+fmt.Sprintf("example.invalid/framefixture.Value%d", index))) + } + assembly, declarations = fixedLocalPointerChoiceProgram(t, "", functions) + original = `package main + +import ( + "fmt" + "math/bits" + "unsafe" +) + +` + fmt.Sprintf("const checkedFlags = %d\n", checkedFlags) + declarations + ` + +type row struct{ op, width, left, right, carry, ds, ss int } + +var rows = []row{` + goRows.String() + `} + +func reference(c row, base, other, a uint64) (uint64, [6]byte) { + values := [3]uint64{a, base, other} + old := values[c.left] + if c.op == 8 { + // The vector adds zero to the spilled native address. Integer FLAGS + // still describe the equal comparison preceding the vector operation. + return old, [6]byte{0, 1, 0, 0, 1, 0} + } + mask := ^uint64(0) + if c.width < 64 { + mask = (uint64(1) << c.width) - 1 + } + x, y := (old>>c.ds)&mask, (values[c.right]>>c.ss)&mask + sign := uint64(1) << (c.width - 1) + var r, cf, of uint64 + switch c.op { + case 0, 1: + carry := uint64(0) + if c.op == 1 { + carry = uint64(c.carry) + } + r, cf = bits.Add64(x, y, carry) + if c.width < 64 && r > mask { + cf = 1 + } + r &= mask + if (^(x ^ y) & (x ^ r) & sign) != 0 { + of = 1 + } + case 2, 3: + carry := uint64(0) + if c.op == 3 { + carry = uint64(c.carry) + } + r, cf = bits.Sub64(x, y, carry) + r &= mask + if ((x ^ y) & (x ^ r) & sign) != 0 { + of = 1 + } + case 4: + r = x & y + case 5: + r = x | y + case 6: + r = x ^ y + case 7: + hi, lo := bits.Mul64(x, y) + r = lo + if hi != 0 || c.width < 64 && lo>>c.width != 0 { + cf, of = 1, 1 + } + case 9: + if x&sign != 0 { + x |= ^mask + } + if y&sign != 0 { + y |= ^mask + } + hi, lo := bits.Mul64(x, y) + if int64(x) < 0 { + hi -= y + } + if int64(y) < 0 { + hi -= x + } + r = lo + low, high := lo&mask, uint64(0) + if low&sign != 0 { + low |= ^mask + high = ^uint64(0) + } + if lo != low || hi != high { + cf, of = 1, 1 + } + default: + panic("unknown reference") + } + flags := [6]byte{byte(cf), 0, 0, byte(of), 0, 0} + if checkedFlags == 6 { + carry := uint64(0) + if c.op == 1 || c.op == 3 { + carry = uint64(c.carry) + } + if c.op < 2 && (x&15)+(y&15)+carry > 15 || + c.op >= 2 && (x&15) < (y&15)+carry { + flags[5] = 16 + } + } + if c.op != 7 && c.op != 9 { + if r == 0 { + flags[1] = 1 + } + if r&sign != 0 { + flags[2] = 1 + } + if bits.OnesCount8(uint8(r))%2 == 0 { + flags[4] = 1 + } + } + if c.op == 7 || c.op == 9 { + if c.width == 8 { + mask = 65535 // Byte multiply writes AX; word multiply also writes AX. + } + r &= mask + } + if c.width < 32 { + r = (old &^ (mask << c.ds)) | (r << c.ds) + } + return r, flags +} +func main() { + functions := []func(*uint64, *uint64, uint64) uint64{` + goFunctions.String() + `} + for index, fn := range functions { + for alias := 0; alias < 2; alias++ { + var left, right [32]uint64 + p, q := &left[4], &right[4] + if alias != 0 { + q = p + } + base, other := uint64(uintptr(unsafe.Pointer(p))), uint64(uintptr(unsafe.Pointer(q))) + inputs := []uint64{ + 0, 1, 7, 8, 127, 128, 255, 256, 32767, 32768, 65535, 65536, + 1<<31 - 1, 1 << 31, 1<<63 - 1, 1 << 63, ^uint64(0), + -base, other - base, base, + } + for sample, a := range inputs { + for i := range left { + left[i] = uint64(i+1)*0x0102030405060708 ^ a + right[i] = ^left[i] + } + wantLeft, wantRight := left, right + want, flags := reference(rows[index], base, other, a) + bytes := (*[256]byte)(unsafe.Pointer(&wantLeft)) + for i, f := range flags[:checkedFlags] { + bytes[32+i] = f + } + got := fn(p, q, a) + if got != want || left != wantLeft || right != wantRight { + panic(fmt.Sprintf("case=%d sample=%d alias=%d got=%016x want=%016x", index, sample, alias, got, want)) + } + fmt.Printf("%d %d %d PASS\n", index, alias, sample) + } + } + } +} +` + driver = "#include \n#include \n#include \n" + + fmt.Sprintf("enum { function_count = %d, checked_flags = %d };\n", len(rows), checkedFlags) + + runtimeCAsmLabelPreamble + cDeclarations.String() + ` +typedef struct { + unsigned op, width, left, right, carry, ds, ss; +} row; +static const row rows[] = { + ` + cRows.String() + ` +}; +static uint64_t reference(row c, uint64_t base, uint64_t other, uint64_t a, + unsigned char flags[6]) { + uint64_t values[] = {a, base, other}, old = values[c.left]; + if (c.op == 8) { + const unsigned char equal_flags[] = {0, 1, 0, 0, 1, 0}; + memcpy(flags, equal_flags, sizeof equal_flags); + return old; + } + uint64_t mask = c.width == 64 ? UINT64_MAX : (UINT64_C(1) << c.width) - 1; + uint64_t x = (old >> c.ds) & mask, y = (values[c.right] >> c.ss) & mask, + sign = UINT64_C(1) << (c.width - 1), r = 0; + unsigned cf = 0, of = 0; + switch (c.op) { + case 0: + case 1: { + __uint128_t wide = (__uint128_t)x + y + (c.op == 1 ? c.carry : 0); + r = (uint64_t)wide & mask; + cf = wide > mask; + of = (~(x ^ y) & (x ^ r) & sign) != 0; + break; + } + case 2: + case 3: { + __uint128_t rhs = (__uint128_t)y + (c.op == 3 ? c.carry : 0); + r = (uint64_t)((__uint128_t)x - rhs) & mask; + cf = (__uint128_t)x < rhs; + of = ((x ^ y) & (x ^ r) & sign) != 0; + break; + } + case 4: + r = x & y; + break; + case 5: + r = x | y; + break; + case 6: + r = x ^ y; + break; + case 7: { + __uint128_t wide = (__uint128_t)x * y; + r = (uint64_t)wide; + cf = of = wide > mask; + break; + } + case 9: { + __int128 sx = x, sy = y; + if (x & sign) + sx -= (__int128)1 << c.width; + if (y & sign) + sy -= (__int128)1 << c.width; + __int128 wide = sx * sy; + r = (uint64_t)wide; + __int128 limit = (__int128)1 << (c.width - 1); + cf = of = wide < -limit || wide >= limit; + break; + } + default: + return 0; + } + flags[0] = cf; + flags[1] = 0; + flags[2] = 0; + flags[3] = of; + flags[4] = 0; + flags[5] = 0; + if (checked_flags == 6) { + unsigned carry = c.op == 1 || c.op == 3 ? c.carry : 0; + unsigned low_left = x & 15, low_right = (y & 15) + carry; + if (c.op < 2 ? low_left + low_right > 15 : low_left < low_right) { + flags[5] = 16; + } + } + if (c.op != 7 && c.op != 9) { + unsigned parity = 0; + for (unsigned b = 0; b < 8; b++) + parity ^= (r >> b) & 1; + flags[1] = r == 0; + flags[2] = (r & sign) != 0; + flags[4] = !parity; + } + if (c.op == 7 || c.op == 9) { + if (c.width == 8) + mask = 65535; + r &= mask; + } + if (c.width < 32) + r = (old & ~(mask << c.ds)) | (r << c.ds); + return r; +} +int main(void) { + uint64_t (*functions[])(uint64_t *, uint64_t *, uint64_t) = { + ` + cFunctions.String() + ` + }; + for (unsigned index = 0; index < function_count; index++) + for (unsigned alias = 0; alias < 2; alias++) { + uint64_t left[32], right[32], wantLeft[32], wantRight[32]; + uint64_t *p = &left[4], *q = alias ? p : &right[4]; + uint64_t base = (uintptr_t)p, other = (uintptr_t)q; + uint64_t inputs[] = {0, + 1, + 7, + 8, + 127, + 128, + 255, + 256, + 32767, + 32768, + 65535, + 65536, + (UINT64_C(1) << 31) - 1, + UINT64_C(1) << 31, + (UINT64_C(1) << 63) - 1, + UINT64_C(1) << 63, + UINT64_MAX, + -base, + other - base, + base}; + for (unsigned sample = 0; sample < 20; sample++) { + uint64_t a = inputs[sample]; + for (unsigned i = 0; i < 32; i++) { + left[i] = (uint64_t)(i + 1) * UINT64_C(0x0102030405060708) ^ a; + right[i] = ~left[i]; + } + memcpy(wantLeft, left, sizeof left); + memcpy(wantRight, right, sizeof right); + unsigned char flags[6]; + uint64_t want = reference(rows[index], base, other, a, flags); + memcpy((unsigned char *)wantLeft + 32, flags, checked_flags); + uint64_t got = functions[index](p, q, a); + if (got != want || memcmp(left, wantLeft, sizeof left) || + memcmp(right, wantRight, sizeof right)) { + fprintf(stderr, "case=%u sample=%u alias=%u got=%016llx want=%016llx\n", index, + sample, alias, (unsigned long long)got, (unsigned long long)want); + return 1; + } + printf("%u %u %u PASS\n", index, alias, sample); + } + } + return 0; +} +` + return +} + +func fixedLocalMachineBitsRuntime(t *testing.T, execution x86FloatCompareExecution, execute bool) { + t.Helper() + fixedLocalMachineRuntime(t, execution, execute, "machine") +} + +func fixedLocalMachineRuntime(t *testing.T, execution x86FloatCompareExecution, execute bool, scope string) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + assembly, declarations, original, driver := fixedLocalMachineBitsRuntimeProgram(t, execution.triple, scope) + count := len(fixedLocalMachineRuntimeScope(scope)) + var want []string + for index := 0; index < count; index++ { + for alias := 0; alias < 2; alias++ { + for sample := 0; sample < 20; sample++ { + want = append(want, fmt.Sprintf("%d %d %d PASS", index, alias, sample)) + } + } + } + check := func(output []byte) { + t.Helper() + if strings.TrimSpace(string(output)) != strings.Join(want, "\n") { + t.Fatalf("incomplete machine-bit arithmetic oracle:\n%s", output) + } + } + if execute { + check(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original)) + } else { + x86COMICompileOriginalGo(t, map[string]string{"go.mod": "module machinebits\ngo 1.20\n", "main.go": original, "original_amd64.s": assembly}, false) + } + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, declarations, runtime.GOOS+"/amd64", execution.triple) + fixedLocalResultsRuntimeAPIs(t, pkg, request, file, options, assembly, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + if execute { + check(execution.compileAndRunLLVM(t, "machine-bits", input, driver, level)) + } else { + execution.compileLLVM(t, "machine-bits", input, driver, level) + } + }) + } + }) + t.Logf("%s: %d complete functions, 20 actual-address samples, two aliases and all 512 canary bytes; execute=%v", scope, count, execute) +} + +func TestGoAMD64FixedLocalMachineBitsRuntimeObjects(t *testing.T) { + fixedLocalMachineBitsRuntime(t, fixedLocalPackedCompileOnlyTools(t), false) +} + +func TestGoAMD64FixedLocalMachineSupplementalRuntimeObjects(t *testing.T) { + for _, scope := range []string{"auxiliary", "vector spill", "multiply"} { + t.Run(scope, func(t *testing.T) { + fixedLocalMachineRuntime(t, fixedLocalPackedCompileOnlyTools(t), false, scope) + }) + } +} diff --git a/amd64_fixed_local_machine_bits_test.go b/amd64_fixed_local_machine_bits_test.go new file mode 100644 index 000000000..b4fd0eb89 --- /dev/null +++ b/amd64_fixed_local_machine_bits_test.go @@ -0,0 +1,152 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalMachineBitCases() []fixedLocalPointerChoiceCase { + var rows []fixedLocalPointerChoiceCase + for _, width := range []string{"B", "W", "L", "Q"} { + for _, stem := range []string{"ADD", "ADC", "SUB", "SBB", "AND", "OR", "XOR"} { + for _, destination := range []string{"AX", "SI"} { + for _, source := range []string{"AX", "DI"} { + body := "MOVQ " + destination + ",BX\nMOVQ " + source + ",CX\nCMPQ AX,AX\n" + + stem + width + " CX,BX\nXORQ $0,BX\nMOVQ BX,AX" + rows = append(rows, fixedLocalPointerChoiceCase{ + fmt.Sprintf("%s%s/%s/%s", stem, width, destination, source), body, + }) + } + } + } + for _, stem := range []string{"NEG", "NOT", "INC", "DEC"} { + rows = append(rows, fixedLocalPointerChoiceCase{stem + width, + "MOVQ SI,BX\n" + stem + width + " BX\nXORQ $0,BX\nMOVQ BX,AX"}) + } + for _, stem := range []string{"CMP", "TEST"} { + rows = append(rows, fixedLocalPointerChoiceCase{stem + width, + stem + width + " SI,AX\nSETCS AL"}) + } + rows = append(rows, + fixedLocalPointerChoiceCase{"spill source/" + width, + "MOVQ SI,8(SP)\nSUB" + width + " 8(SP),AX"}, + fixedLocalPointerChoiceCase{"spill destination/" + width, + "MOVQ SI,8(SP)\nSUB" + width + " AX,8(SP)\nMOVQ 8(SP),AX"}, + ) + } + for _, row := range []fixedLocalPointerChoiceCase{ + {"aliased full address", "SUBQ SI,SI\nMOVQ SI,AX"}, + {"high byte alias", "MOVQ SI,AX\nSUBB AH,AL\nXORQ $0,AX"}, + {"typed pointer FP", "SUBQ q+8(FP),SI\nMOVQ SI,AX"}, + {"typed pointer FP mixed", "SUBQ p+0(FP),AX"}, + {"first address from typed FP", "XORQ SI,SI\nXORQ DI,DI\nMOVQ $0,CX\nADDQ p+0(FP),CX\nMOVQ (CX),AX"}, + {"typed numeric FP affine", "ADDQ a+16(FP),SI\nSUBQ DI,SI\nMOVQ SI,AX"}, + {"pointer FP multiply", "MULQ p+0(FP)"}, + {"address multiply", "MULQ SI"}, + {"address shift", "MOVQ SI,AX\nSHRQ $3,AX"}, + {"pointer compare native numeric", "CMPQ SI,AX\nSETCS AL"}, + {"carry consumed", "STC\nSBBQ SI,AX\nSETCS CL\nMOVBQZX CL,CX\nADDQ CX,AX"}, + {"address flags branch", "SUBQ DI,SI\nJCS done\nINCQ AX\ndone:"}, + {"unaligned spill interior", "MOVQ SI,3(SP)\nSUBL 4(SP),AX"}, + {"unaligned spill partial RMW", "MOVQ SI,3(SP)\nSUBB AX,7(SP)\nMOVQ 3(SP),AX"}, + {"two spill boundary", "MOVQ SI,8(SP)\nMOVQ DI,16(SP)\nSUBQ 12(SP),AX"}, + {"vector machine input", "MOVQ SI,8(SP)\nMOVQ AX,16(SP)\nMOVQ AX,24(SP)\nMOVQ AX,32(SP)\nVPXOR Y0,Y0,Y0\nVPADDQ 8(SP),Y0,Y0\nMOVQ X0,AX"}, + } { + rows = append(rows, row) + } + return append(rows, fixedLocalMachineAddressCases()...) +} + +// Inputs point inside the native oracle's buffers, so both +/- one-byte paths +// remain valid for every original input and alias choice. +func fixedLocalMachineAddressCases() []fixedLocalPointerChoiceCase { + return []fixedLocalPointerChoiceCase{ + {"ADC native CF0", "CLC\nADCQ $0,SI\nMOVQ (SI),AX"}, + {"ADC native CF1", "STC\nADCQ $0,SI\nMOVQ (SI),AX"}, + {"SBB native CF0", "CLC\nSBBQ $0,SI\nMOVQ (SI),AX"}, + {"SBB native CF1", "STC\nSBBQ $0,SI\nMOVQ (SI),AX"}, + {"ADC reversed CF0", "MOVQ $0,CX\nCLC\nADCQ SI,CX\nMOVQ (CX),AX"}, + {"ADC reversed CF1", "MOVQ $0,CX\nSTC\nADCQ SI,CX\nMOVQ (CX),AX"}, + } +} + +func TestGoAMD64FixedLocalMachineBitsCompleteSources(t *testing.T) { + rows := fixedLocalMachineBitCases() + if len(rows) != 166 { + t.Fatal("independent machine-bit source inventory changed", len(rows)) + } + var assembly strings.Builder + for index, row := range rows { + fn, sig, source := fixedLocalInputFixture(t, row.body) + assembly.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("initialized machine input rejected", err) + } + }) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) +} + +func TestGoAMD64FixedLocalMachineBitsRetainCapabilities(t *testing.T) { + for _, row := range []fixedLocalPointerChoiceCase{ + {"difference is not address", "SUBQ DI,SI\nMOVQ (SI),AX"}, + {"sum is not address", "ADDQ DI,SI\nMOVQ (SI),AX"}, + {"number minus address", "SUBQ SI,AX\nMOVQ (AX),AX"}, + {"truncated result is not address", "SUBL AX,SI\nMOVQ (SI),AX"}, + {"private source", "MOVQ SP,CX\nSUBQ CX,AX"}, + {"private destination", "MOVQ SP,CX\nSUBQ SI,CX"}, + {"private source spill", "MOVQ SP,8(SP)\nSUBQ 8(SP),AX"}, + {"private destination spill", "MOVQ SP,8(SP)\nSUBQ AX,8(SP)"}, + {"unknown source", "SUBQ R12,SI"}, + {"unknown byte", "MOVB $1,R12\nSUBQ R12,SI"}, + {"partial revoked spill", "MOVQ SI,8(SP)\nMOVB $1,9(SP)\nSUBQ 8(SP),AX"}, + {"missing carry", "SBBQ SI,AX"}, + {"multiply does not define zero", "MULQ SI\nJE done\ndone:"}, + {"private index", "SUBQ (DI)(SP*1),AX"}, + {"native index", "SUBQ (DI)(SI*1),AX"}, + {"NUM is not address", "SUBQ DI,SI\nSUBQ (SI),AX"}, + {"partial FP pointer", "SUBL p+0(FP),AX"}, + {"changed SP FP", "ADDQ $8,SP\nSUBQ p+0(FP),AX\nADDQ $-8,SP"}, + {"FP padding", "SUBQ p+1(FP),AX"}, + {"FP destination", "SUBQ SI,a+16(FP)"}, + {"undefined spill boundary", "MOVQ SI,8(SP)\nSUBQ 12(SP),AX"}, + {"vector private source", "MOVQ SP,8(SP)\nMOVQ AX,16(SP)\nMOVQ AX,24(SP)\nMOVQ AX,32(SP)\nVPXOR Y0,Y0,Y0\nVPADDQ 8(SP),Y0,Y0"}, + {"vector undefined source", "MOVQ SI,8(SP)\nVPXOR Y0,Y0,Y0\nVPADDQ 8(SP),Y0,Y0"}, + {"carry sum is not address", "STC\nADCQ DI,SI\nMOVQ (SI),AX"}, + {"carry difference is not address", "CLC\nSBBQ DI,SI\nMOVQ (SI),AX"}, + {"carry number minus address", "STC\nSBBQ SI,AX\nMOVQ (AX),AX"}, + {"missing address carry", "ADCQ $0,SI\nMOVQ (SI),AX"}, + {"conditional FP pointer is not NUM", "CMPQ AX,AX\nCMOVQEQ p+0(FP),AX"}, + {"conditional FP number is not address", "CMPQ AX,AX\nCMOVQEQ a+16(FP),SI\nXORQ $0,SI\nMOVQ SI,AX"}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, source := fixedLocalInputFixture(t, row.body) + operation := "" + switch row.name { + case "difference is not address", "sum is not address", "truncated result is not address", "carry sum is not address", "carry difference is not address": + operation = "MOVQ (SI),AX" + case "number minus address", "carry number minus address": + operation = "MOVQ (AX),AX" + case "NUM is not address": + operation = "SUBQ (SI),AX" + } + if operation != "" { + fixedLocalRequireNumericControl(t, fn, sig, source, "func Value(p,q *uint64,a uint64) uint64", + operation, 0, 8, false, "load volatile i64") + return + } + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("machine input acquired unrelated storage, flags or ABI", err) + } + }) + } +} + +func TestGoAMD64FixedLocalMachineBitsCompleteObjects(t *testing.T) { + rows := fixedLocalMachineBitCases() + assembly, declarations := fixedLocalPointerChoiceProgram(t, "", rows) + fixedLocalCompleteVectorObjects(t, assembly, declarations, len(rows)) +} diff --git a/amd64_fixed_local_mixed_entries_test.go b/amd64_fixed_local_mixed_entries_test.go new file mode 100644 index 000000000..f384ce6a3 --- /dev/null +++ b/amd64_fixed_local_mixed_entries_test.go @@ -0,0 +1,169 @@ +package plan9asm + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "strings" + "testing" +) + +// One physical source owns both entries. Ordinary dereferences an incoming Go +// pointer, which is deliberately not a private-frame address or numeric input. +func fixedLocalMixedEntriesFixture(t *testing.T, ordinaryFirst bool) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options, string) { + t.Helper() + _, request, _, _ := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + name := filepath.Join(request.Directory, request.AssemblyFile) + frame, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + ordinary := "TEXT ·Ordinary(SB),4,$0-16\nMOVQ p+0(FP),AX\nMOVQ (AX),AX\nMOVQ AX,ret+8(FP)\nRET\n" + assembly := string(frame) + ordinary + if ordinaryFirst { + assembly = ordinary + string(frame) + } + declaration := "package framefixture\nfunc Value(a, b, c, d uint64) uint64\nfunc Ordinary(p *uint64) uint64\n" + if err := os.WriteFile(name, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + goFile := filepath.Join(request.Directory, "value.go") + if err := os.WriteFile(goFile, []byte(declaration), 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, goFile, declaration, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + typed, err := (&types.Config{Sizes: sizes}).Check("example.invalid/framefixture", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: make(map[string]FuncSig), + ResolveSym: func(symbol string) string { return typed.Path() + "." + strings.TrimPrefix(symbol, "·") }} + for _, name := range []string{"Value", "Ordinary"} { + sig, err := DeriveGoFuncABI0Signature(typed.Path()+"."+name, typed.Scope().Lookup(name).(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + options.Sigs[sig.Name] = sig + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{syntax}}, request, file, options, assembly +} + +func TestGoAMD64FixedLocalMixedOrdinaryEntries(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, ordinaryFirst := range []bool{false, true} { + order := "frame-first" + if ordinaryFirst { + order = "ordinary-first" + } + t.Run(order, func(t *testing.T) { + pkg, request, _, options, _ := fixedLocalMixedEntriesFixture(t, ordinaryFirst) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil || strings.Count(ir, "define ") != 2 || strings.Count(ir, "%fixed_local_native = alloca") != 1 || + !strings.Contains(ir, pkg.Path+".Value") || !strings.Contains(ir, pkg.Path+".Ordinary") { + t.Fatal("mixed full source did not retain exactly one private frame and both ordinary ABI0 entries", err) + } + compileLLVMToObject(t, llc, options.TargetTriple, "mixed.ll", "mixed.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "mixed-opt.ll", "mixed-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +func TestGoAMD64FixedLocalMixedEntryInventoryAndRevocation(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + pkg, request, file, options, assembly := fixedLocalMixedEntriesFixture(t, false) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + frameName, ordinaryName := pkg.Path+".Value", pkg.Path+".Ordinary" + frameBinding := bound.Sigs[frameName].x86FixedLocal + if len(source.functions) != 2 || len(bound.Sigs) != 2 || frameBinding == nil || bound.Sigs[ordinaryName].x86FixedLocal != nil { + t.Fatal("complete inventory was confused with per-function private-frame permission") + } + for _, mutation := range []string{"omit ordinary", "ordinary signature", "borrow frame", "detach frame", "crop source"} { + t.Run(mutation, func(t *testing.T) { + changed := bound + changed.Sigs, err = cloneFixedLocalSigs(bound.Sigs) + if err != nil { + t.Fatal(err) + } + frame, ordinary := changed.Sigs[frameName], changed.Sigs[ordinaryName] + frame.x86FixedLocal = frameBinding + changedFile := *file + switch mutation { + case "ordinary signature": + ordinary.Args[0] = I64 + case "borrow frame": + ordinary.x86FixedLocal = frameBinding + case "detach frame": + frame.x86FixedLocal = nil + case "crop source": + changedFile.Funcs = file.Funcs[:1] + } + changed.Sigs[frameName], changed.Sigs[ordinaryName] = frame, ordinary + if mutation == "omit ordinary" { + delete(changed.Sigs, ordinaryName) + } + ir, err := Translate(&changedFile, changed) + fixedLocalRequireContext(t, ir, err) + }) + } + // A body with no frame capability remains part of the revocable complete + // source identity. Changing it must revoke the sibling's backing as well. + changed := strings.Replace(assembly, "MOVQ (AX),AX", "MOVQ 8(AX),AX", 1) + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(changed), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("ordinary sibling mutation retained the complete source owner", err) + } + ir, err := Translate(file, bound) + fixedLocalRequireContext(t, ir, err) +} + +func TestGoAMD64FixedLocalMixedEntryRejectsUnclosedRoles(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, role := range []string{"source call", "private helper"} { + t.Run(role, func(t *testing.T) { + pkg, request, _, _, assembly := fixedLocalMixedEntriesFixture(t, false) + if role == "source call" { + assembly = strings.Replace(assembly, "TEXT ·Value(SB),4,$160-40", "TEXT ·Value(SB),$160-40\nCALL runtime·morestack_noctxt(SB)", 1) + } else { + assembly += "TEXT helper<>(SB),4,$0-0\nRET\n" + } + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil { + source.Close() + } + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("ordinary sibling invented a source-call or private-entry contract", err) + } + }) + } +} diff --git a/amd64_fixed_local_native_address_objects_go127_test.go b/amd64_fixed_local_native_address_objects_go127_test.go new file mode 100644 index 000000000..187f213da --- /dev/null +++ b/amd64_fixed_local_native_address_objects_go127_test.go @@ -0,0 +1,14 @@ +//go:build go1.27 + +package plan9asm + +import "testing" + +func TestGoAMD64FixedLocalNativeAddressObjects(t *testing.T) { + rows := fixedLocalNativeAddressCases() + if len(rows) != 17 { + t.Fatal("complete native-address source inventory changed", len(rows)) + } + assembly, declarations := fixedLocalPointerChoiceProgram(t, fixedLocalChoiceData, rows) + fixedLocalCompleteVectorObjects(t, assembly, declarations, len(rows)) +} diff --git a/amd64_fixed_local_native_address_test.go b/amd64_fixed_local_native_address_test.go new file mode 100644 index 000000000..10197ad55 --- /dev/null +++ b/amd64_fixed_local_native_address_test.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// Every read below is valid for both runtime choices when the native input +// points to its actual test buffer. A possible literal in the abstract state +// is not proof that an out-of-range literal access is executed. +func fixedLocalNativeAddressCases() []fixedLocalPointerChoiceCase { + selected := "LEAQ choice<>(SB),CX\nCMPQ AX,$0\nCMOVQNE SI,CX\n" + guarded := "CMPQ AX,$0\nJE literal\nMOVOU 128(CX),X0\nJMP read\nliteral:\nMOVOU (CX),X0\nread:\nMOVQ X0,AX" + return []fixedLocalPointerChoiceCase{ + {"two actual read paths", selected + guarded}, + {"unexecuted out-of-range candidate", selected + "CMPQ AX,$0\nJE done\nMOVQ 128(CX),AX\ndone:"}, + {"CFG preserves actual address", "CMPQ AX,$0\nJE choose\nMOVQ SI,CX\nJMP join\nchoose:\nLEAQ choice<>(SB),CX\njoin:\n" + guarded}, + {"private spill preserves actual address", selected + "MOVQ CX,8(SP)\nMOVQ 8(SP),CX\n" + guarded}, + {"exact address excursion returns before read", "LEAQ choice<>(SB),CX\nADDQ $128,CX\nSUBQ $128,CX\nMOVQ (CX),AX"}, + {"source address expression returns before read", "LEAQ choice<>+128(SB),CX\nSUBQ $128,CX\nMOVQ (CX),AX"}, + {"dynamic affine literal address", "MOVQ AX,DX\nANDQ $7,DX\nSHLQ $3,DX\nLEAQ choice<>(SB),CX\nADDQ DX,CX\nMOVQ (CX),AX"}, + {"dynamic affine selected address", selected + "MOVQ AX,DX\nANDQ $7,DX\nSHLQ $3,DX\nADDQ DX,CX\nMOVQ (CX),AX"}, + {"finite address loop converges", selected + "MOVQ $4,R11\nagain:\nADDQ $8,CX\nDECQ R11\nJNE again\nMOVQ (CX),AX"}, + {"native compare retains carry flags", selected + "CMPQ CX,CX\nCMOVQCC DI,CX\nMOVQ (CX),AX"}, + {"native address arithmetic retains carry flags", selected + "ADDQ $0,CX\nCMOVQCC DI,CX\nMOVQ (CX),AX"}, + {"more than four source choices", fixedLocalChoiceCountBody(8)}, + {"former fifth alternative limit", fixedLocalChoiceCountBody(5)}, + {"dynamic indexed selected address", selected + "MOVQ AX,DX\nANDQ $7,DX\nMOVQ (CX)(DX*8),AX"}, + {"indexed joined literals", "LEAQ choice<>(SB),CX\nLEAQ other<>(SB),DX\nCMPQ AX,$0\nCMOVQEQ DX,CX\nMOVQ $1,DX\nMOVQ (CX)(DX*8),AX"}, + {"dynamic indexed literal address", "MOVQ AX,DX\nANDQ $7,DX\nSHLQ $3,DX\nLEAQ choice<>(SB),CX\nMOVQ (CX)(DX*1),AX"}, + {"dynamic indexed literal LEA", "MOVQ AX,DX\nANDQ $7,DX\nLEAQ choice<>(SB),CX\nLEAQ (CX)(DX*8),CX\nMOVQ (CX),AX"}, + } +} + +func TestGoAMD64FixedLocalNativeAddressCompleteSources(t *testing.T) { + var assembly strings.Builder + assembly.WriteString(fixedLocalChoiceData) + for index, row := range fixedLocalNativeAddressCases() { + fn, sig, source := fixedLocalPointerChoiceFixture(t, row.body) + assembly.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("complete original native address operation was rejected", err) + } + }) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) +} + +func TestGoAMD64FixedLocalNativeAddressRetainsBoundaries(t *testing.T) { + t.Run("native address has machine bits", func(t *testing.T) { + fn, sig, _ := fixedLocalPointerChoiceFixture(t, fixedLocalChoicePrefix+"XORQ CX,AX") + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("complete native arithmetic source rejected", err) + } + }) + for _, row := range []fixedLocalPointerChoiceCase{ + {"known literal read exceeds object", "LEAQ choice<>(SB),CX\nMOVQ 64(CX),AX"}, + {"known literal vector crosses object", "LEAQ choice<>(SB),CX\nMOVOU 56(CX),X0\nMOVQ X0,AX"}, + {"known indexed literal exceeds object", "LEAQ choice<>(SB),CX\nMOVQ $8,DX\nMOVQ (CX)(DX*8),AX"}, + {"known readonly write", "LEAQ choice<>(SB),CX\nMOVQ AX,(CX)"}, + {"readonly alternative does not grant write", fixedLocalChoicePrefix + "MOVQ AX,(CX)"}, + {"other object does not establish adjacency", "LEAQ choice<>+64(SB),CX\nLEAQ other<>(SB),DX\nMOVQ (CX),AX"}, + {"private address cannot merge with native", "LEAQ 8(SP),CX\nCMPQ AX,$0\nCMOVQEQ SI,CX\nMOVQ (CX),AX"}, + {"partial register address is undefined", fixedLocalChoicePrefix + "MOVL CX,DX\nMOVQ (DX),AX"}, + {"partial spilled address is undefined", fixedLocalChoicePrefix + "MOVQ CX,8(SP)\nMOVB $1,9(SP)\nMOVQ 8(SP),DX\nMOVQ (DX),AX"}, + {"unknown input is not an address", "CMPQ AX,$0\nCMOVQEQ SI,R12\nMOVQ (R12),AX"}, + {"numeric input is not an address", "MOVQ $1,CX\nMOVQ (CX),AX"}, + {"unknown index is not numerical", fixedLocalChoicePrefix + "MOVQ (CX)(R12*8),AX"}, + {"private index is not numerical", fixedLocalChoicePrefix + "MOVQ (CX)(SP*1),AX"}, + {"address index is not numerical", fixedLocalChoicePrefix + "MOVQ (CX)(DI*8),AX"}, + {"partial index is undefined", fixedLocalChoicePrefix + "MOVW $1,R12\nMOVQ (CX)(R12*8),AX"}, + {"equality does not grant readonly write", fixedLocalChoicePrefix + "CMPQ CX,R8\nJNE done\nMOVQ AX,(CX)\ndone:"}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, source := fixedLocalPointerChoiceFixture(t, row.body) + if row.name == "numeric input is not an address" { + fixedLocalRequireNumericControl(t, fn, sig, source, "func Value(p,q *uint64,a uint64) uint64", + "MOVQ (CX),AX", 0, 8, false, "load volatile i64") + return + } + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native address acquired unrelated storage or numerical permission", err) + } + }) + } +} diff --git a/amd64_fixed_local_native_numeric_runtime_test.go b/amd64_fixed_local_native_numeric_runtime_test.go new file mode 100644 index 000000000..2040899f5 --- /dev/null +++ b/amd64_fixed_local_native_numeric_runtime_test.go @@ -0,0 +1,109 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Called from the existing dead-read fault root, retaining its cross-runtime +// owner. Original Go checks valid unaligned accesses; the shared independent +// byte reference checks NULL and short-page faults in owned child processes. +func fixedLocalNativeNumericAddressRuntime(t *testing.T) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + triple := testTargetTriple(runtime.GOOS, "amd64") + assembly, declarations := fixedLocalNativeNumericRuntimeSources(t) + var goCalls, prototypes, rows, validCalls strings.Builder + prefix := "" + if strings.Contains(execution.triple, "apple") { + prefix = "_" + } + for ordinal := 0; ordinal < 8; ordinal++ { + width := 1 << uint(ordinal%4) + fmt.Fprintf(&goCalls, ` + for i := range buffer { buffer[i] = 0x91 } + if Value%d(&address,&output,0) != 23063 { t.Fatal("original value %d") } +`, ordinal, ordinal) + if ordinal >= 4 { + fmt.Fprintf(&goCalls, ` + for i, value := range buffer { + want := byte(0x91) + if i >= 1 && i < 1+%d { want = 0 } + if i == 1 { want = 17 } + if value != want { t.Fatal("original exact store", %d, i, value, want) } + } +`, width, ordinal) + } + fmt.Fprintf(&prototypes, "extern uint64_t value%d(const uint64_t *,uint64_t *,uint64_t) __asm__(%q);\n", + ordinal, prefix+"example.invalid/framefixture.Value"+fmt.Sprint(ordinal)) + // The first pointer is always a valid local word. Only the loaded + // initialized NUM word contains the faulting effective address. + fmt.Fprintf(&prototypes, ` +static int run%d(const uint8_t *p) { + uint64_t address = (uint64_t)(uintptr_t)p, output = 0; + return value%d(&address, &output, 0) != 23063; +} +`, ordinal, ordinal) + fmt.Fprintf(&rows, "{run%d,%d,%d},\n", ordinal, width, width-1) + fmt.Fprintf(&validCalls, "memset(buffer,0x91,sizeof buffer);\nif(value%d(&address,&output,0)!=23063)return %d;\n", ordinal, 10+ordinal) + if ordinal >= 4 { + fmt.Fprintf(&validCalls, ` +for(unsigned i=0;i=1 && i<1+%d) want=0; + if(i==1) want=17; + if(buffer[i]!=want) return %d; +} +`, width, 30+ordinal) + } else { + validCalls.WriteString("for(unsigned i=0;i"}, + })} + before := fixedLocalOriginMapStorage(previous) + if changed, err := mergeFixedLocalOrigins(previous, current); err != nil || !changed { + t.Fatal("actual address join failed", changed, err) + } + if bytes := fixedLocalOriginMapStorage(previous); bytes != before || bytes != 64 { + t.Fatal("fixed-size address join changed the conservative per-entry charge", bytes) + } + for _, capacity := range []int64{63, 64} { + budget := x86FixedLocalBudget{bytes: capacity} + err := budget.consumeOrigins(before) + if errors.Is(err, ErrProbeNeedsContext) != (capacity < before) { + t.Fatal("cloning addresses bypassed its unchanged work bound", capacity, err) + } + if capacity < before && budget.bytes != capacity { + t.Fatal("failed work accounting partially consumed its budget") + } + } + allocations := testing.AllocsPerRun(10, func() { + previous[AX] = fixedLocalOrigin(x86FixedLocalOrigin{}) + if _, err := mergeFixedLocalOrigins(previous, current); err != nil { + panic(err) + } + }) + if allocations != 0 { + t.Fatal("address join allocated alternative-set storage", allocations) + } +} + +func TestGoAMD64FixedLocalOriginValueStorage(t *testing.T) { + if size := unsafe.Sizeof(x86FixedLocalOrigins{}); size > uintptr(fixedLocalOriginPayloadBytes) || size > 128 { + t.Fatalf("address value is %d bytes, exceeding its charge or inline map threshold", size) + } +} + +func TestGoAMD64FixedLocalOriginSetValueSemantics(t *testing.T) { + set := func(offsets ...int64) x86FixedLocalOrigins { + value := fixedLocalOrigin(x86FixedLocalOrigin{literal: x86StaticSBAddress{symbol: "choice<>", offset: offsets[0]}}) + for _, offset := range offsets[1:] { + var valid bool + value, valid = value.union(fixedLocalOrigin(x86FixedLocalOrigin{literal: x86StaticSBAddress{symbol: "choice<>", offset: offset}})) + if !valid { + t.Fatal("complete native address join rejected") + } + } + return value + } + t.Run("invalid equal values remain unknown", func(t *testing.T) { + for _, value := range []x86FixedLocalOrigins{{}, {roles: 0x80}, { + roles: fixedLocalNativeAddress, address: x86StaticSBAddress{symbol: "foreign<>"}, + }} { + previous := map[Reg]x86FixedLocalOrigins{AX: value} + if _, err := mergeFixedLocalOrigins(previous, map[Reg]x86FixedLocalOrigins{AX: value}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("equal fast path granted an invalid address", err) + } + } + }) + t.Run("independent order and duplicate joins", func(t *testing.T) { + for _, offsets := range [][]int64{{0, 8, 16}, {16, 0, 8}, {8, 16, 8, 0, 16}} { + first, second := set(0, 8, 16), set(offsets...) + previous := map[Reg]x86FixedLocalOrigins{AX: first} + if changed, err := mergeFixedLocalOrigins(previous, map[Reg]x86FixedLocalOrigins{AX: second}); err != nil || changed { + t.Fatal("address join did not converge independently of order", changed, err) + } + if first != second || previous[AX] != first || !first.readonly() { + t.Fatal("joining addresses lost readonly storage or changed a canonical value") + } + } + }) + t.Run("union does not change either input", func(t *testing.T) { + first, second := set(0), fixedLocalOrigin(x86FixedLocalOrigin{}) + merged, valid := first.union(second) + if !valid || !merged.hasLiteral() || merged.roles&fixedLocalNativeAddress == 0 || + first != set(0) || second.hasLiteral() { + t.Fatal("union mutated an input or lost an address role") + } + }) + t.Run("affine write does not change prior state", func(t *testing.T) { + fn, sig, _ := fixedLocalPointerChoiceFixture(t, "MOVQ (SI),AX") + proof := &x86FixedLocalProof{frame: fn.FrameSize, binding: sig.x86FixedLocal} + original := set(8) + state := x86FixedLocalState{ + opaque: map[Reg]x86FixedLocalOrigins{AX: original}, + known: make(map[Reg]uint8), booleans: make(map[Reg]uint8), + } + ins := Instr{Op: "ADDQ", Args: []Operand{{Kind: OpImm, Imm: 8}, {Kind: OpReg, Reg: AX}}} + if handled, err := fixedLocalInputAddressStep(&ins, &state, sig, proof, 0); err != nil || !handled { + t.Fatal("exact affine write rejected", handled, err) + } + if !reflect.DeepEqual(original, set(8)) || !reflect.DeepEqual(state.opaque[AX], set(16)) { + t.Fatal("affine write changed the prior address value") + } + }) +} diff --git a/amd64_fixed_local_packed.go b/amd64_fixed_local_packed.go new file mode 100644 index 000000000..ecf461c88 --- /dev/null +++ b/amd64_fixed_local_packed.go @@ -0,0 +1,138 @@ +package plan9asm + +// Reuse the complete source grammars and their byte effects. This adapter adds +// independently replayed private-frame memory, typed native input addresses +// and separately bound literal reads; it never supplies caller or raw access. +func fixedLocalPackedMoveStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + form, known, err := amd64PackedVectorMoveProperties(*ins, "amd64") + legacy := false + alignment := int64(1) + if !known { + if original, recognized, originalErr := x86NonTemporalVectorProperties(ins.Op, *ins, "amd64"); recognized { + known, err, legacy = true, originalErr, !original.spec.vector + form = original.move + if original.trap { + return true, goAMD64FrameContext("fixed local non-temporal move has no numerical continuation") + } + } + } + if !known { + original, recognized, originalErr := x86LegacyPackedMoveProperties(ins.Op, *ins, "amd64") + if !recognized { + return false, nil + } + known, err, legacy = true, originalErr, true + form = amd64PackedVectorMoveForm{ + byteWidth: 16, destination: 1, mask: -1, + memory: original.memory, memoryWrite: original.memoryWrite, + } + form.spec.aligned = original.aligned + } + if err != nil || form.undefinedOpcode() { + return known, goAMD64FrameContext("fixed local packed move has no numerical continuation") + } + if form.spec.aligned { + alignment = int64(form.byteWidth) + } + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return true, goAMD64FrameContext("fixed local packed move has an undefined mask") + } + width := int64(form.byteWidth) + widthMask := x86CallerVectorByteMask(width) + numerical := widthMask + if form.memory == 0 { + // The private proof does not model runtime mask values: require every + // possible read byte. Lowering still executes the original masked + // opcode, including its alignment and masked fault behavior. + if _, _, err := fixedLocalReadMemory(ins, 0, state, proof, residue, width, alignment); err != nil { + return true, err + } + } else { + numerical = state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] & widthMask + } + if form.memoryWrite { + if numerical != widthMask { + return true, goAMD64FrameContext("fixed local packed store has undefined source bytes") + } + offset, role, err := fixedLocalMemoryAccess(ins, form.memory, state, proof, residue, width, alignment, true, false) + if err != nil { + return true, err + } + // A runtime mask is not an all-one mask. A partial store may invalidate + // a private pointer spill, but cannot initialize its retained bytes. + if role == x86FixedLocalPrivateMemory { + fixedLocalNumericalStore(state, offset, width, form.mask < 0) + } + return true, nil + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + previous := state.vectors[destination] + numerical = form.definedActiveBytes(numerical, previous) + if legacy { + state.vectors[destination] = previous&^widthMask | numerical + } else { + state.vectors[destination] = ^widthMask | numerical + } + return true, nil +} + +func fixedLocalNumericalStore(state *x86FixedLocalState, offset, width int64, complete bool) { + fixedLocalRevokeSpillRoles(state, offset, width) + if complete { + state.bytes.setRange(offset, width, true) + } +} + +// KMOV supplies numerical predicates through its existing four-width grammar. +// Full-word private addresses never become mask values. Local reads/writes +// retain the same byte envelope and owner as every other fixed-frame access. +func fixedLocalMaskMoveStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + bits, known := amd64MaskMoveSpecs[ins.Op] + if !known { + return false, nil + } + fail := func() (bool, error) { + return true, goAMD64FrameContext("fixed local mask move has unproved shape or numerical bytes") + } + if !amd64MaskMoveOperandsValid(*ins, "amd64") { + return fail() + } + width := int64(bits / 8) + source, destination := ins.Args[0], ins.Args[1] + if source.Kind == OpMem || source.Kind == OpSym { + if _, _, err := fixedLocalReadMemory(ins, 0, state, proof, residue, width, 1); err != nil { + return true, err + } + } else if amd64IsKOperand(source) { + if !state.masks[source.Reg] { + return fail() + } + } else if !fixedLocalNumericRead(source, width, state) { + return fail() + } + if destination.Kind == OpMem { + offset, role, err := fixedLocalMemoryAccess(ins, 1, state, proof, residue, width, 1, true, false) + if err != nil { + return true, err + } + if role == x86FixedLocalPrivateMemory { + fixedLocalNumericalStore(state, offset, width, true) + } + return true, nil + } + if amd64IsKOperand(destination) { + state.masks[destination.Reg] = true + return true, nil + } + root, valid := x86CallerGP(destination.Reg) + if !valid || root == SP { + return fail() + } + // Every KMOV GP destination zero-extends to the full architectural word. + delete(state.pointers, root) + delete(state.opaque, root) + delete(state.constants, root) + fixedLocalWriteBooleans(state, destination.Reg, 8, 0) + state.known[root] = 255 + return true, nil +} diff --git a/amd64_fixed_local_packed_effects_test.go b/amd64_fixed_local_packed_effects_test.go new file mode 100644 index 000000000..656261cdb --- /dev/null +++ b/amd64_fixed_local_packed_effects_test.go @@ -0,0 +1,174 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalPackedMasksAndUpperAliases(t *testing.T) { + for _, row := range []struct { + name, body string + valid bool + }{ + {"legacy preserves unknown upper", "MOVOU 64(BP),X0\nVMOVDQU Y0,128(BP)", false}, + {"VEX zeroes upper", "VMOVDQU 64(BP),X0\nVMOVDQU64 Z0,128(BP)", true}, + {"EVEX zeroes upper", "VMOVDQU64 64(BP),Y31\nVMOVDQU64 Z31,128(BP)", true}, + {"legacy retains known upper", "VMOVDQU64 64(BP),Z0\nMOVOU 64(BP),X0\nVMOVDQU64 Z0,128(BP)", true}, + {"merge needs old bytes", "KMOVQ AX,K1\nVMOVDQA64 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", false}, + {"zero needs no old bytes", "KMOVQ AX,K1\nVMOVDQA64.Z 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", true}, + {"zero cannot bless unknown source", "KMOVQ AX,K1\nVMOVDQA64.Z Z1,K1,Z0\nVMOVDQA64 Z0,128(BP)", false}, + {"mask is undefined", "VMOVDQA64.Z 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", false}, + {"unknown source store", "VMOVDQA64 Z0,128(BP)", false}, + {"Go zero store traps", "VMOVDQA64 64(BP),Z0\nKMOVQ AX,K1\nVMOVDQA64.Z Z0,K1,128(BP)", false}, + } { + t.Run(row.name, func(t *testing.T) { + pkg, _, file, options := fixedLocalPackedFixture(t, row.body) + requireX86GoAssemblerResult(t, "amd64", fixedLocalPackedAssembly(row.body), true) + _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]) + if row.valid && err != nil || !row.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("private byte transfer valid=%v: %v", row.valid, err) + } + }) + } +} + +func TestGoAMD64FixedLocalPackedMaskedStoreCannotInitializeUnknownBytes(t *testing.T) { + const body = "VMOVDQA64 64(BP),Z0\nKMOVQ AX,K1\nVMOVDQA64 Z0,K1,128(BP)" + pkg, _, _, options := fixedLocalPackedFixture(t, body) + sig := options.Sigs[pkg.Path+".Value"] + for missing := 0; missing < 64; missing++ { + t.Run(fmt.Sprint(missing), func(t *testing.T) { + assembly := fixedLocalPackedAssembly(body) + var stores strings.Builder + for index := 0; index < 8; index++ { + if index != missing%8 { + fmt.Fprintf(&stores, "MOVB $1,%d(BP)\n", 128+missing/8*8+index) + } + } + assembly = strings.Replace(assembly, fmt.Sprintf("MOVQ AX,%d(BP)\n", 128+missing/8*8), stores.String(), 1) + assembly = strings.Replace(assembly, "MOVQ 128(BP),AX", fmt.Sprintf("MOVQ %d(BP),AX", 128+missing/8*8), 1) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("dynamic mask initialized retained byte %d: %v", missing, err) + } + }) + } +} + +func TestGoAMD64FixedLocalPackedMaskMovesCompleteLocalRoles(t *testing.T) { + for _, opcode := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + t.Run(opcode, func(t *testing.T) { + body := fmt.Sprintf("%s AX,K0\n%s K0,K7\n%s K7,128(BP)\n%s 64(BP),K1\n%s K1,CX\nVMOVDQA64.Z 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", opcode, opcode, opcode, opcode, opcode) + pkg, _, file, options := fixedLocalPackedFixture(t, body) + requireX86GoAssemblerResult(t, "amd64", fixedLocalPackedAssembly(body), true) + sig := options.Sigs[pkg.Path+".Value"] + if _, err := proveX86FixedLocal(file.Funcs[0], sig); err != nil { + t.Fatal("complete local mask transfer", err) + } + for name, changed := range map[string]string{ + "private pointer": strings.Replace(body, opcode+" AX,K0", opcode+" BP,K0", 1), + "unknown GP": strings.Replace(body, opcode+" AX,K0", opcode+" DX,K0", 1), + "unknown K": strings.Replace(body, opcode+" AX,K0", opcode+" K2,K0", 1), + "BP output": strings.Replace(body, opcode+" K1,CX", opcode+" K1,BP", 1), + "SP output": strings.Replace(body, opcode+" K1,CX", opcode+" K1,SP", 1), + "outside frame": strings.Replace(body, "128(BP)", "256(BP)", 1), + "global output": strings.Replace(body, "128(BP)", "external(SB)", 1), + } { + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, fixedLocalPackedAssembly(changed)) + if err != nil { + t.Fatal(err) + } + if name == "BP output" { + fixedLocalRequireNumericControl(t, file.Funcs[0], sig, fixedLocalPackedAssembly(changed), + "func Value(a uint64) uint64", "VMOVDQA64.Z 64(BP),K1,Z0", 0, 64, false, "load volatile i64") + return + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("mask move acquired another native role", err) + } + }) + } + }) + } +} + +func TestGoAMD64FixedLocalPackedCFGMergeMasksAndBytes(t *testing.T) { + for name, body := range map[string]string{ + "mask omitted on one path": "CMPQ AX,AX\nJEQ joined\nKMOVQ AX,K1\njoined:\nVMOVDQA64.Z 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", + "vector omitted on one path": "CMPQ AX,AX\nJEQ joined\nVMOVDQA64 64(BP),Z0\njoined:\nVMOVDQA64 Z0,128(BP)", + "BP differs on one path": "CMPQ AX,AX\nJEQ joined\nADDQ $64,BP\njoined:\nVMOVDQA64 64(BP),Z0\nVMOVDQA64 Z0,128(BP)", + } { + t.Run(name, func(t *testing.T) { + pkg, _, file, options := fixedLocalPackedFixture(t, body) + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("predecessor lost its must-defined/source role", err) + } + }) + } +} + +func TestGoAMD64FixedLocalPackedMaskedStoreRevokesPrivatePointerSpills(t *testing.T) { + for _, mask := range []string{"AX", "$0", "$1", "$-1"} { + for _, use := range []string{"MOVQ 32(BP),R11\nMOVQ 64(R11),AX", "MOVQ 32(BP),AX"} { + t.Run(mask+"/"+use, func(t *testing.T) { + body := "VMOVDQU64 64(BP),Z0\nMOVQ BP,32(BP)\nMOVQ " + mask + ",DX\nKMOVQ DX,K1\nVMOVDQU64 Z0,K1,32(BP)\n" + use + pkg, _, file, options := fixedLocalPackedFixture(t, body) + requireX86GoAssemblerResult(t, "amd64", fixedLocalPackedAssembly(body), true) + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("partial store preserved pointer authority or invented numerical bytes", err) + } + }) + } + } + // This domain proves mask definedness, not constant mask values. Even a + // zero mask must conservatively revoke the overlapping pointer certificate. + state := x86FixedLocalState{bytes: newFixedLocalByteSet(96), booleanBytes: newFixedLocalByteSet(96), addresses: map[int64]int64{32: 8, 80: 16}} + fixedLocalNumericalStore(&state, 36, 16, false) + if _, found := state.addresses[32]; found || state.addresses[80] != 16 { + t.Fatal("partial overlap did not revoke exactly the overlapping pointer spill") + } + for index := int64(0); index < 96; index++ { + if state.bytes.has(index) { + t.Fatalf("masked store invented initialized byte %d", index) + } + } +} + +func TestGoAMD64FixedLocalPackedMergeRequiresEveryDestinationByte(t *testing.T) { + for _, bank := range []struct { + name string + width int64 + }{{"X", 16}, {"Y", 32}, {"Z", 64}} { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$256-0\nVMOVDQU8 "+bank.name+"1,K1,"+bank.name+"0\nVMOVDQU8 "+bank.name+"0,0(BP)\nRET\n") + if err != nil { + t.Fatal(err) + } + for missing := int64(0); missing < bank.width; missing++ { + t.Run(fmt.Sprintf("%s/%d", bank.name, missing), func(t *testing.T) { + state := x86FixedLocalState{ + pointers: map[Reg]int64{BP: 0}, masks: map[Reg]bool{"K1": true}, + vectors: map[Reg]uint64{"Z0": ^(uint64(1) << uint(missing)), "Z1": ^uint64(0)}, + bytes: newFixedLocalByteSet(256), addresses: make(map[int64]int64), + } + proof := &x86FixedLocalProof{frame: 256, accesses: make(map[int]map[int]x86FixedLocalAccess)} + copy := file.Funcs[0].Instrs[1] + if known, err := fixedLocalPackedMoveStep(©, &state, proof, 0); !known || err != nil { + t.Fatalf("register merge: known=%v error=%v", known, err) + } + if got, want := state.vectors["Z0"], ^(uint64(1) << uint(missing)); got != want { + t.Fatalf("merge/upper-zero bytes=%016x want=%016x", got, want) + } + store := file.Funcs[0].Instrs[2] + if _, err := fixedLocalPackedMoveStep(&store, &state, proof, 0); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("merge result exposed an undefined destination byte", err) + } + }) + } + } +} diff --git a/amd64_fixed_local_packed_matrix_test.go b/amd64_fixed_local_packed_matrix_test.go new file mode 100644 index 000000000..507f2ecb2 --- /dev/null +++ b/amd64_fixed_local_packed_matrix_test.go @@ -0,0 +1,258 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalPackedCompleteProgram() (string, string, int) { + return fixedLocalPackedCompleteProgramRange(0, 195) +} + +func fixedLocalPackedCompleteProgramRange(start, end int) (string, string, int) { + var assembly, declarations strings.Builder + count := 0 + emit := func(operation string) { + index := count + count++ + if index < start || index >= end { + return + } + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(fixedLocalPackedAssembly(operation), "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + } + for _, row := range fixedLocalPackedMoveOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if bank == "Z" && !row.evex { + continue + } + for _, high := range []bool{false, true} { + first, last := bank+"0", bank+"1" + if high { + first, last = bank+"15", bank+"14" + if row.evex { + first, last = bank+"31", bank+"30" + } + } + for _, predicate := range []string{"none", "merge", "zero"} { + if predicate != "none" && !row.evex { + continue + } + operation := fmt.Sprintf("%s 64(BP),%s\n%s 128(BP),%s\n", row.name, first, row.name, last) + mask, opcode := "", row.name + if predicate != "none" { + operation += "KMOVQ AX,K7\n" + mask = "K7," + } + if predicate == "zero" { + opcode += ".Z" + } + operation += fmt.Sprintf("%s 64(BP),%s%s\n%s %s,%s%s\n%s %s,%s128(BP)", + opcode, mask, first, opcode, first, mask, last, row.name, last, mask) + emit(operation) + } + } + } + } + // Legacy aliases retain upper bytes and share the exact same local envelope. + for _, name := range []string{"MOVO", "MOVOA", "MOVOU", "MOVAPS", "MOVAPD", "MOVUPS", "MOVUPD"} { + operation := fmt.Sprintf("%s 64(BP),X15\n%s X15,X0\n%s X0,128(BP)", name, name, name) + emit(operation) + } + return assembly.String(), declarations.String(), count +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsLinuxBatch00(t *testing.T) { + fixedLocalPackedActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 0) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsLinuxBatch01(t *testing.T) { + fixedLocalPackedActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 1) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsLinuxBatch02(t *testing.T) { + fixedLocalPackedActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 2) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsLinuxBatch03(t *testing.T) { + fixedLocalPackedActualObjects(t, "linux/amd64", "x86_64-unknown-linux-gnu", 3) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsDarwinBatch00(t *testing.T) { + fixedLocalPackedActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 0) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsDarwinBatch01(t *testing.T) { + fixedLocalPackedActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 1) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsDarwinBatch02(t *testing.T) { + fixedLocalPackedActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 2) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsDarwinBatch03(t *testing.T) { + fixedLocalPackedActualObjects(t, "darwin/amd64", "x86_64-apple-darwin", 3) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsWindowsBatch00(t *testing.T) { + fixedLocalPackedActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 0) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsWindowsBatch01(t *testing.T) { + fixedLocalPackedActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 1) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsWindowsBatch02(t *testing.T) { + fixedLocalPackedActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 2) +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjectsWindowsBatch03(t *testing.T) { + fixedLocalPackedActualObjects(t, "windows/amd64", "x86_64-pc-windows-msvc", 3) +} + +func TestGoAMD64FixedLocalPackedObjectBatchPlan(t *testing.T) { + completeAssembly, completeDeclarations, count := fixedLocalPackedCompleteProgram() + if count != 195 { + t.Fatal("complete independent packed format inventory changed", count) + } + var assembly, declarations strings.Builder + for batch := 0; batch < 4; batch++ { + start, end, ok := fixedLocalObjectBatchRange(count, batch) + if !ok { + t.Fatal("missing packed object batch", batch) + } + body, decls, total := fixedLocalPackedCompleteProgramRange(start, end) + file, err := Parse(ArchAMD64, body) + if err != nil || total != count || len(file.Funcs) != end-start { + t.Fatal("packed object batch lost its exact function range", batch, err) + } + assembly.WriteString(body) + declarations.WriteString(decls) + } + if assembly.String() != completeAssembly || declarations.String() != completeDeclarations { + t.Fatal("packed object batches changed, omitted or duplicated original source bytes") + } +} + +func fixedLocalPackedActualObjects(t *testing.T, target, triple string, batch int) { + t.Helper() + // Keep lazy Go tools in this root's disposable cache, not a global cache + // or a different source/object observation. + t.Setenv("GOCACHE", t.TempDir()) + start, end, ok := fixedLocalObjectBatchRange(195, batch) + if !ok { + t.Fatal("invalid complete packed object batch", batch) + } + assembly, declarations, total := fixedLocalPackedCompleteProgramRange(start, end) + if total != 195 { + t.Fatalf("independent width/register/mask/legacy matrix changed: %d", total) + } + count := end - start + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + t.Run(fmt.Sprintf("batch%d", batch), func(t *testing.T) { + pkg, request, _, options := x86CMOVFixedLocalFixture(t, assembly, declarations, target, triple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil || strings.Count(ir, "%fixed_local_native = alloca") != count { + t.Fatalf("complete original frame inventory lost: %v", err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + var names []string + for index := start; index < end; index++ { + names = append(names, fmt.Sprintf("example.invalid/framefixture.Value%d", index)) + } + x86CompactExpandRequireAllDefinitions(t, text, names) + compileLLVMToObject(t, llc, triple, "complete-packed.ll", "complete-packed.o", text) + } + }) + }) +} + +func TestGoAMD64FixedLocalPackedEveryByteAndUnknownRole(t *testing.T) { + for _, bank := range []struct { + name string + width int + }{{"X", 16}, {"Y", 32}, {"Z", 64}} { + operation := fmt.Sprintf("VMOVDQA64 64(BP),%s0\nVMOVDQA64 %s0,128(BP)", bank.name, bank.name) + pkg, _, _, options := fixedLocalPackedFixture(t, operation) + sig := options.Sigs[pkg.Path+".Value"] + base := fixedLocalPackedAssembly(operation) + for missing := 0; missing < bank.width; missing++ { + t.Run(fmt.Sprintf("%s/byte%d", bank.name, missing), func(t *testing.T) { + var stores strings.Builder + for index := 0; index < 8; index++ { + if index != missing%8 { + fmt.Fprintf(&stores, "MOVB $1,%d(BP)\n", 64+missing/8*8+index) + } + } + assembly := strings.Replace(base, fmt.Sprintf("MOVQ AX,%d(BP)\n", 64+missing/8*8), stores.String(), 1) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing private byte obtained native read authority", err) + } + }) + } + for name, changed := range map[string]string{ + "past frame": strings.Replace(base, operation, strings.Replace(operation, "64(BP)", "256(BP)", 1), 1), + "misaligned": strings.Replace(base, operation, strings.Replace(operation, "64(BP)", "65(BP)", 1), 1), + "unknown base": strings.Replace(base, operation, strings.Replace(operation, "64(BP)", "64(DX)", 1), 1), + "external escape": strings.Replace(base, operation, "MOVQ BP,(AX)\n"+operation, 1), + "private mask": strings.Replace(base, operation, "KMOVQ BP,K1\n"+operation, 1), + "unaligned residue": strings.Replace(base, "ANDQ $-64,BP", "ANDQ $-8,BP", 1), + "unknown address size": strings.Replace(base, "ADDQ $64,BP", "ADDQ DX,BP", 1), + } { + t.Run(bank.name+"/"+name, func(t *testing.T) { + file, err := Parse(ArchAMD64, changed) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native frame boundary was weakened", err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalPackedMod64IndependentAliasReplay(t *testing.T) { + operation := "VMOVDQA64 64(BP),Z0\nVMOVDQA64 Z0,128(BP)" + pkg, _, file, options := fixedLocalPackedFixture(t, operation) + sig := options.Sigs[pkg.Path+".Value"] + proved, err := proveX86FixedLocal(file.Funcs[0], sig) + if err != nil { + t.Fatal(err) + } + for _, ins := range proved.Instrs { + if ins.Op != "VMOVDQA64" { + continue + } + for operand, access := range ins.x86FixedLocalMemory { + for residue, offset := range access.offset { + base := int64(((residue*8 + 64) & -64) - residue*8) + if offset != base+ins.Args[operand].Mem.Off || (offset+int64(residue*8))%64 != 0 { + t.Fatalf("mod64 residue %d: %+v", residue, access) + } + } + } + } + // Only residues six and seven overlap this scalar store with source bytes. + // Checking the old four residues would miss this real alias difference. + changed := strings.Replace(fixedLocalPackedAssembly(operation), operation, "MOVQ $0,80(SP)\n"+operation, 1) + requireX86GoAssemblerResult(t, "amd64", changed, true) + file, err = Parse(ArchAMD64, changed) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "byte identity") { + t.Fatal("residue-dependent byte aliases were relocated", err) + } +} diff --git a/amd64_fixed_local_packed_native_linux_amd64_test.go b/amd64_fixed_local_packed_native_linux_amd64_test.go new file mode 100644 index 000000000..010a04e53 --- /dev/null +++ b/amd64_fixed_local_packed_native_linux_amd64_test.go @@ -0,0 +1,35 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWideBatch00(t *testing.T) { + fixedLocalPackedNativeWideBatch(t, 0) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWideBatch01(t *testing.T) { + fixedLocalPackedNativeWideBatch(t, 1) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWideBatch02(t *testing.T) { + fixedLocalPackedNativeWideBatch(t, 2) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWideBatch03(t *testing.T) { + fixedLocalPackedNativeWideBatch(t, 3) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWideBatch04(t *testing.T) { + fixedLocalPackedNativeWideBatch(t, 4) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWideBatch05(t *testing.T) { + fixedLocalPackedNativeWideBatch(t, 5) +} + +func fixedLocalPackedNativeWideBatch(t *testing.T, batch int) { + t.Helper() + execution := x86PackedDoubleDwordNativeTools(t) + fixedLocalPackedRuntimeAPIs(t, execution, true, true, []string{"Translate", "Module", "GoModule"}, batch) +} diff --git a/amd64_fixed_local_packed_native_partition_test.go b/amd64_fixed_local_packed_native_partition_test.go new file mode 100644 index 000000000..2d3491462 --- /dev/null +++ b/amd64_fixed_local_packed_native_partition_test.go @@ -0,0 +1,81 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/token" + "strconv" + "strings" + "testing" +) + +func fixedLocalPackedRequireNativeBatchOwners(t *testing.T) { + t.Helper() + functions := fixedLocalLeaseFunctions(t, "amd64_fixed_local_packed_native_linux_amd64_test.go") + const prefix = "TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWide" + expected := make(map[string]int) + for batch := 0; batch < 6; batch++ { + expected[fmt.Sprintf("%sBatch%02d", prefix, batch)] = batch + } + for name := range functions { + if strings.HasPrefix(name, prefix) { + if _, ok := expected[name]; !ok { + t.Error("unknown or monolithic native source owner", name) + } + } + } + for name, batch := range expected { + function := functions[name] + if function == nil { + t.Error("missing complete native source owner", name) + continue + } + if len(function.Body.List) != 1 { + t.Fatal("native owner must delegate exactly once", name) + } + statement, ok := function.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("native owner lost its complete source call", name) + } + call, ok := statement.X.(*ast.CallExpr) + if !ok || fixedLocalLeaseCallName(call) != "fixedLocalPackedNativeWideBatch" || len(call.Args) != 2 { + t.Fatal("native owner must retain the shared source/preflight helper", name) + } + fixedLocalLeaseRequireArguments(t, &ast.CallExpr{Fun: call.Fun, Args: call.Args[:1]}, "t") + literal, ok := call.Args[1].(*ast.BasicLit) + if !ok || literal.Kind != token.INT || literal.Value != strconv.Itoa(batch) { + t.Fatal("native owner substituted or repeated a source batch", name) + } + } + helper := functions["fixedLocalPackedNativeWideBatch"] + if helper == nil { + t.Error("missing independently preflighted native source helper") + return + } + calls := fixedLocalLeaseCalls(helper.Body) + if calls["x86PackedDoubleDwordNativeTools"] != 1 || calls["fixedLocalPackedRuntimeAPIs"] != 1 || calls["t.Skip"] != 0 { + t.Fatal("every source owner needs the unchanged actual native preflight and API plan") + } + ast.Inspect(helper.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok || fixedLocalLeaseCallName(call) != "fixedLocalPackedRuntimeAPIs" { + return true + } + if len(call.Args) != 6 { + t.Fatal("native source owner must select one complete batch") + } + fixedLocalLeaseRequireArguments(t, &ast.CallExpr{Fun: call.Fun, Args: call.Args[:4]}, "t", "execution", "true", "true") + fixedLocalLeaseRequireArguments(t, &ast.CallExpr{Fun: call.Fun, Args: call.Args[5:]}, "batch") + apis, ok := call.Args[4].(*ast.CompositeLit) + if !ok || len(apis.Elts) != 3 { + t.Fatal("native source owner lost a public API") + } + for index, want := range []string{"Translate", "Module", "GoModule"} { + literal, ok := apis.Elts[index].(*ast.BasicLit) + if !ok || literal.Value != strconv.Quote(want) { + t.Fatal("native source owner changed its complete three-API plan") + } + } + return true + }) +} diff --git a/amd64_fixed_local_packed_native_partition_unix_test.go b/amd64_fixed_local_packed_native_partition_unix_test.go new file mode 100644 index 000000000..8f6b836bb --- /dev/null +++ b/amd64_fixed_local_packed_native_partition_unix_test.go @@ -0,0 +1,79 @@ +//go:build unix + +package plan9asm + +import ( + "go/ast" + "go/token" + "reflect" + "strings" + "testing" +) + +func TestFixedLocalPackedNativeBatchSelectionPreservesCompleteSourceUnion(t *testing.T) { + helper := fixedLocalLeaseFunctions(t, "amd64_fixed_local_packed_runtime_test.go")["fixedLocalPackedRuntimeAPIs"] + if helper == nil { + t.Fatal("missing complete packed source consumer") + } + var selection, original token.Pos + consumesSelected := false + ast.Inspect(helper.Body, func(node ast.Node) bool { + if loop, ok := node.(*ast.RangeStmt); ok { + if indices, ok := loop.X.(*ast.Ident); ok && indices.Name == "indices" { + consumesSelected = true + } + } + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + switch fixedLocalLeaseCallName(call) { + case "fixedLocalPackedBatchSelection": + selection = call.Pos() + case "fixedLocalPackedRunOriginal": + original = call.Pos() + fixedLocalLeaseRequireArguments(t, call, "t", "execution", "assembly", "declarations", "goOracle", "execute") + } + return true + }) + if selection == 0 || original <= selection || !consumesSelected { + t.Fatal("invalid ownership must fail before running the unchanged whole original, and only selected complete batches may execute") + } + for _, all := range []bool{false, true} { + batches := fixedLocalPackedLeaseBatches(t, all, "x86_64-unknown-linux-gnu") + indices, err := fixedLocalPackedBatchSelection(len(batches)) + if err != nil || len(indices) != len(batches) { + t.Fatal("default compile-only/SSE source selection changed", err) + } + var assembly, declarations, rows, prototypes strings.Builder + for index, batch := range batches { + selected, err := fixedLocalPackedBatchSelection(len(batches), index) + if err != nil || !reflect.DeepEqual(selected, []int{index}) || indices[index] != index { + t.Fatal("one required owner must retain exactly its complete original source batch", index, selected, err) + } + assembly.WriteString(batch.assembly) + declarations.WriteString(batch.declarations) + rows.WriteString(batch.rows) + prototypes.WriteString(batch.prototypes) + } + original, decls, cases := fixedLocalPackedRuntimeProgram(all) + _, oracle := fixedLocalPackedRuntimeOracles(cases, "x86_64-unknown-linux-gnu") + if assembly.String() != original || declarations.String() != decls || + !strings.Contains(oracle, rows.String()) || !strings.Contains(oracle, prototypes.String()) { + t.Fatal("independent source owners cropped, renamed or duplicated the original whole program") + } + } +} + +func TestFixedLocalPackedNativeBatchSelectionRejectsInvalidOwners(t *testing.T) { + for _, selected := range [][]int{{-1}, {6}, {0, 0}, {0, 1}, {5, 4}} { + if _, err := fixedLocalPackedBatchSelection(6, selected...); err == nil { + t.Error("invalid or repeated source selection became an executable owner", selected) + } + } + for _, total := range []int{-1, 0} { + if _, err := fixedLocalPackedBatchSelection(total); err == nil { + t.Error("empty source inventory became a successful owner", total) + } + } +} diff --git a/amd64_fixed_local_packed_runtime_test.go b/amd64_fixed_local_packed_runtime_test.go new file mode 100644 index 000000000..04aa3836d --- /dev/null +++ b/amd64_fixed_local_packed_runtime_test.go @@ -0,0 +1,337 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/nativeoracle" +) + +type fixedLocalPackedRuntimeCase struct { + width, bits, word, mode int +} + +func fixedLocalPackedRuntimeProgram(all bool) (string, string, []fixedLocalPackedRuntimeCase) { + var assembly, declarations strings.Builder + var cases []fixedLocalPackedRuntimeCase + add := func(opcode, bank string, width, bits, mode int) { + for word := -1; word <= width/8; word++ { + index := len(cases) + cases = append(cases, fixedLocalPackedRuntimeCase{width, bits, word, mode}) + fmt.Fprintf(&declarations, "func Value%d(a, mask uint64) uint64\n", index) + fmt.Fprintf(&assembly, "TEXT ·Value%d(SB),4,$512-24\nMOVQ a+0(FP),AX\nMOVQ mask+8(FP),CX\n", index) + assembly.WriteString("MOVQ SP,BP\nADDQ $64,BP\nANDQ $-64,BP\n") + assembly.WriteString("MOVQ $0x5a5a5a5a5a5a5a5a,DX\nMOVQ DX,248(BP)\n") + fmt.Fprintf(&assembly, "MOVQ DX,%d(BP)\n", 256+width) + for offset := 0; offset < width; offset += 8 { + fmt.Fprintf(&assembly, "MOVQ DX,%d(BP)\n", 256+offset) + } + for offset := 0; offset < width; offset += 8 { + fmt.Fprintf(&assembly, "MOVQ $%d,DX\nMOVQ DX,%d(BP)\n", int64(0x102030405060708)+int64(offset), 128+offset) + } + if word >= 0 && word < width/8 { + fmt.Fprintf(&assembly, "MOVQ AX,%d(BP)\n", 128+word*8) + } + first, last := bank+"0", bank+"1" + fmt.Fprintf(&assembly, "%s 256(BP),%s\n%s 256(BP),%s\n", opcode, first, opcode, last) + mask, effective := "", opcode + if mode != 0 { + assembly.WriteString("KMOVQ CX,K7\n") + mask = "K7," + } + if mode == 2 { + effective += ".Z" + } + fmt.Fprintf(&assembly, "%s 128(BP),%s%s\n%s %s,%s%s\n", effective, mask, first, effective, first, mask, last) + if mode == 2 { + mask = "" // Observe inactive zeroed bytes, not retained memory. + } + fmt.Fprintf(&assembly, "%s %s,%s256(BP)\nMOVQ %d(BP),AX\nMOVQ AX,ret+16(FP)\nRET\n", opcode, last, mask, 256+word*8) + } + } + for _, opcode := range []string{"MOVO", "MOVOA", "MOVOU", "MOVAPS", "MOVAPD", "MOVUPS", "MOVUPD"} { + add(opcode, "X", 16, 8, 0) + } + if all { + for _, row := range fixedLocalPackedMoveOracle { + width, bank, bits := 32, "Y", 8 + if row.evex { + width, bank = 64, "Z" + } + switch row.name { + case "VMOVAPD", "VMOVUPD", "VMOVDQA64", "VMOVDQU64": + bits = 64 + case "VMOVAPS", "VMOVUPS", "VMOVDQA32", "VMOVDQU32": + bits = 32 + case "VMOVDQU16": + bits = 16 + } + add(row.name, bank, width, bits, 0) + if row.evex { + add(row.name, bank, width, bits, 1) + add(row.name, bank, width, bits, 2) + } + } + } + return assembly.String(), declarations.String(), cases +} + +func fixedLocalPackedRuntimeOracles(cases []fixedLocalPackedRuntimeCase, triple string) (string, string) { + var goRows, cRows, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, row := range cases { + fmt.Fprintf(&goRows, "{Value%d,%d,%d,%d,%d},\n", index, row.width, row.bits, row.word, row.mode) + fmt.Fprintf(&cRows, "{value%d,%d,%d,%d,%d},\n", index, row.width, row.bits, row.word, row.mode) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t,uint64_t) __asm__(\"%sexample.invalid/framefixture.Value%d\");\n", index, prefix, index) + } + goOracle := `package framefixture +import "testing" +func TestOriginal(t *testing.T) { + cases := []struct { + call func(uint64,uint64) uint64 + width, bits, word, mode int + }{ +` + goRows.String() + ` + } + values := []uint64{0,1,^uint64(0),0x8000000000000000,0x123456789abcdef0,0x7ff81234567890ab} + masks := []uint64{0,1,^uint64(0),0x5555555555555555,0xaaaaaaaaaaaaaaaa,0x8000000000000000} + for index, row := range cases { + for _, value := range values { + for _, mask := range masks { + want := uint64(0x5a5a5a5a5a5a5a5a) + if row.word >= 0 && row.word < row.width/8 { + want = 0 + for bit := 0; bit < 64; bit++ { + input := uint64(0x5a5a5a5a5a5a5a5a) + if row.mode == 0 || mask >> uint((row.word*64+bit)/row.bits)&1 != 0 { + input = value + } else if row.mode == 2 { + input = 0 + } + want |= input & (uint64(1) << uint(bit)) + } + } + if got := row.call(value,mask); got != want { + t.Fatalf("case=%d input=%x mask=%x got=%x want=%x",index,value,mask,got,want) + } + } + } + } +} +` + cOracle := `#include +#include +` + cDeclarations.String() + ` +struct entry { + uint64_t (*call)(uint64_t,uint64_t); + int width,bits,word,mode; +}; +static const struct entry cases[]={ +` + cRows.String() + ` +}; +int main(void) { + const uint64_t values[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000),UINT64_C(0x123456789abcdef0),UINT64_C(0x7ff81234567890ab)}; + const uint64_t masks[]={0,1,UINT64_MAX,UINT64_C(0x5555555555555555),UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_C(0x8000000000000000)}; + for(unsigned n=0;n=0 && row.word>lane)&1) source=values[v]; + else if(row.mode==2) source=0; + want|=source&(UINT64_C(1)< 1 { + return nil, fmt.Errorf("packed source owner needs a nonempty inventory and at most one batch: total=%d selected=%v", + total, selected) + } + if len(selected) == 1 { + if selected[0] < 0 || selected[0] >= total { + return nil, fmt.Errorf("packed source batch outside complete inventory: total=%d selected=%v", total, selected) + } + return []int{selected[0]}, nil + } + var indices []int + for index := 0; index < total; index++ { + indices = append(indices, index) + } + return indices, nil +} + +func fixedLocalPackedRuntimeAPIs(t *testing.T, execution x86FloatCompareExecution, all, execute bool, apis []string, selected ...int) { + t.Helper() + batches := fixedLocalPackedLeaseBatches(t, all, execution.triple) + indices, err := fixedLocalPackedBatchSelection(len(batches), selected...) + if err != nil { + t.Fatal(err) + } + // Share only disposable compiler artifacts, never a source observation. + t.Setenv("GOCACHE", t.TempDir()) + assembly, declarations, cases := fixedLocalPackedRuntimeProgram(all) + goOracle, cOracle := fixedLocalPackedRuntimeOracles(cases, execution.triple) + // Compile/execute the unchanged whole original before starting any API + // lease. The following batches are complete independent source packages. + fixedLocalPackedRunOriginal(t, execution, assembly, declarations, goOracle, execute) + if !execute { + dir := t.TempDir() + input, output := filepath.Join(dir, "oracle.c"), filepath.Join(dir, "oracle.o") + if err := os.WriteFile(input, []byte(cOracle), 0600); err != nil { + t.Fatal(err) + } + command := append([]string(nil), execution.compiler...) + command = append(command, "-O2", "-c", input, "-o", output) + x86FloatCompareCaptureCommand(t, exec.Command(command[0], command[1:]...)) + } + for _, index := range indices { + batch := batches[index] + t.Run(fmt.Sprintf("batch%d", index), func(t *testing.T) { + pkg, request, _, options := x86CMOVFixedLocalFixture(t, batch.assembly, batch.declarations, + runtime.GOOS+"/amd64", execution.triple) + fixedLocalConversionPrepareToolCache(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, apis, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + var names []string + for member := batch.start; member < batch.end; member++ { + names = append(names, fmt.Sprintf("example.invalid/framefixture.Value%d", member)) + } + for _, pipeline := range []struct { + name, level string + optimize bool + }{ + {"backend_O0", "-O0", false}, + {"backend_O2", "-O2", false}, + {"IR_O2/backend_O2", "-O2", true}, + } { + t.Run(pipeline.name, func(t *testing.T) { + input := ir + if pipeline.optimize { + input = x86MulDivRoundOptimize(t, ir) + } + x86CompactExpandRequireAllDefinitions(t, input, names) + if execute { + execution.compileAndRunLLVM(t, "packed-native-frame", input, batch.oracle, pipeline.level) + } else { + x86PackedStringCompileObject(t, execution.llc, execution.triple, input, pipeline.level) + } + t.Logf("complete packed oracle [%d,%d): %d kernels x 6 values x 6 masks x 64 result bits; execute=%v", + batch.start, batch.end, batch.end-batch.start, execute) + }) + } + }) + }) + } + t.Logf("cases=%d original Go + APIs=%v; complete source batches=%d selected=%v execute=%v native-wide=%v", + len(cases), apis, len(batches), indices, execute, all) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedSSE(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + fixedLocalPackedRuntime(t, execution, false, true) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("Rosetta SSE diagnostic, not native AVX/AVX512 execution") + } +} + +func TestGoAMD64FixedLocalPackedWideRuntimeOracleCompileOnly(t *testing.T) { + execution := fixedLocalPackedCompileOnlyTools(t) + fixedLocalPackedRuntimeAPIs(t, execution, true, false, []string{"Translate"}) + t.Log("full wide oracle compiled, not executed; required Linux/amd64 root owns native coverage") +} + +func TestGoAMD64FixedLocalPackedWideRuntimeOracleModuleCompileOnly(t *testing.T) { + execution := fixedLocalPackedCompileOnlyTools(t) + fixedLocalPackedRuntimeAPIs(t, execution, true, false, []string{"Module"}) + t.Log("full wide oracle compiled, not executed; required Linux/amd64 root owns native coverage") +} + +func TestGoAMD64FixedLocalPackedWideRuntimeOracleGoModuleCompileOnly(t *testing.T) { + execution := fixedLocalPackedCompileOnlyTools(t) + fixedLocalPackedRuntimeAPIs(t, execution, true, false, []string{"GoModule"}) + t.Log("full wide oracle compiled, not executed; required Linux/amd64 root owns native coverage") +} + +// Object compilation needs a real compiler and target headers, not an x86 CPU +// or executable transport. This value must never be used to claim execution. +func fixedLocalPackedCompileOnlyTools(t *testing.T) x86FloatCompareExecution { + t.Helper() + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang required for complete packed object compilation") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + compiler := []string{clang, "-target", triple} + if runtime.GOOS == "darwin" { + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + flags, err := nativeoracle.DarwinClangFlags(ctx, os.Environ()) + if err != nil { + t.Fatal(err) + } + compiler = append(compiler, flags...) + } + if runtime.GOOS == "linux" && runtime.GOARCH != "amd64" && runtime.GOARCH != "386" { + const root = "/usr/x86_64-linux-gnu" + include := filepath.Join(root, "include") + if info, err := os.Stat(include); err != nil || !info.IsDir() { + t.Fatalf("required AMD64 C target sysroot headers %s: %v", include, err) + } + compiler = append(compiler, "--sysroot="+root, "-isystem", include) + } + return x86FloatCompareExecution{llc: llc, triple: triple, compiler: compiler} +} + +func TestGoAMD64FixedLocalPackedRuntimeOwnership(t *testing.T) { + for _, row := range []struct { + all bool + count int + }{{false, 28}, {true, 340}} { + _, declarations, cases := fixedLocalPackedRuntimeProgram(row.all) + if len(cases) != row.count || strings.Count(declarations, "func Value") != row.count { + t.Fatalf("runtime row ownership all=%v: got %d want %d", row.all, len(cases), row.count) + } + } + source, err := os.ReadFile("amd64_fixed_local_packed_native_linux_amd64_test.go") + if err != nil { + t.Fatal(err) + } + fixedLocalPackedRequireNativeBatchOwners(t) + if strings.Contains(string(source), "t.Skip") { + t.Fatal("native wide runtime coverage must not silently skip") + } +} diff --git a/amd64_fixed_local_packed_test.go b/amd64_fixed_local_packed_test.go new file mode 100644 index 000000000..22df0e209 --- /dev/null +++ b/amd64_fixed_local_packed_test.go @@ -0,0 +1,166 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "strings" + "testing" +) + +// Independent names/widths, not iteration over the implementation map. +var fixedLocalPackedMoveOracle = []struct { + name string + evex bool +}{ + {"VMOVDQA", false}, {"VMOVDQU", false}, + {"VMOVAPD", true}, {"VMOVAPS", true}, + {"VMOVUPD", true}, {"VMOVUPS", true}, + {"VMOVDQA32", true}, {"VMOVDQA64", true}, + {"VMOVDQU8", true}, {"VMOVDQU16", true}, + {"VMOVDQU32", true}, {"VMOVDQU64", true}, +} + +func fixedLocalPackedAssembly(operation string) string { + var source strings.Builder + source.WriteString("TEXT ·Value(SB),4,$256-16\nMOVQ a+0(FP),AX\n") + source.WriteString("MOVQ SP,BP\nADDQ $64,BP\nANDQ $-64,BP\n") + for offset := 64; offset < 192; offset += 8 { + fmt.Fprintf(&source, "MOVQ AX,%d(BP)\n", offset) + } + source.WriteString(operation + "\n") + source.WriteString("MOVQ 128(BP),AX\nMOVQ AX,ret+8(FP)\nRET\n") + return source.String() +} + +func fixedLocalPackedFixture(t *testing.T, operation string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options) { + t.Helper() + assembly := fixedLocalPackedAssembly(operation) + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a uint64) uint64", assembly) + sig, err := DeriveGoFuncABI0Signature(pkg.Path+".Value", pkg.Types.Scope().Lookup("Value").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + return pkg, request, file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) }, + Sigs: map[string]FuncSig{sig.Name: sig}, + } +} + +func TestGoAMD64FixedLocalPackedMoveTypedFamilyProof(t *testing.T) { + var original strings.Builder + count := 0 + for _, row := range fixedLocalPackedMoveOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if bank == "Z" && !row.evex { + continue + } + for _, masked := range []bool{false, true} { + if masked && !row.evex { + continue + } + name := fmt.Sprintf("%s/%s/masked=%v", row.name, bank, masked) + load, copy, store := "64(BP),"+bank+"0", bank+"0,"+bank+"1", bank+"1,128(BP)" + prefix := "" + if masked { + prefix = "KMOVQ AX,K1\n" + // Defined merge input and destination memory are independent. + prefix += fmt.Sprintf("%s 64(BP),%s0\n%s 64(BP),%s1\n", row.name, bank, row.name, bank) + load, copy, store = "64(BP),K1,"+bank+"0", bank+"0,K1,"+bank+"1", bank+"1,K1,128(BP)" + } + operation := prefix + fmt.Sprintf("%s %s\n%s %s\n%s %s", row.name, load, row.name, copy, row.name, store) + assembly := fixedLocalPackedAssembly(operation) + original.WriteString(strings.Replace(assembly, "·Value", fmt.Sprintf("·Value%d", count), 1)) + count++ + t.Run(name, func(t *testing.T) { + pkg, _, file, options := fixedLocalPackedFixture(t, operation) + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") || + GoAMD64FixedLocalFrameBindingNeeded(file, "386") { + t.Error("physical private-frame transport needs its original backing regardless of alignment") + } + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); err != nil { + t.Fatalf("initialized pure numeric packed-frame proof: %v", err) + } + }) + } + } + } + if count != 64 { + t.Fatalf("independent full family fixture count changed: %d", count) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalPackedMoveActualOriginalObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + for _, operation := range []string{ + "VMOVDQA 64(BP),Y0\nVMOVDQA Y0,128(BP)", + "VMOVDQA64 64(BP),Z31\nVMOVDQA64 Z31,128(BP)", + } { + t.Run(target.name+"/"+strings.Fields(operation)[0], func(t *testing.T) { + pkg, request, file, options := fixedLocalPackedFixture(t, operation) + request.Target, options.TargetTriple = target.name, target.triple + source := requireGoAMD64FrameCapture(t, pkg, request) + observed := source.functions[pkg.Path+".Value"] + if observed.locals != 264 || observed.args != 16 || len(observed.code) == 0 { + t.Fatal("missing original automatic frame/object witness") + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "fixed_local_native") || !strings.Contains(ir, "asm sideeffect") { + t.Fatalf("actual local certificate was not consumed: %v", err) + } + compileLLVMToObject(t, llc, target.triple, "packed-frame.ll", "packed-frame.o", ir) + }) + }) + } + } +} + +func TestGoAMD64FixedLocalBPPrivateOriginOnly(t *testing.T) { + operation := "MOVOU 64(BP),X0\nPADDQ 64(BP),X0\nMOVOU X0,128(BP)" + pkg, _, file, options := fixedLocalPackedFixture(t, operation) + sig := options.Sigs[pkg.Path+".Value"] + if _, err := proveX86FixedLocal(file.Funcs[0], sig); err != nil { + t.Fatalf("BP private affine source: %v", err) + } + for name, assembly := range map[string]string{ + "numeric BP": strings.Replace(fixedLocalPackedAssembly(operation), "MOVQ SP,BP", "MOVQ AX,BP", 1), + "unknown BP": strings.Replace(fixedLocalPackedAssembly(operation), "MOVQ SP,BP", "MOVQ DX,BP", 1), + "narrow BP": strings.Replace(fixedLocalPackedAssembly(operation), "MOVQ SP,BP", "MOVL SP,BP", 1), + "escaped BP": strings.Replace(fixedLocalPackedAssembly(operation), "MOVQ AX,ret+8(FP)", "MOVQ BP,ret+8(FP)", 1), + } { + t.Run(name, func(t *testing.T) { + changed, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if name == "numeric BP" { + fixedLocalRequireNumericControl(t, changed.Funcs[0], sig, assembly, "func Value(a uint64) uint64", + "PADDQ 64(BP),X0", 0, 16, false, "paddq") + return + } + if _, err := proveX86FixedLocal(changed.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown/numeric/escaping BP obtained local authority: %v", err) + } + }) + } +} diff --git a/amd64_fixed_local_pointer_choice_dead_test.go b/amd64_fixed_local_pointer_choice_dead_test.go new file mode 100644 index 000000000..446cd3838 --- /dev/null +++ b/amd64_fixed_local_pointer_choice_dead_test.go @@ -0,0 +1,73 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalFiveWayPointerBody(final string, spill bool) string { + var body strings.Builder + join := "JMP joined\n" + if spill { + join = "MOVQ CX,8(SP)\n" + join + } + for index := 0; index < 4; index++ { + fmt.Fprintf(&body, "CMPQ AX,$%d\nJE choice%d\n", index, index) + } + body.WriteString("LEAQ choice<>+32(SB),CX\n" + join) + for index := 0; index < 4; index++ { + fmt.Fprintf(&body, "choice%d:\nLEAQ choice<>+%d(SB),CX\n%s", index, index*8, join) + } + body.WriteString("joined:\n" + final) + return body.String() +} + +type fixedLocalPointerChoiceCFGCase struct { + name, final string + spill, fails bool +} + +func fixedLocalPointerChoiceCFGCases() []fixedLocalPointerChoiceCFGCase { + return []fixedLocalPointerChoiceCFGCase{ + {"dead address", "", false, false}, + {"fully overwritten address", "MOVQ $99,CX\nMOVQ CX,AX", false, false}, + {"unknown address is not NUM", "MOVQ CX,AX", false, true}, + {"more than four addresses remain readable", "MOVQ (CX),AX", false, false}, + {"dead pointer spill", "", true, false}, + {"fully overwritten spill", "MOVQ $99,8(SP)\nMOVQ 8(SP),AX", true, false}, + {"unknown spill is not NUM", "MOVQ 8(SP),AX", true, true}, + {"more than four spilled addresses remain readable", "MOVQ 8(SP),CX\nMOVQ (CX),AX", true, false}, + } +} + +func fixedLocalPointerChoiceAllPositiveCases() []fixedLocalPointerChoiceCase { + rows := fixedLocalPointerChoiceCases() + for _, row := range fixedLocalPointerChoiceCFGCases() { + if !row.fails { + rows = append(rows, fixedLocalPointerChoiceCase{row.name, fixedLocalFiveWayPointerBody(row.final, row.spill)}) + } + } + return rows +} + +func TestGoAMD64FixedLocalPointerChoiceDeadCFGOrigins(t *testing.T) { + var original strings.Builder + original.WriteString(fixedLocalChoiceData) + for index, row := range fixedLocalPointerChoiceCFGCases() { + fn, sig, source := fixedLocalPointerChoiceFixture(t, fixedLocalFiveWayPointerBody(row.final, row.spill)) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + _, err := proveX86FixedLocal(fn, sig) + if row.fails { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("CFG address acquired a numerical value", err) + } + } else if err != nil { + t.Fatal("unused or overwritten CFG address became a new rejection", err) + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} diff --git a/amd64_fixed_local_pointer_choice_objects_go127_test.go b/amd64_fixed_local_pointer_choice_objects_go127_test.go new file mode 100644 index 000000000..acc40a040 --- /dev/null +++ b/amd64_fixed_local_pointer_choice_objects_go127_test.go @@ -0,0 +1,16 @@ +//go:build go1.27 + +package plan9asm + +import "testing" + +// Mixed DATA uses the existing audited Go 1.27 object producer. Older SDKs +// retain the native-only object gate and an explicit producer rejection test. +func TestGoAMD64FixedLocalPointerChoiceObjects(t *testing.T) { + rows := fixedLocalPointerChoiceAllPositiveCases() + if len(rows) != 25 { + t.Fatal("expected 19 address-choice and 6 complete CFG functions", len(rows)) + } + assembly, declarations := fixedLocalPointerChoiceProgram(t, fixedLocalChoiceData, rows) + fixedLocalCompleteVectorObjects(t, assembly, declarations, len(rows)) +} diff --git a/amd64_fixed_local_pointer_choice_objects_legacy_test.go b/amd64_fixed_local_pointer_choice_objects_legacy_test.go new file mode 100644 index 000000000..f9adbb91d --- /dev/null +++ b/amd64_fixed_local_pointer_choice_objects_legacy_test.go @@ -0,0 +1,41 @@ +//go:build !go1.27 + +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalPointerChoicesNeedAuditedDATAProducer(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + rows := append(fixedLocalPointerChoiceAllPositiveCases(), fixedLocalNativeAddressCases()...) + if len(rows) != 42 { + t.Fatal("complete mixed-DATA source inventory changed", len(rows)) + } + assembly, declarations := fixedLocalPointerChoiceProgram(t, fixedLocalChoiceData, rows) + // Original Go accepts the complete source. Only the independently audited + // object producer is unavailable; this is not an instruction rejection. + requireX86GoAssemblerResult(t, "amd64", assembly, true) + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, _ := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil { + if closeErr := source.Close(); closeErr != nil { + t.Error(closeErr) + } + } + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "mixed DATA needs its independent actual Go object producer") { + t.Fatal("unaudited mixed-DATA producer did not retain its exact boundary", err) + } + }) + } +} diff --git a/amd64_fixed_local_pointer_choice_objects_test.go b/amd64_fixed_local_pointer_choice_objects_test.go new file mode 100644 index 000000000..3810dc158 --- /dev/null +++ b/amd64_fixed_local_pointer_choice_objects_test.go @@ -0,0 +1,32 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "strings" + "testing" +) + +func fixedLocalPointerChoiceProgram(t *testing.T, data string, rows []fixedLocalPointerChoiceCase) (string, string) { + t.Helper() + var assembly, declarations strings.Builder + assembly.WriteString(data) + for index, row := range rows { + _, _, source := fixedLocalPointerChoiceFixture(t, row.body) + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(source, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p,q *uint64,a uint64) uint64\n", name) + t.Logf("complete function example.invalid/framefixture.%s: %s", name, row.name) + } + t.Logf("complete assembly: functions=%d sha256=%x", len(rows), sha256.Sum256([]byte(assembly.String()))) + return assembly.String(), declarations.String() +} + +func TestGoAMD64FixedLocalNativePointerChoiceObjects(t *testing.T) { + rows := fixedLocalNativePointerChoiceCases() + if len(rows) != 3 { + t.Fatal("expected one native-input CMOV and two CFG joins", len(rows)) + } + assembly, declarations := fixedLocalPointerChoiceProgram(t, "", rows) + fixedLocalCompleteVectorObjects(t, assembly, declarations, len(rows)) +} diff --git a/amd64_fixed_local_pointer_choice_runtime_go127_test.go b/amd64_fixed_local_pointer_choice_runtime_go127_test.go new file mode 100644 index 000000000..2118b747a --- /dev/null +++ b/amd64_fixed_local_pointer_choice_runtime_go127_test.go @@ -0,0 +1,152 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func fixedLocalPointerChoiceRuntimeProgram(t *testing.T, triple string) (assembly, declarations, original, driver string) { + t.Helper() + rows := append(fixedLocalPointerChoiceAllPositiveCases(), fixedLocalNativeAddressCases()...) + rows = append(rows, fixedLocalMachineAddressCases()...) + if len(rows) != 48 { + t.Fatal("complete pointer-choice runtime inventory changed", len(rows)) + } + assembly, declarations = fixedLocalPointerChoiceProgram(t, fixedLocalChoiceData, rows) + inputs := []uint64{0, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 1 << 63, ^uint64(0)} + var goInputs, cInputs, goCalls, cCalls, cDeclarations strings.Builder + for _, value := range inputs { + fmt.Fprintf(&goInputs, "0x%x,", value) + fmt.Fprintf(&cInputs, "UINT64_C(0x%x),", value) + } + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index := range rows { + label := runtimeCAsmLabel(prefix + fmt.Sprintf("example.invalid/framefixture.Value%d", index)) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t*,uint64_t*,uint64_t) __asm__(%s);\n", index, label) + fmt.Fprintf(&goCalls, "check(%d,Value%d)\n", index, index) + fmt.Fprintf(&cCalls, "check(%d,value%d);\n", index, index) + } + original = `package main +import "fmt" +var inputs = []uint64{` + goInputs.String() + `} +func check(index int, fn func(*uint64, *uint64, uint64) uint64) { + for sample, input := range inputs { + for alias := 0; alias < 2; alias++ { + var left, right [32]uint64 + for i := range left { + left[i] = uint64(i+1)*0x0102030405060708 ^ input + right[i] = ^left[i] ^ 0x96a5b4c3d2e1f087 + } + + p, q := &left[2], &right[2] + if alias != 0 { + q = p + } + value := fn(p, q, input) + + fmt.Printf("%d %d %d %016x", index, sample, alias, value) + for _, buffer := range [2][32]uint64{left, right} { + for _, value := range buffer { + fmt.Printf(" %016x", value) + } + } + fmt.Println() + } + } +} +` + declarations + "func main() {\n" + goCalls.String() + "}\n" + driver = "#include \n#include \n" + runtimeCAsmLabelPreamble + + cDeclarations.String() + "static const uint64_t inputs[]={" + cInputs.String() + `}; +static void check(int index, uint64_t (*fn)(uint64_t *, uint64_t *, uint64_t)) { + for (unsigned sample = 0; sample < 16; sample++) { + uint64_t input = inputs[sample]; + for (unsigned alias = 0; alias < 2; alias++) { + uint64_t left[32], right[32]; + for (unsigned i = 0; i < 32; i++) { + left[i] = (uint64_t)(i+1)*UINT64_C(0x0102030405060708)^input; + right[i] = ~left[i]^UINT64_C(0x96a5b4c3d2e1f087); + } + + uint64_t *p = &left[2], *q = alias ? p : &right[2]; + uint64_t value = fn(p, q, input); + + printf("%d %u %u %016llx", index, sample, alias, (unsigned long long)value); + for (unsigned i = 0; i < 32; i++) { + printf(" %016llx", (unsigned long long)left[i]); + } + for (unsigned i = 0; i < 32; i++) { + printf(" %016llx", (unsigned long long)right[i]); + } + putchar('\n'); + } + } +} +int main(void) { +` + cCalls.String() + "return 0;\n}\n" + return +} + +func fixedLocalPointerChoiceRuntime(t *testing.T, execution x86FloatCompareExecution, execute bool) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + assembly, declarations, original, driver := fixedLocalPointerChoiceRuntimeProgram(t, execution.triple) + var want []string + if execute { + output := x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original) + want = strings.Split(strings.TrimSpace(string(output)), "\n") + if len(want) != 48*16*2 { + t.Fatal("original Go pointer-choice runtime output inventory changed", len(want)) + } + } else { + x86COMICompileOriginalGo(t, map[string]string{ + "go.mod": "module pointerchoices\ngo 1.20\n", "main.go": original, "original_amd64.s": assembly, + }, false) + } + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, declarations, + runtime.GOOS+"/amd64", execution.triple) + fixedLocalResultsRuntimeAPIs(t, pkg, request, file, options, assembly, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + if !execute { + execution.compileLLVM(t, "pointer-choices", input, driver, level) + return + } + output := execution.compileAndRunLLVM(t, "pointer-choices", input, driver, level) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatal("LLVM pointer-choice output inventory changed", len(got), len(want)) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("pointer-choice kernel=%d sample=%d alias=%d: original=%s LLVM=%s", + index/32, index/2%16, index%2, want[index], got[index]) + } + } + }) + } + }) + t.Logf("48 complete functions, 16 input patterns, two pointer alias relations, return and both 256-byte buffers; execute=%v", execute) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPointerChoices(t *testing.T) { + fixedLocalPointerChoiceRuntime(t, x86FloatCompareRuntimeTools(t, x86FloatCompareSSE), true) +} + +func TestGoAMD64FixedLocalPointerChoiceRuntimeObjects(t *testing.T) { + fixedLocalPointerChoiceRuntime(t, fixedLocalPackedCompileOnlyTools(t), false) +} diff --git a/amd64_fixed_local_pointer_choices_test.go b/amd64_fixed_local_pointer_choices_test.go new file mode 100644 index 000000000..7d79a7837 --- /dev/null +++ b/amd64_fixed_local_pointer_choices_test.go @@ -0,0 +1,120 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +const fixedLocalChoiceData = "DATA choice<>+0(SB)/8,$11\n" + + "DATA choice<>+56(SB)/8,$19\nGLOBL choice<>(SB),8,$64\n" + + "DATA other<>+0(SB)/8,$23\nGLOBL other<>(SB),8,$64\n" + + "DATA twin<>+0(SB)/8,$11\nDATA twin<>+56(SB)/8,$19\nGLOBL twin<>(SB),8,$64\n" + +const fixedLocalChoicePrefix = "LEAQ choice<>(SB),R8\nMOVQ SI,CX\nCMPQ AX,$0\nCMOVQEQ R8,CX\n" + +type fixedLocalPointerChoiceCase struct{ name, body string } + +func fixedLocalChoiceCountBody(count int) string { + var body strings.Builder + body.WriteString("LEAQ choice<>(SB),CX\n") + for index := 1; index < count; index++ { + fmt.Fprintf(&body, "LEAQ choice<>+%d(SB),R8\nCMPQ AX,$%d\nCMOVQEQ R8,CX\n", index*8, index) + } + body.WriteString("MOVQ (CX),AX") + return body.String() +} + +func fixedLocalNativePointerChoiceCases() []fixedLocalPointerChoiceCase { + return []fixedLocalPointerChoiceCase{ + {"two native inputs", "MOVQ SI,CX\nCMPQ AX,$0\nCMOVQEQ DI,CX\nMOVQ (CX),AX"}, + {"affine native input join", "CMPQ AX,$0\nJEQ left\nMOVQ SI,CX\nJMP join\nleft:\nMOVQ DI,CX\njoin:\nMOVQ (CX),AX"}, + {"memory native input join", "CMPQ AX,$1\nJEQ left\nMOVQ SI,CX\nJMP join\nleft:\nMOVQ DI,CX\njoin:\nMOVQ (CX),AX"}, + } +} + +func fixedLocalPointerChoiceCases() []fixedLocalPointerChoiceCase { + rows := []fixedLocalPointerChoiceCase{ + {"conditional read", fixedLocalChoicePrefix + "MOVQ (CX),AX"}, + {"reverse conditional operands", "LEAQ choice<>(SB),CX\nCMPQ AX,$0\nCMOVQNE SI,CX\nMOVQ (CX),AX"}, + } + rows = append(rows, fixedLocalNativePointerChoiceCases()...) + return append(rows, []fixedLocalPointerChoiceCase{ + {"two literals", "LEAQ choice<>(SB),CX\nLEAQ other<>(SB),DX\nCMPQ AX,$0\nCMOVQEQ DX,CX\nMOVQ (CX),AX"}, + {"identical contents retain address identity", "LEAQ choice<>(SB),CX\nLEAQ twin<>(SB),DX\nCMPQ CX,DX\nSETNE AL\nMOVBLZX AL,AX"}, + {"two offsets", "LEAQ choice<>(SB),CX\nLEAQ choice<>+8(SB),DX\nCMPQ AX,$0\nCMOVQEQ DX,CX\nMOVOU 32(CX),X0\nMOVQ X0,AX"}, + {"four bounded choices", fixedLocalChoiceCountBody(4)}, + {"CFG join", "CMPQ AX,$0\nJE literal\nMOVQ SI,CX\nJMP joined\nliteral:\nLEAQ choice<>(SB),CX\njoined:\nMOVQ (CX),AX"}, + {"private pointer spill", fixedLocalChoicePrefix + "MOVQ CX,8(SP)\nMOVQ 8(SP),DX\nMOVQ (DX),AX"}, + {"conditional pointer spill read", "LEAQ choice<>(SB),R8\nMOVQ R8,8(SP)\nMOVQ SI,CX\nCMPQ AX,$0\nCMOVQEQ 8(SP),CX\nMOVQ (CX),AX"}, + {"exchange pointer spill", fixedLocalChoicePrefix + "MOVQ CX,8(SP)\nMOVQ DI,DX\nXCHGQ DX,8(SP)\nMOVQ (DX),AX"}, + {"bounded affine shift", fixedLocalChoicePrefix + "ADDQ $8,CX\nMOVQ 48(CX),AX"}, + {"sentinel excludes readonly write", fixedLocalChoicePrefix + "CMPQ CX,R8\nJE done\nMOVQ AX,(CX)\ndone:"}, + {"sentinel excludes literal extent", fixedLocalChoicePrefix + "CMPQ CX,R8\nJE done\nMOVQ 64(CX),AX\ndone:"}, + {"captured comparator value", fixedLocalChoicePrefix + "CMPQ CX,R8\nMOVQ DI,R8\nJE done\nMOVQ 64(CX),AX\ndone:"}, + {"compared value exchanged", fixedLocalChoicePrefix + "CMPQ CX,R8\nXCHGQ CX,DX\nJE done\nMOVQ 64(DX),AX\ndone:"}, + {"finite sentinel loop", "LEAQ choice<>(SB),R8\nMOVQ SI,CX\nagain:\nCMPQ CX,R8\nJE done\nMOVQ 64(CX),AX\nMOVQ R8,CX\nJMP again\ndone:"}, + }...) +} + +func fixedLocalPointerChoiceFixture(t *testing.T, body string) (Func, FuncSig, string) { + t.Helper() + fn, sig, source := fixedLocalInputFixture(t, "MOVQ AX,DX\n"+body) + // Internal replay only; the public object gates capture complete sources + // through the existing actual Go source/object producer. + sig.x86FixedLocal = &x86FixedLocalBinding{literalData: map[string]x86FixedLocalData{ + "choice<>": {size: 64, alignment: 16}, + "other<>": {size: 64, alignment: 16}, + "twin<>": {size: 64, alignment: 16}, + }} + return fn, sig, source +} + +func TestGoAMD64FixedLocalPointerChoicesCompleteSources(t *testing.T) { + var original strings.Builder + original.WriteString(fixedLocalChoiceData) + for index, row := range fixedLocalPointerChoiceCases() { + fn, sig, assembly := fixedLocalPointerChoiceFixture(t, row.body) + original.WriteString(strings.Replace(assembly, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("individually bounded pointer alternatives rejected", err) + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalPointerChoicesRetainBoundaries(t *testing.T) { + t.Run("native pointer bits are numerical", func(t *testing.T) { + fn, sig, _ := fixedLocalPointerChoiceFixture(t, fixedLocalChoicePrefix+"XORQ CX,AX") + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("complete native arithmetic source rejected", err) + } + }) + for _, row := range []fixedLocalPointerChoiceCase{ + {"exact address exceeds extent", "LEAQ choice<>(SB),CX\nMOVQ 64(CX),AX"}, + {"exact vector access crosses extent", "LEAQ choice<>(SB),CX\nMOVOU 56(CX),X0\nMOVQ X0,AX"}, + {"one candidate is readonly", fixedLocalChoicePrefix + "MOVQ AX,(CX)"}, + {"partial pointer read", fixedLocalChoicePrefix + "MOVL CX,DX\nMOVQ (DX),AX"}, + {"partial pointer spill overwrite", fixedLocalChoicePrefix + "MOVQ CX,8(SP)\nMOVB $1,9(SP)\nMOVQ 8(SP),DX\nMOVQ (DX),AX"}, + {"private relocation is separate", "LEAQ 8(SP),CX\nCMPQ AX,$0\nCMOVQEQ SI,CX\nMOVQ (CX),AX"}, + {"unknown value is not parameter zero", "CMPQ AX,$0\nCMOVQEQ SI,R12\nMOVQ (R12),AX"}, + {"stale equality flags", fixedLocalChoicePrefix + "CMPQ CX,R8\nCMPQ AX,$1\nJE done\nMOVQ AX,(CX)\ndone:"}, + {"compared register overwritten", fixedLocalChoicePrefix + "CMPQ CX,R8\nMOVQ R8,CX\nJE done\nMOVQ 64(CX),AX\ndone:"}, + {"ordered literal comparison", fixedLocalChoicePrefix + "CMPQ CX,R8\nJAE done\nMOVQ AX,(CX)\ndone:"}, + {"other literal cannot remove candidate", fixedLocalChoicePrefix + "LEAQ other<>(SB),R9\nCMPQ CX,R9\nJE done\nMOVQ AX,(CX)\ndone:"}, + {"other offset cannot remove candidate", fixedLocalChoicePrefix + "LEAQ choice<>+8(SB),R9\nCMPQ CX,R9\nJE done\nMOVQ AX,(CX)\ndone:"}, + {"equality does not grant readonly write", fixedLocalChoicePrefix + "CMPQ CX,R8\nJNE done\nMOVQ AX,(CX)\ndone:"}, + {"label does not prove identity", fixedLocalChoicePrefix + "native:\nJNE done\nMOVQ AX,(CX)\ndone:"}, + {"loop reintroduces literal", "LEAQ choice<>(SB),R8\nMOVQ SI,CX\nagain:\nCMPQ CX,R8\nJE done\nMOVQ R8,CX\nMOVQ 64(CX),AX\nJMP again\ndone:"}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, _ := fixedLocalPointerChoiceFixture(t, row.body) + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("conditional pointer invented an origin, access or predicate", err) + } + }) + } +} diff --git a/amd64_fixed_local_portability_test.go b/amd64_fixed_local_portability_test.go new file mode 100644 index 000000000..75f0c1cf9 --- /dev/null +++ b/amd64_fixed_local_portability_test.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "go/ast" + "go/build" + "go/parser" + "go/token" + "os" + "testing" +) + +func TestFixedLocalSharedHelpersBuildOnAllHosts(t *testing.T) { + for _, goos := range []string{"linux", "darwin", "windows"} { + t.Run(goos, func(t *testing.T) { + context := build.Default + context.GOOS, context.GOARCH = goos, "amd64" + found := map[string]int{} + for _, name := range []string{ + "amd64_fixed_local_runtime_test.go", + "amd64_fixed_local_packed_runtime_test.go", + "amd64_fixed_local_original_test.go", + } { + if _, err := os.Stat(name); os.IsNotExist(err) { + continue + } + included, err := context.MatchFile(".", name) + if err != nil { + t.Fatal(err) + } + if !included { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), name, nil, 0) + if err != nil { + t.Fatal(err) + } + for _, declaration := range file.Decls { + switch declaration := declaration.(type) { + case *ast.FuncDecl: + found[declaration.Name.Name]++ + case *ast.GenDecl: + for _, spec := range declaration.Specs { + if value, ok := spec.(*ast.ValueSpec); ok { + for _, name := range value.Names { + found[name.Name]++ + } + } + } + } + } + } + for _, name := range []string{"fixedLocalNativeOperations", "fixedLocalPackedRunOriginal"} { + if found[name] != 1 { + t.Errorf("portable tests require exactly one %s, found %d", name, found[name]) + } + } + }) + } +} diff --git a/amd64_fixed_local_private_leaf.go b/amd64_fixed_local_private_leaf.go new file mode 100644 index 000000000..3280a861e --- /dev/null +++ b/amd64_fixed_local_private_leaf.go @@ -0,0 +1,136 @@ +package plan9asm + +// Both consumers share the source graph, callsite expansion and per-root +// limits. Total work may span more roots; the authenticated source inventory +// and each proof's work bounds remain unchanged. Memory and result facts still +// come from the existing fixed-local transfer and replay. +func newX86FixedLocalLeafGraph(file *File, options Options, binding *x86FixedLocalBinding) (*x86DirectTailGraph, error) { + if len(binding.leaves) == 0 { + return nil, nil + } + helpers := make(map[string]bool, len(binding.leaves)) + for symbol := range binding.leaves { + helpers[symbol] = true + } + graph, byName, err := newX86DirectTailGraph(file, options, helpers, "amd64") + if err != nil { + return nil, err + } + graph.fixedLocal = binding + reached := make(map[string]bool) + for index, fn := range graph.original.Funcs { + if helpers[fn.Sym] || !x86DirectTailRootTargets(fn, helpers, nil) { + continue + } + if !binding.frames[graph.symbols[fn.Sym]] { + return nil, goAMD64FrameContext("private leaf caller lacks its independently observed nonzero root frame") + } + view, err := graph.root(index, byName, 4096, true) + if err != nil { + return nil, err + } + // Keep only the original root identity. Each consumer expands one root + // in its own function scope; the complete source is never cropped. + graph.roots[fn.Sym] = &x86DirectTailRoot{graph: graph, text: index} + for _, point := range view.points { + reached[graph.original.Funcs[point.text].Sym] = true + } + } + for symbol := range helpers { + if !reached[symbol] { + return nil, goAMD64FrameContext("observed private leaf has no complete framed caller") + } + } + return graph, nil +} + +// The shared constructor still checks the 4096-node limit before allocating. +// Its returned view is local to one proof/renderer/reference operation, never +// installed in the complete file or retained in the graph's root directory. +func (graph *x86DirectTailGraph) fixedLocalRoot(symbol string) (*x86DirectTailRoot, error) { + if graph == nil || graph.fixedLocal == nil { + return nil, nil + } + root := graph.roots[symbol] + if root == nil { + return nil, nil + } + if graph.revoked.Load() || root.graph != graph || root.text < 0 || root.text >= len(graph.original.Funcs) || + graph.original.Funcs[root.text].Sym != symbol { + return nil, goAMD64FrameContext("private leaf original root identity changed") + } + return graph.root(root.text, graph.texts, 4096, true) +} + +func proveX86FixedLocalRoot(fn Func, sig FuncSig, graph *x86DirectTailGraph) (Func, error) { + view, err := graph.fixedLocalRoot(fn.Sym) + if err != nil { + return fn, err + } + fn.x86DirectTailRoot = view + return proveX86FixedLocal(fn, sig) +} + +func prepareX86FixedLocalLeafSource(file *File, options Options) (*File, bool, error) { + for _, sig := range options.Sigs { + binding := sig.x86FixedLocal + if binding == nil || len(binding.leaves) == 0 { + continue + } + if graph := file.x86DirectTails; graph != nil { + if graph.fixedLocal != binding { + return nil, true, goAMD64FrameContext("private leaf graph belongs to another frame owner") + } + return file, true, graph.verify(file, options) + } + if _, err := verifyX86FixedLocalBindings(file, options); err != nil { + return nil, true, err + } + graph, err := newX86FixedLocalLeafGraph(file, options, binding) + if err != nil { + return nil, true, err + } + request := x86DirectTailSnapshot(file) + request.x86DirectTails = graph + return request, true, graph.verify(request, options) + } + return file, false, nil +} + +func fixedLocalProofFunction(fn Func) Func { + if view := fn.x86DirectTailRoot; view != nil { + fn.Instrs = view.instructions + } + return fn +} + +// Selection only: the live fixed-local binder and full replay still validate +// this caller. Its canonical result must not be reclassified as a custom call. +func fixedLocalPrivateLeafCaller(fn Func, sig FuncSig) bool { + binding := sig.x86FixedLocal + return binding != nil && len(binding.leaves) != 0 && binding.frames[sig.Name] && binding.symbols[fn.Sym] == sig.Name +} + +// Signature selection only. A transported real root binding identifies which +// source TEXTs must remain non-callable; the complete live binding is still +// verified before preparation/lowering and cannot be manufactured by a name. +func goFixedLocalLeafSymbols(file *File, resolve func(string) string, manual func(string) (FuncSig, bool)) map[string]bool { + if manual == nil { + return nil + } + var leaves map[string]bool + for _, fn := range file.Funcs { + name := resolve(goTextSymbolForResolution(fn.Sym)) + sig, supplied := goLookupManualSig(manual, name) + if !supplied || !fixedLocalPrivateLeafCaller(fn, sig) { + continue + } + if leaves == nil { + leaves = make(map[string]bool) + } + for symbol := range sig.x86FixedLocal.leaves { + leaves[symbol] = true + } + } + return leaves +} diff --git a/amd64_fixed_local_private_leaf_go127_test.go b/amd64_fixed_local_private_leaf_go127_test.go new file mode 100644 index 000000000..4c18805eb --- /dev/null +++ b/amd64_fixed_local_private_leaf_go127_test.go @@ -0,0 +1,285 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "go/types" + "strings" + "testing" +) + +func fixedLocalPrivateLeafCases() []struct{ name, body, helper string } { + return []struct{ name, body, helper string }{ + {"two_calls", "CALL leaf<>(SB)\n MOVQ $3,BX\n CALL leaf<>(SB)", + "ADDQ BX,(BP)\n MOVQ (BP),AX"}, + {"incoming_flags", "CMPQ AX,BX\n CALL leaf<>(SB)\n SETCS CL\n MOVBLZX CL,AX", + "MOVQ AX,DX\n MOVQ (BP),AX"}, + {"outgoing_flags", "CALL leaf<>(SB)\n SETCS CL\n MOVBLZX CL,AX", + "CMPQ AX,BX"}, + {"distinct_call_addresses", "CALL leaf<>(SB)\n MOVQ AX,DX\n LEAQ 24(SP),BP\n CALL leaf<>(SB)\n ADDQ DX,AX", + "ADDQ $1,(BP)\n MOVQ (BP),AX"}, + {"ordinary_address_register", "LEAQ 16(SP),SI\n CALL leaf<>(SB)", + "MOVQ (SI),AX\n ADDQ BX,AX\n MOVQ AX,(SI)"}, + } +} + +func fixedLocalPrivateLeafSource(body, helper string) string { + return `TEXT ·Value(SB),4,$80-16 + MOVQ a+0(FP),AX + MOVQ AX,16(SP) + MOVQ $7,BX + MOVQ BX,24(SP) + LEAQ 16(SP),BP + ` + body + ` + MOVQ AX,ret+8(FP) + RET +TEXT leaf<>(SB),4,$0 + ` + helper + ` + RET +` +} + +func fixedLocalPrivateLeafFixture(t *testing.T, assembly string, declarations ...string) (GoPackage, GoAMD64FrameCaptureRequest, Options) { + t.Helper() + declaration := "func Value(a uint64) uint64" + if len(declarations) == 1 { + declaration = declarations[0] + } else if len(declarations) != 0 { + t.Fatal("expected at most one actual Go declaration") + } + pkg, request := goAMD64FrameTypedFixture(t, declaration, assembly) + decl := pkg.Types.Scope().Lookup("Value").(*types.Func) + sig, err := DeriveGoFuncABI0Signature(pkg.Path+".Value", decl, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + return pkg, request, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: testResolveSym(pkg.Path), Sigs: map[string]FuncSig{sig.Name: sig}, + } +} + +func TestGoAMD64FixedLocalPrivateLeafCanonicalResult(t *testing.T) { + assembly := strings.NewReplacer("$80-16", "$80-9", "MOVQ AX,ret+8(FP)", "MOVB AX,ret+8(FP)").Replace( + fixedLocalPrivateLeafSource("CALL leaf<>(SB)\n SETCS AL", "CMPQ AX,BX")) + x86InsertExtractGoListing(t, "amd64", assembly) + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly, "func Value(a uint64) bool") + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil || strings.Count(ir, "define ") != 1 { + t.Fatal("actual framed canonical result did not retain its ordinary owner", err, ir) + } + fixedLocalPrivateLeafObjects(t, ir) + }) +} + +func TestGoAMD64FixedLocalPrivateLeafBareSymbol(t *testing.T) { + assembly := strings.ReplaceAll(fixedLocalPrivateLeafSource("CALL leaf<>(SB)", "MOVQ (BP),AX"), "leaf<>", "leaf") + x86InsertExtractGoListing(t, "amd64", assembly) + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil || strings.Count(ir, "define ") != 1 { + t.Fatal("same-source bare leaf lost its observed source role", err, ir) + } + fixedLocalPrivateLeafObjects(t, ir) + }) +} + +func TestGoAMD64FixedLocalPrivateLeafWholeVectorResults(t *testing.T) { + // Both qwords cross CALL, change in the leaf, and return separately. + // This exercises private vector transport without claiming that the + // separate GP-to-XMM MOVQ fixed-local owner gap has been implemented. + assembly := `TEXT ·Value(SB),4,$80-24 + MOVQ a+0(FP),AX + MOVQ AX,16(SP) + MOVQ $7,BX + MOVQ BX,24(SP) + MOVOU 16(SP),X0 + CALL leaf<>(SB) + MOVOU X0,32(SP) + MOVQ 32(SP),AX + MOVQ 40(SP),BX + MOVQ AX,first+8(FP) + MOVQ BX,second+16(FP) + RET +TEXT leaf<>(SB),4,$0 + PADDQ X0,X0 + RET +` + x86InsertExtractGoListing(t, "amd64", assembly) + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly, "func Value(a uint64) (first, second uint64)") + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil || strings.Count(ir, "define ") != 1 { + t.Fatal("complete vector did not return both independent typed results", err, ir) + } + fixedLocalPrivateLeafObjects(t, ir) + }) +} + +func TestGoAMD64FixedLocalPrivateLeafCompleteSources(t *testing.T) { + for _, row := range fixedLocalPrivateLeafCases() { + t.Run(row.name, func(t *testing.T) { + assembly := fixedLocalPrivateLeafSource(row.body, row.helper) + // Exact valid FP labels and all original helper TEXTs are assembled + // before testing the observer or the translator's private contract. + listing := x86InsertExtractGoListing(t, "amd64", assembly) + if !strings.Contains(string(listing), "CALL") || !strings.Contains(string(listing), "leaf") { + t.Fatal("original Go omitted the complete helper call") + } + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("nonzero root and original zero-frame leaf are not composed", err) + } + if strings.Count(ir, "define ") != 1 || !strings.Contains(ir, "fixed_local_native") || + strings.Contains(ir, "call void @\"leaf") || strings.Contains(ir, "define void @\"leaf") { + t.Fatal("leaf acquired an independent ABI or lost its root frame", ir) + } + fixedLocalPrivateLeafObjects(t, ir) + }) + }) + } +} + +func fixedLocalPrivateLeafObjects(t *testing.T, ir string) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + const target = "x86_64-unknown-linux-gnu" + compileLLVMToObject(t, llc, target, "private-leaf.ll", "private-leaf.o", ir) + compileLLVMToObject(t, llc, target, "private-leaf-opt.ll", "private-leaf-opt.o", x86MulDivRoundOptimize(t, ir)) +} + +func TestGoAMD64FixedLocalPrivateLeafBoundaries(t *testing.T) { + valid := fixedLocalPrivateLeafSource("CALL leaf<>(SB)", "MOVQ (BP),AX") + for _, row := range []struct{ name, source string }{ + {"undefined_input", strings.Replace(valid, "MOVQ (BP),AX", "MOVQ R8,AX", 1)}, + {"uninitialized_private", strings.Replace(valid, "MOVQ AX,16(SP)", "NOP", 1)}, + {"outside_private_extent", strings.Replace(valid, "MOVQ (BP),AX", "MOVQ 80(BP),AX", 1)}, + {"callee_SP", strings.Replace(valid, "MOVQ (BP),AX", "MOVQ 16(SP),AX", 1)}, + {"callee_FP", strings.Replace(valid, "MOVQ (BP),AX", "MOVQ a+0(FP),AX", 1)}, + {"callee_nonzero_frame", strings.Replace(valid, "TEXT leaf<>(SB),4,$0", "TEXT leaf<>(SB),4,$8", 1)}, + {"callee_negative_frame", strings.Replace(valid, "TEXT leaf<>(SB),4,$0", "TEXT leaf<>(SB),4,$-8", 1)}, + {"callee_nonleaf", strings.Replace(valid, "MOVQ (BP),AX", "CALL second<>(SB)", 1) + "TEXT second<>(SB),4,$0\n RET\n"}, + {"indirect_call", strings.Replace(valid, "CALL leaf<>(SB)", "CALL AX", 1)}, + {"address_escape", strings.Replace(valid, "CALL leaf<>(SB)", "LEAQ leaf<>(SB),AX\n CALL AX", 1)}, + {"dead_unclosed_helper", valid + "TEXT unused<>(SB),4,$8\n RET\n"}, + {"recursive", strings.Replace(valid, "MOVQ (BP),AX", "CALL leaf<>(SB)", 1)}, + {"missing_flags", strings.Replace(valid, "MOVQ (BP),AX", "SETCS CL\n MOVBLZX CL,AX", 1)}, + } { + t.Run(row.name, func(t *testing.T) { + x86InsertExtractGoListing(t, "amd64", row.source) + pkg, request, options := fixedLocalPrivateLeafFixture(t, row.source) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + switch row.name { + case "undefined_input", "uninitialized_private", "outside_private_extent", "missing_flags": + if err != nil { + t.Fatal("semantic negative requires successful complete original capture", err) + } + } + if err == nil { + defer owner.Close() + expanded, readErr := owner.preprocess(request.AssemblyFile, []byte(row.source)) + if readErr != nil { + t.Fatal(readErr) + } + file, parseErr := Parse(ArchAMD64, expanded) + if parseErr != nil { + t.Fatal(parseErr) + } + _, err = owner.BindFixedLocal(ctx, pkg, file, options) + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unclosed helper/frame/state became authorized", err) + } + }) + } +} + +func TestGoAMD64FixedLocalPrivateLeafRejectsSuppliedSignature(t *testing.T) { + for _, symbol := range []string{"leaf<>", "leaf"} { + t.Run(symbol, func(t *testing.T) { + assembly := strings.ReplaceAll(fixedLocalPrivateLeafSource("CALL leaf<>(SB)", "MOVQ (BP),AX"), "leaf<>", symbol) + x86InsertExtractGoListing(t, "amd64", assembly) + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("complete original source precondition", err) + } + defer owner.Close() + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + name := options.ResolveSym(symbol) + options.Sigs[name] = FuncSig{Name: name, Ret: Void} + if _, err := owner.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "private leaf cannot acquire a supplied callable signature") { + t.Fatal("public binding silently discarded a caller-supplied helper ABI", err) + } + }) + } +} + +func TestGoAMD64SDKFramePrivateLeafWholeOriginalSource(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "vendor/golang.org/x/crypto/chacha20poly1305") + original, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal("complete actual SDK precondition", err) + } + defer original.Close() + pkg := sdkAMD64FrameActualPackage(t, original.source) + owner, err := CaptureGoAMD64SDKFrameSource(ctx, pkg, GoAMD64FrameCaptureRequest{ + GoBinary: request.GoBinary, Directory: request.Directory, Target: request.Target, + AssemblyFile: "chacha20poly1305_amd64.s", Environment: request.Environment, + }) + if err != nil { + t.Fatal("complete original leaf and declared root observation", err) + } + defer owner.Close() + if len(owner.functions) != 3 || len(owner.declarations) != 2 { + t.Fatal("complete original TEXT inventory or actual declarations changed", + len(owner.functions), len(owner.declarations)) + } + for _, name := range []string{"chacha20Poly1305Open", "chacha20Poly1305Seal"} { + fn, found := owner.functions[pkg.Path+"."+name] + if !found || fn.signature.Name != pkg.Path+"."+name || fn.locals != 296 { + t.Fatal("root lost its actual Go signature or physical frame", name) + } + } + leaves := 0 + for name, fn := range owner.functions { + if strings.Contains(name, "polyHashADInternal") { + leaves++ + if fn.signature.Name != "" || len(fn.code) == 0 { + t.Fatal("leaf gained a Go signature or lost original bytes", name) + } + } + } + if leaves != 1 { + t.Fatal("complete original inventory lost or duplicated its leaf", leaves) + } + for _, cpp := range owner.sdkFrame.owner.source.observation.CPP { + file, err := Parse(ArchAMD64, cpp.Output) + if err != nil { + t.Fatal(err) + } + for _, fn := range file.Funcs { + t.Logf("complete original TEXT %s: %d parsed non-label instructions", fn.Sym, len(x86DirectTailInstructions(fn))) + } + } + if err := owner.Verify(ctx); err != nil { + t.Fatal(err) + } + t.Logf("all %d original TEXTs observed; this capture alone is not translation coverage", len(owner.functions)) +} diff --git a/amd64_fixed_local_private_leaf_identity_go127_test.go b/amd64_fixed_local_private_leaf_identity_go127_test.go new file mode 100644 index 000000000..d9c6048c4 --- /dev/null +++ b/amd64_fixed_local_private_leaf_identity_go127_test.go @@ -0,0 +1,109 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "encoding/json" + "errors" + "path/filepath" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestGoAMD64FixedLocalPrivateLeafOriginalCallIdentity(t *testing.T) { + assembly := fixedLocalPrivateLeafSource("CALL leaf<>(SB)\n CALL other<>(SB)", "MOVQ (BP),AX") + + "TEXT other<>(SB),4,$0\n ADDQ $2,AX\n RET\n" + x86InsertExtractGoListing(t, "amd64", assembly) + pkg, request, _ := fixedLocalPrivateLeafFixture(t, assembly) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + expanded, err := owner.preprocess(request.AssemblyFile, []byte(assembly)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + data, err := owner.fixedLocalDataObservation(request.AssemblyFile, expanded) + if err != nil { + t.Fatal(err) + } + listing, err := owner.read(filepath.Join(owner.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + check := func(data *gostaticstorage.AMD64AssemblyData, listing []byte, file *File) error { + functions, err := goAMD64FrameListingWithDataObject(listing, file, pkg, owner.origins, data, nil, + request.Target, request.Directory, "") + if err != nil { + return err + } + return goAMD64FrameDataTextJoin(functions, data, request.Directory, owner.origins, "") + } + if err := check(data, listing, file); err != nil { + t.Fatal("complete actual original source/object/listing control", err) + } + root := data.TextOrder[0].ObjectOrdinal + if root == 0 || len(data.Member.Texts[root].Relocations) != 2 { + t.Fatal("fixture did not exercise separate static/nonstatic banks and both CALLs") + } + encoded, err := json.Marshal(data) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{ + "valid_other_leaf", "wrong_bank", "wrong_relative", "unknown_ABI", "call_addend", "call_width", + "call_field_inside_opcode", "missing_CALL", "missing_source_association", "reordered_source_association", "missing_parsed_TEXT", + } { + t.Run(name, func(t *testing.T) { + var changed gostaticstorage.AMD64AssemblyData + if err := json.Unmarshal(encoded, &changed); err != nil { + t.Fatal(err) + } + input := bytes.Clone(listing) + parsed := *file + relocation := &changed.Member.Texts[root].Relocations[0] + switch name { + case "valid_other_leaf": + before := goAMD64FrameDataRelocationLine(*relocation, &changed) + relocation.Target = changed.Member.Texts[root].Relocations[1].Target + after := goAMD64FrameDataRelocationLine(*relocation, &changed) + if before == "" || after == "" || before == after { + t.Fatal("mutation did not retain another valid actual local object target") + } + input = bytes.Replace(input, []byte(before), []byte(after), 1) + case "wrong_bank": + relocation.Target.PackageIndex++ + case "wrong_relative": + relocation.Target.SymbolIndex++ + case "unknown_ABI": + relocation.Target.ABIKnown = false + case "call_addend": + relocation.Add++ + case "call_width": + relocation.Size++ + case "call_field_inside_opcode": + relocation.Offset-- + case "missing_CALL": + changed.Member.Texts[root].Relocations = changed.Member.Texts[root].Relocations[1:] + case "missing_source_association": + changed.TextOrder = changed.TextOrder[:len(changed.TextOrder)-1] + case "reordered_source_association": + changed.TextOrder[1], changed.TextOrder[2] = changed.TextOrder[2], changed.TextOrder[1] + case "missing_parsed_TEXT": + parsed.Funcs = parsed.Funcs[:len(parsed.Funcs)-1] + } + if err := check(&changed, input, &parsed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed complete source/native CALL identity was accepted", err) + } + }) + } +} diff --git a/amd64_fixed_local_private_leaf_lifetime_go127_test.go b/amd64_fixed_local_private_leaf_lifetime_go127_test.go new file mode 100644 index 000000000..5e0285636 --- /dev/null +++ b/amd64_fixed_local_private_leaf_lifetime_go127_test.go @@ -0,0 +1,411 @@ +//go:build go1.27 + +package plan9asm + +import ( + "crypto/sha256" + "errors" + "fmt" + "go/types" + "strings" + "testing" +) + +func fixedLocalLeafLifetimeFixture(t *testing.T, roots, adds, frame int) (GoPackage, GoAMD64FrameCaptureRequest, Options, string) { + t.Helper() + var assembly, declarations strings.Builder + names := make([]string, roots) + for index := range names { + name := "Value" + if index != 0 { + name = fmt.Sprintf("Other%d", index) + } + names[index] = name + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$%d-16\n", name, frame) + assembly.WriteString(" MOVQ a+0(FP),AX\n MOVQ AX,16(SP)\n LEAQ 16(SP),BP\n") + assembly.WriteString(strings.Repeat(" ADDQ $1,AX\n", adds)) + assembly.WriteString(" CALL leaf<>(SB)\n MOVQ AX,ret+8(FP)\n RET\n") + } + assembly.WriteString("TEXT leaf<>(SB),4,$0\n ADDQ AX,(BP)\n MOVQ (BP),AX\n RET\n") + source := assembly.String() + listing := x86InsertExtractGoListing(t, "amd64", source) + t.Logf("actual Go accepted complete source: sha256=%x listing_bytes=%d roots=%d", sha256.Sum256([]byte(source)), len(listing), roots) + pkg, request, options := fixedLocalPrivateLeafFixture(t, source, declarations.String()) + for _, name := range names { + sig, err := DeriveGoFuncABI0Signature(pkg.Path+"."+name, + pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + options.Sigs[sig.Name] = sig + } + return pkg, request, options, source +} + +func fixedLocalLeafLifetimeBinding(t *testing.T, pkg GoPackage, request GoAMD64FrameCaptureRequest, + options Options, assembly string) (*File, Options, error) { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + t.Cleanup(cancel) + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("complete original source observation must precede the graph bound", err) + } + t.Cleanup(func() { + if err := owner.Close(); err != nil { + t.Error(err) + } + }) + expanded, err := owner.preprocess(request.AssemblyFile, []byte(assembly)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + if len(file.Funcs) != len(options.Sigs)+1 || len(owner.functions) != len(file.Funcs) { + t.Fatal("whole original root and leaf inventory was not retained") + } + bound, err := owner.BindFixedLocal(ctx, pkg, file, options) + return file, bound, err +} + +func fixedLocalLeafLifetimeObjects(t *testing.T, expectedRoots int, ir string, err error) { + t.Helper() + if err != nil || strings.Count(ir, "define ") != expectedRoots { + t.Fatal("complete original roots did not all translate", err) + } + fixedLocalPrivateLeafObjects(t, ir) +} + +func TestGoAMD64FixedLocalPrivateLeafPreservesPerRootWorkBudget(t *testing.T) { + pkg, request, options, source := fixedLocalLeafLifetimeFixture(t, 2, 1500, 2048) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { fixedLocalLeafLifetimeObjects(t, 2, ir, err) }) + t.Run("actual_replay_work", func(t *testing.T) { + file, bound, err := fixedLocalLeafLifetimeBinding(t, pkg, request, options, source) + if err != nil { + t.Fatal("old complete source must remain accepted", err) + } + prepared, err := PrepareX86DirectTailSource(file, bound) + if err != nil { + t.Fatal(err) + } + graph := prepared.x86DirectTails + byName := make(map[string]int) + for index, fn := range graph.original.Funcs { + byName[fn.Sym] = index + } + totalNodes, totalBytes := 0, int64(0) + for index, original := range graph.original.Funcs { + if graph.helpers[original.Sym] { + continue + } + // Use the real graph builder and transfer budget. This is a work + // measurement, not a substitute certificate for the public API above. + view, err := graph.root(index, byName, 4096, true) + if err != nil { + t.Fatal(err) + } + fn := original + fn.x86DirectTailRoot = view + fn = fixedLocalProofFunction(fn) + sig := bound.Sigs[bound.ResolveSym(original.Sym)] + proof := &x86FixedLocalProof{binding: sig.x86FixedLocal, frame: fn.FrameSize} + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + for phase := 0; phase < 2; phase++ { + current := fn + current.Instrs = append([]Instr(nil), fn.Instrs...) + effects, err := fixedLocalEffectPlan(current, sig.x86FixedLocal) + if err != nil { + t.Fatal(err) + } + for residue := 0; residue < 8; residue++ { + if err := replayX86FixedLocal(¤t, sig, proof, residue, &budget, effects, nil); err != nil { + t.Fatal("unchanged per-proof work bound rejected the old source", err) + } + } + } + steps, bytes := (1<<18)-budget.steps, (64<<20)-budget.bytes + if steps != 16*len(view.points) || bytes != int64(steps)*fn.FrameSize || budget.bytes <= 0 { + t.Fatal("actual straight-line replay consumption differs", original.Sym, steps, bytes) + } + t.Logf("actual %s expansion=%d replay_steps=%d byte_work=%d", original.Sym, len(view.points), steps, bytes) + totalNodes += len(view.points) + totalBytes += bytes + } + if totalNodes > 4096 || totalBytes <= 64<<20 { + t.Fatal("old accepted source no longer distinguishes per-proof from shared work", totalNodes, totalBytes) + } + }) +} + +func TestGoAMD64FixedLocalPrivateLeafSequentialRootCapacity(t *testing.T) { + pkg, request, options, _ := fixedLocalLeafLifetimeFixture(t, 2, 2200, 80) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { fixedLocalLeafLifetimeObjects(t, 2, ir, err) }) +} + +func TestGoAMD64FixedLocalPrivateLeafSingleRootCapacity(t *testing.T) { + for _, adds := range []int{4087, 4088} { + t.Run(fmt.Sprint(adds+9), func(t *testing.T) { + pkg, request, options, source := fixedLocalLeafLifetimeFixture(t, 1, adds, 80) + file, bound, err := fixedLocalLeafLifetimeBinding(t, pkg, request, options, source) + if adds == 4088 { + if !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "oversized source continuation instruction budget") { + t.Fatal("one oversized original root escaped its unchanged bound", err) + } + return + } + if err != nil { + t.Fatal("root at the actual existing node bound was rejected", err) + } + ir, err := Translate(file, bound) + fixedLocalLeafLifetimeObjects(t, 1, ir, err) + }) + } +} + +// Count the real retained expansion arrays, not original source text, total +// allocation or a GC-dependent heap estimate. Shared backing arrays are counted +// once; independently owned instruction/certificate arrays remain separate. +type fixedLocalLeafRetained struct{ views, points, instructions, ordinals, edges, tails, accesses int } + +func fixedLocalLeafRetainedArrays(file *File) fixedLocalLeafRetained { + var retained fixedLocalLeafRetained + seen := make(map[*x86DirectTailRoot]bool) + points := make(map[*x86DirectTailPoint]bool) + instructions := make(map[*Instr]bool) + ordinals := make(map[*int]bool) + edges := make(map[*[]int]bool) + targets := make(map[*int]bool) + tails := make(map[*bool]bool) + add := func(view *x86DirectTailRoot) { + if view == nil || seen[view] { + return + } + seen[view] = true + if len(view.points) != 0 || len(view.instructions) != 0 { + retained.views++ + } + if cap(view.points) != 0 && !points[&view.points[:cap(view.points)][0]] { + points[&view.points[:cap(view.points)][0]] = true + retained.points += cap(view.points) + } + if cap(view.instructions) != 0 && !instructions[&view.instructions[:cap(view.instructions)][0]] { + instructions[&view.instructions[:cap(view.instructions)][0]] = true + retained.instructions += cap(view.instructions) + for _, ins := range view.instructions { + retained.accesses += len(ins.x86FixedLocalMemory) + } + } + if cap(view.ordinals) != 0 && !ordinals[&view.ordinals[:cap(view.ordinals)][0]] { + ordinals[&view.ordinals[:cap(view.ordinals)][0]] = true + retained.ordinals += cap(view.ordinals) + } + if cap(view.edges) != 0 && !edges[&view.edges[:cap(view.edges)][0]] { + edges[&view.edges[:cap(view.edges)][0]] = true + retained.edges += cap(view.edges) + } + for _, edge := range view.edges { + if cap(edge) != 0 && !targets[&edge[:cap(edge)][0]] { + targets[&edge[:cap(edge)][0]] = true + retained.edges += cap(edge) + } + } + if cap(view.externalTails) != 0 && !tails[&view.externalTails[:cap(view.externalTails)][0]] { + tails[&view.externalTails[:cap(view.externalTails)][0]] = true + retained.tails += cap(view.externalTails) + } + } + if file.x86DirectTails != nil { + for _, view := range file.x86DirectTails.roots { + add(view) + } + } + for _, fn := range file.Funcs { + add(fn.x86DirectTailRoot) + } + return retained +} + +func TestGoAMD64FixedLocalPrivateLeafRetainedRootViews(t *testing.T) { + var scopeBaseline fixedLocalLeafRetained + for _, count := range []int{2, 8} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + pkg, request, options, source := fixedLocalLeafLifetimeFixture(t, count, 8, 80) + file, bound, err := fixedLocalLeafLifetimeBinding(t, pkg, request, options, source) + if err != nil { + t.Fatal(err) + } + prepared, err := PrepareX86DirectTailSource(file, bound) + if err != nil { + t.Fatal(err) + } + measure := func(stage string, current *File) { + retained := fixedLocalLeafRetainedArrays(current) + t.Logf("roots=%d stage=%s actual retained expansion arrays=%+v", count, stage, retained) + if retained != (fixedLocalLeafRetained{}) { + t.Errorf("%s retained root expansions outside a single-function scope: %+v", stage, retained) + } + } + measure("prepared", prepared) + normalized, err := normalizeX86RawFile(prepared, "amd64", bound) + if err != nil { + t.Fatal(err) + } + measure("normalized", normalized) + binding := bound.Sigs[pkg.Path+".Value"].x86FixedLocal + snapshot, err := newX86FixedLocalResultSnapshot(binding, "amd64") + if err != nil { + t.Fatal(err) + } + for _, original := range file.Funcs { + if prepared.x86DirectTailHelper(original.Sym) { + continue + } + retained := snapshot.functions[bound.ResolveSym(original.Sym)].function + if len(retained.Instrs) != len(original.Instrs) || retained.x86DirectTailRoot != nil { + t.Errorf("result snapshot retained expanded instructions for %s: got %d original %d", + original.Sym, len(retained.Instrs), len(original.Instrs)) + } + sig := bound.Sigs[bound.ResolveSym(original.Sym)] + current, err := proveX86FixedLocalRoot(original, sig, prepared.x86DirectTails) + if err != nil { + t.Fatal(err) + } + entry := snapshot.functions[sig.Name] + reference, err := entry.graph.fixedLocalRoot(original.Sym) + if err != nil || reference == nil || len(current.x86DirectTailRoot.points) > 4096 || len(reference.points) > 4096 { + t.Fatal("current root/reference exceeded the unchanged node bound", err) + } + // Measure the actual proof and independent reference owners of a + // single consumer scope. This temporary container is never sent to + // translation and does not crop the authenticated input source. + referenceFunc := entry.function + referenceFunc.x86DirectTailRoot = reference + scope := *prepared + scope.Funcs = []Func{current, referenceFunc} + live := fixedLocalLeafRetainedArrays(&scope) + t.Logf("roots=%d current=%s proof+reference retained arrays=%+v", count, original.Sym, live) + if scopeBaseline.views == 0 { + scopeBaseline = live + } + if live.views != 2 || live != scopeBaseline { + t.Fatal("single-root retained work grew with complete-file root count", live, scopeBaseline) + } + } + measure("after_root_scopes", prepared) + }) + } +} + +func TestGoAMD64FixedLocalPrivateLeafScopedControlIdentity(t *testing.T) { + pkg, request, options, source := fixedLocalLeafLifetimeFixture(t, 1, 2, 80) + file, bound, err := fixedLocalLeafLifetimeBinding(t, pkg, request, options, source) + if err != nil { + t.Fatal(err) + } + prepared, err := PrepareX86DirectTailSource(file, bound) + if err != nil { + t.Fatal(err) + } + sig := bound.Sigs[pkg.Path+".Value"] + fn, err := proveX86FixedLocalRoot(prepared.Funcs[0], sig, prepared.x86DirectTails) + if err != nil { + t.Fatal(err) + } + snapshot, err := newX86FixedLocalResultSnapshot(sig.x86FixedLocal, "amd64") + if err != nil { + t.Fatal(err) + } + entry := snapshot.functions[sig.Name] + check := func(function Func) error { + var output strings.Builder + consumer := newX86Ctx(&output, function, sig, bound.ResolveSym, bound.Sigs, "amd64", bound.TargetTriple, false) + consumer.fixedLocalResult = snapshot + consumer.fpFeatureAttrs, consumer.sig.Attrs = entry.inferred, entry.attributes + return consumer.checkFixedLocalResultInitialization() + } + if err := check(fn); err != nil { + t.Fatal("independent complete source/reference precondition", err) + } + for _, axis := range []string{"text", "point", "ordinal", "edge", "return", "external_tail"} { + t.Run(axis, func(t *testing.T) { + changed := fn + view := *fn.x86DirectTailRoot + changed.x86DirectTailRoot = &view + switch axis { + case "text": + view.text++ + case "point": + view.points = append([]x86DirectTailPoint(nil), view.points...) + view.points[0].instruction++ + case "ordinal": + view.ordinals = append([]int(nil), view.ordinals...) + view.ordinals[0]++ + case "edge", "return": + view.edges = append([][]int(nil), view.edges...) + pc := 0 + if axis == "return" { + pc = len(view.instructions) - 1 // The private helper's RET. + if view.instructions[pc].Op != OpRET || len(view.edges[pc]) != 1 { + t.Fatal("fixture lost its distinct private return continuation") + } + } + view.edges[pc] = []int{0} + case "external_tail": + view.externalTails = append([]bool(nil), view.externalTails...) + view.externalTails[0] = true + } + if err := check(changed); !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "original private root control/source coordinates differ") { + t.Fatal("unchanged instruction bytes hid a changed private control graph", err) + } + }) + } +} + +func TestGoAMD64FixedLocalPrivateLeafRootDirectoryIdentity(t *testing.T) { + pkg, request, options, source := fixedLocalLeafLifetimeFixture(t, 1, 2, 80) + file, bound, err := fixedLocalLeafLifetimeBinding(t, pkg, request, options, source) + if err != nil { + t.Fatal(err) + } + prepared, err := PrepareX86DirectTailSource(file, bound) + if err != nil { + t.Fatal(err) + } + graph := prepared.x86DirectTails + if view, err := graph.fixedLocalRoot("·Value"); err != nil || view == nil { + t.Fatal("complete original directory precondition", err) + } + original := graph.texts + for _, axis := range []string{"missing", "negative", "outside", "different_symbol"} { + t.Run(axis, func(t *testing.T) { + changed := make(map[string]int, len(original)) + for name, index := range original { + changed[name] = index + } + switch axis { + case "missing": + delete(changed, "leaf<>") + case "negative": + changed["leaf<>"] = -1 + case "outside": + changed["leaf<>"] = len(graph.original.Funcs) + case "different_symbol": + changed["leaf<>"] = original["·Value"] + } + graph.texts = changed + t.Cleanup(func() { graph.texts = original }) + if _, err := graph.fixedLocalRoot("·Value"); !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "tail target is not an original source TEXT") { + t.Fatal("retained source directory selected a different or invalid helper index", err) + } + }) + } +} diff --git a/amd64_fixed_local_private_leaf_selection_go127_test.go b/amd64_fixed_local_private_leaf_selection_go127_test.go new file mode 100644 index 000000000..aea8f1502 --- /dev/null +++ b/amd64_fixed_local_private_leaf_selection_go127_test.go @@ -0,0 +1,117 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestGoAMD64FixedLocalPrivateLeafSelectionBoundary(t *testing.T) { + for _, sibling := range []struct { + name, value string + valid bool + }{ + {"defined_sibling", "$17", true}, + {"undefined_sibling", "R8", false}, + } { + t.Run(sibling.name, func(t *testing.T) { + assembly := fixedLocalPrivateLeafSource("CALL leaf<>(SB)", "MOVQ (BP),AX") + + "TEXT ·Other(SB),4,$0-8\n MOVQ " + sibling.value + ",AX\n MOVQ AX,ret+0(FP)\n RET\n" + x86InsertExtractGoListing(t, "amd64", assembly) + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly, + "func Value(a uint64) uint64\nfunc Other() uint64") + other, err := DeriveGoFuncABI0Signature(pkg.Path+".Other", + pkg.Types.Scope().Lookup("Other").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + options.Sigs[other.Name] = other + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("complete original sibling observation must succeed", err) + } + defer owner.Close() + expanded, err := owner.preprocess(request.AssemblyFile, []byte(assembly)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + if len(file.Funcs) != 3 || len(owner.functions) != 3 { + t.Fatal("complete original root, leaf and sibling inventory changed") + } + bound, err := owner.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("sibling semantics must reach the public translation consumer", err) + } + for _, selection := range []string{"complete", "keep_all", "omit_sibling"} { + t.Run(selection, func(t *testing.T) { + queries := make(map[string]int) + opt := GoModuleOptions{ + GOOS: "linux", GOARCH: "amd64", TargetTriple: bound.TargetTriple, ResolveSym: bound.ResolveSym, + ManualSig: func(name string) (FuncSig, bool) { + sig, found := bound.Sigs[name] + return sig, found + }, + } + if selection != "complete" { + opt.KeepFunc = func(symbol, name string) bool { + queries[symbol]++ + return selection == "keep_all" || name != other.Name + } + } + context := llvm.NewContext() + defer context.Dispose() + result, err := translateGoModuleInContext(context, pkg, []byte(expanded), opt) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if selection != "complete" { + if len(queries) != len(file.Funcs) { + t.Fatal("KeepFunc did not observe the complete original inventory", queries) + } + for _, fn := range file.Funcs { + if queries[fn.Sym] != 1 { + t.Fatal("KeepFunc omitted or repeated an original TEXT", fn.Sym, queries) + } + } + } + if selection == "omit_sibling" { + // This route currently rejects even valid partial selection. + // It does not claim semantic auditing before filtering. + if result != nil || !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "fixed local consumer original/normalized source changed") { + t.Fatal("cropping a sibling reused a complete-source frame binding", err) + } + return + } + if !sibling.valid { + if result != nil || !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "result initialization") || + !strings.Contains(err.Error(), "MOVQ R8,AX") { + t.Fatal("complete source hid an undefined ordinary sibling result", err) + } + return + } + if err != nil || result == nil || result.Module.IsNil() { + t.Fatal("complete valid root, leaf and sibling source was rejected", err) + } + ir := result.Module.String() + if strings.Count(ir, "define ") != 2 || !strings.Contains(ir, other.Name) { + t.Fatal("complete valid source omitted its ordinary sibling", ir) + } + fixedLocalPrivateLeafObjects(t, ir) + }) + } + }) + } +} diff --git a/amd64_fixed_local_proof.go b/amd64_fixed_local_proof.go new file mode 100644 index 000000000..d140a67f5 --- /dev/null +++ b/amd64_fixed_local_proof.go @@ -0,0 +1,1288 @@ +package plan9asm + +import ( + "errors" + "fmt" + "math" + "reflect" +) + +var errFixedLocalLayoutJoin = goAMD64FrameContext("fixed local CFG address identity disagrees") + +// Local offsets are from the observed post-prologue Go SP, not incoming FP, +// saved BP/return slots, a caller allocation or an arbitrary native address. +type x86FixedLocalProof struct { + binding *x86FixedLocalBinding + frame int64 + function [32]byte + accesses map[int]map[int]x86FixedLocalAccess + ordinal int // private builder cursor; never trusted by a consumer +} + +type x86FixedLocalAccess struct { + owner *x86FixedLocalProof + shape [32]byte + width int64 + write bool + nativeInput bool // Typed source address, not a private extent/alignment proof. + offset [8]int64 // The first access; a counted loop retains its complete series below. + ordinal int + // Counted iterations and private layouts retain a complete stable series. + iterationOffsets [][8]int64 +} + +type x86FixedLocalState struct { + pointers map[Reg]int64 + opaque map[Reg]x86FixedLocalOrigins + opaqueCells map[int64]x86FixedLocalOrigins + booleans map[Reg]uint8 + booleanBytes x86FixedLocalByteSet + constants map[Reg]int64 + ranges map[Reg]x86UnsignedRange + comparison x86FixedLocalComparison + known map[Reg]uint8 + vectors map[Reg]uint64 + masks map[Reg]bool + bytes x86FixedLocalByteSet + addresses map[int64]int64 + results map[int]bool + flags uint8 + savedBP *x86FixedLocalSavedBP // Immutable facts for the active observed leaf call. +} + +func cloneFixedLocalState(input x86FixedLocalState) x86FixedLocalState { + clone := input + clone.pointers = make(map[Reg]int64, len(input.pointers)) + clone.opaque = cloneFixedLocalOrigins(input.opaque) + clone.opaqueCells = cloneFixedLocalOrigins(input.opaqueCells) + clone.booleans = x86CloneCallerKnown(input.booleans) + clone.booleanBytes = append(x86FixedLocalByteSet(nil), input.booleanBytes...) + clone.constants = make(map[Reg]int64, len(input.constants)) + clone.ranges = cloneFixedLocalRanges(input.ranges) + clone.addresses = make(map[int64]int64, len(input.addresses)) + clone.results = make(map[int]bool, len(input.results)) + for key, value := range input.pointers { + clone.pointers[key] = value + } + for key, value := range input.constants { + clone.constants[key] = value + } + for key, value := range input.addresses { + clone.addresses[key] = value + } + for key, value := range input.results { + clone.results[key] = value + } + clone.known = x86CloneCallerKnown(input.known) + clone.vectors, clone.masks = x86CloneCallerVectorState(input.vectors, input.masks) + clone.bytes = append(x86FixedLocalByteSet(nil), input.bytes...) + return clone +} + +func proveX86FixedLocal(fn Func, sig FuncSig) (Func, error) { + if view := fn.x86DirectTailRoot; view != nil { + if view.graph == nil || view.graph.revoked.Load() || view.graph.fixedLocal != sig.x86FixedLocal { + return fn, goAMD64FrameContext("private leaf CFG has a foreign root/frame owner") + } + root := view.graph.roots[fn.Sym] + if root == nil || root.graph != view.graph || root.text != view.text { + return fn, goAMD64FrameContext("private leaf CFG has a foreign root/frame owner") + } + proven, err := proveX86FixedLocalBody(fixedLocalProofFunction(fn), sig) + if err != nil { + return fn, err + } + copy := *view + copy.instructions = proven.Instrs + fn.x86DirectTailRoot, fn.x86FixedLocal = ©, proven.x86FixedLocal + return fn, nil + } + return proveX86FixedLocalBody(fn, sig) +} + +func proveX86FixedLocalBody(fn Func, sig FuncSig) (Func, error) { + voidReturn := sig.Ret == Void && len(sig.Frame.Results) == 0 + valueReturn := sig.Ret != "" && sig.Ret != Void && len(sig.Frame.Results) > 0 && len(sig.Frame.Results) <= 4096 + if fn.FrameSize <= 0 || fn.FrameSize > 64<<10 || fn.FrameSize%8 != 0 || + len(fn.Instrs) == 0 || len(fn.Instrs) > 8192 || + len(fn.X86RawText) != 0 || len(sig.ArgRegs) != 0 || sig.ARMEntry != nil || + sig.ARM64GoRegisterABI != nil || sig.x86CallerRegions != nil || !voidReturn && !valueReturn { + return fn, goAMD64FrameContext("fixed local source has an unbound frame or entry contract") + } + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + proven, err := proveX86FixedLocalAttempt(fn, sig, &budget, false) + if errors.Is(err, errFixedLocalLayoutJoin) { + return proveX86FixedLocalAttempt(fn, sig, &budget, true) + } + return proven, err +} + +// Only a private-layout join retries the complete residue/final replay, once, +// with the remaining budget. Ordinary sources retain their original schedule. +func proveX86FixedLocalAttempt(fn Func, sig FuncSig, budget *x86FixedLocalBudget, partitioned bool) (Func, error) { + copyFn := fn + copyFn.Instrs = append([]Instr(nil), fn.Instrs...) + proof := &x86FixedLocalProof{binding: sig.x86FixedLocal, frame: fn.FrameSize, accesses: make(map[int]map[int]x86FixedLocalAccess)} + for index := range copyFn.Instrs { + copyFn.Instrs[index].Args = append([]Operand(nil), fn.Instrs[index].Args...) + copyFn.Instrs[index].x86FixedLocalMemory = nil + } + effects, err := fixedLocalEffectPlan(copyFn, proof.binding) + if err != nil { + return fn, err + } + var layouts *x86FixedLocalLayoutGraph + if partitioned { + layouts = &x86FixedLocalLayoutGraph{} + } + for residue := 0; residue < 8; residue++ { + if err := replayX86FixedLocal(©Fn, sig, proof, residue, budget, effects, layouts); err != nil { + return fn, err + } + } + // Independent final replay must recreate exactly the same access set and + // offset in every possible word-aligned original Go SP congruence class. + final := copyFn + final.Instrs = append([]Instr(nil), copyFn.Instrs...) + for index := range final.Instrs { + final.Instrs[index].x86FixedLocalMemory = nil + } + effects, err = fixedLocalEffectPlan(final, proof.binding) + if err != nil { + return fn, err + } + var firstLayouts []x86FixedLocalLayoutNode + if layouts != nil { + firstLayouts = layouts.nodes + *layouts = x86FixedLocalLayoutGraph{retained: int64(cap(firstLayouts))*32 + layouts.series} + } + for residue := 0; residue < 8; residue++ { + if err := replayX86FixedLocal(&final, sig, proof, residue, budget, effects, layouts); err != nil { + return fn, err + } + } + if layouts != nil && !reflect.DeepEqual(firstLayouts, layouts.nodes) { + return fn, goAMD64FrameContext("fixed local final layout replay changed") + } + for index := range final.Instrs { + if !reflect.DeepEqual(final.Instrs[index].x86FixedLocalMemory, copyFn.Instrs[index].x86FixedLocalMemory) { + return fn, goAMD64FrameContext("fixed local final access replay changed") + } + if len(final.Instrs[index].x86FixedLocalMemory) != 0 { + proof.accesses[index] = cloneX86ReplayFixedLocalMemory(final.Instrs[index].x86FixedLocalMemory) + } + } + if err := fixedLocalRelocationAliases(proof.accesses); err != nil { + return fn, err + } + proof.function, err = x86CallerSourceFingerprint(final) + if err != nil { + return fn, goAMD64FrameContext("fixed local normalized function cannot be bound: %v", err) + } + final.x86FixedLocal = proof + return final, nil +} + +// This call-local plan holds forms and immutable DATA byte dependencies, never +// a live source verification result. Each independent eight-residue replay +// constructs its own plan under the original binding's complete lifetime guards. +func fixedLocalEffectPlan(fn Func, binding *x86FixedLocalBinding) (map[int]x86TypedByteEffect, error) { + var effects map[int]x86TypedByteEffect + for index, ins := range fn.Instrs { + if ins.x86Encoded || ins.x86SourceRaw != nil || ins.x86AddressBits != 0 || ins.x86SegmentPrefix != 0 { + return nil, goAMD64FrameContext("fixed local source contains raw/native effects") + } + if fn.x86DirectTailRoot.privateCallTransition(index) { + continue // Complete private source graph owns only this control edge. + } + if _, ordinary := fixedLocalOrdinaryCall(ins, binding); ordinary { + continue // Exact outgoing layout and state are checked by the transfer. + } + if effect, known, err := fixedLocalVectorByteEffect(ins); known { + if err != nil { + return nil, err + } + if effects == nil { + effects = make(map[int]x86TypedByteEffect) + } + effects[index] = fixedLocalLogicalLiteralEffect(ins, effect, binding) + } else { + shape := ins + if x86LEAWidth(ins.Op, "amd64") != 0 && len(ins.Args) == 2 && ins.Args[0].Kind == OpMem { + address, err := fixedLocalMemoryAddress(ins, 0, fn.FrameSize) + if err != nil { + return nil, err + } + shape.Args = append([]Operand(nil), ins.Args...) + shape.Args[0].Mem = address + } + if !fixedLocalInstructionShape(shape) { + return nil, goAMD64FrameContext("fixed local complete source has an unregistered instruction shape: %s", ins.Op) + } + } + } + return effects, nil +} + +// Every reachable access, including initialization stores and private address +// spills, contributes all its bytes. Each original residue must induce a +// bijection on those bytes: equal bytes remain equal and distinct bytes stay +// distinct. Different distances between permanently disjoint accesses are +// unobservable because the source cannot expose numeric addresses or flags. +func fixedLocalRelocationAliases(accesses map[int]map[int]x86FixedLocalAccess) error { + var forward, reverse [8]map[int64]int64 + for residue := range forward { + forward[residue] = make(map[int64]int64) + reverse[residue] = make(map[int64]int64) + } + var total int64 + for _, operands := range accesses { + for _, access := range operands { + offsets := access.iterationOffsets + if len(offsets) == 0 { + offsets = [][8]int64{access.offset} + } + if access.width <= 0 || access.width > ((1<<20)-total)/int64(len(offsets)) { + return goAMD64FrameContext("fixed local complete access-byte budget exceeded") + } + total += access.width * int64(len(offsets)) + if access.nativeInput { + continue // Native input addresses are never relocated. + } + for _, offset := range offsets { + for byteIndex := int64(0); byteIndex < access.width; byteIndex++ { + original := offset[0] + byteIndex + for residue := range forward { + current := offset[residue] + byteIndex + if mapped, found := forward[residue][original]; found && mapped != current { + return goAMD64FrameContext("fixed local overlapping-byte identity depends on original SP residue") + } + if mapped, found := reverse[residue][current]; found && mapped != original { + return goAMD64FrameContext("fixed local distinct-byte identity depends on original SP residue") + } + forward[residue][original] = current + reverse[residue][current] = original + } + } + } + } + } + return nil +} + +func fixedLocalInstructionShape(ins Instr) bool { + if form, known, err := x86GPSIMDTransferProperties(ins, "amd64"); known { + return err == nil && !form.mmx && !form.spec.rawOnly + } + if _, _, known, err := x86SwapProperties(ins, "amd64"); known { + return err == nil + } + if form, known, err := fixedLocalConversionProperties(ins); known { + return err == nil && !form.mmx + } + if _, known, err := fixedLocalScalarLaneProperties(ins); known { + return err == nil + } + if _, known, err := x86ConditionalSetProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if ins.x86NumericalRead != nil { + _, valid := x86NumericalReadEffect(ins, "amd64") + return valid + } + if _, known, err := amd64PackedVectorMoveProperties(ins, "amd64"); known { + return err == nil + } + if form, known, err := x86NonTemporalVectorProperties(ins.Op, ins, "amd64"); known { + return err == nil && !form.trap + } + if _, known, err := x86OrderingHintProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if _, known, err := x86VectorZeroProperties(ins, "amd64"); known { + return err == nil + } + if _, known, err := x86LegacyPackedMoveProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if _, known := amd64MaskMoveSpecs[ins.Op]; known { + return amd64MaskMoveOperandsValid(ins, "amd64") + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && amd64VectorByteWidth(operand.Reg) != 0 { + index, valid := amd64VectorRegisterIndex(operand.Reg, 16) + if !valid || index > 15 { + return false + } + } + } + if ins.Op == OpTEXT || ins.Op == OpLABEL { + return true // Parser/complete CFG separately binds TEXT and label operands. + } + if ins.Op == OpRET { + return len(ins.Args) == 0 + } + if isAMD64ConditionalBranch(ins.Op) || ins.Op == OpJMP { + _, named := fixedLocalBranchLabel(ins) + return named + } + if x86RawReturnScalarMoveBytes(x86SourceMoveInstruction(ins, "amd64").Op) != 0 { + return len(ins.Args) == 2 + } + if _, address := fixedLocalStaticAddress(ins); address { + return true + } + if effect, known, err := fixedLocalGPByteEffect(ins); known { + return err == nil && !effect.dynamicMemory + } + if _, known, err := x86SHAProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if form, known, err := x86AESProperties(ins.Op, ins, "amd64"); known { + return err == nil && form.byteWidth == 16 + } + if _, known, err := fixedLocalVectorByteEffect(ins); known { + return err == nil + } + if form, known, err := x86FloatCompareProperties(ins, "amd64"); known { + return err == nil && form.legacy && !form.scalar && form.byteWidth == 16 + } + return false +} + +func fixedLocalBranchLabel(ins Instr) (string, bool) { + if len(ins.Args) != 1 { + return "", false + } + operand := ins.Args[0] + if operand.Kind == OpLabel && operand.Sym != "" { + return operand.Sym, true + } + if operand.Kind == OpIdent && operand.Ident != "" { + return operand.Ident, true + } + return "", false +} + +func fixedLocalEdges(fn Func) (map[int][]int, []int, error) { + if view := fn.x86DirectTailRoot; view != nil { + if len(fn.Instrs) != len(view.points) || len(view.edges) != len(view.points) { + return nil, nil, goAMD64FrameContext("private leaf CFG instruction inventory differs") + } + edges, instructions := make(map[int][]int, len(view.edges)), make([]int, len(view.points)) + for pc, targets := range view.edges { + instructions[pc], edges[pc] = pc, targets + } + return edges, instructions, nil + } + labels := make(map[string]int) + var instructions []int + for index, ins := range fn.Instrs { + if ins.Op == OpLABEL { + if len(ins.Args) != 1 || ins.Args[0].Kind != OpLabel { + return nil, nil, goAMD64FrameContext("malformed local label") + } + name := ins.Args[0].Sym + if _, exists := labels[name]; exists { + return nil, nil, goAMD64FrameContext("duplicate local label") + } + labels[name] = len(instructions) + } else { + instructions = append(instructions, index) + } + } + if len(instructions) == 0 || len(instructions) > 8192 { + return nil, nil, goAMD64FrameContext("fixed local CFG lacks a bounded instruction inventory") + } + edges := make(map[int][]int) + for pc, index := range instructions { + ins := fn.Instrs[index] + if ins.Op == OpRET { + continue + } + next := []int{pc + 1} + if ins.Op == OpJMP || isAMD64ConditionalBranch(ins.Op) { + label, named := fixedLocalBranchLabel(ins) + if !named { + return nil, nil, goAMD64FrameContext("fixed local branch needs a complete named target") + } + target, found := labels[label] + if !found || target >= len(instructions) { + return nil, nil, goAMD64FrameContext("fixed local branch target is missing an instruction") + } + if ins.Op == OpJMP { + next = []int{target} + } else { + next = append(next, target) + } + } + edges[pc] = next + } + return edges, instructions, nil +} + +// All congruence classes and the independent final replay share these limits. +// A cyclic source cannot obtain an unbounded analysis or a partial certificate. +type x86FixedLocalBudget struct { + steps int + bytes int64 +} + +func (budget *x86FixedLocalBudget) consume(frame int64) error { + if budget.steps <= 0 || frame > budget.bytes { + return goAMD64FrameContext("fixed local CFG work/byte bound exceeded") + } + budget.steps-- + budget.bytes -= frame + return nil +} + +func (budget *x86FixedLocalBudget) consumeOrigins(bytes int64) error { + if bytes > budget.bytes { + return goAMD64FrameContext("fixed local opaque state byte bound exceeded") + } + budget.bytes -= bytes + return nil +} + +// Partitioned replay charges actual state payload at copies and meets. This +// saturating logical charge includes map entries, not allocator/RSS overhead. +func fixedLocalStateWork(state x86FixedLocalState) int64 { + const limit int64 = 64 << 20 + var bytes int64 + if state.savedBP != nil { + bytes = 128 + } + for _, group := range []struct { + unit int64 + counts []int + }{ + {8, []int{len(state.bytes), len(state.booleanBytes)}}, + {64, []int{len(state.pointers), len(state.opaque), len(state.opaqueCells), len(state.booleans), + len(state.constants), len(state.ranges), len(state.known), len(state.vectors), len(state.masks), + len(state.addresses), len(state.results)}}, + } { + for _, count := range group.counts { + if int64(count) > (limit-bytes)/group.unit { + return limit + 1 + } + bytes += int64(count) * group.unit + } + } + return bytes +} + +// Only the existing closed vector plan calls this cost projection. The step +// debit remains mandatory; this grants no input or storage permission. +func fixedLocalVectorRegisterWork(effect x86TypedByteEffect) bool { + if effect.dynamicMemory || effect.vector.memoryAlignment <= 0 || + effect.vector.upperWrite != x86TypedVectorUpperPreserve && effect.vector.upperWrite != x86TypedVectorUpperZero { + return false + } + for _, operand := range effect.operands { + if operand.operand.Kind != OpReg && operand.operand.Kind != OpImm { + return false + } + } + return true +} + +// Cost only: the ordinary transfer still owns pointer, FLAGS and input facts. +// Keep original address syntax conservative even if its effect becomes Regs. +func fixedLocalGPRegisterWork(ins Instr) (x86TypedByteEffect, bool) { + if !x86ResultNamedPredicateInstruction(ins) || ins.x86NumericalRead != nil { + return x86TypedByteEffect{}, false + } + for _, operand := range ins.Args { + if operand.Kind != OpReg && operand.Kind != OpImm { + return x86TypedByteEffect{}, false + } + } + effect, known, err := fixedLocalGPByteEffect(ins) + if !known || err != nil || effect.dynamicMemory { + return x86TypedByteEffect{}, false + } + for _, operand := range effect.operands { + if operand.operand.Kind != OpReg && operand.operand.Kind != OpImm { + return x86TypedByteEffect{}, false + } + } + return effect, true +} + +func replayX86FixedLocal(fn *Func, sig FuncSig, proof *x86FixedLocalProof, residue int, budget *x86FixedLocalBudget, + effects map[int]x86TypedByteEffect, layouts *x86FixedLocalLayoutGraph) error { + edges, instructions, err := fixedLocalEdges(*fn) + if err != nil { + return err + } + linear := fixedLocalLinearEdges(*fn, edges, instructions) + err = replayX86FixedLocalCFG(fn, sig, proof, residue, budget, effects, edges, instructions, linear, nil, layouts) + if err == nil { + return nil // A stronger loop model must not reject existing generic CFG proofs. + } + loops := fixedLocalCountedLoops(*fn, edges, instructions) + if len(loops) == 0 { + return err + } + // Failed attempts publish no access annotations. Reuse the original source, + // transfer and remaining budget; this is not recursive translation or caching. + countedErr := replayX86FixedLocalCFG(fn, sig, proof, residue, budget, effects, edges, instructions, linear, loops, layouts) + if layouts == nil && errors.Is(err, errFixedLocalLayoutJoin) && countedErr != nil { + return fmt.Errorf("%w; counted replay: %v", err, countedErr) + } + return countedErr +} + +// Keep states at control-flow boundaries. A sole fallthrough predecessor has +// no meet: its private transfer state can flow directly to the next instruction. +// Count all CFG predecessors, including unreachable edges and repeated targets; +// branch destinations and PC zero retain their independent entry state. +func fixedLocalLinearEdges(fn Func, edges map[int][]int, instructions []int) []bool { + predecessors := make([]int, len(instructions)) + for _, targets := range edges { + for _, target := range targets { + if target >= 0 && target < len(predecessors) { + predecessors[target]++ + } + } + } + linear := make([]bool, len(instructions)) + for pc, targets := range edges { + op := fn.Instrs[instructions[pc]].Op + linear[pc] = op != OpJMP && op != OpRET && op != OpCALL && !isAMD64ConditionalBranch(op) && + len(targets) == 1 && targets[0] == pc+1 && pc+1 < len(instructions) && predecessors[pc+1] == 1 + } + return linear +} + +// A nil linear mask retains the all-instruction reference schedule. Both paths +// use the same transfer, meet, work budgets and final access publication. +func replayX86FixedLocalCFG(fn *Func, sig FuncSig, proof *x86FixedLocalProof, residue int, budget *x86FixedLocalBudget, + effects map[int]x86TypedByteEffect, edges map[int][]int, instructions []int, linear []bool, + loops map[int]x86FixedLocalCountedLoop, graph *x86FixedLocalLayoutGraph) error { + if proof.frame <= 0 || proof.frame > 1<<20 { + return goAMD64FrameContext("fixed local retained state storage budget exceeded") + } + state := x86FixedLocalState{ + pointers: map[Reg]int64{SP: 0}, constants: make(map[Reg]int64), known: make(map[Reg]uint8), + opaque: make(map[Reg]x86FixedLocalOrigins), opaqueCells: make(map[int64]x86FixedLocalOrigins), booleans: make(map[Reg]uint8), + booleanBytes: newFixedLocalByteSet(proof.frame), + vectors: make(map[Reg]uint64), masks: make(map[Reg]bool), + bytes: newFixedLocalByteSet(proof.frame), addresses: make(map[int64]int64), results: make(map[int]bool), + } + // All GP entry bytes except the private SP origin remain unknown. ABI0 + // does not supply an initialized R14/g value; the observed nonzero-frame + // prologue/epilogue independently preserves BP. Explicit definitions may + // use either register without importing its entry value or native role. + states := map[int]x86FixedLocalState{0: state} + work := []int{0} + queued := map[int]bool{0: true} + accesses := make(map[int]map[int]x86FixedLocalAccess) + var layouts *x86FixedLocalLayouts + var observations map[int]map[int]map[int]x86FixedLocalAccess + returned := false + retainedBytes := proof.frame + if graph != nil { + current := newFixedLocalLayouts(graph.nodes) + layouts = ¤t + observations = make(map[int]map[int]map[int]x86FixedLocalAccess) + retainedBytes += graph.retained + graph.series + int64(len(layouts.nodes))*fixedLocalLayoutNodeBytes + } + if retainedBytes > 1<<20 { + return goAMD64FrameContext("fixed local retained state storage budget exceeded") + } + for head := 0; head < len(work); head++ { + node := work[head] + pc, alternative := node, false + currentAccesses := accesses + if layouts != nil { + pc = layouts.nodes[node].pc + alternative = node != layouts.at[pc][0] + currentAccesses = make(map[int]map[int]x86FixedLocalAccess) + } + queued[node] = false + if pc < 0 || pc >= len(instructions) { + return goAMD64FrameContext("fixed local fallthrough lacks a source return") + } + copyWork := fixedLocalOriginStorage(states[node]) + if layouts != nil { + copyWork = 2 * fixedLocalStateWork(states[node]) + } + if err := budget.consumeOrigins(copyWork); err != nil { + return err + } + current := cloneFixedLocalState(states[node]) + var active *x86FixedLocalCountedLoop + var number uint64 + iteration := 0 + var targets []int + for { + effect, planned := effects[instructions[pc]] + var gp *x86TypedByteEffect + workBytes := proof.frame + if layouts != nil && planned && fixedLocalVectorRegisterWork(effect) { + workBytes = 0 + } else if layouts != nil && !planned { + if parsed, cheap := fixedLocalGPRegisterWork(fn.Instrs[instructions[pc]]); cheap { + workBytes, gp = 0, &parsed + } + } + if err := budget.consume(workBytes); err != nil { + return err + } + if active == nil { + if loop, found := loops[pc]; found { + if value, known := loop.number(¤t); known { + active, number = &loop, value + } + } + } + // Transfer a private instruction copy: provisional loop states must + // never publish accesses, including certificates from another residue. + ins := fn.Instrs[instructions[pc]] + ins.x86FixedLocalMemory = nil + proof.ordinal = instructions[pc] + var err error + transition := fn.x86DirectTailRoot.privateCallTransition(pc) + if transition { + err = fixedLocalFrameTransition(fn.x86DirectTailRoot, pc, ¤t) + } else if planned { + err = fixedLocalTypedByteStep(&ins, ¤t, FuncSig{}, proof, residue, effect) + } else { + err = fixedLocalStepWithGP(&ins, ¤t, sig, proof, residue, gp) + } + if err != nil { + return fmt.Errorf("%s: %q: %w", fn.Sym, ins.Raw, err) + } + if active != nil { + if pc == active.update { + number, err = active.next(number) + if err != nil { + return err + } + } + if value, known := active.number(¤t); !known || value != number { + return goAMD64FrameContext("fixed local counted-loop counter changed outside its unique update") + } + if err := fixedLocalCountedAccesses(currentAccesses[pc], ins.x86FixedLocalMemory, iteration); err != nil { + return err + } + } + if alternative { + retainedBytes += fixedLocalLayoutAccessBytes(ins.x86FixedLocalMemory) - fixedLocalLayoutAccessBytes(currentAccesses[pc]) + if retainedBytes > 1<<20 { + return goAMD64FrameContext("fixed local retained state storage budget exceeded") + } + } + if layouts == nil || len(ins.x86FixedLocalMemory) != 0 { + currentAccesses[pc] = ins.x86FixedLocalMemory + } + returned = returned || ins.Op == OpRET && !transition + if active != nil { + if pc == active.latch { + if active.continues(number) { + iteration++ + pc = active.header + continue + } + targets = edges[pc][:1] // The actual known comparison has reached the false latch. + break + } + pc++ + continue + } + if linear == nil || !linear[pc] { + break + } + pc++ + } + if targets == nil { + targets = edges[pc] + } + if alternative { + for _, old := range observations[node] { + retainedBytes -= fixedLocalLayoutAccessBytes(old) + } + } + if layouts != nil { + observations[node] = currentAccesses + layouts.nodes[node].end = pc + layouts.nodes[node].targets = [2]int{-1, -1} + } + for edge, target := range targets { + if edge >= 2 || target < 0 || target >= len(instructions) { + return goAMD64FrameContext("fixed local fallthrough lacks a source return") + } + next := current + if isAMD64ConditionalBranch(fn.Instrs[instructions[pc]].Op) { + var feasible bool + next, feasible = fixedLocalRangeEdge(current, fn.Instrs[instructions[pc]].Op, edge == 1) + if !feasible { + continue + } + comparison := current.comparison + if comparison.literal.symbol != "" && !next.opaque[comparison.register].equal(current.opaque[comparison.register]) { + if err := budget.consumeOrigins(fixedLocalOriginMapStorage(next.opaque)); err != nil { + return err + } + } + } + terminal := fn.Instrs[instructions[target]] + if terminal.Op == OpRET && !fn.x86DirectTailRoot.privateCallTransition(target) { + // A final return has no successor that could consume dead local + // identities. Check every incoming SP/result state independently; + // a helper return still resumes its caller through the usual CFG. + if err := budget.consume(proof.frame); err != nil { + return err + } + if err := fixedLocalStep(&terminal, &next, sig, proof, residue); err != nil { + return fmt.Errorf("%s: %q: %w", fn.Sym, terminal.Raw, err) + } + returned = true + if layouts != nil { + layouts.nodes[node].targets[edge] = -2 + } + continue + } + if layouts != nil { + previousNodes := len(layouts.nodes) + var err error + target, err = layouts.target(states, node, edge, target, next, budget) + if err != nil { + return err + } + retainedBytes += int64(len(layouts.nodes)-previousNodes) * fixedLocalLayoutNodeBytes + } + changed := true + if previous, found := states[target]; found { + if layouts != nil { + if err := budget.consumeOrigins(2*fixedLocalStateWork(previous) + fixedLocalStateWork(next)); err != nil { + return err + } + } + var err error + changed, err = mergeFixedLocalState(&previous, next) + if err != nil { + return err + } + states[target] = previous + } else { + // Only control-flow boundaries retain complete frame states. + // Keep the original frame-state product cap on actual storage; + // the independent per-instruction work budgets remain unchanged. + originBytes := fixedLocalOriginStorage(next) + if proof.frame+originBytes > (1<<20)-retainedBytes { + return goAMD64FrameContext("fixed local retained state storage budget exceeded") + } + retainedBytes += proof.frame + originBytes + copyWork := originBytes + if layouts != nil { + copyWork = 2 * fixedLocalStateWork(next) + } + if err := budget.consumeOrigins(copyWork); err != nil { + return err + } + states[target] = cloneFixedLocalState(next) + } + if changed && !queued[target] { + queued[target] = true + work = append(work, target) + } + } + } + if !returned { + return goAMD64FrameContext("fixed local source has no reachable return") + } + // Each node contributes only its final transfer, in frozen edge identity + // order. Counted iterations remain ordered within that node's observations. + var seriesBytes int64 + if layouts != nil { + if err := layouts.finish(); err != nil { + return err + } + var err error + accesses, seriesBytes, err = collectFixedLocalLayoutAccesses(len(layouts.nodes), observations, (1<<20)-retainedBytes) + if err != nil { + return err + } + } + // An empty worklist means each retained transfer consumed the final input + // state: every changed meet requeued its consumer. Publish those local + // annotations only now, without repeating every instruction in a DAG. + // Entry state remains a predecessor of PC zero, including backedges. + for pc, index := range instructions { + current, reachable := accesses[pc] + if !reachable { + continue + } + previous := fn.Instrs[index].x86FixedLocalMemory + if residue != 0 && len(previous) != len(current) { + return goAMD64FrameContext("fixed local access inventory depends on original SP residue") + } + for operand, access := range current { + if residue == 0 { + continue + } + prior, found := previous[operand] + if !found || !fixedLocalAccessShapeEqual(prior, access) || + len(prior.iterationOffsets) != len(access.iterationOffsets) { + return goAMD64FrameContext("fixed local access shape depends on original SP residue") + } + prior.offset[residue] = access.offset[residue] + prior.iterationOffsets = append([][8]int64(nil), prior.iterationOffsets...) + for index, offset := range access.iterationOffsets { + prior.iterationOffsets[index][residue] = offset[residue] + } + current[operand] = prior + } + } + for pc, current := range accesses { + fn.Instrs[instructions[pc]].x86FixedLocalMemory = current + } + if graph != nil { + graph.nodes, graph.series = layouts.nodes, seriesBytes + } + return nil +} + +func mergeFixedLocalState(previous *x86FixedLocalState, current x86FixedLocalState) (bool, error) { + // Private relocation identities must agree. Incoming origins are a separate + // must-domain: losing one leaves unknown bytes, never numerical knowledge. + if !reflect.DeepEqual(previous.pointers, current.pointers) || !reflect.DeepEqual(previous.addresses, current.addresses) { + if err := fixedLocalLayoutRoles(*previous, current); err != nil { + return false, err + } + return false, errFixedLocalLayoutJoin + } + changed, err := mergeFixedLocalSavedBP(previous, current) + if err != nil { + return false, err + } + changed = intersectFixedLocalOrigins(previous.constants, current.constants) || changed + changed = mergeFixedLocalRanges(previous, current) || changed + merged, err := mergeFixedLocalOrigins(previous.opaque, current.opaque) + if err != nil { + return false, err + } + changed = changed || merged + merged, err = mergeFixedLocalOrigins(previous.opaqueCells, current.opaqueCells) + if err != nil { + return false, err + } + changed = changed || merged + flags := previous.flags & current.flags + changed = changed || flags != previous.flags + previous.flags = flags + for reg, bits := range previous.known { + value := bits & current.known[reg] + changed = changed || value != bits + previous.known[reg] = value + } + for reg, bits := range previous.booleans { + value := bits & current.booleans[reg] + changed = changed || value != bits + previous.booleans[reg] = value + } + for reg, bits := range previous.vectors { + value := bits & current.vectors[reg] + changed = changed || value != bits + previous.vectors[reg] = value + } + for reg, known := range previous.masks { + value := known && current.masks[reg] + changed = changed || value != known + previous.masks[reg] = value + } + changed = previous.bytes.intersect(current.bytes) || changed + changed = previous.booleanBytes.intersect(current.booleanBytes) || changed + for index, written := range previous.results { + value := written && current.results[index] + changed = changed || value != written + previous.results[index] = value + } + return changed, nil +} + +func fixedLocalMemory(ins *Instr, index int, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int, width, alignment int64, write, pointer bool) (int64, error) { + if index < 0 || index >= len(ins.Args) || ins.Args[index].Kind != OpMem || width <= 0 { + return 0, goAMD64FrameContext("fixed local access lacks native memory") + } + memory, err := fixedLocalMemoryAddress(*ins, index, proof.frame) + if err != nil { + return 0, err + } + offset, found := fixedLocalPrivateOffset(memory, state) + if !found { + return 0, goAMD64FrameContext("fixed local access has unknown/escaping address provenance") + } + if offset < 0 || offset > proof.frame || width > proof.frame-offset || alignment <= 0 || + (int64(residue*8)+offset)%alignment != 0 { + return 0, goAMD64FrameContext("fixed local access crosses storage or has residue-dependent native alignment") + } + if !write && !pointer { + if !state.bytes.all(offset, width) { + return 0, goAMD64FrameContext("fixed local native read contains uninitialized or address bytes") + } + } + return offset, fixedLocalRecordAccess(ins, index, proof, residue, width, write, false, offset) +} + +// Native input addresses retain ordinary faulting memory semantics. Their Go +// pointer type proves only provenance, never a pointee extent or alignment. +func fixedLocalMemoryAccess(ins *Instr, index int, state *x86FixedLocalState, proof *x86FixedLocalProof, + residue int, width, alignment int64, write, pointer bool) (int64, x86FixedLocalMemoryRole, error) { + if index >= 0 && index < len(ins.Args) && ins.Args[index].Kind == OpMem { + memory := ins.Args[index].Mem + if origins, input := state.opaque[memory.Base]; input { + root, valid := x86CallerGP(memory.Base) + displacement, known, numerical := fixedLocalNativeDisplacement(memory, state) + if !valid || !origins.valid() || root != memory.Base || root == SP || width <= 0 || alignment <= 0 || pointer || !numerical { + return 0, x86FixedLocalNativeInput, goAMD64FrameContext("fixed local native address requires a full typed base and numerical index") + } + if !fixedLocalOriginOwners(origins, proof) || write && origins.readonly() { + return 0, x86FixedLocalNativeInput, goAMD64FrameContext("fixed local native address lacks its original storage or write role") + } + if literal, exact := origins.literal(); exact && known { + literal.offset += displacement + if !fixedLocalLiteralRange(proof.binding, literal, width, alignment) { + return 0, x86FixedLocalLiteralMemory, goAMD64FrameContext("fixed local exact DATA read exceeds original identity, extent or alignment") + } + } + return 0, x86FixedLocalNativeInput, fixedLocalRecordAccess(ins, index, proof, residue, width, write, true, 0) + } + } + // Defined machine address bits retain their ordinary native access. They + // are neither a relocated private pointer nor a reusable pointee fact. + if index >= 0 && index < len(ins.Args) && ins.Args[index].Kind == OpMem && + x86ResultNamedPredicateInstruction(*ins) && !pointer && width > 0 && alignment > 0 { + memory := ins.Args[index].Mem + root, valid := x86CallerGP(memory.Base) + if valid && root == memory.Base && fixedLocalNumericRead(Operand{Kind: OpReg, Reg: root}, 8, state) { + if _, _, numerical := fixedLocalNativeDisplacement(memory, state); numerical { + return 0, x86FixedLocalNativeInput, fixedLocalRecordAccess(ins, index, proof, residue, width, write, true, 0) + } + } + } + offset, err := fixedLocalMemory(ins, index, state, proof, residue, width, alignment, write, pointer) + return offset, x86FixedLocalPrivateMemory, err +} + +func fixedLocalRecordAccess(ins *Instr, index int, proof *x86FixedLocalProof, residue int, + width int64, write, nativeInput bool, offset int64) error { + if ins.x86FixedLocalMemory == nil { + ins.x86FixedLocalMemory = make(map[int]x86FixedLocalAccess) + } + shape, err := fixedLocalInstructionDigest(*ins) + if err != nil { + return err + } + access, found := ins.x86FixedLocalMemory[index] + if !found { + access = x86FixedLocalAccess{owner: proof, shape: shape, width: width, write: write, nativeInput: nativeInput, ordinal: proof.ordinal} + } + if access.owner != proof || access.shape != shape || access.width != width || access.write != write || access.nativeInput != nativeInput { + return goAMD64FrameContext("fixed local access shape changed") + } + access.offset[residue] = offset + ins.x86FixedLocalMemory[index] = access + return nil +} + +func fixedLocalStep(ins *Instr, state *x86FixedLocalState, sig FuncSig, proof *x86FixedLocalProof, residue int) error { + return fixedLocalStepWithGP(ins, state, sig, proof, residue, nil) +} + +func fixedLocalStepWithGP(ins *Instr, state *x86FixedLocalState, sig FuncSig, proof *x86FixedLocalProof, residue int, + gp *x86TypedByteEffect) error { + fail := func() error { + return goAMD64FrameContext("unproved fixed local width/initialization/address/control effect: %s", ins.Op) + } + if ins.Op == OpTEXT { + return nil + } + if ins.Op == OpCALL { + return fixedLocalOrdinaryCallStep(*ins, state, proof) + } + if _, known, err := x86OrderingHintProperties(ins.Op, *ins, "amd64"); known { + return err // Exact native fences change no byte, pointer or FLAGS facts. + } + if form, known, err := x86VectorZeroProperties(*ins, "amd64"); known { + if err != nil { + return err + } + x86VectorZeroDefinedBytes(form, state.vectors) + return nil + } + if ins.Op == OpRET { + sp, found := state.pointers[SP] + if len(ins.Args) != 0 || !found || sp != 0 { + return fail() + } + for index := range sig.Frame.Results { + if !state.results[index] { + return fail() + } + } + return nil + } + if _, counter := x86CounterControlPropertiesForOpcode(ins.Op); counter { + return fail() // This storage owner supplies no numerical counter domain. + } + if isAMD64ConditionalBranch(ins.Op) { + needed := x86ByteBranchFlags(ins.Op) + if needed == 0 || state.flags&needed != needed { + return fail() + } + return nil + } + if ins.Op == OpJMP { + return nil + } + if known, err := fixedLocalNumericalReadStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalStaticAddressStep(*ins, state, proof); known { + return err + } + if known, err := fixedLocalGPSIMDTransferStep(ins, state, sig, proof, residue); known { + return err + } + if known, err := fixedLocalPrivateAddressStep(*ins, state, proof); known { + return err + } + if width := x86RawReturnScalarMoveBytes(x86SourceMoveInstruction(*ins, "amd64").Op); width != 0 { + return fixedLocalMove(ins, state, sig, proof, residue, width) + } + if known, err := fixedLocalExchangeStep(ins, state, sig, proof, residue); known { + return err + } + if known, err := fixedLocalConditionalSetStep(ins, state, sig, proof, residue); known { + return err + } + if (ins.Op == "ADDQ" || ins.Op == "ANDQ") && len(ins.Args) == 2 && ins.Args[1].Kind == OpReg { + reg := ins.Args[1].Reg + value := ins.Args[0].Imm + known := ins.Args[0].Kind == OpImm && ins.Args[0].ImmRaw == "" && !ins.Args[0].ImmIsFloat + if ins.Args[0].Kind == OpReg { + value, known = state.constants[ins.Args[0].Reg] + } + if offset, pointer := state.pointers[reg]; pointer { + if !known { + return fail() + } + if ins.Op == "ANDQ" { + // Only an exact power-of-two residue projection preserves the + // same private address origin. No pointer bit becomes numeric. + if value != -8 && value != -16 && value != -32 && value != -64 { + return fail() + } + offset = (offset+int64(residue*8))&value - int64(residue*8) + } else { + if value > 0 && offset > math.MaxInt64-value || value < 0 && offset < math.MinInt64-value { + return fail() + } + offset += value + } + if offset < -64 || offset > proof.frame { + return fail() + } + state.pointers[reg] = offset + state.flags = 0 // Relocated pointer arithmetic flags are opaque. + state.comparison = x86FixedLocalComparison{} + return nil + } + } + if known, err := fixedLocalConditionalMoveStep(ins, state, sig, proof, residue); known { + return err + } + if known, err := fixedLocalInputAddressStep(ins, state, sig, proof, residue); known { + return err + } + var effect x86TypedByteEffect + var err error + known := gp != nil + if known { + effect = *gp + } else { + effect, known, err = fixedLocalGPByteEffect(*ins) + } + if known { + if err != nil { + return err + } + root, value, constant := fixedLocalConstantArithmetic(*ins, effect, state) + rangeReg, bounds, comparison := fixedLocalRangeEffect(*ins, effect, state) + if err := fixedLocalTypedByteStep(ins, state, sig, proof, residue, effect); err != nil { + return err + } + if constant { + state.constants[root] = value + fixedLocalWriteBooleans(state, root, 8, fixedLocalConstantBooleans(value, 8)) + } + if rangeReg != "" { + state.setRange(rangeReg, bounds) + } + if comparison.register != "" { + state.comparison = comparison + } + return nil + } + if known, err := fixedLocalSHAStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalAESByteStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalPackedMoveStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalMaskMoveStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalConversionStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalScalarLaneStep(ins, state, proof, residue); known { + return err + } + return fixedLocalVectorStep(ins, state, proof, residue) +} + +func fixedLocalNumericRead(operand Operand, width int64, state *x86FixedLocalState) bool { + if operand.Kind == OpImm { + return operand.ImmRaw == "" && !operand.ImmIsFloat + } + if operand.Kind != OpReg { + return false + } + root, valid := x86CallerGP(operand.Reg) + _, pointer := state.pointers[root] + _, opaque := state.opaque[root] + mask := x86CallerByteMask(operand.Reg, width) + return valid && root != SP && !pointer && !opaque && mask != 0 && state.known[root]&mask == mask +} + +func fixedLocalMove(ins *Instr, state *x86FixedLocalState, sig FuncSig, proof *x86FixedLocalProof, residue int, width int64) error { + if len(ins.Args) != 2 { + return goAMD64FrameContext("unproved fixed local scalar move") + } + value, err := fixedLocalReadValue(ins, 0, state, sig, proof, residue, width, false) + if err != nil { + return err + } + return fixedLocalWriteValue(ins, 1, 0, value, state, sig, proof, residue, width) +} + +func fixedLocalVectorStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) error { + if effect, known, err := fixedLocalVectorByteEffect(*ins); known { + if err != nil { + return err + } + return fixedLocalTypedByteStep(ins, state, FuncSig{}, proof, residue, effect) + } + width := uint64(65535) + first, second, destination, memory := 0, 1, 1, -1 + if form, known, err := x86FloatCompareProperties(*ins, "amd64"); known { + if err != nil || !form.legacy || form.scalar || form.byteWidth != 16 { + return goAMD64FrameContext("fixed local compare shape is not closed") + } + first, second, destination, memory = form.first, form.second, form.destination, form.memory + } else { + return goAMD64FrameContext("fixed local source has an unregistered typed vector effect: %s", ins.Op) + } + if state.vectors[x86CallerVectorRoot(ins.Args[second].Reg)]&width != width { + return goAMD64FrameContext("fixed local vector operation reads undefined destination bytes") + } + if memory >= 0 { + if _, _, err := fixedLocalReadMemory(ins, memory, state, proof, residue, 16, 16); err != nil { + return err + } + } else if state.vectors[x86CallerVectorRoot(ins.Args[first].Reg)]&width != width { + return goAMD64FrameContext("fixed local vector operation reads undefined source bytes") + } + state.vectors[x86CallerVectorRoot(ins.Args[destination].Reg)] |= width + return nil +} + +func (c *amd64Ctx) fixedLocalMemoryCertified(ins Instr, index int) bool { + access, valid := c.fixedLocalMemoryAccess(ins, index) + return valid && !access.nativeInput +} + +// Structural reachability is an emission projection, never an access role. +// Private expansions and address-taken/indirect entries keep their own graphs. +func (c *amd64Ctx) fixedLocalBlockReachability() ([]bool, error) { + if c.fixedLocal == nil || c.goarch != "amd64" || c.directTail != nil || c.indirectABI0 != nil || + len(c.continuations) != 0 || len(c.indirectLabels) != 0 || c.fpScalarSource.x86ContinuationFlow != nil { + return nil, nil + } + edges, instructions, err := fixedLocalEdges(c.fpScalarSource) + if err != nil { + return nil, err + } + seen := make([]bool, len(instructions)) + work := []int{0} + for head := 0; head < len(work); head++ { + pc := work[head] + if pc < 0 || pc >= len(seen) { + return nil, goAMD64FrameContext("fixed local emission reaches an unbound instruction") + } + if !seen[pc] { + seen[pc] = true + work = append(work, edges[pc]...) + } + } + if len(c.blockBase) != len(c.blocks) { + return nil, goAMD64FrameContext("fixed local emission block mapping differs") + } + reachable := make([]bool, len(c.blocks)) + dead := false + pc := 0 + for index, base := range c.blockBase { + if base != pc || pc > len(seen) { + return nil, goAMD64FrameContext("fixed local emission block has no source instruction") + } + // Empty adjacent labels share the following instruction's PC. A + // trailing unused label has no successor and remains unreachable. + reachable[index] = pc < len(seen) && seen[pc] + dead = dead || !reachable[index] + pc += len(c.blocks[index].instrs) + } + if pc != len(instructions) { + return nil, goAMD64FrameContext("fixed local emission source inventory is incomplete") + } + if !dead { + return nil, nil + } + // Reuse the complete live binding, declaration, fingerprint, grammar and + // independent replay checks before omitting any original instruction. + if err := c.checkFixedLocalResultInitialization(); err != nil { + return nil, err + } + pc = 0 + for _, block := range c.blocks { + for _, ins := range block.instrs { + if pc >= len(instructions) || !reflect.DeepEqual(ins, c.fpScalarSource.Instrs[instructions[pc]]) { + return nil, goAMD64FrameContext("fixed local emission instruction differs from its source") + } + pc++ + } + } + if pc != len(instructions) { + return nil, goAMD64FrameContext("fixed local emission source inventory is incomplete") + } + return reachable, nil +} + +func (c *amd64Ctx) fixedLocalMemoryAccess(ins Instr, index int) (x86FixedLocalAccess, bool) { + access, found := ins.x86FixedLocalMemory[index] + shape, err := fixedLocalInstructionDigest(ins) + if !found || c.fixedLocal == nil || access.owner != c.fixedLocal || ins.x86Encoded || + ins.x86SourceRaw != nil || ins.x86AddressBits != 0 || ins.x86SegmentPrefix != 0 || + err != nil || access.shape != shape || access.width <= 0 { + return x86FixedLocalAccess{}, false + } + expected, bound := c.fixedLocal.accesses[access.ordinal][index] + return access, bound && reflect.DeepEqual(access, expected) +} diff --git a/amd64_fixed_local_range.go b/amd64_fixed_local_range.go new file mode 100644 index 000000000..85aed5dae --- /dev/null +++ b/amd64_fixed_local_range.go @@ -0,0 +1,107 @@ +package plan9asm + +import "math" + +type x86FixedLocalComparison struct { + register Reg + constant uint64 + literal x86StaticSBAddress +} + +func cloneFixedLocalRanges(input map[Reg]x86UnsignedRange) map[Reg]x86UnsignedRange { + if len(input) == 0 { + return nil + } + copy := make(map[Reg]x86UnsignedRange, len(input)) + for reg, value := range input { + copy[reg] = value + } + return copy +} + +func (state *x86FixedLocalState) setRange(reg Reg, value x86UnsignedRange) { + if state.ranges == nil { + state.ranges = make(map[Reg]x86UnsignedRange) + } + state.ranges[reg] = value +} + +func (state *x86FixedLocalState) revokeRange(reg Reg) { + delete(state.ranges, reg) + if state.comparison.register == reg { + state.comparison = x86FixedLocalComparison{} + } +} + +// Called around the existing typed GP transfer, never as a second executor. +// Other writes revoke bounds at the same shared register-write boundaries. +func fixedLocalRangeEffect(ins Instr, effect x86TypedByteEffect, state *x86FixedLocalState) (Reg, x86UnsignedRange, x86FixedLocalComparison) { + // The existing effect distinguishes read-only comparisons from GP writes; + // unrelated arithmetic never re-enters the complete comparison grammar. + if len(effect.operands) == 2 && !effect.operands[0].write && !effect.operands[1].write { + if form, known, err := x86ScalarCompareProperties(ins.Op, ins, "amd64"); known && err == nil && form.bits == 64 { + first, second := form.args[0], form.args[1] + if first.Kind == OpReg && fixedLocalNumericRead(first, 8, state) { + if !form.test && second.Kind == OpImm { + return "", x86UnsignedRange{}, x86FixedLocalComparison{register: first.Reg, constant: uint64(second.Imm)} + } + if form.test && second.Kind == OpReg && first.Reg == second.Reg { + return "", x86UnsignedRange{}, x86FixedLocalComparison{register: first.Reg} + } + } + } + } + stem, bits, shiftFamily := amd64ScalarShiftRotateProperties(string(ins.Op)) + if shiftFamily && bits == 64 && (stem == "SHL" || stem == "SAL") { + if form, known, err := x86ScalarShiftProperties(ins.Op, ins, "amd64"); known && err == nil && + !form.double && form.count.Kind == OpImm && form.destination.Kind == OpReg { + value, bounded := state.ranges[form.destination.Reg] + shift := uint(form.maskedCount()) + if bounded && value.maximum <= math.MaxUint64>>shift { + return form.destination.Reg, x86UnsignedRange{value.minimum << shift, value.maximum << shift}, x86FixedLocalComparison{} + } + } + } + return "", x86UnsignedRange{}, x86FixedLocalComparison{} +} + +func fixedLocalRangeEdge(state x86FixedLocalState, op Op, taken bool) (x86FixedLocalState, bool) { + comparison := state.comparison + if comparison.register == "" { + return state, true + } + if comparison.literal.symbol != "" { + return fixedLocalLiteralEdge(state, op, taken) + } + value, bounded := state.ranges[comparison.register] + if !bounded { + value.maximum = math.MaxUint64 + } + next, feasible := value.refine(op, taken, comparison.constant) + if !feasible || next == value { + return state, feasible + } + state.ranges = cloneFixedLocalRanges(state.ranges) + state.setRange(comparison.register, next) + return state, feasible +} + +func mergeFixedLocalRanges(previous *x86FixedLocalState, current x86FixedLocalState) bool { + changed := false + for reg, old := range previous.ranges { + value, found := current.ranges[reg] + if !found { + delete(previous.ranges, reg) + changed = true + continue + } + value = value.hull(old) + previous.ranges[reg] = value + changed = changed || value != old + } + if previous.comparison != current.comparison && previous.comparison.register != "" { + previous.comparison = x86FixedLocalComparison{} + changed = true + } + return changed +} diff --git a/amd64_fixed_local_range_go127_test.go b/amd64_fixed_local_range_go127_test.go new file mode 100644 index 000000000..2f233d8bc --- /dev/null +++ b/amd64_fixed_local_range_go127_test.go @@ -0,0 +1,97 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalReadonlyRangeProgram(t *testing.T) (string, string, []fixedLocalRangeCase) { + t.Helper() + var assembly, declarations strings.Builder + for offset := 0; offset < 240; offset += 8 { + // Each source qword advances all eight bytes independently. Runtime + // expectations construct a byte array instead of parsing these DATA. + word := uint64(0x0807060504030201) + uint64(offset)*0x0101010101010101 + fmt.Fprintf(&assembly, "DATA literal<>+%d(SB)/8,$0x%016x\n", offset, word) + } + assembly.WriteString("GLOBL literal<>(SB),8,$240\n") + rows := fixedLocalRangeCases() + if len(rows) != 48 { + t.Fatal("complete readonly range source inventory changed", len(rows)) + } + for index, row := range rows { + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(fixedLocalConstantIndexFunction(name, row.body)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + } + return assembly.String(), declarations.String(), rows +} + +func TestGoAMD64FixedLocalReadonlyRangeObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + assembly, declarations, rows := fixedLocalReadonlyRangeProgram(t) + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + var names []string + for index := range rows { + names = append(names, fmt.Sprintf("%s.Value%d", pkg.Path, index)) + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("original bounded readonly DATA index rejected", err) + } + if !strings.Contains(ir, "internal constant [240 x i8]") { + t.Fatal("the original complete readonly DATA allocation was lost") + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + compileLLVMToObject(t, llc, target.triple, "readonly-range.ll", "readonly-range.o", ir) + compileLLVMToObject(t, llc, target.triple, "readonly-range-opt.ll", "readonly-range-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +func TestGoAMD64FixedLocalReadonlyRangeOwnerLifetime(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + assembly := fixedLocalRangeData + fixedLocalConstantIndexFunction("Value", fixedLocalRangePrefix+fixedLocalRangeLoad) + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + fixedLocalDataAPIs(t, pkg, file, assembly, options, fixedLocalRequireContext) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + for _, mutation := range []struct{ old, replacement string }{ + {",8,$240", ",8,$256"}, + {"CMPQ R9,$16", "CMPQ R9,$17"}, + {"0x0102030405060708", "0x0102030405060709"}, + {fixedLocalRangeData, ""}, + } { + input := strings.Replace(assembly, mutation.old, mutation.replacement, 1) + parsed, err := Parse(ArchAMD64, input) + if err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, parsed, input, bound, fixedLocalRequireContext) + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, file, assembly, bound, fixedLocalRequireContext) +} diff --git a/amd64_fixed_local_range_runtime_go127_test.go b/amd64_fixed_local_range_runtime_go127_test.go new file mode 100644 index 000000000..55607c1b9 --- /dev/null +++ b/amd64_fixed_local_range_runtime_go127_test.go @@ -0,0 +1,44 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "encoding/binary" + "fmt" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalReadonlyRange(t *testing.T) { + assembly, declarations, cases := fixedLocalReadonlyRangeProgram(t) + inputs := []uint64{0, 1, 2, 14, 15, 16, 17, 31} + var data [240]byte + for index := range data { + data[index] = byte(index + 1) + } + var expected [][]string + for _, input := range inputs { + var values []string + for _, row := range cases { + index, stride := input, uint64(16) + switch row.name { + case "generic loop before bound": + index %= 16 + case "scale/1": + stride = 1 + case "scale/2": + stride = 2 + case "scale/4": + stride = 4 + case "scale/8": + stride = 8 + } + value := input // Every rejected boundary preserves the original AX. + if index > 0 && index < 16 { + value = binary.LittleEndian.Uint64(data[(index-1)*stride:]) + } + values = append(values, fmt.Sprintf("%016x", value)) + } + expected = append(expected, values) + } + fixedLocalResultsRuntime(t, assembly, declarations, inputs, expected) +} diff --git a/amd64_fixed_local_range_test.go b/amd64_fixed_local_range_test.go new file mode 100644 index 000000000..b4d8d2333 --- /dev/null +++ b/amd64_fixed_local_range_test.go @@ -0,0 +1,275 @@ +package plan9asm + +import ( + "errors" + "fmt" + "math" + "sort" + "strings" + "testing" +) + +const fixedLocalRangeData = "DATA literal<>+0(SB)/8,$0x0102030405060708\n" + + "DATA literal<>+232(SB)/8,$0x3132333435363738\nGLOBL literal<>(SB),8,$240\n" + +const fixedLocalRangePrefix = "MOVQ AX,R9\nCMPQ R9,$16\nJAE done\nTESTQ R9,R9\nJE done\n" +const fixedLocalRangeLoad = "MOVQ R9,R11\nSHLQ $4,R11\nLEAQ literal<>(SB),R10\n" + + "MOVOU -16(R10)(R11*1),X0\nMOVQ X0,AX\ndone:" + +type fixedLocalRangeCase struct{ name, body string } + +func fixedLocalRangeCases() []fixedLocalRangeCase { + var cases []fixedLocalRangeCase + aliases := x86ConditionalBranchAliasOracle() + var names []string + for alias := range aliases { + names = append(names, alias) + } + sort.Strings(names) + for _, alias := range names { + var guard string + switch aliases[alias] { + case "JCS": + guard = "CMPQ R9,$16\n" + alias + " bounded\nJMP done\nbounded:\n" + case "JCC": + guard = "CMPQ R9,$16\n" + alias + " done\n" + case "JLS": + guard = "CMPQ R9,$15\n" + alias + " bounded\nJMP done\nbounded:\n" + case "JHI": + guard = "CMPQ R9,$15\n" + alias + " done\n" + case "JEQ": + guard = "CMPQ R9,$16\nJAE done\nTESTQ R9,R9\n" + alias + " done\n" + case "JNE": + guard = "CMPQ R9,$16\nJAE done\nTESTQ R9,R9\n" + alias + " nonzero\nJMP done\nnonzero:\n" + default: + continue + } + if aliases[alias] != "JEQ" && aliases[alias] != "JNE" { + guard += "TESTQ R9,R9\nJE done\n" + } + cases = append(cases, fixedLocalRangeCase{"condition/" + alias, "MOVQ AX,R9\n" + guard + fixedLocalRangeLoad}) + } + for _, register := range strings.Fields("AX BX CX DX DI SI BP R8 R9 R10 R11 R12 R13 R14 R15") { + body := strings.ReplaceAll(fixedLocalRangePrefix+fixedLocalRangeLoad, "R9", register) + cases = append(cases, fixedLocalRangeCase{"index/" + register, body}) + } + for _, scale := range []int{1, 2, 4, 8} { + body := fixedLocalRangePrefix + fmt.Sprintf("LEAQ literal<>(SB),R10\nMOVOU -%d(R10)(R9*%d),X0\nMOVQ X0,AX\ndone:", scale, scale) + cases = append(cases, fixedLocalRangeCase{fmt.Sprintf("scale/%d", scale), body}) + } + cases = append(cases, + fixedLocalRangeCase{"compare lower endpoint", strings.Replace(fixedLocalRangePrefix, "TESTQ R9,R9\nJE done", "CMPQ R9,$1\nJB done", 1) + fixedLocalRangeLoad}, + fixedLocalRangeCase{"flags preserved before branch", strings.Replace(fixedLocalRangePrefix, "JAE done", "MOVQ AX,R8\nNOP\nJAE done", 1) + fixedLocalRangeLoad}, + fixedLocalRangeCase{"range survives later flags", fixedLocalRangePrefix + "CMPQ AX,$12\nJE join\nNOP\njoin:\n" + fixedLocalRangeLoad}, + fixedLocalRangeCase{"unrelated partial write", fixedLocalRangePrefix + "MOVB $0,CL\n" + fixedLocalRangeLoad}, + fixedLocalRangeCase{"two bounded incoming paths", "MOVQ AX,R9\nTESTQ AX,AX\nJE small\nCMPQ R9,$16\nJAE done\nJMP bounded\nsmall:\nCMPQ R9,$8\nJAE done\nbounded:\nTESTQ R9,R9\nJE done\n" + fixedLocalRangeLoad}, + fixedLocalRangeCase{"generic loop before bound", "MOVQ AX,R9\nagain:\nCMPQ R9,$16\nJB tail\nSUBQ $16,R9\nJMP again\ntail:\nTESTQ R9,R9\nJE done\n" + fixedLocalRangeLoad}, + fixedLocalRangeCase{"wide numerical shift count", fixedLocalRangePrefix + strings.Replace(fixedLocalRangeLoad, "SHLQ $4", "SHLQ $68", 1)}, + fixedLocalRangeCase{"equivalent arithmetic shift spelling", fixedLocalRangePrefix + strings.Replace(fixedLocalRangeLoad, "SHLQ", "SALQ", 1)}, + ) + return cases +} + +func fixedLocalRangeFixture(t *testing.T, body string) (Func, FuncSig, string) { + t.Helper() + fn, sig, source := fixedLocalGPFixture(t, body) + // This is an internal replay fixture, not a public source binding. The + // separate three-API object matrix captures the complete original DATA. + sig.x86FixedLocal = &x86FixedLocalBinding{literalData: map[string]x86FixedLocalData{ + "literal<>": {size: 240, alignment: 16}, + }} + return fn, sig, source +} + +func TestGoAMD64FixedLocalReadonlyRangeComplete(t *testing.T) { + var original strings.Builder + original.WriteString(fixedLocalRangeData) + rows := fixedLocalRangeCases() + if len(rows) != 48 { + t.Fatal("unsigned/equality aliases, complete GP bank and scale inventory changed", len(rows)) + } + for index, row := range rows { + fn, sig, source := fixedLocalRangeFixture(t, row.body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + proved, err := proveX86FixedLocal(fn, sig) + if err != nil { + t.Fatal("complete numerical range lost its readonly DATA access", err) + } + found := false + for _, instruction := range proved.Instrs { + if instruction.Op != "MOVOU" { + continue + } + found = true + access, present := instruction.x86FixedLocalMemory[0] + if !present || access.owner != proved.x86FixedLocal || access.width != 16 || access.write || !access.nativeInput { + t.Fatal("readonly numerical envelope acquired private storage or lost its complete tuple", access) + } + } + if !found { + t.Fatal("fixture has no complete indexed vector read") + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalReadonlyRangeGuards(t *testing.T) { + valid := fixedLocalRangePrefix + fixedLocalRangeLoad + rows := []fixedLocalRangeCase{ + {"zero not excluded", strings.Replace(valid, "TESTQ R9,R9\nJE done\n", "", 1)}, + {"upper edge reversed", strings.Replace(valid, "JAE done", "JB done", 1)}, + {"sixteen admitted", strings.Replace(valid, "CMPQ R9,$16", "CMPQ R9,$17", 1)}, + {"stale unsigned flags", strings.Replace(valid, "JAE done", "CMPQ AX,$32\nJAE done", 1)}, + {"stale zero flags", strings.Replace(valid, "JE done", "CMPQ AX,$32\nJE done", 1)}, + {"compared value overwritten", strings.Replace(valid, "JAE done", "MOVQ 0(SP),R9\nJAE done", 1)}, + {"partial comparison", strings.Replace(valid, "CMPQ R9,$16", "CMPL R9,$16", 1)}, + {"partial index overwrite", fixedLocalRangePrefix + "MOVB $0,R9\n" + fixedLocalRangeLoad}, + {"loaded index loses range", fixedLocalRangePrefix + "MOVQ 0(SP),R9\n" + fixedLocalRangeLoad}, + {"numerical arithmetic loses range", fixedLocalRangePrefix + "ADDQ $16,R9\n" + fixedLocalRangeLoad}, + {"shift overflow", strings.Replace(valid, "SHLQ $4", "SHLQ $63", 1)}, + {"lower bound before object", strings.Replace(valid, "MOVOU -16(", "MOVOU -17(", 1)}, + {"complete vector after object", strings.Replace(valid, "MOVOU -16(", "MOVOU -15(", 1)}, + {"wider complete tuple", strings.Replace(valid, "MOVOU -16(R10)(R11*1),X0", "VMOVDQU -16(R10)(R11*1),Y0", 1)}, + {"alignment not inferred from interval", strings.Replace(valid, "SHLQ $4,R11", "SHLQ $0,R11", 1)}, + {"unbounded incoming path", strings.Replace(valid, "CMPQ R9,$16", "CMPQ AX,$0\nJE unchecked\nCMPQ R9,$16", 1)}, + {"private base is not literal", strings.Replace(valid, "LEAQ literal<>(SB),R10", "MOVQ SP,R10", 1)}, + {"pointer cannot be numerical index", strings.Replace(valid, "MOVQ AX,R9", "MOVQ SP,R9", 1)}, + {"missing upper numerical bytes", strings.Replace(valid, "MOVQ AX,R9", "MOVW AX,R9", 1)}, + {"foreign readonly owner", strings.Replace(valid, "LEAQ literal<>(SB),R10", "LEAQ other<>(SB),R10", 1)}, + {"readonly write", strings.Replace(valid, "MOVOU -16(R10)(R11*1),X0", "PXOR X0,X0\nMOVOU X0,-16(R10)(R11*1)", 1)}, + } + var original strings.Builder + original.WriteString(fixedLocalRangeData + "GLOBL other<>(SB),8,$240\n") + // Dynamic native reads retain their original effective address. Their + // translation does not assert that every possible caller index is safe. + // Private relocation, initialization and storage ownership remain required. + rejected := map[string]bool{ + "private base is not literal": true, + "pointer cannot be numerical index": true, + "missing upper numerical bytes": true, + "foreign readonly owner": true, + "readonly write": true, + } + for index, row := range rows { + if row.name == "unbounded incoming path" { + row.body = strings.Replace(row.body, "MOVQ R9,R11", "unchecked:\nMOVQ R9,R11", 1) + } + if row.name == "alignment not inferred from interval" { + row.body = strings.Replace(row.body, "MOVOU -16(", "MOVO 0(", 1) + } + fn, sig, source := fixedLocalRangeFixture(t, row.body) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + t.Run(row.name, func(t *testing.T) { + _, err := proveX86FixedLocal(fn, sig) + if !rejected[row.name] { + if err != nil { + t.Fatal("dynamic machine address incorrectly required a caller bounds proof", err) + } + return + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("private, undefined or readonly storage acquired permission", err) + } + for _, instruction := range fn.Instrs { + if len(instruction.x86FixedLocalMemory) != 0 { + t.Fatal("failed range proof published provisional accesses") + } + } + }) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoX86SDKLiteralUnsignedRangeEdges(t *testing.T) { + // A small exhaustive value oracle is independent of the interval formulas. + // It includes zero, interior exclusions and the unsigned overflow endpoint. + for alias, canonical := range x86ConditionalBranchAliasOracle() { + if canonical != "JCS" && canonical != "JCC" && canonical != "JLS" && canonical != "JHI" && canonical != "JEQ" && canonical != "JNE" { + continue + } + t.Run(alias, func(t *testing.T) { + for _, extent := range [][2]uint64{{0, 0}, {1, 1}, {15, 15}, {16, 16}, {0, 15}, {0, 31}, {1, 15}, {math.MaxUint64 - 2, math.MaxUint64}} { + for _, bound := range []uint64{0, 1, 15, 16, math.MaxUint64} { + for _, taken := range []bool{false, true} { + minimum, maximum, feasible := uint64(0), uint64(0), false + for value := extent[0]; ; value++ { + var result bool + switch canonical { + case "JCS": + result = value < bound + case "JCC": + result = value >= bound + case "JLS": + result = value <= bound + case "JHI": + result = value > bound + case "JEQ": + result = value == bound + case "JNE": + result = value != bound + } + if result == taken { + if !feasible { + minimum = value + } + maximum, feasible = value, true + } + if value == extent[1] { + break + } + } + state := x86SDKLiteralState{comparison: 0, compared: true, constant: bound} + state.gp[0] = x86SDKLiteralValue{width: 8, minimum: extent[0], maximum: extent[1]} + actual := x86SDKLiteralRefineEdge(Op(alias), taken, &state) + if actual != feasible || feasible && (state.gp[0].minimum != minimum || state.gp[0].maximum != maximum) { + t.Fatal("unsigned edge differs from enumerated values", extent, bound, taken, state.gp[0], actual, minimum, maximum, feasible) + } + } + } + } + }) + } +} + +func TestGoAMD64FixedLocalReadonlyRangeWriteRevocation(t *testing.T) { + for _, row := range []string{ + "MOVB $0,R9", "MOVQ AX,R9", "MOVL AX,R9", "ADDQ $1,R9", + "LEAQ 16(SP),R9", "LEAQ literal<>(SB),R9", "CMOVQEQ AX,R9", + "SETNE R9", "KMOVQ K1,R9", "VPEXTRQ $0,X0,R9", "MOVQ X0,R9", + } { + t.Run(row, func(t *testing.T) { + state := x86FixedLocalState{ + pointers: map[Reg]int64{SP: 0}, opaque: make(map[Reg]x86FixedLocalOrigins), + constants: map[Reg]int64{AX: 7, "R9": 3}, booleans: make(map[Reg]uint8), + known: map[Reg]uint8{AX: 255, "R9": 255}, vectors: map[Reg]uint64{"Z0": ^uint64(0)}, + masks: map[Reg]bool{"K1": true}, flags: x86ByteFlagsAll, + ranges: map[Reg]x86UnsignedRange{"R9": {1, 15}}, comparison: x86FixedLocalComparison{register: "R9", constant: 16}, + } + proof := &x86FixedLocalProof{frame: 64, binding: &x86FixedLocalBinding{ + literalData: map[string]x86FixedLocalData{"literal<>": {size: 240, alignment: 16}}, + }} + ins := x86ScalarLaneInstruction(t, row) + if err := fixedLocalStep(&ins, &state, FuncSig{}, proof, 0); err != nil { + t.Fatal("registered write fixture rejected", err) + } + if _, stale := state.ranges["R9"]; stale || state.comparison.register != "" { + t.Fatal("GP writer retained a stale numerical range or FLAGS relation", state.ranges, state.comparison) + } + }) + } +} + +func TestGoAMD64FixedLocalUnsignedRangeDeadBackedge(t *testing.T) { + // TEST clears CF regardless of the input, so JB cannot revisit the store. + // The previous CMP must neither force three iterations nor reject this + // actual one-pass function merely because its source contains a backedge. + const body = "MOVQ $0,SI\nloop:\nMOVQ AX,16(SP)(SI*8)\nADDQ $1,SI\n" + + "CMPQ SI,$3\nTESTQ AX,AX\nJB loop\nMOVQ AX,AX" + fn, sig, source := fixedLocalCountedLoopFixture(t, body) + requireX86GoAssemblerResult(t, "amd64", source, true) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("a provably dead backedge rejected an ordinary one-pass private access", err) + } +} diff --git a/amd64_fixed_local_replay_test.go b/amd64_fixed_local_replay_test.go new file mode 100644 index 000000000..5f36cff85 --- /dev/null +++ b/amd64_fixed_local_replay_test.go @@ -0,0 +1,306 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func fixedLocalReplaySource(body string) string { + return "TEXT ·Value(SB),4,$64-16\nMOVQ a+0(FP),AX\n" + + "MOVQ AX,0(SP)\nMOVQ AX,8(SP)\nMOVOU 0(SP),X0\nMOVOU X0,X1\n" + + body + "\nMOVQ X0,AX\nMOVQ AX,ret+8(FP)\nRET\n" +} + +// These analysis-only fixtures deliberately exclude source capture, subprocesses +// and LLVM construction. Their complete proof must succeed before timing starts. +func fixedLocalReplayFixture(t testing.TB, body string) (Func, FuncSig) { + t.Helper() + file, err := Parse(ArchAMD64, fixedLocalReplaySource(body)) + if err != nil { + t.Fatal(err) + } + sig := FuncSig{Name: "Value", Args: []LLVMType{I64}, Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }} + return file.Funcs[0], sig +} + +func fixedLocalReplayBody(shape string, length int) string { + const instruction = "PADDQ X1,X0\n" + switch shape { + case "linear": + return strings.Repeat(instruction, length) + case "diamond": + return "CMPQ AX,AX\nJEQ left\n" + strings.Repeat(instruction, length/2) + + "JMP joined\nleft:\n" + strings.Repeat(instruction, length-length/2) + "joined:\n" + case "loop": + return "MOVQ $3,CX\nagain:\n" + strings.Repeat(instruction, length) + + "SUBQ $1,CX\nJNE again\n" + default: + panic("unknown fixed-local replay fixture shape") + } +} + +func TestGoAMD64FixedLocalReplayInstructionBudget(t *testing.T) { + for _, length := range []int{8, 128, 1024, 5000} { + t.Run(fmt.Sprint(length), func(t *testing.T) { + fn, sig := fixedLocalReplayFixture(t, fixedLocalReplayBody("linear", length)) + effects, err := fixedLocalEffectPlan(fn, nil) + if err != nil { + t.Fatal(err) + } + _, instructions, err := fixedLocalEdges(fn) + if err != nil { + t.Fatal(err) + } + count := len(instructions) + for _, deficit := range []string{"none", "step", "byte"} { + t.Run(deficit, func(t *testing.T) { + current := fn + current.Instrs = append([]Instr(nil), fn.Instrs...) + proof := &x86FixedLocalProof{frame: fn.FrameSize} + budget := x86FixedLocalBudget{steps: count, bytes: int64(count) * fn.FrameSize} + if deficit == "step" { + budget.steps-- + } + if deficit == "byte" { + budget.bytes-- + } + err := replayX86FixedLocal(¤t, sig, proof, 0, &budget, effects, nil) + if deficit == "none" { + if err != nil || budget.steps != 0 || budget.bytes != 0 { + t.Fatalf("complete transfer: budget=%+v, error=%v", budget, err) + } + accesses := 0 + for _, ins := range current.Instrs { + accesses += len(ins.x86FixedLocalMemory) + } + if accesses != 3 { + t.Fatalf("got %d private accesses, want both stores and complete vector load", accesses) + } + return + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("exhausted %s budget returned %v", deficit, err) + } + for _, ins := range current.Instrs { + if len(ins.x86FixedLocalMemory) != 0 { + t.Fatal("incomplete replay published a provisional access") + } + } + }) + } + }) + } +} + +func fixedLocalReplayControlCases() []struct { + name string + body string + valid bool +} { + return []struct { + name string + body string + valid bool + }{ + {"linear", fixedLocalReplayBody("linear", 8), true}, + {"diamond", fixedLocalReplayBody("diamond", 8), true}, + {"loop", fixedLocalReplayBody("loop", 8), true}, + {"duplicate successor", "CMPQ AX,AX\nJEQ next\nnext:\nPADDQ X1,X0", true}, + {"interior target", "CMPQ AX,AX\nJEQ middle\nPADDQ X1,X0\nmiddle:\nPADDQ X1,X0", true}, + {"two returns", "CMPQ AX,AX\nJEQ other\nMOVQ AX,ret+8(FP)\nRET\nother:\nPADDQ X1,X0", true}, + {"unreachable read", "JMP done\nMOVQ 24(SP),AX\ndone:\nPADDQ X1,X0", true}, + {"equal private origins", "MOVQ SP,BX\nCMPQ AX,AX\nJEQ left\nADDQ $8,BX\nJMP joined\nleft:\nADDQ $8,BX\njoined:\nNOP", true}, + {"different private origins", "MOVQ SP,BX\nCMPQ AX,AX\nJEQ left\nADDQ $8,BX\nJMP joined\nleft:\nADDQ $16,BX\njoined:\nNOP", true}, + {"different address cells", "MOVQ SP,BX\nMOVQ BX,24(SP)\nCMPQ AX,AX\nJEQ joined\nMOVQ $0,24(SP)\njoined:\nNOP", false}, + {"missing predecessor bytes", "CMPQ AX,AX\nJEQ joined\nMOVQ AX,24(SP)\njoined:\nMOVQ 24(SP),AX", false}, + {"missing first iteration", "again:\nPADDQ X2,X0\nMOVOU X1,X2\nCMPQ AX,AX\nJNE again", false}, + {"lost backedge flags", "CMPQ AX,AX\nagain:\nSETNE CL\nIMULQ $3,AX\nJCS again", false}, + {"drifting private origin", "MOVQ SP,BX\nagain:\nADDQ $8,BX\nCMPQ AX,AX\nJNE again", false}, + {"one return missing result", "CMPQ AX,AX\nJEQ other\nRET\nother:\nPADDQ X1,X0", false}, + {"no reachable return", "again:\nJMP again", false}, + } +} + +func TestGoAMD64FixedLocalReplayControlFlow(t *testing.T) { + for _, row := range fixedLocalReplayControlCases() { + t.Run(row.name, func(t *testing.T) { + fn, sig := fixedLocalReplayFixture(t, row.body) + _, err := proveX86FixedLocal(fn, sig) + if (err == nil) != row.valid { + t.Fatalf("valid=%v: %v", row.valid, err) + } + if err != nil && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unexpected rejection: %v", err) + } + }) + } +} + +func TestGoAMD64FixedLocalReplayDistinctPrivateLayoutsObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, row := range fixedLocalReplayControlCases() { + if row.name != "different private origins" { + continue + } + // Both paths keep BX private, and its different offsets are dead at + // the join. Retain the original stores, vector load and ABI0 result. + source := fixedLocalReplaySource(row.body) + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, source, + "func Value(a uint64) uint64\n", target.name, target.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("complete private-layout source failed", err) + } + x86CompactExpandRequireAllDefinitions(t, ir, []string{pkg.Path + ".Value"}) + compileLLVMToObject(t, llc, target.triple, "layout.ll", "layout.o", ir) + compileLLVMToObject(t, llc, target.triple, "layout-opt.ll", "layout-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + return + } + t.Fatal("original distinct-private-layout source is missing") +} + +func TestGoAMD64FixedLocalReplayLinearBoundaries(t *testing.T) { + for _, row := range []struct { + name string + body string + linear []bool + }{ + {"straight", "NOP\nNOP\nRET", []bool{true, true, true, false}}, + {"label", "NOP\nunused:\nNOP\nRET", []bool{true, true, true, false}}, + {"jump", "JMP next\nnext:\nNOP\nRET", []bool{true, false, true, false}}, + {"duplicate edge", "JEQ next\nnext:\nNOP\nRET", []bool{true, false, true, false}}, + {"interior target", "JEQ next\nNOP\nnext:\nNOP\nRET", []bool{true, false, false, true, false}}, + {"backedge", "again:\nNOP\nNOP\nJMP again", []bool{false, true, true, false}}, + {"unreachable predecessor", "JMP done\nNOP\ndone:\nRET", []bool{true, false, false, false}}, + {"missing return", "NOP", []bool{true, false}}, + } { + t.Run(row.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$64-0\n"+row.body+"\n") + if err != nil { + t.Fatal(err) + } + fn := file.Funcs[0] + edges, instructions, err := fixedLocalEdges(fn) + if err != nil { + t.Fatal(err) + } + if got := fixedLocalLinearEdges(fn, edges, instructions); !reflect.DeepEqual(got, row.linear) { + t.Fatalf("linear edges = %v, want %v", got, row.linear) + } + }) + } +} + +func TestGoAMD64FixedLocalReplayMatchesInstructionSchedule(t *testing.T) { + for _, row := range fixedLocalReplayControlCases() { + t.Run(row.name, func(t *testing.T) { + fn, sig := fixedLocalReplayFixture(t, row.body) + edges, instructions, err := fixedLocalEdges(fn) + if err != nil { + t.Fatal(err) + } + effects, err := fixedLocalEffectPlan(fn, nil) + if err != nil { + t.Fatal(err) + } + proof := &x86FixedLocalProof{frame: fn.FrameSize} + var results [2]Func + for schedule := range results { + current := fn + current.Instrs = append([]Instr(nil), fn.Instrs...) + var linear []bool + if schedule == 1 { + linear = fixedLocalLinearEdges(fn, edges, instructions) + } + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + var layouts x86FixedLocalLayoutGraph + for residue := 0; residue < 8; residue++ { + err = replayX86FixedLocalCFG(¤t, sig, proof, residue, &budget, effects, edges, instructions, linear, nil, &layouts) + if (err == nil) != row.valid { + t.Fatalf("schedule=%d residue=%d valid=%v: %v", schedule, residue, row.valid, err) + } + if err != nil { + if !errors.Is(err, ErrProbeNeedsContext) || !reflect.DeepEqual(current.Instrs, fn.Instrs) { + t.Fatalf("schedule=%d invalid replay changed input or rejection class: %v", schedule, err) + } + break + } + } + results[schedule] = current + } + if !reflect.DeepEqual(results[0], results[1]) { + t.Fatal("linear replay changed a source instruction or its eight-residue access certificate") + } + }) + } +} + +func TestGoAMD64FixedLocalReplayAllocations(t *testing.T) { + fn, sig := fixedLocalReplayFixture(t, fixedLocalReplayBody("linear", 128)) + edges, instructions, err := fixedLocalEdges(fn) + if err != nil { + t.Fatal(err) + } + effects, err := fixedLocalEffectPlan(fn, nil) + if err != nil { + t.Fatal(err) + } + measure := func(linear []bool) float64 { + return testing.AllocsPerRun(3, func() { + current := fn + current.Instrs = append([]Instr(nil), fn.Instrs...) + proof := &x86FixedLocalProof{frame: fn.FrameSize} + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + if err := replayX86FixedLocalCFG(¤t, sig, proof, 0, &budget, effects, edges, instructions, linear, nil, nil); err != nil { + t.Fatal(err) + } + }) + } + reference := measure(nil) + blocks := measure(fixedLocalLinearEdges(fn, edges, instructions)) + if blocks >= reference/2 { + t.Fatalf("linear state copies still dominate: %.0f allocations versus %.0f for instruction scheduling", blocks, reference) + } +} + +func BenchmarkFixedLocalReplayProof(b *testing.B) { + for _, shape := range []string{"linear", "diamond", "loop"} { + for _, length := range []int{8, 128, 1024, 5000} { + b.Run(fmt.Sprintf("%s/%d", shape, length), func(b *testing.B) { + fn, sig := fixedLocalReplayFixture(b, fixedLocalReplayBody(shape, length)) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + b.Fatalf("fixture must complete before timing: %v", err) + } + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + if _, err := proveX86FixedLocal(fn, sig); err != nil { + b.Fatal(err) + } + } + }) + } + } +} diff --git a/amd64_fixed_local_result.go b/amd64_fixed_local_result.go new file mode 100644 index 000000000..9a9d0106f --- /dev/null +++ b/amd64_fixed_local_result.go @@ -0,0 +1,237 @@ +package plan9asm + +import ( + "path/filepath" + "reflect" +) + +// Fixed-local physical SP and its initialized result already have a complete +// native source/CFG proof. Revalidate that exact role instead of reinterpreting +// its private access annotations as logical AUTO bytes. A proof pointer alone +// is not permission, and this route supplies no new native/caller capability. +func (c *amd64Ctx) checkFixedLocalResultInitialization() error { + fn, sig, proof := c.fpScalarSource, c.fpSourceSig, c.fixedLocal + binding := sig.x86FixedLocal + fail := func(reason string) error { return goAMD64FrameContext("fixed local result consumer: %s", reason) } + if binding == nil || proof == nil || fn.x86FixedLocal != proof || proof.binding != binding || + c.goarch != "amd64" || c.targetTriple != binding.target || fn.FrameSize != proof.frame || c.frameSize != fn.FrameSize { + return fail("source/proof/target/frame role differs") + } + fn = fixedLocalProofFunction(fn) + if c.fixedLocalResult == nil { + if err := binding.verify(); err != nil { + return err + } + } + emitted := c.sig + emitted.Attrs = sig.Attrs // Checked against the actual complete renderer inventory below. + if !reflect.DeepEqual(emitted, sig) { + return fail("emitted logical signature differs from the original declaration") + } + expected, declared := binding.signatures[binding.symbols[fn.Sym]] + sig.x86FixedLocal = nil + if !declared || !reflect.DeepEqual(sig, expected) { + return fail("actual declaration/signature differs") + } + sig.x86FixedLocal = binding + snapshot := c.fixedLocalResult + if snapshot == nil { + var err error + snapshot, err = newX86FixedLocalResultSnapshot(binding, c.goarch) + if err != nil { + return err + } + } + if snapshot.binding != binding || snapshot.file != binding.file { + return fail("complete source snapshot belongs to a different binding") + } + original, valid := snapshot.functions[sig.Name] + if !valid || c.sig.Attrs != original.attributes || c.fpFeatureAttrs != original.inferred { + return fail("emitted attributes are not the exact complete-source renderer projection") + } + fingerprint, err := x86CallerSourceFingerprint(fn) + if err != nil || fingerprint != proof.function { + return fail("complete normalized function fingerprint differs") + } + plain := fn + plain.Instrs = append([]Instr(nil), fn.Instrs...) + for ordinal, ins := range fn.Instrs { + if len(ins.x86FixedLocalMemory) != len(proof.accesses[ordinal]) { + return fail("instruction access inventory differs") + } + for operand, access := range ins.x86FixedLocalMemory { + _, valid := c.fixedLocalMemoryAccess(ins, operand) + if access.ordinal != ordinal || !valid { + return fail("instruction access owner/shape/ordinal differs") + } + } + plain.Instrs[ordinal].x86FixedLocalMemory = nil + } + for ordinal := range proof.accesses { + if ordinal < 0 || ordinal >= len(fn.Instrs) { + return fail("access ordinal is outside the complete function") + } + } + instructions, err := x86ResultAUTOReplayInstructions(plain, c.goarch) + if err != nil { + return err + } + reference := original.function + view, err := original.graph.fixedLocalRoot(reference.Sym) + if err != nil { + return err + } + if view != nil { + actual := fn.x86DirectTailRoot + if actual == nil || view.text != actual.text || !reflect.DeepEqual(view.points, actual.points) || + !reflect.DeepEqual(view.ordinals, actual.ordinals) || !reflect.DeepEqual(view.edges, actual.edges) || + !reflect.DeepEqual(view.externalTails, actual.externalTails) { + return fail("original private root control/source coordinates differ") + } + reference = view.features() + } else if fn.x86DirectTailRoot != nil { + return fail("ordinary source acquired a private root view") + } + if reference.Sym != fn.Sym || reference.FrameSize != fn.FrameSize || + reference.ArgSize != fn.ArgSize || !reflect.DeepEqual(reference.Instrs, instructions) { + return fail("original TEXT/source instruction roles differ") + } + replayed, err := proveX86FixedLocal(plain, sig) + if err != nil { + return err + } + for ordinal, accesses := range proof.accesses { + if len(accesses) != len(replayed.x86FixedLocal.accesses[ordinal]) { + return fail("recomputed access inventory differs") + } + for operand, access := range accesses { + recomputed, found := replayed.x86FixedLocal.accesses[ordinal][operand] + access.owner, recomputed.owner = nil, nil // Compare effects, not new builder identity. + if !found || !reflect.DeepEqual(access, recomputed) { + return fail("recomputed access byte effects differ") + } + } + } + if len(proof.accesses) != len(replayed.x86FixedLocal.accesses) { + return fail("recomputed function access inventory differs") + } + if c.fixedLocalResult == nil { + return binding.verify() + } + return nil +} + +// A module-call-local index of independently parsed source, not a live source +// verification result or a new permission. Only the module's existing complete +// pre/post guards permit sharing it between function consumers. Standalone +// consumers construct their own index inside the original live guards. +type x86FixedLocalResultSnapshot struct { + binding *x86FixedLocalBinding + file [32]byte + functions map[string]x86FixedLocalResultFunction +} + +type x86FixedLocalResultFunction struct { + function Func + graph *x86DirectTailGraph // Original source/indices only; no retained expansion. + attributes string + inferred string +} + +func newX86FixedLocalResultSnapshot(binding *x86FixedLocalBinding, arch string) (*x86FixedLocalResultSnapshot, error) { + fail := func(reason string) (*x86FixedLocalResultSnapshot, error) { + return nil, goAMD64FrameContext("fixed local result consumer: %s", reason) + } + if binding == nil || binding.source == nil { + return fail("missing complete original source binding") + } + // Replay actual guarded CPP, not a mutually consistent replacement of the + // binding's private assembly/fingerprint. No caller callback runs under mu. + binding.source.mu.Lock() + if !binding.source.ready { + binding.source.mu.Unlock() + return fail("original source is closed or revoked") + } + original, err := binding.source.read(filepath.Join(binding.source.request.Directory, binding.source.request.AssemblyFile)) + var expanded string + if err == nil { + expanded, err = binding.source.preprocess(binding.source.request.AssemblyFile, original) + } + binding.source.mu.Unlock() + if err != nil || expanded != binding.assembly { + return fail("expanded assembly differs from the actual guarded source") + } + actual, err := Parse(ArchAMD64, expanded) + if err != nil { + return fail("complete original assembly cannot be parsed") + } + fingerprint, err := x86CallerSourceFingerprint(actual) + if err != nil || fingerprint != binding.file { + return fail("complete original assembly fingerprint differs") + } + functions, valid := fixedLocalResultFunctions(actual, binding, arch) + if !valid { + return fail("complete source feature inventory is missing or ambiguous") + } + return &x86FixedLocalResultSnapshot{binding: binding, file: fingerprint, functions: functions}, nil +} + +// Reproduce the renderer's complete source-order feature allocation once. The +// reference spelling alone is not proof, and supplied attributes never pass +// through the inference registry or become a removable source difference. +func fixedLocalResultFunctions(file *File, binding *x86FixedLocalBinding, arch string) (map[string]x86FixedLocalResultFunction, bool) { + if len(binding.leaves) != 0 { + signatures, err := cloneFixedLocalSigs(binding.signatures) + if err != nil { + return nil, false + } + for name, sig := range signatures { + if binding.frames[name] { + sig.x86FixedLocal = binding + signatures[name] = sig + } + } + options := Options{Goarch: arch, Sigs: signatures, + ResolveSym: func(symbol string) string { return binding.symbols[symbol] }} + graph, err := newX86FixedLocalLeafGraph(file, options, binding) + if err != nil { + return nil, false + } + copy := *file + copy.x86DirectTails = graph + file = © + } + registry := newFeatureAttrRegistry() + functions := make(map[string]x86FixedLocalResultFunction, len(file.Funcs)) + for _, fn := range file.Funcs { + if file.x86DirectTailHelper(fn.Sym) { + continue + } + resolved, found := binding.symbols[fn.Sym] + sig, declared := binding.signatures[resolved] + _, duplicate := functions[resolved] + if !found || !declared || duplicate || len(fn.X86RawText) != 0 || amd64IsRelocationAnchor(fn, sig) { + return nil, false + } + features := fn + view, err := file.x86DirectTails.fixedLocalRoot(fn.Sym) + if err != nil { + return nil, false + } + if view != nil { + features = view.features() + } + derived := "" + if sig.Attrs == "" { + derived = registry.ref(inferFuncTargetFeaturesForGOARCH(file.Arch, arch, features)) + } + attributes := sig.Attrs + if attributes == "" { + attributes = derived + } + functions[resolved] = x86FixedLocalResultFunction{ + function: fn, graph: file.x86DirectTails, attributes: attributes, inferred: derived, + } + } + return functions, true +} diff --git a/amd64_fixed_local_result_selection_test.go b/amd64_fixed_local_result_selection_test.go new file mode 100644 index 000000000..35787d4f7 --- /dev/null +++ b/amd64_fixed_local_result_selection_test.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestGoAMD64FixedLocalResultCompleteMembersAndSelectionBoundary(t *testing.T) { + pkg, request, _, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + first, _, _ := x86PrivateAlignedFrameAuditFixture(t, "PADDQ 16(SP),X0") + second, _, _ := x86PrivateAlignedFrameAuditFixture(t, "AESENC 16(SP),X0") + second = strings.Replace(second, "·Value", "·Other", 1) + second = strings.Replace(second, "#include \"textflag.h\"\n", "", 1) + assembly := first + "\n" + second + declarations := "package framefixture\nfunc Value(a, b, c, d uint64) uint64\nfunc Other(a, b, c, d uint64) uint64\n" + for name, data := range map[string]string{"value.go": declarations, request.AssemblyFile: assembly} { + if err := os.WriteFile(filepath.Join(request.Directory, name), []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "value.go", declarations, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + typed, err := (&types.Config{Sizes: sizes}).Check(pkg.Path, fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + pkg.Types, pkg.Syntax = typed, []*ast.File{syntax} + for _, name := range []string{"Value", "Other"} { + sig, err := DeriveGoFuncABI0Signature(pkg.Path+"."+name, typed.Scope().Lookup(name).(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + options.Sigs[sig.Name] = sig + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("complete two-member actual original object/declaration precondition", err) + } + for _, api := range []string{"text", "context", "go-module"} { + t.Run(api, func(t *testing.T) { + var ir string + if api == "text" { + ir, err = Translate(file, bound) + } else { + context := llvm.NewContext() + defer context.Dispose() + var module llvm.Module + if api == "context" { + module, err = TranslateModuleInContext(context, file, bound) + } else { + result, translateErr := translateGoModuleInContext(context, pkg, []byte(assembly), GoModuleOptions{ + GOOS: "linux", GOARCH: "amd64", TargetTriple: bound.TargetTriple, ResolveSym: bound.ResolveSym, + ManualSig: func(name string) (FuncSig, bool) { sig, found := bound.Sigs[name]; return sig, found }, + }) + err = translateErr + if result != nil { + module = result.Module + } + } + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + } + if err != nil || !strings.Contains(ir, pkg.Path+".Value") || !strings.Contains(ir, pkg.Path+".Other") { + t.Fatal("complete original member/renderer projection failed", err) + } + }) + } + context := llvm.NewContext() + defer context.Dispose() + queries := 0 + result, err := translateGoModuleInContext(context, pkg, []byte(assembly), GoModuleOptions{ + GOOS: "linux", GOARCH: "amd64", TargetTriple: bound.TargetTriple, ResolveSym: bound.ResolveSym, + ManualSig: func(name string) (FuncSig, bool) { sig, found := bound.Sigs[name]; return sig, found }, + KeepFunc: func(_, name string) bool { queries++; return name == pkg.Path+".Other" }, + }) + if result != nil { + if !result.Module.IsNil() { + result.Module.Dispose() + } + t.Fatal("partial selection retained a whole-file native capability") + } + if !errors.Is(err, ErrProbeNeedsContext) || queries != 2 { + t.Fatal("unsupported partial native selection was admitted or KeepFunc was repeated", queries, err) + } +} diff --git a/amd64_fixed_local_result_test.go b/amd64_fixed_local_result_test.go new file mode 100644 index 000000000..3a11e8160 --- /dev/null +++ b/amd64_fixed_local_result_test.go @@ -0,0 +1,160 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalResultFeatureAttrs(file *File, binding *x86FixedLocalBinding, name, arch string) (string, string, bool) { + functions, valid := fixedLocalResultFunctions(file, binding, arch) + function, found := functions[name] + return function.attributes, function.inferred, valid && found +} + +func TestX86FixedLocalFeatureProjectionUsesCompleteSourceOrder(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·First(SB),4,$0\nAESENC X0,X1\nRET\nTEXT ·Second(SB),4,$0\nPSHUFB X0,X1\nRET") + if err != nil { + t.Fatal(err) + } + for _, arch := range []string{"amd64", "386"} { + binding := &x86FixedLocalBinding{symbols: map[string]string{"·First": "First", "·Second": "Second"}, + signatures: map[string]FuncSig{"First": {Name: "First"}, "Second": {Name: "Second"}}} + for index, name := range []string{"First", "Second"} { + attrs, inferred, valid := fixedLocalResultFeatureAttrs(file, binding, name, arch) + if !valid || attrs != fmt.Sprintf("#%d", featureAttrBase+index) || inferred != attrs { + t.Fatal("complete renderer reference/order differs", arch, name, attrs, inferred, valid) + } + } + binding.signatures["First"] = FuncSig{Name: "First", Attrs: "nounwind"} + attrs, inferred, valid := fixedLocalResultFeatureAttrs(file, binding, "First", arch) + if !valid || attrs != "nounwind" || inferred != "" { + t.Fatal("supplied attributes became removable inferred metadata") + } + attrs, inferred, valid = fixedLocalResultFeatureAttrs(file, binding, "Second", arch) + if !valid || attrs != fmt.Sprintf("#%d", featureAttrBase) || inferred != attrs { + t.Fatal("supplied attributes incorrectly consumed a renderer reference") + } + delete(binding.symbols, "·First") + if _, _, valid := fixedLocalResultFeatureAttrs(file, binding, "Second", arch); valid { + t.Fatal("incomplete unrelated source inventory accepted") + } + } +} + +func TestGoAMD64FixedLocalResultConsumerExactProofHandoff(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", bound) + if err != nil { + t.Fatal(err) + } + fn, sig := normalized.Funcs[0], bound.Sigs[pkg.Path+".Value"] + attributes, inferred, valid := fixedLocalResultFeatureAttrs(file, sig.x86FixedLocal, sig.Name, "amd64") + if !valid { + t.Fatal("actual complete renderer inventory precondition") + } + snapshot, err := newX86FixedLocalResultSnapshot(sig.x86FixedLocal, "amd64") + if err != nil || snapshot == nil || len(snapshot.functions) != 1 { + t.Fatal("independent call-local original source index", err) + } + create := func(fn Func) *amd64Ctx { + var output strings.Builder + lower := newX86Ctx(&output, fn, sig, bound.ResolveSym, bound.Sigs, "amd64", bound.TargetTriple, false) + lower.fpFeatureAttrs, lower.sig.Attrs = inferred, attributes + return lower + } + if err := create(fn).checkLoweredScalarFPResultInitialization(); err != nil { + t.Fatal("exact existing fixed-local result proof was stolen by logical AUTO replay", err) + } + scoped := create(fn) + scoped.fixedLocalResult = snapshot + if err := scoped.checkLoweredScalarFPResultInitialization(); err != nil { + t.Fatal("call-local original source index changed result proof", err) + } + if err := checkX86ScalarFPResultInitialization(fn, sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("private source query gained physical-frame permission", err) + } + for name, mutate := range map[string]func(*amd64Ctx){ + "wrong target": func(c *amd64Ctx) { c.targetTriple = "x86_64-pc-windows-msvc" }, + "wrong mode": func(c *amd64Ctx) { c.goarch = "386" }, + "missing proof": func(c *amd64Ctx) { c.fixedLocal = nil }, + "changed declaration": func(c *amd64Ctx) { c.fpSourceSig.Ret = I32 }, + "changed emitted type": func(c *amd64Ctx) { c.sig.Ret = I32 }, + "changed emitted attrs": func(c *amd64Ctx) { c.sig.Attrs = "readnone" }, + "changed inferred attrs": func(c *amd64Ctx) { c.fpFeatureAttrs = "#999" }, + "changed supplied attrs": func(c *amd64Ctx) { c.fpSourceSig.Attrs = "nounwind" }, + "changed frame": func(c *amd64Ctx) { c.fpScalarSource.FrameSize += 8 }, + "changed emitter frame": func(c *amd64Ctx) { c.frameSize += 8 }, + "changed raw role": func(c *amd64Ctx) { c.fpScalarSource.Instrs[1].x86Encoded = true }, + "unknown literal role": func(c *amd64Ctx) { c.fpScalarSource.Instrs[1].x86SDKLiteral = &x86SDKLiteralCertificate{} }, + "foreign snapshot binding": func(c *amd64Ctx) { + copy, foreign := *snapshot, *snapshot.binding + copy.binding = &foreign + c.fixedLocalResult = © + }, + "foreign snapshot file": func(c *amd64Ctx) { + copy := *snapshot + copy.file[0] ^= 1 + c.fixedLocalResult = © + }, + "changed complete source": func(c *amd64Ctx) { + last := len(c.fpScalarSource.Instrs) - 1 + c.fpScalarSource.Instrs[last] = Instr{Op: "NOP", Raw: "NOP"} + }, + "missing access": func(c *amd64Ctx) { + for index := range c.fpScalarSource.Instrs { + if len(c.fpScalarSource.Instrs[index].x86FixedLocalMemory) != 0 { + c.fpScalarSource.Instrs[index].x86FixedLocalMemory = nil + return + } + } + t.Fatal("actual access missing") + }, + "changed access ordinal": func(c *amd64Ctx) { + for index := range c.fpScalarSource.Instrs { + ins := &c.fpScalarSource.Instrs[index] + if len(ins.x86FixedLocalMemory) != 0 { + ins.x86FixedLocalMemory = cloneX86ReplayFixedLocalMemory(ins.x86FixedLocalMemory) + for operand, access := range ins.x86FixedLocalMemory { + access.ordinal++ + ins.x86FixedLocalMemory[operand] = access + return + } + } + } + t.Fatal("actual access missing") + }, + } { + t.Run(name, func(t *testing.T) { + for _, scoped := range []bool{false, true} { + copy := fn + copy.Instrs = append([]Instr(nil), fn.Instrs...) + lower := create(copy) + if scoped { + lower.fixedLocalResult = snapshot + } + mutate(lower) + if err := lower.checkFixedLocalResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed native/source/private role borrowed original result proof", scoped, err) + } + } + }) + } + if err := source.Verify(ctx); err != nil { + t.Fatal("negative consumers changed actual source owner", err) + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + if err := create(fn).checkFixedLocalResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("closed actual frame owner retained result proof", err) + } +} diff --git a/amd64_fixed_local_runtime_test.go b/amd64_fixed_local_runtime_test.go new file mode 100644 index 000000000..f02cf83d7 --- /dev/null +++ b/amd64_fixed_local_runtime_test.go @@ -0,0 +1,267 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func fixedLocalCompleteFixture(t *testing.T, target string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options, string) { + t.Helper() + _, request := goAMD64FrameSourceFixture(t) + request.Target = target + var declarations, assembly strings.Builder + declarations.WriteString("package framefixture\n") + for index, operation := range fixedLocalNativeOperations { + fmt.Fprintf(&declarations, "func Value%d(a, b, c, d uint64) uint64\n", index) + source, _, _ := x86PrivateAlignedFrameAuditFixture(t, operation) + assembly.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + for name, content := range map[string]string{"value.go": declarations.String(), "value_amd64.s": assembly.String()} { + if err := os.WriteFile(filepath.Join(request.Directory, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "value.go", declarations.String(), 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + typed, err := (&types.Config{Sizes: sizes}).Check("example.invalid/framefixture", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for _, name := range typed.Scope().Names() { + sig, err := DeriveGoFuncABI0Signature(typed.Path()+"."+name, typed.Scope().Lookup(name).(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + sigs[sig.Name] = sig + } + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", typed.Path()+".", 1) } + triple := "x86_64-unknown-linux-gnu" + if strings.HasPrefix(target, "darwin/") { + triple = "x86_64-apple-darwin" + } + if strings.HasPrefix(target, "windows/") { + triple = "x86_64-pc-windows-msvc" + } + return GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{syntax}}, request, file, + Options{Goarch: "amd64", TargetTriple: triple, ResolveSym: resolve, Sigs: sigs}, declarations.String() +} + +// Integer-only independent expected values; the comparison inputs remain raw +// IEEE bits, including signed zero, infinities, NaNs and subnormals. +const fixedLocalGoMath = ` +func equalBits(a,b uint64,bits uint, daz bool) bool { + exp,frac,sign:=uint64(0x7ff0000000000000),uint64(0xfffffffffffff),uint64(1)<<63 + if bits==32 { exp,frac,sign=0x7f800000,0x7fffff,1<<31; a=uint64(uint32(a)); b=uint64(uint32(b)) } + if a&exp==exp && a&frac!=0 || b&exp==exp && b&frac!=0 { return false } + if daz && a&exp==0 { a &= sign }; if daz && b&exp==0 { b &= sign } + return a==b || a&^sign==0 && b&^sign==0 +} +func expected(op int,a,c uint64) uint64 { + if op<4 { + width:=uint(8)<>shift)+(c>>shift))&mask<>shift)),int64(int16(a>>(shift+16))) + u,v:=int64(int16(c>>shift)),int64(int16(c>>(shift+16))) + result|=uint64(uint32(x*u+y*v))<>shift,c>>shift,32,false) { result|=uint64(0xffffffff)< +#include +#include +#include +` + declarations + ` +static __attribute__((noinline)) uint64_t readword(const volatile uint64_t *p) { return *p; } +static __attribute__((noinline)) uint64_t reference(int op,const uint64_t *data) { + _Alignas(16) uint64_t input[2]={data[0],data[1]}; + __m128i value=_mm_set_epi64x((long long)data[3],(long long)data[2]); + switch(op) { +` + opcodes.String() + ` + } + return (uint64_t)_mm_cvtsi128_si64(value); +} +int main(void) { + const uint64_t values[]={0,1,UINT64_MAX,UINT64_C(0x8000800080008000),UINT64_C(0x7fff7fff7fff7fff),UINT64_C(0x3f8000003f800000),UINT64_C(0x7fc012347f801234),UINT64_C(0x3ff0000000000000),UINT64_C(0x8000000000000000),UINT64_C(0x123456789abcdef0)}; + unsigned saved=_mm_getcsr(); + /* Controlled first-party storage realizes all four word-aligned source + residues. This is an independent exact-SSE witness of the refused alias + fragment, not a claim that these are the actual Go runtime entry SPs. */ + const uint64_t aliasWant[4][2]={{6,10},{7,8},{7,10},{7,10}}; + for(int residue=0;residue<4;residue++) { + _Alignas(32) unsigned char memory[256]={0}; + unsigned char *sp=memory+32+residue*8; + unsigned char *aligned=(unsigned char*)(((uintptr_t)sp&~(uintptr_t)31)+32); + uint64_t input[2]={2,3}, other[2]={5,7}, one=1, observed[2]; + memcpy(aligned+16,input,16); memcpy(aligned+48,other,16); + memcpy(sp+48,&one,8); + __m128i vector; memcpy(&vector,aligned+48,16); + __asm__ volatile("paddq %1,%0" : "+x"(vector) : "m"(*(const unsigned char (*)[16])(aligned+16)) : "memory"); + memcpy(observed,&vector,16); + if(observed[0]!=aliasWant[residue][0] || observed[1]!=aliasWant[residue][1]) return 3; + } + for(int seed=0;seed<128;seed++) { + volatile uint64_t guarded[12]={11,17,29,53,values[seed%10],values[(seed+1)%10],values[(seed*3)%10],values[(seed*7+1)%10],0xaabbccdd,0x12345678,0x87654321,UINT64_MAX}; + uint64_t data[4],before[12]; + for(int i=0;i<12;i++) before[i]=readword(&guarded[i]); + for(int i=0;i<4;i++) data[i]=readword(&guarded[i+4]); + for(int op=0;op<7;op++) for(int rc=0;rc<4;rc++) for(int ctl=0;ctl<4;ctl++) for(int sticky=0;sticky<3;sticky++) { + unsigned start=0x1f80u|(unsigned)rc<<13|(ctl&1?0x40u:0)|(ctl&2?0x8000u:0)|(sticky==1?1u:sticky==2?0x3fu:0); + _mm_setcsr(start); uint64_t want=reference(op,data); unsigned wantCSR=_mm_getcsr(); + _mm_setcsr(start); uint64_t got=0; + switch(op) { +` + cases.String() + ` + } + unsigned gotCSR=_mm_getcsr(); + if(got!=want || gotCSR!=wantCSR) { fprintf(stderr,"op=%d seed=%d RC=%d ctl=%d sticky=%d got=%llx/%x want=%llx/%x\n",op,seed,rc,ctl,sticky,(unsigned long long)got,gotCSR,(unsigned long long)want,wantCSR); return 1; } + for(int i=0;i<12;i++) if(readword(&guarded[i])!=before[i]) return 2; + } + } + _mm_setcsr(saved); return 0; +} +` +} diff --git a/amd64_fixed_local_scalar_lane.go b/amd64_fixed_local_scalar_lane.go new file mode 100644 index 000000000..e7bcd7aa2 --- /dev/null +++ b/amd64_fixed_local_scalar_lane.go @@ -0,0 +1,77 @@ +package plan9asm + +// Reuse the complete scalar-lane grammar, including Go's GP-number spellings. +// This owner binds only low physical X registers and ordinary memory. Logical +// FP fields and EVEX high banks retain their independent entry contracts. +func fixedLocalScalarLaneProperties(ins Instr) (x86ScalarLaneForm, bool, error) { + form, known, err := x86ScalarLaneProperties(ins, "amd64") + if !known { + return form, false, nil + } + valid := err == nil + if valid && form.memory >= 0 { + kind := ins.Args[form.memory].Kind + valid = kind == OpMem || kind == OpSym + } + if valid { + vectors := [2]int{form.source, -1} + if form.insert { + vectors = [2]int{form.base, form.destination} + } + for _, index := range vectors { + if index >= 0 { + register, physical := amd64ParseXReg(ins.Args[index].Reg) + valid = valid && physical && register < 16 + } + } + } + if !valid { + return form, true, goAMD64FrameContext("fixed local scalar-lane form lacks a bound physical role: %s", ins.Op) + } + return form, true, nil +} + +func fixedLocalScalarLaneStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + form, known, err := fixedLocalScalarLaneProperties(*ins) + if !known || err != nil { + return known, err + } + width := int64(form.bits / 8) + selected := x86CallerVectorByteMask(width) + if form.insert { + if form.memory >= 0 { + // Scalar m8/m16/m32/m64 accesses stay exact and unaligned. Their + // native volatile emitter does not grant future pointee facts. + if _, _, err := fixedLocalReadMemory(ins, form.memory, state, proof, residue, width, 1); err != nil { + return true, err + } + } else if !fixedLocalNumericRead(Operand{Kind: OpReg, Reg: form.transferReg}, width, state) { + return true, goAMD64FrameContext("fixed local scalar insert reads undefined or address bytes") + } + base := state.vectors[x86CallerVectorRoot(ins.Args[form.base].Reg)] + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + state.vectors[destination] = x86ScalarLaneInsertKnown(form, base, selected, state.vectors[destination]) + return true, nil + } + source := state.vectors[x86CallerVectorRoot(ins.Args[form.source].Reg)] + if source>>uint(form.offset)&selected != selected { + return true, goAMD64FrameContext("fixed local scalar extract reads undefined selected bytes") + } + if form.memory >= 0 { + offset, role, err := fixedLocalMemoryAccess(ins, form.memory, state, proof, residue, width, 1, true, false) + if err != nil { + return true, err + } + if role == x86FixedLocalPrivateMemory { + fixedLocalNumericalStore(state, offset, width, true) + } + return true, nil + } + // GP extraction reuses the ordinary dword/qword writer, including zero + // extension and revocation of constants, booleans and address identities. + effect := x86TypedByteEffect{operands: []x86TypedByteOperand{{ + index: form.destination, operand: Operand{Kind: OpReg, Reg: form.transferReg}, + width: int64(form.gpBits / 8), write: true, + }}} + return true, fixedLocalTypedByteStep(ins, state, FuncSig{}, proof, residue, effect) +} diff --git a/amd64_fixed_local_scalar_lane_test.go b/amd64_fixed_local_scalar_lane_test.go new file mode 100644 index 000000000..bdb37fad4 --- /dev/null +++ b/amd64_fixed_local_scalar_lane_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalScalarLaneCases() []string { + var rows []string + for _, prefix := range []string{"", "V"} { + for _, suffix := range []string{"B", "W", "D", "Q"} { + for _, selector := range []int{0, 255} { + for _, transfer := range []string{"AX", "65(BP)", "1(SI)"} { + insert := fmt.Sprintf("%sPINSR%s $%d,%s,X1", prefix, suffix, selector, transfer) + if prefix != "" { + insert += ",X2" + } + rows = append(rows, insert, fmt.Sprintf("%sPEXTR%s $%d,X1,%s", prefix, suffix, selector, transfer)) + } + if prefix != "" { + rows = append(rows, fmt.Sprintf("VPINSR%s $%d,AX,X1,X1", suffix, selector)) + } else if suffix != "W" { + // These Go Ymm spellings encode GP registers, not MMX reads. + rows = append(rows, fmt.Sprintf("PINSR%s $%d,M0,X1", suffix, selector), + fmt.Sprintf("PEXTR%s $%d,X1,M0", suffix, selector)) + } + } + if prefix != "" { + rows = append(rows, fmt.Sprintf("VPEXTR%s $-128,X1,AX", suffix), fmt.Sprintf("VPEXTR%s $-1,X1,65(BP)", suffix)) + } + // Only the inserted lane is observed. Unrelated old X7 bytes + // remain unknown through lowering as well as the direct proof. + partial := fmt.Sprintf("%sPINSR%s $255,AX,X7", prefix, suffix) + if prefix != "" { + partial += ",X7" + } + rows = append(rows, partial+fmt.Sprintf("\n%sPEXTR%s $255,X7,AX\nMOVQ AX,128(BP)", prefix, suffix)) + } + } + return rows +} + +func TestGoAMD64FixedLocalScalarLaneCompleteFamilies(t *testing.T) { + rows := fixedLocalScalarLaneCases() + if len(rows) != 132 { + t.Fatal("scalar-lane source/selector/alias inventory changed", len(rows)) + } + var original strings.Builder + for index, row := range rows { + original.WriteString(strings.Replace(fixedLocalVEXAssembly(row), "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) + for _, row := range rows { + t.Run(row, func(t *testing.T) { + fn, sig, _ := fixedLocalVEXFixture(t, row) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("original Go scalar-lane source lacks fixed-local support", err) + } + if !strings.Contains(row, "\n") && strings.Contains(row, "(BP)") { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$256-0\n"+row+"\nRET\n") + if err != nil || !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") { + t.Fatal("isolated physical scalar-lane access did not request frame capture", err) + } + } + }) + } +} + +func TestGoAMD64FixedLocalScalarLaneInsertDefinedBytes(t *testing.T) { + for _, prefix := range []string{"", "V"} { + for size, suffix := range []string{"B", "W", "D", "Q"} { + width := 1 << uint(size) + for _, destination := range []string{"X1", "X2"} { + if prefix == "" && destination != "X1" { + continue + } + t.Run(prefix+suffix+"/"+destination, func(t *testing.T) { + for immediate := 0; immediate < 256; immediate++ { + row := fmt.Sprintf("%sPINSR%s $%d,AX,X1", prefix, suffix, immediate) + if prefix != "" { + row += "," + destination + } + ins := x86ScalarLaneInstruction(t, row) + root := x86CallerVectorRoot(Reg(destination)) + offset := (immediate % (16 / width)) * width + selected := (uint64(1)<= 0 { + state.known[AX] &^= 1 << uint(missing) + } + input := state.known[AX] + old := state.vectors[root] + err := fixedLocalStep(&ins, &state, FuncSig{}, nil, 0) + if missing >= 0 && missing < width { + if !errors.Is(err, ErrProbeNeedsContext) || state.vectors[root] != old { + t.Fatal("undefined GP source acquired vector bytes", row, missing, err) + } + continue + } + want := uint64(0x5555)&^selected | selected + if prefix == "" { + want |= old &^ 0xffff + } else { + want |= ^uint64(0xffff) + } + if err != nil || state.vectors[root] != want || state.flags != x86ByteFlagsAll || state.known[AX] != input { + t.Fatal("insert lost preserved unknown lanes, upper aliases or FLAGS", row, missing, err, state) + } + if destination == "X2" && state.vectors["Z1"] != 0x1000000000005555 { + t.Fatal("insert changed its separate base register", row) + } + } + } + }) + } + } + } +} + +func TestGoAMD64FixedLocalScalarLaneExtractSelectedBytes(t *testing.T) { + for _, prefix := range []string{"", "V"} { + for size, suffix := range []string{"B", "W", "D", "Q"} { + width := 1 << uint(size) + t.Run(prefix+suffix, func(t *testing.T) { + for immediate := 0; immediate < 256; immediate++ { + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%sPEXTR%s $%d,X1,AX", prefix, suffix, immediate)) + offset := (immediate % (16 / width)) * width + selected := (uint64(1)<= 0 { + state.vectors["Z1"] &^= 1 << uint(missing) + } + before := state.vectors["Z1"] + err := fixedLocalStep(&ins, &state, FuncSig{}, nil, 0) + if missing >= offset && missing < offset+width { + if !errors.Is(err, ErrProbeNeedsContext) || state.known[AX] != 0 { + t.Fatal("undefined selected lane acquired GP result", ins, missing, err) + } + continue + } + _, constant := state.constants[AX] + zeroBytes := uint8(0xf0) + if width == 8 { + zeroBytes = 0 + } + if err != nil || state.known[AX] != 255 || constant || state.booleans[AX] != zeroBytes || state.vectors["Z1"] != before || state.flags != x86ByteFlagsAll { + t.Fatal("extract read unselected bytes or changed unrelated state", ins, missing, err, state) + } + } + } + }) + } + } +} + +func TestGoAMD64FixedLocalScalarLaneExactMemory(t *testing.T) { + for _, prefix := range []string{"", "V"} { + for size, suffix := range []string{"B", "W", "D", "Q"} { + width := 1 << uint(size) + for _, insert := range []bool{false, true} { + row, memory := prefix+"PEXTR"+suffix+" $255,X1,(BP)", 2 + if insert { + row, memory = prefix+"PINSR"+suffix+" $255,(BP),X1", 1 + if prefix != "" { + row += ",X2" + } + } + t.Run(row, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, row) + native := x86FixedLocalState{ + opaque: map[Reg]x86FixedLocalOrigins{BP: fixedLocalOrigin(x86FixedLocalOrigin{})}, vectors: map[Reg]uint64{"Z1": 0xffff}, + bytes: newFixedLocalByteSet(32), flags: x86ByteFlagsAll, + } + proof := &x86FixedLocalProof{frame: 32} + if err := fixedLocalStep(&ins, &native, FuncSig{}, proof, 0); err != nil { + t.Fatal("ordinary native scalar access was rejected", err) + } + access := ins.x86FixedLocalMemory[memory] + if access.owner != proof || access.width != int64(width) || access.write == insert || !access.nativeInput || native.flags != x86ByteFlagsAll { + t.Fatal("native scalar role became private extent or wrong width", access) + } + for index := int64(0); index < 32; index++ { + if native.bytes.has(index) { + t.Fatal("native access acquired private initialized bytes") + } + } + for _, offset := range []int{1, 32 - width, 33 - width} { + for missing := -1; missing < 32; missing++ { + ins := x86ScalarLaneInstruction(t, row) + state := x86FixedLocalState{ + pointers: map[Reg]int64{BP: int64(offset)}, vectors: map[Reg]uint64{"Z1": 0xffff}, + bytes: newFixedLocalByteSet(32), booleanBytes: newFixedLocalByteSet(32), flags: x86ByteFlagsAll, + } + for index := 0; index < 32; index++ { + state.bytes.set(int64(index), index != missing) + } + proof := &x86FixedLocalProof{frame: 32} + err := fixedLocalStep(&ins, &state, FuncSig{}, proof, 0) + wantPass := offset+width <= 32 && (!insert || missing < offset || missing >= offset+width) + if (err == nil) != wantPass || !wantPass && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("exact native tuple/initialization mismatch", offset, missing, err) + } + if wantPass { + access := ins.x86FixedLocalMemory[memory] + if access.owner != proof || access.width != int64(width) || access.write == insert || access.nativeInput || state.flags != x86ByteFlagsAll { + t.Fatal("scalar memory certificate widened or changed FLAGS", access) + } + } + if missing >= 0 && state.bytes.has(int64(missing)) != (wantPass && !insert && missing >= offset && missing < offset+width) { + t.Fatal("unrelated private byte initialized", offset, missing, err) + } + } + } + }) + } + } + } +} + +func TestGoAMD64FixedLocalScalarLaneBoundaries(t *testing.T) { + for _, row := range []string{ + "PINSRB $0,SI,X1", "PINSRQ $0,BP,X1", "PINSRB $0,R15,X1", + "PINSRB $0,(AX),X1", "PINSRQ $0,256(BP),X1", "PEXTRQ $0,X1,256(BP)", + "PINSRB $0,a+8(FP),X1", "PEXTRB $0,X1,a+8(FP)", "PEXTRB $0,X1,(AX)", + "PEXTRB $0,X7,AX", "PINSRB $0,AX,X7\nPEXTRB $1,X7,AX", + "PINSRB $0,AX,X7\nMOVOU X7,128(BP)", "PEXTRQ $0,X1,SP", "PEXTRD $0,X1,BP", + "PINSRW $0,M0,X1", "PEXTRW $0,X1,M0", "VPINSRB $0,M0,X1,X2", + "VPINSRB $0,AX,X1,X16", "VPEXTRB $0,X16,AX", "VPINSRD $0,AX,X1,K1,X2", + } { + t.Run(row, func(t *testing.T) { + fn, sig, source := fixedLocalVEXFixture(t, row) + if row == "PINSRB $0,(AX),X1" || row == "PEXTRB $0,X1,(AX)" { + operand, write := 1, false + emitted := "load volatile i8" + if row == "PEXTRB $0,X1,(AX)" { + operand, write = 2, true + emitted = "store volatile i8" + } + fixedLocalRequireNumericControl(t, fn, sig, source, "func Value(p *uint64,a uint64) uint64", + row, operand, 1, write, emitted) + return + } + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound scalar-lane form acquired frame permissions", err) + } + }) + } +} + +func TestGoAMD64FixedLocalScalarLaneObjects(t *testing.T) { + var assembly, declarations strings.Builder + rows := fixedLocalScalarLaneCases() + for index, row := range rows { + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(fixedLocalVEXAssembly(row), "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p *uint64, a uint64) uint64\n", name) + } + fixedLocalCompleteVectorObjects(t, assembly.String(), declarations.String(), len(rows)) +} diff --git a/amd64_fixed_local_shift_test.go b/amd64_fixed_local_shift_test.go new file mode 100644 index 000000000..3bf456fd7 --- /dev/null +++ b/amd64_fixed_local_shift_test.go @@ -0,0 +1,203 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func fixedLocalLegacyShiftNames() []string { + return strings.Fields("PSLLW PSLLL PSLLQ PSRLW PSRLL PSRLQ PSRAW PSRAL PSLLO PSLLDQ PSRLO PSRLDQ") +} + +func TestGoAMD64FixedLocalShiftOfficialRows(t *testing.T) { + catalog, err := goasmrules.LoadX86LegacyRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + uniform := []goasmrules.X86LegacyOperand{ + {RowIndex: 0, Classes: []string{"Ymm", "Ymr"}, ZCase: "Zm_r_xm", ZOffset: 1, UnmatchedOffset: 0}, + {RowIndex: 1, Classes: []string{"Yi8", "Ymr"}, ZCase: "Zibo_m_xm", ZOffset: 2, UnmatchedOffset: 1}, + {RowIndex: 2, Classes: []string{"Yxm", "Yxr"}, ZCase: "Zm_r_xm", ZOffset: 2, UnmatchedOffset: 3}, + {RowIndex: 3, Classes: []string{"Yi8", "Yxr"}, ZCase: "Zibo_m_xm", ZOffset: 3, UnmatchedOffset: 5}, + } + byteRows := []goasmrules.X86LegacyOperand{ + {RowIndex: 0, Classes: []string{"Yi8", "Yxr"}, ZCase: "Zibo_m", ZOffset: 2, UnmatchedOffset: 0}, + } + seen := make(map[string]bool) + for _, opcode := range catalog.Optab { + if !strings.HasPrefix(opcode.Opcode, "PSLL") && !strings.HasPrefix(opcode.Opcode, "PSRL") && !strings.HasPrefix(opcode.Opcode, "PSRA") { + continue + } + want, table, prefix := uniform, "yps", "Py3" + if opcode.Opcode == "PSLLO" || opcode.Opcode == "PSRLO" { + want, table, prefix = byteRows, "ypsdq", "Pq" + } + if seen[opcode.Opcode] || opcode.Ytab != table || opcode.Prefix != prefix || !reflect.DeepEqual(opcode.Rows, want) { + t.Fatalf("unmodeled original packed-shift rows: %+v", opcode) + } + seen[opcode.Opcode] = true + } + if len(seen) != 10 { + t.Fatal("original packed-shift denominator changed", len(seen)) + } + for _, pair := range [][2]string{{"PSLLO", "PSLLDQ"}, {"PSRLO", "PSRLDQ"}} { + code, pcs := x86NumericalBoundaryGoBytes(t, "amd64", pair[0]+" $7,X1\n"+pair[1]+" $7,X1\nRET") + if len(pcs) != 3 || !bytes.Equal(code[pcs[0]:pcs[1]], code[pcs[1]:pcs[2]]) { + t.Fatal("Go byte-shift alias no longer selects identical original bytes", pair) + } + } +} + +func fixedLocalLegacyShiftCases() []string { + var operations []string + for index, name := range fixedLocalLegacyShiftNames() { + for _, count := range []int{-128, 0, 1, 15, 31, 63, 127} { + operations = append(operations, fmt.Sprintf("%s $%d,X2", name, count)) + } + if index < 8 { + for _, count := range []string{"X1", "X2", "64(BP)", "(SI)"} { + operations = append(operations, name+" "+count+",X2") + } + } + } + return operations +} + +func TestGoAMD64FixedLocalShiftCompleteFamilies(t *testing.T) { + operations := fixedLocalLegacyShiftCases() + if len(operations) != 116 { + t.Fatal("complete legacy packed-shift inventory changed", len(operations)) + } + var assembly strings.Builder + for index, operation := range operations { + assembly.WriteString(strings.Replace(fixedLocalVEXAssembly(operation), "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + for _, operation := range operations { + t.Run(operation, func(t *testing.T) { + fn, sig, _ := fixedLocalVEXFixture(t, operation) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("original Go accepted packed-shift source but its frame effect is missing", err) + } + }) + } +} + +func TestGoAMD64FixedLocalShiftImmediateDomainAndAliases(t *testing.T) { + for index, name := range fixedLocalLegacyShiftNames() { + t.Run(name, func(t *testing.T) { + for count := -128; count <= 127; count++ { + ins := x86YMMReadSource(t, fmt.Sprintf("%s $%d,X2", name, count)) + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z2": 0xffff | uint64(1)<<60, "Z7": 123}, flags: x86ByteFlagsAll} + // Oversized logical/byte shifts do not consume data bits. + if index != 6 && index != 7 && uint8(count) >= 128 { + state.vectors["Z2"] = uint64(1) << 60 + } + if err := fixedLocalVectorStep(&ins, &state, nil, 0); err != nil { + t.Fatal("valid immediate domain or constant-zero result rejected", count, err) + } + if state.vectors["Z2"] != 0xffff|uint64(1)<<60 || state.vectors["Z7"] != 123 || state.flags != x86ByteFlagsAll { + t.Fatal("legacy shift changed upper aliases, unrelated storage or FLAGS", count, state) + } + } + if index >= 8 { + return + } + ins := x86YMMReadSource(t, name+" X1,X2") + for _, known := range []uint64{255, 127} { + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z1": known, "Z2": 0xffff}} + err := fixedLocalVectorStep(&ins, &state, nil, 0) + if (err == nil) != (known == 255) { + t.Fatal("uniform register count must consume exactly its low qword", known, err) + } + } + }) + } +} + +func TestGoAMD64FixedLocalShiftGuards(t *testing.T) { + for index, name := range fixedLocalLegacyShiftNames() { + forms := []string{"$128,X2", "$-129,X2", "$1,Y2", "$1,X16", "$1,M2", "$1,X7"} + if index < 8 { + forms = append(forms, "X7,X2", "65(BP),X2", "256(BP),X2", "(AX),X2") + } else { + forms = append(forms, "X1,X2", "64(BP),X2") + } + for _, args := range forms { + t.Run(name+" "+args, func(t *testing.T) { + operation := name + " " + args + fn, sig, source := fixedLocalVEXFixture(t, operation) + if args == "(AX),X2" { + mnemonic := strings.ToLower(name) + if strings.HasSuffix(mnemonic, "l") { + mnemonic = strings.TrimSuffix(mnemonic, "l") + "d" + } + fixedLocalRequireNumericControl(t, fn, sig, source, "func Value(p *uint64,a uint64) uint64", + operation, 0, 16, false, mnemonic) + return + } + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound form, memory or register acquired a frame effect", err) + } + }) + } + if index < 8 { + t.Run(name+"/whole-count-memory", func(t *testing.T) { + _, sig, source := fixedLocalVEXFixture(t, name+" 64(BP),X2") + changed := strings.ReplaceAll(source, "VMOVDQU 64(BP)", "VMOVDQU 128(BP)") + if changed == source { + t.Fatal("register initialization was not moved away from count memory") + } + source = strings.Replace(changed, "MOVQ AX,72(BP)\n", "", 1) + if source == changed { + t.Fatal("upper count qword initialization was not removed") + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("low-qword count replaced the full m128 read", err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalShiftVEXSharedDependencies(t *testing.T) { + for _, name := range strings.Fields("VPSLLW VPSLLD VPSLLQ VPSRLW VPSRLD VPSRLQ VPSRAW VPSRAD") { + for _, bank := range []string{"X", "Y"} { + t.Run(name+"/"+bank, func(t *testing.T) { + ins := x86YMMReadSource(t, name+" X1,"+bank+"2,"+bank+"3") + for _, known := range []uint64{255, 127} { + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z1": known, "Z2": ^uint64(0), "Z3": 0}, flags: x86ByteFlagsAll} + err := fixedLocalVectorStep(&ins, &state, nil, 0) + if (err == nil) != (known == 255) { + t.Fatal("VEX count must share the complete low-qword dependency", known, err) + } + if err == nil && (state.vectors["Z3"] != ^uint64(0) || state.flags != x86ByteFlagsAll) { + t.Fatal("VEX shared shift lost upper zeros or FLAGS", state) + } + } + }) + } + } +} + +func TestGoAMD64FixedLocalShiftObjects(t *testing.T) { + operations := fixedLocalLegacyShiftCases() + var assembly, declarations strings.Builder + for index, operation := range operations { + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(fixedLocalVEXAssembly(operation), "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p *uint64,a uint64) uint64\n", name) + } + fixedLocalCompleteVectorObjects(t, assembly.String(), declarations.String(), len(operations)) +} diff --git a/amd64_fixed_local_shuffle_test.go b/amd64_fixed_local_shuffle_test.go new file mode 100644 index 000000000..772cb068d --- /dev/null +++ b/amd64_fixed_local_shuffle_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// Independent source names, including Go's separate PSHUFL spelling. +// The physical MMX-only PSHUFW row is inventoried below, not authorized. +func fixedLocalShuffleNames() []string { + return []string{"PSHUFB", "PSHUFD", "PSHUFL", "PSHUFHW", "PSHUFLW"} +} + +func TestGoAMD64FixedLocalShuffleOfficialRows(t *testing.T) { + catalog, err := goasmrules.LoadX86LegacyRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + wanted := map[string]string{"PSHUFB": "Zm2_r:Yxm,Yxr", "PSHUFW": "Zibm_r:Yi8,Ymm,Ymr"} + for _, name := range fixedLocalShuffleNames()[1:] { + wanted[name] = "Zibm_r:Yu8,Yxm,Yxr" + } + seen := make(map[string]bool) + for _, opcode := range catalog.Optab { + if !strings.HasPrefix(opcode.Opcode, "PSHUF") { + continue + } + shape, found := wanted[opcode.Opcode] + if !found { + t.Fatalf("new original legacy shuffle opcode requires an explicit row audit: %+v", opcode) + } + if seen[opcode.Opcode] || len(opcode.Rows) != 1 || + opcode.Rows[0].ZCase+":"+strings.Join(opcode.Rows[0].Classes, ",") != shape { + t.Fatalf("unmodeled original shuffle row: %+v", opcode) + } + seen[opcode.Opcode] = true + } + if len(seen) != 6 { + t.Fatalf("official shuffle denominator changed: %d, want 5 XMM + 1 MMX", len(seen)) + } +} + +func fixedLocalShuffleOperation(name, input, immediate string) string { + if name == "PSHUFB" { + return name + " " + input + ",X0" + } + return name + " $" + immediate + "," + input + ",X0" +} + +func fixedLocalShuffleFixture(t *testing.T, operation string, oldDestination bool) (string, FuncSig) { + t.Helper() + source, sig, _ := x86PrivateAlignedFrameAuditFixture(t, "MOVOU 16(SP),X1\n"+operation) + if !oldDestination { + changed := strings.Replace(source, "MOVOU 48(SP),X0\n", "", 1) + if changed == source { + t.Fatal("old-destination fixture did not change the source") + } + source = changed + } + return source, sig +} + +type fixedLocalShuffleCase struct { + operation string + oldDestination bool +} + +func fixedLocalShuffleCases() []fixedLocalShuffleCase { + var cases []fixedLocalShuffleCase + for _, name := range fixedLocalShuffleNames() { + immediates := []string{"0", "1", "27", "228", "255"} + if name == "PSHUFB" { + immediates = []string{""} + } + for _, immediate := range immediates { + for _, input := range []string{"16(SP)", "X1", "X0"} { + operation := fixedLocalShuffleOperation(name, input, immediate) + cases = append(cases, fixedLocalShuffleCase{operation, true}) + if name != "PSHUFB" && input != "X0" { + cases = append(cases, fixedLocalShuffleCase{operation, false}) + } + } + } + } + return cases +} + +func fixedLocalShuffleProgram(t *testing.T) (string, string, int) { + t.Helper() + var assembly, declarations strings.Builder + cases := fixedLocalShuffleCases() + for index, row := range cases { + source, _ := fixedLocalShuffleFixture(t, row.operation, row.oldDestination) + function := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(source, "·Value", "·"+function, 1)) + fmt.Fprintf(&declarations, "func %s(a,b,c,d uint64) uint64\n", function) + } + return assembly.String(), declarations.String(), len(cases) +} + +func TestGoAMD64FixedLocalShuffleCompleteFamilies(t *testing.T) { + assembly, _, count := fixedLocalShuffleProgram(t) + if count != 103 { + t.Fatal("complete shuffle register/memory/immediate/old-destination denominator changed", count) + } + requireX86GoAssemblerResult(t, "amd64", assembly, true) + for _, row := range fixedLocalShuffleCases() { + t.Run(fmt.Sprintf("%s/old=%t", row.operation, row.oldDestination), func(t *testing.T) { + source, sig := fixedLocalShuffleFixture(t, row.operation, row.oldDestination) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + proved, err := proveX86FixedLocal(file.Funcs[0], sig) + if err != nil { + t.Fatal("original Go accepted complete shuffle source but fixed-frame effect is missing", err) + } + observed := 0 + for _, ins := range proved.Instrs { + if !strings.HasPrefix(row.operation, string(ins.Op)+" ") { + continue + } + observed++ + index := 1 + if ins.Op == "PSHUFB" { + index = 0 + } + access, found := ins.x86FixedLocalMemory[index] + if found != strings.Contains(row.operation, "16(SP)") || found && + (access.width != 16 || access.write || access.nativeInput || access.owner != proved.x86FixedLocal) { + t.Fatalf("shuffle changed the exact private source extent or owner: %+v", access) + } + } + if observed != 1 { + t.Fatal("complete proof did not retain exactly one original shuffle", observed) + } + }) + } +} + +func TestGoAMD64FixedLocalShuffleImmediateDomainAndAliases(t *testing.T) { + for _, name := range fixedLocalShuffleNames() { + t.Run(name, func(t *testing.T) { + limit := 256 + if name == "PSHUFB" { + limit = 1 + } + for immediate := 0; immediate < limit; immediate++ { + operation := fixedLocalShuffleOperation(name, "X1", fmt.Sprint(immediate)) + ins := x86YMMReadSource(t, operation) + for _, upper := range []uint64{0, 0x8040201000000000, ^uint64(0xffff)} { + old := upper + if name == "PSHUFB" { + old |= 0xffff // Byte shuffle reads its old data destination. + } + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z0": old, "Z1": 0xffff}, flags: x86ByteFlagsAll} + if err := fixedLocalVectorStep(&ins, &state, nil, 0); err != nil { + t.Fatalf("%s immediate=%d: %v", name, immediate, err) + } + if state.vectors["Z0"] != upper|0xffff || state.vectors["Z1"] != 0xffff || state.flags != x86ByteFlagsAll { + t.Fatalf("%s changed upper aliases, source or FLAGS: %+v", operation, state) + } + } + } + }) + } +} + +func TestGoAMD64FixedLocalShuffleSharedMetadata(t *testing.T) { + for _, name := range fixedLocalShuffleNames() { + ins := x86YMMReadSource(t, fixedLocalShuffleOperation(name, "X1", "0")) + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || effect.vector.upperWrite != x86TypedVectorUpperPreserve || + effect.vector.memoryAlignment != 16 { + t.Fatalf("%s is not described by the common typed family: %+v %v", name, effect, err) + } + readSource, readDestination, writeDestination := false, false, false + for _, operand := range effect.operands { + if operand.operand.Kind != OpReg { + continue + } + if operand.width != 16 { + t.Fatal("shuffle lost the complete XMM input/output width", name, operand) + } + switch operand.operand.Reg { + case "X1": + readSource = readSource || operand.read + case "X0": + readDestination = readDestination || operand.read + writeDestination = writeDestination || operand.write + } + } + if !readSource || !writeDestination || readDestination != (name == "PSHUFB") { + t.Fatal("shuffle confused old destination with its sole immediate source", name, effect) + } + } +} + +func TestGoAMD64FixedLocalShuffleBoundaries(t *testing.T) { + for _, name := range fixedLocalShuffleNames() { + for _, row := range []struct{ name, input, from, to string }{ + {"unknown source", "X2", "", ""}, + {"unknown aliased source", "X0", "MOVOU 48(SP),X0", "MOVOU 48(SP),X2"}, + {"missing memory byte", "16(SP)", "MOVQ CX,24(SP)", "MOVB CX,24(SP)"}, + {"unaligned memory", "17(SP)", "", ""}, + {"outside frame", "160(SP)", "", ""}, + {"MMX source", "M0", "", ""}, + {"high bank", "X16", "", ""}, + {"unknown dynamic control", "X1", "MOVOU 16(SP),X1", "MOVOU 16(SP),X2"}, + } { + t.Run(name+"/"+row.name, func(t *testing.T) { + operation := fixedLocalShuffleOperation(name, row.input, "0") + source, sig := fixedLocalShuffleFixture(t, operation, true) + if row.from != "" { + changed := strings.Replace(source, row.from, row.to, 1) + if changed == source { + t.Fatal("negative fixture did not change the complete source") + } + source = changed + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved shuffle input acquired private frame authority", err) + } + }) + } + } + invalid := []string{"PSHUFW $0,M0,M1", "PSHUFB.Z X1,X0", "PSHUFB $0,X1,X0"} + for _, name := range fixedLocalShuffleNames()[1:] { + for _, immediate := range []string{"-129", "-128", "-1", "256", "257", "1.0", "missing"} { + invalid = append(invalid, fixedLocalShuffleOperation(name, "X1", immediate)) + } + } + for _, operation := range invalid { + t.Run(operation, func(t *testing.T) { + ins := x86YMMReadSource(t, operation) + effect, known, err := fixedLocalVectorByteEffect(ins) + if known && err == nil { + t.Fatalf("non-XMM or invalid immediate/format gained effect: %+v", effect) + } + }) + } + t.Run("byte shuffle still reads old data", func(t *testing.T) { + source, sig := fixedLocalShuffleFixture(t, "PSHUFB X1,X0", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("PSHUFB acquired numerical data from its initialized control alone", err) + } + }) +} + +func TestGoAMD64FixedLocalShuffleObjects(t *testing.T) { + assembly, declarations, count := fixedLocalShuffleProgram(t) + if count != 103 { + t.Fatal("complete-source shuffle denominator changed", count) + } + fixedLocalCompleteVectorObjects(t, assembly, declarations, count) +} diff --git a/amd64_fixed_local_snapshot_test.go b/amd64_fixed_local_snapshot_test.go new file mode 100644 index 000000000..62a3ccece --- /dev/null +++ b/amd64_fixed_local_snapshot_test.go @@ -0,0 +1,141 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +type fixedLocalCountingContext struct { + context.Context + deadlines atomic.Int64 + onDeadline func(int64) +} + +func (ctx *fixedLocalCountingContext) Deadline() (time.Time, bool) { + samples := ctx.deadlines.Add(1) + if ctx.onDeadline != nil { + ctx.onDeadline(samples) + } + return ctx.Context.Deadline() +} + +func TestGoAMD64FixedLocalSnapshotFinalCallbackMutationRevokes(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, api := range []string{"Translate", "Module", "GoModule"} { + t.Run(api, func(t *testing.T) { + pkg, request, file, options := fixedLocalSnapshotFixture(t, 4) + source := requireGoAMD64FrameCapture(t, pkg, request) + stage, cancel := runtimeOracleStageContext(t) + defer cancel() + ctx := &fixedLocalCountingContext{Context: stage} + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + ctx.deadlines.Store(0) + if err := bound.Sigs[pkg.Path+".Value0"].x86FixedLocal.verify(); err != nil { + t.Fatal(err) + } + perRound := ctx.deadlines.Swap(0) + if perRound == 0 { + t.Fatal("live source verification omitted its caller lifetime") + } + mutated := false + ctx.onDeadline = func(samples int64) { + if samples == 2*perRound { + // With a call-local snapshot this is the terminal callback in + // module completion, after IR emission and the locked commands. + path := filepath.Join(request.Directory, request.AssemblyFile) + if err := os.WriteFile(path, []byte("changed original source\n"), 0600); err != nil { + t.Fatal(err) + } + mutated = true + } + } + fixedLocalAPI(t, pkg, file, bound, api, func(t *testing.T, ir string, err error) { + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("late source mutation returned translated IR: %v", err) + } + }) + if !mutated || source.ready { + t.Fatal("terminal callback was omitted or its mutated source remained live") + } + }) + } +} + +func fixedLocalSnapshotFixture(t *testing.T, functions int) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options) { + t.Helper() + var assembly, declarations strings.Builder + for index := 0; index < functions; index++ { + fmt.Fprintf(&declarations, "func Value%d(a uint64) uint64\n", index) + fmt.Fprintf(&assembly, `TEXT ·Value%d(SB),4,$64-16 +MOVQ a+0(FP),AX +MOVQ AX,0(SP) +ADDQ $1,0(SP) +MOVQ 0(SP),AX +MOVQ AX,ret+8(FP) +RET +`, index) + } + return fixedLocalTransportFixture(t, assembly.String(), declarations.String(), + "linux/amd64", "x86_64-unknown-linux-gnu") +} + +func TestGoAMD64FixedLocalSnapshotVerificationDoesNotScaleWithFunctions(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, api := range []string{"Translate", "Module", "GoModule"} { + t.Run(api, func(t *testing.T) { + for _, functions := range []int{1, 4} { + t.Run(fmt.Sprint(functions), func(t *testing.T) { + pkg, request, file, options := fixedLocalSnapshotFixture(t, functions) + source := requireGoAMD64FrameCapture(t, pkg, request) + stage, cancel := runtimeOracleStageContext(t) + defer cancel() + ctx := &fixedLocalCountingContext{Context: stage} + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + // Calibrate one real verification round. Only this source owns + // the original context; subprocesses use private standard ones. + // These samples measure rounds, not elapsed time or tool bytes. + ctx.deadlines.Store(0) + binding := bound.Sigs[pkg.Path+".Value0"].x86FixedLocal + if err := binding.verify(); err != nil { + t.Fatal(err) + } + perRound := ctx.deadlines.Swap(0) + if perRound == 0 { + t.Fatal("live source verification did not sample its caller lifetime") + } + started := time.Now() + fixedLocalAPI(t, pkg, file, bound, api, func(t *testing.T, ir string, err error) { + if err != nil || ir == "" { + t.Fatalf("complete fixed-local translation: %v", err) + } + for index := 0; index < functions; index++ { + name := fmt.Sprintf("%s.Value%d", pkg.Path, index) + if !strings.Contains(ir, llvmGlobal(name)) { + t.Fatalf("translation omitted %s", name) + } + } + }) + t.Logf("%d functions: API duration %s, %d caller lifetime samples (%d per live verification round)", + functions, time.Since(started), ctx.deadlines.Load(), perRound) + if samples := ctx.deadlines.Load(); samples != 2*perRound { + t.Fatalf("%d functions sampled %d lifetimes, want two live module boundaries (%d samples per round)", + functions, samples, perRound) + } + }) + } + }) + } +} diff --git a/amd64_fixed_local_state_budget_test.go b/amd64_fixed_local_state_budget_test.go new file mode 100644 index 000000000..1e597d234 --- /dev/null +++ b/amd64_fixed_local_state_budget_test.go @@ -0,0 +1,52 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalLargeLinearFrameState(t *testing.T) { + for _, shape := range []string{"linear", "diamond"} { + t.Run(shape, func(t *testing.T) { + fn, sig := fixedLocalReplayFixture(t, fixedLocalReplayBody(shape, 1600)) + fn.FrameSize = 1408 + if fn.FrameSize*int64(len(fn.Instrs)) <= 1<<20 { + t.Fatal("fixture no longer exceeds the old per-instruction state estimate") + } + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("long basic blocks must not reserve a frame state per instruction", err) + } + }) + } +} + +func fixedLocalRetainedStateBudgetFixture(t *testing.T) (Func, FuncSig) { + t.Helper() + var body strings.Builder + for index := 0; index < 20; index++ { + fmt.Fprintf(&body, "CMPQ AX,AX\nJEQ next%d\nNOP\nnext%d:\n", index, index) + } + fn, sig := fixedLocalReplayFixture(t, body.String()) + fn.FrameSize = 64 << 10 + return fn, sig +} + +func TestGoAMD64FixedLocalRetainedStateBudgetIsAtomic(t *testing.T) { + fn, sig := fixedLocalRetainedStateBudgetFixture(t) + proof := &x86FixedLocalProof{frame: fn.FrameSize} + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + err := replayX86FixedLocal(&fn, sig, proof, 0, &budget, nil, nil) + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "state storage budget") { + t.Fatal("actual retained frame states exceeded the storage cap", err) + } + if budget.steps <= 0 || budget.bytes <= 0 { + t.Fatal("fixture must hit retained storage before the independent work budgets") + } + for _, ins := range fn.Instrs { + if len(ins.x86FixedLocalMemory) != 0 { + t.Fatal("state-budget failure published a provisional access") + } + } +} diff --git a/amd64_fixed_local_target_test.go b/amd64_fixed_local_target_test.go new file mode 100644 index 000000000..afb2a4046 --- /dev/null +++ b/amd64_fixed_local_target_test.go @@ -0,0 +1,61 @@ +package plan9asm + +import ( + "errors" + "testing" +) + +func TestGoAMD64FixedLocalDarwinTripleClosure(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + request.Target = "darwin/amd64" + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-apple-macosx"} { + t.Run(triple, func(t *testing.T) { + options.TargetTriple = triple + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatalf("actual Darwin frame rejected its exact LLVM target spelling: %v", err) + } + if bound.TargetTriple != triple { + t.Fatal("binding rewrote the consumer's actual triple") + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "triple.ll", "triple.o", ir) + compileLLVMToObject(t, llc, triple, "triple-opt.ll", "triple-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + // Equivalent initial target spellings cannot mutate an already + // bound consumer. Every certificate still pins its exact options. + changed := bound + changed.TargetTriple = "x86_64-apple-macosx" + if changed.TargetTriple == triple { + changed.TargetTriple = "x86_64-apple-darwin" + } + if _, err := Translate(file, changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("retargeted fixed-local certificate survived: %v", err) + } + }) + } + for _, triple := range []string{ + "", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + "arm64-apple-macosx", "i386-apple-macosx", "aarch64-apple-darwin", + "x86_64-unknown-macosx", "x86_64-apple-ios", "x86_64-apple-macosx-junk", + "x86_64-apple-darwin-junk", "x86_64-apple-macosx11.0.0", "x86_64-apple-darwin20.0.0", + "prefix-x86_64-apple-macosx", "x86_64-apple-macosx ", + } { + t.Run("reject/"+triple, func(t *testing.T) { + options.TargetTriple = triple + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("foreign or unregistered triple acquired a Darwin native frame: %q: %v", triple, err) + } + }) + } +} diff --git a/amd64_fixed_local_terminal_native_linux_amd64_test.go b/amd64_fixed_local_terminal_native_linux_amd64_test.go new file mode 100644 index 000000000..62917d3be --- /dev/null +++ b/amd64_fixed_local_terminal_native_linux_amd64_test.go @@ -0,0 +1,31 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalTerminalReturns(t *testing.T) { + assembly, declarations := fixedLocalTerminalReturnProgram() + inputs := []uint64{0, 1, 2, ^uint64(0)} + var expected [][]string + for range inputs { + expected = append(expected, []string{"0000000000000007", "0000000000000007"}) + } + fixedLocalResultsRuntime(t, assembly, declarations, inputs, expected) + t.Log("native terminal inventory: functions=2 inputs=4 observations_per_program=8 APIs=3 optimizations=2 execute=true") +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalLayoutPartitions(t *testing.T) { + assembly, declarations := fixedLocalLayoutPartitionProgram() + inputs := []uint64{0, 1, 2, 3, 4, 9, 16} + var expected [][]string + for _, input := range inputs { + value := fmt.Sprintf("%016x", input+1) + expected = append(expected, []string{value, value, value}) + } + fixedLocalResultsRuntime(t, assembly, declarations, inputs, expected) + t.Log("native private-layout inventory: functions=3 inputs=7 observations_per_program=21 APIs=3 optimizations=2 execute=true") +} diff --git a/amd64_fixed_local_transport.go b/amd64_fixed_local_transport.go new file mode 100644 index 000000000..c170dcb58 --- /dev/null +++ b/amd64_fixed_local_transport.go @@ -0,0 +1,135 @@ +package plan9asm + +// Opaque inputs identify ABI0 pointer components, not pointee allocations. +// They may be copied through full-width GP/local cells and pointer results. +// They never enter the private-SP address map or numerical initialized bytes. +// Boolean masks independently record bytes whose values are exactly zero/one. +// This is needed because an ABI0 bool occupies one byte, but LLVM uses i1. +func fixedLocalTransportRole(role goAMD64FrameTransport) bool { + return role == goAMD64FrameNumericTransport || role == goAMD64FrameCompositeTransport +} + +func fixedLocalTransportSlot(offset, width int64, slots []FrameSlot) (FrameSlot, int, bool) { + for index, slot := range slots { + switch slot.Type { + case I1, I8, I16, I32, I64, Ptr: + default: + continue + } + if slot.Offset == offset && frameTypeSize(slot.Type, 8) == width { + return slot, index, true + } + } + return FrameSlot{}, 0, false +} + +func fixedLocalBooleanShift(reg Reg) uint { + if _, bits, alias := amd64ByteAlias(reg); alias { + return bits / 8 + } + return 0 +} + +func fixedLocalConstantBooleans(value int64, width int64) uint8 { + var mask uint8 + for index := int64(0); index < width; index++ { + if byte(uint64(value)>>uint(index*8)) <= 1 { + mask |= 1 << uint(index) + } + } + return mask +} + +func fixedLocalBooleanRead(operand Operand, width int64, state *x86FixedLocalState) uint8 { + if operand.Kind == OpImm { + return fixedLocalConstantBooleans(operand.Imm, width) + } + if operand.Kind == OpReg { + root, _ := x86CallerGP(operand.Reg) + return state.booleans[root] >> fixedLocalBooleanShift(operand.Reg) & uint8((1<0 && &s[0]!=&a[0] { t.Fatal("slice identity") } + if a==nil && s!=nil { t.Fatal("nil identity") } + if buffer!=[4]byte{0x11,0x22,0x33,0x44}||word!=0x123456789abcdef0 { t.Fatal("pointee mutation") } + } + } + } + } +} +` + goConditions + fixedLocalPackedRunOriginal(t, execution, assembly+originalObservers, declarations+observerDeclarations, goOracle, true) + // Call the translated aggregate ABI from an exact LLVM wrapper, not a + // guessed C struct ABI. C only crosses a scalar wrapper signature. + const wrapper = ` +define i32 @transport_probe(ptr %data, i64 %len, i64 %cap, ptr %p, i1 %b, i64 %n) { +entry: + %a0 = insertvalue {ptr,i64,i64} zeroinitializer, ptr %data, 0 + %a1 = insertvalue {ptr,i64,i64} %a0, i64 %len, 1 + %a2 = insertvalue {ptr,i64,i64} %a1, i64 %cap, 2 + %b2 = call i1 @"example.invalid/framefixture.Bool"({ptr,i64,i64} %a2, ptr %p, i1 %b, i64 %n) + %p2 = call ptr @"example.invalid/framefixture.Pointer"({ptr,i64,i64} %a2, ptr %p, i1 %b, i64 %n) + %s2 = call {ptr,i64,i64} @"example.invalid/framefixture.Slice"({ptr,i64,i64} %a2, ptr %p, i1 %b, i64 %n) + %d2 = extractvalue {ptr,i64,i64} %s2, 0 + %l2 = extractvalue {ptr,i64,i64} %s2, 1 + %c2 = extractvalue {ptr,i64,i64} %s2, 2 + %badb = icmp ne i1 %b2, %b + %badp = icmp ne ptr %p2, %p + %badd = icmp ne ptr %d2, %data + %badl = icmp ne i64 %l2, %len + %badc = icmp ne i64 %c2, %cap + %bad0 = or i1 %badb, %badp + %bad1 = or i1 %bad0, %badd + %bad2 = or i1 %bad1, %badl + %bad3 = or i1 %bad2, %badc + %result = zext i1 %bad3 to i32 + ret i32 %result +} +` + oracle := `#include +#include +#include +` + cConditions + ` +extern int transport_probe(void *,uint64_t,uint64_t,void *,bool,uint64_t); +int main(void) { + if(check_conditions()) return 3; + volatile uint8_t buffer[4]={0x11,0x22,0x33,0x44}; + volatile uint64_t word=UINT64_C(0x123456789abcdef0); + uint64_t inputs[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000)}; + for(unsigned a=0;a<3;a++) for(unsigned p=0;p<2;p++) + for(unsigned b=0;b<2;b++) for(unsigned n=0;n<4;n++) { + void *data=a?(void *)buffer:0; + uint64_t len=a==1?2:a==2?4:0,cap=a?4:0; + if(transport_probe(data,len,cap,p?(void *)&word:0,b,inputs[n])) return 1; + if(buffer[0]!=0x11||buffer[1]!=0x22||buffer[2]!=0x33||buffer[3]!=0x44||word!=UINT64_C(0x123456789abcdef0)) return 2; + } + return 0; +} +` + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + input, level := ir+wrapper+llvmConditions, "-O0" + if optimized { + input, level = x86MulDivRoundOptimize(t, input), "-O2" + } + execution.compileAndRunLLVM(t, fmt.Sprintf("transport-%v", optimized), input, oracle, level) + } + }) + t.Log("48 inputs x bool/pointer/slice plus all16 SETcc true/false and original physical bool-byte canaries; LLVM three APIs O0/O2; pointees unchanged") + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("Rosetta SSE diagnostic, not native AVX/AVX512 execution") + } +} + +func fixedLocalTransportConditionOracles() (string, string, string, string, string, string, string) { + var assembly, declarations, observers, observerDeclarations strings.Builder + var goRows, llvmRows, cDeclarations, cRows strings.Builder + names := strings.Fields("SETCC SETCS SETEQ SETGE SETGT SETHI SETLE SETLS SETLT SETMI SETNE SETOC SETOS SETPC SETPL SETPS") + var bodies []string + for _, name := range names { + bodies = append(bodies, "MOVQ a_len+8(FP),DX\nCMPQ AX,DX\n"+name+" BX\nMOVB BX,ret+48(FP)") + } + bodies = append(bodies, fixedLocalCanonicalBoolBodies...) + bodies = append(bodies, "MOVQ $-1,BX\nMOVB $1,BX\nMOVB BX,ret+48(FP)") + for index, body := range bodies { + function := fmt.Sprintf("Condition%d", index) + kernel := fixedLocalTransportAssembly(function, 49, body) + kernel = strings.ReplaceAll(strings.ReplaceAll(kernel, "VMOVDQA", "MOVOA"), "Y14", "X14") + assembly.WriteString(kernel) + fmt.Fprintf(&declarations, "func %s(%s) bool\n", function, fixedLocalTransportParams) + fmt.Fprintf(&observerDeclarations, "func Observe%d(%s) (raw, adjacent, input uint64)\n", index, fixedLocalTransportParams) + // Original-only ABI0 adapter. The callee and this NOSPLIT observer + // contain no Go calls, allocation or pointee access. It snapshots the + // actual output word and neighbors, not a Go-normalized bool value. + fmt.Fprintf(&observers, `TEXT ·Observe%d(SB),4,$64-72 +MOVQ a_base+0(FP),AX +MOVQ AX,0(SP) +MOVQ a_len+8(FP),AX +MOVQ AX,8(SP) +MOVQ a_cap+16(FP),AX +MOVQ AX,16(SP) +MOVQ p+24(FP),AX +MOVQ AX,24(SP) +MOVB b+32(FP),AX +MOVB AX,32(SP) +MOVQ n+40(FP),AX +MOVQ AX,40(SP) +MOVQ $0x5a5a5a5a5a5a5a5a,AX +MOVQ AX,48(SP) +MOVQ $0x6b6b6b6b6b6b6b6b,AX +MOVQ AX,56(SP) +CALL ·%s(SB) +MOVQ 48(SP),AX +MOVQ AX,raw+48(FP) +MOVQ 56(SP),AX +MOVQ AX,adjacent+56(FP) +MOVQ 40(SP),AX +MOVQ AX,input+64(FP) +RET +`, index, function) + fmt.Fprintf(&goRows, "{Condition%d,Observe%d},\n", index, index) + fmt.Fprintf(&llvmRows, ` +define i32 @condition_probe%d(i64 %%n, i64 %%len) { + %%a0 = insertvalue {ptr,i64,i64} zeroinitializer, i64 %%len, 1 + %%a1 = insertvalue {ptr,i64,i64} %%a0, i64 %%len, 2 + %%flag = trunc i64 %%len to i1 + %%b = call i1 @"example.invalid/framefixture.%s"({ptr,i64,i64} %%a1, ptr null, i1 %%flag, i64 %%n) + %%result = zext i1 %%b to i32 + ret i32 %%result +} +`, index, function) + fmt.Fprintf(&cDeclarations, "extern int condition_probe%d(uint64_t,uint64_t);\n", index) + fmt.Fprintf(&cRows, "condition_probe%d,", index) + } + goOracle := ` +func checkConditions(t *testing.T) { + rows:=[]struct{ + call func([]byte,*uint64,bool,uint64) bool + observe func([]byte,*uint64,bool,uint64)(uint64,uint64,uint64) + }{ +` + goRows.String() + ` + } + inputs:=[]uint64{0,1,2,3,1<<63,(1<<63)-1,^uint64(0)} + buffer:=[4]byte{1,2,3,4} + for i,row:=range rows { + seen:=0 + for _,a:=range inputs { for b:=uint64(0);b<4;b++ { + r:=a-b + cf,zf,sf:=a>63!=0 + of:=(a^b)&(a^r)>>63!=0 + parity:=byte(r); parity^=parity>>4; parity^=parity>>2; parity^=parity>>1 + pf:=parity&1==0 + want:=[]bool{!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf,false,true,true,b&1!=0,a==0,a==0,a==0,true}[i] + if row.call(buffer[:b],nil,b&1!=0,a)!=want { t.Fatalf("condition %%d",i) } + value:=uint64(0); if want { value=1 } + raw,adjacent,input:=row.observe(buffer[:b],nil,b&1!=0,a) + if raw!=0x5a5a5a5a5a5a5a00|value||adjacent!=0x6b6b6b6b6b6b6b6b||input!=a { t.Fatalf("physical byte/canary %%d %%x %%x %%x",i,raw,adjacent,input) } + seen|=1<>63,of=((a^b)&(a^r))>>63; + uint8_t parity=r; parity^=parity>>4; parity^=parity>>2; parity^=parity>>1; + bool pf=(parity&1)==0; + bool want[]={!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf,false,true,true,(b&1)!=0,a==0,a==0,a==0,true}; + int actual=condition_probes[i](a,b); + if(actual!=want[i]) { + fprintf(stderr,"condition=%u a=%llx b=%llx actual=%d expected=%d\n",i,(unsigned long long)a,(unsigned long long)b,(int)actual,(int)want[i]); + return 1; + } + seen|=1U<(SB)", false}, + } { + name := fmt.Sprintf("%s/%s/%s", opcode, direction, row.address) + t.Run(name, func(t *testing.T) { + operands := row.address + ",X0" + if direction == "store" { + operands = "X0," + row.address + } + source := "TEXT ·Value(SB),4,$160-8\n" + opcode + " " + operands + + "\nMOVQ X0,AX\nMOVQ AX,ret0+0(FP)\nRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if got := GoAMD64FixedLocalFrameBindingNeeded(file, "amd64"); got != row.want { + t.Fatalf("physical scratch query = %v, want %v", got, row.want) + } + if GoAMD64FixedLocalFrameBindingNeeded(file, "386") { + t.Fatal("AMD64 source observer claimed mode32 backing") + } + if row.want { + _, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"Value": sig}, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("candidate query supplied backing or initialized bytes: %v", err) + } + } + }) + } + } + } +} diff --git a/amd64_fixed_local_unrelated_types_test.go b/amd64_fixed_local_unrelated_types_test.go new file mode 100644 index 000000000..2bbc50dfe --- /dev/null +++ b/amd64_fixed_local_unrelated_types_test.go @@ -0,0 +1,76 @@ +package plan9asm + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "testing" +) + +func TestGoAMD64FixedLocalUnrelatedGoTypesObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + const declarations = ` +type XOF interface { Read([]byte) (int, error); Clone() XOF } +type Callback func([]byte) error +type Cache map[string][]byte +type Queue chan []byte +type Scalar float64 +type Node struct { Next *Node; Value uint64 } +type Box[T any] struct { Value T } +type Reader = interface { Read([]byte) (int, error) } +` + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + name := filepath.Join(request.Directory, "value.go") + original, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + data := append(original, []byte(declarations)...) + if err := os.WriteFile(name, data, 0600); err != nil { + t.Fatal(err) + } + set := token.NewFileSet() + syntax, err := parser.ParseFile(set, name, data, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + pkg.Syntax = []*ast.File{syntax} + pkg.Types, err = (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check(pkg.Path, set, pkg.Syntax, nil) + if err != nil { + t.Fatal(err) + } + request.Target = target.name + options.TargetTriple = target.triple + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "types.ll", "types.o", ir) + compileLLVMToObject(t, llc, target.triple, "types-opt.ll", "types-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + if len(bound.Sigs) != 1 || len(source.declarations) != 1 || len(source.functions) != 1 { + t.Fatal("unrelated ordinary types acquired assembly transport or frame authority") + } + }) + } +} diff --git a/amd64_fixed_local_value.go b/amd64_fixed_local_value.go new file mode 100644 index 000000000..0dfad0f3f --- /dev/null +++ b/amd64_fixed_local_value.go @@ -0,0 +1,210 @@ +package plan9asm + +// Scalar transport carries one complete typed value, never additional memory +// permission. MOV, CMOV and XCHG share the same read/write boundaries; exchanges +// snapshot both operands before changing any register or private spill. +type fixedLocalScalarValue struct { + address int64 + pointer, opaque bool + identity x86FixedLocalOrigins + booleans uint8 + constant int64 + constantKnown bool + bounds x86UnsignedRange + bounded bool +} + +// Selection retains only facts true of both possible values. Private offsets +// must agree. Opaque addresses share the same machine-address domain; +// joining them never supplies a numerical value or private-frame permission. +func mergeFixedLocalScalarValues(first, last fixedLocalScalarValue) (fixedLocalScalarValue, bool) { + if first.pointer != last.pointer || first.opaque != last.opaque || + first.pointer && first.address != last.address { + return fixedLocalScalarValue{}, false + } + if first.opaque { + var valid bool + first.identity, valid = first.identity.union(last.identity) + if !valid { + return fixedLocalScalarValue{}, false + } + } + first.booleans &= last.booleans + first.constantKnown = first.constantKnown && last.constantKnown && first.constant == last.constant + first.bounded = first.bounded && last.bounded + if first.bounded { + first.bounds = first.bounds.hull(last.bounds) + } + return first, true +} + +func fixedLocalReadValue(ins *Instr, index int, state *x86FixedLocalState, sig FuncSig, + proof *x86FixedLocalProof, residue int, width int64, write bool) (fixedLocalScalarValue, error) { + var value fixedLocalScalarValue + fail := func() error { return goAMD64FrameContext("unproved fixed local scalar read") } + source := ins.Args[index] + switch source.Kind { + case OpReg: + value.address, value.pointer = state.pointers[source.Reg] + value.identity, value.opaque = state.opaque[source.Reg] + if value.opaque && !value.identity.valid() { + return value, fail() + } + if value.pointer || value.opaque { + if width != 8 { + return value, fail() + } + } else if amd64VectorByteWidth(source.Reg) == 16 { + if width != 8 || state.vectors[x86CallerVectorRoot(source.Reg)]&255 != 255 { + return value, fail() + } + } else { + if !fixedLocalNumericRead(source, width, state) { + return value, fail() + } + value.constant, value.constantKnown = state.constants[source.Reg] + // Narrow moves truncate or preserve a destination's old upper + // bytes. They cannot transport a complete 64-bit address mask. + value.constantKnown = value.constantKnown && width == 8 + value.bounds, value.bounded = state.ranges[source.Reg] + value.bounded = value.bounded && width == 8 + value.booleans = fixedLocalBooleanRead(source, width, state) + } + case OpImm: + if !fixedLocalNumericRead(source, width, state) { + return value, fail() + } + value.constant, value.constantKnown = source.Imm, width == 8 + value.booleans = fixedLocalBooleanRead(source, width, state) + case OpFP: + sp, original := state.pointers[SP] + slot, _, found := fixedLocalTransportSlot(source.FPOffset, width, sig.Frame.Params) + if !original || sp != 0 || !found { + return value, fail() + } + if slot.Type == Ptr { + value.opaque, value.identity = true, fixedLocalOrigin(x86FixedLocalOrigin{parameter: slot.Offset}) + } else if slot.Type == I1 { + value.booleans = 1 + } + case OpSym: + if write { + return value, fail() + } + if _, _, err := fixedLocalReadMemory(ins, index, state, proof, residue, width, 1); err != nil { + return value, err + } + case OpMem: + memory, err := fixedLocalMemoryAddress(*ins, index, proof.frame) + if err != nil { + return value, err + } + offset, found := fixedLocalPrivateOffset(memory, state) + if found { + value.address, value.pointer = state.addresses[offset] + value.identity, value.opaque = state.opaqueCells[offset] + } + if value.opaque && !value.identity.valid() { + return value, fail() + } + if (value.pointer || value.opaque) && width != 8 { + return value, fail() + } + offset, role, err := fixedLocalMemoryAccess(ins, index, state, proof, residue, width, 1, write, value.pointer || value.opaque) + if err != nil { + return value, err + } + if role == x86FixedLocalPrivateMemory && !value.pointer && !value.opaque { + if write { + if !state.bytes.all(offset, width) { + return value, fail() + } + } + value.booleans = fixedLocalMemoryBooleans(state, offset, width) + } + default: + return value, fail() + } + return value, nil +} + +func fixedLocalWriteValue(ins *Instr, index, sourceIndex int, value fixedLocalScalarValue, state *x86FixedLocalState, + sig FuncSig, proof *x86FixedLocalProof, residue int, width int64) error { + fail := func() error { return goAMD64FrameContext("unproved fixed local scalar write") } + destination, source := ins.Args[index], ins.Args[sourceIndex] + switch destination.Kind { + case OpReg: + root, valid := x86CallerGP(destination.Reg) + if !valid || (value.pointer || value.opaque) && (width != 8 || destination.Reg != root) { + return fail() + } + if root == SP && !value.pointer { + return fail() + } + if _, oldPointer := state.pointers[root]; oldPointer && width != 8 { + return fail() + } + delete(state.pointers, root) + delete(state.opaque, root) + delete(state.constants, root) + fixedLocalWriteBooleans(state, destination.Reg, width, value.booleans) + if value.pointer { + state.pointers[root] = value.address + state.known[root] = 0 + } else if value.opaque { + state.opaque[root] = value.identity + state.known[root] = 0 + } else { + mask := x86CallerByteMask(destination.Reg, width) + if mask == 0 { + return fail() + } + if width == 4 { + mask = 255 + } + state.known[root] |= mask + if value.constantKnown { + state.constants[root] = value.constant + } + if value.bounded { + state.setRange(root, value.bounds) + } + } + case OpMem: + if source.Kind == OpMem { + return fail() + } + offset, role, err := fixedLocalMemoryAccess(ins, index, state, proof, residue, width, 1, true, false) + if err != nil { + return err + } + if role == x86FixedLocalNativeInput { + if value.pointer || value.opaque { + return fail() // Neither private nor input pointers escape as bytes. + } + return nil + } + fixedLocalRevokeSpillRoles(state, offset, width) + state.bytes.setRange(offset, width, !value.pointer && !value.opaque) + for index := offset; index < offset+width; index++ { + state.booleanBytes.set(index, value.booleans&(1< 0 + case OpImm: + valid = valid && operand.read && !operand.write && fixedLocalNumericRead(operand.operand, operand.width, nil) + default: + valid = false + } + } + if !valid { + return x86TypedByteEffect{}, true, goAMD64FrameContext("fixed local vector effect has an unbound storage or alias role: %s", ins.Op) + } + return effect, true, nil +} diff --git a/amd64_fixed_local_vex_effect_test.go b/amd64_fixed_local_vex_effect_test.go new file mode 100644 index 000000000..34d7c8238 --- /dev/null +++ b/amd64_fixed_local_vex_effect_test.go @@ -0,0 +1,391 @@ +package plan9asm + +import ( + "errors" + "fmt" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +type fixedLocalVEXWitness struct { + owner, source string +} + +// The scope is independent of the translator's registration maps. Keep every +// original VEX row, including overlapping signed/unsigned immediate classes. +// EVEX rows are inventoried separately and do not acquire permission here. +func fixedLocalVEXWitnesses(t *testing.T) []fixedLocalVEXWitness { + return fixedLocalVEXFamilyWitnesses(t, + `^(VPADD(B|W|D|Q|SB|SW|USB|USW)|VPALIGNR|V(BLENDP[DS]|PBLEND[WD])|V(ANDN?|OR|XOR)P[DS]|VP(ANDN?|OR|XOR)|VMOV(AP[DS]|UP[DS]|DQ[AU])|VPERM2[FI]128|VPS(LL|RL|RA)(W|D|Q|V[WDQ]|DQ)|VPSHUFB|VZERO(UPPER|ALL))$`, + 146, 354) +} + +func fixedLocalVEXBinaryWitnesses(t *testing.T) []fixedLocalVEXWitness { + return fixedLocalVEXFamilyWitnesses(t, + `^VP(SUB(B|W|D|Q|SB|SW|USB|USW)|MUL(LD|LQ|LW|HW|HUW|HRSW|DQ|UDQ)|AVG(B|W)|MADD(WD|UBSW)|(MIN|MAX)[SU][BWDQ])$`, + 62, 186) +} + +func fixedLocalVEXFamilyWitnesses(t *testing.T, pattern string, wantRows, wantWitnesses int) []fixedLocalVEXWitness { + t.Helper() + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + family := regexp.MustCompile(pattern) + var witnesses []fixedLocalVEXWitness + rows := 0 + for _, rule := range rules { + if !family.MatchString(rule.Opcode) || !strings.HasPrefix(rule.ZCase, "Zvex") { + continue + } + rows++ + if rule.Prefix != "Pavx" || rule.EncodingOffset < 0 || rule.EncodingOffset+2 > len(rule.OpBytes) { + t.Fatalf("unmodeled official VEX attributes: %+v", rule) + } + var args []string + memory := -1 + for index, class := range rule.Classes { + switch class { + case "Yxr", "Yxm", "Yyr", "Yym": + bank := "X" + if class[1] == 'y' { + bank = "Y" + } + args = append(args, fmt.Sprintf("%s%d", bank, index)) + if class[2] == 'm' { + if memory >= 0 { + t.Fatalf("unexpected two-memory VEX row: %+v", rule) + } + memory = index + } + case "Yi8": + args = append(args, "$-128") + case "Yu8": + args = append(args, "$255") + default: + t.Fatalf("unmodeled original VEX operand class: %+v", rule) + } + } + inputs := []string{""} + if memory >= 0 { + inputs = []string{args[memory], "64(BP)", "(SI)"} + } + for _, input := range inputs { + if memory >= 0 { + args[memory] = input + } + witnesses = append(witnesses, fixedLocalVEXWitness{ + owner: fmt.Sprintf("%s/row%d/%s", rule.Opcode, rule.RowIndex, input), + source: strings.TrimSpace(rule.Opcode + " " + strings.Join(args, ",")), + }) + } + } + if rows != wantRows || len(witnesses) != wantWitnesses { + t.Fatalf("official non-masked VEX inventory changed: rows=%d witnesses=%d, want %d/%d", rows, len(witnesses), wantRows, wantWitnesses) + } + return witnesses +} + +func fixedLocalVEXFixture(t *testing.T, operation string) (Func, FuncSig, string) { + t.Helper() + source := fixedLocalVEXAssembly(operation) + _, _, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *uint64, a uint64) uint64", "RET") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + return file.Funcs[0], sig, source +} + +func fixedLocalVEXAssembly(operation string) string { + // Reuse the complete aligned scratch fixture. Input pointers are typed, + // not inferred from numerical register contents or private frame offsets. + source := fixedLocalPackedAssembly("VMOVDQU 64(BP),Y0\nVMOVDQU 64(BP),Y1\n" + + "VMOVDQU 64(BP),Y2\nVMOVDQU 64(BP),Y3\n" + operation) + source = strings.Replace(source, "$256-16", "$256-24", 1) + source = strings.Replace(source, "MOVQ a+0(FP),AX", "MOVQ p+0(FP),SI\nMOVQ a+8(FP),AX", 1) + source = strings.Replace(source, "ret+8(FP)", "ret+16(FP)", 1) + return source +} + +func TestGoAMD64FixedLocalVEXOfficialEncodings(t *testing.T) { + fixedLocalCheckVEXEncodings(t, fixedLocalVEXWitnesses(t)) +} + +func TestGoAMD64FixedLocalVEXBinaryOfficialEncodings(t *testing.T) { + fixedLocalCheckVEXEncodings(t, fixedLocalVEXBinaryWitnesses(t)) +} + +func fixedLocalCheckVEXEncodings(t *testing.T, witnesses []fixedLocalVEXWitness) { + t.Helper() + rows := make([]string, len(witnesses)) + for index, row := range witnesses { + rows[index] = row.source + } + code, pcs := x86NumericalBoundaryGoBytes(t, "amd64", strings.Join(rows, "\n")+"\nRET") + if len(pcs) != len(rows)+1 { + t.Fatal("Go changed the complete source instruction inventory", len(pcs), len(rows)) + } + for index, row := range witnesses { + instruction := code[pcs[index]:pcs[index+1]] + if len(instruction) < 3 || instruction[0] != 0xc4 && instruction[0] != 0xc5 { + t.Fatalf("%s selected non-VEX bytes %x", row.owner, instruction) + } + } + t.Logf("%d legal original Go source witnesses; VEX generation check, not exact row-encoding or execution proof", len(witnesses)) +} + +func TestGoAMD64FixedLocalVEXCompleteFamilies(t *testing.T) { + fixedLocalCheckVEXFamilies(t, fixedLocalVEXWitnesses(t)) +} + +func TestGoAMD64FixedLocalVEXBinaryCompleteFamilies(t *testing.T) { + fixedLocalCheckVEXFamilies(t, fixedLocalVEXBinaryWitnesses(t)) +} + +func fixedLocalCheckVEXFamilies(t *testing.T, witnesses []fixedLocalVEXWitness) { + t.Helper() + for _, row := range witnesses { + t.Run(row.owner, func(t *testing.T) { + fn, sig, _ := fixedLocalVEXFixture(t, row.source) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("initialized complete VEX source rejected", err) + } + }) + } +} + +func TestGoAMD64FixedLocalVEXPhysicalBoundaries(t *testing.T) { + fixedLocalCheckVEXPhysicalBoundaries(t, fixedLocalVEXWitnesses(t)) +} + +func TestGoAMD64FixedLocalVEXBinaryPhysicalBoundaries(t *testing.T) { + fixedLocalCheckVEXPhysicalBoundaries(t, fixedLocalVEXBinaryWitnesses(t)) +} + +func fixedLocalCheckVEXPhysicalBoundaries(t *testing.T, witnesses []fixedLocalVEXWitness) { + t.Helper() + rows := make([]string, len(witnesses)) + for index, row := range witnesses { + rows[index] = row.source + } + code, pcs := x86NumericalBoundaryGoBytes(t, "amd64", strings.Join(rows, "\n")+"\nRET") + if len(pcs) != len(rows)+1 { + t.Fatal("Go changed the complete source instruction inventory", len(pcs), len(rows)) + } + for index, row := range witnesses { + t.Run(row.owner, func(t *testing.T) { + instruction := code[pcs[index]:pcs[index+1]] + boundaries, scalar, err := x86NumericalNativeBoundaries(instruction, 64) + if err != nil || len(boundaries) != 1 || boundaries[0].Length != len(instruction) || len(scalar) != 0 { + t.Fatalf("original Go boundary for %s (%x): %v %v %v", row.source, instruction, boundaries, scalar, err) + } + }) + } +} + +func TestGoAMD64FixedLocalVEXBoundaries(t *testing.T) { + for _, row := range []struct{ name, operation string }{ + {"unknown source", "VPADDQ Y7,Y1,Y2"}, + {"missing count upper qword", "VPSRLQ 184(BP),Y1,Y2"}, + {"outside private frame", "VPSHUFB 256(BP),Y1,Y2"}, + {"numeric pointer", "VPOR (AX),Y1,Y2"}, + {"unbound FP", "VPAND a+8(FP),Y1,Y2"}, + {"EVEX high register", "VPADDQ X16,X1,X2"}, + {"EVEX width", "VPADDQ Z0,Z1,Z2"}, + {"EVEX mask", "KMOVQ AX,K1\nVPADDQ Y0,Y1,K1,Y2"}, + {"EVEX broadcast", "VPADDQ.BCST (SI),Y1,Y2"}, + {"EVEX-only member", "VPSRAQ $1,Y1,Y2"}, + {"EVEX immediate memory", "VPSRLQ $1,64(BP),Y2"}, + {"MMX", "PADDQ M0,M1"}, + } { + t.Run(row.name, func(t *testing.T) { + fn, sig, source := fixedLocalVEXFixture(t, row.operation) + if row.name == "numeric pointer" { + fixedLocalRequireNumericControl(t, fn, sig, source, "func Value(p *uint64,a uint64) uint64", + row.operation, 0, 32, false, "load volatile <32 x i8>") + return + } + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved form acquired fixed-local permission", err) + } + }) + } +} + +func TestGoAMD64FixedLocalVEXUpperAliases(t *testing.T) { + for _, operation := range []string{ + "VPADDQ X0,X1,X2", "VPOR X0,X1,X2", "VPSHUFB X0,X1,X2", + "VPSRLQ $3,X1,X2", "VPALIGNR $3,X0,X1,X2", "VPBLENDD $3,X0,X1,X2", + "VPERM2F128 $3,Y0,Y1,Y2", "VPERM2I128 $3,Y0,Y1,Y2", + } { + t.Run(operation, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, operation) + state := x86FixedLocalState{vectors: map[Reg]uint64{ + "Z0": 0xffffffff, "Z1": 0xffffffff, "Z2": 0, + }, flags: x86ByteFlagCF} + if err := fixedLocalVectorStep(&ins, &state, nil, 0); err != nil { + t.Fatal(err) + } + if state.vectors["Z2"] != ^uint64(0) || state.vectors["Z0"] != 0xffffffff || + state.vectors["Z1"] != 0xffffffff || state.flags != x86ByteFlagCF { + t.Fatal("VEX did not define upper zeros independently of the old destination", state) + } + }) + } + for _, op := range []string{"VZEROALL", "VZEROUPPER"} { + t.Run(op, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, op) + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z0": 3, "Z16": 5}, flags: x86ByteFlagCF} + if err := fixedLocalStep(&ins, &state, FuncSig{}, nil, 0); err != nil { + t.Fatal(err) + } + want := ^uint64(0) + if op == "VZEROUPPER" { + want = ^uint64(65535) | 3 + } + if state.vectors["Z0"] != want || state.vectors["Z16"] != 5 || state.flags != x86ByteFlagCF { + t.Fatal("VZERO changed an unaffected low/high-bank byte or FLAGS", state) + } + }) + } +} + +func TestGoAMD64FixedLocalVEXBinaryInputAndAliasEffects(t *testing.T) { + names := make(map[string]bool) + for _, row := range fixedLocalVEXBinaryWitnesses(t) { + names[strings.Fields(row.source)[0]] = true + } + if len(names) != 31 { + t.Fatalf("official VEX binary members=%d, want 31", len(names)) + } + for name := range names { + for width, bank := range map[int]string{16: "X", 32: "Y"} { + t.Run(name+"/"+bank, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s %s0,%s1,%s2", name, bank, bank, bank)) + mask := x86CallerVectorByteMask(int64(width)) + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z0": mask, "Z1": mask}, flags: x86ByteFlagCF} + if err := fixedLocalVectorStep(&ins, &state, nil, 0); err != nil { + t.Fatal("complete VEX inputs rejected", err) + } + if state.vectors["Z2"] != ^uint64(0) || state.vectors["Z0"] != mask || + state.vectors["Z1"] != mask || state.flags != x86ByteFlagCF { + t.Fatal("VEX did not read inputs before defining result/upper zeros", state) + } + state.vectors["Z0"] &^= uint64(1) << (width - 1) + state.vectors["Z2"] = 0 + if err := fixedLocalVectorStep(&ins, &state, nil, 0); !errors.Is(err, ErrProbeNeedsContext) || + state.vectors["Z2"] != 0 { + t.Fatal("missing input byte published VEX result or upper-zero knowledge", err, state) + } + }) + } + } + for _, operation := range []string{ + "VPMULLQ X0,X1,X2", "VPMINSQ X0,X1,X2", "VPMINUQ Y0,Y1,Y2", + "VPMAXSQ X0,X1,X2", "VPMAXUQ Y0,Y1,Y2", + "VPSUBW Z0,Z1,Z2", "VPMULHW X16,X1,X2", "VPAVGB X0,X1,K1,X2", + "VPMADDWD.Z X0,X1,K1,X2", "VPMINSD.BCST (SI),X1,X2", + } { + t.Run(operation, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, operation) + if _, known, err := fixedLocalVectorByteEffect(ins); known && err == nil { + t.Fatal("separate EVEX form acquired a VEX effect before input checks") + } + fn, sig, _ := fixedLocalVEXFixture(t, operation) + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("separate EVEX width/mask/broadcast acquired VEX permission", err) + } + }) + } +} + +func TestGoAMD64FixedLocalVEXCompleteObjects(t *testing.T) { + fixedLocalCheckVEXObjects(t, fixedLocalVEXWitnesses(t)) +} + +func TestGoAMD64FixedLocalVEXBinaryCompleteObjects(t *testing.T) { + fixedLocalCheckVEXObjects(t, fixedLocalVEXBinaryWitnesses(t)) +} + +func fixedLocalCheckVEXObjects(t *testing.T, witnesses []fixedLocalVEXWitness) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + passed := 0 + for start := 0; start < len(witnesses); start += 32 { + end := start + 32 + if end > len(witnesses) { + end = len(witnesses) + } + if t.Run(fmt.Sprintf("batch%d", start/32), func(t *testing.T) { + var assembly, declarations strings.Builder + for index := start; index < end; index++ { + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(fixedLocalVEXAssembly(witnesses[index].source), "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p *uint64, a uint64) uint64\n", name) + } + const triple = "x86_64-unknown-linux-gnu" + pkg, request, _, options := fixedLocalTransportFixture(t, assembly.String(), declarations.String(), "linux/amd64", triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "vex.ll", "vex.o", ir) + compileLLVMToObject(t, llc, triple, "vex-opt.ll", "vex-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) { + passed += end - start + } + } + if passed != len(witnesses) { + t.Fatalf("three-API LLVM22 object witnesses passed=%d total=%d", passed, len(witnesses)) + } + t.Logf("all %d non-masked VEX source witnesses passed three APIs and LLVM22 objects; not execution evidence", passed) +} + +func BenchmarkGoAMD64FixedLocalVEXCandidate(b *testing.B) { + // This is the candidate query, not successful source proof, LLVM or + // native execution. No earlier scalar scratch store may short-circuit + // the register-only negative filter this benchmark measures. + source := "TEXT ·Value(SB),4,$256-0\n" + strings.Repeat("VPADDQ Y0,Y0,Y1\n", 32) + + "VMOVDQU 64(BP),Y0\nRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for index := 0; index < b.N; index++ { + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") { + b.Fatal("complete scratch source lost its candidate role") + } + } +} + +func TestGoAMD64FixedLocalVEXCandidateAllocations(t *testing.T) { + source := "TEXT ·Value(SB),4,$256-0\n" + strings.Repeat("VPADDQ Y0,Y0,Y1\n", 32) + + "VMOVDQU 64(BP),Y0\nRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") { + t.Fatal("complete scratch source lost its candidate role") + } + allocations := testing.AllocsPerRun(100, func() { + GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") + }) + if allocations != 0 { + t.Fatalf("candidate query allocated for register-only noncandidates: %g", allocations) + } +} diff --git a/amd64_fixed_local_vex_native_linux_amd64_test.go b/amd64_fixed_local_vex_native_linux_amd64_test.go new file mode 100644 index 000000000..6725da6dd --- /dev/null +++ b/amd64_fixed_local_vex_native_linux_amd64_test.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "os" + "path/filepath" + "testing" +) + +// This required native owner is never registered on a Windows/macOS host. +// Objects from those hosts cannot substitute for original Go execution. +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalVEX(t *testing.T) { + fixedLocalVEXRuntime(t, fixedLocalVEXNativeTools(t), true) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalShuffle(t *testing.T) { + kernels := fixedLocalShuffleRuntimeKernels() + if len(kernels) != 41 { + t.Fatal("legacy and VEX shuffle runtime inventory changed", len(kernels)) + } + fixedLocalVectorRuntime(t, fixedLocalVEXNativeTools(t), true, kernels) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalShift(t *testing.T) { + fixedLocalVectorRuntime(t, fixedLocalVEXNativeTools(t), true, fixedLocalShiftRuntimeKernels(t)) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalScalarLane(t *testing.T) { + fixedLocalVectorRuntime(t, fixedLocalVEXNativeTools(t), true, fixedLocalScalarLaneRuntimeKernels(t)) + x87IA32RequireNativeHost(t) +} + +func fixedLocalVEXNativeTools(t *testing.T) x86FloatCompareExecution { + t.Helper() + x87IA32RequireNativeHost(t) + execution := x86FloatCompareExecutionTools(t) + if len(execution.prefix) != 0 || execution.preserveArgv0 { + t.Fatal("fixed-frame VEX differential execution requires native AMD64") + } + directory := t.TempDir() + source, executable := filepath.Join(directory, "avx2.c"), filepath.Join(directory, "avx2") + const preflight = `#include +__attribute__((target("avx2"))) static int original_avx2(void) { + unsigned bits; + __asm__ volatile("vpcmpeqb %%ymm0,%%ymm0,%%ymm0; " + "vpmovmskb %%ymm0,%0; vzeroupper" + : "=r"(bits) :: "ymm0"); + return bits==0xffffffffu; +} +int main(void) { + __builtin_cpu_init(); + int avx=!!__builtin_cpu_supports("avx"); + int avx2=!!__builtin_cpu_supports("avx2"); + printf("fixed-frame native AVX=%d AVX2=%d\n",avx,avx2); + return avx && avx2 && original_avx2() ? 0 : 1; +} +` + if err := os.WriteFile(source, []byte(preflight), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + t.Logf("%s", execution.run(t, executable)) + return execution +} diff --git a/amd64_fixed_local_vex_runtime_test.go b/amd64_fixed_local_vex_runtime_test.go new file mode 100644 index 000000000..eebae066a --- /dev/null +++ b/amd64_fixed_local_vex_runtime_test.go @@ -0,0 +1,456 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +type fixedLocalVEXRuntimeKernel struct { + shape string + operations []string +} + +// A kernel owns one real register/width/immediate shape. Equivalent signed +// and unsigned immediate table rows share it; register/private/native memory +// sources and destination aliases remain separate observable operations. +func fixedLocalVEXRuntimeKernels(t *testing.T) []fixedLocalVEXRuntimeKernel { + t.Helper() + witnesses := append(fixedLocalVEXWitnesses(t), fixedLocalVEXBinaryWitnesses(t)...) + var kernels []fixedLocalVEXRuntimeKernel + rows, shapes := make(map[string]string), make(map[string]int) + seen := make(map[string]bool) + for _, witness := range witnesses { + parts := strings.SplitN(witness.source, " ", 2) + var args []string + if len(parts) == 2 { + args = strings.Split(parts[1], ",") + } + for index, argument := range args { + if strings.HasPrefix(argument, "$") { + args[index] = "$3" + if strings.HasPrefix(parts[0], "VPERM2") { + args[index] = "$49" + } else if parts[0] == "VPALIGNR" { + args[index] = "$17" + } + } + } + operation := strings.TrimSpace(parts[0] + " " + strings.Join(args, ",")) + owner := strings.Split(witness.owner, "/") + row := owner[0] + "/" + owner[1] + shape, exists := rows[row] + if !exists { + // The official witness builder emits the register choice first. + if strings.Contains(operation, "(") { + t.Fatal("runtime row lost its independent register witness", witness.owner) + } + shape, rows[row] = operation, operation + } + index, exists := shapes[shape] + if !exists { + index, shapes[shape] = len(kernels), len(kernels) + kernels = append(kernels, fixedLocalVEXRuntimeKernel{shape: shape}) + } + add := func(source string) { + key := shape + "\n" + source + if !seen[key] { + seen[key] = true + kernels[index].operations = append(kernels[index].operations, source) + } + } + add(operation) + if len(args) < 2 || (!strings.HasPrefix(args[len(args)-1], "X") && !strings.HasPrefix(args[len(args)-1], "Y")) { + continue + } + for _, argument := range args[:len(args)-1] { + if !strings.HasPrefix(argument, "X") && !strings.HasPrefix(argument, "Y") { + continue + } + alias := append([]string(nil), args...) + alias[len(alias)-1] = args[len(args)-1][:1] + argument[1:] + add(parts[0] + " " + strings.Join(alias, ",")) + } + } + return kernels +} + +// Each operation observes Y0..Y3 in full, including the upper 128 bits of an +// X destination, and all three scratch vectors (left canary/source/right). +// Input memory and the outer output canaries are printed by both drivers. +func fixedLocalVEXRuntimeProgram(kernels []fixedLocalVEXRuntimeKernel, triple string) (string, string, string, string) { + var assembly, declarations, goRows, cRows, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, kernel := range kernels { + name := fmt.Sprintf("Value%d", index) + fmt.Fprintf(&declarations, "func %s(out *uint64, input *[16]uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$256-16\nMOVQ out+0(FP),DI\nMOVQ input+8(FP),SI\n", name) + assembly.WriteString("MOVQ SP,BP\nADDQ $64,BP\nANDQ $-64,BP\n") + for observation, operation := range kernel.operations { + for register := 0; register < 4; register++ { + fmt.Fprintf(&assembly, "VMOVDQU %d(SI),Y%d\n", register*32, register) + } + assembly.WriteString("VMOVDQU Y3,32(BP)\nVMOVDQU Y0,64(BP)\nVMOVDQU Y2,96(BP)\n") + assembly.WriteString(operation + "\n") + for register := 0; register < 4; register++ { + fmt.Fprintf(&assembly, "VMOVDQU Y%d,%d(DI)\n", register, observation*224+register*32) + } + for vector := 0; vector < 3; vector++ { + fmt.Fprintf(&assembly, "VMOVDQU %d(BP),Y4\nVMOVDQU Y4,%d(DI)\n", + 32+vector*32, observation*224+128+vector*32) + } + } + assembly.WriteString("VZEROUPPER\nRET\n") + fmt.Fprintf(&goRows, "{%s,%d},\n", name, len(kernel.operations)*28) + fmt.Fprintf(&cRows, "{value%d,%d},\n", index, len(kernel.operations)*28) + label := runtimeCAsmLabel(prefix + "example.invalid/framefixture." + name) + fmt.Fprintf(&cDeclarations, "extern void value%d(uint64_t*,uint64_t*) __asm__(%s);\n", index, label) + } + const values = `0,1,7,15,16,31,32,63,64,65,0x7f7f7f7f7f7f7f7f, +0x8080808080808080,0x7fff7fff7fff7fff,0x8000800080008000, +0xffffffffffffffff,0x0123456789abcdef` + original := `package main +import ("fmt"; "unsafe") +` + declarations.String() + ` +func main() { + rows := []struct { call func(*uint64,*[16]uint64); words int }{ +` + goRows.String() + ` + } + values := []uint64{` + values + `} + for row, entry := range rows { + for sample := range values { + var storage [24]uint64 + input := (*[16]uint64)(unsafe.Pointer((uintptr(unsafe.Pointer(&storage[0]))+63)&^uintptr(63))) + for index := range input { + input[index] = values[(sample+index)%len(values)] + } + output := make([]uint64,entry.words+8) + for index := range output { + output[index] = 0xc3c3c3c3c3c3c300+uint64(index) + } + entry.call(&output[4],input) + fmt.Printf("%d %d",row,sample) + for _, value := range output { + fmt.Printf(" %016x",value) + } + for _, value := range input { + fmt.Printf(" %016x",value) + } + fmt.Println() + } + } +} +` + driver := `#include +#include +#include +` + runtimeCAsmLabelPreamble + cDeclarations.String() + ` +int main(void) { + const struct { void (*call)(uint64_t*,uint64_t*); size_t words; } rows[] = { +` + cRows.String() + ` + }; + const uint64_t values[] = {` + values + `}; + const size_t samples = sizeof(values)/sizeof(values[0]); + for (size_t row = 0; row < sizeof(rows)/sizeof(rows[0]); row++) { + for (size_t sample = 0; sample < samples; sample++) { + _Alignas(64) uint64_t input[16]; + for (size_t index = 0; index < 16; index++) { + input[index] = values[(sample+index)%samples]; + } + const size_t words = rows[row].words+8; + uint64_t *output = malloc(words*sizeof(*output)); + if (!output) return 2; + for (size_t index = 0; index < words; index++) { + output[index] = UINT64_C(0xc3c3c3c3c3c3c300)+index; + } + rows[row].call(output+4,input); + printf("%zu %zu",row,sample); + for (size_t index = 0; index < words; index++) { + printf(" %016llx",(unsigned long long)output[index]); + } + for (size_t index = 0; index < 16; index++) { + printf(" %016llx",(unsigned long long)input[index]); + } + puts(""); + free(output); + } + } + return 0; +} +` + return assembly.String(), declarations.String(), original, driver +} + +func fixedLocalVEXRuntime(t *testing.T, execution x86FloatCompareExecution, execute bool) { + t.Helper() + fixedLocalVectorRuntime(t, execution, execute, fixedLocalVEXRuntimeKernels(t)) +} + +func fixedLocalVectorRuntime(t *testing.T, execution x86FloatCompareExecution, execute bool, kernels []fixedLocalVEXRuntimeKernel) { + t.Helper() + goAMD64FramePrepareOwnedToolCache(t) + for start := 0; start < len(kernels); start += 32 { + end := start + 32 + if end > len(kernels) { + end = len(kernels) + } + t.Run(fmt.Sprintf("batch%d", start/32), func(t *testing.T) { + assembly, declarations, original, driver := fixedLocalVEXRuntimeProgram(kernels[start:end], execution.triple) + var want []string + if execute { + output := x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original) + want = strings.Split(strings.TrimSpace(string(output)), "\n") + if len(want) != (end-start)*16 { + t.Fatalf("original Go emitted %d observations, want %d", len(want), (end-start)*16) + } + } else { + x86COMICompileOriginalGo(t, map[string]string{ + "go.mod": "module vexframe\ngo 1.20\n", "main.go": original, "original_amd64.s": assembly, + }, false) + } + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, + runtime.GOOS+"/amd64", execution.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + if !execute { + execution.compileLLVM(t, "private-vex", input, driver, level) + return + } + output := execution.compileAndRunLLVM(t, "private-vex", input, driver, level) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM emitted %d observations, original Go %d", len(got), len(want)) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("kernel=%s sample=%d original=%s LLVM=%s", kernels[start+index/16].shape, + index%16, want[index], got[index]) + } + } + }) + } + }) + }) + } + t.Logf("%d vector shape kernels, 16 input patterns; original Go and three LLVM APIs execute=%v", len(kernels), execute) +} + +// Immediate-domain tests cover all selectors separately. These kernels compare +// original execution for four distinct selector patterns, both widths, memory and +// alias forms, and preserved/zeroed upper bytes. Memory canaries remain visible. +func fixedLocalShuffleRuntimeKernels() []fixedLocalVEXRuntimeKernel { + var kernels []fixedLocalVEXRuntimeKernel + names := []string{"PSHUFB", "PSHUFD", "PSHUFL", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} + for _, name := range names { + banks := []string{"X"} + if strings.HasPrefix(name, "V") { + banks = append(banks, "Y") + } + selectors := []string{"0", "27", "78", "255"} + if name == "PSHUFB" { + selectors = []string{""} + } + for _, bank := range banks { + for _, selector := range selectors { + prefix := name + " " + if selector != "" { + prefix += "$" + selector + "," + } + operations := []string{ + prefix + bank + "0," + bank + "2", + prefix + bank + "0," + bank + "0", + prefix + "64(BP)," + bank + "2", + prefix + "0(SI)," + bank + "2", + } + if strings.HasPrefix(name, "V") { + operations = append(operations, prefix+"65(BP),"+bank+"2", prefix+"1(SI),"+bank+"2") + } + kernels = append(kernels, fixedLocalVEXRuntimeKernel{shape: operations[0], operations: operations}) + } + } + } + return kernels +} + +func TestGoAMD64FixedLocalShuffleRuntimeObjects(t *testing.T) { + kernels := fixedLocalShuffleRuntimeKernels() + if len(kernels) != 41 { + t.Fatal("legacy and VEX shuffle runtime inventory changed", len(kernels)) + } + fixedLocalVectorRuntime(t, fixedLocalPackedCompileOnlyTools(t), false, kernels) +} + +func fixedLocalCLMULRuntimeKernels(t *testing.T) []fixedLocalVEXRuntimeKernel { + t.Helper() + var kernels []fixedLocalVEXRuntimeKernel + operations := 0 + for _, kind := range []string{"legacy", "X", "Y"} { + for _, selector := range []int{0, 1, 16, 17, 238, 255} { + var rows []string + if kind == "legacy" { + for _, source := range []string{"X1", "X2", "64(BP)", "(SI)"} { + rows = append(rows, fmt.Sprintf("PCLMULQDQ $%d,%s,X2", selector, source)) + } + } else { + for _, source := range []string{kind + "1", "64(BP)", "(SI)", "65(BP)", "1(SI)"} { + rows = append(rows, fmt.Sprintf("VPCLMULQDQ $%d,%s,%s2,%s3", selector, source, kind, kind)) + } + for _, destination := range []string{kind + "1", kind + "2"} { + rows = append(rows, fmt.Sprintf("VPCLMULQDQ $%d,%s1,%s2,%s", selector, kind, kind, destination)) + } + for _, destination := range []string{kind + "3", kind + "2"} { + rows = append(rows, fmt.Sprintf("VPCLMULQDQ $%d,%s2,%s2,%s", selector, kind, kind, destination)) + } + } + kernels = append(kernels, fixedLocalVEXRuntimeKernel{shape: rows[0], operations: rows}) + operations += len(rows) + } + } + if len(kernels) != 18 || operations != 132 { + t.Fatal("CLMUL original-execution inventory changed", len(kernels), operations) + } + return kernels +} + +// AVX2 does not imply VPCLMULQDQ's Y-width instruction. The native owner +// checks both features and executes actual legacy/X and VEX/Y instructions; +// compile-only hosts build this preflight without claiming CPU availability. +func fixedLocalCLMULRuntimePreflight(t *testing.T, execution x86FloatCompareExecution) string { + t.Helper() + directory := t.TempDir() + source, executable := filepath.Join(directory, "clmul.c"), filepath.Join(directory, "clmul") + const program = `#include +__attribute__((target("avx2,pclmul,vpclmulqdq"))) static int original_clmul(void) { + unsigned legacy, wide; + __asm__ volatile("pxor %%xmm0,%%xmm0; " + "pclmulqdq $0,%%xmm0,%%xmm0; pmovmskb %%xmm0,%0; " + "vpxor %%ymm0,%%ymm0,%%ymm0; " + "vpclmulqdq $0,%%ymm0,%%ymm0,%%ymm0; " + "vpmovmskb %%ymm0,%1; vzeroupper" + : "=r"(legacy), "=r"(wide) :: "ymm0"); + return legacy==0 && wide==0; +} +int main(void) { + __builtin_cpu_init(); + int pclmul=!!__builtin_cpu_supports("pclmul"); + int vpclmul=!!__builtin_cpu_supports("vpclmulqdq"); + printf("fixed-frame native PCLMUL=%d VPCLMULQDQ=%d\n",pclmul,vpclmul); + return pclmul && vpclmul && original_clmul() ? 0 : 1; +} +` + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + return executable +} + +func TestGoAMD64FixedLocalCLMULRuntimeObjects(t *testing.T) { + execution := fixedLocalPackedCompileOnlyTools(t) + fixedLocalCLMULRuntimePreflight(t, execution) + fixedLocalVectorRuntime(t, execution, false, fixedLocalCLMULRuntimeKernels(t)) +} + +func TestGoAMD64FixedLocalVEXRuntimeObjects(t *testing.T) { + fixedLocalVEXRuntime(t, fixedLocalPackedCompileOnlyTools(t), false) +} + +func TestGoAMD64FixedLocalVEXRuntimeInventory(t *testing.T) { + kernels := fixedLocalVEXRuntimeKernels(t) + var witnesses []fixedLocalVEXWitness + for _, kernel := range kernels { + if len(kernel.operations) == 0 { + t.Fatal("runtime kernel has no observable operation", kernel.shape) + } + for _, source := range kernel.operations { + witnesses = append(witnesses, fixedLocalVEXWitness{owner: kernel.shape, source: source}) + } + } + fixedLocalCheckVEXEncodings(t, witnesses) + t.Logf("540 official source witnesses grouped into %d real shape kernels, %d original-Go VEX operations including destination aliases", + len(kernels), len(witnesses)) +} + +func fixedLocalShiftRuntimeKernels(t *testing.T) []fixedLocalVEXRuntimeKernel { + t.Helper() + var kernels []fixedLocalVEXRuntimeKernel + operations := 0 + for index, name := range fixedLocalLegacyShiftNames() { + for _, count := range []int{0, 1, 15, 127} { + operation := fmt.Sprintf("%s $%d,X2", name, count) + kernels = append(kernels, fixedLocalVEXRuntimeKernel{shape: operation, operations: []string{operation}}) + operations++ + } + if index >= 8 { + continue // Whole-byte shifts have only an immediate count. + } + var rows []string + for _, count := range []string{"X0", "X2", "64(BP)", "(SI)"} { + rows = append(rows, name+" "+count+",X2") + } + kernels = append(kernels, fixedLocalVEXRuntimeKernel{shape: rows[0], operations: rows}) + operations += len(rows) + } + if len(kernels) != 56 || operations != 80 { + t.Fatal("complete legacy shift runtime inventory changed", len(kernels), operations) + } + return kernels +} + +func TestGoAMD64FixedLocalShiftRuntimeObjects(t *testing.T) { + fixedLocalVectorRuntime(t, fixedLocalPackedCompileOnlyTools(t), false, fixedLocalShiftRuntimeKernels(t)) +} + +func fixedLocalScalarLaneRuntimeKernels(t *testing.T) []fixedLocalVEXRuntimeKernel { + t.Helper() + rows := fixedLocalScalarLaneCases() + var kernels []fixedLocalVEXRuntimeKernel + owners := make(map[string]int) + for _, row := range rows { + first, _, _ := strings.Cut(row, "\n") + name, operands, _ := strings.Cut(first, " ") + index, exists := owners[name] + if !exists { + index, owners[name] = len(kernels), len(kernels) + kernels = append(kernels, fixedLocalVEXRuntimeKernel{shape: name}) + } + operation := "MOVQ (SI),AX\n" + row + if strings.Contains(name, "INSR") { + args := strings.Split(operands, ",") + extract := strings.Replace(name, "INSR", "EXTR", 1) + operation += fmt.Sprintf("\n%s %s,%s,AX", extract, args[0], args[len(args)-1]) + } + // The existing scratch observation includes this separate full-GP + // result slot. It exposes the high zeros of byte/word/dword extracts. + // Y0 immediately observes native stores; Y1/Y2 retain the insert's + // lane and upper-alias effects. Private transfers use bytes 65..72, + // so neither the GP slot nor its surrounding canaries overlap them. + operation += "\nMOVQ AX,88(BP)\nVMOVDQU (SI),Y0" + kernels[index].operations = append(kernels[index].operations, operation) + } + if len(rows) != 132 || len(kernels) != 16 { + t.Fatal("complete scalar-lane original-execution inventory changed", len(rows), len(kernels)) + } + return kernels +} + +func TestGoAMD64FixedLocalScalarLaneRuntimeObjects(t *testing.T) { + fixedLocalVectorRuntime(t, fixedLocalPackedCompileOnlyTools(t), false, fixedLocalScalarLaneRuntimeKernels(t)) +} diff --git a/amd64_fixed_local_vex_shuffle_test.go b/amd64_fixed_local_vex_shuffle_test.go new file mode 100644 index 000000000..9ebaab477 --- /dev/null +++ b/amd64_fixed_local_vex_shuffle_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalVEXShuffleWitnesses(t *testing.T) []fixedLocalVEXWitness { + t.Helper() + // This existing audit checks all 30 VEX/EVEX rows, including the + // dword-only broadcast attribute; only 12 VEX rows enter this scope. + x86OfficialImmediateShuffleRows(t) + return fixedLocalVEXFamilyWitnesses(t, `^VPSHUF(D|HW|LW)$`, 12, 36) +} + +func TestGoAMD64FixedLocalVEXShuffleOfficialEncodings(t *testing.T) { + fixedLocalCheckVEXEncodings(t, fixedLocalVEXShuffleWitnesses(t)) +} + +func TestGoAMD64FixedLocalVEXShufflePhysicalBoundaries(t *testing.T) { + fixedLocalCheckVEXPhysicalBoundaries(t, fixedLocalVEXShuffleWitnesses(t)) +} + +func TestGoAMD64FixedLocalVEXShuffleCompleteFamilies(t *testing.T) { + fixedLocalCheckVEXFamilies(t, fixedLocalVEXShuffleWitnesses(t)) + for _, name := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for _, bank := range []string{"X", "Y"} { + t.Run(name+"/unaligned/"+bank, func(t *testing.T) { + fn, sig, _ := fixedLocalVEXFixture(t, name+" $255,65(BP),"+bank+"2") + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("VEX full tuple incorrectly inherited legacy alignment", err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalVEXShuffleImmediateDomainAndUpperZeros(t *testing.T) { + for _, name := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for _, bank := range []string{"X", "Y"} { + for _, alias := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/alias=%t", name, bank, alias), func(t *testing.T) { + destination := bank + "2" + if alias { + destination = bank + "1" + } + width := 16 + if bank == "Y" { + width = 32 + } + input := uint64(0x8040201000000000) | x86CallerVectorByteMask(int64(width)) + for immediate := -128; immediate <= 255; immediate++ { + operation := fmt.Sprintf("%s $%d,%s1,%s", name, immediate, bank, destination) + ins := x86YMMReadSource(t, operation) + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z1": input, "Z2": 0}, flags: x86ByteFlagsAll} + if err := fixedLocalVectorStep(&ins, &state, nil, 0); err != nil { + t.Fatal("complete VEX immediate domain or source-only dependency rejected", operation, err) + } + if state.vectors[x86CallerVectorRoot(Reg(destination))] != ^uint64(0) || + !alias && state.vectors["Z1"] != input || state.flags != x86ByteFlagsAll { + t.Fatal("VEX shuffle failed upper-zero/source/FLAGS contract", operation, state) + } + } + }) + } + } + } +} + +func TestGoAMD64FixedLocalVEXShuffleBoundaries(t *testing.T) { + for _, name := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + t.Run(name+"/dispatch identity", func(t *testing.T) { + ins := x86YMMReadSource(t, "PSHUFD $0,X1,X2") + if _, known, err := x86UnmaskedVEXByteEffect(Op(name), ins, "amd64"); !known || err == nil { + t.Fatal("VEX dispatch changed a legacy source's upper-write contract", known, err) + } + }) + for _, suffix := range []string{ + " $0,Z1,Z2", " $0,X16,X2", " $0,X1,X16", " $0,X1,K1,X2", + ".Z $0,X1,K1,X2", ".BCST $0,(SI),X2", " $0,M0,M1", + " $-129,X1,X2", " $256,X1,X2", " $0.5,X1,X2", " $missing,X1,X2", + } { + operation := name + suffix + t.Run(operation, func(t *testing.T) { + ins := x86YMMReadSource(t, operation) + if effect, known, err := fixedLocalVectorByteEffect(ins); known && err == nil { + t.Fatal("separate EVEX/MMX or invalid syntax acquired VEX metadata", operation, effect) + } + }) + } + for _, args := range []string{ + "$0,X7,X2", "$0,184(BP),X2", "$0,184(BP),Y2", "$0,256(BP),X2", + "$0,(AX),X2", "$0,a+8(FP),X2", + } { + t.Run(name+"/unproved/"+args, func(t *testing.T) { + operation := name + " " + args + fn, sig, source := fixedLocalVEXFixture(t, operation) + if args == "$0,(AX),X2" { + fixedLocalRequireNumericControl(t, fn, sig, source, "func Value(p *uint64,a uint64) uint64", + operation, 1, 16, false, "shufflevector <16 x i8>") + return + } + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized vector/private extent/numerical address gained backing", err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalVEXShuffleObjects(t *testing.T) { + witnesses := fixedLocalVEXShuffleWitnesses(t) + for _, name := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for _, bank := range []string{"X", "Y"} { + witnesses = append(witnesses, fixedLocalVEXWitness{source: name + " $255,65(BP)," + bank + "2"}) + } + } + if len(witnesses) != 42 { + t.Fatal("complete VEX shuffle source denominator changed", len(witnesses)) + } + var assembly, declarations strings.Builder + for index, row := range witnesses { + source := fixedLocalVEXAssembly(row.source) + changed := strings.Replace(source, "VMOVDQU 64(BP),Y2\n", "", 1) + if changed == source { + t.Fatal("old destination removal did not change complete source") + } + name := fmt.Sprintf("Value%d", index) + assembly.WriteString(strings.Replace(changed, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(p *uint64,a uint64) uint64\n", name) + } + fixedLocalCompleteVectorObjects(t, assembly.String(), declarations.String(), len(witnesses)) +} diff --git a/amd64_fixed_local_vex_zero_runtime_linux_amd64_test.go b/amd64_fixed_local_vex_zero_runtime_linux_amd64_test.go new file mode 100644 index 000000000..047d25d42 --- /dev/null +++ b/amd64_fixed_local_vex_zero_runtime_linux_amd64_test.go @@ -0,0 +1,27 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalVEXLogicalZero(t *testing.T) { + // The Y-width integer identities require AVX2. The existing native probe + // checks the CPU, OS register state and actual AVX2 execution before Go. + fixedLocalVEXNativeTools(t) + assembly, declarations, count := fixedLocalVEXZeroProgram(t) + inputs := []uint64{0, 1, ^uint64(0), 0x0123456789abcdef} + var expected [][]string + for _, input := range inputs { + values := make([]string, count) + for index := range values { + // All four Y qwords must be zero, including an X result's upper + // alias. Only the independently stored input canary contributes. + values[index] = fmt.Sprintf("%016x", input) + } + expected = append(expected, values) + } + fixedLocalResultsRuntimeISA(t, assembly, declarations, inputs, expected, x86FloatCompareVEX) +} diff --git a/amd64_fixed_local_vex_zero_test.go b/amd64_fixed_local_vex_zero_test.go new file mode 100644 index 000000000..567ee5540 --- /dev/null +++ b/amd64_fixed_local_vex_zero_test.go @@ -0,0 +1,154 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +type fixedLocalVEXZeroCase struct { + instruction string + bank string + source, destination int +} + +func fixedLocalVEXZeroCases() []fixedLocalVEXZeroCase { + var cases []fixedLocalVEXZeroCase + for _, name := range x86LegacyLogicalMemoryNames { + if !fixedLocalLogicalZeroName(name) { + continue + } + for _, bank := range []string{"X", "Y"} { + for source := 0; source < 16; source++ { + for _, destination := range []int{source, (source + 1) % 16} { + instruction := fmt.Sprintf("V%s %s%d,%s%d,%s%d", name, bank, source, bank, source, bank, destination) + cases = append(cases, fixedLocalVEXZeroCase{instruction, bank, source, destination}) + } + } + } + } + return cases +} + +func fixedLocalVEXZeroBody(row fixedLocalVEXZeroCase) string { + // Read the complete Y alias even after an X-width zero, then consume all + // four qwords and the separately initialized input canary at 0(SP). + // OR prevents incorrect nonzero lanes from cancelling one another. + return row.instruction + fmt.Sprintf("\nVMOVDQU Y%d,16(SP)\n", row.destination) + + "MOVQ 16(SP),AX\nORQ 24(SP),AX\nORQ 32(SP),AX\nORQ 40(SP),AX\nADDQ 0(SP),AX" +} + +func TestGoAMD64FixedLocalVEXLogicalZeroOfficialRows(t *testing.T) { + witnesses := fixedLocalVEXFamilyWitnesses(t, + `^V(ANDN?P[DS]|ORP[DS]|XORP[DS]|PANDN?|POR|PXOR)$`, 24, 72) + fixedLocalCheckVEXEncodings(t, witnesses) + fixedLocalCheckVEXPhysicalBoundaries(t, witnesses) +} + +func TestGoAMD64FixedLocalVEXLogicalZeroComplete(t *testing.T) { + cases := fixedLocalVEXZeroCases() + if len(cases) != 384 { + t.Fatal("six zero identities by two widths, 16 sources and two destination aliases changed", len(cases)) + } + var original strings.Builder + for index, row := range cases { + _, _, source := fixedLocalGPFixture(t, fixedLocalVEXZeroBody(row)) + original.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) + for _, row := range cases { + t.Run(row.instruction, func(t *testing.T) { + fn, sig, _ := fixedLocalGPFixture(t, fixedLocalVEXZeroBody(row)) + if _, err := proveX86FixedLocal(fn, sig); err != nil { + t.Fatal("same-input logical zero acquired an undefined-input obligation", err) + } + }) + } +} + +func TestGoAMD64FixedLocalVEXLogicalZeroConsumers(t *testing.T) { + for _, name := range x86LegacyLogicalMemoryNames { + for _, bank := range []string{"X", "Y"} { + for _, first := range []int{0, 1} { + for _, destination := range []int{0, 2} { + instruction := fmt.Sprintf("V%s %s%d,%s0,%s%d", name, bank, first, bank, bank, destination) + zero := first == 0 && fixedLocalLogicalZeroName(name) + t.Run(instruction, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, instruction) + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value() uint64", + instruction+fmt.Sprintf("\nMOVQ X%d,AX\nMOVQ AX,ret0+0(FP)\nRET", destination)) + resultErr := checkX86ScalarFPResultInitialization(file.Funcs[0], sig, "amd64") + if zero && resultErr != nil || !zero && !errors.Is(resultErr, ErrProbeNeedsContext) { + t.Fatal("ordinary result consumer changed its logical dependency", resultErr) + } + effect, known, err := x86UnmaskedVEXByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || len(effect.operands) != 3 { + t.Fatal("complete VEX logical source lacks its typed effect", known, err) + } + for _, operand := range effect.operands { + if operand.read && zero { + t.Fatal("a logical zero still reads undefined register bytes", operand) + } + } + state := x86FixedLocalState{vectors: map[Reg]uint64{"Z0": 0, "Z1": 0, "Z2": 0, "Z16": 0xa55a}, flags: x86ByteFlagsAll} + before := cloneFixedLocalState(state) + err = fixedLocalVectorStep(&ins, &state, nil, 0) + if !zero { + if !errors.Is(err, ErrProbeNeedsContext) || !reflect.DeepEqual(state.vectors, before.vectors) || state.flags != before.flags { + t.Fatal("nonzero logical input was invented or partially published", err, state) + } + return + } + before.vectors[Reg(fmt.Sprintf("Z%d", destination))] = ^uint64(0) + if err != nil || !reflect.DeepEqual(state.vectors, before.vectors) || state.flags != before.flags { + t.Fatal("logical zero changed a source alias, upper bytes or FLAGS", err, state) + } + }) + } + } + } + } +} + +func TestGoAMD64FixedLocalVEXLogicalZeroKeepsMemoryAndEncodingRoles(t *testing.T) { + for _, row := range []string{ + "VPXOR 16(SP),Y0,Y0", "VANDNPS 16(SP),Y0,Y0", + "VXORPS X16,X16,X16", "VXORPS Y16,Y16,Y16", "VXORPS Z0,Z0,Z0", + "KMOVQ AX,K1\nVXORPS X0,X0,K1,X1", "KMOVQ AX,K1\nVXORPS.Z X0,X0,K1,X1", + } { + t.Run(row, func(t *testing.T) { + fn, sig, source := fixedLocalGPFixture(t, row+"\nMOVQ $0,AX") + requireX86GoAssemblerResult(t, "amd64", source, true) + if _, err := proveX86FixedLocal(fn, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("logical zero granted memory, mask or high-bank authority", err) + } + }) + } +} + +func fixedLocalVEXZeroProgram(t *testing.T) (string, string, int) { + t.Helper() + var assembly, declarations strings.Builder + count := 0 + for _, row := range fixedLocalVEXZeroCases() { + if row.source != 15 { + continue + } + _, _, source := fixedLocalGPFixture(t, fixedLocalVEXZeroBody(row)) + name := fmt.Sprintf("Value%d", count) + assembly.WriteString(strings.Replace(source, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + count++ + } + if count != 24 { + t.Fatal("six zero identities by two widths and two destination aliases changed", count) + } + return assembly.String(), declarations.String(), count +} + +func TestGoAMD64FixedLocalVEXLogicalZeroObjects(t *testing.T) { + assembly, declarations, count := fixedLocalVEXZeroProgram(t) + fixedLocalCompleteVectorObjects(t, assembly, declarations, count) +} diff --git a/amd64_fixed_local_void_test.go b/amd64_fixed_local_void_test.go new file mode 100644 index 000000000..ea410b05d --- /dev/null +++ b/amd64_fixed_local_void_test.go @@ -0,0 +1,352 @@ +package plan9asm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "strings" + "testing" +) + +func fixedLocalVoidFixture(t *testing.T, operation string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options) { + t.Helper() + original, _, _ := x86PrivateAlignedFrameAuditFixture(t, operation) + assembly := strings.Replace(original, "$160-40", "$160-32", 1) + assembly = strings.Replace(assembly, "MOVQ AX,ret+32(FP)\n", "", 1) + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a, b, c, d uint64)", assembly) + sig, err := DeriveGoFuncABI0Signature(pkg.Path+".Value", pkg.Types.Scope().Lookup("Value").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + return pkg, request, file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) }, + Sigs: map[string]FuncSig{sig.Name: sig}, + } +} + +func fixedLocalVoidAssembly(file *File) string { + var source strings.Builder + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + source.WriteString(ins.Raw + "\n") + } + } + return source.String() +} + +func TestGoAMD64FixedLocalVoidActualTargetBoundaries(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ goTarget, triple string }{ + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + {"linux/386", "i386-unknown-linux-gnu"}, + {"windows/386", "i686-pc-windows-msvc"}, + } { + t.Run(target.goTarget, func(t *testing.T) { + pkg, request, file, options := fixedLocalVoidFixture(t, "PADDQ 16(SP),X0") + request.Target, options.TargetTriple = target.goTarget, target.triple + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if strings.HasSuffix(target.goTarget, "/386") || target.goTarget == "windows/amd64" && err != nil { + if source != nil || !errors.Is(err, ErrProbeNeedsContext) || + target.goTarget == "windows/amd64" && !strings.Contains(err.Error(), "SSEHUNWINDINFO") { + t.Fatalf("unclosed target unexpectedly acquired void backing: %v", err) + } + t.Logf("unchanged original target observation boundary, not an LLVM object: %v", err) + return + } + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "void-target.ll", "void-target.o", ir) + }) + }) + } +} + +func TestGoAMD64FixedLocalVoidAllPathMemoryAndReturnNegatives(t *testing.T) { + pkg, _, file, options := fixedLocalVoidFixture(t, "PADDQ 16(SP),X0") + base, sig := fixedLocalVoidAssembly(file), options.Sigs[pkg.Path+".Value"] + mutations := map[string]string{ + "SP not restored": strings.Replace(base, "MOVQ R11,SP", "MOVQ $0,R10", 1), + "incoming BP read": strings.Replace(base, "MOVQ AX,16(SP)", "MOVQ BP,AX\nMOVQ AX,16(SP)", 1), + "incoming R14 read": strings.Replace(base, "MOVQ AX,16(SP)", "MOVQ R14,AX\nMOVQ AX,16(SP)", 1), + "input FP store": strings.Replace(base, "MOVQ X0,AX", "MOVQ AX,a+0(FP)", 1), + "private FP escape": strings.Replace(base, "MOVQ X0,AX", "MOVQ R11,a+0(FP)", 1), + "restored input store": strings.Replace(base, "MOVQ R11,SP\nRET", "MOVQ R11,SP\nMOVQ AX,a+0(FP)\nRET", 1), + "restored FP escape": strings.Replace(base, "MOVQ R11,SP\nRET", "MOVQ R11,SP\nMOVQ R11,a+0(FP)\nRET", 1), + "external escape": strings.Replace(base, "MOVQ X0,AX", "MOVQ R11,(AX)", 1), + "unbound local alias": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 16(SI)", 1), + "private read extent": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 152(SP)", 1), + "saved BP overlap": strings.Replace(base, "MOVQ AX,16(SP)", "MOVQ AX,160(SP)", 1), + "bad alignment": strings.Replace(base, "ADDQ $32,SP", "ADDQ $33,SP", 1), + "pointer as numeric": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 96(SP)", 1), + "pointer spill torn": strings.Replace(base, "MOVQ 96(SP),R11", "MOVB $1,96(SP)\nMOVQ 96(SP),R11", 1), + "raw instruction": strings.Replace(base, "PADDQ", "BYTE $0x90\nPADDQ", 1), + "unknown native call": strings.Replace(base, "PADDQ", "CALL ·Other(SB)\nPADDQ", 1), + "unknown dead effect": base + "SYSCALL\n", + "loop unknown input": strings.Replace(base, "MOVQ AX,16(SP)", "again:\nCMPQ AX,R10\nJNE again\nMOVQ AX,16(SP)", 1), + "overflow address": strings.Replace(base, "ADDQ $32,SP", "ADDQ $9223372036854775807,SP", 1), + } + for missing := 0; missing < 16; missing++ { + var stores strings.Builder + for index := 0; index < 16; index++ { + if index != missing { + fmt.Fprintf(&stores, "MOVB $1,%d(SP)\n", 16+index) + } + } + mutations[fmt.Sprintf("missing local byte %02d", missing)] = strings.Replace(base, "MOVQ AX,16(SP)\nMOVQ CX,24(SP)\n", stores.String(), 1) + } + for name, source := range mutations { + t.Run(name, func(t *testing.T) { + changed, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(changed.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("void return bypassed complete source safety: %v", err) + } + }) + } + for _, mutation := range []string{"nonvoid no result", "void result slot", "empty return", "multiple result slots"} { + t.Run(mutation, func(t *testing.T) { + changed := sig + switch mutation { + case "nonvoid no result": + changed.Ret = I64 + case "empty return": + changed.Ret = "" + case "void result slot": + changed.Frame.Results = []FrameSlot{{Offset: 32, Type: I64}} + case "multiple result slots": + changed.Ret = I64 + changed.Frame.Results = []FrameSlot{{Offset: 32, Type: I64}, {Offset: 40, Type: I64}} + } + if _, err := proveX86FixedLocal(file.Funcs[0], changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("inconsistent result contract obtained a void exemption: %v", err) + } + }) + } +} + +func TestGoAMD64FixedLocalVoidActualDAGInitialization(t *testing.T) { + pkg, request, file, options := fixedLocalVoidFixture(t, "PADDQ 16(SP),X0") + base := fixedLocalVoidAssembly(file) + for _, missing := range []bool{false, true} { + t.Run(fmt.Sprintf("missing=%v", missing), func(t *testing.T) { + join := "MOVQ CX,24(SP)\nCMPQ AX,CX\nJEQ joined\nMOVQ CX,24(SP)\njoined:" + if missing { + join = "CMPQ AX,CX\nJEQ joined\nMOVQ CX,24(SP)\njoined:" + } + assembly := strings.Replace(base, "MOVQ CX,24(SP)", join, 1) + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + actual, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + // Complete actual Go objects accept both bodies. Only the all-path + // initialized body may obtain the separate private-byte certificate. + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, actual, options) + if missing { + if !errors.Is(err, ErrProbeNeedsContext) || len(bound.Sigs) != 0 { + t.Fatalf("void DAG bypassed a missing predecessor write: %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, actual, bound, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "fixed_local_native") { + t.Fatalf("actual initialized void DAG did not consume its certificate: %v", err) + } + }) + }) + } +} + +func TestGoAMD64FixedLocalVoidCompositeNeedsCompleteBinding(t *testing.T) { + for _, test := range []struct { + name, declaration string + args, n int64 + }{ + {"pointer", "func Value(a *uint64, n uint64)", 16, 8}, + {"slice", "func Value(a []byte, n uint64)", 32, 24}, + {"array pointer", "func Value(a *[16]byte, n uint64)", 16, 8}, + } { + t.Run(test.name, func(t *testing.T) { + assembly := fixedLocalTransportAssembly("Value", int(test.args), "") + assembly = strings.Replace(assembly, "n+40(FP)", fmt.Sprintf("n+%d(FP)", test.n), 1) + pkg, request := goAMD64FrameTypedFixture(t, test.declaration, assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + observed := source.functions[pkg.Path+".Value"] + if observed.transport != goAMD64FrameCompositeTransport || observed.signature.Ret != Void || len(observed.code) == 0 { + t.Fatal("void composite observation lost its original role/object") + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + options := Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) }, + Sigs: map[string]FuncSig{observed.signature.Name: observed.signature}, + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + fixedLocalAPIs(t, pkg, file, options, fixedLocalRequireContext) + if bound, err := source.BindFixedLocal(ctx, pkg, file, options); err != nil || len(bound.Sigs) != 1 { + t.Fatalf("complete void composite transport lacks its independent frame proof: %v", err) + } + }) + } +} + +func TestGoAMD64FixedLocalVoidThreeAPIsRequireExactLiveWitness(t *testing.T) { + pkg, request, file, options := fixedLocalVoidFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + name := pkg.Path + ".Value" + for _, mutation := range []string{"JSON", "return", "input slot", "source"} { + t.Run(mutation, func(t *testing.T) { + changed := bound + changed.Sigs, err = cloneFixedLocalSigs(bound.Sigs) + if err != nil { + t.Fatal(err) + } + sig := changed.Sigs[name] + if mutation != "JSON" { + sig.x86FixedLocal = bound.Sigs[name].x86FixedLocal + } + changedFile := *file + changedFile.Funcs = append([]Func(nil), file.Funcs...) + switch mutation { + case "return": + sig.Ret = I64 + case "input slot": + sig.Frame.Params[0].Offset++ + case "source": + changedFile.Funcs[0].FrameSize += 8 + changedFile.Funcs[0].Instrs = append([]Instr(nil), file.Funcs[0].Instrs...) + changedFile.Funcs[0].Instrs[0].Raw = strings.Replace(changedFile.Funcs[0].Instrs[0].Raw, "$160-32", "$168-32", 1) + } + changed.Sigs[name] = sig + fixedLocalAPIs(t, pkg, &changedFile, changed, fixedLocalRequireContext) + }) + } + for _, mutation := range fixedLocalTypedRoleMutations { + t.Run(mutation, func(t *testing.T) { + fixedLocalTypedRoleMutation(t, source, name, mutation) + fixedLocalAPIs(t, pkg, file, bound, fixedLocalRequireContext) + }) + } + encoded, err := json.Marshal(source) + if err != nil { + t.Fatal(err) + } + var clone GoAMD64FrameSource + if err := json.Unmarshal(encoded, &clone); err != nil { + t.Fatal(err) + } + if _, err := clone.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("JSON clone recreated void source authority: %v", err) + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, fixedLocalRequireContext) + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("closed observer rebound void source: %v", err) + } + // Cancellation revokes a separately captured live void binding too, not + // merely the already closed source used above. + pkg, request, file, options = fixedLocalVoidFixture(t, "PADDQ 16(SP),X0") + source = requireGoAMD64FrameCapture(t, pkg, request) + child, stop := context.WithCancel(ctx) + defer stop() + bound, err = source.BindFixedLocal(child, pkg, file, options) + if err != nil { + t.Fatal(err) + } + stop() + fixedLocalAPIs(t, pkg, file, bound, fixedLocalRequireContext) + if source.ready { + t.Fatal("cancelled binding retained a live source token") + } +} + +func TestGoAMD64FixedLocalVoidActualNumericSourceAndObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, operation := range fixedLocalNativeOperations { + t.Run(strings.Fields(operation)[0], func(t *testing.T) { + pkg, request, file, options := fixedLocalVoidFixture(t, operation) + source := requireGoAMD64FrameCapture(t, pkg, request) + observed := source.functions[pkg.Path+".Value"] + if observed.transport != goAMD64FrameNumericTransport || observed.signature.Ret != Void || + len(observed.signature.Frame.Results) != 0 || observed.args != 32 || observed.locals != 168 || len(observed.code) == 0 { + t.Fatalf("void fixture lacks complete actual numeric Go object identity: %+v", observed) + } + object := filepath.Join(source.owned, request.AssemblyFile+".o") + t.Logf("actual stock-Go void object: args=%d locals=%d bytes=%d object_sha=%x listing_sha=%x", observed.args, observed.locals, len(observed.code), source.inputs[object], source.inputs[object+".listing"]) + // The ordinary translation API must still lack native backing before + // the live source token binds this complete numeric function. + fixedLocalAPIs(t, pkg, file, options, fixedLocalRequireContext) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatalf("actual Go accepted the numeric void source but its local backing is rejected: %v", err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "define void") || !strings.Contains(ir, "fixed_local_native") || !strings.Contains(ir, "asm sideeffect") { + t.Fatalf("sealed void source failed its compile-only native consumer: %v", err) + } + compileLLVMToObject(t, llc, bound.TargetTriple, "void-local.ll", "void-local.o", ir) + compileLLVMToObject(t, llc, bound.TargetTriple, "void-local-opt.ll", "void-local-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + if _, err := os.Stat(object); err != nil { + t.Fatal(err) + } + t.Log("compile-only void local operations; no observable numerical result or external-library runtime claim") + }) + } +} diff --git a/amd64_fixed_local_work_charge_test.go b/amd64_fixed_local_work_charge_test.go new file mode 100644 index 000000000..6a9179eb1 --- /dev/null +++ b/amd64_fixed_local_work_charge_test.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalOrdinaryFullWorkBoundary(t *testing.T) { + fn, sig := fixedLocalReplayFixture(t, strings.Repeat("NOP\n", 55)) + fn.FrameSize = 64 << 10 + _, instructions, err := fixedLocalEdges(fn) + if err != nil || len(instructions) != 64 { + t.Fatal("independent 64-transfer ordinary fixture changed", len(instructions), err) + } + for _, deficit := range []int64{0, 1} { + t.Run(fmt.Sprint(deficit), func(t *testing.T) { + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: (64 << 20) - deficit} + proven, err := proveX86FixedLocalAttempt(fn, sig, &budget, false) + if deficit == 0 { + if err != nil || proven.x86FixedLocal == nil || budget.bytes != 0 || budget.steps != (1<<18)-64*16 { + t.Fatalf("ordinary original full allowance: %+v, %v", budget, err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || !reflect.DeepEqual(proven, fn) { + t.Fatal("one-byte-short ordinary replay must reject atomically", err) + } + }) + } +} + +func TestGoAMD64FixedLocalPartitionVectorWork(t *testing.T) { + body := strings.Replace(fixedLocalLayoutPingPong, "ret+16(FP)", "ret+8(FP)", 1) + body = strings.Replace(body, "loop:\n", "loop:\n"+strings.Repeat("PADDQ X0,X0\n", 1024), 1) + source := "TEXT ·Value(SB),4,$4096-16\nMOVQ a+0(FP),CX\nMOVQ $0,AX\n" + + "MOVQ AX,0(SP)\nMOVQ AX,8(SP)\nMOVOU 0(SP),X0\n" + body + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil || len(file.Funcs) != 1 { + t.Fatal(err) + } + fn := file.Funcs[0] + _, sig := fixedLocalReplayFixture(t, "") + budget := x86FixedLocalBudget{steps: 1 << 18, bytes: 64 << 20} + proven, err := proveX86FixedLocalAttempt(fn, sig, &budget, true) + t.Logf("partitioned vector replay: remaining_steps=%d remaining_bytes=%d error=%v", budget.steps, budget.bytes, err) + if err != nil || proven.x86FixedLocal == nil { + t.Fatal("initialized register work must not scan a whole frame per instruction", err) + } + used := (1 << 18) - budget.steps + if used < 16*2*1024 || budget.bytes <= 0 { + t.Fatal("both layouts and sixteen replays must retain instruction and byte accounting", used, budget) + } + budget = x86FixedLocalBudget{steps: used - 1, bytes: 64 << 20} + proven, err = proveX86FixedLocalAttempt(fn, sig, &budget, true) + t.Logf("partitioned step deficit: remaining_steps=%d remaining_bytes=%d error=%v", budget.steps, budget.bytes, err) + if !errors.Is(err, ErrProbeNeedsContext) || budget.steps != 0 || !reflect.DeepEqual(proven, fn) { + t.Fatal("register-only work still requires every step, with atomic failure", budget, err) + } +} + +func TestGoAMD64FixedLocalPartitionClonePrecharge(t *testing.T) { + fn, sig := fixedLocalReplayFixture(t, "") + proof := &x86FixedLocalProof{frame: 64} + // Initial state: two one-word bitmaps (16 bytes), one SP map entry (64). + // Reading and copying that payload costs 160 before the first instruction. + budget := x86FixedLocalBudget{steps: 100, bytes: 159} + var layouts x86FixedLocalLayoutGraph + err := replayX86FixedLocal(&fn, sig, proof, 0, &budget, nil, &layouts) + if !errors.Is(err, ErrProbeNeedsContext) || budget.steps != 100 || budget.bytes != 159 || len(layouts.nodes) != 0 { + t.Fatal("state copy must be charged before executing or publishing", budget, err) + } + for _, ins := range fn.Instrs { + if len(ins.x86FixedLocalMemory) != 0 { + t.Fatal("failed clone debit published an access") + } + } +} + +func TestGoAMD64FixedLocalPartitionCandidateWork(t *testing.T) { + for _, count := range []int{3, 17} { + for _, deficit := range []int64{0, 1} { + t.Run(fmt.Sprintf("%d/%d", count, deficit), func(t *testing.T) { + states := make(map[int]x86FixedLocalState) + layouts := x86FixedLocalLayouts{at: map[int][]int{5: {}}, nodes: make([]x86FixedLocalLayoutNode, count)} + for index := 0; index < count; index++ { + states[index] = x86FixedLocalState{ + pointers: map[Reg]int64{SP: 0, "R14": int64(index * 8)}, known: map[Reg]uint8{AX: 255}, + bytes: x86FixedLocalByteSet{0}, booleanBytes: x86FixedLocalByteSet{0}, + } + layouts.nodes[index].pc = 5 + layouts.at[5] = append(layouts.at[5], index) + } + // Each state has 16 bitmap bytes + three 64-byte map entries. + // One comparison costs 64 lookup + 2*208 prior + 208 incoming. + const perCandidate = 688 + budget := x86FixedLocalBudget{steps: 100, bytes: int64(count*count*perCandidate) - deficit} + for query := 0; query < count; query++ { + before := append([]x86FixedLocalLayoutNode(nil), layouts.nodes...) + target, err := layouts.target(states, 0, 0, 5, states[count-1], &budget) + if deficit != 0 && query == count-1 { + if !errors.Is(err, ErrProbeNeedsContext) || !reflect.DeepEqual(before, layouts.nodes) || budget.bytes != perCandidate-1 { + t.Fatal("last candidate must reject before changing an edge", budget, err) + } + } else if err != nil || target != count-1 { + t.Fatal("complete candidate inventory must be inspected", target, err) + } + } + if budget.steps != 100 || deficit == 0 && budget.bytes != 0 { + t.Fatal("candidate comparisons require quadratic work, not instruction steps", budget) + } + }) + } + } +} + +func TestGoAMD64FixedLocalPartitionUnboundCandidateWork(t *testing.T) { + state := x86FixedLocalState{pointers: map[Reg]int64{SP: 0}, known: map[Reg]uint8{AX: 255}} + layouts := x86FixedLocalLayouts{ + at: map[int][]int{5: {0, 1}}, nodes: []x86FixedLocalLayoutNode{{pc: 5}, {pc: 5}}, + } + // One bound candidate costs 64+3*128, one missing state 64+128. + budget := x86FixedLocalBudget{steps: 10, bytes: 640} + target, err := layouts.target(map[int]x86FixedLocalState{0: state}, 0, 0, 5, state, &budget) + if err != nil || target != 0 || budget.bytes != 0 || budget.steps != 10 { + t.Fatal("missing candidates still require lookup and incoming-state work", target, budget, err) + } +} + +func TestGoAMD64FixedLocalPartitionStatePayload(t *testing.T) { + state := x86FixedLocalState{ + bytes: make(x86FixedLocalByteSet, 2), booleanBytes: make(x86FixedLocalByteSet, 3), + pointers: map[Reg]int64{SP: 0}, opaque: map[Reg]x86FixedLocalOrigins{AX: {}}, + opaqueCells: map[int64]x86FixedLocalOrigins{0: {}}, booleans: map[Reg]uint8{BX: 1}, + constants: map[Reg]int64{CX: 0}, ranges: map[Reg]x86UnsignedRange{DX: {}}, + known: map[Reg]uint8{SI: 255}, vectors: map[Reg]uint64{"X0": 255}, masks: map[Reg]bool{"K1": true}, + addresses: map[int64]int64{8: 0}, results: map[int]bool{0: true}, savedBP: &x86FixedLocalSavedBP{}, + } + if got := fixedLocalStateWork(state); got != 5*8+11*64+128 { + t.Fatal("both bitmaps, all eleven maps and saved BP must be counted", got) + } + if got := fixedLocalStateWork(x86FixedLocalState{}); got != 0 { + t.Fatal("an absent state has no payload", got) + } + // A single bounded allocation exercises saturation before multiplication. + large := x86FixedLocalState{bytes: make(x86FixedLocalByteSet, (64<<20)/8+1)} + if got := fixedLocalStateWork(large); got != (64<<20)+1 { + t.Fatal("oversized state work must saturate above the unchanged cap", got) + } +} + +func TestGoAMD64FixedLocalPartitionVectorWorkProjection(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$64-0\nPADDQ X0,X0\nRET\n") + if err != nil { + t.Fatal(err) + } + effects, err := fixedLocalEffectPlan(file.Funcs[0], nil) + if err != nil || len(effects) != 1 { + t.Fatal("complete vector grammar did not select the single effect", err) + } + var original x86TypedByteEffect + for _, effect := range effects { + original = effect + } + for _, kind := range []OperandKind{OpReg, OpImm, OpMem, OpSym, OpFP} { + effect := original + effect.operands = append([]x86TypedByteOperand(nil), original.operands...) + effect.operands[0].operand.Kind = kind + if got := fixedLocalVectorRegisterWork(effect); got != (kind == OpReg || kind == OpImm) { + t.Fatal("only the register/immediate projection has bounded register work", kind, got) + } + } + for _, contract := range []string{"dynamic", "unknown upper", "invalid upper", "missing alignment"} { + effect := original + switch contract { + case "dynamic": + effect.dynamicMemory = true + case "unknown upper": + effect.vector.upperWrite = x86TypedVectorUpperUnknown + case "invalid upper": + effect.vector.upperWrite = 255 + case "missing alignment": + effect.vector.memoryAlignment = 0 + } + if fixedLocalVectorRegisterWork(effect) { + t.Fatal("an open or memory effect acquired the register-only discount", contract) + } + } +} diff --git a/amd64_fixup_old_destination_test.go b/amd64_fixup_old_destination_test.go new file mode 100644 index 000000000..5f1b930ed --- /dev/null +++ b/amd64_fixup_old_destination_test.go @@ -0,0 +1,170 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "reflect" + "strings" + "testing" +) + +const x86FixupOriginalSourceSHA = "42757d53a686c410e0c26dd1a0b3fc046098d5b96a431553e3604bcdbf52703c" + +const x86FixupRuntimeDeclaration = `func fixupimmsemantics(out *[289]byte, + src32 *[16]uint32, table32 *[16]uint32, old32 *[16]uint32, + src64 *[8]uint64, table64 *[8]uint64, old64 *[8]uint64, mask uint64)` + +func x86FixupOriginalRuntimeSource(t *testing.T) string { + t.Helper() + source := x86X87FunctionLiteral(t, "amd64_lower_fixupimm_test.go", "qualifyAMD64FixupImmediateRuntimeSemantics", "source") + if actual := fmt.Sprintf("%x", sha256.Sum256([]byte(source))); actual != x86FixupOriginalSourceSHA { + t.Fatal("complete original FIXUPIMM source changed", actual) + } + return source +} + +// Table response 0000 retains the active old DEST even with zero masking. +// Supply the old32 input expected by the unchanged independent C reference; +// the complete historical source remains an independent Context negative. +func x86FixupRuntimeSource(t *testing.T, original string) string { + t.Helper() + if actual := fmt.Sprintf("%x", sha256.Sum256([]byte(original))); actual != x86FixupOriginalSourceSHA { + t.Fatal("foreign source acquired FIXUPIMM runtime fixture transport", actual) + } + const instruction = "\tVFIXUPIMMPS.Z $0, Z0, Z1, K1, Z3\n" + if strings.Count(original, instruction) != 1 { + t.Fatal("complete FIXUPIMM runtime lost its unique old32 destination consumer") + } + return strings.Replace(original, instruction, "\tVMOVDQU32 0(DX), Z3\n"+instruction, 1) +} + +func TestX86FixupRuntimeOriginalSourceRequiresOldDestination(t *testing.T) { + source := x86FixupOriginalRuntimeSource(t) + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, x86FixupRuntimeDeclaration, + "amd64", "x86_64-unknown-linux-gnu", api) + x86VectorScratchCodegenControl(t, source, x86FixupRuntimeDeclaration, + "amd64", "x86_64-unknown-linux-gnu", ir, err) + }) + } +} + +func TestX86FixupRuntimeInputMatchesIndependentOldDestinationReference(t *testing.T) { + original := x86FixupOriginalRuntimeSource(t) + source := x86FixupRuntimeSource(t, original) + // Check the complete actual program before invoking any translator API. + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, x86FixupRuntimeDeclaration, + "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil || ir == "" { + t.Fatal("actual runtime input does not provide the old32 required by its independent C reference", err) + } + }) + } +} + +func TestX86FixupCorrectedRuntimeCompleteThreeAPITargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + original := x86FixupOriginalRuntimeSource(t) + source := x86FixupRuntimeSource(t, original) + const load = "\tVMOVDQU32 0(DX), Z3\n" + if strings.Count(source, load) != 1 || strings.Replace(source, load, "", 1) != original { + t.Fatal("runtime repair changed more than the unique old32 input load") + } + // These are complete public-program objects, not executed native semantics. + for _, triple := range []string{ + "x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, "amd64", triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, x86FixupRuntimeDeclaration, + "amd64", triple, api) + if err != nil || ir == "" { + t.Fatal("corrected complete FIXUPIMM program did not translate", err) + } + for _, optimize := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimize), func(t *testing.T) { + input := ir + if optimize { + input = x86DwordProductIROptimize(t, ir) + } + x86PackedStringCompileObject(t, llc, triple, input, "-O0") + }) + } + }) + } + }) + } +} + +func TestX86FixupImmediateZeroingStillReadsOldActiveDestination(t *testing.T) { + var directory strings.Builder + directory.WriteString("TEXT fixupoldinputs(SB),$0-0\n") + var rows []struct { + name, instruction string + active int + } + for _, member := range go127FixupImmediateSpecs() { + banks := []string{"X", "Y", "Z"} + if member.scalar { + banks = banks[:1] + } + for _, bank := range banks { + width := map[string]int{"X": 16, "Y": 32, "Z": 64}[bank] + active := width + if member.scalar { + active = member.laneBits / 8 + } + for _, mode := range []string{"plain", "merge", "zero"} { + op, mask := member.op, "" + if mode != "plain" { + mask = ", K1" + } + if mode == "zero" { + op += ".Z" + } + instruction := fmt.Sprintf("%s $0, %s0, %s1%s, %s2", op, bank, bank, mask, bank) + fmt.Fprintf(&directory, "\t%s\n", instruction) + rows = append(rows, struct { + name, instruction string + active int + }{member.op + "/" + bank + "/" + mode, instruction, active}) + } + } + } + directory.WriteString("\tRET\n") + // Go's full four-name, eight-width register directory is independent of the + // result model. Zeroing applies to inactive lanes, not table response 0000. + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", directory.String()) + for _, row := range rows { + t.Run(row.name, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, row.instruction) + state := x86FloatingHelperInitializedState() + before := state.clone() + if accepted, known := x86ResultFloatingHelperStep(ins, &state, nil, "amd64", true); !accepted || !known { + t.Fatal("complete old destination lost its initialized transfer") + } + if !reflect.DeepEqual(state, before) { + t.Fatal("FIXUPIMM changed independent inputs, FLAGS, DF, pointers or upper-byte knowledge") + } + for missing := 0; missing < row.active; missing++ { + state = before.clone() + state.writeVector(2, 64, state.vectorBytes(2)&^(uint64(1)< 0 { + flags |= 1 + } + fmt.Fprintf(&tests, "if got,flags:=%s(0x%x,0x%x,0x1f80);got!=0x%x || flags!=0x%x {t.Fatalf(\"%s NaN got=%%x flags=%%x\",got,flags)}\n", name, vector[0], vector[1], vector[2], flags, name) + } + one, half, next := uint64(0x3f800000), uint64(0x33800000), uint64(0x3f800001) + if bits == 64 { + one, half, next = 0x3ff0000000000000, 0x3ca0000000000000, 0x3ff0000000000001 + } + if operation == "SUB" { + half |= uint64(1) << (bits - 1) // 1-(-half ULP) has the same tie + } + for mode := 0; mode < 4; mode++ { + want := one + if mode == 2 { + want = next + } + fmt.Fprintf(&tests, "if got,flags:=%s(0x%x,0x%x,0x%x);got!=0x%x || flags!=0x%x {t.Fatalf(\"%s mode%d got=%%x flags=%%x\",got,flags)}\n", name, one, half, 0x1f80|mode<<13, want, 0x1fa0|mode<<13, name, mode) + } + } + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go SSE bit/round oracle: %v\n%s", err, output) + } +} diff --git a/amd64_float_add_sub_literal.go b/amd64_float_add_sub_literal.go new file mode 100644 index 000000000..1fb07c91d --- /dev/null +++ b/amd64_float_add_sub_literal.go @@ -0,0 +1,41 @@ +package plan9asm + +import ( + "math" + "strings" +) + +func x86FloatAddSubLiteral(ins Instr, spec x86FloatArithmeticSpec) bool { + return x86FloatArithmeticLiteral(ins, spec.legacy, spec.scalar) +} + +// obj6.progedit rewrites legacy scalar ADD/SUB/MUL/DIV constants to +// Float32Sym/Float64Sym. A literal is not caller memory or a register input. +// Both reviewed Go encoders use the same arithmetic rewrite, including ±0. +func x86FloatArithmeticLiteral(ins Instr, legacy, scalar bool) bool { + if !legacy || !scalar || len(ins.Args) != 2 { + return false + } + operand := ins.Args[0] + if operand.Kind != OpImm || !operand.ImmIsFloat || operand.ImmRaw != "" { + return false + } + value := math.Float64frombits(uint64(operand.Imm)) + if math.IsInf(value, 0) || math.IsNaN(value) { + return false // these are not stock Go FLOAT tokens + } + if ins.Raw == "" { + return true // explicitly typed finite value, without source syntax + } + _, rest := splitOpcode(ins.Raw) + arguments := splitTopLevelCSV(rest) + if len(arguments) != 2 { + return false + } + text := strings.TrimSpace(arguments[0]) + if !strings.HasPrefix(text, "$") { + return false + } + parsed, ok := parseGoFloatLiteral(strings.TrimSpace(text[1:])) + return ok && math.Float64bits(parsed) == uint64(operand.Imm) +} diff --git a/amd64_float_add_sub_literal_runtime_test.go b/amd64_float_add_sub_literal_runtime_test.go new file mode 100644 index 000000000..0319cb882 --- /dev/null +++ b/amd64_float_add_sub_literal_runtime_test.go @@ -0,0 +1,229 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86AddSubLiteralNumericalSource() (declarations, assembly string) { + var goSource, machine strings.Builder + for _, opcode := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + for _, value := range x86AddSubLiteralValues { + name := strings.ToLower(opcode) + value.name + fmt.Fprintf(&goSource, "func %s(input *[32]byte, output *[80]byte, csr uint32)\n", name) + fmt.Fprintf(&machine, `TEXT ·%s(SB),4,$8-20 +STMXCSR saved-4(SP) +LDMXCSR csr+16(FP) +MOVQ input+0(FP),AX +MOVQ output+8(FP),BX +VMOVUPS (AX),Y0 +%s $%s,X0 +MOVUPS X0,8(BX) +VMOVUPS Y0,24(BX) +STMXCSR 56(BX) +LDMXCSR saved-4(SP) +RET +`, name, opcode, value.text) + } + } + return goSource.String(), machine.String() +} + +func x86AddSubLiteralCOracle() string { + var declarations, checks strings.Builder + index := 0 + for _, opcode := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + bits := 32 + if strings.HasSuffix(opcode, "SD") { + bits = 64 + } + for _, value := range x86AddSubLiteralValues { + literal := value.bits64 + if bits == 32 { + literal = uint64(value.bits32) + } + name := strings.ToLower(opcode) + value.name + fmt.Fprintf(&declarations, "extern void %s(void*,void*,unsigned);\n", name) + fmt.Fprintf(&checks, " if (check_literal(%s,%q,%d,UINT64_C(0x%x),%d)) return %d;\n", name, name, bits, literal, x86FloatAddSubBoolInt(strings.HasPrefix(opcode, "SUB")), index+1) + index++ + } + } + return x86FloatAddSubSSEOracle + declarations.String() + ` +typedef void (*literal_fn)(void*,void*,unsigned); +static int check_literal(literal_fn fn,const char *name,unsigned bits,uint64_t literal,int subtract) { + const uint64_t edge32[]={0,0x80000000,1,0x80000001,0x3f800000,0xbf800000, + 0x7f7fffff,0x7f800000,0xff800000,0x7fc12345,0x7f812345}; + const uint64_t edge64[]={0,UINT64_C(0x8000000000000000),1,UINT64_C(0x8000000000000001), + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x7fefffffffffffff), + UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000), + UINT64_C(0x7ff8123456789abc),UINT64_C(0x7ff0123456789abc)}; + const unsigned controls[]={0x1f80,0x1f85,0x1fc0,0x9f80,0x9fc0,0x3f80,0x5f80,0x7f80}; + for(unsigned sample=0;sample<11;sample++) { + unsigned char input[32],before[32],output[80],want[80]; + for(unsigned byte=0;byte<32;byte++) input[byte]=(unsigned char)(byte*37+19); + uint64_t left=bits==32?edge32[sample]:edge64[sample]; + memcpy(input,&left,bits/8); + memcpy(before,input,32); + for(unsigned mode=0;mode<8;mode++) { + unsigned ambient=0x3f85,after; + memset(output,0xa7,80); + memset(want,0xa7,80); + uint64_t result=sse_result(bits,left,literal,subtract,controls[mode]); + unsigned expectedCSR=last_sse_csr; + memcpy(want+8,input,16); + memcpy(want+24,input,32); + memcpy(want+8,&result,bits/8); + memcpy(want+24,&result,bits/8); + memcpy(want+56,&expectedCSR,4); + __asm__ volatile("ldmxcsr %0"::"m"(ambient)); + fn(input,output,controls[mode]); + __asm__ volatile("stmxcsr %0":"=m"(after)); + if(memcmp(input,before,32)||memcmp(output,want,80)||after!=ambient) { + fprintf(stderr,"%s bits=%u sample=%u csr=%x after=%x\n",name,bits,sample,controls[mode],after); + return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + ` + unsigned csr=0x1f80; + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + return 0; +} +` +} + +func TestCrossLinuxRuntimeMatrixX86FloatAddSubLiteralNumerical(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + declarations, assembly := x86AddSubLiteralNumericalSource() + x86ScalarLiteralGoListing(t, "amd64", assembly) + pkg := x86AddSubLiteralPackage(t, "amd64", declarations) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + resolve := func(sym string) string { return strings.TrimPrefix(sym, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var translation *GoModuleTranslation + translation, err = translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{GOARCH: "amd64", TargetTriple: triple, ResolveSym: resolve}) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + ir = translation.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "addsub-literal-"+api, triple, ir, x86AddSubLiteralCOracle(), prefix) + }) + } + t.Log("56 original literal forms × 11 independent SSE bit inputs × 8 full MXCSR controls × 3 translation APIs; whole X/Y bytes and canaries") +} + +func TestCrossLinuxRuntimeMatrixX86FloatAddSubLiteralOriginalGo(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + declarations, assembly := x86AddSubLiteralNumericalSource() + var checks strings.Builder + for _, opcode := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + bits := 32 + if strings.HasSuffix(opcode, "SD") { + bits = 64 + } + for _, value := range x86AddSubLiteralValues { + literal := value.bits64 + if bits == 32 { + literal = uint64(value.bits32) + } + fmt.Fprintf(&checks, " check(t,%s,%d,0x%x,%t)\n", strings.ToLower(opcode)+value.name, bits, literal, strings.HasPrefix(opcode, "SUB")) + } + } + goTest := `package literal +import("encoding/binary";"math";"testing") +type literalFn func(*[32]byte,*[80]byte,uint32) +func check(t *testing.T,fn literalFn,bits int,literal uint64,subtract bool) { + t.Helper() + for _,negative:=range []bool{false,true} { + var input [32]byte + var output [80]byte + for i:=range input {input[i]=byte(i*37+19)} + for i:=range output {output[i]=0xa7} + var result uint64 + if bits==32 { + left:=float32(4) + if negative {left=-left} + binary.LittleEndian.PutUint32(input[:4],math.Float32bits(left)) + right:=math.Float32frombits(uint32(literal)) + if subtract {left-=right}else{left+=right} + result=uint64(math.Float32bits(left)) + }else{ + left:=float64(4) + if negative {left=-left} + binary.LittleEndian.PutUint64(input[:8],math.Float64bits(left)) + right:=math.Float64frombits(literal) + if subtract {left-=right}else{left+=right} + result=math.Float64bits(left) + } + before:=input + fn(&input,&output,0x1f80) + if input!=before {t.Fatal("input changed")} + for i,got:=range output { + if i>=56&&i<60 {continue} // CSR is checked independently by the C SSE oracle. + want:=byte(0xa7) + if i>=8&&i<24 {want=input[i-8]} + if i>=24&&i<56 {want=input[i-24]} + if i>=8&&i<8+bits/8 {want=byte(result>>(8*(i-8)))} + if i>=24&&i<24+bits/8 {want=byte(result>>(8*(i-24)))} + if got!=want {t.Fatalf("bits=%d literal=%x byte=%d got=%x want=%x",bits,literal,i,got,want)} + } + } +} +func TestOriginalLiteralNumerical(t *testing.T) { +` + checks.String() + "}\n" + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module literal\n\ngo 1.20\n", "literal.go": "package literal\n" + declarations, + "literal_amd64.s": assembly, "literal_test.go": goTest, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go original literal numerical: %v\n%s", err, output) + } + t.Log("actual Go original 56 literal kernels × 2 initialized finite inputs; X/Y aliases and whole output canaries (host execution may be Rosetta)") +} diff --git a/amd64_float_add_sub_literal_test.go b/amd64_float_add_sub_literal_test.go new file mode 100644 index 000000000..9eee3c08e --- /dev/null +++ b/amd64_float_add_sub_literal_test.go @@ -0,0 +1,360 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86AddSubLiteralValues = []x86ScalarLiteralValue{ + {name: "one", text: "1.0", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "plus", text: "+1.0", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "minus", text: "-1.0", bits32: 0xbf800000, bits64: 0xbff0000000000000}, + {name: "zero", text: "0.0"}, + {name: "negativeZero", text: "-0.0", bits32: 0x80000000, bits64: 0x8000000000000000}, + {name: "hex", text: "0x1.4p+0", bits32: 0x3fa00000, bits64: 0x3ff4000000000000}, + {name: "scientific", text: "6.25e-1", bits32: 0x3f200000, bits64: 0x3fe4000000000000}, + {name: "parentheses", text: "(1.25)", bits32: 0x3fa00000, bits64: 0x3ff4000000000000}, + {name: "signedParentheses", text: "(-2.5)", bits32: 0xc0200000, bits64: 0xc004000000000000}, + {name: "roundEven", text: "1.000000059604644775390625", bits32: 0x3f800000, bits64: 0x3ff0000010000000}, + {name: "underflow32", text: "1e-50", bits64: 0x358dee7a4ad4b81f}, + {name: "overflow32", text: "1e40", bits32: 0x7f800000, bits64: 0x483d6329f1c35ca5}, + {name: "underscore", text: "1_000.0", bits32: 0x447a0000, bits64: 0x408f400000000000}, + {name: "nestedSign", text: "+(-(+1.25))", bits32: 0xbfa00000, bits64: 0xbff4000000000000}, +} + +func x86AddSubLiteralPackage(t *testing.T, arch, declarations string) GoPackage { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "literal.go", "package literal\n"+declarations, 0) + if err != nil { + t.Fatal(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("literal", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: "literal", Types: pkg, Syntax: []*ast.File{file}} +} + +func TestX86FloatAddSubLiteralActualGoObjects(t *testing.T) { + // This shared oracle authenticates the invoked SDK/assembler and hashes + // obj6/asm6/AVX tables before and after these original-source objects. + x86ScalarLiteralActualGoRewriteSet(t) + for _, name := range []string{"obj6.go", "asm6.go"} { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86", name) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if name == "obj6.go" { + if err := x86AddSubLiteralRewriteSource(data); err != nil { + t.Fatal(err) + } + } else { + for _, op := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + if !bytes.Contains(data, []byte("{A"+op+", yxm,")) { + t.Fatalf("actual Go no longer uses yxm for %s", op) + } + } + } + } + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + for _, op := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + for _, value := range x86AddSubLiteralValues { + for _, register := range []int{0, 7, 8, 15} { + row := fmt.Sprintf("%s $%s,X%d", op, value.text, register) + entry := fmt.Sprintf("TEXT literal%d(SB),4,$0-0\n%s\nRET\n", source.Len(), row) + if arch == "386" && register >= 8 { + x86ScalarLiteralExpectedGoListing(t, arch, entry, false) + continue + } + source.WriteString(entry) + } + } + } + listing, err := x86ScalarLiteralGoListing(t, arch, source.String()) + if err != nil { + t.Fatal(err) + } + for _, value := range x86AddSubLiteralValues { + for _, pool := range []string{fmt.Sprintf("$f32.%08x", value.bits32), fmt.Sprintf("$f64.%016x", value.bits64)} { + if !strings.Contains(listing, pool) { + t.Fatalf("actual Go pool rounding/sign missing %s", pool) + } + } + } + if strings.Contains(listing, "XORPS") { + t.Fatal("arithmetic zero literal incorrectly acquired MOV zero rewrite") + } + } +} + +// Independently bind the actual encoder's complete scalar pool-rewrite body, +// not just the presence of an opcode name somewhere in the source file. +func x86AddSubLiteralRewriteSource(source []byte) error { + file, err := parser.ParseFile(token.NewFileSet(), "obj6.go", source, 0) + if err != nil { + return err + } + seen := make(map[string]int) + var progedit *ast.FuncDecl + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == "progedit" { + if progedit != nil || function.Body == nil { + return fmt.Errorf("missing or duplicate progedit body") + } + progedit = function + } + } + if progedit == nil { + return fmt.Errorf("actual Go progedit missing") + } + allowed := make(map[*ast.CaseClause]bool) + for _, statement := range progedit.Body.List { + switchStatement, ok := statement.(*ast.SwitchStmt) + if !ok || switchStatement.Init != nil { + continue + } + selector, ok := switchStatement.Tag.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "As" { + continue + } + base, ok := selector.X.(*ast.Ident) + if !ok || base.Name != "p" { + continue + } + for _, entry := range switchStatement.Body.List { + if clause, ok := entry.(*ast.CaseClause); ok { + allowed[clause] = true + } + } + } + var failure error + ast.Inspect(progedit.Body, func(node ast.Node) bool { + clause, ok := node.(*ast.CaseClause) + if !ok { + return true + } + for _, expression := range clause.List { + name, ok := expression.(*ast.Ident) + if !ok || name.Name != "AADDSS" && name.Name != "AADDSD" && name.Name != "ASUBSS" && name.Name != "ASUBSD" { + continue + } + seen[name.Name]++ + if !allowed[clause] { + failure = fmt.Errorf("nested or conditional literal rewrite %s", name.Name) + } + bits := 64 + if strings.HasSuffix(name.Name, "SS") { + bits = 32 + } + expected := fmt.Sprintf(`package p +func f() { +if p.From.Type == obj.TYPE_FCONST { + f%d := %s + p.From.Type = obj.TYPE_MEM + p.From.Name = obj.NAME_EXTERN + p.From.Sym = ctxt.Float%dSym(f%d) + p.From.Offset = 0 +} +}`, bits, map[int]string{32: "float32(p.From.Val.(float64))", 64: "p.From.Val.(float64)"}[bits], bits, bits) + fixture, err := parser.ParseFile(token.NewFileSet(), "expected.go", expected, 0) + if err != nil { + failure = err + continue + } + body, err := x86ScalarLiteralBodySyntax(clause.Body) + want, wantErr := x86ScalarLiteralBodySyntax(fixture.Decls[0].(*ast.FuncDecl).Body.List) + if err != nil || wantErr != nil || body != want { + failure = fmt.Errorf("unrecognized actual Go %s literal rewrite body", name.Name) + } + } + return true + }) + for _, name := range []string{"AADDSS", "AADDSD", "ASUBSS", "ASUBSD"} { + if seen[name] != 1 { + return fmt.Errorf("actual Go literal rewrite registration %s count=%d", name, seen[name]) + } + } + return failure +} + +func TestX86FloatAddSubLiteralRewriteSourceRejectsMutation(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/obj6.go") + source, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := x86AddSubLiteralRewriteSource(source); err != nil { + t.Fatal(err) + } + before := sha256.Sum256(source) + for _, replacement := range [][2]string{ + {"func progedit(", "func unrelated("}, + {"AADDSS,", "AADDPS,"}, {"ASUBSD,", "ASUBPD,"}, + {"ctxt.Float32Sym(f32)", "ctxt.Float64Sym(f32)"}, {"p.From.Offset = 0", "p.From.Offset = 4"}, + } { + mutated := bytes.ReplaceAll(source, []byte(replacement[0]), []byte(replacement[1])) + if bytes.Equal(mutated, source) || x86AddSubLiteralRewriteSource(mutated) == nil { + t.Fatalf("unknown source rewrite accepted: %q", replacement) + } + } + after, err := os.ReadFile(path) + if err != nil || sha256.Sum256(after) != before { + t.Fatalf("source mutation oracle changed the real SDK: %v", err) + } +} + +func TestX86FloatAddSubLiteralOriginalInstructionBytes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + opcode string + prefix, opcodeByte byte + }{ + {"ADDSS", 0xf3, 0x58}, {"ADDSD", 0xf2, 0x58}, + {"SUBSS", 0xf3, 0x5c}, {"SUBSD", 0xf2, 0x5c}, + } { + for _, literal := range []string{"0.0", "-0.0", "1.0", "0x1.4p0"} { + code := x86PackedFirstGoInstructionBytes(t, arch, row.opcode+" $"+literal+",X0") + if !bytes.Equal(code[:4], []byte{row.prefix, 0x0f, row.opcodeByte, 0x05}) { + t.Fatalf("%s %s literal %s original bytes=%x", arch, row.opcode, literal, code) + } + } + } + } +} + +func TestX86FloatAddSubLiteralCallerKeepsDefinedBytes(t *testing.T) { + for _, opcode := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + file, err := Parse(ArchAMD64, "TEXT literal(SB),4,$0-0\n"+opcode+" $-0.0,X0\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86FloatAddSubProperties(ins, "amd64") + if !recognized || err != nil || !form.literal || form.memory != -1 || form.memoryBytes != 0 { + t.Fatalf("literal confused with caller memory: %+v / %v", form, err) + } + low := x86CallerVectorByteMask(int64(form.bits / 8)) + for _, old := range []uint64{0, low &^ 1, low, low | 0xf0000000, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": old}, masks: map[Reg]bool{}} + before, _ := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerFloatAddSubStep(&ins, form, &state, nil) + if accepted != (old&low == low) || !accepted && !reflect.DeepEqual(before, state.vectors) || accepted && state.vectors["Z0"] != old { + t.Fatalf("%s invented destination/upper knowledge: old=%x state=%x accepted=%t", opcode, old, state.vectors["Z0"], accepted) + } + } + // Source spelling is not permission to use a different typed value. + ins.Args[0].Imm++ + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + _, _, err = ctx.lowerFloatAddSub(ins.Op, ins) + if err == nil || output.String() != before { + t.Fatal("mismatched literal/source bits emitted IR") + } + } +} + +func TestX86FloatAddSubLiteralThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var declarations, source strings.Builder + for _, op := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + for _, value := range x86AddSubLiteralValues { + name := strings.ToLower(op) + value.name + fmt.Fprintf(&declarations, "func %s()\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\nPXOR X0,X0\n%s $%s,X0\nRET\n", name, op, value.text) + } + } + pkg := x86AddSubLiteralPackage(t, target.arch, declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + resolve := func(sym string) string { return strings.Replace(sym, "·", "literal.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, target.arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var result *GoModuleTranslation + result, err = translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + ir = result.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "add-sub-literal.ll", "add-sub-literal.o", ir) + }) + } + }) + } +} + +func TestX86FloatAddSubLiteralRejectsNonGoFormats(t *testing.T) { + for _, row := range []string{ + "ADDSD $1,X0", "ADDSS $0,X0", "SUBSD $0x3ff0000000000000,X0", + "ADDSD $(1.0+2.0),X0", "SUBSS $(1.0/2.0),X0", "ADDSS $1e999,X0", + "ADDPS $1.0,X0", "ADDPD $1.0,X0", "SUBPS $1.0,X0", "SUBPD $1.0,X0", + "VADDSS $1.0,X0,X1", "VADDSD $1.0,X0,X1", "VSUBSS $1.0,X0,X1", "VSUBSD $1.0,X0,X1", + "ADDSD.Z $1.0,X0", "ADDSS $1.0,Y0", "SUBSD $1.0,X16", "ADDSD X0,$1.0", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT literal(SB),4,$0-0\n" + row + "\nRET\n" + x86ScalarLiteralExpectedGoListing(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerFloatAddSub(ins.Op, ins) + if !known || term || err == nil || before != output.String() { + t.Fatalf("non-Go literal emitted IR: %t/%t/%v", known, term, err) + } + }) + } +} diff --git a/amd64_float_add_sub_numerical_test.go b/amd64_float_add_sub_numerical_test.go new file mode 100644 index 000000000..94efa6b03 --- /dev/null +++ b/amd64_float_add_sub_numerical_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// This oracle executes real legacy SSE under an explicitly saved/restored +// MXCSR. It is independent of the translated opcode table and numerical +// accumulator, including NaN payload priority and DAZ/FTZ signed zeros. +const x86FloatAddSubSSEOracle = `#include +#include +#include +static unsigned last_sse_csr; +static uint64_t sse_result(unsigned width,uint64_t left,uint64_t right,int sub,unsigned csr) { + unsigned saved; __asm__ volatile("stmxcsr %0":"=m"(saved)); + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + uint64_t bits=0; + if(width==32) { + uint32_t l=(uint32_t)left,r=(uint32_t)right,out; float a,b; + memcpy(&a,&l,4);memcpy(&b,&r,4); + if(sub) __asm__ volatile("subss %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("addss %1,%0":"+x"(a):"x"(b)); + memcpy(&out,&a,4);bits=out; + } else { + double a,b;memcpy(&a,&left,8);memcpy(&b,&right,8); + if(sub) __asm__ volatile("subsd %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("addsd %1,%0":"+x"(a):"x"(b)); + memcpy(&bits,&a,8); + } + __asm__ volatile("stmxcsr %0":"=m"(last_sse_csr)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); return bits; +} +` + +func x86FloatAddSubRuntimeTools(t *testing.T) (llc, clang, triple string, prefix []string) { + t.Helper() + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires amd64 numerical counterpart") + } + var found bool + llc, clang, found = findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang missing") + } + triple = testTargetTriple(runtime.GOOS, "amd64") + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + return +} + +func TestCrossLinuxRuntimeMatrixX86FloatAddSubFixedRoundSSEExtremes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var ir strings.Builder + ir.WriteString("declare i128 @llvm.ctlz.i128(i128, i1)\n") + for _, bits := range []int{32, 64} { + typ := amd64FMA3LLVMType(1, bits) + fmt.Fprintf(&ir, "declare %s @llvm.fma.f%d(%s,%s,%s)\n", typ, bits, typ, typ, typ) + emitAMD64FMA3RoundingHelper(&ir, bits) + emitX86FloatAddSubRoundingHelper(&ir, bits) + fmt.Fprintf(&ir, "define i%d @addsub_round%d(i%d %%a,i%d %%b,i1 %%sub,i32 %%mode,i32 %%csr){\n %%a_float=bitcast i%d %%a to %s\n %%b_float=bitcast i%d %%b to %s\n %%value=call %s @__plan9asm_add_sub_round_f%d(%s %%a_float,%s %%b_float,i1 %%sub,i32 %%mode,i32 %%csr)\n %%bits=bitcast %s %%value to i%d\n ret i%d %%bits\n}\n", bits, bits, bits, bits, bits, typ, bits, typ, typ, bits, typ, typ, typ, bits, bits) + } + main := x86FloatAddSubSSEOracle + ` +extern uint32_t addsub_round32(uint32_t,uint32_t,_Bool,unsigned,unsigned); +extern uint64_t addsub_round64(uint64_t,uint64_t,_Bool,unsigned,unsigned); +int main(void) { + const uint64_t edges32[]={0,0x80000000,1,0x80000001,0x7fffff,0x800000, + 0x7f7fffff,0xff7fffff,0x3f800000,0xbf800000,0x3f800001,0x33800000, + 0x7f800000,0xff800000,0x7fc12345,0xffc45678,0x7f812345,0xff845678}; + const uint64_t edges64[]={0,UINT64_C(0x8000000000000000),1, + UINT64_C(0x8000000000000001),UINT64_C(0xfffffffffffff),UINT64_C(0x10000000000000), + UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff),UINT64_C(0x3ff0000000000000), + UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000001),UINT64_C(0x3ca0000000000000), + UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000),UINT64_C(0x7ff8123456789abc), + UINT64_C(0xfff8fedcba987654),UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff0fedcba987654)}; + const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0}; + uint64_t state=UINT64_C(0x7182329ab3293821); + for(unsigned width=32;width<=64;width+=32) for(unsigned pair=0;pair<1000;pair++) { + const uint64_t *edges=width==32?edges32:edges64; + uint64_t a=edges[pair%18],b=edges[(pair/18)%18]; + if(pair>=324){ state^=state<<13;state^=state>>7;state^=state<<17;a=state; + state^=state<<13;state^=state>>7;state^=state<<17;b=state; } + for(unsigned sub=0;sub<2;sub++) for(unsigned mode=0;mode<4;mode++) + for(unsigned ambient=0;ambient<4;ambient++) for(unsigned control=0;control<6;control++) { + unsigned csr=controls[control],ambientCSR=csr|(ambient<<13),after; + __asm__ volatile("ldmxcsr %0"::"m"(ambientCSR)); + uint64_t got=width==32?addsub_round32(a,b,sub,mode,csr):addsub_round64(a,b,sub,mode,csr); + __asm__ volatile("stmxcsr %0":"=m"(after)); + uint64_t want=sse_result(width,a,b,sub,csr|(mode<<13)); + if(got!=want || after!=ambientCSR) { + fprintf(stderr,"binary width/sub/mode/ambient/csr=%u/%u/%u/%u/%x a=%llx b=%llx got=%llx want=%llx after=%x\n", + width,sub,mode,ambient,csr,(unsigned long long)a,(unsigned long long)b,(unsigned long long)got,(unsigned long long)want,after);return 1; + } + } + } + unsigned csr=0x1f80;__asm__ volatile("ldmxcsr %0"::"m"(csr));return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "binary-fixed-sse", triple, ir.String(), main, prefix) +} + +func TestCrossLinuxRuntimeMatrixX86FloatAddSubCompletePhysicalSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, row := range x86FloatAddSubFormats("amd64") { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + form, _, err := x86FloatAddSubProperties(file.Funcs[0].Instrs[1], "amd64") + if err != nil { + t.Fatal(err) + } + fixedMode := -1 + for mode, suffix := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + if form.rounding == suffix { + fixedMode = mode + } + } + name := fmt.Sprintf("binary%d", count) + count++ + row = strings.Replace(row, "8(AX)", "(BX)", 1) + row = strings.Replace(row, "(AX)", "(BX)", 1) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nRET\n", name, row, map[bool]int{true: 1, false: 2}[form.spec.legacy]) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*,uint64_t);\n", name) + arithmeticBytes := form.byteWidth + if form.spec.scalar { + arithmeticBytes = form.spec.bits / 8 + } + opcode := strings.Split(strings.Fields(row)[0], ".")[0] + independentSubtract := strings.Contains(opcode, "SUB") + // Only legacy packed native memory has m128 alignment semantics. + // Keep scalar/VEX/EVEX inputs unaligned; the separate exact-opcode + // fault oracle retains every old mod16 case instead of erasing it. + aligned := form.legacy && !form.scalar && form.memory >= 0 + fmt.Fprintf(&checks, ` + for(int round=0;round<4;round++) for(int m=0;m<4;m++) for(int control=0;control<6;control++) for(int ambient=0;ambient<4;ambient++) { + uint8_t aStorage[96] __attribute__((aligned(16))),bStorage[68],out[67];uint8_t *a=aStorage+(%[13]d?16:1),*b=bStorage+3; + memset(aStorage,0xa7,96);memset(bStorage,0xb9,68);memset(out,0xc3,67); + for(int i=0;i<64/%[2]d;i++) { uint64_t av=%[3]d==32?UINT64_C(0x3f000001)+i*0x12345+round*0x10000:UINT64_C(0x3fe0000000000001)+i*0x123456789; + uint64_t bv=%[3]d==32?UINT64_C(0xbf800001)+i*0x12345:UINT64_C(0xbff0000000000001)+i*0x2345678; + if(round==1){av=%[3]d==32?UINT64_C(0x7fc12345)+i:UINT64_C(0x7ff8123456789abc)+i;bv=%[3]d==32?UINT64_C(0xffc45678)+i:UINT64_C(0xfff8fedcba987654)+i;} + if(round==2){av=%[3]d==32?UINT64_C(0x7f812345)+i:UINT64_C(0x7ff0123456789abc)+i;bv=%[3]d==32?UINT64_C(0xff800000):UINT64_C(0xfff0000000000000);} + if(round==3){av=1+i;bv=(%[3]d==32?UINT64_C(0x80000000):UINT64_C(0x8000000000000000))|i;} + memcpy(a+i*%[2]d,&av,%[2]d);memcpy(b+i*%[2]d,&bv,%[2]d); } + unsigned csr=controls[control]|(ambient<<13),observed,expectedFlags=0; + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + uint64_t mask=masks[m];%[1]s(out+1,a,b,mask); + __asm__ volatile("stmxcsr %%0":"=m"(observed)); + for(int i=0;i<64/%[2]d;i++) { + uint64_t av=0,bv=0,got=0;memcpy(&av,a+(%[4]d?0:i*%[2]d),%[2]d);memcpy(&bv,b+i*%[2]d,%[2]d);memcpy(&got,out+1+i*%[2]d,%[2]d); + int active=%[5]d<0 || ((mask>>i)&1); + unsigned operationCSR=%[12]d<0?csr:((csr&~0x6000u)|((unsigned)%[12]d<<13)); + uint64_t want=i*%[2]d<%[6]d?(active?sse_result(%[3]d,bv,av,%[7]d,operationCSR):(%[8]d?0:bv)):(%[9]d || (%[10]d && i*%[2]d<16)?bv:0); + if(i*%[2]d<%[6]d && active && %[12]d<0)expectedFlags|=last_sse_csr&63; + if(got!=want){fprintf(stderr,"%[1]s lane=%%d got=%%llx want=%%llx\n",i,(unsigned long long)got,(unsigned long long)want);return %[11]d;} + } + if(observed!=(csr|expectedFlags)){fprintf(stderr,"%[1]s sticky observed=%%x expected=%%x\n",observed,csr|expectedFlags);return 3;} + if(out[0]!=0xc3 || out[65]!=1 || out[66]!=0xc3 || a[-1]!=0xa7 || a[64]!=0xa7 || bStorage[2]!=0xb9 || bStorage[67]!=0xb9)return 2; + } +`, name, form.spec.bits/8, form.spec.bits, x86FloatAddSubBoolInt(form.broadcast), form.mask, arithmeticBytes, x86FloatAddSubBoolInt(independentSubtract), x86FloatAddSubBoolInt(form.zeroing), x86FloatAddSubBoolInt(form.spec.legacy), x86FloatAddSubBoolInt(form.spec.scalar), count+10, fixedMode, x86FloatAddSubBoolInt(aligned)) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := x86FloatAddSubSSEOracle + declarations.String() + "int main(void){const uint64_t masks[]={0,1,0x55aa55aa55aa55aaULL,~0ULL}; const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0};\n" + checks.String() + "unsigned csr=0x1f80;__asm__ volatile(\"ldmxcsr %0\"::\"m\"(csr));return 0;}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "binary-complete-semantics", triple, ir, main, prefix) + t.Logf("%d independent opcode/width/memory/mask programs x384 IEEE/MXCSR/canary inputs", count) +} + +func x86FloatAddSubBoolInt(value bool) int { + if value { + return 1 + } + return 0 +} diff --git a/amd64_float_add_sub_runtime_test.go b/amd64_float_add_sub_runtime_test.go new file mode 100644 index 000000000..fc24a049c --- /dev/null +++ b/amd64_float_add_sub_runtime_test.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86FloatAddSubPhysicalAliasesAndInactiveMemory(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang missing") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + for _, fixture := range []struct { + name, body, checks string + }{ + {"packed upper aliases", "VADDPS X1,X0,X2", "for(int i=0;i<16;i++) if(out[i]!=(i<4?5.0f:0.0f)) return 11;"}, + {"scalar second-source passthrough", "VSUBSS X1,X0,X2", "for(int i=0;i<16;i++) if(out[i]!=(i==0?-1.0f:i<4?2.0f:0.0f)) return 12;"}, + {"inactive memory", "MOVQ $0,DX\nKMOVQ DX,K1\nVADDPS.Z (CX),X0,K1,X2", "for(int i=0;i<16;i++) if(out[i]!=0.0f) return 13;"}, + {"inactive scalar memory", "MOVQ $0,DX\nKMOVQ DX,K1\nVSUBSD.Z (CX),X0,K1,X2", "for(int i=0;i<16;i++) if(out[i]!=(i<2?0.0f:i<4?2.0f:0.0f)) return 14;"}, + {"inactive broadcast memory", "MOVQ $0,DX\nKMOVQ DX,K1\nVADDPD.BCST.Z (CX),Z0,K1,Z2", "for(int i=0;i<16;i++) if(out[i]!=0.0f) return 15;"}, + } { + t.Run(fixture.name, func(t *testing.T) { + source := "TEXT addsub(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nVMOVUPS (BX),Z0\nVMOVUPS (BX),Z2\n" + if !strings.HasPrefix(fixture.name, "inactive") { + source += "VMOVUPS (CX),Z1\n" + } + source += fixture.body + "\nVMOVUPS Z2,(AX)\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{ + "addsub": {Name: "addsub", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}}, + }}) + if err != nil { + t.Fatal(err) + } + argument := "b" + if strings.HasPrefix(fixture.name, "inactive") { + argument = "0" + } + main := fmt.Sprintf("extern void addsub(float*,const float*,const float*);\nint main(void){float a[16],b[16],out[16]; for(int i=0;i<16;i++){a[i]=2;b[i]=3;} addsub(out,a,%s); %s return 0;}\n", argument, fixture.checks) + compileAndRunRuntimeTestForTarget(t, llc, clang, "addsub-alias", triple, ir, strings.TrimSpace(main), prefix) + }) + } +} + +func TestX86FloatAddSubFixedRoundingIgnoresAmbientMode(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang missing") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for mode, suffix := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + name := fmt.Sprintf("roundadd%d", mode) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVSS (BX),X0\nMOVSS (CX),X1\nVADDSS.%s X1,X0,X2\nMOVSS X2,(AX)\nRET\n", name, suffix) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint32_t*,const uint32_t*,const uint32_t*);\n", name) + expected := uint32(0x3f800000) + if mode == 2 { + expected++ + } + fmt.Fprintf(&checks, "for(int ambient=0;ambient<4;ambient++){if(fesetround(modes[ambient]))return 2; uint32_t a=0x3f800000,b=0x33800000,out=0; %s(&out,&a,&b); if(out!=0x%x)return %d;}\n", name, expected, mode+10) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := "#include \n#include \n" + declarations.String() + "int main(void){const int modes[]={FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO};\n" + checks.String() + "return fesetround(FE_TONEAREST)!=0;}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "addsub-fixed-ambient", triple, ir, main, prefix) +} diff --git a/amd64_float_arithmetic_forms.go b/amd64_float_arithmetic_forms.go new file mode 100644 index 000000000..c76ffea5a --- /dev/null +++ b/amd64_float_arithmetic_forms.go @@ -0,0 +1,136 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86FloatArithmeticFamily string + +const ( + x86FloatAddSubtract x86FloatArithmeticFamily = "add/sub" + x86FloatMultiplyDivide x86FloatArithmeticFamily = "mul/div" + x86FloatMinimumMaximum x86FloatArithmeticFamily = "min/max" +) + +type x86FloatArithmeticSpec struct { + bits int + legacy, scalar bool + mode amd64BinaryFloatingMode +} + +// These are the complete six yxm/VEX/EVEX arithmetic pairs. Operand shapes +// share one grammar; operation tags retain distinct arithmetic and FP effects. +var x86FloatArithmeticSpecs = map[Op]x86FloatArithmeticSpec{ + "ADDPS": {32, true, false, amd64BinaryFloatingAdd}, "ADDPD": {64, true, false, amd64BinaryFloatingAdd}, + "ADDSS": {32, true, true, amd64BinaryFloatingAdd}, "ADDSD": {64, true, true, amd64BinaryFloatingAdd}, + "SUBPS": {32, true, false, amd64BinaryFloatingSub}, "SUBPD": {64, true, false, amd64BinaryFloatingSub}, + "SUBSS": {32, true, true, amd64BinaryFloatingSub}, "SUBSD": {64, true, true, amd64BinaryFloatingSub}, + "VADDPS": {32, false, false, amd64BinaryFloatingAdd}, "VADDPD": {64, false, false, amd64BinaryFloatingAdd}, + "VADDSS": {32, false, true, amd64BinaryFloatingAdd}, "VADDSD": {64, false, true, amd64BinaryFloatingAdd}, + "VSUBPS": {32, false, false, amd64BinaryFloatingSub}, "VSUBPD": {64, false, false, amd64BinaryFloatingSub}, + "VSUBSS": {32, false, true, amd64BinaryFloatingSub}, "VSUBSD": {64, false, true, amd64BinaryFloatingSub}, + "MULPS": {32, true, false, amd64BinaryFloatingMul}, "MULPD": {64, true, false, amd64BinaryFloatingMul}, + "MULSS": {32, true, true, amd64BinaryFloatingMul}, "MULSD": {64, true, true, amd64BinaryFloatingMul}, + "DIVPS": {32, true, false, amd64BinaryFloatingDiv}, "DIVPD": {64, true, false, amd64BinaryFloatingDiv}, + "DIVSS": {32, true, true, amd64BinaryFloatingDiv}, "DIVSD": {64, true, true, amd64BinaryFloatingDiv}, + "VMULPS": {32, false, false, amd64BinaryFloatingMul}, "VMULPD": {64, false, false, amd64BinaryFloatingMul}, + "VMULSS": {32, false, true, amd64BinaryFloatingMul}, "VMULSD": {64, false, true, amd64BinaryFloatingMul}, + "VDIVPS": {32, false, false, amd64BinaryFloatingDiv}, "VDIVPD": {64, false, false, amd64BinaryFloatingDiv}, + "VDIVSS": {32, false, true, amd64BinaryFloatingDiv}, "VDIVSD": {64, false, true, amd64BinaryFloatingDiv}, + "MINPS": {32, true, false, amd64BinaryFloatingMin}, "MINPD": {64, true, false, amd64BinaryFloatingMin}, + "MINSS": {32, true, true, amd64BinaryFloatingMin}, "MINSD": {64, true, true, amd64BinaryFloatingMin}, + "MAXPS": {32, true, false, amd64BinaryFloatingMax}, "MAXPD": {64, true, false, amd64BinaryFloatingMax}, + "MAXSS": {32, true, true, amd64BinaryFloatingMax}, "MAXSD": {64, true, true, amd64BinaryFloatingMax}, + "VMINPS": {32, false, false, amd64BinaryFloatingMin}, "VMINPD": {64, false, false, amd64BinaryFloatingMin}, + "VMINSS": {32, false, true, amd64BinaryFloatingMin}, "VMINSD": {64, false, true, amd64BinaryFloatingMin}, + "VMAXPS": {32, false, false, amd64BinaryFloatingMax}, "VMAXPD": {64, false, false, amd64BinaryFloatingMax}, + "VMAXSS": {32, false, true, amd64BinaryFloatingMax}, "VMAXSD": {64, false, true, amd64BinaryFloatingMax}, +} + +func (spec x86FloatArithmeticSpec) family() x86FloatArithmeticFamily { + switch spec.mode { + case amd64BinaryFloatingAdd, amd64BinaryFloatingSub: + return x86FloatAddSubtract + case amd64BinaryFloatingMul, amd64BinaryFloatingDiv: + return x86FloatMultiplyDivide + case amd64BinaryFloatingMin, amd64BinaryFloatingMax: + return x86FloatMinimumMaximum + } + panic("non-arithmetic floating operation in arithmetic catalog") +} + +type x86FloatArithmeticForm struct { + x86FloatBinaryForm + spec x86FloatArithmeticSpec +} + +func x86FloatArithmeticProperties(ins Instr, arch string, family x86FloatArithmeticFamily) (x86FloatArithmeticForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86FloatArithmeticSpecs[Op(base)] + if !known || family != "" && spec.family() != family { + return x86FloatArithmeticForm{}, false, nil + } + family = spec.family() + form := x86FloatArithmeticForm{spec: spec} + shape := ins + literal := false + sae := false + if family == x86FloatMinimumMaximum { + controls, valid := parseAMD64BinaryFloatingSuffix(suffix) + sae, form.sae = controls.sae, controls.sae + if !valid || dotted && suffix == "" || controls.rounding != "" || spec.legacy && dotted { + return form, true, fmt.Errorf("%s %s has invalid MIN/MAX operands or controls: %q", arch, ins.Op, ins.Raw) + } + // Strip only the independently validated SAE control from this local + // shape. Original operands/opcode remain the source and emitter input. + if sae { + shape.Op = Op(base) + if controls.zeroing { + shape.Op += ".Z" + } + } + } else { + literal = x86FloatArithmeticLiteral(ins, spec.legacy, spec.scalar) + } + var err error + form.x86FloatBinaryForm, err = x86FloatBinaryOperandProperties(shape, arch, spec.bits, spec.legacy, spec.scalar, literal, string(family)) + form.sae = sae + if err != nil { + return form, true, err + } + if family == x86FloatMinimumMaximum { + if sae && (ins.Args[0].Kind != OpReg || !spec.scalar && form.byteWidth != 64) { + return form, true, fmt.Errorf("%s %s has invalid MIN/MAX operands or controls: %q", arch, ins.Op, ins.Raw) + } + } + return form, true, x86Mode32VectorRegisterContext(ins, arch) +} + +func (c *amd64Ctx) lowerFloatArithmetic(op Op, ins Instr, family x86FloatArithmeticFamily) (bool, bool, error) { + local := ins + local.Op = op + form, known, err := x86FloatArithmeticProperties(local, c.goarch, family) + if !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + name := string(form.spec.family()) + if form.spec.family() == x86FloatMinimumMaximum { + name = "MIN/MAX" + } + return true, false, fmt.Errorf("floating %s dispatch/source opcode mismatch: %s/%s", name, op, ins.Op) + } + if err != nil { + return true, false, err + } + switch form.spec.family() { + case x86FloatAddSubtract: + return c.lowerFloatAddSubForm(local, form) + case x86FloatMultiplyDivide: + return c.lowerFloatMulDivForm(local, form) + case x86FloatMinimumMaximum: + return c.lowerFloatMinMaxForm(local, form) + } + panic("unregistered floating arithmetic family") +} diff --git a/amd64_float_arithmetic_mode32_test.go b/amd64_float_arithmetic_mode32_test.go new file mode 100644 index 000000000..b65f288d7 --- /dev/null +++ b/amd64_float_arithmetic_mode32_test.go @@ -0,0 +1,55 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86FloatArithmeticMode32PhysicalRegisterBoundary(t *testing.T) { + names := append(x86FloatAddSubOpcodes(), x86FloatMulDivOpcodes()...) + for _, member := range x86FloatingMinMaxCases { + names = append(names, member.op) + } + var rows []string + for _, name := range names { + if !strings.HasPrefix(name, "V") { + continue + } + banks := []string{"X"} + if strings.HasSuffix(name, "PS") || strings.HasSuffix(name, "PD") { + banks = append(banks, "Y") + } + for _, bank := range banks { + for high := 0; high < 3; high++ { + registers := []string{bank + "0", bank + "1", bank + "2"} + registers[high] = fmt.Sprint(bank, high+8) + rows = append(rows, name+" "+strings.Join(registers, ",")) + } + } + } + if len(rows) != 108 { + t.Fatal("complete arithmetic operation/width/input-output role inventory changed", len(rows)) + } + // A stock-Go spelling is not proof of physical high-register identity in + // mode32. Keep that frontend observation separate from both source modes. + assembly := "TEXT boundary(SB),4,$0-0\n" + strings.Join(rows, "\n") + "\nRET\n" + requireX86GoAssemblerResult(t, "386", assembly, true) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + for _, row := range rows { + t.Run(row, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT boundary(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + if _, known, err := x86FloatArithmeticProperties(instruction, "386", ""); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mode32 spelling acquired physical high-register authority: known=%v err=%v", known, err) + } + if _, known, err := x86FloatArithmeticProperties(instruction, "amd64", ""); !known || err != nil { + t.Fatalf("amd64 high-register positive control changed: known=%v err=%v", known, err) + } + }) + } +} diff --git a/amd64_float_arithmetic_pipeline_bench_test.go b/amd64_float_arithmetic_pipeline_bench_test.go new file mode 100644 index 000000000..ba606f760 --- /dev/null +++ b/amd64_float_arithmetic_pipeline_bench_test.go @@ -0,0 +1,49 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Keep the public translation workload independent of the production registry. +// Every arithmetic family and precision remains present before and after a +// normalization change; no source/tool subprocess runs inside the timer. +func BenchmarkX86FloatingArithmeticTranslate(b *testing.B) { + var source strings.Builder + source.WriteString("TEXT arithmetic(SB),4,$0-0\nVPXOR Y0,Y0,Y0\nVPXOR Y1,Y1,Y1\nVPXOR Y2,Y2,Y2\n") + var names []string + names = append(names, x86FloatAddSubOpcodes()...) + names = append(names, x86FloatMulDivOpcodes()...) + for _, member := range x86FloatingMinMaxCases { + names = append(names, member.op) + } + if len(names) != 48 { + b.Fatal("complete named floating arithmetic inventory changed", len(names)) + } + for _, name := range names { + operands := "X1,X2" + if strings.HasPrefix(name, "V") { + operands = "X0,X1,X2" + } + fmt.Fprintf(&source, "%s %s\n", name, operands) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + b.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"arithmetic": {Name: "arithmetic", Ret: Void}}} + if _, err := Translate(file, options); err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.SetBytes(int64(source.Len())) + b.ResetTimer() + for index := 0; index < b.N; index++ { + if _, err := Translate(file, options); err != nil { + b.Fatal(err) + } + } +} diff --git a/amd64_float_binary_forms.go b/amd64_float_binary_forms.go new file mode 100644 index 000000000..3222c9d64 --- /dev/null +++ b/amd64_float_binary_forms.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's yxm, _yvaddpd and _yvaddsd shapes are shared by binary floating +// arithmetic. Operation semantics are deliberately not part of this grammar. +type x86FloatBinaryForm struct { + bits int + legacy, scalar bool + first, second, destination, mask, memory int + byteWidth, memoryBytes int + broadcast, zeroing bool + literal bool // family-authorized Go constant-pool rewrite + sae bool // independently validated standalone suppression + rounding string +} + +// Parse only shared operand axes. The registered family supplies precision +// and encoding generation; its lowerer retains operation-specific semantics. +func x86FloatBinaryOperandProperties(ins Instr, goarch string, bits int, legacy, scalar, literal bool, family string) (x86FloatBinaryForm, error) { + return x86FloatOperandProperties(ins, goarch, bits, legacy, scalar, literal, false, family) +} + +// Unary packed arithmetic shares the width, memory and EVEX control axes, +// but has no second numerical input and allows three-operand masks on 386. +func x86FloatOperandProperties(ins Instr, goarch string, bits int, legacy, scalar, literal, unary bool, family string) (x86FloatBinaryForm, error) { + _, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + form := x86FloatBinaryForm{ + bits: bits, legacy: legacy, scalar: scalar, literal: literal, + first: 0, second: 1, destination: len(ins.Args) - 1, mask: -1, memory: -1, + } + if unary { + form.second = form.destination + } + fail := func() (x86FloatBinaryForm, error) { + return form, fmt.Errorf("%s %s operands/suffix outside Go floating %s grammar: %q", goarch, ins.Op, family, ins.Raw) + } + if goarch != "386" && goarch != "amd64" { + return fail() + } + if legacy { + if dotted || len(ins.Args) != 2 { + return fail() + } + form.byteWidth = 16 + } else { + operands := 3 + if unary { + operands = 2 + } + properties, valid := parseAMD64FMA3Suffix(suffix) + if !valid || dotted && suffix == "" || scalar && properties.broadcast || len(ins.Args) != operands && len(ins.Args) != operands+1 { + return fail() + } + form.broadcast, form.zeroing, form.rounding = properties.broadcast, properties.zeroing, properties.rounding + if len(ins.Args) == operands+1 { + if goarch == "386" && operands == 3 && !ins.x86Encoded || !amd64NonzeroKOperand(ins.Args[operands-1]) { + return fail() + } + form.mask = operands - 1 + } else if form.zeroing { + return fail() + } + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if scalar && form.byteWidth != 16 || form.rounding != "" && !scalar && form.byteWidth != 64 { + return fail() + } + } + register := func(operand Operand) bool { + if legacy { + index, valid := amd64ParseXReg(operand.Reg) + return operand.Kind == OpReg && valid && (goarch == "amd64" && index < 16 || goarch == "386" && index < 8) + } + return amd64GoPackedVectorMoveRegister(operand, form.byteWidth, goarch) + } + if !register(ins.Args[form.second]) || !register(ins.Args[form.destination]) { + return fail() + } + first := ins.Args[form.first] + if literal { + if !legacy || !scalar || first.Kind != OpImm || !first.ImmIsFloat || first.ImmRaw != "" { + return fail() + } + } else if isAMD64MemoryOperand(first) { + if form.rounding != "" { + return fail() + } + if first.Kind == OpSym { + symbol := strings.TrimSpace(first.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if first.Kind == OpMem { + displacement := first.Mem.Off + if displacement != int64(int32(displacement)) && (goarch != "386" || displacement != int64(uint32(displacement))) || + !x86MemoryRegistersValidForArch(first.Mem, goarch) || first.Mem.Index != "" && + (first.Mem.Index == SP || first.Mem.Scale != 1 && first.Mem.Scale != 2 && first.Mem.Scale != 4 && first.Mem.Scale != 8) { + return fail() + } + } + form.memory, form.memoryBytes = 0, form.byteWidth + if scalar || form.broadcast { + form.memoryBytes = bits / 8 + } + } else if form.broadcast || !register(first) { + return fail() + } + return form, nil +} diff --git a/amd64_float_binary_fp_load.go b/amd64_float_binary_fp_load.go new file mode 100644 index 000000000..a2569bbfd --- /dev/null +++ b/amd64_float_binary_fp_load.go @@ -0,0 +1,94 @@ +package plan9asm + +import "fmt" + +// FP operands describe checked virtual ABI0 fields. Loading one field using +// the whole instruction width loses later fields (or reads past an alloca). +func (c *amd64Ctx) loadFloatBinaryFPBits(source Operand, bits int) (string, error) { + bytes, err := c.loadFPVectorBytes(source.FPOffset, bits/8) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", value, bits/8, bytes, bits) + return "%" + value, nil +} + +func (c *amd64Ctx) loadFloatBinaryScalar(source Operand, bits int, mask string) (string, error) { + if source.Kind != OpFP { + if x86OrdinaryNativeMemoryOperand(source, c.goarch) { + // Preserve the source read even for a dead numerical result or + // a literal null address. It supplies no pointee-validity fact. + return c.loadFMA3ObservableScalar(source, bits, mask) + } + return c.loadFMA3Scalar(source, bits, mask) + } + if err := c.validateFPVectorSpan(source.FPOffset, bits/8); err != nil { + return "", err + } + load := func() (string, error) { return c.loadFloatBinaryFPBits(source, bits) } + var value string + var err error + if mask == "" { + value, err = load() + } else { + value, err = c.loadX86ScalarIf(bits, amd64MaskBitI1(c, mask, 0), load) + } + if err != nil { + return "", err + } + floating := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %s to %s\n", floating, bits, value, amd64FMA3LLVMType(1, bits)) + return "%" + floating, nil +} + +func (c *amd64Ctx) loadFloatBinaryPacked(source Operand, width, bits int, broadcast bool, mask string) (string, error) { + if source.Kind != OpFP { + return c.loadMaskedPackedCompareLanes(source, width, bits, broadcast, mask) + } + memoryBytes := width + if broadcast { + memoryBytes = bits / 8 + } + if err := c.validateFPVectorSpan(source.FPOffset, memoryBytes); err != nil { + return "", err + } + lanes := width * 8 / bits + if broadcast { + load := func() (string, error) { return c.loadFloatBinaryFPBits(source, bits) } + var value string + var err error + if mask == "" { + value, err = load() + } else { + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)< %s, i%d %s, i32 %d\n", inserted, lanes, bits, result, bits, value, lane) + result = "%" + inserted + } + return result, nil +} diff --git a/amd64_float_binary_fp_span_test.go b/amd64_float_binary_fp_span_test.go new file mode 100644 index 000000000..46777003e --- /dev/null +++ b/amd64_float_binary_fp_span_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +type x86FloatBinaryFPFixture struct { + name, opcode, operation, declaration, assembly string + bits, bytes, resultOffset int + sig FuncSig +} + +// Derive these layouts from actual Go declarations. Aggregate byte fields have +// separate virtual slots, not a pointer to one contiguous native FP word. +func x86FloatBinaryFPFixtures(t *testing.T) []x86FloatBinaryFPFixture { + t.Helper() + var fixtures []x86FloatBinaryFPFixture + var declarations strings.Builder + declarations.WriteString("package oracle\n") + for _, operation := range []string{"ADD", "SUB", "MUL", "DIV"} { + for _, bits := range []int{32, 64} { + for _, spelling := range []string{"legacy", "vex", "broadcast"} { + fixture := x86FloatBinaryFPFixture{operation: operation, bits: bits, bytes: bits / 8} + fixture.name = fmt.Sprintf("%s%d%s", strings.ToLower(operation), bits, spelling) + fixture.declaration = fmt.Sprintf("func %s(source [%d]byte,left uint%d) uint%d\n", fixture.name, fixture.bytes, bits, bits) + declarations.WriteString(fixture.declaration) + fixtures = append(fixtures, fixture) + } + } + } + set := token.NewFileSet() + file, err := parser.ParseFile(set, "value.go", declarations.String(), 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + pkg, err := (&types.Config{Sizes: sizes}).Check("oracle", set, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + for i := range fixtures { + fixture := &fixtures[i] + fixture.sig, err = DeriveGoFuncABI0Signature(fixture.name, pkg.Scope().Lookup(fixture.name).(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + fixture.resultOffset = int(fixture.sig.Frame.Results[0].Offset) + if fixture.resultOffset != 2*fixture.bytes || len(fixture.sig.Frame.Params) != fixture.bytes+1 { + t.Fatalf("independent Go ABI0 frame changed: %+v", fixture.sig.Frame) + } + move, suffix := "MOVSS", "SS" + if fixture.bits == 64 { + move, suffix = "MOVSD", "SD" + } + fixture.opcode = fixture.operation + suffix + row, destination := fixture.opcode+" source+0(FP),X0", "X0" + if strings.HasSuffix(fixture.name, "vex") { + fixture.opcode = "V" + fixture.opcode + row, destination = fixture.opcode+" source+0(FP),X0,X2", "X2" + } else if strings.HasSuffix(fixture.name, "broadcast") { + packedSuffix := "PS" + if fixture.bits == 64 { + packedSuffix = "PD" + } + fixture.opcode = "V" + fixture.operation + packedSuffix + ".BCST" + row, destination = fixture.opcode+" source+0(FP),X0,X2", "X2" + } + fixture.assembly = fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n%s left+%d(FP),X0\n%s\n%s %s,ret+%d(FP)\nRET\n", + fixture.name, fixture.resultOffset+fixture.bytes, move, fixture.bytes, row, move, destination, fixture.resultOffset) + } + return fixtures +} + +func TestCrossLinuxRuntimeMatrixX86FloatBinaryOriginalGoFPByteFields(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package oracle\n") + checks.WriteString("package oracle\nimport \"testing\"\nfunc TestOriginalByteFields(t *testing.T){\n") + for _, fixture := range x86FloatBinaryFPFixtures(t) { + declarations.WriteString(fixture.declaration) + // asmdecl refuses scalar arithmetic on array types even when its byte + // size matches. A vetted LEAQ exposes the same original Go FP bytes; + // the floating opcode, operand order and independently known bits stay + // unchanged. The translator test below keeps the direct FP spelling. + memoryRow := fixture.opcode + " source+0(FP)" + addressedRow := "LEAQ source+0(FP),CX\n" + fixture.opcode + " (CX)" + assembly.WriteString(strings.Replace(fixture.assembly, memoryRow, addressedRow, 1)) + // The original executable uses baseline SSE only. Go still assembles + // every VEX/EVEX form; LLVM software lanes are checked separately. + if !strings.HasSuffix(fixture.name, "legacy") { + continue + } + left, source := uint64(0x3f800000), uint64(0x40200000) + want := map[string]uint64{"ADD": 0x40600000, "SUB": 0xbfc00000, "MUL": 0x40200000, "DIV": 0x3ecccccd}[fixture.operation] + if fixture.bits == 64 { + left, source = 0x3ff0000000000000, 0x4004000000000000 + want = map[string]uint64{"ADD": 0x400c000000000000, "SUB": 0xbff8000000000000, "MUL": 0x4004000000000000, "DIV": 0x3fd999999999999a}[fixture.operation] + } + var args []string + for i := 0; i < fixture.bytes; i++ { + args = append(args, fmt.Sprint(byte(source>>uint(i*8)))) + } + arguments := fmt.Sprintf("[%d]byte{%s},uint%d(0x%x)", fixture.bytes, strings.Join(args, ","), fixture.bits, left) + fmt.Fprintf(&checks, "if got:=%s(%s);got!=0x%x{t.Fatalf(%q,got)}\n", fixture.name, arguments, want, fixture.name+" got=%x") + } + checks.WriteString("}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": checks.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=asmdecl", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go cross-field floating memory oracle: %v\n%s", err, output) + } +} + +func TestCrossLinuxRuntimeMatrixX86FloatBinaryFPByteFields(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, wrappers, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for _, fixture := range x86FloatBinaryFPFixtures(t) { + assembly.WriteString(fixture.assembly) + sigs[fixture.name] = fixture.sig + // A bit-carrier wrapper constructs the LLVM array argument explicitly; + // C aggregate calling conventions are not the translated Go signature. + fmt.Fprintf(&wrappers, "define i%d @check_%s(i%d %%right,i%d %%left){\n", fixture.bits, fixture.name, fixture.bits, fixture.bits) + packed := "poison" + for i := 0; i < fixture.bytes; i++ { + fmt.Fprintf(&wrappers, " %%shift%d=lshr i%d %%right,%d\n %%byte%d=trunc i%d %%shift%d to i8\n %%pack%d=insertvalue [%d x i8] %s,i8 %%byte%d,%d\n", + i, fixture.bits, i*8, i, fixture.bits, i, i, fixture.bytes, packed, i, i) + packed = fmt.Sprintf("%%pack%d", i) + } + fmt.Fprintf(&wrappers, " %%result=call i%d @%s([%d x i8] %s,i%d %%left)\n ret i%d %%result\n}\n", fixture.bits, fixture.name, fixture.bytes, packed, fixture.bits, fixture.bits) + fmt.Fprintf(&declarations, "extern uint%d_t check_%s(uint%d_t,uint%d_t);\n", fixture.bits, fixture.name, fixture.bits, fixture.bits) + oracle, flags, secondOperation := "sse_result", "last_sse_csr", fixture.operation == "SUB" + if fixture.operation == "MUL" || fixture.operation == "DIV" { + oracle, flags, secondOperation = "sse_muldiv_result", "last_sse_muldiv_csr", fixture.operation == "DIV" + } + fmt.Fprintf(&checks, ` + for(unsigned rc=0;rc<4;rc++) for(unsigned index=0;index<64;index++) { + uint64_t left=%[1]d==32?UINT64_C(0x3f800001)+index*3571:UINT64_C(0x3ff0000000000001)+index*7912347; + uint64_t right=%[1]d==32?UINT64_C(0x40200001)+index*761:UINT64_C(0x4004000000000001)+index*1234567; + unsigned csr=0x1f80|(rc<<13),after; + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + uint64_t got=check_%[2]s(right,left); + __asm__ volatile("stmxcsr %%0":"=m"(after)); + uint64_t want=%[3]s(%[1]d,left,right,%[4]d,csr); + if(got!=want || after!=%[5]s){fprintf(stderr,"%[2]s rc=%%u got=%%llx want=%%llx after=%%x expectedCSR=%%x\n",rc,(unsigned long long)got,(unsigned long long)want,after,%[5]s);return 1;} + } +`, fixture.bits, fixture.name, oracle, x86FloatAddSubBoolInt(secondOperation), flags) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(name string) string { return strings.TrimPrefix(name, "·") }, + }) + if err != nil { + t.Fatal(err) + } + mulDivOracle := strings.NewReplacer("subss", "divss", "subsd", "divsd", "addss", "mulss", "addsd", "mulsd", + "sse_result", "sse_muldiv_result", "last_sse_csr", "last_sse_muldiv_csr").Replace(x86FloatAddSubSSEOracle) + main := x86FloatAddSubSSEOracle + mulDivOracle + declarations.String() + "int main(void){\n" + checks.String() + + "unsigned csr=0x1f80;__asm__ volatile(\"ldmxcsr %0\"::\"m\"(csr));return 0;}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "float-binary-fp-bytes", triple, ir+wrappers.String(), main, prefix) +} diff --git a/amd64_float_compare_forms.go b/amd64_float_compare_forms.go new file mode 100644 index 000000000..3f2a08064 --- /dev/null +++ b/amd64_float_compare_forms.go @@ -0,0 +1,153 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86FloatCompareSpec struct { + bits int + legacy, scalar bool +} + +var x86FloatCompareSpecs = map[Op]x86FloatCompareSpec{ + "CMPPS": {32, true, false}, "CMPPD": {64, true, false}, + "CMPSS": {32, true, true}, "CMPSD": {64, true, true}, + "VCMPPS": {32, false, false}, "VCMPPD": {64, false, false}, + "VCMPSS": {32, false, true}, "VCMPSD": {64, false, true}, +} + +// Compare has an immediate position and a K-result generation distinct from +// binary arithmetic. The shared source-width axes do not imply merge/zero +// destination semantics or shared floating exception policy. +type x86FloatCompareForm struct { + spec x86FloatCompareSpec + x86FloatBinaryForm + immediate uint8 + maskResult, sae bool +} + +func (form x86FloatCompareForm) inputByteWidth() int { + if form.scalar { + return form.bits / 8 + } + return form.byteWidth +} + +// Both callers and result analysis use the same architectural byte writes. +// These are definedness facts, not predicate values or native memory rights. +func (form x86FloatCompareForm) resultDefinedBytes(second, previous uint64) uint64 { + active := x86CallerVectorByteMask(int64(form.inputByteWidth())) + if form.legacy { + return previous | active + } + if form.scalar { + return ^uint64(0xffff) | active | second&(0xffff&^active) + } + return ^uint64(0) +} + +func x86FloatCompareProperties(ins Instr, goarch string) (x86FloatCompareForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86FloatCompareSpecs[Op(base)] + if !known { + return x86FloatCompareForm{}, false, nil + } + form := x86FloatCompareForm{spec: spec, x86FloatBinaryForm: x86FloatBinaryForm{ + bits: spec.bits, legacy: spec.legacy, scalar: spec.scalar, + mask: -1, memory: -1, + }} + fail := func() (x86FloatCompareForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go floating compare grammar: %q", goarch, ins.Op, ins.Raw) + } + if goarch != "386" && goarch != "amd64" { + return fail() + } + if spec.legacy { + if dotted || len(ins.Args) != 3 { + return fail() + } + if ins.Args[2].Kind == OpImm && (ins.Args[2].ImmIsFloat || ins.Args[2].ImmRaw != "") { + return fail() + } + immediate, valid := amd64LegacyFloatingCompareImmediate(ins.Args[2]) + if !valid || immediate < -128 || immediate > 127 { + return fail() + } + form.immediate = uint8(immediate) + form.first, form.second, form.destination, form.byteWidth = 0, 1, 1, 16 + } else { + if goarch == "386" && !ins.x86Encoded || len(ins.Args) != 4 && len(ins.Args) != 5 || + dotted && suffix != "BCST" && suffix != "SAE" || ins.Args[0].Kind != OpImm || ins.Args[0].ImmIsFloat || ins.Args[0].ImmRaw != "" || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return fail() + } + form.immediate = uint8(ins.Args[0].Imm) + form.first, form.second, form.destination = 1, 2, len(ins.Args)-1 + form.broadcast, form.sae = suffix == "BCST", suffix == "SAE" + form.maskResult = amd64IsKOperand(ins.Args[form.destination]) + if form.maskResult { + form.byteWidth = amd64VectorByteWidth(ins.Args[form.second].Reg) + if len(ins.Args) == 5 { + if !amd64NonzeroKOperand(ins.Args[3]) { + return fail() + } + form.mask = 3 + } + } else { + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if dotted || len(ins.Args) != 4 || form.byteWidth != 16 && form.byteWidth != 32 { + return fail() + } + } + if spec.scalar && form.byteWidth != 16 || form.byteWidth != 16 && form.byteWidth != 32 && form.byteWidth != 64 { + return fail() + } + } + register := func(operand Operand) bool { + if operand.Kind != OpReg || amd64VectorByteWidth(operand.Reg) != form.byteWidth { + return false + } + index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + limit := 16 + if goarch == "386" { + limit = 8 + } else if form.maskResult { + limit = 32 + } + return valid && index < limit + } + if !register(ins.Args[form.second]) || !form.maskResult && !register(ins.Args[form.destination]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) { + if form.sae { + return fail() + } + if first.Kind == OpSym { + symbol := strings.TrimSpace(first.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if first.Kind == OpMem { + displacement := first.Mem.Off + if displacement != int64(int32(displacement)) && (goarch != "386" || displacement != int64(uint32(displacement))) || + !x86MemoryRegistersValidForArch(first.Mem, goarch) || first.Mem.Index != "" && + (first.Mem.Index == SP || first.Mem.Scale != 1 && first.Mem.Scale != 2 && first.Mem.Scale != 4 && first.Mem.Scale != 8) { + return fail() + } + } + form.memory, form.memoryBytes = form.first, form.byteWidth + if spec.scalar || form.broadcast { + form.memoryBytes = spec.bits / 8 + } + } else if !register(first) || form.broadcast { + return fail() + } + if form.broadcast && (spec.scalar || !form.maskResult) || form.sae && (!form.maskResult || !spec.scalar && form.byteWidth != 64) { + return fail() + } + return form, true, nil +} diff --git a/amd64_float_compare_forms_test.go b/amd64_float_compare_forms_test.go new file mode 100644 index 000000000..7174dcf3c --- /dev/null +++ b/amd64_float_compare_forms_test.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86FloatCompareOpcodes() []string { + return []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} +} + +// Enumerate independently from the production grammar: yxcmpi has one +// signed-imm8 row, _yvcmppd has eight rows, and _yvcmpsd has three rows. +func x86FloatCompareGoForms(arch string) []string { + var rows []string + for _, opcode := range x86FloatCompareOpcodes() { + scalar := strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") + if !strings.HasPrefix(opcode, "V") { + for _, source := range []string{"X0", "8(AX)", "(AX)(CX*4)"} { + for _, immediate := range []int{-128, -1, 0, 7, 127} { + rows = append(rows, fmt.Sprintf("%s %s,X1,$%d", opcode, source, immediate)) + } + } + continue + } + if arch == "386" { + continue // Go rejects named VCMP; physical raw forms are separate. + } + widths := []string{"X", "Y"} + if scalar { + widths = widths[:1] + } + for _, width := range widths { + for _, source := range []string{width + "0", "8(AX)", "(AX)(CX*4)"} { + for _, immediate := range []int{0, 31, 255} { + rows = append(rows, fmt.Sprintf("%s $%d,%s,%s1,%s2", opcode, immediate, source, width, width)) + } + } + } + widths = []string{"X", "Y", "Z"} + if scalar { + widths = widths[:1] + } + for _, width := range widths { + for _, mask := range []string{"", "K1,"} { + for _, destination := range []string{"K0", "K7"} { + for _, source := range []string{width + "20", "8(AX)", "(AX)(CX*4)"} { + rows = append(rows, fmt.Sprintf("%s $255,%s,%s21,%s%s", opcode, source, width, mask, destination)) + } + if !scalar { + rows = append(rows, fmt.Sprintf("%s.BCST $31,8(AX),%s21,%s%s", opcode, width, mask, destination)) + } + if scalar || width == "Z" { + rows = append(rows, fmt.Sprintf("%s.SAE $31,%s20,%s21,%s%s", opcode, width, width, mask, destination)) + } + } + } + } + } + return rows +} + +func TestX86FloatingCompareCompleteGoGrammarFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + for index, row := range x86FloatCompareGoForms(arch) { + name := fmt.Sprintf("compare_form%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + triples := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-macosx", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir := x86FloatingCompareUnboundFormatObject(t, source.String(), arch, triple) + compileLLVMToObject(t, llc, triple, "compare-formats.ll", "compare-formats.o", ir) + } + t.Logf("%s: %d original Go compare formats; public Context and emitter-only objects", arch, len(x86FloatCompareGoForms(arch))) + } +} + +func TestX86FloatingCompareMalformedAndDispatchIdentityBeforeEmission(t *testing.T) { + for _, row := range []string{ + "CMPPS X0,X1,$128", "CMPSD X0,Y1,$0", "CMPSS $0,X1,$0", "CMPPD.SAE X0,X1,$0", + "VCMPPS $256,X0,X1,X2", "VCMPPD $0,X0,X1,Z2", "VCMPSS.Z $0,X0,X1,K2", + "VCMPPD $0,X0,X1,K0,K2", "VCMPPS.BCST $0,X0,X1,K2", "VCMPSS.BCST $0,(AX),X1,K2", + "VCMPPS.SAE $0,Y0,Y1,K2", "VCMPSS.SAE $0,(AX),X1,K2", "VCMPPS $0,X0,X1,X17", + "CMPPS (AX)(SP*2),X1,$0", "VCMPPS $0,(AX)(SP*2),X1,X2", "VCMPPS $0,$data(SB),X1,X2", + } { + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + return + } + context, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + context.goarch = "amd64" + instruction := file.Funcs[0].Instrs[1] + before := output.Len() + known, term, err := context.lowerFloatingCompare(instruction.Op, instruction) + if !known || term || err == nil || output.Len() != before { + t.Fatalf("malformed compare emitted IR: %t/%t/%v", known, term, err) + } + }) + } + for _, opcode := range x86FloatCompareOpcodes() { + row := opcode + " X0,X1,$0" + if strings.HasPrefix(opcode, "V") { + row = opcode + " $0,X0,X1,X2" + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + context, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + context.goarch = "amd64" + instruction := file.Funcs[0].Instrs[1] + instruction.Op = "MULSS" + before := output.Len() + if known, term, err := context.lowerFloatingCompare(Op(opcode), instruction); !known || term || err == nil || output.Len() != before { + t.Fatalf("conflicting compare identity emitted IR: %s/%t/%t/%v", opcode, known, term, err) + } + } +} diff --git a/amd64_float_compare_fp_runtime_test.go b/amd64_float_compare_fp_runtime_test.go new file mode 100644 index 000000000..a81a5d3bd --- /dev/null +++ b/amd64_float_compare_fp_runtime_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" +) + +// Only fixture layout uses the ordinary Go ABI derivation. Neither numerical +// truth nor MXCSR expectations use the production compare implementation. +type x86FloatCompareFPOracle struct { + name, declaration, row string + bits, predicate int + mask bool + axis x86FloatCompareRuntimeISA + sig FuncSig +} + +func x86FloatCompareFPOracles(t *testing.T) []x86FloatCompareFPOracle { + t.Helper() + var fixtures []x86FloatCompareFPOracle + var declarations strings.Builder + declarations.WriteString("package oracle\n") + for _, bits := range []int{32, 64} { + for _, spelling := range []string{"legacy", "vex", "evex"} { + for predicate := 0; predicate < 8; predicate++ { + fixture := x86FloatCompareFPOracle{ + name: fmt.Sprintf("fp%d%s%d", bits, spelling, predicate), + bits: bits, predicate: predicate, mask: spelling == "evex", + axis: map[string]x86FloatCompareRuntimeISA{"legacy": x86FloatCompareSSE, "vex": x86FloatCompareVEX, "evex": x86FloatCompareEVEX}[spelling], + } + fixture.declaration = fmt.Sprintf("func %s(out *[20]byte,source [%d]byte,left uint%d)\n", fixture.name, bits/8, bits) + declarations.WriteString(fixture.declaration) + opcode := "CMPSS" + if bits == 64 { + opcode = "CMPSD" + } + fixture.row = fmt.Sprintf("%s source+8(FP),X0,$%d", opcode, predicate) + if spelling == "vex" { + fixture.row = fmt.Sprintf("V%s $%d,source+8(FP),X0,X2", opcode, predicate) + } else if spelling == "evex" { + fixture.row = fmt.Sprintf("V%s $%d,source+8(FP),X0,K1,K2", opcode, predicate) + } + fixtures = append(fixtures, fixture) + } + } + } + set := token.NewFileSet() + file, err := parser.ParseFile(set, "oracle.go", declarations.String(), 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + pkg, err := (&types.Config{Sizes: sizes}).Check("oracle", set, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + for index := range fixtures { + fixture := &fixtures[index] + fixture.sig, err = DeriveGoFuncABI0Signature(fixture.name, pkg.Scope().Lookup(fixture.name).(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + if len(fixture.sig.Frame.Params) != fixture.bits/8+2 { + t.Fatalf("Go array did not derive independent byte fields: %+v", fixture.sig.Frame) + } + } + return fixtures +} + +func TestCrossLinuxRuntimeMatrixX86FloatingCompareFPScalarByteSpans(t *testing.T) { + x86FloatCompareFPScalarByteSpansRuntime(t, x86FloatCompareSSE) +} + +func x86FloatCompareFPAssembly(fixtures []x86FloatCompareFPOracle) (string, map[string]FuncSig) { + var assembly strings.Builder + sigs := map[string]FuncSig{} + for _, fixture := range fixtures { + move, destination := "MOVSS", "X0" + if fixture.bits == 64 { + move = "MOVSD" + } + if fixture.axis == x86FloatCompareVEX { + destination = "X2" + } + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-%d\nMOVQ out+0(FP),AX\n%s left+%d(FP),X0\nMOVQ $1,DX\nKMOVQ DX,K1\n%s\n", + fixture.name, 8+2*(fixture.bits/8), move, 8+fixture.bits/8, fixture.row) + if fixture.mask { + assembly.WriteString("KMOVQ K2,(AX)\n") + } else { + fmt.Fprintf(&assembly, "MOVOU %s,(AX)\n", destination) + } + assembly.WriteString("STMXCSR 16(AX)\nRET\n") + sigs[fixture.name] = fixture.sig + } + return assembly.String(), sigs +} + +func x86FloatCompareFPScalarByteSpansRuntime(t *testing.T, axis x86FloatCompareRuntimeISA) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, axis) + var fixtures []x86FloatCompareFPOracle + for _, fixture := range x86FloatCompareFPOracles(t) { + if fixture.axis == axis { + fixtures = append(fixtures, fixture) + } + } + if len(fixtures) != 16 { + t.Fatal("incomplete scalar FP-byte-field execution axis", axis, len(fixtures)) + } + assembly, sigs := x86FloatCompareFPAssembly(fixtures) + var wrappers, declarations, checks strings.Builder + for index, fixture := range fixtures { + + // A C bit carrier is converted into the precise LLVM array value. C's + // aggregate calling convention is not assumed to match Go's signature. + fmt.Fprintf(&wrappers, "define void @check_%s(ptr %%out,i64 %%right,i64 %%left){\n", fixture.name) + packed := "poison" + for field := 0; field < fixture.bits/8; field++ { + fmt.Fprintf(&wrappers, " %%shift%d=lshr i64 %%right,%d\n %%byte%d=trunc i64 %%shift%d to i8\n %%pack%d=insertvalue [%d x i8] %s,i8 %%byte%d,%d\n", + field, field*8, field, field, field, fixture.bits/8, packed, field, field) + packed = fmt.Sprintf("%%pack%d", field) + } + left := "%left" + if fixture.bits == 32 { + wrappers.WriteString(" %left32=trunc i64 %left to i32\n") + left = "%left32" + } + fmt.Fprintf(&wrappers, " call void @%s(ptr %%out,[%d x i8] %s,i%d %s)\n ret void\n}\n", fixture.name, fixture.bits/8, packed, fixture.bits, left) + fmt.Fprintf(&declarations, "extern void check_%s(void*,uint64_t,uint64_t);\n", fixture.name) + fmt.Fprintf(&checks, "if(check(check_%s,%d,%d,%d))return %d;\n", fixture.name, fixture.bits, fixture.predicate, + x86FloatAddSubBoolInt(fixture.mask), index+1) + } + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if err != nil { + t.Fatal(err) + } + const check = ` +typedef void(*compare_fp)(void*,uint64_t,uint64_t); +static int check(compare_fp call,unsigned bits,unsigned predicate,unsigned mask_result){ + const uint64_t *edges=bits==32?edges32:edges64; + unsigned saved;__asm__ volatile("stmxcsr %0":"=m"(saved)); + for(unsigned i=0;i<18;i++)for(unsigned j=0;j<18;j++)for(unsigned c=0;c<6;c++){ + uint8_t output[22],expected[20];memset(output,0xc3,sizeof(output));memset(expected,0,20); + unsigned bytes=bits/8,csr=controls[c]; + int truth=independent_compare(bits,edges[i],edges[j],predicate,csr); + if(mask_result){memset(expected+8,0xc3,8);expected[0]=(uint8_t)truth;} + else memset(expected,truth?255:0,bytes); + unsigned effect=original_effect(bits,edges[i],edges[j],predicate,csr); + memcpy(expected+16,&effect,4); + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + call(output+1,edges[j],edges[i]); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + if(output[0]!=0xc3||output[21]!=0xc3||memcmp(output+1,expected,20)){ + fprintf(stderr,"FP compare bits=%u pred=%u i=%u j=%u csr=%x mask=%u observed=%llx wanted=%llx\n", + bits,predicate,i,j,csr,mask_result,(unsigned long long)read_bits(output+1,8),(unsigned long long)read_bits(expected,8)); + return 1; + } + } + return 0; +} +` + main := x86FloatCompareIndependentC + declarations.String() + check + "int main(void){\n" + checks.String() + "return 0;}\n" + execution.compileAndRunLLVM(t, "compare-fp-scalar-bytes", ir+wrappers.String(), main, "-O2") + t.Logf("%d scalar compare programs ISA-axis=%d, independently allocated 4/8 FP byte fields ×1944 inputs; actual transport=%v", len(fixtures), axis, execution.prefix) +} diff --git a/amd64_float_compare_immediate_class_test.go b/amd64_float_compare_immediate_class_test.go new file mode 100644 index 000000000..fe6a5a9dc --- /dev/null +++ b/amd64_float_compare_immediate_class_test.go @@ -0,0 +1,35 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86FloatingCompareIntegerImmediateClassBeforeEmission(t *testing.T) { + for _, opcode := range x86FloatCompareOpcodes() { + row, index := opcode+" X0,X1,$0.0", 2 + if strings.HasPrefix(opcode, "V") { + row, index = opcode+" $0.0,X0,X1,X2", 0 + } + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, unresolved := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/unresolved=%t", opcode, unresolved), func(t *testing.T) { + ins := file.Funcs[0].Instrs[1] + ins.Args = append([]Operand(nil), ins.Args...) + if unresolved { + ins.Args[index] = Operand{Kind: OpImm, ImmRaw: "$UNRESOLVED_PREDICATE"} + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + if known, term, err := ctx.lowerFloatingCompare(ins.Op, ins); !known || term || err == nil || output.Len() != before { + t.Fatalf("floating/symbolic predicate acquired integer comparison or emitted IR: %t/%t/%v", known, term, err) + } + }) + } + } +} diff --git a/amd64_float_compare_isa_audit_test.go b/amd64_float_compare_isa_audit_test.go new file mode 100644 index 000000000..fd67bad84 --- /dev/null +++ b/amd64_float_compare_isa_audit_test.go @@ -0,0 +1,53 @@ +package plan9asm + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +// Observation only: no ISA permission or substitute numerical verdict. +func TestX86FloatingCompareActualExecutionISAAudit(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + directory := t.TempDir() + source := filepath.Join(directory, "isa.c") + executable := filepath.Join(directory, "isa") + const program = `#include +#include +#include +int main(void) { + unsigned a, b, c, d; + uint64_t xcr0 = 0; + unsigned osxsave = 0, raw_avx = 0, raw_f = 0, raw_vl = 0; + if (__get_cpuid(1, &a, &b, &c, &d)) { + osxsave = (c >> 27) & 1; + raw_avx = (c >> 28) & 1; + if (osxsave) { + unsigned low, high; + __asm__ volatile("xgetbv" : "=a"(low), "=d"(high) : "c"(0)); + xcr0 = ((uint64_t)high << 32) | low; + } + } + if (__get_cpuid_count(7, 0, &a, &b, &c, &d)) { + raw_f = (b >> 16) & 1; + raw_vl = (b >> 31) & 1; + } + __builtin_cpu_init(); + printf("SSE2=%d AVX=%d AVX2=%d AVX512F=%d AVX512VL=%d " + "rawAVX=%u rawAVX512F=%u rawAVX512VL=%u OSXSAVE=%u XCR0=%llx\n", + !!__builtin_cpu_supports("sse2"), !!__builtin_cpu_supports("avx"), + !!__builtin_cpu_supports("avx2"), !!__builtin_cpu_supports("avx512f"), + !!__builtin_cpu_supports("avx512vl"), raw_avx, raw_f, raw_vl, + osxsave, (unsigned long long)xcr0); + return 0; +} +` + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + output := execution.run(t, executable) + t.Logf("actual execution host=%s/%s triple=%s prefix=%q: %s", + runtime.GOOS, runtime.GOARCH, execution.triple, execution.prefix, output) +} diff --git a/amd64_float_compare_native.go b/amd64_float_compare_native.go new file mode 100644 index 000000000..eecfa705a --- /dev/null +++ b/amd64_float_compare_native.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// A relation-only fcmp (including constrained fcmp/fcmps) is not the source +// CMP immediate's MXCSR contract. Keep a single sideeffecting native instruction: +// its complete vector, immediate, active mask and SAE decide IA/DE and traps. +// In particular, a per-lane loop changes packed exception priority. +func (c *amd64Ctx) emitNativeFloatingCompare(ins Instr, f x86FloatCompareForm) error { + first := ins.Args[f.first] + if first.Kind == OpFP && !f.scalar { + return fmt.Errorf("%w: %s packed FP comparison needs original contiguous native storage", ErrProbeNeedsContext, ins.Op) + } + if !f.scalar && first.Kind == OpMem && !c.nativeMemoryOperand(ins, f.first) && !c.fixedLocalMemoryCertified(ins, f.first) { + // A word-aligned virtual source stack is neither the original native + // address nor an m128 alignment witness. Do not strengthen its alloca + // alignment or synthesize a native shadow to suppress source faults. + return fmt.Errorf("%w: %s packed source stack lacks original native backing", ErrProbeNeedsContext, ins.Op) + } + if ins.x86AddressBits != 0 { + return fmt.Errorf("%w: %s native compare needs its original address-size binding", ErrProbeNeedsContext, ins.Op) + } + if first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, f.bits/8); err != nil { + return err + } + } + + mask := "" + if f.mask >= 0 { + value, err := c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return err + } + mask = value + } + typ := amd64FloatingVectorType(f.byteWidth*8/f.bits, f.bits) + second, err := c.loadFloatingVectorOperand(ins.Args[f.second], f.byteWidth, f.byteWidth*8/f.bits, f.bits) + if err != nil { + return err + } + firstValue, firstType, firstConstraint, firstTemplate, err := c.nativeFloatingCompareSource(first, f, typ, mask) + if err != nil { + return err + } + if !f.maskResult && firstConstraint == "v" { + // A mixed function can also contain EVEX and therefore enable AVX512. + // Legacy/VEX still cannot encode registers 16-31 in this operand. + firstConstraint = "x" + } + mnemonic := strings.ToLower(strings.Split(string(ins.Op), ".")[0]) + immediate := f.immediate + result := c.newTmp() + clobbers := "~{memory},~{fpsr}" + + if f.maskResult { + // The K result is private physical scratch. Inputs/outputs remain + // virtual; real k1/k2 do not become public caller transports. + assembly := "" + constraints := "=r," + firstConstraint + ",v" + args := fmt.Sprintf("%s %s, %s %s", firstType, firstValue, typ, second) + if mask != "" { + assembly = "kmovw ${3:k}, %k2; " + constraints += ",r,~{k2}" + args += ", i32 " + c.approximateMathMask32(mask) + } + assembly += fmt.Sprintf("%s $$%d, ", mnemonic, immediate) + if f.sae { + assembly += "{sae}, " + } + if f.broadcast { + firstTemplate += fmt.Sprintf(" {1to%d}", f.byteWidth*8/f.bits) + } + assembly += firstTemplate + ", $2, %k1" + if mask != "" { + assembly += " {%k2}" + } + assembly += "; kmovw %k1, ${0:k}" + fmt.Fprintf(c.b, " %%%s = call i32 asm sideeffect %q, %q(%s)\n", result, assembly, constraints+",~{k1},"+clobbers, args) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, result) + return c.storeK(ins.Args[f.destination].Reg, "%"+wide) + } + + assembly := fmt.Sprintf("%s $$%d, ", mnemonic, immediate) + constraints := "=x," + args := "" + if f.legacy { + // Tie the full old X value: scalar compares preserve upper X bytes. + assembly += strings.Replace(firstTemplate, "$1", "$2", 1) + ", $0" + constraints += "0," + firstConstraint + args = fmt.Sprintf("%s %s, %s %s", typ, second, firstType, firstValue) + } else { + assembly += firstTemplate + ", $2, $0" + constraints += firstConstraint + ",x" + args = fmt.Sprintf("%s %s, %s %s", firstType, firstValue, typ, second) + } + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, typ, assembly, constraints+","+clobbers, args) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, typ, result, f.byteWidth) + if f.legacy { + return c.storeLegacyXViews(ins.Args[f.destination].Reg, "%"+bytes) + } + return c.storePackedMoveOperand(ins.Args[f.destination], f.byteWidth, "%"+bytes) +} + +func (c *amd64Ctx) nativeFloatingCompareSource(source Operand, f x86FloatCompareForm, typ, mask string) (value, sourceType, constraint, template string, err error) { + if source.Kind == OpFP { + // Scalar compare observes only the low field. Assemble its checked + // bytes as integer data, not a fabricated contiguous native FP pointer. + bits, loadErr := c.loadFloatBinaryScalar(source, f.bits, mask) + if loadErr != nil { + return "", "", "", "", loadErr + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s zeroinitializer, %s %s, i32 0\n", inserted, typ, amd64FloatingScalarType(f.bits), bits) + return "%" + inserted, typ, "v", "$1", nil + } + if source.Kind == OpReg { + value, err = c.loadFloatingVectorOperand(source, f.byteWidth, f.byteWidth*8/f.bits, f.bits) + return value, typ, "v", "$1", err + } + if source.Kind != OpMem && source.Kind != OpSym { + return "", "", "", "", fmt.Errorf("%w: native compare source lacks original memory binding", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", "", "", "", err + } + segment := "" + if source.Kind == OpMem && source.Mem.Segment != "" { + segment = "%" + strings.ToLower(string(source.Mem.Segment)) + ":" + } + return pointer, fmt.Sprintf("%s elementtype([%d x i8])", pointerType, f.memoryBytes), "*m", segment + "$1", nil +} diff --git a/amd64_float_compare_native_test.go b/amd64_float_compare_native_test.go new file mode 100644 index 000000000..c8b858851 --- /dev/null +++ b/amd64_float_compare_native_test.go @@ -0,0 +1,311 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +func TestX86FloatCompareNativeOriginalInstruction(t *testing.T) { + for _, row := range []string{ + "CMPPS X0,X1,$3", "CMPSD 3(AX),X1,$1", + "VCMPPS $11,Y0,Y1,Y2", "VCMPPD $15,Z0,Z1,K3,K2", + "VCMPPS.BCST $31,3(AX),Z1,K3,K2", "VCMPSS.SAE $31,X0,X1,K3,K2", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + ir := x86FloatingCompareUnboundFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + opcode := strings.ToLower(strings.Split(row, " ")[0]) + opcode = strings.Split(opcode, ".")[0] + if !strings.Contains(ir, "asm sideeffect") || !strings.Contains(ir, opcode+" $$") || strings.Contains(ir, " = fcmp ") { + t.Fatalf("original whole instruction/CSR effect missing:\n%s", ir) + } + }) + } +} + +func TestX86FloatCompareNativeAllPredicatesObjectsAndDeadO2(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + var source strings.Builder + count := 0 + for _, opcode := range x86FloatCompareOpcodes() { + for _, row := range x86FloatCompareOracleCases(opcode) { + name := fmt.Sprintf("native_compare%d", count) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row.row) + count++ + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + for _, triple := range []string{"x86_64-apple-macosx", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + ir := x86FloatingCompareUnboundFormatObject(t, source.String(), "amd64", triple) + if strings.Count(ir, "asm sideeffect") != count { + t.Fatalf("whole-instruction sideeffects=%d want=%d", strings.Count(ir, "asm sideeffect"), count) + } + compileLLVMToObject(t, llc, triple, "native-compare.ll", "native-compare.o", ir) + path := filepath.Join(t.TempDir(), "dead.ll") + if err := os.WriteFile(path, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + command := exec.CommandContext(ctx, llc, "-O2", "-mtriple="+triple, "-filetype=asm", path, "-o", "-") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("dead native effect object: %v\n%s", err, output) + } + // LLVM's assembly printer canonicalizes CMP-immediate mnemonics to + // predicate aliases (cmpeqps, vcmptrue_uspd, ...). Count those actual + // instructions, not the source spelling, after optimizing dead results. + compares := regexp.MustCompile(`(?m)^\s+v?cmp[a-z0-9_]*(ps|pd|ss|sd)\s+`).FindAll(output, -1) + if len(compares) != count { + t.Fatalf("dead O2 retained %d native comparisons, want %d", len(compares), count) + } + } + t.Logf("%d independent 8/32-predicate forms: 3 objects plus dead O2 native emission", count) +} + +func TestX86FloatCompareNativePhysicalMemoryAndPrefixContract(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + for index, row := range []string{ + "CMPPS 16(AX)(CX*4),X1,$127", "CMPSS 3(FS),X1,$-128", + "CMPPD 16(GS),X1,$-1", "CMPSD 256,X1,$7", + } { + name := fmt.Sprintf("memory%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + triples := []string{"x86_64-apple-macosx", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir := x86FloatingCompareUnboundFormatObject(t, source.String(), arch, triple) + for _, want := range []string{"%fs:$2", "%gs:$2", "elementtype([16 x i8])", "elementtype([4 x i8])", "elementtype([8 x i8])", "$$127", "$$128", "$$255"} { + if !strings.Contains(ir, want) { + t.Fatalf("%s original memory/predicate contract lost %s:\n%s", arch, want, ir) + } + } + compileLLVMToObject(t, llc, triple, "native-memory.ll", "native-memory.o", ir) + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nCMPPS (AX),X1,$0\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "amd64" + ins := file.Funcs[0].Instrs[1] + ins.x86AddressBits = 32 + before := output.Len() + if _, _, err := ctx.lowerFloatingCompare(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || output.Len() != before { + t.Fatalf("unbound native address prefix emitted compare IR: %v", err) + } +} + +func TestX86FloatCompareNativeRawMode32HighBanksRemainContext(t *testing.T) { + for _, code := range [][]byte{ + encodeX86VEXVectorFloatCompare(0, false, 8, 1, 0, 0), + encodeX86VEXVectorFloatCompare(1, true, 2, 8, 0, 0), + encodeX86VEXVectorFloatCompare(2, false, 2, 1, 8, 0), + encodeX86EVEXVectorFloatCompare(0, 2, false, 3, 2, 8, 0, 0), + encodeX86EVEXVectorFloatCompare(1, 0, false, 3, 2, 1, 8, 0), + encodeX86EVEXVectorFloatCompare(2, 0, false, 3, 2, 1, 16, 0), + } { + if _, _, known, err := decodedX86VectorFloatCompareInstruction(code, 32); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mode32 high-bank %x gained a native binding: %v", code, err) + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nVCMPPS $0,X0,X1,X2\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "386", Sigs: map[string]FuncSig{"f": {Ret: Void}}}); err == nil { + t.Fatal("named mode32 source impersonated private raw decoding") + } +} + +func TestX86FloatCompareNativeRawMode32PhysicalForms(t *testing.T) { + mc, llc := findLLVM22Tool("llvm-mc"), findLLVM22Tool("llc") + if mc == "" || llc == "" { + t.Fatal("LLVM22 mc/llc required") + } + var source strings.Builder + count := 0 + for pp := byte(0); pp < 4; pp++ { + widths := []byte{0, 1, 2} + if pp >= 2 { + widths = widths[:1] + } + for _, width := range widths { + for predicate := byte(0); predicate < 32; predicate++ { + var codes [][]byte + if width != 2 { + codes = append(codes, encodeX86VEXVectorFloatCompare(pp, width == 1, 2, 1, 0, predicate)) + } + codes = append(codes, encodeX86EVEXVectorFloatCompare(pp, width, false, 3, 2, 1, 0, predicate)) + if pp >= 2 || width == 2 { + codes = append(codes, encodeX86EVEXVectorFloatCompare(pp, width, true, 3, 2, 1, 0, predicate)) + } + for _, code := range codes { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=i386-unknown-linux-gnu", "--mattr=+avx,+avx512f,+avx512vl") + var hex strings.Builder + for _, b := range code { + fmt.Fprintf(&hex, "0x%02x ", b) + } + command.Stdin = strings.NewReader(hex.String()) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || strings.Contains(string(output), "warning:") || !strings.Contains(string(output), "vcmp") { + t.Fatalf("physical i386 compare %x: %v\n%s", code, err, output) + } + name := fmt.Sprintf("raw_native%d", count) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", name) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $%d\n", b) + } + source.WriteString("RET\n") + count++ + } + } + } + } + goAssemblerOracleOutput(t, "386", source.String(), "", true) + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir := x86FloatingCompareUnboundFormatObject(t, source.String(), "386", triple) + compileLLVMToObject(t, llc, triple, "raw-native-compare.ll", "raw-native-compare.o", ir) + } + t.Logf("%d physical low-bank i386 forms independently decoded by LLVM22; 2 objects", count) +} + +func TestX86FloatCompareNativePackedVirtualFPNeedsContext(t *testing.T) { + for _, row := range []string{ + "CMPPS a+0(FP),X1,$0", "VCMPPD $3,a+0(FP),Y1,Y2", "VCMPPS.BCST $3,a+0(FP),Z1,K2", + "CMPPS 16(SP),X1,$0", "CMPPD 16(BP),X1,$0", "VCMPPS $0,16(SP),Z1,K2", "VCMPPD $0,(AX)(BP*4),Y1,Y2", + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{"f": {Ret: Void}}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s fabricated native backing: %v", row, err) + } + } +} + +func TestX86FloatCompareNativeScalarFPIndependentByteFields(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, opcode := range []string{"CMPSS", "CMPSD", "VCMPSS", "VCMPSD"} { + width := 4 + if strings.HasSuffix(opcode, "SD") { + width = 8 + } + row := opcode + " a+0(FP),X1,$3" + if strings.HasPrefix(opcode, "V") { + row = opcode + " $3,a+0(FP),X1,K3,K2" + } + original := "TEXT f(SB),4,$0-8\n" + row + "\nRET\n" + sig := FuncSig{Ret: Void} + var names []string + for index := 0; index < width; index++ { + sig.Args = append(sig.Args, I8) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(index), Type: I8, Index: index, Field: -1}) + names = append(names, string(rune('a'+index))) + } + declaration := "func f(" + strings.Join(names, ",") + " uint8)" + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", original) + prefix := "PXOR X1,X1\n" + if strings.HasPrefix(opcode, "V") { + prefix += "MOVL $1,AX\nKMOVW AX,K3\n" + } + source := strings.Replace(original, row+"\n", prefix+row+"\n", 1) + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, original, declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if strings.HasPrefix(opcode, "V") { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original FP format invented its mask input", opcode, api, err) + } + } else { + x86VectorScratchCodegenControl(t, original, declaration, "amd64", "x86_64-unknown-linux-gnu", ir, err) + } + ir, err = x86ScalarFPBitViewAPI(t, source, declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal(err) + } + if strings.Contains(ir, "elementtype(") || !strings.Contains(ir, "asm sideeffect") || strings.Count(ir, "load i8, ptr %fp_") != width { + t.Fatalf("%s invented contiguous native FP backing or omitted field bytes:\n%s", row, ir) + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "scalar-fp.ll", "scalar-fp.o", ir) + } + // Even an inactive EVEX mask cannot bless an invalid source span. + sig.Frame.Params = sig.Frame.Params[:width-1] + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{"f": sig}}); err == nil { + t.Fatalf("%s missing final FP byte accepted", row) + } + } +} + +func TestX86FloatCompareNativeMixedISAUsesLowVEXConstraints(t *testing.T) { + const source = "TEXT f(SB),4,$0-0\nCMPSS X0,X1,$3\nVCMPPS $3,Y0,Y1,Y2\nVCMPPS $3,Z0,Z1,K2\nRET\n" + ir := x86FloatingCompareUnboundFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + if !strings.Contains(ir, `"=x,0,x,`) || !strings.Contains(ir, `"=x,x,x,`) || !strings.Contains(ir, `"=r,v,v,`) { + t.Fatalf("mixed EVEX function widened a legacy/VEX physical register class:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "mixed-isa.ll", "mixed-isa.o", ir) +} + +func TestX86FloatCompareNativeValidatedPrivateFeatures(t *testing.T) { + for _, tc := range []struct{ row, want string }{ + {"CMPPS X0,X1,$0", "+sse"}, + {"CMPSD X0,X1,$0", "+sse2"}, + {"VCMPPS $0,Y0,Y1,Y2", "+avx"}, + {"VCMPPS $0,X0,X1,K2", "+avx512f,+avx512vl"}, + {"VCMPSS $0,X0,X1,K2", "+avx512f"}, + {"VCMPPS $0,Z0,Z1,K2", "+avx512f"}, + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+tc.row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", file.Funcs[0]); got != tc.want { + t.Errorf("%s features=%q want=%q", tc.row, got, tc.want) + } + } + for _, row := range []string{"VCMPPS $256,Z0,Z1,K2", "VCMPPS.Z $0,Z0,Z1,K2", "VCMPSS $0,X0,Y1,K2"} { + file, err := Parse(ArchAMD64, fmt.Sprintf("TEXT f(SB),4,$0-0\n%s\nRET\n", row)) + if err != nil { + t.Fatal(err) + } + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", file.Funcs[0]); got != "" { + t.Errorf("malformed source gained ISA permission: %s/%s", row, got) + } + } +} diff --git a/amd64_float_compare_numerical_test.go b/amd64_float_compare_numerical_test.go new file mode 100644 index 000000000..4bd3cf317 --- /dev/null +++ b/amd64_float_compare_numerical_test.go @@ -0,0 +1,257 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86FloatCompareOracleCase struct { + row string + bits, width, predicate int + scalar, legacy, maskResult bool + memory, broadcast, suppressed bool + destination int +} + +// Enumerate this matrix from explicit architectural axes, not the production +// grammar, predicate table, memory-shape classifier or caller effect registry. +func x86FloatCompareOracleCases(opcode string) []x86FloatCompareOracleCase { + bits := 32 + if strings.HasSuffix(opcode, "PD") || strings.HasSuffix(opcode, "SD") { + bits = 64 + } + scalar := strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") + legacy := !strings.HasPrefix(opcode, "V") + predicates := 32 + if legacy { + predicates = 8 + } + var cases []x86FloatCompareOracleCase + for predicate := 0; predicate < predicates; predicate++ { + for variant := 0; variant < 2; variant++ { + width := 16 + if !legacy && !scalar && predicate%2 != 0 { + width = 32 + } + bank := map[int]string{16: "X", 32: "Y"}[width] + first, destination := bank+"1", predicate%3 + if variant == 1 { + first = "(CX)" + } + row := fmt.Sprintf("%s $%d,%s,%s0,%s%d", opcode, predicate, first, bank, bank, destination) + if legacy { + destination = 0 + row = fmt.Sprintf("%s %s,X0,$%d", opcode, first, predicate) + } + cases = append(cases, x86FloatCompareOracleCase{row: row, bits: bits, width: width, + predicate: predicate, scalar: scalar, legacy: legacy, memory: variant == 1, destination: destination}) + } + if legacy { + continue + } + for variant := 0; variant < 4; variant++ { + width := []int{16, 32, 64}[predicate%3] + if scalar { + width = 16 + } + if variant == 3 && !scalar { + width = 64 // Go only permits packed SAE at Z width. + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + first, operation := bank+"1", opcode + memory, broadcast := variant == 1 || variant == 2, variant == 2 && !scalar + if memory { + first = "(CX)" + } + if broadcast { + operation += ".BCST" + } + if variant == 3 { + operation += ".SAE" + } + row := fmt.Sprintf("%s $%d,%s,%s0,K1,K2", operation, predicate, first, bank) + cases = append(cases, x86FloatCompareOracleCase{row: row, bits: bits, width: width, + predicate: predicate, scalar: scalar, maskResult: true, memory: memory, + broadcast: broadcast, suppressed: variant == 3}) + } + } + return cases +} + +func x86FloatCompareOracleAssembly(cases []x86FloatCompareOracleCase, dead bool) (string, map[string]FuncSig) { + var source strings.Builder + sigs := map[string]FuncSig{} + for index, c := range cases { + name := fmt.Sprintf("compare%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (BX),Z0\nVMOVUPS (BX),Z2\n", name) + if !c.memory { + source.WriteString("VMOVUPS (CX),Z1\n") + } + source.WriteString("KMOVQ DX,K1\nKMOVQ DX,K2\nSTC\n" + c.row + "\n") + if !dead { + if c.maskResult { + source.WriteString("KMOVQ K2,8(AX)\n") + } else { + fmt.Fprintf(&source, "VMOVUPS Z%d,8(AX)\n", c.destination) + } + } + source.WriteString("SETCS 72(AX)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + } + return source.String(), sigs +} + +func x86FloatCompareOracleMain(cases []x86FloatCompareOracleCase, dead bool) string { + var declarations, descriptors strings.Builder + for index, c := range cases { + fmt.Fprintf(&declarations, "extern void compare%d(void*,const void*,const void*,uint64_t);\n", index) + fmt.Fprintf(&descriptors, "{compare%d,%d,%d,%d,%d,%d,%d,%d,%d,%d},\n", index, c.bits, c.width, c.predicate, + x86FloatAddSubBoolInt(c.scalar), x86FloatAddSubBoolInt(c.legacy), x86FloatAddSubBoolInt(c.maskResult), + x86FloatAddSubBoolInt(c.memory), x86FloatAddSubBoolInt(c.broadcast), x86FloatAddSubBoolInt(c.suppressed)) + } + return x86FloatCompareIndependentC + declarations.String() + ` +typedef void (*compare_fn)(void*,const void*,const void*,uint64_t); +struct comparison {compare_fn call;unsigned bits,width,predicate,scalar,legacy,mask_result,memory,broadcast,sae;}; +static const struct comparison comparisons[]={ +` + descriptors.String() + fmt.Sprintf(` +}; +int main(void){ + const uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX}; + unsigned saved;__asm__ volatile("stmxcsr %%0":"=m"(saved)); + for(unsigned index=0;indexbits/8; + const uint64_t *edges=c->bits==32?edges32:edges64; + for(unsigned pair=0;pair<18;pair++) for(unsigned m=0;m<4;m++) for(unsigned control=0;control<6;control++){ + _Alignas(64) uint8_t aStorage[192],bStorage[192]; + uint8_t *a=aStorage+64,*b=bStorage+64,out[81],want[64]; + memset(aStorage,0xa7,sizeof(aStorage));memset(bStorage,0xb9,sizeof(bStorage));memset(out,0xc3,sizeof(out)); + for(unsigned lane=0;lane<64/bytes;lane++){ + uint64_t av=edges[(pair+lane)%%18],bv=edges[(pair*7+lane*3+1)%%18]; + // Extended predicates have no actual AVX512 execution oracle here. Keep + // denormal and NaN coverage, but do not mix them in a single extended + // comparison: the diagnostic SSE class representative is not a proof of + // that exception-priority corner (the original legacy matrix retains it). + if(c->predicate>=8 && ((is_denormal(c->bits,av)&&is_nan(c->bits,bv)) || + (is_nan(c->bits,av)&&is_denormal(c->bits,bv)))) { + if(is_denormal(c->bits,av))av=0; + if(is_denormal(c->bits,bv))bv=0; + } + memcpy(a+lane*bytes,&av,bytes);memcpy(b+lane*bytes,&bv,bytes); + } + uint8_t beforeA[192],beforeB[192];memcpy(beforeA,aStorage,192);memcpy(beforeB,bStorage,192); + unsigned csr=controls[control],after,expectedCSR=csr;uint64_t expectedMask=0; + memset(want,0,64); + if(c->legacy)memcpy(want,a,64);else if(c->scalar)memcpy(want,a,16); + unsigned lanes=c->scalar?1:c->width/bytes; + for(unsigned lane=0;lanemask_result || ((masks[m]>>lane)&1); + if(!active)continue; + uint64_t av=read_bits(a+lane*bytes,bytes),bv=read_bits(b+(c->broadcast?0:lane*bytes),bytes); + int truth=independent_compare(c->bits,av,bv,c->predicate,csr); + if(truth)expectedMask|=UINT64_C(1)<sae)expectedCSR|=original_effect(c->bits,av,bv,c->predicate,csr)&63; + } + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + const uint8_t *input=c->mask_result&&c->memory&&m==0?0:b; + c->call(out,a,input,masks[m]); + __asm__ volatile("stmxcsr %%0":"=m"(after)); + __asm__ volatile("ldmxcsr %%0"::"m"(saved)); + if(after!=expectedCSR){fprintf(stderr,"compare%%u pred=%%u mxcsr=%%x want=%%x mask=%%llx\n",index,c->predicate,after,expectedCSR,(unsigned long long)masks[m]);return 10;} + if(out[72]!=1||memcmp(aStorage,beforeA,192)||memcmp(bStorage,beforeB,192))return 11; + for(unsigned byte=0;byte<81;byte++){ + uint8_t expected=0xc3; + if(byte==72)expected=1; + if(!%d&&byte>=8&&byte<72){ + if(c->mask_result){if(byte<16)expected=(uint8_t)(expectedMask>>(8*(byte-8)));} + else expected=want[byte-8]; + } + if(out[byte]!=expected){fprintf(stderr,"compare%%u pred=%%u byte=%%u got=%%x want=%%x\n",index,c->predicate,byte,out[byte],expected);return 12;} + } + } + } + __asm__ volatile("ldmxcsr %%0"::"m"(saved));return 0; +} +`, x86FloatAddSubBoolInt(dead)) +} + +// Keep optimized-away results separate: a compare still has MXCSR/trap and +// memory effects. Both optimization levels execute the same independent C. +func x86FloatCompareRunOptimization(t *testing.T, execution x86FloatCompareExecution, ir, main, level string) { + t.Helper() + execution.compileAndRunLLVM(t, "compare-effects", ir, main, level) +} + +func TestCrossLinuxRuntimeMatrixX86FloatingCompareIndependentIEEEAndMXCSR(t *testing.T) { + x86FloatCompareIndependentRuntime(t, x86FloatCompareSSE) +} + +func x86FloatCompareIndependentRuntime(t *testing.T, axis x86FloatCompareRuntimeISA) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, axis) + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + cases := x86FloatCompareRuntimeCases(opcode, axis) + if len(cases) == 0 { + continue + } + t.Run(opcode, func(t *testing.T) { + assembly, sigs := x86FloatCompareOracleAssembly(cases, false) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := x86FloatCompareOracleMain(cases, false) + execution.compileAndRunLLVM(t, "compare-ieee", ir, main, "-O2") + t.Logf("%s ISA-axis=%d: %d independently enumerated programs ×432 IEEE/DAZ/full-MXCSR/mask/upper-alias/canary inputs; extended NaN+denormal priority is not covered; actual transport=%v", opcode, axis, len(cases), execution.prefix) + }) + } +} + +func TestCrossLinuxRuntimeMatrixX86FloatingCompareDeadResultEffects(t *testing.T) { + x86FloatCompareDeadResultRuntime(t, x86FloatCompareSSE) +} + +func x86FloatCompareDeadPredicate(predicate int) bool { + return predicate == 0 || predicate == 1 || predicate == 11 || predicate == 15 || predicate == 27 || predicate == 31 +} + +func x86FloatCompareDeadResultRuntime(t *testing.T, axis x86FloatCompareRuntimeISA) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, axis) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + var cases []x86FloatCompareOracleCase + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + for _, c := range x86FloatCompareRuntimeCases(opcode, axis) { + if x86FloatCompareDeadPredicate(c.predicate) { + cases = append(cases, c) + } + } + } + assembly, sigs := x86FloatCompareOracleAssembly(cases, true) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + x86FloatCompareRunOptimization(t, execution, ir, x86FloatCompareOracleMain(cases, true), level) + t.Logf("dead-result ISA-axis=%d: %d complete programs ×432 inputs, %s, actual transport=%v", axis, len(cases), level, execution.prefix) + }) + } +} diff --git a/amd64_float_compare_oracle_test.go b/amd64_float_compare_oracle_test.go new file mode 100644 index 000000000..b70309cee --- /dev/null +++ b/amd64_float_compare_oracle_test.go @@ -0,0 +1,229 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +// Intel SDM Vol.2A Table 3-1 supplies the relation truth columns and signaling +// column. The relation reference uses only integer IEEE encodings; neither it +// nor the native original scalar CMP exception oracle calls a production helper. +// COMIS/UCOMIS and quiet imm0 are not substituted for legacy predicates: their +// observed exception priority differs for NaN/denormal pairs under Rosetta. +const x86FloatCompareIndependentC = ` +#include +#include +#include +static const unsigned relation_truth[32]={ + 4,2,6,8,11,13,9,7,12,10,14,0,3,5,1,15, + 4,2,6,8,11,13,9,7,12,10,14,0,3,5,1,15}; +static int is_nan(unsigned bits,uint64_t a) { + uint64_t fraction=bits==32?0x7fffff:UINT64_C(0xfffffffffffff); + uint64_t exponent=bits==32?0x7f800000:UINT64_C(0x7ff0000000000000); + return (a&exponent)==exponent && (a&fraction)!=0; +} +static uint64_t normalized(unsigned bits,uint64_t a,unsigned csr) { + uint64_t sign=bits==32?0x80000000:UINT64_C(0x8000000000000000); + uint64_t exponent=bits==32?0x7f800000:UINT64_C(0x7ff0000000000000); + if(bits==32) a&=UINT64_C(0xffffffff); + if((csr&64) && !(a&exponent)) a&=sign; + return a; +} +static int independent_compare(unsigned bits,uint64_t a,uint64_t b,unsigned pred,unsigned csr) { + a=normalized(bits,a,csr); b=normalized(bits,b,csr); + unsigned relation; + if(is_nan(bits,a)||is_nan(bits,b)) relation=8; + else { + uint64_t sign=bits==32?0x80000000:UINT64_C(0x8000000000000000); + uint64_t mask=bits==32?UINT64_C(0xffffffff):UINT64_MAX; + if(a==b || (!(a&~sign)&&!(b&~sign))) relation=4; + else { + uint64_t ak=(a&sign)?(~a&mask):(a^sign); + uint64_t bk=(b&sign)?(~b&mask):(b^sign); + relation=ak>(pred&31))&1); +#define CMP_EFFECT(op) do { \ + switch(native_pred) { \ + case 0: __asm__ volatile(op " $0,%1,%0":"+x"(x):"x"(y));break; \ + case 1: __asm__ volatile(op " $1,%1,%0":"+x"(x):"x"(y));break; \ + case 2: __asm__ volatile(op " $2,%1,%0":"+x"(x):"x"(y));break; \ + case 3: __asm__ volatile(op " $3,%1,%0":"+x"(x):"x"(y));break; \ + case 4: __asm__ volatile(op " $4,%1,%0":"+x"(x):"x"(y));break; \ + case 5: __asm__ volatile(op " $5,%1,%0":"+x"(x):"x"(y));break; \ + case 6: __asm__ volatile(op " $6,%1,%0":"+x"(x):"x"(y));break; \ + case 7: __asm__ volatile(op " $7,%1,%0":"+x"(x):"x"(y));break; \ + } \ +} while(0) + if(bits==32) { + uint32_t av=(uint32_t)a,bv=(uint32_t)b; float x,y; + memcpy(&x,&av,4); memcpy(&y,&bv,4); + CMP_EFFECT("cmpss"); + } else { + double x,y; memcpy(&x,&a,8); memcpy(&y,&b,8); + CMP_EFFECT("cmpsd"); + } +#undef CMP_EFFECT + __asm__ volatile("stmxcsr %0":"=m"(after)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + return after; +} +static int is_denormal(unsigned bits,uint64_t a) { + uint64_t fraction=bits==32?0x7fffff:UINT64_C(0xfffffffffffff); + uint64_t exponent=bits==32?0x7f800000:UINT64_C(0x7ff0000000000000); + return !(a&exponent) && (a&fraction)!=0; +} +static const uint64_t edges32[]={0,0x80000000,1,0x80000001,0x7fffff,0x800000, + 0x3f800000,0xbf800000,0x3f800001,0x7f7fffff,0xff7fffff,0x7f800000, + 0xff800000,0x7fc12345,0xffc45678,0x7f812345,0xff845678,0x7fffffff}; +static const uint64_t edges64[]={0,UINT64_C(0x8000000000000000),1, + UINT64_C(0x8000000000000001),UINT64_C(0xfffffffffffff),UINT64_C(0x10000000000000), + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000001), + UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff),UINT64_C(0x7ff0000000000000), + UINT64_C(0xfff0000000000000),UINT64_C(0x7ff8123456789abc),UINT64_C(0xfff8fedcba987654), + UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff0fedcba987654),UINT64_C(0x7fffffffffffffff)}; +static const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1fbf,0x7fc5}; +static uint64_t read_bits(const uint8_t *p,unsigned bytes) { + uint64_t value=0; memcpy(&value,p,bytes); return value; +} +` + +// An actual SSE instruction has a compile-time immediate. This switch covers +// all eight legacy predicates, independently of LLVM's predicate selection. +func x86FloatCompareOriginalC() string { + var source strings.Builder + source.WriteString(x86FloatCompareIndependentC) + for index, op := range []string{"cmpps", "cmppd", "cmpss", "cmpsd"} { + fmt.Fprintf(&source, "static void original%d(uint8_t *out,const uint8_t *a,const uint8_t *b,unsigned pred,unsigned csr){\n unsigned saved,after; __asm__ volatile(\"stmxcsr %%0\":\"=m\"(saved));\n __asm__ volatile(\"ldmxcsr %%0\"::\"m\"(csr));\n typedef uint8_t v16 __attribute__((vector_size(16))); v16 x,y; memcpy(&x,a,16); memcpy(&y,b,16);\n switch(pred){\n", index) + for pred := 0; pred < 8; pred++ { + fmt.Fprintf(&source, " case %d: __asm__ volatile(\"%s $%d,%%1,%%0\":\"+x\"(x):\"x\"(y)); break;\n", pred, op, pred) + } + source.WriteString(" }\n __asm__ volatile(\"stmxcsr %0\":\"=m\"(after));\n __asm__ volatile(\"ldmxcsr %0\"::\"m\"(saved)); memcpy(out,&x,16); memcpy(out+16,&after,4);\n}\n") + } + source.WriteString("typedef void (*original_fn)(uint8_t*,const uint8_t*,const uint8_t*,unsigned,unsigned);\n static original_fn originals[]={original0,original1,original2,original3};\n") + return source.String() +} + +func x86FloatCompareCaptureCommand(t *testing.T, command *exec.Cmd) []byte { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + // Keep one lifecycle/deadline per actual command, including the original-Go + // driver and the independently compiled native SSE reference. + bounded := exec.CommandContext(ctx, command.Path, command.Args[1:]...) + bounded.Dir, bounded.Env = command.Dir, command.Env + output, err := runtimeOracleCombinedOutput(ctx, bounded) + if err != nil { + t.Fatalf("compare oracle command %q: %v\n%s", command.Args, err, output) + } + return output +} + +func TestCrossLinuxRuntimeMatrixX86FloatingCompareOriginalLegacyPredicates(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + var assembly, declarations strings.Builder + assembly.WriteString("#include \"textflag.h\"\n") + for index, op := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD"} { + for pred := 0; pred < 8; pred++ { + name := fmt.Sprintf("legacy%d_%d", index, pred) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),NOSPLIT,$8-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nSTMXCSR 0(SP)\nLDMXCSR csr+24(FP)\nMOVOU (BX),X0\nMOVOU (CX),X1\n%s X1,X0,$%d\nMOVOU X0,(AX)\nSTMXCSR 16(AX)\nLDMXCSR 0(SP)\nRET\n", name, op, pred) + fmt.Fprintf(&declarations, "func %s(out,a,b *[20]byte,csr uint32)\n", name) + } + } + var functions strings.Builder + for index := 0; index < 4; index++ { + for pred := 0; pred < 8; pred++ { + fmt.Fprintf(&functions, "legacy%d_%d,", index, pred) + } + } + const goDriver = `package main +import("encoding/binary";"fmt") +func main(){ + functions:=[]func(*[20]byte,*[20]byte,*[20]byte,uint32){FUNCTIONS} + edges32:=[]uint64{0,0x80000000,1,0x80000001,0x7fffff,0x800000,0x3f800000,0xbf800000,0x3f800001,0x7f7fffff,0xff7fffff,0x7f800000,0xff800000,0x7fc12345,0xffc45678,0x7f812345,0xff845678,0x7fffffff} + edges64:=[]uint64{0,0x8000000000000000,1,0x8000000000000001,0xfffffffffffff,0x10000000000000,0x3ff0000000000000,0xbff0000000000000,0x3ff0000000000001,0x7fefffffffffffff,0xffefffffffffffff,0x7ff0000000000000,0xfff0000000000000,0x7ff8123456789abc,0xfff8fedcba987654,0x7ff0123456789abc,0xfff0fedcba987654,0x7fffffffffffffff} + controls:=[]uint32{0x1f80,0x1fc0,0x9f80,0x9fc0,0x1fbf,0x7fc5} + hash:=uint64(1469598103934665603) + for index,fn:=range functions{ + bytes,edges:=4,edges32 + if index/8==1||index/8==3{bytes,edges=8,edges64} + for _,av:=range edges{for _,bv:=range edges{for _,csr:=range controls{ + var a,b [20]byte + out:=struct{before byte;data [20]byte;after byte}{before:0xa7,after:0xb9} + for i:=0;i<16;i+=bytes{ + if bytes==4{binary.LittleEndian.PutUint32(a[i:],uint32(av));binary.LittleEndian.PutUint32(b[i:],uint32(bv))}else{binary.LittleEndian.PutUint64(a[i:],av);binary.LittleEndian.PutUint64(b[i:],bv)} + } + beforeA,beforeB:=a,b;fn(&out.data,&a,&b,csr) + if a!=beforeA||b!=beforeB||out.before!=0xa7||out.after!=0xb9{panic("input/output canary mutation")} + for _,v:=range out.data{hash^=uint64(v);hash*=1099511628211} + }}} + } + fmt.Printf("%016x\n",hash) +} +` + for name, source := range map[string]string{ + "go.mod": "module originalcompare\n\ngo 1.20\n", + "main.go": strings.Replace(goDriver, "FUNCTIONS", functions.String(), 1) + declarations.String(), + "original_amd64.s": assembly.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + goExecutable := filepath.Join(dir, gotoolchain.HostExecutable("original-go")) + command := exec.Command("go", "build", "-p=2", "-o", goExecutable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH=amd64", "CGO_ENABLED=0", "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local") + x86FloatCompareCaptureCommand(t, command) + actualGo := execution.run(t, goExecutable) + const main = ` +int main(void){ + uint64_t hash=UINT64_C(1469598103934665603); + for(unsigned op=0;op<4;op++) for(unsigned pred=0;pred<8;pred++){ + unsigned bytes=(op==1||op==3)?8:4; const uint64_t *edges=bytes==4?edges32:edges64; + for(unsigned i=0;i<18;i++) for(unsigned j=0;j<18;j++) for(unsigned c=0;c<6;c++){ + uint8_t a[20]={0},b[20]={0},storage[22],want[16]; + uint8_t *out=storage+1;memset(storage,0xc3,sizeof(storage)); + for(unsigned lane=0;lane<16;lane+=bytes){memcpy(a+lane,edges+i,bytes);memcpy(b+lane,edges+j,bytes);} + uint8_t beforeA[20],beforeB[20];memcpy(beforeA,a,20);memcpy(beforeB,b,20); + originals[op](out,a,b,pred,controls[c]); memcpy(want,a,16); + if(storage[0]!=0xc3||storage[21]!=0xc3||memcmp(a,beforeA,20)||memcmp(b,beforeB,20))return 2; + unsigned compared=op>=2?bytes:16; + for(unsigned lane=0;lane=4?bytes:width; + for(unsigned lane=0;lane x86FloatCompareEVEX { + t.Fatal("unknown compare execution ISA", axis) + } + if axis != x86FloatCompareSSE && (runtime.GOOS != "linux" || runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64") { + t.Fatal("VEX/EVEX comparison must consume its required Linux execution counterpart") + } + if axis == x86FloatCompareEVEX && runtime.GOARCH != "amd64" { + t.Fatal("EVEX requires actual native Linux AMD64, not an emulator or Rosetta") + } + execution := x86FloatCompareExecutionTools(t) + if axis == x86FloatCompareEVEX && (len(execution.prefix) != 0 || execution.preserveArgv0) { + t.Fatal("EVEX native counterpart cannot use a guest execution route") + } + x86FloatCompareVerifyArguments(t, execution) + if axis == x86FloatCompareVEX { + execution.requireAVX(t) + } + x86FloatCompareRequireOpcodeISA(t, execution, axis) + return execution +} + +// CPUID and enabled OS register state are both required. The original whole +// opcodes execute only after this check; their results are consumed. This is +// an execution preflight, not a production source/ISA authorization token. +const x86FloatCompareOpcodePreflightC = `#include +#include +#include +#if !defined(_WIN32) +#include +#endif + +static int required_state(unsigned axis, unsigned sse2, unsigned avx, + unsigned osxsave, unsigned f, unsigned vl, uint64_t xcr0) { + if (axis==1) return sse2!=0; + if (!sse2 || !avx || !osxsave || (xcr0&6)!=6) return 0; + if (axis==2) return 1; + return axis==3 && f && vl && (xcr0&0xe6)==0xe6; +} + +__attribute__((target("avx"))) static unsigned original_vex(void) { + unsigned mask; + __asm__ volatile("vxorps %%xmm0,%%xmm0,%%xmm0; " + "vcmpps $0,%%xmm0,%%xmm0,%%xmm1; " + "vmovmskps %%xmm1,%0" : "=r"(mask) :: "xmm0","xmm1"); + return mask; +} + +__attribute__((target("avx512f,avx512vl"))) static int original_evex(void) { + unsigned ps,pd,ss,sd,active; + __asm__ volatile("vxorps %%zmm0,%%zmm0,%%zmm0; " + "vcmpps $0,%%zmm0,%%zmm0,%%k1; kmovw %%k1,%0; " + "vcmppd $0,%%zmm0,%%zmm0,%%k1; kmovw %%k1,%1; " + "vcmpss $0,%%xmm0,%%xmm0,%%k1; kmovw %%k1,%2; " + "vcmpsd $0,%%xmm0,%%xmm0,%%k1; kmovw %%k1,%3; " + "movl $1,%4; kmovw %4,%%k2; " + "vcmpps $0,%%zmm0,%%zmm0,%%k1%{%%k2%}; kmovw %%k1,%4" + : "=r"(ps),"=r"(pd),"=r"(ss),"=r"(sd),"=r"(active) + :: "zmm0","k1","k2"); + return ps==0xffff && pd==0xff && ss==1 && sd==1 && active==1; +} + +int main(void) { +#if !defined(_WIN32) + struct rlimit no_core={0,0}; + if (setrlimit(RLIMIT_CORE,&no_core)) return 77; +#endif + unsigned a,b,c,d,sse2=0,avx=0,osxsave=0,f=0,vl=0; + uint64_t xcr0=0; + if (__get_cpuid(1,&a,&b,&c,&d)) { + sse2=(d>>26)&1; avx=(c>>28)&1; osxsave=(c>>27)&1; + if (osxsave) { + unsigned low,high; + __asm__ volatile("xgetbv" : "=a"(low),"=d"(high) : "c"(0)); + xcr0=((uint64_t)high<<32)|low; + } + } + if (__get_cpuid_count(7,0,&a,&b,&c,&d)) { + f=(b>>16)&1; vl=(b>>31)&1; + } + unsigned axis=REQUIRED_AXIS; + printf("axis=%u SSE2=%u AVX=%u OSXSAVE=%u AVX512F=%u AVX512VL=%u XCR0=%llx\n", + axis,sse2,avx,osxsave,f,vl,(unsigned long long)xcr0); + fflush(stdout); + if (!required_state(axis,sse2,avx,osxsave,f,vl,xcr0)) return 78; + if (axis==2 && original_vex()!=15) return 79; + if (axis==3 && !original_evex()) return 80; + return 0; +} +` + +func x86FloatCompareRequireOpcodeISA(t *testing.T, execution x86FloatCompareExecution, axis x86FloatCompareRuntimeISA) { + t.Helper() + dir := t.TempDir() + source, executable := filepath.Join(dir, "preflight.c"), filepath.Join(dir, "preflight") + program := fmt.Sprintf("#define REQUIRED_AXIS %d\n", axis) + x86FloatCompareOpcodePreflightC + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + output := execution.run(t, executable) + t.Logf("actual CPU/OS-state and original-opcode preflight: %s", output) +} + +func TestX86FloatingCompareRuntimeCaseUnion(t *testing.T) { + counts, dead := [3]int{}, [3]int{} + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + complete := x86FloatCompareOracleCases(opcode) + // Scalar variant 1/2 intentionally have identical memory spellings. + // Preserve their original multiplicity instead of dropping a fixture. + remaining := make(map[x86FloatCompareOracleCase]int) + for _, fixture := range complete { + remaining[fixture]++ + } + for _, axis := range []x86FloatCompareRuntimeISA{x86FloatCompareSSE, x86FloatCompareVEX, x86FloatCompareEVEX} { + for _, fixture := range x86FloatCompareRuntimeCases(opcode, axis) { + if remaining[fixture] == 0 { + t.Fatal("duplicated runtime case", fixture.row) + } + remaining[fixture]-- + counts[axis-1]++ + if x86FloatCompareDeadPredicate(fixture.predicate) { + dead[axis-1]++ + } + } + } + for fixture, count := range remaining { + if count != 0 { + t.Fatal("missing runtime case", opcode, fixture.row, count) + } + } + } + if counts != [3]int{64, 256, 512} || dead != [3]int{16, 48, 96} { + t.Fatalf("changed complete fixture denominators: IEEE=%v dead=%v", counts, dead) + } + var fp [3]int + for _, fixture := range x86FloatCompareFPOracles(t) { + fp[fixture.axis-1]++ + } + if fp != [3]int{16, 16, 16} { + t.Fatal("changed FP byte-field runtime denominator", fp) + } +} + +func TestX86FloatingCompareCompleteRuntimeOriginalGoInventory(t *testing.T) { + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + t.Run(opcode, func(t *testing.T) { + assembly, _ := x86FloatCompareOracleAssembly(x86FloatCompareOracleCases(opcode), false) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + }) + } + var dead []x86FloatCompareOracleCase + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + for _, fixture := range x86FloatCompareOracleCases(opcode) { + if x86FloatCompareDeadPredicate(fixture.predicate) { + dead = append(dead, fixture) + } + } + } + assembly, _ := x86FloatCompareOracleAssembly(dead, true) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + assembly, _ = x86FloatCompareFPAssembly(x86FloatCompareFPOracles(t)) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + t.Log("actual stock Go objects bind 832 independent IEEE, 160 dead-result and all 48 scalar FP-byte-field programs; object evidence does not imply ISA execution") +} + +func TestX86FloatingCompareOpcodePreflightCapabilities(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, executable := filepath.Join(dir, "capabilities.c"), filepath.Join(dir, "capabilities") + // Execute the exact preflight predicate against independent missing-feature + // cases, without substituting synthetic capabilities for an actual ISA run. + const checks = ` +#undef main +int main(void) { + if (!required_state(1,1,0,0,0,0,0) || required_state(1,0,1,1,1,1,0xe6)) return 1; + if (!required_state(2,1,1,1,0,0,6) || !required_state(3,1,1,1,1,1,0xe6)) return 2; + if (required_state(0,1,1,1,1,1,0xe6) || required_state(4,1,1,1,1,1,0xe6)) return 3; + for (unsigned axis=2;axis<=3;axis++) { + if (required_state(axis,0,1,1,1,1,0xe6) || required_state(axis,1,0,1,1,1,0xe6) || + required_state(axis,1,1,0,1,1,0xe6)) return 4; + if (required_state(axis,1,1,1,1,1,0xe6&~2) || + required_state(axis,1,1,1,1,1,0xe6&~4)) return 5; + } + if (required_state(3,1,1,1,0,1,0xe6) || required_state(3,1,1,1,1,0,0xe6)) return 6; + for (unsigned bit=5;bit<=7;bit++) { + if (required_state(3,1,1,1,1,1,0xe6&~(1u< +int main(void) { + __builtin_cpu_init(); + int avx=!!__builtin_cpu_supports("avx"); + int avx2=!!__builtin_cpu_supports("avx2"); + int avx512=!!__builtin_cpu_supports("avx512f"); + printf("AVX=%d AVX2=%d AVX512F=%d\n",avx,avx2,avx512); + return avx?0:1; +} +` + if err := os.WriteFile(source, []byte(preflight), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + t.Logf("actual guest CPUID/XCR0 preflight: %s", strings.TrimSpace(string(execution.run(t, executable)))) +} + +func TestX86FloatingCompareExecutionTransport(t *testing.T) { + for _, test := range []struct { + name string + execution x86FloatCompareExecution + want []string + }{ + {"native", x86FloatCompareExecution{}, []string{"/guest", "numeric", "with space"}}, + {"rosetta", x86FloatCompareExecution{prefix: []string{"arch", "-x86_64"}}, + []string{"arch", "-x86_64", "/guest", "numeric", "with space"}}, + {"pinned", x86FloatCompareExecution{prefix: []string{"qemu", "-cpu", "max"}, preserveArgv0: true}, + []string{"qemu", "-cpu", "max", "/guest", "/guest", "numeric", "with space"}}, + } { + t.Run(test.name, func(t *testing.T) { + arguments := []string{"numeric", "with space"} + before := append([]string(nil), test.execution.prefix...) + if actual := test.execution.command("/guest", arguments...); !reflect.DeepEqual(actual, test.want) { + t.Fatalf("guest arguments: got %q, want %q", actual, test.want) + } + if !reflect.DeepEqual(before, test.execution.prefix) || !reflect.DeepEqual(arguments, []string{"numeric", "with space"}) { + t.Fatal("execution argument construction mutated its inputs") + } + }) + } +} + +func x86FloatCompareVerifyArguments(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module comparearguments\n\ngo 1.20\n", + "main.go": "package main\nimport(\"fmt\";\"os\")\nfunc main(){fmt.Printf(\"%q\\n\",os.Args)}\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, gotoolchain.HostExecutable("arguments")) + command := exec.Command("go", "build", "-p=2", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH=amd64", "CGO_ENABLED=0", + "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local") + x86FloatCompareCaptureCommand(t, command) + for _, arguments := range [][]string{nil, {"numeric", "with space"}} { + want := fmt.Sprintf("%q\n", append([]string{executable}, arguments...)) + if actual := string(execution.run(t, executable, arguments...)); actual != want { + t.Fatalf("actual guest argument witness: got %q, want %q", actual, want) + } + } +} diff --git a/amd64_float_compare_vex_numerical_linux_test.go b/amd64_float_compare_vex_numerical_linux_test.go new file mode 100644 index 000000000..7fbf00d0b --- /dev/null +++ b/amd64_float_compare_vex_numerical_linux_test.go @@ -0,0 +1,11 @@ +//go:build linux && (amd64 || arm64) + +package plan9asm + +import "testing" + +// This mandatory axis executes the translated actual whole VEX compare against +// the independent original opcode. QEMU has AVX/AVX2 but not native AVX512. +func TestCrossLinuxRuntimeMatrixX86FloatingCompareVEXWholeIEEEAndMXCSR(t *testing.T) { + x86FloatCompareVEXWholeRuntime(t) +} diff --git a/amd64_float_compare_vex_numerical_test.go b/amd64_float_compare_vex_numerical_test.go new file mode 100644 index 000000000..8cfb3d4aa --- /dev/null +++ b/amd64_float_compare_vex_numerical_test.go @@ -0,0 +1,365 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// This matrix is independent of the production operand/predicate registries. +// All six native VEX widths have every predicate and both first-source roles; +// destination aliases rotate through both sources and an independent register. +func x86FloatCompareVEXOracleCases() []x86FloatCompareOracleCase { + var cases []x86FloatCompareOracleCase + for _, fixture := range x86FloatCompareOriginalVectors() { + bank := "X" + if fixture.width == 32 { + bank = "Y" + } + for predicate := 0; predicate < 32; predicate++ { + for _, memory := range []bool{false, true} { + first := bank + "1" + if memory { + first = "(CX)" + } + cases = append(cases, x86FloatCompareOracleCase{ + row: fmt.Sprintf("%s $%d,%s,%s0,%s%d", fixture.opcode, predicate, first, bank, bank, predicate%3), + bits: fixture.bits, width: fixture.width, predicate: predicate, + scalar: fixture.scalar, memory: memory, destination: predicate % 3, + }) + } + } + } + return cases +} + +func x86FloatCompareVEXOracleAssembly(cases []x86FloatCompareOracleCase) (string, map[string]FuncSig) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, fixture := range cases { + name := fmt.Sprintf("vex_compare%d", index) + fmt.Fprintf(&source, `TEXT %s(SB),4,$0-24 +MOVQ out+0(FP),AX +MOVQ a+8(FP),BX +MOVQ b+16(FP),CX +VMOVUPS (BX),Y0 +VMOVUPS (BX),Y2 +`, name) + if !fixture.memory { + source.WriteString("VMOVUPS (CX),Y1\n") + } + source.WriteString("STC\n" + fixture.row + "\n") + // Z is a translated shadow-view observation, not a native AVX512 + // instruction in the original-opcode reference. The expected high bytes + // are independently zero for VEX X/Y writes. No EVEX compare is run. + fmt.Fprintf(&source, "VMOVUPS Z%d,8(AX)\nSETCS 72(AX)\nRET\n", fixture.destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + } + return source.String(), sigs +} + +// Emit the actual requested whole VCMP opcode and full native memory tuple. +// Its MXCSR is an independent original-instruction reference, not a union of +// per-lane quiet/signaling representatives. IEEE truth remains integer-only. +func x86FloatCompareVEXWholeReferenceC() string { + var source strings.Builder + source.WriteString(x86FloatCompareIndependentC) + for index, fixture := range x86FloatCompareOriginalVectors() { + for variant := 0; variant < 2; variant++ { + fmt.Fprintf(&source, ` +__attribute__((target("avx"))) static void reference%d(uint8_t *out,const uint8_t *a,const uint8_t *b,unsigned predicate,unsigned csr,unsigned *after){ + typedef uint8_t vector __attribute__((vector_size(%d))); + vector left,right,result; + memcpy(&left,a,sizeof(left));memcpy(&right,b,sizeof(right)); + unsigned saved;uint8_t carry; + __asm__ volatile("stmxcsr %%0":"=m"(saved)); + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + switch(predicate){ +`, index*2+variant, fixture.width) + operand := "\"x\"(right)" + if variant == 1 { + bytes := fixture.width + if fixture.scalar { + bytes = fixture.bits / 8 + } + operand = fmt.Sprintf("\"m\"(*(const uint8_t(*)[%d])b)", bytes) + } + for predicate := 0; predicate < 32; predicate++ { + fmt.Fprintf(&source, " case %d: __asm__ volatile(\"stc; %s $%d,%%3,%%2,%%0; setc %%1\":\"=x\"(result),\"=qm\"(carry):\"x\"(left),%s:\"cc\");break;\n", + predicate, strings.ToLower(fixture.opcode), predicate, operand) + } + source.WriteString(` } + __asm__ volatile("stmxcsr %0":"=m"(*after)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + memset(out,0,64);memcpy(out,&result,sizeof(result));out[64]=carry; +} +`) + } + } + source.WriteString("typedef void(*reference_fn)(uint8_t*,const uint8_t*,const uint8_t*,unsigned,unsigned,unsigned*);\nstatic reference_fn references[]={") + for index := 0; index < 12; index++ { + fmt.Fprintf(&source, "reference%d,", index) + } + source.WriteString("};\n") + return source.String() +} + +func x86FloatCompareVEXWholeMain(cases []x86FloatCompareOracleCase) string { + var declarations, descriptors strings.Builder + for index, fixture := range cases { + fmt.Fprintf(&declarations, "extern void vex_compare%d(void*,const void*,const void*);\n", index) + fmt.Fprintf(&descriptors, "{vex_compare%d,%d,%d,%d,%d,%d},\n", index, index/64*2+x86FloatAddSubBoolInt(fixture.memory), + fixture.bits, fixture.width, fixture.predicate, x86FloatAddSubBoolInt(fixture.scalar)) + } + return x86FloatCompareVEXWholeReferenceC() + declarations.String() + ` +typedef void(*translated_fn)(void*,const void*,const void*); +struct comparison{translated_fn call;unsigned reference,bits,width,predicate,scalar;}; +static const struct comparison comparisons[]={ +` + descriptors.String() + ` +}; +int main(void){ + unsigned saved;__asm__ volatile("stmxcsr %0":"=m"(saved)); + uint64_t samples=0; + for(unsigned index=0;indexbits/8;const uint64_t *edges=bytes==4?edges32:edges64; + for(unsigned mode=0;mode<2;mode++)for(unsigned i=0;i<18;i++)for(unsigned j=0;j<18;j++)for(unsigned control=0;control<6;control++){ + uint8_t aStorage[192],bStorage[192],out[81],expected[65],truth[64]; + uint8_t *a=aStorage+65,*b=bStorage+65; + memset(aStorage,0xa7,sizeof(aStorage));memset(bStorage,0xb9,sizeof(bStorage));memset(out,0xc3,sizeof(out)); + for(unsigned lane=0;lane<64/bytes;lane++){ + uint64_t av=edges[(i+(mode?lane:0))%18],bv=edges[(j+(mode?lane*5:0))%18]; + memcpy(a+lane*bytes,&av,bytes);memcpy(b+lane*bytes,&bv,bytes); + } + uint8_t beforeA[192],beforeB[192];memcpy(beforeA,aStorage,192);memcpy(beforeB,bStorage,192); + unsigned csr=controls[control],expectedCSR,after; + references[c->reference](expected,a,b,c->predicate,csr,&expectedCSR); + memset(truth,0,64);if(c->scalar)memcpy(truth,a,16); + unsigned compared=c->scalar?bytes:c->width; + for(unsigned lane=0;lanebits,read_bits(a+lane,bytes),read_bits(b+lane,bytes),c->predicate,csr); + memset(truth+lane,value?255:0,bytes); + } + if(memcmp(truth,expected,64)||expected[64]!=1){ + fprintf(stderr,"original VEX truth/CF index=%u pred=%u mode=%u i=%u j=%u csr=%x\n",index,c->predicate,mode,i,j,csr);return 10; + } + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + c->call(out,a,b); + __asm__ volatile("stmxcsr %0":"=m"(after)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + if(after!=expectedCSR||memcmp(aStorage,beforeA,192)||memcmp(bStorage,beforeB,192)){ + fprintf(stderr,"translated VEX MXCSR/input index=%u pred=%u mode=%u i=%u j=%u csr=%x got=%x want=%x\n",index,c->predicate,mode,i,j,csr,after,expectedCSR);return 11; + } + for(unsigned byte=0;byte<81;byte++){ + uint8_t want=byte>=8&&byte<=72?expected[byte-8]:0xc3; + if(out[byte]!=want){ + fprintf(stderr,"translated VEX alias/canary index=%u pred=%u mode=%u i=%u j=%u byte=%u got=%x want=%x\n",index,c->predicate,mode,i,j,byte,out[byte],want);return 12; + } + } + samples++; + } + } + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + printf("384 VEX forms, %llu full IEEE/MXCSR/CF/64-alias/canary samples\n",(unsigned long long)samples); + return samples==UINT64_C(1492992)?0:13; +} +` +} + +func TestX86FloatingCompareVEXWholeOracleInventory(t *testing.T) { + cases := x86FloatCompareVEXOracleCases() + if len(cases) != 384 { + t.Fatalf("VEX forms=%d want384", len(cases)) + } + source, _ := x86FloatCompareVEXOracleAssembly(cases) + if strings.Contains(source, "KMOV") || strings.Contains(source, ".SAE") || strings.Contains(source, ".BCST") { + t.Fatal("non-VEX compare prerequisites entered native VEX oracle") + } + goAssemblerOracleOutput(t, "amd64", source, "", true) + if strings.Contains(x86FloatCompareVEXWholeMain(cases), "original_effect(") { + // The shared relation source declares this older diagnostic helper but + // the whole-vector driver must not call it or substitute sample values. + main := strings.Split(x86FloatCompareVEXWholeMain(cases), "int main(void){")[1] + if strings.Contains(main, "original_effect(") || strings.Contains(main, "is_denormal(") { + t.Fatal("whole native effect reference was replaced by a class/lane policy") + } + } +} + +func TestX86FloatingCompareVEXWholeEffectsIR(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + cases := x86FloatCompareVEXOracleCases() + var physicalSource strings.Builder + var physicalBytes strings.Builder + for index, fixture := range cases { + name := fmt.Sprintf("physical_vex%d", index) + fmt.Fprintf(&physicalSource, "TEXT %s(SB),4,$0-0\n", name) + for _, b := range x86FloatCompareVEXOriginalBytes(t, fixture.row) { + fmt.Fprintf(&physicalSource, "BYTE $%d\n", b) + fmt.Fprintf(&physicalBytes, "0x%02x ", b) + } + physicalBytes.WriteString("\n") + // This is only a physical mode32 format harness. Unknown incoming + // pointers cannot borrow a native return contract from that byte marker. + physicalSource.WriteString("UNDEF\n") + } + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM22 llvm-mc required") + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=i386-unknown-linux-gnu", "--mattr=+avx") + command.Stdin = strings.NewReader(physicalBytes.String()) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("original Go mode32 byte validation: %v\n%s", err, output) + } + physical := regexp.MustCompile(`(?m)^\s+vcmp[a-z0-9_]*(ps|pd|ss|sd)\s+`).FindAll(output, -1) + if len(physical) != 384 || strings.Contains(string(output), "warning:") { + t.Fatalf("original AMD64 Go bytes lack complete physical mode32 LLVM-MC identity:\n%s", output) + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-macosx"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + source, sigs := x86FloatCompareVEXOracleAssembly(cases) + if target.arch == "386" { + // The original mode32 byte fixture has neither initialized + // vectors nor native pointer inputs. Keep its original Go + // objects and public Context controls, then check only its + // complete physical syntax through the private emitter. + ir := x86FloatingCompareUnboundFormatObject(t, physicalSource.String(), target.arch, target.triple) + if strings.Contains(ir, " = fcmp ") || strings.Count(ir, "asm sideeffect \"vcmp") != len(cases) || + strings.Count(ir, "asm sideeffect \"ud2\"") != len(cases) { + t.Fatal("physical mode32 VEX formats lost original comparison side effects") + } + compileLLVMToObject(t, llc, target.triple, "vex-whole-format.ll", "vex-whole-format.o", ir) + return + } + goAssemblerOracleOutput(t, target.arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir := func() string { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs}) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + return module.String() + }() + if strings.Contains(ir, " = fcmp ") || strings.Count(ir, "asm sideeffect") < 384 { + t.Fatal("full native VEX predicate/width side effects were replaced by relation-only fcmp") + } + compileLLVMToObject(t, llc, target.triple, "vex-whole.ll", "vex-whole.o", ir) + }) + } +} + +func x86FloatCompareVEXOriginalBytes(t *testing.T, row string) []byte { + t.Helper() + listing, err := x86ScalarLiteralGoListing(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatalf("original Go VEX encoding: %v\n%s", err, listing) + } + length := 0 + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) >= 4 && fields[len(fields)-1] == "RET" { + length, err = strconv.Atoi(fields[1]) + if err != nil || length < 1 || length > 15 { + t.Fatalf("original VEX native width: %v\n%s", err, listing) + } + } + } + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if length == 0 || len(fields) <= length || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + code := make([]byte, length) + for index := range code { + value, err := strconv.ParseUint(fields[index+1], 16, 8) + if err != nil { + t.Fatal(err) + } + code[index] = byte(value) + } + return code + } + t.Fatalf("original VEX machine bytes unavailable:\n%s", listing) + return nil +} + +func TestX86FloatingCompareVEXWholeOriginalCObject(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, object := filepath.Join(dir, "original.c"), filepath.Join(dir, "original.o") + contents := x86FloatCompareVEXWholeMain(x86FloatCompareVEXOracleCases()) + // These two headers belong to older fault fixtures, not this numerical + // reference. Keep the ordinary C object portable to Windows as well. + contents = strings.NewReplacer("#include \n", "", "#include \n", "").Replace(contents) + if err := os.WriteFile(source, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + arguments := append(append([]string(nil), execution.compiler...), "-O2", "-c", source, "-o", object) + command := exec.Command(arguments[0], arguments[1:]...) + x86FloatCompareCaptureCommand(t, command) +} + +func x86FloatCompareVEXWholeRuntime(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + execution.requireAVX(t) + x86FloatCompareVerifyArguments(t, execution) + cases := x86FloatCompareVEXOracleCases() + source, sigs := x86FloatCompareVEXOracleAssembly(cases) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir := func() string { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + return module.String() + }() + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "vex-whole-compare", ir, x86FloatCompareVEXWholeMain(cases), level) + const expected = "384 VEX forms, 1492992 full IEEE/MXCSR/CF/64-alias/canary samples" + if strings.TrimSpace(string(output)) != expected { + t.Fatalf("whole native VEX oracle output: %q", output) + } + t.Logf("%s: %s transport=%v", level, expected, execution.prefix) + }) + } + t.Log("384 register/memory VEX forms ×3888 full Cartesian/mixed-lane samples per optimization level; no EVEX execution") +} diff --git a/amd64_float_min_max_forms.go b/amd64_float_min_max_forms.go new file mode 100644 index 000000000..26b07709e --- /dev/null +++ b/amd64_float_min_max_forms.go @@ -0,0 +1,22 @@ +package plan9asm + +type x86FloatMinMaxForm = x86FloatArithmeticForm + +func x86FloatMinMaxNativeEVEX(ins Instr, f x86FloatMinMaxForm) bool { + if f.legacy { + return false + } + if ins.x86FloatMinMaxEVEX || f.byteWidth == 64 || f.mask >= 0 || f.broadcast || f.sae { + return true + } + for _, operand := range ins.Args { + if index, valid := amd64VectorRegisterIndex(operand.Reg, f.byteWidth); operand.Kind == OpReg && valid && index >= 16 { + return true + } + } + return false +} + +func x86FloatMinMaxProperties(ins Instr, arch string) (x86FloatMinMaxForm, bool, error) { + return x86FloatArithmeticProperties(ins, arch, x86FloatMinimumMaximum) +} diff --git a/amd64_float_min_max_forms_test.go b/amd64_float_min_max_forms_test.go new file mode 100644 index 000000000..dbcb1a1d9 --- /dev/null +++ b/amd64_float_min_max_forms_test.go @@ -0,0 +1,213 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func x86FloatingMinMaxFormats(arch string) []string { + var rows []string + for _, member := range x86FloatingMinMaxCases { + if member.legacy { + rows = append(rows, member.op+" X0,X1", member.op+" (AX),X1") + continue + } + banks := []string{"X", "Y", "Z"} + if member.scalar { + banks = banks[:1] + } + for _, bank := range banks { + for _, mode := range []string{"", "merge", "zero"} { + if arch == "386" && mode != "" { + continue // Go's named four-operand frontend cap, not a physical ISA claim. + } + mask, zero := "", "" + if mode != "" { + mask = "K1," + } + if mode == "zero" { + zero = ".Z" + } + registers := bank + "0," + bank + "1," + mask + bank + "2" + memory := "(AX)," + bank + "1," + mask + bank + "2" + rows = append(rows, member.op+zero+" "+registers, member.op+zero+" "+memory) + if !member.scalar { + rows = append(rows, member.op+".BCST"+zero+" "+memory) + } + if member.scalar || bank == "Z" { + rows = append(rows, member.op+".SAE"+zero+" "+registers) + } + } + } + } + return rows +} + +func TestX86FloatingMinMaxIndependentCompleteEncoderRows(t *testing.T) { + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(legacy), "var yxm = []ytab{\n\t{Zm_r_xm, 1, argList{Yxm, Yxr}},\n}") { + t.Fatal("complete original MIN/MAX yxm table changed") + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + if x86FloatArithmeticFamilyCount(x86FloatMinimumMaximum) != len(x86FloatingMinMaxCases) { + t.Fatal("incomplete typed MIN/MAX family") + } + for _, member := range x86FloatingMinMaxCases { + prefix, w, opcode := "", 0, "0x5D" + if member.bits == 64 { + prefix, w = "66", 1 + } + if member.scalar { + prefix = map[int]string{32: "F3", 64: "F2"}[member.bits] + } + if strings.Contains(member.op, "MAX") { + opcode = "0x5F" + } + if member.legacy { + p := map[string]string{"": "Pm", "66": "Pe", "F2": "Pf2", "F3": "Pf3"}[prefix] + row := fmt.Sprintf("{A%s, yxm, %s, opBytes{%s}}", member.op, p, strings.ToLower(opcode)) + if !strings.Contains(string(legacy), row) { + t.Fatal("original legacy MIN/MAX prefix or opcode changed", row) + } + continue + } + table := "_yvaddpd" + vexWidths, evexWidths := []int{128, 256}, []int{512, 128, 256} + if member.scalar { + table, vexWidths, evexWidths = "_yvaddsd", []int{128}, []int{128} + } + var expected []goasmrules.X86AVXRule + var encoding []string + offset := 0 + add := func(zcase string, advance int, classes string) { + expected = append(expected, goasmrules.X86AVXRule{Opcode: member.op, Ytab: table, Prefix: "Pavx", + Classes: strings.Split(classes, ","), RowIndex: len(expected), ZCase: zcase, + ZOffset: advance, EncodingOffset: offset}) + offset += advance + } + pp := func(generation string) string { + if prefix == "" { + return "" + } + return " | " + generation + prefix + } + for _, width := range vexWidths { + bank := map[int]string{128: "x", 256: "y"}[width] + add("Zvex_rm_v_r", 2, "Y"+bank+"m,Y"+bank+"r,Y"+bank+"r") + encoding = append(encoding, fmt.Sprintf("avxEscape | vex%d%s | vex0F | vexW0", width, pp("vex")), opcode) + } + for _, width := range evexWidths { + bank := map[int]string{128: "x", 256: "y", 512: "z"}[width] + memory, register := "Y"+bank+"mEvex", "Y"+bank+"rEvex" + if width == 512 { + memory, register = "Yzm", "Yzr" + } + add("Zevex_rm_v_r", 0, memory+","+register+","+register) + add("Zevex_rm_v_k_r", 3, memory+","+register+",Yknot0,"+register) + attributes := fmt.Sprintf("evexN%d | evexBcstN%d", width/8, member.bits/8) + if member.scalar { + attributes = fmt.Sprintf("evexN%d", member.bits/8) + } + if member.scalar || width == 512 { + attributes += " | evexSaeEnabled" + } + attributes += " | evexZeroingEnabled" + encoding = append(encoding, fmt.Sprintf("avxEscape | evex%d%s | evex0F | evexW%d", width, pp("evex"), w), attributes, opcode) + } + for index := range expected { + expected[index].OpBytes = encoding + } + var actual []goasmrules.X86AVXRule + for _, rule := range rules { + if rule.Opcode == member.op { + actual = append(actual, rule) + } + } + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("%s complete original encoding changed:\ngot %+v\nwant %+v", member.op, actual, expected) + } + } +} + +func TestX86FloatingMinMaxCompleteNamedGrammar(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + rows := x86FloatingMinMaxFormats(arch) + want := 68 + if arch == "amd64" { + want = 172 + } + if len(rows) != want { + t.Fatal("independent MIN/MAX format denominator changed", arch, len(rows)) + } + requireX86GoAssemblerResult(t, arch, "TEXT Forms(SB),4,$0-0\n"+strings.Join(rows, "\n")+"\nRET\n", true) + for _, row := range rows { + ins := x86ScalarLaneInstruction(t, row) + f, known, err := x86FloatMinMaxProperties(ins, arch) + if !known || err != nil || f.byteWidth == 0 || f.rounding != "" { + t.Fatal("complete typed MIN/MAX grammar lost an original row", arch, row, err) + } + if f.memory >= 0 { + logical := ins + logical.Args = append([]Operand(nil), ins.Args...) + logical.Args[f.memory] = Operand{Kind: OpFP} + width, write, valid := X86FPAccessBytes(logical, f.memory, arch) + if !valid || write || width != int64(f.memoryBytes) { + t.Fatal("SDK/native-width observer diverged from the complete typed tuple", arch, row, width, valid) + } + } + } + } +} + +func TestX86FloatingMinMaxMalformedAndDispatchBeforeIR(t *testing.T) { + for _, row := range []string{ + "MINPS X0,Y1", "MAXSD X0,AX", "MINSS X0,X16", "MAXPD.SAE X0,X1", + "VMINPS X0,X1", "VMAXPD X0,Y1,Y2", "VMINSS Y0,Y1,Y2", "VMAXSD X0,X1,K0,X2", + "VMINPS.Z X0,X1,X2", "VMAXPS.BCST X0,X1,X2", "VMINSS.BCST (AX),X1,X2", + "VMAXPS.RN_SAE Z0,Z1,Z2", "VMINPS.SAE X0,X1,X2", "VMAXSD.SAE (AX),X1,X2", + "VMINPS. X0,X1,X2", "VMAXPS.Z.BCST (AX),Z1,K1,Z2", "VMINSD $1.5,X1,X2", + "VMINPS $data(SB),X1,X2", "VMAXSS (AX)(SP*2),X1,X2", "VMINPD 2147483648(AX),X1,X2", + } { + t.Run(row, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT Value(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + return + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, terminated, err := ctx.lowerFloatMinMax(ins.Op, ins) + if !known || terminated || err == nil || output.String() != before { + t.Fatal("malformed MIN/MAX emitted IR", known, terminated, err) + } + }) + } + for _, member := range x86FloatingMinMaxCases { + row := member.op + " X0,X1,X2" + if member.legacy { + row = member.op + " X0,X1" + } + ins := x86ScalarLaneInstruction(t, row) + file := mustParseX86RawReturn(t, "TEXT Value(SB),4,$0-0\n"+row+"\nRET\n") + ins.Op = "UNKNOWN" + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, terminated, err := ctx.lowerFloatMinMax(Op(member.op), ins) + if !known || terminated || err == nil || output.String() != before { + t.Fatal("MIN/MAX dispatch changed original identity", member.op, err) + } + } +} diff --git a/amd64_float_mul_div_forms.go b/amd64_float_mul_div_forms.go new file mode 100644 index 000000000..5390a7ddc --- /dev/null +++ b/amd64_float_mul_div_forms.go @@ -0,0 +1,7 @@ +package plan9asm + +type x86FloatMulDivForm = x86FloatArithmeticForm + +func x86FloatMulDivProperties(ins Instr, arch string) (x86FloatMulDivForm, bool, error) { + return x86FloatArithmeticProperties(ins, arch, x86FloatMultiplyDivide) +} diff --git a/amd64_float_mul_div_forms_test.go b/amd64_float_mul_div_forms_test.go new file mode 100644 index 000000000..388791be1 --- /dev/null +++ b/amd64_float_mul_div_forms_test.go @@ -0,0 +1,243 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86FloatMulDivOpcodes() []string { + return []string{ + "MULPS", "MULPD", "MULSS", "MULSD", + "DIVPS", "DIVPD", "DIVSS", "DIVSD", + "VMULPS", "VMULPD", "VMULSS", "VMULSD", + "VDIVPS", "VDIVPD", "VDIVSS", "VDIVSD", + } +} + +func x86FloatMulDivFormats(arch string) []string { + // All eight opcode attributes are independently checked below. Go uses + // the same yxm/_yvaddpd/_yvaddsd operand grammars as ADD/SUB, not merely + // the same vector widths. Reuse the independent shape-axis inventory. + rows := x86FloatAddSubFormats(arch) + for i, row := range rows { + rows[i] = strings.NewReplacer("ADD", "MUL", "SUB", "DIV").Replace(row) + } + return rows +} + +func TestX86FloatMulDivEveryGoOpcodeAttribute(t *testing.T) { + if x86FloatArithmeticFamilyCount(x86FloatMultiplyDivide) != 16 { + t.Fatal("incomplete floating MUL/DIV family") + } + for _, opcode := range x86FloatMulDivOpcodes() { + bits := 32 + if strings.HasSuffix(opcode, "PD") || strings.HasSuffix(opcode, "SD") { + bits = 64 + } + mode := amd64BinaryFloatingMul + if strings.Contains(opcode, "DIV") { + mode = amd64BinaryFloatingDiv + } + want := x86FloatArithmeticSpec{bits, !strings.HasPrefix(opcode, "V"), + strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD"), mode} + if got, found := x86FloatArithmeticSpecs[Op(opcode)]; !found || got != want { + t.Fatalf("%s typed specification differs from independent mnemonic: %+v/%+v", opcode, got, want) + } + } + if !goToolchainAtLeast(runtime.Version(), 1, 27) { + return // The current-toolchain gate owns encoder-table completeness. + } + dir := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + for _, filename := range []string{"asm6.go", "avx_optabs.go"} { + data, err := os.ReadFile(filepath.Join(dir, filename)) + if err != nil { + t.Fatal(err) + } + for _, opcode := range x86FloatMulDivOpcodes() { + legacy := !strings.HasPrefix(opcode, "V") + if (filename == "asm6.go") != legacy { + continue + } + if legacy { + if !strings.Contains(string(data), "{A"+opcode+", yxm,") { + t.Fatalf("Go legacy grammar changed: %s", opcode) + } + continue + } + scalar := strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") + bits := 32 + if strings.HasSuffix(opcode, "PD") || strings.HasSuffix(opcode, "SD") { + bits = 64 + } + prefix := "{as: A" + opcode + ", ytab: " + start := strings.Index(string(data), prefix) + if start < 0 { + t.Fatalf("Go omits %s", opcode) + } + row := strings.SplitN(string(data)[start:], "\n\t}},", 2)[0] + table := "_yvaddpd" + if scalar { + table = "_yvaddsd" + } + if !strings.HasPrefix(row, prefix+table+",") { + t.Fatalf("Go grammar changed: %s", row) + } + seen := 0 + for _, line := range strings.Split(row, "\n") { + if !strings.Contains(line, "avxEscape | evex") { + continue + } + seen++ + if !strings.Contains(line, "evexZeroingEnabled") || strings.Contains(line, "evexSaeEnabled") || + strings.Contains(line, "evexRoundingEnabled") != (scalar || strings.Contains(line, "evex512")) || + strings.Contains(line, fmt.Sprintf("evexBcstN%d", bits/8)) != !scalar { + t.Fatalf("%s opcode attribute changed: %s", opcode, line) + } + } + if scalar && seen != 1 || !scalar && seen != 3 { + t.Fatalf("%s incomplete EVEX row count %d", opcode, seen) + } + } + } +} + +func TestX86FloatMulDivCompleteGoGrammarAndFiveLLVMTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + var declarations strings.Builder + declarations.WriteString("package shapes\n") + sigs := map[string]FuncSig{} + for index, row := range x86FloatMulDivFormats(arch) { + name := fmt.Sprintf("muldiv%d", index) + entry := fmt.Sprintf("TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + file, err := Parse(ArchAMD64, entry) + if err != nil { + t.Fatal(err) + } + form, recognized, err := x86FloatMulDivProperties(file.Funcs[0].Instrs[1], arch) + if !recognized || err != nil || form.bits == 0 || form.byteWidth == 0 { + t.Fatalf("%s/%s: %v", arch, row, err) + } + source.WriteString(entry) + fmt.Fprintf(&declarations, "func %s()\n", name) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triples := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + options := Options{Goarch: arch, TargetTriple: triple, Sigs: sigs} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + func() { + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + goSource := strings.ReplaceAll(source.String(), "TEXT muldiv", "TEXT ·muldiv") + result, err := translateGoModuleInContext(context, x86FloatMulDivShapesPackage(t, declarations.String(), arch), + []byte(goSource), GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + for api, body := range []string{ir, module.String(), result.Module.String()} { + name := fmt.Sprintf("muldiv-formats-api%d", api) + compileLLVMToObject(t, llc, triple, name+".ll", name+".o", body) + } + }() + } + t.Logf("%s: %d independent Go MUL/DIV formats", arch, len(sigs)) + } +} + +func x86FloatMulDivShapesPackage(t *testing.T, source, arch string) GoPackage { + t.Helper() + set := token.NewFileSet() + file, err := parser.ParseFile(set, "shapes.go", source, 0) + if err != nil { + t.Fatal(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("shapes", set, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: pkg.Path(), Types: pkg, Syntax: []*ast.File{file}} +} + +func TestX86FloatMulDivRejectsMalformedBeforeEmission(t *testing.T) { + for _, row := range []string{ + "MULPS X0,Y1", "MULSD X0,AX", "DIVSS X0,X16", "DIVPD.SAE X0,X1", + "VMULPS X0,X1", "VMULPD X0,Y1,Y2", "VMULSS Y0,Y1,Y2", "VDIVSD X0,X1,K0,X2", + "VMULPS.Z X0,X1,X2", "VMULPS.BCST X0,X1,X2", "VMULSS.BCST (AX),X1,X2", + "VDIVPS.SAE Z0,Z1,Z2", "VMULPS.RD_SAE X0,X1,X2", "VDIVSD.RU_SAE (AX),X1,X2", + "VMULPS. X0,X1,X2", "VDIVPS.Z.BCST (AX),Z1,K1,Z2", + "VMULPS $data(SB),X1,X2", "VDIVSS (AX)(SP*2),X1,X2", "VMULPD 2147483648(AX),X1,X2", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + oldReservedZeroing := row == "VMULPS.Z X0,X1,X2" && !goToolchainAtLeast(runtime.Version(), 1, 21) + goAssemblerOracleOutput(t, "amd64", source, "", oldReservedZeroing) + if oldReservedZeroing { + listing := x86InsertExtractGoListing(t, "amd64", source) + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + if !bytes.Equal(code, []byte{0x62, 0xf1, 0x74, 0x88, 0x59, 0xd0, 0xc3}) { + t.Fatalf("old Go reserved EVEX.z/aaa=0 observation changed: %x", code) + } + if _, _, known, err := decodedX86EVEXBinaryFloatInstruction(code, 0, 64, "", false); !known || err == nil { + t.Fatal("old Go reserved zeroing bytes acquired valid raw semantics") + } + t.Log("old Go accepted reserved EVEX.z/aaa=0; both named and raw validators still reject it") + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + return // Parser rejection also occurs before IR emission. + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + var known, term bool + if strings.HasPrefix(row, "V") { + known, term, err = ctx.lowerBinaryFloating(ins.Op, ins) + } else { + known, term, err = ctx.lowerFP(ins.Op, ins) + } + if !known || term || err == nil || output.Len() != before { + t.Fatalf("invalid MUL/DIV emitted IR: %t/%t/%v", known, term, err) + } + }) + } +} diff --git a/amd64_float_mul_div_literal_runtime_test.go b/amd64_float_mul_div_literal_runtime_test.go new file mode 100644 index 000000000..5d7a6f535 --- /dev/null +++ b/amd64_float_mul_div_literal_runtime_test.go @@ -0,0 +1,256 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86MulDivLiteralNumericalSource() (declarations, assembly string) { + var goSource, machine strings.Builder + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + for _, value := range x86MulDivLiteralValues { + name := strings.ToLower(opcode) + value.name + fmt.Fprintf(&goSource, "func %s(input *[32]byte, output *[80]byte, csr uint32)\n", name) + fmt.Fprintf(&machine, `TEXT ·%s(SB),4,$8-20 +STMXCSR saved-4(SP) +LDMXCSR csr+16(FP) +MOVQ input+0(FP),AX +MOVQ output+8(FP),BX +VMOVUPS (AX),Y0 +MOVQ $0x7fffffffffffffff,DX +ADDQ $1,DX +STC +%s $%s,X0 +MOVUPS X0,8(BX) +VMOVUPS Y0,24(BX) +STMXCSR 56(BX) +SETCS 60(BX) +SETOS 61(BX) +LDMXCSR saved-4(SP) +RET +`, name, opcode, value.text) + } + } + return goSource.String(), machine.String() +} + +func x86MulDivLiteralOptimizedObject(t *testing.T, llc, triple, ir string) { + t.Helper() + directory := t.TempDir() + source, object := filepath.Join(directory, "literal.ll"), filepath.Join(directory, "literal.o") + if err := os.WriteFile(source, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, llc, "-O2", "-mtriple="+triple, "-filetype=obj", source, "-o", object) + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("LLVM22 optimized literal object: %v\n%s", err, output) + } + info, err := os.Stat(object) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatalf("LLVM22 optimized literal object missing/nonregular/empty: %v", err) + } +} + +func x86MulDivLiteralCOracle() string { + var declarations, checks strings.Builder + index := 0 + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + bits := 32 + if strings.HasSuffix(opcode, "SD") { + bits = 64 + } + for _, value := range x86MulDivLiteralValues { + literal := value.bits64 + if bits == 32 { + literal = uint64(value.bits32) + } + name := strings.ToLower(opcode) + value.name + fmt.Fprintf(&declarations, "extern void %s(void*,void*,unsigned);\n", name) + fmt.Fprintf(&checks, " if (check_literal(%s,%q,%d,UINT64_C(0x%x),%d)) return %d;\n", name, name, bits, literal, x86FloatAddSubBoolInt(strings.HasPrefix(opcode, "DIV")), index+1) + index++ + } + } + return x86FloatMulDivSSEOracle + declarations.String() + ` +typedef void (*literal_fn)(void*,void*,unsigned); +static int check_literal(literal_fn fn,const char *name,unsigned bits,uint64_t literal,int divide) { + const uint64_t edge32[]={0,0x80000000,1,0x80000001,0x3f800000,0xbf800000, + 0x7f7fffff,0x7f800000,0xff800000,0x7fc12345,0x7f812345}; + const uint64_t edge64[]={0,UINT64_C(0x8000000000000000),1,UINT64_C(0x8000000000000001), + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x7fefffffffffffff), + UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000), + UINT64_C(0x7ff8123456789abc),UINT64_C(0x7ff0123456789abc)}; + const unsigned controls[]={0x1f80,0x1f85,0x1fc0,0x9f80,0x9fc0,0x3f80,0x5f80,0x7f80}; + for(unsigned sample=0;sample<11;sample++) { + unsigned char input[32],before[32],output[80],want[80]; + for(unsigned byte=0;byte<32;byte++) input[byte]=(unsigned char)(byte*37+19); + uint64_t left=bits==32?edge32[sample]:edge64[sample]; + memcpy(input,&left,bits/8); + memcpy(before,input,32); + for(unsigned mode=0;mode<8;mode++) { + unsigned ambient=0x3f85,after; + memset(output,0xa7,80); + memset(want,0xa7,80); + uint64_t result=sse_result(bits,left,literal,divide,controls[mode]); + unsigned expectedCSR=last_sse_csr; + memcpy(want+8,input,16); + memcpy(want+24,input,32); + memcpy(want+8,&result,bits/8); + memcpy(want+24,&result,bits/8); + memcpy(want+56,&expectedCSR,4); + want[60]=1; + want[61]=1; + __asm__ volatile("ldmxcsr %0"::"m"(ambient)); + fn(input,output,controls[mode]); + __asm__ volatile("stmxcsr %0":"=m"(after)); + if(memcmp(input,before,32)||memcmp(output,want,80)||after!=ambient) { + fprintf(stderr,"%s bits=%u sample=%u csr=%x after=%x\n",name,bits,sample,controls[mode],after); + return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + ` + unsigned csr=0x1f80; + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + return 0; +} +` +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivLiteralNumerical(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + declarations, assembly := x86MulDivLiteralNumericalSource() + x86ScalarLiteralGoListing(t, "amd64", assembly) + pkg := x86MulDivLiteralPackage(t, "amd64", declarations) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + resolve := func(sym string) string { return strings.TrimPrefix(sym, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var translation *GoModuleTranslation + translation, err = translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{GOARCH: "amd64", TargetTriple: triple, ResolveSym: resolve}) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + ir = translation.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "muldiv-literal-"+api, triple, ir, x86MulDivLiteralCOracle(), prefix) + }) + } + t.Logf("%d original literal forms × 11 independent SSE bit inputs × 8 full MXCSR controls × 3 translation APIs; whole X/Y bytes, CF/OF=1 and canaries", 4*len(x86MulDivLiteralValues)) +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivLiteralOriginalGo(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + declarations, assembly := x86MulDivLiteralNumericalSource() + var checks strings.Builder + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + bits := 32 + if strings.HasSuffix(opcode, "SD") { + bits = 64 + } + for _, value := range x86MulDivLiteralValues { + literal := value.bits64 + if bits == 32 { + literal = uint64(value.bits32) + } + fmt.Fprintf(&checks, " check(t,%s,%d,0x%x,%t)\n", strings.ToLower(opcode)+value.name, bits, literal, strings.HasPrefix(opcode, "DIV")) + } + } + goTest := `package literal +import("encoding/binary";"math";"testing") +type literalFn func(*[32]byte,*[80]byte,uint32) +func check(t *testing.T,fn literalFn,bits int,literal uint64,divide bool) { + t.Helper() + for _,negative:=range []bool{false,true} { + var input [32]byte + var output [80]byte + for i:=range input {input[i]=byte(i*37+19)} + for i:=range output {output[i]=0xa7} + var result uint64 + if bits==32 { + left:=float32(4) + if negative {left=-left} + binary.LittleEndian.PutUint32(input[:4],math.Float32bits(left)) + right:=math.Float32frombits(uint32(literal)) + if divide {left/=right}else{left*=right} + result=uint64(math.Float32bits(left)) + }else{ + left:=float64(4) + if negative {left=-left} + binary.LittleEndian.PutUint64(input[:8],math.Float64bits(left)) + right:=math.Float64frombits(literal) + if divide {left/=right}else{left*=right} + result=math.Float64bits(left) + } + before:=input + fn(&input,&output,0x1f80) + if input!=before {t.Fatal("input changed")} + for i,got:=range output { + if i>=56&&i<60 {continue} // CSR is checked independently by the C SSE oracle. + want:=byte(0xa7) + if i>=8&&i<24 {want=input[i-8]} + if i>=24&&i<56 {want=input[i-24]} + if i>=8&&i<8+bits/8 {want=byte(result>>(8*(i-8)))} + if i>=24&&i<24+bits/8 {want=byte(result>>(8*(i-24)))} + if i==60||i==61 {want=1} + if got!=want {t.Fatalf("bits=%d literal=%x byte=%d got=%x want=%x",bits,literal,i,got,want)} + } + } +} +func TestOriginalLiteralNumerical(t *testing.T) { +` + checks.String() + "}\n" + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module literal\n\ngo 1.20\n", "literal.go": "package literal\n" + declarations, + "literal_amd64.s": assembly, "literal_test.go": goTest, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go original literal numerical: %v\n%s", err, output) + } + t.Logf("actual Go original %d literal kernels × 2 initialized finite inputs; X/Y aliases, CF/OF=1 and whole output canaries (host execution may be Rosetta)", 4*len(x86MulDivLiteralValues)) +} diff --git a/amd64_float_mul_div_literal_test.go b/amd64_float_mul_div_literal_test.go new file mode 100644 index 000000000..75f91dbb3 --- /dev/null +++ b/amd64_float_mul_div_literal_test.go @@ -0,0 +1,506 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86MulDivLiteralValues = []x86ScalarLiteralValue{ + {name: "reduce", text: "0.0625", bits32: 0x3d800000, bits64: 0x3fb0000000000000}, + {name: "one", text: "1.0", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "plus", text: "+1.0", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "minus", text: "-1.0", bits32: 0xbf800000, bits64: 0xbff0000000000000}, + {name: "zero", text: "0.0"}, + {name: "negativeZero", text: "-0.0", bits32: 0x80000000, bits64: 0x8000000000000000}, + {name: "hex", text: "0x1.4p+0", bits32: 0x3fa00000, bits64: 0x3ff4000000000000}, + {name: "scientific", text: "6.25e-1", bits32: 0x3f200000, bits64: 0x3fe4000000000000}, + {name: "parentheses", text: "(1.25)", bits32: 0x3fa00000, bits64: 0x3ff4000000000000}, + {name: "signedParentheses", text: "(-2.5)", bits32: 0xc0200000, bits64: 0xc004000000000000}, + {name: "roundEven", text: "1.000000059604644775390625", bits32: 0x3f800000, bits64: 0x3ff0000010000000}, + {name: "underflow32", text: "1e-50", bits64: 0x358dee7a4ad4b81f}, + {name: "overflow32", text: "1e40", bits32: 0x7f800000, bits64: 0x483d6329f1c35ca5}, + {name: "underscore", text: "1_000.0", bits32: 0x447a0000, bits64: 0x408f400000000000}, + {name: "nestedSign", text: "+(-(+1.25))", bits32: 0xbfa00000, bits64: 0xbff4000000000000}, + {name: "leadingDot", text: ".5", bits32: 0x3f000000, bits64: 0x3fe0000000000000}, + {name: "trailingDot", text: "1.", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "exponent", text: "1E+2", bits32: 0x42c80000, bits64: 0x4059000000000000}, + {name: "hexExponent", text: "0x1p-4", bits32: 0x3d800000, bits64: 0x3fb0000000000000}, + {name: "doubleNegativeZero", text: "--0.0"}, + {name: "unaryNegativeZero", text: "-(0.0)", bits32: 0x80000000, bits64: 0x8000000000000000}, + {name: "negativeUnderflow32", text: "-1e-50", bits32: 0x80000000, bits64: 0xb58dee7a4ad4b81f}, + {name: "negativeOverflow32", text: "-1e40", bits32: 0xff800000, bits64: 0xc83d6329f1c35ca5}, +} + +func TestX86FloatMulDivLiteralMathArchExp(t *testing.T) { + x86ScalarLiteralActualGoRewriteSet(t) + path := filepath.Join(runtime.GOROOT(), "src/math/exp_amd64.s") + original, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + before := sha256.Sum256(original) + row := "MULSD $0.0625, X0" + if bytes.Count(original, []byte(row)) != 2 { + t.Fatal("original math.archExp no longer contains both literal reductions") + } + // The source row is original. The surrounding first-party function has + // an explicit initialized scalar parameter/result, not guessed math ABI. + source := "TEXT ·Reduce(SB),4,$0-16\nMOVSD input+0(FP),X0\n" + row + "\nMOVSD X0,ret+8(FP)\nRET\n" + x86ScalarLiteralGoListing(t, "amd64", source) + pkg := x86MulDivLiteralPackage(t, "amd64", "func Reduce(input float64) float64\n") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "literal.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}); err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(path) + if err != nil || sha256.Sum256(after) != before { + t.Fatalf("actual original math source changed: %v", err) + } + t.Logf("original math.archExp source SHA256=%x; both reduction rows retain initialized scalar semantics", before) +} + +func x86MulDivLiteralPackage(t *testing.T, arch, declarations string) GoPackage { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "literal.go", "package literal\n"+declarations, 0) + if err != nil { + t.Fatal(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("literal", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: "literal", Types: pkg, Syntax: []*ast.File{file}} +} + +func TestX86FloatMulDivLiteralActualGoObjects(t *testing.T) { + // This shared oracle authenticates the invoked SDK/assembler and hashes + // obj6/asm6/AVX tables before and after these original-source objects. + x86ScalarLiteralActualGoRewriteSet(t) + for _, name := range []string{"obj6.go", "asm6.go"} { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86", name) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if name == "obj6.go" { + if err := x86MulDivLiteralRewriteSource(data); err != nil { + t.Fatal(err) + } + } else { + for _, op := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + if !bytes.Contains(data, []byte("{A"+op+", yxm,")) { + t.Fatalf("actual Go no longer uses yxm for %s", op) + } + } + } + } + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + for _, op := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + for _, value := range x86MulDivLiteralValues { + for _, register := range []int{0, 7, 8, 15} { + row := fmt.Sprintf("%s $%s,X%d", op, value.text, register) + entry := fmt.Sprintf("TEXT literal%d(SB),4,$0-0\n%s\nRET\n", source.Len(), row) + if arch == "386" && register >= 8 { + x86ScalarLiteralExpectedGoListing(t, arch, entry, false) + continue + } + source.WriteString(entry) + } + } + } + listing, err := x86ScalarLiteralGoListing(t, arch, source.String()) + if err != nil { + t.Fatal(err) + } + for _, value := range x86MulDivLiteralValues { + for _, pool := range []struct { + name string + bits uint64 + width int + }{ + {fmt.Sprintf("$f32.%08x", value.bits32), uint64(value.bits32), 4}, + {fmt.Sprintf("$f64.%016x", value.bits64), value.bits64, 8}, + } { + if !strings.Contains(listing, pool.name) { + t.Fatalf("actual Go pool rounding/sign missing %s", pool.name) + } + start := strings.Index(listing, pool.name+" SRODATA") + if start < 0 { + t.Fatalf("actual Go pool has no original DATA payload: %s", pool.name) + } + var payload []byte + for _, line := range strings.Split(listing[start:], "\n")[1:] { + data := x86InsertExtractListedBytes(line) + if len(data) == 0 { + break + } + payload = append(payload, data...) + } + var want [8]byte + binary.LittleEndian.PutUint64(want[:], pool.bits) + if !bytes.Equal(payload, want[:pool.width]) { + t.Fatalf("actual original pool %s bytes=%x want=%x", pool.name, payload, want[:pool.width]) + } + } + } + if strings.Contains(listing, "XORPS") { + t.Fatal("arithmetic zero literal incorrectly acquired MOV zero rewrite") + } + } +} + +func TestX86FloatMulDivLiteralCallerSourceContracts(t *testing.T) { + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + move, result := "MOVSS", "float32" + if strings.HasSuffix(opcode, "SD") { + move, result = "MOVSD", "float64" + } + body := fmt.Sprintf("%s (DI),X15\n%s $0.0625,X15\n%s X15,(DX)", move, opcode, move) + for _, target := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + t.Run(opcode+"/"+target, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, strings.ReplaceAll(x86CallerFixtureSource, "float32", result), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, target); err != nil { + t.Fatal(err) + } + }) + } + for name, unsafe := range map[string]string{ + "undefined low bytes": fmt.Sprintf("%s $1.0,X0\n%s X0,(DX)", opcode, move), + "pointer input": fmt.Sprintf("MOVQ DI,X0\n%s $1.0,X0\n%s X0,(DX)", opcode, move), + "private stack bits": fmt.Sprintf("MOVQ SP,X0\n%s $1.0,X0\n%s X0,(DX)", opcode, move), + "read-only input store": strings.Replace(body, "(DX)", "(DI)", 1), + "one-byte result overflow": strings.Replace(body, "(DX)", "1(DX)", 1), + "unproved source memory": strings.Replace(body, "(DI)", "(BX)", 1), + "FP pointer contents": fmt.Sprintf("%s a+0(FP),X0\n%s $1.0,X0\n%s X0,(DX)", move, opcode, move), + "join lacks input": fmt.Sprintf("CMPQ AX,$0\nJE joined\n%s (DI),X15\njoined:\n%s $1.0,X15\n%s X15,(DX)", move, opcode, move), + } { + t.Run(opcode+"/"+name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", unsafe, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, strings.ReplaceAll(x86CallerFixtureSource, "float32", result), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("literal invented source/extent/escape evidence: %v", err) + } + }) + } + } +} + +// Independently bind the actual encoder's complete scalar pool-rewrite body, +// not just the presence of an opcode name somewhere in the source file. +func x86MulDivLiteralRewriteSource(source []byte) error { + file, err := parser.ParseFile(token.NewFileSet(), "obj6.go", source, 0) + if err != nil { + return err + } + seen := make(map[string]int) + var progedit *ast.FuncDecl + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == "progedit" { + if progedit != nil || function.Body == nil { + return fmt.Errorf("missing or duplicate progedit body") + } + progedit = function + } + } + if progedit == nil { + return fmt.Errorf("actual Go progedit missing") + } + allowed := make(map[*ast.CaseClause]bool) + for _, statement := range progedit.Body.List { + switchStatement, ok := statement.(*ast.SwitchStmt) + if !ok || switchStatement.Init != nil { + continue + } + selector, ok := switchStatement.Tag.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "As" { + continue + } + base, ok := selector.X.(*ast.Ident) + if !ok || base.Name != "p" { + continue + } + for _, entry := range switchStatement.Body.List { + if clause, ok := entry.(*ast.CaseClause); ok { + allowed[clause] = true + } + } + } + var failure error + ast.Inspect(progedit.Body, func(node ast.Node) bool { + clause, ok := node.(*ast.CaseClause) + if !ok { + return true + } + for _, expression := range clause.List { + name, ok := expression.(*ast.Ident) + if !ok || name.Name != "AMULSS" && name.Name != "AMULSD" && name.Name != "ADIVSS" && name.Name != "ADIVSD" { + continue + } + seen[name.Name]++ + if !allowed[clause] { + failure = fmt.Errorf("nested or conditional literal rewrite %s", name.Name) + } + bits := 64 + if strings.HasSuffix(name.Name, "SS") { + bits = 32 + } + expected := fmt.Sprintf(`package p +func f() { +if p.From.Type == obj.TYPE_FCONST { + f%d := %s + p.From.Type = obj.TYPE_MEM + p.From.Name = obj.NAME_EXTERN + p.From.Sym = ctxt.Float%dSym(f%d) + p.From.Offset = 0 +} +}`, bits, map[int]string{32: "float32(p.From.Val.(float64))", 64: "p.From.Val.(float64)"}[bits], bits, bits) + fixture, err := parser.ParseFile(token.NewFileSet(), "expected.go", expected, 0) + if err != nil { + failure = err + continue + } + body, err := x86ScalarLiteralBodySyntax(clause.Body) + want, wantErr := x86ScalarLiteralBodySyntax(fixture.Decls[0].(*ast.FuncDecl).Body.List) + if err != nil || wantErr != nil || body != want { + failure = fmt.Errorf("unrecognized actual Go %s literal rewrite body", name.Name) + } + } + return true + }) + for _, name := range []string{"AMULSS", "AMULSD", "ADIVSS", "ADIVSD"} { + if seen[name] != 1 { + return fmt.Errorf("actual Go literal rewrite registration %s count=%d", name, seen[name]) + } + } + return failure +} + +func TestX86FloatMulDivLiteralRewriteSourceRejectsMutation(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/obj6.go") + source, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := x86MulDivLiteralRewriteSource(source); err != nil { + t.Fatal(err) + } + before := sha256.Sum256(source) + for _, replacement := range [][2]string{ + {"func progedit(", "func unrelated("}, + {"AMULSS,", "AMULPS,"}, {"ADIVSD,", "ADIVPD,"}, + {"ctxt.Float32Sym(f32)", "ctxt.Float64Sym(f32)"}, {"p.From.Offset = 0", "p.From.Offset = 4"}, + } { + mutated := bytes.ReplaceAll(source, []byte(replacement[0]), []byte(replacement[1])) + if bytes.Equal(mutated, source) || x86MulDivLiteralRewriteSource(mutated) == nil { + t.Fatalf("unknown source rewrite accepted: %q", replacement) + } + } + after, err := os.ReadFile(path) + if err != nil || sha256.Sum256(after) != before { + t.Fatalf("source mutation oracle changed the real SDK: %v", err) + } +} + +func TestX86FloatMulDivLiteralOriginalInstructionBytes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + opcode string + prefix, opcodeByte byte + }{ + {"MULSS", 0xf3, 0x59}, {"MULSD", 0xf2, 0x59}, + {"DIVSS", 0xf3, 0x5e}, {"DIVSD", 0xf2, 0x5e}, + } { + for _, literal := range []string{"0.0", "-0.0", "1.0", "0x1.4p0"} { + code := x86PackedFirstGoInstructionBytes(t, arch, row.opcode+" $"+literal+",X0") + if !bytes.Equal(code[:4], []byte{row.prefix, 0x0f, row.opcodeByte, 0x05}) { + t.Fatalf("%s %s literal %s original bytes=%x", arch, row.opcode, literal, code) + } + } + } + } +} + +func TestX86FloatMulDivLiteralCallerKeepsDefinedBytes(t *testing.T) { + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + file, err := Parse(ArchAMD64, "TEXT literal(SB),4,$0-0\n"+opcode+" $-0.0,X0\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86FloatMulDivProperties(ins, "amd64") + if !recognized || err != nil || !form.literal || form.memory != -1 || form.memoryBytes != 0 { + t.Fatalf("literal confused with caller memory: %+v / %v", form, err) + } + low := x86CallerVectorByteMask(int64(form.bits / 8)) + for _, old := range []uint64{0, low &^ 1, low, low | 0xf0000000, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": old}, masks: map[Reg]bool{}} + before, _ := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerFloatBinaryStep(&ins, form.x86FloatBinaryForm, &state, nil) + if accepted != (old&low == low) || !accepted && !reflect.DeepEqual(before, state.vectors) || accepted && state.vectors["Z0"] != old { + t.Fatalf("%s invented destination/upper knowledge: old=%x state=%x accepted=%t", opcode, old, state.vectors["Z0"], accepted) + } + } + // Source spelling is not permission to use a different typed value. + ins.Args[0].Imm++ + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + _, _, err = ctx.lowerFloatMulDiv(ins.Op, ins) + if err == nil || output.String() != before { + t.Fatal("mismatched literal/source bits emitted IR") + } + } +} + +func TestX86FloatMulDivLiteralThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var declarations, source strings.Builder + for _, op := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + for _, value := range x86MulDivLiteralValues { + name := strings.ToLower(op) + value.name + fmt.Fprintf(&declarations, "func %s()\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\nPXOR X0,X0\n%s $%s,X0\nRET\n", name, op, value.text) + } + } + pkg := x86MulDivLiteralPackage(t, target.arch, declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + resolve := func(sym string) string { return strings.Replace(sym, "·", "literal.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, target.arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var result *GoModuleTranslation + result, err = translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + ir = result.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mul-div-literal.ll", "mul-div-literal.o", ir) + x86MulDivLiteralOptimizedObject(t, llc, target.triple, ir) + }) + } + }) + } +} + +func TestX86FloatMulDivLiteralRejectsNonGoFormats(t *testing.T) { + for _, row := range []string{ + "MULSD $1,X0", "MULSS $0,X0", "DIVSD $0x3ff0000000000000,X0", + "MULSD $(1.0+2.0),X0", "DIVSS $(1.0/2.0),X0", "MULSS $1e999,X0", + "MULPS $1.0,X0", "MULPD $1.0,X0", "DIVPS $1.0,X0", "DIVPD $1.0,X0", + "VMULSS $1.0,X0,X1", "VMULSD $1.0,X0,X1", "VDIVSS $1.0,X0,X1", "VDIVSD $1.0,X0,X1", + "MULSD.Z $1.0,X0", "MULSS $1.0,Y0", "DIVSD $1.0,X16", "MULSD X0,$1.0", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT literal(SB),4,$0-0\n" + row + "\nRET\n" + x86ScalarLiteralExpectedGoListing(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerFloatMulDiv(ins.Op, ins) + if !known || term || err == nil || before != output.String() { + t.Fatalf("non-Go literal emitted IR: %t/%t/%v", known, term, err) + } + }) + } +} + +// Keep this consumer's lifecycle inventory independent of the shared file, +// which has another writer. The actual original-Go test still uses the same +// bounded owned command helper as all other native numerical oracles. +func TestX86FloatMulDivLiteralCommandLifecycle(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "amd64_float_mul_div_literal_runtime_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + commands, consumers := 0, 0 + ast.Inspect(file, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if function, ok := call.Fun.(*ast.Ident); ok && function.Name == "runtimeOracleCombinedOutput" { + consumers++ + } + if method, ok := call.Fun.(*ast.SelectorExpr); ok { + if owner, ok := method.X.(*ast.Ident); ok && owner.Name == "exec" && method.Sel.Name == "CommandContext" { + commands++ + } + switch method.Sel.Name { + case "CombinedOutput", "Output", "Start", "Run": + if owner, ok := method.X.(*ast.Ident); !ok || owner.Name != "t" { + t.Errorf("literal oracle bypasses owned lifecycle: %s", method.Sel.Name) + } + } + } + return true + }) + if commands != 2 || consumers != 2 { + t.Fatalf("original-Go literal command lifecycle missing: commands=%d consumers=%d", commands, consumers) + } +} diff --git a/amd64_float_mul_div_mode32_test.go b/amd64_float_mul_div_mode32_test.go new file mode 100644 index 000000000..a8123ca5e --- /dev/null +++ b/amd64_float_mul_div_mode32_test.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestX86FloatMulDivMode32HighNamesStayUnlowered(t *testing.T) { + for _, opcode := range x86FloatMulDivOpcodes() { + if !strings.HasPrefix(opcode, "V") { + continue + } + widths := []string{"X", "Y", "Z"} + if strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") { + widths = widths[:1] + } + for _, width := range widths { + for _, index := range []int{8, 15, 16, 29, 31} { + row := fmt.Sprintf("%s %s%d,%s1,%s2", opcode, width, index, width, width) + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAccepted := width != "Z" + goAssemblerOracleOutput(t, "386", source, "", goAccepted) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + context, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + context.goarch = "386" + before := output.Len() + known, term, err := context.lowerFloatMulDiv(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]) + if !known || term || err == nil || goAccepted && !errors.Is(err, ErrProbeNeedsContext) || output.Len() != before { + t.Fatalf("mode32 high bank acquired a physical value: %t/%t/%v", known, term, err) + } + }) + } + } + } +} + +func TestX86FloatMulDivMode32KeepsActualMXCSRInstructions(t *testing.T) { + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 llc/llvm-objdump required") + } + var source strings.Builder + sigs := map[string]FuncSig{} + for _, opcode := range x86FloatMulDivOpcodes() { + if strings.HasSuffix(opcode, "PS") || strings.HasSuffix(opcode, "PD") { + continue + } + bits, typ := 32, I32 + load, resultOffset := "MOVSS", 8 + if strings.HasSuffix(opcode, "SD") { + bits, typ, load, resultOffset = 64, I64, "MOVSD", 16 + } + name := strings.ToLower(opcode) + row, destination := opcode+" X1,X0", "X0" + if strings.HasPrefix(opcode, "V") { + row, destination = opcode+" X1,X0,X2", "X2" + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-%d\n%s a+0(FP),X0\n%s b+%d(FP),X1\n%s\n%s %s,result+%d(FP)\nRET\n", + name, resultOffset+bits/8, load, load, bits/8, row, load, destination, resultOffset) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{typ, typ}, Ret: typ, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}, {Offset: int64(bits / 8), Type: typ, Index: 1, Field: -1}}, + Results: []FrameSlot{{Offset: int64(resultOffset), Type: typ, Field: -1}}, + }} + } + goAssemblerOracleOutput(t, "386", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + input, object := filepath.Join(dir, "mode32.ll"), filepath.Join(dir, "mode32.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, llc, "-O2", "-mtriple="+triple, "-filetype=obj", input, "-o", object) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("actual mode32 object: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + command = exec.CommandContext(ctx, disassembler, "-d", object) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("actual mode32 machine instructions: %v\n%s", err, output) + } + for _, opcode := range []string{"mulss", "mulsd", "divss", "divsd"} { + if strings.Count(string(output), "\t"+opcode+"\t") != 2 { + t.Fatalf("%s lost %s/MXCSR to another FP environment:\n%s", triple, opcode, output) + } + } + if strings.Contains(string(output), "fmul") || strings.Contains(string(output), "fdiv") { + t.Fatalf("mode32 MUL/DIV used x87 instead of source MXCSR:\n%s", output) + } + } +} diff --git a/amd64_float_mul_div_numerical_test.go b/amd64_float_mul_div_numerical_test.go new file mode 100644 index 000000000..8f042cb27 --- /dev/null +++ b/amd64_float_mul_div_numerical_test.go @@ -0,0 +1,100 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The arithmetic reference is actual independent legacy SSE. No production +// opcode/rounding helper supplies expected numerical bits or MXCSR flags. +var x86FloatMulDivSSEOracle = strings.NewReplacer("subss", "divss", "subsd", "divsd", "addss", "mulss", "addsd", "mulsd").Replace(x86FloatAddSubSSEOracle) + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivDefaultPhysicalSemantics(t *testing.T) { + testX86FloatMulDivPhysicalSemantics(t, false) +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivFixedRoundPhysicalSemantics(t *testing.T) { + testX86FloatMulDivPhysicalSemantics(t, true) +} + +func testX86FloatMulDivPhysicalSemantics(t *testing.T, fixed bool) { + t.Helper() + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, row := range x86FloatMulDivFormats("amd64") { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + form, _, err := x86FloatMulDivProperties(file.Funcs[0].Instrs[1], "amd64") + if err != nil { + t.Fatal(err) + } + if (form.rounding != "") != fixed { + continue + } + fixedMode := -1 + for mode, suffix := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + if form.rounding == suffix { + fixedMode = mode + } + } + name := fmt.Sprintf("binary%d", count) + count++ + row = strings.Replace(row, "8(AX)", "(BX)", 1) + row = strings.Replace(row, "(AX)", "(BX)", 1) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nRET\n", name, row, map[bool]int{true: 1, false: 2}[form.spec.legacy]) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*,uint64_t);\n", name) + arithmeticBytes := form.byteWidth + if form.spec.scalar { + arithmeticBytes = form.spec.bits / 8 + } + opcode := strings.Split(strings.Fields(row)[0], ".")[0] + independentDivide := strings.Contains(opcode, "DIV") + aligned := form.legacy && !form.scalar && form.memory >= 0 + fmt.Fprintf(&checks, ` + for(int round=0;round<4;round++) for(int m=0;m<4;m++) for(int control=0;control<6;control++) for(int ambient=0;ambient<4;ambient++) { + uint8_t aStorage[96] __attribute__((aligned(16))),bStorage[68],out[67];uint8_t *a=aStorage+(%[13]d?16:1),*b=bStorage+3; + memset(aStorage,0xa7,96);memset(bStorage,0xb9,68);memset(out,0xc3,67); + for(int i=0;i<64/%[2]d;i++) { uint64_t av=%[3]d==32?UINT64_C(0x3f000001)+i*0x12345+round*0x10000:UINT64_C(0x3fe0000000000001)+i*0x123456789; + uint64_t bv=%[3]d==32?UINT64_C(0xbf800001)+i*0x12345:UINT64_C(0xbff0000000000001)+i*0x2345678; + if(round==1){av=%[3]d==32?UINT64_C(0x7fc12345)+i:UINT64_C(0x7ff8123456789abc)+i;bv=%[3]d==32?UINT64_C(0xffc45678)+i:UINT64_C(0xfff8fedcba987654)+i;} + if(round==2){av=%[3]d==32?UINT64_C(0x7f812345)+i:UINT64_C(0x7ff0123456789abc)+i;bv=%[3]d==32?UINT64_C(0xff800000):UINT64_C(0xfff0000000000000);} + if(round==3){av=1+i;bv=(%[3]d==32?UINT64_C(0x80000000):UINT64_C(0x8000000000000000))|i;} + memcpy(a+i*%[2]d,&av,%[2]d);memcpy(b+i*%[2]d,&bv,%[2]d); } + unsigned csr=controls[control]|(ambient<<13),observed,expectedFlags=0; + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + uint64_t mask=masks[m];%[1]s(out+1,a,b,mask); + __asm__ volatile("stmxcsr %%0":"=m"(observed)); + for(int i=0;i<64/%[2]d;i++) { + uint64_t av=0,bv=0,got=0;memcpy(&av,a+(%[4]d?0:i*%[2]d),%[2]d);memcpy(&bv,b+i*%[2]d,%[2]d);memcpy(&got,out+1+i*%[2]d,%[2]d); + int active=%[5]d<0 || ((mask>>i)&1); + unsigned operationCSR=%[12]d<0?csr:((csr&~0x6000u)|((unsigned)%[12]d<<13)); + uint64_t want=i*%[2]d<%[6]d?(active?sse_result(%[3]d,bv,av,%[7]d,operationCSR):(%[8]d?0:bv)):(%[9]d || (%[10]d && i*%[2]d<16)?bv:0); + if(i*%[2]d<%[6]d && active && %[12]d<0)expectedFlags|=last_sse_csr&63; + if(got!=want){fprintf(stderr,"%[1]s lane=%%d got=%%llx want=%%llx\n",i,(unsigned long long)got,(unsigned long long)want);return %[11]d;} + } + if(observed!=(csr|expectedFlags)){fprintf(stderr,"%[1]s sticky observed=%%x expected=%%x\n",observed,csr|expectedFlags);return 3;} + if(out[0]!=0xc3 || out[65]!=1 || out[66]!=0xc3 || a[-1]!=0xa7 || a[64]!=0xa7 || bStorage[2]!=0xb9 || bStorage[67]!=0xb9)return 2; + } +`, name, form.spec.bits/8, form.spec.bits, x86FloatAddSubBoolInt(form.broadcast), form.mask, arithmeticBytes, x86FloatAddSubBoolInt(independentDivide), x86FloatAddSubBoolInt(form.zeroing), x86FloatAddSubBoolInt(form.spec.legacy), x86FloatAddSubBoolInt(form.spec.scalar), count+10, fixedMode, x86FloatAddSubBoolInt(aligned)) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := x86FloatMulDivSSEOracle + declarations.String() + "int main(void){const uint64_t masks[]={0,1,0x55aa55aa55aa55aaULL,~0ULL}; const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0};\n" + checks.String() + "unsigned csr=0x1f80;__asm__ volatile(\"ldmxcsr %0\"::\"m\"(csr));return 0;}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "mul-div-default-semantics", triple, ir, main, prefix) + t.Logf("%d independent opcode/width/memory/mask programs x384 IEEE/MXCSR/canary inputs", count) +} diff --git a/amd64_float_mul_div_regression_test.go b/amd64_float_mul_div_regression_test.go new file mode 100644 index 000000000..57df156f8 --- /dev/null +++ b/amd64_float_mul_div_regression_test.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestX86GoCallerFloatMulDivOriginalSourceReachesObjects(t *testing.T) { + for _, opcode := range []string{"MULSS", "DIVSS", "VMULSS", "VDIVSS"} { + t.Run(opcode, func(t *testing.T) { + row := opcode + " X0,X1" + if strings.HasPrefix(opcode, "V") { + row += ",X2" + } + destination := "X1" + if strings.HasPrefix(opcode, "V") { + destination = "X2" + } + body := "VMOVSS (DI),X0\nVMOVSS (SI),X1\n" + row + "\nVMOVSS " + destination + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivConstantMXCSR(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for _, opcode := range []string{"MULSS", "DIVSS", "VMULSS", "VDIVSS"} { + name := strings.ToLower(opcode) + left, right := uint32(0x3f800001), uint32(0x3f800001) + if strings.Contains(opcode, "DIV") { + left, right = 0x3f800000, 0x40400000 + } + row, destination := opcode+" X1,X0", "X0" + if strings.HasPrefix(opcode, "V") { + row, destination = opcode+" X1,X0,X2", "X2" + } + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-4\nMOVL $0x%x,AX\nMOVD AX,X0\nMOVL $0x%x,AX\nMOVD AX,X1\n%s\nMOVSS %s,ret+0(FP)\nRET\n", name, left, right, row, destination) + sigs[name] = FuncSig{Name: name, Ret: I32, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I32, Field: -1}}}} + fmt.Fprintf(&declarations, "extern uint32_t %s(void);\n", name) + fmt.Fprintf(&checks, "failed|=check(%s,0x%x,0x%x,%d,%q);\n", name, left, right, x86FloatAddSubBoolInt(strings.Contains(opcode, "DIV")), opcode) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const oracle = `#include +#include +#include +typedef uint32_t(*binary_constant)(void); +static int check(binary_constant fn,uint32_t left,uint32_t right,int divide,const char *name) { + unsigned saved; __asm__ volatile("stmxcsr %0":"=m"(saved)); + int failed=0; + for(unsigned mode=0;mode<4;mode++) { + unsigned csr=0x1f80|(mode<<13),wantCSR,gotCSR; + float a,b;memcpy(&a,&left,4);memcpy(&b,&right,4); + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + if(divide)__asm__ volatile("divss %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("mulss %1,%0":"+x"(a):"x"(b)); + uint32_t want;memcpy(&want,&a,4); + __asm__ volatile("stmxcsr %0":"=m"(wantCSR)); + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + uint32_t got=fn();__asm__ volatile("stmxcsr %0":"=m"(gotCSR)); + if(got!=want||gotCSR!=wantCSR) { + fprintf(stderr,"%s RC%u got=%x/%x original-SSE=%x/%x\n",name,mode,got,gotCSR,want,wantCSR); + failed=1; + } + } + __asm__ volatile("ldmxcsr %0"::"m"(saved));return failed; +} +` + main := oracle + declarations.String() + "int main(void){int failed=0;\n" + checks.String() + "return failed;}\n" + t.Run("original-ir", func(t *testing.T) { + compileAndRunRuntimeTestForTarget(t, llc, clang, "mul-div-constant-mxcsr", triple, ir, main, prefix) + }) + t.Run("optimized-ir", func(t *testing.T) { + opt := findLLVM22Tool("opt") + if opt == "" { + t.Fatal("LLVM22 opt missing") + } + dir := t.TempDir() + input, output := filepath.Join(dir, "source.ll"), filepath.Join(dir, "optimized.ll") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, opt, "-S", "-passes=default", input, "-o", output) + message, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("LLVM22 O2: %v\n%s", err, message) + } + optimized, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "mul-div-constant-mxcsr-o2", triple, string(optimized), main, prefix) + }) +} diff --git a/amd64_float_zero_mask_epoch_test.go b/amd64_float_zero_mask_epoch_test.go new file mode 100644 index 000000000..144f6120b --- /dev/null +++ b/amd64_float_zero_mask_epoch_test.go @@ -0,0 +1,125 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// The Go encoder tables do not express this helper rule. Go 1.20 could +// assemble EVEX.z=1/aaa=0; issue 57952 closed that bug in Go 1.21. Those +// historical objects are observations of invalid bytes, never legal forms. +func TestX86BinaryFloatZeroingRequiresNonzeroMaskAcrossAssemblerEpochs(t *testing.T) { + oldAssembler := !goToolchainAtLeast(runtime.Version(), 1, 21) + for _, arch := range []string{"386", "amd64"} { + for operation, opcode := range map[string]byte{ + "ADD": 0x58, "SUB": 0x5c, "MUL": 0x59, "DIV": 0x5e, + } { + for pp, suffix := range []string{"PS", "PD", "SS", "SD"} { + widths := []string{"X", "Y", "Z"} + if pp >= 2 { + widths = widths[:1] + } + for width, bank := range widths { + name := "V" + operation + suffix + row := fmt.Sprintf("%s.Z %s0,%s1,%s2", name, bank, bank, bank) + t.Run(arch+"/"+row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", oldAssembler) + + // Independent register encoding: source0, source1, + // destination2; EVEX.v'=1 and the requested vector length. + p1 := byte(0x74 | pp) + if pp&1 != 0 { + p1 |= 0x80 + } + code := []byte{0x62, 0xf1, p1, 0x88 | byte(width<<5), opcode, 0xd0} + if oldAssembler { + listing := x86InsertExtractGoListing(t, arch, source) + var observed []byte + for _, line := range strings.Split(string(listing), "\n") { + observed = append(observed, x86InsertExtractListedBytes(line)...) + } + want := append(append([]byte(nil), code...), 0xc3) + if !bytes.Equal(observed, want) { + t.Fatalf("historical assembler observation changed: got %x want %x", observed, want) + } + t.Logf("actual old Go emitted reserved z=1/aaa=0 bytes %x", code) + } + mode := 64 + if arch == "386" { + mode = 32 + } + if _, _, known, err := decodedX86EVEXBinaryFloatInstruction(code, 0, mode, "", false); !known || err == nil || + !strings.Contains(err.Error(), "zeroing requires a nonzero mask") { + t.Fatalf("reserved zeroing acquired raw semantics: known=%t err=%v", known, err) + } + x86RequireZeroMaskAPIOutcome(t, arch, source, false) + x86RequireZeroMaskAPIOutcome(t, arch, x86RawINSERTPSSource("f", code), false) + + explicit := fmt.Sprintf("%s.Z %s0,%s1,K0,%s2", name, bank, bank, bank) + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+explicit+"\nRET\n", "", false) + x86RequireZeroMaskAPIOutcome(t, arch, "TEXT f(SB),4,$0-0\n"+explicit+"\nRET\n", false) + + if arch == "amd64" { + valid := fmt.Sprintf("%s.Z %s0,%s1,K1,%s2", name, bank, bank, bank) + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+valid+"\nRET\n", "", true) + control := append([]byte(nil), code...) + control[3] |= 1 + ins, size, known, err := decodedX86EVEXBinaryFloatInstruction(control, 0, 64, "", false) + if !known || err != nil || size != len(control) || ins.Op != Op(name+".Z") || + len(ins.Args) != 4 || ins.Args[2].Reg != "K1" { + t.Fatalf("nonzero-mask control rejected: %+v size=%d known=%t err=%v", ins, size, known, err) + } + x86RequireZeroMaskAPIOutcome(t, arch, "TEXT f(SB),4,$0-0\n"+valid+"\nRET\n", true) + x86RequireZeroMaskAPIOutcome(t, arch, x86RawINSERTPSSource("f", control), true) + } + }) + } + } + } + } +} + +func x86RequireZeroMaskAPIOutcome(t *testing.T, arch, source string, wantSuccess bool) { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal("fixture must reach the semantic validators", err) + } + triples := []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + if arch == "amd64" { + triples = []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} + } + pkg := x86FloatMulDivShapesPackage(t, "package shapes\nfunc f()\n", arch) + for _, triple := range triples { + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}} + if ir, err := Translate(file, options); (err == nil) != wantSuccess || err == nil && ir == "" { + t.Fatalf("%s text API outcome wantSuccess=%t err=%v", triple, wantSuccess, err) + } + func() { + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if (err == nil) != wantSuccess || err == nil && module.IsNil() { + t.Fatalf("%s module API outcome wantSuccess=%t err=%v", triple, wantSuccess, err) + } + goSource := strings.Replace(source, "TEXT f(SB)", "TEXT ·f(SB)", 1) + result, err := translateGoModuleInContext(context, pkg, []byte(goSource), + GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if (err == nil) != wantSuccess || err == nil && (result == nil || result.Module.IsNil()) { + t.Fatalf("%s Go API outcome wantSuccess=%t err=%v", triple, wantSuccess, err) + } + }() + } +} diff --git a/amd64_floating_helper_forms.go b/amd64_floating_helper_forms.go new file mode 100644 index 000000000..d934e7043 --- /dev/null +++ b/amd64_floating_helper_forms.go @@ -0,0 +1,222 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// These families share operand and native-effect axes, not floating-point +// arithmetic. Existing operation registrations remain the mnemonic inventory. +type x86FloatingHelperSpec struct { + bits, bytes int + scalar, immediate, second bool + readDestination, maskResult, round, dq bool + estimate14, estimate28 bool +} + +func x86FloatingHelperSpecFor(op Op) (x86FloatingHelperSpec, bool) { + if spec, known := amd64ApproximateMathSpecs[op]; known { + return x86FloatingHelperSpec{bits: spec.laneBits, scalar: spec.scalar, estimate28: true}, true + } + if spec, known := amd64Reciprocal14Specs[op]; known { + return x86FloatingHelperSpec{bits: spec.laneBits, scalar: spec.scalar, estimate14: true}, true + } + if spec, known := amd64GetExpSpecs[op]; known { + return x86FloatingHelperSpec{bits: spec.laneBits, scalar: spec.scalar}, true + } + if spec, known := amd64GetMantSpecs[op]; known { + return x86FloatingHelperSpec{bits: spec.laneBits, scalar: spec.scalar, immediate: true}, true + } + if spec, known := amd64BinaryFloatingSpecs[op]; known && spec.mode == amd64BinaryFloatingScale { + return x86FloatingHelperSpec{bits: spec.laneBits, scalar: spec.scalar, second: true, round: true}, true + } + if spec, known := amd64FloatingRangeSpecs[op]; known { + return x86FloatingHelperSpec{bits: spec.laneBits, scalar: spec.scalar, immediate: true, second: true, dq: true}, true + } + if spec, known := amd64FixupImmediateSpecs[op]; known { + return x86FloatingHelperSpec{bits: spec.laneBits, scalar: spec.scalar, immediate: true, second: true, readDestination: true}, true + } + if spec, known := amd64ScaledRoundSpecs[op]; known { + return x86FloatingHelperSpec{bits: spec.laneBits, scalar: spec.scalar, immediate: true, dq: spec.mode == amd64Reduce}, true + } + if spec, known := amd64FPClassSpecs[op]; known { + return x86FloatingHelperSpec{bits: spec.laneBits, bytes: spec.bytes, scalar: spec.scalar, immediate: true, maskResult: true, dq: true}, true + } + return x86FloatingHelperSpec{}, false +} + +type x86FloatingHelperForm struct { + spec x86FloatingHelperSpec + first, second, destination, mask, memory int + byteWidth, memoryBytes int + immediate uint8 + properties amd64BinaryFloatingSuffix +} + +func x86FloatingHelperProperties(ins Instr, arch string) (x86FloatingHelperForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86FloatingHelperSpecFor(Op(base)) + form := x86FloatingHelperForm{spec: spec, second: -1, mask: -1, memory: -1} + if !known { + return form, false, nil + } + fail := func() (x86FloatingHelperForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go floating-helper grammar: %q", arch, ins.Op, ins.Raw) + } + properties, valid := parseAMD64BinaryFloatingSuffix(suffix) + if !valid || dotted && suffix == "" || arch != "386" && arch != "amd64" || + spec.scalar && properties.broadcast || spec.round && properties.sae || + !spec.round && properties.rounding != "" || spec.estimate14 && properties.sae || spec.maskResult && (properties.sae || properties.zeroing) { + return fail() + } + form.properties = properties + if spec.immediate { + if len(ins.Args) == 0 || ins.Args[0].Kind != OpImm || ins.Args[0].ImmIsFloat || ins.Args[0].ImmRaw != "" || + ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return fail() + } + form.first, form.immediate = 1, uint8(ins.Args[0].Imm) + } + operands := form.first + 2 + if !spec.maskResult && (spec.second || spec.scalar) { + form.second = form.first + 1 + operands++ + } + if len(ins.Args) != operands && len(ins.Args) != operands+1 || arch == "386" && len(ins.Args) > 3 && !ins.x86Encoded { + return fail() + } + form.destination = len(ins.Args) - 1 + if len(ins.Args) == operands+1 { + form.mask = form.destination - 1 + if !amd64NonzeroKOperand(ins.Args[form.mask]) { + return fail() + } + } else if properties.zeroing { + return fail() + } + destination := ins.Args[form.destination] + form.byteWidth = amd64VectorByteWidth(destination.Reg) + if spec.maskResult { + if !amd64IsKOperand(destination) { + return fail() + } + form.byteWidth = spec.bytes + } + if form.byteWidth != 16 && form.byteWidth != 32 && form.byteWidth != 64 || spec.scalar && form.byteWidth != 16 || + spec.estimate28 && !spec.scalar && form.byteWidth != 64 || + (properties.sae || properties.rounding != "") && !spec.scalar && form.byteWidth != 64 { + return fail() + } + register := func(operand Operand) bool { + return amd64GoPackedVectorMoveRegister(operand, form.byteWidth, arch) + } + if !spec.maskResult && !register(destination) || form.second >= 0 && !register(ins.Args[form.second]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) { + if properties.sae || properties.rounding != "" { + return fail() + } + if first.Kind == OpMem { + if x86SymbolIndexedMemoryError(first.Mem, arch) != nil || first.Mem.OffRaw != "" || first.Mem.IndexExt != "" || + first.Mem.Off != int64(int32(first.Mem.Off)) && (arch != "386" || first.Mem.Off != int64(uint32(first.Mem.Off))) || + !x86MemoryRegistersValidForArch(first.Mem, arch) || first.Mem.Index != "" && + (first.Mem.Index == SP || first.Mem.Scale != 1 && first.Mem.Scale != 2 && first.Mem.Scale != 4 && first.Mem.Scale != 8) { + return fail() + } + } + if first.Kind == OpSym { + symbol := strings.TrimSpace(first.Sym) + address, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil || + absolute == nil && address != int64(int32(address)) && (arch != "386" || address != int64(uint32(address))) { + return fail() + } + } + form.memory, form.memoryBytes = form.first, form.byteWidth + if spec.scalar || properties.broadcast { + form.memoryBytes = spec.bits / 8 + } + } else if properties.broadcast || !register(first) { + return fail() + } + // Go's named high banks on 386 are pre-encoding syntax, not initialized + // physical registers. Check this only after validating the entire grammar. + if arch == "386" { + for _, operand := range ins.Args { + if index, vector := amd64VectorRegisterIndex(operand.Reg, form.byteWidth); operand.Kind == OpReg && vector && index >= 8 { + return form, true, fmt.Errorf("%w: %s has no physical mode32 binding for %s", ErrProbeNeedsContext, ins.Op, operand.Reg) + } + } + } + return form, true, nil +} + +func (form x86FloatingHelperForm) activeBytes() int { + if form.spec.scalar { + return form.spec.bits / 8 + } + return form.byteWidth +} + +func (form x86FloatingHelperForm) nativeMnemonic(ins Instr) string { + name, _, _ := strings.Cut(strings.ToLower(string(ins.Op)), ".") + if form.spec.maskResult && !form.spec.scalar && (form.memory < 0 || form.properties.broadcast) { + name = name[:len(name)-1] // Go specializes FPCLASS's source width. + } + return name +} + +func x86TypedFloatingHelperByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + local := ins + local.Op = op + form, known, err := x86FloatingHelperProperties(local, arch) + if known && ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + err = fmt.Errorf("floating-helper dispatch/source opcode mismatch") + } + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + firstWidth := int64(form.activeBytes()) + if form.memory >= 0 { + firstWidth = int64(form.memoryBytes) + } + outputWidth := int64(form.byteWidth) + if form.spec.maskResult { + outputWidth = 8 + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: firstWidth, read: true}, + }} + if form.spec.estimate14 || form.spec.estimate28 { + effect.vector.upperWrite = x86TypedVectorUpperZero + // Native estimates accept unaligned memory. Their original + // masked access/fault behavior still executes in the native helper; + // this alignment metadata supplies no private/caller storage authority. + effect.vector.memoryAlignment = 1 + } + if form.second >= 0 { + // A scalar upper source is copied as bits. Sparse numerical knowledge + // is retained separately by the precise result-byte transfer. + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: form.second, operand: ins.Args[form.second], width: int64(form.byteWidth), read: true, + }) + if (form.spec.estimate14 || form.spec.estimate28) && form.spec.scalar { + effect.operands[len(effect.operands)-1].requiredBytes = 0xffff &^ x86CallerVectorByteMask(int64(form.activeBytes())) + } + } + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: form.destination, operand: ins.Args[form.destination], width: outputWidth, + read: !form.spec.maskResult && (form.spec.readDestination || form.mask >= 0 && !form.properties.zeroing), write: true, + }) + if (form.spec.estimate14 || form.spec.estimate28) && form.spec.scalar && form.mask >= 0 && !form.properties.zeroing { + effect.operands[len(effect.operands)-1].requiredBytes = x86CallerVectorByteMask(int64(form.activeBytes())) + } + if form.mask >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: form.mask, operand: ins.Args[form.mask], width: 8, read: true, + }) + } + return effect, true, nil +} diff --git a/amd64_floating_helper_go_rows_test.go b/amd64_floating_helper_go_rows_test.go new file mode 100644 index 000000000..60981c7fd --- /dev/null +++ b/amd64_floating_helper_go_rows_test.go @@ -0,0 +1,146 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86FloatingHelperCompleteIndependentGoRows(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + members := make(map[string]x86FloatingHelperOracleMember) + for _, member := range x86FloatingHelperOracleMembers() { + members[member.name] = member + } + counts := make(map[string]int) + for _, rule := range rules { + member, wanted := members[rule.Opcode] + if !wanted { + continue + } + ordinal := counts[rule.Opcode] + counts[rule.Opcode]++ + widths := []int{512, 128, 256} + if member.scalar { + widths = []int{128} + } else if member.maskResult { + widths = []int{map[string]int{"X": 128, "Y": 256, "Z": 512}[member.bank]} + } + if ordinal >= len(widths)*2 { + t.Fatal("unexpected extra floating-helper Go row", rule) + } + bits, word := 32, "0" + if strings.Contains(member.name, "PD") || strings.HasSuffix(member.name, "SD") { + bits, word = 64, "1" + } + family, opcode, opcodeScalar, encodingMap := "", "", "", "evex0F3A" + switch { + case strings.HasPrefix(member.name, "VGETEXP"): + family, opcode, opcodeScalar, encodingMap = "_yvcvtpd2qq", "0x42", "0x43", "evex0F38" + case strings.HasPrefix(member.name, "VGETMANT"): + family, opcode, opcodeScalar = "_yvgetmantpd", "0x26", "0x27" + case strings.HasPrefix(member.name, "VSCALEF"): + family, opcode, opcodeScalar, encodingMap = "_yvscalefpd", "0x2C", "0x2D", "evex0F38" + case strings.HasPrefix(member.name, "VRANGE"): + family, opcode, opcodeScalar = "_yvfixupimmpd", "0x50", "0x51" + case strings.HasPrefix(member.name, "VFIXUPIMM"): + family, opcode, opcodeScalar = "_yvfixupimmpd", "0x54", "0x55" + case strings.HasPrefix(member.name, "VRNDSCALE"): + family, opcode, opcodeScalar = "_yvgetmantpd", "0x08", "0x0A" + if bits == 64 { + opcode, opcodeScalar = "0x09", "0x0B" + } + case strings.HasPrefix(member.name, "VREDUCE"): + family, opcode, opcodeScalar = "_yvgetmantpd", "0x56", "0x57" + case member.maskResult: + family = "_yvfpclasspd" + strings.ToLower(member.bank) + opcode, opcodeScalar = "0x66", "0x67" + } + if member.scalar { + family, opcode = "_yvgetexpsd", opcodeScalar + if member.immediate { + family = "_yvfixupimmsd" + } + if member.maskResult { + family = "_yvfpclasspdx" + } + } + var bytes []string + for _, width := range widths { + tuple := width / 8 + if member.scalar { + tuple = bits / 8 + } + attributes := fmt.Sprintf("evexN%d", tuple) + if !member.scalar { + attributes += fmt.Sprintf(" | evexBcstN%d", bits/8) + } + if !member.maskResult { + if width == 512 || member.scalar { + control := "evexSaeEnabled" + if strings.HasPrefix(member.name, "VSCALEF") { + control = "evexRoundingEnabled" + } + attributes += " | " + control + } + attributes += " | evexZeroingEnabled" + } + bytes = append(bytes, fmt.Sprintf("avxEscape | evex%d | evex66 | %s | evexW%s", width, encodingMap, word), attributes, opcode) + } + width := widths[ordinal/2] + memory, vector := "YxmEvex", "YxrEvex" + if width == 256 { + memory, vector = "YymEvex", "YyrEvex" + } else if width == 512 { + memory, vector = "Yzm", "Yzr" + } + var classes []string + zcase := "Zevex_" + if member.immediate { + classes = append(classes, "Yu8") + zcase += "i_" + } + classes = append(classes, memory) + zcase += "rm_" + if !member.maskResult && (member.second || member.scalar) { + classes = append(classes, vector) + zcase += "v_" + } else if !member.immediate && ordinal%2 == 0 { + zcase += "v_" + } + offset := 0 + if ordinal%2 != 0 { + classes = append(classes, "Yknot0") + zcase += "k_" + offset = 3 + } + zcase += "r" + destination := vector + if member.maskResult { + destination = "Yk" + } + classes = append(classes, destination) + if rule.RowIndex != ordinal || rule.Ytab != family || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != offset || rule.EncodingOffset != ordinal/2*3 || + !reflect.DeepEqual(rule.Classes, classes) || !reflect.DeepEqual(rule.OpBytes, bytes) { + t.Fatal("complete independent floating-helper Go row/attribute changed", rule, + "want", family, zcase, classes, bytes) + } + } + for name, member := range members { + want := 6 + if member.scalar || member.maskResult { + want = 2 + } + if counts[name] != want { + t.Fatal("floating-helper family row denominator changed", name, counts[name], want) + } + } +} diff --git a/amd64_floating_helper_native.go b/amd64_floating_helper_native.go new file mode 100644 index 000000000..61109bc2a --- /dev/null +++ b/amd64_floating_helper_native.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// One complete native instruction preserves packed exception priority, ambient +// MXCSR, immediate exception controls, masked access and dead-result effects. +// A software value formula is not a replacement for any of those obligations. +func (c *amd64Ctx) lowerNativeFloatingHelper(op Op, ins Instr) (bool, bool, error) { + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("floating-helper dispatch/source mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, known, err := x86FloatingHelperProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return true, false, c.emitNativeFloatingHelper(ins, form) +} + +func (c *amd64Ctx) emitNativeFloatingHelper(ins Instr, form x86FloatingHelperForm) error { + first := ins.Args[form.first] + if ins.x86AddressBits != 0 || first.Kind == OpFP && !form.spec.scalar || first.Kind == OpMem && + !c.nativeMemoryOperand(ins, form.first) && !c.fixedLocalMemoryCertified(ins, form.first) { + return fmt.Errorf("%w: %s needs its original native source backing", ErrProbeNeedsContext, ins.Op) + } + if form.spec.maskResult { + return c.emitNativeFloatingClass(ins, form) + } + typ := amd64FloatingVectorType(form.byteWidth*8/form.spec.bits, form.spec.bits) + load := func(index int) (string, error) { + bytes, err := c.loadPackedCompareBytes(ins.Args[index], form.byteWidth) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", value, form.byteWidth, bytes, typ) + return "%" + value, nil + } + old := "zeroinitializer" + if form.spec.readDestination || form.mask >= 0 && !form.properties.zeroing { + var err error + old, err = load(form.destination) + if err != nil { + return err + } + } + mask := "" + if form.mask >= 0 { + var err error + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + } + input, inputType, inputConstraint, inputTemplate, err := c.nativeFloatingCompareSource(first, + x86FloatCompareForm{x86FloatBinaryForm: x86FloatBinaryForm{ + bits: form.spec.bits, byteWidth: form.byteWidth, memoryBytes: form.memoryBytes, scalar: form.spec.scalar, + }}, typ, mask) + if err != nil { + return err + } + // Source helper's $1 refers to its own first explicit argument. Here + // operand one is the tied old destination, and the source is operand two. + inputTemplate = strings.ReplaceAll(inputTemplate, "$1", "$2") + args := fmt.Sprintf("%s %s, %s %s", typ, old, inputType, input) + constraints := "=v,0," + inputConstraint + operands := inputTemplate + maskOrdinal := 3 + if form.second >= 0 { + second, err := load(form.second) + if err != nil { + return err + } + args += ", " + typ + " " + second + constraints += ",v" + operands += ", $3" + maskOrdinal++ + } + if form.properties.broadcast { + operands = strings.Replace(operands, inputTemplate, + inputTemplate+fmt.Sprintf(" {1to%d}", form.byteWidth*8/form.spec.bits), 1) + } + assembly := "{evex} " + form.nativeMnemonic(ins) + " " + if form.spec.immediate { + assembly += fmt.Sprintf("$$%d, ", form.immediate) + } + if form.properties.sae { + assembly += "{sae}, " + } else if form.properties.rounding != "" { + assembly += "{" + strings.ToLower(strings.TrimSuffix(form.properties.rounding, "_SAE")) + "-sae}, " + } + assembly += operands + ", $0" + if form.mask >= 0 { + assembly = fmt.Sprintf("kmovw ${%d:k}, %%k1; ", maskOrdinal) + assembly + " {%k1}" + if form.properties.zeroing { + assembly += " {z}" + } + constraints += ",r,~{k1}" + args += ", i32 " + c.approximateMathMask32(mask) + } + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, typ, assembly, + constraints+",~{memory},~{fpsr}", args) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, typ, result, form.byteWidth) + return c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, "%"+bytes) +} + +func (c *amd64Ctx) emitNativeFloatingClass(ins Instr, form x86FloatingHelperForm) error { + typ := amd64FloatingVectorType(form.byteWidth*8/form.spec.bits, form.spec.bits) + mask := "" + if form.mask >= 0 { + var err error + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + } + input, inputType, inputConstraint, inputTemplate, err := c.nativeFloatingCompareSource(ins.Args[form.first], + x86FloatCompareForm{x86FloatBinaryForm: x86FloatBinaryForm{ + bits: form.spec.bits, byteWidth: form.byteWidth, memoryBytes: form.memoryBytes, scalar: form.spec.scalar, + }}, typ, mask) + if err != nil { + return err + } + if form.properties.broadcast { + inputTemplate += fmt.Sprintf(" {1to%d}", form.byteWidth*8/form.spec.bits) + } + args := inputType + " " + input + constraints := "=r," + inputConstraint + assembly := fmt.Sprintf("{evex} %s $$%d, %s, %%k1", form.nativeMnemonic(ins), form.immediate, inputTemplate) + if form.mask >= 0 { + assembly = "kmovw ${2:k}, %k2; " + assembly + " {%k2}" + constraints += ",r,~{k2}" + args += ", i32 " + c.approximateMathMask32(mask) + } + assembly += "; kmovw %k1, ${0:k}" + result, wide := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 asm sideeffect %q, %q(%s)\n", result, assembly, + constraints+",~{k1},~{memory},~{fpsr}", args) + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, result) + return c.storeK(ins.Args[form.destination].Reg, "%"+wide) +} diff --git a/amd64_floating_helper_native_test.go b/amd64_floating_helper_native_test.go new file mode 100644 index 000000000..2c226c046 --- /dev/null +++ b/amd64_floating_helper_native_test.go @@ -0,0 +1,228 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// This is an independent inventory, not an iteration over the implementation +// registration. FPCLASS's X/Y/Z opcode spellings differ from LLVM's mnemonic. +type x86FloatingHelperOracleMember struct { + name, mnemonic string + immediate bool + second bool + scalar bool + maskResult bool + bank string +} + +func TestX86FloatingHelperInitializedFiveTargetThreeAPIObjects(t *testing.T) { + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, member := range x86FloatingHelperOracleMembers() { + instruction := x86FloatingHelperOracleInstruction(member, false) + ins := x86ScalarLaneInstruction(t, instruction) + if target.arch == "386" && len(ins.Args) > 3 { + continue + } + name := fmt.Sprintf("F%d", count) + count++ + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-4\nVPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\n%s\n", name, instruction) + if member.maskResult { + source.WriteString("KMOVD K2,AX\n") + } else { + source.WriteString("PEXTRD $0,X2,AX\n") + } + source.WriteString("MOVL AX,ret+0(FP)\nRET\n") + fmt.Fprintf(&declarations, "func %s() uint32\n", name) + } + want := 36 + if target.arch == "386" { + want = 22 + } + if count != want { + t.Fatal("initialized named family omitted an applicable member", target, count, want) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("initialized floating family failed public translation", api, err) + } + if got := strings.Count(ir, "asm sideeffect"); got != count { + t.Fatal("public translation lost original floating instructions", api, got, count) + } + compileLLVMToObject(t, llc, target.triple, api+"-helpers.ll", api+"-helpers.o", ir) + } + }) + } +} + +func TestX86FloatingHelperVoidAndUnusedBoolCannotSupplyInputs(t *testing.T) { + for _, member := range x86FloatingHelperOracleMembers() { + instruction := x86FloatingHelperOracleInstruction(member, false) + for _, unusedBool := range []bool{false, true} { + declaration, extent := "func Missing()", 0 + if unusedBool { + declaration, extent = "func Missing(unused bool)", 1 + } + source := fmt.Sprintf("TEXT ·Missing(SB),4,$0-%d\n%s\nRET\n", extent, instruction) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if unusedBool { + x86VectorScratchCodegenControl(t, source, declaration, "amd64", "x86_64-unknown-linux-gnu", ir, err) + continue + } + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("empty Void invented missing floating inputs", member.name, unusedBool, api, err) + } + } + } + } +} + +func x86FloatingHelperOracleMembers() []x86FloatingHelperOracleMember { + var members []x86FloatingHelperOracleMember + for _, family := range []struct { + name string + immediate bool + second bool + }{ + {"VGETEXP", false, false}, + {"VGETMANT", true, false}, + {"VSCALEF", false, true}, + {"VRANGE", true, true}, + {"VFIXUPIMM", true, true}, + {"VRNDSCALE", true, false}, + {"VREDUCE", true, false}, + } { + for _, suffix := range []string{"PS", "PD", "SS", "SD"} { + members = append(members, x86FloatingHelperOracleMember{ + name: family.name + suffix, mnemonic: strings.ToLower(family.name + suffix), + immediate: family.immediate, second: family.second, + scalar: suffix == "SS" || suffix == "SD", bank: "X", + }) + } + } + for _, precision := range []string{"PS", "PD"} { + for _, bank := range []string{"X", "Y", "Z"} { + members = append(members, x86FloatingHelperOracleMember{ + name: "VFPCLASS" + precision + bank, mnemonic: "vfpclass" + strings.ToLower(precision), + immediate: true, maskResult: true, bank: bank, + }) + } + } + for _, precision := range []string{"SS", "SD"} { + members = append(members, x86FloatingHelperOracleMember{ + name: "VFPCLASS" + precision, mnemonic: "vfpclass" + strings.ToLower(precision), + immediate: true, scalar: true, maskResult: true, bank: "X", + }) + } + return members +} + +// Retain the entire original Go source and its public Context classification. +// An unbound high-bank mode32 source cannot supply a physical LLVM object; +// independent initialized low-bank sources own that separate positive axis. +func x86FloatingHelperOriginalFormatObject(t *testing.T, source, name, arch, triple string) string { + t.Helper() + file, signatures := x86UnboundVoidFormatSource(t, source, name, arch, triple) + if arch == "386" { + for _, function := range file.Funcs { + for _, ins := range function.Instrs { + if x86Mode32VectorRegisterContext(ins, arch) != nil { + t.Log("complete original Go object and public Context retained; named high vector bank has no physical mode32 binding") + return "" + } + } + } + } + return x86ScalarFormatObjectUnit(t, file, signatures, arch, triple, 0) +} + +func x86FloatingHelperOracleInstruction(member x86FloatingHelperOracleMember, memory bool) string { + var args []string + if member.immediate { + args = append(args, "$255") + } + source := member.bank + "0" + if memory { + source = "8(AX)" + } + args = append(args, source) + if !member.maskResult && (member.second || member.scalar) { + args = append(args, member.bank+"1") + } + destination := member.bank + "2" + if member.maskResult { + destination = "K2" + } + return member.name + " " + strings.Join(append(args, destination), ",") +} + +func TestX86FloatingHelperCompleteNativeOpcodes(t *testing.T) { + members := x86FloatingHelperOracleMembers() + if len(members) != 36 { + t.Fatal("independent floating-helper family denominator changed", len(members)) + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + signatures := make(map[string]FuncSig) + var expected []string + for _, member := range members { + // Go's named mode32 frontend has no four-operand forms. Its + // physical encodings are a separate decoder test obligation. + if target.arch == "386" && (member.immediate && (member.second || member.scalar) && !member.maskResult) { + continue + } + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("F%d", len(signatures)) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\n%s\nRET\n", name, + x86FloatingHelperOracleInstruction(member, memory)) + signatures[name] = FuncSig{Name: name, Ret: Void} + mnemonic := member.mnemonic + if memory && member.maskResult && !member.scalar { + mnemonic += strings.ToLower(member.bank) + } + expected = append(expected, mnemonic) + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + // These deliberately unbound original formats own only emitter + // objects. They do not claim typed inputs or a public translation. + ir := x86ScalarFormatObjectUnit(t, file, signatures, target.arch, target.triple, 0) + if got := strings.Count(ir, "asm sideeffect"); got != len(expected) { + t.Fatalf("floating helpers lost original native MXCSR/exception/access effects: %d native instructions, want %d", got, len(expected)) + } + for ordinal, mnemonic := range expected { + start := strings.Index(ir, fmt.Sprintf("@InstructionFormat%d(", ordinal)) + if start < 0 { + t.Fatal("missing original format function", ordinal) + } + body, _, _ := strings.Cut(ir[start:], "\n}") + if !strings.Contains(body, mnemonic+" ") || !strings.Contains(body, "~{fpsr}") || + !strings.Contains(body, "~{memory}") { + t.Fatal("native source opcode or observable clobbers lost", ordinal, mnemonic) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "floating-helper-native.ll", "floating-helper-native.o", ir) + }) + } +} diff --git a/amd64_floating_helper_runtime_policy_test.go b/amd64_floating_helper_runtime_policy_test.go new file mode 100644 index 000000000..a7f89e4de --- /dev/null +++ b/amd64_floating_helper_runtime_policy_test.go @@ -0,0 +1,224 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "strconv" + "strings" + "testing" +) + +// Bind the existing complete assembly builders and independent C references. +// A transport repair must not replace them with a smaller passing program. +var x86FloatingHelperOriginalRuntimeOwners = []struct { + root, file, sourceSHA, cSHA string +}{ + { + root: "TestAMD64GetExpRuntimeSemantics", file: "amd64_lower_getexp_test.go", + sourceSHA: "e984e60362c11b28189a43d2e24efc2a25d9ef7df5d7ba750632dffc2ada582c", + cSHA: "755b471dca11961ef94e8103f354d27b1b737d265886465f6cca9ce18a953f88", + }, + { + root: "TestAMD64GetMantRuntimeSemantics", file: "amd64_lower_getmant_test.go", + sourceSHA: "631e669e6d8ff1c65192ad74a54f7d5ba3bc847e3b0c64dac64d94f44c1dcbbe", + cSHA: "c9bf0ab8f11e0317569d87c4ec72a9815afddcdf599c6b05195f4ecd9f94a19e", + }, + { + root: "TestAMD64ScaleFRuntimeSemantics", file: "amd64_lower_scalef_test.go", + sourceSHA: "9e0547ef11ff18488d9f093641beff6f2f88405e6c6aa8884d7b79bde75307ab", + cSHA: "3cec89474463e1884cc2979aea89520dd011589ab73d3c6051bfebc61bcc31c9", + }, + { + root: "TestAMD64FloatingRangeRuntimeSemantics", file: "amd64_lower_range_test.go", + sourceSHA: "5ec0b3b10d651607d57d18991cb9dd1d91aeea18c6bbbaaa540c0c2970370cbb", + cSHA: "f8b2c3f77cb8f4c6902308df47a1a832c3d03a981471ffe44a837c93b457b359", + }, + { + root: "TestAMD64FloatingRangeDoubleAndScalarRuntimeSemantics", file: "amd64_lower_range_test.go", + sourceSHA: "a5fac93bf84e828725448f9d6fdef423baf0ac0f1e1bdc7ce30537bbeb4f4377", + cSHA: "5dfe99148f760b1df90c2633526437cf02e932f2f98068d6fdb15eb6cee67409", + }, + { + root: "TestAMD64FixupImmediateRuntimeSemantics", file: "amd64_lower_fixupimm_test.go", + sourceSHA: "6ba976350729649f866150d52bc624bc5e5628524295d544c3d5df2bbe79b3fd", + cSHA: "147083a3eb84d95f929c67ac71d7ef42932b2bbdc4c8d5b2d3eee81ed7ac5a27", + }, + { + root: "TestAMD64ScaledRoundRuntimeSemantics", file: "amd64_lower_scaled_round_test.go", + sourceSHA: "c53a42b278d22ca13a3144f75dd58be991b8971ab70754f4bcffbbe952f2c413", + cSHA: "090f1b1990362807062dfe8709bd4094cfad1fcb2b6b9470ea0c56e516429db9", + }, + { + root: "TestAMD64FPClassRuntimeSemantics", file: "amd64_lower_fpclass_test.go", + sourceSHA: "754b23dc03e28506f8036180859683c6a07f49355752f846d8f5afcb56e4f078", + cSHA: "92e86c32eddff2266c607b5ca64b9616e86c46be09e1a9ca2948d1e51306bff8", + }, +} + +func TestX86FloatingHelperRuntimeNativeOwnerAndPipelinePolicy(t *testing.T) { + if len(x86FloatingHelperOriginalRuntimeOwners) != 8 { + t.Fatal("complete original runtime owner inventory changed") + } + for _, owner := range x86FloatingHelperOriginalRuntimeOwners { + t.Run(owner.root, func(t *testing.T) { + data, err := os.ReadFile(owner.file) + if err != nil { + t.Fatal(err) + } + positions := token.NewFileSet() + file, err := parser.ParseFile(positions, owner.file, data, 0) + if err != nil { + t.Fatal(err) + } + function := x86OriginalFloatingPolicyFunction(t, file, owner.root) + if function == nil || function.Body == nil { + t.Fatal("missing original runtime owner") + } + start, end := token.NoPos, token.NoPos + cSource := "" + calls := make(map[string]int) + for _, statement := range function.Body.List { + ast.Inspect(statement, func(node ast.Node) bool { + if identifier, ok := node.(*ast.Ident); ok && identifier.Name == "source" && start == token.NoPos { + start = statement.Pos() + } + if call, ok := node.(*ast.CallExpr); ok { + if identifier, ok := call.Fun.(*ast.Ident); ok { + calls[identifier.Name]++ + if identifier.Name == "requireX86GoAssemblerResult" && end == token.NoPos { + end = statement.Pos() + } + } + } + if specification, ok := node.(*ast.ValueSpec); ok && len(specification.Names) == 1 && + specification.Names[0].Name == "mainC" && len(specification.Values) == 1 { + if literal, ok := specification.Values[0].(*ast.BasicLit); ok && literal.Kind == token.STRING { + cSource, err = strconv.Unquote(literal.Value) + } + } + return true + }) + } + if start == token.NoPos || end <= start || cSource == "" || err != nil { + t.Fatal("original complete source/reference boundaries lost", err) + } + assemblyBuilder := strings.TrimSpace(string(data[positions.Position(start).Offset:positions.Position(end).Offset])) + if actual := fmt.Sprintf("%x", sha256.Sum256([]byte(assemblyBuilder))); actual != owner.sourceSHA { + t.Fatal("original complete assembly source builder changed", actual, owner.sourceSHA) + } + if actual := fmt.Sprintf("%x", sha256.Sum256([]byte(cSource))); actual != owner.cSHA { + t.Fatal("independent original C values/reference changed", actual, owner.cSHA) + } + if calls["x86FloatingHelperRuntimeTools"] != 1 || calls["x86FloatingHelperRunNativePipelines"] != 1 || + calls["rosettaAvailable"] != 0 || calls["compileAndRunRuntimeTestForTarget"] != 0 { + t.Fatal("original runtime must use its required native EVEX/DQ owner and all three execution pipelines", calls) + } + }) + } +} + +func TestX86FloatingHelperRuntimeNativeDriverPolicy(t *testing.T) { + data, err := os.ReadFile("amd64_floating_helper_runtime_test.go") + if err != nil { + t.Fatal(err) + } + positions := token.NewFileSet() + file, err := parser.ParseFile(positions, "amd64_floating_helper_runtime_test.go", data, 0) + if err != nil { + t.Fatal(err) + } + functions := make(map[string]*ast.FuncDecl) + for _, declaration := range file.Decls { + if function, ok := declaration.(*ast.FuncDecl); ok { + functions[function.Name.Name] = function + } + } + tools := functions["x86FloatingHelperRuntimeTools"] + pipelines := functions["x86FloatingHelperRunNativePipelines"] + if tools == nil || pipelines == nil { + t.Fatal("required native tools or complete pipeline driver missing") + } + calls := func(function *ast.FuncDecl) map[string]int { + result := make(map[string]int) + ast.Inspect(function.Body, func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok { + switch target := call.Fun.(type) { + case *ast.Ident: + result[target.Name]++ + case *ast.SelectorExpr: + result[target.Sel.Name]++ + } + } + return true + }) + return result + } + toolsCalls, pipelineCalls := calls(tools), calls(pipelines) + if toolsCalls["x86FloatCompareRuntimeTools"] != 1 || toolsCalls["compileC"] != 1 || toolsCalls["run"] != 1 || + toolsCalls["Skip"] != 1 || pipelineCalls["compileAndRunLLVM"] != 1 || + pipelineCalls["x86DwordProductIROptimize"] != 1 || pipelineCalls["Run"] != 1 || pipelineCalls["Skip"] != 0 { + t.Fatal("native preflight or actual compiler/run obligations changed", toolsCalls, pipelineCalls) + } + if len(tools.Body.List) < 3 { + t.Fatal("native host and EVEX checks missing") + } + host, ok := tools.Body.List[1].(*ast.IfStmt) + if !ok { + t.Fatal("native host exclusion is not an explicit platform boundary") + } + condition := string(data[positions.Position(host.Cond.Pos()).Offset:positions.Position(host.Cond.End()).Offset]) + if condition != `runtime.GOOS != "linux" || runtime.GOARCH != "amd64"` { + t.Fatal("native platform exclusion became a CPU/tool skip", condition) + } + var actual []string + ast.Inspect(pipelines.Body, func(node ast.Node) bool { + loop, ok := node.(*ast.RangeStmt) + if !ok { + return true + } + rows, ok := loop.X.(*ast.CompositeLit) + if !ok { + t.Fatal("native execution pipelines lost their complete explicit inventory") + } + for _, element := range rows.Elts { + row, ok := element.(*ast.CompositeLit) + if !ok { + t.Fatal("native pipeline row is not a complete fixture") + } + values := make(map[string]string) + for _, field := range row.Elts { + pair, ok := field.(*ast.KeyValueExpr) + if !ok { + t.Fatal("native pipeline lacks explicit axes") + } + key, ok := pair.Key.(*ast.Ident) + if !ok { + t.Fatal("native pipeline key is not bound") + } + values[key.Name] = string(data[positions.Position(pair.Value.Pos()).Offset:positions.Position(pair.Value.End()).Offset]) + } + actual = append(actual, values["name"]+"/"+values["backend"]+"/"+values["optimizeIR"]) + } + return true + }) + want := []string{`"backend_O0"/"-O0"/`, `"backend_O2"/"-O2"/`, `"IR_O2/backend_O2"/"-O2"/true`} + if strings.Join(actual, "\n") != strings.Join(want, "\n") { + t.Fatal("native execution narrowed or changed backend/IR optimization axes", actual) + } + for _, required := range []string{ + `x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX)`, + `if (!f || !dq || !vl || !osxsave || (xcr0&0xe6)!=0xe6) return 1;`, + `if (!original_dq()) return 2;`, + `vfpclassps $2,%%zmm0,%%k1;`, + `vfpclassps $2,%%xmm0,%%k1;`, + `vfpclasssd $2,%%xmm0,%%k1;`, + } { + if !strings.Contains(string(data), required) { + t.Fatal("native CPU, original opcode or OS state obligation missing", required) + } + } +} diff --git a/amd64_floating_helper_runtime_test.go b/amd64_floating_helper_runtime_test.go new file mode 100644 index 000000000..05a120990 --- /dev/null +++ b/amd64_floating_helper_runtime_test.go @@ -0,0 +1,91 @@ +package plan9asm + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +func x86FloatingHelperRuntimeTools(t *testing.T) x86FloatCompareExecution { + t.Helper() + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Skip("required native Linux/amd64 floating-helper execution counterpart; objects remain tested on every target") + } + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + // The shared EVEX owner already checks actual F/VL, OSXSAVE, XCR0, + // original opcodes and native argv transport. RANGE/REDUCE/FPCLASS also + // require DQ; neither an old receipt nor the host model name supplies it. + directory := t.TempDir() + source, executable := filepath.Join(directory, "dq.c"), filepath.Join(directory, "dq") + if err := os.WriteFile(source, []byte(x86FloatingHelperDQPreflightC), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + t.Logf("actual native floating-helper DQ/original-opcode preflight: %s", execution.run(t, executable)) + return execution +} + +const x86FloatingHelperDQPreflightC = `#include +#include +#include + +__attribute__((target("avx512f,avx512dq,avx512vl"))) +static int original_dq(void) { + unsigned packed, narrow, scalar; + __asm__ volatile("vpxord %%zmm0,%%zmm0,%%zmm0; " + "vfpclassps $2,%%zmm0,%%k1; kmovw %%k1,%0; " + "vfpclassps $2,%%xmm0,%%k1; kmovw %%k1,%1; " + "vfpclasssd $2,%%xmm0,%%k1; kmovw %%k1,%2" + : "=r"(packed),"=r"(narrow),"=r"(scalar) + :: "zmm0","k1","memory"); + return packed==0xffff && narrow==0xf && scalar==1; +} + +int main(void) { + unsigned a,b,c,d,f=0,dq=0,vl=0,osxsave=0; + uint64_t xcr0=0; + if (__get_cpuid(1,&a,&b,&c,&d)) { + osxsave=(c>>27)&1; + if (osxsave) { + unsigned low,high; + __asm__ volatile("xgetbv" : "=a"(low),"=d"(high) : "c"(0)); + xcr0=((uint64_t)high<<32)|low; + } + } + if (__get_cpuid_count(7,0,&a,&b,&c,&d)) { + f=(b>>16)&1; dq=(b>>17)&1; vl=(b>>31)&1; + } + printf("AVX512F=%u AVX512DQ=%u AVX512VL=%u OSXSAVE=%u XCR0=%llx\n", + f,dq,vl,osxsave,(unsigned long long)xcr0); + fflush(stdout); + if (!f || !dq || !vl || !osxsave || (xcr0&0xe6)!=0xe6) return 1; + if (!original_dq()) return 2; + puts("FLOATING_HELPER_NATIVE_DQ_OK"); + return 0; +} +` + +func x86FloatingHelperRunNativePipelines(t *testing.T, execution x86FloatCompareExecution, name, ir, mainC string) { + t.Helper() + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" || execution.triple != "x86_64-unknown-linux-gnu" || + len(execution.prefix) != 0 || execution.preserveArgv0 { + t.Fatal("floating-helper native execution cannot borrow a guest or foreign target") + } + for _, pipeline := range []struct { + name, backend string + optimizeIR bool + }{ + {name: "backend_O0", backend: "-O0"}, + {name: "backend_O2", backend: "-O2"}, + {name: "IR_O2/backend_O2", backend: "-O2", optimizeIR: true}, + } { + t.Run(pipeline.name, func(t *testing.T) { + input := ir + if pipeline.optimizeIR { + input = x86DwordProductIROptimize(t, ir) + } + execution.compileAndRunLLVM(t, name, input, mainC, pipeline.backend) + }) + } +} diff --git a/amd64_fma3_forms.go b/amd64_fma3_forms.go new file mode 100644 index 000000000..ed3e1d26c --- /dev/null +++ b/amd64_fma3_forms.go @@ -0,0 +1,163 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FMA3Form struct { + spec amd64FMA3Spec + suffix amd64FMA3Suffix + byteWidth int + memoryBytes int + destination int + mask int // -1 when unmasked + memory int // -1 for a register first source +} + +// Original physical generation wins over an indistinguishable named spelling. +// Named controls/register banks remain the fallback only when that observation +// is absent. Neither this selector nor CPU features admit raw numerical inputs. +func x86FMA3NativeEVEX(ins Instr, form amd64FMA3Form) bool { + switch ins.x86FMA3Encoding { + case x86RawFMA3EVEX: + return true + case x86RawFMA3VEX: + return false + } + if form.byteWidth == 64 || form.mask >= 0 || form.suffix.broadcast || form.suffix.rounding != "" { + return true + } + for _, operand := range ins.Args { + if index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth); operand.Kind == OpReg && valid && index >= 16 { + return true + } + } + return false +} + +func x86TypedFMA3ByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + if ins.Op != "" && !strings.EqualFold(string(op), string(ins.Op)) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64FMA3Specs[Op(base)]; known { + return x86TypedByteEffect{}, true, fmt.Errorf("FMA3 effect/source opcode mismatch: %s/%s", op, ins.Op) + } + return x86TypedByteEffect{}, false, nil + } + ins.Op = op + form, known, err := amd64FMA3Properties(ins, arch) + if !known || form.spec.rawOnly { + return x86TypedByteEffect{}, false, nil + } + if err != nil { + return x86TypedByteEffect{}, true, err + } + active := int64(form.byteWidth) + if form.spec.scalar { + active = int64(form.spec.laneBits / 8) + } + effect := x86TypedByteEffect{requiresResultProof: true} + for index, operand := range ins.Args { + width := active + if index == form.memory { + width = int64(form.memoryBytes) + } else if index == form.mask { + width = 8 + } + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: index, operand: operand, width: width, read: true, + }) + } + // Scalar active inputs and the full destination write are distinct axes. + // Numerical upper-X passthrough is handled by the shared result transfer. + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), write: true, + }) + return effect, true, nil +} + +// The complete _yvaddpd/_yvaddsd axes are independent of numerical input, +// caller-memory, initialized-mask and source-stack continuation proofs. FP16 +// shares the architectural grammar but keeps its raw-encoding-only boundary. +func amd64FMA3Properties(ins Instr, goarch string) (amd64FMA3Form, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := amd64FMA3Specs[Op(base)] + if !recognized { + return amd64FMA3Form{}, false, nil + } + form := amd64FMA3Form{ + spec: spec, destination: len(ins.Args) - 1, mask: -1, memory: -1, + } + fail := func() (amd64FMA3Form, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix are outside the FMA3 grammar: %q", goarch, base, ins.Raw) + } + if goarch != "amd64" && goarch != "386" || spec.rawOnly && !ins.x86Encoded || + dotted && suffix == "" || len(ins.Args) != 3 && len(ins.Args) != 4 { + return fail() + } + if ins.x86FMA3Encoding != x86RawFMA3Absent && + (spec.rawOnly || ins.x86FMA3Encoding != x86RawFMA3VEX && ins.x86FMA3Encoding != x86RawFMA3EVEX) { + return fail() + } + properties, valid := parseAMD64FMA3Suffix(suffix) + if !valid || spec.scalar && properties.broadcast { + return fail() + } + form.suffix = properties + if len(ins.Args) == 4 { + if goarch == "386" && !ins.x86Encoded && ins.x86FMA3Encoding != x86RawFMA3EVEX || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + form.mask = 2 + } else if properties.zeroing { + return fail() + } + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if spec.scalar && form.byteWidth != 16 || + !amd64GoPackedVectorMoveRegister(ins.Args[form.destination], form.byteWidth, goarch) || + !amd64GoPackedVectorMoveRegister(ins.Args[1], form.byteWidth, goarch) || + properties.rounding != "" && !spec.scalar && form.byteWidth != 64 { + return fail() + } + first := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(first) { + return fail() + } + } else if first.Kind == OpReg { + if !amd64GoPackedVectorMoveRegister(first, form.byteWidth, goarch) { + return fail() + } + } else if properties.rounding != "" || !isAMD64MemoryOperand(first) { + return fail() + } + if isAMD64MemoryOperand(first) { + if !x86PackedShiftMemoryValid(first, goarch) { + return fail() + } + if first.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(first.Mem, goarch); err != nil { + return form, true, err + } + if first.Mem.OffRaw != "" || first.Mem.IndexExt != "" { + return fail() + } + } + form.memory = 0 + form.memoryBytes = form.byteWidth + if spec.scalar || properties.broadcast { + form.memoryBytes = spec.laneBits / 8 + } + } + if ins.x86FMA3Encoding == x86RawFMA3VEX { + if form.byteWidth == 64 || form.mask >= 0 || properties.broadcast || properties.rounding != "" { + return fail() + } + for _, operand := range ins.Args { + if index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth); operand.Kind == OpReg && valid && index >= 16 { + return fail() + } + } + } + return form, true, x86Mode32VectorRegisterContext(ins, goarch) +} diff --git a/amd64_fma3_forms_test.go b/amd64_fma3_forms_test.go new file mode 100644 index 000000000..a189705af --- /dev/null +++ b/amd64_fma3_forms_test.go @@ -0,0 +1,232 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86FMA3GrammarMatchesEveryCurrentGoTableMember(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go"), nil, 0) + if err != nil { + t.Fatal(err) + } + rows := make(map[string]int) + wanted := make(map[Op]bool) + ast.Inspect(file, func(node ast.Node) bool { + if value, ok := node.(*ast.ValueSpec); ok && len(value.Names) == 1 && len(value.Values) == 1 { + name := value.Names[0].Name + if name == "_yvaddpd" || name == "_yvaddsd" { + literal, ok := value.Values[0].(*ast.CompositeLit) + if !ok { + t.Fatalf("Go FMA3 grammar changed shape: %s", name) + } + rows[name] = len(literal.Elts) + } + } + literal, ok := node.(*ast.CompositeLit) + if !ok { + return true + } + var opcode, table string + for _, item := range literal.Elts { + pair, ok := item.(*ast.KeyValueExpr) + if !ok { + continue + } + key, keyOK := pair.Key.(*ast.Ident) + value, valueOK := pair.Value.(*ast.Ident) + if !keyOK || !valueOK { + continue + } + if key.Name == "as" { + opcode = value.Name + } else if key.Name == "ytab" { + table = value.Name + } + } + if strings.HasPrefix(opcode, "AVFM") || strings.HasPrefix(opcode, "AVFNM") { + if table != "_yvaddpd" && table != "_yvaddsd" { + t.Fatalf("Go FMA3 acquired an unreviewed grammar: %s/%s", opcode, table) + } + wanted[Op(strings.TrimPrefix(opcode, "A"))] = table == "_yvaddsd" + } + return true + }) + if len(wanted) != 60 || !reflect.DeepEqual(rows, map[string]int{"_yvaddpd": 8, "_yvaddsd": 3}) { + t.Fatalf("current Go FMA3 family changed: opcodes=%d rows=%v", len(wanted), rows) + } + named := 0 + for opcode, spec := range amd64FMA3Specs { + if spec.rawOnly { + if _, exists := wanted[opcode]; exists { + t.Fatalf("raw-only opcode is now a Go named format: %s", opcode) + } + continue + } + scalar, exists := wanted[opcode] + if !exists || scalar != spec.scalar { + t.Fatalf("Go opcode omitted or changed: %s/%+v", opcode, spec) + } + named++ + } + if named != len(wanted) { + t.Fatalf("named family count mismatch: %d/%d", named, len(wanted)) + } + x86FMA3CheckCompleteEncoderAttributes(t) +} + +// Independent opcode enumeration and format axes. Every form has a separate +// entry, so an earlier trap/return cannot remove a later LLVM object witness. +func TestX86FMA3FixedRoundingDefaultArchitecture(t *testing.T) { + const source = "TEXT defaultFMA(SB),4,$0-0\nVFMADD132SD.RD_SAE X0,X1,X2\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"defaultFMA": {Name: "defaultFMA", Ret: Void}}, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("default-architecture format source invented its missing FMA inputs", err) + } + file, signatures := x86UnboundVoidFormatSource(t, source, "defaultFMA", "amd64", "x86_64-unknown-linux-gnu") + ir = x86ScalarFormatObjectUnit(t, file, signatures, "amd64", "x86_64-unknown-linux-gnu", 0) + llc, _, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "default-fma.ll", "default-fma.o", ir) +} + +func x86FMA3NamedFormats(arch string) []string { + var instructions []string + ops := append(go127FMA3PackedOps(), go127FMA3ScalarOps()...) + for _, opcode := range ops { + scalar := strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") + banks := []string{"X", "Y", "Z"} + if scalar { + banks = banks[:1] + } + for _, bank := range banks { + for _, mode := range []string{"", "merge", "zero"} { + if arch == "386" && mode != "" { + continue + } + mask, zero := "", "" + if mode != "" { + mask = "K1," + } + if mode == "zero" { + zero = ".Z" + } + operands := bank + "0," + bank + "1," + mask + bank + "2" + memory := "(AX)," + bank + "1," + mask + bank + "2" + instructions = append(instructions, opcode+zero+" "+operands, opcode+zero+" "+memory) + if !scalar { + instructions = append(instructions, opcode+".BCST"+zero+" "+memory) + } + if scalar || bank == "Z" { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + instructions = append(instructions, opcode+"."+rounding+zero+" "+operands) + } + } + } + } + } + return instructions +} + +func TestX86FMA3GrammarAxesActualGoAndFiveLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, instruction := range x86FMA3NamedFormats(arch) { + name := fmt.Sprintf("fmaformat%d", index) + entry := fmt.Sprintf("TEXT %s(SB),4,$0-0\n%s\nRET\n", name, instruction) + file, err := Parse(ArchAMD64, entry) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := amd64FMA3Properties(ins, arch) + if err != nil || !recognized || form.spec.rawOnly || form.destination != len(ins.Args)-1 { + t.Fatalf("%s %s: incomplete grammar %+v/%t/%v", arch, instruction, form, recognized, err) + } + if form.memory >= 0 { + want := form.byteWidth + if strings.Contains(instruction, ".BCST") || strings.HasSuffix(strings.Split(string(ins.Op), ".")[0], "SS") || + strings.HasSuffix(strings.Split(string(ins.Op), ".")[0], "SD") { + want = 4 + if strings.Contains(string(ins.Op), "PD") || strings.Contains(string(ins.Op), "SD") { + want = 8 + } + } + if form.memory != 0 || form.memoryBytes != want { + t.Fatalf("%s changed native read extent: %+v; want %d", instruction, form, want) + } + } + source.WriteString(entry) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + wanted := map[string]int{"amd64": 1836, "386": 612}[arch] + if len(sigs) != wanted { + t.Fatal("complete original FMA3 format denominator changed", arch, len(sigs), wanted) + } + requireX86GoAssemblerResult(t, arch, source.String(), true) + triples := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + file, observed := x86UnboundVoidFormatSources(t, source.String(), arch, triple) + if !reflect.DeepEqual(observed, sigs) { + t.Fatal("complete original FMA format inventory changed its function signatures") + } + ir := x86ScalarFormatObjectUnit(t, file, sigs, arch, triple, 0) + compileLLVMToObject(t, llc, triple, "fma-formats.ll", "fma-formats.o", ir) + } + t.Logf("%s: %d independently emitted original-Go format entries", arch, len(sigs)) + } +} + +func TestX86FMA3DispatchPreservesOriginalSourceIdentity(t *testing.T) { + for opcode, spec := range amd64FMA3Specs { + for _, identity := range []Op{"", opcode, Op(strings.ToLower(string(opcode)))} { + ins := Instr{Op: identity, x86Encoded: spec.rawOnly, Args: []Operand{ + {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X2"}, + }} + original := ins + ctx, _ := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + recognized, terminated, err := ctx.lowerFMA3(opcode, ins) + if !recognized || terminated || err != nil || !reflect.DeepEqual(ins, original) { + t.Fatalf("%s/%s legacy dispatch changed source/failed: %t/%t/%v", opcode, identity, recognized, terminated, err) + } + } + for _, conflict := range []Op{"VMOVUPS", "VFMADD132PS.BCST", opcode + ".Z"} { + ins := Instr{Op: conflict, x86Encoded: spec.rawOnly, Args: []Operand{ + {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X2"}, + }} + original := ins + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before := output.String() + recognized, _, err := ctx.lowerFMA3(opcode, ins) + if !recognized || err == nil || output.String() != before || !reflect.DeepEqual(ins, original) { + t.Fatalf("%s/%s conflicting dispatch changed source/emitted IR: %t/%v", opcode, conflict, recognized, err) + } + } + } +} diff --git a/amd64_fma3_round_runtime_test.go b/amd64_fma3_round_runtime_test.go new file mode 100644 index 000000000..3adcddfd2 --- /dev/null +++ b/amd64_fma3_round_runtime_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// This separately exercises the production numerical helper on every host. +// Original instruction acceptance and caller/LLVM objects are independently +// covered by the complete Go-format oracle, not inferred from this helper. +func TestCrossLinuxRuntimeMatrixX86FMA3FixedRoundingExtremes(t *testing.T) { + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + var ir strings.Builder + ir.WriteString("declare i128 @llvm.ctlz.i128(i128, i1)\n") + for _, bits := range []int{32, 64} { + floatType := amd64FMA3LLVMType(1, bits) + fmt.Fprintf(&ir, "declare %s @llvm.fma.f%d(%s, %s, %s)\n", floatType, bits, floatType, floatType, floatType) + emitAMD64FMA3RoundingHelper(&ir, bits) + fmt.Fprintf(&ir, "define i%d @fma_round%d(i%d %%a, i%d %%b, i%d %%c, i32 %%mode, i32 %%mxcsr) {\n", bits, bits, bits, bits, bits) + for _, operand := range []string{"a", "b", "c"} { + fmt.Fprintf(&ir, " %%%s_float = bitcast i%d %%%s to %s\n", operand, bits, operand, floatType) + } + fmt.Fprintf(&ir, " %%value = call %s @__plan9asm_fma3_round_f%d(%s %%a_float, %s %%b_float, %s %%c_float, i32 %%mode, i32 %%mxcsr)\n", floatType, bits, floatType, floatType, floatType) + fmt.Fprintf(&ir, " %%bits = bitcast %s %%value to i%d\n ret i%d %%bits\n}\n", floatType, bits, bits) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + compileAndRunRuntimeTestForTarget(t, llc, clang, "fma3-fixed-round-extremes", triple, ir.String(), x86FMA3RoundingExtremeOracle, nil) +} + +const x86FMA3RoundingExtremeOracle = `#include +#include +#include +#include +#include +#pragma STDC FENV_ACCESS ON + +extern uint32_t fma_round32(uint32_t,uint32_t,uint32_t,uint32_t,uint32_t); +extern uint64_t fma_round64(uint64_t,uint64_t,uint64_t,uint32_t,uint32_t); + +static uint64_t state=UINT64_C(0x71b32dd678098deb); +static uint64_t random_bits(void) { + state^=state<<13; state^=state>>7; state^=state<<17; + return state; +} + +static uint64_t flush_input(uint64_t bits, uint64_t exponent, uint64_t sign, unsigned csr) { + return (csr&64) && !(bits&exponent) ? bits&sign : bits; +} + +static int check(unsigned width, uint64_t a, uint64_t b, uint64_t c, + unsigned mode, unsigned ambient, unsigned csr) { + const int modes[]={FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO}; + uint64_t sign=width==32 ? UINT64_C(0x80000000) : UINT64_C(0x8000000000000000); + uint64_t exponent=width==32 ? UINT64_C(0x7f800000) : UINT64_C(0x7ff0000000000000); + uint64_t got, expected=0; + if (fesetround(modes[ambient])) return 1; + if (feclearexcept(FE_ALL_EXCEPT)) return 5; + if (width==32) got=fma_round32((uint32_t)a,(uint32_t)b,(uint32_t)c,mode,csr); + else got=fma_round64(a,b,c,mode,csr); + if (fegetround()!=modes[ambient]) return 2; + if (fetestexcept(FE_ALL_EXCEPT)) return 6; + a=flush_input(a,exponent,sign,csr); + b=flush_input(b,exponent,sign,csr); + c=flush_input(c,exponent,sign,csr); + if (fesetround(modes[mode])) return 3; + if (width==32) { + uint32_t aa=(uint32_t)a, bb=(uint32_t)b, cc=(uint32_t)c, result_bits; + float first, second, addend; + memcpy(&first,&aa,4); memcpy(&second,&bb,4); memcpy(&addend,&cc,4); + float result=fmaf(first,second,addend); + memcpy(&result_bits,&result,4); expected=result_bits; + } else { + double first, second, addend; + memcpy(&first,&a,8); memcpy(&second,&b,8); memcpy(&addend,&c,8); + double result=fma(first,second,addend); + memcpy(&expected,&result,8); + } + if ((csr&32768) && !(expected&exponent)) expected&=sign; + if (got!=expected) { + fprintf(stderr,"FMA rounding width/mode/ambient/csr %u/%u/%u/%x inputs=%llx,%llx,%llx got=%llx expected=%llx\n", + width,mode,ambient,csr,(unsigned long long)a,(unsigned long long)b, + (unsigned long long)c,(unsigned long long)got,(unsigned long long)expected); + return 4; + } + return 0; +} + +int main(void) { + const uint64_t edges32[]={0,0x80000000,1,0x80000001,0x7fffff,0x800000, + 0x800001,0x7f7fffff,0xff7fffff,0x3f000000,0xbf000000,0x3f800000, + 0xbf800000,0x3f800001,0xbf800002}; + const uint64_t edges64[]={0,UINT64_C(0x8000000000000000),1, + UINT64_C(0x8000000000000001),UINT64_C(0xfffffffffffff), + UINT64_C(0x10000000000000),UINT64_C(0x10000000000001), + UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff), + UINT64_C(0x3fe0000000000000),UINT64_C(0xbfe0000000000000), + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000), + UINT64_C(0x3ff0000000000001),UINT64_C(0xbff0000000000002)}; + const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0}; + for (unsigned width=32; width<=64; width+=32) { + const uint64_t *edges=width==32 ? edges32 : edges64; + const uint64_t exponent=width==32 ? UINT64_C(0x7f800000) : UINT64_C(0x7ff0000000000000); + const uint64_t exponent_low=width==32 ? UINT64_C(0x800000) : UINT64_C(0x10000000000000); + for (unsigned i=0; i<5000; i++) { + uint64_t a, b, c; + if (i<3375) { + a=edges[i%15]; b=edges[(i/15)%15]; c=edges[(i/225)%15]; + } else { + a=random_bits(); b=random_bits(); c=random_bits(); + if (width==32) { a=(uint32_t)a; b=(uint32_t)b; c=(uint32_t)c; } + if ((a&exponent)==exponent) a^=exponent_low; + if ((b&exponent)==exponent) b^=exponent_low; + if ((c&exponent)==exponent) c^=exponent_low; + } + for (unsigned mode=0; mode<4; mode++) { + for (unsigned ambient=0; ambient<4; ambient++) { + for (unsigned control=0; control<4; control++) { + if (check(width,a,b,c,mode,ambient,controls[control])) return 1; + } + } + } + } + } + return fesetround(FE_TONEAREST) ? 1 : 0; +} +` diff --git a/amd64_four_source_forms.go b/amd64_four_source_forms.go new file mode 100644 index 000000000..c91b5d8ba --- /dev/null +++ b/amd64_four_source_forms.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FourSourceMode uint8 + +const ( + amd64FourSourceFMA amd64FourSourceMode = iota + amd64FourSourceDotWords +) + +type amd64FourSourceSpec struct { + mode amd64FourSourceMode + scalar bool + negative bool + saturating bool +} + +// These are all six Go AVX512_4FMAPS/4VNNIW names. Their complete shared +// grammar is m128, a four-register range, optional K1-K7 and accumulator; +// only masked register zeroing is supported, not broadcast, ER or SAE. +var amd64FourSourceSpecs = map[Op]amd64FourSourceSpec{ + "V4FMADDPS": {mode: amd64FourSourceFMA}, + "V4FMADDSS": {mode: amd64FourSourceFMA, scalar: true}, + "V4FNMADDPS": {mode: amd64FourSourceFMA, negative: true}, + "V4FNMADDSS": {mode: amd64FourSourceFMA, scalar: true, negative: true}, + "VP4DPWSSD": {mode: amd64FourSourceDotWords}, + "VP4DPWSSDS": {mode: amd64FourSourceDotWords, saturating: true}, +} + +type amd64FourSourceForm struct { + spec amd64FourSourceSpec + memory Operand + sourceBase int + destination Operand + maskIndex int + zeroing bool +} + +func (form amd64FourSourceForm) byteWidth() int { + if form.spec.scalar { + return 16 + } + return 64 +} + +func (form amd64FourSourceForm) inputBytes() int64 { + if form.spec.scalar { + return 4 + } + return 64 +} + +func (form amd64FourSourceForm) source(index int) Operand { + bank := "Z" + if form.spec.scalar { + bank = "X" + } + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", bank, form.sourceBase+index))} +} + +func x86FourSourceProperties(op Op, ins Instr, arch string) (amd64FourSourceForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64FourSourceSpecs[Op(base)] + form := amd64FourSourceForm{spec: spec, maskIndex: -1, zeroing: suffix == "Z"} + if !known { + return form, false, nil + } + fail := func() (amd64FourSourceForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go four-source grammar: %q", arch, op, ins.Raw) + } + if arch != "386" && arch != "amd64" || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || + dotted && suffix == "" || suffix != "" && suffix != "Z" || len(ins.Args) != 3 && len(ins.Args) != 4 || + arch == "386" && len(ins.Args) == 4 || suffix == "Z" && len(ins.Args) != 4 { + return fail() + } + form.memory, form.destination = ins.Args[0], ins.Args[len(ins.Args)-1] + if !isAMD64MemoryOperand(form.memory) || !x86PackedShiftMemoryValid(form.memory, arch) || + x86NamedMultiplyMemoryError(form.memory, arch) != nil { + return fail() + } + if form.memory.Kind == OpMem && form.memory.Mem.IndexExt != "" { + return fail() + } + rangeOperand := ins.Args[1] + if rangeOperand.Kind != OpRegList || !rangeOperand.RegListRange || len(rangeOperand.RegList) != 4 { + return fail() + } + first := -1 + for position, register := range rangeOperand.RegList { + index, valid := amd64VectorRegisterIndex(register, form.byteWidth()) + if !valid || arch == "386" && index >= 8 || position != 0 && index != first+position { + return fail() + } + if position == 0 { + first = index + } + } + index, valid := amd64VectorRegisterIndex(form.destination.Reg, form.byteWidth()) + if form.destination.Kind != OpReg || !valid || arch == "386" && index >= 8 { + return fail() + } + if len(ins.Args) == 4 { + if !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + form.maskIndex = 2 + } + // Hardware selects the aligned block containing the encoded first + // register, not necessarily the literal spelling's four registers. + form.sourceBase = first &^ 3 + return form, true, nil +} + +func x86TypedFourSourceByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86FourSourceProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: 0, operand: form.memory, width: 16, read: true}, + {index: len(ins.Args) - 1, operand: form.destination, width: int64(form.byteWidth()), read: true, write: true}, + }} + for position := 0; position < 4; position++ { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: 1, operand: form.source(position), width: form.inputBytes(), read: true, + }) + } + if form.maskIndex >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: form.maskIndex, operand: ins.Args[form.maskIndex], width: 8, read: true, + }) + } + return effect, true, nil +} diff --git a/amd64_fp16_binary_forms.go b/amd64_fp16_binary_forms.go new file mode 100644 index 000000000..f8bdd11a2 --- /dev/null +++ b/amd64_fp16_binary_forms.go @@ -0,0 +1,99 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86FP16BinaryForm struct { + x86FloatBinaryForm + spec amd64BinaryFloatingSpec + sae bool +} + +// The twelve raw MAP5 binary members share the existing floating operand +// grammar. MIN/MAX allow SAE, not arithmetic's embedded rounding modes. +func x86FP16BinaryProperties(ins Instr, arch string) (x86FP16BinaryForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := amd64BinaryFloatingSpecs[Op(base)] + if !known || !spec.rawOnly || spec.mode == amd64BinaryFloatingSqrt { + return x86FP16BinaryForm{}, false, nil + } + controls, valid := parseAMD64BinaryFloatingSuffix(suffix) + f := x86FP16BinaryForm{spec: spec, sae: controls.sae} + fail := func() (x86FP16BinaryForm, bool, error) { + return f, true, fmt.Errorf("%s %s has invalid raw FP16 binary operands or controls: %q", arch, ins.Op, ins.Raw) + } + if !ins.x86Encoded || !valid || dotted && suffix == "" || + spec.sae && controls.rounding != "" || !spec.sae && controls.sae { + return fail() + } + shape := ins + if controls.sae { + shape.Op = Op(base) + if controls.zeroing { + shape.Op += ".Z" + } + } + var err error + f.x86FloatBinaryForm, err = x86FloatBinaryOperandProperties(shape, arch, 16, false, spec.scalar, false, "FP16 binary") + if err != nil { + return f, true, err + } + if controls.sae && (ins.Args[0].Kind != OpReg || !spec.scalar && f.byteWidth != 64) { + return fail() + } + if ins.Args[0].Kind == OpMem { + memory := ins.Args[0].Mem + if err := x86SymbolIndexedMemoryError(memory, arch); err != nil { + return f, true, err + } + if memory.OffRaw != "" || memory.IndexExt != "" { + return fail() + } + } + return f, true, x86Mode32VectorRegisterContext(ins, arch) +} + +func (c *amd64Ctx) lowerFP16Binary(op Op, ins Instr) (bool, bool, error) { + local := ins + local.Op = op + f, known, err := x86FP16BinaryProperties(local, c.goarch) + if !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(op), string(ins.Op)) { + return true, false, fmt.Errorf("FP16 binary dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + if err != nil { + return true, false, err + } + ins = local + controls := amd64FMA3Suffix{broadcast: f.broadcast, zeroing: f.zeroing, rounding: f.rounding} + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + lanes := f.byteWidth / 2 + if f.scalar { + lanes = 1 + } + return true, false, c.emitNativeFP16ThreeOperand(ins, x86NativeFP16ThreeOperand{ + byteWidth: f.byteWidth, lanes: lanes, memoryBytes: f.memoryBytes, + second: f.second, destination: f.destination, mask: f.mask, + controls: controls, sae: f.sae, + }) + } + // Keep cross-target numerical values separate from original x86 effects. + mask := "" + if f.mask >= 0 { + mask, err = c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + } + properties := amd64BinaryFloatingSuffix{ + broadcast: f.broadcast, zeroing: f.zeroing, rounding: f.rounding, sae: f.sae, + } + if f.scalar { + return c.lowerScalarBinaryFloating(f.spec, properties, ins, ins.Args[f.destination], mask) + } + return c.lowerPackedBinaryFloating(f.spec, properties, ins, ins.Args[f.destination], f.byteWidth, mask) +} diff --git a/amd64_fp16_conversion_forms.go b/amd64_fp16_conversion_forms.go new file mode 100644 index 000000000..7c4bb11c9 --- /dev/null +++ b/amd64_fp16_conversion_forms.go @@ -0,0 +1,96 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// The six raw-only FP16 floating conversions share source, mask and control +// axes. Physical carrier widths are distinct from active input/output bytes. +type x86FP16ConversionForm struct { + spec amd64HalfConversionSpec + controls amd64FMA3Suffix + sae bool + lanes, inputBytes, outputBytes int + sourceBytes, byteWidth, memoryBytes int + destination, mask, upper int +} + +func x86FP16ConversionProperties(ins Instr, arch string) (x86FP16ConversionForm, bool, error) { + base, suffix, dotted := strings.Cut(string(ins.Op), ".") + spec, known := amd64HalfConversionSpecs[base] + if !known || !spec.rawOnly { + return x86FP16ConversionForm{}, false, nil + } + f := x86FP16ConversionForm{spec: spec, destination: len(ins.Args) - 1, mask: -1, upper: -1} + fail := func() (x86FP16ConversionForm, bool, error) { + return f, true, fmt.Errorf("%s %s has invalid raw FP16 conversion operands or controls: %q", arch, ins.Op, ins.Raw) + } + if arch != "386" && arch != "amd64" || !ins.x86Encoded || dotted && suffix == "" { + return fail() + } + operands := 2 + f.lanes = ins.x86VectorBytes / 4 + if spec.scalar { + operands, f.lanes, f.upper = 3, 1, 1 + } else if ins.x86VectorBytes != 16 && ins.x86VectorBytes != 32 && ins.x86VectorBytes != 64 { + return fail() + } + if len(ins.Args) != operands && len(ins.Args) != operands+1 { + return fail() + } + f.sae = suffix == "SAE" || suffix == "SAE.Z" + if f.sae { + suffix = strings.TrimPrefix(strings.TrimPrefix(suffix, "SAE"), ".") + } + var valid bool + f.controls, valid = parseAMD64FMA3Suffix(suffix) + if !valid || spec.inputBits == 16 && f.controls.rounding != "" || spec.outputBits == 16 && f.sae || + spec.scalar && f.controls.broadcast { + return fail() + } + if len(ins.Args) == operands+1 { + f.mask = operands - 1 + if !amd64NonzeroKOperand(ins.Args[f.mask]) { + return fail() + } + } else if f.controls.zeroing { + return fail() + } + f.inputBytes, f.outputBytes = f.lanes*spec.inputBits/8, f.lanes*spec.outputBits/8 + f.sourceBytes, f.byteWidth = f.inputBytes, f.outputBytes + if f.sourceBytes < 16 { + f.sourceBytes = 16 + } + if f.byteWidth < 16 { + f.byteWidth = 16 + } + if !amd64GoPackedVectorMoveRegister(ins.Args[f.destination], f.byteWidth, arch) || + f.upper >= 0 && !amd64GoPackedVectorMoveRegister(ins.Args[f.upper], 16, arch) { + return fail() + } + source := ins.Args[0] + if source.Kind == OpReg { + if !amd64GoPackedVectorMoveRegister(source, f.sourceBytes, arch) || f.controls.broadcast { + return fail() + } + } else if !x86PackedShiftMemoryValid(source, arch) || f.sae || f.controls.rounding != "" { + return fail() + } + if source.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(source.Mem, arch); err != nil { + return f, true, err + } + if source.Mem.OffRaw != "" || source.Mem.IndexExt != "" { + return fail() + } + } + if !spec.scalar && (f.sae || f.controls.rounding != "") && ins.x86VectorBytes != 64 { + return fail() + } + f.memoryBytes = f.inputBytes + if f.controls.broadcast { + f.memoryBytes = spec.inputBits / 8 + } + return f, true, x86Mode32VectorRegisterContext(ins, arch) +} diff --git a/amd64_fp16_conversion_native.go b/amd64_fp16_conversion_native.go new file mode 100644 index 000000000..9b85e40e4 --- /dev/null +++ b/amd64_fp16_conversion_native.go @@ -0,0 +1,40 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) emitNativeFP16Conversion(ins Instr, f x86FP16ConversionForm) error { + source := ins.Args[0] + if source.Kind == OpFP || ins.x86AddressBits != 0 || source.Kind == OpMem && + !c.nativeMemoryOperand(ins, 0) { + return fmt.Errorf("%w: FP16 conversion needs original native backing", ErrProbeNeedsContext) + } + if f.spec.scalar { + return c.emitNativeFP16ThreeOperand(ins, x86NativeFP16ThreeOperand{ + byteWidth: 16, lanes: 1, memoryBytes: f.memoryBytes, second: f.upper, + destination: f.destination, mask: f.mask, + controls: f.controls, sae: f.sae, + }) + } + mnemonic, _, _ := strings.Cut(strings.ToLower(string(ins.Op)), ".") + if source.Kind != OpReg && f.spec.outputBits == 16 && !f.controls.broadcast && f.sourceBytes < 64 { + mnemonic += map[int]string{16: "x", 32: "y"}[f.sourceBytes] + } + return c.emitNativePackedConversion(ins, x86NativePackedConversion{ + byteWidth: f.byteWidth, sourceBytes: f.sourceBytes, memoryBytes: f.memoryBytes, + outputBits: 32, // Bit carrier only: the original opcode selects FP16/FP32. + destination: f.destination, mask: f.mask, lanes: f.lanes, evex: true, + broadcast: f.controls.broadcast, zeroing: f.controls.zeroing, + rounding: f.controls.rounding, sae: f.sae, mnemonic: mnemonic, + }) +} + +// Existing FP16 clients retain their own grammar and source permissions. Only +// the actual native operand transport is shared with other floating families. +type x86NativeFP16ThreeOperand = x86NativeFloatingOperands + +func (c *amd64Ctx) emitNativeFP16ThreeOperand(ins Instr, f x86NativeFP16ThreeOperand) error { + return c.emitNativeFloatingOperands(ins, f) +} diff --git a/amd64_fp_address.go b/amd64_fp_address.go new file mode 100644 index 000000000..0724d17be --- /dev/null +++ b/amd64_fp_address.go @@ -0,0 +1,56 @@ +package plan9asm + +import "fmt" + +// boundFPAddress addresses a declared field's backing storage, never its +// current value. An unbound source address needs a physical frame contract; +// inventing a zero pointer or extending an alloca into padding is not one. +func (c *amd64Ctx) boundFPAddress(off int64) (string, error) { + slot, result, ok := c.fpVectorSlot(off) + if !ok { + return "", fmt.Errorf("%w: %s FP address at +%d(FP) requires bound typed frame storage", ErrProbeNeedsContext, c.goarch, off) + } + if x86FrameScalarBytes(slot.Type, c.goarch) == 0 { + // The vector lookup's conservative aggregate-size fallback is not a + // storage-width proof. Scalar bool fields use canonical i8 backing. + return "", fmt.Errorf("%w: %s FP address at +%d(FP) lacks a scalar byte-storage contract for %s", ErrProbeNeedsContext, c.goarch, off, slot.Type) + } + if result { + c.markFPResultAddrTaken(slot.Offset) + } + var pointer string + if c.classicFrame != "" { + pointer = c.classicFramePtr(off) + } else { + if result { + pointer, _, ok = c.fpResultAlloca(slot.Offset) + } else { + pointer = c.fpParamAlloca[slot.Offset] + ok = pointer != "" + } + if !ok { + return "", fmt.Errorf("%w: %s FP address at +%d(FP) has no backing storage", ErrProbeNeedsContext, c.goarch, off) + } + if off != slot.Offset { + adjusted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr inbounds i8, ptr %s, i64 %d\n", adjusted, pointer, off-slot.Offset) + pointer = "%" + adjusted + } + } + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", address, pointer) + return "%" + address, nil +} + +func x86FrameScalarBytes(typ LLVMType, goarch string) int64 { + switch typ { + case I1, I8, I16, I32, I64, Ptr, LLVMType("float"), LLVMType("double"): + pointerSize := int64(8) + if goarch == "386" { + pointerSize = 4 + } + return frameTypeSize(typ, pointerSize) + default: + return 0 + } +} diff --git a/amd64_fp_address_test.go b/amd64_fp_address_test.go new file mode 100644 index 000000000..65240549c --- /dev/null +++ b/amd64_fp_address_test.go @@ -0,0 +1,93 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86LEAFPAddressesRequireBoundStorage(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + widths := []string{"W", "L"} + if target.arch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + for _, prefix := range []string{"", "$"} { + for _, offset := range []int64{0, 1, 8, 9, 4, 12} { + t.Run(fmt.Sprintf("%s/%s/%s/%d", target.triple, width, prefix, offset), func(t *testing.T) { + // Storage-address validity is independent of result + // initialization. The original unwritten-result source is + // retained as a separate complete negative regression. + source := fmt.Sprintf("TEXT Address(SB),4,$0-16\nMOVL $7,r+8(FP)\nLEA%s %sr+%d(FP),AX\nRET\n", width, prefix, offset) + requireX86GoAssemblerResult(t, target.arch, source, prefix == "") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Address": { + Name: "Address", Args: []LLVMType{I32}, Ret: I32, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I32, Index: 0, Field: -1, Name: "r"}}, + }, + }}, + } + ir, err := Translate(file, opt) + if prefix == "$" { + if err == nil { + t.Fatal("accepted TYPE_ADDR outside Go's LEA Ym/Yrl row") + } + return + } + if offset == 4 || offset == 12 { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unbound FP address became success/zero: %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "ptrtoint ptr") { + t.Fatalf("LEA used the parameter value instead of its storage:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "fp-address.ll", "fp-address.o", ir) + }) + } + } + } + } +} + +func TestX86LEAFPAddressCannotInventScalarBacking(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + lea, triple := "LEAQ", "x86_64-unknown-linux-gnu" + if arch == "386" { + lea, triple = "LEAL", "i386-unknown-linux-gnu" + } + for _, typ := range []LLVMType{"{ i8 }", "{}", "<8 x i8>"} { + t.Run(arch+"/"+string(typ), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT Address(SB),4,$0-16\n"+lea+" x+0(FP),AX\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"Address": { + Name: "Address", Args: []LLVMType{typ}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}}}, + }}, + }) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("address lacks a scalar/canonical-byte contract, got %v", err) + } + }) + } + } +} diff --git a/amd64_fp_bool_storage_test.go b/amd64_fp_bool_storage_test.go new file mode 100644 index 000000000..1a8981bb1 --- /dev/null +++ b/amd64_fp_bool_storage_test.go @@ -0,0 +1,213 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// A Go bool occupies one canonical byte in ABI0 memory, even though its LLVM +// argument and result have type i1. These sources exercise the actual byte +// backing, including indirect writes through LEA, not just the SSA value. +func x86BoolFrameSource(arch string) string { + lea, word := "LEAQ", 8 + if arch == "386" { + lea, word = "LEAL", 4 + } + source := fmt.Sprintf(`TEXT ·ReadByte(SB),4,$0-%d +%s x+0(FP),AX +MOVB (AX),BX +MOVB BX,ret+%d(FP) +RET +TEXT ·WriteParam(SB),4,$0-%d +%s x+0(FP),AX +MOVB $1,(AX) +MOVB x+0(FP),BX +MOVB BX,ret+%d(FP) +RET +TEXT ·WriteResult(SB),4,$0-%d +MOVB x+0(FP),BX +%s ret+%d(FP),AX +MOVB BX,(AX) +RET +TEXT ·WriteDirect(SB),4,$0-%d +MOVB $0,x+0(FP) +MOVB x+0(FP),BX +MOVB BX,ret+%d(FP) +RET +`, word+1, lea, word, word+1, lea, word, word+1, lea, word, word+1, word) + source += fmt.Sprintf(`TEXT ·BoolCallee(SB),4,$0-%d +%s x+0(FP),AX +MOVB (AX),BX +MOVB BX,ret+%d(FP) +RET +TEXT ·BoolCaller(SB),4,$16-%d +MOVB x+0(FP),AX +MOVB AX,0(SP) +CALL ·BoolCallee(SB) +MOVB %d(SP),AX +MOVB AX,ret+%d(FP) +RET +TEXT ·BoolField(SB),4,$0-%d +%s p_flag+1(FP),AX +MOVB (AX),BX +MOVB BX,ret+%d(FP) +RET +`, word+1, lea, word, word+1, word, word, word+1, lea, word) + return source +} + +func x86BoolFrameIR(t *testing.T, arch, triple string) string { + t.Helper() + source := x86BoolFrameSource(arch) + requireX86GoAssemblerResult(t, arch, source, true) + pkg := mustGoPackage(t, "example.com/boolframe", `package boolframe +func ReadByte(x bool) uint8 +func WriteParam(x bool) bool +func WriteResult(x bool) bool +func WriteDirect(x bool) bool +func BoolCallee(x bool) bool +func BoolCaller(x bool) bool +func BoolField(p struct { Pad uint8; Flag bool }) bool +`) + ctx := llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: arch, TargetTriple: triple, + ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + ir := translation.Module.String() + for _, unsafeBacking := range []string{ + "%fp_arg_0 = alloca i1", "%fp_ret_0 = alloca i1", + } { + if strings.Contains(ir, unsafeBacking) { + t.Fatalf("Go bool uses a sub-byte backing store: %s", unsafeBacking) + } + } + // This LLVM caller keeps the exact aggregate type. A C struct parameter + // would use C's ABI coercions and is not an oracle for this LLVM boundary. + return ir + ` +define i1 @BoolFieldWrapper(i1 %flag) { +entry: + %pad = insertvalue { i8, i1 } undef, i8 17, 0 + %input = insertvalue { i8, i1 } %pad, i1 %flag, 1 + %result = call i1 @BoolField({ i8, i1 } %input) + ret i1 %result +} +` +} + +func TestX86BoolFrameCanonicalByteObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + ir := x86BoolFrameIR(t, target.arch, target.triple) + compileLLVMToObject(t, llc, target.triple, "bool-frame.ll", "bool-frame.o", ir) + }) + } +} + +func x86BoolFrameGoOracle(t *testing.T, arch string, runPrefix []string) { + t.Helper() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module example.com/boolframe\n\ngo 1.20\n", + "bool_" + arch + ".s": x86BoolFrameSource(arch), + "bool_test.go": `package boolframe +import "testing" +func ReadByte(x bool) uint8 +func WriteParam(x bool) bool +func WriteResult(x bool) bool +func WriteDirect(x bool) bool +func BoolCallee(x bool) bool +func BoolCaller(x bool) bool +func BoolField(p struct { Pad uint8; Flag bool }) bool +func TestCanonicalBytes(t *testing.T) { + for i := 0; i < 1024; i++ { + for _, x := range []bool{false, true} { + want := uint8(0) + if x { want = 1 } + if ReadByte(x) != want || !WriteParam(x) || WriteResult(x) != x || WriteDirect(x) { + t.Fatalf("canonical Go bool frame failed for %v", x) + } + if BoolCallee(x) != x || BoolCaller(x) != x || BoolField(struct { Pad uint8; Flag bool }{17, x}) != x { + t.Fatalf("canonical Go bool call/aggregate failed for %v", x) + } + } + } +} +`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + args := []string{"test", "-vet=off", "-count=1"} + if len(runPrefix) != 0 { + args = append(args, "-exec", strings.Join(runPrefix, " ")) + } + cmd := exec.Command("go", append(args, ".")...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOWORK=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0", "GOARCH="+arch) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("Go %s bool byte oracle: %v\n%s", arch, err, out) + } +} + +func TestCrossLinuxRuntimeMatrixX86BoolFrameCanonicalBytes(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("x86 byte ABI oracle runs in the required Linux/amd64 cross-runtime gate") + } + x86BoolFrameGoOracle(t, "amd64", nil) + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var runPrefix []string + if crossRosetta { + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + const main = `#include +#include +extern uint8_t ReadByte(bool); +extern bool WriteParam(bool), WriteResult(bool), WriteDirect(bool); +extern bool BoolCallee(bool), BoolCaller(bool), BoolFieldWrapper(bool); +int main(void) { + for (unsigned i = 0; i < 1024; i++) { + for (unsigned x = 0; x < 2; x++) { + volatile uint64_t canary = UINT64_C(0x123456789abcdef0); + if (ReadByte(x) != x || !WriteParam(x) || WriteResult(x) != x || WriteDirect(x)) return 1; + if (canary != UINT64_C(0x123456789abcdef0)) return 2; + if (BoolCallee(x) != x || BoolCaller(x) != x || BoolFieldWrapper(x) != x) return 3; + } + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "bool_frame", triple, + x86BoolFrameIR(t, "amd64", triple), main, runPrefix) + if os.Getenv("PLAN9ASM_CROSS_EXEC") == "1" { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("PLAN9ASM_CROSS_EXEC bool matrix requires Linux/amd64") + } + prefix := []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"} + x86BoolFrameGoOracle(t, "386", prefix) + compileAndRunRuntimeTestWithCompiler(t, llc, []string{"i686-linux-gnu-gcc", "-no-pie"}, + "bool_frame386", "i386-unknown-linux-gnu", x86BoolFrameIR(t, "386", "i386-unknown-linux-gnu"), main, prefix) + } +} diff --git a/amd64_fp_storage.go b/amd64_fp_storage.go new file mode 100644 index 000000000..8d8991f9a --- /dev/null +++ b/amd64_fp_storage.go @@ -0,0 +1,33 @@ +package plan9asm + +import "fmt" + +// Go ABI0 bool fields occupy a whole canonical byte. LLVM's i1 describes the +// logical argument, not byte-addressable storage: writing i1 alone does not +// establish the other seven bits for subsequent assembly byte loads. +func x86FPStorageType(typ LLVMType) LLVMType { + if typ == I1 { + return I8 + } + return typ +} + +func (c *amd64Ctx) storeFPStorage(typ LLVMType, value, pointer, alignment, metadata string) { + if typ == I1 { + byteValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to i8\n", byteValue, value) + value = "%" + byteValue + } + fmt.Fprintf(c.b, " store %s %s, ptr %s%s%s\n", x86FPStorageType(typ), value, pointer, alignment, metadata) +} + +func (c *amd64Ctx) loadFPStorage(typ LLVMType, pointer, alignment string) string { + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s%s\n", loaded, x86FPStorageType(typ), pointer, alignment) + if typ == I1 { + logical := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i8 %%%s to i1\n", logical, loaded) + return "%" + logical + } + return "%" + loaded +} diff --git a/amd64_frame_input_digest_test.go b/amd64_frame_input_digest_test.go new file mode 100644 index 000000000..2d7d586ed --- /dev/null +++ b/amd64_frame_input_digest_test.go @@ -0,0 +1,109 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "errors" + "os" + "path/filepath" + "runtime" + "testing" +) + +// Private verifier unit only. This does not construct a public source/frame +// observation, actual compiler selection or an ASM permission. +func frameInputDigestFixture(t *testing.T, size int) (*GoAMD64FrameSource, string, byte) { + t.Helper() + directory, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + name := filepath.Join(directory, "input") + data := bytes.Repeat([]byte{0x5a}, size) + if err := os.WriteFile(name, data, 0600); err != nil { + t.Fatal(err) + } + info, err := os.Lstat(directory) + if err != nil { + t.Fatal(err) + } + source := &GoAMD64FrameSource{request: GoAMD64FrameCaptureRequest{Directory: directory}, + owned: directory, ownedInfo: info, directoryInfo: info, + entries: []string{"input"}, inputs: map[string][32]byte{name: sha256.Sum256(data)}} + if err := source.verifyInputs(); err != nil { + t.Fatal("unchanged complete verifier control failed", err) + } + return source, name, 0x5a +} + +func TestGoAMD64FrameInputVerificationHasBoundedAllocations(t *testing.T) { + source, _, _ := frameInputDigestFixture(t, 8<<20) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + for round := 0; round < 3; round++ { + if err := source.verifyInputs(); err != nil { + t.Fatal(err) + } + } + runtime.ReadMemStats(&after) + allocated := after.TotalAlloc - before.TotalAlloc + t.Logf("three complete 8-MiB file verifications allocated %d bytes", allocated) + if allocated > 4<<20 { + t.Fatal("digest-only verification allocated whole-file buffers", allocated) + } +} + +func TestGoAMD64FrameInputVerificationReadsTailDespiteSameMetadata(t *testing.T) { + source, name, original := frameInputDigestFixture(t, 1<<20) + info, err := os.Stat(name) + if err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(name, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + _, writeErr := file.WriteAt([]byte{original ^ 1}, info.Size()-1) + if err := errors.Join(writeErr, file.Close()); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(name, info.ModTime(), info.ModTime()); err != nil { + t.Fatal(err) + } + if err := source.verifyInputs(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("same-size same-time tail mutation escaped complete content verification", err) + } +} + +func TestGoAMD64FrameInputVerificationRetainsInputRole(t *testing.T) { + for _, mutation := range []string{"missing", "directory", "growth", "truncation"} { + t.Run(mutation, func(t *testing.T) { + source, name, _ := frameInputDigestFixture(t, 128) + switch mutation { + case "missing": + if err := os.Remove(name); err != nil { + t.Fatal(err) + } + case "directory": + if err := os.Remove(name); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(name, 0700); err != nil { + t.Fatal(err) + } + case "growth": + if err := os.Truncate(name, 129); err != nil { + t.Fatal(err) + } + case "truncation": + if err := os.Truncate(name, 127); err != nil { + t.Fatal(err) + } + } + if err := source.verifyInputs(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed physical input retained its verifier role", err) + } + }) + } +} diff --git a/amd64_frame_vector.go b/amd64_frame_vector.go new file mode 100644 index 000000000..d170435ad --- /dev/null +++ b/amd64_frame_vector.go @@ -0,0 +1,231 @@ +package plan9asm + +import "fmt" + +// fpVectorSlot locates the scalar ABI0 field containing one byte of a vector +// memory operand. Aggregate parameters have several independently allocated +// fields on amd64; treating the first field as a contiguous pointer is wrong. +func (c *amd64Ctx) fpVectorSlot(off int64) (FrameSlot, bool, bool) { + for _, slot := range c.sig.Frame.Params { + size := x86FrameTypeSize(slot.Type) + if slot.Type == Ptr { + size = int64(goWordSize(c.goarch)) + } + if off >= slot.Offset && off < slot.Offset+size { + return slot, false, true + } + } + for _, slot := range c.fpResults { + size := x86FrameTypeSize(slot.Type) + if slot.Type == Ptr { + size = int64(goWordSize(c.goarch)) + } + if off >= slot.Offset && off < slot.Offset+size { + return slot, true, true + } + } + return FrameSlot{}, false, false +} + +func (c *amd64Ctx) validateFPVectorSpan(off int64, byteWidth int) error { + for i := 0; i < byteWidth; i++ { + if _, _, ok := c.fpVectorSlot(off + int64(i)); !ok { + return fmt.Errorf("undeclared FP vector byte at +%d(FP)", off+int64(i)) + } + } + return nil +} + +func (c *amd64Ctx) loadFPVectorBytes(off int64, byteWidth int) (string, error) { + return c.loadFPVectorBytesMode(off, byteWidth, false) +} + +// Integer byte views may observe an intermediate noncanonical bool byte, but +// only after their independent whole-function canonical result proof. Existing +// SIMD views keep their old logical-field policy and never acquire this mode. +func (c *amd64Ctx) loadFPVectorField(slot FrameSlot, result, physicalBool bool) (string, LLVMType, error) { + if physicalBool && slot.Type == I1 { + if !c.abi0BoolResultByteStorage() { + return "", "", fmt.Errorf("%w: FP bool byte lacks a checked storage role", ErrProbeNeedsContext) + } + pointer := c.fpParamAlloca[slot.Offset] + if result { + pointer, _, _ = c.fpResultAlloca(slot.Offset) + } + if c.classicFrame != "" { + pointer = c.classicFramePtr(slot.Offset) + } + if pointer == "" { + return "", "", fmt.Errorf("missing physical FP bool storage at +%d(FP)", slot.Offset) + } + return c.loadFPStorage(I8, pointer, ", align 1"), I8, nil + } + if result { + value, err := c.loadFPResult(slot) + return value, slot.Type, err + } + value, err := c.loadFPParamValue(slot) + return value, slot.Type, err +} + +func (c *amd64Ctx) loadFPVectorBytesMode(off int64, byteWidth int, physicalBool bool) (string, error) { + if err := c.validateFPVectorSpan(off, byteWidth); err != nil { + return "", err + } + value := "zeroinitializer" + words := make(map[int64]string) + for i := 0; i < byteWidth; i++ { + slot, result, _ := c.fpVectorSlot(off + int64(i)) + word, found := words[slot.Offset] + if !found { + scalar, typ, err := c.loadFPVectorField(slot, result, physicalBool) + if err != nil { + return "", err + } + word, err = c.fpVectorScalarBits(typ, scalar) + if err != nil { + return "", err + } + words[slot.Offset] = word + } + shift := (off + int64(i) - slot.Offset) * 8 + if shift != 0 { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", shifted, word, shift) + word = "%" + shifted + } + part := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", part, word) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i8> %s, i8 %%%s, i32 %d\n", inserted, byteWidth, value, part, i) + value = "%" + inserted + } + return value, nil +} + +func (c *amd64Ctx) fpVectorScalarBits(typ LLVMType, scalar string) (string, error) { + switch typ { + case I64: + return scalar, nil + case Ptr: + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", converted, scalar) + return "%" + converted, nil + case LLVMType("double"): + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", converted, scalar) + return "%" + converted, nil + case LLVMType("float"): + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", bits, scalar) + scalar = "%" + bits + typ = I32 + } + bits, ok := amd64IntegerTypeBits(typ) + if !ok { + return "", fmt.Errorf("unsupported FP vector field type %s", typ) + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %s to i64\n", wide, bits, scalar) + return "%" + wide, nil +} + +func (c *amd64Ctx) storeFPVectorBytes(off int64, byteWidth int, value, metadata string) error { + return c.storeFPVectorBytesMode(off, byteWidth, value, metadata, false) +} + +func (c *amd64Ctx) storeFPVectorBytesMode(off int64, byteWidth int, value, metadata string, physicalBool bool) error { + if err := c.validateFPVectorSpan(off, byteWidth); err != nil { + return err + } + if physicalBool && !c.abi0BoolResultByteStorage() { + for i := 0; i < byteWidth; i++ { + if slot, _, _ := c.fpVectorSlot(off + int64(i)); slot.Type == I1 { + return fmt.Errorf("%w: FP bool byte lacks a checked storage role", ErrProbeNeedsContext) + } + } + } + if c.classicFrame != "" { + for i := 0; i < byteWidth; i++ { + part := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", part, byteWidth, value, i) + fmt.Fprintf(c.b, " store i8 %%%s, ptr %s, align 1%s\n", part, c.classicFramePtr(off+int64(i)), metadata) + if slot, result, ok := c.fpVectorSlot(off + int64(i)); ok && result { + c.markFPResultWritten(slot.Offset) + } + } + return nil + } + + updated := make(map[int64]string) + order := make([]FrameSlot, 0, byteWidth/4) + for i := 0; i < byteWidth; i++ { + current := off + int64(i) + slot, result, _ := c.fpVectorSlot(current) + bits, found := updated[slot.Offset] + if !found { + scalar, typ, err := c.loadFPVectorField(slot, result, physicalBool) + if err != nil { + return err + } + bits, err = c.fpVectorScalarBits(typ, scalar) + if err != nil { + return err + } + order = append(order, slot) + } + part := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", part, byteWidth, value, i) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", wide, part) + shift := (current - slot.Offset) * 8 + inserted := "%" + wide + if shift != 0 { + tmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i64 %s, %d\n", tmp, inserted, shift) + inserted = "%" + tmp + } + cleared := c.newTmp() + mask := ^(uint64(0xff) << uint(shift)) + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", cleared, bits, mask) + merged := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %s\n", merged, cleared, inserted) + updated[slot.Offset] = "%" + merged + } + for _, slot := range order { + bits := updated[slot.Offset] + converted := bits + typ := slot.Type + if physicalBool && typ == I1 { + typ = I8 + } + switch typ { + case I64: + case Ptr: + tmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", tmp, bits) + converted = "%" + tmp + case LLVMType("double"): + tmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to double\n", tmp, bits) + converted = "%" + tmp + case LLVMType("float"): + narrow, tmp := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", narrow, bits) + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", tmp, narrow) + converted = "%" + tmp + default: + width, ok := amd64IntegerTypeBits(typ) + if !ok || width > 64 { + return fmt.Errorf("unsupported FP vector field type %s", slot.Type) + } + tmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", tmp, bits, width) + converted = "%" + tmp + } + if err := c.storeFPResultWithMetadata(slot.Offset, typ, converted, metadata); err != nil { + return err + } + } + return nil +} diff --git a/amd64_gfni_forms.go b/amd64_gfni_forms.go new file mode 100644 index 000000000..884100d25 --- /dev/null +++ b/amd64_gfni_forms.go @@ -0,0 +1,77 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86GFNIForm struct { + x86PackedBinaryForm + spec amd64GFNISpec + immediate uint8 +} + +func x86GFNIProperties(op Op, ins Instr, arch string) (x86GFNIForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64GFNISpecs[Op(base)] + form := x86GFNIForm{spec: spec} + if !known { + return form, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return form, true, fmt.Errorf("GFNI dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + offset := 0 + if spec.immediate { + if len(ins.Args) == 0 || !amd64UnsignedImmediate(ins.Args[0], 8) { + return form, true, fmt.Errorf("%s %s requires a resolved unsigned imm8: %q", arch, op, ins.Raw) + } + if arch == "386" && !ins.x86Encoded { + return form, true, fmt.Errorf("386 %s exceeds cmd/asm's named operand limit: %q", op, ins.Raw) + } + form.immediate, offset = uint8(ins.Args[0].Imm), 1 + local.Args = ins.Args[1:] + } + local, contextual := x86VectorValueMemoryInput(local, arch) + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true} + if spec.broadcast { + grammar.broadcastBytes = 8 + } + var err error + form.x86PackedBinaryForm, err = x86PackedBinaryOperandGrammar(local, arch, grammar, ins.x86Encoded) + if err != nil { + return form, true, err + } + form.first += offset + form.second += offset + form.destination += offset + if form.mask >= 0 { + form.mask += offset + } + if form.memory >= 0 { + form.memory += offset + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && amd64VectorByteWidth(operand.Reg) != 0 && + !amd64GoPackedVectorMoveRegister(operand, form.byteWidth, arch) { + return form, true, fmt.Errorf("%s %s has an unbound physical vector register: %q", arch, op, ins.Raw) + } + } + if form.memory >= 0 && !contextual && !x86PackedShiftMemoryValid(ins.Args[form.memory], arch) { + return form, true, fmt.Errorf("%s %s has an unbound memory operand: %q", arch, op, ins.Raw) + } + if contextual { + return form, true, x86VectorValueMemoryContext(ins) + } + return form, true, nil +} + +func x86ResultGFNIStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, known, err := x86GFNIProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + return x86ResultFixedVectorInputs(ins, form.x86PackedBinaryForm, false, state, owned, arch), true +} diff --git a/amd64_go_argument_alias_go122_test.go b/amd64_go_argument_alias_go122_test.go new file mode 100644 index 000000000..e4113eb89 --- /dev/null +++ b/amd64_go_argument_alias_go122_test.go @@ -0,0 +1,141 @@ +//go:build go1.22 + +package plan9asm + +import ( + "errors" + "fmt" + "go/token" + "go/types" + "reflect" + "testing" +) + +func amd64ArgumentAlias(typ types.Type, count int) types.Type { + for index := 0; index < count; index++ { + typ = types.NewAlias(types.NewTypeName(token.NoPos, nil, "Alias", nil), typ) + } + return typ +} + +func amd64ArgumentAliasSignature(typ types.Type) *types.Signature { + return types.NewSignatureType(nil, nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "value", typ)), nil, false) +} + +func TestAMD64GoArgumentPlanAliasesPreserveConcreteIdentity(t *testing.T) { + for _, count := range []int{1, 2, 32} { + typ := amd64ArgumentAlias(types.Typ[types.Uint32], count) + plan, err := planAMD64GoArguments(amd64ArgumentAliasSignature(typ)) + if !amd64ArgumentAliasHasActualRHS(typ) { + if plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("opaque alias acquired a concrete argument plan at depth %d: %#v, %v", count, plan, err) + } + continue + } + if err != nil { + t.Fatal(err) + } + argument := plan.Arguments[0] + if argument.Type != typ || argument.Stack || len(argument.Carriers) != 1 || + argument.Carriers[0].Type != I32 || argument.Carriers[0].Width != 4 || argument.Carriers[0].Register != AX { + t.Fatalf("concrete alias/group identity was lost at depth %d: %#v", count, argument) + } + } + boolean := amd64ArgumentAlias(types.Typ[types.Bool], 2) + floating := amd64ArgumentAlias(types.Typ[types.Float32], 1) + aggregate := amd64ArgumentAlias(types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, nil, "B", boolean, false), + types.NewField(token.NoPos, nil, "F", floating, false), + }, nil), 2) + plan, err := planAMD64GoArguments(amd64ArgumentAliasSignature(aggregate)) + if !amd64ArgumentAliasHasActualRHS(aggregate) { + if plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("opaque alias descendants acquired concrete byte kinds: %#v, %v", plan, err) + } + return + } + if err != nil { + t.Fatal(err) + } + want := []amd64GoArgumentCarrier{ + {Path: []int{0}, Offset: 0, Width: 1, Type: I1, Kind: amd64GoArgumentBoolean, Register: AX}, + {Path: []int{1}, Offset: 4, Width: 4, Type: "float", Kind: amd64GoArgumentFloating, Register: "X0"}, + } + if plan.Arguments[0].Type != aggregate || !reflect.DeepEqual(plan.Arguments[0].Carriers, want) { + t.Fatalf("alias descendants changed their concrete byte kinds: %#v", plan) + } +} + +func TestAMD64GoArgumentPlanAliasesRejectUnresolvedAndUnboundedRHS(t *testing.T) { + parameter := types.NewTypeParam(types.NewTypeName(token.NoPos, nil, "T", nil), types.NewInterfaceType(nil, nil).Complete()) + generic := types.NewNamed(types.NewTypeName(token.NoPos, nil, "Generic", nil), types.Typ[types.Uint64], nil) + generic.SetTypeParams([]*types.TypeParam{types.NewTypeParam(types.NewTypeName(token.NoPos, nil, "U", nil), types.NewInterfaceType(nil, nil).Complete())}) + for _, test := range []struct { + name string + typ types.Type + }{ + {"type-parameter", amd64ArgumentAlias(parameter, 1)}, + {"alias-of-alias-type-parameter", amd64ArgumentAlias(parameter, 2)}, + {"generic-named-RHS", amd64ArgumentAlias(generic, 2)}, + {"alias-depth", amd64ArgumentAlias(types.Typ[types.Uint64], 33)}, + } { + t.Run(test.name, func(t *testing.T) { + plan, err := planAMD64GoArguments(amd64ArgumentAliasSignature(test.typ)) + if plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unsupported alias RHS acquired a partial plan: %#v, %v", plan, err) + } + }) + } +} + +func TestAMD64GoArgumentIssuerAliasesRetainResultBoundary(t *testing.T) { + for _, count := range []int{1, 2, 32} { + for _, scalar := range []types.Type{types.Typ[types.Bool], types.Typ[types.Uint8], types.Typ[types.Uint16], + types.Typ[types.Uint32], types.Typ[types.Uint64]} { + t.Run(fmt.Sprintf("%s/depth-%d", scalar, count), func(t *testing.T) { + input := amd64ArgumentEntryModel(t, "a float64") + result := amd64ArgumentAlias(scalar, count) + pkg := types.NewPackage(input.Package.Path(), input.Package.Name()) + signature := types.NewSignatureType(nil, nil, nil, + input.Declaration.Type().(*types.Signature).Params(), + types.NewTuple(types.NewVar(token.NoPos, nil, "r", result)), false) + input.Declaration = types.NewFunc(token.NoPos, pkg, "Entry", signature) + pkg.Scope().Insert(input.Declaration) + input.Package = pkg + var err error + input.Signature, err = DeriveGoFuncABI0Signature(input.Native, input.Declaration, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + language, err := numericalSDKDeclarationLanguage(input.Declaration, types.SizesFor("gc", "amd64"), true) + if !amd64ArgumentAliasHasActualRHS(result) { + if language != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("opaque scalar result alias acquired a language identity", language, err) + } + if plan, err := amd64GoArgumentEntryPlan(input); plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("opaque scalar result alias acquired an independent transport", plan, err) + } + return + } + if err != nil { + t.Fatal("concrete scalar result alias lost its language identity", err) + } + if _, err := amd64GoArgumentEntryPlan(input); err != nil { + t.Fatal("concrete scalar result alias lost its independent transport", err) + } + }) + } + } + for _, count := range []int{1, 2} { + calls := 0 + result := amd64ArgumentAlias(&amd64ArgumentCallbackType{calls: &calls}, count) + signature := types.NewSignatureType(nil, nil, nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "r", result)), false) + function := types.NewFunc(token.NoPos, nil, "Entry", signature) + calls = 0 // Only issuer sampling, not fixture construction, is observed. + if _, err := numericalSDKDeclarationLanguage(function, types.SizesFor("gc", "amd64"), true); !errors.Is(err, ErrProbeNeedsContext) || calls != 0 { + t.Fatalf("alias RHS reached caller type callbacks at depth %d: err=%v, calls=%d", count, err, calls) + } + } +} diff --git a/amd64_go_argument_alias_rhs_go122_test.go b/amd64_go_argument_alias_rhs_go122_test.go new file mode 100644 index 000000000..bc888ffb6 --- /dev/null +++ b/amd64_go_argument_alias_rhs_go122_test.go @@ -0,0 +1,76 @@ +//go:build go1.22 + +package plan9asm + +import ( + "errors" + "fmt" + "go/token" + "go/types" + "testing" +) + +// Go1.22 exposes Alias but not its immediate RHS. Underlying/Unalias cannot +// replace the independently bounded complete chain observed by these roles. +func amd64ArgumentAliasHasActualRHS(typ types.Type) bool { + _, available := typ.(interface{ Rhs() types.Type }) + return goAMD64FrameIsAlias(typ) && available +} + +func TestAMD64GoArgumentAliasConsumersRequireActualRHS(t *testing.T) { + sizes := types.SizesFor("gc", "amd64") + named := types.NewNamed(types.NewTypeName(token.NoPos, nil, "NamedWord", nil), types.Typ[types.Uint32], nil) + for _, count := range []int{1, 2, 32} { + for _, scalar := range []types.Type{types.Typ[types.Bool], types.Typ[types.Uint32], + types.Typ[types.Float32], types.Typ[types.Float64], named} { + t.Run(fmt.Sprintf("%s/depth-%d", scalar, count), func(t *testing.T) { + typ := amd64ArgumentAlias(scalar, count) + available := amd64ArgumentAliasHasActualRHS(typ) + check := func(role string, expectConcrete, complete bool, err error) { + t.Helper() + if expectConcrete { + if !complete || err != nil { + t.Fatal("actual concrete alias RHS was lost", role, complete, err) + } + } else if complete || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("alias acquired an unobserved or over-depth RHS contract", role, complete, err) + } + } + plan, err := planAMD64GoArguments(amd64ArgumentAliasSignature(typ)) + check("argument layout", available, plan != nil, err) + // A language identity retains the Named node and its underlying + // node separately. After 32 aliases, the latter is depth 33; + // the argument layout's scalar shape does not add that node. + languageAvailable := available && (scalar != named || count < 32) + language, err := goAMD64LanguageTypeRole(typ, sizes, true, true) + check("argument language", languageAvailable, language != "", err) + language, err = goAMD64NumericalScalarLanguageType(typ, sizes, true) + check("ordinary numerical scalar language", languageAvailable, language != "", err) + if scalar == types.Typ[types.Float32] || scalar == types.Typ[types.Float64] { + if _, err := goAMD64FrameLanguageType(typ, sizes); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("alias API availability expanded the ordinary frame's floating role", err) + } + } + }) + } + } + for _, count := range []int{1, 2, 32} { + t.Run(fmt.Sprintf("no-callback/depth-%d", count), func(t *testing.T) { + calls := 0 + typ := amd64ArgumentAlias(&amd64ArgumentCallbackType{calls: &calls}, count) + calls = 0 // Fixture construction is not an observer or source capability. + plan, err := planAMD64GoArguments(amd64ArgumentAliasSignature(typ)) + if plan != nil || !errors.Is(err, ErrProbeNeedsContext) || calls != 0 { + t.Fatal("opaque argument alias invoked caller callbacks", plan, err, calls) + } + language, err := goAMD64LanguageTypeRole(typ, sizes, true, true) + if language != "" || !errors.Is(err, ErrProbeNeedsContext) || calls != 0 { + t.Fatal("opaque argument language invoked caller callbacks", language, err, calls) + } + language, err = goAMD64NumericalScalarLanguageType(typ, sizes, true) + if language != "" || !errors.Is(err, ErrProbeNeedsContext) || calls != 0 { + t.Fatal("opaque numerical scalar language invoked caller callbacks", language, err, calls) + } + }) + } +} diff --git a/amd64_go_argument_binding.go b/amd64_go_argument_binding.go new file mode 100644 index 000000000..aaf6f12e9 --- /dev/null +++ b/amd64_go_argument_binding.go @@ -0,0 +1,134 @@ +package plan9asm + +import ( + "crypto/sha256" + "encoding/json" + "go/types" + "reflect" +) + +// Only a complete numerical source binder or the call-local GoModule issuer +// creates this private token. It authenticates declared argument carriers, +// not physical stack, closure, caller, call target, pointer extent or result +// transport permissions. +type amd64GoArgumentBinding struct { + identity *amd64GoArgumentBinding + owner *GoX86NumericalReadBinding + virtual *amd64GoVirtualArguments + declaration *types.Func + symbol string + signature FuncSig + plan *amd64GoArgumentPlan + seal [32]byte +} + +func (entry *amd64GoArgumentBinding) digest() ([32]byte, error) { + type argument struct { + Index int + Carriers []amd64GoArgumentCarrier + } + var arguments []argument + if entry.plan != nil { + for _, input := range entry.plan.Arguments { + arguments = append(arguments, argument{input.Index, input.Carriers}) + } + } + data, err := json.Marshal(struct { + Symbol string + Signature FuncSig + Arguments []argument + }{entry.symbol, entry.signature, arguments}) + return sha256.Sum256(data), err +} + +func (entry *amd64GoArgumentBinding) valid(fn Func, sig FuncSig, arch string) bool { + if entry != nil && entry.virtual != nil { + if entry.identity != entry || entry.owner != nil || entry.declaration == nil || entry.plan == nil || entry.virtual.entries[sig.Name] != entry || + entry.virtual.live() != nil || arch != "amd64" || fn.Sym != entry.symbol || fn.FrameSize != 0 || + len(fn.X86RawText) != 0 || fn.x86FixedLocal != nil || fn.x86SourceStack != nil { + return false + } + return entry.matches(sig) + } + if entry == nil || entry.identity != entry || entry.owner == nil || arch != "amd64" || fn.Sym != entry.symbol || + entry.owner.originalBinding != entry.owner || entry.owner.source == nil || entry.owner.source != entry.owner.originalSource || + entry.declaration == nil || entry.plan == nil || entry.owner.originalTypes == nil || + entry.owner.originalTypes.Scope().Lookup(entry.declaration.Name()) != entry.declaration || + entry.owner.capabilities[sig.Name].arguments != entry || entry.owner.options.Sigs[sig.Name].amd64GoArguments != entry { + return false + } + source := entry.owner.source + source.mu.Lock() + ready := source.identity == source && source.ready && !source.revoked.Load() + source.mu.Unlock() + if !ready || fn.FrameSize != 0 || len(fn.X86RawText) != 0 || fn.x86FixedLocal != nil || fn.x86SourceStack != nil { + return false + } + return entry.matches(sig) +} + +func (entry *amd64GoArgumentBinding) matches(sig FuncSig) bool { + // Feature references are rendered after the complete public options check. + // They are not part of a language argument's physical carrier assignment. + shape := numericalSignatureShape(sig) + shape.Attrs = entry.signature.Attrs + if !reflect.DeepEqual(shape, entry.signature) { + return false + } + declaration, valid := entry.declaration.Type().(*types.Signature) + if !valid { + return false + } + plan, err := planAMD64GoArguments(declaration) + if err != nil || plan.requireRegisters() != nil || !reflect.DeepEqual(plan, entry.plan) { + return false + } + for index, argument := range plan.Arguments { + if argument.Type != entry.plan.Arguments[index].Type { + return false + } + } + digest, err := entry.digest() + if entry.virtual != nil { + return err == nil && digest == entry.seal && entry.virtual.live() == nil + } + source := entry.owner.source + source.mu.Lock() + ready := source.identity == source && source.ready && !source.revoked.Load() + source.mu.Unlock() + return err == nil && digest == entry.seal && ready +} + +func amd64GoArgumentInitialState(plan *amd64GoArgumentPlan) (x86ResultState, bool) { + var initial x86ResultState + if plan == nil || plan.requireRegisters() != nil { + return initial, false + } + for _, argument := range plan.Arguments { + for _, carrier := range argument.Carriers { + if carrier.Kind == amd64GoArgumentFloating { + index, valid := amd64ParseXReg(carrier.Register) + if !valid || index >= 15 || carrier.Width != 4 && carrier.Width != 8 { + return x86ResultState{}, false + } + initial.xmm[index] = uint16(1< 32<<20 { + return nil, fmt.Errorf("missing/oversized compiler listing") + } + functions := make(map[string]*amd64ArgumentCompilerFunction) + var current *amd64ArgumentCompilerFunction + for _, line := range strings.Split(string(listing), "\n") { + if row := amd64ArgumentCompilerText.FindStringSubmatch(line); row != nil { + flags := make(map[string]bool) + for _, flag := range strings.Split(row[2], "|") { + if flags[flag] { + return nil, fmt.Errorf("duplicate compiler TEXT flag") + } + flags[flag] = true + switch flag { + case "ABIInternal", "NOSPLIT", "NOFRAME", "DUPOK": + default: + return nil, fmt.Errorf("unregistered compiler TEXT flag %s", flag) + } + } + if !flags["ABIInternal"] { + return nil, fmt.Errorf("compiler argument fixture has a non-ABIInternal entry") + } + if functions[row[1]] != nil || len(functions) >= 256 { + return nil, fmt.Errorf("duplicate/oversized compiler TEXT inventory") + } + current = &amd64ArgumentCompilerFunction{calls: make(map[string]int64)} + functions[row[1]] = current + } else if len(line) != 0 && line[0] != ' ' && line[0] != '\t' { + current = nil // A new named data or TEXT record, not the prior body. + } + if current == nil { + continue + } + if row := amd64ArgumentCompilerStore.FindStringSubmatch(line); row != nil { + register := Reg(row[2]) + if base, valid := amd64FullRegBase(register); valid { + register = base + } + offset, err := strconv.ParseInt(row[4], 10, 64) + if err != nil || len(current.spills) >= 4096 { + return nil, fmt.Errorf("invalid compiler argument store") + } + width := map[string]int64{"MOVB": 1, "MOVW": 2, "MOVL": 4, "MOVQ": 8, "MOVSS": 4, "MOVSD": 8}[row[1]] + current.spills = append(current.spills, amd64ArgumentOracleCarrier{row[3], offset, register, width}) + } + if row := amd64ArgumentCompilerBytes.FindStringSubmatch(line); row != nil { + offset, err := strconv.ParseInt(row[1], 16, 64) + bytes, decode := hex.DecodeString(strings.ReplaceAll(row[2], " ", "")) + if err != nil || decode != nil || offset != int64(len(current.code)) || len(current.code) > 1<<20 { + return nil, fmt.Errorf("noncontiguous compiler code") + } + current.code = append(current.code, bytes...) + } + if row := amd64ArgumentCompilerCall.FindStringSubmatch(line); row != nil { + offset, err := strconv.ParseInt(row[1], 10, 64) + if _, duplicate := current.calls[row[2]]; err != nil || duplicate { + return nil, fmt.Errorf("duplicate/invalid compiler call") + } + current.calls[row[2]] = offset + } + } + return functions, nil +} diff --git a/amd64_go_argument_compiler_go127_test.go b/amd64_go_argument_compiler_go127_test.go new file mode 100644 index 000000000..8f58ec5c4 --- /dev/null +++ b/amd64_go_argument_compiler_go127_test.go @@ -0,0 +1,119 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +func TestAMD64GoArgumentCompilerCallsiteBanksAndWidths(t *testing.T) { + fixtures := amd64ArgumentOracleFixtures() + source, err := amd64ArgumentOracleSource(fixtures) + if err != nil { + t.Fatal(err) + } + directory := t.TempDir() + input, object := filepath.Join(directory, "arguments.go"), filepath.Join(directory, "arguments.o") + if err := os.WriteFile(input, source, 0600); err != nil { + t.Fatal(err) + } + binary, err := exec.LookPath("go") + if err != nil { + t.Fatal(err) + } + binary, err = sourcepath.Resolve(binary) + if err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, binary, "tool", "compile", "-N", "-l", "-S", "-p", "example.invalid/carrieroracle", "-o", object, input) + command.Dir = directory + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "GOAMD64=v1", "GOEXPERIMENT=", "GOFLAGS=", "GOENV=off", "GOWORK=off", "CGO_ENABLED=0", "GOTOOLCHAIN=local") + stdout, stderr, err := runtimeOracleMachineOutput(ctx, command) + if err != nil { + t.Fatalf("actual independent ordinary compiler fixture: %v\n%s\n%s", err, stdout, stderr) + } + listing := append(append([]byte(nil), stdout...), stderr...) + functions, err := amd64ArgumentCompilerFunctions(listing) + if err != nil { + t.Fatalf("complete actual compiler listing: %v\n%s", err, listing) + } + compiled, err := os.ReadFile(object) + if err != nil || len(compiled) == 0 { + t.Fatal("missing complete actual compiler object", err) + } + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, input, source, 0) + if err != nil { + t.Fatal(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check("example.invalid/carrieroracle", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + for _, fixture := range fixtures { + t.Run(fixture.name, func(t *testing.T) { + name := "example.invalid/carrieroracle.Target" + fixture.name + target, caller := functions[name], functions["example.invalid/carrieroracle.Caller"+fixture.name] + if target == nil || caller == nil || len(target.code) == 0 || len(caller.code) == 0 || + !bytes.Contains(compiled, target.code) || !bytes.Contains(compiled, caller.code) { + t.Fatalf("original named ABIInternal compiler bodies/code are absent\n%s", listing) + } + call, found := caller.calls[name] + if !found || call < 1 || call > int64(len(caller.code))-4 || caller.code[call-1] != 0xe8 { + t.Fatal("ordinary original caller lost its unique actual E8/R_CALL target") + } + got := append([]amd64ArgumentOracleCarrier(nil), target.spills...) + bases := make(map[string]int64) + for _, spill := range got { + base, found := bases[spill.parameter] + if !found || spill.offset < base { + bases[spill.parameter] = spill.offset + } + } + for index := range got { + got[index].offset -= bases[got[index].parameter] + } + if !reflect.DeepEqual(got, fixture.carriers) { + t.Fatalf("independent actual incoming register spill observations\n got %#v\nwant %#v\n%s", got, fixture.carriers, listing) + } + sig := pkg.Scope().Lookup("Target" + fixture.name).Type().(*types.Signature) + plan, err := planAMD64GoArguments(sig) + if err != nil { + t.Fatal(err) + } + var modeled []amd64ArgumentOracleCarrier + for _, argument := range plan.Arguments { + for _, carrier := range argument.Carriers { + modeled = append(modeled, amd64ArgumentOracleCarrier{sig.Params().At(argument.Index).Name(), carrier.Offset, carrier.Register, carrier.Width}) + } + } + if !reflect.DeepEqual(modeled, fixture.carriers) { + t.Fatalf("compiler-independent parameter grouping/bank/width matrix differs: %#v", modeled) + } + t.Logf("actual original ABIInternal target=%s receivingStores=%+v uniqueE8CallOffset=%d callerCodeSHA=%x targetCodeSHA=%x; no incoming stack/carrier permission", name, got, call, sha256.Sum256(caller.code), sha256.Sum256(target.code)) + if strings.Contains(fixture.name, "Rollback") && plan.requireRegisters() == nil { + t.Fatal("oracle spill observations became a register-only entry permission") + } + }) + } + after, err := os.ReadFile(input) + if err != nil || sha256.Sum256(after) != sha256.Sum256(source) { + t.Fatal("original ordinary fixture changed", err) + } + t.Logf("complete original ordinary fixture: %d source functions, %d observed compiler TEXTs, sourceSHA=%x objectSHA=%x; receiving spill stores are NOT incoming stack authority", 2*len(fixtures), len(functions), sha256.Sum256(source), sha256.Sum256(compiled)) +} diff --git a/amd64_go_argument_compiler_model_test.go b/amd64_go_argument_compiler_model_test.go new file mode 100644 index 000000000..1d97ff2c5 --- /dev/null +++ b/amd64_go_argument_compiler_model_test.go @@ -0,0 +1,76 @@ +package plan9asm + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "reflect" + "strings" + "testing" +) + +func TestAMD64GoArgumentCompilerFixtureAndListingGuards(t *testing.T) { + fixtures := amd64ArgumentOracleFixtures() + source, err := amd64ArgumentOracleSource(fixtures) + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "arguments.go", source, 0) + if err != nil { + t.Fatal(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check("example.invalid/carrieroracle", fset, []*ast.File{file}, nil) + if err != nil || len(fixtures) != 9 || len(pkg.Scope().Names()) != 18 { + t.Fatal("whole independent ordinary fixture inventory", err) + } + for _, fixture := range fixtures { + t.Run("independent-matrix/"+fixture.name, func(t *testing.T) { + sig := pkg.Scope().Lookup("Target" + fixture.name).Type().(*types.Signature) + plan, err := planAMD64GoArguments(sig) + if err != nil { + t.Fatal(err) + } + var got []amd64ArgumentOracleCarrier + for _, argument := range plan.Arguments { + for _, carrier := range argument.Carriers { + got = append(got, amd64ArgumentOracleCarrier{sig.Params().At(argument.Index).Name(), carrier.Offset, carrier.Register, carrier.Width}) + } + } + if !reflect.DeepEqual(got, fixture.carriers) { + t.Fatalf("independent ordinary matrix: %#v != %#v", got, fixture.carriers) + } + if strings.Contains(fixture.name, "Rollback") && plan.requireRegisters() == nil { + t.Fatal("compiler oracle enabled incoming spill storage") + } + }) + } + // Synthetic parser controls never become source/compiler/entry authority. + const listing = "target STEXT size=1\n\t0x0000 00000 TEXT target(SB), ABIInternal, $0-8\n\t0x0000 00000 MOVB AL, fixture.a+16(SP)\n\t0x0000 c3 \ncaller STEXT size=6\n\t0x0000 00000 TEXT caller(SB), ABIInternal, $0-8\n\t0x0000 e8 00 00 00 00 c3 \n\trel 1+4 t=R_CALL target+0\n" + observed, err := amd64ArgumentCompilerFunctions([]byte(listing)) + if err != nil || len(observed) != 2 || len(observed["target"].spills) != 1 || observed["target"].spills[0].register != AX || observed["caller"].calls["target"] != 1 { + t.Fatal("complete compiler parser model", err) + } + for _, flags := range []string{"NOSPLIT|ABIInternal", "ABIInternal|NOSPLIT", "DUPOK|NOSPLIT|NOFRAME|ABIInternal"} { + withFlags := strings.Replace(listing, "TEXT target(SB), ABIInternal,", "TEXT target(SB), "+flags+",", 1) + if functions, err := amd64ArgumentCompilerFunctions([]byte(withFlags)); err != nil || len(functions) != 2 || functions["target"] == nil { + t.Errorf("compiler TEXT flags lost an original ABIInternal target (%s): %v", flags, err) + } + } + for name, invalid := range map[string]string{ + "duplicate-flags": strings.Replace(listing, "TEXT target(SB), ABIInternal,", "TEXT target(SB), ABIInternal|ABIInternal,", 1), + "unknown-flags": strings.Replace(listing, "TEXT target(SB), ABIInternal,", "TEXT target(SB), ABIInternal|UNKNOWN,", 1), + "ABI0": strings.Replace(listing, "TEXT target(SB), ABIInternal,", "TEXT target(SB), ABI0,", 1), + "duplicate-TEXT": listing + "\t0x0000 00000 TEXT target(SB), ABIInternal, $0-8\n", + "code-hole": strings.Replace(listing, "0x0000 c3", "0x0001 c3", 1), + "duplicate-call": listing + "\trel 1+4 t=R_CALL target+0\n", + "empty": "", + } { + t.Run(name, func(t *testing.T) { + if _, err := amd64ArgumentCompilerFunctions([]byte(invalid)); err == nil { + t.Fatal("partial/ambiguous compiler listing was accepted") + } + }) + } +} diff --git a/amd64_go_argument_consumer_test.go b/amd64_go_argument_consumer_test.go new file mode 100644 index 000000000..7afdd4337 --- /dev/null +++ b/amd64_go_argument_consumer_test.go @@ -0,0 +1,227 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/token" + "go/types" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Deliberately synthetic private comparison state. Public consumers below +// reject it: it has no live tool/source guard, archive or compiler producer. +func amd64ArgumentBoundModel(t *testing.T, parameters, body string, result ...types.Type) (Func, FuncSig, *amd64GoArgumentBinding) { + t.Helper() + input := amd64ArgumentEntryModel(t, parameters) + if len(result) == 1 { + pkg := types.NewPackage(input.Package.Path(), input.Package.Name()) + sig := types.NewSignatureType(nil, nil, nil, input.Declaration.Type().(*types.Signature).Params(), + types.NewTuple(types.NewVar(token.NoPos, nil, "", result[0])), false) + input.Package = pkg + input.Declaration = types.NewFunc(token.NoPos, pkg, "Entry", sig) + pkg.Scope().Insert(input.Declaration) + var err error + input.Signature, err = DeriveGoFuncABI0Signature(input.Native, input.Declaration, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + } + plan, err := amd64GoArgumentEntryPlan(input) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, "TEXT ·Entry(SB),4,$0-128\n"+body+"\nRET\n") + if err != nil { + t.Fatal(err) + } + source := &GoX86NumericalReadSource{ready: true} + source.identity = source + owner := &GoX86NumericalReadBinding{source: source, originalSource: source, originalTypes: input.Package, + options: Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: make(map[string]FuncSig)}} + owner.originalBinding = owner + entry := &amd64GoArgumentBinding{owner: owner, declaration: input.Declaration, symbol: file.Funcs[0].Sym, signature: input.Signature, plan: plan} + entry.identity = entry + entry.seal, err = entry.digest() + if err != nil { + t.Fatal(err) + } + sig := input.Signature + sig.amd64GoArguments = entry + owner.options.Sigs[sig.Name] = sig + owner.capabilities = numericalCapabilities(owner.options.Sigs) + return file.Funcs[0], sig, entry +} + +func TestAMD64GoArgumentEntryDeclaredBytesAndConsumerRevocation(t *testing.T) { + input := amd64ArgumentEntryModel(t, "a byte, b bool, p *byte, f float32, d float64") + plan, err := amd64GoArgumentEntryPlan(input) + if err != nil { + t.Fatal(err) + } + state, valid := amd64GoArgumentInitialState(plan) + if !valid || state.gp[0] != 1 || state.gp[3] != 1 || state.canonicalGP[3] != 1 || + !state.pointerGP[1] || !state.callerAddressGP[1] || state.gp[1] != 0 || + state.xmm[0] != 0xf || state.xmm[1] != 0xff || state.gp[2] != 0 || state.pointerGP[2] || + len(state.fp) != 0 || len(state.auto) != 0 || state.flags != 0 { + t.Fatalf("entry gained upper bytes, native backing, DX, FLAGS or stack facts: %#v", state) + } + for _, test := range []struct { + name, parameters, body string + accept bool + }{ + {"slice-length", "a []byte", "MOVQ BX,AX", true}, + {"string-length", "a string", "MOVQ BX,AX", true}, + {"float64-bits", "a float64", "MOVQ X0,AX", true}, + {"integer32-overwrite", "a uint32", "MOVL AX,AX", true}, + {"bool-byte", "a bool", "MOVBQZX AL,AX", true}, + {"unknown-byte-upper", "a byte", "MOVQ AX,AX", false}, + {"unknown-float-upper", "a float32", "MOVQ X0,AX", false}, + {"pointer-not-numeric", "a *byte", "MOVQ AX,AX", false}, + {"unknown-DX-closure", "a []byte", "MOVQ 8(DX),AX", false}, + {"unknown-stack", "a []byte", "MOVQ (SP),AX", false}, + {"no-ABI0-parameter-home", "a []byte", "MOVQ a+0(FP),AX", false}, + {"unknown-FLAGS", "a bool", "SETEQ AL\nMOVBQZX AL,AX", false}, + } { + t.Run(test.name, func(t *testing.T) { + fn, sig, _ := amd64ArgumentBoundModel(t, test.parameters, test.body) + err := checkAMD64RegisterResultInitialization(fn, sig, + func(_ int, ins Instr, state *x86ResultState, owned []bool, offset int64, arch string) bool { + return x86ScalarFPResultTransfer(ins, state, owned, offset, arch) + }) + if test.accept && err != nil || !test.accept && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("declared-byte-only result proof: %v", err) + } + }) + } + for _, name := range []string{"closed", "revoked-restored", "copied-token", "signature", "original-type", "carrier", "capability", "frame"} { + t.Run(name, func(t *testing.T) { + fn, sig, entry := amd64ArgumentBoundModel(t, "a string", "MOVQ BX,AX") + if !entry.valid(fn, sig, "amd64") { + t.Fatal("complete private comparison model was not usable") + } + switch name { + case "closed": + entry.owner.source.ready = false + case "revoked-restored": + entry.owner.source.revoked.Store(true) + entry.owner.source.ready = true + case "copied-token": + copied := *entry + entry = &copied + case "signature": + sig.Args = []LLVMType{I64} + case "original-type": + entry.plan.Arguments[0].Type = nil + case "carrier": + entry.plan.Arguments[0].Carriers[1].Register = DX + case "capability": + entry.owner.capabilities = nil + case "frame": + fn.FrameSize = 8 + } + if entry.valid(fn, sig, "amd64") { + t.Fatal("changed/revoked private argument association survived") + } + }) + } + for _, test := range []struct { + body string + accept bool + }{{"MOVB AL,AL", true}, {"XORL AX,AX", true}, {"MOVB $2,AL", false}} { + t.Run("bool-result/"+test.body, func(t *testing.T) { + fn, sig, _ := amd64ArgumentBoundModel(t, "a bool", test.body, types.Typ[types.Bool]) + ctx := newX86Ctx(new(strings.Builder), fn, sig, nil, nil, "amd64", "x86_64-unknown-linux-gnu", false) + err := ctx.scalarFPResultInitialized() + if test.accept && err != nil || !test.accept && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("argument role widened canonical bool result", err) + } + }) + } +} + +func TestAMD64GoArgumentEntryEmissionUsesGroupedIndependentBanks(t *testing.T) { + fn, sig, _ := amd64ArgumentBoundModel(t, "f float64, a struct { N byte; V [1]float32 }, b bool, s string", + "MOVBQZX AL,AX\nMOVBQZX BL,BX\nMOVQ CX,CX\nMOVQ DI,DI\nMOVQ X0,R8\nMOVL X1,R9\nMOVQ $1,AX") + var output strings.Builder + ctx := newX86Ctx(&output, fn, sig, nil, nil, "amd64", "x86_64-unknown-linux-gnu", false) + if err := ctx.emitAMD64GoArguments(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("prologue bypassed whole-source proof", err) + } + if err := ctx.scalarFPResultInitialized(); err != nil { + t.Fatal("complete typed argument source proof", err) + } + if err := ctx.emitEntryAllocas(); err != nil { + t.Fatal(err) + } + ir := output.String() + for _, fragment := range []string{ + "bitcast double %arg0 to i64", "extractvalue { i8, [1 x float] } %arg1, 0", + "extractvalue { i8, [1 x float] } %arg1, 1, 0", "zext i1 %arg2 to i64", + "extractvalue { ptr, i64 } %arg3, 0", "extractvalue { ptr, i64 } %arg3, 1", + "ptr %reg_AX", "ptr %reg_BX", "ptr %reg_CX", "ptr %reg_DI", "ptr %x0", "ptr %x1", + } { + if !strings.Contains(ir, fragment) { + t.Errorf("missing grouped carrier %q in\n%s", fragment, ir) + } + } + if strings.Contains(ir, "DX") || strings.Contains(ir, "inbounds") || strings.Contains(ir, "noalias") { + t.Fatal("argument emission invented hidden/backing/alias authority", ir) + } +} + +func TestAMD64GoArgumentEntryPublicConsumersRequireLiveBinding(t *testing.T) { + fn, sig, _ := amd64ArgumentBoundModel(t, "a string", "MOVQ BX,AX") + file := &File{Arch: ArchAMD64, Funcs: []Func{fn}} + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: func(string) string { return sig.Name }, Sigs: map[string]FuncSig{sig.Name: sig}} + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + var err error + switch api { + case "text": + _, err = Translate(file, options) + case "module": + context := llvm.NewContext() + defer context.Dispose() + module, failure := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + module.Dispose() + } + err = failure + case "Go": + context := llvm.NewContext() + defer context.Dispose() + module, failure := translateGoModuleInContext(context, GoPackage{Path: sig.amd64GoArguments.declaration.Pkg().Path(), Types: sig.amd64GoArguments.declaration.Pkg()}, + []byte("TEXT ·Entry(SB),4,$0-24\nMOVQ BX,AX\nRET\n"), GoModuleOptions{GOARCH: "amd64", TargetTriple: options.TargetTriple, + ResolveSym: options.ResolveSym, ManualSig: func(string) (FuncSig, bool) { return sig, true }}) + if module != nil && !module.Module.IsNil() { + module.Module.Dispose() + } + err = failure + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s borrowed a private comparison model without live original compiler/source proof: %v", api, err) + } + }) + } + t.Run("no-direct-module-bypass", func(t *testing.T) { + context := llvm.NewContext() + defer context.Dispose() + module, err := translateModuleDirectInContext(context, file, options) + if !module.IsNil() { + module.Dispose() + } + if !errors.Is(err, errDirectModuleUnsupported) { + t.Fatal("linear module path bypassed typed whole-source proof", err) + } + }) + clone, err := numericalCloneSignatures(options.Sigs) + if err != nil || clone[sig.Name].amd64GoArguments != sig.amd64GoArguments || + !reflect.DeepEqual(numericalSignatureShape(clone[sig.Name]), numericalSignatureShape(sig)) { + t.Fatal("owned private identity and language shape were confused", fmt.Sprint(err)) + } +} diff --git a/amd64_go_argument_emit.go b/amd64_go_argument_emit.go new file mode 100644 index 000000000..dba3d38c6 --- /dev/null +++ b/amd64_go_argument_emit.go @@ -0,0 +1,63 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) emitAMD64GoArguments() error { + entry := c.sig.amd64GoArguments + if entry != nil && entry.virtual != nil { + if err := c.scalarFPResultInitialized(); err != nil { + return err + } + } + if entry == nil || !entry.valid(c.fpScalarSource, c.sig, c.goarch) || !c.fpScalarChecked || c.fpScalarCheckErr != nil { + return amd64GoArgumentContext("entry emission lacks its complete live source-byte proof") + } + for _, argument := range entry.plan.Arguments { + for _, carrier := range argument.Carriers { + value := fmt.Sprintf("%%arg%d", argument.Index) + if len(carrier.Path) != 0 { + var path []string + for _, field := range carrier.Path { + path = append(path, fmt.Sprint(field)) + } + temporary := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %s\n", temporary, c.sig.Args[argument.Index], value, strings.Join(path, ", ")) + value = "%" + temporary + } + if carrier.Kind != amd64GoArgumentFloating { + slot, used := c.regSlot[carrier.Register] + if !used { + continue + } + integer, valid, err := amd64ValueAsI64(c, carrier.Type, value) + if err != nil || !valid { + return amd64GoArgumentContext("declared GP carrier lost its LLVM scalar transport") + } + fmt.Fprintf(c.b, " store i64 %s, ptr %s\n", integer, slot) + continue + } + index, valid := amd64ParseXReg(carrier.Register) + if !valid { + return amd64GoArgumentContext("invalid declared FP carrier") + } + if _, used := c.xRegSlot[index]; !used { + continue + } + bits := carrier.Width * 8 + integer, lane, bytes := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", integer, carrier.Type, value, bits) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> zeroinitializer, i%d %%%s, i32 0\n", lane, 128/bits, bits, bits, integer) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <16 x i8>\n", bytes, 128/bits, bits, lane) + if err := c.storeX(carrier.Register, "%"+bytes); err != nil { + return err + } + } + } + // The virtual slots use a deterministic representative for unused bytes. + // The complete source proof initialized only each declared scalar width, + // never the representative upper GP/X bytes or the native ABI spill homes. + return nil +} diff --git a/amd64_go_argument_entry.go b/amd64_go_argument_entry.go new file mode 100644 index 000000000..9ecd4a708 --- /dev/null +++ b/amd64_go_argument_entry.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "go/types" + "reflect" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "github.com/xgo-dev/plan9asm/internal/gosym" +) + +// This comparison model cannot create a binding or source token. The live +// factory supplies its complete original type graph and independently observed +// assembler definition, after verifying the registered compiler ABI rules. +type amd64GoArgumentEntryInput struct { + Package *types.Package + Declaration *types.Func + Signature FuncSig + Native, Target, Goarch string + GOAMD64, GOEXPERIMENT, GOFLAGS string + SymABI []byte + Original, Expanded gostaticstorage.TextRecord + Aliases *numericalCompilerAliasDirectory + Standard bool +} + +func amd64GoArgumentEntryPlan(input amd64GoArgumentEntryInput) (*amd64GoArgumentPlan, error) { + fail := func() (*amd64GoArgumentPlan, error) { + return nil, amd64GoArgumentContext("original declaration/compiler ABI/profile association differs") + } + fn, pkg := input.Declaration, input.Package + if pkg == nil || fn == nil || fn.Pkg() != pkg || pkg.Scope().Lookup(fn.Name()) != fn || + input.Goarch != "amd64" || !strings.HasSuffix(input.Target, "/amd64") || numericalSDKTargetTriple(input.Target) == "" || + input.GOAMD64 != "v1" || input.GOEXPERIMENT != "" || input.GOFLAGS != "" { + return fail() + } + declaration, valid := fn.Type().(*types.Signature) + if !valid { + return fail() + } + plan, err := planAMD64GoArguments(declaration) + if err != nil || plan.requireRegisters() != nil { + return fail() // Neither actual stack arguments nor reserved spill homes. + } + if declaration.Results().Len() != 1 { + return fail() + } + resultBuilder := amd64GoArgumentBuilder{sizes: types.SizesFor("gc", "amd64")} + result, err := resultBuilder.shape(declaration.Results().At(0).Type(), 0) + if err != nil || result.stack || len(result.leaves) != 1 || len(result.leaves[0].Path) != 0 || + (result.leaves[0].Kind != amd64GoArgumentInteger && result.leaves[0].Kind != amd64GoArgumentBoolean) { + return fail() // LLVM flattening is not a scalar Go result declaration. + } + expected, err := DeriveGoFuncABI0Signature(input.Native, fn, "amd64", types.SizesFor("gc", "amd64")) + if err != nil || !reflect.DeepEqual(numericalSignatureShape(input.Signature), expected) || len(expected.Frame.Results) != 1 { + return fail() + } + switch expected.Ret { + case I1, I8, I16, I32, I64: + default: + return fail() // Results retain their independently closed existing role. + } + if input.Native != gosym.PackagePrefix(pkg.Path())+"."+fn.Name() && + (input.Aliases == nil || input.Aliases.association(input.Native, 1, input.Standard) != nil) { + return fail() + } + if !amd64GoArgumentDefinition(input.SymABI, input.Native) { + return fail() + } + original := input.Original + if original.Name != input.Native || original.ABI != 1 || original.Kind != 1 || + original.Namespace != gostaticstorage.TextSymbolNonpkgDefinition || original.Size <= 0 || + original.Size != int64(len(original.Code)) || !reflect.DeepEqual(original, input.Expanded) { + return fail() + } + return plan, nil +} + +func amd64GoArgumentDefinition(symABI []byte, native string) bool { + if len(symABI) == 0 || len(symABI) > 16<<20 || native == "" { + return false + } + definitions := make(map[string]bool) + found := false + for _, line := range strings.Split(string(symABI), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + if len(fields) != 3 || fields[0] != "def" && fields[0] != "ref" || + fields[2] != "ABI0" && fields[2] != "ABIInternal" { + return false + } + if fields[0] != "def" { + continue + } + if definitions[fields[1]] || len(definitions) >= gostaticstorage.ObjectSymbolInventoryLimit { + return false + } + definitions[fields[1]] = true + if fields[1] == native { + found = fields[2] == "ABIInternal" + } + } + return found +} diff --git a/amd64_go_argument_entry_test.go b/amd64_go_argument_entry_test.go new file mode 100644 index 000000000..f0612e95c --- /dev/null +++ b/amd64_go_argument_entry_test.go @@ -0,0 +1,140 @@ +package plan9asm + +import ( + "errors" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +// This is an independent comparison model, not a live compiler/source token. +func amd64ArgumentEntryModel(t *testing.T, parameters string) amd64GoArgumentEntryInput { + t.Helper() + typed := amd64ArgumentTestSignature(t, parameters) + pkg := types.NewPackage("example.invalid/carriers", "carriers") + fn := types.NewFunc(token.NoPos, pkg, "Entry", typed) + pkg.Scope().Insert(fn) + native := pkg.Path() + ".Entry" + sig, err := DeriveGoFuncABI0Signature(native, fn, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + text := gostaticstorage.TextRecord{Name: native, ABI: 1, Kind: 1, + Namespace: gostaticstorage.TextSymbolNonpkgDefinition, Size: 1, Code: []byte{0xc3}} + return amd64GoArgumentEntryInput{Package: pkg, Declaration: fn, Signature: sig, + Native: native, Target: "linux/amd64", Goarch: "amd64", GOAMD64: "v1", GOEXPERIMENT: "", GOFLAGS: "", + SymABI: []byte("def " + native + " ABIInternal\n"), Original: text, Expanded: text} +} + +func TestAMD64GoArgumentEntryCompleteCompilerAssociation(t *testing.T) { + for _, parameters := range []string{ + "a,b string", "a,b []byte", "a float64, b bool, c struct { X byte; Y [1]float32; P *byte }", + "a,b,c,d,e,f,g,h,i uint64", "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o float64", + } { + t.Run(parameters, func(t *testing.T) { + input := amd64ArgumentEntryModel(t, parameters) + plan, err := amd64GoArgumentEntryPlan(input) + if err != nil || plan == nil || plan.requireRegisters() != nil || len(plan.Arguments) != len(input.Signature.Args) { + t.Fatalf("complete declared compiler-derived transport: %#v, %v", plan, err) + } + }) + } + for _, standard := range []bool{false, true} { + input := amd64ArgumentEntryModel(t, "a,b string") + alias := numericalCompilerAliasModel(standard) + input.Native, input.Signature.Name, input.Standard = alias.Native, alias.Native, standard + input.Original.Name, input.Expanded.Name, input.SymABI = alias.Native, alias.Native, alias.SymABI + input.Aliases, _ = numericalCompilerAliasIndex(alias.SymABI, alias.Generated, alias.Selected) + if _, err := amd64GoArgumentEntryPlan(input); err != nil { + t.Fatal("complete separately observed foreign compiler association did not join", err) + } + input.Aliases = nil + if plan, err := amd64GoArgumentEntryPlan(input); plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("foreign spelling/SymABI without original compiler alias acquired arguments", err) + } + } +} + +func TestAMD64GoArgumentEntryLanguageRoleDoesNotWidenOrdinaryFrame(t *testing.T) { + sizes := types.SizesFor("gc", "amd64") + for _, parameters := range []string{"a float32", "a float64", "a string", "a struct { S [1]string; F float64 }"} { + t.Run(parameters, func(t *testing.T) { + typ := amd64ArgumentTestSignature(t, parameters).Params().At(0).Type() + if _, err := goAMD64FrameLanguageTypeRole(typ, sizes, true); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("ordinary frame language query acquired the private argument transport", err) + } + first, err := goAMD64LanguageTypeRole(typ, sizes, true, true) + if err != nil || first == "" { + t.Fatal("bounded argument language comparison rejected its declared type", err) + } + secondType := amd64ArgumentTestSignature(t, strings.ReplaceAll(parameters, "float64", "uint64")).Params().At(0).Type() + second, err := goAMD64LanguageTypeRole(secondType, sizes, true, true) + if strings.Contains(parameters, "float64") && (err != nil || first == second) { + t.Fatal("same-size integer/FP typegraph drift was erased", err) + } + }) + } + for _, result := range []types.Type{types.Typ[types.Float64], types.NewPointer(types.Typ[types.Byte]), types.NewStruct(nil, nil)} { + input := amd64ArgumentEntryModel(t, "a string") + declaration := types.NewSignatureType(nil, nil, nil, input.Declaration.Type().(*types.Signature).Params(), + types.NewTuple(types.NewVar(token.NoPos, nil, "", result)), false) + pkg := types.NewPackage(input.Package.Path(), input.Package.Name()) + input.Declaration = types.NewFunc(token.NoPos, pkg, "Entry", declaration) + pkg.Scope().Insert(input.Declaration) + input.Package = pkg + input.Signature, _ = DeriveGoFuncABI0Signature(input.Native, input.Declaration, "amd64", sizes) + if plan, err := amd64GoArgumentEntryPlan(input); plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("argument type comparison silently widened result transport", err) + } + } +} + +func TestAMD64GoArgumentEntryRejectsChangedSourceTypeABIProfile(t *testing.T) { + for name, mutate := range map[string]func(*amd64GoArgumentEntryInput){ + "missing-package": func(v *amd64GoArgumentEntryInput) { v.Package = nil }, + "foreign-package": func(v *amd64GoArgumentEntryInput) { v.Package = types.NewPackage(v.Package.Path(), v.Package.Name()) }, + "shadowed-declaration": func(v *amd64GoArgumentEntryInput) { + v.Declaration = types.NewFunc(token.NoPos, v.Package, "Entry", v.Declaration.Type().(*types.Signature)) + }, + "foreign-native": func(v *amd64GoArgumentEntryInput) { v.Native = "runtime.other" }, + "missing-def": func(v *amd64GoArgumentEntryInput) { v.SymABI = []byte("ref " + v.Native + " ABIInternal\n") }, + "ABI0-def": func(v *amd64GoArgumentEntryInput) { v.SymABI = []byte("def " + v.Native + " ABI0\n") }, + "duplicate-def": func(v *amd64GoArgumentEntryInput) { v.SymABI = append(v.SymABI, v.SymABI...) }, + "unknown-row": func(v *amd64GoArgumentEntryInput) { v.SymABI = append(v.SymABI, []byte("garbage\n")...) }, + "native-ABI0": func(v *amd64GoArgumentEntryInput) { v.Original.ABI = 0 }, + "changed-expanded": func(v *amd64GoArgumentEntryInput) { v.Expanded.Flags ^= 1 }, + "missing-code": func(v *amd64GoArgumentEntryInput) { v.Original.Code = nil }, + "private-TEXT": func(v *amd64GoArgumentEntryInput) { v.Original.Namespace = gostaticstorage.TextSymbolSelf }, + "wrong-target": func(v *amd64GoArgumentEntryInput) { v.Target = "linux/386" }, + "wrong-mode": func(v *amd64GoArgumentEntryInput) { v.Goarch = "386" }, + "feature-profile": func(v *amd64GoArgumentEntryInput) { v.GOAMD64 = "v3" }, + "experiment": func(v *amd64GoArgumentEntryInput) { v.GOEXPERIMENT = "regabiargs" }, + "flags": func(v *amd64GoArgumentEntryInput) { v.GOFLAGS = "-N" }, + "argument-group": func(v *amd64GoArgumentEntryInput) { v.Signature.Args = []LLVMType{Ptr, I64, Ptr, I64} }, + "result-width": func(v *amd64GoArgumentEntryInput) { v.Signature.Ret = I32 }, + "manual-registers": func(v *amd64GoArgumentEntryInput) { v.Signature.ArgRegs = []Reg{AX, BX} }, + "frame-layout": func(v *amd64GoArgumentEntryInput) { v.Signature.Frame.Params[0].Offset++ }, + } { + t.Run(name, func(t *testing.T) { + input := amd64ArgumentEntryModel(t, "a,b string") + mutate(&input) + if plan, err := amd64GoArgumentEntryPlan(input); plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed association gained a carrier plan: %#v, %v", plan, err) + } + }) + } + for _, parameters := range []string{ + "a,b,c,d,e,f,g,h uint64, s string, last uint64", + "a,b,c,d,e,f,g,h,i,j,k,l,m,n float64, z complex128, last float64", + "a [2]uint32", "a struct{}", + } { + t.Run("stack/"+strings.ReplaceAll(parameters, "/", "_"), func(t *testing.T) { + if plan, err := amd64GoArgumentEntryPlan(amd64ArgumentEntryModel(t, parameters)); plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("spill homes or true stack arguments became entry authority: %#v, %v", plan, err) + } + }) + } +} diff --git a/amd64_go_argument_go_module_test.go b/amd64_go_argument_go_module_test.go new file mode 100644 index 000000000..6869089ca --- /dev/null +++ b/amd64_go_argument_go_module_test.go @@ -0,0 +1,91 @@ +package plan9asm + +import ( + "errors" + "reflect" + "testing" +) + +func TestAMD64GoArgumentGoModuleGraphSignatureTransport(t *testing.T) { + // This model exercises only the existing signature/graph comparison seam. + // It cannot pass a public source consumer without the real live producer. + fn, bound, entry := amd64ArgumentBoundModel(t, "a string, f float32", "MOVQ BX,AX") + original, err := numericalCloneSignatures(entry.owner.options.Sigs) + if err != nil { + t.Fatal(err) + } + derived := map[string]FuncSig{bound.Name: numericalSignatureShape(bound)} + if derived[bound.Name].amd64GoArguments != nil { + t.Fatal("a fresh language signature retained private entry metadata") + } + + transported, err := numericalGoSignatures(entry.owner, derived) + if err != nil || transported[bound.Name].amd64GoArguments != entry || + !entry.valid(fn, transported[bound.Name], "amd64") { + t.Fatal("the authenticated signature identity was lost before the graph consumer", err) + } + if derived[bound.Name].amd64GoArguments != nil || + !reflect.DeepEqual(entry.owner.options.Sigs, original) { + t.Fatal("transport changed the caller's fresh signatures or the owner's inventory") + } + + file := &File{Arch: ArchAMD64, Funcs: []Func{fn}} + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: transported, ResolveSym: func(string) string { return bound.Name }} + graph, _, err := newX86DirectTailGraph(file, options, nil, "amd64") + if err != nil || graph.signatures[bound.Name].amd64GoArguments != entry { + t.Fatal("a graph snapshot replaced its existing bound argument identity", err) + } + if err := graph.verify(file, options); err != nil { + t.Fatal("the exact signature snapshot did not survive the final graph comparison", err) + } + + options.Sigs = derived + if !errors.Is(graph.verify(file, options), ErrProbeNeedsContext) { + t.Fatal("the final graph comparison accepted language shapes in place of private identities") + } + if !graph.revoked.Load() { + t.Fatal("signature substitution did not revoke the graph") + } + + for _, name := range []string{"missing", "extra", "name", "argument", "result", "layout"} { + t.Run(name, func(t *testing.T) { + changed, err := numericalCloneSignatures(derived) + if err != nil { + t.Fatal(err) + } + sig := changed[bound.Name] + switch name { + case "missing": + delete(changed, bound.Name) + case "extra": + changed["unrelated"] = sig + case "name": + sig.Name = "unrelated" + case "argument": + sig.Args[1] = I32 + case "result": + sig.Ret = Ptr + case "layout": + sig.Frame.Params[0].Offset++ + } + if name != "missing" && name != "extra" { + changed[bound.Name] = sig + } + if _, err := numericalGoSignatures(entry.owner, changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go signature transport accepted changed declarations/layout", err) + } + }) + } + + t.Run("supplied-token-is-not-authority", func(t *testing.T) { + forged := *entry + sig := derived[bound.Name] + sig.amd64GoArguments = &forged + transported, err := numericalGoSignatures(entry.owner, map[string]FuncSig{bound.Name: sig}) + if err != nil || transported[bound.Name].amd64GoArguments != entry || + transported[bound.Name].amd64GoArguments == &forged { + t.Fatal("caller metadata replaced the original live binding identity", err) + } + }) +} diff --git a/amd64_go_argument_ir_test.go b/amd64_go_argument_ir_test.go new file mode 100644 index 000000000..6b8cee8be --- /dev/null +++ b/amd64_go_argument_ir_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Inspect the actual LLVM values. LLVM legitimately omits quotes around +// runtime.cmpstring while retaining them around internal/bytealg.Compare. +func amd64ArgumentCompareEntryTypes(module llvm.Module) error { + for _, row := range []struct { + name string + fields int + }{ + {"internal/bytealg.Compare", 3}, + {"runtime.cmpstring", 2}, + } { + function := module.NamedFunction(row.name) + if function.IsNil() || function.IsDeclaration() || function.BasicBlocksCount() == 0 { + return fmt.Errorf("missing complete entry definition %s", row.name) + } + typ := function.GlobalValueType() + result := typ.ReturnType() + if result.TypeKind() != llvm.IntegerTypeKind || result.IntTypeWidth() != 64 || + typ.IsFunctionVarArg() || typ.ParamTypesCount() != 2 { + return fmt.Errorf("entry signature differs for %s", row.name) + } + for _, parameter := range typ.ParamTypes() { + if parameter.TypeKind() != llvm.StructTypeKind || parameter.IsStructPacked() || + parameter.IsStructOpaque() || parameter.StructElementTypesCount() != row.fields { + return fmt.Errorf("argument grouping differs for %s", row.name) + } + for index, field := range parameter.StructElementTypes() { + valid := field.TypeKind() == llvm.PointerTypeKind + if index != 0 { + valid = field.TypeKind() == llvm.IntegerTypeKind && field.IntTypeWidth() == 64 + } + if !valid { + return fmt.Errorf("argument field %d differs for %s", index, row.name) + } + } + } + } + return nil +} + +func TestAMD64GoArgumentIRUsesActualEntryIdentityAndTypes(t *testing.T) { + const source = `define i64 @"internal/bytealg.Compare"({ ptr, i64, i64 } %a, { ptr, i64, i64 } %b) { + ret i64 0 +} +define i64 @runtime.cmpstring({ ptr, i64 } %a, { ptr, i64 } %b) { + ret i64 0 +} +` + for _, row := range []struct { + name, ir string + failure bool + }{ + {"canonical-printer", source, false}, + {"quoted-identifier", strings.ReplaceAll(source, "@runtime.cmpstring", "@\"runtime.cmpstring\""), false}, + {"renamed-entry", strings.ReplaceAll(source, "runtime.cmpstring", "runtime.other"), true}, + {"missing-entry", strings.Split(source, "define i64 @runtime")[0], true}, + {"wrong-result", strings.ReplaceAll(source, "i64", "i32"), true}, + {"wrong-length-width", strings.ReplaceAll(source, "ptr, i64", "ptr, i32"), true}, + {"pointer-bits", strings.ReplaceAll(source, "ptr,", "i64,"), true}, + {"collapsed-group", strings.ReplaceAll(source, "{ ptr, i64, i64 }", "{ ptr, i64 }"), true}, + {"packed-group", strings.ReplaceAll(source, "{ ptr, i64 }", "<{ ptr, i64 }>"), true}, + } { + t.Run(row.name, func(t *testing.T) { + context := llvm.NewContext() + defer context.Dispose() + module, err := parseIRModuleInContext(context, row.ir) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal("signature-only test fixture is not valid LLVM IR", err) + } + if err := amd64ArgumentCompareEntryTypes(module); (err != nil) != row.failure { + t.Fatalf("signature check error=%v; want failure=%t", err, row.failure) + } + }) + } +} diff --git a/amd64_go_argument_issuer_test.go b/amd64_go_argument_issuer_test.go new file mode 100644 index 000000000..a73acad3e --- /dev/null +++ b/amd64_go_argument_issuer_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/token" + "go/types" + "testing" +) + +func TestAMD64GoArgumentIssuerDoesNotExtendResultLanguage(t *testing.T) { + sizes := types.SizesFor("gc", "amd64") + for _, parameter := range []string{"byte", "uint16", "uint32", "uint64", "*byte"} { + for _, result := range []struct{ name, declarations, spelling string }{ + {"float32", "", "float32"}, + {"float64", "", "float64"}, + {"complex64", "", "complex64"}, + {"complex128", "", "complex128"}, + {"struct-float", "", "struct { Value float64 }"}, + {"array-complex", "", "[1]complex128"}, + {"pointer-float", "", "*float64"}, + {"slice-float", "", "[]float32"}, + {"named-float", "type Result float64\n", "Result"}, + {"alias-complex", "type Result = complex64\n", "Result"}, + } { + t.Run(parameter+"/"+result.name, func(t *testing.T) { + pkg := mustGoPackage(t, "example.invalid/issuer", + "package issuer\n"+result.declarations+fmt.Sprintf("func Entry(a %s) %s\n", parameter, result.spelling)) + fn := pkg.Types.Scope().Lookup("Entry").(*types.Func) + sig, err := DeriveGoFuncABI0Signature("example.invalid/issuer.Entry", fn, "amd64", sizes) + if err != nil { + t.Fatal("valid original Go language declaration did not derive", err) + } + if amd64GoArgumentNeedsBinding(sig) { + t.Fatal("fixture no longer covers the scalar-only issuer branch") + } + if _, err := numericalSDKDeclarationLanguage(fn, sizes, false); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("the original result-language role unexpectedly accepted this fixture", err) + } + if _, err := numericalSDKDeclarationLanguage(fn, sizes, true); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("argument-only issuer extended result language while its result entry guard was skipped", err) + } + }) + } + } +} + +func TestAMD64GoArgumentIssuerKeepsParameterAndResultRolesIndependent(t *testing.T) { + sizes := types.SizesFor("gc", "amd64") + for _, parameter := range []string{"byte", "uint16", "uint32", "uint64", "*byte"} { + for _, result := range []string{"bool", "uint8", "uint16", "uint32", "uint64"} { + t.Run("old-scalar/"+parameter+"/"+result, func(t *testing.T) { + pkg := mustGoPackage(t, "example.invalid/issuer", fmt.Sprintf("package issuer\nfunc Entry(a %s) %s\n", parameter, result)) + fn := pkg.Types.Scope().Lookup("Entry").(*types.Func) + original, err := numericalSDKDeclarationLanguage(fn, sizes, false) + if err != nil { + t.Fatal("the original scalar language role was lost", err) + } + actual, err := numericalSDKDeclarationLanguage(fn, sizes, true) + if err != nil || actual != original { + t.Fatal("argument role changed the old scalar/result identity", err) + } + }) + } + } + for _, parameter := range []string{"a bool", "a float32", "a float64", "a complex64", "a complex128", + "a string", "a []byte", "a [1]float64", "a struct { F float32; N uint64 }"} { + t.Run("argument-only/"+parameter, func(t *testing.T) { + input := amd64ArgumentEntryModel(t, parameter) + if _, err := numericalSDKDeclarationLanguage(input.Declaration, sizes, true); err != nil { + t.Fatal("the separately validated parameter language role was removed", err) + } + if _, err := amd64GoArgumentEntryPlan(input); err != nil { + t.Fatal("original integer result and register-only argument model were lost", err) + } + }) + } + for _, result := range []string{"bool", "uint8", "uint16", "uint32", "uint64"} { + t.Run("named-scalar-result/"+result, func(t *testing.T) { + input := amd64ArgumentEntryModel(t, "a float64") + pkg := mustGoPackage(t, input.Package.Path(), fmt.Sprintf("package args\ntype Result %s\nfunc Entry(a float64) Result\n", result)) + input.Package = pkg.Types + input.Declaration = pkg.Types.Scope().Lookup("Entry").(*types.Func) + var err error + input.Signature, err = DeriveGoFuncABI0Signature(input.Native, input.Declaration, "amd64", sizes) + if err != nil { + t.Fatal(err) + } + if _, err := numericalSDKDeclarationLanguage(input.Declaration, sizes, true); err != nil { + t.Fatal("the original named scalar language identity was lost", err) + } + if _, err := amd64GoArgumentEntryPlan(input); err != nil { + t.Fatal("named scalar result was confused with an aggregate transport", err) + } + }) + } + for _, result := range []string{"*byte", "string", "[]byte", "struct { N uint64 }", "[1]uint64", + "struct { B bool }", "[1]bool", "(uint64, uint64)"} { + t.Run("language-is-not-result-transport/"+result, func(t *testing.T) { + input := amd64ArgumentEntryModel(t, "a float64") + pkg := mustGoPackage(t, input.Package.Path(), fmt.Sprintf("package args\nfunc Entry(a float64) %s\n", result)) + input.Package = pkg.Types + input.Declaration = pkg.Types.Scope().Lookup("Entry").(*types.Func) + var err error + input.Signature, err = DeriveGoFuncABI0Signature(input.Native, input.Declaration, "amd64", sizes) + if err != nil { + t.Fatal(err) + } + if _, err := numericalSDKDeclarationLanguage(input.Declaration, sizes, true); err != nil { + t.Fatal("old result language identity was confused with entry transport", err) + } + if _, err := amd64GoArgumentEntryPlan(input); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a language comparison granted a pointer/aggregate/multiple result transport", err) + } + }) + } +} + +func TestAMD64GoArgumentIssuerRejectsResultCallbacksBeforeSampling(t *testing.T) { + for _, role := range []string{"language", "entry"} { + t.Run(role, func(t *testing.T) { + calls := 0 + result := &amd64ArgumentCallbackType{calls: &calls} + input := amd64ArgumentEntryModel(t, "a uint64") + signature := types.NewSignatureType(nil, nil, nil, + input.Declaration.Type().(*types.Signature).Params(), + types.NewTuple(types.NewVar(token.NoPos, nil, "r", result)), false) + pkg := types.NewPackage(input.Package.Path(), input.Package.Name()) + input.Declaration = types.NewFunc(token.NoPos, pkg, "Entry", signature) + pkg.Scope().Insert(input.Declaration) + input.Package = pkg + var err error + if role == "language" { + value, failure := numericalSDKDeclarationLanguage(input.Declaration, types.SizesFor("gc", "amd64"), true) + err = failure + if value != "" { + t.Fatal("caller-controlled type gained language identity", value) + } + } else { + plan, failure := amd64GoArgumentEntryPlan(input) + err = failure + if plan != nil { + t.Fatal("caller-controlled result acquired an entry plan", plan) + } + } + if !errors.Is(err, ErrProbeNeedsContext) || calls != 0 { + t.Fatalf("result type reached a callback before rejection: err=%v, calls=%d", err, calls) + } + }) + } +} diff --git a/amd64_go_argument_plan.go b/amd64_go_argument_plan.go new file mode 100644 index 000000000..9c0944257 --- /dev/null +++ b/amd64_go_argument_plan.go @@ -0,0 +1,308 @@ +package plan9asm + +import ( + "fmt" + "go/types" +) + +// amd64GoArgumentPlan is a private language-layout calculation, not an entry +// certificate. In particular, it supplies no initialized bytes, native pointer +// backing, FP/spill storage, closure context, or permission to lower a source. +// The private entry consumer separately authenticates the actual declaration, +// selected compiler/ABI, LLVM transport and complete source before using it. +type amd64GoArgumentPlan struct { + Arguments []amd64GoArgumentAssignment + GPUsed int + FPUsed int +} + +type amd64GoArgumentAssignment struct { + Index int + Type types.Type // Preserve the original Go parameter grouping and identity. + Stack bool // Descriptive only: there is no stack/backing model here. + Carriers []amd64GoArgumentCarrier +} + +type amd64GoArgumentKind uint8 + +const ( + amd64GoArgumentInteger amd64GoArgumentKind = iota + 1 + amd64GoArgumentBoolean + amd64GoArgumentPointer + amd64GoArgumentFloating +) + +type amd64GoArgumentCarrier struct { + Path []int // extractvalue path relative to this one LLVM argument. + Offset int64 // Go language-model byte offset, not a native memory grant. + Width int64 // Only these physical bytes carry the declared scalar value. + Type LLVMType + Kind amd64GoArgumentKind + Register Reg +} + +const amd64GoArgumentPlanLimit = 4096 + +func amd64GoArgumentContext(reason string) error { + return fmt.Errorf("%w: AMD64 Go argument plan: %s", ErrProbeNeedsContext, reason) +} + +func planAMD64GoArguments(sig *types.Signature) (*amd64GoArgumentPlan, error) { + if sig == nil || sig.Recv() != nil || sig.Variadic() || sig.TypeParams().Len() != 0 || sig.RecvTypeParams().Len() != 0 { + return nil, amd64GoArgumentContext("requires a fixed monomorphic direct function") + } + if sig.Params().Len() > amd64GoArgumentPlanLimit { + return nil, amd64GoArgumentContext("parameter inventory exceeds the bounded plan") + } + plan := &amd64GoArgumentPlan{} + builder := amd64GoArgumentBuilder{sizes: types.SizesFor("gc", "amd64")} + registers := [...]Reg{AX, BX, CX, DI, SI, "R8", "R9", "R10", "R11"} + for index := 0; index < sig.Params().Len(); index++ { + typ := sig.Params().At(index).Type() + shape, err := builder.shape(typ, 0) + if err != nil { + return nil, err + } + argument := amd64GoArgumentAssignment{Index: index, Type: typ} + gp, fp := 0, 0 + for _, leaf := range shape.leaves { + if leaf.Kind == amd64GoArgumentFloating { + fp++ + } else { + gp++ + } + } + // Go's abiutils.tryAllocRegs checks the complete parameter before + // allocateRegs advances either bank. A failure consumes neither bank; + // a subsequent parameter may still use the remaining registers. + argument.Stack = shape.size == 0 || shape.stack || gp > len(registers)-plan.GPUsed || fp > 15-plan.FPUsed + if !argument.Stack { + argument.Carriers = shape.leaves + for leaf := range argument.Carriers { + carrier := &argument.Carriers[leaf] + if carrier.Kind == amd64GoArgumentFloating { + carrier.Register = Reg(fmt.Sprintf("X%d", plan.FPUsed)) + plan.FPUsed++ + } else { + carrier.Register = registers[plan.GPUsed] + plan.GPUsed++ + } + } + } + plan.Arguments = append(plan.Arguments, argument) + } + return plan, nil +} + +type amd64GoArgumentShape struct { + size, align int64 + stack bool + leaves []amd64GoArgumentCarrier +} + +type amd64GoArgumentBuilder struct { + sizes types.Sizes + nodes, leaves int +} + +// This models the ordinary Go language types in cmd/compile's ABI assignment: +// abiutils.allocateRegs/tryAllocRegs and types.CalcArraySize. Compiler-intrinsic +// SIMD types and an actual compiler's physical layout still need their own +// producer validation. go/types sizes below are language-model offsets only; +// notably, an older go/types model is not native pointee or spill extent. +func (builder *amd64GoArgumentBuilder) shape(typ types.Type, depth int) (amd64GoArgumentShape, error) { + var shape amd64GoArgumentShape + builder.nodes++ + if typ == nil || depth > 32 || builder.nodes > amd64GoArgumentPlanLimit { + return shape, amd64GoArgumentContext("recursive or unbounded parameter type") + } + // Preserve argument.Type above, but inspect the actual RHS before using + // Underlying: an alias of a TypeParam otherwise looks like its constraint + // interface and can reach Sizes with no concrete layout. Rhs is expressed + // as a private interface so the base planner still compiles with Go1.20. + for { + // A user-supplied Type implementation can execute arbitrary callbacks + // from Underlying/String/TypeParams. Only concrete parser/compiler + // go/types nodes may reach a live source's serialized declaration query. + switch typ.(type) { + case *types.Basic, *types.Named, *types.Pointer, *types.Array, *types.Slice, + *types.Struct, *types.Map, *types.Chan, *types.Signature, *types.Interface, *types.TypeParam: + default: + if !goAMD64FrameIsAlias(typ) { + return shape, amd64GoArgumentContext("argument type is not an original concrete go/types node") + } + } + if generic, ok := typ.(interface{ TypeParams() *types.TypeParamList }); ok && generic.TypeParams().Len() != 0 { + return shape, amd64GoArgumentContext("generic parameter needs its separate type contract") + } + if !goAMD64FrameIsAlias(typ) { + break + } + alias, valid := typ.(interface{ Rhs() types.Type }) + if !valid { + return shape, amd64GoArgumentContext("alias lacks its actual RHS") + } + typ = alias.Rhs() + depth++ + builder.nodes++ + if typ == nil || depth > 32 || builder.nodes > amd64GoArgumentPlanLimit { + return shape, amd64GoArgumentContext("recursive or unbounded alias RHS") + } + } + if _, parameter := typ.(*types.TypeParam); parameter { + return shape, amd64GoArgumentContext("unresolved type parameter has no concrete argument layout") + } + add := func(llvm LLVMType, kind amd64GoArgumentKind, width, offset int64, path []int) error { + builder.leaves++ + if builder.leaves > amd64GoArgumentPlanLimit { + return amd64GoArgumentContext("scalar carrier inventory exceeds the bounded plan") + } + shape.leaves = append(shape.leaves, amd64GoArgumentCarrier{ + Path: path, Offset: offset, Width: width, Type: llvm, Kind: kind, + }) + return nil + } + finishLeaf := func(llvm LLVMType, kind amd64GoArgumentKind, width, offset int64, path []int) (amd64GoArgumentShape, error) { + err := add(llvm, kind, width, offset, path) + return builder.finish(typ, shape, err) + } + appendChild := func(child amd64GoArgumentShape, index int, offset int64) { + shape.stack = shape.stack || child.stack + for _, leaf := range child.leaves { + leaf.Path = append([]int{index}, leaf.Path...) + leaf.Offset += offset + shape.leaves = append(shape.leaves, leaf) + } + } + switch underlying := typ.Underlying().(type) { + case *types.Basic: + var llvm LLVMType + kind, width := amd64GoArgumentInteger, int64(0) + switch underlying.Kind() { + case types.Bool: + llvm, kind, width = I1, amd64GoArgumentBoolean, 1 + case types.Int8, types.Uint8: + llvm, width = I8, 1 + case types.Int16, types.Uint16: + llvm, width = I16, 2 + case types.Int32, types.Uint32: + llvm, width = I32, 4 + case types.Int, types.Uint, types.Uintptr, types.Int64, types.Uint64: + llvm, width = I64, 8 + case types.UnsafePointer: + llvm, kind, width = Ptr, amd64GoArgumentPointer, 8 + case types.Float32: + llvm, kind, width = "float", amd64GoArgumentFloating, 4 + case types.Float64: + llvm, kind, width = "double", amd64GoArgumentFloating, 8 + case types.Complex64, types.Complex128: + llvm, kind, width = "float", amd64GoArgumentFloating, 4 + if underlying.Kind() == types.Complex128 { + llvm, width = "double", 8 + } + if err := add(llvm, kind, width, 0, []int{0}); err != nil { + return shape, err + } + return finishLeaf(llvm, kind, width, width, []int{1}) + case types.String: + if err := add(Ptr, amd64GoArgumentPointer, 8, 0, []int{0}); err != nil { + return shape, err + } + return finishLeaf(I64, amd64GoArgumentInteger, 8, 8, []int{1}) + default: + return shape, amd64GoArgumentContext("unsupported or untyped scalar parameter") + } + return finishLeaf(llvm, kind, width, 0, nil) + case *types.Pointer, *types.Map, *types.Chan, *types.Signature: + // Pointer-shaped values are opaque carriers. Do not inspect or grant + // numerical bytes, canonical bools, callable bodies or pointee bounds. + return finishLeaf(Ptr, amd64GoArgumentPointer, 8, 0, nil) + case *types.Interface: + if !underlying.IsMethodSet() { + return shape, amd64GoArgumentContext("constraint interface is not a concrete argument value") + } + for index := 0; index < 2; index++ { + if err := add(Ptr, amd64GoArgumentPointer, 8, int64(index)*8, []int{index}); err != nil { + return shape, err + } + } + case *types.Slice: + if err := add(Ptr, amd64GoArgumentPointer, 8, 0, []int{0}); err != nil { + return shape, err + } + for index := 1; index < 3; index++ { + if err := add(I64, amd64GoArgumentInteger, 8, int64(index)*8, []int{index}); err != nil { + return shape, err + } + } + case *types.Array: + child, err := builder.shape(underlying.Elem(), depth+1) + if err != nil { + return shape, err + } + n := underlying.Len() + const maxInt64 = int64(1<<63 - 1) + if n < 0 || child.size > maxInt64-(child.align-1) { + return shape, amd64GoArgumentContext("unrepresentable array parameter layout") + } + stride := (child.size + child.align - 1) / child.align * child.align + if n > 0 && stride != 0 && n-1 > (maxInt64-child.size)/stride { + return shape, amd64GoArgumentContext("array parameter layout overflows") + } + switch n { + case 0: + case 1: + appendChild(child, 0, 0) + default: + shape.stack = true + } + case *types.Struct: + if underlying.NumFields() > amd64GoArgumentPlanLimit-builder.nodes { + return shape, amd64GoArgumentContext("field inventory exceeds the bounded plan") + } + fields := make([]*types.Var, underlying.NumFields()) + children := make([]amd64GoArgumentShape, len(fields)) + for index := range fields { + fields[index] = underlying.Field(index) + child, err := builder.shape(fields[index].Type(), depth+1) + if err != nil { + return shape, err + } + children[index] = child + } + // Descendants are concrete and bounded before go/types traverses them. + offsets := builder.sizes.Offsetsof(fields) + for index, child := range children { + if offsets[index] < 0 || child.size > (1<<63-1)-offsets[index] { + return shape, amd64GoArgumentContext("struct parameter layout overflows") + } + appendChild(child, index, offsets[index]) + } + default: + return shape, amd64GoArgumentContext("unsupported concrete parameter type") + } + return builder.finish(typ, shape, nil) +} + +func (builder *amd64GoArgumentBuilder) finish(typ types.Type, shape amd64GoArgumentShape, err error) (amd64GoArgumentShape, error) { + if err != nil { + return shape, err + } + shape.size, shape.align = builder.sizes.Sizeof(typ), builder.sizes.Alignof(typ) + if shape.size < 0 || shape.align <= 0 { + return shape, amd64GoArgumentContext("parameter lacks a representable language layout") + } + return shape, nil +} + +func (plan *amd64GoArgumentPlan) requireRegisters() error { + if plan == nil { + return amd64GoArgumentContext("missing complete argument plan") + } + for _, argument := range plan.Arguments { + if argument.Stack { + return amd64GoArgumentContext("stack-assigned argument needs an independent backing contract") + } + } + return nil +} diff --git a/amd64_go_argument_plan_test.go b/amd64_go_argument_plan_test.go new file mode 100644 index 000000000..79774b6ab --- /dev/null +++ b/amd64_go_argument_plan_test.go @@ -0,0 +1,417 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "reflect" + "strings" + "testing" +) + +func amd64ArgumentTestSignature(t *testing.T, parameters string) *types.Signature { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "arguments.go", "package arguments\nfunc Entry("+parameters+") int", 0) + if err != nil { + t.Fatal(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check("example.invalid/arguments", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return pkg.Scope().Lookup("Entry").Type().(*types.Signature) +} + +func TestAMD64GoArgumentPlanIndependentTypedCarriers(t *testing.T) { + type expected struct { + parameter int + path []int + offset int64 + width int64 + typ LLVMType + kind amd64GoArgumentKind + register Reg + } + tests := []struct { + name string + parameters string + gp, fp int + want []expected + }{ + {"strings", "a, b string", 4, 0, []expected{ + {0, []int{0}, 0, 8, Ptr, amd64GoArgumentPointer, AX}, + {0, []int{1}, 8, 8, I64, amd64GoArgumentInteger, BX}, + {1, []int{0}, 0, 8, Ptr, amd64GoArgumentPointer, CX}, + {1, []int{1}, 8, 8, I64, amd64GoArgumentInteger, DI}, + }}, + {"slices", "a, b []byte", 6, 0, []expected{ + {0, []int{0}, 0, 8, Ptr, amd64GoArgumentPointer, AX}, + {0, []int{1}, 8, 8, I64, amd64GoArgumentInteger, BX}, + {0, []int{2}, 16, 8, I64, amd64GoArgumentInteger, CX}, + {1, []int{0}, 0, 8, Ptr, amd64GoArgumentPointer, DI}, + {1, []int{1}, 8, 8, I64, amd64GoArgumentInteger, SI}, + {1, []int{2}, 16, 8, I64, amd64GoArgumentInteger, "R8"}, + }}, + {"independent-banks", "f float64, n uint32, b bool, s string", 4, 1, []expected{ + {0, nil, 0, 8, "double", amd64GoArgumentFloating, "X0"}, + {1, nil, 0, 4, I32, amd64GoArgumentInteger, AX}, + {2, nil, 0, 1, I1, amd64GoArgumentBoolean, BX}, + {3, []int{0}, 0, 8, Ptr, amd64GoArgumentPointer, CX}, + {3, []int{1}, 8, 8, I64, amd64GoArgumentInteger, DI}, + }}, + {"nested-padding-and-array-one", "a struct { B byte; F struct { X float32; N uint16 }; P [1]*byte }", 3, 1, []expected{ + {0, []int{0}, 0, 1, I8, amd64GoArgumentInteger, AX}, + {0, []int{1, 0}, 4, 4, "float", amd64GoArgumentFloating, "X0"}, + {0, []int{1, 1}, 8, 2, I16, amd64GoArgumentInteger, BX}, + {0, []int{2, 0}, 16, 8, Ptr, amd64GoArgumentPointer, CX}, + }}, + {"complex-and-opaque-handles", "z complex64, p *byte, m map[int]int, c chan byte, f func(), i interface{}", 6, 2, []expected{ + {0, []int{0}, 0, 4, "float", amd64GoArgumentFloating, "X0"}, + {0, []int{1}, 4, 4, "float", amd64GoArgumentFloating, "X1"}, + {1, nil, 0, 8, Ptr, amd64GoArgumentPointer, AX}, + {2, nil, 0, 8, Ptr, amd64GoArgumentPointer, BX}, + {3, nil, 0, 8, Ptr, amd64GoArgumentPointer, CX}, + {4, nil, 0, 8, Ptr, amd64GoArgumentPointer, DI}, + {5, []int{0}, 0, 8, Ptr, amd64GoArgumentPointer, SI}, + {5, []int{1}, 8, 8, Ptr, amd64GoArgumentPointer, "R8"}, + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + sig := amd64ArgumentTestSignature(t, test.parameters) + plan, err := planAMD64GoArguments(sig) + if err != nil { + t.Fatal(err) + } + if plan.GPUsed != test.gp || plan.FPUsed != test.fp || len(plan.Arguments) != sig.Params().Len() { + t.Fatalf("incorrect complete grouping/banks: %#v", plan) + } + var got []expected + for _, argument := range plan.Arguments { + if argument.Type != sig.Params().At(argument.Index).Type() || argument.Stack { + t.Fatalf("original parameter identity/group lost: %#v", argument) + } + for _, carrier := range argument.Carriers { + got = append(got, expected{argument.Index, carrier.Path, carrier.Offset, carrier.Width, carrier.Type, carrier.Kind, carrier.Register}) + } + } + if !reflect.DeepEqual(got, test.want) { + t.Fatalf("carrier map\n got %#v\nwant %#v", got, test.want) + } + if err := plan.requireRegisters(); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestAMD64GoArgumentPlanWholeArgumentSpillRollback(t *testing.T) { + for _, test := range []struct { + name, parameters string + spilled, next int + register Reg + gp, fp int + }{ + {"gp-string", "a,b,c,d,e,f,g,h uint64, s string, last uint64", 8, 9, "R11", 9, 0}, + {"fp-complex", "a,b,c,d,e,f,g,h,i,j,k,l,m,n float64, z complex128, last float64", 14, 15, "X14", 0, 15}, + {"both-bank-rollback", "a,b,c,d,e,f,g,h uint64, s struct { F float64; A,B uint64 }, last float64, n uint64", 8, 9, "X0", 9, 1}, + {"fp-failure-does-not-consume-gp", "a,b,c,d,e,f,g,h,i,j,k,l,m,n float64, s struct { P *byte; Z complex128 }, last uint64", 14, 15, AX, 1, 14}, + {"nontrivial-array", "a [2]uint32, n uint64", 0, 1, AX, 1, 0}, + {"nested-array", "a struct { N uint64; V [2]float64 }, n uint64", 0, 1, AX, 1, 0}, + {"zero-size", "a struct{}, n uint64", 0, 1, AX, 1, 0}, + } { + t.Run(test.name, func(t *testing.T) { + plan, err := planAMD64GoArguments(amd64ArgumentTestSignature(t, test.parameters)) + if err != nil { + t.Fatal(err) + } + spill := plan.Arguments[test.spilled] + if !spill.Stack || len(spill.Carriers) != 0 { + t.Fatalf("whole argument was partially assigned: %#v", spill) + } + next := plan.Arguments[test.next] + if next.Stack || len(next.Carriers) != 1 || next.Carriers[0].Register != test.register || plan.GPUsed != test.gp || plan.FPUsed != test.fp { + t.Fatalf("failed parameter consumed registers: %#v", plan) + } + if err := plan.requireRegisters(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("stack assignment acquired register/backing permission: %v", err) + } + }) + } +} + +func TestAMD64GoArgumentPlanPreservesGroupingAndZeroFields(t *testing.T) { + grouped, err := planAMD64GoArguments(amd64ArgumentTestSignature(t, "a,b,c,d,e,f,g,h uint64, pair struct { X,Y uint64 }")) + if err != nil { + t.Fatal(err) + } + separate, err := planAMD64GoArguments(amd64ArgumentTestSignature(t, "a,b,c,d,e,f,g,h uint64, x,y uint64")) + if err != nil { + t.Fatal(err) + } + if len(grouped.Arguments) != 9 || !grouped.Arguments[8].Stack || grouped.GPUsed != 8 || + len(separate.Arguments) != 10 || separate.Arguments[8].Stack || !separate.Arguments[9].Stack || + separate.Arguments[8].Carriers[0].Register != "R11" { + t.Fatal("flattening changed the original all-or-nothing parameter boundary") + } + for _, test := range []struct { + parameters string + stack bool + path []int + }{ + {"a struct { Z [0]uint64; N uint64 }", false, []int{1}}, + {"a struct { Z struct{}; N uint64 }", false, []int{1}}, + {"a struct { Z [1]struct{}; N uint64 }", false, []int{1}}, + // CalcArraySize makes even a zero-sized nontrivial array unsuitable + // for recursive register assignment of this nonzero-sized struct. + {"a struct { Z [2]struct{}; N uint64 }", true, nil}, + } { + t.Run(test.parameters, func(t *testing.T) { + plan, err := planAMD64GoArguments(amd64ArgumentTestSignature(t, test.parameters)) + if err != nil { + t.Fatal(err) + } + argument := plan.Arguments[0] + if argument.Stack != test.stack { + t.Fatalf("incorrect zero-field classification: %#v", argument) + } + if !test.stack && (len(argument.Carriers) != 1 || !reflect.DeepEqual(argument.Carriers[0].Path, test.path) || argument.Carriers[0].Offset != 0) { + t.Fatalf("zero field acquired storage/carriers or changed the field path: %#v", argument) + } + }) + } +} + +func TestAMD64GoArgumentPlanCompleteScalarKindsAndRegisterBanks(t *testing.T) { + for _, test := range []struct { + typ types.BasicKind + width int64 + llvm LLVMType + kind amd64GoArgumentKind + }{ + {types.Bool, 1, I1, amd64GoArgumentBoolean}, + {types.Int8, 1, I8, amd64GoArgumentInteger}, {types.Uint8, 1, I8, amd64GoArgumentInteger}, + {types.Int16, 2, I16, amd64GoArgumentInteger}, {types.Uint16, 2, I16, amd64GoArgumentInteger}, + {types.Int32, 4, I32, amd64GoArgumentInteger}, {types.Uint32, 4, I32, amd64GoArgumentInteger}, + {types.Int64, 8, I64, amd64GoArgumentInteger}, {types.Uint64, 8, I64, amd64GoArgumentInteger}, + {types.Int, 8, I64, amd64GoArgumentInteger}, {types.Uint, 8, I64, amd64GoArgumentInteger}, + {types.Uintptr, 8, I64, amd64GoArgumentInteger}, {types.UnsafePointer, 8, Ptr, amd64GoArgumentPointer}, + {types.Float32, 4, "float", amd64GoArgumentFloating}, {types.Float64, 8, "double", amd64GoArgumentFloating}, + } { + t.Run(types.Typ[test.typ].Name(), func(t *testing.T) { + parameter := types.NewVar(token.NoPos, nil, "value", types.Typ[test.typ]) + sig := types.NewSignatureType(nil, nil, nil, types.NewTuple(parameter), nil, false) + plan, err := planAMD64GoArguments(sig) + if err != nil { + t.Fatal(err) + } + carriers := plan.Arguments[0].Carriers + if len(carriers) != 1 || carriers[0].Width != test.width || carriers[0].Type != test.llvm || carriers[0].Kind != test.kind { + t.Fatalf("scalar kind/byte extent differs: %#v", carriers) + } + }) + } + for _, test := range []struct { + name string + typ string + registers []Reg + }{ + {"GP", "uint64", []Reg{AX, BX, CX, DI, SI, "R8", "R9", "R10", "R11"}}, + {"FP", "float64", []Reg{"X0", "X1", "X2", "X3", "X4", "X5", "X6", "X7", "X8", "X9", "X10", "X11", "X12", "X13", "X14"}}, + } { + t.Run(test.name, func(t *testing.T) { + parameters := strings.TrimSuffix(strings.Repeat(test.typ+",", len(test.registers)), ",") + plan, err := planAMD64GoArguments(amd64ArgumentTestSignature(t, parameters)) + if err != nil { + t.Fatal(err) + } + for index, argument := range plan.Arguments { + if argument.Stack || len(argument.Carriers) != 1 || argument.Carriers[0].Register != test.registers[index] { + t.Fatalf("incorrect bank order at %d: %#v", index, argument) + } + } + }) + } +} + +func TestAMD64GoArgumentPlanRejectsUnsupportedAndUnboundedTypes(t *testing.T) { + signature := func(typ types.Type) *types.Signature { + return types.NewSignatureType(nil, nil, nil, types.NewTuple(types.NewVar(token.NoPos, nil, "value", typ)), nil, false) + } + deep := types.Type(types.Typ[types.Uint64]) + for index := 0; index < 34; index++ { + deep = types.NewArray(deep, 1) + } + parameter := types.NewTypeParam(types.NewTypeName(token.NoPos, nil, "T", nil), types.NewInterfaceType(nil, nil).Complete()) + generic := types.NewNamed(types.NewTypeName(token.NoPos, nil, "Generic", nil), types.Typ[types.Uint64], nil) + generic.SetTypeParams([]*types.TypeParam{types.NewTypeParam(types.NewTypeName(token.NoPos, nil, "U", nil), types.NewInterfaceType(nil, nil).Complete())}) + constraint := types.NewInterfaceType(nil, []types.Type{types.NewUnion([]*types.Term{types.NewTerm(true, types.Typ[types.Int])})}).Complete() + tests := map[string]*types.Signature{ + "nil": nil, + "variadic": amd64ArgumentTestSignature(t, "values ...byte"), + "type-parameter": signature(parameter), + "generic-named": signature(generic), + "constraint": signature(constraint), + "untyped-integer": signature(types.Typ[types.UntypedInt]), + "invalid": signature(types.Typ[types.Invalid]), + "unbounded-depth": signature(deep), + "array-overflow": signature(types.NewArray(types.Typ[types.Uint64], 1<<62)), + "method": types.NewSignatureType(types.NewVar(token.NoPos, nil, "receiver", types.Typ[types.Int]), nil, nil, + types.NewTuple(), nil, false), + "generic-function": types.NewSignatureType(nil, nil, []*types.TypeParam{parameter}, types.NewTuple(), nil, false), + } + for name, sig := range tests { + t.Run(name, func(t *testing.T) { + plan, err := planAMD64GoArguments(sig) + if plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unsupported shape got a partial plan: %#v, %v", plan, err) + } + }) + } + if err := (*amd64GoArgumentPlan)(nil).requireRegisters(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("nil plan acquired a register role: %v", err) + } +} + +func TestAMD64GoArgumentPlanIndependentInventoryBudgets(t *testing.T) { + makeSignature := func(count int, typ types.Type) *types.Signature { + parameters := make([]*types.Var, count) + for index := range parameters { + parameters[index] = types.NewVar(token.NoPos, nil, "", typ) + } + return types.NewSignatureType(nil, nil, nil, types.NewTuple(parameters...), nil, false) + } + for _, count := range []int{4096, 4097} { + t.Run(fmt.Sprintf("parameters-%d", count), func(t *testing.T) { + plan, err := planAMD64GoArguments(makeSignature(count, types.Typ[types.Uint64])) + if count == 4096 { + if err != nil || len(plan.Arguments) != count || plan.GPUsed != 9 { + t.Fatalf("exact parameter boundary: %#v, %v", plan, err) + } + } else if plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("oversized parameter inventory: %#v, %v", plan, err) + } + }) + } + for _, count := range []int{2048, 2049} { + t.Run(fmt.Sprintf("complex-carriers-%d", count), func(t *testing.T) { + plan, err := planAMD64GoArguments(makeSignature(count, types.Typ[types.Complex128])) + if count == 2048 { + if err != nil || len(plan.Arguments) != count || plan.FPUsed != 14 { + t.Fatalf("exact carrier boundary: %#v, %v", plan, err) + } + } else if plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("oversized carrier inventory: %#v, %v", plan, err) + } + }) + } + for _, count := range []int{4095, 4096} { + t.Run(fmt.Sprintf("struct-nodes-%d", count+1), func(t *testing.T) { + fields := make([]*types.Var, count) + for index := range fields { + fields[index] = types.NewField(token.NoPos, nil, fmt.Sprintf("N%d", index), types.Typ[types.Uint64], false) + } + plan, err := planAMD64GoArguments(makeSignature(1, types.NewStruct(fields, nil))) + if count == 4095 { + if err != nil || len(plan.Arguments) != 1 || !plan.Arguments[0].Stack { + t.Fatalf("exact node boundary: %#v, %v", plan, err) + } + } else if plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("oversized node inventory: %#v, %v", plan, err) + } + }) + } + for _, count := range []int{32, 33} { + t.Run(fmt.Sprintf("depth-%d", count), func(t *testing.T) { + typ := types.Type(types.Typ[types.Uint64]) + for index := 0; index < count; index++ { + typ = types.NewArray(typ, 1) + } + plan, err := planAMD64GoArguments(makeSignature(1, typ)) + if count == 32 { + if err != nil || len(plan.Arguments[0].Carriers[0].Path) != count { + t.Fatalf("exact depth boundary: %#v, %v", plan, err) + } + } else if plan != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("oversized depth: %#v, %v", plan, err) + } + }) + } +} + +func TestAMD64GoArgumentPlanRetainsNamedIdentityAndIndependentPaths(t *testing.T) { + first := types.NewNamed(types.NewTypeName(token.NoPos, nil, "First", nil), types.Typ[types.Uint32], nil) + second := types.NewNamed(types.NewTypeName(token.NoPos, nil, "Second", nil), types.Typ[types.Uint32], nil) + aggregate := types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, nil, "A", first, false), + types.NewField(token.NoPos, nil, "B", second, false), + }, nil) + sig := types.NewSignatureType(nil, nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "value", aggregate)), nil, false) + plan, err := planAMD64GoArguments(sig) + if err != nil { + t.Fatal(err) + } + if plan.Arguments[0].Type != aggregate || aggregate.Field(0).Type() != first || aggregate.Field(1).Type() != second { + t.Fatal("same-width original language identities were replaced") + } + plan.Arguments[0].Carriers[0].Path[0] = 99 + again, err := planAMD64GoArguments(sig) + if err != nil || again.Arguments[0].Carriers[0].Path[0] != 0 || plan.Arguments[0].Carriers[1].Path[0] != 1 { + t.Fatal("one plan or sibling carrier mutated another owned path") + } +} + +func TestAMD64GoArgumentPlanDoesNotAuthorizeExistingSourceConsumers(t *testing.T) { + for _, parameters := range []string{"value string", "value []byte", "value float64", "value bool", "value struct { N uint64 }"} { + t.Run(parameters, func(t *testing.T) { + declaration := amd64ArgumentTestSignature(t, parameters) + plan, err := planAMD64GoArguments(declaration) + if err != nil || plan.requireRegisters() != nil { + t.Fatalf("valid private model was not calculated: %#v, %v", plan, err) + } + fn := types.NewFunc(token.NoPos, nil, "Entry", declaration) + sig, err := DeriveGoFuncABI0Signature("Entry", fn, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, "TEXT ·Entry(SB),4,$0-64\nMOVQ $1,AX\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx := &amd64Ctx{fpScalarSource: file.Funcs[0], sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("private allocation plan widened the existing register entry: %v", err) + } + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("private allocation plan became ABI0 result storage: %v", err) + } + }) + } +} + +func TestAMD64GoArgumentPlanEveryScalarWidthAndNoHiddenDX(t *testing.T) { + sig := amd64ArgumentTestSignature(t, "a int8, b uint16, c int32, d uint64, e bool, p *byte, f float32, g float64") + plan, err := planAMD64GoArguments(sig) + if err != nil { + t.Fatal(err) + } + widths := []int64{1, 2, 4, 8, 1, 8, 4, 8} + kinds := []amd64GoArgumentKind{amd64GoArgumentInteger, amd64GoArgumentInteger, amd64GoArgumentInteger, amd64GoArgumentInteger, + amd64GoArgumentBoolean, amd64GoArgumentPointer, amd64GoArgumentFloating, amd64GoArgumentFloating} + for index, argument := range plan.Arguments { + if len(argument.Carriers) != 1 || argument.Carriers[0].Width != widths[index] || argument.Carriers[0].Kind != kinds[index] { + t.Fatalf("incorrect declared byte kind/width at %d: %#v", index, argument) + } + if argument.Carriers[0].Register == DX { + t.Fatal("hidden DX context was assigned as an ordinary argument") + } + } + if strings.Contains(types.TypeString(sig, nil), "DX") { + t.Fatal("planner mutated the original logical declaration") + } +} diff --git a/amd64_go_argument_sdk_go127_test.go b/amd64_go_argument_sdk_go127_test.go new file mode 100644 index 000000000..f0ce92925 --- /dev/null +++ b/amd64_go_argument_sdk_go127_test.go @@ -0,0 +1,186 @@ +//go:build go1.27 + +package plan9asm + +import ( + "crypto/sha256" + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func amd64SDKArgumentTranslate(t *testing.T, api string, pkg GoPackage, file *File, options Options) (string, error) { + t.Helper() + if api == "text" { + return Translate(file, options) + } + context := llvm.NewContext() + defer context.Dispose() + var module llvm.Module + var err error + if api == "module" { + module, err = TranslateModuleInContext(context, file, options) + } else { + source := options.NumericalReads.source + original, failure := os.ReadFile(filepath.Join(source.request.Directory, source.request.AssemblyFile)) + if failure != nil { + t.Fatal(failure) + } + goos, _, _ := strings.Cut(source.request.Target, "/") + result, failure := translateGoModuleInContext(context, pkg, original, GoModuleOptions{ + GOARCH: options.Goarch, GOOS: goos, TargetTriple: options.TargetTriple, FileName: source.request.AssemblyFile, + NumericalReads: options.NumericalReads, ResolveSym: options.ResolveSym, + }) + err = failure + if result != nil { + module = result.Module + } + } + if !module.IsNil() { + defer module.Dispose() + if err == nil { + return module.String(), nil + } + } + return "", err +} + +func amd64SDKTypedArgumentsWholeSource(t *testing.T, member string, apis ...string) { + t.Helper() + if len(apis) == 0 { + apis = []string{"text", "module", "go-module"} + } + seen := make(map[string]bool) + for _, api := range apis { + if api != "text" && api != "module" && api != "go-module" || seen[api] { + t.Fatal("invalid or repeated complete-source API", api) + } + seen[api] = true + } + var llc string + if member == "compare_amd64.s" { + llc = findLLVM22Tool("llc") + if llc == "" { + t.Fatal("complete original Compare objects require LLVM22 llc") + } + } + ctx, request, metadata, _ := numericalSDKAssemblyFixture(t) + request.AssemblyFile = member + original, err := os.ReadFile(filepath.Join(request.Directory, member)) + if err != nil { + t.Fatal(err) + } + source, err := CaptureGoX86SDKNumericalReadSource(ctx, request) + if err != nil { + t.Fatal("unchanged complete original SDK source/compiler/CPU join", err) + } + defer closeNumericalConsumerSource(t, source) + pkg, file, options := numericalSDKOriginalTypedPackage(t, metadata, member) + if len(file.Funcs) != 3 || len(file.Data)+len(file.Globl) != 0 || len(source.reads) != 1 || + len(metadata.observation.Objects.Sources) != len(metadata.guard.selection.SFiles) { + t.Fatal("complete original three TEXT/all selected ASM/CPU inventory changed") + } + sizes := types.SizesFor("gc", "amd64") + options.Sigs, err = goSigsForAsmFile(pkg, file, options.ResolveSym, "amd64", sizes, sizes, nil) + if err != nil || len(options.Sigs) != 2 { + t.Fatal("actual full original declaration inventory", err) + } + before, err := x86CallerSourceFingerprint(file) + if err != nil { + t.Fatal(err) + } + bound, err := source.BindNumericalReads(ctx, pkg, file, options) + if err != nil { + t.Fatal("complete original typed carrier/compiler/source binding", err) + } + for _, api := range apis { + t.Run(api, func(t *testing.T) { + ir, err := amd64SDKArgumentTranslate(t, api, pkg, file, bound) + if member == "equal_amd64.s" { + // The complete source also has a custom tail-entry obligation; + // either may reject before the unbound 8(DX) closure read. + // The declared-byte consumer matrix independently checks that + // exact hidden read. Neither rejection is whole-file success. + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("whole equal source acquired an unproved entry or closure contract", err) + } + t.Log("complete original equal source remains Context (NOT file pass):", err) + return + } + if err != nil || ir == "" { + t.Fatal("whole original Compare source still needs an independent contract", err) + } + context := llvm.NewContext() + defer context.Dispose() + module, err := parseIRModuleInContext(context, ir) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal("complete translated Compare module did not parse", err) + } + if err := amd64ArgumentCompareEntryTypes(module); err != nil { + t.Fatal("complete original grouped entry transport differs", err) + } + for _, optimizeIR := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimizeIR), func(t *testing.T) { + input := ir + if optimizeIR { + input = x86MulDivRoundOptimize(t, input) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run("backend"+level, func(t *testing.T) { + x86PackedStringCompileObject(t, llc, options.TargetTriple, input, level) + }) + } + }) + } + }) + } + after, err := x86CallerSourceFingerprint(file) + if err != nil || after != before { + t.Fatal("consumer changed the original source", err) + } + if err := source.Verify(ctx); err != nil { + t.Fatal("final original numerical source Verify", err) + } + if err := metadata.Verify(ctx); err != nil { + t.Fatal("final original complete header/archive Verify", err) + } + afterBytes, err := os.ReadFile(filepath.Join(request.Directory, member)) + if err != nil || sha256.Sum256(afterBytes) != sha256.Sum256(original) { + t.Fatal("original SDK bytes changed", err) + } + role := "Context only; no object/native/closure/stack proof" + if member == "compare_amd64.s" { + role = fmt.Sprintf("%d LLVM22 object axes; no native/closure/stack proof", 4*len(apis)) + } + t.Logf("member=%s APIs=%v completeTEXTs=3 declaredEntries=2 allSelectedASM=%d sourceSHA=%x; %s", member, apis, len(metadata.guard.selection.SFiles), sha256.Sum256(original), role) +} + +// Each API owns a fresh complete capture, before/final verification and all +// four object axes. Scheduling them independently avoids spending one shared +// five-minute source lease across three expensive compiler observations. +// Together these roots retain the entire original three-TEXT, twelve-object +// scope; they do not cache or share mutable source observations. +func TestGoX86SDKAMD64CompareTypedArgumentsWholeSource(t *testing.T) { + amd64SDKTypedArgumentsWholeSource(t, "compare_amd64.s", "text") +} + +func TestGoX86SDKAMD64CompareTypedArgumentsWholeSourceModule(t *testing.T) { + amd64SDKTypedArgumentsWholeSource(t, "compare_amd64.s", "module") +} + +func TestGoX86SDKAMD64CompareTypedArgumentsWholeSourceGoModule(t *testing.T) { + amd64SDKTypedArgumentsWholeSource(t, "compare_amd64.s", "go-module") +} + +func TestGoX86SDKAMD64EqualTypedArgumentsWholeSource(t *testing.T) { + amd64SDKTypedArgumentsWholeSource(t, "equal_amd64.s") +} diff --git a/amd64_go_argument_sdk_partitions_test.go b/amd64_go_argument_sdk_partitions_test.go new file mode 100644 index 000000000..6c6d027ce --- /dev/null +++ b/amd64_go_argument_sdk_partitions_test.go @@ -0,0 +1,67 @@ +package plan9asm + +import ( + "go/ast" + "go/parser" + "go/token" + "strconv" + "testing" +) + +// Check scheduling ownership without loading LLVM or capturing an SDK source. +// Actual source, translation, object and lifetime checks remain in each owner. +func TestAMD64SDKCompareAPIIndependentOwners(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "amd64_go_argument_sdk_go127_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + functions := make(map[string]*ast.FuncDecl) + for _, declaration := range file.Decls { + if function, ok := declaration.(*ast.FuncDecl); ok { + functions[function.Name.Name] = function + } + } + for _, owner := range []struct{ name, api string }{ + {"TestGoX86SDKAMD64CompareTypedArgumentsWholeSource", "text"}, + {"TestGoX86SDKAMD64CompareTypedArgumentsWholeSourceModule", "module"}, + {"TestGoX86SDKAMD64CompareTypedArgumentsWholeSourceGoModule", "go-module"}, + } { + t.Run(owner.api, func(t *testing.T) { + function := functions[owner.name] + if function == nil { + t.Fatal("missing independent complete-source API owner", owner.name) + } + calls := 0 + ast.Inspect(function.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + callee, ok := call.Fun.(*ast.Ident) + if !ok || callee.Name != "amd64SDKTypedArgumentsWholeSource" { + return true + } + calls++ + if len(call.Args) != 3 { + t.Error("owner must select exactly one complete-source API") + return false + } + for index, want := range map[int]string{1: "compare_amd64.s", 2: owner.api} { + literal, ok := call.Args[index].(*ast.BasicLit) + if !ok || literal.Kind != token.STRING { + t.Error("member/API selection must be a literal") + continue + } + got, err := strconv.Unquote(literal.Value) + if err != nil || got != want { + t.Errorf("selection = %q, want %q", got, want) + } + } + return false + }) + if calls != 1 { + t.Errorf("complete-source helper calls = %d, want one", calls) + } + }) + } +} diff --git a/amd64_go_argument_type_callback_test.go b/amd64_go_argument_type_callback_test.go new file mode 100644 index 000000000..8b681f82a --- /dev/null +++ b/amd64_go_argument_type_callback_test.go @@ -0,0 +1,32 @@ +package plan9asm + +import ( + "errors" + "go/token" + "go/types" + "testing" +) + +type amd64ArgumentCallbackType struct{ calls *int } + +func (typ *amd64ArgumentCallbackType) Underlying() types.Type { + *typ.calls++ + return types.Typ[types.Uint64] +} +func (typ *amd64ArgumentCallbackType) String() string { *typ.calls++; return "uint64" } +func (typ *amd64ArgumentCallbackType) TypeParams() *types.TypeParamList { *typ.calls++; return nil } + +func TestAMD64GoArgumentEntryRejectsCallerTypeCallbacksBeforeSampling(t *testing.T) { + calls := 0 + typ := &amd64ArgumentCallbackType{&calls} + sig := types.NewSignatureType(nil, nil, nil, types.NewTuple(types.NewVar(token.NoPos, nil, "x", typ)), nil, false) + defer func() { + if failure := recover(); failure != nil { + t.Errorf("caller-controlled type reached compiler layout/panicked: %v (callbacks=%d)", failure, calls) + } + }() + plan, err := planAMD64GoArguments(sig) + if plan != nil || !errors.Is(err, ErrProbeNeedsContext) || calls != 0 { + t.Fatalf("non-source go/types implementation reached callback under a future owner lock: %#v, %v, calls=%d", plan, err, calls) + } +} diff --git a/amd64_go_argument_wrapper_inventory_go127_test.go b/amd64_go_argument_wrapper_inventory_go127_test.go new file mode 100644 index 000000000..182a408ea --- /dev/null +++ b/amd64_go_argument_wrapper_inventory_go127_test.go @@ -0,0 +1,119 @@ +//go:build go1.27 + +package plan9asm + +import ( + "crypto/sha256" + "go/types" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +// Reuse the existing complete live SDK producer. No extra assembly reference, +// declaration, compiler flag or synthetic wrapper is added to the SDK. This is +// an inventory, not a carrier-placement proof or successful file translation. +func amd64SDKArgumentWrapperInventory(t *testing.T, member string) { + t.Helper() + ctx, request, metadata, _ := numericalSDKAssemblyFixture(t) + request.AssemblyFile = member + before, err := os.ReadFile(filepath.Join(request.Directory, member)) + if err != nil { + t.Fatal(err) + } + source, err := CaptureGoX86SDKNumericalReadSource(ctx, request) + if err != nil { + t.Fatal("complete original compiler/source/CPU observation", err) + } + defer closeNumericalConsumerSource(t, source) + pkg, file, options := numericalSDKOriginalTypedPackage(t, metadata, member) + if len(file.Funcs) != 3 || len(file.Data) != 0 || len(file.Globl) != 0 { + t.Fatal("original complete assembly member changed; no TEXT may be cropped") + } + if metadata.observation.Objects == nil || len(metadata.observation.Objects.Members) < 2 || + len(metadata.observation.Objects.Sources) != len(metadata.guard.selection.SFiles) || + len(metadata.guard.selection.SFiles) == 0 || len(source.native.ArchiveMembers) != len(metadata.observation.Objects.Members) || + !reflect.DeepEqual(source.native.ArchiveMembers[1].Texts, metadata.observation.Objects.Members[1].Texts) { + t.Fatal("inventory lacks the complete selected package assembly/compiler archive") + } + sizes := types.SizesFor("gc", "amd64") + options.Sigs, err = goSigsForAsmFile(pkg, file, options.ResolveSym, "amd64", sizes, sizes, nil) + if err != nil { + t.Fatal("actual complete declaration signatures", err) + } + bound, err := source.BindNumericalReads(ctx, pkg, file, options) + if err != nil || bound.NumericalReads == nil { + t.Fatal("original declaration/compiler/native association", err) + } + names, err := numericalGoDeclarationLinknames(pkg.Syntax, pkg.Types) + if err != nil { + t.Fatal(err) + } + entries, _, err := numericalTypedSourceDeclarations(file, pkg.Path, "amd64", source.role, names, source.native.CompilerAliases) + if err != nil || len(entries) != len(options.Sigs) { + t.Fatal("actual bodyless entry inventory differs from original source", err) + } + generated, err := gostaticstorage.ObserveX86SDKCompilerText(metadata.observation.Header.CompilerObject, + metadata.observation.ObjectInputs.BuiltinSource, request.Target, metadata.observation.ObjectInputs.Referenced) + if err != nil { + t.Fatal("existing complete header compiler object", err) + } + selected := metadata.observation.Objects.Members[1].Texts + var ordered []string + for native := range entries { + ordered = append(ordered, native) + } + sort.Strings(ordered) + present := 0 + var absent []string + for _, native := range ordered { + entry := entries[native] + declaration, valid := pkg.Types.Scope().Lookup(entry.local).(*types.Func) + if !valid { + t.Fatal("inventory entry lacks its original typed declaration", native) + } + plan, err := planAMD64GoArguments(declaration.Type().(*types.Signature)) + if err != nil || plan.requireRegisters() != nil { + t.Fatal("original argument shape exceeds this private register-only model", native, err) + } + wrapper, found, err := amd64ArgumentWrapperInventory(generated, selected, native) + if err != nil { + t.Fatal("actual generated/selected wrapper inventory differs", native, err) + } + if !found { + absent = append(absent, native) + t.Logf("native=%q local=%q parameters=%d GP=%d FP=%d wrapper=ABSENT; no carrier evidence", native, entry.local, len(plan.Arguments), plan.GPUsed, plan.FPUsed) + continue + } + present++ + t.Logf("native=%q local=%q parameters=%d GP=%d FP=%d wrapperABI=%d namespace=%d flags=%d/%d bytes=%x relocations=%+v; placement UNPROVED", + native, entry.local, len(plan.Arguments), plan.GPUsed, plan.FPUsed, wrapper.ABI, wrapper.Namespace, wrapper.Flags, wrapper.Flags2, wrapper.Code, wrapper.Relocations) + } + if present+len(absent) != len(entries) { + t.Fatal("wrapper inventory omitted an original declared entry") + } + if err := source.Verify(ctx); err != nil { + t.Fatal("complete original numerical source final verification", err) + } + if err := metadata.Verify(ctx); err != nil { + t.Fatal("borrowed complete SDK source/header/archive final verification", err) + } + after, err := os.ReadFile(filepath.Join(request.Directory, member)) + if err != nil || sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("inventory changed the unchanged original assembly", err) + } + t.Logf("inventory-only member=%s originalTEXTs=%d selectedASM=%d entries=%d wrapperCandidates=%d explicitlyAbsent=%v; no translation or entry permission", + member, len(file.Funcs), len(metadata.guard.selection.SFiles), len(entries), present, absent) +} + +func TestGoX86SDKAMD64CompareArgumentWrapperInventory(t *testing.T) { + amd64SDKArgumentWrapperInventory(t, "compare_amd64.s") +} + +func TestGoX86SDKAMD64EqualArgumentWrapperInventory(t *testing.T) { + amd64SDKArgumentWrapperInventory(t, "equal_amd64.s") +} diff --git a/amd64_go_argument_wrapper_inventory_test.go b/amd64_go_argument_wrapper_inventory_test.go new file mode 100644 index 000000000..6d3ccbb3d --- /dev/null +++ b/amd64_go_argument_wrapper_inventory_test.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "bytes" + "errors" + "reflect" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +// This test-only inventory checks retained records, not argument placement or +// source authority. In particular, no wrapper is manufactured when one is +// absent, and a returned record never becomes a carrier or entry certificate. +func amd64ArgumentWrapperInventory(generated, selected []gostaticstorage.TextRecord, native string) (gostaticstorage.TextRecord, bool, error) { + fail := func() (gostaticstorage.TextRecord, bool, error) { + return gostaticstorage.TextRecord{}, false, amd64GoArgumentContext("incomplete or ambiguous original wrapper inventory") + } + if native == "" || len(generated) == 0 || len(selected) == 0 || + len(generated) > gostaticstorage.ObjectSymbolInventoryLimit || len(selected) > gostaticstorage.ObjectSymbolInventoryLimit { + return fail() + } + find := func(records []gostaticstorage.TextRecord) (gostaticstorage.TextRecord, int) { + var result gostaticstorage.TextRecord + count := 0 + for _, record := range records { + if record.Name == native && record.ABI == 0 { + result, count = record, count+1 + } + } + return result, count + } + first, firstCount := find(generated) + second, secondCount := find(selected) + if firstCount == 0 && secondCount == 0 { + return gostaticstorage.TextRecord{}, false, nil // An explicit absence, not a successful placement proof. + } + if firstCount != 1 || secondCount != 1 || first.Kind != 1 || first.Flags2&64 == 0 || + first.Namespace != gostaticstorage.TextSymbolSelf && first.Namespace != gostaticstorage.TextSymbolNonpkgDefinition || + first.Size <= 0 || first.Size != int64(len(first.Code)) || first.Namespace != second.Namespace || + first.Kind != second.Kind || first.Flags != second.Flags || first.Flags2 != second.Flags2 || + first.Size != second.Size || !bytes.Equal(first.Code, second.Code) || !reflect.DeepEqual(first.Relocations, second.Relocations) { + return fail() + } + calls := 0 + for _, relocation := range first.Relocations { + if relocation.Target.Name != native { + continue + } + // Go1.27's already registered R_CALL record must identify an actual + // direct near CALL field. This does not decode or certify its inputs. + if relocation.Type != 7 || relocation.Size != 4 || relocation.Add != 0 || + relocation.Offset < 1 || relocation.Offset > int64(len(first.Code))-4 || + first.Code[relocation.Offset-1] != 0xe8 || !relocation.Target.ABIKnown || relocation.Target.ABI != 1 { + return fail() + } + calls++ + } + if calls != 1 { + return fail() + } + return first, true, nil +} + +func TestAMD64GoArgumentWrapperInventoryCompleteRecords(t *testing.T) { + for _, namespace := range []gostaticstorage.TextSymbolNamespace{gostaticstorage.TextSymbolSelf, gostaticstorage.TextSymbolNonpkgDefinition} { + input := numericalCompilerAliasModel(false) + input.Generated[0].Namespace = namespace + input.Selected[0].Namespace = namespace + if namespace == gostaticstorage.TextSymbolSelf { + input.Generated[0].Flags2, input.Selected[0].Flags2 = 64, 64 + } + if _, present, err := amd64ArgumentWrapperInventory(input.Generated, input.Selected, input.Native); err != nil || !present { + t.Fatal("matched original candidate was not inventoried", err) + } + if record, present, err := amd64ArgumentWrapperInventory(input.Generated, input.Selected, "example.invalid/remote.Absent"); err != nil || present || record.Name != "" { + t.Fatal("absence was replaced with a manufactured wrapper", record, present, err) + } + } +} + +func TestAMD64GoArgumentWrapperInventoryRejectsPartialOrAmbiguousRecords(t *testing.T) { + for name, mutate := range map[string]func(*numericalCompilerAliasInput){ + "missing-generated": func(input *numericalCompilerAliasInput) { input.Generated = nil }, + "missing-selected": func(input *numericalCompilerAliasInput) { input.Selected = nil }, + "one-sided-absence": func(input *numericalCompilerAliasInput) { input.Generated[0].Name += "Other" }, + "duplicate-generated": func(input *numericalCompilerAliasInput) { + input.Generated = append(input.Generated, input.Generated[0]) + }, + "duplicate-selected": func(input *numericalCompilerAliasInput) { + input.Selected = append(input.Selected, input.Selected[0]) + }, + "not-wrapper": func(input *numericalCompilerAliasInput) { input.Generated[0].Flags2 &^= 64 }, + "wrong-kind": func(input *numericalCompilerAliasInput) { input.Generated[0].Kind = 2 }, + "wrong-code": func(input *numericalCompilerAliasInput) { input.Generated[0].Code = []byte{0xcc} }, + "namespace": func(input *numericalCompilerAliasInput) { + input.Generated[0].Namespace = gostaticstorage.TextSymbolHashed + }, + "not-call-opcode": func(input *numericalCompilerAliasInput) { + input.Generated[0].Code = []byte{0xe9, 0, 0, 0, 0, 0xc3} + input.Selected[0].Code = append([]byte(nil), input.Generated[0].Code...) + }, + "wrong-call-relocation": func(input *numericalCompilerAliasInput) { + input.Generated[0].Relocations = append([]gostaticstorage.TextRelocation(nil), input.Generated[0].Relocations...) + input.Generated[0].Relocations[0].Type = 14 + input.Selected[0].Relocations = append([]gostaticstorage.TextRelocation(nil), input.Generated[0].Relocations...) + }, + "unknown-call-ABI": func(input *numericalCompilerAliasInput) { + input.Generated[0].Relocations = append([]gostaticstorage.TextRelocation(nil), input.Generated[0].Relocations...) + input.Generated[0].Relocations[0].Target.ABIKnown = false + input.Selected[0].Relocations = append([]gostaticstorage.TextRelocation(nil), input.Generated[0].Relocations...) + }, + } { + t.Run(name, func(t *testing.T) { + input := numericalCompilerAliasModel(false) + mutate(&input) + if _, present, err := amd64ArgumentWrapperInventory(input.Generated, input.Selected, input.Native); present || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("partial records became an original wrapper candidate", present, err) + } + }) + } +} diff --git a/amd64_go_assembler_test.go b/amd64_go_assembler_test.go new file mode 100644 index 000000000..16f2a8485 --- /dev/null +++ b/amd64_go_assembler_test.go @@ -0,0 +1,18 @@ +package plan9asm + +import ( + "runtime" + "testing" +) + +// requireX86GoAssemblerResult keeps the Go 1.27 table oracle available to +// tests that must remain visible to llgo. Older compatibility toolchains still +// exercise parsing, translation, and LLVM compilation, but cannot define the +// current operand grammar. +func requireX86GoAssemblerResult(t *testing.T, goarch, src string, wantSuccess bool) { + t.Helper() + if !goToolchainAtLeast(runtime.Version(), 1, 27) { + return + } + goAssemblerOracleOutput(t, goarch, src, "", wantSuccess) +} diff --git a/amd64_go_caller_acyclic_domain_test.go b/amd64_go_caller_acyclic_domain_test.go new file mode 100644 index 000000000..8ef5a191e --- /dev/null +++ b/amd64_go_caller_acyclic_domain_test.go @@ -0,0 +1,213 @@ +package plan9asm + +import ( + "math" + "reflect" + "testing" +) + +func x86CallerJoinDomainState() (x86SourceStackState, *x86GoCallerRegionProof) { + proof := &x86GoCallerRegionProof{arguments: []x86CallerArgumentRegion{ + {kind: x86CallerSliceInput, width: 4}, {kind: x86CallerSliceInput, width: 4}, + {kind: x86CallerFreshResult, width: 4}, {kind: x86CallerLocalLength, width: 8}, + }} + state := x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, numericFlags: true, gOriginal: true, + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}, SI: {argument: 1}, DX: {argument: 2}, CX: {argument: 3}}, + known: map[Reg]uint8{AX: 0xff}, vectors: map[Reg]uint64{"Z0": ^uint64(0)}, masks: map[Reg]bool{"K1": true}, + } + return state, proof +} + +func TestX86GoCallerAcyclicJoinDomainIntersectionAndNoAliasing(t *testing.T) { + left, proof := x86CallerJoinDomainState() + left.cells = []x86SourceStackCell{{offset: -8, width: 8, + bp: &x86SourceStackAddress{original: true}, caller: &x86CallerPointerValue{argument: 0}}} + right := x86CloneCallerState(left) + right.callers[DI] = x86CallerPointerValue{argument: 0, offset: 4} + right.known[AX] = 1 + right.known[BX] = 0xff + right.vectors["Z0"] = 0xf + right.vectors["Z1"] = ^uint64(0) + delete(right.masks, "K1") + right.masks["K2"] = true + beforeLeft, beforeRight := x86CloneCallerState(left), x86CloneCallerState(right) + merged, valid := x86CallerAcyclicMerge(left, right, proof) + if !valid || merged.callers[DI] != (x86CallerPointerValue{argument: 0, span: 4}) || + merged.known[AX] != 1 || merged.known[BX] != 0 || merged.vectors["Z0"] != 0xf || + merged.vectors["Z1"] != 0 || merged.masks["K1"] || merged.masks["K2"] { + t.Fatalf("not an all-path same-origin intersection: %+v, %t", merged, valid) + } + reversed, valid := x86CallerAcyclicMerge(right, left, proof) + if !valid || !reflect.DeepEqual(merged, reversed) { + t.Fatal("merge depends on predecessor arrival order") + } + merged.callers[DI] = x86CallerPointerValue{} + merged.known[AX] = 0 + merged.vectors["Z0"] = 0 + merged.masks["K1"] = true + merged.cells[0].bp.offset = 7 + merged.cells[0].caller.offset = 7 + if !reflect.DeepEqual(left, beforeLeft) || !reflect.DeepEqual(right, beforeRight) { + t.Fatal("merged state aliases a predecessor map, cell or saved pointer") + } +} + +func TestX86GoCallerAcyclicJoinDomainRejectsChangedFixedOrigins(t *testing.T) { + for _, change := range []string{ + "missing origin", "different argument", "result offset", "length offset", "SP", "BP", "g", "flags", "saved cell", + "span on result", "negative span", "overflow span", "past offset cap", "past end cap", + } { + t.Run(change, func(t *testing.T) { + left, proof := x86CallerJoinDomainState() + right := x86CloneCallerState(left) + switch change { + case "missing origin": + delete(right.callers, DI) + case "different argument": + right.callers[DI] = x86CallerPointerValue{argument: 1} + case "result offset": + right.callers[DX] = x86CallerPointerValue{argument: 2, offset: 1} + case "length offset": + right.callers[CX] = x86CallerPointerValue{argument: 3, offset: 1} + case "SP": + right.sp = -8 + case "BP": + right.bp.original = false + case "g": + right.gOriginal = false + case "flags": + right.numericFlags = false + case "saved cell": + right.cells = []x86SourceStackCell{{offset: -8, width: 8}} + case "span on result": + left.callers[DX] = x86CallerPointerValue{argument: 2, span: 1} + right.callers[DX] = left.callers[DX] + case "negative span": + right.callers[DI] = x86CallerPointerValue{argument: 0, span: -1} + case "overflow span": + right.callers[DI] = x86CallerPointerValue{argument: 0, offset: 1, span: math.MaxInt64} + case "past offset cap": + right.callers[DI] = x86CallerPointerValue{argument: 0, offset: -x86SourceStackLimit - 1} + case "past end cap": + right.callers[DI] = x86CallerPointerValue{argument: 0, offset: x86SourceStackLimit, span: 1} + } + if _, valid := x86CallerAcyclicMerge(left, right, proof); valid { + t.Fatal("changed fixed state or invalid span acquired a join") + } + }) + } +} + +func TestX86GoCallerAcyclicJoinPointerArithmeticBounds(t *testing.T) { + for _, test := range []struct { + pointer x86CallerPointerValue + delta int64 + valid bool + }{ + {x86CallerPointerValue{offset: -4, span: 8}, 4, true}, + {x86CallerPointerValue{offset: -x86SourceStackLimit, span: 2 * x86SourceStackLimit}, 0, true}, + {x86CallerPointerValue{offset: x86SourceStackLimit - 4, span: 4}, 1, false}, + {x86CallerPointerValue{offset: -x86SourceStackLimit, span: 4}, -1, false}, + {x86CallerPointerValue{offset: 1, span: 4}, math.MaxInt64, false}, + {x86CallerPointerValue{offset: -1, span: 4}, math.MinInt64, false}, + {x86CallerPointerValue{offset: math.MaxInt64, span: 1}, 0, false}, + } { + shifted, valid := x86CallerPointerShift(test.pointer, test.delta) + if valid != test.valid || valid && shifted.span != test.pointer.span { + t.Fatalf("shift %+v by %d: %+v/%t", test.pointer, test.delta, shifted, valid) + } + } +} + +func TestX86GoCallerAcyclicJoinDomainCannotRecoverUnknownFacts(t *testing.T) { + left, proof := x86CallerJoinDomainState() + unknown := x86CloneCallerState(left) + delete(unknown.known, AX) + delete(unknown.vectors, "Z0") + delete(unknown.masks, "K1") + joined, valid := x86CallerAcyclicMerge(left, unknown, proof) + if !valid { + t.Fatal("same origins with unknown numerical bytes should intersect") + } + joined, valid = x86CallerAcyclicMerge(joined, left, proof) + if !valid || joined.known[AX] != 0 || joined.vectors["Z0"] != 0 || joined.masks["K1"] { + t.Fatal("later predecessor recovered an erased all-path fact") + } +} + +func TestX86GoCallerAcyclicJoinDomainAllPredecessorPermutations(t *testing.T) { + first, proof := x86CallerJoinDomainState() + second, third := x86CloneCallerState(first), x86CloneCallerState(first) + second.callers[DI] = x86CallerPointerValue{argument: 0, offset: -4} + third.callers[DI] = x86CallerPointerValue{argument: 0, offset: 12} + second.known[AX] = 0xf + third.known[AX] = 3 + second.vectors["Z0"] = 0xff + third.vectors["Z0"] = 0xf + second.known[BX], third.known[BX] = 0xff, 0xff + second.vectors["Z1"], third.vectors["Z1"] = 0xff, 0xff + second.masks["K2"], third.masks["K2"] = true, true + third.masks["K1"] = false + states := []x86SourceStackState{first, second, third} + var expected *x86SourceStackState + for _, order := range [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}} { + joined, valid := x86CallerAcyclicMerge(states[order[0]], states[order[1]], proof) + if valid { + joined, valid = x86CallerAcyclicMerge(joined, states[order[2]], proof) + } + if !valid || joined.callers[DI] != (x86CallerPointerValue{argument: 0, offset: -4, span: 16}) || + joined.known[AX] != 3 || joined.known[BX] != 0 || joined.vectors["Z0"] != 0xf || + joined.vectors["Z1"] != 0 || joined.masks["K1"] || joined.masks["K2"] { + t.Fatalf("permutation %v recovered a missing fact: %+v/%t", order, joined, valid) + } + if expected != nil && !reflect.DeepEqual(*expected, joined) { + t.Fatalf("three-predecessor merge depends on order %v", order) + } + copy := x86CloneCallerState(joined) + expected = © + } +} + +func TestX86GoCallerAcyclicJoinGraphRequiresCompleteReachableDAG(t *testing.T) { + for _, test := range []struct { + name, body string + acyclic bool + }{ + {"fallthrough", "MOVL $1,AX\nRET", true}, + {"diamond", "CMPQ AX,$0\nJE other\nMOVL $1,AX\nJMP done\nother:\nMOVL $2,AX\ndone:\nRET", true}, + {"dead cycle", "JMP done\ndead:\nJMP dead\ndone:\nRET", true}, + {"reachable cycle", "loop:\nCMPQ AX,$0\nJNE loop\nRET", false}, + {"cycle beyond join", "CMPQ AX,$0\nJE joined\nMOVL $1,AX\njoined:\nloop:\nJMP loop", false}, + {"PC forward", "JMP 1(PC)\nRET", true}, + {"PC self", "JMP 0(PC)", false}, + {"PC out of source", "JMP 3(PC)\nRET", false}, + {"unknown indirect target", "JMP AX\nRET", false}, + {"unknown label", "JMP missing\nRET", false}, + {"fall off source", "MOVL $1,AX", false}, + } { + t.Run(test.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·graph(SB),4,$0-0\n"+test.body+"\n") + if err != nil { + t.Fatal(err) + } + fn := file.Funcs[0] + labels := map[string]int{} + var instructions []int + for index, ins := range fn.Instrs { + if ins.Op == OpLABEL { + labels[ins.Args[0].Sym] = len(instructions) + } else { + instructions = append(instructions, index) + } + } + if got := x86CallerReachableAcyclic(fn, instructions, labels); got != test.acyclic { + t.Fatalf("reachable graph acyclic=%t, want %t", got, test.acyclic) + } + }) + } + if x86CallerReachableAcyclic(Func{}, nil, nil) || + x86CallerReachableAcyclic(Func{}, make([]int, 8193), nil) { + t.Fatal("empty or over-budget source acquired a new merge") + } +} diff --git a/amd64_go_caller_acyclic_join.go b/amd64_go_caller_acyclic_join.go new file mode 100644 index 000000000..82c687f7f --- /dev/null +++ b/amd64_go_caller_acyclic_join.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "math" + "reflect" +) + +// A span is an abstract set of offsets from the same authenticated input. +// It is not an allocation extent or a promise that an unsafe read cannot fault. +func x86CallerPointerEnd(pointer x86CallerPointerValue) (int64, bool) { + if pointer.span < 0 || pointer.offset > math.MaxInt64-pointer.span { + return 0, false + } + end := pointer.offset + pointer.span + return end, pointer.offset >= -x86SourceStackLimit && end <= x86SourceStackLimit +} + +func x86CallerPointerShift(pointer x86CallerPointerValue, delta int64) (x86CallerPointerValue, bool) { + if _, valid := x86CallerPointerEnd(pointer); !valid || + delta > 0 && pointer.offset > math.MaxInt64-delta || + delta < 0 && pointer.offset < math.MinInt64-delta { + return pointer, false + } + pointer.offset += delta + _, valid := x86CallerPointerEnd(pointer) + return pointer, valid +} + +func x86CloneCallerState(state x86SourceStackState) x86SourceStackState { + if state.cells != nil { + cells := make([]x86SourceStackCell, len(state.cells)) + copy(cells, state.cells) + state.cells = cells + } + for index := range state.cells { + if state.cells[index].bp != nil { + value := *state.cells[index].bp + state.cells[index].bp = &value + } + if state.cells[index].caller != nil { + value := *state.cells[index].caller + state.cells[index].caller = &value + } + } + state.callers = x86CloneCallerPointers(state.callers) + state.known = x86CloneCallerKnown(state.known) + state.vectors, state.masks = x86CloneCallerVectorState(state.vectors, state.masks) + return state +} + +// Only numerical definedness and same-origin read-only input offsets merge. +// Neither a missing origin nor a fixed local, stack or continuation can borrow +// this domain. Return a fresh deep clone, never edit either predecessor. +func x86CallerAcyclicMerge(left, right x86SourceStackState, proof *x86GoCallerRegionProof) (x86SourceStackState, bool) { + if proof == nil || left.sp != right.sp || left.bp != right.bp || + left.numericFlags != right.numericFlags || left.partialNumericFlags != right.partialNumericFlags || left.gOriginal != right.gOriginal || + !reflect.DeepEqual(left.cells, right.cells) || len(left.callers) != len(right.callers) { + return x86SourceStackState{}, false + } + merged := x86CloneCallerState(left) + for reg, a := range left.callers { + b, present := right.callers[reg] + if !present || a.argument != b.argument || a.argument < 0 || a.argument >= len(proof.arguments) { + return x86SourceStackState{}, false + } + aEnd, aValid := x86CallerPointerEnd(a) + bEnd, bValid := x86CallerPointerEnd(b) + if !aValid || !bValid { + return x86SourceStackState{}, false + } + if proof.arguments[a.argument].kind != x86CallerSliceInput { + if a != b || a.span != 0 { + return x86SourceStackState{}, false + } + continue + } + if b.offset < a.offset { + a.offset = b.offset + } + if bEnd > aEnd { + aEnd = bEnd + } + a.span = aEnd - a.offset // Both endpoints already have the 1 MiB bound. + merged.callers[reg] = a + } + // Include absent facts as zero, including on a later predecessor. Retain + // zero entries so replays do not depend on which predecessor arrived first. + for reg, value := range right.known { + merged.known[reg] = left.known[reg] & value + } + for reg, value := range left.known { + merged.known[reg] = value & right.known[reg] + } + for reg, value := range right.vectors { + merged.vectors[reg] = left.vectors[reg] & value + } + for reg, value := range left.vectors { + merged.vectors[reg] = value & right.vectors[reg] + } + for reg, value := range right.masks { + merged.masks[reg] = left.masks[reg] && value + } + for reg, value := range left.masks { + merged.masks[reg] = value && right.masks[reg] + } + return merged, true +} + +// A new merge cannot establish any cyclic invariant. This deliberately checks +// the entire reachable graph, not only the current edge or label's spelling. +func x86CallerReachableAcyclic(fn Func, instructions []int, labels map[string]int) bool { + if len(instructions) == 0 || len(instructions) > 8192 { + return false + } + edges := make(map[int][]int) + seen := make(map[int]bool) + work := []int{0} + for len(work) != 0 { + pc := work[0] + work = work[1:] + if pc < 0 || pc >= len(instructions) { + return false + } + if seen[pc] { + continue + } + seen[pc] = true + ins := fn.Instrs[instructions[pc]] + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpRET { + continue + } + next := []int{pc + 1} + if op == OpJMP || isAMD64ConditionalBranch(op) { + if len(ins.Args) != 1 { + return false + } + operand := ins.Args[0] + label := operand.Sym + if operand.Kind == OpIdent { + label = operand.Ident + } + target, valid := labels[label] + if operand.Kind == OpMem && operand.Mem.Base == PC && operand.Mem.Index == "" && + operand.Mem.Sym == "" && operand.Mem.Off >= -int64(len(instructions)) && operand.Mem.Off <= int64(len(instructions)) { + target, valid = pc+int(operand.Mem.Off), true + } else if operand.Kind != OpLabel && operand.Kind != OpIdent { + return false + } + if !valid { + return false + } + if op == OpJMP { + next = []int{target} + } else { + next = append(next, target) + } + } + edges[pc] = next + work = append(work, next...) + } + indegree := make(map[int]int, len(seen)) + for _, next := range edges { + for _, pc := range next { + indegree[pc]++ + } + } + work = work[:0] + for pc := range seen { + if indegree[pc] == 0 { + work = append(work, pc) + } + } + removed := 0 + for len(work) != 0 { + pc := work[0] + work = work[1:] + removed++ + for _, next := range edges[pc] { + indegree[next]-- + if indegree[next] == 0 { + work = append(work, next) + } + } + } + return removed == len(seen) +} diff --git a/amd64_go_caller_acyclic_join_test.go b/amd64_go_caller_acyclic_join_test.go new file mode 100644 index 000000000..e6b0647ed --- /dev/null +++ b/amd64_go_caller_acyclic_join_test.go @@ -0,0 +1,286 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerAcyclicJoinAssembly(body string) string { + return strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) +} + +func TestX86GoCallerAcyclicJoinActualSourceInputs(t *testing.T) { + for name, body := range map[string]string{ + "same input offsets": "MOVQ (CX),AX\nCMPQ AX,$1\nJE other\nADDQ $4,DI\nCMPQ AX,$1\nother:\nMOVL (DI),AX\nMOVL AX,(DX)", + "copy and affine range": "MOVQ (CX),AX\nCMPQ AX,$1\nJE other\nADDQ $4,DI\nCMPQ AX,$1\nother:\nMOVQ DI,BX\nLEAQ 4(BX),R11\nSUBQ $4,R11\nMOVL (R11),AX\nMOVL AX,(DX)", + "all path low GP bytes": "MOVQ (CX),AX\nCMPQ AX,$1\nJE narrow\nMOVL $42,BX\nJMP joined\nnarrow:\nMOVB $42,BL\njoined:\nMOVB BL,(DX)", + "unused SIMD definition": "VMOVUPS (DI),X0\nMOVQ (CX),AX\nCMPQ AX,$1\nJE joined\nVMOVUPS (SI),X1\njoined:\nVMOVSS X0,(DX)", + "unused mask definition": "MOVQ (CX),AX\nCMPQ AX,$1\nJE joined\nKMOVQ AX,K1\njoined:\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := x86CallerAcyclicJoinAssembly(body) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatalf("complete actual source acyclic join %s: %v", triple, err) + } + } + }) + } +} + +func TestX86GoCallerCountedLoopOriginalBackedgeNeedsFiniteProof(t *testing.T) { + // These are unchanged consecutive original directives and the actual + // backedge in Weaviate dot_avx256_amd64.s. The acyclic mechanism still cannot + // prove them; the new private finite replay proves only this bounded region, + // not the complete external function or its later unbounded loops. + body := `MOVQ (CX),R9 +LONG $0xff418d45 +LONG $0x03c1f641 +JE done +WORD $0x8944; BYTE $0xc8 +WORD $0xe083; BYTE $0x03 +LONG $0xc057f8c5 +WORD $0xc931 +LBB0_4: +LONG $0x0f10fac5 +LONG $0xb971e2c4; BYTE $0x06 +LONG $0x04c78348 +LONG $0x04c68348 +LONG $0x01c18348 +WORD $0xc839 +JNE LBB0_4 +done: +MOVL $7,AX +MOVL AX,(DX)` + assembly := x86CallerAcyclicJoinAssembly(body) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + var instructions []int + labels := map[string]int{} + for index, ins := range decoded.Instrs { + if ins.Op == OpLABEL { + labels[ins.Args[0].Sym] = len(instructions) + } else { + instructions = append(instructions, index) + } + } + if x86CallerReachableAcyclic(decoded, instructions, labels) { + t.Fatal("a natural backedge acquired DAG admission") + } + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("source-bound finite replay of only the first region: %v", err) + } + // Preserve the bounded but outside-budget variant as a real source Context, + // without hiding its original directives or adding a wrapper length check. + outsideBudget := strings.Replace(assembly, "WORD $0xe083; BYTE $0x03", "WORD $0xe083; BYTE $0x07", 1) + unsafePkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, outsideBudget) + if err := x86CallerCoreTranslate(t, unsafePkg, outsideBudget, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bound outside finite mechanism admitted: %v", err) + } +} + +func TestX86GoCallerAcyclicJoinRetainsUnknownPermissions(t *testing.T) { + for name, body := range map[string]string{ + "different base": "CMPQ AX,$0\nJE joined\nMOVQ SI,DI\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", + "lost pointer": "CMPQ AX,$0\nJE joined\nMOVQ $0,DI\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", + "readonly write": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVL AX,(DI)", + "result offset": "CMPQ AX,$0\nJE joined\nADDQ $1,DX\nCMPQ AX,$0\njoined:\nMOVB AX,(DX)", + "length offset": "CMPQ AX,$0\nJE joined\nADDQ $1,CX\nCMPQ AX,$0\njoined:\nMOVL (CX),AX\nMOVL AX,(DX)", + "upper range cap": "CMPQ AX,$0\nJE joined\nADDQ $1048576,DI\nCMPQ AX,$0\njoined:\nLEAQ 1(DI),BX\nMOVL (BX),AX\nMOVL AX,(DX)", + "partial pointer": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVW DI,BX\nMOVL (BX),AX\nMOVL AX,(DX)", + "truncated pointer": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nLEAL (DI),DI\nMOVL (DI),AX\nMOVL AX,(DX)", + "stack escape": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVQ DI,SP\nMOVL AX,(DX)", + "input escapes as data": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVQ DI,(DX)", + "unknown length": "MOVQ R11,AX\nCMPQ AX,$0\nJE joined\nADDQ $4,DI\njoined:\nMOVL AX,(DX)", + "late unknown GP": "CMPQ AX,$0\nJE missing\nMOVQ $7,BX\nJMP joined\nmissing:\nMOVB $7,BL\njoined:\nMOVQ BX,AX\nMOVL AX,(DX)", + "late unknown vector": "CMPQ AX,$0\nJE missing\nVMOVUPS (DI),Y0\nJMP joined\nmissing:\nMOVSS (DI),X0\njoined:\nVEXTRACTF128 $1,Y0,X1\nVMOVSS X1,(DX)", + "late unknown mask": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nKMOVQ AX,K1\njoined:\nVPADDD.Z Z0,Z0,K1,Z1\nVMOVSS X1,(DX)", + "mask0 cannot grant source": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nCMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nVPADDD.Z (DX),Z0,K1,Z1\nVMOVSS X1,(DX)", + "stale address flags": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", + "same state loop cannot borrow span": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nCMPQ AX,$0\nJNE joined\nMOVL (DI),AX\nMOVL AX,(DX)", + "unknown dead instruction": "JMP joined\nRDTSC\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := x86CallerAcyclicJoinAssembly(body) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown permission acquired a finite input span: %v", err) + } + }) + } +} + +func TestX86GoCallerAcyclicJoinRevokesProvisionalCertificatesForLateUnknown(t *testing.T) { + // The short predecessor can reach and certify the input load before the + // other predecessor arrives. A longer path then removes numerical bytes; + // neither the early certificate nor arrival order may admit the function. + for _, knownFirst := range []bool{true, false} { + fast, slow := "MOVQ $7,BX", "MOVB $7,BL" + if !knownFirst { + fast, slow = slow, fast + } + body := "CMPQ AX,$0\nJE slow\n" + fast + "\nJMP joined\nslow:\n" + + "MOVL $1,R10\nMOVL $2,R10\nMOVL $3,R10\nMOVL $4,R10\n" + slow + + "\njoined:\nMOVL (DI),AX\nMOVQ BX,AX\nMOVL AX,(DX)" + assembly := x86CallerAcyclicJoinAssembly(body) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + normalized := normalizeX86RawReturnSourceStack(decoded, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs, + }, true, file.Funcs[0]) + if normalized.x86SourceStack != nil || !strings.Contains(normalized.x86SourceStackFailure, "unproved caller-region effect: MOVQ") { + t.Fatalf("late unknown path acquired source permission: %s", normalized.x86SourceStackFailure) + } + for _, ins := range normalized.Instrs { + if len(ins.x86CallerMemory) != 0 || len(ins.x86SourceStackFP) != 0 { + t.Fatal("failed final source proof leaked a provisional instruction certificate") + } + } + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("late unknown predecessor was hidden by a prior proof: %v", err) + } + } +} + +func TestX86GoCallerAcyclicJoinFinalCertificatesAndSourceMutations(t *testing.T) { + body := "MOVQ (CX),AX\nCMPQ AX,$1\nJE joined\nADDQ $4,DI\nCMPQ AX,$1\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)" + for _, change := range []string{"none", "access width", "operand", "missing caller", "actual caller bytes"} { + t.Run(change, func(t *testing.T) { + assembly := x86CallerAcyclicJoinAssembly(body) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil || normalized.Funcs[0].x86SourceStack == nil { + t.Fatalf("complete final source proof: %v", err) + } + found := false + for index := range normalized.Funcs[0].Instrs { + ins := &normalized.Funcs[0].Instrs[index] + if ins.Op != "MOVL" || len(ins.Args) != 2 || ins.Args[0].Kind != OpMem { + continue + } + found = true + if !x86CallerMemoryCertified(*ins, 0) { + t.Fatal("final merged input read lacks a recomputed complete-width certificate") + } + if change == "access width" { + access := ins.x86CallerMemory[0] + access.width++ + ins.x86CallerMemory[0] = access + if x86CallerMemoryCertified(*ins, 0) || x86RawReturnMemoryBound(normalized.Funcs[0], normalized, options, true) { + t.Fatal("changed provisional access acquired final admission") + } + // Editing a private certificate also invalidates the sealed + // normalized snapshot; recapture must start from original source. + } else if change == "operand" { + ins.Args[0].Mem.Base = DX + } + } + if !found { + t.Fatal("missing final original input load") + } + if change == "missing caller" { + sig := sigs["caller.kernel"] + sig.x86CallerRegions = nil + sigs[sig.Name] = sig + } + if change == "actual caller bytes" { + pkg.SourceSelection.GoSources["caller.go"] = []byte(x86CallerFixtureSource + "\nvar exposed = kernel\n") + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed producer caller source retained its original permission: %v", err) + } + return + } + _, err = Translate(normalized, options) + if change == "none" { + if err != nil { + t.Fatal(err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("edited %s acquired a final caller proof: %v", change, err) + } + if change == "access width" { + fresh, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(fresh, options); err != nil { + t.Fatalf("recapture from unchanged original source: %v", err) + } + requireX86GoAssemblerResult(t, "amd64", assembly, true) + if err := x86CallerCoreTranslate(t, pkg, assembly, options.TargetTriple); err != nil { + t.Fatalf("fresh actual caller source and LLVM object: %v", err) + } + } + }) + } +} + +func TestX86GoCallerAcyclicJoinFiveTargetsRetain386Boundary(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + if target.arch == "amd64" { + assembly := x86CallerAcyclicJoinAssembly("CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)") + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, target.triple); err != nil { + t.Fatal(err) + } + return + } + // The actual caller producer remains AMD64-only. A 386 object is + // the existing independent scalar ABI0 control, not a copied span. + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "386"); !errors.Is(err, ErrProbeNeedsContext) || proof != nil { + t.Fatal("AMD64 input-origin proof was borrowed by mode32") + } + source, scalarSig, _ := x86SourceStackNumericalFixture("386", "@MOV @WORD(SP),AX\nCMPL AX,$0\nJE done\nADDL $7,AX\ndone:\n@MOV AX,@RESULT(SP)\nBYTE $0xc3") + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, x86SourceStackOptions("386", target.triple, scalarSig)) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mode32-control.ll", "mode32-control.o", module.String()) + }) + } +} diff --git a/amd64_go_caller_acyclic_runtime_test.go b/amd64_go_caller_acyclic_runtime_test.go new file mode 100644 index 000000000..3a8290e32 --- /dev/null +++ b/amd64_go_caller_acyclic_runtime_test.go @@ -0,0 +1,144 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerAcyclicRuntimeSources() (goSource, assembly, goTest, cOracle string) { + forms := []struct { + name, body string + width int + }{ + {"inputjoin", "CMPQ AX,$1\nJE joined\nADDQ $4,DI\nCMPQ AX,$1\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", 4}, + {"copyjoin", "CMPQ AX,$1\nJE joined\nADDQ $4,DI\nCMPQ AX,$1\njoined:\nMOVQ DI,BX\nLEAQ 8(BX),R11\nSUBQ $8,R11\nMOVL (R11),AX\nMOVL AX,(DX)", 4}, + {"reversedjoin", "CMPQ AX,$1\nJNE changed\nJMP joined\nchanged:\nADDQ $4,DI\nCMPQ AX,$1\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", 4}, + {"lowjoin", "CMPQ AX,$1\nJE narrow\nMOVL $42,BX\nJMP joined\nnarrow:\nMOVB $42,BL\njoined:\nMOVB BL,(DX)", 1}, + } + var declarations, machine, tests, c strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + tests.WriteString(`package caller +import "testing" +func TestOriginalAcyclicCallerValues(t *testing.T) { + values := []uint32{0,1,7,0x80000000,0xffffffff,0x12345678} + for _, value := range values { + for length:=1; length<=4; length++ { + x := []uint32{value,value^0xaabbccdd,19,23} + y := []uint32{31,37,41,43} + originalX, originalY := [4]uint32{x[0],x[1],x[2],x[3]}, [4]uint32{31,37,41,43} + canary := [4]uint64{11,17,29,53} + want := x[0] + if length!=1 { want=x[1] } +`) + c.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range forms { + declaration := strings.ReplaceAll(x86CallerFixtureSource, "float32", "uint32") + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", form.name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+form.name+"(", 1) + declarations.WriteString(declaration) + source := strings.ReplaceAll(x86CallerFixtureAssembly, "·kernel", "·"+form.name) + source = strings.Replace(source, "MOVL $7,AX", "MOVQ (CX),AX", 1) + machine.WriteString(strings.Replace(source, "MOVL AX,(DX)", form.body, 1)) + want := "want" + if form.width == 1 { + want = "uint32(42)" + } + fmt.Fprintf(&tests, ` + if got:=Call%s(x[:length],y); got!=%s || + [4]uint32{x[0],x[1],x[2],x[3]}!=originalX || [4]uint32{y[0],y[1],y[2],y[3]}!=originalY || + canary!=[4]uint64{11,17,29,53} { t.Fatalf("%s length=%%d got=%%#x",length,got) } +`, form.name, want, form.name) + fmt.Fprintf(&c, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", form.name, form.name) + } + tests.WriteString(" }\n }\n}\n") + c.WriteString(`int main(void) { + const uint32_t values[]={0,1,7,0x80000000u,0xffffffffu,0x12345678u}; + for (unsigned i=0;i= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := int64(form.byteWidth) + if form.scalar { + width = int64(form.bits / 8) + } + required := x86CallerVectorByteMask(width) + if form.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + for _, index := range []int{form.first, form.second} { + if form.literal && index == form.first { + continue + } + if index != form.memory && state.vectors[x86CallerVectorRoot(ins.Args[index].Reg)]&required != required { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + // A merely initialized mask does not prove any lane active. Merge needs + // the old numerical lane(s), zeroing does not; neither invents inputs. + if form.mask >= 0 && !form.zeroing && old&required != required { + return false + } + known := ^uint64(0) + if form.legacy { + known = old | required + } else if form.scalar { + upperX := x86CallerVectorByteMask(16) &^ required + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] + known = ^x86CallerVectorByteMask(16) | required | second&upperX + } + state.vectors[destination] = known + return true +} diff --git a/amd64_go_caller_add_sub_test.go b/amd64_go_caller_add_sub_test.go new file mode 100644 index 000000000..f085f46a6 --- /dev/null +++ b/amd64_go_caller_add_sub_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86GoCallerFloatAddSubOriginalSourceReachesObjects(t *testing.T) { + for _, fixture := range []struct { + name string + body string + }{ + {"original VADDPS LONG", "VMOVUPS (DI),Y2\nVMOVUPS (SI),Y3\nLONG $0xd358ecc5\nVMOVSS X2,(DX)"}, + {"named VADDPS", "VMOVUPS (DI),Y2\nVMOVUPS (SI),Y3\nVADDPS Y3,Y2,Y2\nVMOVSS X2,(DX)"}, + {"original VSUBSS family", "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVSUBSS X0,X1,X2\nVMOVSS X2,(DX)"}, + } { + t.Run(fixture.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", fixture.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerFloatAddSubWholeFamilyByteEffects(t *testing.T) { + for _, row := range x86FloatAddSubFormats("amd64") { + if strings.Contains(row, "(AX)") || strings.Contains(row, "8(AX)") { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86FloatAddSubProperties(ins, "amd64") + if !recognized || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatalf("missing shared family effect %s: %v", row, err) + } + for _, missing := range []int{-1, form.first, form.second, form.destination, 4} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0)}, masks: map[Reg]bool{"K1": true}} + if missing >= 0 && missing < len(ins.Args) && ins.Args[missing].Kind == OpReg { + state.vectors[x86CallerVectorRoot(ins.Args[missing].Reg)] &^= 1 + } + if missing == 4 { + state.masks["K1"] = false + } + before, beforeMasks := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerFloatAddSubStep(&ins, form, &state, nil) + want := missing == -1 || missing == form.destination && !form.spec.legacy && (form.mask < 0 || form.zeroing) || missing == 4 && form.mask < 0 + if accepted != want || !reflect.DeepEqual(state.masks, beforeMasks) || !accepted && !reflect.DeepEqual(state.vectors, before) { + t.Fatalf("%s missing %d: %t want %t, invented numerical/mask evidence", row, missing, accepted, want) + } + } + width := int64(form.byteWidth) + if form.spec.scalar { + width = int64(form.spec.bits / 8) + } + low := x86CallerVectorByteMask(width) + for _, old := range []uint64{0, 0xff00, 0xff00000000, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": low, "Z1": low | old, "Z2": low | old}, masks: map[Reg]bool{"K1": true}} + if !x86CallerFloatAddSubStep(&ins, form, &state, nil) { + t.Fatalf("low arithmetic requires unrelated unknown bytes: %s", row) + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + want := ^uint64(0) + if form.spec.legacy { + want = low | old + } else if form.spec.scalar { + want = ^x86CallerVectorByteMask(16) | low | old&(x86CallerVectorByteMask(16)&^low) + } + if state.vectors[destination] != want { + t.Fatalf("%s physical byte definitions %x want %x", row, state.vectors[destination], want) + } + } + } +} + +func TestX86GoCallerFloatAddSubDoesNotInventValuesOrCallerPermissions(t *testing.T) { + for name, body := range map[string]string{ + "unknown arithmetic source": "VADDPS X0,X1,X2\nVMOVSS X2,(DX)", + "SUB is not an integer zero idiom": "VSUBPS X0,X0,X2\nVMOVSS X2,(DX)", + "unknown destructive destination": "VMOVUPS (DI),X0\nADDPS X0,X1\nVMOVSS X1,(DX)", + "unknown merge": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nKMOVQ AX,K1\nVADDPS X0,X1,K1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVADDPS.Z X0,X1,K1,X2\nVMOVSS X2,(DX)", + "mask pointer bits": "VMOVUPS (DI),X0\nKMOVQ DI,K1\nVSUBPS.Z X0,X0,K1,X2\nVMOVSS X2,(DX)", + "unknown memory": "VMOVUPS (DI),X0\nVADDPS (BX),X0,X2\nVMOVSS X2,(DX)", + "physical stack": "VMOVUPS (DI),X0\nVADDPS 0(SP),X0,X2\nVMOVSS X2,(DX)", + "FP pointer contents": "VMOVUPS (DI),X0\nVADDPS a+0(FP),X0,X2\nVMOVSS X2,(DX)", + "read length as vector": "VMOVUPS (DI),X0\nVADDPS (CX),X0,X2\nVMOVSS X2,(DX)", + "read g": "VMOVUPS (DI),X0\nVADDPS (R14),X0,X2\nVMOVSS X2,(DX)", + "wide output": "VMOVUPS (DI),X0\nVADDPS X0,X0,X2\nVMOVUPS X2,(DX)", + "write input": "VMOVUPS (DI),X0\nVADDPS X0,X0,X2\nVMOVSS X2,(DI)", + "unknown CFG join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),X0\njoined:\nVADDPS X0,X0,X2\nVMOVSS X2,(DX)", + "scalar unknown upper reused": "VMOVSS (DI),X0\nVADDSS X0,X3,X2\nVADDPS X2,X2,X2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved add/sub source passed: %v", err) + } + }) + } +} + +func TestX86GoCallerFloatAddSubAllOpcodesInitializedMemory(t *testing.T) { + for _, opcode := range x86FloatAddSubOpcodes() { + t.Run(opcode, func(t *testing.T) { + args := "(DI),X1" + destination := "X1" + if strings.HasPrefix(opcode, "V") { + args += ",X2" + destination = "X2" + } + body := fmt.Sprintf("VMOVUPS (SI),Z1\n%s %s\nVMOVSS %s,(DX)", opcode, args, destination) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_caller_aes.go b/amd64_go_caller_aes.go new file mode 100644 index 000000000..f502016f5 --- /dev/null +++ b/amd64_go_caller_aes.go @@ -0,0 +1,24 @@ +package plan9asm + +func x86CallerAESByteStep(ins *Instr, f x86AESForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + width := x86CallerVectorByteMask(int64(f.byteWidth)) + source := state.vectors[x86CallerVectorRoot(ins.Args[f.source].Reg)] & width + if f.memory >= 0 { + if !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.byteWidth), false) { + return false + } + source = width + } + input := uint64(0) + if f.state >= 0 { + input = state.vectors[x86CallerVectorRoot(ins.Args[f.state].Reg)] & width + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + defined := x86AESDefinedBytes(f, source, input) + if !f.spec.vector { + state.vectors[destination] = state.vectors[destination]&^width | defined + } else { + state.vectors[destination] = ^width | defined + } + return true +} diff --git a/amd64_go_caller_binding.go b/amd64_go_caller_binding.go new file mode 100644 index 000000000..339cf9377 --- /dev/null +++ b/amd64_go_caller_binding.go @@ -0,0 +1,48 @@ +package plan9asm + +import "fmt" + +// GoX86RawReturnCallerBindingNeeded is only a loader cost predicate. A true +// result proves neither a caller region nor a valid source continuation. +func GoX86RawReturnCallerBindingNeeded(file *File, goarch string, resolve func(string) string, sigs map[string]FuncSig) bool { + if file == nil || goarch != "amd64" || resolve == nil { + return false + } + for _, fn := range file.Funcs { + if x86GoCallerRegionCandidate(fn, sigs[resolve(goTextSymbolForResolution(fn.Sym))]) { + return true + } + } + return false +} + +// BindGoX86RawReturnCallerRegions consumes the compiler's complete actual +// typed package selection. It returns new signatures with private source +// evidence; the whole-source normalizer must still prove every stack, memory +// effect and return. Pointer/noescape types or JSON hashes are insufficient. +func BindGoX86RawReturnCallerRegions(pkg GoPackage, file *File, goarch string, resolve func(string) string, sigs map[string]FuncSig) (map[string]FuncSig, error) { + if !GoX86RawReturnCallerBindingNeeded(file, goarch, resolve, sigs) { + return sigs, nil + } + if pkg.SourceSelection == nil { + return nil, fmt.Errorf("%w: bounded raw-return caller binding requires actual selected source", ErrProbeNeedsContext) + } + bound := make(map[string]FuncSig, len(sigs)) + for name, sig := range sigs { + bound[name] = sig + } + for _, fn := range file.Funcs { + name := resolve(goTextSymbolForResolution(fn.Sym)) + sig := bound[name] + if !x86GoCallerRegionCandidate(fn, sig) { + continue + } + proof, err := deriveX86GoCallerRegions(pkg, *pkg.SourceSelection, fn, sig, goarch) + if err != nil { + return nil, err + } + sig.x86CallerRegions = proof + bound[name] = sig + } + return bound, nil +} diff --git a/amd64_go_caller_binding_test.go b/amd64_go_caller_binding_test.go new file mode 100644 index 000000000..30b05c53a --- /dev/null +++ b/amd64_go_caller_binding_test.go @@ -0,0 +1,43 @@ +package plan9asm + +import ( + "errors" + "go/types" + "strings" + "testing" +) + +func TestX86GoCallerRegionBindingRequiresCompleteTypedSelection(t *testing.T) { + for _, change := range []string{"none", "missing selection", "missing sizes", "wrong sizes", "changed source", "missing assembly"} { + t.Run(change, func(t *testing.T) { + pkg, selected, _, sig := x86CallerFixture(t, x86CallerFixtureSource) + file := selected.Assembly["kernel_amd64.s"] + pkg.SourceSelection = &selected + sigs := map[string]FuncSig{sig.Name: sig} + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + switch change { + case "missing selection": + pkg.SourceSelection = nil + case "missing sizes": + selected.Sizes = nil + case "wrong sizes": + selected.Sizes = types.SizesFor("gc", "386") + case "changed source": + selected.GoSources["caller.go"] = []byte(x86CallerFixtureSource + "\nvar exposed = kernel\n") + case "missing assembly": + selected.SelectedAssemblyFiles = nil + } + bound, err := BindGoX86RawReturnCallerRegions(pkg, file, "amd64", resolve, sigs) + if change == "none" { + if err != nil || bound[sig.Name].x86CallerRegions == nil { + t.Fatalf("actual selection acquired no private binding: %v", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || bound != nil { + t.Fatalf("%s obtained complete/partial binding: %+v, %v", change, bound, err) + } + if sigs[sig.Name].x86CallerRegions != nil { + t.Fatal("caller-owned signature map was mutated") + } + }) + } +} diff --git a/amd64_go_caller_counted_loop.go b/amd64_go_caller_counted_loop.go new file mode 100644 index 000000000..d8901ae41 --- /dev/null +++ b/amd64_go_caller_counted_loop.go @@ -0,0 +1,496 @@ +package plan9asm + +import ( + "fmt" + "math" + "reflect" + "strings" +) + +// This private plan partitions numerical low bits, not allocation bounds. It +// never supplies a pointer, initial GP value, public option or serialized proof. +type x86CallerCountedPlan struct { + edges [][]int + loops []x86CallerCountedLoop + headers, latches map[int]int + original, normalized [32]byte +} + +type x86CallerCountedLoop struct { + body map[int]bool + header, latch, update int + counter, bound Reg +} + +func x86CallerCountedEdges(fn Func, instructions []int, labels map[string]int) ([][]int, bool) { + if len(instructions) == 0 || len(instructions) > 8192 { + return nil, false + } + edges := make([][]int, len(instructions)) + for pc, index := range instructions { + ins := fn.Instrs[index] + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpRET { + continue + } + edges[pc] = []int{pc + 1} + if op == OpJMP || isAMD64ConditionalBranch(op) { + if len(ins.Args) != 1 { + return nil, false + } + arg := ins.Args[0] + label := arg.Sym + if arg.Kind == OpIdent { + label = arg.Ident + } + target, valid := labels[label] + if arg.Kind == OpMem && arg.Mem.Base == PC && arg.Mem.Index == "" && arg.Mem.Sym == "" && + arg.Mem.Off >= -int64(len(instructions)) && arg.Mem.Off <= int64(len(instructions)) { + target, valid = pc+int(arg.Mem.Off), true + } else if arg.Kind != OpLabel && arg.Kind != OpIdent { + return nil, false + } + if !valid { + return nil, false + } + if op == OpJMP { + edges[pc] = []int{target} + } else { + edges[pc] = append(edges[pc], target) + } + } + for _, next := range edges[pc] { + if next < 0 || next >= len(instructions) { + return nil, false + } + } + } + return edges, true +} + +// Each SCC must retain the original one-loop rule: one entry edge, one exact +// adjacent comparison/latch, an immutable bound and one unavoidable +1 update. +// Its remaining edges must be a DAG, not an inferred cyclic merge invariant. +func deriveX86CallerCountedLoop(fn Func, instructions []int, edges [][]int, reachable []bool, + body map[int]bool, header, latchPC int) *x86CallerCountedLoop { + plan := &x86CallerCountedLoop{body: body, header: header, latch: latchPC, update: -1} + if plan.latch == 0 || !plan.body[plan.latch-1] { + return nil + } + latch := fn.Instrs[instructions[plan.latch]] + compare := fn.Instrs[instructions[plan.latch-1]] + if latch.Op != "JNE" || len(edges[plan.latch]) != 2 || edges[plan.latch][1] != plan.header || + (compare.Op != "CMPL" && compare.Op != "CMPQ") || len(compare.Args) != 2 || + compare.Args[0].Kind != OpReg || compare.Args[1].Kind != OpReg { + return nil + } + left, leftOK := x86CallerGP(compare.Args[0].Reg) + right, rightOK := x86CallerGP(compare.Args[1].Reg) + if !leftOK || !rightOK || left == right || left != compare.Args[0].Reg || right != compare.Args[1].Reg { + return nil + } + entries, updates := 0, 0 + for pc, nexts := range edges { + if !reachable[pc] { + continue + } + for _, next := range nexts { + if next == plan.latch && pc != plan.latch-1 { + return nil // The latch must consume this exact adjacent comparison. + } + if !plan.body[pc] && plan.body[next] { + if next != plan.header { + return nil + } + entries++ + } + if plan.body[pc] && !plan.body[next] && pc != plan.latch { + return nil + } + } + if !plan.body[pc] { + continue + } + ins := fn.Instrs[instructions[pc]] + if (ins.Op == "ADDL" || ins.Op == "ADDQ") && len(ins.Args) == 2 && + ins.Args[0].Kind == OpImm && ins.Args[0].ImmRaw == "" && ins.Args[0].Imm == 1 && + ins.Args[1].Kind == OpReg && (ins.Args[1].Reg == left || ins.Args[1].Reg == right) { + updates++ + plan.update, plan.counter = pc, ins.Args[1].Reg + } + } + if entries != 1 || updates != 1 { + return nil + } + plan.bound = left + if plan.counter == left { + plan.bound = right + } + // Every body path must pass the one update; both live values stay immutable + // otherwise. Conservatively reject even an unmodeled read of these registers. + for pc := range plan.body { + ins := fn.Instrs[instructions[pc]] + if pc == plan.update || ins.Op == "CMPL" || ins.Op == "CMPQ" || isAMD64ConditionalBranch(ins.Op) || ins.Op == OpJMP { + continue + } + for _, arg := range ins.Args { + if arg.Kind == OpReg { + root, _ := x86CallerGP(arg.Reg) + if root == plan.counter || root == plan.bound { + return nil + } + } + } + } + seen := map[int]bool{} + work := []int{plan.header} + for len(work) != 0 { + pc := work[0] + work = work[1:] + if pc == plan.update || seen[pc] { + continue + } + if pc == plan.latch { + return nil + } + seen[pc] = true + work = append(work, edges[pc]...) + } + return plan +} + +type x86CallerCountedNumber struct { + source, version int + mask, value uint64 + known bool +} + +type x86CallerCountedFlags struct { + pc int + op Op + regs [2]Reg + versions [2]int + equal, known bool +} + +type x86CallerCountedNumbers struct { + regs map[Reg]x86CallerCountedNumber + flags x86CallerCountedFlags + source, sequence int +} + +func (facts x86CallerCountedNumbers) clone() x86CallerCountedNumbers { + copy := facts + copy.regs = make(map[Reg]x86CallerCountedNumber, len(facts.regs)) + for reg, value := range facts.regs { + copy.regs[reg] = value + } + return copy +} + +// Relational evidence is independent of numericFlags. Unknown upper bits stay +// unknown; only a real readonly local-length load supplies the partition source. +func (facts *x86CallerCountedNumbers) step(pc, residue int, ins Instr, before x86SourceStackState, proof *x86GoCallerRegionProof) { + op := Op(normalizeInstructionOpcode(x86SourceMoveInstruction(ins, "amd64").Op)) + if effect, known, err := x86TypedCounterControlByteEffect(ins.Op, ins, "amd64"); known { + if err != nil { + for register := range facts.regs { + delete(facts.regs, register) + } + facts.flags = x86CallerCountedFlags{} + facts.sequence++ + return + } + for _, operand := range effect.operands { + if operand.write && operand.operand.Kind == OpReg { + facts.sequence++ + if register, valid := x86CallerGP(operand.operand.Reg); valid { + delete(facts.regs, register) + } + } + } + return // Preserve actual FLAGS, but never retain a decremented exact value/version. + } + if op == OpTEXT || op == OpRET || op == OpJMP || isAMD64ConditionalBranch(op) { + return + } + facts.sequence++ + if (op == "CMPL" || op == "CMPQ" || op == "TESTB" || op == "TESTW" || op == "TESTL" || op == "TESTQ") && len(ins.Args) == 2 { + facts.flags = x86CallerCountedFlags{pc: pc, op: op} + values := [2]x86CallerCountedNumber{} + for index, arg := range ins.Args { + if arg.Kind == OpImm && arg.ImmRaw == "" { + values[index] = x86CallerCountedNumber{source: -1, value: uint64(arg.Imm), known: true} + } else if arg.Kind == OpReg { + root, valid := x86CallerGP(arg.Reg) + if !valid || arg.Reg != root { + return + } + var present bool + values[index], present = facts.regs[root] + if !present { + return + } + facts.flags.regs[index], facts.flags.versions[index] = root, values[index].version + } else { + return + } + } + if strings.HasPrefix(string(op), "TEST") { + for index := range values { + other := values[1-index] + if !values[index].known && values[index].source == facts.source && facts.source >= 0 && other.known && other.value <= 3 { + values[index].value, values[index].known = uint64(residue), true + } + } + mask := uint64(math.MaxUint64) + switch op { + case "TESTB": + mask = math.MaxUint8 + case "TESTW": + mask = math.MaxUint16 + case "TESTL": + mask = math.MaxUint32 + } + facts.flags.equal = values[0].value&values[1].value&mask == 0 + } else { + if op == "CMPL" { + values[0].value, values[1].value = uint64(uint32(values[0].value)), uint64(uint32(values[1].value)) + } + facts.flags.equal = values[0].value == values[1].value + } + facts.flags.known = values[0].known && values[1].known + return + } + move := op == "MOVL" || op == "MOVQ" + _, conditionalMove, conditionalMoveError := x86ConditionalMoveProperties(op, ins, "amd64") + if !move && !(conditionalMove && conditionalMoveError == nil) && x86LEAWidth(op, "amd64") == 0 && op != "PUSHQ" && op != "POPQ" && + !strings.HasPrefix(string(op), "MOV") && !strings.HasPrefix(string(op), "VMOV") { + facts.flags = x86CallerCountedFlags{} + } + if len(ins.Args) == 0 || ins.Args[len(ins.Args)-1].Kind != OpReg { + return + } + arg := ins.Args[len(ins.Args)-1] + reg, valid := x86CallerGP(arg.Reg) + if !valid { + return + } + old, oldPresent := facts.regs[reg] + delete(facts.regs, reg) + if arg.Reg != reg || len(ins.Args) != 2 { + return + } + source := ins.Args[0] + value := x86CallerCountedNumber{source: -1, version: facts.sequence} + if move { + switch source.Kind { + case OpImm: + if source.ImmRaw != "" || source.ImmIsFloat { + return + } + value.value, value.known = uint64(source.Imm), true + case OpReg: + root, ok := x86CallerGP(source.Reg) + if !ok || root != source.Reg { + return + } + if root == reg { + value = old + } else { + var present bool + value, present = facts.regs[root] + if !present { + return + } + } + if !oldPresent && root == reg { + return + } + case OpMem: + pointer, present := before.callers[source.Mem.Base] + if op != "MOVQ" || !present || pointer.offset != 0 || pointer.span != 0 || + pointer.argument < 0 || pointer.argument >= len(proof.arguments) || proof.arguments[pointer.argument].kind != x86CallerLocalLength || + source.Mem.Off != 0 || source.Mem.Index != "" || source.Mem.Sym != "" || source.Mem.Segment != "" { + return + } + value.source = pc + if facts.source < 0 { + facts.source = pc + } + default: + return + } + if op == "MOVL" && value.known { + value.value = uint64(uint32(value.value)) + } + } else if (op == "XORL" || op == "XORQ") && source.Kind == OpReg && source.Reg == reg { + value.value, value.known = 0, true + } else if (op == "ANDL" || op == "ANDQ") && oldPresent && source.Kind == OpImm && source.ImmRaw == "" { + value = old + if old.known { + value.value &= uint64(source.Imm) + } else if old.source == facts.source && facts.source >= 0 && source.Imm > 0 && source.Imm <= 3 { + value.mask, value.value, value.known = uint64(source.Imm), uint64(residue)&uint64(source.Imm), true + } else { + return + } + } else if (op == "ADDL" || op == "ADDQ") && oldPresent && old.known && source.Kind == OpImm && source.ImmRaw == "" { + limit := uint64(math.MaxUint64) + if op == "ADDL" { + limit = math.MaxUint32 + } + if source.Imm < 0 || old.value > limit || uint64(source.Imm) > limit-old.value { + return + } + value.value, value.known = old.value+uint64(source.Imm), true + } else { + return + } + if (op == "ANDL" || op == "ADDL") && value.known { + value.value = uint64(uint32(value.value)) + } + value.version = facts.sequence + facts.regs[reg] = value +} + +type x86CallerCountedVisit struct { + pc, residue, iteration, loop int + state x86SourceStackState + numbers x86CallerCountedNumbers +} + +type x86CallerCountedWork struct { + x86CallerCountedVisit + bound x86CallerCountedNumber + entry x86SourceStackState +} + +func x86CallerCountedOrigins(entry, current x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if entry.sp != current.sp || entry.bp != current.bp || entry.gOriginal != current.gOriginal || + !reflect.DeepEqual(entry.cells, current.cells) || len(entry.callers) != len(current.callers) { + return false + } + for reg, pointer := range entry.callers { + other, present := current.callers[reg] + if !present || pointer.argument != other.argument || pointer.argument < 0 || pointer.argument >= len(proof.arguments) { + return false + } + if _, valid := x86CallerPointerEnd(other); !valid { + return false + } + if proof.arguments[pointer.argument].kind != x86CallerSliceInput && pointer != other { + return false + } + } + return true +} + +type x86CallerCountedTransfer func(int, *Instr, *x86SourceStackState) (bool, string) + +// Each feasible path keeps a fresh state, so no predecessor can supply another +// path's permission. The finite visit cap fails; it never widens a loop state. +func replayX86CallerCountedPlan(plan *x86CallerCountedPlan, fn *Func, instructions []int, initial x86SourceStackState, + proof *x86GoCallerRegionProof, transfer x86CallerCountedTransfer) ([]x86CallerCountedVisit, string) { + var visits []x86CallerCountedVisit + returned := 0 + for residue := 0; residue <= 3; residue++ { + work := []x86CallerCountedWork{{x86CallerCountedVisit: x86CallerCountedVisit{ + residue: residue, iteration: -1, loop: -1, state: x86CloneCallerState(initial), + numbers: x86CallerCountedNumbers{regs: map[Reg]x86CallerCountedNumber{}, source: -1}, + }}} + for len(work) != 0 { + if len(visits) >= 8192 { + return nil, "counted-loop finite path budget exhausted" + } + item := work[0] + work = work[1:] + pc := item.pc + if loopIndex, header := plan.headers[pc]; header { + loop := plan.loops[loopIndex] + bound, bOK := item.numbers.regs[loop.bound] + counter, cOK := item.numbers.regs[loop.counter] + if item.loop != -1 && item.loop != loopIndex { + return nil, "counted-loop entry crosses an active SCC" + } + if item.loop == -1 { + item.loop = loopIndex + item.iteration, item.bound = 0, bound + item.entry = x86CloneCallerState(item.state) + } + if !bOK || !cOK || !bound.known || !counter.known || bound.value == 0 || bound.value > 3 || + bound.source != item.numbers.source || bound.source < 0 || bound.mask == 0 || bound.mask > 3 || + !reflect.DeepEqual(bound, item.bound) || counter.value != uint64(item.iteration) || item.iteration >= 3 { + return nil, "counted-loop entry lacks immutable bounded value and zero-based counter" + } + } + visits = append(visits, item.x86CallerCountedVisit) + state, numbers := x86CloneCallerState(item.state), item.numbers.clone() + ins := &fn.Instrs[instructions[pc]] + ins.x86CallerMemory, ins.x86SourceStackFP = nil, nil + isReturn, reason := transfer(pc, ins, &state) + if reason != "" { + return nil, fmt.Sprintf("normalized instruction %d: %s", instructions[pc], reason) + } + if isReturn { + if item.loop != -1 { + return nil, "counted-loop return bypasses its exact latch" + } + returned++ + continue + } + numbers.step(pc, residue, *ins, item.state, proof) + nexts := plan.edges[pc] + if loopIndex, latch := plan.latches[pc]; latch { + loop := plan.loops[loopIndex] + flags, counter, bound := numbers.flags, numbers.regs[loop.counter], numbers.regs[loop.bound] + if item.loop != loopIndex { + return nil, "counted-loop latch lacks its original SCC entry" + } + if !flags.known || flags.pc != pc-1 || !counter.known || counter.value != uint64(item.iteration+1) || + !reflect.DeepEqual(bound, item.bound) || !x86CallerCountedOrigins(item.entry, state, proof) { + return nil, "counted-loop latch lacks exact counter, bound and comparison versions" + } + for index, reg := range flags.regs { + if (reg != loop.counter && reg != loop.bound) || flags.versions[index] != numbers.regs[reg].version { + return nil, "counted-loop comparison observes stale operand versions" + } + } + } + if (ins.Op == "JE" || ins.Op == "JNE") && numbers.flags.known { + taken := numbers.flags.equal == (ins.Op == "JE") + if taken { + nexts = nexts[1:] + } else { + nexts = nexts[:1] + } + } + for _, next := range nexts { + iteration, loopIndex := item.iteration, item.loop + bound, entry := item.bound, item.entry + if item.loop >= 0 { + loop := plan.loops[item.loop] + if pc == loop.latch && next == loop.header { + iteration++ + } else if !loop.body[next] { + if pc != loop.latch { + return nil, "counted-loop exit bypasses its exact latch" + } + // Clear only this loop's control witness. All source origins, + // initialized bytes, numerical versions and flags survive. + iteration, loopIndex = -1, -1 + bound, entry = x86CallerCountedNumber{}, x86SourceStackState{} + } + } + work = append(work, x86CallerCountedWork{x86CallerCountedVisit: x86CallerCountedVisit{ + pc: next, residue: residue, iteration: iteration, loop: loopIndex, + state: x86CloneCallerState(state), numbers: numbers.clone(), + }, bound: bound, entry: x86CloneCallerState(entry)}) + } + } + } + if returned == 0 { + return nil, "counted-loop replay has no original source return" + } + return visits, "" +} diff --git a/amd64_go_caller_counted_loop_boundary_test.go b/amd64_go_caller_counted_loop_boundary_test.go new file mode 100644 index 000000000..7aefb5bc0 --- /dev/null +++ b/amd64_go_caller_counted_loop_boundary_test.go @@ -0,0 +1,270 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerCountedLoopRejectedChanges() map[string][2]string { + return map[string][2]string{ + "missing nonzero guard": {"JE done\n", ""}, + "reversed guard": {"JE done", "JNE done"}, + "stale guard flags": {"JE done", "ADDL $1,R10\nJE done"}, + "different guard bits": {"TESTB $3,R9", "TESTB $4,R9"}, + "byte flags truncate": {"TESTB $3,R9\nJE done", "MOVL $256,R8\nTESTB R8,R8\nJNE done"}, + "unmapped length": {"MOVQ (CX),R9", "MOVQ (SI),R9"}, + "unknown length": {"MOVQ (CX),R9", "MOVQ R12,R9"}, + "bound four": {"ANDL $3,AX", "ANDL $7,AX"}, + "unrelated bound": {"MOVL R9,AX", "MOVL $2,AX"}, + "partial bound": {"MOVL R9,AX", "MOVW R9,AX"}, + "bound overwritten": {"MOVL (DI),BX", "ANDL $3,AX\nMOVL (DI),BX"}, + "bound alias": {"ANDL $3,AX", "ANDL $3,CX"}, + "counter absent": {"XORL CX,CX", ""}, + "counter nonzero": {"XORL CX,CX", "MOVQ $1,CX"}, + "counter high bits": {"XORL CX,CX", "MOVQ $0x100000000,CX"}, + "partial counter": {"XORL CX,CX", "XORW CX,CX"}, + "zero stride": {"ADDQ $1,CX", "ADDQ $0,CX"}, + "negative stride": {"ADDQ $1,CX", "SUBQ $1,CX"}, + "two stride": {"ADDQ $1,CX", "ADDQ $2,CX"}, + "double update": {"ADDQ $1,CX", "ADDQ $1,CX\nADDQ $1,CX"}, + "update bypass": {"ADDQ $1,CX", "TESTL BX,BX\nJE compare\nADDQ $1,CX\ncompare:"}, + "stale latch flags": {"JNE again", "ADDL $1,R10\nJNE again"}, + "different latch operands": {"CMPL CX,AX", "CMPL BX,AX"}, + "side entry": {"JE done", "JE again"}, + "nested loop": {"MOVL (DI),BX", "inner:\nCMPQ R9,$0\nJNE inner\nMOVL (DI),BX"}, + "unknown indirect target": {"JNE again", "JNE again\nJMP R12"}, + "unknown dead effect": {"again:", "JMP again\nRDTSC\nagain:"}, + "different input origin": {"ADDQ $4,DI", "MOVQ SI,DI"}, + "missing input origin": {"ADDQ $4,DI", "MOVQ $0,DI"}, + "readonly write": {"MOVL (DI),BX", "MOVL R10,(DI)\nMOVL (DI),BX"}, + "result displacement": {"ADDQ $4,DI", "ADDQ $1,DX\nADDQ $4,DI"}, + "source stack escape": {"ADDQ $4,DI", "MOVQ DI,SP\nADDQ $4,DI"}, + "input cap": {"ADDQ $4,DI", "ADDQ $1048576,DI"}, + "pointer truncation": {"ADDQ $4,DI", "LEAL 4(DI),DI"}, + "unknown first GP": {"ADDL R11,BX", "ADDL R12,BX"}, + "late first GP": {"MOVL (DI),BX", "ADDL BX,R10\nMOVL (DI),BX"}, + "unknown first vector": {"MOVL (DI),BX", "VEXTRACTF128 $1,Y0,X1\nMOVL (DI),BX"}, + "unknown first mask": {"MOVL (DI),BX", "VMOVUPS (DI),Z0\nVPADDD.Z Z0,Z0,K1,Z1\nMOVL (DI),BX"}, + "mask zero no rights": {"MOVL (DI),BX", "VMOVUPS (DI),Z0\nMOVL $0,BX\nKMOVQ BX,K1\nVPADDD.Z (DX),Z0,K1,Z1\nMOVL (DI),BX"}, + } +} + +func TestX86GoCallerCountedLoopRetainsUnprovedPaths(t *testing.T) { + original := x86CallerCountedLoopScalarAssembly(false) + for name, change := range x86CallerCountedLoopRejectedChanges() { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(original, change[0], change[1], 1) + if assembly == original { + t.Fatal("negative failed to change its original source") + } + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved %s admitted: %v", name, err) + } + }) + } +} + +func TestX86GoCallerCountedLoopSealsAllPartitionsAndSource(t *testing.T) { + for _, change := range []string{"none", "original", "normalized", "missing caller", "certificate width"} { + t.Run(change, func(t *testing.T) { + assembly := x86CallerCountedLoopScalarAssembly(true) + pkg, file := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil || normalized.Funcs[0].x86SourceStack == nil { + t.Fatalf("complete finite source proof: %v", err) + } + proof := normalized.Funcs[0].x86SourceStack + if proof.countedIterations != [4]int{0, 1, 2, 3} { + t.Fatalf("missing or altered feasible residue visits: %v", proof.countedIterations) + } + actualHash, err := x86CallerSourceFingerprint(normalized.Funcs[0]) + if err != nil || actualHash != proof.normalized { + t.Fatal("complete normalized identity was not sealed") + } + if change == "original" { + file.Funcs[0].Instrs[1].Args[0].FPOffset++ + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + failed := normalizeX86RawReturnSourceStack(decoded, options, true, file.Funcs[0]) + if failed.x86SourceStack != nil { + t.Fatal("changed original source borrowed its old caller identity") + } + return + } + for index := range normalized.Funcs[0].Instrs { + ins := &normalized.Funcs[0].Instrs[index] + if ins.Op != "MOVL" || len(ins.Args) != 2 || ins.Args[0].Kind != OpMem { + continue + } + if !x86CallerMemoryCertified(*ins, 0) { + t.Fatal("original full-width read lacks a final replay certificate") + } + if change == "certificate width" { + access := ins.x86CallerMemory[0] + access.width++ + ins.x86CallerMemory[0] = access + if x86CallerMemoryCertified(*ins, 0) || x86RawReturnMemoryBound(normalized.Funcs[0], normalized, options, true) { + t.Fatal("mutated private width acquired admission") + } + } else if change == "normalized" { + ins.Args[0].Mem.Base = DX + } + } + if change == "missing caller" { + sig := sigs["caller.kernel"] + sig.x86CallerRegions = nil + sigs[sig.Name] = sig + } + _, err = Translate(normalized, options) + if change == "none" { + if err != nil { + t.Fatal(err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("edited %s retained final proof: %v", change, err) + } + if change == "certificate width" { + fresh, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(fresh, options); err != nil { + t.Fatalf("recapture from unchanged original source: %v", err) + } + requireX86GoAssemblerResult(t, "amd64", assembly, true) + if err := x86CallerCoreTranslate(t, pkg, assembly, options.TargetTriple); err != nil { + t.Fatalf("fresh actual caller source and LLVM object: %v", err) + } + } + }) + } +} + +func TestX86GoCallerCountedLoopFinitePathBudgetFailsClosed(t *testing.T) { + var body strings.Builder + for index := 0; index < 14; index++ { + fmt.Fprintf(&body, "CMPL BX,$0\nJE branch%d\nMOVL $7,R11\nbranch%d:\n", index, index) + } + original := x86CallerCountedLoopScalarAssembly(false) + assembly := strings.Replace(original, "MOVL (DI),BX", "MOVL (DI),BX\n"+body.String(), 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "budget exhausted") { + t.Fatalf("unproved/exhausted path set acquired a certificate: %v", err) + } +} + +func TestX86GoCallerCountedLoopStatusWidthsUseActualLowBits(t *testing.T) { + for _, item := range []struct { + op Op + value uint64 + }{ + {"TESTB", 1 << 8}, {"TESTW", 1 << 16}, {"TESTL", 1 << 32}, {"TESTQ", 0}, + } { + t.Run(string(item.op), func(t *testing.T) { + assembly := fmt.Sprintf("TEXT ·status(SB),4,$0-0\nMOVQ $%d,R8\n%s R8,R8\nRET\n", item.value, item.op) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + facts := x86CallerCountedNumbers{regs: map[Reg]x86CallerCountedNumber{}, source: -1} + for pc, ins := range file.Funcs[0].Instrs { + facts.step(pc, 0, ins, x86SourceStackState{}, nil) + } + if !facts.flags.known || !facts.flags.equal { + t.Fatalf("actual %s truncation to zero did not prove ZF: %+v", item.op, facts.flags) + } + }) + } +} + +func TestX86GoCallerCountedLoopAbsentNumericalFactStaysUnknown(t *testing.T) { + facts := x86CallerCountedNumbers{ + regs: map[Reg]x86CallerCountedNumber{}, source: 0, + } + facts.step(1, 0, Instr{Op: "TESTQ", Args: []Operand{ + {Kind: OpImm, Imm: 3}, {Kind: OpReg, Reg: Reg("R8")}, + }}, x86SourceStackState{}, nil) + if facts.flags.known { + t.Fatal("an absent numerical fact borrowed the first partition source") + } +} + +func TestX86GoCallerCountedLoopAlternateRegistersAndWidths(t *testing.T) { + for _, counterWidth := range []string{"L", "Q"} { + for _, compareWidth := range []string{"L", "Q"} { + t.Run(counterWidth+compareWidth, func(t *testing.T) { + assembly := x86CallerCountedLoopScalarAssembly(false) + assembly = strings.NewReplacer( + "JE done", "JE empty_final", "done:", "empty_final:", + "again:", "generic_body:", "JNE again", "JNE generic_body", + "MOVL R9,AX", "MOVL R9,R13", "ANDL $3,AX", "ANDL $3,R13", + "XORL CX,CX", "XORL R12,R12", "ADDQ $1,CX", "ADD"+counterWidth+" $1,R12", + "CMPL CX,AX", "CMP"+compareWidth+" R12,R13", + ).Replace(assembly) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } + } +} + +func TestX86GoCallerCountedLoopFiveTargetsKeepAMD64Authority(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + if target.arch == "amd64" { + assembly := x86CallerCountedLoopScalarAssembly(false) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, target.triple); err != nil { + t.Fatal(err) + } + return + } + // These two objects are the existing ordinary 386 scalar contract. + // They are not copied AMD64 input or finite-loop authority. + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerCountedLoopScalarSource()) + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "386"); !errors.Is(err, ErrProbeNeedsContext) || proof != nil { + t.Fatal("mode32 borrowed the AMD64 caller input contract") + } + source, scalarSig, _ := x86SourceStackNumericalFixture("386", "@MOV @WORD(SP),AX\nADDL $7,AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3") + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, x86SourceStackOptions("386", target.triple, scalarSig)) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mode32-counted-boundary.ll", "mode32-counted-boundary.o", module.String()) + }) + } +} diff --git a/amd64_go_caller_counted_loop_runtime_test.go b/amd64_go_caller_counted_loop_runtime_test.go new file mode 100644 index 000000000..5cdee233e --- /dev/null +++ b/amd64_go_caller_counted_loop_runtime_test.go @@ -0,0 +1,217 @@ +package plan9asm + +import ( + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86GoCallerCountedLoopOriginalInitializedValues(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the separately required Linux/amd64 caller runtime counterpart") + } + runX86CallerCountedLoopOriginalValues(t, []string{"MOVQ"}) +} + +func runX86CallerCountedLoopOriginalValues(t *testing.T, aliases []string) { + t.Helper() + var source, assembly, calls strings.Builder + source.WriteString("package caller\nimport \"unsafe\"\n") + for _, alias := range aliases { + for _, raw := range []bool{false, true} { + for _, separateExit := range []bool{false, true} { + name := fmt.Sprintf("counted%s%t%t", alias, raw, separateExit) + declaration := x86CallerCountedLoopScalarSource() + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1) + source.WriteString(declaration) + machine := x86CallerCountedLoopScalarAssembly(raw) + if separateExit { + machine = x86CallerCountedLoopSeparateExitAssembly(raw) + } + machine = x86SourceAliasCountedAssembly(machine, alias) + assembly.WriteString(strings.ReplaceAll(machine, "·kernel", "·"+name)) + fmt.Fprintf(&calls, ` + if got:=Call%s(x.data[:length],y.data[:]); got!=want || + snapshot(&x)!=originalX || snapshot(&y)!=originalY { + t.Fatalf("%s length=%%d got=%%#x want=%%#x",length,got,want) + } +`, name, name) + } + } + } + goTest := `package caller +import "testing" + +type guardedInput struct { + before [4]uint32 + data [16]uint32 + after [4]uint32 +} + +//go:noinline +func snapshot(input *guardedInput) guardedInput { return *input } + +func TestOriginalInitializedCountedValues(t *testing.T) { + for repeat:=0; repeat<4; repeat++ { + for _, value:=range []uint32{0,1,7,0x80000000,0xffffffff,0x12345678} { + for length:=0; length<16; length++ { + x := guardedInput{ + before: [4]uint32{11,17,29,53}, + data: [16]uint32{value,value^0xaabbccdd,19,23,29,31,37,41,43,47,53,59,61,67,71,73}, + after: [4]uint32{0xaabbccdd,0x12345678,0x87654321,0xffffffff}, + } + y := guardedInput{ + before: x.after, + data: [16]uint32{31,37,value,43,47,53,59,61,67,71,73,79,83,89,97,101}, + after: x.before, + } + originalX,originalY:=snapshot(&x),snapshot(&y) + var want uint32 + for index:=0;index +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + var calls strings.Builder + for _, alias := range aliases { + for _, raw := range []bool{false, true} { + for _, separateExit := range []bool{false, true} { + name := fmt.Sprintf("counted%s%t%t", alias, raw, separateExit) + declaration := x86CallerCountedLoopScalarSource() + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + source.WriteString(strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1)) + machine := x86CallerCountedLoopScalarAssembly(raw) + if separateExit { + machine = x86CallerCountedLoopSeparateExitAssembly(raw) + } + machine = x86SourceAliasCountedAssembly(machine, alias) + assembly.WriteString(strings.ReplaceAll(machine, "·kernel", "·"+name)) + fmt.Fprintf(&oracle, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + fmt.Fprintf(&calls, ` + memset(output,0xc9,sizeof(output));memcpy(expected,output,sizeof(output)); + memcpy(expected+5,&want,sizeof(want)); + %s(a+1,b+3,output+5,&length); + if (memcmp(output,expected,sizeof(output)) || memcmp(a,original_a,sizeof(a)) || + memcmp(b,original_b,sizeof(b)) || length!=original_length) return 1; + calls++; +`, name) + } + } + } + oracle.WriteString(`int main(void) { + const uint32_t values[]={0,1,7,0x80000000u,0xffffffffu,0x12345678u}; + unsigned calls=0; + for (unsigned repeat=0;repeat<4;repeat++) { + for (unsigned value=0;value 1 + for _, next := range edges[component[0]] { + cyclic = cyclic || next == component[0] + } + if !cyclic { + continue + } + body := make(map[int]bool, len(component)) + for _, pc := range component { + body[pc] = true + } + header, latch := -1, -1 + entries, exits := 0, 0 + for pc, nexts := range edges { + if !reachable[pc] { + continue + } + for _, next := range nexts { + if !body[pc] && body[next] { + entries++ + header = next + } + if body[pc] && !body[next] { + exits++ + latch = pc + } + } + } + if entries != 1 || exits != 1 { + return nil + } + loop := deriveX86CallerCountedLoop(fn, instructions, edges, reachable, body, header, latch) + if loop == nil || !x86CallerCountedBodyDAG(*loop, edges) { + return nil + } + plan.headers[header], plan.latches[latch] = len(plan.loops), len(plan.loops) + plan.loops = append(plan.loops, *loop) + } + if len(plan.loops) == 0 { + return nil + } + var err error + plan.original, err = x86CallerSourceFingerprint(original) + if err != nil { + return nil + } + plan.normalized, err = x86CallerSourceFingerprint(fn) + if err != nil { + return nil + } + return plan +} + +// Removing exactly the proved latch backedge must eliminate every body cycle. +// A nested/alternate cycle cannot borrow another SCC's finite iteration cap. +func x86CallerCountedBodyDAG(loop x86CallerCountedLoop, edges [][]int) bool { + indegree := map[int]int{} + for pc := range loop.body { + for _, next := range edges[pc] { + if loop.body[next] && !(pc == loop.latch && next == loop.header) { + indegree[next]++ + } + } + } + var work []int + for pc := range loop.body { + if indegree[pc] == 0 { + work = append(work, pc) + } + } + removed := 0 + for len(work) > 0 { + pc := work[0] + work = work[1:] + removed++ + for _, next := range edges[pc] { + if loop.body[next] && !(pc == loop.latch && next == loop.header) { + indegree[next]-- + if indegree[next] == 0 { + work = append(work, next) + } + } + } + } + return removed == len(loop.body) +} diff --git a/amd64_go_caller_counted_scc_boundary_test.go b/amd64_go_caller_counted_scc_boundary_test.go new file mode 100644 index 000000000..b6de21566 --- /dev/null +++ b/amd64_go_caller_counted_scc_boundary_test.go @@ -0,0 +1,257 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerCountedSCCRejectThreeAPIs(t *testing.T, assembly string) { + t.Helper() + pkg, file := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + before := fmt.Sprintf("%#v %#v", file, sigs) + t.Run("Translate", func(t *testing.T) { + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved SCC source acquired text permission: %v", err) + } + }) + t.Run("Module", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved SCC source acquired module permission: %v", err) + } + }) + t.Run("GoModule", func(t *testing.T) { + if err := x86CallerCoreTranslate(t, pkg, assembly, options.TargetTriple); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved SCC source acquired Go module permission: %v", err) + } + }) + if before != fmt.Sprintf("%#v %#v", file, sigs) { + t.Fatal("failed SCC replay mutated caller/source inputs") + } +} + +func x86CallerCountedSCCRejectedChanges() map[string][2]string { + return map[string][2]string{ + "second missing guard": {"JE done\n", ""}, + "second reversed guard": {"JE done", "JNE done"}, + "second unknown bound": {"MOVL R9,R13", "MOVL BX,R13"}, + "second unrelated bound": {"MOVL R9,R13", "MOVL $2,R13"}, + "second fresh length source": {"MOVL R9,R13", "MOVQ length+24(FP),R8\nMOVQ (R8),R9\nMOVL R9,R13"}, + "second wider partition": {"ANDL $3,R13", "ANDL $7,R13"}, + "second partial bound": {"MOVL R9,R13", "MOVW R9,R13"}, + "second missing counter": {"XORL R12,R12", ""}, + "second nonzero counter": {"XORL R12,R12", "MOVL $1,R12"}, + "second partial counter": {"XORL R12,R12", "XORW R12,R12"}, + "second stride zero": {"ADDQ $1,R12", "ADDQ $0,R12"}, + "second stride two": {"ADDQ $1,R12", "ADDQ $2,R12"}, + "second decrement": {"ADDQ $1,R12", "SUBQ $1,R12"}, + "second double update": {"ADDQ $1,R12", "ADDQ $1,R12\nADDQ $1,R12"}, + "second update bypass": {"ADDQ $1,R12", "TESTL BX,BX\nJE second_compare\nADDQ $1,R12\nsecond_compare:"}, + "second stale flags": {"JNE second_body", "ADDL $1,R10\nJNE second_body"}, + "second side entry": {"JE done", "JE second_body"}, + "second early exit": {"second_body:", "second_body:\nTESTL BX,BX\nJE done"}, + "second nested cycle": {"second_body:", "second_body:\ninner:\nTESTL BX,BX\nJNE inner"}, + "dead unknown effect": {"second_body:", "JMP second_body\nRDTSC\nsecond_body:"}, + "first unknown GP": {"ADDL R11,BX", "ADDL R8,BX"}, + "diamond missing GP": {"MOVL $5,R11", "MOVL R8,R11"}, + "diamond missing vector": {"MOVL $5,R11", "VEXTRACTF128 $1,Y0,X1\nMOVL $5,R11"}, + "diamond missing mask": {"MOVL $5,R11", "VMOVUPS (DI),Z0\nVPADDD.Z Z0,Z0,K1,Z1\nMOVL $5,R11"}, + "diamond readonly write": {"MOVL $5,R11", "MOVL $5,(DI)\nMOVL $5,R11"}, + "diamond unknown origin": {"MOVL $5,R11", "MOVQ R8,DI\nMOVL $5,R11"}, + "diamond lost result": {"MOVL $5,R11", "MOVQ DI,DX\nMOVL $5,R11"}, + "diamond displaced result": {"MOVL $5,R11", "ADDQ $1,DX\nMOVL $5,R11"}, + "diamond SP escape": {"MOVL $5,R11", "MOVQ DI,SP\nMOVL $5,R11"}, + "diamond BP escape": {"MOVL $5,R11", "MOVQ DI,BP\nMOVL $5,R11"}, + "diamond g clobber": {"MOVL $5,R11", "MOVQ $0,R14\nMOVL $5,R11"}, + } +} + +func TestX86GoCallerCountedSCCRetainsEveryPathAndOrigin(t *testing.T) { + original := x86CallerCountedSCCAssembly(false, true) + for name, change := range x86CallerCountedSCCRejectedChanges() { + t.Run(name, func(t *testing.T) { + if strings.Count(original, change[0]) == 0 { + t.Fatal("negative does not name original source") + } + assembly := strings.Replace(original, change[0], change[1], 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + x86CallerCountedSCCRejectThreeAPIs(t, assembly) + }) + } +} + +func TestX86GoCallerCountedSCCCannotCloseUnknownCallers(t *testing.T) { + for name, change := range map[string][2]string{ + "indirect exit": {"done:", "done:\nJMP R8"}, + "unknown call": {"first_done:", "first_done:\nCALL ·unregistered(SB)"}, + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerCountedSCCAssembly(false, false), change[0], change[1], 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + if _, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unclosed caller graph supplied a producer: %v", err) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unclosed caller graph translated: %v", err) + } + }) + } +} + +func TestX86GoCallerCountedSCCDoesNotMergeDifferentReadonlyPaths(t *testing.T) { + // These paths are individually legal. Replay must keep their different + // authenticated input origins rather than deriving a synthetic common one. + assembly := strings.Replace(x86CallerCountedSCCAssembly(false, true), "MOVL $5,R11", "MOVQ SI,DI\nMOVL $5,R11", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + x86CallerCountedSCCObjects(t, x86CallerCountedLoopScalarSource(), assembly, "x86_64-unknown-linux-gnu") +} + +func TestX86GoCallerCountedSCCPreservesResidueAndSourceSeals(t *testing.T) { + assembly := x86CallerCountedSCCAssembly(true, false) + pkg, file := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil || normalized.Funcs[0].x86SourceStack == nil { + t.Fatalf("complete SCC replay proof missing: %v", err) + } + proof := normalized.Funcs[0].x86SourceStack + if proof.countedIterations != [4]int{0, 2, 4, 6} { + t.Fatalf("multiple SCCs changed residue denominator: %v", proof.countedIterations) + } + if fingerprint, err := x86CallerSourceFingerprint(normalized.Funcs[0]); err != nil || fingerprint != proof.normalized { + t.Fatal("normalized identity not sealed") + } + for _, name := range []string{"original source", "normalized source", "missing caller", "access width"} { + t.Run(name, func(t *testing.T) { + _, original := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + copy, err := normalizeX86RawFile(original, "amd64", options) + if err != nil { + t.Fatal(err) + } + switch name { + case "original source": + original.Funcs[0].Instrs[1].Args[0].FPOffset++ + decoded, err := decodeX86RawDirectives(original.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + failed := normalizeX86RawReturnSourceStack(decoded, options, true, original.Funcs[0]) + if failed.x86SourceStack != nil { + t.Fatal("modified original acquired stale SCC permission") + } + return + case "missing caller": + privateOptions := options + privateOptions.Sigs = map[string]FuncSig{} + for symbol, sig := range sigs { + sig.x86CallerRegions = nil + privateOptions.Sigs[symbol] = sig + } + if _, err := Translate(copy, privateOptions); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("lost caller identity acquired SCC permission: %v", err) + } + return + } + for index := range copy.Funcs[0].Instrs { + ins := ©.Funcs[0].Instrs[index] + if ins.Op != "MOVL" || len(ins.Args) != 2 || ins.Args[0].Kind != OpMem || ins.x86CallerMemory == nil { + continue + } + if !x86CallerMemoryCertified(*ins, 0) { + t.Fatal("complete read certificate missing") + } + if name == "access width" { + access := ins.x86CallerMemory[0] + access.width++ + ins.x86CallerMemory[0] = access + if x86CallerMemoryCertified(*ins, 0) || x86RawReturnMemoryBound(copy.Funcs[0], copy, options, true) { + t.Fatal("tampered read width borrowed SCC permission") + } + return + } + ins.Args[0].Mem.Base = DX + if _, err := Translate(copy, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("modified normalized source retained SCC permission: %v", err) + } + return + } + t.Fatal("no source access selected for negative") + }) + } +} + +func TestX86GoCallerCountedSCCMasksAndWidths(t *testing.T) { + for _, mask := range []int{1, 2, 3} { + for _, width := range []string{"L", "Q"} { + t.Run(fmt.Sprintf("mask%d/%s", mask, width), func(t *testing.T) { + assembly := strings.ReplaceAll(x86CallerCountedSCCAssembly(false, false), "$3,", fmt.Sprintf("$%d,", mask)) + assembly = strings.NewReplacer("ADDQ $1,CX", "ADD"+width+" $1,CX", + "ADDQ $1,R12", "ADD"+width+" $1,R12", "CMPL CX,AX", "CMP"+width+" CX,AX", + "CMPL R12,R13", "CMP"+width+" R12,R13").Replace(assembly) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + x86CallerCountedSCCObjects(t, x86CallerCountedLoopScalarSource(), assembly, "x86_64-unknown-linux-gnu") + }) + } + } +} + +func TestX86GoCallerCountedSCCIndependentMasksAndBodyDiamonds(t *testing.T) { + for _, masks := range [][2]int{{1, 2}, {2, 3}, {3, 1}} { + t.Run(fmt.Sprintf("%d/%d", masks[0], masks[1]), func(t *testing.T) { + assembly := x86CallerCountedSCCAssembly(false, false) + assembly = strings.NewReplacer("TESTB $3,R9\nJE first_done", fmt.Sprintf("TESTB $%d,R9\nJE first_done", masks[0]), + "ANDL $3,AX", fmt.Sprintf("ANDL $%d,AX", masks[0]), + "TESTB $3,R9\nJE done", fmt.Sprintf("TESTB $%d,R9\nJE done", masks[1]), + "ANDL $3,R13", fmt.Sprintf("ANDL $%d,R13", masks[1])).Replace(assembly) + // Both arms of this internal diamond are known numerical writes. + // The +1 update still dominates every path to the exact latch. + assembly = strings.Replace(assembly, "ADDQ $1,R12", `TESTL BX,BX +JE zero_value +MOVL $3,R11 +JMP body_join +zero_value: +MOVL $5,R11 +body_join: +ADDL R11,R10 +ADDQ $1,R12`, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + x86CallerCountedSCCObjects(t, x86CallerCountedLoopScalarSource(), assembly, "x86_64-unknown-linux-gnu") + }) + } +} + +func TestX86GoCallerCountedSCCKeepsWholeReplayBudget(t *testing.T) { + var diamond strings.Builder + for index := 0; index < 14; index++ { + fmt.Fprintf(&diamond, "TESTL BX,BX\nJE budget%d\nMOVL $7,R11\nbudget%d:\n", index, index) + } + assembly := strings.Replace(x86CallerCountedSCCAssembly(false, false), "ADDQ $1,R12", diamond.String()+"ADDQ $1,R12", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "finite path budget exhausted") { + t.Fatalf("SCC composition expanded or bypassed the finite replay budget: %v", err) + } +} diff --git a/amd64_go_caller_counted_scc_runtime_test.go b/amd64_go_caller_counted_scc_runtime_test.go new file mode 100644 index 000000000..72660020e --- /dev/null +++ b/amd64_go_caller_counted_scc_runtime_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Both existing required counted-loop runtime roots consume these fixtures. +// Their host/runner policy is unchanged; Rosetta remains diagnostic rather +// than evidence of native Linux execution or physical source-stack identity. +func x86CallerCountedSCCRuntimeFixtures(aliases ...string) (string, string, []string) { + if len(aliases) == 0 { + aliases = []string{"MOVQ"} + } + var source, assembly strings.Builder + source.WriteString("package caller\nimport \"unsafe\"\n") + var names []string + for _, alias := range aliases { + for _, raw := range []bool{false, true} { + for _, diamond := range []bool{false, true} { + name := fmt.Sprintf("scc%s%t%t", alias, raw, diamond) + declaration := x86CallerCountedLoopScalarSource() + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + source.WriteString(strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1)) + machine := x86SourceAliasCountedAssembly(x86CallerCountedSCCAssembly(raw, diamond), alias) + assembly.WriteString(strings.ReplaceAll(machine, "·kernel", "·"+name)) + names = append(names, name) + } + } + } + return source.String(), assembly.String(), names +} + +func runX86CallerCountedSCCOriginalValues(t *testing.T, aliases ...string) { + source, assembly, names := x86CallerCountedSCCRuntimeFixtures(aliases...) + var calls strings.Builder + for index, name := range names { + fmt.Fprintf(&calls, ` + { + want := sum + if %t { + if x.data[bound] == 0 { want += 5 } else { want += 3 } + } + if got := Call%s(x.data[:length],y.data[:]); got != want || + snapshot(&x) != originalX || snapshot(&y) != originalY { + t.Fatalf("%s length=%%d got=%%#x want=%%#x",length,got,want) + } + calls++ + } +`, index%2 == 1, name, name) + } + testSource := `package caller +import "testing" +type guardedInput struct { before [4]uint32; data [16]uint32; after [4]uint32 } +//go:noinline +func snapshot(input *guardedInput) guardedInput { return *input } +func TestOriginalSCCValues(t *testing.T) { + calls := 0 + for repeat := 0; repeat < 4; repeat++ { + for _, value := range []uint32{0,1,7,0x80000000,0xffffffff,0x12345678} { + for length := 0; length < 16; length++ { + x := guardedInput{ + before: [4]uint32{11,17,29,53}, + data: [16]uint32{value,0,value^0xaabbccdd,0,29,31,37,41,43,47,53,59,61,67,71,73}, + after: [4]uint32{0xaabbccdd,0x12345678,0x87654321,0xffffffff}, + } + y := guardedInput{before:x.after, + data:[16]uint32{31,37,value,43,47,53,59,61,67,71,73,79,83,89,97,101},after:x.before} + originalX,originalY := snapshot(&x),snapshot(&y) + bound := length & 3 + var sum uint32 + for index := 0; index < 2*bound; index++ { sum += x.data[index]+y.data[index] } +` + calls.String() + ` + } + } + } +` + fmt.Sprintf(" if calls != %d { t.Fatalf(\"calls=%%d\", calls) }\n", 384*len(names)) + ` +} +` + dir := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module caller\n\ngo 1.20\n", "value.go": source, + "value_amd64.s": assembly, "value_test.go": testSource, + } { + bytes := []byte(content) + if strings.HasSuffix(name, ".go") { + var err error + bytes, err = format.Source(bytes) + if err != nil { + t.Fatalf("format generated %s: %v", name, err) + } + } + if err := os.WriteFile(filepath.Join(dir, name), bytes, 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("first-party original Go serial SCC integer/canary oracle: %v\n%s", err, output) + } + t.Logf("%d original Go calls: two serial bounded loops, raw/named controls, both diamond arms, u32 wrap and complete guarded input snapshots", 384*len(names)) +} + +func runX86CallerCountedSCCLLVMValues(t *testing.T, aliases ...string) { + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang required") + } + source, assembly, names := x86CallerCountedSCCRuntimeFixtures(aliases...) + var oracle, calls strings.Builder + oracle.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for index, name := range names { + fmt.Fprintf(&oracle, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + fmt.Fprintf(&calls, ` + { + uint32_t want = sum; + if (%d) want += x[bound] == 0 ? 5 : 3; + memset(output,0xc9,sizeof(output)); memcpy(expected,output,sizeof(output)); + memcpy(expected+5,&want,sizeof(want)); + %s(a+1,b+3,output+5,&length); + if (memcmp(output,expected,sizeof(output)) || memcmp(a,original_a,sizeof(a)) || + memcmp(b,original_b,sizeof(b)) || length != original_length) return 1; + calls++; + } +`, index%2, name) + } + oracle.WriteString(`int main(void) { + const uint32_t values[]={0,1,7,0x80000000u,0xffffffffu,0x12345678u}; + unsigned calls=0; + for (unsigned repeat=0;repeat<4;repeat++) { + for (unsigned value=0;value= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(form.source.Reg)] & width + if form.memory >= 0 { + // Even dwords are the numerical inputs, but the possible native read + // remains m128/m256/m512 or m64bcst, independent of the mask value. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + var known uint64 + for start := 0; start < form.byteWidth; start += 8 { + input := uint64(15) << uint(start) + output := uint64(255) << uint(start) + if first&input == input && second&input == input { + known |= output + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.mask >= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_dword_product_test.go b/amd64_go_caller_dword_product_test.go new file mode 100644 index 000000000..fb34ac7a3 --- /dev/null +++ b/amd64_go_caller_dword_product_test.go @@ -0,0 +1,270 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerDwordProductIndependentSourceEffect(t *testing.T) { + for _, row := range []string{"PMULDQ X0,X1", "PMULULQ X0,X1", "VPMULDQ Y0,Y1,Y2", "VPMULUDQ Y0,Y1,Y2"} { + t.Run(row, func(t *testing.T) { + result := "X2" + if !strings.HasPrefix(row, "V") { + result = "X1" + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerDwordProductCompleteMaskBroadcastAndMemoryForms(t *testing.T) { + rows := []string{"PMULDQ X0,X1", "PMULDQ X1,X1", "PMULDQ (DI),X1", "PMULULQ X0,X1", "PMULULQ X1,X1", "PMULULQ (DI),X1"} + for _, name := range x86DwordProductOracleNames[2:] { + for _, row := range x86OfficialPackedBinaryWitnesses(t)[name] { + rows = append(rows, strings.Replace(row, "(AX)", "(DI)", 1)) + } + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + result := "X2" + if !strings.HasPrefix(row, "V") { + result = "X1" + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("complete signed/unsigned family caller failed", row, triple, err) + } + } + }) + } +} + +func TestX86GoCallerDwordProductEveryPotentialByteAndUpperAliases(t *testing.T) { + for _, row := range []string{"PMULDQ X0,X1", "PMULULQ X0,X1", "VPMULDQ X0,X1,X2", "VPMULUDQ Y0,Y1,Y2", "VPMULDQ Z0,Z1,K1,Z2", "VPMULUDQ.Z Z0,Z1,K1,Z2"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86DwordProductProperties(ins.Op, ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, missing := range []Reg{"Z0", "Z1", "Z2"} { + for index := 0; index < 64; index++ { + t.Run(fmt.Sprintf("%s/missing%s/byte%d", row, missing, index), func(t *testing.T) { + state := &x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0)}, masks: map[Reg]bool{"K1": true}} + state.vectors[missing] &^= uint64(1) << uint(index) + old := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)] + if !x86CallerDwordProductStep(&ins, form, state, nil) { + t.Fatal("register-only numerical effect failed") + } + var want uint64 + for output := 0; output < 64; output++ { + known := true + if output >= form.byteWidth { + known = !form.legacy || old&(uint64(1)<= start+4 + } + if form.mask >= 0 && !form.zeroing && missing == x86CallerVectorRoot(ins.Args[form.destination].Reg) && index == output { + known = false + } + } + if known { + want |= uint64(1) << uint(output) + } + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatalf("potential input/merge/upper byte lost: got=%016x want=%016x", got, want) + } + }) + } + } + } +} + +func TestX86GoCallerDwordProductExactMemoryCertificates(t *testing.T) { + for _, row := range []string{"PMULDQ (DI),X0", "PMULULQ (DI),X0", "VPMULDQ (DI),Y0,Y2", "VPMULUDQ.Z (DI),Z0,K1,Z2", "VPMULDQ.BCST.Z (DI),Z0,K1,Z2", "VPMULUDQ.BCST (DI),X0,X2"} { + result := "X2" + if !strings.HasPrefix(row, "V") { + result = "X0" + } + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86DwordProductProperties(ins.Op, ins, "amd64") + if !known || err != nil || form.memory < 0 { + continue + } + found = true + index := form.memory + access, exists := ins.x86CallerMemory[index] + wantWidth := form.byteWidth + if strings.Contains(row, ".BCST") { + wantWidth = 8 + } + if !exists || access.width != int64(wantWidth) || access.write || !x86CallerMemoryCertified(ins, index) { + t.Fatal("zero mask/selected dwords shrank native envelope", row, access) + } + for _, mutation := range []string{"opcode", "width", "direction", "address", "destination", "mask", "address size"} { + changed := ins + changedAccess := access + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch mutation { + case "opcode": + changed.Op = "VPMULDQ" + if strings.Contains(row, "PMULDQ") { + changed.Op = "VPMULUDQ" + } + case "width": + changedAccess.width-- + changed.x86CallerMemory[index] = changedAccess + case "direction": + changedAccess.write = true + changed.x86CallerMemory[index] = changedAccess + case "address": + changed.Args[index].Mem.Off++ + case "destination": + changed.Args[form.destination].Reg = "X7" + case "mask": + if form.mask >= 0 { + changed.Args[form.mask].Reg = "K7" + } else { + changed.Args = append(changed.Args, Operand{Kind: OpReg, Reg: "K7"}) + } + case "address size": + changed.x86AddressBits = 32 + } + if x86CallerMemoryCertified(changed, index) { + t.Fatal("stale native read certificate survived", row, mutation) + } + } + } + if !found { + t.Fatal("normalized source omitted dword product", row) + } + } +} + +func TestX86GoCallerDwordProductUnprovedInputsAndGuardsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPMULDQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPMULUDQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + "partial even dword": "MOVSS (DI),X3\nMOVSS X3,X0\nVMOVUPS (SI),Z1\nVPMULDQ Z0,Z1,Z2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVPMULDQ.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVPMULUDQ Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPMULDQ.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask full tuple": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPMULDQ.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "broadcast full qword": "VMOVUPS (DI),Z0\nVPMULUDQ.BCST (DX),Z0,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPMULDQ (BX),Z0,Z2\nVMOVSS X2,(DX)", + "private SP": "VMOVUPS (DI),Z0\nVPMULDQ (SP),Z0,Z2\nVMOVSS X2,(DX)", + "SP alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPMULDQ (BX),X0\nVMOVSS X0,(DX)", + "BP alias": "VMOVUPS (DI),Z0\nMOVQ BP,BX\nPMULULQ (BX),X0\nVMOVSS X0,(DX)", + "g use": "VMOVUPS (DI),Z0\nVPMULDQ (R14),Z0,Z2\nVMOVSS X2,(DX)", + "input write": "VMOVUPS (DI),Z0\nVPMULDQ Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX caller": "PMULULQ M0,M1\nVMOVSS X1,(DX)", + "unknown CFG join": "VMOVUPS (DI),Y0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Y1\njoined:\nVPMULDQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved input/native/caller guard acquired permission", err) + } + }) + } +} + +func TestX86GoCallerDwordProductRawTuplesBindNativeReadWidths(t *testing.T) { + for _, name := range []string{"VPMULDQ", "VPMULUDQ"} { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + op, tuple := name, width + if broadcast { + op, tuple = name+".BCST", 8 + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, offset := range []int{-tuple, 64 - tuple, 64} { + row := fmt.Sprintf("%s %d(DI),%s1,K1,%s2", op, offset, bank, bank) + t.Run(row, func(t *testing.T) { + original := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+row+"\nRET\n") + body := "VMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVQ $1,AX\nKMOVQ AX,K1\n" + x86DwordProductDirectives(original[:len(original)-1]) + "VMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("original raw tuple lost existing readonly slice role", row, err) + } + // The input is a dynamic slice role, not an observed fixed + // 64-byte allocation. Certify only the exact possible native + // instruction read, preserving native faulting addresses. + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86DwordProductProperties(ins.Op, ins, "amd64") + if !known || err != nil { + continue + } + found = true + access, ok := ins.x86CallerMemory[0] + if !ins.x86Encoded || form.memory != 0 || ins.Args[0].Mem.Base != DI || ins.Args[0].Mem.Off != int64(offset) || !ok || access.width != int64(tuple) || access.write || !x86CallerMemoryCertified(ins, 0) { + t.Fatal("raw source tuple/address/native-width certificate drift", row, ins, access) + } + } + if !found { + t.Fatal("original raw source omitted from normalized proof", row) + } + // Conversely the fresh result is a real bounded four-byte + // region: no full-vector or m64bcst read fits this role. + boundedRow := strings.Replace(row, "(DI)", "(DX)", 1) + bounded := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+boundedRow+"\nRET\n") + boundedBody := strings.Replace(body, x86DwordProductDirectives(original[:len(original)-1]), x86DwordProductDirectives(bounded[:len(bounded)-1]), 1) + boundedAssembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", boundedBody, 1) + boundedPkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, boundedAssembly) + if err := x86CallerCoreTranslate(t, boundedPkg, boundedAssembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("actual raw tuple gained bounded result-object permission", boundedRow, err) + } + }) + } + } + } + } +} diff --git a/amd64_go_caller_explicit_imul.go b/amd64_go_caller_explicit_imul.go new file mode 100644 index 000000000..75e05bfda --- /dev/null +++ b/amd64_go_caller_explicit_imul.go @@ -0,0 +1,52 @@ +package plan9asm + +// Only the existing validated explicit IMUL descriptor is consumed here. This +// supplies byte-definedness, not a new pointer, frame or memory capability. +func x86CallerExplicitIMULStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (recognized, valid bool) { + if _, known, err := x86ExplicitIMULProperties(ins.Op, *ins, "amd64"); !known || err != nil { + return known, false + } + effect, known, err := x86TypedGPByteEffect(ins.Op, *ins, "amd64") + if !known || err != nil || effect.dynamicMemory { + return true, false + } + // All ordered source bytes (including the two-operand old destination) + // must be defined before any destination can overwrite their evidence. + for _, operand := range effect.operands { + if !operand.read { + continue + } + if operand.operand.Kind == OpMem { + if proof == nil || !x86CallerOrdinaryMemoryStep(ins, operand.index, state, proof, operand.width, false) { + return true, false + } + } else if !x86CallerNumericRead(operand.operand, operand.width, state) { + return true, false + } + } + for _, operand := range effect.operands { + if operand.write && (operand.operand.Kind != OpReg || !x86CallerNumericWrite(operand.operand.Reg, operand.width, state)) { + return true, false + } + } + return true, true +} + +func x86SourceStackNumericFlagMask(state *x86SourceStackState) uint8 { + if state.numericFlags { + return x86ByteFlagsAll // Preserve the pre-existing complete-flags witness. + } + return state.partialNumericFlags +} + +func x86SourceStackNumericFlagsRead(state *x86SourceStackState, required uint8) bool { + return required != 0 && x86SourceStackNumericFlagMask(state)&required == required +} + +func x86SourceStackSetNumericFlags(state *x86SourceStackState, mask uint8) { + state.numericFlags = mask == x86ByteFlagsAll + state.partialNumericFlags = 0 + if !state.numericFlags { + state.partialNumericFlags = mask + } +} diff --git a/amd64_go_caller_explicit_imul_test.go b/amd64_go_caller_explicit_imul_test.go new file mode 100644 index 000000000..34d8f5884 --- /dev/null +++ b/amd64_go_caller_explicit_imul_test.go @@ -0,0 +1,268 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// This directory is independent of the typed effect descriptor. It exercises +// all explicit yimul/yimul3 source classes; one-operand IMUL is not in it. +var x86CallerExplicitIMULRows = []struct { + name, instruction string + three bool +}{ + {"register", "IMUL@ AX,BX", false}, + {"memory", "IMUL@ (DI),BX", false}, + {"immediate8", "IMUL@ $-7,BX", false}, + {"immediate32", "IMUL@ $65537,BX", false}, + {"three-register8", "IMUL3@ $-7,AX,BX", true}, + {"three-register32", "IMUL3@ $65537,AX,BX", true}, + {"three-memory8", "IMUL3@ $-7,(DI),BX", true}, + {"three-memory32", "IMUL3@ $65537,(DI),BX", true}, +} + +func x86CallerExplicitIMULFixture(t *testing.T, body string) (GoPackage, *File, string, Options) { + t.Helper() + declaration := strings.ReplaceAll(x86CallerFixtureSource, "float32", "uint64") + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX\nMOVL AX,(DX)", body+"\nMOVQ BX,(DX)", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, declaration, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + // goSigsForAsmFile already binds the complete actual SourceSelection. + if sigs["caller.kernel"].x86CallerRegions == nil { + t.Fatalf("actual complete typed caller precondition failed: %v", err) + } + if !x86CallerRegionSourceMatches(sigs["caller.kernel"], file.Funcs[0], "amd64") { + t.Fatal("actual complete typed caller does not match its original source/signature") + } + return pkg, file, assembly, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} +} + +func TestX86GoCallerExplicitIMULRawReturnRegisteredForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, suffix := range []string{"W", "L", "Q"} { + for _, row := range x86CallerExplicitIMULRows { + t.Run(suffix+"/"+row.name, func(t *testing.T) { + body := "MOVQ $7,AX\n" + // W preserves the old upper bytes. Two-operand IMUL also reads + // the old low result; IMUL3L/Q must not require that old result. + if suffix == "W" || !row.three { + body += "MOVQ $3,BX\n" + } + body += strings.ReplaceAll(row.instruction, "@", suffix) + pkg, file, assembly, options := x86CallerExplicitIMULFixture(t, body) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + // Separate format and API/target axes: two representative + // rows cover all APIs/targets; every row covers text/Linux. + allAPIs := suffix == "Q" && row.name == "register" || suffix == "L" && row.name == "three-memory32" + if !allAPIs && triple != "x86_64-unknown-linux-gnu" { + continue + } + options.TargetTriple = triple + t.Run(triple+"/text", func(t *testing.T) { + ir, err := Translate(file, options) + if err != nil { + t.Fatalf("Go-accepted explicit multiply lacks source/caller effect: %v", err) + } + compileLLVMToObject(t, llc, triple, "imul-caller.ll", "imul-caller.o", ir) + }) + if !allAPIs { + continue + } + t.Run(triple+"/module", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "imul-caller-module.ll", "imul-caller-module.o", module.String()) + }) + t.Run(triple+"/go", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "kernel_amd64.s", GOARCH: "amd64", TargetTriple: triple, ResolveSym: options.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "imul-caller-go.ll", "imul-caller-go.o", translated.Module.String()) + }) + } + }) + } + } +} + +func TestX86SourceStackExplicitIMULKillsOldComparisonFlags(t *testing.T) { + for _, op := range []string{"IMULW", "IMULL", "IMULQ", "IMUL3W", "IMUL3L", "IMUL3Q"} { + t.Run(op, func(t *testing.T) { + row := op + " AX,BX" + if strings.HasPrefix(op, "IMUL3") { + row = op + " $7,AX,BX" + } + ins := x86YMMReadSource(t, row) + state := x86SourceStackState{numericFlags: true, callers: map[Reg]x86CallerPointerValue{}} + if !x86SourceStackFlagsStep(ins, &state) || state.numericFlags { + t.Fatal("IMUL retained complete old CMP flags instead of only CF/OF") + } + for _, branch := range []string{"JE", "JNE", "JLT", "JGE", "JLE", "JGT", "JMI", "JPL", "JHI", "JLS", "JPS", "JPC", "JCS", "JCC", "JOS", "JOC"} { + accepted := x86SourceStackFlagsStep(Instr{Op: Op(branch)}, &state) + want := branch == "JCS" || branch == "JCC" || branch == "JOS" || branch == "JOC" + if accepted != want { + t.Fatalf("%s after %s: accepted=%t want=%t", branch, op, accepted, want) + } + } + }) + } +} + +func TestX86GoCallerExplicitIMULRetainsContext(t *testing.T) { + for _, test := range []struct{ name, body string }{ + {"undefined-input", "MOVQ $7,BX\nIMULQ AX,BX"}, + {"undefined-old-destination", "MOVQ $7,AX\nIMULQ AX,BX"}, + {"partial-input", "MOVB $7,AX\nIMULQ AX,BX"}, + {"partial-word-result", "MOVQ $7,AX\nIMUL3W $7,AX,BX"}, + {"pointer-input", "MOVQ $7,BX\nIMULQ DI,BX"}, + {"pointer-old-destination", "MOVQ DI,BX\nIMULQ $7,BX"}, + {"pointer-survives-word-result", "MOVQ DI,BX\nMOVQ $7,AX\nIMUL3W $7,AX,BX"}, + {"unknown-memory", "MOVQ $7,BX\nIMULQ (R11),BX"}, + {"wide-length-read", "MOVQ $7,BX\nIMULQ 1(CX),BX"}, + {"native-return-pc", "MOVQ $7,BX\nIMULQ 8(BP),BX"}, + {"FP-pointer-input", "MOVQ $7,BX\nIMULQ a+0(FP),BX"}, + {"g-input", "MOVQ $7,BX\nIMULQ R14,BX"}, + {"g-unrestored", "MOVQ $7,AX\nIMUL3Q $7,AX,R14\nMOVQ R14,BX"}, + {"SP-destination", "MOVQ $7,BX\nIMULQ BX,SP"}, + {"BP-input", "MOVQ $7,BX\nIMULQ BP,BX"}, + {"implicit-IMUL", "MOVQ $7,AX\nMOVQ $7,BX\nIMULQ BX"}, + {"implicit-MUL", "MOVQ $7,AX\nMOVQ $7,BX\nMULQ BX"}, + {"implicit-DIV", "MOVQ $7,AX\nMOVQ $7,BX\nMOVQ $0,DX\nDIVQ BX"}, + {"implicit-IDIV", "MOVQ $7,AX\nMOVQ $7,BX\nMOVQ $0,DX\nIDIVQ BX"}, + {"unknown-CFG-input", "MOVQ $7,BX\nCMPQ BX,$0\nJE joined\nMOVQ $7,AX\njoined:\nIMULQ AX,BX"}, + } { + t.Run(test.name, func(t *testing.T) { + pkg, file, assembly, options := x86CallerExplicitIMULFixture(t, test.body) + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved IMUL effect acquired authority: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module accepted unproved IMUL: %v", err) + } + ctx = llvm.NewContext() + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "kernel_amd64.s", GOARCH: "amd64", TargetTriple: options.TargetTriple, ResolveSym: options.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + translated.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go API accepted unproved IMUL: %v", err) + } + }) + } +} + +func TestX86GoCallerExplicitIMULActualInitializedLocalRead(t *testing.T) { + // The retained actual caller declares `var res uint64`: Go initializes + // those complete numerical bytes before the call. This is not an unknown + // native entry region, nor permission to read an arbitrary result pointer. + _, file, _, options := x86CallerExplicitIMULFixture(t, "MOVQ $7,BX\nIMULQ (DX),BX") + if _, err := Translate(file, options); err != nil { + t.Fatal(err) + } +} + +func TestX86GoCallerExplicitIMULFlagsAndSavedFlags(t *testing.T) { + for _, condition := range []string{"CC", "CS", "EQ", "NE", "GE", "LT", "GT", "LE", "HI", "LS", "MI", "PL", "OC", "OS", "PC", "PS"} { + for _, consumer := range []string{"SET", "CMOVQ", "J"} { + t.Run(consumer+condition, func(t *testing.T) { + body := "MOVQ $7,AX\nMOVQ $3,BX\nCMPQ AX,$0\nIMULQ AX,BX\n" + switch consumer { + case "SET": + body += "SET" + condition + " BL" + case "CMOVQ": + body += "CMOVQ" + condition + " AX,BX" + case "J": + body += "J" + condition + " joined\njoined:" + } + _, file, _, options := x86CallerExplicitIMULFixture(t, body) + _, err := Translate(file, options) + defined := condition == "CC" || condition == "CS" || condition == "OC" || condition == "OS" + if defined && err != nil || !defined && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("IMUL flag consumer defined=%t: %v", defined, err) + } + }) + } + } + for _, test := range []struct { + name, body string + accepted bool + }{ + {"restore-complete", "CMPQ AX,$0\nPUSHFQ\nIMULQ AX,BX\nPOPFQ\nSETEQ BL", true}, + {"restore-partial", "IMULQ AX,BX\nPUSHFQ\nCMPQ AX,$0\nPOPFQ\nSETEQ BL", false}, + {"restore-CF", "IMULQ AX,BX\nPUSHFQ\nCMPQ AX,$0\nPOPFQ\nSETCS BL", true}, + {"carry-after-multiply", "IMULQ AX,BX\nADCQ $1,BX", true}, + {"CMP-redefines-flags", "IMULQ AX,BX\nCMPQ AX,$0\nSETEQ BL", true}, + {"shift-does-not-resurrect-OF", "IMULQ AX,BX\nSHLQ $2,BX\nSETOS BL", false}, + } { + t.Run(test.name, func(t *testing.T) { + _, file, _, options := x86CallerExplicitIMULFixture(t, "MOVQ $7,AX\nMOVQ $3,BX\n"+test.body) + _, err := Translate(file, options) + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("FLAGS dependency accepted=%t: %v", test.accepted, err) + } + }) + } +} + +func TestX86SourceStackExplicitIMULWithoutCallerRemainsContext(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, suffix := range []string{"W", "L", "Q"} { + if arch == "386" && suffix == "Q" { + continue // Independently Go-rejected Q rows are in the encoder oracle. + } + for _, three := range []bool{false, true} { + body := "@MOV $7,AX\n@MOV $3,BX\nIMUL" + suffix + " AX,BX\n@MOV BX,@RESULT(SP)\nBYTE $0xc3" + if three { + body = strings.Replace(body, "IMUL"+suffix+" AX,BX", "IMUL3"+suffix+" $7,AX,BX", 1) + } + t.Run(fmt.Sprintf("%s/%s/three=%t", arch, suffix, three), func(t *testing.T) { + source, sig, _ := x86SourceStackNumericalFixture(arch, body) + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, x86SourceStackOptions(arch, "", sig)); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("declaration-only source acquired complete numerical input proof: %v", err) + } + }) + } + } + } +} diff --git a/amd64_go_caller_float_compare.go b/amd64_go_caller_float_compare.go new file mode 100644 index 000000000..b96dbcda5 --- /dev/null +++ b/amd64_go_caller_float_compare.go @@ -0,0 +1,27 @@ +package plan9asm + +// Comparison writes either complete predicate lanes or a fully defined K +// result. A K destination is never an old merge input; its optional source +// mask must still be initialized, even when source and destination alias. +func x86CallerFloatCompareStep(ins *Instr, form x86FloatCompareForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + required := x86CallerVectorByteMask(int64(form.inputByteWidth())) + if form.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + for _, index := range []int{form.first, form.second} { + if index != form.memory && state.vectors[x86CallerVectorRoot(ins.Args[index].Reg)]&required != required { + return false + } + } + if form.maskResult { + state.masks[ins.Args[form.destination].Reg] = true + return true + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + second := x86CallerVectorRoot(ins.Args[form.second].Reg) + state.vectors[destination] = form.resultDefinedBytes(state.vectors[second], state.vectors[destination]) + return true +} diff --git a/amd64_go_caller_float_compare_test.go b/amd64_go_caller_float_compare_test.go new file mode 100644 index 000000000..d73712371 --- /dev/null +++ b/amd64_go_caller_float_compare_test.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86GoCallerFloatingCompareCompleteInitializedForms(t *testing.T) { + for _, original := range x86FloatCompareGoForms("amd64") { + t.Run(original, func(t *testing.T) { + row := strings.NewReplacer("8(AX)", "8(DI)", "(AX)(CX*4)", "(DI)").Replace(original) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86FloatCompareProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatalf("independent source form not recognized: %v", err) + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z20\nVMOVUPS (SI),Z21\nKMOVQ AX,K1\n" + row + destination := ins.Args[form.destination].Reg + if form.maskResult { + body += "\nKMOVD " + string(destination) + ",(DX)" + } else { + index, _ := amd64VectorRegisterIndex(destination, form.byteWidth) + body += fmt.Sprintf("\nVMOVSS X%d,(DX)", index) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerFloatingCompareRegisterKnowledgeAndMasks(t *testing.T) { + rows := x86FloatCompareGoForms("amd64") + rows = append(rows, "VCMPPS $0,Z0,Z1,K1,K1", "VCMPSS $255,X0,X1,K1,K1") + for _, row := range rows { + if strings.Contains(row, "(AX)") { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86FloatCompareProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + width := form.byteWidth + if form.scalar { + width = form.bits / 8 + } + required := x86CallerVectorByteMask(int64(width)) + first, second := x86CallerVectorRoot(ins.Args[form.first].Reg), x86CallerVectorRoot(ins.Args[form.second].Reg) + for seed := 0; seed < 64; seed++ { + upper := uint64(seed) * 0x391072d5e64b8a0f + state := x86SourceStackState{ + vectors: map[Reg]uint64{first: required, second: required | upper}, + masks: map[Reg]bool{"K1": true}, + } + if !x86CallerVectorMoveKnown(ins) || !x86CallerVectorMoveStep(&ins, &state, nil) { + t.Fatalf("complete registered compare effect rejected %s", row) + } + if form.maskResult { + if !state.masks[ins.Args[form.destination].Reg] || state.vectors[first] != required || state.vectors[second] != required|upper { + t.Fatalf("K-result changed input vectors or required old mask contents: %s", row) + } + continue + } + want := ^uint64(0) + if form.legacy { + want = required | upper + } else if form.scalar { + want = ^x86CallerVectorByteMask(16) | required | upper&(x86CallerVectorByteMask(16)&^required) + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatalf("%s seed%d defined bytes=%#x want=%#x", row, seed, got, want) + } + } + for _, missing := range []int{form.first, form.second, form.mask} { + if missing < 0 { + continue + } + state := x86SourceStackState{ + vectors: map[Reg]uint64{first: required, second: required}, + masks: map[Reg]bool{"K1": true}, + } + if missing == form.mask { + state.masks[ins.Args[missing].Reg] = false + } else { + state.vectors[x86CallerVectorRoot(ins.Args[missing].Reg)] &^= 1 + } + vectors, masks := x86CloneCallerVectorState(state.vectors, state.masks) + if x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state.vectors, vectors) || !reflect.DeepEqual(state.masks, masks) { + t.Fatalf("%s missing role%d invented evidence or updated partial state", row, missing) + } + } + } +} + +func TestX86GoCallerFloatingCompareRejectsUnprovedInputAndEnvelope(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "CMPPS X0,X1,$0\nVMOVSS X1,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVCMPPS $0,Z0,Z1,K1,K2\nKMOVD K2,(DX)", + "unknown old K is read": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVCMPPS $0,Z0,Z1,K1,K1\nKMOVD K1,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ DI,K1\nVCMPPS $0,Z0,Z1,K1,K2\nKMOVD K2,(DX)", + "unknown base": "VMOVUPS (SI),X1\nCMPPS (BX),X1,$0\nVMOVSS X1,(DX)", + "native stack": "VMOVUPS (SI),X1\nCMPPS (SP),X1,$0\nVMOVSS X1,(DX)", + "pointer FP": "VMOVUPS (SI),X1\nCMPPS a+0(FP),X1,$0\nVMOVSS X1,(DX)", + "read scalar length": "VMOVUPS (SI),X1\nCMPPS (CX),X1,$0\nVMOVSS X1,(DX)", + "read output": "VMOVUPS (SI),X1\nCMPPS (DX),X1,$0\nVMOVSS X1,(DX)", + "write source": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nCMPPS X0,X1,$0\nVMOVSS X1,(DI)", + "wide output": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nCMPPS X0,X1,$0\nVMOVUPS X1,(DX)", + // A VEX memory load defines zero upper bytes. Legacy register MOVSS + // preserves X1's unknown upper bytes, which scalar VCMP copies and + // the following packed VCMP genuinely reads. + "unknown upper reused": "VMOVSS (DI),X0\nMOVSS X0,X1\nVCMPSS $0,X0,X1,X2\nVCMPPS $0,X2,X2,X3\nVMOVSS X3,(DX)", + "mixed CFG": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),X0\njoined:\nVMOVUPS (SI),X1\nCMPPS X0,X1,$0\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved compare acquired caller continuation: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_fma3.go b/amd64_go_caller_fma3.go new file mode 100644 index 000000000..33564eaeb --- /dev/null +++ b/amd64_go_caller_fma3.go @@ -0,0 +1,37 @@ +package plan9asm + +func x86CallerFMA3Step(ins *Instr, form amd64FMA3Form, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.spec.rawOnly || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := int64(form.byteWidth) + if form.spec.scalar { + width = int64(form.spec.laneBits / 8) + } + required := x86CallerVectorByteMask(width) + if form.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + // All three operands participate in every order, including the old + // destination in zeroing forms. An initialized unknown-valued mask is + // not a proof of inactive lanes; arithmetic cannot invent missing bytes. + for _, index := range []int{0, 1, form.destination} { + if index == form.memory { + continue + } + root := x86CallerVectorRoot(ins.Args[index].Reg) + if state.vectors[root]&required != required { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + known := ^uint64(0) + if form.spec.scalar { + // Scalar FMA preserves the old destination's remaining XMM bytes, + // independently of its low-lane mask, and zeroes higher Y/Z aliases. + upperX := x86CallerVectorByteMask(16) &^ required + known = ^x86CallerVectorByteMask(16) | required | state.vectors[destination]&upperX + } + state.vectors[destination] = known + return true +} diff --git a/amd64_go_caller_fma3_runtime_test.go b/amd64_go_caller_fma3_runtime_test.go new file mode 100644 index 000000000..2e6a9b097 --- /dev/null +++ b/amd64_go_caller_fma3_runtime_test.go @@ -0,0 +1,208 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func qualifyCrossLinuxRuntimeMatrixX86FMA3GoCallerNumericalFamily(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the separately required Linux/amd64 FMA3 caller runtime oracle") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + formats := x86FMA3NamedFormats("amd64") + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + x86OriginalFloatingRuntimeRequireCPU(t, "fma-caller", clang, testTargetTriple(runtime.GOOS, "amd64"), runner) + // Caller proofs fingerprint the complete source selection. Keep that + // selection bounded instead of repeatedly hashing a giant fixture for + // every function. Each format still has its own actual typed Go caller, + // translation and numerical oracle; every batch releases its LLVM objects. + const batchSize = 64 + attempted, passed := 0, 0 + for start := 0; start < len(formats); start += batchSize { + end := start + batchSize + if end > len(formats) { + end = len(formats) + } + t.Run(fmt.Sprintf("formats_%04d_%04d", start, end), func(t *testing.T) { + attempted += end - start + runX86FMA3GoCallerNumericalBatch(t, llc, clang, formats[start:end], start, runner) + passed += end - start + }) + } + t.Logf("%d/%d attempted FMA3 formats passed (%d available): numerical low result, directed rounding, unknown-valued initialized mask, read-only inputs and full output canaries", passed, attempted, len(formats)) +} + +func runX86FMA3GoCallerNumericalBatch(t *testing.T, llc, clang string, formats []string, first int, runner []string) { + t.Helper() + var goSource, assembly, declarations, checks strings.Builder + goSource.WriteString("package caller\nimport \"unsafe\"\n") + for index, instruction := range formats { + name := fmt.Sprintf("fmacase%d", first+index) + opcode := strings.Fields(instruction)[0] + base := strings.Split(opcode, ".")[0] + bits := 32 + if strings.HasSuffix(base, "PD") || strings.HasSuffix(base, "SD") { + bits = 64 + } + declaration := x86CallerFixtureSource[strings.Index(x86CallerFixtureSource, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1) + if bits == 64 { + declaration = strings.ReplaceAll(declaration, "float32", "float64") + } + goSource.WriteString(declaration) + + bank := "X" + if strings.Contains(instruction, "Y1") { + bank = "Y" + } else if strings.Contains(instruction, "Z1") { + bank = "Z" + } + body := fmt.Sprintf("VMOVUPS (DI),%s0\nVMOVUPS (SI),%s1\nVMOVUPS 64(SI),%s2\nKMOVQ AX,K1\n", bank, bank, bank) + body += strings.ReplaceAll(instruction, "(AX)", "(DI)") + "\n" + store := "VMOVSS X2,(DX)" + if bits == 64 { + store = "VMOVSD X2,(DX)" + } + body += store + machine := strings.ReplaceAll(x86CallerFixtureAssembly, "·kernel", "·"+name) + machine = strings.Replace(machine, "MOVL $7,AX", "MOVQ (CX),AX", 1) + assembly.WriteString(strings.Replace(machine, "MOVL AX,(DX)", body, 1)) + fmt.Fprintf(&declarations, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + + order, err := strconv.Atoi(base[len(base)-5 : len(base)-2]) + if err != nil { + t.Fatal(err) + } + mode := 0 + switch { + case strings.HasPrefix(base, "VFMADDSUB"): + mode = 4 + case strings.HasPrefix(base, "VFMSUBADD"): + mode = 5 + case strings.HasPrefix(base, "VFNMADD"): + mode = 2 + case strings.HasPrefix(base, "VFNMSUB"): + mode = 3 + case strings.HasPrefix(base, "VFMSUB"): + mode = 1 + } + rounding := 0 + for suffix, value := range map[string]int{".RD_SAE": 1, ".RU_SAE": 2, ".RZ_SAE": 3} { + if strings.Contains(opcode, suffix) { + rounding = value + } + } + fmt.Fprintf(&checks, " if (check(%s,%q,%d,%d,%d,%t,%t,%d)) return 1;\n", + name, instruction, bits, order, mode, strings.Contains(instruction, "K1"), strings.HasSuffix(opcode, ".Z"), rounding) + } + // Actual Go accepts every original instruction and supplies the exact + // typed caller. Portable LLVM/C execution is not native AVX512 execution + // and is not a whole third-party library/ledger pass. + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + pkg := x86CallerRuntimePackage(t, goSource.String(), assembly.String()) + ctx := llvm.NewContext() + defer ctx.Dispose() + triple := testTargetTriple(runtime.GOOS, "amd64") + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly.String()), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + oracle := x86FMA3CallerCOracle + declarations.String() + "\nint main(void) {\n" + checks.String() + " return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "fma3-caller-numerical", triple, translation.Module.String(), oracle, runner) +} + +const x86FMA3CallerCOracle = `#include +#include +#include +#include +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif + +#pragma STDC FENV_ACCESS ON +static int check(void (*fn)(void*,void*,void*,void*), const char *instruction, unsigned bits, + unsigned order, unsigned mode, bool masked, bool zero, + unsigned rounding) { + const int modes[] = {FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO}; + for (uint64_t length=64; length<68; length++) { + uint8_t first[512], second[512], before_first[512], before_second[512]; + uint8_t output[48], expected[48]; + for (unsigned i=0; i<512/(bits/8); i++) { + if (bits == 32) { + float a=1.00000011920928955078125f; + float b=-1.0000002384185791015625f; + if (i != 0) { a=(float)((int)(i%7)-3)/4; b=(float)((int)(i%5)-2)/8; } + memcpy(first+i*4,&a,4); memcpy(second+i*4,&b,4); + } else { + double a=1.0000000000000002220446049250313; + double b=-1.0000000000000004440892098500626; + if (i != 0) { a=(double)((int)(i%7)-3)/4; b=(double)((int)(i%5)-2)/8; } + memcpy(first+i*8,&a,8); memcpy(second+i*8,&b,8); + } + } + memcpy(before_first,first,512); memcpy(before_second,second,512); + memset(output,0xa5,48); memcpy(expected,output,48); + const uint64_t original_length=length; + fn(first,second,output+16,&length); + if (length != original_length) return 1; + if (fesetround(modes[rounding])) return 2; + if (bits == 32) { + float first_value, second_value, old, a, b, c, result; + memcpy(&first_value,first,4); memcpy(&second_value,second,4); memcpy(&old,second+64,4); + a=old; b=first_value; c=second_value; + if (order == 213) { a=second_value; b=old; c=first_value; } + if (order == 231) { a=second_value; b=first_value; c=old; } + if (mode == 2 || mode == 3) a=-a; + if (mode == 1 || mode == 3 || mode == 4) c=-c; + result=fmaf(a,b,c); + if (masked && !(length&1)) result=zero ? 0 : old; + memcpy(expected+16,&result,4); + } else { + double first_value, second_value, old, a, b, c, result; + memcpy(&first_value,first,8); memcpy(&second_value,second,8); memcpy(&old,second+64,8); + a=old; b=first_value; c=second_value; + if (order == 213) { a=second_value; b=old; c=first_value; } + if (order == 231) { a=second_value; b=first_value; c=old; } + if (mode == 2 || mode == 3) a=-a; + if (mode == 1 || mode == 3 || mode == 4) c=-c; + result=fma(a,b,c); + if (masked && !(length&1)) result=zero ? 0 : old; + memcpy(expected+16,&result,8); + } + if (fesetround(FE_TONEAREST)) return 3; + if (memcmp(output,expected,48) || memcmp(first,before_first,512) || + memcmp(second,before_second,512)) { + fprintf(stderr,"FMA3 %s bits/order/mode/mask/zero/rounding/length %u/%u/%u/%u/%u/%u/%llu\n", + instruction,bits,order,mode,masked,zero,rounding,(unsigned long long)length); + for (unsigned i=16; i<16+bits/8; i++) { + fprintf(stderr,"%02x/%02x ",output[i],expected[i]); + } + fprintf(stderr,"\n"); + return 4; + } + } + return 0; +} +` diff --git a/amd64_go_caller_fma3_test.go b/amd64_go_caller_fma3_test.go new file mode 100644 index 000000000..707d6bb78 --- /dev/null +++ b/amd64_go_caller_fma3_test.go @@ -0,0 +1,134 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86GoCallerFMA3InitializedNumericalInputs(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVFMADD231SS (DI),X0,X1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Y0\nVMOVUPS (SI),Y1\nVFMADD132PS (DI),Y0,Y1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVFNMADD213PD.BCST (DI),Z0,Z1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nVFMADDSUB231PS.Z Z0,Z0,K1,Z1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVFMSUBADD132PD.RD_SAE Z0,Z0,Z1\nVMOVSS X1,(DX)", + } { + t.Run(strings.ReplaceAll(body, "\n", ";"), func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerFMA3CompleteFamilyRegisterEffects(t *testing.T) { + for _, instruction := range x86FMA3NamedFormats("amd64") { + if strings.Contains(instruction, "(AX)") { + continue // Actual typed caller-memory/objects are checked separately. + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + if !x86CallerVectorMoveKnown(ins) { + t.Fatalf("complete named FMA3 format has no shared effect: %s", instruction) + } + form, _, err := amd64FMA3Properties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, missing := range []int{-1, 0, 1, 2} { + state := x86SourceStackState{vectors: map[Reg]uint64{ + "Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0), + }, masks: map[Reg]bool{"K1": true}} + if missing >= 0 { + state.vectors[Reg(fmt.Sprintf("Z%d", missing))] &^= 1 + } + before, masks := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerFMA3Step(&ins, form, &state, nil) + if accepted != (missing == -1) || !reflect.DeepEqual(state.masks, masks) || + !accepted && !reflect.DeepEqual(state.vectors, before) { + t.Fatalf("%s missing input %d: invented numerical data or changed masks: %t/%v", instruction, missing, accepted, state.vectors) + } + } + } + for opcode, spec := range amd64FMA3Specs { + if !spec.rawOnly { + continue + } + ins := Instr{Op: opcode, x86Encoded: true, Args: []Operand{ + {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X2"}, + }} + if x86CallerVectorMoveKnown(ins) { + t.Fatalf("raw-only FP16 acquired a Go-named caller contract: %s", opcode) + } + } +} + +func TestX86GoCallerFMA3ScalarPreservesUnknownUpperXBytes(t *testing.T) { + for _, instruction := range []string{ + "VFMADD132SS X0,X1,X2", "VFMSUB213SD.RD_SAE X0,X1,X2", + "VFNMADD231SS.Z X0,X1,K1,X2", "VFNMSUB132SD.RU_SAE.Z X0,X1,K1,X2", + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := amd64FMA3Properties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + low := x86CallerVectorByteMask(int64(form.spec.laneBits / 8)) + for _, oldUpper := range []uint64{0, 0xff00, 0xff000, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{ + "Z0": low, "Z1": low, "Z2": low | oldUpper, + }, masks: map[Reg]bool{"K1": true}} + if !x86CallerFMA3Step(&ins, form, &state, nil) { + t.Fatalf("initialized low inputs incorrectly require unknown upper X data: %s", instruction) + } + var expected uint64 + for index := 0; index < 64; index++ { + bit := uint64(1) << uint(index) + if index >= 16 || index < form.spec.laneBits/8 || oldUpper&bit != 0 { + expected |= bit + } + } + if state.vectors["Z2"] != expected { + t.Fatalf("%s changed byte %x initialization: got %x want %x", instruction, oldUpper, state.vectors["Z2"], expected) + } + } + } +} + +func TestX86GoCallerFMA3CannotInventNumericalInputs(t *testing.T) { + for _, body := range []string{ + "VFMADD231SS (DI),X0,X1\nVMOVSS X1,(DX)", + "VMOVSS (DI),X0\nVFMADD231SS X0,X0,X1\nVMOVSS X1,(DX)", + "VMOVSS (DI),X0\nVFMADD231SS.Z X0,X0,K1,X0\nVMOVSS X0,(DX)", + "VMOVSS (DI),X0\nKMOVQ DI,K1\nVFMADD231SS.Z X0,X0,K1,X0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS (BX),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS a+0(FP),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS (CX),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS (R14),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS 0(SP),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS (DI),Y0,Y0\nVMOVUPS Y0,(DX)", + "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),Y0\njoined:\nVFMADD231PS (DI),Y0,Y0\nVMOVSS X0,(DX)", + } { + t.Run(strings.ReplaceAll(body, "\n", ";"), func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved FMA input acquired caller continuation: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_gp_simd_transfer.go b/amd64_go_caller_gp_simd_transfer.go new file mode 100644 index 000000000..2f087815b --- /dev/null +++ b/amd64_go_caller_gp_simd_transfer.go @@ -0,0 +1,37 @@ +package plan9asm + +func x86CallerGPSIMDTransferStep(ins *Instr, f x86GPSIMDTransferForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if f.mmx || f.spec.rawOnly { + return false + } + width := int64(f.bits / 8) + lowMask := x86CallerVectorByteMask(width) + var known uint64 + if f.sourceX { + known = state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] & lowMask + } else if f.memory < 0 { + root, valid := x86CallerGP(ins.Args[0].Reg) + _, pointer := state.callers[root] + if ins.Args[0].Kind != OpReg || !valid || root == SP || root == BP || pointer { + return false + } + known = uint64(state.known[root]) & lowMask + } else { + known = lowMask + } + if !f.destinationX && known != lowMask { + return false + } + if f.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, width, f.memoryWrite) { + return false + } + if f.destinationX { + root := x86CallerVectorRoot(ins.Args[1].Reg) + state.vectors[root] = f.definedVectorBytes(known, state.vectors[root]) + return true + } + if f.memoryWrite { + return true + } + return x86CallerNumericWrite(ins.Args[1].Reg, width, state) +} diff --git a/amd64_go_caller_horizontal_float.go b/amd64_go_caller_horizontal_float.go new file mode 100644 index 000000000..36c23d6b4 --- /dev/null +++ b/amd64_go_caller_horizontal_float.go @@ -0,0 +1,24 @@ +package plan9asm + +func x86CallerHorizontalFloatStep(ins *Instr, form x86HorizontalFloatForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.memory >= 0 { + // SSE3's legacy m128 requires alignment. A slice/nonalias witness + // does not establish that; VEX forms allow unaligned memory. + if !form.spec.vector || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.byteWidth), false) { + return false + } + } + required := x86CallerVectorByteMask(int64(form.byteWidth)) + for _, index := range []int{form.first, form.second} { + if index != form.memory && state.vectors[x86CallerVectorRoot(ins.Args[index].Reg)]&required != required { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + known := ^uint64(0) + if !form.spec.vector { + known = state.vectors[destination] | required + } + state.vectors[destination] = known + return true +} diff --git a/amd64_go_caller_horizontal_float_test.go b/amd64_go_caller_horizontal_float_test.go new file mode 100644 index 000000000..bd7b4ca70 --- /dev/null +++ b/amd64_go_caller_horizontal_float_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86GoCallerHorizontalFloatOriginalSourceReachesObjects(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),Y0\nLONG $0xc07cffc5\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVMOVUPS (SI),Y1\nVHADDPS Y0,Y1,Y2\nVMOVSS X2,(DX)", + "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nHSUBPD X0,X1\nVMOVSS X1,(DX)", + } { + t.Run(strings.Split(body, "\n")[1], func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerHorizontalFloatWholeFamilyByteEffects(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + if strings.Contains(row, "(BX)") { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86HorizontalFloatProperties(ins, "amd64") + if !recognized || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatalf("missing shared effect %s: %v", row, err) + } + required := x86CallerVectorByteMask(int64(form.byteWidth)) + for _, missing := range []int{-1, form.first, form.second, form.destination} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": required, "Z1": required, "Z2": 0}} + if missing >= 0 { + state.vectors[x86CallerVectorRoot(ins.Args[missing].Reg)] &^= uint64(1) << uint(form.byteWidth-1) + } + before, _ := x86CloneCallerVectorState(state.vectors, nil) + accepted := x86CallerHorizontalFloatStep(&ins, form, &state, nil) + want := missing == -1 || missing == form.destination && form.spec.vector + if accepted != want || !accepted && !reflect.DeepEqual(state.vectors, before) { + t.Fatalf("%s missing %d: %t want %t, invented high source bytes", row, missing, accepted, want) + } + } + for _, upper := range []uint64{0, 0xff00000000, ^required} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": required, "Z1": required | upper, "Z2": upper}} + if !x86CallerHorizontalFloatStep(&ins, form, &state, nil) { + t.Fatal("fully initialized sources rejected") + } + want := ^uint64(0) + if !form.spec.vector { + want = required | upper + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatalf("%s definitions %x want %x", row, got, want) + } + } + } +} + +func TestX86GoCallerHorizontalFloatAllOpcodesAndMemoryBounds(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + memory := strings.Contains(row, "(BX)") + legacy := !strings.HasPrefix(row, "V") + row = strings.Replace(row, "(BX)", "(DI)", 1) + destination := "X1" + if !legacy { + destination = "X2" + } + body := fmt.Sprintf("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\n%s\nVMOVSS %s,(DX)", row, destination) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + err := x86CallerCoreTranslate(t, pkg, assembly, triple) + if legacy && memory { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved legacy alignment: %s: %v", row, err) + } + } else if err != nil { + t.Fatalf("%s/%s: %v", row, triple, err) + } + } + } +} + +func TestX86GoCallerHorizontalFloatDoesNotInventPermissions(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "VHADDPS X0,X1,X2\nVMOVSS X2,(DX)", + "not a zero idiom": "VHSUBPS X0,X0,X2\nVMOVSS X2,(DX)", + "unknown destructive destination": "VMOVUPS (DI),X0\nHSUBPD X0,X1\nVMOVSS X1,(DX)", + "partially defined source": "VMOVSS X0,X1,X2\nVMOVUPS (DI),X0\nVHADDPS X0,X2,X1\nVMOVSS X1,(DX)", + "unknown input": "VMOVUPS (DI),X0\nVHADDPD (BX),X0,X1\nVMOVSS X1,(DX)", + "physical SP": "VMOVUPS (DI),X0\nVHADDPS 0(SP),X0,X1\nVMOVSS X1,(DX)", + "FP pointer bits": "VMOVUPS (DI),X0\nVHSUBPS a+0(FP),X0,X1\nVMOVSS X1,(DX)", + "length as vector": "VMOVUPS (DI),X0\nVHADDPD (CX),X0,X1\nVMOVSS X1,(DX)", + "g as input": "VMOVUPS (DI),X0\nVHADDPS (R14),X0,X1\nVMOVSS X1,(DX)", + "result wider than output": "VMOVUPS (DI),Y0\nVHADDPS Y0,Y0,Y1\nVMOVUPS Y1,(DX)", + "write original input": "VMOVUPS (DI),X0\nVHADDPS X0,X0,X1\nVMOVSS X1,(DI)", + "CFG missing incoming value": "CMPQ AX,$0\nJE join\nVMOVUPS (DI),X0\njoin:\nVHADDPS X0,X0,X1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved horizontal state passed: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_horizontal_integer_test.go b/amd64_go_caller_horizontal_integer_test.go new file mode 100644 index 000000000..0a87f3f68 --- /dev/null +++ b/amd64_go_caller_horizontal_integer_test.go @@ -0,0 +1,229 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// This first-party caller closes its own source/ABI boundary. The horizontal +// operation has no exemption from the original raw-return stack certificate. +func TestX86GoCallerHorizontalIntegerOriginalSourceThreeAPIs(t *testing.T) { + for _, body := range []string{ + "VMOVDQU (DI),Y0\nVMOVDQU (SI),Y1\nVPHADDD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "VMOVDQU (SI),Y1\nVPHADDD (DI),Y1,Y2\nVMOVSS X2,(DX)", + } { + t.Run(strings.Split(body, "\n")[1], func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs, ResolveSym: resolve} + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + if ir, err := Translate(file, options); err != nil { + t.Error("text", err) + } else { + compileLLVMToObject(t, llc, options.TargetTriple, "horizontal-caller-text.ll", "horizontal-caller-text.o", ir) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Error("module", err) + } else { + compileLLVMToObject(t, llc, options.TargetTriple, "horizontal-caller-module.ll", "horizontal-caller-module.o", module.String()) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, options.TargetTriple); err != nil { + t.Error("selected Go caller", err) + } + }) + } +} + +func TestX86GoCallerHorizontalIntegerCompleteNamedAndRawThreeAPIs(t *testing.T) { + for _, raw := range []bool{false, true} { + var body strings.Builder + for _, op := range x86HorizontalIntegerOpcodes { + rows := []string{op + " X0,X1"} + for _, bank := range []string{"X", "Y"} { + rows = append(rows, fmt.Sprintf("V%s %s0,%s1,%s2", op, bank, bank, bank), fmt.Sprintf("V%s (DI),%s1,%s2", op, bank, bank)) + } + for _, row := range rows { + body.WriteString("VMOVDQU (DI),Y0\nVMOVDQU (SI),Y1\n") + if raw { + code := assembleX87ControlBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if len(code) < 2 || code[len(code)-1] != 0xc3 { + t.Fatal("original Go horizontal bytes have no exact near RET", row, code) + } + for _, value := range code[:len(code)-1] { + fmt.Fprintf(&body, "BYTE $0x%02x\n", value) + } + } else { + body.WriteString(row + "\n") + } + result := "X2" + if !strings.HasPrefix(row, "V") { + result = "X1" + } + fmt.Fprintf(&body, "VMOVSS %s,(DX)\n", result) + } + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body.String(), 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(fmt.Sprint(raw, "/", triple), func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "horizontal-caller-text.ll", "horizontal-caller-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "horizontal-caller-module.ll", "horizontal-caller-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestX86GoCallerHorizontalIntegerCompleteMemoryCertificates(t *testing.T) { + for _, op := range x86HorizontalIntegerOpcodes { + for _, bank := range []string{"X", "Y"} { + row := fmt.Sprintf("V%s (DI),%s0,%s1", op, bank, bank) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVDQU (SI),Y0\n"+row+"\nVMOVSS X1,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs, ResolveSym: resolve}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86HorizontalIntegerProperties(ins, "amd64") + if !known || err != nil || form.memory < 0 { + continue + } + found = true + access, exists := ins.x86CallerMemory[form.memory] + if !exists || access.width != int64(form.byteWidth) || access.write || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatal("full horizontal source envelope lost its instruction identity", row) + } + for _, change := range []string{"opcode", "source", "destination", "width", "address", "direction", "address size"} { + access := access + local := ins + local.Args = append([]Operand(nil), ins.Args...) + local.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch change { + case "opcode": + local.Op = "VPHADDSW" + if local.Op == ins.Op { + local.Op = "VPHSUBSW" + } + case "source": + local.Args[form.second].Reg = Reg(bank + "7") + case "destination": + local.Args[form.destination].Reg = Reg(bank + "7") + case "width": + access.width-- + local.x86CallerMemory[form.memory] = access + case "address": + local.Args[form.memory].Mem.Off++ + case "direction": + access.write = true + local.x86CallerMemory[form.memory] = access + case "address size": + local.x86AddressBits = 32 + } + if x86CallerMemoryCertified(local, form.memory) { + t.Fatal("edited horizontal instruction borrowed stale memory identity", row, change) + } + } + } + if !found { + t.Fatal("fixture did not consume horizontal memory", row) + } + } + } +} + +func TestX86GoCallerHorizontalIntegerRetainsContextBoundaries(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVDQU (DI),Y1\nVPHADDD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVDQU (DI),Y0\nVPHSUBW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown destructive destination": "VMOVDQU (DI),X0\nPHADDSW X0,X1\nVMOVSS X1,(DX)", + "unknown same subtract": "VPHSUBD Y0,Y0,Y2\nVMOVSS X2,(DX)", + "partial upper input": "MOVUPS (DI),X0\nVMOVDQU (SI),Y1\nVPHADDW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown pointer": "VMOVDQU (SI),Y0\nVPHADDD (BX),Y0,Y1\nVMOVSS X1,(DX)", + "full width length envelope": "VMOVDQU (SI),Y0\nVPHADDW (CX),Y0,Y1\nVMOVSS X1,(DX)", + "full width scalar result envelope": "VMOVDQU (SI),Y0\nVPHSUBSW (DX),Y0,Y1\nVMOVSS X1,(DX)", + "physical SP": "VMOVDQU (SI),Y0\nVPHADDD (SP),Y0,Y1\nVMOVSS X1,(DX)", + "physical BP": "VMOVDQU (SI),Y0\nVPHSUBD (BP),Y0,Y1\nVMOVSS X1,(DX)", + "private stack alias": "MOVQ SP,BX\nVMOVDQU (SI),Y0\nVPHADDW (BX),Y0,Y1\nVMOVSS X1,(DX)", + "g memory": "VMOVDQU (SI),Y0\nVPHADDSW (R14),Y0,Y1\nVMOVSS X1,(DX)", + "FP pointer data": "VMOVDQU (SI),X0\nVPHADDD a+0(FP),X0,X1\nVMOVSS X1,(DX)", + "legacy native alignment": "VMOVDQU (SI),X0\nPHADDW (DI),X0\nVMOVSS X0,(DX)", + "MMX state": "PHADDD M0,M1\nMOVL AX,(DX)", + "pointer input write": "VMOVDQU (DI),Y0\nVPHADDD Y0,Y0,Y1\nVMOVSS X1,(DI)", + "wide result write": "VMOVDQU (DI),Y0\nVPHADDD Y0,Y0,Y1\nVMOVDQU Y1,(DX)", + "unrestored SP": "VMOVDQU (DI),Y0\nVPHADDD Y0,Y0,Y1\nVMOVSS X1,(DX)", + "CFG missing source": "VMOVDQU (DI),Y0\nCMPQ AX,$0\nJE join\nVMOVDQU (SI),Y1\njoin:\nVPHADDD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown effect": "VMOVDQU (DI),Y0\nVPHADDD Y0,Y0,Y1\nXCHGL AX,(DI)\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + if name == "unrestored SP" { + assembly = strings.Replace(assembly, "BYTE $0xc3", "SUBQ $8,SP\nBYTE $0xc3", 1) + } + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved %s acquired source/caller privilege: %v", name, err) + } + }) + } + ins := x86YMMReadSource(t, "PHADDD M0,M1") + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0)}} + state = x86CloneCallerState(state) + before := x86CloneCallerState(state) + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(before, state) { + t.Fatal("MMX grammar inherited a physical caller state proof") + } +} diff --git a/amd64_go_caller_immediate_shuffle.go b/amd64_go_caller_immediate_shuffle.go new file mode 100644 index 000000000..f52ff0c46 --- /dev/null +++ b/amd64_go_caller_immediate_shuffle.go @@ -0,0 +1,36 @@ +package plan9asm + +// Byte availability follows the exact immediate permutation. No unused byte, +// initialized mask, or shadow zero value establishes additional input data. +func x86CallerImmediateShuffleStep(ins *Instr, form x86ImmediateShuffleForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.spec.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + var source uint64 + if form.memory >= 0 { + // Caller-region identity does not promise legacy m128 alignment. + // E4NF memory keeps its full native tuple, including at mask zero. + if form.spec.legacy || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + source = x86CallerVectorByteMask(int64(form.memoryBytes)) + } else { + source = state.vectors[x86CallerVectorRoot(form.source.Reg)] + } + known := uint64(0) + for byteIndex := 0; byteIndex < form.byteWidth; byteIndex++ { + known |= (source >> uint(form.sourceByte(byteIndex)) & 1) << uint(byteIndex) + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.mask >= 0 && !form.zeroing { + known &= old // Unknown initialized K may retain any old lane. + } + if form.spec.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_insert_extract.go b/amd64_go_caller_insert_extract.go new file mode 100644 index 000000000..fb4e57e32 --- /dev/null +++ b/amd64_go_caller_insert_extract.go @@ -0,0 +1,62 @@ +package plan9asm + +func x86CallerInsertExtractStep(ins *Instr, f x86InsertExtractForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if f.mask >= 0 && !state.masks[ins.Args[f.mask].Reg] || f.mask >= 0 && f.memoryWrite { + return false + } + if f.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.spec.chunk), f.memoryWrite) { + return false + } + source := state.vectors[x86CallerVectorRoot(ins.Args[f.source].Reg)] + low := x86CallerVectorByteMask(int64(f.spec.chunk)) + if !f.spec.insert { + if source>>uint(f.offset)&low != low { + return false + } + if f.memoryWrite { + return true + } + if f.spec.scalar { + return x86CallerNumericWrite(ins.Args[f.destination].Reg, 8, state) + } + known := low + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + if f.mask >= 0 && !f.zeroing { + known &= state.vectors[destination] + } + state.vectors[destination] = ^low | known + return true + } + base := state.vectors[x86CallerVectorRoot(ins.Args[f.base].Reg)] + outputMask := x86CallerVectorByteMask(int64(f.output)) + if f.spec.scalar { + insertLane := int(f.immediate>>4) & 3 + insertMask := uint64(15) << uint(insertLane*4) + knownSource := source >> uint((f.immediate>>6)*4) & 15 + if f.memory >= 0 { + knownSource = 15 + } + known := base&^insertMask | knownSource<= 0 && !f.zeroing { + known &= state.vectors[x86CallerVectorRoot(ins.Args[f.destination].Reg)] + } + state.vectors[x86CallerVectorRoot(ins.Args[f.destination].Reg)] = ^outputMask | known&outputMask + return true +} diff --git a/amd64_go_caller_insert_extract_test.go b/amd64_go_caller_insert_extract_test.go new file mode 100644 index 000000000..268e58c29 --- /dev/null +++ b/amd64_go_caller_insert_extract_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerInsertExtractOriginalSourceReachesObjects(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),Y0\nLONG $0x197de3c4\nWORD $0x01c1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Z0\nVEXTRACTF64X4 $1,Z0,Y1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Y0\nVMOVUPS (SI),X1\nVINSERTF128 $1,X1,Y0,Y2\nVMOVSS X2,(DX)", + "VMOVUPS (DI),X0\nVEXTRACTPS $3,X0,AX\nMOVL AX,(DX)", + "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVINSERTPS $64,X1,X0,X2\nVMOVSS X2,(DX)", + } { + t.Run(strings.Split(body, "\n")[1], func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerInsertExtractWholeFamilyByteKnowledge(t *testing.T) { + for _, row := range x86InsertExtractFormats() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + f, recognized, err := x86InsertExtractProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatal(err) + } + if f.memory >= 0 { + continue + } + if f.spec.scalar && !f.spec.insert { + continue // full typed scalar-state coverage is exercised via public API below + } + for _, missing := range []int{-1, 0, 7, 15, 16, 31, 32, 63} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0)}, masks: map[Reg]bool{"K1": true}} + if missing >= 0 { + state.vectors[x86CallerVectorRoot(ins.Args[f.source].Reg)] &^= uint64(1) << uint(missing) + } + accepted := x86CallerInsertExtractStep(&ins, f, &state, nil) + if !f.spec.insert { + want := missing < f.offset || missing >= f.offset+f.output + if accepted != want { + t.Fatalf("%s missing source byte %d: %t want %t", row, missing, accepted, want) + } + } else if !f.spec.scalar && accepted != (missing < 0 || missing >= f.spec.chunk) { + t.Fatalf("%s missing insertion byte %d: %t", row, missing, accepted) + } + } + } + for _, row := range []string{"VEXTRACTF128 $1,Y1,X0", "VINSERTI128 $1,X0,Y1,Y2", "VINSERTPS $0,X0,X1,X2"} { + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + f, _, _ := x86InsertExtractProperties(ins, "amd64") + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0xffff, "Z1": ^uint64(0), "Z2": 0}, masks: map[Reg]bool{}} + if !x86CallerInsertExtractStep(&ins, f, &state, nil) || state.vectors[x86CallerVectorRoot(ins.Args[f.destination].Reg)] != ^uint64(0) { + t.Fatalf("complete inputs did not establish physical upper-zero: %s", row) + } + } +} + +func TestX86GoCallerInsertExtractAllOpcodesAndUnknownPermissions(t *testing.T) { + for _, row := range x86InsertExtractFormats() { + row = strings.Replace(row, "(BX)", "(DI)", 1) + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + f, _, _ := x86InsertExtractProperties(ins, "amd64") + if f.memoryWrite { + continue // caller's scalar result is only four bytes, not vector output + } + store := fmt.Sprintf("VMOVSS X%d,(DX)", ins.Args[f.destination].Reg[len(ins.Args[f.destination].Reg)-1]-'0') + if f.spec.scalar && !f.spec.insert { + store = "MOVL AX,(DX)" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nKMOVQ AX,K1\n"+row+"\n"+store, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("%s: %v", row, err) + } + } + for name, body := range map[string]string{ + "unknown selected extract": "VEXTRACTF128 $1,Y0,X1\nVMOVSS X1,(DX)", + "unknown inserted bytes": "VMOVUPS (DI),Y1\nVINSERTF128 $1,X0,Y1,Y2\nVEXTRACTF128 $1,Y2,X2\nVMOVSS X2,(DX)", + "uninitialized retained base": "VMOVUPS (DI),X0\nVINSERTI128 $1,X0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVEXTRACTI32X4.Z $0,Z0,K1,X1\nVMOVSS X1,(DX)", + "unknown masked merge destination": "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVEXTRACTF32X4 $0,Z0,K1,X1\nVMOVSS X1,(DX)", + "pointer FP not numerical": "VMOVUPS (DI),Y1\nVINSERTF128 $0,a+0(FP),Y1,Y2\nVMOVSS X2,(DX)", + "unknown memory even zero mask": "VMOVUPS (DI),Z1\nMOVQ $0,AX\nKMOVQ AX,K1\nVINSERTF32X4.Z $0,(BX),Z1,K1,Z2\nVMOVSS X2,(DX)", + "read source SP": "VMOVUPS (DI),Y1\nVINSERTI128 $0,(SP),Y1,Y2\nVMOVSS X2,(DX)", + "source length too narrow": "VMOVUPS (DI),Y1\nVINSERTF128 $0,(CX),Y1,Y2\nVMOVSS X2,(DX)", + "write input": "VMOVUPS (DI),X0\nVEXTRACTPS $0,X0,(DI)", + "write g": "VMOVUPS (DI),X0\nVEXTRACTPS $0,X0,R14", + "CFG incomplete": "CMPQ AX,$0\nJE join\nVMOVUPS (DI),Y0\njoin:\nVEXTRACTF128 $1,Y0,X1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown source permission invented: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_lea_runtime_test.go b/amd64_go_caller_lea_runtime_test.go new file mode 100644 index 000000000..a1fc2e2ac --- /dev/null +++ b/amd64_go_caller_lea_runtime_test.go @@ -0,0 +1,216 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86CallerLEARuntimeForm struct { + name, suffix string + bits, scale int +} + +func x86CallerLEARuntimeForms() []x86CallerLEARuntimeForm { + var forms []x86CallerLEARuntimeForm + for _, width := range []struct { + suffix string + bits int + }{{"W", 16}, {"L", 32}, {"Q", 64}} { + for _, scale := range []int{1, 2, 4, 8} { + forms = append(forms, x86CallerLEARuntimeForm{ + name: fmt.Sprintf("kernel%s%d", width.suffix, scale), suffix: width.suffix, + bits: width.bits, scale: scale, + }) + } + } + return forms +} + +func x86CallerLEARuntimeSources() (goSource, assembly, goTest, cOracle string) { + var declarations, machine, tests, c strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + tests.WriteString(`package caller +import "testing" +func TestOriginalLEAMathematics(t *testing.T) { + values := []uint64{0, 1, 7, 19, 65535, 1<<16, 1<<31, 1<<32-1, 1<<32, 1<<63, ^uint64(0)} +`) + c.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range x86CallerLEARuntimeForms() { + fmt.Fprintf(&declarations, ` +//go:noescape +func %[1]s(a, b, res, length unsafe.Pointer) +func Call%[1]s(x, y []uint64) uint64 { + var res uint64 + length := len(x) + %[1]s(unsafe.Pointer(unsafe.SliceData(x)), unsafe.Pointer(unsafe.SliceData(y)), + unsafe.Pointer(&res), unsafe.Pointer(&length)) + return res +} +`, form.name) + fmt.Fprintf(&machine, `TEXT ·%s(SB),4,$0-32 +MOVQ a+0(FP),DI +MOVQ b+8(FP),SI +MOVQ res+16(FP),DX +MOVQ length+24(FP),CX +BYTE $0x55 +WORD $0x8948 +BYTE $0xe5 +LONG $0xf8e48348 +MOVQ (DI),R9 +MOVQ (SI),BX +MOVQ $0x1122334455667788,AX +LEA%s -7(R9)(BX*%d),AX +MOVQ AX,(DX) +WORD $0x8948 +BYTE $0xec +BYTE $0x5d +BYTE $0xc3 +`, form.name, form.suffix, form.scale) + fmt.Fprintf(&tests, ` + for _, x := range values { + for _, y := range values { + a, b := []uint64{x, 23}, []uint64{y, 37} + canary := [4]uint64{11, 41, 53, 67} + want := x + y*%[1]d - 7 +`, form.scale) + switch form.bits { + case 16: + tests.WriteString(" want = 0x1122334455660000 | uint64(uint16(want))\n") + case 32: + tests.WriteString(" want = uint64(uint32(want))\n") + } + fmt.Fprintf(&tests, ` + if got := Call%[1]s(a, b); got != want || a[0] != x || a[1] != 23 || b[0] != y || b[1] != 37 || + canary != [4]uint64{11, 41, 53, 67} { + t.Fatalf("%[1]s x=%%#x y=%%#x got=%%#x want=%%#x", x, y, got, want) + } + } + } +`, form.name) + fmt.Fprintf(&c, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", form.name, form.name) + } + tests.WriteString("}\n") + c.WriteString(`int main(void) { + const uint64_t values[] = {0,1,7,19,65535,UINT64_C(1)<<16,UINT64_C(1)<<31,UINT32_MAX, + UINT64_C(1)<<32,UINT64_C(1)<<63,UINT64_MAX}; +`) + for _, form := range x86CallerLEARuntimeForms() { + fmt.Fprintf(&c, ` + for (unsigned i=0; i= 0 { + // Preserve the existing packed-move boundary: ordinary numerical + // SliceData is not proof of the aligned opcode's native address. + if form.aligned { + return false + } + if form.memoryWrite && state.vectors[x86CallerVectorRoot(form.source.Reg)]&mask != mask { + return false + } + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, width, form.memoryWrite) { + return false + } + if form.memoryWrite { + return true + } + known = mask + } else { + // Register copies propagate unknown source bytes without making + // them numeric. A later observable consumer still needs its bytes. + known = state.vectors[x86CallerVectorRoot(form.source.Reg)] & mask + } + destination := x86CallerVectorRoot(form.destination.Reg) + state.vectors[destination] = state.vectors[destination]&^mask | known + return true // Legacy writes preserve every upper Y/Z byte and all FLAGS. +} diff --git a/amd64_go_caller_legacy_moves_test.go b/amd64_go_caller_legacy_moves_test.go new file mode 100644 index 000000000..e3dc579a9 --- /dev/null +++ b/amd64_go_caller_legacy_moves_test.go @@ -0,0 +1,215 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +// The independent Go directory includes all twelve yxmov opcodes and MOVOA's +// frontend alias. Sharing yxmov does not share lane, source or memory effects. +var x86CallerLegacyMoveDirectory = []struct { + name, prefix, bytes string + width int64 + half, aligned bool +}{ + {"MOVAPD", "Pe", "0x28, 0x29", 16, false, true}, + {"MOVAPS", "Pm", "0x28, 0x29", 16, false, true}, + {"MOVO", "Pe", "0x6f, 0x7f", 16, false, true}, + {"MOVOU", "Pf3", "0x6f, 0x7f", 16, false, false}, + {"MOVHPD", "Pe", "0x16, 0x17", 8, true, false}, + {"MOVHPS", "Pm", "0x16, 0x17", 8, true, false}, + {"MOVLPD", "Pe", "0x12, 0x13", 8, true, false}, + {"MOVLPS", "Pm", "0x12, 0x13", 8, true, false}, + {"MOVSD", "Pf2", "0x10, 0x11", 8, false, false}, + {"MOVSS", "Pf3", "0x10, 0x11", 4, false, false}, + {"MOVUPD", "Pe", "0x10, 0x11", 16, false, false}, + {"MOVUPS", "Pm", "0x10, 0x11", 16, false, false}, + {"MOVOA", "Pe", "0x6f, 0x7f", 16, false, true}, +} + +func TestX86GoCallerLegacyMoveCompleteGoRows(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`\{A(MOV\w+), yxmov, (\w+), opBytes\{([^}]+)\}\}`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 12 { + t.Fatalf("actual Go yxmov opcode/attribute inventory changed: %d", len(rows)) + } + for index, row := range rows { + want := x86CallerLegacyMoveDirectory[index] + if row[1] != want.name || row[2] != want.prefix || row[3] != want.bytes { + t.Fatalf("actual Go opcode attributes differ: %v want %+v", row, want) + } + } + table := regexp.MustCompile(`(?s)var yxmov = \[\]ytab\{\n(.*?)\n\}`).FindSubmatch(data) + if len(table) != 2 || strings.Join(strings.Fields(string(table[1])), " ") != + "{Zm_r_xm, 1, argList{Yxm, Yxr}}, {Zr_m_xm, 1, argList{Yxr, Yxm}}," { + t.Fatal("actual Go operand classes, zcases or row-byte advances changed") + } + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil || !strings.Contains(string(aliases), `instructions["MOVOA"] = x86.AMOVO`) { + t.Fatal("actual Go MOVOA frontend identity changed", err) + } +} + +func x86CallerLegacyMoveInstruction(t *testing.T, text string) Instr { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\n"+text+"\nRET\n") + if err != nil { + t.Fatal(err) + } + return file.Funcs[0].Instrs[1] +} + +func TestX86GoCallerLegacyMoveRegisteredEffects(t *testing.T) { + // A transfer-domain model, not a package/caller authority producer. Complete + // actual source/signature binding is tested separately below. Input slices + // retain the existing numerical-origin/nonalias contract, not allocation bounds. + proof := &x86GoCallerRegionProof{arguments: []x86CallerArgumentRegion{ + {kind: x86CallerSliceInput, width: 4}, + {kind: x86CallerFreshResult, width: 8}, + }} + for _, row := range x86CallerLegacyMoveDirectory { + for _, operands := range []string{"X0,X1", "(DI),X1", "X0,(DX)"} { + t.Run(row.name+"/"+operands, func(t *testing.T) { + ins := x86CallerLegacyMoveInstruction(t, row.name+" "+operands) + original := fmt.Sprintf("%#v", ins) + if known := x86CallerVectorMoveKnown(ins); known == row.half { + t.Errorf("caller known axis: got %t, half-unregistered=%t", known, row.half) + } + if x86SourceStackInstructionKnown(ins, "MOVQ", "Q", "amd64") { + t.Error("a move shape alone acquired non-caller source-stack admission") + } + if original != fmt.Sprintf("%#v", ins) { + t.Fatal("known/grammar observation changed original instruction") + } + state := x86SourceStackState{ + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}, DX: {argument: 1}}, + vectors: map[Reg]uint64{"Z0": 0xffff, "Z1": 1 << 40}, + known: map[Reg]uint8{}, masks: map[Reg]bool{}, partialNumericFlags: x86ByteFlagCF | x86ByteFlagOF, + } + wantValid := !row.half && (!row.aligned || operands == "X0,X1") && + (operands != "X0,(DX)" || row.width <= 8) + if valid := x86CallerVectorMoveStep(&ins, &state, proof); valid != wantValid { + t.Errorf("caller step axis: got %t want %t", valid, wantValid) + } + if !x86SourceStackFlagsStep(ins, &state) || x86SourceStackNumericFlagMask(&state) != x86ByteFlagCF|x86ByteFlagOF { + t.Error("legacy move changed or fabricated numerical flags") + } + memory := -1 + if operands == "(DI),X1" { + memory = 0 + } else if operands == "X0,(DX)" { + memory = 1 + } + if memory >= 0 { + width, write, known := x86InstructionMemoryShape(ins, memory, "amd64") + if known == row.half || known && (width != row.width || write != (memory == 1)) { + t.Errorf("memory-shape axis differs: %d/%t/%t", width, write, known) + } + if wantValid && (!x86CallerMemoryCertified(ins, memory) || ins.x86CallerMemory[memory].width != row.width) { + t.Error("accepted read/write lacks its exact instruction-local native width") + } + } else if !row.half { + want := uint64(1)<<40 | 0xffff + if row.width < 16 { + want = uint64(1)<<40 | x86CallerVectorByteMask(row.width) + } + if state.vectors["Z1"] != want { + t.Errorf("register transfer fabricated/cleared old X/Y/Z bytes: %x want %x", state.vectors["Z1"], want) + } + } + }) + } + } +} + +func TestX86GoCallerLegacyMoveActualSourceProducer(t *testing.T) { + for _, row := range x86CallerLegacyMoveDirectory { + t.Run(row.name, func(t *testing.T) { + body := row.name + " (DI),X1" + if row.aligned { + // The current caller contract has no native alignment witness. + // Register recognition must not authorize aligned memory. + body = "VMOVUPS (DI),X1\n" + row.name + " X1,X2" + } + assembly := "TEXT ·kernel(SB),4,$0-32\nMOVQ a+0(FP),DI\n" + body + "\nBYTE $0xc3\n" + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + sig := sigs["caller.kernel"] + if sig.Ret != Void || len(sig.Frame.Results) != 0 || sig.x86CallerRegions == nil || + !x86CallerRegionSourceMatches(sig, file.Funcs[0], "amd64") { + t.Fatal("actual original source and pointer-only Void signature prerequisite missing") + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", ResolveSym: resolve, Sigs: sigs} + proved := normalizeX86RawReturnSourceStack(decoded, options, true, file.Funcs[0]) + if row.half { + if proved.x86SourceStack != nil { + t.Fatal("half-vector move inherited an unproved caller byte effect") + } + return + } + if proved.x86SourceStack == nil || !x86CallerNormalizedSourceMatches(proved, sig) { + t.Fatalf("existing typed move lacks whole-source caller producer hook: %s", proved.x86SourceStackFailure) + } + for _, ins := range proved.Instrs { + if !row.aligned && ins.Op == Op(row.name) && (!x86CallerMemoryCertified(ins, 0) || ins.x86CallerMemory[0].width != row.width) { + t.Fatal("actual instruction did not retain exact numerical-source read width") + } + } + }) + } +} + +func TestX86GoCallerLegacyMoveDoesNotBorrowUnknownBytesOrRegions(t *testing.T) { + for _, row := range x86CallerLegacyMoveDirectory { + if row.half { + continue + } + for _, text := range []string{row.name + " X0,X1", row.name + " (BX),X1", row.name + " X0,(DI)", row.name + " X0,1(DX)"} { + t.Run(text, func(t *testing.T) { + ins := x86CallerLegacyMoveInstruction(t, text) + proof := &x86GoCallerRegionProof{arguments: []x86CallerArgumentRegion{ + {kind: x86CallerSliceInput, width: 4}, {kind: x86CallerFreshResult, width: row.width}, + }} + state := x86SourceStackState{ + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}, DX: {argument: 1}}, + vectors: map[Reg]uint64{"Z0": x86CallerVectorByteMask(row.width) &^ 1}, + known: map[Reg]uint8{}, masks: map[Reg]bool{}, + } + valid := x86CallerVectorMoveStep(&ins, &state, proof) + if text == row.name+" X0,X1" { + // A register copy may propagate unknown bytes. It may never + // turn those bytes into a numerically initialized output. + if valid && state.vectors["Z1"]&x86CallerVectorByteMask(row.width) == x86CallerVectorByteMask(row.width) { + t.Fatal("register transfer fabricated a missing source byte") + } + ins = x86CallerLegacyMoveInstruction(t, row.name+" X1,(DX)") + valid = x86CallerVectorMoveStep(&ins, &state, proof) + } + if valid { + t.Fatal("move acquired unknown numerical byte/address or exceeded the fresh result envelope") + } + if !reflect.DeepEqual(state.callers, map[Reg]x86CallerPointerValue{DI: {argument: 0}, DX: {argument: 1}}) { + t.Fatal("failed numerical transfer changed pointer provenance") + } + }) + } + } +} diff --git a/amd64_go_caller_logical.go b/amd64_go_caller_logical.go new file mode 100644 index 000000000..1ea3a8ccc --- /dev/null +++ b/amd64_go_caller_logical.go @@ -0,0 +1,31 @@ +package plan9asm + +// Packed bitwise operations do not write flags or GP/stack state. Their byte +// availability is independent of the shared operand grammar. MMX has no +// reviewed private caller move bank and must not be aliased to virtual Z0. +func x86CallerPackedLogicalStep(ins *Instr, form x86PackedLogicalForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := uint64(0) + if form.memory >= 0 { + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } else { + first = state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + known := x86PackedLogicalDefinedBytes(*ins, form, first, second, old) + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + // Every VEX/EVEX width clears all higher physical register bits. + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_logical_runtime_test.go b/amd64_go_caller_logical_runtime_test.go new file mode 100644 index 000000000..16eefe33c --- /dev/null +++ b/amd64_go_caller_logical_runtime_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86GoLogicalAndNotOperandOrder(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires Linux/amd64 runtime counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools missing") + } + var declarations, assembly, tests, oracle strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + tests.WriteString(`package caller +import "testing" +func TestOriginalLogicalMathematics(t *testing.T) { + values:=[]uint64{0,1,7,0x0ff00ff0a55aa55a,0x33cc55aa5aa5f00f,1<<63,^uint64(0)} +`) + oracle.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, op := range []string{"ANDNPS", "ANDNPD", "PANDN"} { + name := strings.ToLower(op) + fmt.Fprintf(&declarations, ` +//go:noescape +func %[1]s(a,b,res,length unsafe.Pointer) +func Call%[1]s(x,y []uint64) uint64 { + var res uint64 + length:=len(x) + %[1]s(unsafe.Pointer(unsafe.SliceData(x)),unsafe.Pointer(unsafe.SliceData(y)),unsafe.Pointer(&res),unsafe.Pointer(&length)) + return res +} +`, name) + body := "MOVSD (DI),X0\nMOVSD (SI),X1\n" + op + " X0,X1\nMOVSD X1,(DX)" + machine := strings.Replace(x86CallerFixtureAssembly, "·kernel", "·"+name, 1) + machine = strings.Replace(machine, "MOVL AX,(DX)", body, 1) + assembly.WriteString(machine) + fmt.Fprintf(&tests, ` + for _,a:=range values { for _,b:=range values { + x,y:=[]uint64{a,17},[]uint64{b,23} + got:=Call%[1]s(x[:1],y[:1]); want:=a &^ b + if got!=want || x[0]!=a || x[1]!=17 || y[0]!=b || y[1]!=23 { t.Fatalf("%[1]s(%%x,%%x)=%%x want %%x",a,b,got,want) } + } } +`, name) + fmt.Fprintf(&oracle, "extern void %s(const void*,const void*,void*,const void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + } + tests.WriteString("}\n") + oracle.WriteString(`int main(void) { + const uint64_t values[]={0,1,7,UINT64_C(0x0ff00ff0a55aa55a),UINT64_C(0x33cc55aa5aa5f00f),UINT64_C(1)<<63,UINT64_MAX}; +`) + for _, name := range []string{"andnps", "andnpd", "pandn"} { + fmt.Fprintf(&oracle, ` + for(int i=0;i<7;++i) for(int j=0;j<7;++j) { + uint8_t a[18],b[18],out[18],before[18]; uint64_t length=1,got; + memset(a,0xa7,18); memset(b,0xb9,18); memset(out,0xc3,18); memcpy(before,out,18); + memcpy(a+1,values+i,8); memcpy(b+3,values+j,8); + %s(a+1,b+3,out+5,&length); + uint64_t want=values[i]&~values[j]; memcpy(&got,out+5,8); memcpy(before+5,&want,8); + if(got!=want || memcmp(before,out,18) || length!=1 || a[0]!=0xa7 || a[9]!=0xa7 || b[2]!=0xb9 || b[11]!=0xb9) return 1; + } +`, name) + } + oracle.WriteString("return 0;\n}\n") + dir := t.TempDir() + for name, src := range map[string]string{"go.mod": "module caller\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(src), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("original Go ANDNOT operand order: %v\n%s", err, output) + } + pkg := x86CallerRuntimePackage(t, declarations.String(), assembly.String()) + llvmContext := llvm.NewContext() + defer llvmContext.Dispose() + triple := testTargetTriple(runtime.GOOS, "amd64") + translation, err := translateGoModuleInContext(llvmContext, pkg, []byte(assembly.String()), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "logical_andnot_order", triple, translation.Module.String(), oracle.String(), prefix) +} diff --git a/amd64_go_caller_logical_test.go b/amd64_go_caller_logical_test.go new file mode 100644 index 000000000..5d2e0f76a --- /dev/null +++ b/amd64_go_caller_logical_test.go @@ -0,0 +1,217 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerLogicalOriginalZeroLiterals(t *testing.T) { + for _, literal := range []struct{ name, word string }{ + {"VXORPS", "0xc057f8c5"}, {"VPXOR", "0xc0eff9c5"}, + } { + t.Run(literal.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "LONG $"+literal.word+"\nVMOVSS X0,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerLogicalCompleteFamilyInitializedForms(t *testing.T) { + for _, op := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[op] + for _, row := range x86LogicalAcceptedForms(op, "amd64") { + if strings.Contains(row, "M7") { + continue + } // No private MMX bank. + t.Run(row, func(t *testing.T) { + row = strings.NewReplacer("(AX)", "(DI)", "X15", "X2", "X31", "X2", "Y31", "Y2", "Z31", "Z2").Replace(row) + instruction := strings.Split(row, " ")[1] + operands := strings.Split(instruction, ",") + destination := operands[len(operands)-1] + index := destination[1:] + init := fmt.Sprintf("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z%s\nKMOVQ AX,K1\nKMOVQ AX,K7\n", index) + // Legacy destructive destinations must be initialized just as + // masked merge destinations; register spelling grants nothing. + if spec.encoding <= x86PackedLogicalLegacyInteger && destination == "X2" { + init += "VMOVUPS (SI),Z2\n" + } + body := init + row + "\nVMOVSS X" + index + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestX86GoCallerLogicalRetainsUnknownSourceAndEnvelopeFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "VANDPS X0,X1,X2\nVMOVSS X2,(DX)", + "unknown destructive destination": "VMOVUPS (DI),X0\nANDPS X0,X1\nVMOVSS X1,(DX)", + "masked unknown merge": "KMOVQ AX,K1\nVXORPS X0,X0,K1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VXORPS.Z X0,X0,K1,X2\nVMOVSS X2,(DX)", + "mask from pointer": "KMOVQ DI,K1\nVXORPS.Z X0,X0,K1,X2\nVMOVSS X2,(DX)", + "private stack": "VXORPS 0(SP),X0,X0\nVMOVSS X0,(DX)", + "unknown address": "VXORPS (BX),X0,X0\nVMOVSS X0,(DX)", + "FP pointer bits": "VXORPS a+0(FP),X0,X0\nVMOVSS X0,(DX)", + "length full vector": "VMOVUPS (DI),X0\nVPAND (CX),X0,X0\nVMOVSS X0,(DX)", + "oversized broadcast": "VMOVUPS (DI),X0\nVANDPD.BCST (DX),X0,X0\nVMOVSS X0,(DX)", + "source data write": "VXORPS X0,X0,X0\nVMOVSS X0,(DI)", + "wide result": "VXORPS Y0,Y0,Y0\nVMOVUPS Y0,(DX)", + "CFG undefined join": "CMPQ AX,$0\nJE joined\nVXORPS X0,X0,X0\njoined:\nVORPS X0,X0,X1\nVMOVSS X1,(DX)", + "legacy undefined other input": "XORPS X0,X0\nVORPS Y0,Y1,Y2\nVMOVSS X2,(DX)", + "MMX zero cannot borrow Z0": "PXOR M0,M0\nVMOVSS X0,(DX)", + "MMX memory bank": "PAND (DI),M0\nMOVQ M0,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved logical value/region passed: %v", err) + } + }) + } +} + +func TestX86GoCallerLogicalByteAvailabilityMatchesPhysicalEncoding(t *testing.T) { + for _, op := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[op] + for _, width := range []int{16, 32, 64} { + if spec.encoding <= x86PackedLogicalLegacyInteger && width != 16 || spec.encoding == x86PackedLogicalVEX && width == 64 { + continue + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, maskMode := range []string{"none", "merge", "zero"} { + if maskMode != "none" && spec.broadcastBytes == 0 { + continue + } + for _, zeroIdiom := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%d/%s/zero=%v", op, width, maskMode, zeroIdiom), func(t *testing.T) { + row := string(op) + " " + bank + "0," + bank + "1" + if zeroIdiom { + row = string(op) + " " + bank + "0," + bank + "0" + } + if spec.encoding > x86PackedLogicalLegacyInteger { + if maskMode != "none" { + row += ",K1" + } + row += "," + bank + "2" + if maskMode == "zero" { + row = strings.Replace(row, string(op), string(op)+".Z", 1) + } + } + file, err := Parse(ArchAMD64, "TEXT one(SB),$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedLogicalProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0x0f, "Z1": 0x07, "Z2": 0x05}, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedLogicalStep(&ins, form, &state, nil) { + t.Fatal("register effect rejected") + } + known := uint64(0x07) + if zeroIdiom { + known = 0x0f + if strings.Contains(string(op), "XOR") || strings.Contains(string(op), "ANDN") { + known = x86CallerVectorByteMask(int64(width)) + } + } + dest := "Z2" + if form.legacy { + dest = "Z1" + if zeroIdiom { + dest = "Z0" + } + } else { + if maskMode == "merge" { + known &= 0x05 + } + known |= ^x86CallerVectorByteMask(int64(width)) + } + if state.vectors[Reg(dest)] != known { + t.Fatalf("defined bytes %#x want %#x", state.vectors[Reg(dest)], known) + } + }) + } + } + } + } +} + +func TestX86GoCallerLogicalReadCertificatesRetainExactSource(t *testing.T) { + for _, row := range []string{"VPXOR (DI),X0,X1", "VANDPD.BCST (DI),Z0,K1,Z1"} { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVUPS (SI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\n"+row+"\nVMOVSS X1,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, instruction := range normalized.Funcs[0].Instrs { + form, recognized, err := x86PackedLogicalProperties(instruction, "amd64") + if !recognized || err != nil || form.memory < 0 { + continue + } + index := form.memory + access, exists := instruction.x86CallerMemory[index] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(instruction, index) { + t.Fatal("missing exact read certificate") + } + found = true + for _, change := range []string{"opcode", "suffix", "width", "mask", "address", "source", "direction", "address-size"} { + mutated := instruction + mutated.Args = append([]Operand(nil), instruction.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VANDPS" + case "suffix": + mutated.Op = Op(string(mutated.Op) + ".Z") + case "width": + access.width++ + mutated.x86CallerMemory[index] = access + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K2" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K2"}) + } + case "address": + mutated.Args[index].Mem.Off++ + case "source": + mutated.Args[1].Reg = "X4" + case "direction": + access.write = true + mutated.x86CallerMemory[index] = access + case "address-size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s retained stale read certificate: %s", row, change) + } + } + } + if !found { + t.Fatal("logical source never acquired its independent actual read certificate") + } + } +} diff --git a/amd64_go_caller_low_multiply_test.go b/amd64_go_caller_low_multiply_test.go new file mode 100644 index 000000000..3324f1303 --- /dev/null +++ b/amd64_go_caller_low_multiply_test.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerLowMultiplyUnknownInputsRemainContext(t *testing.T) { + for _, name := range []string{"VPMULLD", "VPMULLQ"} { + for reason, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Z1\n%s Z0,Z1,Z2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Z0\n%s Z0,Z1,Z2\nVMOVSS X2,(DX)", + "partial source": "MOVSS (DI),X0\nVMOVUPS (SI),Z1\n%s Z0,Z1,Z2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\n%s.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nMOVL $1,AX\nKMOVQ AX,K1\n%s Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\n%s.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask lacks region": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n%s.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\n%s (BX),Z0,Z2\nVMOVSS X2,(DX)", + "input write": "VMOVUPS (DI),Z0\n%s Z0,Z0,Z2\nVMOVSS X2,(DI)", + "join missing second": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\n%s Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name+"/"+reason, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", fmt.Sprintf(body, name), 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("low product acquired unproved input/caller memory", err) + } + }) + } + } +} + +// Keep this independent of the external package which exposed the missing +// source-stack effect. The original Go caller owns every input/result region. +func TestX86GoCallerLowMultiplyCompleteForms(t *testing.T) { + for _, name := range []string{"PMULLW", "VPMULLW", "PMULLD", "VPMULLD", "VPMULLQ"} { + banks := []string{"X"} + vector := strings.HasPrefix(name, "V") + if vector { + banks = append(banks, "Y", "Z") + } + for _, bank := range banks { + for _, memory := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && (!memory || !vector || strings.HasSuffix(name, "W")) { + continue + } + for mode := 0; mode < 3; mode++ { + if !vector && mode != 0 { + continue + } + first, opcode := bank+"0", name + if memory { + first = "(DI)" + } + if broadcast { + opcode += ".BCST" + } + args, result := first+","+bank+"1", "X1" + if vector { + if mode != 0 { + args += ",K1" + } + if mode == 2 { + opcode += ".Z" + } + args += "," + bank + "2" + result = "X2" + } + row := opcode + " " + args + t.Run(row, func(t *testing.T) { + body := fmt.Sprintf("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n%s\nVMOVSS %s,(DX)", row, result) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } + } + } + } + } +} diff --git a/amd64_go_caller_maddword.go b/amd64_go_caller_maddword.go new file mode 100644 index 000000000..05cbbb55b --- /dev/null +++ b/amd64_go_caller_maddword.go @@ -0,0 +1,38 @@ +package plan9asm + +func x86CallerWordMultiplyAddStep(ins *Instr, form x86PackedMultiplyAddForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + if form.memory >= 0 { + // E4NF.nb retains the complete source read even for a zero mask. The + // native legacy bridge consumes this same actual pointer and opcode; + // the caller witness grants a read envelope, not alignment/no-fault. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + var known uint64 + groupBytes := form.spec.outputBits / 8 + for byteIndex := 0; byteIndex < form.byteWidth; byteIndex += groupBytes { + pair := (uint64(1)<= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_maddword_runtime_test.go b/amd64_go_caller_maddword_runtime_test.go new file mode 100644 index 000000000..05e7908aa --- /dev/null +++ b/amd64_go_caller_maddword_runtime_test.go @@ -0,0 +1,112 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddActualTypedCaller(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + checks.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for index, bank := range []string{"legacy", "X", "Y", "Z"} { + name := fmt.Sprintf("madd%d", index) + goSource := strings.ReplaceAll(x86CallerFixtureSource, "kernel", name) + goSource = strings.TrimPrefix(goSource, "package caller\nimport \"unsafe\"\n") + goSource = strings.Replace(goSource, "func Call(", "func Call"+name+"(", 1) + goSource = strings.Replace(goSource, ") float32 {", ") uint32 {", 1) + goSource = strings.Replace(goSource, "var res float32", "var res uint32", 1) + declarations.WriteString(goSource) + body := "MOVSS (DI),X0\nMOVSS (SI),X1\nPMADDWL X0,X1\nMOVSS X1,(DX)" + if bank != "legacy" { + body = "VMOVUPS (DI)," + bank + "0\nVMOVUPS (SI)," + bank + "1\nVPMADDWD " + bank + "0," + bank + "1," + bank + "2\nVMOVSS X2,(DX)" + } + machine := strings.Replace(x86CallerFixtureAssembly, "·kernel", "·"+name, 1) + machine = strings.Replace(machine, "MOVL AX,(DX)", body, 1) + assembly.WriteString(machine) + fmt.Fprintf(&checks, "extern void %s(const void*,const void*,void*,const void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + } + goTests := `package caller +import("testing";"unsafe") +func TestOriginalSSEPair(t *testing.T) { + values:=[]uint32{0,1,0x80008000,0x7fff7fff,0xffff0001,0x1234abcd,0xfffe8000} + signed:=func(v uint32) int64 {x:=int64(v&65535);if x>=32768{x-=65536};return x} + for _,a:=range values {for _,b:=range values { + var x,y [16]float32 + *(*uint32)(unsafe.Pointer(&x[0]))=a;*(*uint32)(unsafe.Pointer(&y[0]))=b + got:=Callmadd0(x[:],y[:]);want:=uint32(signed(a)*signed(b)+signed(a>>16)*signed(b>>16)) + if got!=want || *(*uint32)(unsafe.Pointer(&x[0]))!=a || *(*uint32)(unsafe.Pointer(&y[0]))!=b {t.Fatalf("pair %x/%x got%x want%x",a,b,got,want)} + }} +} +` + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module caller\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": goTests} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("original actual typed SSE caller: %v\n%s", err, output) + } + checks.WriteString(` +static int64_t signedWord(uint32_t bits) {uint32_t v=bits&65535;return v<32768 ? (int64_t)v : (int64_t)v-65536;} +int main(void) { + const uint32_t values[]={0,1,0x80008000,0x7fff7fff,0xffff0001,0x1234abcd,0xfffe8000}; +`) + for index := 0; index < 4; index++ { + fmt.Fprintf(&checks, ` + for(int i=0;i<7;++i) for(int j=0;j<7;++j) { + uint8_t a[70],b[70],out[12],before[12];uint64_t length=16;uint32_t got; + memset(a,0xa7,sizeof(a));memset(b,0xb9,sizeof(b));memset(out,0xc3,sizeof(out));memcpy(before,out,sizeof(out)); + memcpy(a+1,values+i,4);memcpy(b+3,values+j,4); + madd%d(a+1,b+3,out+5,&length); + uint32_t want=(uint32_t)(signedWord(values[i])*signedWord(values[j])+signedWord(values[i]>>16)*signedWord(values[j]>>16)); + memcpy(&got,out+5,4);memcpy(before+5,&want,4); + if(got!=want || memcmp(before,out,sizeof(out)) || length!=16 || a[0]!=0xa7 || a[65]!=0xa7 || b[2]!=0xb9 || b[67]!=0xb9) return 1; + } +`, index) + } + checks.WriteString("return 0;\n}\n") + pkg := x86CallerRuntimePackage(t, declarations.String(), assembly.String()) + llvmContext := llvm.NewContext() + defer llvmContext.Dispose() + translation, err := translateGoModuleInContext(llvmContext, pkg, []byte(assembly.String()), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + if translation != nil && !translation.Module.IsNil() { + translation.Module.Dispose() + } + t.Fatal(err) + } + defer translation.Module.Dispose() + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := translation.Module.String() + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "word-madd-actual-caller", triple, text, checks.String(), prefix) + }) + } +} diff --git a/amd64_go_caller_maddword_test.go b/amd64_go_caller_maddword_test.go new file mode 100644 index 000000000..29ae6ed7c --- /dev/null +++ b/amd64_go_caller_maddword_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86GoCallerWordMultiplyAddOriginalRawEffect(t *testing.T) { + // This is the unchanged raw instruction from Weaviate's dot_byte_256. + // The independent fixture supplies actual typed, complete selected callers; + // it does not borrow the external function's unclosed pointer loop. + body := "VMOVDQU (DI),Y2\nVMOVDQU (SI),Y4\nLONG $0xd4f5edc5\nVMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } +} + +func TestX86GoCallerWordMultiplyAddOriginalFollowingAddEffect(t *testing.T) { + // Consecutive unchanged LONGs from dot_byte_avx256.s. Each arithmetic + // family now has its separate complete typed caller effect. + body := "VMOVDQU (DI),Y2\nVMOVDQU (SI),Y4\nVPXOR Y0,Y0,Y0\nLONG $0xd4f5edc5\nLONG $0xc0feedc5\nVMOVSS X0,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } +} + +func TestX86GoCallerWordMultiplyAddCompleteActualSourceConsumers(t *testing.T) { + rows := []string{"PMADDWL X0,X1", "PMADDWL X1,X1", "PMADDWL (DI),X1"} + for _, bank := range []string{"X", "Y", "Z"} { + for _, first := range []string{bank + "0", "(DI)"} { + rows = append(rows, + "VPMADDWD "+first+","+bank+"1,"+bank+"2", + "VPMADDWD "+first+","+bank+"1,K1,"+bank+"2", + "VPMADDWD.Z "+first+","+bank+"1,K1,"+bank+"2") + } + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + result := "X2" + if strings.HasPrefix(row, "PMADDWL") { + result = "X1" + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatalf("actual typed caller %s: %v", triple, err) + } + } + }) + } +} + +func TestX86GoCallerWordMultiplyAddRetainsUnknownSourceFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPMADDWD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPMADDWD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown upper pairs": "MOVSS (DI),X0\nMOVSS (SI),X1\nVPMADDWD Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVPMADDWD.Z Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nVPMADDWD Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPMADDWD.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask full read": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPMADDWD.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPMADDWD (BX),Z0,Z2\nVMOVSS X2,(DX)", + "private stack": "VMOVUPS (DI),Z0\nVPMADDWD (SP),Z0,Z2\nVMOVSS X2,(DX)", + "legacy SP alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPMADDWL (BX),X0\nVMOVSS X0,(DX)", + "legacy BP alias": "VMOVUPS (DI),Z0\nMOVQ BP,BX\nPMADDWL (BX),X0\nVMOVSS X0,(DX)", + "input write": "VMOVUPS (DI),Z0\nVPMADDWD Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX separate bank": "PMADDWL M0,M1\nVMOVSS X1,(DX)", + "undefined branch": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPMADDWD Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved packed multiply-add became caller permission: %v", err) + } + }) + } +} + +func TestX86GoCallerWordMultiplyAddMemoryCertificatesRetainExactFullRead(t *testing.T) { + for _, row := range []string{"PMADDWL (DI),X0", "VPMADDWD.Z (DI),Z0,K1,Z2", "PMADDUBSW (DI),X0", "VPMADDUBSW.Z (DI),Z0,K1,Z2"} { + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS X2,(DX)" + if strings.HasPrefix(row, "PMADD") { + body = strings.Replace(body, "VMOVSS X2", "VMOVSS X0", 1) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, recognized, err := x86PackedMultiplyAddProperties(ins.Op, ins, "amd64") + if !recognized || err != nil || form.memory < 0 { + continue + } + found = true + index := form.memory + access, exists := ins.x86CallerMemory[index] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(ins, index) { + t.Fatal("original legacy or zero-mask E4NF lost exact native full-read witness") + } + for _, change := range []string{"opcode", "width", "address", "destination", "direction", "mask", "address size"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VPMADDUBSW" + if strings.Contains(row, "UBSW") { + mutated.Op = "VPMADDWD" + } + case "width": + access.width-- + mutated.x86CallerMemory[index] = access + case "address": + mutated.Args[index].Mem.Off++ + case "destination": + mutated.Args[form.destination].Reg = "X7" + case "direction": + access.write = true + mutated.x86CallerMemory[index] = access + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K7" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K1"}) + } + case "address size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s stale %s native read permission survived", row, change) + } + } + } + if !found { + t.Fatal("normalized source omitted actual multiply-add") + } + } +} diff --git a/amd64_go_caller_mask_vector_test.go b/amd64_go_caller_mask_vector_test.go new file mode 100644 index 000000000..81b842ff7 --- /dev/null +++ b/amd64_go_caller_mask_vector_test.go @@ -0,0 +1,355 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// The denominator comes from every original Go encoder row, not the caller +// registry or the library which first exposed its missing effect registration. +func x86OfficialMaskVectorRows(t *testing.T) []string { + t.Helper() + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + family := map[string]bool{ + "VPMOVM2B": true, "VPMOVM2W": true, "VPMOVM2D": true, "VPMOVM2Q": true, + "VPMOVB2M": true, "VPMOVW2M": true, "VPMOVD2M": true, "VPMOVQ2M": true, + } + var rows []string + counts := map[string]int{} + for _, rule := range rules { + if !family[rule.Opcode] { + continue + } + toVector := strings.HasPrefix(rule.Opcode, "VPMOVM2") + vectorIndex, maskIndex := 0, 1 + if toVector { + vectorIndex, maskIndex = 1, 0 + } + if len(rule.Classes) != 2 || rule.Classes[maskIndex] != "Yk" || + rule.Prefix != "Pavx" || rule.ZCase != "Zevex_rm_v_r" || + rule.ZOffset != 3 || rule.EncodingOffset != rule.RowIndex*3 || len(rule.OpBytes) != 9 { + t.Fatalf("unmodeled official mask/vector row: %+v", rule) + } + bank := map[string]string{"YxrEvex": "X", "YyrEvex": "Y", "Yzr": "Z"}[rule.Classes[vectorIndex]] + bits := map[string]string{"X": "128", "Y": "256", "Z": "512"}[bank] + w := "evexW0" + if strings.HasSuffix(rule.Opcode, "W") || strings.HasSuffix(rule.Opcode, "Q") || + strings.Contains(rule.Opcode, "W2M") || strings.Contains(rule.Opcode, "Q2M") { + w = "evexW1" + } + terms := strings.Split(rule.OpBytes[rule.EncodingOffset], " | ") + opcode := "0x28" + if strings.Contains(rule.Opcode, "D") || strings.Contains(rule.Opcode, "Q") { + opcode = "0x38" + } + if !toVector { + opcode = map[string]string{"0x28": "0x29", "0x38": "0x39"}[opcode] + } + if bank == "" || !reflect.DeepEqual(terms, []string{"avxEscape", "evex" + bits, "evexF3", "evex0F38", w}) || + rule.OpBytes[rule.EncodingOffset+1] != "0" || !strings.EqualFold(rule.OpBytes[rule.EncodingOffset+2], opcode) { + t.Fatalf("unmodeled encoding generation/attributes: %+v", rule) + } + operands := bank + "1,K1" + if toVector { + operands = "K1," + bank + "1" + } + rows = append(rows, rule.Opcode+" "+operands) + counts[rule.Opcode]++ + } + for opcode := range family { + if counts[opcode] != 3 { + t.Fatalf("%s complete X/Y/Z denominator changed: %d", opcode, counts[opcode]) + } + } + if len(rows) != 24 { + t.Fatalf("complete mask/vector Go-row denominator changed: %d", len(rows)) + } + return rows +} + +func TestX86GoCallerMaskVectorCompleteOfficialEffectRegistration(t *testing.T) { + for _, row := range x86OfficialMaskVectorRows(t) { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !x86CallerVectorMoveKnown(file.Funcs[0].Instrs[1]) { + t.Fatal("Go-accepted complete mask/vector row lacks its caller effect") + } + }) + } +} + +func TestX86GoCallerMaskVectorActualTypedSourceAndObjects(t *testing.T) { + for index, row := range x86OfficialMaskVectorRows(t) { + t.Run(fmt.Sprintf("%02d/%s", index, row), func(t *testing.T) { + body := "KMOVQ AX,K1\nVMOVUPS (DI),Z1\n" + row + "\n" + if strings.HasPrefix(row, "VPMOVM2") { + body += "VMOVSS X1,(DX)" + } else { + body += "KMOVQ K1,AX\nMOVL AX,(DX)" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "mask-vector-text.ll", "mask-vector-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "mask-vector-module.ll", "mask-vector-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("actual selected typed caller/LLVM object failed", err) + } + } + }) + } +} + +func TestX86MaskVectorGrammarKeepsModeAndDispatchBoundaries(t *testing.T) { + for _, row := range x86OfficialMaskVectorRows(t) { + for _, bank := range []string{"X", "Y", "Z"} { + if !strings.Contains(row, bank+"1") { + continue + } + for _, index := range []int{0, 7, 8, 31} { + spelling := strings.Replace(row, bank+"1", fmt.Sprint(bank, index), 1) + t.Run(spelling, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+spelling+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + for _, arch := range []string{"amd64", "386"} { + _, known, err := x86MaskVectorProperties(ins, arch) + if !known { + t.Fatal("recognized family fell through") + } + if arch == "386" && index >= 8 { + if bank == "Z" { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+spelling+"\nRET\n", "", false) + if err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go-invalid mode32 Z bank became a physical binding") + } + } else { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+spelling+"\nRET\n", "", true) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved physical mode32 X/Y binding was accepted", err) + } + } + } else if err != nil { + t.Fatal("valid physical bank rejected", err) + } + } + }) + } + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nVPMOVM2B K1,X1\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + ins.Op = "vpmovm2b" + var output strings.Builder + ctx := amd64Ctx{b: &output, goarch: "amd64"} + if known, _, err := ctx.lowerMaskVectorMove("VPMOVM2B", ins); !known || err != nil { + t.Fatal("equivalent case-normalized opcode lost its existing dispatch", err) + } + output.Reset() + if known, _, err := ctx.lowerMaskVectorMove("VPMOVM2W", ins); !known || err == nil || output.Len() != 0 { + t.Fatal("contradictory dispatch emitted a different opcode", err) + } +} + +func TestX86GoCallerMaskVectorInitializationAndUnaffectedState(t *testing.T) { + for _, row := range x86OfficialMaskVectorRows(t) { + t.Run(row, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86MaskVectorProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatal("shared grammar lost official row", err) + } + initial := x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, gOriginal: true, numericFlags: true, + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}}, known: map[Reg]uint8{AX: 0xff}, + vectors: map[Reg]uint64{"Z1": ^uint64(0), "Z31": 0x55}, + masks: map[Reg]bool{"K1": true, "K0": false, "K7": true}, + } + state, want := x86CloneCallerState(initial), x86CloneCallerState(initial) + if form.spec.maskToVector { + want.vectors["Z1"] = ^uint64(0) + } else { + want.masks["K1"] = true + } + if !x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, want) { + t.Fatalf("mask/vector effect changed unrelated state: got=%+v want=%+v", state, want) + } + for missing := 0; missing < form.byteWidth; missing++ { + state = x86CloneCallerState(initial) + if form.spec.maskToVector { + state.masks["K1"] = false + } else { + state.vectors["Z1"] &^= uint64(1) << uint(missing) + } + before := x86CloneCallerState(state) + required := form.spec.maskToVector || x86MaskVectorSignBytes(row, form.byteWidth)&(uint64(1)<= 0 && missing < len(ins.Args) && ins.Args[missing].Kind == OpReg { + state.vectors[x86CallerVectorRoot(ins.Args[missing].Reg)] &^= 1 + } + if missing == 4 { + state.masks["K1"] = false + } + before, masksBefore := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerVectorMoveStep(&ins, &state, nil) + want := missing == -1 || missing == form.destination && !form.legacy && (form.mask < 0 || form.zeroing) || missing == 4 && form.mask < 0 + if accepted != want || !reflect.DeepEqual(state.masks, masksBefore) || !accepted && !reflect.DeepEqual(state.vectors, before) { + t.Fatalf("%s missing %d: %t want %t, invented numerical/mask evidence", row, missing, accepted, want) + } + } + } +} + +func TestX86GoCallerFloatMulDivCannotInventValuesOrPermissions(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "VMULPS X0,X1,X2\nVMOVSS X2,(DX)", + "division no identity": "VDIVPS X0,X0,X2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nKMOVQ AX,K1\nVMULPS X0,X1,K1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVDIVPS.Z X0,X1,K1,X2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),X0\nKMOVQ DI,K1\nVMULPS.Z X0,X0,K1,X2\nVMOVSS X2,(DX)", + "physical stack": "VMOVUPS (DI),X0\nVMULPS (SP),X0,X2\nVMOVSS X2,(DX)", + "FP pointer": "VMOVUPS (DI),X0\nVDIVPS a+0(FP),X0,X2\nVMOVSS X2,(DX)", + "unknown memory": "VMOVUPS (DI),X0\nVMULPS (R11),X0,X2\nVMOVSS X2,(DX)", + "g read": "VMOVUPS (DI),X0\nVDIVPS (R14),X0,X2\nVMOVSS X2,(DX)", + "wide result": "VMOVUPS (DI),X0\nVMULPS X0,X0,X2\nVMOVUPS X2,(DX)", + "input write": "VMOVUPS (DI),X0\nVDIVPS X0,X0,X2\nVMOVSS X2,(DI)", + "unknown CFG join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),X0\njoined:\nVMULPS X0,X0,X2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved MUL/DIV source passed: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_packed_add.go b/amd64_go_caller_packed_add.go new file mode 100644 index 000000000..c2d68c8cd --- /dev/null +++ b/amd64_go_caller_packed_add.go @@ -0,0 +1,44 @@ +package plan9asm + +func x86CallerPackedWrappingAddStep(ins *Instr, form x86PackedAddForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + return x86CallerPackedLaneArithmeticStep(ins, form.x86PackedBinaryForm, form.spec.laneBits, state, proof) +} + +// Each complete input lane determines one output lane for wrapping or +// saturating integer add/subtract or low-product multiplication. Form sharing +// grants no address permission. +func x86CallerPackedLaneArithmeticStep(ins *Instr, form x86PackedBinaryForm, bits int, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + if form.memory >= 0 { + // Source permission still requires the full original possible access + // envelope; an initialized K mask does not prove a smaller region. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + var known uint64 + step := bits / 8 + for index := 0; index < form.byteWidth; index += step { + lane := x86CallerVectorByteMask(int64(step)) << uint(index) + if first&lane == lane && second&lane == lane { + known |= lane + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.mask >= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_packed_add_runtime_test.go b/amd64_go_caller_packed_add_runtime_test.go new file mode 100644 index 000000000..4f8b41678 --- /dev/null +++ b/amd64_go_caller_packed_add_runtime_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86PackedWrappingAddActualTypedCaller(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var goSource, assembly, declarations, checks strings.Builder + goSource.WriteString("package caller\nimport \"unsafe\"\n") + declarations.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + count := 0 + for opcodeIndex, opcode := range x86WrappingAddOpcodes { + banks := []string{"X"} + if strings.HasPrefix(opcode, "V") { + banks = []string{"X", "Y", "Z"} + } + for _, bank := range banks { + name := fmt.Sprintf("calleradd%d", count) + count++ + declaration := x86CallerFixtureSource[strings.Index(x86CallerFixtureSource, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1) + declaration = strings.Replace(declaration, ") float32 {", ") uint32 {", 1) + declaration = strings.Replace(declaration, "var res float32", "var res uint32", 1) + goSource.WriteString(declaration) + body := "MOVSS (DI),X0\nMOVSS (SI),X1\n" + opcode + " X0,X1\nMOVSS X1,(DX)" + if strings.HasPrefix(opcode, "V") { + body = "VMOVUPS (DI)," + bank + "0\nVMOVUPS (SI)," + bank + "1\n" + opcode + " " + bank + "0," + bank + "1," + bank + "2\nVMOVSS X2,(DX)" + } + machine := strings.Replace(x86CallerFixtureAssembly, "·kernel", "·"+name, 1) + assembly.WriteString(strings.Replace(machine, "MOVL AX,(DX)", body, 1)) + fmt.Fprintf(&declarations, "extern void %s(const void*,const void*,void*,const void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + fmt.Fprintf(&checks, ` + for(int round=0;round<32;++round) { + uint8_t a[70],b[70],out[12],before[12];uint64_t length=16; + memset(a,0xa7,70);memset(b,0xb9,70);memset(out,0xc3,12);memcpy(before,out,12); + for(int i=0;i<64;++i){a[i+1]=(uint8_t)(i*17+round*31);b[i+3]=(uint8_t)(i*23+round*41);} + %[1]s(a+1,b+3,out+5,&length); + for(int i=0;i<4;i+=%[2]d) { + uint64_t sum=0,av=0,bv=0;int bytes=%[2]d; + if(bytes>4)bytes=4; + for(int j=0;j>(8*j)); + } + if(memcmp(out,before,12)||length!=16||a[0]!=0xa7||a[65]!=0xa7||b[2]!=0xb9||b[67]!=0xb9)return 1; + } +`, name, 1<= 0 { + root := x86CallerVectorRoot(instruction.Args[operand].Reg) + vectors[root] &^= uint64(1) << uint(missing) + } + state := &x86SourceStackState{vectors: vectors, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedWrappingAddStep(&instruction, form, state, nil) { + t.Fatal("register grammar rejected") + } + want := ^uint64(0) + if missing >= 0 { + want &^= x86CallerVectorByteMask(int64(step)) << uint(missing/step*step) + } + got := state.vectors[x86CallerVectorRoot(instruction.Args[form.destination].Reg)] + // Legacy preserves its old upper view, while each vector form + // defines it to zero. Both start fully known in this axis. + if got != want || got&^width != want&^width { + t.Fatalf("%s/%s missing%d input%d known=%x want=%x", opcode, mode, missing, operand, got, want) + } + } + } + } + } +} + +func TestX86GoCallerPackedAddUnknownBoundariesRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPADDD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPADDQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown high byte": "MOVSS (DI),X0\nMOVSS (SI),X1\nVPADDQ Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVPADDB.Z Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nVPADDW Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPADDQ.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask no permission": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPADDD.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPADDB (BX),Z0,Z2\nVMOVSS X2,(DX)", + "private stack": "VMOVUPS (DI),Z0\nVPADDW (SP),Z0,Z2\nVMOVSS X2,(DX)", + "legacy SP alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPADDL (BX),X0\nVMOVSS X0,(DX)", + "input store": "VMOVUPS (DI),Z0\nVPADDQ Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX": "PADDB M0,M1\nVMOVSS X1,(DX)", + "saturating separate": "VMOVUPS (DI),Z0\nVPADDSW Z0,Z0,Z2\nVMOVSS X2,(DX)", + "unknown joined bank": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPADDW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "different pointer offset": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nADDQ $4,DI\njoined:\nVPADDD (DI),Z0,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s escaped actual source boundary: %v", name, err) + } + }) + } + // Independent caller effect shape does not allow another opcode or a + // narrowed masked source to reuse a private instruction certificate. + for _, row := range []string{"PADDL (DI),X0", "VPADDQ.BCST.Z (DI),Z0,K1,Z2"} { + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS X0,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86PackedWrappingAddProperties(ins, "amd64") + if !known || err != nil { + continue + } + access := ins.x86CallerMemory[form.memory] + if access.width != int64(form.memoryBytes) || access.write || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatalf("missing full possible source extent for %s", row) + } + for _, mutation := range []string{"opcode", "width", "direction", "address"} { + changedAccess := access + changed := ins + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch mutation { + case "opcode": + changed.Op = "VPSUBQ" + case "width": + changedAccess.width-- + changed.x86CallerMemory[form.memory] = changedAccess + case "direction": + changedAccess.write = true + changed.x86CallerMemory[form.memory] = changedAccess + case "address": + changed.Args[form.memory].Mem.Off++ + } + if x86CallerMemoryCertified(changed, form.memory) { + t.Fatal(fmt.Sprintf("%s stale %s witness", row, mutation)) + } + } + } + } +} + +func TestX86PackedAddLegacyVirtualMemoryCannotForgeNativeAlignment(t *testing.T) { + for _, row := range []string{"PADDB source+0(FP),X0", "PADDW (SP),X0", "PADDL (BP),X0", "PADDQ source+0(FP),X0"} { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-64\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + sig := x86WordMaddFPType(t) + _, err = Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": sig}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("virtual storage acquired physical aligned m128 permission: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_packed_moves_test.go b/amd64_go_caller_packed_moves_test.go new file mode 100644 index 000000000..309f424c7 --- /dev/null +++ b/amd64_go_caller_packed_moves_test.go @@ -0,0 +1,193 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "sort" + "strings" + "testing" +) + +func TestX86GoCallerPackedMovesPreserveNumericalInputBytes(t *testing.T) { + for _, form := range []struct{ name, body string }{ + {"VEX input", "VMOVDQU (DI),Y0\nVMOVSS X0,(DX)"}, + {"EVEX input", "VMOVDQU8 (DI),Z31\nVMOVSS X31,(DX)"}, + {"EVEX zero mask", "KMOVQ AX,K1\nVMOVUPS.Z (DI),K1,Z0\nVMOVSS X0,(DX)"}, + } { + t.Run(form.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", form.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerPackedMovesCompleteFamilyRegisterCopies(t *testing.T) { + var opcodes []string + for opcode := range amd64PackedVectorMoveSpecs { + opcodes = append(opcodes, string(opcode)) + } + sort.Strings(opcodes) + for _, opcode := range opcodes { + spec := amd64PackedVectorMoveSpecs[Op(opcode)] + for _, bank := range []string{"X", "Y", "Z"} { + if bank == "Z" && spec.encoding == amd64PackedMoveVEX { + continue + } + for _, mode := range []string{"unmasked", "merge", "zero"} { + if mode != "unmasked" && spec.encoding == amd64PackedMoveVEX { + continue + } + body := fmt.Sprintf("VMOVDQU8 (DI),%s0\n", bank) + name, mask := opcode, "" + if mode != "unmasked" { + body += fmt.Sprintf("VMOVDQU8 (SI),%s1\nKMOVQ AX,K1\n", bank) + mask = "K1," + if mode == "zero" { + name += ".Z" + } + } + body += fmt.Sprintf("%s %s0,%s%s1\nVMOVSS X1,(DX)", name, bank, mask, bank) + t.Run(opcode+"/"+bank+"/"+mode, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } + } +} + +func TestX86GoCallerPackedMovesRetainUnknownAndBoundedMemoryFailures(t *testing.T) { + for _, example := range []struct{ name, body string }{ + {"unknown source", "VMOVUPS X0,X1\nVMOVSS X1,(DX)"}, + {"unknown mask", "VMOVUPS.Z (DI),K1,Z0\nVMOVSS X0,(DX)"}, + {"pointer mask", "KMOVQ DI,K1\nVMOVUPS.Z (DI),K1,Z0\nVMOVSS X0,(DX)"}, + {"unknown merge destination", "KMOVQ AX,K1\nVMOVUPS (DI),K1,Z0\nVMOVSS X0,(DX)"}, + {"unknown source in zero mask", "KMOVQ AX,K1\nVMOVUPS.Z Z0,K1,Z1\nVMOVSS X1,(DX)"}, + {"unknown base", "VMOVUPS (BX),Y0\nVMOVSS X0,(DX)"}, + {"pointer argument bits", "VMOVUPS a+0(FP),X0\nVMOVSS X0,(DX)"}, + {"saved BP bytes", "VMOVUPS 0(SP),X0\nVMOVSS X0,(DX)"}, + {"unknown input alignment", "VMOVAPS (DI),X0\nVMOVSS X0,(DX)"}, + {"unknown integer alignment", "VMOVDQA32 (DI),Y0\nVMOVSS X0,(DX)"}, + {"input write", "VMOVUPS (DI),Y0\nVMOVUPS Y0,(SI)\nVMOVSS X0,(DX)"}, + {"length read too wide", "VMOVUPS (CX),X0\nVMOVSS X0,(DX)"}, + {"length write", "VMOVUPS (DI),X0\nVMOVUPS X0,(CX)\nVMOVSS X0,(DX)"}, + {"wide fresh result", "VMOVUPS (DI),X0\nVMOVUPS X0,(DX)"}, + {"masked wide fresh result", "KMOVQ AX,K1\nVMOVUPS (DI),X0\nVMOVUPS X0,K1,(DX)"}, + {"invalid zero memory store", "KMOVQ AX,K1\nVMOVUPS (DI),X0\nVMOVUPS.Z X0,K1,(DX)"}, + {"indexed fresh result", "VMOVUPS (DI),X0\nVMOVUPS X0,(DX)(AX*1)"}, + {"SP escape", "MOVQ SP,BX\nVMOVUPS (BX),X0\nVMOVSS X0,(DX)"}, + {"g carrier", "VMOVUPS (R14),X0\nVMOVSS X0,(DX)"}, + {"different initialized bytes at join", "CMPQ AX,$0\nJE join\nVMOVUPS (DI),Z0\njoin:\nVMOVSS X0,(DX)"}, + } { + t.Run(example.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", example.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved packed move acquired continuation: %v", err) + } + }) + } +} + +func TestX86PackedMoveInvalidStoreCannotAcquireCallerMemoryCertificate(t *testing.T) { + for _, opcode := range x86PackedZeroStoreOpcodes { + for _, bank := range []string{"X", "Y", "Z"} { + t.Run(opcode+"/"+bank, func(t *testing.T) { + instruction := fmt.Sprintf("%s.Z %s0,K1,(DX)", opcode, bank) + parsed, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := parsed.Funcs[0].Instrs[1] + state := x86SourceStackState{ + vectors: map[Reg]uint64{"Z0": ^uint64(0)}, masks: map[Reg]bool{"K1": true}, + } + if strings.Contains(opcode, "COMPRESS") { + form, known, err := x86PackedCompactProperties(ins.Op, ins, "amd64") + if !known || err != nil || !form.undefinedOpcode() { + t.Fatal("compact syntax lost its independent #UD effect", err) + } + if _, known, err := x86TypedCompactExpandByteEffect(ins.Op, ins, "amd64"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("#UD compact form acquired ordinary numerical effects", err) + } + } else { + form, known, err := amd64PackedVectorMoveProperties(ins, "amd64") + if !known || err != nil || !form.undefinedOpcode() { + t.Fatal("move syntax lost its independent #UD effect", err) + } + if x86CallerPackedVectorMoveStep(&ins, form, &state, nil) { + t.Fatal("#UD instruction acquired caller continuation") + } + } + if x86CallerVectorMoveKnown(ins) { + t.Fatal("#UD instruction acquired an ordinary move effect") + } + if _, _, valid := x86CallerMoveMemoryShape(ins, 2); valid { + t.Fatal("#UD instruction acquired a memory certificate shape") + } + if x86CallerVectorMoveStep(&ins, &state, nil) || x86CallerMemoryCertified(ins, 2) { + t.Fatal("#UD instruction acquired a caller memory certificate") + } + }) + } + } +} + +func TestX86GoCallerPackedMoveByteKnowledgeIncludesMaskAndZeroUpperViews(t *testing.T) { + for _, encoding := range []struct { + op string + width int + }{ + {"VMOVUPS", 16}, {"VMOVUPD", 32}, {"VMOVDQU8", 64}, + {"VMOVDQU16", 16}, {"VMOVDQU32", 32}, {"VMOVDQU64", 64}, + } { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[encoding.width] + for _, sourceKnown := range []uint64{0, 0xf, 0xff, ^uint64(0)} { + for _, oldKnown := range []uint64{0, 0xff, ^uint64(0)} { + for _, mode := range []string{"", "merge", "zero"} { + name, mask := encoding.op, "" + if mode != "" { + mask = "K1," + if mode == "zero" { + name += ".Z" + } + } + instruction := fmt.Sprintf("%s %s0,%s%s1", name, bank, mask, bank) + parsed, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := parsed.Funcs[0].Instrs[1] + form, recognized, err := amd64PackedVectorMoveProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatalf("known transfer form: %v", err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": sourceKnown, "Z1": oldKnown}, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedVectorMoveStep(&ins, form, &state, nil) { + t.Fatal("register-only effect unexpectedly requires a caller region") + } + var expected uint64 + for index := 0; index < 64; index++ { + bit := uint64(1) << uint(index) + if index >= encoding.width || sourceKnown&bit != 0 && (mode != "merge" || oldKnown&bit != 0) { + expected |= bit + } + } + if state.vectors["Z1"] != expected || state.vectors["Z0"] != sourceKnown || !reflect.DeepEqual(state.masks, map[Reg]bool{"K1": true}) { + t.Fatalf("%s changed initialized-byte/mask state: %+v; want %x", instruction, state, expected) + } + } + } + } + } +} diff --git a/amd64_go_caller_packed_narrow.go b/amd64_go_caller_packed_narrow.go new file mode 100644 index 000000000..c4c3757fa --- /dev/null +++ b/amd64_go_caller_packed_narrow.go @@ -0,0 +1,27 @@ +package plan9asm + +func x86CallerPackedNarrowStep(ins *Instr, form x86PackedNarrowForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + if form.memory >= 0 { + // E4NF/.nb requires the complete tuple, even when every output mask + // bit is zero. The form supplies a width, never a pointer permission. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + known := x86PackedNarrowDefinedBytes(form, first, second, old) + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_packed_narrow_alias_test.go b/amd64_go_caller_packed_narrow_alias_test.go new file mode 100644 index 000000000..180e03057 --- /dev/null +++ b/amd64_go_caller_packed_narrow_alias_test.go @@ -0,0 +1,84 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerPackedNarrowDestinationAliasesKeepOriginalInputs(t *testing.T) { + // Whole-source initialization isolates concatenation and alias ordering + // from the finer per-input-byte dependency test. Expected half masks are + // independent constants, repeated once for each 128-bit chunk. + for _, name := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for bankIndex, bank := range []string{"X", "Y", "Z"} { + for _, mask := range []string{"", "merge", "zero"} { + for destination := 0; destination < 3; destination++ { + opcode, operands := name, bank+"0,"+bank+"1" + if mask != "" { + operands += ",K1" + } + if mask == "zero" { + opcode += ".Z" + } + operands += fmt.Sprintf(",%s%d", bank, destination) + row := opcode + " " + operands + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + width := x86CallerVectorByteMask(int64(16 << bankIndex)) + for _, sample := range []struct{ first, second, halves uint64 }{ + {0, ^uint64(0), 0x00ff00ff00ff00ff}, + {^uint64(0), 0, 0xff00ff00ff00ff00}, + {^uint64(0), ^uint64(0), ^uint64(0)}, + {0, 0, 0}, + } { + ins := file.Funcs[0].Instrs[1] + state := &x86SourceStackState{ + vectors: map[Reg]uint64{"Z0": sample.first, "Z1": sample.second, "Z2": 0xaaaaaaaaaaaaaaaa}, + masks: map[Reg]bool{"K1": true}, + } + root := Reg(fmt.Sprintf("Z%d", destination)) + old := state.vectors[root] + want := sample.halves & width + if mask == "merge" { + want &= old + } + want |= ^width + if !x86CallerVectorMoveStep(&ins, state, nil) || state.vectors[root] != want { + t.Fatalf("source/destination alias grant %s first=%x second=%x got=%x want=%x", + row, sample.first, sample.second, state.vectors[root], want) + } + } + }) + } + } + } + } +} + +func TestX86GoCallerPackedNarrowLegacyUpperBytesDoNotBecomeInitialized(t *testing.T) { + for _, name := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB"} { + row := name + " X0,X1" + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + state := &x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": 0xffff}} + ins := file.Funcs[0].Instrs[1] + if !x86CallerVectorMoveStep(&ins, state, nil) || state.vectors["Z1"] != 0xffff { + t.Fatal("legacy PACK borrowed VEX upper-zero initialization", row, state.vectors["Z1"]) + } + // A following YMM use cannot inherit initialized upper bytes from + // the instruction's legal XMM-only grammar. + next := Instr{Op: "VPACKSSDW", Args: []Operand{ + {Kind: OpReg, Reg: "Y1"}, {Kind: OpReg, Reg: "Y1"}, {Kind: OpReg, Reg: "Y2"}, + }} + if !x86CallerVectorMoveStep(&next, state, nil) || state.vectors["Z2"]&0xffff0000 != 0 { + t.Fatal(strings.Join([]string{"legacy PACK initialized an unrelated YMM upper chunk", row}, ": ")) + } + } +} diff --git a/amd64_go_caller_packed_narrow_permissions_test.go b/amd64_go_caller_packed_narrow_permissions_test.go new file mode 100644 index 000000000..b665df192 --- /dev/null +++ b/amd64_go_caller_packed_narrow_permissions_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerPackedNarrowUnknownValuesAndRegionsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown second": "VMOVUPS (DI),Z0\nVPACKSSDW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "unknown first high half": "VMOVUPS (SI),Z1\nVPACKSSDW X0,X1,X2\nVPSRLDQ $8,X2,X3\nVMOVSS X3,(DX)", + "partial second": "MOVSS (DI),X3\nMOVSS X3,X1\nVMOVUPS (SI),Z0\nVPACKSSDW X0,X1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVPACKUSWB.Z Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nMOVL $1,AX\nKMOVQ AX,K1\nVPACKSSWB Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPACKUSDW.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask full tuple": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPACKSSDW.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask broadcast span": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPACKUSDW.BCST.Z 1(DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPACKUSWB (BX),Z0,Z2\nVMOVSS X2,(DX)", + "stack alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPACKSSWB (BX),X0\nVMOVSS X0,(DX)", + "BP alias": "VMOVUPS (DI),Z0\nMOVQ BP,BX\nPACKUSWB (BX),X0\nVMOVSS X0,(DX)", + "input write": "VMOVUPS (DI),Z0\nVPACKSSDW Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX separate bank": "PACKSSLW M0,M1\nVMOVSS X1,(DX)", + "join missing second": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPACKSSDW Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved PACK acquired caller permission", err) + } + }) + } + // The first input contributes only the high half of each 128-bit chunk. + // Unknown unused bytes must not forbid this initialized low-half write. + body := "VMOVUPS (SI),Z1\nVPACKSSDW Z0,Z1,Z2\nVMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("independent initialized low half was lost", err) + } +} + +func TestX86GoCallerPackedNarrowFullReadCertificateIdentity(t *testing.T) { + for _, name := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB", + "VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + banks := []string{"X"} + if strings.HasPrefix(name, "V") { + banks = append(banks, "Y", "Z") + } + for index, bank := range banks { + broadcasts := []bool{false} + if name == "VPACKSSDW" || name == "VPACKUSDW" { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + row, result, width := name+" (DI),X0", "X0", int64(16<= 0 { + mutated.Args[form.mask].Reg = "K7" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K7"}) + } + case "address size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, form.memory) { + t.Fatal(fmt.Sprintf("stale complete instruction certificate survived %s", change)) + } + } + } + if !found { + t.Fatal("actual PACK memory proof absent") + } + }) + } + } + } +} diff --git a/amd64_go_caller_packed_narrow_test.go b/amd64_go_caller_packed_narrow_test.go new file mode 100644 index 000000000..e739ee395 --- /dev/null +++ b/amd64_go_caller_packed_narrow_test.go @@ -0,0 +1,156 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86NarrowCallerRows(t *testing.T) []string { + t.Helper() + rows := x86OfficialPackedNarrowRows(t, "amd64") + for _, name := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB"} { + rows = append(rows, name+" X0,X1", name+" (DI),X1") + } + if len(rows) != 98 { + t.Fatalf("independent current Go narrow denominator changed: %d", len(rows)) + } + return rows +} + +func TestX86GoCallerPackedNarrowRecognizesEveryOfficialVectorForm(t *testing.T) { + for _, row := range x86NarrowCallerRows(t) { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !x86CallerVectorMoveKnown(file.Funcs[0].Instrs[1]) { + t.Fatal("accepted complete narrow grammar has no caller consumer") + } + }) + } +} + +func TestX86GoCallerPackedNarrowActualTypedCallerAndObjects(t *testing.T) { + for index, original := range x86NarrowCallerRows(t) { + t.Run(fmt.Sprintf("%d/%s", index, original), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+original+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + var args []string + for position, arg := range ins.Args { + if arg.Kind == OpReg && amd64VectorByteWidth(arg.Reg) != 0 { + // Rebind only the physical index; bank, memory, mask and + // opcode attributes remain the independent Go row shape. + arg.Reg = Reg(fmt.Sprintf("%c%d", arg.Reg[0], position)) + } + if arg.Kind == OpMem { + arg.Mem.Base, arg.Mem.Index, arg.Mem.Sym = DI, "", "" + arg.Mem.Off, arg.Mem.Scale = 0, 0 + } + args = append(args, arg.String()) + } + row := string(ins.Op) + " " + strings.Join(args, ",") + result := "X" + fmt.Sprint(len(ins.Args)-1) + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nVMOVUPS (SI),Z3\n" + + "MOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("actual typed caller/LLVM object consumer failed", err) + } + } + }) + } +} + +func TestX86GoCallerPackedNarrowIndependentPerOutputByteDependencies(t *testing.T) { + for _, name := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB", + "VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + legacy := !strings.HasPrefix(name, "V") + banks := []string{"X", "Y", "Z"} + if legacy { + banks = banks[:1] + } + inputBytes := 2 + if strings.HasSuffix(name, "DW") || name == "PACKSSLW" { + inputBytes = 4 + } + for bankIndex, bank := range banks { + for _, mask := range []string{"", "merge", "zero"} { + if legacy && mask != "" { + continue + } + opcode, args := name, bank+"0,"+bank+"1" + if !legacy { + if mask != "" { + args += ",K1" + } + if mask == "zero" { + opcode += ".Z" + } + args += "," + bank + "2" + } + row := opcode + " " + args + t.Run(row, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + width := 16 << bankIndex + widthMask := x86CallerVectorByteMask(int64(width)) + for _, first := range []uint64{0, 1, 3, 15, 0xffff, 0xff00ff00ff00ff00, ^uint64(0)} { + for _, second := range []uint64{0, 7, 0xffff, 0x5555555555555555, ^uint64(0)} { + old := uint64(0xaaaaaaaaaaaaaaaa) + destination := Reg("Z2") + if legacy { + destination, old = "Z1", second + } + state := &x86SourceStackState{ + vectors: map[Reg]uint64{"Z0": first, "Z1": second, "Z2": old}, + masks: map[Reg]bool{"K1": true}, + } + // Each 128-bit chunk contains second's narrowed + // low half, then first's narrowed high half. One + // missing input byte revokes its whole output lane. + var want uint64 + for output := 0; output < width; output += inputBytes / 2 { + chunk, local := output/16*16, output%16 + input, source := chunk+(local%8)/(inputBytes/2)*inputBytes, second + if local >= 8 { + source = first + } + needed := uint64(1< +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +` + declarations.String() + ` +static int check(void (*fn)(void*,void*,void*,void*), unsigned lane, bool unmasked, bool zero) { + for (uint64_t length = 64; length < 68; length++) { + uint8_t input[512], old[512], output[48], before[48]; + for (unsigned i = 0; i < 512; i++) { + input[i] = (uint8_t)(i*7 + length*13); + old[i] = (uint8_t)(i*11 + length*17); + } + memset(output, 0xa5, sizeof(output)); + memcpy(before, output, sizeof(output)); + const uint64_t original_length = length; + fn(input, old, output+16, &length); + if (length != original_length) return 1; + for (unsigned i = 0; i < sizeof(output); i++) { + uint8_t want = before[i]; + if (i >= 16 && i < 20) { + const unsigned byte = i-16; + const bool active = unmasked || ((length >> (byte/lane)) & 1); + want = active ? input[byte] : zero ? 0 : old[byte]; + } + if (output[i] != want) return 2; + } + for (unsigned i = 0; i < 512; i++) { + if (input[i] != (uint8_t)(i*7 + length*13) || + old[i] != (uint8_t)(i*11 + length*17)) return 3; + } + } + return 0; +} + +int main(void) { +` + checks.String() + " return 0;\n}\n" + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-caller-numerical", triple, translation.Module.String(), oracle, runner) +} diff --git a/amd64_go_caller_packed_shift.go b/amd64_go_caller_packed_shift.go new file mode 100644 index 000000000..86bb33928 --- /dev/null +++ b/amd64_go_caller_packed_shift.go @@ -0,0 +1,48 @@ +package plan9asm + +// Availability is separate from encoding. No count, mask, or zero-valued +// shadow register is evidence that an incoming vector contains numeric data. +func x86CallerPackedShiftStep(ins *Instr, form x86PackedShiftForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + if form.memory >= 0 { + // Legacy XMM uniform count loads still need an independently bound + // native-address/fault contract. A slice identity alone does not + // establish it; the original opcode, not MOVDQA, governs faults. + if form.legacy || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + data := width + if form.data != form.memory { + data = state.vectors[x86CallerVectorRoot(ins.Args[form.data].Reg)] & width + } + if data != width { + return false + } + counts := ^uint64(0) + if ins.Args[form.count].Kind != OpImm && form.count != form.memory { + countWidth := int64(form.byteWidth) + if form.countMode == x86PackedShiftUniform { + countWidth = 8 // The native register's upper count qword is ignored. + } + needed := x86CallerVectorByteMask(countWidth) + counts = state.vectors[x86CallerVectorRoot(ins.Args[form.count].Reg)] + if counts&needed != needed { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + // The caller-entry contract deliberately retains its complete input + // requirements above. Sharing numerical dependencies does not relax them. + known := x86PackedShiftDefinedBytes(form, data, counts, old) + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_packed_shift_test.go b/amd64_go_caller_packed_shift_test.go new file mode 100644 index 000000000..73284a23e --- /dev/null +++ b/amd64_go_caller_packed_shift_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86GoCallerPackedShiftWeaviateOriginalBytes(t *testing.T) { + // Original dot_byte_avx256.s uses c5 ed 71 d1 08 at its first + // VPSRLW. Keep the complete encoded instruction, not a named surrogate. + body := "VMOVDQU (DI),Y1\nLONG $0xd171edc5\nBYTE $0x08\nVMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + for _, bound := range []bool{true, false} { + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if !bound { + pkg.SourceSelection = nil + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: triple, ResolveSym: resolve, Sigs: sigs} + ir, err := Translate(file, options) + if bound { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "original.ll", "original.o", ir) + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("uncontracted Translate: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if bound && err != nil || !bound && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bound=%t module: %v", bound, err) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); bound && err != nil || !bound && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bound=%t GoModule: %v", bound, err) + } + } + } +} + +func TestX86GoCallerPackedShiftWholeFamilyEffects(t *testing.T) { + for _, op := range x86PackedShiftNames() { + for _, row := range x86PackedShiftRows(op, "amd64") { + if strings.Contains(row, "(AX)") || strings.Contains(row, "M0") || strings.Contains(row, "M1") { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !recognized || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatalf("missing effect %s: %v", row, err) + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + for _, missing := range []string{"none", "data", "count", "mask"} { + state := x86SourceStackState{ + vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": 0}, + masks: map[Reg]bool{"K1": true, "K7": true}, + } + applicable := true + switch missing { + case "data": + state.vectors[x86CallerVectorRoot(ins.Args[form.data].Reg)] &^= uint64(1) << uint(form.byteWidth-1) + case "count": + applicable = ins.Args[form.count].Kind != OpImm + if applicable { + state.vectors[x86CallerVectorRoot(ins.Args[form.count].Reg)] &^= 1 + } + case "mask": + applicable = form.mask >= 0 + if applicable { + delete(state.masks, ins.Args[form.mask].Reg) + } + } + if !applicable { + continue + } + before, _ := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerPackedShiftStep(&ins, form, &state, nil) + if accepted != (missing == "none") || !accepted && !reflect.DeepEqual(before, state.vectors) { + t.Fatalf("%s missing %s accepted=%t", row, missing, accepted) + } + if accepted { + want := ^uint64(0) + if form.mask >= 0 && !form.zeroing { + want = ^width + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatalf("%s known=%x want %x", row, got, want) + } + } + } + } + } +} + +func TestX86GoCallerPackedShiftMemoryShapesAndTargets(t *testing.T) { + for _, op := range x86PackedShiftNames() { + for _, row := range x86PackedShiftRows(op, "amd64") { + if strings.Contains(row, "M0") || strings.Contains(row, "M1") { + continue + } + row = strings.ReplaceAll(row, "(AX)", "(DI)") + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if err != nil { + t.Fatal(err) + } + if form.memory >= 0 { + width, write, valid := x86CallerMoveMemoryShape(ins, form.memory) + if !valid || write || int(width) != form.memoryBytes { + t.Fatalf("incorrect read envelope %s: %d/%t/%t", row, width, write, valid) + } + } + body := fmt.Sprintf("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVQ $-1,AX\nKMOVQ AX,K1\nKMOVQ AX,K7\n%s\nVMOVSS X2,(DX)", row) + if form.legacy { + body = strings.Replace(body, "VMOVSS X2,(DX)", "VMOVSS X1,(DX)", 1) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + // Compile one bounded corpus per target below; individual forms here + // also exercise the actual caller transport, not a public Ptr noalias. + err = x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if form.legacy && form.memory >= 0 { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved legacy native-memory contract passed %s: %v", row, err) + } + } else if err != nil { + t.Fatalf("%s: %v", row, err) + } + } + } +} + +func TestX86GoCallerPackedShiftUnprovedInputsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown data": "VPSRLW $8,Y1,Y2\nVMOVSS X2,(DX)", + "zero count unknown data": "VPSRLW $0,Y1,Y2\nVMOVSS X2,(DX)", + "saturated unknown data": "VPSRLW $255,Y1,Y2\nVMOVSS X2,(DX)", + "unknown count": "VMOVUPS (DI),Y1\nVPSRLW X0,Y1,Y2\nVMOVSS X2,(DX)", + "partial count": "MOVL $1,AX\nMOVD AX,X0\nVMOVUPS (DI),Y1\nVPSRLVW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),Y1\nVPSRLW.Z $8,Y1,K1,Y2\nVMOVSS X2,(DX)", + "merge unknown destination": "VMOVUPS (DI),Y1\nMOVL $1,AX\nKMOVW AX,K1\nVPSRLW $8,Y1,K1,Y2\nVMOVSS X2,(DX)", + "physical SP": "VMOVUPS (DI),Y1\nVPSRLW 0(SP),Y1,Y2\nVMOVSS X2,(DX)", + "FP pointer bytes": "VMOVUPS (DI),Y1\nVPSRLW a+0(FP),Y1,Y2\nVMOVSS X2,(DX)", + "g": "VMOVUPS (DI),Y1\nVPSRLW (R14),Y1,Y2\nVMOVSS X2,(DX)", + "unknown base": "VMOVUPS (DI),Y1\nVPSRLW (BX),Y1,Y2\nVMOVSS X2,(DX)", + "length m128": "VMOVUPS (DI),Y1\nVPSRLW (CX),Y1,Y2\nVMOVSS X2,(DX)", + "partial output": "VMOVUPS (DI),Y1\nVPSRLW $8,Y1,Y2\nVMOVUPS Y2,(DX)", + "input write": "VMOVUPS (DI),Y1\nVPSRLW $8,Y1,Y2\nVMOVSS X2,(DI)", + "branch join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),Y1\njoined:\nVPSRLW $8,Y1,Y2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source accepted: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_packed_subtract_test.go b/amd64_go_caller_packed_subtract_test.go new file mode 100644 index 000000000..f8315051f --- /dev/null +++ b/amd64_go_caller_packed_subtract_test.go @@ -0,0 +1,255 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// These rows come from cmd/internal/obj/x86's yxm and _yvandnpd tables. +// Saturation changes arithmetic, not the operand availability obligation. +func x86PackedSubtractCallerRows() []string { + var rows []string + for _, opcode := range []string{ + "PSUBB", "PSUBW", "PSUBL", "PSUBQ", + "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW", + } { + rows = append(rows, opcode+" X0,X15", opcode+" (DI),X15") + } + for _, opcode := range []string{ + "VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", + "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW", + } { + for _, bank := range []string{"X", "Y", "Z"} { + rows = append(rows, + fmt.Sprintf("%s %s0,%s1,%s2", opcode, bank, bank, bank), + fmt.Sprintf("%s (DI),%s1,%s2", opcode, bank, bank), + fmt.Sprintf("%s %s20,%s21,%s22", opcode, bank, bank, bank), + ) + for _, suffix := range []string{"", ".Z"} { + rows = append(rows, + fmt.Sprintf("%s%s %s20,%s21,K1,%s22", opcode, suffix, bank, bank, bank), + fmt.Sprintf("%s%s (DI),%s21,K1,%s22", opcode, suffix, bank, bank), + ) + } + if opcode == "VPSUBD" || opcode == "VPSUBQ" { + rows = append(rows, + fmt.Sprintf("%s.BCST (DI),%s21,%s22", opcode, bank, bank), + fmt.Sprintf("%s.BCST (DI),%s21,K1,%s22", opcode, bank, bank), + fmt.Sprintf("%s.BCST.Z (DI),%s21,K1,%s22", opcode, bank, bank), + ) + } + } + } + return rows +} + +func TestX86GoCallerPackedSubtractCompleteOriginalTypedConsumers(t *testing.T) { + rows := x86PackedSubtractCallerRows() + if len(rows) != 202 { + t.Fatalf("complete Go subtract surface changed: %d rows", len(rows)) + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + operands := strings.Split(strings.Fields(row)[1], ",") + destination := operands[len(operands)-1] + result := "X" + destination[1:] + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\n" + + "VMOVUPS (SI),Z15\nVMOVUPS (DI),Z20\nVMOVUPS (SI),Z21\n" + + "VMOVUPS (DI),Z22\nMOVL $1,AX\nKMOVQ AX,K1\n" + + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } +} + +func TestX86PackedSubtractMaskedLoadsRetainActiveLaneEffects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, opcode := range []string{ + "VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", + "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW", + } { + for _, bank := range []string{"X", "Y", "Z"} { + for _, suffix := range []string{".Z", ".BCST.Z"} { + if suffix == ".BCST.Z" && opcode != "VPSUBD" && opcode != "VPSUBQ" { + continue + } + row := fmt.Sprintf("%s%s (AX),%s1,K1,%s2", opcode, suffix, bank, bank) + t.Run(row, func(t *testing.T) { + assembly := "TEXT maskedsubtract(SB),4,$0-0\n" + row + "\nRET\n" + ir := x86UnboundVoidScalarFormatObject(t, assembly, "maskedsubtract", "amd64", "x86_64-unknown-linux-gnu") + if !strings.Contains(ir, "load volatile") || !strings.Contains(ir, "br i1") { + t.Fatal("masked source needs conditional volatile active-lane reads, not an unconditional vector load") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "masked-subtract.ll", "masked-subtract.o", ir) + }) + } + } + } +} + +func TestX86GoCallerPackedSubtractEveryLaneByteKnowledge(t *testing.T) { + for _, row := range x86PackedSubtractCallerRows() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + form, _, err := x86PackedSubtractProperties(instruction, "amd64") + if err != nil { + t.Fatal(err) + } + if form.memory >= 0 { + continue // Actual source extents are covered by the typed consumers. + } + for operand := 0; operand < 2; operand++ { + for missing := -1; missing < form.byteWidth; missing++ { + vectors := make(map[Reg]uint64) + for _, arg := range instruction.Args { + if amd64VectorByteWidth(arg.Reg) != 0 { + vectors[x86CallerVectorRoot(arg.Reg)] = ^uint64(0) + } + } + if missing >= 0 { + vectors[x86CallerVectorRoot(instruction.Args[operand].Reg)] &^= uint64(1) << uint(missing) + } + state := &x86SourceStackState{vectors: vectors, masks: map[Reg]bool{"K1": true}} + if !x86CallerVectorMoveStep(&instruction, state, nil) { + t.Fatalf("registered subtract shape rejected: %s", row) + } + want := ^uint64(0) + if missing >= 0 { + bytes := form.laneBits / 8 + want &^= x86CallerVectorByteMask(int64(bytes)) << uint(missing/bytes*bytes) + } + got := state.vectors[x86CallerVectorRoot(instruction.Args[form.destination].Reg)] + if got != want { + t.Fatalf("%s missing byte%d input%d: known=%x want=%x", row, missing, operand, got, want) + } + } + } + } +} + +func TestX86GoCallerPackedSubtractUnknownBoundariesRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPSUBD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPSUBQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown high byte": "MOVSS (DI),X0\nMOVSS (SI),X1\nVPSUBQ Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVPSUBSB.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nMOVL $1,AX\nKMOVQ AX,K1\nVPSUBSW Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPSUBUSB.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask no permission": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPSUBUSW.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPSUBB (BX),Z0,Z2\nVMOVSS X2,(DX)", + "private stack": "VMOVUPS (DI),Z0\nVPSUBW (SP),Z0,Z2\nVMOVSS X2,(DX)", + "legacy SP alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPSUBL (BX),X0\nVMOVSS X0,(DX)", + "input store": "VMOVUPS (DI),Z0\nVPSUBQ Z0,Z0,Z2\nVMOVSS X2,(DI)", + "unknown joined bank": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPSUBW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "different pointer offset": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nADDQ $4,DI\njoined:\nVPSUBD (DI),Z0,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source boundary accepted: %v", err) + } + }) + } +} + +func TestX86PackedSubtractLegacyVirtualMemoryCannotForgeNativeAlignment(t *testing.T) { + for _, opcode := range []string{ + "PSUBB", "PSUBW", "PSUBL", "PSUBQ", "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW", + } { + for _, source := range []string{"source+0(FP)", "(SP)", "(BP)"} { + row := opcode + " " + source + ",X0" + t.Run(row, func(t *testing.T) { + assembly := "TEXT f(SB),4,$0-64\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"f": x86WordMaddFPType(t)}, + }) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("virtual storage acquired native alignment permission: %v", err) + } + }) + } + } +} + +func TestX86GoCallerPackedSubtractMemoryWitnessBindsExactInstruction(t *testing.T) { + for _, row := range []string{"PSUBL (DI),X0", "VPSUBQ.BCST.Z (DI),Z0,K1,Z2"} { + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS X0,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs, + }) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86PackedSubtractProperties(ins, "amd64") + if !known || err != nil { + continue + } + found = true + access := ins.x86CallerMemory[form.memory] + if access.width != int64(form.memoryBytes) || access.write || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatalf("missing complete possible source extent for %s", row) + } + for _, mutation := range []string{"opcode", "width", "direction", "address"} { + changedAccess := access + changed := ins + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch mutation { + case "opcode": + changed.Op = "PSUBQ" + if form.broadcast { + changed.Op = "VPSUBQ.BCST" + } + case "width": + changedAccess.width-- + changed.x86CallerMemory[form.memory] = changedAccess + case "direction": + changedAccess.write = true + changed.x86CallerMemory[form.memory] = changedAccess + case "address": + changed.Args[form.memory].Mem.Off++ + } + if x86CallerMemoryCertified(changed, form.memory) { + t.Fatalf("%s reused a stale %s witness", row, mutation) + } + } + } + if !found { + t.Fatalf("source subtract disappeared from normalization: %s", row) + } + } +} diff --git a/amd64_go_caller_region_core.go b/amd64_go_caller_region_core.go new file mode 100644 index 000000000..a0e93abe0 --- /dev/null +++ b/amd64_go_caller_region_core.go @@ -0,0 +1,418 @@ +package plan9asm + +import "reflect" + +// A pointer is never recovered from arbitrary integer data. The region and +// affine offset survive only exact full-register copies/address arithmetic. +type x86CallerPointerValue struct { + argument int + offset int64 + span int64 +} + +// Instruction-local access, not a function-wide memory exemption. The final +// guard compares the complete operands, effect direction and native width. +type x86CallerMemoryAccess struct { + op Op + args []Operand + width int64 + write bool +} + +func x86CallerSignatureMatches(sig FuncSig, proof *x86GoCallerRegionProof) bool { + if proof == nil || proof.calls == 0 || proof.symbol != sig.Name || len(proof.arguments) != len(sig.Args) { + return false + } + sig.x86CallerRegions = nil + return reflect.DeepEqual(sig, proof.signature) +} + +func x86CallerRegionSourceMatches(sig FuncSig, source Func, arch string) bool { + if arch != "amd64" || !x86CallerSignatureMatches(sig, sig.x86CallerRegions) { + return false + } + fingerprint, err := x86CallerSourceFingerprint(source) + if err != nil { + return false + } + if fingerprint == sig.x86CallerRegions.source { + return true + } + // Module/text lowering normalize more than once. Only the exact result of + // an earlier complete proof may be re-proved; edited public operands cannot + // inherit its private metadata or the original source fingerprint. + previous := source.x86SourceStack + return previous != nil && previous.caller == sig.x86CallerRegions && fingerprint == previous.normalized +} + +func x86CallerNormalizedSourceMatches(fn Func, sig FuncSig) bool { + proof := fn.x86SourceStack + if proof == nil || proof.caller == nil || proof.caller != sig.x86CallerRegions || !x86CallerSignatureMatches(sig, proof.caller) { + return false + } + fingerprint, err := x86CallerSourceFingerprint(fn) + return err == nil && fingerprint == proof.normalized +} + +func x86CallerGP(reg Reg) (Reg, bool) { + // This class includes R14, not numerical knowledge of incoming g. The + // complete source-stack proof separately requires its opaque identity at + // every return; only actual source definitions add numerical/pointer bits. + root, valid := amd64FullRegBase(reg) + if !valid { + return "", false + } + switch root { + case AX, BX, CX, DX, DI, SI, SP, BP, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15": + return root, true + default: + return "", false + } +} + +func x86CallerScalarOperands(ins Instr) bool { + for _, operand := range ins.Args { + if operand.Kind == OpImm && (operand.ImmIsFloat || operand.ImmRaw != "") { + return false + } + if operand.Kind == OpReg { + if _, valid := x86CallerGP(operand.Reg); !valid { + return false + } + } + if operand.Kind == OpMem { + for _, reg := range []Reg{operand.Mem.Base, operand.Mem.Index} { + if reg != "" && reg != PC { + if _, valid := x86CallerGP(reg); !valid { + return false + } + } + } + } + } + return true +} + +func x86CloneCallerPointers(source map[Reg]x86CallerPointerValue) map[Reg]x86CallerPointerValue { + copy := make(map[Reg]x86CallerPointerValue, len(source)) + for reg, value := range source { + copy[reg] = value + } + return copy +} + +func x86CloneCallerKnown(source map[Reg]uint8) map[Reg]uint8 { + copy := make(map[Reg]uint8, len(source)) + for reg, value := range source { + copy[reg] = value + } + return copy +} + +func x86CallerByteMask(reg Reg, width int64) uint8 { + if width < 1 || width > 8 { + return 0 + } + shift := uint(0) + if _, bits, alias := amd64ByteAlias(reg); alias { + if width != 1 { + return 0 + } + shift = bits / 8 + } + return uint8((uint16(1)<= int64(len(proof.arguments)) || + ins.Args[1].Kind != OpReg { + return false + } + root, valid := x86CallerGP(ins.Args[1].Reg) + if !valid || root != ins.Args[1].Reg || root == SP || root == BP { + return false + } + x86CallerRecordMemory(ins, index, width, false) + case OpMem: + // The existing stack proof separately maps complete physical SP + // argument words to actual FP slots, excluding the native return PC. + if offset, mapped := ins.x86SourceStackFP[index]; mapped { + if index != 0 || len(ins.Args) != 2 || width != 8 || offset < 0 || offset%8 != 0 || + offset/8 >= int64(len(proof.arguments)) || ins.Args[1].Kind != OpReg { + return false + } + root, valid := x86CallerGP(ins.Args[1].Reg) + if !valid || root != ins.Args[1].Reg || root == SP || root == BP { + return false + } + x86CallerRecordMemory(ins, index, width, false) + continue + } + if len(ins.Args) != 2 || !x86CallerOrdinaryMemoryStep(ins, index, state, proof, width, index == 1) { + return false + } + } + } + return true +} + +func x86CallerOrdinaryMemoryStep(ins *Instr, index int, state *x86SourceStackState, proof *x86GoCallerRegionProof, width int64, write bool) bool { + if index < 0 || index >= len(ins.Args) || ins.Args[index].Kind != OpMem || width <= 0 { + return false + } + if _, parameter := ins.x86SourceStackFP[index]; parameter { + return false // Only the scalar GP import above can consume pointer slots. + } + memory := ins.Args[index].Mem + if memory.Sym != "" || memory.Segment != "" || ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + memory.Off < -x86SourceStackLimit || memory.Off > x86SourceStackLimit { + return false + } + base, valid := x86CallerGP(memory.Base) + pointer, known := state.callers[base] + if !valid || base != memory.Base || !known || pointer.argument < 0 || pointer.argument >= len(proof.arguments) { + return false + } + region := proof.arguments[pointer.argument] + if _, valid := x86CallerPointerEnd(pointer); !valid || pointer.span != 0 && region.kind != x86CallerSliceInput { + return false + } + if memory.Index != "" { + if write || region.kind != x86CallerSliceInput || + memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8 || + !x86CallerNumericRead(Operand{Kind: OpReg, Reg: memory.Index}, 8, state) { + return false + } + } + offset := pointer.offset + memory.Off + if write { + if region.kind != x86CallerFreshResult || offset < 0 || offset > region.width || width > region.width-offset { + return false + } + } else if region.kind != x86CallerSliceInput { + if offset < 0 || offset > region.width || width > region.width-offset { + return false + } + } + x86CallerRecordMemory(ins, index, width, write) + return true +} + +// The existing complete source stack handles all SP/BP transformations and +// returns. This additional transfer tracks only actual caller pointer regions +// and numerical byte initialization; it grants no unknown instruction effects. +func x86CallerScalarStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (pointerFlags, valid bool) { + if accepted, recognized := x86CallerNumericalReadStep(*ins, state); recognized { + return false, accepted + } + if _, known, err := x86OrderingHintProperties(ins.Op, *ins, "amd64"); known { + return false, err == nil + } + op := Op(normalizeInstructionOpcode(x86SourceMoveInstruction(*ins, "amd64").Op)) + if effect, known, err := x86TypedCounterControlByteEffect(ins.Op, *ins, "amd64"); known { + if err != nil { + return false, false + } + // The already registered caller supplies NUM; the descriptor grants no role. + for _, operand := range effect.operands { + if operand.read && !x86CallerNumericRead(operand.operand, operand.width, state) { + return false, false + } + } + for _, operand := range effect.operands { + if operand.write && (operand.operand.Kind != OpReg || + !x86CallerNumericWrite(operand.operand.Reg, operand.width, state)) { + return false, false + } + } + return false, true // The source-stack FLAGS consumer separately checks flagsRead. + } + if op == OpTEXT || op == OpLABEL || op == OpRET || op == OpJMP || isAMD64ConditionalBranch(op) { + return false, !isAMD64CounterZeroBranch(op) || x86CallerNumericRead(Operand{Kind: OpReg, Reg: CX}, 8, state) + } + if _, stack := amd64StackWidthSpecs[op]; stack { + return false, true // Exact saved values/flags are tracked by the stack step. + } + if recognized, valid := x86CallerExplicitIMULStep(ins, state, proof); recognized { + return false, valid + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + return false, true // Only the preceding exact address step admits this. + } + } + if recognized, valid := x86CallerExtensionStep(ins, state, proof); recognized { + return false, valid + } + if recognized, valid := x86CallerConditionalSetStep(ins, state, proof); recognized { + return false, valid + } + if recognized, valid := x86CallerConditionalMoveStep(ins, state, proof); recognized { + return false, valid + } + width := x86RawReturnScalarMoveBytes(op) + if width != 0 { + if len(ins.Args) != 2 || !x86CallerMemoryStep(ins, state, proof, width) { + return false, false + } + source, destination := ins.Args[0], ins.Args[1] + if isAMD64MemoryOperand(source) && isAMD64MemoryOperand(destination) { + return false, false + } + var pointer x86CallerPointerValue + hasPointer := false + if source.Kind == OpFP { + pointer = x86CallerPointerValue{argument: int(source.FPOffset / 8)} + hasPointer = true + } else if source.Kind == OpMem { + if offset, mapped := ins.x86SourceStackFP[0]; mapped { + pointer = x86CallerPointerValue{argument: int(offset / 8)} + hasPointer = true + } + } else if source.Kind == OpReg { + root, _ := x86CallerGP(source.Reg) + pointer, hasPointer = state.callers[root] + } + if hasPointer { + root, full := x86CallerGP(destination.Reg) + sourceRoot, _ := x86CallerGP(source.Reg) + if width != 8 || source.Kind == OpReg && source.Reg != sourceRoot || + destination.Kind != OpReg || !full || root != destination.Reg { + return false, false // Pointer-as-data stores and partial copies stay unknown. + } + state.callers[root] = pointer + delete(state.known, root) + return false, true + } + if source.Kind != OpMem && !x86CallerNumericRead(source, width, state) { + return false, false + } + if destination.Kind == OpReg { + return false, x86CallerNumericWrite(destination.Reg, width, state) + } + return false, destination.Kind == OpMem + } + if x86LEAWidth(ins.Op, "amd64") != 0 { + return false, x86CallerLEAStep(ins, state) + } + name := string(op) + if len(name) < 2 { + return false, false + } + base, suffix := name[:len(name)-1], name[len(name)-1:] + switch suffix { + case "B": + width = 1 + case "W": + width = 2 + case "L": + width = 4 + case "Q": + width = 8 + default: + return false, false + } + if len(ins.Args) == 2 && (op == "ADDQ" || op == "SUBQ") && ins.Args[0].Kind == OpImm && + ins.Args[0].ImmRaw == "" && !ins.Args[0].ImmIsFloat && ins.Args[1].Kind == OpReg { + root, _ := x86CallerGP(ins.Args[1].Reg) + if pointer, present := state.callers[root]; present { + if root != ins.Args[1].Reg { + return false, false + } + delta := int64(ins.Args[0].Imm) + if delta < -x86SourceStackLimit || delta > x86SourceStackLimit { + return false, false + } + if op == "SUBQ" { + delta = -delta + } + var valid bool + pointer, valid = x86CallerPointerShift(pointer, delta) + if !valid { + return false, false + } + state.callers[root] = pointer + return true, true // Private local addresses cannot define numerical flags. + } + } + zero := base == "XOR" && len(ins.Args) == 2 && ins.Args[0].Kind == OpReg && reflect.DeepEqual(ins.Args[0], ins.Args[1]) + for _, argument := range ins.Args { + if !zero && !x86CallerNumericRead(argument, width, state) { + return false, false + } + } + if base == "CMP" || base == "TEST" { + return false, true + } + if len(ins.Args) == 0 || ins.Args[len(ins.Args)-1].Kind != OpReg { + return false, false + } + return false, x86CallerNumericWrite(ins.Args[len(ins.Args)-1].Reg, width, state) +} + +func x86CallerMemoryCertified(ins Instr, index int) bool { + access, found := ins.x86CallerMemory[index] + width, write, valid := x86CallerMoveMemoryShape(ins, index) + // Producer memory proofs admit only native/default addressing. A later + // address-size override cannot reuse even otherwise identical operands. + return found && valid && ins.x86AddressBits == 0 && !ins.x86UnboundAddressOverride && access.op == ins.Op && reflect.DeepEqual(access.args, ins.Args) && + access.width == width && width > 0 && access.write == write +} diff --git a/amd64_go_caller_region_core_test.go b/amd64_go_caller_region_core_test.go new file mode 100644 index 000000000..59950c26c --- /dev/null +++ b/amd64_go_caller_region_core_test.go @@ -0,0 +1,296 @@ +package plan9asm + +import ( + "errors" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerCoreFixture(t *testing.T, source, assembly string) (GoPackage, *File) { + t.Helper() + pkg, selected, _, _ := x86CallerFixture(t, source) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + selected.Assembly["kernel_amd64.s"] = file + pkg.SourceSelection = &selected + return pkg, file +} + +func x86CallerCoreTranslate(t *testing.T, pkg GoPackage, assembly, triple string) error { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "kernel_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + return err + } + defer translated.Module.Dispose() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "caller-regions.ll", "caller-regions.o", translated.Module.String()) + return nil +} + +func TestX86GoCallerRegionCoreConsumesFreshScalarWrites(t *testing.T) { + for _, form := range []struct{ typ, move string }{ + {"uint8", "MOVB"}, {"uint16", "MOVW"}, {"uint32", "MOVL"}, {"uint64", "MOVQ"}, + } { + source := strings.Replace(x86CallerFixtureSource, "var res float32", "var res "+form.typ, 1) + source = strings.Replace(source, ") float32 {", ") "+form.typ+" {", 1) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX\nMOVL AX,(DX)", form.move+" $7,AX\n"+form.move+" AX,(DX)", 1) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(form.move+"/"+triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestX86GoCallerRegionCoreDoesNotCertifyBadStackOrWrites(t *testing.T) { + for name, replacement := range map[string]string{ + "input write": "MOVL AX,(DI)", + "length write": "MOVL AX,(CX)", + "wide result": "MOVQ AX,(DX)", + "one byte overlap": "MOVL AX,1(DX)", + "negative offset": "MOVL AX,-1(DX)", + "unknown base": "MOVL AX,(BX)", + "indexed result": "MOVL AX,(DX)(BX*1)", + "pointer as data": "MOVL DI,(DX)", + "partial pointer": "MOVL DX,BX\nMOVL AX,(BX)", + "fresh pointer join": "CMPQ AX,$0\nJE joined\nMOVQ DI,DX\njoined:\nMOVL AX,(DX)", + "return PC": "MOVQ AX,8(BP)", + "saved BP": "MOVB AX,0(BP)", + "SP escape": "MOVQ SP,AX\nMOVL AX,(DX)", + "input pointer to SP": "MOVQ DI,SP\nMOVL AX,(DX)", + "input pointer to BP": "MOVQ DI,BP\nMOVL AX,(DX)", + "input address to SP": "LEAQ 0(DI),SP\nMOVL AX,(DX)", + "input address to BP": "LEAQ 0(DI),BP\nMOVL AX,(DX)", + "g use": "MOVQ R14,AX\nMOVL AX,(DX)", + "g overwrite": "MOVQ $0,R14\nMOVL AX,(DX)", + "pointer FP store": "MOVQ AX,res+16(FP)\nMOVL AX,(DX)", + "unknown numeric input": "MOVL BX,(DX)", + "partial address clobber": "MOVW $0,DX\nMOVL AX,(DX)", + "pointer flags": "ADDQ $0,DX\nJE joined\njoined:\nMOVL AX,(DX)", + "different pointer offsets": "CMPQ AX,$0\nJE joined\nADDQ $1,DX\njoined:\nMOVB AX,(DX)", + "unrestored SP": "MOVL AX,(DX)", + "unknown store effect": "XCHGL AX,(DX)", + "vector effect": "VMOVSS X0,(DX)", + "floating immediate": "MOVQ $1.25,AX\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", replacement, 1) + if name == "unrestored SP" { + assembly = strings.Replace(assembly, "BYTE $0xc3", "SUBQ $8,SP\nBYTE $0xc3", 1) + } + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if name == "floating immediate" { + // Go's FLOAT class has no MOVQ row; this is invalid source, + // not a native caller privilege that a later producer can grant. + goAssemblerOracleOutput(t, "amd64", assembly, "", false) + if err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go-rejected FLOAT source became Context or success: %v", err) + } + return + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bad source acquired caller-region continuation: %v", err) + } + }) + } + // An actual producer snapshot is mandatory; a pointer signature remains + // the original failure rather than acquiring source memory privileges. + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, x86CallerFixtureAssembly) + pkg.SourceSelection = nil + if err := x86CallerCoreTranslate(t, pkg, x86CallerFixtureAssembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("missing caller selection acquired pointer-store proof: %v", err) + } +} + +func TestX86GoCallerRegionCoreDoesNotStoreInputPointerAsFullWidthData(t *testing.T) { + source := strings.NewReplacer( + "var res float32", "var res uint64", ") float32 {", ") uint64 {", + ).Replace(x86CallerFixtureSource) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "MOVQ DI,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("input pointer escaped through a correctly sized fresh result: %v", err) + } +} + +func TestX86GoCallerRegionCoreTracksScalarCopiesOffsetsAndInputs(t *testing.T) { + for name, replacement := range map[string]string{ + "input load": "MOVL (DI),AX\nADDL $7,AX\nMOVL AX,(DX)", + "length load": "MOVQ (CX),AX\nMOVL AX,(DX)", + "pointer copy": "MOVQ DX,BX\nMOVL AX,(BX)", + "pointer offset": "ADDQ $1,DX\nMOVB AX,(DX)", + "LEA offset": "LEAQ 3(DX),BX\nMOVB AX,(BX)", + "safe adjacent": "MOVB AX,3(DX)", + "byte alias": "MOVB $7,AH\nMOVB AH,(DX)", + "indexed input": "MOVQ $0,BX\nMOVL (DI)(BX*4),AX\nMOVL AX,(DX)", + "copy input offset": "MOVQ DI,BX\nADDQ $4,BX\nMOVL (BX),AX\nMOVL AX,(DX)", + "saved result pointer": "PUSHQ DX\nMOVQ DI,DX\nPOPQ DX\nMOVL AX,(DX)", + "identical join": "MOVQ DX,BX\nCMPQ AX,$0\nJE joined\nMOVQ DX,BX\njoined:\nMOVL AX,(BX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", replacement, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + physical := strings.NewReplacer( + "a+0(FP)", "8(SP)", "b+8(FP)", "16(SP)", + "res+16(FP)", "24(SP)", "length+24(FP)", "32(SP)", + ).Replace(x86CallerFixtureAssembly) + leaf := strings.NewReplacer( + "BYTE $0x55\nWORD $0x8948\nBYTE $0xe5\nLONG $0xf8e48348\n", "", + "WORD $0x8948\nBYTE $0xec\nBYTE $0x5d\n", "", + ).Replace(x86CallerFixtureAssembly) + for name, assembly := range map[string]string{"actual physical FP": physical, "stack-unobserving leaf": leaf} { + t.Run(name, func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerRegionCoreRejectsDifferentCompilerDataLayout(t *testing.T) { + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, x86CallerFixtureAssembly) + ctx := llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(x86CallerFixtureAssembly), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sizes: types.SizesFor("gc", "386"), FrameSizes: types.SizesFor("gc", "amd64"), + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if translation != nil || !errors.Is(err, ErrProbeNeedsContext) { + if translation != nil { + translation.Module.Dispose() + } + t.Fatalf("typed Args/FP slots concealed a different local int/pointer layout: %v", err) + } +} + +func TestX86GoCallerRegionCoreRejectsStaleSourceSignatureAndAccess(t *testing.T) { + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + for _, change := range []string{"original source", "normalized source", "signature", "slot", "memory access", "missing caller"} { + t.Run(change, func(t *testing.T) { + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, x86CallerFixtureAssembly) + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + if change != "original source" { + file = normalized + } + sig := sigs["caller.kernel"] + switch change { + case "original source", "normalized source": + for index := range file.Funcs[0].Instrs { + instruction := &file.Funcs[0].Instrs[index] + if instruction.Op == "MOVL" && len(instruction.Args) == 2 && instruction.Args[1].Kind == OpMem { + instruction.Args[1].Mem.Off++ + } + } + case "signature": + sig.Args[0] = I64 + case "slot": + sig.Frame.Params[0].Offset = 8 + case "missing caller": + sig.x86CallerRegions = nil + case "memory access": + for index := range file.Funcs[0].Instrs { + instruction := &file.Funcs[0].Instrs[index] + for operand, access := range instruction.x86CallerMemory { + access.width++ + instruction.x86CallerMemory[operand] = access + } + } + if x86RawReturnMemoryBound(file.Funcs[0], file, options, true) { + t.Fatal("edited access inherited a whole-function memory permission") + } + // This checks private access revocation. Public normalization + // requires a fresh original source, not the edited sealed snapshot. + return + } + sigs["caller.kernel"] = sig + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed %s borrowed a caller continuation: %v", change, err) + } + }) + } +} + +func TestX86GoCallerRegionCoreCannotBorrowPhysicalFPWithoutAccessProof(t *testing.T) { + assembly := strings.NewReplacer( + "a+0(FP)", "8(SP)", "b+8(FP)", "16(SP)", + "res+16(FP)", "24(SP)", "length+24(FP)", "32(SP)", + ).Replace(x86CallerFixtureAssembly) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + removed := false + for index := range normalized.Funcs[0].Instrs { + instruction := &normalized.Funcs[0].Instrs[index] + for operand := range instruction.x86SourceStackFP { + delete(instruction.x86CallerMemory, operand) + removed = true + } + } + if !removed { + t.Fatal("fixture did not exercise a physical-SP imported parameter") + } + if x86RawReturnMemoryBound(normalized.Funcs[0], normalized, options, true) { + t.Fatal("missing caller access borrowed the old numerical FP continuation") + } +} + +func TestX86GoCallerRegionCoreDoesNotExportScalarOrFloatingStackPermissions(t *testing.T) { + for name, assembly := range map[string]string{ + "external scalar write": strings.Replace(x86CallerFixtureAssembly, + "MOVL AX,(DX)", "MOVL AX,global(SB)\nMOVL AX,(DX)", 1) + "GLOBL global(SB),8,$4\n", + "floating stack adjustment": strings.Replace(x86CallerFixtureAssembly, + "BYTE $0xc3", "ADDQ $0.5,SP\nBYTE $0xc3", 1), + } { + t.Run(name, func(t *testing.T) { + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved permission acquired a caller continuation: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_region_diagnostic_test.go b/amd64_go_caller_region_diagnostic_test.go new file mode 100644 index 000000000..adc8b5dbc --- /dev/null +++ b/amd64_go_caller_region_diagnostic_test.go @@ -0,0 +1,37 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86GoCallerRegionRejectedEffectsRetainInstructionDiagnostic(t *testing.T) { + for _, example := range []struct { + name, replacement, diagnostic string + }{ + { + name: "uninitialized vector arithmetic", replacement: "VFMADD231SS (DI),X0,X1\nVMOVSS X1,(DX)", + diagnostic: "unproved caller-region effect: VFMADD231SS", + }, + { + name: "pointer as scalar data", replacement: "MOVL DI,(DX)", + diagnostic: "unproved caller-region effect: MOVL", + }, + { + name: "different input origins", + replacement: "CMPQ AX,$0\nJE joined\nMOVQ SI,DI\nCMPQ AX,$0\njoined:\nMOVL AX,(DX)", + diagnostic: "different source-stack or caller-region states at join", + }, + } { + t.Run(example.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", example.replacement, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), example.diagnostic) { + t.Fatalf("source rejection lost its actual instruction/state cause: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_region_runtime_test.go b/amd64_go_caller_region_runtime_test.go new file mode 100644 index 000000000..7f6bb287b --- /dev/null +++ b/amd64_go_caller_region_runtime_test.go @@ -0,0 +1,239 @@ +package plan9asm + +import ( + "context" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" +) + +type x86CallerRuntimeForm struct { + name string + bits int + move string + stack string +} + +func x86CallerRuntimeForms() []x86CallerRuntimeForm { + var forms []x86CallerRuntimeForm + for _, width := range []struct { + bits int + move string + }{{8, "MOVB"}, {16, "MOVW"}, {32, "MOVL"}, {64, "MOVQ"}} { + for _, stack := range []string{"Leaf", "Balanced", "SavedPointer"} { + forms = append(forms, x86CallerRuntimeForm{ + name: fmt.Sprintf("kernel%d%s", width.bits, stack), + bits: width.bits, move: width.move, stack: stack, + }) + } + } + return forms +} + +func x86CallerRuntimeSource() (goSource, assembly string) { + var declarations, machine strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + for _, form := range x86CallerRuntimeForms() { + fmt.Fprintf(&declarations, ` +//go:noescape +func %[1]s(a, b, res, length unsafe.Pointer) + +func Call%[1]s(x, y []uint64) uint%[2]d { + var res uint%[2]d + length := len(x) + %[1]s( + unsafe.Pointer(unsafe.SliceData(x)), + unsafe.Pointer(unsafe.SliceData(y)), + unsafe.Pointer(&res), + unsafe.Pointer(&length)) + return res +} +`, form.name, form.bits) + fmt.Fprintf(&machine, "TEXT ·%s(SB),4,$0-32\n", form.name) + machine.WriteString("MOVQ a+0(FP),DI\nMOVQ b+8(FP),SI\nMOVQ res+16(FP),DX\nMOVQ length+24(FP),CX\n") + if form.stack != "Leaf" { + machine.WriteString("BYTE $0x55\nWORD $0x8948\nBYTE $0xe5\nLONG $0xf8e48348\n") + } + if form.stack == "SavedPointer" { + machine.WriteString("PUSHQ DX\nMOVQ DI,DX\nPOPQ DX\n") + } + suffix := strings.TrimPrefix(form.move, "MOV") + fmt.Fprintf(&machine, "%s (DI),AX\n%s 8(SI),BX\nADD%s BX,AX\nMOVQ (CX),BX\nADD%s BX,AX\nADD%s $7,AX\n%s AX,(DX)\n", + form.move, form.move, suffix, suffix, suffix, form.move) + if form.stack != "Leaf" { + machine.WriteString("WORD $0x8948\nBYTE $0xec\nBYTE $0x5d\n") + } + machine.WriteString("BYTE $0xc3\n") + } + return declarations.String(), machine.String() +} + +func x86CallerRuntimePackage(t *testing.T, goSource, assembly string) GoPackage { + t.Helper() + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "value.go", goSource, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + info := &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + } + typed, err := (&types.Config{ + Importer: x86CallerTestImporter{}, Sizes: types.SizesFor("gc", "amd64"), + }).Check("caller", fset, []*ast.File{syntax}, info) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + return GoPackage{ + Path: "caller", Types: typed, Syntax: []*ast.File{syntax}, + SourceSelection: &GoSourceSelection{ + Fset: fset, TypesInfo: info, Sizes: types.SizesFor("gc", "amd64"), + SelectedGoFiles: []string{"value.go"}, GoSources: map[string][]byte{"value.go": []byte(goSource)}, + SelectedAssemblyFiles: []string{"value_amd64.s"}, Assembly: map[string]*File{"value_amd64.s": file}, + }, + } +} + +func x86CallerActualGoOracle(t *testing.T, goSource, assembly string) { + t.Helper() + var tests strings.Builder + tests.WriteString("package caller\nimport \"testing\"\nfunc TestActualCallers(t *testing.T) {\n") + for _, form := range x86CallerRuntimeForms() { + fmt.Fprintf(&tests, ` + for iteration := 0; iteration < 64; iteration++ { + for _, input := range []uint64{0, 1, 127, 255, 65535, 1<<31, 1<<63, ^uint64(0)} { + x := []uint64{input, 0x1122334455667788} + y := []uint64{0x8877665544332211, input ^ 0x9876543210abcdef} + originalX := [2]uint64{x[0], x[1]} + originalY := [2]uint64{y[0], y[1]} + canary := [4]uint64{11, 23, 37, 41} + want := uint%[1]d(x[0]) + uint%[1]d(y[1]) + uint%[1]d(len(x)) + 7 + got := Call%[2]s(x, y) + if got != want || x[0] != originalX[0] || x[1] != originalX[1] || + y[0] != originalY[0] || y[1] != originalY[1] || canary != [4]uint64{11, 23, 37, 41} { + t.Fatalf("%[2]s got=%%#x want=%%#x x=%%#v y=%%#v canary=%%#v", got, want, x, y, canary) + } + } + } +`, form.bits, form.name) + } + tests.WriteString("}\n") + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module caller\n\ngo 1.20\n", "value.go": goSource, + "value_amd64.s": assembly, "value_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + command.WaitDelay = 2 * time.Second + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("actual Go original caller-region oracle: %v\n%s", err, output) + } +} + +func x86CallerCOracle() string { + var source strings.Builder + source.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range x86CallerRuntimeForms() { + fmt.Fprintf(&source, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", form.name, form.name) + } + source.WriteString("int main(void) {\n") + for _, form := range x86CallerRuntimeForms() { + fmt.Fprintf(&source, ` + for (unsigned iteration = 0; iteration < 64; iteration++) { + const uint64_t inputs[] = {0, 1, 127, 255, 65535, UINT64_C(1)<<31, UINT64_C(1)<<63, UINT64_MAX}; + for (unsigned index = 0; index < sizeof(inputs)/sizeof(inputs[0]); index++) { + uint64_t x[] = {inputs[index], UINT64_C(0x1122334455667788)}; + uint64_t y[] = {UINT64_C(0x8877665544332211), inputs[index] ^ UINT64_C(0x9876543210abcdef)}; + uint64_t original_x[2], original_y[2], length = 2; + unsigned char output[48], before[48]; + memcpy(original_x, x, sizeof(x)); + memcpy(original_y, y, sizeof(y)); + for (unsigned byte = 0; byte < sizeof(output); byte++) { + output[byte] = byte + iteration; + } + memset(output + 16, 0, %[1]d); + memcpy(before, output, sizeof(output)); + const uint64_t want = (x[0] + y[1] + length + 7) & UINT%[2]d_MAX; + %[3]s(x, y, output + 16, &length); + for (unsigned byte = 0; byte < sizeof(output); byte++) { + unsigned char expected = before[byte]; + if (byte >= 16 && byte < 16 + %[1]d) { + expected = want >> (8*(byte-16)); + } + if (output[byte] != expected) { + return 1; + } + } + if (memcmp(x, original_x, sizeof(x)) || memcmp(y, original_y, sizeof(y)) || length != 2) { + return 2; + } + } + } +`, form.bits/8, form.bits, form.name) + } + source.WriteString("\treturn 0;\n}\n") + return source.String() +} + +func TestCrossLinuxRuntimeMatrixX86GoCallerRegions(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("caller-region execution requires the Linux/amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + goSource, assembly := x86CallerRuntimeSource() + x86CallerActualGoOracle(t, goSource, assembly) + pkg := x86CallerRuntimePackage(t, goSource, assembly) + ctx := llvm.NewContext() + defer ctx.Dispose() + triple := testTargetTriple(runtime.GOOS, "amd64") + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "caller_regions", triple, + translation.Module.String(), x86CallerCOracle(), runner) +} diff --git a/amd64_go_caller_regions.go b/amd64_go_caller_regions.go new file mode 100644 index 000000000..991da4dbd --- /dev/null +++ b/amd64_go_caller_regions.go @@ -0,0 +1,519 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "go/types" + "reflect" + "strings" +) + +// GoSourceSelection carries a compiler's complete, actual selected package, +// not a user-supplied noalias assertion. The inventories must come from the +// same load/profile as Types, Syntax and TypesInfo. Every selected source is +// mandatory; a per-translation subset cannot establish a closed caller graph. +// Assembly contains the corresponding fully preprocessed source files. +type GoSourceSelection struct { + Fset *token.FileSet + TypesInfo *types.Info + Sizes types.Sizes + SelectedGoFiles []string + GoSources map[string][]byte + SelectedAssemblyFiles []string + Assembly map[string]*File +} + +type x86CallerRegionKind uint8 + +const ( + x86CallerSliceInput x86CallerRegionKind = iota + 1 + x86CallerFreshResult + x86CallerLocalLength +) + +type x86CallerArgumentRegion struct { + kind x86CallerRegionKind + width int64 + slot int64 +} + +// Only the actual Go producer can construct this private proof. It does not +// establish a source stack, an instruction effect or a return continuation. +// The subsequent whole-source CFG proof must consume the exact signature, +// parameter slots and output envelope before it can admit a raw return. +type x86GoCallerRegionProof struct { + symbol string + arguments []x86CallerArgumentRegion + calls int + goSources map[string][32]byte + assembly map[string][32]byte + signature FuncSig + source [32]byte +} + +func x86GoCallerRegionCandidate(fn Func, sig FuncSig) bool { + if sig.Ret != Void || len(sig.Args) == 0 { + return false + } + for _, argument := range sig.Args { + if argument != Ptr { + return false + } + } + decoded, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + return false // The normal decoder still reports the actual failure. + } + for _, ins := range decoded.Instrs { + if ins.x86RawNearReturn != nil { + return true + } + } + return false +} + +func deriveX86GoCallerRegions(pkg GoPackage, selected GoSourceSelection, fn Func, sig FuncSig, arch string) (*x86GoCallerRegionProof, error) { + fail := func(reason string) (*x86GoCallerRegionProof, error) { + return nil, fmt.Errorf("%w: bounded raw-return caller source: %s", ErrProbeNeedsContext, reason) + } + if arch != "amd64" || pkg.Types == nil || selected.Fset == nil || selected.TypesInfo == nil || !x86CallerTargetSizesValid(selected.Sizes) || + len(sig.ArgRegs) != 0 || sig.Ret != Void || len(sig.Frame.Results) != 0 || + sig.ARMEntry != nil || sig.ARM64GoRegisterABI != nil || sig.ARM64ClosureABI != nil || sig.WASMNative || sig.WASMContext != "" || + fn.FrameSize != 0 || strings.Contains(fn.Sym, "<") || len(sig.Args) == 0 || len(sig.Args) > 16 { + return fail("needs complete ordinary AMD64 ABI0 pointer/void types and source") + } + name := strings.TrimPrefix(sig.Name, pkg.Types.Path()+".") + declaration, ok := pkg.Types.Scope().Lookup(name).(*types.Func) + if !ok || name == sig.Name || declaration.Exported() { + return fail("entry must be an unexported actual package declaration") + } + actual, ok := declaration.Type().(*types.Signature) + if !ok || actual.Recv() != nil || actual.Variadic() || actual.TypeParams().Len() != 0 || + actual.Results().Len() != 0 || actual.Params().Len() != len(sig.Args) || len(sig.Frame.Params) != len(sig.Args) || + fn.ArgSize != int64(8*len(sig.Args)) { + return fail("signature is not the complete pointer-only ABI0 declaration") + } + for index, argument := range sig.Args { + bound := false + for _, slot := range sig.Frame.Params { + bound = bound || slot.Offset == int64(index*8) && slot.Index == index && slot.Type == Ptr && len(frameSlotFields(slot)) == 0 + } + if argument != Ptr || !types.Identical(actual.Params().At(index).Type(), types.Typ[types.UnsafePointer]) || + !bound || !x86RawReturnFPBound(int64(index*8), 8, sig, arch) { + return fail("parameter types or exact FP slots differ from the declaration") + } + } + if err := x86ValidateCallerSelection(pkg, selected); err != nil { + return fail(err.Error()) + } + if err := x86RejectAssemblyCallers(selected, fn, name, sig.Name, arch); err != nil { + return fail(err.Error()) + } + + proof := &x86GoCallerRegionProof{ + symbol: sig.Name, signature: sig, + goSources: make(map[string][32]byte), assembly: make(map[string][32]byte), + } + // Slices and maps supplied by the compiler remain mutable. The producer + // owns an independent signature and byte fingerprint, never an alias of + // an operand or layout that a later normalization may change. + proof.signature.Args = append([]LLVMType(nil), sig.Args...) + proof.signature.Frame.Params = append([]FrameSlot(nil), sig.Frame.Params...) + var err error + proof.source, err = x86CallerSourceFingerprint(fn) + if err != nil { + return fail("translated source cannot be pinned: " + err.Error()) + } + for file, source := range selected.GoSources { + proof.goSources[file] = sha256.Sum256(source) + } + for file, assembly := range selected.Assembly { + fingerprint, err := x86CallerSourceFingerprint(assembly) + if err != nil { + return fail("selected assembly cannot be pinned: " + err.Error()) + } + proof.assembly[file] = fingerprint + } + allowedUses := map[*ast.Ident]bool{} + declarations := 0 + for _, file := range pkg.Syntax { + for _, group := range file.Comments { + for _, comment := range group.List { + if strings.Contains(comment.Text, "go:linkname") { + for _, field := range strings.Fields(comment.Text) { + if field == name || field == sig.Name { + return fail("linkname prevents package-local caller closure") + } + } + } + } + } + for _, node := range file.Decls { + function, ok := node.(*ast.FuncDecl) + if !ok { + continue + } + if selected.TypesInfo.Defs[function.Name] == declaration { + if function.Body != nil { + return fail("assembly declaration has a Go body") + } + declarations++ + continue + } + if function.Body == nil { + continue + } + for statementIndex, statement := range function.Body.List { + expression, ok := statement.(*ast.ExprStmt) + if !ok { + continue + } + call, ok := expression.X.(*ast.CallExpr) + if !ok { + continue + } + identifier, ok := call.Fun.(*ast.Ident) + if !ok || selected.TypesInfo.Uses[identifier] != declaration { + continue + } + regions, err := x86DeriveCallerArguments(function, statementIndex, call, selected.TypesInfo) + if err != nil { + return fail(err.Error()) + } + for index := range regions { + regions[index].slot = int64(index * 8) + } + if proof.arguments != nil && !reflect.DeepEqual(proof.arguments, regions) { + return fail("different callers do not share the same parameter/write envelope") + } + proof.arguments = regions + proof.calls++ + allowedUses[identifier] = true + } + } + } + if declarations != 1 || proof.calls == 0 { + return fail("missing bodyless declaration or closed direct caller") + } + closed := true + for _, file := range pkg.Syntax { + ast.Inspect(file, func(node ast.Node) bool { + identifier, ok := node.(*ast.Ident) + if ok && selected.TypesInfo.Uses[identifier] == declaration && !allowedUses[identifier] { + closed = false // Address-taking, indirect and nested calls never disappear. + } + return true + }) + } + if !closed { + return fail("entry has a non-direct, nested or unregistered caller") + } + return proof, nil +} + +func x86CallerTargetSizesValid(sizes types.Sizes) bool { + if sizes == nil || sizes.Sizeof(types.Typ[types.UnsafePointer]) != 8 { + return false + } + for _, kind := range []types.BasicKind{ + types.Int8, types.Uint8, types.Int16, types.Uint16, types.Int32, types.Uint32, + types.Int64, types.Uint64, types.Int, types.Uint, types.Float32, types.Float64, + } { + object := types.NewVar(token.NoPos, nil, "", types.Typ[kind]) + if sizes.Sizeof(object.Type()) != x86CallerNumericWidth(object) { + return false + } + } + return true +} + +func x86ValidateCallerSelection(pkg GoPackage, selected GoSourceSelection) error { + if len(selected.SelectedGoFiles) == 0 || len(selected.SelectedGoFiles) != len(selected.GoSources) || + len(pkg.Syntax) != len(selected.SelectedGoFiles) || len(selected.SelectedAssemblyFiles) == 0 || + len(selected.SelectedAssemblyFiles) != len(selected.Assembly) { + return fmt.Errorf("complete selected Go and assembly inventory is required") + } + seen := map[string]bool{} + for _, name := range selected.SelectedGoFiles { + source, ok := selected.GoSources[name] + if name == "" || seen[name] || !ok { + return fmt.Errorf("missing/duplicate selected Go source") + } + seen[name] = true + var syntax *ast.File + for _, file := range pkg.Syntax { + if selected.Fset.Position(file.Pos()).Filename == name { + if syntax != nil { + return fmt.Errorf("duplicate selected Go syntax") + } + syntax = file + } + } + if syntax == nil { + return fmt.Errorf("selected Go source has no actual typed syntax") + } + identitiesMatch := true + ast.Inspect(syntax, func(node ast.Node) bool { + if identifier, ok := node.(*ast.Ident); ok { + object := selected.TypesInfo.ObjectOf(identifier) + if object != nil && object.Name() != identifier.Name { + identitiesMatch = false + } + } + return true + }) + if !identitiesMatch { + return fmt.Errorf("selected Go identifier differs from its actual typechecked identity") + } + fset := token.NewFileSet() + parsed, err := parser.ParseFile(fset, name, source, parser.ParseComments) + if err != nil { + return fmt.Errorf("selected Go source is not the typed source: %w", err) + } + var original, typed bytes.Buffer + if err := format.Node(&original, fset, parsed); err != nil { + return err + } + if err := format.Node(&typed, selected.Fset, syntax); err != nil { + return err + } + if !bytes.Equal(original.Bytes(), typed.Bytes()) { + return fmt.Errorf("selected Go bytes differ from the actual typed syntax") + } + } + seen = map[string]bool{} + for _, name := range selected.SelectedAssemblyFiles { + if name == "" || seen[name] || selected.Assembly[name] == nil || selected.Assembly[name].Arch != ArchAMD64 { + return fmt.Errorf("missing/duplicate selected assembly source") + } + seen[name] = true + } + return nil +} + +func x86RejectAssemblyCallers(selected GoSourceSelection, target Func, localName, resolvedName, arch string) error { + entries := 0 + for _, file := range selected.Assembly { + if len(file.UnlinkedPrelude) != 0 { + return fmt.Errorf("unregistered assembly prelude prevents caller closure") + } + for _, data := range file.Data { + base, _, reference := parseSBRef(strings.TrimPrefix(data.Addr, "$")) + if reference && (base == target.Sym || base == "·"+localName || base == resolvedName) { + return fmt.Errorf("assembly DATA exposes the Go entry") + } + } + for _, fn := range file.Funcs { + if len(fn.X86RawText) != 0 { + return fmt.Errorf("address-sensitive raw assembly prevents caller closure") + } + if fn.Sym == target.Sym { + if !reflect.DeepEqual(fn, target) { + return fmt.Errorf("selected assembly entry differs from the translated source") + } + entries++ + } + decoded, err := decodeX86RawDirectives(fn, arch) + if err != nil { + return fmt.Errorf("selected assembly caller effects are not known: %w", err) + } + for _, ins := range decoded.Instrs { + for _, argument := range ins.Args { + symbol := argument.Sym + if argument.Kind == OpMem { + symbol = argument.Mem.Sym + } + symbol = strings.TrimPrefix(strings.TrimSpace(symbol), "$") + base, _, reference := parseSBRef(symbol) + if reference && (base == target.Sym || base == "·"+localName || base == resolvedName) && ins.Op != OpTEXT { + return fmt.Errorf("selected assembly calls or exposes the Go entry") + } + } + if ins.Op == OpCALL { + return fmt.Errorf("unknown assembly call cannot close package-local callers") + } + if ins.Op == OpJMP { + if len(ins.Args) != 1 { + return fmt.Errorf("incomplete assembly branch prevents caller closure") + } + argument := ins.Args[0] + if argument.Kind != OpLabel && argument.Kind != OpIdent && + !(argument.Kind == OpMem && argument.Mem.Base == PC && argument.Mem.Index == "" && argument.Mem.Sym == "") { + return fmt.Errorf("indirect or external assembly branch prevents caller closure") + } + } + } + } + } + if entries != 1 { + return fmt.Errorf("translated entry is not unique in the complete selected assembly") + } + return nil +} + +func x86DeriveCallerArguments(function *ast.FuncDecl, index int, call *ast.CallExpr, info *types.Info) ([]x86CallerArgumentRegion, error) { + if call.Ellipsis.IsValid() || len(call.Args) == 0 { + return nil, fmt.Errorf("direct call has incomplete argument flow") + } + parameters := map[types.Object]bool{} + if function.Type.Params != nil { + for _, field := range function.Type.Params.List { + for _, name := range field.Names { + parameters[info.Defs[name]] = true + } + } + } + locals := map[types.Object]x86CallerArgumentRegion{} + for _, statement := range function.Body.List[:index] { + switch statement := statement.(type) { + case *ast.DeclStmt: + declaration, ok := statement.Decl.(*ast.GenDecl) + if !ok || declaration.Tok != token.VAR { + continue + } + for _, spec := range declaration.Specs { + value, ok := spec.(*ast.ValueSpec) + if !ok || len(value.Values) != 0 { + continue + } + for _, name := range value.Names { + object := info.Defs[name] + if width := x86CallerNumericWidth(object); width != 0 { + locals[object] = x86CallerArgumentRegion{kind: x86CallerFreshResult, width: width} + } + } + } + case *ast.AssignStmt: + if statement.Tok != token.DEFINE || len(statement.Lhs) != 1 || len(statement.Rhs) != 1 { + continue + } + name, ok := statement.Lhs[0].(*ast.Ident) + length, call := statement.Rhs[0].(*ast.CallExpr) + if !ok || !call || len(length.Args) != 1 { + continue + } + builtin, ok := length.Fun.(*ast.Ident) + slice, sliceOK := length.Args[0].(*ast.Ident) + if ok && sliceOK && info.Uses[builtin] == types.Universe.Lookup("len") && parameters[info.Uses[slice]] { + if _, ok := info.Uses[slice].Type().Underlying().(*types.Slice); ok { + locals[info.Defs[name]] = x86CallerArgumentRegion{kind: x86CallerLocalLength, width: 8} + } + } + } + } + var regions []x86CallerArgumentRegion + allowedAddresses := map[*ast.Ident]bool{} + localObjects := map[types.Object]bool{} + results := 0 + for _, argument := range call.Args { + conversion, ok := argument.(*ast.CallExpr) + if !ok || len(conversion.Args) != 1 || conversion.Ellipsis.IsValid() || + !info.Types[conversion.Fun].IsType() || !types.Identical(info.TypeOf(conversion.Fun), types.Typ[types.UnsafePointer]) { + return nil, fmt.Errorf("argument is not an actual unsafe.Pointer conversion") + } + switch value := conversion.Args[0].(type) { + case *ast.UnaryExpr: + identifier, ok := value.X.(*ast.Ident) + if value.Op != token.AND || !ok { + return nil, fmt.Errorf("result/length address is not a fresh direct local") + } + object := info.Uses[identifier] + region, ok := locals[object] + if !ok || localObjects[object] { + return nil, fmt.Errorf("result/length local is missing, aliased or not fresh") + } + regions = append(regions, region) + localObjects[object] = true + allowedAddresses[identifier] = true + if region.kind == x86CallerFreshResult { + results++ + } + case *ast.CallExpr: + selector, ok := value.Fun.(*ast.SelectorExpr) + if !ok || info.Uses[selector.Sel] != types.Unsafe.Scope().Lookup("SliceData") || len(value.Args) != 1 { + return nil, fmt.Errorf("input is not actual unsafe.SliceData") + } + identifier, ok := value.Args[0].(*ast.Ident) + if !ok || !parameters[info.Uses[identifier]] { + return nil, fmt.Errorf("slice input is not an independent typed caller parameter") + } + slice, ok := info.Uses[identifier].Type().Underlying().(*types.Slice) + if !ok { + return nil, fmt.Errorf("input is not a typed slice") + } + width := x86CallerNumericWidth(types.NewVar(token.NoPos, nil, "", slice.Elem())) + if width == 0 { + return nil, fmt.Errorf("slice element layout is not a supported numerical input") + } + regions = append(regions, x86CallerArgumentRegion{kind: x86CallerSliceInput, width: width}) + default: + return nil, fmt.Errorf("unsafe arithmetic or non-direct argument flow") + } + } + if results == 0 { + return nil, fmt.Errorf("caller has no independent fresh result envelope") + } + valid := true + ast.Inspect(function.Body, func(node ast.Node) bool { + if address, ok := node.(*ast.UnaryExpr); ok && address.Op == token.AND { + if identifier, ok := address.X.(*ast.Ident); ok && localObjects[info.Uses[identifier]] && !allowedAddresses[identifier] { + valid = false + } + } + if assignment, ok := node.(*ast.AssignStmt); ok && assignment.Tok != token.DEFINE { + for _, target := range assignment.Lhs { + if identifier, ok := target.(*ast.Ident); ok && localObjects[info.Uses[identifier]] { + valid = false + } + } + } + if increment, ok := node.(*ast.IncDecStmt); ok { + if identifier, ok := increment.X.(*ast.Ident); ok && localObjects[info.Uses[identifier]] { + valid = false + } + } + return true + }) + if !valid { + return nil, fmt.Errorf("fresh local address escaped or its value was reassigned") + } + return regions, nil +} + +func x86CallerSourceFingerprint(source any) ([32]byte, error) { + // The parser's exported source operands/bytes contain only JSON-encodable + // values. Private decoder certificates are deliberately not source input. + encoded, err := json.Marshal(source) + if err != nil { + return [32]byte{}, err + } + return sha256.Sum256(encoded), nil +} + +func x86CallerNumericWidth(object types.Object) int64 { + if object == nil { + return 0 + } + basic, ok := object.Type().(*types.Basic) + if !ok { + return 0 + } + switch basic.Kind() { + case types.Uint8, types.Int8: + return 1 + case types.Uint16, types.Int16: + return 2 + case types.Uint32, types.Int32, types.Float32: + return 4 + case types.Uint64, types.Int64, types.Float64, types.Int, types.Uint: + return 8 + default: + return 0 + } +} diff --git a/amd64_go_caller_regions_test.go b/amd64_go_caller_regions_test.go new file mode 100644 index 000000000..327e72646 --- /dev/null +++ b/amd64_go_caller_regions_test.go @@ -0,0 +1,319 @@ +package plan9asm + +import ( + "crypto/sha256" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" +) + +type x86CallerTestImporter struct{} + +func (x86CallerTestImporter) Import(path string) (*types.Package, error) { + if path == "unsafe" { + return types.Unsafe, nil + } + panic("unexpected caller-region fixture import: " + path) +} + +const x86CallerFixtureSource = `package caller +import "unsafe" + +//go:noescape +func kernel(a, b, res, length unsafe.Pointer) + +func Call(x, y []float32) float32 { + var res float32 + length := len(x) + kernel( + unsafe.Pointer(unsafe.SliceData(x)), + unsafe.Pointer(unsafe.SliceData(y)), + unsafe.Pointer(&res), + unsafe.Pointer(&length)) + return res +} +` + +const x86CallerFixtureAssembly = `TEXT ·kernel(SB),4,$0-32 +MOVQ a+0(FP),DI +MOVQ b+8(FP),SI +MOVQ res+16(FP),DX +MOVQ length+24(FP),CX +BYTE $0x55 +WORD $0x8948 +BYTE $0xe5 +LONG $0xf8e48348 +MOVL $7,AX +MOVL AX,(DX) +WORD $0x8948 +BYTE $0xec +BYTE $0x5d +BYTE $0xc3 +` + +func x86CallerFixture(t *testing.T, source string) (GoPackage, GoSourceSelection, Func, FuncSig) { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "caller.go", source, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + info := &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + } + pkg, err := (&types.Config{ + Importer: x86CallerTestImporter{}, + Sizes: types.SizesFor("gc", "amd64"), + }).Check("caller", fset, []*ast.File{file}, info) + if err != nil { + t.Fatal(err) + } + assembly, err := Parse(ArchAMD64, x86CallerFixtureAssembly) + if err != nil { + t.Fatal(err) + } + goPkg := GoPackage{Path: "caller", Types: pkg, Syntax: []*ast.File{file}} + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(goPkg, assembly, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + selected := GoSourceSelection{ + Fset: fset, + TypesInfo: info, + Sizes: types.SizesFor("gc", "amd64"), + SelectedGoFiles: []string{"caller.go"}, + GoSources: map[string][]byte{"caller.go": []byte(source)}, + SelectedAssemblyFiles: []string{"kernel_amd64.s"}, + Assembly: map[string]*File{"kernel_amd64.s": assembly}, + } + return goPkg, selected, assembly.Funcs[0], sigs["caller.kernel"] +} + +func TestX86GoCallerRegionsDeriveActualTypedArguments(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64") + if err != nil { + t.Fatal(err) + } + if proof == nil || proof.symbol != "caller.kernel" || len(proof.arguments) != 4 || proof.calls != 1 { + t.Fatalf("missing actual caller/parameter binding: %+v", proof) + } + wantKinds := []x86CallerRegionKind{x86CallerSliceInput, x86CallerSliceInput, x86CallerFreshResult, x86CallerLocalLength} + wantWidths := []int64{4, 4, 4, 8} + for index, argument := range proof.arguments { + if argument.kind != wantKinds[index] || argument.width != wantWidths[index] || argument.slot != int64(index*8) { + t.Fatalf("argument %d: %+v", index, argument) + } + } + // A caller producer by itself must not turn this into a raw-return pass. + file, err := Parse(ArchAMD64, x86CallerFixtureAssembly) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{sig.Name: sig}, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }); err == nil { + t.Fatal("region metadata without a whole-source control proof became a pass") + } +} + +func TestX86GoCallerRegionsBindNumericalResultWidthsAndEveryCaller(t *testing.T) { + for _, result := range []struct { + name string + width int64 + }{ + {name: "uint8", width: 1}, {name: "int8", width: 1}, + {name: "uint16", width: 2}, {name: "int16", width: 2}, + {name: "uint32", width: 4}, {name: "int32", width: 4}, {name: "float32", width: 4}, + {name: "uint64", width: 8}, {name: "int64", width: 8}, {name: "float64", width: 8}, + } { + t.Run(result.name, func(t *testing.T) { + source := strings.ReplaceAll(x86CallerFixtureSource, "float32", result.name) + caller := source[strings.Index(source, "func Call("):] + source += strings.Replace(caller, "func Call(", "func SecondCall(", 1) + pkg, selected, fn, sig := x86CallerFixture(t, source) + proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64") + if err != nil { + t.Fatal(err) + } + if proof.calls != 2 || proof.arguments[2].width != result.width { + t.Fatalf("actual typed callers/result envelope: %+v", proof) + } + }) + } +} + +func TestX86GoCallerRegionsRejectIncompleteAndChangedSelection(t *testing.T) { + for _, change := range []string{ + "missing types", "missing type info", "missing Go bytes", "missing Go AST", "changed Go bytes", + "extra Go bytes", "missing ASM", "extra ASM", "changed signature", "wrong parameter index", "custom registers", "wrong architecture", "missing sizes", "wrong actual sizes", + } { + t.Run(change, func(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + arch := "amd64" + switch change { + case "missing types": + pkg.Types = nil + case "missing type info": + selected.TypesInfo = nil + case "missing Go bytes": + delete(selected.GoSources, "caller.go") + case "missing Go AST": + pkg.Syntax = nil + case "changed Go bytes": + selected.GoSources["caller.go"] = []byte(strings.Replace(x86CallerFixtureSource, "&res", "unsafe.SliceData(x)", 1)) + case "extra Go bytes": + selected.GoSources["unselected.go"] = []byte("package caller\n") + case "missing ASM": + delete(selected.Assembly, "kernel_amd64.s") + case "extra ASM": + selected.Assembly["unselected.s"] = selected.Assembly["kernel_amd64.s"] + case "changed signature": + sig.Args = []LLVMType{I64, Ptr, Ptr, Ptr} + case "wrong parameter index": + sig.Frame.Params[0].Index, sig.Frame.Params[1].Index = 1, 0 + case "custom registers": + sig.ArgRegs = []Reg{AX, BX, CX, DX} + case "wrong architecture": + arch = "arm64" + case "missing sizes": + selected.Sizes = nil + case "wrong actual sizes": + selected.Sizes = types.SizesFor("gc", "386") + } + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, arch); err == nil || proof != nil { + t.Fatalf("incomplete/changed actual source got a caller certificate: %+v, %v", proof, err) + } + }) + } +} + +func TestX86GoCallerRegionsRejectUnclosedCallsAndUnsafeRegions(t *testing.T) { + for _, source := range []string{ + x86CallerFixtureSource + "\nvar addressTaken = kernel\n", + strings.Replace(x86CallerFixtureSource, "//go:noescape", "//go:linkname kernel external.kernel", 1), + strings.Replace(x86CallerFixtureSource, "kernel(\n", "indirect := kernel\n\tindirect(\n", 1), + strings.Replace(x86CallerFixtureSource, "unsafe.Pointer(&res)", "unsafe.Pointer(unsafe.SliceData(x))", 1), + strings.Replace(x86CallerFixtureSource, "unsafe.Pointer(&res)", "unsafe.Pointer(uintptr(unsafe.Pointer(&res)) + 1)", 1), + strings.Replace(x86CallerFixtureSource, "length := len(x)", "length := 7", 1), + strings.Replace(x86CallerFixtureSource, "length := len(x)", "length := len(x)\n\tlength++", 1), + strings.Replace(x86CallerFixtureSource, "length := len(x)", "length := len(x)\n\tEscape(&res)", 1) + "\nfunc Escape(*float32) {}\n", + strings.Replace(x86CallerFixtureSource, "kernel(\n", "func() { kernel(\n", 1), + } { + // The last source is intentionally replaced with a valid closure below. + if strings.Contains(source, "func() { kernel") { + source = strings.Replace(source, "unsafe.Pointer(&length))", "unsafe.Pointer(&length)) }()", 1) + } + pkg, selected, fn, sig := x86CallerFixture(t, source) + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("unclosed/unsafe source got a caller certificate:\n%s\n%+v, %v", source, proof, err) + } + } +} + +func TestX86GoCallerRegionsRejectAssemblyCallersAndAddressTaking(t *testing.T) { + for _, body := range []string{ + "CALL ·kernel(SB)", + "MOVQ $·kernel(SB),AX", + "JMP ·kernel(SB)", + "JMP external·unknown(SB)", + "BYTE $0xe8\nLONG $0", + "JMP AX", + } { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + sibling, err := Parse(ArchAMD64, "TEXT ·Sibling(SB),4,$0-0\n"+body+"\nRET\n") + if err != nil { + t.Fatal(err) + } + selected.SelectedAssemblyFiles = append(selected.SelectedAssemblyFiles, "sibling_amd64.s") + selected.Assembly["sibling_amd64.s"] = sibling + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("unregistered assembly caller got a certificate: %s: %+v, %v", body, proof, err) + } + } +} + +func TestX86GoCallerRegionsRejectAssemblyMemorySymbolExposure(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + sibling, err := Parse(ArchAMD64, "TEXT ·Sibling(SB),4,$0-0\nMOVQ (AX),BX\nRET\n") + if err != nil { + t.Fatal(err) + } + // The actual parser usually spells SB references as OpSym. A compiler's + // selected File may also carry a symbolic memory operand; closure must + // inspect that representation rather than silently treating it as numeric. + sibling.Funcs[0].Instrs[1].Args[0].Mem.Sym = "·kernel(SB)" + selected.SelectedAssemblyFiles = append(selected.SelectedAssemblyFiles, "sibling_amd64.s") + selected.Assembly["sibling_amd64.s"] = sibling + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("assembly memory exposed entry without closing its callers: %+v, %v", proof, err) + } +} + +func TestX86GoCallerRegionsKeepIndependentSourceAndSignatureSnapshots(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64") + if err != nil { + t.Fatal(err) + } + before := proof.source + fn.Instrs[1].Args[0].FPOffset = 8 + sig.Args[0] = I64 + sig.Frame.Params[0].Offset = 8 + selected.GoSources["caller.go"][0] = 'X' + after, err := x86CallerSourceFingerprint(fn) + if err != nil { + t.Fatal(err) + } + if proof.source != before || proof.source == after || + proof.signature.Args[0] != Ptr || proof.signature.Frame.Params[0].Offset != 0 || + proof.goSources["caller.go"] == sha256.Sum256(selected.GoSources["caller.go"]) { + t.Fatal("mutable compiler inputs changed an already-produced witness") + } +} + +func TestX86CallerSourceFingerprintRejectsOpaqueInput(t *testing.T) { + if _, err := x86CallerSourceFingerprint(make(chan int)); err == nil { + t.Fatal("opaque input acquired a source fingerprint") + } +} + +func TestX86GoCallerRegionsRejectEditedASTWithStaleTypeInfo(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + changed := false + ast.Inspect(pkg.Syntax[0], func(node ast.Node) bool { + if changed { + return false + } + address, ok := node.(*ast.UnaryExpr) + if !ok || address.Op != token.AND { + return true + } + identifier, ok := address.X.(*ast.Ident) + if ok && identifier.Name == "res" { + identifier.Name = "length" + changed = true + } + return true + }) + selected.GoSources["caller.go"] = []byte(strings.Replace(x86CallerFixtureSource, "&res", "&length", 1)) + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("edited AST retained stale typechecked local identity: %+v, %v", proof, err) + } +} + +func TestX86GoCallerRegionsRejectAssemblyDataAddressTaking(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + selected.Assembly["kernel_amd64.s"].Data = []DataStmt{{ + Sym: "entryAddress", Width: 8, Addr: "·kernel(SB)", + }} + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("assembly DATA exposed a closed entry: %+v, %v", proof, err) + } +} diff --git a/amd64_go_caller_sad.go b/amd64_go_caller_sad.go new file mode 100644 index 000000000..cebdbd4be --- /dev/null +++ b/amd64_go_caller_sad.go @@ -0,0 +1,38 @@ +package plan9asm + +func x86CallerSADStep(ins *Instr, form x86PackedSADForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + if form.memory >= 0 { + // A selected dword or zero mask does not shrink the native access. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + var known uint64 + for output := 0; output < form.byteWidth; output += 2 { + a, b := form.wordInputs(output) + if first&a == a && second&b == b { + known |= uint64(3) << uint(output) + } + } + if form.constantZero(*ins) { + known = width + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.mask >= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_sad_test.go b/amd64_go_caller_sad_test.go new file mode 100644 index 000000000..363137290 --- /dev/null +++ b/amd64_go_caller_sad_test.go @@ -0,0 +1,239 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerSADOriginalRawEffect(t *testing.T) { + // Unchanged instruction from Weaviate v1.39.6 hamming_avx256_amd64.s:77. + // The independent fixture binds its own complete selected typed callers; + // it does not borrow the external kernel's unresolved loop/frame contract. + body := "VMOVDQU (DI),Y2\nVMOVDQU (SI),Y5\nLONG $0xeaf6d5c5\nVMOVSS X5,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } +} + +func TestX86GoCallerSADCompleteMemoryMaskAndAliasForms(t *testing.T) { + for _, row := range x86SADRows("amd64") { + // This caller fixture binds ordinary slice data, not arbitrary SB or + // segment sources. Those accepted table forms have separate objects. + if strings.Contains(row, "(FS)") || strings.Contains(row, "(GS)") || strings.Contains(row, "(SB)") { + continue + } + // Rebind only register/address tokens, never the immediate selector. + row = strings.NewReplacer( + "16(AX)(BX*2)", "(DI)", "(AX)", "(DI)", + "X15", "X2", "Y15", "Y2", "Z15", "Z2", + "X31", "X1", "Y31", "Y1", "Z31", "Z1", + "X30", "X2", "Y30", "Y2", "Z30", "Z2", + "X7", "X2", "Y7", "Y2", "Z7", "Z2", + ).Replace(row) + args := strings.Split(strings.SplitN(row, " ", 2)[1], ",") + index := args[len(args)-1][1:] + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\nKMOVQ AX,K7\n" + row + "\nVMOVSS X" + index + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } +} + +func TestX86GoCallerSADMemoryCertificatesBindCompleteInstruction(t *testing.T) { + for _, row := range []string{"PSADBW (DI),X0", "MPSADBW $5,(DI),X0", "VPSADBW (DI),Y0,Y2", "VMPSADBW $37,(DI),Y0,Y2", "VDBPSADBW.Z $228,(DI),Z0,K1,Z2"} { + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS X2,(DX)" + if !strings.HasPrefix(row, "V") { + body = strings.Replace(body, "VMOVSS X2", "VMOVSS X0", 1) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86PackedSADProperties(ins, "amd64") + if !known || err != nil || form.memory < 0 { + continue + } + found = true + access, exists := ins.x86CallerMemory[form.memory] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatal("SAD lost its exact native full-read witness") + } + for _, change := range []string{"opcode", "immediate", "width", "address", "destination", "direction", "mask", "address size"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch change { + case "opcode": + mutated.Op = "VPMADDUBSW" + case "immediate": + if form.spec.kind != amd64PackedSADBlockSum { + mutated.Args[0].Imm++ + } else { + mutated.Args = append([]Operand{{Kind: OpImm, Imm: 1}}, mutated.Args...) + } + case "width": + access.width-- + mutated.x86CallerMemory[form.memory] = access + case "address": + mutated.Args[form.memory].Mem.Off++ + case "destination": + mutated.Args[form.destination].Reg = "X7" + case "direction": + access.write = true + mutated.x86CallerMemory[form.memory] = access + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K7" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K1"}) + } + case "address size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, form.memory) { + t.Fatalf("%s stale %s permission survived", row, change) + } + } + } + if !found { + t.Fatal("no consumed SAD memory instruction") + } + } +} + +func TestX86GoCallerSADRetainsUnprovedSourcesAndEnvelopes(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPSADBW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPSADBW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown legacy": "MOVUPS (DI),X0\nMPSADBW $5,X0,X1\nVMOVSS X1,(DX)", + "unknown upper": "MOVUPS (DI),X0\nMOVUPS (SI),X1\nVPSADBW Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVDBPSADBW.Z $228,Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nVDBPSADBW $228,Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVDBPSADBW.Z $228,Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask full read": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVDBPSADBW.Z $0,(DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "selected full read": "VMOVUPS (DI),X0\nMPSADBW $0,(CX),X0\nVMOVSS X0,(DX)", + "unknown address": "VMOVUPS (DI),Y0\nVPSADBW (BX),Y0,Y1\nVMOVSS X1,(DX)", + "private SP": "VMOVUPS (DI),X0\nPSADBW (SP),X0\nVMOVSS X0,(DX)", + "private alias": "VMOVUPS (DI),X0\nMOVQ BP,BX\nMPSADBW $0,(BX),X0\nVMOVSS X0,(DX)", + "input write": "VMOVUPS (DI),X0\nPSADBW X0,X0\nVMOVSS X0,(DI)", + "undefined join": "VMOVUPS (DI),Y0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Y1\njoined:\nVPSADBW Y0,Y1,Y2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("SAD became unproved source permission: %v", err) + } + }) + } +} + +func TestX86GoCallerSADIndependentByteDependencies(t *testing.T) { + // All selectors and widths are compared to perturbations of an independent + // numeric reference, not expectations generated by the production masks. + for _, kind := range []string{"PSADBW", "MPSADBW", "VDBPSADBW"} { + for _, width := range []int{16, 32, 64} { + if kind == "MPSADBW" && width == 64 { + continue + } + for control := 0; control < 256; control++ { + form := x86PackedSADForm{spec: amd64PackedSADSpecs[Op(kind)], immediate: uint8(control)} + form.byteWidth = width + for output := 0; output < width; output += 2 { + var first, second uint64 + for input := 0; input < width; input++ { + a, b := make([]byte, width), make([]byte, width) + a[input] = 1 + if x86SADReference(kind, width, control, a, b)[output/2] != 0 { + first |= uint64(1) << uint(input) + } + a[input], b[input] = 0, 1 + if x86SADReference(kind, width, control, a, b)[output/2] != 0 { + second |= uint64(1) << uint(input) + } + } + a, b := form.wordInputs(output) + if first != a || second != b { + t.Fatalf("%s width%d imm%d word%d: %x/%x != %x/%x", kind, width, control, output/2, a, b, first, second) + } + } + } + } + } + for _, row := range []string{"PSADBW X0,X1", "VPSADBW Y0,Y1,Y2", "MPSADBW $5,X0,X1", "VMPSADBW $37,Y0,Y1,Y2", "VDBPSADBW.Z $228,Z0,Z1,K1,Z2"} { + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedSADProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, defined := range []uint64{0, 15, 255, 65535, 0xffffffff, ^uint64(0)} { + state := &x86SourceStackState{vectors: map[Reg]uint64{"Z0": defined, "Z1": defined, "Z2": 0}, masks: map[Reg]bool{"K1": true}} + if !x86CallerSADStep(&ins, form, state, nil) { + t.Fatal("register effect failed") + } + got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)] + var want uint64 + for output := 0; output < form.byteWidth; output += 2 { + first, second := form.wordInputs(output) + if defined&first == first && defined&second == second { + want |= uint64(3) << uint(output) + } + } + if !form.legacy { + want |= ^x86CallerVectorByteMask(int64(form.byteWidth)) + } else { + want |= defined &^ x86CallerVectorByteMask(int64(form.byteWidth)) + } + if got != want { + t.Fatal(fmt.Sprintf("ordered %s: %x != %x", row, got, want)) + } + } + } +} + +func TestX86GoCallerSADCompleteNamedFamily(t *testing.T) { + for _, row := range []struct { + instruction string + result string + }{ + {"PSADBW X0,X1", "X1"}, + {"VPSADBW Y0,Y1,Y2", "X2"}, + {"MPSADBW $5,X0,X1", "X1"}, + {"VMPSADBW $37,Y0,Y1,Y2", "X2"}, + {"VDBPSADBW $228,Z0,Z1,K1,Z2", "X2"}, + } { + t.Run(row.instruction, func(t *testing.T) { + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row.instruction + "\nVMOVSS " + row.result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_caller_saved_g.go b/amd64_go_caller_saved_g.go new file mode 100644 index 000000000..f64e8f339 --- /dev/null +++ b/amd64_go_caller_saved_g.go @@ -0,0 +1,41 @@ +package plan9asm + +import "strings" + +// Incoming g is opaque, not an initialized number or caller data pointer. +// Only an exact native-width private PUSH/POP may save/restore its identity. +// All non-stack uses still consume the actual caller's byte/provenance proof. +func x86SourceGRegister(register Reg) bool { + root, valid := amd64FullRegBase(register) + return valid && root == "R14" +} + +func x86SourceGOperand(instruction Instr) bool { + for _, operand := range instruction.Args { + if operand.Kind == OpReg && x86SourceGRegister(operand.Reg) || + operand.Kind == OpMem && (x86SourceGRegister(operand.Mem.Base) || x86SourceGRegister(operand.Mem.Index)) { + return true + } + } + return false +} + +func x86CallerGDefinition(instruction Instr, state *x86SourceStackState) { + op := Op(normalizeInstructionOpcode(instruction.Op)) + if _, stack := amd64StackWidthSpecs[op]; stack { + return // Stack transfer already consumed its exact opaque/numeric cell. + } + if op == OpTEXT || op == OpLABEL || op == OpRET || op == OpJMP || isAMD64ConditionalBranch(op) || + strings.HasPrefix(string(op), "CMP") || strings.HasPrefix(string(op), "TEST") { + return + } + if count := len(instruction.Args); count != 0 { + destination := instruction.Args[count-1] + if destination.Kind == OpReg && x86SourceGRegister(destination.Reg) { + // Called only after the registered family's complete must-defined + // and pointer effects succeeded. No instruction can define the + // original g; every source return needs an exact saved token POP. + state.gOriginal = false + } + } +} diff --git a/amd64_go_caller_saved_g_runtime_test.go b/amd64_go_caller_saved_g_runtime_test.go new file mode 100644 index 000000000..52c769c4c --- /dev/null +++ b/amd64_go_caller_saved_g_runtime_test.go @@ -0,0 +1,119 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerSavedGRuntimeAssembly(assembly, mode string) string { + // Reuse independent W/L/Q LEA mathematics, including preservation of the + // initialized high bytes. The incoming g never supplies numerical data. + assembly = strings.ReplaceAll(assembly, "AX", "R14") + save, restore := "WORD $0x5641\n", "WORD $0x5e41\n" + if mode == "named" { + save, restore = "PUSHQ R14\n", "POPQ R14\n" + } else if mode == "nested" { + save = "PUSHQ R14\nMOVQ $91,R14\nPUSHQ R14\n" + restore = "POPQ R14\nPOPQ R14\n" + } + assembly = strings.ReplaceAll(assembly, "LONG $0xf8e48348\n", "LONG $0xf8e48348\n"+save) + return strings.ReplaceAll(assembly, "WORD $0x8948\nBYTE $0xec", restore+"WORD $0x8948\nBYTE $0xec") +} + +func x86CallerSavedGOriginalObserver() (goTest, assembly string) { + var source strings.Builder + source.WriteString(` +// This native observer is a separate test-only oracle, never a numerical +// input or declaration used to authorize the translated caller's source. +func savedGContext() (gvalue, spvalue, bpvalue uint64) +func TestOriginalSavedGContext(t *testing.T) { + for iteration := 0; iteration < 64; iteration++ { +`) + for _, form := range x86CallerLEARuntimeForms() { + fmt.Fprintf(&source, ` + { + x, y := []uint64{11,23}, []uint64{29,31} + beforeG, beforeSP, beforeBP := savedGContext() + _ = Call%s(x,y) + afterG, afterSP, afterBP := savedGContext() + if beforeG == 0 || beforeG != afterG || beforeSP != afterSP || beforeBP != afterBP { + t.Fatalf("%s changed native caller context g=%%x/%%x SP=%%x/%%x BP=%%x/%%x", + beforeG,afterG,beforeSP,afterSP,beforeBP,afterBP) + } + } +`, form.name, form.name) + } + source.WriteString(" }\n}\n") + return source.String(), `TEXT ·savedGContext(SB),4,$0-24 +MOVQ R14,AX +MOVQ AX,gvalue+0(FP) +MOVQ SP,AX +MOVQ AX,spvalue+8(FP) +MOVQ BP,AX +MOVQ AX,bpvalue+16(FP) +RET +` +} + +func TestCrossLinuxRuntimeMatrixX86GoCallerPrivateSavedG(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("saved-g execution requires the Linux/amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + for _, mode := range []string{"raw", "named", "nested"} { + t.Run(mode, func(t *testing.T) { + goSource, original, goTest, cOracle := x86CallerLEARuntimeSources() + assembly := x86CallerSavedGRuntimeAssembly(original, mode) + observer, observerAssembly := x86CallerSavedGOriginalObserver() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module caller\n\ngo 1.20\n", "value.go": goSource, + "value_amd64.s": assembly, "value_test.go": goTest + observer, + "context_test_amd64.s": observerAssembly, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + stage, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(stage, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", + "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(stage, command); err != nil { + t.Fatalf("actual Go saved-g numerical/context oracle: %v\n%s", err, output) + } + + pkg := x86CallerRuntimePackage(t, goSource, assembly) + ctx := llvm.NewContext() + defer ctx.Dispose() + triple := testTargetTriple(runtime.GOOS, "amd64") + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "saved_g", triple, + translation.Module.String(), cOracle, runner) + }) + } +} diff --git a/amd64_go_caller_saved_g_test.go b/amd64_go_caller_saved_g_test.go new file mode 100644 index 000000000..d917be64b --- /dev/null +++ b/amd64_go_caller_saved_g_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86GoCallerPrivateSavedGOriginalRawPushPop(t *testing.T) { + for _, form := range []struct{ name, body string }{ + // The original dot_byte_256 / hamming_512 two-byte raw words. + {"original raw", "WORD $0x5641\nMOVL $7,R14\nMOVL R14,AX\nWORD $0x5e41"}, + {"named", "PUSHQ R14\nMOVL $7,R14\nMOVL R14,AX\nPOPQ R14"}, + {"nested numerical", "PUSHQ R14\nMOVQ $19,R14\nPUSHQ R14\nMOVL $7,R14\nPOPQ R14\nMOVL R14,AX\nPOPQ R14"}, + {"nested original", "PUSHQ R14\nPUSHQ R14\nMOVL $7,R14\nPOPQ R14\nPOPQ R14\nMOVL $7,AX"}, + {"defined byte", "PUSHQ R14\nMOVL $0,AX\nMOVB $7,R14\nMOVB R14,AX\nPOPQ R14"}, + {"defined word", "PUSHQ R14\nMOVL $0,AX\nMOVW $7,R14\nMOVW R14,AX\nPOPQ R14"}, + {"defined zero", "PUSHQ R14\nXORQ R14,R14\nMOVL R14,AX\nPOPQ R14"}, + {"actual caller pointer", "PUSHQ R14\nMOVQ DI,R14\nMOVL (R14),AX\nPOPQ R14"}, + {"identical joins", "PUSHQ R14\nMOVL $7,R14\nCMPQ R14,$0\nJE joined\nMOVL $7,R14\njoined:\nMOVL R14,AX\nPOPQ R14"}, + {"both CFG arms restore", "WORD $0x5641\nMOVL $7,R14\nCMPQ R14,$0\nJE other\nWORD $0x5e41\nJMP joined\nother:\nWORD $0x5e41\njoined:\nMOVL $7,AX"}, + } { + t.Run(form.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX", form.body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } +} + +func TestX86GoCallerPrivateSavedGDoesNotExposeOpaqueIdentity(t *testing.T) { + for _, form := range []struct{ name, body string }{ + {"entry number", "MOVL R14,AX"}, + {"entry address", "MOVL (R14),AX"}, + {"entry flags", "CMPQ R14,$0\nMOVL $7,AX"}, + {"no restoration", "MOVL $7,R14\nMOVL R14,AX"}, + {"zero is not g", "XORQ R14,R14\nMOVL R14,AX"}, + {"saved then discarded", "PUSHQ R14\nMOVL $7,R14\nMOVL R14,AX\nADDQ $8,SP"}, + {"saved token to ordinary GP", "PUSHQ R14\nPOPQ BX\nMOVL $7,AX"}, + {"restored token read", "PUSHQ R14\nMOVL $7,R14\nPOPQ R14\nMOVL R14,AX"}, + {"restored token address", "PUSHQ R14\nMOVQ DI,R14\nPOPQ R14\nMOVL (R14),AX"}, + {"narrow original save", "PUSHW R14\nMOVW $7,R14\nMOVW R14,AX\nPOPW R14"}, + {"narrow restoration", "PUSHQ R14\nMOVL $7,R14\nMOVL R14,AX\nPOPW R14\nADDQ $6,SP"}, + {"unknown upper numerical bytes", "PUSHQ R14\nMOVW $7,R14\nMOVQ R14,AX\nPOPQ R14"}, + {"byte copy cannot define full result", "PUSHQ R14\nMOVB $7,R14\nMOVB R14,AX\nPOPQ R14"}, + {"word copy cannot define full result", "PUSHQ R14\nMOVW $7,R14\nMOVW R14,AX\nPOPQ R14"}, + {"opaque saved stack read", "PUSHQ R14\nMOVQ 0(SP),AX\nPOPQ R14"}, + {"opaque saved stack overwrite", "PUSHQ R14\nMOVQ $0,0(SP)\nPOPQ R14\nMOVL $7,AX"}, + {"unknown native call", "PUSHQ R14\nCALL ·unknown(SB)\nPOPQ R14\nMOVL $7,AX"}, + {"one unbalanced CFG arm", "PUSHQ R14\nMOVL $7,R14\nCMPQ R14,$0\nJE joined\nPOPQ R14\njoined:\nMOVL $7,AX"}, + {"one CFG arm loses g", "WORD $0x5641\nMOVL $7,R14\nCMPQ R14,$0\nJE other\nWORD $0x5e41\nJMP joined\nother:\nADDQ $8,SP\njoined:\nMOVL $7,AX"}, + {"g is not the input pointer", "PUSHQ R14\nMOVQ DI,R14\nMOVQ R14,(DX)\nPOPQ R14\nMOVL $7,AX"}, + } { + t.Run(form.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX", form.body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("opaque or unrestored g acquired source continuation: %v", err) + } + }) + } +} + +func TestX86GoCallerPrivateSavedGRequiresActualCallerSelection(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX", + "PUSHQ R14\nMOVL $7,R14\nMOVL R14,AX\nPOPQ R14", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + pkg.SourceSelection = nil + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bare declaration acquired numerical g permission: %v", err) + } +} + +func TestX86GoCallerPrivateSavedGRejectsUnprovedLeafFallback(t *testing.T) { + for _, body := range []string{ + "MOVQ R14,AX", + "MOVQ $7,R14\nMOVQ R14,AX", + "XORQ R14,R14\nMOVQ R14,AX", + } { + t.Run(body, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture("amd64", + body+"\nMOVQ AX,ret+8(FP)\nBYTE $0xc3") + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig) + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("text API trusted unproved incoming/clobbered g: %v", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if module != (llvm.Module{}) { + module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("module API trusted unproved incoming/clobbered g: %v", err) + } + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + }) + if translation != nil { + translation.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("declaration API trusted unproved incoming/clobbered g: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_sign_extend_test.go b/amd64_go_caller_sign_extend_test.go new file mode 100644 index 000000000..9da80c197 --- /dev/null +++ b/amd64_go_caller_sign_extend_test.go @@ -0,0 +1,56 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerPackedSignExtendIndependentInitializedNamedRole(t *testing.T) { + // This exact body was the old "signed not inherited" negative. With the + // independently registered named SX family it is a valid initialized + // program, not permission inherited from the still-disjoint ZX registry. + const body = "VMOVUPS (DI),X0\nVPMOVSXBD X0,Y1\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + ins := x86ScalarLaneInstruction(t, "VPMOVSXBD X0,Y1") + if _, known, _ := x86PackedZeroExtendProperties(ins, "amd64"); known || x86RawZeroExtendSource(ins, "amd64") { + t.Fatal("genuine named SX silently entered the ZX membership or raw identity") + } + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("original complete initialized signed caller program rejected", err) + } +} + +func TestX86GoCallerPackedSignExtendPartialSignByteKnowledge(t *testing.T) { + for _, ratio := range x86PackedSignExtendOracleRatios { + for _, mode := range []string{"signed-known", "signed-missing", "zero-known-independent"} { + t.Run(ratio.suffix+"/"+mode, func(t *testing.T) { + at, stem := ratio.input-1, "VPMOVSX" + if mode != "signed-known" { + at++ // Only an unrelated byte is available; the first lane's sign byte is absent. + } + if mode == "zero-known-independent" { + stem = "VPMOVZX" + } + // Observe only the first lane's first extension byte. SX depends + // on its actual last input byte even when lower bytes are absent; + // ZX defines the same extension byte independently as zero. + body := fmt.Sprintf("PINSRB $%d,AX,X0\n%s%s X0,X1\nPEXTRB $%d,X1,AX\nMOVL AX,(DX)", at, stem, ratio.suffix, ratio.input) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if mode == "signed-missing" { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a missing sign byte acquired a numerical caller result", err) + } + } else if err != nil { + t.Fatal("independent selected sign/constant-zero byte control rejected", err) + } + }) + } + } +} diff --git a/amd64_go_caller_signed_integer_float.go b/amd64_go_caller_signed_integer_float.go new file mode 100644 index 000000000..45a4623b4 --- /dev/null +++ b/amd64_go_caller_signed_integer_float.go @@ -0,0 +1,20 @@ +package plan9asm + +func x86CallerSignedIntegerFloatStep(ins *Instr, f x86SignedIntegerFloatForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if f.memory >= 0 { + if !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.sourceBits/8), false) { + return false + } + } else if !x86CallerNumericRead(ins.Args[0], int64(f.sourceBits/8), state) { + return false + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + low := x86CallerVectorByteMask(int64(f.spec.resultBits / 8)) + if f.spec.legacy { + state.vectors[destination] |= low + } else { + upper := state.vectors[x86CallerVectorRoot(ins.Args[f.upper].Reg)] + state.vectors[destination] = ^x86CallerVectorByteMask(16) | low | upper&(x86CallerVectorByteMask(16)&^low) + } + return true +} diff --git a/amd64_go_caller_signed_integer_float_runtime_test.go b/amd64_go_caller_signed_integer_float_runtime_test.go new file mode 100644 index 000000000..f87f2ed5e --- /dev/null +++ b/amd64_go_caller_signed_integer_float_runtime_test.go @@ -0,0 +1,69 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86SignedIntegerScalarFloatActualCaller(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + for _, op := range []string{"CVTSL2SS", "CVTSL2SD", "CVTSQ2SS", "CVTSQ2SD", "VCVTSI2SSL", "VCVTSI2SDL", "VCVTSI2SSQ", "VCVTSI2SDQ"} { + t.Run(op, func(t *testing.T) { + goSource, bits, resultBits := x86CallerFixtureSource, 32, 32 + load, store, args := "MOVL (DI),AX", "MOVSS X1,(DX)", "AX,X1" + if strings.Contains(op, "Q") { + bits, load = 64, "MOVQ (DI),AX" + } + if strings.Contains(op, "SD") { + resultBits, store = 64, "MOVSD X1,(DX)" + goSource = strings.Replace(goSource, ") float32 {", ") float64 {", 1) + goSource = strings.Replace(goSource, "var res float32", "var res float64", 1) + } + if strings.HasPrefix(op, "V") { + args = "AX,X0,X1" + } + body := load + "\n" + op + " " + args + "\n" + store + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, goSource, assembly) + ctx := llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "kernel_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + main := x86SignedIntegerFloatSSEOracle + fmt.Sprintf(` +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +extern void kernel(const void*,const void*,void*,const void*) __asm__(LABEL("caller.kernel")); +int main(void){ + for(unsigned sample=0;sample<256;sample++)for(unsigned mode=0;mode<4;mode++){ + uint64_t a[2]={UINT64_C(0xd31733a6287fbbc3)*sample+UINT64_C(0x1000001),UINT64_C(0x789abcdef1234567)}; + uint64_t b[2]={12,13},aCopy[2],bCopy[2];memcpy(aCopy,a,sizeof(a));memcpy(bCopy,b,sizeof(b)); + uint8_t out[16];memset(out,0xc3,sizeof(out));uint64_t length=4; + unsigned csr=0x9fc5|(mode<<13),after;uint64_t want=sse_convert(a[0],%d,%d,csr);unsigned expectedCSR=sse_csr; + __asm__ volatile("ldmxcsr %%0"::"m"(csr):"memory");kernel(a,b,out+1,&length); + __asm__ volatile("stmxcsr %%0":"=m"(after));uint64_t got=0;memcpy(&got,out+1,%d); + if(got!=want||after!=expectedCSR){fprintf(stderr,"actual caller %s input=%%llx got=%%llx want=%%llx csr=%%x want=%%x\n",(unsigned long long)a[0],(unsigned long long)got,(unsigned long long)want,after,expectedCSR);return 1;} + for(unsigned i=0;i%d)&&out[i]!=0xc3)return 2; + if(memcmp(a,aCopy,sizeof(a))||memcmp(b,bCopy,sizeof(b))||length!=4)return 3; + } + unsigned csr=0x1f80;__asm__ volatile("ldmxcsr %%0"::"m"(csr));return 0; +} +`, bits, resultBits, resultBits/8, op, resultBits/8) + compileAndRunRuntimeTestForTarget(t, llc, clang, "signed-float-caller", triple, x86SignedIntegerFloatOptimize(t, translation.Module.String()), main, prefix) + }) + } +} diff --git a/amd64_go_caller_signed_integer_float_test.go b/amd64_go_caller_signed_integer_float_test.go new file mode 100644 index 000000000..708f54922 --- /dev/null +++ b/amd64_go_caller_signed_integer_float_test.go @@ -0,0 +1,93 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerSignedIntegerFloatCompleteFamily(t *testing.T) { + for _, row := range x86SignedIntegerFloatRows("amd64") { + row = strings.ReplaceAll(strings.Replace(row, "8(BX)", "(DI)", 1), "(BX)(SI*4)", "(DI)") + body := "VMOVUPS (SI),Z0\nVMOVUPS (SI),Z1\n" + row + "\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("%s: %v", row, err) + } + } +} + +func TestX86GoCallerSignedIntegerFloatByteKnowledge(t *testing.T) { + for _, row := range x86SignedIntegerFloatRows("amd64") { + if strings.Contains(row, "(BX)") { + continue + } + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + f, _, err := x86SignedIntegerFloatProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for known := uint8(0); ; known++ { + state := x86SourceStackState{known: map[Reg]uint8{AX: known}, callers: map[Reg]x86CallerPointerValue{}, vectors: map[Reg]uint64{"Z0": 0x5500, "Z1": 0xaaff0000}, masks: map[Reg]bool{}} + accepted := x86CallerSignedIntegerFloatStep(&ins, f, &state, nil) + mask := uint8((1 << uint(f.sourceBits/8)) - 1) + if accepted != (known&mask == mask) { + t.Fatalf("%s GP knowledge %x: %t", row, known, accepted) + } + if accepted { + low := x86CallerVectorByteMask(int64(f.spec.resultBits / 8)) + want := uint64(0xaaff0000) | low + if !f.spec.legacy { + want = ^uint64(0xffff) | low | (0x5500 & (uint64(0xffff) &^ low)) + } + if state.vectors["Z1"] != want { + t.Fatalf("%s invented upper byte knowledge: %x want %x", row, state.vectors["Z1"], want) + } + } + if known == 255 { + break + } + } + } +} + +func TestX86GoCallerSignedIntegerFloatUnknownPermissionsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "pointer bits": "VCVTSI2SSL DI,X0,X1\nVMOVSS X1,(DX)", + "stack bits": "CVTSQ2SS SP,X1\nVMOVSS X1,(DX)", + "g bits": "VCVTSI2SSL R14,X0,X1\nVMOVSS X1,(DX)", + "undefined GP": "VCVTSI2SSQ R11,X0,X1\nVMOVSS X1,(DX)", + "partial GP": "MOVB $7,R11\nCVTSL2SS R11,X1\nVMOVSS X1,(DX)", + "source SP": "VCVTSI2SSL (SP),X0,X1\nVMOVSS X1,(DX)", + "unknown memory": "VCVTSI2SSL (R11),X0,X1\nVMOVSS X1,(DX)", + "pointer FP": "VCVTSI2SSL a+0(FP),X0,X1\nVMOVSS X1,(DX)", + "unknown upper passthrough": "VCVTSI2SSL AX,X0,X1\nVEXTRACTPS $3,X1,AX\nMOVL AX,(DX)", + "CFG undefined GP": "CMPQ AX,$0\nJE join\nMOVL $7,R11\njoin:\nVCVTSI2SSL R11,X0,X1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown permission became success: %v", err) + } + }) + } + // A numerical low result is independent of undefined passthrough bytes. + for _, op := range []string{"CVTSL2SS", "VCVTSI2SSL"} { + args := "AX,X1" + if strings.HasPrefix(op, "V") { + args = "AX,X0,X1" + } + body := fmt.Sprintf("%s %s\nVMOVSS X1,(DX)", op, args) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + } +} diff --git a/amd64_go_caller_signed_packed_float.go b/amd64_go_caller_signed_packed_float.go new file mode 100644 index 000000000..35f30ff85 --- /dev/null +++ b/amd64_go_caller_signed_packed_float.go @@ -0,0 +1,39 @@ +package plan9asm + +func x86CallerSignedPackedFloatStep(ins *Instr, f x86SignedPackedFloatForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + // Virtual MMX has no complete public caller x87/tag/physical alias contract. + if f.mmx || f.mask >= 0 && !state.masks[ins.Args[f.mask].Reg] { + return false + } + if f.memory >= 0 { + source := ins.Args[f.memory] + if source.Kind == OpFP || source.Kind == OpMem && + (source.Mem.Base == SP || source.Mem.Base == BP || source.Mem.Index == BP) { + return false + } + // The full potential tuple, even for inactive masks, is read-only input. + // The actual native opcode retains lane fault suppression; initialized + // masks do not authorize missing bytes or writable input pointers. + if !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.memoryBytes), false) { + return false + } + } else { + input := state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] + mask := x86CallerVectorByteMask(int64(f.inputBytes)) + if input&mask != mask { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + width := x86CallerVectorByteMask(int64(f.byteWidth)) + old := state.vectors[destination] + if f.mask >= 0 && !f.zeroing && old&width != width { + return false + } + if f.spec.legacy { + state.vectors[destination] = old&^width | width + } else { + state.vectors[destination] = ^uint64(0) + } + return true +} diff --git a/amd64_go_caller_signed_packed_float_test.go b/amd64_go_caller_signed_packed_float_test.go new file mode 100644 index 000000000..26dfafee9 --- /dev/null +++ b/amd64_go_caller_signed_packed_float_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86SignedPackedIntegerFloatCallerCompleteInitializedForms(t *testing.T) { + for _, op := range []string{"CVTPL2PS", "CVTPL2PD", "VCVTDQ2PS", "VCVTDQ2PD"} { + banks := []string{"X"} + if strings.HasPrefix(op, "V") { + banks = []string{"X", "Y", "Z"} + } + for _, bank := range banks { + src := bank + if strings.HasSuffix(op, "PD") { + src = "X" + if bank == "Z" { + src = "Y" + } + } + modes := []string{""} + if strings.HasPrefix(op, "V") { + modes = []string{"", "K1,", "K7,"} + } + for _, input := range []string{src + "0", "(DI)"} { + for _, mode := range modes { + name := op + if mode == "K7," { + name += ".Z" + } + row := fmt.Sprintf("%s %s,%s%s1", name, input, mode, bank) + t.Run(row, func(t *testing.T) { + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nKMOVQ AX,K7\n" + row + "\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } + } + } +} + +func TestX86SignedPackedIntegerFloatCallerRetainsFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown input": "VCVTDQ2PS Y0,Y1\nVMOVSS X1,(DX)", + "partial dword": "MOVSS (DI),X2\nMOVSS X2,X0\nCVTPL2PS X0,X1\nVMOVSS X1,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVCVTDQ2PS Z0,K1,Z1\nVMOVSS X1,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVCVTDQ2PS.Z Z0,K1,Z1\nVMOVSS X1,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVCVTDQ2PS.Z Z0,K1,Z1\nVMOVSS X1,(DX)", + "unknown base": "VCVTDQ2PD (BX),Y1\nVMOVSS X1,(DX)", + "private stack": "VCVTDQ2PS (SP),Y1\nVMOVSS X1,(DX)", + "FP pointer data": "CVTPL2PD a+0(FP),X1\nVMOVSS X1,(DX)", + "zero mask envelope": "MOVL $0,AX\nKMOVQ AX,K1\nVCVTDQ2PS.Z (DX),K1,Z1\nVMOVSS X1,(DX)", + "source write": "VCVTDQ2PS (DI),Y1\nVMOVSS X1,(DI)", + "wide result": "VCVTDQ2PD (DI),Y1\nVMOVUPS Y1,(DX)", + "join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),Y0\njoined:\nVCVTDQ2PS Y0,Y1\nVMOVSS X1,(DX)", + "unknown upper alias": "VMOVUPS (DI),X0\nCVTPL2PS X0,X1\nVCVTDQ2PS Y1,Y2\nVMOVSS X2,(DX)", + "MMX transport": "MOVQ (DI),M0\nCVTPL2PS M0,X1\nVMOVSS X1,(DX)", + "unsigned unknown upper": "MOVOU (DI),X0\nVCVTUDQ2PS Y0,Y1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved conversion got caller permission: %v", err) + } + }) + } +} + +func TestX86SignedPackedIntegerFloatGPTransferDefinesCompleteXInput(t *testing.T) { + // MOVQ GP -> X initializes the low qword and zeroes all remaining X + // bytes. In contrast, the register MOVSS negative above preserves the + // unknown upper twelve bytes of its destination. + body := "MOVQ AX,X0\nCVTPL2PS X0,X1\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("complete GP/X input lost its caller-defined bytes", err) + } +} + +func TestX86SignedPackedIntegerFloatIndependentByteEffects(t *testing.T) { + for _, row := range x86SignedPackedFloatRows("amd64") { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + f, _, err := x86SignedPackedFloatProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + if f.memory >= 0 || f.mmx { + continue + } + for missing := -1; missing < f.inputBytes; missing++ { + input := x86CallerVectorByteMask(int64(f.inputBytes)) + if missing >= 0 { + input &^= uint64(1) << uint(missing) + } + for _, old := range []uint64{0, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{x86CallerVectorRoot(ins.Args[0].Reg): input}, masks: map[Reg]bool{"K1": true, "K3": true, "K7": true}} + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + if destination != x86CallerVectorRoot(ins.Args[0].Reg) { + state.vectors[destination] = old + } else { + old = input + } + valid := missing < 0 && (f.mask < 0 || f.zeroing || old&x86CallerVectorByteMask(int64(f.byteWidth)) == x86CallerVectorByteMask(int64(f.byteWidth))) + if got := x86CallerSignedPackedFloatStep(&ins, f, &state, nil); got != valid { + t.Fatalf("%s missing%d merge%016x got%t want%t", row, missing, old, got, valid) + } + if valid { + want := ^uint64(0) + if f.spec.legacy { + want = old | x86CallerVectorByteMask(16) + } + if state.vectors[destination] != want { + t.Fatal("alias/upper byte effect", row) + } + } + } + } + } +} diff --git a/amd64_go_caller_sqrt.go b/amd64_go_caller_sqrt.go new file mode 100644 index 000000000..a459d71d4 --- /dev/null +++ b/amd64_go_caller_sqrt.go @@ -0,0 +1,26 @@ +package plan9asm + +func x86CallerSqrtStep(ins *Instr, f x86FloatBinaryForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if f.bits == 16 { + return false // Raw FP16 retains its separate original-source continuation. + } + if f.mask >= 0 && !state.masks[ins.Args[f.mask].Reg] { + return false + } + input := state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] + if f.memory >= 0 { + // Ordinary caller bounds do not establish legacy m128 alignment. + if f.legacy && !f.scalar || !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.memoryBytes), false) { + return false + } + input = ^uint64(0) + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + upper := state.vectors[x86CallerVectorRoot(ins.Args[f.second].Reg)] + value, valid := x86SqrtDefinedBytes(f, input, upper, state.vectors[destination]) + if !valid { + return false + } + state.vectors[destination] = value + return true +} diff --git a/amd64_go_caller_unpack.go b/amd64_go_caller_unpack.go new file mode 100644 index 000000000..985082b1c --- /dev/null +++ b/amd64_go_caller_unpack.go @@ -0,0 +1,29 @@ +package plan9asm + +func x86CallerPackedUnpackStep(ins *Instr, form x86PackedUnpackForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := uint64(0) + if form.memory >= 0 { + // Legacy m128 needs native alignment, not just a slice-data identity. + // No initialized mask shrinks an E4NF read's complete native extent. + if form.legacy || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } else { + first = state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + known := x86PackedUnpackDefinedBytes(form, first, second, old) + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_unpack_test.go b/amd64_go_caller_unpack_test.go new file mode 100644 index 000000000..f7311d298 --- /dev/null +++ b/amd64_go_caller_unpack_test.go @@ -0,0 +1,278 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerUnpackOpcodes() []string { + return []string{ + "PUNPCKLBW", "PUNPCKHBW", "PUNPCKLWL", "PUNPCKHWL", + "PUNPCKLLQ", "PUNPCKHLQ", "PUNPCKLQDQ", "PUNPCKHQDQ", + "UNPCKLPS", "UNPCKHPS", "UNPCKLPD", "UNPCKHPD", + "VPUNPCKLBW", "VPUNPCKHBW", "VPUNPCKLWD", "VPUNPCKHWD", + "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ", + "VUNPCKLPS", "VUNPCKHPS", "VUNPCKLPD", "VUNPCKHPD", + } +} + +func TestX86GoCallerUnpackCompleteInitializedGrammar(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + for _, original := range x86UnpackAcceptedForms(opcode, "amd64") { + t.Run(original, func(t *testing.T) { + row := strings.NewReplacer("8(AX)", "8(DI)", "X15", "X2", "X31", "X2", "Y31", "Y2", "Z31", "Z2").Replace(original) + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nKMOVQ AX,K1\nKMOVQ AX,K7\n" + row + "\nVMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + needsContext := strings.Contains(row, "M7") || !strings.HasPrefix(opcode, "V") && strings.Contains(row, "(DI)") + if needsContext && !errors.Is(err, ErrProbeNeedsContext) || !needsContext && err != nil { + t.Fatalf("Context=%t: %v", needsContext, err) + } + }) + } + } +} + +func TestX86GoCallerUnpackRetainsUnknownAndMemoryFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),X1\nVPUNPCKLBW X0,X1,X2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),X0\nVPUNPCKLBW X0,X1,X2\nVMOVSS X2,(DX)", + "unknown selected high bytes": "MOVSS (DI),X0\nMOVSS X0,X1\nVUNPCKHPS X0,X1,X2\nVMOVSS X2,(DX)", + "unknown destructive input": "VMOVUPS (DI),X0\nPUNPCKLBW X0,X1\nVMOVSS X1,(DX)", + "unknown merge": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nKMOVQ AX,K1\nVPUNPCKLBW X0,X1,K1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVPUNPCKLBW.Z X0,X1,K1,X2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nKMOVQ DI,K1\nVPUNPCKLBW.Z X0,X1,K1,X2\nVMOVSS X2,(DX)", + "zero mask still full E4NF read": "VMOVUPS (DI),X0\nMOVL $0,AX\nKMOVQ AX,K1\nVPUNPCKLDQ.Z (DX),X0,K1,X1\nVMOVSS X1,(DX)", + "zero mask broadcast still full read": "VMOVUPS (DI),X0\nMOVL $0,AX\nKMOVQ AX,K1\nVPUNPCKLQDQ.BCST.Z (DX),X0,K1,X1\nVMOVSS X1,(DX)", + "legacy alignment absent": "VMOVUPS (DI),X0\nPUNPCKLBW (DI),X0\nVMOVSS X0,(DX)", + "private stack": "VMOVUPS (DI),X0\nVPUNPCKLBW (SP),X0,X0\nVMOVSS X0,(DX)", + "unknown address": "VMOVUPS (DI),X0\nVPUNPCKLBW (BX),X0,X0\nVMOVSS X0,(DX)", + "read beyond scalar length": "VMOVUPS (DI),X0\nVPUNPCKLDQ (CX),X0,X0\nVMOVSS X0,(DX)", + "source write": "VMOVUPS (DI),X0\nVPUNPCKLBW X0,X0,X0\nVMOVSS X0,(DI)", + "wide result": "VMOVUPS (DI),X0\nVPUNPCKLBW X0,X0,X0\nVMOVUPS X0,(DX)", + "undefined CFG join": "VMOVUPS (DI),X0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),X1\njoined:\nVPUNPCKLBW X0,X1,X2\nVMOVSS X2,(DX)", + "MMX is a separate bank": "PUNPCKLBW M0,M1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved interleave acquired caller permission: %v", err) + } + }) + } +} + +func TestX86GoCallerUnpackIndependentByteKnowledge(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + widths := []int{16} + if strings.HasPrefix(opcode, "V") { + widths = []int{16, 32, 64} + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, destination := range []int{0, 1, 2} { + if !strings.HasPrefix(opcode, "V") && destination != 1 { + continue + } + for _, mode := range []string{"unmasked", "merge", "zero"} { + if !strings.HasPrefix(opcode, "V") && mode != "unmasked" { + continue + } + row := opcode + " " + bank + "0," + bank + "1" + if strings.HasPrefix(opcode, "V") { + if mode != "unmasked" { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + if mode == "zero" { + row = strings.Replace(row, opcode, opcode+".Z", 1) + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86PackedUnpackProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatalf("%s: %v", row, err) + } + for seed := 0; seed < 64; seed++ { + initial := []uint64{0x19ca786fe32a907b, 0xf6210c89d8ba3547, 0x4a3b27e806c591df} + for index := range initial { + initial[index] ^= uint64(seed+index) * 0x102030405060708 + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": initial[0], "Z1": initial[1], "Z2": initial[2]}, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedUnpackStep(&ins, form, &state, nil) { + t.Fatalf("register effect refused %s", row) + } + // Calculate each output byte from the source spelling and + // Intel's independent per-128-bit interleave definition. + var want uint64 + lane := x86UnpackTestLaneBytes(opcode) + high := strings.Contains(opcode, "UNPCKH") + for output := 0; output < 64; output++ { + known := false + if output >= width { + known = strings.HasPrefix(opcode, "V") || initial[destination]>>uint(output)&1 != 0 + } else { + pair := (output % 16) / (2 * lane) + sourceByte := output/16*16 + pair*lane + output%lane + if high { + sourceByte += 8 + } + sourceRegister := 1 + if output/lane%2 != 0 { + sourceRegister = 0 + } + known = initial[sourceRegister]>>uint(sourceByte)&1 != 0 + if mode == "merge" { + known = known && initial[destination]>>uint(output)&1 != 0 + } + } + if known { + want |= uint64(1) << uint(output) + } + } + if got := state.vectors[Reg(fmt.Sprintf("Z%d", destination))]; got != want { + t.Fatalf("%s seed=%d: byte knowledge %#x want %#x", row, seed, got, want) + } + } + } + } + } + } +} + +func TestX86GoCallerUnpackCompleteFamilyThreeAPIs(t *testing.T) { + var body strings.Builder + body.WriteString("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\n") + for _, opcode := range x86CallerUnpackOpcodes() { + if strings.HasPrefix(opcode, "V") { + fmt.Fprintf(&body, "%s Z0,Z1,Z2\nVMOVSS X2,(DX)\n", opcode) + } else { + fmt.Fprintf(&body, "%s X0,X1\nVMOVSS X1,(DX)\n", opcode) + } + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body.String(), 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "unpack.ll", "unpack.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, triple, "unpack-module.ll", "unpack-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerUnpackMemoryCertificatesBindFullE4NFSource(t *testing.T) { + for _, row := range []string{"VPUNPCKLBW (DI),X0,X2", "VUNPCKHPD.BCST.Z (DI),Z0,K1,Z2"} { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n"+row+"\nVMOVSS X2,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, recognized, err := x86PackedUnpackProperties(ins, "amd64") + if !recognized || err != nil || form.memory < 0 { + continue + } + found = true + index := form.memory + access, exists := ins.x86CallerMemory[index] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(ins, index) { + t.Fatal("missing full-width native read certificate") + } + for _, change := range []string{"opcode", "width", "address", "destination", "direction", "broadcast", "mask"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VPUNPCKHBW" + case "width": + access.width-- + mutated.x86CallerMemory[index] = access + case "address": + mutated.Args[index].Mem.Off++ + case "destination": + mutated.Args[form.destination].Reg = "X3" + case "direction": + access.write = true + mutated.x86CallerMemory[index] = access + case "broadcast": + mutated.Op = Op(string(mutated.Op) + ".BCST") + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K2" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K2"}) + } + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s mutation reused a stale interleave certificate", change) + } + } + } + if !found { + t.Fatal("no interleave instruction consumed its caller proof") + } + } +} + +func TestX86GoCallerUnpackCompleteFamilyOriginalSource(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + t.Run(opcode, func(t *testing.T) { + operands, destination := "X0,X1", "X1" + if strings.HasPrefix(opcode, "V") { + operands, destination = "Y0,Y1,Y2", "X2" + } + body := fmt.Sprintf("VMOVUPS (DI),Y0\nVMOVUPS (SI),Y1\n%s %s\nVMOVSS %s,(DX)", opcode, operands, destination) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_caller_vector_moves.go b/amd64_go_caller_vector_moves.go new file mode 100644 index 000000000..a532a3a18 --- /dev/null +++ b/amd64_go_caller_vector_moves.go @@ -0,0 +1,545 @@ +package plan9asm + +import "fmt" + +func x86CallerVectorMoveKnown(ins Instr) bool { + if _, known, err := x86Permute128Properties(ins.Op, ins, "amd64"); known { + return err == nil && x86ResultNamedPredicateInstruction(ins) && !ins.x86Permute128VEX + } + if _, known, err := x86PackedDotProperties(ins.Op, ins, "amd64"); known { + return err == nil && x86ResultNamedPredicateInstruction(ins) + } + if _, known, err := x86ExplicitMaskMoveProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if _, known, err := x86PackedAlignProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if form, known, err := x86SqrtProperties(ins, "amd64"); known { + return err == nil && form.bits != 16 + } + if _, known, err := x86ByteShuffleProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if _, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if form, known, err := x86PackedConversionProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86CLMULProperties(ins.Op, ins, "amd64"); known { + return err == nil && form.memory < 0 + } + if _, known, err := x86LegacyPackedMoveProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if _, known, err := x86ScalarLaneProperties(ins, "amd64"); known { + return err == nil + } + if _, known, err := x86AESProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if form, known, err := x86HorizontalIntegerProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86GPSIMDTransferProperties(ins, "amd64"); known { + return err == nil && !form.mmx && !form.spec.rawOnly + } + if form, known, err := x86ImmediateShuffleProperties(ins, "amd64"); known { + return err == nil && !form.spec.mmx + } + if _, known, err := x86PackedFloatShuffleProperties(ins.Op, ins, "amd64"); known { + return err == nil && x86ResultNamedPredicateInstruction(ins) + } + if form, known, err := x86IntegerMinMaxProperties(ins.Op, ins, "amd64"); known && form.legacy { + return err == nil && form.memory < 0 && x86ResultNamedPredicateInstruction(ins) + } + if _, known, err := x86MaskVectorProperties(ins, "amd64"); known { + return err == nil + } + if form, known, err := x86PackedNarrowProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86PackedWordMultiplyProperties(ins.Op, ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86DwordProductProperties(ins.Op, ins, "amd64"); known { + return err == nil && !form.mmx + } + if _, known, err := x86PackedSADProperties(ins, "amd64"); known { + return err == nil + } + if form, known, err := x86PackedSubtractProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if _, recognized, err := x86FloatCompareProperties(ins, "amd64"); recognized { + return err == nil + } + if form, known, err := x86PackedWrappingAddProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86PackedMultiplyAddProperties(ins.Op, ins, "amd64"); known { + return err == nil && !form.mmx + } + if f, known, err := x86SignedPackedFloatProperties(ins, "amd64"); known { + return err == nil && !f.mmx + } + if _, recognized, err := x86FloatArithmeticProperties(ins, "amd64", ""); recognized { + return err == nil + } + if form, recognized, err := x86PackedExtendProperties(ins, "amd64"); recognized { + return err == nil && (!form.signed || x86ResultNamedPredicateInstruction(ins)) + } + if _, recognized, err := x86VectorZeroProperties(ins, "amd64"); recognized { + return err == nil + } + if _, recognized, err := x86SignedIntegerFloatProperties(ins, "amd64"); recognized { + return err == nil + } + if f, recognized, err := x86InsertExtractProperties(ins, "amd64"); recognized { + return err == nil && !(f.mask >= 0 && f.memoryWrite) + } + if form, recognized, err := x86PackedShiftProperties(ins.Op, ins, "amd64"); recognized { + return err == nil && !form.mmx + } + if _, recognized, err := x86HorizontalFloatProperties(ins, "amd64"); recognized { + return err == nil + } + if form, recognized, err := x86PackedUnpackProperties(ins, "amd64"); recognized { + return err == nil && !form.mmx + } + if form, recognized, err := x86PackedLogicalProperties(ins, "amd64"); recognized { + return err == nil && !form.mmx + } + if amd64MaskMoveOperandsValid(ins, "amd64") { + return true + } + if form, recognized, err := amd64PackedVectorMoveProperties(ins, "amd64"); recognized { + return err == nil && !form.undefinedOpcode() + } + if form, recognized, err := amd64FMA3Properties(ins, "amd64"); recognized { + return err == nil && !form.spec.rawOnly + } + form, recognized, err := amd64ScalarVectorMoveProperties(ins, "amd64") + // The shared lowerer also models raw-only FP16 moves. Their caller-bank + // permission needs its own complete source/runtime family, not inheritance + // from the reviewed Go-named single/double-precision forms. + return recognized && err == nil && !form.spec.rawOnly +} + +func x86CloneCallerVectorState(vectors map[Reg]uint64, masks map[Reg]bool) (map[Reg]uint64, map[Reg]bool) { + vectorCopy := make(map[Reg]uint64, len(vectors)) + for reg, known := range vectors { + vectorCopy[reg] = known + } + maskCopy := make(map[Reg]bool, len(masks)) + for reg, known := range masks { + maskCopy[reg] = known + } + return vectorCopy, maskCopy +} + +func x86CallerVectorRoot(reg Reg) Reg { + index, _ := amd64VectorRegisterIndex(reg, amd64VectorByteWidth(reg)) + return Reg(fmt.Sprintf("Z%d", index)) +} + +func x86CallerVectorByteMask(width int64) uint64 { + if width <= 0 || width > 64 { + return 0 + } + if width == 64 { + return ^uint64(0) + } + return uint64(1)<= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := int64(form.spec.bits / 8) + lowMask := x86CallerVectorByteMask(width) + if form.memory >= 0 { + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, width, form.memoryWrite) { + return false + } + if form.memoryWrite { + source := x86CallerVectorRoot(ins.Args[form.low].Reg) + return state.vectors[source]&lowMask == lowMask + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.spec.legacy || form.legacyZero { + if form.memory >= 0 || form.literal { + // A legacy memory load zeroes the rest of XMM but preserves the + // upper Y/Z aliases. Register copies preserve the old upper XMM. + state.vectors[destination] = old | x86CallerVectorByteMask(16) + } else { + source := x86CallerVectorRoot(ins.Args[form.low].Reg) + state.vectors[destination] = old&^lowMask | state.vectors[source]&lowMask + } + return true + } + known := ^uint64(0) + if form.memory < 0 && !form.literal { + low := state.vectors[x86CallerVectorRoot(ins.Args[form.low].Reg)] & lowMask + upperMask := x86CallerVectorByteMask(16) &^ lowMask + upper := state.vectors[x86CallerVectorRoot(ins.Args[form.upper].Reg)] & upperMask + known = ^x86CallerVectorByteMask(16) | low | upper + } + if form.mask >= 0 && !form.zeroing { + // The mask is initialized, not assumed all-one: either active data + // or the old low element may survive. Higher lanes follow the move + // form's upper merge/zero rules independently of that mask bit. + known = known&^lowMask | known&old&lowMask + } + state.vectors[destination] = known + return true +} + +func x86CallerMaskMoveStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + bits := amd64MaskMoveSpecs[ins.Op] + width := int64(bits / 8) + source, destination := ins.Args[0], ins.Args[1] + if source.Kind == OpMem { + if !x86CallerOrdinaryMemoryStep(ins, 0, state, proof, width, false) { + return false + } + } else if amd64IsKOperand(source) { + if !state.masks[source.Reg] { + return false + } + } else if !x86CallerNumericRead(source, width, state) { + return false // In particular, GP pointer bits cannot become mask data. + } + if destination.Kind == OpMem { + return x86CallerOrdinaryMemoryStep(ins, 1, state, proof, width, true) + } + if amd64IsKOperand(destination) { + state.masks[destination.Reg] = true + return true + } + // All KMOV GP destinations zero-extend the transferred value, including + // B/W/D, so the complete GP result is numeric; SP/BP/g stay forbidden. + return x86CallerNumericWrite(destination.Reg, 8, state) +} + +func x86CallerMoveMemoryShape(ins Instr, index int) (width int64, write, valid bool) { + return x86InstructionMemoryShape(ins, index, "amd64") +} + +func x86InstructionMemoryShape(ins Instr, index int, arch string) (width int64, write, valid bool) { + if f, known, err := x86SingleSourceNarrowProperties(ins.Op, ins, arch); known { + return int64(f.outputBytes), true, err == nil && !f.undefinedOpcode() && f.memory >= 0 && f.memory == index + } + if form, known, err := x86PackedPrecisionProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86ExplicitMaskMoveProperties(ins.Op, ins, arch); known { + return int64(form.byteWidth), !form.load, err == nil && form.memory == index + } + if form, known, err := x86SqrtProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory >= 0 && form.memory == index + } + if known, _ := x86SizedNOPProperties(ins, arch); known { + return 0, false, false // An encoding-only ModRM operand grants no access or caller envelope. + } + if form, known, err := x86NonTemporalVectorProperties(ins.Op, ins, arch); known { + return int64(form.move.byteWidth), !form.spec.load, + err == nil && !form.trap && form.move.memory == index + } + if form, known, err := x86CRC32Properties(ins.Op, ins, arch); known { + return int64(form.bits / 8), false, err == nil && index == 0 && isAMD64MemoryOperand(form.source) + } + if form, known, err := x86ScalarBitFieldProperties(ins.Op, ins, arch); known { + return int64(form.bits / 8), false, err == nil && index == 1 && isAMD64MemoryOperand(form.source) + } + if form, known, err := x86BLSProperties(ins.Op, ins, arch); known { + return int64(form.spec.bits / 8), false, err == nil && index == 0 && isAMD64MemoryOperand(form.source) + } + if form, known, err := x86ByteShuffleProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86CLMULProperties(ins.Op, ins, arch); known { + return int64(form.byteWidth), false, err == nil && form.memory >= 0 && index == form.memory + } + if form, known, err := x86ExplicitIMULProperties(ins.Op, ins, arch); known { + sourceIndex := 0 + if form.three { + sourceIndex = 1 + } + return int64(form.bits / 8), false, err == nil && index == sourceIndex && isAMD64MemoryOperand(form.source) + } + if form, known, err := x86AESProperties(ins.Op, ins, arch); known { + return int64(form.byteWidth), false, err == nil && form.memory >= 0 && form.memory == index + } + if form, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch); known { + return int64(form.bits / 8), false, err == nil && index == 0 && isAMD64MemoryOperand(form.source) + } + if form, known, err := x86ScalarLaneProperties(ins, arch); known { + return int64(form.bits / 8), !form.insert, err == nil && form.memory == index + } + if form, known, err := x86HorizontalIntegerProperties(ins, arch); known { + return int64(form.byteWidth), false, err == nil && !form.mmx && form.spec.vector && form.memory == index + } + if form, known, err := x86GPSIMDTransferProperties(ins, arch); known { + return int64(form.bits / 8), form.memoryWrite, err == nil && !form.mmx && form.memory == index + } + if form, recognized, err := x86LegacyPackedMoveProperties(ins.Op, ins, arch); recognized { + return 16, form.memoryWrite, err == nil && x86Mode32VectorRegisterContext(ins, arch) == nil && form.memory == index + } + if form, recognized, err := x86LegacyIntegerArithmeticProperties(ins.Op, ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if index < 0 || index >= len(ins.Args) || !isAMD64MemoryOperand(ins.Args[index]) { + return 0, false, false + } + if form, known, err := x86ImmediateShuffleProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.spec.legacy && form.memory == index + } + if form, known, err := x86PackedFloatShuffleProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.legacy && form.memory == index + } + if form, known, err := x86PackedAlignProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.legacy && form.memory == index + } + if form, known, err := x86Permute128Properties(ins.Op, ins, arch); known { + return 32, false, err == nil && form.memory == index + } + if form, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86PackedPhysicalProperties(ins.Op, ins, arch, ""); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index && + (!form.mmx || form.spec.family == x86PackedAverage) + } + if form, known, err := x86PackedIntegerCompareProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if form, known, err := x86PackedTestMaskProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); recognized { + return 1, true, err == nil && index == 0 + } + if form, known, err := x86PackedNarrowProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86PackedSADProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86PackedSubtractProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86PackedWrappingAddProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if f, known, err := x86PackedConversionProperties(ins, arch); known { + return int64(f.memoryBytes), false, err == nil && !f.mmx && f.memory == index + } + if f, known, err := x86SignedPackedFloatProperties(ins, arch); known { + return int64(f.memoryBytes), false, err == nil && !f.mmx && f.memory == index + } + if form, recognized, err := x86PackedExtendProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && form.memory == index && + (!form.signed || x86ResultNamedPredicateInstruction(ins)) + } + if form, recognized, err := x86FloatCompareProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if f, recognized, err := x86SignedIntegerFloatProperties(ins, arch); recognized { + return int64(f.sourceBits / 8), false, err == nil && f.memory == index + } + if f, recognized, err := x86UnsignedIntegerFloatProperties(ins, arch); recognized { + return int64(f.sourceBits / 8), false, err == nil && f.memory == index + } + if form, recognized, err := x86FloatArithmeticProperties(ins, arch, ""); recognized { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if f, recognized, err := x86InsertExtractProperties(ins, arch); recognized { + return int64(f.spec.chunk), f.memoryWrite, err == nil && f.memory == index && !(f.mask >= 0 && f.memoryWrite) + } + if form, recognized, err := x86PackedShiftProperties(ins.Op, ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if form, recognized, err := x86HorizontalFloatProperties(ins, arch); recognized { + return int64(form.byteWidth), false, err == nil && form.spec.vector && form.memory == index + } + if form, recognized, err := x86PackedUnpackProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && !form.mmx && !form.legacy && form.memory == index + } + if form, recognized, err := x86PackedLogicalProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if width = x86RawReturnScalarMoveBytes(Op(normalizeInstructionOpcode(x86SourceMoveInstruction(ins, arch).Op))); width != 0 && len(ins.Args) == 2 { + return width, index == 1, true + } + if form, recognized, err := x86ScalarExtensionProperties(ins.Op, ins, arch); recognized { + return int64(form.spec.sourceBits / 8), false, err == nil && index == 0 + } + if amd64MaskMoveOperandsValid(ins, arch) { + return int64(amd64MaskMoveSpecs[ins.Op] / 8), index == 1, true + } + if form, recognized, err := amd64PackedVectorMoveProperties(ins, arch); recognized { + return int64(form.byteWidth), form.memoryWrite, err == nil && !form.undefinedOpcode() && form.memory == index + } + if form, recognized, err := amd64FMA3Properties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && !form.spec.rawOnly && form.memory == index + } + form, recognized, err := amd64ScalarVectorMoveProperties(ins, arch) + if !recognized || err != nil || form.memory != index { + return 0, false, false + } + return int64(form.spec.bits / 8), form.memoryWrite, true +} + +func x86CallerPackedVectorMoveStep(ins *Instr, form amd64PackedVectorMoveForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.undefinedOpcode() || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + widthMask := x86CallerVectorByteMask(int64(form.byteWidth)) + var known uint64 + if form.memory >= 0 { + // A slice-data/nonalias witness is not a SIMD alignment promise. + // Aligned register copies are valid, but aligned memory requires a + // separate actual-address proof before borrowing this continuation. + if form.spec.aligned || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.byteWidth), form.memoryWrite) { + return false + } + if form.memoryWrite { + return state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)]&widthMask == widthMask + } + known = widthMask + } else { + known = state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] & widthMask + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + // No initialized mask is assumed all-one. Both potential inputs matter. + known = form.definedActiveBytes(known, state.vectors[destination]) + state.vectors[destination] = ^widthMask | known + return true +} diff --git a/amd64_go_caller_vector_moves_test.go b/amd64_go_caller_vector_moves_test.go new file mode 100644 index 000000000..0f74db41c --- /dev/null +++ b/amd64_go_caller_vector_moves_test.go @@ -0,0 +1,173 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerVectorMovesFreshScalarResults(t *testing.T) { + for _, width := range []struct { + suffix, typ string + }{{"SS", "float32"}, {"SD", "float64"}} { + legacy, vector := "MOV"+width.suffix, "VMOV"+width.suffix + for _, form := range []struct { + name, body string + }{ + {name: "legacy load/copy/store", body: fmt.Sprintf("%s (DI),X0\n%s X0,X1\n%s X1,(DX)", legacy, legacy, legacy)}, + {name: "vector load/merge/store", body: fmt.Sprintf("%s (DI),X0\n%s X0,X0,X1\n%s X1,(DX)", vector, vector, vector)}, + {name: "masked load zero", body: fmt.Sprintf("KMOVW AX,K1\n%s.Z (DI),K1,X0\n%s X0,(DX)", vector, vector)}, + {name: "masked store", body: fmt.Sprintf("KMOVW AX,K1\n%s (DI),X0\n%s X0,K1,(DX)", vector, vector)}, + {name: "masked merge", body: fmt.Sprintf("KMOVW AX,K1\n%s (DI),X0\n%s (SI),X1\n%s (DI),X2\n%s X0,X1,K1,X2\n%s X2,(DX)", vector, vector, vector, vector, vector)}, + {name: "masked merge zero", body: fmt.Sprintf("KMOVW AX,K1\n%s (DI),X0\n%s (SI),X1\n%s.Z X0,X1,K1,X2\n%s X2,(DX)", vector, vector, vector, vector)}, + {name: "extended vector register", body: fmt.Sprintf("%s (DI),X31\n%s X31,(DX)", vector, vector)}, + {name: "complete numeric overwrite", body: fmt.Sprintf("%s X0,X1,X2\n%s (DI),X2\n%s X2,(DX)", vector, vector, vector)}, + } { + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + t.Run(width.suffix+"/"+form.name+"/"+triple, func(t *testing.T) { + source := strings.ReplaceAll(x86CallerFixtureSource, "float32", width.typ) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", form.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } + } + } +} + +func TestX86GoCallerMaskMovesAllWidthsAndDirections(t *testing.T) { + for _, width := range []struct{ suffix, typ, move string }{ + {"B", "uint8", "MOVB"}, {"W", "uint16", "MOVW"}, + {"D", "uint32", "MOVL"}, {"Q", "uint64", "MOVQ"}, + } { + maskMove := "KMOV" + width.suffix + for _, form := range []struct{ name, body string }{ + {"GP to mask to memory", maskMove + " AX,K1\n" + maskMove + " K1,(DX)"}, + {"input to mask to memory", maskMove + " (DI),K1\n" + maskMove + " K1,(DX)"}, + {"length to mask to memory", maskMove + " (CX),K1\n" + maskMove + " K1,(DX)"}, + {"mask copy to memory", maskMove + " AX,K1\n" + maskMove + " K1,K2\n" + maskMove + " K2,(DX)"}, + {"mask to GP to memory", maskMove + " AX,K1\n" + maskMove + " K1,BX\n" + width.move + " BX,(DX)"}, + } { + t.Run(width.suffix+"/"+form.name, func(t *testing.T) { + source := strings.ReplaceAll(x86CallerFixtureSource, "float32", width.typ) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", form.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } + } +} + +func TestX86GoCallerVectorMovesDoNotWashPointersOrInventInitializedBytes(t *testing.T) { + for _, example := range []struct{ name, body, result string }{ + {"unknown mask", "VMOVSS.Z (DI),K1,X0\nVMOVSS X0,(DX)", "float32"}, + {"unknown mask source", "KMOVW BX,K1\nVMOVSS.Z (DI),K1,X0\nVMOVSS X0,(DX)", "float32"}, + {"pointer to mask", "KMOVW DI,K1\nVMOVSS.Z (DI),K1,X0\nVMOVSS X0,(DX)", "float32"}, + {"fixed g to mask", "KMOVQ R14,K1\nKMOVQ K1,(DX)", "uint64"}, + {"mask to fixed g", "KMOVW AX,K1\nKMOVQ K1,R14\nMOVL AX,(DX)", "float32"}, + {"mask to SP", "KMOVW AX,K1\nKMOVQ K1,SP\nMOVL AX,(DX)", "float32"}, + {"mask to BP", "KMOVW AX,K1\nKMOVQ K1,BP\nMOVL AX,(DX)", "float32"}, + {"mask replaces input pointer", "KMOVW AX,K1\nKMOVQ K1,DI\nVMOVSS (DI),X0\nVMOVSS X0,(DX)", "float32"}, + {"GP pointer as vector data", "MOVQ DI,X0\nMOVSD X0,(DX)", "float64"}, + {"private SP as vector data", "MOVQ SP,X0\nMOVSD X0,(DX)", "float64"}, + {"fixed g as vector data", "MOVQ R14,X0\nMOVSD X0,(DX)", "float64"}, + {"FP pointer as vector data", "MOVSD a+0(FP),X0\nMOVSD X0,(DX)", "float64"}, + {"saved BP as vector data", "MOVSD 0(SP),X0\nMOVSD X0,(DX)", "float64"}, + {"unknown vector bytes", "VMOVSS X0,(DX)", "float32"}, + {"unknown low source", "VMOVSS (DI),X0\nVMOVSS X1,X0,X2\nVMOVSS X2,(DX)", "float32"}, + {"legacy unknown upper bytes", "MOVSS (DI),X0\nMOVSS X0,X1\nMOVSD X1,(DX)", "float64"}, + {"masked merge unknown destination", "KMOVW AX,K1\nVMOVSS (DI),X0\nVMOVSS (SI),X1\nVMOVSS X0,X1,K1,X2\nVMOVSS X2,(DX)", "float32"}, + {"wide result", "VMOVSD (DI),X0\nVMOVSD X0,(DX)", "float32"}, + {"result one byte overlap", "VMOVSS (DI),X0\nVMOVSS X0,1(DX)", "float32"}, + {"result negative offset", "VMOVSS (DI),X0\nVMOVSS X0,-1(DX)", "float32"}, + {"result indexed", "VMOVSS (DI),X0\nVMOVSS X0,(DX)(AX*1)", "float32"}, + {"input write", "VMOVSS (DI),X0\nVMOVSS X0,(DI)", "float32"}, + {"length write", "VMOVSS (DI),X0\nVMOVSS X0,(CX)", "float32"}, + {"unknown input base", "VMOVSS (BX),X0\nVMOVSS X0,(DX)", "float32"}, + {"mask input write", "KMOVW AX,K1\nKMOVW K1,(DI)", "uint16"}, + {"mask wide result", "KMOVQ AX,K1\nKMOVQ K1,(DX)", "uint32"}, + {"literal result one byte overlap", "MOVSS $1.25,X0\nMOVSS X0,1(DX)", "float32"}, + {"literal unknown mask", "VMOVSS.Z $1.25,K1,X0\nVMOVSS X0,(DX)", "float32"}, + {"literal mask from pointer", "KMOVW DI,K1\nVMOVSS.Z $1.25,K1,X0\nVMOVSS X0,(DX)", "float32"}, + } { + t.Run(example.name, func(t *testing.T) { + source := strings.ReplaceAll(x86CallerFixtureSource, "float32", example.result) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", example.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved vector/mask state acquired raw-RET continuation: %v", err) + } + }) + } +} + +func TestX86GoCallerVectorMemoryCertificatesBindMaskWidthAndEveryOperand(t *testing.T) { + for _, body := range []string{ + "KMOVW AX,K1\nVMOVSS (DI),X0\nVMOVSS X0,K1,(DX)", + "KMOVD AX,K1\nKMOVD K1,(DX)", + } { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + found := false + for _, instruction := range normalized.Funcs[0].Instrs { + for index, access := range instruction.x86CallerMemory { + if !access.write || !x86CallerMemoryCertified(instruction, index) { + continue + } + found = true + for _, change := range []string{"opcode", "source", "address", "address-bits", "width", "direction", "mask"} { + mutated := instruction + mutated.Args = append([]Operand(nil), instruction.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VMOVSD" + case "source": + mutated.Args[0].Reg = "X3" + case "address": + mutated.Args[index].Mem.Off++ + case "address-bits": + mutated.x86AddressBits = 32 + case "width": + access.width++ + mutated.x86CallerMemory[index] = access + case "direction": + access.write = false + mutated.x86CallerMemory[index] = access + case "mask": + if len(mutated.Args) != 3 { + continue + } + mutated.Args[1].Reg = "K2" + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s borrowed a %s memory certificate", change, instruction.Op) + } + } + } + } + if !found { + t.Fatal("fixture produced no complete vector/mask write certificate") + } + } +} diff --git a/amd64_go_caller_vector_zero_test.go b/amd64_go_caller_vector_zero_test.go new file mode 100644 index 000000000..881aacc13 --- /dev/null +++ b/amd64_go_caller_vector_zero_test.go @@ -0,0 +1,124 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86GoCallerVectorZeroOriginalSource(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),Z0\nVZEROUPPER\nVMOVSS X0,(DX)", + "VZEROALL\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Z0\nWORD $0xf8c5\nBYTE $0x77\nVMOVSS X0,(DX)", + "WORD $0xfcc5\nBYTE $0x77\nVMOVSS X0,(DX)", + } { + t.Run(strings.ReplaceAll(body, "\n", "_"), func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerVectorZeroInitializedBytesAndUnaffectedState(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, opcode := range []Op{"VZEROUPPER", "VZEROALL"} { + for _, initial := range []uint64{0, 1, 0xffff, 0xaaaaaaaaaaaaaaaa, ^uint64(0)} { + state := x86SourceStackState{ + vectors: map[Reg]uint64{}, masks: map[Reg]bool{"K1": true, "K2": false}, + known: map[Reg]uint8{AX: 0xff}, numericFlags: true, gOriginal: true, + } + for register := 0; register < 32; register++ { + state.vectors[Reg(fmt.Sprintf("Z%d", register))] = initial + } + known, masks := x86CloneCallerKnown(state.known), map[Reg]bool{"K1": true, "K2": false} + form, recognized, err := x86VectorZeroProperties(Instr{Op: opcode}, arch) + if !recognized || err != nil || !x86CallerVectorZeroStep(form, &state) { + t.Fatalf("%s/%s grammar/effect: %t/%v", arch, opcode, recognized, err) + } + limit, added := 16, ^uint64(0xffff) + if arch == "386" { + limit = 8 + } + if opcode == "VZEROALL" { + added = ^uint64(0) + } + for register := 0; register < 32; register++ { + want := initial + if register < limit { + want |= added + } + if state.vectors[Reg(fmt.Sprintf("Z%d", register))] != want { + t.Fatalf("%s/%s invented/lost register%d byte knowledge", arch, opcode, register) + } + } + if !reflect.DeepEqual(state.known, known) || !reflect.DeepEqual(state.masks, masks) || !state.numericFlags || !state.gOriginal { + t.Fatal("vector zero changed GP/mask/flags/g source state") + } + } + } + } +} + +func TestX86GoCallerVectorZeroDoesNotInventPreservedLowOrHighRegisters(t *testing.T) { + for _, body := range []string{ + "VZEROUPPER\nVMOVSS X0,(DX)", + "VZEROALL\nVMOVSS X16,(DX)", + "VZEROUPPER\nVMOVSS X31,(DX)", + "VZEROALL\nKMOVQ K1,AX\nMOVL AX,(DX)", + } { + t.Run(strings.ReplaceAll(body, "\n", "_"), func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown low/high/mask source acquired return proof: %v", err) + } + }) + } +} + +func TestX86GoCallerVectorZeroCompleteFamilyThreeAPIs(t *testing.T) { + const body = "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z16\nVZEROUPPER\nVMOVSS X0,(DX)\nVZEROALL\nVMOVSS X15,(DX)\nVMOVSS X16,(DX)\nWORD $0xfcc5\nBYTE $0x77\nVMOVSS X1,(DX)\n" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "zero-caller.ll", "zero-caller.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, triple, "zero-module.ll", "zero-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_caller_word_multiply_test.go b/amd64_go_caller_word_multiply_test.go new file mode 100644 index 000000000..ef6ebb643 --- /dev/null +++ b/amd64_go_caller_word_multiply_test.go @@ -0,0 +1,178 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerWordMultiplyCompleteForms(t *testing.T) { + for _, name := range x86WordMultiplyNames { + banks := []string{"X"} + if strings.HasPrefix(name, "V") { + banks = append(banks, "Y", "Z") + } + for _, bank := range banks { + for _, first := range []string{bank + "0", "(DI)"} { + for _, mode := range []int{0, 1, 2} { + if !strings.HasPrefix(name, "V") && mode != 0 { + continue + } + opcode, args, result := name, first+","+bank+"1", "X1" + if strings.HasPrefix(name, "V") { + args, result = first+","+bank+"1", "X2" + if mode != 0 { + args += ",K1" + } + if mode == 2 { + opcode += ".Z" + } + args += "," + bank + "2" + } + row := opcode + " " + args + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } + } + } + } +} + +func TestX86GoCallerWordMultiplyUnknownAndMemoryPermissionsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Z1\nVPMULLW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Z0\nVPMULHW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "partial source": "MOVSS (DI),X0\nVMOVUPS (SI),Z1\nVPMULHUW Z0,Z1,Z2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVPMULHRSW.Z Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nMOVL $1,AX\nKMOVQ AX,K1\nVPMULHW Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPMULLW.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask lacks region": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPMULHW.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPMULHUW (BX),Z0,Z2\nVMOVSS X2,(DX)", + "stack alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPMULLW (BX),X0\nVMOVSS X0,(DX)", + "BP alias": "VMOVUPS (DI),Z0\nMOVQ BP,BX\nPMULHW (BX),X0\nVMOVSS X0,(DX)", + "input write": "VMOVUPS (DI),Z0\nVPMULHRSW Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX separate bank": "PMULLW M0,M1\nVMOVSS X1,(DX)", + "join missing second": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPMULHW Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved word multiply acquired caller permission", err) + } + }) + } +} + +func TestX86GoCallerWordMultiplyFullReadCertificatesAndByteAliases(t *testing.T) { + names := append(append([]string(nil), x86WordMultiplyNames...), "PMULLD", "VPMULLD", "VPMULLQ") + for _, name := range names { + row, result := name+" (DI),X0", "X0" + if strings.HasPrefix(name, "V") { + row, result = name+".Z (DI),Z0,K1,Z2", "X2" + } + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86LowAndWordMultiplyTestForm(ins) + if !known || err != nil || form.memory < 0 { + continue + } + found = true + access, exists := ins.x86CallerMemory[form.memory] + if !exists || access.width != int64(form.memoryBytes) || access.write || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatal("zero mask shrank the independent possible read envelope") + } + for _, change := range []string{"op", "base", "width", "direction", "destination", "mask", "address size"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch change { + case "op": + mutated.Op = "MOVOU" + case "base": + mutated.Args[form.memory].Mem.Base = BX + case "width": + access.width-- + mutated.x86CallerMemory[form.memory] = access + case "direction": + access.write = true + mutated.x86CallerMemory[form.memory] = access + case "destination": + mutated.Args[form.destination].Reg = "X7" + case "mask": + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K7"}) + case "address size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, form.memory) { + t.Fatal("stale original instruction memory certificate survived", change) + } + } + } + if !found { + t.Fatal("no multiply memory proof") + } + } + for _, name := range names { + operands := "X0,X1" + if strings.HasPrefix(name, "V") { + operands = "X0,X1,X2" + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+name+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86LowAndWordMultiplyTestForm(ins) + if err != nil { + t.Fatal(err) + } + state := &x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0xfffd, "Z1": 0xffff}, masks: map[Reg]bool{}} + if !x86CallerVectorMoveStep(&ins, state, nil) { + t.Fatal("registered pure vector effect missing") + } + want := uint64(0xfffc) // One absent source byte kills its whole word. + if name == "PMULLD" || name == "VPMULLD" { + want = 0xfff0 + } else if name == "VPMULLQ" { + want = 0xff00 + } + if !form.legacy { + want |= ^uint64(0xffff) + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatal(fmt.Sprintf("incorrect word-byte/upper alias transfer %s got=%x want=%x", name, got, want)) + } + } +} + +func x86LowAndWordMultiplyTestForm(ins Instr) (x86PackedBinaryForm, bool, error) { + if form, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, "amd64"); known { + return form.x86PackedBinaryForm, known, err + } + form, known, err := x86PackedWordMultiplyProperties(ins.Op, ins, "amd64") + return form.x86PackedBinaryForm, known, err +} diff --git a/amd64_go_caller_zero_extend.go b/amd64_go_caller_zero_extend.go new file mode 100644 index 000000000..c4c6afca5 --- /dev/null +++ b/amd64_go_caller_zero_extend.go @@ -0,0 +1,62 @@ +package plan9asm + +func x86CallerPackedZeroExtendStep(ins *Instr, form x86PackedZeroExtendForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + return x86CallerPackedExtendStep(ins, form, state, proof) +} + +func x86CallerPackedExtendStep(ins *Instr, form x86PackedExtendForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + var input uint64 + if form.memory >= 0 { + // Mask knowledge grants no address/alignment/envelope permission. + // Keep the complete potential read bound even when lanes can fault- + // suppress. Input regions never become writable scalar results. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + input = x86CallerVectorByteMask(int64(form.memoryBytes)) + } else { + input = state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + known := x86PackedExtendDefinedBytes(form, input, old) + width := x86CallerVectorByteMask(int64(form.byteWidth)) + if form.spec.vector { + state.vectors[destination] = ^width | known + } else { + state.vectors[destination] = old&^width | known + } + return true +} + +// This pure byte dependency grants no memory or caller permission. Upper +// zero bits are known independently of input bytes; mask/alias writes remain +// distinct from the register and memory observations that precede this step. +func x86PackedZeroExtendDefinedBytes(form x86PackedZeroExtendForm, input, old uint64) uint64 { + return x86PackedExtendDefinedBytes(form, input, old) +} + +func x86PackedExtendDefinedBytes(form x86PackedExtendForm, input, old uint64) uint64 { + var known uint64 + inputBytes, outputBytes := form.spec.inputBits/8, form.spec.outputBits/8 + for output := 0; output < form.byteWidth; output++ { + part := output % outputBytes + if part >= inputBytes && !form.signed { + known |= uint64(1) << uint(output) + continue // ZX high bytes are numerical zero independently of input. + } + if part >= inputBytes { + part = inputBytes - 1 // SX high bytes each depend on the actual sign byte. + } + if input>>uint(output/outputBytes*inputBytes+part)&1 != 0 { + known |= uint64(1) << uint(output) + } + } + if form.mask >= 0 && !form.zeroing { + known &= old // An initialized mask is not an all-active mask promise. + } + return known +} diff --git a/amd64_go_caller_zero_extend_test.go b/amd64_go_caller_zero_extend_test.go new file mode 100644 index 000000000..5784c0db0 --- /dev/null +++ b/amd64_go_caller_zero_extend_test.go @@ -0,0 +1,161 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerPackedZeroExtendCurrentGap(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", + "VMOVUPS (DI),X0\nVPMOVZXBD X0,Y1\nVMOVSS X1,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("Go-accepted bounded byte widening with initialized input: %v", err) + } +} + +func TestX86GoCallerPackedZeroExtendCompleteInitializedForms(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, original := range x86ZeroExtendAcceptedForms(prefix+"MOVZX"+ratio.suffix, "amd64") { + t.Run(original, func(t *testing.T) { + row := strings.NewReplacer("(AX)", "(DI)", "X15", "X2", "X31", "X2", "Y31", "Y2", "Z31", "Z2").Replace(original) + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nKMOVQ AX,K1\nKMOVQ AX,K7\n" + row + if strings.HasSuffix(row, "X1") || strings.HasSuffix(row, "Y1") { + body += "\nVMOVSS X1,(DX)" + } else { + body += "\nVMOVSS X2,(DX)" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } + } +} + +func TestX86GoCallerPackedZeroExtendRetainsUnknownAndEnvelopeFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown input": "VPMOVZXBD X0,Y1\nVMOVSS X1,(DX)", + "unknown merge": "VMOVUPS (DI),X0\nKMOVQ AX,K1\nVPMOVZXBD X0,K1,Y1\nVMOVSS X1,(DX)", + "unknown mask": "VMOVUPS (DI),X0\nVPMOVZXBD.Z X0,K1,Y1\nVMOVSS X1,(DX)", + "pointer mask": "VMOVUPS (DI),X0\nKMOVQ DI,K1\nVPMOVZXBD.Z X0,K1,Y1\nVMOVSS X1,(DX)", + "unknown base": "VPMOVZXBD (BX),Y1\nVMOVSS X1,(DX)", + "private stack": "VPMOVZXBD (SP),Y1\nVMOVSS X1,(DX)", + "pointer FP data": "VPMOVZXBD a+0(FP),Y1\nVMOVSS X1,(DX)", + "overread length": "VPMOVZXBW (CX),Y1\nVMOVSS X1,(DX)", + "zero mask envelope": "MOVL $0,AX\nKMOVQ AX,K1\nVPMOVZXBW.Z (DX),K1,Y1\nVMOVSS X1,(DX)", + "source write": "VPMOVZXBD (DI),Y1\nVMOVSS X1,(DI)", + "wide result": "VPMOVZXBD (DI),Y1\nVMOVUPS Y1,(DX)", + "undefined CFG join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),X0\njoined:\nVPMOVZXBD X0,Y1\nVMOVSS X1,(DX)", + "unknown upper alias": "VMOVUPS (DI),X0\nPMOVZXBD X0,X1\nVPMOVZXBW Y1,Z2\nVMOVSS X2,(DX)", + "signed sign byte absent": "PINSRB $0,AX,X0\nVPMOVSXWQ X0,X1\nPEXTRD $1,X1,AX\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + // The upper-alias negative must observe an unknown part, not the + // correctly initialized low X bytes of the first widening result. + if name == "unknown upper alias" { + assembly = strings.Replace(assembly, "VMOVSS X2,(DX)", "VEXTRACTI128 $1,Y1,X2\nVMOVSS X2,(DX)", 1) + } + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved zero-extension acquired caller continuation: %v", err) + } + }) + } +} + +func TestX86GoCallerPackedZeroExtendIndependentByteKnowledge(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, vector := range []bool{false, true} { + for _, width := range []int{16, 32, 64} { + if !vector && width != 16 { + continue + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + source := "X0" + if width*ratio.input/ratio.output > 16 { + source = "Y0" + } + for _, destination := range []int{0, 1} { + for mode := 0; mode < 3; mode++ { + if !vector && mode != 0 { + continue + } + name, mask := "PMOVZX"+ratio.suffix, "" + if vector { + name = "V" + name + } + if mode != 0 { + mask = "K1," + if mode == 2 { + name += ".Z" + } + } + row := fmt.Sprintf("%s %s,%s%s%d", name, source, mask, bank, destination) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedZeroExtendProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for seed := 0; seed < 64; seed++ { + input, old := uint64(seed)*0x391072d5e64b8a0f, uint64(seed+7)*0x6a89e14f30725bc1 + initial := map[Reg]uint64{"Z0": input, "Z1": old} + before := initial[Reg(fmt.Sprintf("Z%d", destination))] + result := x86ResultState{flags: x86ByteFlagsAll} + result.writeVector(0, 64, input, false) + result.writeVector(1, 64, old, false) + result.masks[1] = true + if !x86ResultTransfer(ins, &result, nil, 0, "amd64") { + t.Fatal("independent result observer lacks widening byte effects", row) + } + state := x86SourceStackState{vectors: initial, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedZeroExtendStep(&ins, form, &state, nil) { + t.Fatal("register byte effect refused") + } + var want uint64 + for output := 0; output < 64; output++ { + known := false + if output >= width { + known = vector || before>>uint(output)&1 != 0 + } else { + part := output % ratio.output + known = part >= ratio.input || input>>uint(output/ratio.output*ratio.input+part)&1 != 0 + if mode == 1 { + known = known && before>>uint(output)&1 != 0 + } + } + if known { + want |= uint64(1) << uint(output) + } + } + if got := state.vectors[Reg(fmt.Sprintf("Z%d", destination))]; got != want { + t.Fatalf("%s seed%d: defined bytes %#x want %#x", row, seed, got, want) + } + if result.vectorBytes(destination) != want || result.flags != x86ByteFlagsAll || + result.vectorBytes(1-destination) != map[int]uint64{0: input, 1: old}[1-destination] { + t.Fatalf("%s seed%d: result bytes %#x want %#x, or changed another register/FLAGS", row, seed, result.vectorBytes(destination), want) + } + if state.vectors[Reg(fmt.Sprintf("Z%d", 1-destination))] != map[int]uint64{0: input, 1: old}[1-destination] { + t.Fatal("widening changed another SIMD register") + } + } + } + } + } + } + } +} diff --git a/amd64_go_downloaded_frame_imports.go b/amd64_go_downloaded_frame_imports.go new file mode 100644 index 000000000..2798b47b2 --- /dev/null +++ b/amd64_go_downloaded_frame_imports.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "bytes" + "context" + "encoding/json" + "io" + "path/filepath" + + "github.com/xgo-dev/plan9asm/internal/golegacymod" + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +// The frame factory independently observes exact dependency downloads. The +// outer package query and caller-supplied types cannot supply these origins. +func (source *GoAMD64FrameSource) captureDownloadedDeclarationOrigins(ctx context.Context, output []byte) error { + if source.downloadedModule == nil { + return nil + } + if len(output) > 16<<20 { + return goAMD64FrameContext("declaration import graph exceeds its bound") + } + if err := source.verifyDownloadedOrigin(); err != nil { + return err + } + decoder := json.NewDecoder(bytes.NewReader(output)) + for count := 0; ; count++ { + var selected goAMD64FrameSelection + if err := decoder.Decode(&selected); err == io.EOF { + break + } else if err != nil { + return err + } + if count >= 4096 { + return goAMD64FrameContext("downloaded declaration import graph exceeds its count bound") + } + if selected.Standard { + continue // The SDK source/export consumer validates this role. + } + actual := selected.Module + if actual == nil || len(actual.Replace) != 0 && string(actual.Replace) != "null" { + return goAMD64FrameContext("downloaded dependency has an unbound module/replacement role") + } + if !actual.Main { + key := actual.Path + "@" + actual.Version + if _, exists := source.downloadedImports[key]; !exists { + if source.ready { + return goAMD64FrameContext("actual declaration import graph acquired a new module") + } + original, err := source.observeDownloadedModule(ctx, source.downloadedOrigin, + goAMD64DownloadedIdentity{module: actual.Path, version: actual.Version}, source.env["GOTMPDIR"]) + if err != nil { + return err + } + if source.downloadedImports == nil { + source.downloadedImports = make(map[string]gooriginalsrc.Module) + } + source.downloadedImports[key] = original + } + } + if _, err := source.downloadedDeclarationDirectory(selected); err != nil { + return err + } + } + return source.verifyDownloadedOrigin() +} + +func (source *GoAMD64FrameSource) downloadedDeclarationDirectory(selected goAMD64FrameSelection) (string, error) { + actual := selected.Module + if source.downloadedOrigin == nil || source.downloadedModule == nil || selected.Standard || actual == nil || + len(actual.Replace) != 0 && string(actual.Replace) != "null" { + return "", goAMD64FrameContext("foreign declaration dependency needs authenticated original handoff") + } + original := *source.downloadedModule + version := "" // Go's actual cache-CWD main role is not its download version. + if !actual.Main { + var found bool + original, found = source.downloadedImports[actual.Path+"@"+actual.Version] + if !found { + return "", goAMD64FrameContext("declaration dependency lacks its exact original download") + } + version = original.SourceVersion + } + root, rootErr := sourcepath.Resolve(actual.Dir) + metadata, metadataErr := sourcepath.Resolve(actual.GoMod) + if rootErr != nil || metadataErr != nil || root != original.Root || + actual.Path != original.ModulePath || actual.Version != version || + actual.GoVersion != gooriginalsrc.SelectedGoVersion(original.GoVersion, actual.Main) || + metadata != original.GoMod && metadata != filepath.Join(original.Root, "go.mod") { + return "", goAMD64FrameContext("declaration dependency differs from its original module metadata") + } + if original.GoModOrigin == golegacymod.ProxyGoModProtocol && (actual.Main || metadata != original.GoMod) { + return "", goAMD64FrameContext("legacy proxy metadata requires its exact dependency role") + } + directory, err := sourcepath.Resolve(selected.Dir) + if err != nil { + return "", err + } + path, err := goAMD64DownloadedPackagePath(original, directory) + if err != nil || path != selected.ImportPath { + return "", goAMD64FrameContext("declaration dependency escaped its original package namespace") + } + return directory, nil +} diff --git a/amd64_go_downloaded_frame_imports_test.go b/amd64_go_downloaded_frame_imports_test.go new file mode 100644 index 000000000..da780be22 --- /dev/null +++ b/amd64_go_downloaded_frame_imports_test.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" +) + +func TestGoAMD64DownloadedDeclarationMetadataRoles(t *testing.T) { + root := t.TempDir() + primary := gooriginalsrc.Module{Root: filepath.Join(root, "main"), ModulePath: "example.invalid/main", GoVersion: "1.20"} + foreign := gooriginalsrc.Module{Root: filepath.Join(root, "dependency"), ModulePath: "example.invalid/dependency", + SourceModule: "example.invalid/dependency", SourceVersion: "v1.0.0", GoVersion: "1.20"} + for _, original := range []*gooriginalsrc.Module{&primary, &foreign} { + if err := os.Mkdir(original.Root, 0700); err != nil { + t.Fatal(err) + } + canonical, err := filepath.EvalSymlinks(original.Root) + if err != nil { + t.Fatal(err) + } + original.Root = canonical + original.GoMod = filepath.Join(canonical, "go.mod") + if err := os.WriteFile(original.GoMod, []byte("module "+original.ModulePath+"\ngo 1.20\n"), 0600); err != nil { + t.Fatal(err) + } + } + for _, axis := range []string{"foreign", "primary", "unknown version", "namespace", "language", "main role", "root", + "metadata", "package directory", "package path", "replacement", "standard", "missing module", "missing owner"} { + t.Run(axis, func(t *testing.T) { + original, version, main := foreign, foreign.SourceVersion, false + if axis == "primary" { + original, version, main = primary, "", true + } + data, err := json.Marshal(map[string]any{ + "Dir": original.Root, "ImportPath": original.ModulePath, + "Module": map[string]any{"Path": original.ModulePath, "Version": version, "Main": main, + "Dir": original.Root, "GoMod": original.GoMod, "GoVersion": original.GoVersion}, + }) + if err != nil { + t.Fatal(err) + } + var selected goAMD64FrameSelection + if err := json.Unmarshal(data, &selected); err != nil { + t.Fatal(err) + } + // This is metadata predicate data, deliberately not a sealed source + // or a real OriginSet. It cannot Verify or supply frame authority. + source := &GoAMD64FrameSource{downloadedOrigin: &gooriginalsrc.OriginSet{}, downloadedModule: &primary, + downloadedImports: map[string]gooriginalsrc.Module{foreign.ModulePath + "@" + foreign.SourceVersion: foreign}} + switch axis { + case "unknown version": + selected.Module.Version = "v2.0.0" + case "namespace": + selected.Module.Path = primary.ModulePath + case "language": + selected.Module.GoVersion = "1.21" + case "main role": + selected.Module.Main = true + case "root": + selected.Module.Dir = primary.Root + case "metadata": + selected.Module.GoMod = primary.GoMod + case "package directory": + selected.Dir = primary.Root + case "package path": + selected.ImportPath += "/unobserved" + case "replacement": + selected.Module.Replace = json.RawMessage(`{}`) + case "standard": + selected.Standard = true + case "missing module": + selected.Module = nil + case "missing owner": + source.downloadedOrigin = nil + } + directory, err := source.downloadedDeclarationDirectory(selected) + if axis == "foreign" || axis == "primary" { + if err != nil || directory != original.Root { + t.Fatalf("consistent actual role lost metadata identity: %q, %v", directory, err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || directory != "" { + t.Fatalf("changed original module role was accepted: %q, %v", directory, err) + } + if err := source.Verify(context.Background()); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("metadata predicate manufactured actual frame authority", err) + } + }) + } +} diff --git a/amd64_go_downloaded_frame_source.go b/amd64_go_downloaded_frame_source.go new file mode 100644 index 000000000..9728c9229 --- /dev/null +++ b/amd64_go_downloaded_frame_source.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "time" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/golegacymod" + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" + "github.com/xgo-dev/plan9asm/internal/sourcepath" + "golang.org/x/mod/module" +) + +// GoAMD64DownloadedFrameCaptureRequest names original exact cache inputs. It +// carries no observation, checker, object, signature or permission callback. +type GoAMD64DownloadedFrameCaptureRequest struct { + Original GoAMD64FrameCaptureRequest + SourceModule, SourceVersion string +} + +type goAMD64DownloadedIdentity struct { + module, version string +} + +// CaptureGoAMD64DownloadedFrameSource independently executes stock Go and +// closes the original ZIP, extracted members, declarations and whole objects. +// Downloaded roles retain their primary module and independently observed +// original import graph. Capture itself grants no memory permission. +func CaptureGoAMD64DownloadedFrameSource(ctx context.Context, pkg GoPackage, request GoAMD64DownloadedFrameCaptureRequest) (*GoAMD64FrameSource, error) { + if module.Check(request.SourceModule, request.SourceVersion) != nil || + module.CanonicalVersion(request.SourceVersion) != request.SourceVersion { + return nil, goAMD64FrameContext("downloaded capture requires a canonical exact source identity") + } + return captureGoAMD64FrameSource(ctx, pkg, request.Original, + &goAMD64DownloadedIdentity{module: request.SourceModule, version: request.SourceVersion}) +} + +func (source *GoAMD64FrameSource) captureDownloadedOrigin(ctx context.Context, exact goAMD64DownloadedIdentity, temporary string) (result error) { + cache, err := sourcepath.Resolve(source.env["GOMODCACHE"]) + if err != nil { + return err + } + origins, err := gooriginalsrc.NewOriginSet(cache, source.request.GoBinary) + if err != nil { + return err + } + original, err := source.observeDownloadedModule(ctx, origins, exact, temporary) + if err != nil { + return err + } + if _, err := goAMD64DownloadedPackagePath(original, source.request.Directory); err != nil { + return err + } + source.downloadedOrigin, source.downloadedModule = origins, &original + source.downloadedImports = map[string]gooriginalsrc.Module{original.SourceModule + "@" + original.SourceVersion: original} + return source.verifyDownloadedOrigin() +} + +func (source *GoAMD64FrameSource) legacyDownloadedModule() bool { + return source.downloadedModule != nil && source.downloadedModule.GoModOrigin == golegacymod.ProxyGoModProtocol +} + +func (source *GoAMD64FrameSource) runDownloadedList(ctx context.Context, args []string) ([]byte, error) { + if source.numericalBinding != nil { + return nil, goAMD64FrameContext("legacy dependency query needs its separate numerical-command binding") + } + original := source.downloadedModule + path, err := goAMD64DownloadedPackagePath(*original, source.request.Directory) + if err != nil { + return nil, err + } + env, err := gooriginalsrc.DependencyQueryEnvironment(source.request.Environment) + if err != nil { + return nil, err + } + arguments := append([]string(nil), args...) + for index, argument := range arguments { + switch argument { + case ".": + arguments[index] = path + } + } + return gooriginalsrc.RunDependencyList(ctx, source.env["GOTMPDIR"], + original.SourceModule, original.SourceVersion, arguments, func(directory string, arguments []string) ([]byte, []byte, error) { + observed, err := goasmcmd.Observe(ctx, directory, env, source.request.GoBinary, arguments...) + return observed.Stdout, observed.Stderr, err + }) +} + +func (source *GoAMD64FrameSource) observeDownloadedModule(ctx context.Context, origins *gooriginalsrc.OriginSet, + exact goAMD64DownloadedIdentity, temporary string, +) (_ gooriginalsrc.Module, result error) { + if module.Check(exact.module, exact.version) != nil || module.CanonicalVersion(exact.version) != exact.version { + return gooriginalsrc.Module{}, goAMD64FrameContext("downloaded dependency requires a canonical exact source identity") + } + boundary, err := gooriginalsrc.NewDownloadBoundary(temporary, "plan9asm-frame-download-", "example.invalid/plan9asm-frame-download") + if err != nil { + return gooriginalsrc.Module{}, err + } + defer func() { + result = errors.Join(result, boundary.Close()) + }() + // This private command boundary is not a copied main/source module. Its + // sole purpose is to prevent stock Go mutating an ancestor go.mod/go.sum. + if err := boundary.Prepare(); err != nil { + return gooriginalsrc.Module{}, err + } + commandContext, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + data, err := goasmcmd.Run(commandContext, boundary.Directory(), source.request.Environment, + source.request.GoBinary, "mod", "download", "-json", exact.module+"@"+exact.version) + err = errors.Join(err, commandContext.Err()) + if err != nil { + return gooriginalsrc.Module{}, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + var receipt gooriginalsrc.DownloadReceipt + if err := decoder.Decode(&receipt); err != nil { + return gooriginalsrc.Module{}, err + } + var extra any + if decoder.Decode(&extra) != io.EOF { + return gooriginalsrc.Module{}, goAMD64FrameContext("actual exact download receipt has incomplete or additional JSON") + } + var proxyMetadata *golegacymod.ProxyGoModProof + if _, err := os.Lstat(filepath.Join(receipt.Dir, "go.mod")); os.IsNotExist(err) { + // Source capture pins GOPROXY=off, but authenticated public checksum + // metadata remains independently readable. This neither creates a ZIP + // member nor turns a legacy dependency into an original main module. + proxyMetadata, _, err = golegacymod.CaptureCachedProxyGoMod(commandContext, + source.env["GOMODCACHE"], "off", exact.module, exact.version, + receipt.Sum, receipt.GoModSum, receipt.GoMod, nil) + if err != nil { + return gooriginalsrc.Module{}, err + } + } + original, err := origins.ObserveLegacy(ctx, exact.module, exact.version, receipt, proxyMetadata) + if err != nil { + return gooriginalsrc.Module{}, errors.Join(goAMD64FrameContext("exact original archive/source is not closed"), err) + } + return original, nil +} + +func (source *GoAMD64FrameSource) verifyDownloadedOrigin() error { + if source.downloadedOrigin == nil && source.downloadedModule == nil { + return nil + } + if source.downloadedOrigin == nil || source.downloadedModule == nil { + return goAMD64FrameContext("missing exact original download/source role") + } + if _, err := goAMD64DownloadedPackagePath(*source.downloadedModule, source.request.Directory); err != nil { + return err + } + if err := source.downloadedOrigin.Verify(); err != nil { + return errors.Join(goAMD64FrameContext("exact original source was changed or revoked"), err) + } + return nil +} + +// The original archive namespace and the declared Go package namespace are +// distinct. Derive the latter only from the guarded module's own go.mod and +// the actual canonical package directory, never from an import-path prefix. +func goAMD64DownloadedPackagePath(original gooriginalsrc.Module, directory string) (string, error) { + relative, err := filepath.Rel(original.Root, directory) + if err != nil || !filepath.IsAbs(original.Root) || !filepath.IsAbs(directory) || !filepath.IsLocal(relative) { + return "", goAMD64FrameContext("downloaded package must remain inside its actual exact primary cache root") + } + path := original.ModulePath + if relative != "." { + path += "/" + filepath.ToSlash(relative) + } + if module.CheckImportPath(path) != nil { + return "", goAMD64FrameContext("downloaded package has an invalid original module-relative identity") + } + return path, nil +} diff --git a/amd64_go_downloaded_package_path_test.go b/amd64_go_downloaded_package_path_test.go new file mode 100644 index 000000000..6ce9dcbd1 --- /dev/null +++ b/amd64_go_downloaded_package_path_test.go @@ -0,0 +1,37 @@ +package plan9asm + +import ( + "path/filepath" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" +) + +func TestGoAMD64DownloadedPackageIdentity(t *testing.T) { + root := filepath.Join(t.TempDir(), "archive.example", "mirror@v1.0.0") + original := gooriginalsrc.Module{Root: root, ModulePath: "example.invalid/declared"} + for _, test := range []struct { + name, directory, want string + }{ + {"root alias", root, "example.invalid/declared"}, + {"nested alias", filepath.Join(root, "internal", "kernel"), "example.invalid/declared/internal/kernel"}, + {"parent", filepath.Dir(root), ""}, + {"prefix sibling", root + "-other", ""}, + {"traversal", filepath.Join(root, "..", "other"), ""}, + {"relative", "internal/kernel", ""}, + {"invalid import", filepath.Join(root, "bad path"), ""}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := goAMD64DownloadedPackagePath(original, test.directory) + if test.want == "" { + if err == nil || got != "" { + t.Fatalf("unbound path acquired an original package identity: %q, %v", got, err) + } + return + } + if err != nil || got != test.want { + t.Fatalf("original declared package = %q, %v; want %q", got, err, test.want) + } + }) + } +} diff --git a/amd64_go_frame_adx_test.go b/amd64_go_frame_adx_test.go new file mode 100644 index 000000000..778f964fb --- /dev/null +++ b/amd64_go_frame_adx_test.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "strconv" + "strings" + "testing" +) + +func TestGoAMD64FrameADXMachineBoundaries(t *testing.T) { + for _, op := range []string{"ADCXL", "ADCXQ", "ADOXL", "ADOXQ"} { + for _, source := range []string{"AX", "8(SP)", "128(BX)(CX*4)"} { + t.Run(op+"/"+source, func(t *testing.T) { + assembly := "TEXT value(SB),4,$64-0\n" + op + " " + source + ",DX\nRET\n" + listing := x86InsertExtractGoListing(t, "amd64", assembly) + fn := goAMD64FrameFunction{positions: make(map[int]string)} + adxPC := -1 + for _, line := range strings.Split(string(listing), "\n") { + fn.code = append(fn.code, x86InsertExtractListedBytes(line)...) + row := goAMD64FrameRow.FindStringSubmatch(line) + if row == nil || strings.HasPrefix(row[3], "TEXT") || strings.HasPrefix(row[3], "PCDATA") { + continue + } + pc, err := strconv.ParseInt(row[1], 16, 64) + if err != nil { + t.Fatal(err) + } + fn.positions[int(pc)] = row[3] + if strings.HasPrefix(row[3], op) { + adxPC = int(pc) + } + } + if adxPC < 0 { + t.Fatal("original Go listing lost ADX") + } + if err := goAMD64FrameMachineShape(fn, 64); err != nil { + t.Error("ordinary ADX must retain its original instruction boundary", err) + } + for _, mutation := range []string{"missing position", "interior position", "bad prefix", "truncated", "unbound CALL"} { + t.Run(mutation, func(t *testing.T) { + changed := fn + changed.code = append([]byte(nil), fn.code...) + changed.positions = make(map[int]string, len(fn.positions)) + for pc, position := range fn.positions { + changed.positions[pc] = position + } + switch mutation { + case "missing position": + delete(changed.positions, adxPC) + case "interior position": + changed.positions[adxPC+1] = "not an instruction boundary" + case "bad prefix": + changed.code[adxPC] = 0xf2 + case "truncated": + changed.code = changed.code[:adxPC+3] + case "unbound CALL": + copy(changed.code[adxPC:], []byte{0xe8, 0, 0, 0, 0}) + } + if err := goAMD64FrameMachineShape(changed, 64); err == nil { + t.Fatal("ADX boundary observation granted invalid machine shape") + } + }) + } + }) + } + } +} diff --git a/amd64_go_frame_byte_listing_test.go b/amd64_go_frame_byte_listing_test.go new file mode 100644 index 000000000..47f9919b5 --- /dev/null +++ b/amd64_go_frame_byte_listing_test.go @@ -0,0 +1,37 @@ +package plan9asm + +import ( + "bytes" + "context" + "testing" + "time" +) + +func TestGoAMD64FrameListingHexASCII(t *testing.T) { + const assembly = `TEXT ·Value(SB),4,$16-16 + MOVQ a+0(FP),AX + MOVQ AX,0(SP) + MOVQ 0(SP),AX + MOVQ AX,ret+8(FP) + RET +TEXT ·HexASCII(SB),4,$0-0 + XORB (R8),AH + RET +` + pkg, request, _, _ := fixedLocalTransportFixture(t, assembly, + "func Value(a uint64) uint64\nfunc HexASCII()\n", "linux/amd64", "x86_64-unknown-linux-gnu") + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("complete original Go listing with hex-looking ASCII", err) + } + defer source.Close() + fn, found := source.functions[pkg.Path+".HexASCII"] + if !found || !bytes.Equal(fn.code, []byte{0x41, 0x32, 0x20, 0xc3}) { + t.Fatalf("Go listing ASCII was interpreted as an instruction byte: %x", fn.code) + } + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } +} diff --git a/amd64_go_frame_call_observation_go127_test.go b/amd64_go_frame_call_observation_go127_test.go new file mode 100644 index 000000000..f6b8f401b --- /dev/null +++ b/amd64_go_frame_call_observation_go127_test.go @@ -0,0 +1,192 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +// The outgoing words and declarations describe the source. Observing CALL's +// relocation does not prove a callee ABI, reachable inputs or stack ownership. +const frameOrdinaryCallAssembly = `TEXT ·Caller(SB),4,$24-24 + MOVQ dst+0(FP),AX + MOVQ AX,0(SP) + MOVQ src+8(FP),AX + MOVQ AX,8(SP) + MOVQ n+16(FP),AX + MOVQ AX,16(SP) + CALL ·First(SB) + CALL ·Second(SB) + RET +` + +func TestGoAMD64FrameOrdinaryCallObjectListingIdentity(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + for _, data := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/DATA=%t", target, data), func(t *testing.T) { + assembly := frameOrdinaryCallAssembly + if data { + assembly = "DATA literal<>+0(SB)/8,$7\nGLOBL literal<>(SB),8,$8\n" + assembly + } + declarations := "func Caller(dst,src *byte,n uintptr)\nfunc First(dst,src *byte,n uintptr)\nfunc Second(dst,src *byte,n uintptr)\n" + pkg, request := goAMD64FrameTypedFixture(t, declarations, assembly) + request.Target = target + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + objectPath := filepath.Join(t.TempDir(), "original.o") + env := append(append([]string(nil), request.Environment...), + "GOOS="+strings.Split(target, "/")[0], "GOARCH=amd64", "CGO_ENABLED=0", + "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=") + listing, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, + "tool", "asm", "-S", "-p", pkg.Path, "-o", objectPath, request.AssemblyFile) + if err != nil { + t.Fatalf("original Go CALL object: %v\n%s", err, listing) + } + object, err := os.ReadFile(objectPath) + if err != nil { + t.Fatal(err) + } + builtin, err := os.ReadFile(filepath.Join(filepath.Dir(filepath.Dir(request.GoBinary)), "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + t.Fatal(err) + } + observed, err := gostaticstorage.ObserveAMD64AssemblyData([]byte(assembly), object, builtin, pkg.Path, target) + if err != nil || len(observed.Member.Texts) != 1 || len(observed.Member.Texts[0].Relocations) != 2 { + t.Fatal("independent complete CALL object observation", err) + } + for ordinal, relocation := range observed.Member.Texts[0].Relocations { + t.Logf("actual original relocation %d: %+v", ordinal, relocation) + want := fmt.Sprintf("rel %d+4 t=R_CALL %s+0", relocation.Offset, relocation.Target.Name) + if relocation.Type != 7 || relocation.Size != 4 || relocation.Add != 0 || + !relocation.Target.ABIKnown || relocation.Target.ABI != 0 || + !strings.Contains(string(listing), want) { + t.Fatal("the original Go producer did not emit the expected direct ABI0 CALL format", relocation, string(listing)) + } + if got := goAMD64FrameDataRelocationLine(relocation, &observed); got != want { + t.Error("ordinary CALL record lost exact object/listing association", got, want) + } + } + // This is a deliberately distinct source-contract negative. A + // correct observation still cannot make these external CALLs into + // the existing same-source private-leaf frame contract. + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if err := goAMD64FrameSourceInstructions(file.Funcs[0], map[string]bool{}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("object observation granted an ordinary callee/frame role", err) + } + if target != "windows/amd64" { + frameOrdinaryCallListingGuards(t, pkg, request, file, assembly, listing, &observed) + } + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if owner != nil { + if closeErr := owner.Close(); closeErr != nil { + t.Error(closeErr) + } + } + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "original CALL needs its complete same-source leaf") { + t.Fatal("ordinary CALL acquired an unaudited fixed-local frame", err) + } + }) + } + } +} + +func frameOrdinaryCallListingGuards(t *testing.T, pkg GoPackage, request GoAMD64FrameCaptureRequest, + file *File, assembly string, listing []byte, observed *gostaticstorage.AMD64AssemblyData) { + t.Helper() + input := listing + if len(observed.Layouts) != 0 { + var err error + input, err = goAMD64FrameDataListing(listing, observed) + if err != nil { + t.Fatal(err) + } + } + check := func(value []byte) error { + _, err := goAMD64FrameListingWithDataObject(value, file, pkg, + map[string]int{request.AssemblyFile: strings.Count(assembly, "\n") + 1}, + observed, nil, request.Target, request.Directory, "") + return err + } + if err := check(input); !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "original CALL needs its complete same-source leaf") { + t.Fatal("exact complete CALL listing did not reach the independent source permission check", err) + } + relocations := observed.Member.Texts[0].Relocations + first := []byte("\t" + goAMD64FrameDataRelocationLine(relocations[0], observed) + "\n") + second := []byte("\t" + goAMD64FrameDataRelocationLine(relocations[1], observed) + "\n") + for name, replacement := range map[string][]byte{ + "foreign": bytes.Replace(first, []byte(pkg.Path+".First"), []byte("foreign.First"), 1), + "offset": bytes.Replace(first, []byte(fmt.Sprintf("rel %d+", relocations[0].Offset)), []byte(fmt.Sprintf("rel %d+", relocations[0].Offset+1)), 1), + "width": bytes.Replace(first, []byte("+4 t="), []byte("+8 t="), 1), + "type": bytes.Replace(first, []byte("R_CALL"), []byte("R_PCREL"), 1), + "addend": bytes.Replace(first, []byte("First+0"), []byte("First+1"), 1), + "missing": nil, + "duplicate": append(append([]byte(nil), first...), first...), + "reordered": second, + } { + t.Run(name, func(t *testing.T) { + changed := bytes.Replace(input, first, replacement, 1) + err := check(changed) + if bytes.Equal(changed, input) || !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "object/listing relocation") { + t.Fatal("changed CALL listing retained its exact original association", err) + } + }) + } +} + +func TestGoAMD64FrameCallRelocationRetainsPrivateIdentity(t *testing.T) { + assembly := fixedLocalPrivateLeafSource("CALL leaf<>(SB)\nCALL other<>(SB)", "MOVQ (BP),AX") + + "TEXT other<>(SB),4,$0\nADDQ $1,AX\nRET\n" + pkg, request, _ := fixedLocalPrivateLeafFixture(t, assembly) + owner := requireGoAMD64FrameCapture(t, pkg, request) + expanded, err := owner.preprocess(request.AssemblyFile, []byte(assembly)) + if err != nil { + t.Fatal(err) + } + observed, err := owner.fixedLocalDataObservation(request.AssemblyFile, expanded) + if err != nil { + t.Fatal(err) + } + root := observed.TextOrder[0].ObjectOrdinal + if len(observed.Member.Texts[root].Relocations) != 2 { + t.Fatal("the private CALL control lost its two exact source/object targets") + } + original := observed.Member.Texts[root].Relocations[0] + if goAMD64FrameDataRelocationLine(original, observed) == "" { + t.Fatal("existing private CALL lost its observation") + } + for _, test := range []struct { + name string + edit func(*gostaticstorage.TextRelocation) + }{ + {"bank", func(r *gostaticstorage.TextRelocation) { r.Target.PackageIndex++ }}, + {"relative", func(r *gostaticstorage.TextRelocation) { r.Target.SymbolIndex++ }}, + {"ABI", func(r *gostaticstorage.TextRelocation) { r.Target.ABI++ }}, + {"unknown-ABI", func(r *gostaticstorage.TextRelocation) { r.Target.ABIKnown = false }}, + {"width", func(r *gostaticstorage.TextRelocation) { r.Size = 8 }}, + {"addend", func(r *gostaticstorage.TextRelocation) { r.Add = 1 }}, + {"kind", func(r *gostaticstorage.TextRelocation) { r.Type = 10 }}, + } { + t.Run(test.name, func(t *testing.T) { + changed := original + test.edit(&changed) + if goAMD64FrameDataRelocationLine(changed, observed) != "" { + t.Fatal("ordinary CALL formatting reclassified a mismatched local identity", changed) + } + }) + } +} diff --git a/amd64_go_frame_command_root_test.go b/amd64_go_frame_command_root_test.go new file mode 100644 index 000000000..8d0523bab --- /dev/null +++ b/amd64_go_frame_command_root_test.go @@ -0,0 +1,76 @@ +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +func TestGoAMD64FrameSourceRetainsCommandSDKIdentity(t *testing.T) { + for _, mutation := range []string{"command-spelling", "physical-root"} { + t.Run(mutation, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + physical, err := sourcepath.Resolve(filepath.Dir(filepath.Dir(request.GoBinary))) + if err != nil { + t.Fatal(err) + } + alias := filepath.Join(t.TempDir(), "selected-sdk") + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + link := func(target string) { + t.Helper() + if runtime.GOOS == "windows" { + output, err := goasmcmd.Run(ctx, "", nil, "cmd", "/c", "mklink", "/J", alias, target) + if err != nil { + t.Fatalf("create actual SDK junction: %v; %s", err, output) + } + } else if err := os.Symlink(target, alias); err != nil { + t.Fatal(err) + } + } + link(physical) + request.Environment = append(request.Environment, "GOROOT="+alias) + original, err := goasmcmd.Run(ctx, request.Directory, request.Environment, request.GoBinary, "env", "GOROOT") + if err != nil || strings.TrimSpace(string(original)) != alias || alias == physical { + t.Fatalf("actual driver did not retain a distinct SDK spelling: %q; %v", original, err) + } + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + if source.env["command/GOROOT"] != alias || source.env["GOROOT"] != physical { + t.Fatalf("command spelling and physical SDK were conflated: %v", source.env) + } + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + switch mutation { + case "command-spelling": + // The same physical SDK under a new command spelling can change + // compiled builtin tools; it must not reuse the observation. + source.request.Environment = append(source.request.Environment, "GOROOT="+physical) + case "physical-root": + if err := os.Remove(alias); err != nil { + t.Fatal(err) + } + link(t.TempDir()) + } + if err := source.Verify(ctx); err == nil || source.ready { + t.Fatalf("changed %s retained source observation: %v", mutation, err) + } else if mutation == "command-spelling" && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed command identity was not rejected as source context: %v", err) + } + }) + } +} diff --git a/amd64_go_frame_complete_source_test.go b/amd64_go_frame_complete_source_test.go new file mode 100644 index 000000000..49448d041 --- /dev/null +++ b/amd64_go_frame_complete_source_test.go @@ -0,0 +1,127 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// Original Go compilation is the acceptance oracle. Additional ordinary Go +// declarations do not themselves grant caller memory or fixed-local backing. +func TestGoAMD64FrameSourceCompleteOrdinaryGoDeclarations(t *testing.T) { + // Reuse only this root's owned compiler cache, not source identities. + // Every case still executes actual Go and consumes its selected export. + // Recent Go versions build objdump lazily, even for `go tool -n`. + // Prepare only the compiler artifact before any bounded source lease; + // each case still independently discovers and verifies the actual tool. + goAMD64FramePrepareOwnedToolCache(t) + + for _, fixture := range []struct { + name, imports, declarations string + }{ + {"body", "", "func Reference(a uint64) uint64 { return a + 1 }"}, + {"values", "", "const Bias uint64 = 1\nvar Counter uint64"}, + {"unused-type", "", "type Record struct { Bytes []byte; Value uint64 }"}, + {"unused-interface", "", "type XOF interface { Read([]byte) (int, error); Clone() XOF }"}, + {"unused-function", "", "type Callback func([]byte) error"}, + {"unused-map", "", "type Cache map[string][]byte"}, + {"unused-channel", "", "type Queue chan []byte"}, + {"unused-float", "", "type Scalar float64"}, + {"unused-recursive", "", "type Node struct { Next *Node; Value uint64 }"}, + {"unused-generic", "", "type Box[T any] struct { Value T }"}, + {"unused-interface-alias", "", "type XOF = interface { Read([]byte) (int, error) }"}, + {"sdk-import", "import \"math/bits\"", "func Reference(a uint64) uint64 { return bits.RotateLeft64(a, 1) }"}, + {"unrelated-body-transport", "", "func Reference(a float64) float64 { return a + 1 }"}, + {"combined", "import \"math/bits\"", "const Bias = 1\nvar Counter uint64\nfunc Reference(a uint64) uint64 { return bits.RotateLeft64(a, Bias) }"}, + } { + t.Run(fixture.name, func(t *testing.T) { + _, request := goAMD64FrameSourceFixture(t) + text := "package framefixture\n" + fixture.imports + "\nfunc Value(a uint64) uint64\n" + fixture.declarations + "\n" + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte(text), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), + "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOPROXY=off") + command := exec.CommandContext(ctx, request.GoBinary, + "list", "-deps", "-export", "-compiled", "-json", "-mod=readonly", ".") + command.Dir, command.Env = request.Directory, env + output, diagnostics, err := runtimeOracleMachineOutput(ctx, command) + if err != nil { + t.Fatalf("original Go rejected complete ordinary source: %v\n%s\n%s", err, output, diagnostics) + } + var selected goAMD64FrameSelection + exports := make(map[string]string) + decoder := json.NewDecoder(bytes.NewReader(output)) + for { + var member goAMD64FrameSelection + if err := decoder.Decode(&member); err == io.EOF { + break + } else if err != nil || member.ImportPath == "" || len(member.Error)+len(member.DepsErrors) != 0 { + t.Fatalf("original Go compiler graph is incomplete: %+v: %v", member, err) + } + exports[member.ImportPath] = member.Export + if member.Module != nil && member.Module.Main { + if selected.ImportPath != "" { + t.Fatal("original Go selected more than one owned package") + } + selected = member + } + } + if selected.ImportPath != "example.invalid/framefixture" || len(selected.CompiledGoFiles) != 1 || len(selected.SFiles) != 1 { + t.Fatalf("original Go omitted the complete Go/ASM source pair: %+v", selected) + } + export, err := os.ReadFile(selected.Export) + if err != nil || len(export) == 0 { + t.Fatalf("original Go omitted compiled package/assembly export: %v", err) + } + t.Logf("ORIGINAL_GO_COMPLETE_FRAME_SOURCE role=%s go_sha=%x export_sha=%x", fixture.name, + sha256.Sum256([]byte(text)), sha256.Sum256(export)) + set := token.NewFileSet() + syntax, err := parser.ParseFile(set, filepath.Join(request.Directory, "value.go"), text, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + actualImporter := importer.ForCompiler(set, "gc", func(path string) (io.ReadCloser, error) { + return os.Open(exports[path]) + }) + typed, err := (&types.Config{Importer: actualImporter, Sizes: types.SizesFor("gc", "amd64")}).Check( + selected.ImportPath, set, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + pkg := GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{syntax}} + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatalf("Go-accepted ordinary declarations lost original frame observation: %v", err) + } + defer func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }() + function, found := source.functions[pkg.Path+".Value"] + if !found || function.locals != 168 || function.args != 16 || function.transport != goAMD64FrameNumericTransport { + t.Fatalf("complete source changed the independently captured numeric ASM frame: %+v", function) + } + if len(source.functions) != 1 || len(source.declarations) != 1 { + t.Fatal("unrelated ordinary Go declarations became ASM frame capabilities") + } + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_frame_context.go b/amd64_go_frame_context.go new file mode 100644 index 000000000..6a0aeac07 --- /dev/null +++ b/amd64_go_frame_context.go @@ -0,0 +1,75 @@ +package plan9asm + +import ( + "context" + "errors" + "time" +) + +// Caller contexts are callbacks, not safe command contexts under an observer +// lock. Capture only their actual lifetime outside locks; commands consume a +// standard private context with the same (never extended) deadline/Done signal. +// The original context remains the published binding's lifetime authority. +type goAMD64FramePrivateContext struct { + context.Context + original context.Context + deadline time.Time + bounded bool + done <-chan struct{} + cancel context.CancelFunc + stop func() +} + +func captureGoAMD64FramePrivateContext(original context.Context) (*goAMD64FramePrivateContext, error) { + if original == nil { + return nil, goAMD64FrameContext("missing caller context") + } + if err := original.Err(); err != nil { + return nil, err + } + deadline, bounded := original.Deadline() + done := original.Done() + if _, err := sampleGoSourceContext(context.Background(), original, + goSourceContextLifetime{deadline: deadline, bounded: bounded, done: done, checkDone: true}, false, + goAMD64FrameContext("caller context lifetime changed during capture")); err != nil { + return nil, err + } + var private context.Context + var cancel context.CancelFunc + if bounded { + private, cancel = context.WithDeadline(context.Background(), deadline) + } else { + private, cancel = context.WithCancel(context.Background()) + } + finished, stopped := make(chan struct{}), make(chan struct{}) + go func() { + defer close(stopped) + select { + case <-done: + cancel() + case <-private.Done(): + case <-finished: + } + }() + return &goAMD64FramePrivateContext{Context: private, original: original, + deadline: deadline, bounded: bounded, done: done, cancel: cancel, + stop: func() { + close(finished) + cancel() + <-stopped + }}, nil +} + +// Always called outside observer locks. Retain actual cancellation causes and +// reject a caller that changes its captured lifetime while work is in flight. +func (ctx *goAMD64FramePrivateContext) verify() error { + _, err := sampleGoSourceContext(ctx.Context, ctx.original, + goSourceContextLifetime{deadline: ctx.deadline, bounded: ctx.bounded, done: ctx.done, checkDone: true}, false, + goAMD64FrameContext("caller context lifetime changed during observation")) + if goSourceContextDone(ctx.done) { + ctx.cancel() + } + // Context is the privately constructed standard command context, not the + // original caller implementation. No user callback follows the checkpoint. + return errors.Join(err, ctx.Context.Err()) +} diff --git a/amd64_go_frame_context_test.go b/amd64_go_frame_context_test.go new file mode 100644 index 000000000..b1f1a914a --- /dev/null +++ b/amd64_go_frame_context_test.go @@ -0,0 +1,222 @@ +package plan9asm + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" +) + +type goAMD64FrameCallbackContext struct { + context.Context + method string + armed atomic.Bool + called atomic.Bool + locked atomic.Bool + callback func() +} + +func (ctx *goAMD64FrameCallbackContext) call(method string) { + if ctx.method == method && ctx.armed.CompareAndSwap(true, false) { + ctx.called.Store(true) + ctx.callback() + } +} + +func (ctx *goAMD64FrameCallbackContext) Err() error { + ctx.call("Err") + return ctx.Context.Err() +} + +func (ctx *goAMD64FrameCallbackContext) Deadline() (time.Time, bool) { + ctx.call("Deadline") + return ctx.Context.Deadline() +} + +func (ctx *goAMD64FrameCallbackContext) Done() <-chan struct{} { + ctx.call("Done") + return ctx.Context.Done() +} + +func goAMD64FrameCloseCallback(t *testing.T, original context.Context, source *GoAMD64FrameSource, method string) *goAMD64FrameCallbackContext { + t.Helper() + ctx := &goAMD64FrameCallbackContext{Context: original, method: method} + ctx.callback = func() { + // A synchronous Close here would deadlock if the callback owns mu. + // Record the actual lock boundary without leaving a blocked goroutine + // or a deferred cleanup that can never complete in the RED test. + if !source.mu.TryLock() { + ctx.locked.Store(true) + return + } + source.mu.Unlock() + if err := source.Close(); err != nil { + t.Errorf("context callback could not close its original source: %v", err) + } + } + return ctx +} + +func TestGoAMD64FrameVerifyContextCallbacksMayClose(t *testing.T) { + for _, method := range []string{"Err", "Deadline", "Done"} { + t.Run(method, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + original, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(original, pkg, request) + if err != nil { + t.Fatal(err) + } + defer source.Close() + ctx := goAMD64FrameCloseCallback(t, original, source, method) + ctx.armed.Store(true) + err = source.Verify(ctx) + if ctx.locked.Load() || !ctx.called.Load() { + t.Fatalf("caller %s callback ran under frame lock (Close would deadlock) or was never sampled", method) + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("callback-closed source verified", err) + } + if err := source.Verify(original); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("callback source revocation was not permanent", err) + } + }) + } +} + +func TestGoAMD64FrameBindRetainsOriginalContextLifetime(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + original, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(original, pkg, file, options) + if err != nil { + t.Fatal(err) + } + binding := bound.Sigs[pkg.Path+".Value"].x86FixedLocal + if binding == nil || binding.context != original { + t.Fatal("binding retained temporary private command context instead of caller lifetime") + } + if err := binding.verify(); err != nil { + t.Fatal("stopping private binder commands revoked the live original binding", err) + } + cancel() + if err := binding.verify(); !errors.Is(err, context.Canceled) { + t.Fatal("original caller cancellation was ignored", err) + } + if err := source.Verify(context.Background()); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("fresh context revived a source revoked by original binding cancellation", err) + } +} + +type goAMD64FramePostContext struct { + context.Context + samples int + callback func() +} + +func (ctx *goAMD64FramePostContext) Err() error { + ctx.samples++ + if ctx.samples == 3 { + // The first two samples capture the caller lifetime. This sample is + // the final callback boundary after the complete locked command round. + ctx.callback() + } + return ctx.Context.Err() +} + +func TestGoAMD64FrameVerifyPostContextMutationRevokes(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + original, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(original, pkg, request) + if err != nil { + t.Fatal(err) + } + defer source.Close() + ctx := &goAMD64FramePostContext{Context: original, callback: func() { + if !source.mu.TryLock() { + t.Fatal("post-context callback held the frame lock") + } + source.mu.Unlock() + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte("changed original source\n"), 0600); err != nil { + t.Fatal(err) + } + }} + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) || ctx.samples < 3 { + t.Fatal("final caller callback changed source after the last complete input guard", err) + } + if err := source.Verify(original); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("final callback mutation did not permanently revoke source", err) + } +} + +type goAMD64FrameChangingContext struct { + context.Context + shift time.Duration + done <-chan struct{} +} + +func (ctx *goAMD64FrameChangingContext) Deadline() (time.Time, bool) { + deadline, bounded := ctx.Context.Deadline() + return deadline.Add(ctx.shift), bounded +} + +func (ctx *goAMD64FrameChangingContext) Done() <-chan struct{} { + return ctx.done +} + +func TestGoAMD64FramePrivateContextExactLifetime(t *testing.T) { + if private, err := captureGoAMD64FramePrivateContext(nil); private != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("nil caller acquired a private command context", err) + } + for _, bounded := range []bool{false, true} { + original := context.Background() + cancel := func() {} + if bounded { + original, cancel = context.WithTimeout(original, time.Minute) + } + private, err := captureGoAMD64FramePrivateContext(original) + if err != nil { + cancel() + t.Fatal(err) + } + want, wantBounded := original.Deadline() + got, gotBounded := private.Deadline() + if gotBounded != wantBounded || gotBounded && !got.Equal(want) || private.verify() != nil { + t.Fatal("private context changed actual ordinary caller lifetime") + } + private.stop() + if original.Err() != nil || !errors.Is(private.Err(), context.Canceled) { + t.Fatal("owned private cleanup canceled its original caller") + } + cancel() + } + for _, axis := range []string{"deadline", "done", "cancel"} { + t.Run(axis, func(t *testing.T) { + original, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + caller := &goAMD64FrameChangingContext{Context: original, done: original.Done()} + private, err := captureGoAMD64FramePrivateContext(caller) + if err != nil { + t.Fatal(err) + } + defer private.stop() + switch axis { + case "deadline": + caller.shift = time.Second + case "done": + caller.done = make(chan struct{}) + case "cancel": + cancel() + } + if err := private.verify(); axis == "cancel" && !errors.Is(err, context.Canceled) || + axis != "cancel" && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed actual caller lifetime was accepted", err) + } + }) + } +} diff --git a/amd64_go_frame_data.go b/amd64_go_frame_data.go new file mode 100644 index 000000000..ee072f79b --- /dev/null +++ b/amd64_go_frame_data.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "path/filepath" + "reflect" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +func (source *GoAMD64FrameSource) frameDataObject(expanded string, object []byte) (*gostaticstorage.AMD64AssemblyData, error) { + if !gotoolchain.AuditedGo127(source.env["GOVERSION"]) || source.numericalBinding != nil { + return nil, goAMD64FrameContext("mixed DATA needs its independent actual Go object producer") + } + builtin, err := source.read(filepath.Join(source.env["GOROOT"], "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + return nil, err + } + observed, err := gostaticstorage.ObserveAMD64AssemblyData([]byte(expanded), object, builtin, + source.selection.ImportPath, source.request.Target) + if err != nil { + return nil, goAMD64FrameContext("complete original DATA/source object: %v", err) + } + return &observed, nil +} + +// The SDK owner has already joined every original/expanded object to its actual +// compiler archive. Reuse that same observation, not a new standalone object +// with different producer flags. As with preprocessSDKFrame, these immutable +// fields remain covered by the enclosing live source verification. +func (source *GoAMD64FrameSource) fixedLocalDataObservation(name, expanded string) (*gostaticstorage.AMD64AssemblyData, error) { + if origin := source.sdkFrame; origin != nil { + if origin.identity != source || origin.owner == nil || origin.owner.source == nil { + return nil, goAMD64FrameContext("fixed local DATA has a foreign SDK source") + } + observation := &origin.owner.source.observation + for ordinal, input := range observation.ObjectInputs.Sources { + if input.Name != name { + continue + } + if observation.Objects == nil || ordinal >= len(observation.Objects.Sources) || + ordinal >= len(observation.CPP) || string(input.Expanded) != expanded || observation.CPP[ordinal].Output != expanded { + return nil, goAMD64FrameContext("fixed local DATA lost its complete SDK source/object pairing") + } + object := observation.Objects.Sources[ordinal] + if object.Name != input.Name { + return nil, goAMD64FrameContext("fixed local DATA SDK object order changed") + } + return &gostaticstorage.AMD64AssemblyData{ + Member: object.Original, Layouts: object.ExpandedLayouts, TextOrder: object.TextOrder, + }, nil + } + return nil, goAMD64FrameContext("fixed local DATA is absent from the complete SDK selection") + } + object, err := source.read(filepath.Join(source.owned, name+".o")) + if err != nil { + return nil, err + } + return source.frameDataObject(expanded, object) +} + +// Consume exact complete DATA blocks, never arbitrary listing lines. TEXT and +// Windows metadata remain in the original order for their independent parser. +// This observation grants no permission to use a DATA address or load its bytes. +func goAMD64FrameDataListing(listing []byte, observed *gostaticstorage.AMD64AssemblyData) ([]byte, error) { + if observed == nil || len(observed.Layouts) == 0 || len(listing) == 0 || listing[len(listing)-1] != '\n' { + return nil, goAMD64FrameContext("mixed DATA listing lacks a complete object/source inventory") + } + blocks := make(map[string][]string, len(observed.Layouts)) + var headers []string + for _, layout := range observed.Layouts { + block, err := goAMD64FrameDataBlock(layout) + if err != nil { + return nil, err + } + if _, duplicate := blocks[block[0]]; duplicate { + return nil, goAMD64FrameContext("mixed DATA listing has an ambiguous complete symbol header") + } + blocks[block[0]] = block + headers = append(headers, block[0]) + } + lines := strings.Split(string(listing), "\n") + var result strings.Builder + next := 0 + for index := 0; index < len(lines)-1; index++ { + block, found := blocks[lines[index]] + if !found { + result.WriteString(lines[index] + "\n") + continue + } + if next >= len(headers) || block[0] != headers[next] || + index+len(block) > len(lines)-1 || !reflect.DeepEqual(lines[index:index+len(block)], block) { + return nil, goAMD64FrameContext("mixed DATA listing bytes, extent or ordered relocations differ") + } + delete(blocks, block[0]) + next++ + index += len(block) - 1 + } + if len(blocks) != 0 { + return nil, goAMD64FrameContext("mixed DATA listing omits an original definition") + } + return []byte(result.String()), nil +} + +func goAMD64FrameDataBlock(layout gostaticstorage.ASMGlobalLayout) ([]string, error) { + definition := layout.Definition + kinds := map[uint8]string{3: "SRODATA", 4: "SRODATAFIPS", 5: "SNOPTRDATA", 6: "SNOPTRDATAFIPS", + 7: "SDATA", 8: "SDATAFIPS", 9: "SBSS", 10: "SNOPTRBSS", 11: "STLSBSS"} + kind := kinds[definition.Kind] + if kind == "" || definition.Name == "" || definition.Name != layout.Name || definition.ABI != layout.ABI { + return nil, goAMD64FrameContext("mixed DATA header lacks its exact source/object identity") + } + header := definition.Name + " " + kind + " " + if layout.Static { + header += "static " + } + if definition.Flags&1 != 0 { + header += "dupok " + } + // Go's GloblPos consumes DUPOK/RODATA/NOPTR/TLSBSS only; unrelated + // TEXT flags, including CFunc, do not change the DATA listing header. + header += fmt.Sprintf("size=%d align=%#x", definition.Size, definition.Align) + block := []string{header} + for offset := 0; offset < len(definition.Data); offset += 16 { + var row strings.Builder + fmt.Fprintf(&row, "\t%#04x", offset) + end := offset + 16 + if end > len(definition.Data) { + end = len(definition.Data) + } + for _, value := range definition.Data[offset:end] { + fmt.Fprintf(&row, " %02x", value) + } + row.WriteString(strings.Repeat(" ", offset+16-end) + " ") + for _, value := range definition.Data[offset:end] { + if value < ' ' || value > '~' { + value = '.' + } + row.WriteByte(value) + } + block = append(block, row.String()) + } + for _, relocation := range definition.Relocations { + if relocation.Type != 1 || !relocation.Target.ABIKnown { + return nil, goAMD64FrameContext("mixed DATA initializer has an unbound relocation") + } + block = append(block, fmt.Sprintf("\trel %d+%d t=R_ADDR %s+%d", + relocation.Offset, relocation.Size, relocation.Target.Name, relocation.Add)) + } + return block, nil +} + +func goAMD64FrameDataRelocationLine(relocation gostaticstorage.TextRelocation, observed *gostaticstorage.AMD64AssemblyData) string { + if line := goAMD64FramePrivateCallRelocation(relocation, observed); line != "" { + return line // Exact local TEXT identity only; the caller validates its leaf role. + } + if relocation.Type == 7 && relocation.Size == 4 && relocation.Add == 0 && + relocation.Target.ABIKnown && relocation.Target.ABI == 0 && relocation.Target.Name != "" { + for _, text := range observed.Member.Texts { + if text.Name == relocation.Target.Name { + return "" // A mismatched local TEXT cannot become an external call. + } + } + // Observe the original direct CALL spelling only. Its source, callee + // signature and frame permissions are validated independently. + return fmt.Sprintf("rel %d+%d t=R_CALL %s+0", + relocation.Offset, relocation.Size, relocation.Target.Name) + } + if relocation.Type != 14 || relocation.Size != 4 || !relocation.Target.ABIKnown { + return "" + } + packageIndices := map[gostaticstorage.TextSymbolNamespace]uint32{ + gostaticstorage.TextSymbolSelf: 2147483643, + gostaticstorage.TextSymbolHashed64: 2147483646, + gostaticstorage.TextSymbolHashed: 2147483645, + gostaticstorage.TextSymbolNonpkgDefinition: 2147483647, + } + for _, layout := range observed.Layouts { + definition := layout.Definition + if definition.Name != relocation.Target.Name { + continue + } + if definition.ABI == relocation.Target.ABI && + packageIndices[definition.Namespace] == relocation.Target.PackageIndex && + definition.Relative == relocation.Target.SymbolIndex { + return fmt.Sprintf("rel %d+%d t=R_PCREL %s+%d", + relocation.Offset, relocation.Size, relocation.Target.Name, relocation.Add) + } + return "" // A mismatched private definition cannot become an external role. + } + if relocation.Target.ABI == 0 && relocation.Target.Name != "" { + // Preserve ordinary references in the complete original TEXT inventory. + // This is only its exact object/listing spelling, not numerical access or + // address permission. Such a target is never added to private literalData. + return fmt.Sprintf("rel %d+%d t=R_PCREL %s+%d", + relocation.Offset, relocation.Size, relocation.Target.Name, relocation.Add) + } + return "" +} + +func goAMD64FrameDataTextJoin(functions map[string]goAMD64FrameFunction, observed *gostaticstorage.AMD64AssemblyData, + directory string, origins map[string]int, root string) error { + if len(functions) != len(observed.Member.Texts) { + return goAMD64FrameContext("mixed DATA object/listing complete TEXT inventories differ") + } + for _, text := range observed.Member.Texts { + fn, found := functions[text.Name] + if !found || text.ABI != fn.nativeABI || text.Kind != 1 && text.Kind != 2 || text.Flags != fn.nativeFlags() || text.Flags2 != 0 || !bytes.Equal(text.Code, fn.code) { + return goAMD64FrameContext("mixed DATA object/listing TEXT ABI, flags or complete bytes differ") + } + if err := goAMD64FrameSourcePCs(text, fn, directory, origins, root); err != nil { + return err + } + } + return nil +} diff --git a/amd64_go_frame_data_boundaries_go127_test.go b/amd64_go_frame_data_boundaries_go127_test.go new file mode 100644 index 000000000..fc78ed9dd --- /dev/null +++ b/amd64_go_frame_data_boundaries_go127_test.go @@ -0,0 +1,53 @@ +//go:build go1.27 + +package plan9asm + +import ( + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestGoAMD64FrameDataWholeMaskAESBoundaries(t *testing.T) { + var rows []string + for _, op := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + rows = append(rows, op+" literal<>+1(SB),K1", op+" K1,literal<>+1(SB)") + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, op := range []string{"VAESENC", "VAESENCLAST", "VAESDEC", "VAESDECLAST"} { + rows = append(rows, op+" literal<>+1(SB),"+bank+"0,"+bank+"1") + if bank != "Z" { + // Force the distinct EVEX.128/256 rows, not only VEX. + rows = append(rows, op+" literal<>+1(SB),"+bank+"0,"+bank+"16") + } + } + } + rows = append(rows, "VAESIMC literal<>+1(SB),X0", + "VAESKEYGENASSIST $255,literal<>+1(SB),X0", "VAESKEYGENASSIST $-128,literal<>+1(SB),X0") + for _, row := range rows { + t.Run(row, func(t *testing.T) { + code, pcs := x86NumericalBoundaryGoBytes(t, "amd64", row+"\nRET") + if len(pcs) != 2 { + t.Fatal("independent original instruction inventory changed", pcs) + } + decode := decodedX86MaskMoveInstruction + if strings.HasPrefix(row, "VAES") { + decode = decodedX86AESInstruction + } + if _, _, known, err := decode(code, 64); !known || err == nil || !strings.Contains(err.Error(), "RIP-relative") { + t.Fatal("ordinary raw decoding borrowed a physical RIP source", known, err) + } + boundaries, scalar, err := x86NumericalNativeBoundaries(code, 64) + if err != nil || len(boundaries) != 2 || boundaries[0].Length != int(pcs[1]) || + len(scalar) != 1 || scalar[pcs[1]].Op != x86asm.RET { + t.Fatal("whole original mask/AES boundary lost physical RIP", boundaries, scalar, err) + } + for cut := 1; cut < int(pcs[1]); cut++ { + if values, _, err := x86NumericalNativeBoundaries(code[:cut], 64); err == nil || values != nil { + t.Fatal("truncated original mask/AES acquired a boundary", cut, err) + } + } + }) + } +} diff --git a/amd64_go_frame_data_go127_test.go b/amd64_go_frame_data_go127_test.go new file mode 100644 index 000000000..5dc40aafa --- /dev/null +++ b/amd64_go_frame_data_go127_test.go @@ -0,0 +1,211 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestGoAMD64FrameDataOrdinaryReferenceIsNotPermission(t *testing.T) { + const assembly = `DATA literal<>+0(SB)/8,$7 +GLOBL literal<>(SB),8,$8 +TEXT ·Value(SB),4,$16-16 + MOVQ a+0(FP),AX + MOVQ AX,0(SP) + MOVQ literal<>(SB),CX + MOVQ $·Global(SB),BX + MOVQ 0(SP),AX + MOVQ AX,ret+8(FP) + RET +` + pkg, request, file, options := fixedLocalTransportFixture(t, assembly, + "func Global() {}\nfunc Value(a uint64) uint64\n", "linux/amd64", "x86_64-unknown-linux-gnu") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if err := source.Verify(ctx); err != nil { + t.Fatal("complete original DATA and ordinary reference", err) + } + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("observing an ordinary code reference granted a variable address role", err) + } +} + +// DATA coexistence is an original-object observation, not a new memory, +// address, native-entry or fixed-local permission. +func TestGoAMD64FrameSourceMixedDataOriginalObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + for _, used := range []bool{false, true} { + name := "unused" + if used { + name = "referenced" + } + t.Run(target+"/"+name, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = target + assembly := "DATA literal<>+0(SB)/8,$0x102030405060708\n" + + "DATA literal<>+16(SB)/4,$\"abcd\"\n" + + "GLOBL literal<>(SB),8,$32\n" + + "DATA mutable<>+0(SB)/8,$0x55667788\n" + + "GLOBL mutable<>(SB),16,$8\n" + + "GLOBL zero<>(SB),16,$24\n" + + "DATA pointer<>+0(SB)/8,$literal<>+8(SB)\n" + + "GLOBL pointer<>(SB),8,$8\n" + + "TEXT ·Value(SB),4,$160-16\n" + + "MOVQ a+0(FP),AX\nMOVQ AX,16(SP)\n" + if used { + assembly += "MOVQ literal<>+8(SB),CX\n" + } + assembly += "MOVQ 16(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), + "GOOS="+strings.Split(target, "/")[0], "GOARCH=amd64", "CGO_ENABLED=0", + "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=") + listing, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, + "tool", "asm", "-S", "-p", pkg.Path, "-o", filepath.Join(t.TempDir(), "original.o"), request.AssemblyFile) + if err != nil { + t.Fatalf("independent original Go mixed TEXT/DATA control: %v\n%s", err, listing) + } + t.Logf("original Go listing:\n%s", listing) + source := requireGoAMD64FrameCapture(t, pkg, request) + if len(source.functions) != 1 || source.functions[pkg.Path+".Value"].locals != 168 { + t.Fatal("mixed DATA lost the original complete frame inventory") + } + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + goAMD64FrameMixedDataJoinGuards(t, pkg, source, used) + changed := strings.Replace(assembly, "0x102030405060708", "0x102030405060709", 1) + path := filepath.Join(request.Directory, request.AssemblyFile) + if err := os.WriteFile(path, []byte(changed), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed original DATA retained the frame observation", err) + } + if err := os.WriteFile(path, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("restored DATA revived a revoked frame observation", err) + } + }) + } + } +} + +func goAMD64FrameMixedDataJoinGuards(t *testing.T, pkg GoPackage, source *GoAMD64FrameSource, used bool) { + t.Helper() + objectPath := filepath.Join(source.owned, source.request.AssemblyFile+".o") + object, err := source.read(objectPath) + if err != nil { + t.Fatal(err) + } + listing, err := source.read(objectPath + ".listing") + if err != nil { + t.Fatal(err) + } + original, err := source.read(filepath.Join(source.request.Directory, source.request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + expanded, err := source.preprocess(source.request.AssemblyFile, original) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + observed, err := source.frameDataObject(expanded, object) + if err != nil { + t.Fatal(err) + } + var unwind *gostaticstorage.WindowsAMD64AssemblyUnwind + if source.request.Target == "windows/amd64" { + value, err := source.observeWindowsUnwind(object) + if err != nil { + t.Fatal(err) + } + unwind = &value + } + check := func(input []byte) error { + filtered, err := goAMD64FrameDataListing(input, observed) + if err != nil { + return err + } + functions, err := goAMD64FrameListingWithDataObject(filtered, file, pkg, source.origins, observed, unwind, + source.request.Target, source.request.Directory, "") + if err != nil { + return err + } + return goAMD64FrameDataTextJoin(functions, observed, source.request.Directory, source.origins, "") + } + if err := check(listing); err != nil { + t.Fatal("complete unmodified source/object/listing control", err) + } + changes := map[string][2]string{ + "DATA kind": {"literal SRODATA", "literal SDATA"}, + "DATA static": {"literal SRODATA static", "literal SRODATA"}, + "DATA dupok": {"literal SRODATA static", "literal SRODATA static dupok"}, + "DATA size": {"size=32 align=0x0", "size=33 align=0x0"}, + "DATA alignment": {"size=32 align=0x0", "size=32 align=0x8"}, + "DATA byte": {"08 07 06 05", "09 07 06 05"}, + "DATA ASCII": {"abcd\n", "abce\n"}, + "DATA offset": {"0x0010 61 62 63 64", "0x0011 61 62 63 64"}, + "DATA relocation": {"R_ADDR literal+8", "R_ADDR literal+16"}, + "DATA target": {"R_ADDR literal+8", "R_ADDR mutable+8"}, + "DATA BSS extent": {"zero SNOPTRBSS static size=24", "zero SNOPTRBSS static size=25"}, + "TEXT bytes": {"55 48 89 e5", "90 48 89 e5"}, + } + if used { + changes["TEXT relocation"] = [2]string{"R_PCREL literal+8", "R_PCREL literal+16"} + changes["TEXT target"] = [2]string{"R_PCREL literal+8", "R_PCREL mutable+8"} + } + for name, change := range changes { + t.Run(name, func(t *testing.T) { + changed := bytes.Replace(listing, []byte(change[0]), []byte(change[1]), 1) + if bytes.Equal(changed, listing) { + t.Fatal("listing mutation did not touch the independent Go control") + } + if err := check(changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed complete listing acquired an observation", err) + } + }) + } + for _, suffix := range []string{"UNKNOWN_NATIVE_RECORD\n", "\trel 0+8 t=R_ADDR literal+8\n", "zero SNOPTRBSS static size=24 align=0x0\n"} { + if err := check(append(bytes.Clone(listing), suffix...)); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("extra DATA/listing record was ignored", err) + } + } + if used { + relocation := observed.Member.Texts[0].Relocations[0] + for _, mutate := range []func(*gostaticstorage.TextRelocation){ + func(r *gostaticstorage.TextRelocation) { r.Type = 1 }, + func(r *gostaticstorage.TextRelocation) { r.Size = 8 }, + func(r *gostaticstorage.TextRelocation) { r.Target.ABIKnown = false }, + func(r *gostaticstorage.TextRelocation) { r.Target.ABI = 1 }, + func(r *gostaticstorage.TextRelocation) { r.Target.PackageIndex = 2147483647 }, + func(r *gostaticstorage.TextRelocation) { r.Target.SymbolIndex++ }, + } { + changed := relocation + mutate(&changed) + if changed == relocation || goAMD64FrameDataRelocationLine(changed, observed) != "" { + t.Fatal("DATA relocation lost exact object-local identity", changed) + } + } + } +} diff --git a/amd64_go_frame_fixture_flags_test.go b/amd64_go_frame_fixture_flags_test.go new file mode 100644 index 000000000..2a495be80 --- /dev/null +++ b/amd64_go_frame_fixture_flags_test.go @@ -0,0 +1,34 @@ +package plan9asm + +import ( + "errors" + "os" + "testing" +) + +func TestGoAMD64FrameFixtureSeparatesHarnessBuildTags(t *testing.T) { + // The Windows LLVM harness needs this tag; the independent, pure-Go + // source fixture does not import LLVM and must have its own build role. + t.Setenv("GOFLAGS", "-tags=byollvm") + for _, target := range []string{"linux/amd64", "windows/amd64"} { + t.Run(target, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = target + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + if os.Getenv("GOFLAGS") != "-tags=byollvm" { + t.Fatal("fixture changed its parent harness environment") + } + // An explicit caller request is still nonordinary. This fixture + // separation cannot relax production profile/source validation. + request.Environment = append(request.Environment, "GOFLAGS=-tags=foreign") + if changed, err := CaptureGoAMD64FrameSource(ctx, pkg, request); changed != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("explicit foreign source profile was accepted", err) + } + }) + } +} diff --git a/amd64_go_frame_foreign_call_test.go b/amd64_go_frame_foreign_call_test.go new file mode 100644 index 000000000..232c1b3ab --- /dev/null +++ b/amd64_go_frame_foreign_call_test.go @@ -0,0 +1,358 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +type foreignABI0CallFixture struct { + name, target, declarations, outgoing string + args []LLVMType +} + +func foreignABI0CallFixtures() []foreignABI0CallFixture { + return []foreignABI0CallFixture{ + { + name: "SDK-copy-pointer-pointer-word", target: "runtime.memmove", + declarations: "import _ \"runtime\"\nfunc Value(dst, src *byte, n uintptr) uint64\n", + outgoing: "MOVQ AX,0(SP)\nMOVQ BX,8(SP)\nMOVQ CX,16(SP)\n", + args: []LLVMType{Ptr, Ptr, I64}, + }, + { + name: "SDK-clear-pointer-word", target: "runtime.memclrNoHeapPointers", + declarations: "import _ \"runtime\"\nfunc Value(dst, src *byte, n uintptr) uint64\n", + outgoing: "MOVQ AX,0(SP)\nMOVQ CX,8(SP)\n", + args: []LLVMType{Ptr, I64}, + }, + { + name: "local-Go-body-pointer-pointer-word", target: "example.invalid/framefixture.Callee", + declarations: "func Value(dst, src *byte, n uintptr) uint64\n" + + "//go:noinline\nfunc Callee(dst, src *byte, n uintptr) { if n != 0 { *dst = *src } }\n", + outgoing: "MOVQ AX,0(SP)\nMOVQ BX,8(SP)\nMOVQ CX,16(SP)\n", + args: []LLVMType{Ptr, Ptr, I64}, + }, + } +} + +func foreignABI0CallAssembly(fixture foreignABI0CallFixture) string { + target := strings.ReplaceAll(fixture.target, ".", "·") + if strings.HasPrefix(fixture.target, "example.invalid/framefixture.") { + target = "·Callee" + } + return "TEXT ·Value(SB),4,$64-32\n" + + "MOVQ dst+0(FP),AX\nMOVQ src+8(FP),BX\nMOVQ n+16(FP),CX\n" + fixture.outgoing + + "MOVQ AX,24(SP)\nMOVQ BX,32(SP)\nMOVQ CX,40(SP)\nMOVQ $37,48(SP)\n" + + "CALL " + target + "(SB)\n" + + "MOVQ 24(SP),R8\nMOVQ 32(SP),R9\nMOVQ 40(SP),AX\nADDQ 48(SP),AX\n" + + "MOVQ AX,ret+24(FP)\nRET\n" +} + +func foreignABI0Fixture(t *testing.T, fixture foreignABI0CallFixture, assembly string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options) { + t.Helper() + _, request := goAMD64FrameSourceFixture(t) + filename := filepath.Join(request.Directory, "value.go") + source := "package framefixture\n" + fixture.declarations + if err := os.WriteFile(filename, []byte(source), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, filename, source, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", "amd64"), Importer: importer.Default()}).Check( + "example.invalid/framefixture", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + pkg := GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{syntax}, Imports: make(map[string]*types.Package)} + for _, dependency := range typed.Imports() { + pkg.Imports[dependency.Path()] = dependency + } + caller, err := DeriveGoFuncABI0Signature(pkg.Path+".Value", typed.Scope().Lookup("Value").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + callee := FuncSig{Name: fixture.target, Args: append([]LLVMType(nil), fixture.args...), Ret: Void} + for index, typ := range fixture.args { + callee.Frame.Params = append(callee.Frame.Params, FrameSlot{Offset: int64(index) * 8, Type: typ, Index: index, Field: -1}) + } + // ABI0 parameter metadata intentionally has empty Name fields; the real + // producer retains source names on flattened results, not parameters. + if fixture.target == pkg.Path+".Callee" { + actual, err := DeriveGoFuncABI0Signature(fixture.target, typed.Scope().Lookup("Callee").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil || !reflect.DeepEqual(actual, callee) { + t.Fatalf("actual/consumer complete ABI0 metadata differs: err=%v\nactual=%#v\nconsumer=%#v", err, actual, callee) + } + callee = actual + } + // This independent literal layout is a consumer expectation, never an SDK + // source/ABI certificate. Capture must derive and authenticate its own callee. + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + return pkg, request, file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: testResolveSym(pkg.Path), + Sigs: map[string]FuncSig{caller.Name: caller, callee.Name: callee}, + } +} + +func foreignABI0OriginalLink(t *testing.T, request GoAMD64FrameCaptureRequest) { + t.Helper() + // A package archive alone leaves cross-package ABI0 references unresolved. + // Link an owned complete main so a missing SDK adapter is fixture failure, + // not translator RED. The child is not a declaration/source authority. + directory := filepath.Join(request.Directory, "linkprobe") + if err := os.Mkdir(directory, 0700); err != nil { + t.Fatal(err) + } + main := "package main\nimport probe \"example.invalid/framefixture\"\n" + + "func main() { var a, b byte; if probe.Value(&a, &b, 0) != 37 { panic(\"call result\") } }\n" + if err := os.WriteFile(filepath.Join(directory, "main.go"), []byte(main), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", + "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOPROXY=off") + binary := filepath.Join(t.TempDir(), "original-linked") + output, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, "build", "-p=1", "-o", binary, "./linkprobe") + if err != nil { + t.Fatalf("original Go complete main did not link the actual ABI0 target (not translator RED): %v\n%s", err, output) + } + if info, err := os.Stat(binary); err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatal("original Go omitted complete linked executable", err) + } +} + +func TestGoAMD64FrameForeignABI0OriginalGoLink(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, fixture := range foreignABI0CallFixtures() { + t.Run(fixture.name, func(t *testing.T) { + _, request, _, _ := foreignABI0Fixture(t, fixture, foreignABI0CallAssembly(fixture)) + foreignABI0OriginalLink(t, request) + }) + } +} + +func TestGoAMD64FrameForeignABI0PointersCaptureAndObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, fixture := range foreignABI0CallFixtures() { + t.Run(fixture.name, func(t *testing.T) { + pkg, request, file, options := foreignABI0Fixture(t, fixture, foreignABI0CallAssembly(fixture)) + foreignABI0OriginalLink(t, request) + t.Run("signatures-alone", func(t *testing.T) { + fixedLocalAPIs(t, pkg, file, options, func(t *testing.T, ir string, err error) { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("callee signature alone authorized a physical frame/call", err, ir) + } + }) + }) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + foreignABI0CallObjects(t, fixture, ir, err) + }) + }) + } +} + +func foreignABI0CallObjects(t *testing.T, fixture foreignABI0CallFixture, ir string, err error) { + t.Helper() + global := llvmGlobal(fixture.target) + // LLVM's printer can remove optional identifier quotes. Only canonicalize + // this exact target token to the existing formatter. + normalized := strings.ReplaceAll(ir, "@"+fixture.target+"(", global+"(") + call := "call void " + global + "(ptr " + if err != nil || !strings.Contains(normalized, call) || strings.Contains(normalized, "define void "+global+"(") { + t.Fatal("an actual separate typed ABI0 callee was rejected, dropped or privately inlined", err, ir) + } + ordinaryGoFrameCallObjects(t, ir) +} + +func TestGoAMD64FrameForeignABI0SDKDefaultsStayOwnedAndStrict(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + fixture := foreignABI0CallFixtures()[0] + pkg, request, file, options := foreignABI0Fixture(t, fixture, foreignABI0CallAssembly(fixture)) + expected := options.Sigs[fixture.target] + delete(options.Sigs, fixture.target) + foreignABI0OriginalLink(t, request) + t.Run("owner-derived-three-APIs", func(t *testing.T) { + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + foreignABI0CallObjects(t, fixture, ir, err) + }) + if _, changed := options.Sigs[fixture.target]; changed { + t.Fatal("binding inserted a callee into the consumer's original map") + } + }) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := owner.Close(); err != nil { + t.Error(err) + } + }() + bound, err := owner.BindFixedLocal(ctx, pkg, file, options) + if err != nil || !reflect.DeepEqual(bound.Sigs[fixture.target], expected) { + t.Fatalf("owner default differs from independently expected complete ABI0 metadata: %v\nactual=%#v\nexpected=%#v", err, bound.Sigs[fixture.target], expected) + } + t.Run("public-default-slices-cannot-change-owner", func(t *testing.T) { + changed := bound.Sigs[fixture.target] + changed.Frame.Params[0].Offset++ + bound.Sigs[fixture.target] = changed + if !reflect.DeepEqual(owner.callees[fixture.target].signature, expected) { + t.Fatal("public default slices alias the private source declaration") + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("mutated derived metadata retained a callable signature", err, ir) + } + }) + }) + t.Run("caller-signature-is-not-derived", func(t *testing.T) { + missing := options + missing.Sigs = nil + if _, err := owner.BindFixedLocal(ctx, pkg, file, missing); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a foreign SDK callee granted the caller's missing signature", err) + } + }) + t.Run("same-package-callee-stays-explicit", func(t *testing.T) { + local := foreignABI0CallFixtures()[2] + pkg, request, file, options := foreignABI0Fixture(t, local, foreignABI0CallAssembly(local)) + delete(options.Sigs, local.target) + actual, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := actual.Close(); err != nil { + t.Error(err) + } + }() + if _, err := actual.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("the foreign SDK default widened same-package consumer metadata", err) + } + }) +} + +func TestGoAMD64FrameForeignABI0SuppliedLayoutAndSourceBoundaries(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + fixture := foreignABI0CallFixtures()[0] + pkg, request, file, options := foreignABI0Fixture(t, fixture, foreignABI0CallAssembly(fixture)) + foreignABI0OriginalLink(t, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("actual SDK callee must capture before its independent consumer negatives", err) + } + defer func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }() + t.Logf("actual SDK callee complete signature=%#v\nconsumer=%#v", source.callees[fixture.target].signature, options.Sigs[fixture.target]) + for _, name := range []string{"wrong-register-ABI", "wrong-slot", "wrong-pointer-width", "foreign-resolver"} { + t.Run(name, func(t *testing.T) { + changed := options + changed.Sigs, err = cloneFixedLocalSigs(options.Sigs) + if err != nil { + t.Fatal(err) + } + sig := changed.Sigs[fixture.target] + switch name { + case "wrong-register-ABI": + sig.ArgRegs = []Reg{AX, BX, CX} + case "wrong-slot": + sig.Frame.Params[1].Offset++ + case "wrong-pointer-width": + sig.Args[0], sig.Frame.Params[0].Type = I32, I32 + case "foreign-resolver": + changed.ResolveSym = func(string) string { return "other.Callee" } + } + changed.Sigs[fixture.target] = sig + if _, err := source.BindFixedLocal(ctx, pkg, file, changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("supplied ABI/resolver substituted an actual SDK call", err) + } + }) + } + // A rejected consumer does not substitute for this independent source test: + // use a fresh producer before changing the actual declaration on disk. + mutationSource, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("fresh producer needed before source-mutation negative", err) + } + defer func() { + if err := mutationSource.Close(); err != nil { + t.Error(err) + } + }() + path := filepath.Join(request.Directory, "value.go") + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + after := strings.Replace(string(before), "n uintptr", "n uint32", 1) + if after == string(before) { + t.Fatal("source mutation did not change the real declaration") + } + if err := os.WriteFile(path, []byte(after), 0600); err != nil { + t.Fatal(err) + } + if !errors.Is(mutationSource.Verify(ctx), ErrProbeNeedsContext) { + t.Fatal("mutated source retained its call lease") + } +} + +func TestGoAMD64FrameForeignABI0SDKOwnerCannotBeRetaggedOrRevived(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + fixture := foreignABI0CallFixtures()[0] + pkg, request, _, _ := foreignABI0Fixture(t, fixture, foreignABI0CallAssembly(fixture)) + foreignABI0OriginalLink(t, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("actual SDK call must capture before source-owner retagging", err) + } + defer func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }() + selected, found := source.imports["runtime"] + if !found || !selected.Standard || selected.Export == "" || !filepath.IsAbs(selected.Dir) { + t.Fatal("SDK target lost its actual private selected-source owner") + } + changed := selected + changed.Standard = false + source.imports["runtime"] = changed + if !errors.Is(source.Verify(ctx), ErrProbeNeedsContext) { + t.Fatal("retagged SDK package still authorized the callee") + } + source.imports["runtime"] = selected + if !errors.Is(source.Verify(ctx), ErrProbeNeedsContext) { + t.Fatal("restoring selected metadata revived a revoked SDK call owner") + } +} diff --git a/amd64_go_frame_foreign_call_transfer_test.go b/amd64_go_frame_foreign_call_transfer_test.go new file mode 100644 index 000000000..1400cba3a --- /dev/null +++ b/amd64_go_frame_foreign_call_transfer_test.go @@ -0,0 +1,57 @@ +package plan9asm + +import ( + "errors" + "testing" +) + +func TestGoAMD64FrameForeignABI0TypedPointerTransfer(t *testing.T) { + sig := FuncSig{Name: "foreign.Consume", Args: []LLVMType{Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}}} + binding := &x86FixedLocalBinding{symbols: map[string]string{"foreign·Consume": sig.Name}, + signatures: map[string]FuncSig{sig.Name: sig}, + callees: map[string]goAMD64FrameDeclaration{sig.Name: {signature: sig, transport: goAMD64FrameCompositeTransport}}} + proof := &x86FixedLocalProof{binding: binding, frame: 64} + call := x86ScalarLaneInstruction(t, "CALL foreign·Consume(SB)") + before := fixedLocalExchangeState() + origin := fixedLocalOrigin(x86FixedLocalOrigin{parameter: 0}) + before.opaqueCells[0], before.opaqueCells[24] = origin, origin + before.bytes.setRange(8, 8, true) + before.bytes.setRange(40, 8, true) + before.addresses[48] = 56 + before.known[AX], before.vectors["Z0"], before.masks["K1"] = 255, ^uint64(0), true + before.flags = x86ByteFlagCF + t.Run("opaque-word-and-independent-spills", func(t *testing.T) { + state := cloneFixedLocalState(before) + if err := fixedLocalOrdinaryCallStep(call, &state, proof); err != nil { + t.Fatal("typed opaque pointer argument was not accepted", err) + } + if state.opaqueCells[0].valid() || !state.opaqueCells[24].equal(origin) || !state.bytes.all(40, 8) || + state.addresses[48] != 56 || state.bytes.all(8, 8) || len(state.known)+len(state.opaque)+len(state.vectors)+len(state.masks) != 0 || state.flags != 0 { + t.Fatal("CALL lost private spills or retained outgoing/scratch authority", state) + } + }) + for _, name := range []string{"missing-pointer", "partial-numeric-word", "private-frame-pointer", "unknown-callee", "changed-SP"} { + t.Run(name, func(t *testing.T) { + state := cloneFixedLocalState(before) + instruction := call + switch name { + case "missing-pointer": + delete(state.opaqueCells, 0) + case "partial-numeric-word": + delete(state.opaqueCells, 0) + state.bytes.setRange(0, 7, true) + case "private-frame-pointer": + delete(state.opaqueCells, 0) + state.addresses[0] = 40 + case "unknown-callee": + instruction = x86ScalarLaneInstruction(t, "CALL foreign·Other(SB)") + case "changed-SP": + state.pointers[SP] = 8 + } + if err := fixedLocalOrdinaryCallStep(instruction, &state, proof); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized/unbound/private pointer acquired a call contract", err) + } + }) + } +} diff --git a/amd64_go_frame_global_go127_test.go b/amd64_go_frame_global_go127_test.go new file mode 100644 index 000000000..4071c5fdf --- /dev/null +++ b/amd64_go_frame_global_go127_test.go @@ -0,0 +1,104 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "errors" + "path/filepath" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +const fixedLocalGlobalAssembly = `TEXT ·Value(SB),4,$32-16 + MOVQ a+0(FP),AX + MOVQ AX,0(SP) + MOVQ $·Table(SB),SI + MOVQ SI,8(SP) + MOVQ 8(SP),DX + MOVOU (DX),X0 + ADDQ $16,DX + MOVOU (DX),X1 + PADDQ X1,X0 + MOVQ X0,CX + MOVQ 0(SP),AX + ADDQ CX,AX + MOVQ AX,ret+8(FP) + RET +` + +const fixedLocalGlobalDeclarations = "var Table [4]uint64\nfunc Value(a uint64) uint64\n" + +func TestGoAMD64FrameOrdinaryRelocationWithoutData(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + t.Run(target, func(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, fixedLocalGlobalDeclarations, fixedLocalGlobalAssembly) + request.Target = target + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + object, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o")) + if err != nil { + t.Fatal(err) + } + listing, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + expanded, err := source.preprocess(request.AssemblyFile, []byte(fixedLocalGlobalAssembly)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + observed, err := source.frameDataObject(expanded, object) + if err != nil || len(observed.Layouts) != 0 || len(observed.Member.Texts) != 1 || len(observed.Member.Texts[0].Relocations) != 1 { + t.Fatal("ordinary reference changed the complete no-DATA inventory", err) + } + var unwind *gostaticstorage.WindowsAMD64AssemblyUnwind + if target == "windows/amd64" { + value, err := source.observeWindowsUnwind(object) + if err != nil { + t.Fatal(err) + } + unwind = &value + } + check := func(input []byte) error { + functions, err := goAMD64FrameListingWithDataObject(input, file, pkg, source.origins, observed, unwind, target, request.Directory, "") + if err != nil { + return err + } + return goAMD64FrameDataTextJoin(functions, observed, request.Directory, source.origins, "") + } + if err := check(listing); err != nil { + t.Fatal("complete unmodified original source/object/listing", err) + } + for name, replacement := range map[string]string{ + "target": "foreign.Table+0", "addend": pkg.Path + ".Table+1", "missing zero": pkg.Path + ".Table", + } { + t.Run(name, func(t *testing.T) { + changed := bytes.Replace(listing, []byte(pkg.Path+".Table+0"), []byte(replacement), 1) + if bytes.Equal(changed, listing) || !errors.Is(check(changed), ErrProbeNeedsContext) { + t.Fatal("changed ordinary relocation retained the complete original observation") + } + }) + } + relocation := []byte("\t" + goAMD64FrameDataRelocationLine(observed.Member.Texts[0].Relocations[0], observed) + "\n") + for name, replacement := range map[string][]byte{"missing": nil, "duplicate": append(append([]byte(nil), relocation...), relocation...)} { + t.Run(name, func(t *testing.T) { + changed := bytes.Replace(listing, relocation, replacement, 1) + if bytes.Equal(changed, listing) || !errors.Is(check(changed), ErrProbeNeedsContext) { + t.Fatal("incomplete or duplicate relocation inventory accepted") + } + }) + } + }) + } +} diff --git a/amd64_go_frame_identity_test.go b/amd64_go_frame_identity_test.go new file mode 100644 index 000000000..6e3da0cc9 --- /dev/null +++ b/amd64_go_frame_identity_test.go @@ -0,0 +1,83 @@ +package plan9asm + +import ( + "context" + "errors" + "os" + "reflect" + "testing" +) + +func TestGoOriginalInputGuardCannotSupplyFrameAuthority(t *testing.T) { + request := GoAMD64FrameCaptureRequest{Environment: []string{"GOFLAGS="}} + first, second := newGoOriginalInputGuard(request), newGoOriginalInputGuard(request) + request.Environment[0] = "GOFLAGS=-overlay=foreign.json" + if first.request.Environment[0] != "GOFLAGS=" || second.request.Environment[0] != "GOFLAGS=" { + t.Fatal("query-only guard retained caller environment storage") + } + first.inputs["first-only"] = [32]byte{1} + if len(second.inputs) != 0 { + t.Fatal("independent query-only guards shared input maps") + } + for _, source := range []*GoAMD64FrameSource{first, second} { + if source.ready || source.owned != "" || len(source.functions) != 0 || len(source.declarations) != 0 || + source.downloadedOrigin != nil || source.downloadedModule != nil { + t.Fatal("input initialization created an object or original-source capability") + } + if !errors.Is(source.Verify(context.Background()), ErrProbeNeedsContext) { + t.Fatal("query-only guard verified as a frame") + } + if _, err := source.BindFixedLocal(context.Background(), GoPackage{}, nil, Options{}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("query-only guard admitted fixed-local memory", err) + } + if err := source.Close(); err != nil { + t.Fatal("original query guard could not close its empty resource role", err) + } + } +} + +func TestGoAMD64FrameSourceCopyCannotBorrowOrCleanOriginal(t *testing.T) { + for _, operation := range []string{"Verify", "Close"} { + t.Run(operation, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }() + if err := source.Verify(ctx); err != nil { + t.Fatal("actual original public factory did not verify", err) + } + owned := source.owned + before, err := os.Lstat(owned) + if err != nil { + t.Fatal(err) + } + // Deliberately model the public whole-value operation without hiding + // a mutex copy behind a constructor or serializing private fields. + copied := new(GoAMD64FrameSource) + reflect.ValueOf(copied).Elem().Set(reflect.ValueOf(source).Elem()) + if operation == "Verify" { + err = copied.Verify(ctx) + } else { + err = copied.Close() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("whole-value copied frame %s borrowed its original owner role: %v", operation, err) + } + after, err := os.Lstat(owned) + if err != nil || !os.SameFile(before, after) { + t.Error("copied frame operation removed or replaced original artifacts", err) + } + if err := source.Verify(ctx); err != nil { + t.Error("copied frame operation revoked its original object", err) + } + }) + } +} diff --git a/amd64_go_frame_macro_test.go b/amd64_go_frame_macro_test.go new file mode 100644 index 000000000..ecdb769e3 --- /dev/null +++ b/amd64_go_frame_macro_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestGoAMD64FrameSourceConditionalMacrosMatchActualObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for targetIndex, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + for _, profile := range []string{"v1", "v2", "v3", "v4"} { + t.Run(target+"/"+profile, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = target + request.Environment = append(request.Environment, "GOAMD64="+profile) + assembly := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(assembly) + if err != nil { + t.Fatal(err) + } + header := `#ifndef PROFILE_GUARD +#define PROFILE_GUARD +#ifdef GOAMD64_v2 +#define PROFILE_VALUE 7 +#else +#define PROFILE_VALUE 11 +#endif +#ifdef GOOS_linux +#define TARGET_VALUE 1 +#else +#ifdef GOOS_darwin +#define TARGET_VALUE 2 +#else +#define TARGET_VALUE 3 +#endif +#endif +#endif +` + headerFile := filepath.Join(request.Directory, "profile.h") + if err := os.WriteFile(headerFile, []byte(header), 0600); err != nil { + t.Fatal(err) + } + prefix := `#include "profile.h" +#include "profile.h" +#ifdef UNDEFINED_EXTERNAL_MACRO +#include "missing_generated.h" +#endif +` + text := prefix + strings.Replace(string(original), "MOVQ AX,ret+8(FP)", "MOVQ $(PROFILE_VALUE+TARGET_VALUE),AX\nMOVQ AX,ret+8(FP)", 1) + if err := os.WriteFile(assembly, []byte(text), 0600); err != nil { + t.Fatal(err) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + function, found := source.functions[pkg.Path+".Value"] + if !found { + t.Fatal("complete original function is missing") + } + want := int64(11 + targetIndex + 1) + if profile == "v2" { + want -= 4 + } + matched := 0 + for code := function.code; len(code) != 0; { + instruction, err := x86asm.Decode(code, 64) + if err != nil || instruction.Len == 0 { + t.Fatal("decode actual conditional Go object", err) + } + _, immediate := instruction.Args[1].(x86asm.Imm) + // Current Go encodes these small MOVQ constants as a + // zero-extending EAX write; Go 1.20 retains RAX's prefix. + valueRegister := instruction.Args[0] == x86asm.RAX || instruction.Args[0] == x86asm.EAX + if instruction.Op == x86asm.MOV && valueRegister && immediate { + if instruction.Args[1] != x86asm.Imm(want) { + t.Fatal("conditional branch differs from independent profile/target", instruction, want) + } + matched++ + } + code = code[instruction.Len:] + } + if matched != 1 { + t.Fatal("missing or repeated independently observed conditional result", matched) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + expanded, err := source.preprocess(request.AssemblyFile, []byte(text)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + triple := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"}[targetIndex] + options := Options{Goarch: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) }, + Sigs: map[string]FuncSig{function.signature.Name: function.signature}, + } + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "conditional.ll", "conditional.o", ir) + compileLLVMToObject(t, llc, triple, "conditional-opt.ll", "conditional-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(headerFile, []byte(header+"// late mutation\n"), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); err == nil { + t.Fatal("changed conditional header retained source authority") + } + if err := os.WriteFile(headerFile, []byte(header), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); err == nil { + t.Fatal("restored header revived a revoked observation") + } + t.Logf("actual %s/%s selected MOVQ $%d,AX", target, profile, want) + }) + } + } +} + +func TestGoAMD64FrameSourceConditionalMacrosGrantNoNumericEffects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, operand := range []string{"$7", "R11"} { + t.Run(operand, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + name := filepath.Join(request.Directory, request.AssemblyFile) + text, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + // Retain the original ADDQ $7 source as a positive now that its + // typed family is implemented. Macro selection still cannot supply + // the missing numerical bytes of an uninitialized register. + body := "#ifdef GOARCH_amd64\n" + strings.Replace(string(text), "MOVQ AX,ret+8(FP)", + "ADDQ "+operand+",AX\nMOVQ AX,ret+8(FP)", 1) + "#endif\n" + if err := os.WriteFile(name, []byte(body), 0600); err != nil { + t.Fatal(err) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + expanded, err := source.preprocess(request.AssemblyFile, []byte(body)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + sig := source.functions[pkg.Path+".Value"].signature + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) }, + Sigs: map[string]FuncSig{sig.Name: sig}, + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if operand == "R11" { + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "ADDQ") { + t.Fatal("macro selection supplied missing ADDQ input bytes", err) + } + return + } + if err != nil { + t.Fatal("initialized ADDQ frame binding failed", err) + } + ir, err := Translate(file, bound) + if err != nil || ir == "" { + t.Fatal("original initialized ADDQ lost its independently bound translation", err) + } + }) + } +} + +func TestGoAMD64FrameSourceCPPRequiresActualRegistration(t *testing.T) { + source := newGoOriginalInputGuard(GoAMD64FrameCaptureRequest{Directory: t.TempDir()}) + source.env = map[string]string{"GOOS": "linux", "GOARCH": "amd64", "GOAMD64": "v2"} + if _, err := source.preprocess("value.s", []byte("#ifdef GOAMD64_v2\nTEXT value(SB),4,$0-0\nRET\n#endif\n")); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("environment strings impersonated registered actual macro sources", err) + } +} diff --git a/amd64_go_frame_numerical_source.go b/amd64_go_frame_numerical_source.go new file mode 100644 index 000000000..edc44c01d --- /dev/null +++ b/amd64_go_frame_numerical_source.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "path/filepath" + "reflect" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "golang.org/x/arch/x86/x86asm" +) + +func verifyGoAMD64FrameNumericalRequest(ctx context.Context, pkg GoPackage, request GoAMD64FrameCaptureRequest) error { + binding := request.NumericalReads + if err := binding.verify(); err != nil { + return err + } + if ctx != binding.context || request.Target != "linux/amd64" || request.Target != binding.source.request.Target || + request.GoBinary != binding.source.request.GoBinary || request.Directory != binding.source.request.Directory || + request.AssemblyFile != binding.source.request.AssemblyFile || pkg.Path != binding.packageInput.Path || + pkg.Types != binding.originalTypes || pkg.SourceSelection != nil || !numericalSameImports(pkg.Imports, binding.originalImports) { + return goAMD64FrameContext("numerical frame join has foreign complete source/package/tool/profile/context") + } + environment := func(values []string) (map[string]string, bool) { + result := make(map[string]string, len(values)) + for _, pair := range values { + key, value, valid := strings.Cut(pair, "=") + if _, duplicate := result[key]; !valid || key == "" || duplicate { + return nil, false + } + result[key] = value + } + return result, true + } + provided, valid := environment(request.Environment) + original, originalValid := environment(binding.source.request.Environment) + if !valid || !originalValid || !reflect.DeepEqual(provided, original) { + return goAMD64FrameContext("numerical frame join command environment differs") + } + binding.source.mu.Lock() + err := numericalDeclarationSyntax(binding.source, pkg) + binding.source.mu.Unlock() + return err +} + +func (source *GoAMD64FrameSource) revokeNumericalFrame() { + source.mu.Lock() + source.ready = false + source.mu.Unlock() +} + +// No Frame lock survives an external numerical/owner verification. Close can +// linearize during the independent frame round, but its final guard must fail. +func (source *GoAMD64FrameSource) verifyNumericalFrameInputs(ctx context.Context) error { + source.mu.Lock() + request, binding, seal, pkg, ready, origin := source.request, source.numericalBinding, source.numericalSeal, source.numericalPackage, source.ready, source.numericalOrigin + selection := source.selection + inputs := make(map[string][32]byte, len(source.inputs)) + for name, sum := range source.inputs { + inputs[name] = sum + } + environment := make(map[string]string, len(source.env)) + for name, value := range source.env { + environment[name] = value + } + source.mu.Unlock() + if binding == nil && request.NumericalReads == nil && origin == nil { + return nil // Preserve every existing unbound/default role. + } + if ctx == nil { + return goAMD64FrameContext("numerical frame verification requires a live context") + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("verify numerical frame context: %w", err) + } + if !ready || origin != source || binding == nil || request.NumericalReads != binding || binding.seal != seal { + return goAMD64FrameContext("numerical frame generation/token is closed, missing or changed") + } + if err := verifyGoAMD64FrameNumericalRequest(binding.context, pkg, request); err != nil { + return err + } + binding.source.mu.Lock() + actualSelection := binding.source.guard.selection + actualEnvironment := binding.source.guard.env + var mismatch bool + for _, key := range []string{"GOROOT", "GOTOOLDIR", "GOVERSION", "GOOS", "GOARCH", "GOAMD64", "GOFLAGS", "GOEXPERIMENT", "GOCACHE"} { + mismatch = mismatch || environment[key] != actualEnvironment[key] + } + for name, data := range binding.source.native.Sources { + sum, exists := inputs[name] + mismatch = mismatch || !exists || sum != sha256.Sum256(data) + } + binding.source.mu.Unlock() + if mismatch || !reflect.DeepEqual(selection, actualSelection) || len(selection.SFiles) != 1 || selection.SFiles[0] != request.AssemblyFile { + return goAMD64FrameContext("independent complete numerical/frame source selection or tool bytes differ") + } + // Catch Close that linearized while the independent owner checks ran. + source.mu.Lock() + live := source.ready && source.request.NumericalReads == binding && source.numericalBinding == binding + source.mu.Unlock() + if !live { + return goAMD64FrameContext("numerical frame closed during complete verification") + } + return nil +} + +func (source *GoAMD64FrameSource) numericalFrameObject(object string, file *File) ([]gostaticstorage.TextRecord, error) { + data, err := source.read(object) + if err != nil { + return nil, err + } + builtin, err := source.read(filepath.Join(source.env["GOROOT"], "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + return nil, err + } + texts, err := gostaticstorage.ObserveX86AssemblyText(data, builtin, source.request.Target) + if err != nil { + return nil, err + } + binding := source.numericalBinding + if err := binding.verify(); err != nil { + return nil, err + } + fingerprint, err := x86CallerSourceFingerprint(file) + if err != nil || fingerprint != binding.file { + return nil, goAMD64FrameContext("independent numerical frame parsed source differs") + } + if err := goAMD64FrameNumericalObjectJoin(texts, binding.source.native.OriginalText, binding.reads); err != nil { + return nil, err + } + return texts, nil +} + +// Both TEXT collections came from independently decoded actual original +// objects. Whole records (including positions and auxiliaries) must agree; +// every relocation must also be exactly one certified scalar read's field. +func goAMD64FrameNumericalObjectJoin(texts, numerical []gostaticstorage.TextRecord, reads []numericalSBReadFact) error { + if len(texts) == 0 || !reflect.DeepEqual(texts, numerical) { + return goAMD64FrameContext("independent complete TEXT/code/source/ABI0/relocations differ") + } + readIndex := 0 + for _, text := range texts { + if text.ABI != 0 || text.Size != int64(len(text.Code)) { + return goAMD64FrameContext("numerical frame has foreign ABI0 or incomplete code") + } + lastPC, lastInstruction := int64(-1), -1 + for _, relocation := range text.Relocations { + if readIndex >= len(reads) { + return goAMD64FrameContext("missing complete numerical frame read") + } + read := reads[readIndex] + if read.Function != text.Name || !reflect.DeepEqual(read.Relocation, relocation) || + read.PC <= lastPC || read.Instruction <= lastInstruction || read.PC >= text.Size || + read.Variable.NativeName != relocation.Target.Name { + return goAMD64FrameContext("foreign, duplicate or reordered numerical frame read/owner") + } + native, err := x86asm.Decode(text.Code[read.PC:], 64) + if err != nil || native.Op == 0 || native.Len <= 0 || read.PC+int64(native.Len) > text.Size || + !bytes.Equal(read.Code, text.Code[read.PC:read.PC+int64(native.Len)]) || native.MemBytes != int(read.Width) { + return goAMD64FrameContext("numerical frame instruction boundary/code/width differs") + } + actual, offset, err := numericalInstructionRelocation(text, read.PC, native, "amd64") + if err != nil || !reflect.DeepEqual(actual, relocation) || offset != read.Offset { + return goAMD64FrameContext("numerical frame PC displacement/type/target/ABI/addend differs") + } + lastPC, lastInstruction = read.PC, read.Instruction + readIndex++ + } + } + if readIndex != len(reads) { + return goAMD64FrameContext("numerical frame omitted a certified original read") + } + return nil +} + +func goAMD64FrameNumericalRelocationLine(relocation gostaticstorage.TextRelocation) string { + if relocation.Type != 14 || relocation.Size != 4 || !relocation.Target.ABIKnown || relocation.Target.ABI != 0 { + return "" // No other listing relocation has been authorized. + } + line := fmt.Sprintf("rel %d+%d t=R_PCREL %s", relocation.Offset, relocation.Size, relocation.Target.Name) + if relocation.Add != 0 { + line += fmt.Sprintf("%+d", relocation.Add) + } + return line +} diff --git a/amd64_go_frame_ordinary_call.go b/amd64_go_frame_ordinary_call.go new file mode 100644 index 000000000..10a7ae654 --- /dev/null +++ b/amd64_go_frame_ordinary_call.go @@ -0,0 +1,183 @@ +package plan9asm + +import ( + "reflect" + "strings" + + "github.com/xgo-dev/plan9asm/internal/goabi" +) + +// A Go implementation is a separately compiled callable, not an assembly +// entry or an inlineable private leaf. Select only actual direct references; +// unrelated Go declarations retain their existing transport roles. +func (source *GoAMD64FrameSource) ordinaryCallDeclarations(pkg GoPackage, file *File) (map[string]goAMD64FrameDeclaration, error) { + leaves, err := goAMD64FramePrivateLeaves(file, pkg) + if err != nil { + return nil, err + } + entries := make(map[string]bool) + foreign := make(map[string]map[string]bool) + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + if ins.Op != OpCALL || len(ins.Args) != 1 || ins.Args[0].Kind != OpSym { + continue + } + name, offset, valid := parseSBRef(ins.Args[0].Sym) + if valid && offset == 0 && leaves[name] { + continue + } + path, local := goReadOnlyImportedName(goAMD64FrameCallName(pkg.Path, name)) + if !valid || offset != 0 || path == "" { + return nil, goAMD64FrameContext("ordinary CALL needs an exact selected Go target") + } + if path == pkg.Path { + entries[local] = true + } else { + if foreign[path] == nil { + foreign[path] = make(map[string]bool) + } + foreign[path][local] = true + } + } + } + if len(entries)+len(foreign) == 0 { + return nil, nil + } + declarations := make(map[string]goAMD64FrameDeclaration) + if len(entries) != 0 { + declarations, err = source.typedSourceDeclarations(pkg, false, "amd64", true, entries, false, true) + if err != nil { + return nil, err + } + } + for path, names := range foreign { + selected, err := source.ordinarySDKCallDeclarations(path, names) + if err != nil { + return nil, err + } + for name, declaration := range selected { + declarations[name] = declaration + } + } + for _, declaration := range declarations { + if !goAMD64FrameScalarCall(declaration.signature) { + return nil, goAMD64FrameContext("ordinary CALL has a non-scalar transport") + } + } + return declarations, nil +} + +func goAMD64FrameCallName(path, symbol string) string { + if strings.HasPrefix(symbol, "·") { + symbol = path + "." + strings.TrimPrefix(symbol, "·") + } + owner, name := goReadOnlyImportedName(symbol) + if owner == "" { + return "" + } + return owner + "." + name +} + +func goAMD64FrameScalarCall(sig FuncSig) bool { + scalar := func(typ LLVMType) bool { + return typ == Ptr || typ == I1 || typ == I8 || typ == I16 || typ == I32 || typ == I64 + } + for _, typ := range sig.Args { + if !scalar(typ) { + return false + } + } + if sig.Ret == Void || scalar(sig.Ret) { + return true + } + fields, valid := parseLiteralStructFields(sig.Ret) + for _, typ := range fields { + valid = valid && scalar(typ) + } + return valid +} + +func fixedLocalOrdinaryCall(ins Instr, binding *x86FixedLocalBinding) (FuncSig, bool) { + if binding == nil || ins.Op != OpCALL || len(ins.Args) != 1 || ins.Args[0].Kind != OpSym { + return FuncSig{}, false + } + symbol, offset, valid := parseSBRef(ins.Args[0].Sym) + name := binding.symbols[symbol] + declaration, found := binding.callees[name] + sig, supplied := binding.signatures[name] + return sig, valid && offset == 0 && found && supplied && + (declaration.transport == goAMD64FrameNumericTransport || declaration.transport == goAMD64FrameCompositeTransport) && + goAMD64FrameScalarCall(sig) && reflect.DeepEqual(sig, declaration.signature) +} + +// The existing ABI0 emitter reads outgoing slots and writes only typed result +// slots. Opaque pointer words supply no private-frame or pointee authority. +// A Go callee may overwrite its entire outgoing extent and every scratch bank; +// only unescaped caller locals outside that extent remain initialized. +func fixedLocalOrdinaryCallStep(ins Instr, state *x86FixedLocalState, proof *x86FixedLocalProof) error { + sig, found := fixedLocalOrdinaryCall(ins, proof.binding) + sp, original := state.pointers[SP] + if !found || !original || sp != 0 || !x86ABI0FrameLayout(sig, proof.frame, "amd64") { + return goAMD64FrameContext("ordinary CALL lacks its exact bound ABI0 outgoing layout") + } + _, changedBP := state.known[BP] + if _, privateBP := state.pointers[BP]; changedBP || privateBP || state.opaque[BP].valid() { + return goAMD64FrameContext("ordinary Go CALL requires the unchanged automatic frame pointer") + } + var extent int64 + for group, slots := range [][]FrameSlot{sig.Frame.Params, sig.Frame.Results} { + for _, slot := range slots { + width := frameTypeSize(slot.Type, 8) + if width <= 0 || slot.Offset < 0 || slot.Offset > proof.frame-width { + return goAMD64FrameContext("ordinary CALL slot exceeds its exact outgoing frame") + } + initialized := state.bytes.all(slot.Offset, width) + if slot.Type == Ptr && group == 0 { + origin := state.opaqueCells[slot.Offset] + _, private := state.addresses[slot.Offset] + initialized = width == 8 && !private && (initialized || origin.valid() && !origin.readonly()) + } + if group == 0 && (!initialized || slot.Type == I1 && !state.booleanBytes.all(slot.Offset, width)) { + return goAMD64FrameContext("ordinary CALL reads uninitialized or noncanonical outgoing bytes") + } + if end := slot.Offset + width; end > extent { + extent = end + } + } + } + var valid bool + extent, valid = goabi.ABI0StackSize("amd64", extent) + if !valid || extent > proof.frame { + return goAMD64FrameContext("ordinary CALL outgoing aligned extent exceeds its private frame") + } + state.bytes.setRange(0, extent, false) + fixedLocalRevokeSpillRoles(state, 0, extent) + for _, slot := range sig.Frame.Results { + width := frameTypeSize(slot.Type, 8) + state.bytes.setRange(slot.Offset, width, slot.Type != Ptr) + state.booleanBytes.setRange(slot.Offset, width, slot.Type == I1) + if slot.Type == Ptr { + state.opaqueCells[slot.Offset] = fixedLocalOrigin(x86FixedLocalOrigin{}) + } + } + // This is the existing architectural GP bank, not a native-MMX grant. + for _, reg := range x86NativeMMXGPRegisters { + if reg == SP || reg == BP { + continue + } + delete(state.known, reg) + delete(state.pointers, reg) + delete(state.opaque, reg) + delete(state.booleans, reg) + delete(state.constants, reg) + delete(state.ranges, reg) + } + for reg := range state.vectors { + delete(state.vectors, reg) + } + for reg := range state.masks { + delete(state.masks, reg) + } + state.flags, state.comparison = 0, x86FixedLocalComparison{} + return nil +} diff --git a/amd64_go_frame_ordinary_call_contract_go127_test.go b/amd64_go_frame_ordinary_call_contract_go127_test.go new file mode 100644 index 000000000..44e1f6add --- /dev/null +++ b/amd64_go_frame_ordinary_call_contract_go127_test.go @@ -0,0 +1,175 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestGoAMD64FrameOrdinaryCallKeepsPrivateLeafAndPrototypeBoundaries(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + t.Run("existing-private-asm", func(t *testing.T) { + assembly := fixedLocalPrivateLeafSource("CALL leaf<>(SB)", "MOVQ (BP),AX") + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly) + ordinaryGoFrameCallOriginalPackage(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil || strings.Count(ir, "define ") != 1 || strings.Contains(ir, "call void @\"leaf") { + t.Fatal("the established same-file private leaf acquired an ordinary callable ABI", err, ir) + } + fixedLocalPrivateLeafObjects(t, ir) + }) + }) + for _, test := range []struct{ name, declaration string }{ + {"unknown-callee", "func Value(a uint64) uint64\n"}, + {"bodyless-prototype-only", "func Value(a uint64) uint64\nfunc Callee(a uint64) uint64\n"}, + {"pointer-callee", "func Value(a uint64) uint64\nfunc Callee(a *uint64) uint64 { return *a }\n"}, + {"floating-callee", "func Value(a uint64) uint64\nfunc Callee(a float64) float64 { return a }\n"}, + {"variadic-callee", "func Value(a uint64) uint64\nfunc Callee(a ...uint64) uint64 { return uint64(len(a)) }\n"}, + } { + t.Run(test.name, func(t *testing.T) { + var pkg GoPackage + var request GoAMD64FrameCaptureRequest + var options Options + if test.name == "variadic-callee" { + // The actual source producer must reject this role. Do not ask + // the independent nonvariadic ABI builder to accept it first. + pkg, request = goAMD64FrameTypedFixture(t, test.declaration, ordinaryGoFrameCallAssembly) + } else { + pkg, request, _, options = ordinaryGoFrameCallFixture(t, test.declaration, ordinaryGoFrameCallAssembly) + } + frameRelocationOriginal(t, pkg, request) + if test.name == "bodyless-prototype-only" && options.Sigs[pkg.Path+".Callee"].Name == "" { + t.Fatal("the negative lost its caller-supplied typed signature") + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if owner != nil { + defer func() { + if err := owner.Close(); err != nil { + t.Error(err) + } + }() + } + if test.name == "pointer-callee" { + if err != nil { + t.Fatal("actual typed pointer callee source was rejected", err) + } + file, err := Parse(ArchAMD64, ordinaryGoFrameCallAssembly) + if err != nil { + t.Fatal(err) + } + t.Run("signatures-alone", func(t *testing.T) { + fixedLocalAPIs(t, pkg, file, options, func(t *testing.T, ir string, err error) { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a pointer callee signature granted an unobserved physical caller frame", err, ir) + } + }) + }) + t.Run("source-bound", func(t *testing.T) { + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "call i64 "+llvmGlobal(pkg.Path+".Callee")+"(ptr ") { + t.Fatal("an initialized word did not retain its exact opaque-pointer call", err, ir) + } + ordinaryGoFrameCallObjects(t, ir) + }) + }) + return + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("an unresolved target or prototype alone granted a callee/source contract", err) + } + }) + } +} + +func TestGoAMD64FrameOrdinaryCallObjectRejectsRelocationAndABIChanges(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + pkg, request, file, _ := ordinaryGoFrameCallFixture(t, + ordinaryGoFrameCallDeclarations, ordinaryGoFrameCallAssembly) + ordinaryGoFrameCallOriginalPackage(t, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + objectPath := filepath.Join(t.TempDir(), "original.o") + env := append(append([]string(nil), request.Environment...), + "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOPROXY=off") + listing, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, + "tool", "asm", "-S", "-p", pkg.Path, "-o", objectPath, request.AssemblyFile) + if err != nil { + t.Fatalf("original call object: %v\n%s", err, listing) + } + object, err := os.ReadFile(objectPath) + if err != nil { + t.Fatal(err) + } + builtin, err := os.ReadFile(filepath.Join(filepath.Dir(filepath.Dir(request.GoBinary)), "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + t.Fatal(err) + } + observed, err := gostaticstorage.ObserveAMD64AssemblyData([]byte(ordinaryGoFrameCallAssembly), object, builtin, pkg.Path, request.Target) + if err != nil || len(observed.Member.Texts) != 1 || len(observed.Member.Texts[0].Relocations) != 1 { + t.Fatal("the independent object lost its single complete CALL", err) + } + original := observed.Member.Texts[0].Relocations[0] + if !original.Target.ABIKnown || original.Target.ABI != 0 || original.Target.Name != pkg.Path+".Callee" || + !bytes.Contains(listing, []byte("t=R_CALL "+original.Target.Name+"+0")) || + goAMD64FrameDataRelocationLine(original, &observed) == "" { + t.Fatal("canonical ordinary CALL no longer has an exact ABI0 object/listing observation", original) + } + for _, test := range []struct { + name string + edit func(*gostaticstorage.TextRelocation) + }{ + {"wrong-ABI", func(r *gostaticstorage.TextRelocation) { r.Target.ABI = 1 }}, + {"unknown-ABI", func(r *gostaticstorage.TextRelocation) { r.Target.ABIKnown = false }}, + {"wrong-relocation", func(r *gostaticstorage.TextRelocation) { r.Type = 10 }}, + {"wrong-width", func(r *gostaticstorage.TextRelocation) { r.Size = 8 }}, + {"nonzero-addend", func(r *gostaticstorage.TextRelocation) { r.Add = 1 }}, + } { + t.Run(test.name, func(t *testing.T) { + changed := original + test.edit(&changed) + if goAMD64FrameDataRelocationLine(changed, &observed) != "" { + t.Fatal("a changed object role retained its ordinary ABI0 CALL observation", changed) + } + }) + } + // Independently reject changed raw relocation text against the unchanged + // actual object. The formatter above is not the permission/admission test. + line := []byte("\t" + goAMD64FrameDataRelocationLine(original, &observed) + "\n") + if bytes.Count(listing, line) != 1 { + t.Fatal("the raw listing does not contain exactly one canonical CALL record") + } + for name, changed := range map[string][]byte{ + "raw-foreign-target": bytes.Replace(line, []byte(pkg.Path+".Callee"), []byte("foreign.Callee"), 1), + "raw-width": bytes.Replace(line, []byte("+4 t="), []byte("+8 t="), 1), + "raw-relocation": bytes.Replace(line, []byte("R_CALL"), []byte("R_PCREL"), 1), + "raw-addend": bytes.Replace(line, []byte("Callee+0"), []byte("Callee+1"), 1), + "raw-missing": nil, + "raw-duplicate": append(append([]byte(nil), line...), line...), + } { + t.Run(name, func(t *testing.T) { + mutated := bytes.Replace(listing, line, changed, 1) + if bytes.Equal(mutated, listing) { + t.Fatal("the negative did not change the producer's raw relocation") + } + _, err := goAMD64FrameListingWithDataObject(mutated, file, pkg, + map[string]int{request.AssemblyFile: strings.Count(ordinaryGoFrameCallAssembly, "\n") + 1}, + &observed, nil, request.Target, request.Directory, "") + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "object/listing relocation") { + t.Fatal("changed raw CALL listing was not rejected by the exact-object join", err) + } + }) + } +} diff --git a/amd64_go_frame_ordinary_call_contract_test.go b/amd64_go_frame_ordinary_call_contract_test.go new file mode 100644 index 000000000..d85619d46 --- /dev/null +++ b/amd64_go_frame_ordinary_call_contract_test.go @@ -0,0 +1,228 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "errors" + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "go/types" + "os" + "path/filepath" + "strings" + "testing" +) + +const ordinaryGoFrameCallAssembly = `TEXT ·Value(SB),4,$48-16 + MOVQ a+0(FP),AX + MOVQ AX,0(SP) + MOVQ $37,32(SP) + CALL ·Callee(SB) + MOVQ 8(SP),AX + ADDQ 32(SP),AX + MOVQ AX,ret+8(FP) + RET +` + +const ordinaryGoFrameCallDeclarations = `func Value(a uint64) uint64 +//go:noinline +func Callee(a uint64) uint64 { return (a ^ 11) + 17 } +` + +func ordinaryGoFrameCallObjects(t *testing.T, ir string) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + const target = "x86_64-unknown-linux-gnu" + compileLLVMToObject(t, llc, target, "ordinary-call.ll", "ordinary-call.o", ir) + compileLLVMToObject(t, llc, target, "ordinary-call-opt.ll", "ordinary-call-opt.o", x86MulDivRoundOptimize(t, ir)) +} + +func ordinaryGoFrameCallFixture(t *testing.T, declaration, assembly string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options) { + t.Helper() + pkg, request := goAMD64FrameTypedFixture(t, declaration, assembly) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + signatures := make(map[string]FuncSig) + for _, name := range []string{"Value", "Callee"} { + if fn, ok := pkg.Types.Scope().Lookup(name).(*types.Func); ok { + sig, err := DeriveGoFuncABI0Signature(pkg.Path+"."+name, fn, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + signatures[sig.Name] = sig + } + } + return pkg, request, file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: testResolveSym(pkg.Path), Sigs: signatures, + } +} + +func ordinaryGoFrameCallOriginalPackage(t *testing.T, request GoAMD64FrameCaptureRequest) { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), + "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOPROXY=off") + archive := filepath.Join(t.TempDir(), "original.a") + output, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, + "build", "-o", archive, ".") + if err != nil { + t.Fatalf("stock Go rejected the complete Go-body/assembly package: %v\n%s", err, output) + } + if info, err := os.Stat(archive); err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatal("stock Go did not produce the complete package archive", err) + } +} + +func TestGoAMD64FrameOrdinaryGoCalleeCaptureAndObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + pkg, request, file, options := ordinaryGoFrameCallFixture(t, + ordinaryGoFrameCallDeclarations, ordinaryGoFrameCallAssembly) + callee := options.Sigs[pkg.Path+".Callee"] + if len(callee.Args) != 1 || callee.Args[0] != I64 || callee.Ret != I64 || + len(callee.Frame.Params) != 1 || callee.Frame.Params[0].Offset != 0 || + len(callee.Frame.Results) != 1 || callee.Frame.Results[0].Offset != 8 { + t.Fatal("the independently expected ABI0 outgoing/result slots changed", callee) + } + // This compiles both the real Go callee and its assembly caller. The + // archive is not a claim of runtime execution or memory-effect proof. + ordinaryGoFrameCallOriginalPackage(t, request) + listing := frameRelocationOriginal(t, pkg, request) + if !strings.Contains(string(listing), "t=R_CALL "+pkg.Path+".Callee+0") { + t.Fatal("the independent assembler omitted the direct ABI0 callee identity", string(listing)) + } + leaves, err := goAMD64FramePrivateLeaves(file, pkg) + if err != nil || len(leaves) != 0 { + t.Fatal("a Go implementation was reclassified as a private assembly leaf", leaves, err) + } + t.Run("unbound-three-APIs", func(t *testing.T) { + fixedLocalAPIs(t, pkg, file, options, func(t *testing.T, ir string, err error) { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("callee signatures alone authorized the original physical source frame", err, ir) + } + }) + }) + // Each API owns its bounded capture/binding lifetime. Old production must + // fail here at Capture, not pass as an intentional source rejection. + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "call i64 @\""+pkg.Path+".Callee\"(i64 ") || + strings.Contains(ir, "define i64 @\""+pkg.Path+".Callee\"") { + t.Fatal("ordinary Go call was dropped, inlined as a private leaf, or assigned another ABI", err, ir) + } + ordinaryGoFrameCallObjects(t, ir) + }) + t.Run("narrow-ABI0-slots", func(t *testing.T) { + declaration := "func Value(a uint64) uint64\nfunc Callee(a uint8) uint8 { return (a ^ 11) + 17 }\n" + assembly := strings.Replace(ordinaryGoFrameCallAssembly, "MOVQ 8(SP),AX", "MOVBQZX 8(SP),AX", 1) + pkg, request, _, options := ordinaryGoFrameCallFixture(t, declaration, assembly) + ordinaryGoFrameCallOriginalPackage(t, request) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "call i8 @\""+pkg.Path+".Callee\"(i8 ") { + t.Fatal("narrow argument/result transport lost its exact typed CALL", err, ir) + } + ordinaryGoFrameCallObjects(t, ir) + }) + }) +} + +func TestGoAMD64FrameOrdinaryCallOutgoingAndClobberBoundaries(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, test := range []struct { + name, before, after string + }{ + {"missing-argument", "\tMOVQ AX,0(SP)\n", ""}, + {"partial-argument", "\tMOVQ AX,0(SP)\n", "\tMOVL AX,0(SP)\n"}, + {"wrong-argument-slot", "\tMOVQ AX,0(SP)\n", "\tMOVQ AX,16(SP)\n"}, + {"private-address-argument", "\tMOVQ AX,0(SP)\n", "\tLEAQ 32(SP),AX\n\tMOVQ AX,0(SP)\n"}, + {"old-GP-result", "\tMOVQ 8(SP),AX\n", ""}, + {"old-vector", "\tCALL ·Callee(SB)\n\tMOVQ 8(SP),AX\n", "\tPXOR X0,X0\n\tCALL ·Callee(SB)\n\tMOVQ X0,AX\n"}, + {"old-FLAGS", "\tCALL ·Callee(SB)\n\tMOVQ 8(SP),AX\n", "\tCMPQ AX,$0\n\tCALL ·Callee(SB)\n\tMOVQ $0,AX\n\tSETCS AL\n"}, + {"old-outgoing-input", "\tMOVQ 8(SP),AX\n", "\tMOVQ 0(SP),AX\n"}, + {"overwritten-frame-pointer", "\tCALL ·Callee(SB)\n", "\tMOVQ $0,BP\n\tCALL ·Callee(SB)\n"}, + } { + t.Run(test.name, func(t *testing.T) { + assembly := strings.Replace(ordinaryGoFrameCallAssembly, test.before, test.after, 1) + pkg, request, file, options := ordinaryGoFrameCallFixture(t, ordinaryGoFrameCallDeclarations, assembly) + ordinaryGoFrameCallOriginalPackage(t, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("a legal original CALL did not reach its independent state boundary", err) + } + defer func() { + if err := owner.Close(); err != nil { + t.Error(err) + } + }() + if _, err := owner.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("ordinary Go CALL acquired undefined bytes or retained physical scratch facts", err) + } + }) + } +} + +func TestGoAMD64FrameOrdinaryCallSourceAndSuppliedABIStayBound(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, change := range []string{"Go-body-mutation", "register-ABI", "result-type", "parameter-slot"} { + t.Run(change, func(t *testing.T) { + pkg, request, file, options := ordinaryGoFrameCallFixture(t, + ordinaryGoFrameCallDeclarations, ordinaryGoFrameCallAssembly) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("canonical ordinary CALL must capture before testing an independent boundary", err) + } + defer func() { + if err := owner.Close(); err != nil { + t.Error(err) + } + }() + if callee, found := owner.functions[pkg.Path+".Callee"]; found && callee.privateLeaf { + t.Fatal("the Go body acquired a private physical entry") + } + if change == "Go-body-mutation" { + path := filepath.Join(request.Directory, "value.go") + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + after := bytes.Replace(before, []byte("+ 17"), []byte("+ 19"), 1) + if bytes.Equal(before, after) { + t.Fatal("source mutation did not alter the actual implementation") + } + if err := os.WriteFile(path, after, 0600); err != nil { + t.Fatal(err) + } + if !errors.Is(owner.Verify(ctx), ErrProbeNeedsContext) { + t.Fatal("a changed implementation retained the captured call source") + } + return + } + callee := options.Sigs[pkg.Path+".Callee"] + switch change { + case "register-ABI": + callee.ArgRegs = []Reg{AX} + case "result-type": + callee.Ret = I32 + case "parameter-slot": + callee.Frame.Params = append([]FrameSlot(nil), callee.Frame.Params...) + callee.Frame.Params[0].Offset++ + } + options.Sigs[callee.Name] = callee + if _, err := owner.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("supplied signature changed the actual ABI0 callee contract", err) + } + }) + } +} diff --git a/amd64_go_frame_ordinary_call_runtime_test.go b/amd64_go_frame_ordinary_call_runtime_test.go new file mode 100644 index 000000000..72b0899f3 --- /dev/null +++ b/amd64_go_frame_ordinary_call_runtime_test.go @@ -0,0 +1,63 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Actual Go archives/ABI0 objects are checked by the source owner. This +// independent C callee executes the translated logical call on the host; +// it does not claim original native AMD64 or Go runtime execution. +func TestGoAMD64FrameOrdinaryCallPortableValues(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + execution := x86ScalarCountHostExecution(t) + pkg, request, _, options := ordinaryGoFrameCallFixture(t, + ordinaryGoFrameCallDeclarations, ordinaryGoFrameCallAssembly) + prefix := "" + if strings.Contains(execution.triple, "apple") { + prefix = "_" + } + driver := fmt.Sprintf(`#include +#include +static unsigned calls; +uint64_t callee(uint64_t) __asm__(%q); +uint64_t callee(uint64_t a) { ++calls; return (a ^ UINT64_C(11)) + UINT64_C(17); } +extern uint64_t value(uint64_t) __asm__(%q); +int main(void) { + const uint64_t edges[] = {0, 1, 10, 11, 12, UINT64_MAX, UINT64_C(0x8000000000000000), UINT64_C(0x7fffffffffffffff)}; + uint64_t random = UINT64_C(0x893701ab2468cdfe); + for (unsigned i = 0; i < 264; ++i) { + random ^= random << 13; random ^= random >> 7; random ^= random << 17; + uint64_t input = i < 8 ? edges[i] : random; + uint64_t expected = (input ^ UINT64_C(11)) + UINT64_C(17) + UINT64_C(37); + uint64_t got = value(input); + if (got != expected || calls != i + 1) { + fprintf(stderr, "ordinary call input=%%llu got=%%llu expected=%%llu calls=%%u\n", + (unsigned long long)input, (unsigned long long)got, (unsigned long long)expected, calls); + return 1; + } + } + return 0; +} +`, prefix+pkg.Path+".Callee", prefix+pkg.Path+".Value") + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR-O2=%t/backend%s", optimized, level), func(t *testing.T) { + execution.compileAndRunLLVM(t, "ordinary-call", text, driver, level) + }) + } + } + }) +} diff --git a/amd64_go_frame_ordinary_call_transfer_test.go b/amd64_go_frame_ordinary_call_transfer_test.go new file mode 100644 index 000000000..b4e63b3f5 --- /dev/null +++ b/amd64_go_frame_ordinary_call_transfer_test.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "errors" + "testing" +) + +// Pure typed-state tests run on the translator's minimum SDK too. They do not +// impersonate the separately audited Go1.27 object/source producer. +func TestGoAMD64FrameOrdinaryCallTypedTransfer(t *testing.T) { + sig := FuncSig{Name: "example.Callee", Args: []LLVMType{I64}, Ret: I64, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I64, Field: -1}}, Results: []FrameSlot{{Offset: 8, Type: I64, Field: -1}}}} + binding := &x86FixedLocalBinding{ + symbols: map[string]string{"·Callee": sig.Name}, signatures: map[string]FuncSig{sig.Name: sig}, + callees: map[string]goAMD64FrameDeclaration{sig.Name: {signature: sig, transport: goAMD64FrameNumericTransport}}, + } + proof := &x86FixedLocalProof{binding: binding, frame: 48} + call := x86ScalarLaneInstruction(t, "CALL ·Callee(SB)") + state := fixedLocalExchangeState() + state.bytes.setRange(0, 48, true) + state.booleanBytes.setRange(0, 48, true) + state.bytes.setRange(8, 8, false) + state.addresses[8] = 32 + state.opaqueCells[8] = fixedLocalOrigin(x86FixedLocalOrigin{parameter: 0}) + state.known[AX], state.booleans[AX], state.constants[AX] = 255, 255, 1 + state.ranges = map[Reg]x86UnsignedRange{AX: {minimum: 1, maximum: 1}} + state.comparison = x86FixedLocalComparison{register: AX, constant: 1} + state.pointers[BX] = 32 + state.opaque[CX] = fixedLocalOrigin(x86FixedLocalOrigin{parameter: 0}) + state.vectors["Z0"], state.masks["K1"] = ^uint64(0), true + before := cloneFixedLocalState(state) + if err := fixedLocalOrdinaryCallStep(call, &state, proof); err != nil { + t.Fatal(err) + } + if state.bytes.all(0, 8) || !state.bytes.all(8, 8) || !state.bytes.all(32, 8) || + state.booleanBytes.all(0, 16) || !state.booleanBytes.all(32, 8) || + len(state.addresses) != 0 || len(state.opaqueCells) != 0 || + len(state.known)+len(state.booleans)+len(state.constants)+len(state.ranges)+len(state.opaque) != 0 || + len(state.pointers) != 1 || state.pointers[SP] != 0 || len(state.vectors)+len(state.masks) != 0 || state.flags != 0 || + state.comparison != (x86FixedLocalComparison{}) { + t.Fatal("CALL retained outgoing aliases/predicates/scratch or lost unescaped private bytes", state) + } + for index := int64(0); index < 16; index++ { + if state.booleanBytes.has(index) { + t.Fatal("an old outgoing bool byte survived a nonboolean call result", index) + } + } + for _, test := range []string{"missing-byte", "private-pointer", "unknown-call", "wrong-SP", "short-frame"} { + t.Run(test, func(t *testing.T) { + state := cloneFixedLocalState(before) + ins, localProof := call, *proof + switch test { + case "missing-byte": + state.bytes.set(7, false) + case "private-pointer": + state.bytes.setRange(0, 8, false) + state.addresses[0] = 32 + case "unknown-call": + ins = x86ScalarLaneInstruction(t, "CALL ·Foreign(SB)") + case "wrong-SP": + state.pointers[SP] = 8 + case "short-frame": + localProof.frame = 8 + } + if err := fixedLocalOrdinaryCallStep(ins, &state, &localProof); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("an incomplete/unbound outgoing call acquired a result", err) + } + }) + } + t.Run("narrow-result-padding", func(t *testing.T) { + narrow := sig + narrow.Args, narrow.Ret = []LLVMType{I8}, I8 + narrow.Frame = FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I8, Field: -1}}, Results: []FrameSlot{{Offset: 8, Type: I8, Field: -1}}} + localBinding := *binding + localBinding.signatures = map[string]FuncSig{sig.Name: narrow} + localBinding.callees = map[string]goAMD64FrameDeclaration{sig.Name: {signature: narrow, transport: goAMD64FrameNumericTransport}} + localProof := *proof + localProof.binding = &localBinding + state := cloneFixedLocalState(before) + if err := fixedLocalOrdinaryCallStep(call, &state, &localProof); err != nil { + t.Fatal(err) + } + for index := int64(0); index < 16; index++ { + if state.bytes.has(index) != (index == 8) { + t.Fatal("narrow result or ABI0 trailing padding acquired initialized bytes", index) + } + } + }) +} diff --git a/amd64_go_frame_package_directory_test.go b/amd64_go_frame_package_directory_test.go new file mode 100644 index 000000000..f017be148 --- /dev/null +++ b/amd64_go_frame_package_directory_test.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestGoAMD64FrameSourceChildDirectoriesStayOutsidePackage(t *testing.T) { + for _, change := range []string{"contents", "replacement", "regular file", "symlink", "removed", "new sibling", "source bytes"} { + t.Run(change, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + child := filepath.Join(request.Directory, "testdata") + if err := os.Mkdir(child, 0700); err != nil { + t.Fatal(err) + } + unused := filepath.Join(child, "unused.bin") + file, err := os.Create(unused) + if err != nil { + t.Fatal(err) + } + truncateErr := file.Truncate(17 << 20) + if err := errors.Join(truncateErr, file.Close()); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source := requireGoAMD64FrameCapture(t, pkg, request) + if _, retained := source.inputs[unused]; retained { + t.Fatal("unimported descendant became a package input") + } + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + switch change { + case "contents": + err = os.WriteFile(unused, []byte("not a compiler input"), 0600) + case "replacement", "regular file", "symlink", "removed": + // Keep the old inode alive so replacement cannot reuse it. + err = os.Rename(child, filepath.Join(t.TempDir(), "original")) + if err == nil { + switch change { + case "replacement": + err = os.Mkdir(child, 0700) + case "regular file": + err = os.WriteFile(child, []byte("changed role"), 0600) + case "symlink": + err = os.Symlink(t.TempDir(), child) + } + } + case "new sibling": + err = os.Mkdir(filepath.Join(request.Directory, "new"), 0700) + case "source bytes": + err = os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package changed\n"), 0600) + } + if err != nil { + t.Fatal(err) + } + err = source.Verify(ctx) + if change == "contents" { + if err != nil { + t.Fatalf("unimported contents changed the package witness: %v", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed original input role remained valid: %v", err) + } + }) + } +} diff --git a/amd64_go_frame_private_leaf.go b/amd64_go_frame_private_leaf.go new file mode 100644 index 000000000..f327f4856 --- /dev/null +++ b/amd64_go_frame_private_leaf.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "go/types" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +// This classifies original source nodes, not callable ABIs or initialized +// inputs. A leaf cannot observe its physical frame or CALL's stack change. +// A nonzero frame's automatic BP/FLAGS effects are checked independently. +func goAMD64FramePrivateLeaves(file *File, pkg GoPackage) (map[string]bool, error) { + if file == nil || pkg.Types == nil { + return nil, goAMD64FrameContext("private source roles need complete parsed source and actual types") + } + leaves := make(map[string]bool) + for _, fn := range file.Funcs { + if _, declared := pkg.Types.Scope().Lookup(strings.TrimPrefix(fn.Sym, "·")).(*types.Func); declared { + continue + } + name := strings.TrimSuffix(fn.Sym, "<>") + if name == "" || strings.ContainsAny(name, "·./<>") || fn.FrameSize < 0 || fn.FrameSize > 64<<10 || + fn.FrameSize%8 != 0 || fn.ArgSize != 0 || len(fn.Instrs) == 0 { + return nil, goAMD64FrameContext("undeclared TEXT is not a bounded same-source leaf: %s", fn.Sym) + } + _, header := splitOpcode(fn.Instrs[0].Raw) + parts := strings.Split(header, ",") + if len(parts) != 3 { + return nil, goAMD64FrameContext("private leaf needs explicit original NOSPLIT flags") + } + flags, err := nativeFlags(parts[1], "NOSPLIT", "NOFRAME") + if err != nil || !flags["NOSPLIT"] { + return nil, goAMD64FrameContext("private leaf has an unclosed original prologue") + } + for _, ins := range fn.Instrs { + if ins.Op == OpCALL || isX86RawDirective(ins) { + return nil, goAMD64FrameContext("private leaf has raw or non-leaf effects") + } + if _, stack, _ := x86StackWidthProperties(ins, "amd64"); stack { + return nil, goAMD64FrameContext("private leaf observes implicit stack storage") + } + for _, operand := range ins.Args { + if operand.Kind == OpFP || operand.Kind == OpFPAddr || operand.Kind == OpReg && operand.Reg == SP || + operand.Kind == OpMem && (operand.Mem.Base == SP || operand.Mem.Index == SP) { + return nil, goAMD64FrameContext("private leaf observes its incoming physical frame") + } + } + } + leaves[fn.Sym] = true + } + graph := &x86DirectTailGraph{original: file, helpers: leaves} + if err := graph.auditReferences(); err != nil { + return nil, err + } + reached := make(map[string]bool) + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + if ins.Op != OpCALL && ins.Op != OpJMP || len(ins.Args) != 1 || ins.Args[0].Kind != OpSym { + continue + } + name, offset, valid := parseSBRef(ins.Args[0].Sym) + if valid && offset == 0 && leaves[name] { + if ins.Op != OpCALL { + return nil, goAMD64FrameContext("framed root cannot tail-enter a private leaf") + } + reached[name] = true + } + } + } + for name := range leaves { + if !reached[name] { + return nil, goAMD64FrameContext("private leaf lacks a same-source declared caller: %s", name) + } + } + return leaves, nil +} + +// Reuse the independent complete source/object association. Go orders static +// and nonstatic symbols in separate banks; listings retain source order. +func goAMD64FrameSourceTexts(listing []byte, data *gostaticstorage.AMD64AssemblyData) ([]gostaticstorage.TextRecord, error) { + if _, err := sdkAMD64FrameNativeRecordsForSource(listing, data.Member.Texts, data.TextOrder); err != nil { + return nil, err + } + texts := make([]gostaticstorage.TextRecord, len(data.TextOrder)) + for index, association := range data.TextOrder { + texts[index] = data.Member.Texts[association.ObjectOrdinal] + } + return texts, nil +} + +func goAMD64FrameLocalCallTarget(r gostaticstorage.TextRelocation, texts []gostaticstorage.TextRecord) bool { + if r.Type != 7 || r.Size != 4 || r.Add != 0 || !r.Target.ABIKnown { + return false + } + for _, text := range texts { + bank := uint32(2147483647) // PkgIdxNone: non-package definitions. + if text.Namespace == gostaticstorage.TextSymbolSelf { + bank = 2147483643 + } else if text.Namespace != gostaticstorage.TextSymbolNonpkgDefinition { + continue + } + if r.Target.Name == text.Name && r.Target.ABI == text.ABI && + r.Target.PackageIndex == bank && r.Target.SymbolIndex == text.Relative { + return true + } + } + return false +} + +// Machine CALL fields must match the complete original source call order and +// exact observed object target. Each caller supplies independently established +// source roles; this association grants no private-leaf or ordinary-call ABI. +// Generated split calls are outside this projection. +func goAMD64FrameSourceCallPCs(fn goAMD64FrameFunction, original Func, text gostaticstorage.TextRecord, + allowed map[string]bool, names map[string]string) (map[int]bool, error) { + var targets []string + for _, ins := range original.Instrs { + if ins.Op != OpCALL { + continue + } + if len(ins.Args) != 1 || ins.Args[0].Kind != OpSym { + return nil, goAMD64FrameContext("original CALL lacks a direct observed target") + } + name, offset, valid := parseSBRef(ins.Args[0].Sym) + if !valid || offset != 0 || !allowed[name] || names[name] == "" { + return nil, goAMD64FrameContext("original CALL target has no complete observed source role") + } + targets = append(targets, names[name]) + } + calls := make(map[int]bool) + for _, relocation := range text.Relocations { + pc := int(relocation.Offset) - fn.prologue - 1 + if relocation.Type != 7 || pc < 0 || pc >= len(fn.code) { + continue + } + if len(calls) >= len(targets) || relocation.Target.Name != targets[len(calls)] || + relocation.Size != 4 || relocation.Add != 0 || pc+5 > len(fn.code) || + fn.code[pc] != 0xe8 || !bytes.Equal(fn.code[pc+1:pc+5], []byte{0, 0, 0, 0}) || + !strings.Contains(fn.positions[pc], "\tCALL\t") || calls[pc] { + return nil, goAMD64FrameContext("original source CALL and exact machine relocation differ") + } + calls[pc] = true + } + if len(calls) != len(targets) { + return nil, goAMD64FrameContext("original source CALL inventory is incomplete") + } + return calls, nil +} + +func goAMD64FramePrivateCallRelocation(r gostaticstorage.TextRelocation, data *gostaticstorage.AMD64AssemblyData) string { + if goAMD64FrameLocalCallTarget(r, data.Member.Texts) { + return fmt.Sprintf("rel %d+4 t=R_CALL %s+0", r.Offset, r.Target.Name) + } + return "" +} diff --git a/amd64_go_frame_profile_test.go b/amd64_go_frame_profile_test.go new file mode 100644 index 000000000..29fee414c --- /dev/null +++ b/amd64_go_frame_profile_test.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +func TestGoAMD64FrameSourceActualCPUProfiles(t *testing.T) { + // Share only this root's owned build cache. Go 1.27 builds its objdump + // helper there; rebuilding that tool for every profile adds no coverage. + // Every fixture retains its own source/module/temp directories and still + // runs actual Go selection, export, object and post-capture verification. + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + for _, profile := range []string{"v1", "v2", "v3", "v4"} { + t.Run(target.name+"/"+profile, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + request.Target = target.name + request.Environment = append(request.Environment, "GOAMD64="+profile) + options.TargetTriple = target.triple + assembly := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(assembly) + if err != nil { + t.Fatal(err) + } + // Go must select the exact feature-tagged original file before + // the frame factory independently observes its native object. + tagged := fmt.Sprintf("//go:build amd64.%s\n\n%s", profile, original) + if err := os.WriteFile(assembly, []byte(tagged), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), + "GOOS="+strings.Split(target.name, "/")[0], "GOARCH=amd64", + "CGO_ENABLED=0", "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=") + output, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, + "list", "-json", "-compiled", "-export", "-mod=readonly", ".") + if err != nil { + t.Fatalf("independent actual Go profile control: %v\n%s", err, output) + } + var selected goAMD64FrameSelection + if err := json.Unmarshal(output, &selected); err != nil || selected.Export == "" || + len(selected.SFiles) != 1 || selected.SFiles[0] != request.AssemblyFile { + t.Fatalf("actual Go omitted profile-selected assembly/export: %v\n%s", err, output) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + if source.env["GOAMD64"] != profile || source.env["GOOS"]+"/"+source.env["GOARCH"] != target.name { + t.Fatal("capture substituted a different actual CPU/target profile") + } + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "profile.ll", "profile.o", ir) + compileLLVMToObject(t, llc, target.triple, "profile-opt.ll", "profile-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestGoAMD64FrameSourceCPUProfileRevocation(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, profile := range []string{"v1", "v2", "v3", "v4"} { + t.Run(profile, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Environment = append(request.Environment, "GOAMD64="+profile) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + before := append([]string(nil), source.request.Environment...) + replacement := "v1" + if profile == "v1" { + replacement = "v2" + } + source.request.Environment = append(source.request.Environment, "GOAMD64="+replacement) + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a different actual CPU profile retained the original observation", err) + } + source.request.Environment = before + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("restoring the CPU profile revived a revoked observation", err) + } + }) + } +} + +func TestGoAMD64FrameSourceCPUProfilesDoNotGrantGeneratedCPP(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, profile := range []string{"v1", "v2", "v3", "v4"} { + t.Run(profile, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Environment = append(request.Environment, "GOAMD64="+profile) + name := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + conditional := fmt.Sprintf("#ifdef GOAMD64_%s\n#include \"go_asm.h\"\n%s\n#endif\n", profile, original) + if err := os.WriteFile(name, []byte(conditional), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil { + source.Close() + t.Fatal("conditional generated header acquired an ordinary profile observation") + } + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "qualified metadata") { + t.Fatal("generated CPP still needs its own source mapping, not CPU rejection", err) + } + }) + } +} diff --git a/amd64_go_frame_relocation_kinds_test.go b/amd64_go_frame_relocation_kinds_test.go new file mode 100644 index 000000000..876522f4a --- /dev/null +++ b/amd64_go_frame_relocation_kinds_test.go @@ -0,0 +1,65 @@ +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +func frameRelocationOriginal(t *testing.T, pkg GoPackage, request GoAMD64FrameCaptureRequest) []byte { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), + "GOOS="+strings.Split(request.Target, "/")[0], "GOARCH=amd64", "CGO_ENABLED=0", + "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=") + object := filepath.Join(t.TempDir(), "original.o") + listing, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, + "tool", "asm", "-S", "-p", pkg.Path, "-o", object, request.AssemblyFile) + if err != nil { + t.Fatalf("original Go assembly failed: %v\n%s", err, listing) + } + info, err := os.Stat(object) + if err != nil || info.Size() == 0 { + t.Fatal("successful original assembler did not produce a complete object", err) + } + t.Logf("original Go %s %s listing:\n%s", runtime.Version(), request.Target, listing) + return listing +} + +func TestGoAMD64FrameOrdinaryCallDoesNotMintPermission(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + t.Run(target, func(t *testing.T) { + const assembly = "TEXT ·Caller(SB),4,$24-24\nMOVQ dst+0(FP),AX\nMOVQ AX,0(SP)\n" + + "MOVQ src+8(FP),AX\nMOVQ AX,8(SP)\nMOVQ n+16(FP),AX\nMOVQ AX,16(SP)\nCALL ·External(SB)\nRET\n" + pkg, request := goAMD64FrameTypedFixture(t, "func Caller(dst,src *byte,n uintptr)\nfunc External(dst,src *byte,n uintptr)\n", assembly) + request.Target = target + listing := frameRelocationOriginal(t, pkg, request) + call := "t=R_CALL " + pkg.Path + ".External+0" + if !gotoolchain.AuditedGo127(runtime.Version()) { + call = "t=7 \"\".External+0" + } + if !strings.Contains(string(listing), call) { + t.Fatal("original ordinary direct CALL relocation was not observed", string(listing)) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if owner != nil { + if closeErr := owner.Close(); closeErr != nil { + t.Error(closeErr) + } + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("an ordinary relocation acquired a private-call or callee frame contract", err) + } + }) + } +} diff --git a/amd64_go_frame_requested_member_test.go b/amd64_go_frame_requested_member_test.go new file mode 100644 index 000000000..7b890c585 --- /dev/null +++ b/amd64_go_frame_requested_member_test.go @@ -0,0 +1,120 @@ +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +// Complete package inputs remain live observations even when only one assembly +// member is requested for a frame binding. These ordinary Go formats work on +// both supported endpoint SDKs; no current-only object decoder is called here. +func TestGoAMD64FrameRequestedMemberRetainsSiblingInventory(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + for _, change := range []string{"ASM", "CPP"} { + t.Run(target+"/"+change, func(t *testing.T) { + const declarations = "func Value(a uint64) uint64\nfunc Unused(dst,src *byte,n uintptr)\nfunc External(dst,src *byte,n uintptr)\n" + const value = "TEXT ·Value(SB),4,$24-16\nMOVQ a+0(FP),AX\nMOVQ AX,0(SP)\nMOVQ 0(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + const unused = "#include \"textflag.h\"\n#include \"sibling.h\"\nTEXT ·Unused(SB),NOSPLIT,$24-24\nMOVQ dst+0(FP),AX\nMOVQ AX,0(SP)\nMOVQ src+8(FP),AX\nMOVQ AX,8(SP)\nMOVQ n+16(FP),AX\nMOVQ AX,16(SP)\nADDQ $EXTRA,AX\nCALL ·External(SB)\nRET\n" + pkg, request := goAMD64FrameTypedFixture(t, declarations, value) + request.Target = target + sibling := filepath.Join(request.Directory, "a_unused_amd64.s") + header := filepath.Join(request.Directory, "sibling.h") + if err := os.WriteFile(sibling, []byte(unused), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(header, []byte("#define EXTRA 0\n"), 0600); err != nil { + t.Fatal(err) + } + callRequest := request + callRequest.AssemblyFile = filepath.Base(sibling) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), + "GOOS="+strings.Split(target, "/")[0], "GOARCH=amd64", "CGO_ENABLED=0", + "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=") + for _, member := range []string{request.AssemblyFile, callRequest.AssemblyFile} { + object := filepath.Join(t.TempDir(), member+".o") + listing, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, + "tool", "asm", "-S", "-p", pkg.Path, + "-I", filepath.Join(filepath.Dir(filepath.Dir(request.GoBinary)), "pkg", "include"), + "-o", object, member) + if err != nil { + t.Fatalf("independent original member %s: %v\n%s", member, err, listing) + } + if info, err := os.Stat(object); err != nil || info.Size() == 0 { + t.Fatal("original member lacks its complete Go object", member, err) + } + } + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("unrequested valid assembly sibling acquired a frame-proof obligation", err) + } + defer func() { + if err := owner.Close(); err != nil { + t.Error(err) + } + }() + if len(owner.selection.SFiles) != 2 || len(owner.functions) != 1 || + owner.functions[pkg.Path+".Value"].sourceSymbol != "·Value" { + t.Fatal("frame observation lost the complete selection or exact requested member") + } + toolHeader := filepath.Join(owner.env["GOROOT"], "pkg", "include", "textflag.h") + for _, name := range []string{sibling, header, toolHeader} { + if _, found := owner.inputs[name]; !found { + t.Fatal("unrequested sibling input escaped live source verification", name) + } + } + if owner.origins[toolHeader] == 0 || owner.origins[header] == 0 { + t.Fatal("sibling-only CPP dependencies lost their actual source association") + } + if err := owner.Verify(ctx); err != nil { + t.Fatal(err) + } + callOwner, err := CaptureGoAMD64FrameSource(ctx, pkg, callRequest) + if callOwner != nil { + if closeErr := callOwner.Close(); closeErr != nil { + t.Error(closeErr) + } + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a requested ordinary CALL acquired a callee/frame contract", err) + } + expanded, err := owner.preprocess(callRequest.AssemblyFile, []byte(unused)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + triple := map[string]string{"linux/amd64": "x86_64-unknown-linux-gnu", "darwin/amd64": "x86_64-apple-darwin", "windows/amd64": "x86_64-pc-windows-msvc"}[target] + name := pkg.Path + ".Unused" + _, err = owner.BindFixedLocal(ctx, pkg, file, Options{Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{name: owner.declarations[name].signature}, + ResolveSym: func(symbol string) string { return pkg.Path + "." + strings.TrimPrefix(symbol, "·") }}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("requested member owner authorized a different sibling's source", err) + } + if err := owner.Verify(ctx); err != nil { + t.Fatal("negative sibling binding invalidated the positive mutation control", err) + } + changed, data := sibling, unused+"\n" + if change == "CPP" { + changed, data = header, "#define EXTRA 1\n" + } + if err := os.WriteFile(changed, []byte(data), 0600); err != nil { + t.Fatal(err) + } + if err := owner.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unrequested selected sibling mutation retained source ownership", err) + } + }) + } + } +} diff --git a/amd64_go_frame_return_cfg_go127_test.go b/amd64_go_frame_return_cfg_go127_test.go new file mode 100644 index 000000000..dbf7ee594 --- /dev/null +++ b/amd64_go_frame_return_cfg_go127_test.go @@ -0,0 +1,58 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func fixedLocalReturnCFGProgram() (string, string, []string) { + var assembly, declarations strings.Builder + var expected []string + for _, flags := range []string{"", "4,"} { + for _, value := range []int{0, 7} { + name := fmt.Sprintf("Value%d", len(expected)) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),%s$160-16\nMOVQ $%d,AX\nMOVQ AX,8(SP)\n", name, flags, value) + assembly.WriteString("loop:\nTESTQ AX,AX\nJNE work\nMOVQ 8(SP),AX\nMOVQ AX,ret0+8(FP)\nRET\nwork:\nXORQ AX,AX\nJMP loop\n") + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + expected = append(expected, fmt.Sprintf("%016x", value)) + } + } + return assembly.String(), declarations.String(), expected +} + +func TestGoAMD64FixedLocalReturnCFGObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + assembly, declarations, expected := fixedLocalReturnCFGProgram() + if len(expected) != 4 { + t.Fatal("complete split/NOSPLIT and branch inventory changed", len(expected)) + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, _, options := fixedLocalTransportFixture(t, assembly, declarations, target.name, target.triple) + var names []string + for index := range expected { + names = append(names, fmt.Sprintf("%s.Value%d", pkg.Path, index)) + } + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("original source with early RET and final local backedge", err) + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + compileLLVMToObject(t, llc, target.triple, "return-cfg.ll", "return-cfg.o", ir) + compileLLVMToObject(t, llc, target.triple, "return-cfg-opt.ll", "return-cfg-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} diff --git a/amd64_go_frame_return_cfg_runtime_go127_test.go b/amd64_go_frame_return_cfg_runtime_go127_test.go new file mode 100644 index 000000000..988da06a8 --- /dev/null +++ b/amd64_go_frame_return_cfg_runtime_go127_test.go @@ -0,0 +1,13 @@ +//go:build unix && go1.27 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalReturnCFG(t *testing.T) { + assembly, declarations, expected := fixedLocalReturnCFGProgram() + if len(expected) != 4 { + t.Fatal("complete return CFG runtime inventory changed", len(expected)) + } + fixedLocalConstantResultsRuntime(t, assembly, declarations, expected) +} diff --git a/amd64_go_frame_return_cfg_test.go b/amd64_go_frame_return_cfg_test.go new file mode 100644 index 000000000..1716814e3 --- /dev/null +++ b/amd64_go_frame_return_cfg_test.go @@ -0,0 +1,85 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestGoAMD64FrameReturnBeforeFinalLocalBranch(t *testing.T) { + for _, frame := range []int64{64, 128, 160} { + t.Run(fmt.Sprint(frame), func(t *testing.T) { + assembly := fmt.Sprintf("TEXT value(SB),4,$%d-0\nMOVQ $1,AX\nloop:\nTESTQ AX,AX\nJNE work\nRET\nwork:\nXORQ AX,AX\nJMP loop\n", frame) + listing := x86InsertExtractGoListing(t, "amd64", assembly) + fn := goAMD64FrameFunction{positions: make(map[int]string)} + for _, line := range strings.Split(string(listing), "\n") { + fn.code = append(fn.code, x86InsertExtractListedBytes(line)...) + } + last := 0 + var starts []int + returnIndex := -1 + for pc := 0; pc < len(fn.code); { + ins, err := x86asm.Decode(fn.code[pc:], 64) + if err != nil || ins.Len <= 0 { + t.Fatal("original Go machine fixture", err) + } + // Machine-only unit fixture; the public API matrix separately + // captures actual original source and object positions. + fn.positions[pc] = "machine-shape unit fixture" + if ins.Op == x86asm.RET { + returnIndex = len(starts) + } + starts = append(starts, pc) + last = pc + pc += ins.Len + } + if len(fn.code)-last != 2 || fn.code[last] != 0xeb { + t.Fatalf("expected original short local JMP, got %x", fn.code[last:]) + } + if err := goAMD64FrameMachineShape(fn, frame); err != nil { + t.Error("conventional RET may precede a closed local backedge", err) + } + if returnIndex < 2 { + t.Fatal("original fixture has no complete return epilogue") + } + for _, target := range []int{0, 1, starts[1], starts[2], starts[returnIndex-1], starts[returnIndex], + last + 1, len(fn.code), len(fn.code) + 1, -1} { + changed := fn + changed.code = append([]byte(nil), fn.code...) + changed.code[last+1] = byte(target - len(fn.code)) + if err := goAMD64FrameMachineShape(changed, frame); err == nil { + t.Errorf("terminal branch to prologue/nonboundary/outside target %d accepted", target) + } + } + changed := fn + changed.code = append([]byte(nil), fn.code...) + changed.code[last] = 0x75 // JNE has an unclosed fallthrough after the function. + if err := goAMD64FrameMachineShape(changed, frame); err == nil { + t.Error("terminal conditional branch accepted an escaping fallthrough") + } + changed = fn + changed.code = append([]byte(nil), fn.code...) + changed.code[last+1] = byte(starts[returnIndex-2] - len(fn.code)) + if err := goAMD64FrameMachineShape(changed, frame); err != nil { + t.Error("branch to the complete automatic epilogue entry rejected", err) + } + for _, mutation := range []string{"no RET", "wrong POP", "wrong adjustment"} { + changed := fn + changed.code = append([]byte(nil), fn.code...) + switch mutation { + case "no RET": + changed.code[starts[returnIndex]] = 0x90 + case "wrong POP": + changed.code[starts[returnIndex-1]] = 0x5b + case "wrong adjustment": + changed.code[starts[returnIndex-1]-1] ^= 1 + } + if err := goAMD64FrameMachineShape(changed, frame); err == nil { + t.Errorf("terminal local JMP accepted %s", mutation) + } + } + }) + } +} diff --git a/amd64_go_frame_sdk_call.go b/amd64_go_frame_sdk_call.go new file mode 100644 index 000000000..ae349e4bb --- /dev/null +++ b/amd64_go_frame_sdk_call.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "path/filepath" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +// The existing selected dependency graph supplies both language declarations +// and actual ABI0 entries. Neither a supplied signature nor an SDK spelling +// creates this role; no pointee extent or noescape property follows from it. +func (source *GoAMD64FrameSource) ordinarySDKCallDeclarations(path string, entries map[string]bool) (map[string]goAMD64FrameDeclaration, error) { + selected, found := source.imports[path] + if !found || !selected.Standard || selected.Module != nil || selected.ImportPath != path || selected.Export == "" { + return nil, goAMD64FrameContext("ordinary foreign CALL lacks its actual selected SDK source: %s", path) + } + fset := token.NewFileSet() + var syntax []*ast.File + for _, name := range selected.GoFiles { + filename := filepath.Join(selected.Dir, name) + data, err := source.read(filename) + if err != nil { + return nil, err + } + file, err := parser.ParseFile(fset, filename, data, parser.ParseComments) + if err != nil { + return nil, err + } + syntax = append(syntax, file) + } + actualImporter := importer.ForCompiler(fset, "gc", func(imported string) (io.ReadCloser, error) { + if target, mapped := selected.ImportMap[imported]; mapped { + imported = target + } + dependency, present := source.imports[imported] + if !present || dependency.Export == "" { + return nil, goAMD64FrameContext("SDK CALL declaration lacks its captured compiler import: %s", imported) + } + data, err := source.read(dependency.Export) + return io.NopCloser(bytes.NewReader(data)), err + }) + sizes := types.SizesFor("gc", "amd64") + config := &types.Config{Sizes: sizes, Importer: actualImporter, IgnoreFuncBodies: true} + typed, err := config.Check(path, fset, syntax, nil) + if err != nil { + return nil, goAMD64FrameContext("actual selected SDK CALL declarations: %v", err) + } + texts, err := source.ordinarySDKCallTexts(selected, entries) + if err != nil { + return nil, err + } + declarations := make(map[string]goAMD64FrameDeclaration, len(entries)) + for name := range entries { + fn, valid := typed.Scope().Lookup(name).(*types.Func) + if !valid { + return nil, goAMD64FrameContext("SDK CALL has no actual Go function declaration: %s", name) + } + sig := fn.Type().(*types.Signature) + if sig.Recv() != nil || sig.Variadic() || sig.TypeParams().Len() != 0 { + return nil, goAMD64FrameContext("SDK CALL excludes method, variadic or generic transport") + } + var language strings.Builder + for _, tuple := range []*types.Tuple{sig.Params(), sig.Results()} { + if tuple.Len() > 4096 { + return nil, goAMD64FrameContext("SDK CALL declaration tuple bound") + } + language.WriteString("[") + for index := 0; index < tuple.Len(); index++ { + identity, err := goAMD64FrameLanguageTypeRole(tuple.At(index).Type(), sizes, true) + if err != nil { + return nil, err + } + fmt.Fprintf(&language, "%q:%s;", tuple.At(index).Name(), identity) + if language.Len() > 64<<10 { + return nil, goAMD64FrameContext("SDK CALL language byte bound") + } + } + language.WriteString("]") + } + actual, err := DeriveGoFuncABI0Signature(path+"."+name, fn, "amd64", sizes) + if err != nil || !goAMD64FrameScalarCall(actual) || !goAMD64FrameSDKABI0Entry(texts, actual.Name) { + return nil, goAMD64FrameContext("SDK CALL lacks its exact scalar ABI0 declaration/entry: %s: %v", name, err) + } + declarations[actual.Name] = goAMD64FrameDeclaration{signature: actual, + transport: goAMD64FrameCompositeTransport, languageType: language.String()} + } + return declarations, nil +} + +func (source *GoAMD64FrameSource) ordinarySDKCallTexts(selected goAMD64FrameSelection, entries map[string]bool) ([]gostaticstorage.TextRecord, error) { + builtin, err := source.read(filepath.Join(source.env["GOROOT"], "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + return nil, err + } + referenced := make(map[string]gostaticstorage.SDKReferencedArchive) + remaining := int64(64 << 20) // Same complete-archive budget as the decoder. + paths := []string{selected.ImportPath} + for len(paths) != 0 { + path := paths[len(paths)-1] + paths = paths[:len(paths)-1] + if _, captured := referenced[path]; captured || path == "unsafe" { + continue + } + dependency, found := source.imports[path] + if !found || !dependency.Standard || dependency.Module != nil || dependency.Export == "" { + return nil, goAMD64FrameContext("SDK CALL archive lacks its actual dependency: %s", path) + } + archive, err := source.readBound(dependency.Export, remaining) + if err != nil { + return nil, err + } + remaining -= int64(len(archive)) + referenced[path] = gostaticstorage.SDKReferencedArchive{PackagePath: path, Sources: dependency.SFiles, Archive: archive} + paths = append(paths, dependency.Imports...) + } + archive := referenced[selected.ImportPath].Archive + if len(selected.SFiles) == 0 { + return gostaticstorage.ObserveX86SDKCompilerText(archive, builtin, source.request.Target, referenced) + } + targets := make(map[string]bool, len(entries)) + for name := range entries { + targets[selected.ImportPath+"."+name] = true + } + members, err := gostaticstorage.ObserveX86SDKCallArchiveReferences(archive, builtin, source.request.Target, selected.SFiles, referenced, targets) + if err != nil { + return nil, goAMD64FrameContext("actual SDK CALL archive: %v", err) + } + var texts []gostaticstorage.TextRecord + for _, member := range members { + texts = append(texts, member.Texts...) + } + return texts, nil +} + +func goAMD64FrameSDKABI0Entry(texts []gostaticstorage.TextRecord, name string) bool { + // These roles belong to the existing decoder's pinned current goobj + // schema (objabi.IsText, goobj.SymFlagABIWrapper and objabi.R_CALL). + const ( + textKind = 1 + fipsTextKind = 2 + abiWrapperFlag = 1 << 6 + callRelocation = 7 + ) + var entry, internal *gostaticstorage.TextRecord + for index := range texts { + text := &texts[index] + if text.Name != name { + continue + } + if text.Kind != textKind && text.Kind != fipsTextKind || text.Size <= 0 || text.Size != int64(len(text.Code)) { + return false + } + switch text.ABI { + case 0: + if entry != nil { + return false + } + entry = text + case 1: + if internal != nil { + return false + } + internal = text + default: + return false + } + } + if entry == nil { + return false + } + // An original ABI0 assembly entry needs no adapter; an actual generated + // adapter must retain its unique ABIInternal target in the same archive. + if entry.Flags2&abiWrapperFlag == 0 { + return true + } + calls := 0 + for _, relocation := range entry.Relocations { + if relocation.Type == callRelocation && relocation.Target.Name == name { + if !relocation.Target.ABIKnown || relocation.Target.ABI != 1 || relocation.Size != 4 || relocation.Add != 0 { + return false + } + calls++ + } + } + return internal != nil && calls == 1 +} diff --git a/amd64_go_frame_sdk_call_budget_test.go b/amd64_go_frame_sdk_call_budget_test.go new file mode 100644 index 000000000..b006fd1a8 --- /dev/null +++ b/amd64_go_frame_sdk_call_budget_test.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestGoAMD64FrameForeignABI0SDKArchiveBudgetBeforeRead(t *testing.T) { + // These are deliberately invalid object bytes, not SDK source evidence. + // Observe whether the second file was read before format validation: the + // aggregate bound must stop an oversized graph before retaining that input. + for _, excess := range []bool{false, true} { + name := "exact-budget" + if excess { + name = "over-budget" + } + t.Run(name, func(t *testing.T) { + directory, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + builtin := filepath.Join(directory, "src/cmd/internal/goobj/builtinlist.go") + if err := os.MkdirAll(filepath.Dir(builtin), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(builtin, []byte("fixture"), 0600); err != nil { + t.Fatal(err) + } + first, second := filepath.Join(directory, "first.a"), filepath.Join(directory, "second.a") + file, err := os.Create(first) + if err != nil { + t.Fatal(err) + } + truncateErr := file.Truncate((64 << 20) - 1) + closeErr := file.Close() + if truncateErr != nil || closeErr != nil { + t.Fatal(truncateErr, closeErr) + } + last := []byte{0} + if excess { + last = append(last, 0) + } + if err := os.WriteFile(second, last, 0600); err != nil { + t.Fatal(err) + } + selected := goAMD64FrameSelection{ImportPath: "fixture/first", Standard: true, Export: first, Imports: []string{"fixture/second"}} + source := &GoAMD64FrameSource{env: map[string]string{"GOROOT": directory}, inputs: make(map[string][32]byte), + request: GoAMD64FrameCaptureRequest{Target: "linux/amd64"}, + imports: map[string]goAMD64FrameSelection{selected.ImportPath: selected, + "fixture/second": {ImportPath: "fixture/second", Standard: true, Export: second}}} + if _, err := source.ordinarySDKCallTexts(selected, map[string]bool{"Value": true}); err == nil { + t.Fatal("invalid archive fixture acquired an SDK call observation") + } else if excess && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("oversized input was not stopped by the source read bound", err) + } + if _, captured := source.inputs[first]; !captured { + t.Fatal("the first fitting file was not observed") + } + if _, captured := source.inputs[second]; captured == excess { + t.Fatal("the aggregate budget did not bound the next file before observation", captured, excess) + } + }) + } +} diff --git a/amd64_go_frame_source.go b/amd64_go_frame_source.go new file mode 100644 index 000000000..6558b19d7 --- /dev/null +++ b/amd64_go_frame_source.go @@ -0,0 +1,2069 @@ +package plan9asm + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/xgo-dev/plan9asm/internal/goabi" + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/goasmmacros" + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "github.com/xgo-dev/plan9asm/internal/gotoolchain" + "github.com/xgo-dev/plan9asm/internal/sourcepath" + "golang.org/x/arch/x86/x86asm" +) + +// GoAMD64FrameCaptureRequest describes actual input locations, not permissions. +type GoAMD64FrameCaptureRequest struct { + GoBinary, Directory, AssemblyFile, Target string + Environment []string + // NumericalReads joins one already authenticated original numerical-read + // source. It observes relocations; it grants no additional memory role. + NumericalReads *GoX86NumericalReadBinding +} + +// GoAMD64FrameSource is a sealed actual-object observation. It does not grant +// a native backing, caller region, memory access or return certificate. +type GoAMD64FrameSource struct { + mu sync.Mutex + identity *GoAMD64FrameSource + ready bool + request GoAMD64FrameCaptureRequest + selection goAMD64FrameSelection + env map[string]string + assemblerDefines []string + inputs map[string][32]byte + entries []string + directoryInfo os.FileInfo + childDirectories map[string]os.FileInfo + directoryInventory *gooriginalsrc.Snapshot // Immutable complete SDK membership; no source or ASM authority. + owned string + ownedInfo os.FileInfo + functions map[string]goAMD64FrameFunction + declarations map[string]goAMD64FrameDeclaration + callees map[string]goAMD64FrameDeclaration // Actual separate Go/SDK callees, never private leaves. + imports map[string]goAMD64FrameSelection + origins map[string]int + downloadedOrigin *gooriginalsrc.OriginSet + downloadedModule *gooriginalsrc.Module + downloadedImports map[string]gooriginalsrc.Module + numericalBinding *GoX86NumericalReadBinding + numericalOrigin *GoAMD64FrameSource + numericalSeal [32]byte + numericalPackage GoPackage + numericalCapturing bool + numericalCommands []numericalSBCommand // Actual capture output only; never an authority. + sdkFrame *goAMD64SDKFrameOrigin +} + +// newGoOriginalInputGuard initializes only private input/cleanup ownership. +// It has no ready object, declaration, source-origin or frame capability. +func newGoOriginalInputGuard(request GoAMD64FrameCaptureRequest) *GoAMD64FrameSource { + request.Environment = append([]string(nil), request.Environment...) + source := &GoAMD64FrameSource{request: request, + inputs: make(map[string][32]byte), origins: make(map[string]int)} + source.identity = source + return source +} + +// Transport is an observation role, never a memory or alias permission. +type goAMD64FrameTransport uint8 + +const ( + goAMD64FrameNumericTransport goAMD64FrameTransport = iota + 1 + goAMD64FrameCompositeTransport + goAMD64FrameFloatingScalarTransport +) + +type goAMD64FrameDeclaration struct { + signature FuncSig + transport goAMD64FrameTransport + languageType string // go/types model identity, not a native pointee extent +} + +type goAMD64FrameFunction struct { + objectSymbol, listingSymbol string + sourceSymbol string + privateLeaf bool + nativeABI uint16 + locals, args int64 + split bool + prologue int // original byte offset, after any generated stack check + code []byte + positions map[int]string + sourcePositions map[int][]goAMD64FrameSourcePosition + signature FuncSig + transport goAMD64FrameTransport + languageType string +} + +func (fn goAMD64FrameFunction) nativeFlags() uint8 { + if fn.split { + return 0 + } + return 16 // goobj.SymFlagNoSplit, not an assembly-origin flag. +} + +type goAMD64FrameSourcePosition struct { + file string + line int +} + +// This is decoded only from the actual driver stdout, never from a proof file. +type goAMD64FrameSelection struct { + Dir, ImportPath, Name, Export, ForTest string + GoFiles, CompiledGoFiles, SFiles, HFiles, CgoFiles, Imports, EmbedFiles, EmbedPatterns []string + IgnoredGoFiles, IgnoredOtherFiles []string + Standard bool + ImportMap map[string]string + CFiles, CXXFiles, MFiles, FFiles, SwigFiles, SwigCXXFiles, SysoFiles []string + Module *struct { + Path, Version, Dir, GoMod, GoVersion string + Main bool + Replace json.RawMessage + } + Error json.RawMessage + DepsErrors []json.RawMessage +} + +func (selected goAMD64FrameSelection) nonordinaryRole() bool { + for _, files := range [][]string{selected.CgoFiles, selected.CFiles, selected.CXXFiles, selected.MFiles, + selected.FFiles, selected.SwigFiles, selected.SwigCXXFiles, selected.SysoFiles, selected.EmbedFiles, selected.EmbedPatterns} { + if len(files) != 0 { + return true + } + } + return false +} + +func CaptureGoAMD64FrameSource(ctx context.Context, pkg GoPackage, request GoAMD64FrameCaptureRequest) (*GoAMD64FrameSource, error) { + return captureGoAMD64FrameSource(ctx, pkg, request, nil) +} + +func captureGoAMD64FrameSource(ctx context.Context, pkg GoPackage, request GoAMD64FrameCaptureRequest, exact *goAMD64DownloadedIdentity) (_ *GoAMD64FrameSource, captureErr error) { + if ctx != nil && ctx.Err() != nil { + if request.NumericalReads != nil { + return nil, fmt.Errorf("capture numerical frame context: %w", ctx.Err()) + } + return nil, errors.Join(goAMD64FrameContext("requires live capture context"), ctx.Err()) + } + if ctx == nil || pkg.Types == nil || pkg.Path != pkg.Types.Path() || + !filepath.IsAbs(request.GoBinary) || !filepath.IsAbs(request.Directory) || + filepath.Base(request.AssemblyFile) != request.AssemblyFile || filepath.Ext(request.AssemblyFile) != ".s" { + return nil, goAMD64FrameContext("requires live context, actual package and bounded absolute inputs") + } + parts := strings.Split(request.Target, "/") + if len(parts) != 2 || parts[1] != "amd64" || parts[0] != "linux" && parts[0] != "darwin" && parts[0] != "windows" { + return nil, goAMD64FrameContext("initial object capture supports ordinary AMD64 targets only") + } + var err error + request.Directory, err = sourcepath.Resolve(request.Directory) + if err != nil { + return nil, err + } + source := newGoOriginalInputGuard(request) + if request.NumericalReads != nil { + if err := verifyGoAMD64FrameNumericalRequest(ctx, pkg, source.request); err != nil { + return nil, err + } + source.numericalBinding = request.NumericalReads + source.numericalOrigin = source + source.numericalSeal = request.NumericalReads.seal + source.numericalPackage = pkg + source.numericalCapturing = true + } + provided := make(map[string]string) + for _, value := range request.Environment { + key, setting, valid := strings.Cut(value, "=") + if !valid || key == "" { + return nil, goAMD64FrameContext("malformed command environment") + } + provided[key] = setting + if (key == "GOFLAGS" || key == "GOEXPERIMENT") && setting != "" || key == "GOPACKAGESDRIVER" && setting != "" && setting != "off" { + return nil, goAMD64FrameContext("effective overlay, custom flags, experiments or driver require their own source mapping") + } + } + for _, key := range []string{"GOPATH", "GOMODCACHE", "GOCACHE", "TMPDIR", "GOTMPDIR"} { + if !filepath.IsAbs(provided[key]) { + return nil, goAMD64FrameContext("explicit private %s directory is required", key) + } + } + settings := map[string]string{"GOOS": parts[0], "GOARCH": "amd64", "CGO_ENABLED": "0", "GOTOOLCHAIN": "local", "GOENV": "off", "GOWORK": "off", "GOFLAGS": "", "GOPROXY": "off"} + var env []string + for _, value := range source.request.Environment { + key, _, _ := strings.Cut(value, "=") + if _, replace := settings[key]; !replace { + env = append(env, value) + } + } + for key, value := range settings { + env = append(env, key+"="+value) + } + source.request.Environment = env + if err := source.snapshotDirectory(); err != nil { + return nil, err + } + if err := source.observe(ctx); err != nil { + return nil, err + } + if exact == nil { + if err := source.verifyOwnedSourceDirectory(); err != nil { + return nil, err + } + } else { + inside, err := gostaticstorage.SourceDirectoryWithinRoot(request.Directory, source.env["GOROOT"]) + if err != nil || inside { + return nil, errors.Join(goAMD64FrameContext("SDK source cannot acquire an exact downloaded role"), err) + } + if err := source.captureDownloadedOrigin(ctx, *exact, provided["GOTMPDIR"]); err != nil { + return nil, err + } + } + selected, err := source.load(ctx) + if err != nil { + return nil, err + } + source.selection = selected + if source.numericalBinding != nil && (len(selected.SFiles) != 1 || selected.SFiles[0] != request.AssemblyFile) { + return nil, goAMD64FrameContext("numerical frame join requires one complete original assembly selection") + } + moduleRoot := request.Directory + if source.downloadedModule != nil { + moduleRoot = source.downloadedModule.Root + } + mainRole, moduleVersion := true, "" + metadata := filepath.Join(moduleRoot, "go.mod") + if source.legacyDownloadedModule() { + mainRole, moduleVersion = false, source.downloadedModule.SourceVersion + metadata = source.downloadedModule.GoMod + } + if selected.Dir != request.Directory || selected.ImportPath != pkg.Path || selected.ForTest != "" || selected.Module == nil || + selected.Module.Main != mainRole || selected.Module.Version != moduleVersion || selected.Module.Dir != moduleRoot || len(selected.Module.Replace) != 0 && string(selected.Module.Replace) != "null" || + selected.nonordinaryRole() || len(selected.GoFiles) == 0 || len(selected.SFiles) == 0 { + return nil, goAMD64FrameContext("capture needs complete ordinary original module sources") + } + if selected.Module.GoMod != metadata { + return nil, goAMD64FrameContext("actual module metadata is outside the original source directory") + } + if source.downloadedModule != nil { + original := source.downloadedModule + path, err := goAMD64DownloadedPackagePath(*original, selected.Dir) + if err != nil { + return nil, err + } + if selected.Module.Path != original.ModulePath || selected.Module.GoVersion != gooriginalsrc.SelectedGoVersion(original.GoVersion, mainRole) || + selected.ImportPath != path { + return nil, goAMD64FrameContext("exact downloaded capture requires its original primary module package") + } + } + source.imports, err = source.loadOriginalDeclarationImports(ctx) + if err != nil { + return nil, err + } + source.declarations, err = source.typedDeclarations(pkg, false) + if err != nil { + return nil, err + } + rules, err := goasmmacros.Capture(source.env["GOROOT"], source.env["GOVERSION"], selected.ImportPath, source.read) + if err != nil || rules.PackageRole != "ordinary_path" && rules.PackageRole != "legacy_no_experiment_macros" { + return nil, goAMD64FrameContext("ordinary assembler macro source registration: %v", err) + } + source.assemblerDefines, err = GoAssemblerDefinesForEnvironment(source.env["GOOS"], source.env["GOARCH"], source.env) + if err != nil { + return nil, err + } + source.owned, err = os.MkdirTemp(provided["TMPDIR"], "plan9asm-go-frame-object-") + if err != nil { + return nil, err + } + defer func() { + if captureErr != nil { + captureErr = errors.Join(captureErr, source.Close()) + } + }() + source.ownedInfo, err = sourcepath.Lstat(source.owned) + if err != nil { + return nil, err + } + canonicalOwned, err := sourcepath.Resolve(source.owned) + if err != nil { + return nil, err + } + source.owned = canonicalOwned + if err := source.verifyOwned(); err != nil { + return nil, err + } + source.functions = make(map[string]goAMD64FrameFunction) + found := false + for _, name := range selected.SFiles { + found = found || name == request.AssemblyFile + original, err := source.read(filepath.Join(request.Directory, name)) + if err != nil { + return nil, err + } + expanded, err := source.preprocess(name, original) + if err != nil { + return nil, err + } + if name != request.AssemblyFile { + continue // Preserve sibling inputs/CPP, without acquiring its frame role. + } + file, err := ParseWithDefines(ArchAMD64, expanded, nil) + if err != nil || len(file.Funcs) == 0 || len(file.UnlinkedPrelude) != 0 { + return nil, goAMD64FrameContext("full original assembly lacks complete TEXT: %v", err) + } + source.callees, err = source.ordinaryCallDeclarations(pkg, file) + if err != nil { + return nil, err + } + object := filepath.Join(source.owned, name+".o") + assemblyInput := name + if source.numericalBinding != nil { + assemblyInput = filepath.Join(request.Directory, name) + } + // The independent actual assembler and source preprocessor consume + // the same registered ordinary profile macros. Generated headers keep + // their separate source mapping; CPU identity grants no memory role. + args := []string{"tool", "asm", "-S", "-p", pkg.Path, + "-I", request.Directory, "-I", filepath.Join(source.env["GOROOT"], "pkg", "include")} + for _, define := range source.assemblerDefines { + args = append(args, "-D", define) + } + args = append(args, "-o", object, assemblyInput) + listing, err := source.run(ctx, args...) + if err != nil { + return nil, err + } + objectBytes, err := source.read(object) + if err != nil || len(objectBytes) == 0 { + if err == nil { + err = goAMD64FrameContext("actual assembler produced an empty object") + } + return nil, err + } + if err := os.WriteFile(object+".listing", listing, 0600); err != nil { + return nil, err + } + if _, err := source.read(object + ".listing"); err != nil { + return nil, err + } + var texts []gostaticstorage.TextRecord + if source.numericalBinding != nil { + texts, err = source.numericalFrameObject(object, file) + if err != nil { + return nil, err + } + } + var functions map[string]goAMD64FrameFunction + // An ordinary symbolic reference also needs its complete original + // relocation inventory, even when this source defines no ASM DATA. + if len(file.Data)+len(file.Globl) != 0 || source.numericalBinding == nil && bytes.Contains(listing, []byte("\trel ")) { + observed, readErr := source.frameDataObject(expanded, objectBytes) + if readErr != nil { + return nil, readErr + } + filtered := listing + if len(observed.Layouts) != 0 { + filtered, readErr = goAMD64FrameDataListing(listing, observed) + if readErr != nil { + return nil, readErr + } + } + var unwind *gostaticstorage.WindowsAMD64AssemblyUnwind + if source.request.Target == "windows/amd64" { + leaves, readErr := goAMD64WindowsSourceLeafTexts(file, pkg, listing, observed.Member.Texts, observed.TextOrder) + if readErr != nil { + return nil, readErr + } + value, readErr := source.observeWindowsUnwind(objectBytes, leaves) + if readErr != nil { + return nil, readErr + } + unwind = &value + } + functions, err = goAMD64FrameListingWithDataObject(filtered, file, pkg, source.origins, observed, unwind, + source.request.Target, source.request.Directory, "", source.callees) + if err == nil { + err = goAMD64FrameDataTextJoin(functions, observed, source.request.Directory, source.origins, "") + } + if err == nil && unwind != nil { + err = goAMD64FrameWindowsUnwindFunctions(functions, source.origins, source.request.Directory, *unwind) + } + } else if source.request.Target == "windows/amd64" && (gotoolchain.AuditedGo127(source.env["GOVERSION"]) || + bytes.Contains(listing, []byte(" SSEHUNWINDINFO "))) { + builtin, readErr := source.windowsUnwindBuiltin() + if readErr != nil { + return nil, readErr + } + texts, readErr := gostaticstorage.ObserveWindowsAMD64AssemblyText(objectBytes, builtin) + if readErr != nil { + return nil, readErr + } + // The shared reader checked the exact FromAssembly object flag. + order, readErr := gostaticstorage.ASMSourceTextOrder([]byte(expanded), pkg.Path, + gostaticstorage.ASMDataMember{Flags: 4, Texts: texts}) + if readErr != nil { + return nil, readErr + } + leaves, readErr := goAMD64WindowsSourceLeafTexts(file, pkg, listing, texts, order) + if readErr != nil { + return nil, readErr + } + unwind, readErr := source.observeWindowsUnwind(objectBytes, leaves) + if readErr != nil { + return nil, readErr + } + functions, err = goAMD64FrameWindowsUnwindListing(listing, file, pkg, source.origins, + source.request.Directory, unwind) + } else { + if source.numericalBinding == nil && goAMD64FrameSplitListing(listing) { + observed, readErr := source.frameDataObject(expanded, objectBytes) + if readErr != nil { + return nil, readErr + } + functions, err = goAMD64FrameListingWithSplitObject(listing, file, pkg, source.origins, + observed.Member.Texts, nil, source.request.Target, source.request.Directory, "") + if err == nil { + err = goAMD64FrameDataTextJoin(functions, observed, source.request.Directory, source.origins, "") + } + } else { + functions, err = goAMD64FrameListingWithNumericalObject(listing, file, pkg, source.origins, texts) + } + } + if err != nil { + return nil, err + } + if source.numericalBinding != nil && !bytes.Equal(listing, source.numericalBinding.source.native.OriginalListing) { + return nil, goAMD64FrameContext("independent complete original numerical/frame listing differs") + } + for _, text := range texts { + if !bytes.Equal(functions[text.Name].code, text.Code) { + return nil, goAMD64FrameContext("independent complete object/listing code bytes differ") + } + } + if err := source.verifyObjectListing(ctx, object, functions); err != nil { + return nil, err + } + for symbol, fn := range functions { + if _, duplicate := source.functions[symbol]; duplicate { + return nil, goAMD64FrameContext("duplicate original TEXT symbol") + } + declaration, exists := source.declarations[symbol] + if fn.privateLeaf { + if exists { + return nil, goAMD64FrameContext("private leaf acquired a Go declaration") + } + } else if !exists || !reflect.DeepEqual(fn.signature, declaration.signature) { + return nil, goAMD64FrameContext("object lacks its independently checked declaration/layout") + } + fn.transport, fn.languageType = declaration.transport, declaration.languageType + source.functions[symbol] = fn + } + } + if !found { + return nil, goAMD64FrameContext("requested assembly is absent from actual complete selection") + } + withFrame := false + for _, fn := range source.functions { + withFrame = withFrame || fn.locals > 0 + } + if !withFrame { + return nil, goAMD64FrameContext("complete source has no nonzero native frame") + } + source.ready = true + source.numericalCapturing = false + if err := source.Verify(ctx); err != nil { + return nil, err + } + return source, nil +} + +// Verify rechecks actual routing/selection and all original/tool/export/artifact +// bytes. Failure permanently revokes this observation; it grants no memory. +func (source *GoAMD64FrameSource) Verify(ctx context.Context) (verifyErr error) { + if source == nil || source.identity != source { + return goAMD64FrameContext("missing sealed actual-object observation") + } + if source.sdkFrame != nil { + return source.verifySDKFrame(ctx) + } + if err := source.verifyNumericalFrameInputs(ctx); err != nil { + source.revokeNumericalFrame() + return err + } + defer func() { + if err := source.verifyNumericalFrameInputs(ctx); err != nil { + source.revokeNumericalFrame() + // The inner observer already revokes on failure. Its own closed + // generation must not relabel an actual command/cancellation + // failure as a source exclusion during the outer postcheck. + if verifyErr == nil || !errors.Is(err, ErrProbeNeedsContext) { + verifyErr = errors.Join(verifyErr, err) + } + } + }() + return source.verifyFrameOnly(ctx) +} + +func (source *GoAMD64FrameSource) verifyFrameOnly(ctx context.Context) (verifyErr error) { + if source == nil { + return goAMD64FrameContext("missing sealed actual-object observation") + } + private, contextErr := captureGoAMD64FramePrivateContext(ctx) + if contextErr != nil { + source.mu.Lock() + source.ready = false + source.mu.Unlock() + if source.numericalOrigin != nil { + return fmt.Errorf("verify numerical frame observer context: %w", contextErr) + } + return errors.Join(goAMD64FrameContext("expired actual-object observation"), contextErr) + } + defer private.stop() + defer func() { + // Run every caller callback after the locked command/input round. + // A callback may Close or change original inputs; no authority may + // survive that final boundary even if its Err method returns nil. + contextErr := private.verify() + source.mu.Lock() + defer source.mu.Unlock() + if contextErr != nil { + if source.numericalOrigin != nil { + verifyErr = errors.Join(verifyErr, fmt.Errorf("numerical frame caller context expired after actual commands: %w", contextErr)) + } else { + verifyErr = errors.Join(verifyErr, goAMD64FrameContext("caller context expired after actual commands"), contextErr) + } + } + if !source.ready && verifyErr == nil { + verifyErr = goAMD64FrameContext("source closed during caller context callbacks") + } + if verifyErr == nil { + verifyErr = errors.Join(source.verifyInputs(), source.verifyDownloadedOrigin()) + } + if verifyErr != nil { + source.ready = false + } + }() + ctx = private + source.mu.Lock() + defer source.mu.Unlock() + defer func() { + if verifyErr != nil { + source.ready = false + } + }() + if ctx != nil && ctx.Err() != nil { + if source.numericalOrigin != nil { + return fmt.Errorf("verify numerical frame observer context: %w", ctx.Err()) + } + return errors.Join(goAMD64FrameContext("expired actual-object observation"), ctx.Err()) + } + if !source.ready || ctx == nil { + return goAMD64FrameContext("closed, uninitialized or expired actual-object observation") + } + if err := source.verifyDownloadedOrigin(); err != nil { + return err + } + before := source.env + if err := source.observe(ctx); err != nil { + return err + } + if !reflect.DeepEqual(before, source.env) { + return goAMD64FrameContext("actual Go environment changed") + } + if source.downloadedOrigin == nil { + if err := source.verifyOwnedSourceDirectory(); err != nil { + return err + } + } + if err := source.verifyInputs(); err != nil { + return err + } + selected, err := source.load(ctx) + if err != nil { + return err + } + if !reflect.DeepEqual(source.selection, selected) { + return goAMD64FrameContext("actual complete package selection/export changed") + } + imports, err := source.loadOriginalDeclarationImports(ctx) + if err != nil && source.numericalOrigin != nil { + return err + } + if err != nil || !reflect.DeepEqual(source.imports, imports) { + return goAMD64FrameContext("actual complete declaration import graph changed: %v", err) + } + if err := ctx.Err(); err != nil { + if source.numericalOrigin != nil { + return fmt.Errorf("numerical frame verification context expired after actual commands: %w", err) + } + return errors.Join(goAMD64FrameContext("verification context expired after actual commands"), err) + } + // The final complete input/origin check follows caller callbacks in the + // outer deferred boundary; do not repeat a whole-map read before it. + return nil +} + +func (source *GoAMD64FrameSource) verifyOwnedSourceDirectory() error { + if err := gostaticstorage.ValidateOwnedSourceDirectory(source.request.Directory, source.env["GOROOT"], source.env["GOMODCACHE"]); err != nil { + return errors.Join(goAMD64FrameContext("owned source needs a distinct physical SDK/cache role"), err) + } + return nil +} + +func (source *GoAMD64FrameSource) verifyInputs() error { + if err := source.verifyOwned(); err != nil { + return err + } + maximum := 256 // Ordinary owned-main retains its original member bound. + for name, original := range source.childDirectories { + info, err := os.Lstat(name) + if err != nil || !info.IsDir() || !os.SameFile(original, info) { + return goAMD64FrameContext("original child directory identity changed") + } + } + if inventory := source.directoryInventory; inventory != nil { + if err := inventory.Verify(); err != nil { + return goAMD64FrameContext("complete SDK directory membership changed: %v", err) + } + view := inventory.View() + original := view.Directories[source.request.Directory] + if original == nil || source.directoryInfo == nil || !os.SameFile(original, source.directoryInfo) || + len(view.Entries[source.request.Directory]) != len(source.entries) { + return goAMD64FrameContext("complete SDK directory inventory lost its original role") + } + maximum = len(view.Entries[source.request.Directory]) + } + entries, directoryInfo, err := gooriginalsrc.ReadDirectory(source.request.Directory, maximum) + if err != nil || source.directoryInfo == nil || !os.SameFile(source.directoryInfo, directoryInfo) { + return goAMD64FrameContext("original package directory inventory or identity changed: %v", err) + } + var names []string + for _, entry := range entries { + names = append(names, entry.Name()) + } + if !reflect.DeepEqual(names, source.entries) { + return goAMD64FrameContext("original package directory inventory changed") + } + for name, expected := range source.inputs { + digest, err := gooriginalsrc.HashRegularFile(name, 64<<20) + if err != nil || digest != expected { + return goAMD64FrameContext("original/tool/export/object input changed: %s: %v", filepath.Base(name), err) + } + } + return nil +} + +func (source *GoAMD64FrameSource) verifyOwned() error { + info, err := os.Lstat(source.owned) + if err != nil || source.ownedInfo == nil || !info.IsDir() || !os.SameFile(source.ownedInfo, info) { + return goAMD64FrameContext("factory artifact directory identity changed or is inaccessible: %v", err) + } + return nil +} + +// Close invalidates the token before removing only its factory-owned artifacts. +// Calling it again is harmless; no source, tool or cache-owned file is removed. +func (source *GoAMD64FrameSource) Close() error { + if source == nil { + return nil + } + if source.identity != source { + return goAMD64FrameContext("copied or uninitialized observation cannot dispose original artifacts") + } + if source.sdkFrame != nil { + return source.closeSDKFrame() + } + source.mu.Lock() + defer source.mu.Unlock() + source.ready = false + if source.numericalOrigin != nil && source.numericalOrigin != source { + return goAMD64FrameContext("copied numerical frame cannot close the original artifact owner") + } + if source.owned == "" { + return nil + } + if err := source.verifyOwned(); err != nil { + return err + } + err := os.RemoveAll(source.owned) + if err == nil { + source.owned = "" + source.ownedInfo = nil + } + return err +} + +func goAMD64FrameContext(format string, args ...any) error { + return fmt.Errorf("%w: actual Go frame source: %s", ErrProbeNeedsContext, fmt.Sprintf(format, args...)) +} + +func (source *GoAMD64FrameSource) run(ctx context.Context, args ...string) ([]byte, error) { + if ctx == nil { + return nil, goAMD64FrameContext("actual command requires live context") + } + if err := ctx.Err(); err != nil { + if source.numericalOrigin != nil { + return nil, fmt.Errorf("numerical frame actual command context expired: %w", err) + } + return nil, errors.Join(goAMD64FrameContext("actual command context expired"), err) + } + if err := source.verifyDownloadedOrigin(); err != nil { + return nil, err + } + // Capture has not published the source and holds no Frame lock. Verify's + // existing locked command path is guarded outside the entire round instead. + if source.numericalCapturing { + if err := source.numericalBinding.verify(); err != nil { + return nil, err + } + } + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + var output []byte + var err error + if source.legacyDownloadedModule() && len(args) > 0 && args[0] == "list" { + output, err = source.runDownloadedList(ctx, args) + } else { + output, err = goasmcmd.Run(ctx, source.request.Directory, source.request.Environment, source.request.GoBinary, args...) + } + if source.numericalCapturing { + source.numericalCommands = append(source.numericalCommands, numericalSBCommand{ + Directory: source.request.Directory, Binary: source.request.GoBinary, + Arguments: append([]string(nil), args...), Stdout: append([]byte(nil), output...), + }) + err = errors.Join(err, source.numericalBinding.verify()) + } + // Preserve both independent origins and cancellation when commands race + // revocation. Neither observation grants the other's source permission. + return output, errors.Join(err, ctx.Err(), source.verifyDownloadedOrigin()) +} + +func goAMD64FrameRead(name string) (_ []byte, readErr error) { + return goAMD64FrameReadBound(name, 64<<20) +} + +func goAMD64FrameReadBound(name string, maximum int64) (_ []byte, readErr error) { + resolved, err := sourcepath.Resolve(name) + if err != nil || resolved != name { + return nil, goAMD64FrameContext("input path is redirected or inaccessible: %s", filepath.Base(name)) + } + before, err := os.Lstat(name) + if err != nil || !before.Mode().IsRegular() { + return nil, goAMD64FrameContext("input role changed before opening") + } + file, err := os.Open(name) + if err != nil { + return nil, err + } + defer func() { readErr = errors.Join(readErr, file.Close()) }() + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || !os.SameFile(before, info) || maximum < 0 || maximum > 64<<20 || info.Size() > maximum { + return nil, goAMD64FrameContext("input is not a bounded regular file") + } + data, err := io.ReadAll(io.LimitReader(file, maximum+1)) + if err != nil || int64(len(data)) > maximum { + return nil, goAMD64FrameContext("input byte bound/read failure: %v", err) + } + after, err := os.Lstat(name) + if err != nil || !os.SameFile(info, after) || !after.Mode().IsRegular() || int64(len(data)) != info.Size() { + afterSize, sameFile, regular := int64(-1), false, false + if err == nil { + afterSize, sameFile, regular = after.Size(), os.SameFile(info, after), after.Mode().IsRegular() + } + return nil, goAMD64FrameContext("input role/extent changed while reading %q: opened=%d read=%d after=%d same_file=%t regular=%t after_stat=%v", + filepath.Base(name), info.Size(), len(data), afterSize, sameFile, regular, err) + } + return data, nil +} + +func (source *GoAMD64FrameSource) read(name string) ([]byte, error) { + return source.readBound(name, 64<<20) +} + +func (source *GoAMD64FrameSource) readBound(name string, maximum int64) ([]byte, error) { + data, err := goAMD64FrameReadBound(name, maximum) + if err != nil { + return nil, err + } + digest := sha256.Sum256(data) + if before, exists := source.inputs[name]; exists && before != digest { + return nil, goAMD64FrameContext("input changed during capture: %s", filepath.Base(name)) + } + source.inputs[name] = digest + return data, nil +} + +func (source *GoAMD64FrameSource) snapshotDirectory() error { + resolved, err := sourcepath.Resolve(source.request.Directory) + if err != nil || resolved != source.request.Directory { + return goAMD64FrameContext("requires canonical actual package directory") + } + entries, directoryInfo, err := gooriginalsrc.ReadDirectory(resolved, 256) + if err != nil || len(entries) == 0 || len(entries) > 256 { + return goAMD64FrameContext("package directory is unavailable or exceeds its bound") + } + source.directoryInfo = directoryInfo + source.childDirectories = make(map[string]os.FileInfo) + var total int + for _, entry := range entries { + source.entries = append(source.entries, entry.Name()) + if entry.IsDir() { + info, err := entry.Info() + if err != nil || !info.IsDir() { + return goAMD64FrameContext("original child directory role changed during capture") + } + // Child contents belong to separately selected packages/includes. + // Retain identity without recursively reading unrelated testdata. + source.childDirectories[filepath.Join(resolved, entry.Name())] = info + continue + } + data, err := source.read(filepath.Join(resolved, entry.Name())) + if err != nil { + return err + } + total += len(data) + if total > 16<<20 { + return goAMD64FrameContext("original package byte inventory exceeds its bound") + } + } + return nil +} + +func (source *GoAMD64FrameSource) observe(ctx context.Context) error { + if _, err := source.read(source.request.GoBinary); err != nil { + return err + } + data, err := source.run(ctx, "env", "-json", "GOROOT", "GOTOOLDIR", "GOCACHE", "GOMODCACHE", "GOPATH", "GOTMPDIR", "GOVERSION", "GOOS", "GOARCH", "GOFLAGS", "GOEXPERIMENT", "GOAMD64") + if err != nil { + return err + } + var env map[string]string + if err := json.Unmarshal(data, &env); err != nil { + return err + } + // Commands must keep their observed SDK spelling: changing it can rebuild + // builtin tools with different bytes. File reads and driver ownership use + // the independently resolved physical SDK. Verify compares both identities. + root, err := sourcepath.Resolve(env["GOROOT"]) + if err != nil || !filepath.IsAbs(env["GOROOT"]) { + return goAMD64FrameContext("actual SDK root is not an accessible absolute directory: %v", err) + } + env["command/GOROOT"], env["GOROOT"] = env["GOROOT"], root + if env["GOOS"]+"/"+env["GOARCH"] != source.request.Target || env["GOFLAGS"] != "" || env["GOEXPERIMENT"] != "" || + filepath.Join(env["GOROOT"], "bin", gotoolchain.HostExecutable("go")) != source.request.GoBinary { + return goAMD64FrameContext("actual driver/target/CPU/source routing needs a separate audited contract") + } + switch env["GOAMD64"] { + case "v1", "v2", "v3", "v4": + // Each actual profile retains its own selected original Go sources, + // exports and object bytes. Verify compares this environment exactly; + // no capture is relabeled or reused across different CPU profiles. + default: + return goAMD64FrameContext("unregistered actual GOAMD64 profile %q", env["GOAMD64"]) + } + version, err := source.read(filepath.Join(env["GOROOT"], "VERSION")) + if err != nil || strings.Split(string(version), "\n")[0] != env["GOVERSION"] { + return goAMD64FrameContext("actual driver differs from source VERSION") + } + for _, tool := range []string{"asm", "compile", "objdump"} { + executable := gotoolchain.HostExecutable(tool) + data, err := source.run(ctx, "tool", "-n", tool) + if err != nil { + return err + } + name := strings.TrimSpace(string(data)) + if !filepath.IsAbs(name) || strings.ContainsAny(name, "\r\n") || tool != "objdump" && name != filepath.Join(env["GOTOOLDIR"], executable) { + return goAMD64FrameContext("unrecognized actual %s route", tool) + } + resolved, err := sourcepath.Resolve(name) + if err != nil { + return err + } + if tool == "objdump" && name != filepath.Join(env["GOTOOLDIR"], executable) { + cache, err := sourcepath.Resolve(env["GOCACHE"]) + if err != nil || !goAssemblyPathWithinRoot(cache, resolved) || filepath.Base(resolved) != executable { + return goAMD64FrameContext("unknown builtin object observer origin") + } + } + name = resolved + env["route/"+tool] = name + if _, err := source.read(name); err != nil { + return err + } + } + for _, name := range []string{"src/cmd/internal/obj/x86/obj6.go", "src/cmd/internal/obj/x86/asm6.go", "src/cmd/go/internal/work/gc.go", "src/cmd/asm/main.go", "src/cmd/go/internal/tool/tool.go", "src/cmd/internal/objabi/stack.go"} { + if _, err := source.read(filepath.Join(env["GOROOT"], filepath.FromSlash(name))); err != nil { + return err + } + } + source.env = env + return nil +} + +func (source *GoAMD64FrameSource) load(ctx context.Context) (goAMD64FrameSelection, error) { + selected, _, err := source.loadWithRaw(ctx) + return selected, err +} + +func (source *GoAMD64FrameSource) loadWithRaw(ctx context.Context) (goAMD64FrameSelection, []byte, error) { + return source.loadWithRawPackage(ctx, ".") +} + +func (source *GoAMD64FrameSource) loadWithRawPackage(ctx context.Context, packageQuery string) (goAMD64FrameSelection, []byte, error) { + data, err := source.run(ctx, "list", "-json", "-compiled", "-export", "-mod=readonly", packageQuery) + if err != nil { + return goAMD64FrameSelection{}, data, err + } + return source.readRawPackageSelection(data) +} + +// Both root-only and dependency queries retain the same complete root record +// and physical export read. This does not authenticate a caller-supplied JSON. +func (source *GoAMD64FrameSource) readRawPackageSelection(data []byte) (goAMD64FrameSelection, []byte, error) { + var selected goAMD64FrameSelection + decoder := json.NewDecoder(bytes.NewReader(data)) + if err := decoder.Decode(&selected); err != nil { + return selected, data, err + } + var extra any + if decoder.Decode(&extra) != io.EOF || len(selected.Error) != 0 || len(selected.DepsErrors) != 0 { + return selected, data, goAMD64FrameContext("incomplete or ambiguous actual package export") + } + if selected.Export == "" { + return selected, data, goAMD64FrameContext("actual package has no compiled export") + } + export, err := sourcepath.Resolve(selected.Export) + selected.Export = export + cache, cacheErr := sourcepath.Resolve(source.env["GOCACHE"]) + if err != nil || cacheErr != nil || !goAssemblyPathWithinRoot(cache, selected.Export) { + return selected, data, goAMD64FrameContext("compiled export is outside the actual build cache") + } + if export, err := source.read(selected.Export); err != nil || len(export) == 0 { + if err == nil { + err = goAMD64FrameContext("actual package export is empty") + } + return selected, data, err + } + return selected, data, nil +} + +func (source *GoAMD64FrameSource) numericDeclarations(pkg GoPackage) error { + _, err := source.typedDeclarations(pkg, true) + return err +} + +// The numeric-only query remains strict. The fixed-local binder separately +// checks the closed transport role and replays each actual typed component; +// observation itself never supplies local or incoming memory permission. +func (source *GoAMD64FrameSource) typedDeclarations(pkg GoPackage, numericOnly bool) (map[string]goAMD64FrameDeclaration, error) { + return source.typedDeclarationsForArch(pkg, numericOnly, "amd64") +} + +// The numerical SB binding also observes 386 declarations. This helper only +// rechecks language/ABI0 identity; it does not extend the AMD64 frame factory. +func (source *GoAMD64FrameSource) typedDeclarationsForArch(pkg GoPackage, numericOnly bool, arch string) (map[string]goAMD64FrameDeclaration, error) { + return source.typedDeclarationsForArchRole(pkg, numericOnly, arch, false, nil) +} + +func (source *GoAMD64FrameSource) typedDeclarationsForArchRole(pkg GoPackage, numericOnly bool, arch string, + opaqueUnsafePointer bool, entries map[string]bool, +) (map[string]goAMD64FrameDeclaration, error) { + return source.typedDeclarationsForSourceRole(pkg, numericOnly, arch, opaqueUnsafePointer, entries, false) +} + +// Numerical-source binding observes ordinary F32/F64 declaration identity and +// ABI0 slots, not the frame factory's integer transport. This separate route +// supplies no physical frame, caller, register-entry or numerical input facts. +func (source *GoAMD64FrameSource) typedNumericalDeclarationsForArchRole(pkg GoPackage, arch string, + opaqueUnsafePointer bool, entries map[string]bool, +) (map[string]goAMD64FrameDeclaration, error) { + return source.typedDeclarationsForSourceRole(pkg, false, arch, opaqueUnsafePointer, entries, true) +} + +func (source *GoAMD64FrameSource) typedDeclarationsForSourceRole(pkg GoPackage, numericOnly bool, arch string, + opaqueUnsafePointer bool, entries map[string]bool, floatingScalars bool, globals ...map[string]goReadOnlyObject, +) (map[string]goAMD64FrameDeclaration, error) { + return source.typedSourceDeclarations(pkg, numericOnly, arch, opaqueUnsafePointer, entries, floatingScalars, false, globals...) +} + +func (source *GoAMD64FrameSource) typedSourceDeclarations(pkg GoPackage, numericOnly bool, arch string, + opaqueUnsafePointer bool, entries map[string]bool, floatingScalars, bodies bool, globals ...map[string]goReadOnlyObject, +) (map[string]goAMD64FrameDeclaration, error) { + if len(globals) > 1 { + return nil, goAMD64FrameContext("ambiguous original Go variable inventory") + } + fset := token.NewFileSet() + var syntax []*ast.File + var compiled []string + bodyless := make(map[string]bool) + var implemented map[string]bool + if bodies { + implemented = make(map[string]bool) + } + for _, name := range source.selection.GoFiles { + filename := filepath.Join(source.request.Directory, name) + data, err := source.read(filename) + if err != nil { + return nil, err + } + file, err := parser.ParseFile(fset, filename, data, parser.ParseComments) + if err != nil { + return nil, err + } + for _, declaration := range file.Decls { + switch declaration := declaration.(type) { + case *ast.FuncDecl: + if declaration.Body == nil && declaration.Recv == nil { + bodyless[declaration.Name.Name] = true + } + if bodies && declaration.Body != nil && declaration.Recv == nil { + implemented[declaration.Name.Name] = true + } + case *ast.GenDecl: + if declaration.Tok != token.TYPE && declaration.Tok != token.IMPORT && + declaration.Tok != token.CONST && declaration.Tok != token.VAR { + return nil, goAMD64FrameContext("unknown original declaration role") + } + default: + return nil, goAMD64FrameContext("unknown original declaration role") + } + } + syntax = append(syntax, file) + compiled = append(compiled, filename) + } + if len(pkg.Syntax) != len(syntax) { + return nil, goAMD64FrameContext("supplied syntax omitted complete ordinary source files") + } + if len(bodyless) == 0 { + return nil, goAMD64FrameContext("original source has no bodyless assembly declaration") + } + sort.Strings(compiled) + actual := append([]string(nil), source.selection.CompiledGoFiles...) + for i, name := range actual { + if !filepath.IsAbs(name) { + actual[i] = filepath.Join(source.request.Directory, name) + } + } + sort.Strings(actual) + if !reflect.DeepEqual(compiled, actual) { + return nil, goAMD64FrameContext("effective/generated compiled source mapping is unbound") + } + sizes := types.SizesFor("gc", arch) + actualImporter := importer.ForCompiler(fset, "gc", func(path string) (io.ReadCloser, error) { + if target, mapped := source.selection.ImportMap[path]; mapped { + path = target + } + dependency, present := source.imports[path] + if !present || dependency.Export == "" { + return nil, goAMD64FrameContext("missing captured actual compiler export: %s", path) + } + data, err := source.read(dependency.Export) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(data)), nil + }) + config := &types.Config{Sizes: sizes, Importer: actualImporter} + if source.legacyDownloadedModule() { + config.GoVersion = "go1.16" + } + typed, err := config.Check(pkg.Path, fset, syntax, nil) + if err != nil { + return nil, err + } + if !reflect.DeepEqual(typed.Scope().Names(), pkg.Types.Scope().Names()) { + return nil, goAMD64FrameContext("actual selected declarations differ from supplied types") + } + if bodies { + names, err := numericalGoDeclarationLinknames(syntax, typed) + if err != nil { + return nil, err + } + for name := range entries { + if _, linked := names.localToNative[name]; linked { + return nil, goAMD64FrameContext("ordinary Go CALL requires its original unaliased implementation") + } + } + } + if len(globals) == 1 { + if err := joinFixedLocalGlobalDeclarations(typed, pkg.Types, syntax, sizes, globals[0]); err != nil { + return nil, err + } + } + languageType := goAMD64FrameLanguageTypeRole + if floatingScalars { + languageType = goAMD64NumericalScalarLanguageType + } + declarations := make(map[string]goAMD64FrameDeclaration) + for _, name := range typed.Scope().Names() { + if named, ok := typed.Scope().Lookup(name).(*types.TypeName); ok { + previous, supplied := pkg.Types.Scope().Lookup(name).(*types.TypeName) + if !supplied || named.IsAlias() != previous.IsAlias() { + return nil, goAMD64FrameContext("actual local type declaration differs: %s", name) + } + // The complete original Go program has already been typechecked. + // Unrelated interfaces, callbacks, maps or recursive/generic types + // do not cross an assembly ABI and must not enter its restricted + // transport grammar. Every type reachable from an actual assembly + // parameter/result is independently compared in full below. + continue + } + // The default table contains only bodyless assembly declarations. + // A requested Go-body table is independently typed and kept separate. + if !bodies && !bodyless[name] || bodies && !implemented[name] { + continue + } + if entries != nil && !entries[name] { + // All original sources, bodies, declarations and complete scope + // names were independently checked above. An unrelated bodyless + // runtime declaration does not enter this selected ASM entry set. + continue + } + fn, ok := typed.Scope().Lookup(name).(*types.Func) + previous, supplied := pkg.Types.Scope().Lookup(name).(*types.Func) + if !ok || !supplied { + return nil, goAMD64FrameContext("missing actual typed declaration") + } + signature := fn.Type().(*types.Signature) + priorSignature, valid := previous.Type().(*types.Signature) + if !valid || priorSignature == nil || signature.Recv() != nil || signature.Variadic() || signature.TypeParams().Len() != 0 || + priorSignature.Recv() != nil || priorSignature.Variadic() || priorSignature.TypeParams().Len() != 0 { + return nil, goAMD64FrameContext("custom, generic or variadic declaration transport") + } + role := goAMD64FrameNumericTransport + var language strings.Builder + for _, tuple := range []*types.Tuple{signature.Params(), signature.Results()} { + if tuple.Len() > 4096 { + return nil, goAMD64FrameContext("declaration tuple bound") + } + language.WriteString("[") + for i := 0; i < tuple.Len(); i++ { + typ := tuple.At(i).Type() + switch underlying := typ.Underlying().(type) { + case *types.Basic: + if floatingScalars && goAMD64NumericalFloatingScalar(typ) { + if role == goAMD64FrameNumericTransport { + role = goAMD64FrameFloatingScalarTransport + } + } else if underlying.Kind() == types.Bool || opaqueUnsafePointer && underlying.Kind() == types.UnsafePointer { + role = goAMD64FrameCompositeTransport + } else if underlying.Info()&types.IsInteger == 0 { + return nil, goAMD64FrameContext("observation excludes noninteger scalar transport") + } + case *types.Pointer, *types.Slice: + role = goAMD64FrameCompositeTransport + default: + return nil, goAMD64FrameContext("by-value aggregate transport needs a separate observation contract") + } + identity, err := languageType(typ, sizes, opaqueUnsafePointer) + if err != nil { + return nil, err + } + fmt.Fprintf(&language, "%q:%s;", tuple.At(i).Name(), identity) + if language.Len() > 64<<10 { + return nil, goAMD64FrameContext("declaration language byte bound") + } + } + language.WriteString("]") + } + if numericOnly && role != goAMD64FrameNumericTransport { + return nil, goAMD64FrameContext("fixed-local consumer requires numeric transport, not pointer/slice observation") + } + // Named types from two independent typechecks have different object + // identities. Compare their complete language/layout trees, not Ptr or + // qualified spelling alone; underlying and pointee changes must fail. + priorLanguage := strings.Builder{} + for _, tuple := range []*types.Tuple{priorSignature.Params(), priorSignature.Results()} { + if tuple.Len() > 4096 { + return nil, goAMD64FrameContext("supplied declaration tuple bound") + } + priorLanguage.WriteString("[") + for i := 0; i < tuple.Len(); i++ { + identity, err := languageType(tuple.At(i).Type(), sizes, opaqueUnsafePointer) + if err != nil { + return nil, err + } + fmt.Fprintf(&priorLanguage, "%q:%s;", tuple.At(i).Name(), identity) + if priorLanguage.Len() > 64<<10 { + return nil, goAMD64FrameContext("supplied declaration language byte bound") + } + } + priorLanguage.WriteString("]") + } + if language.String() != priorLanguage.String() { + return nil, goAMD64FrameContext("actual language type/pointee/field identity mismatch: %s", name) + } + actual, err := DeriveGoFuncABI0Signature(pkg.Path+"."+name, fn, arch, sizes) + prior, priorErr := DeriveGoFuncABI0Signature(pkg.Path+"."+name, previous, arch, sizes) + if err != nil || priorErr != nil || !reflect.DeepEqual(actual, prior) { + return nil, goAMD64FrameContext("actual typed declaration/layout mismatch: %s", name) + } + declarations[actual.Name] = goAMD64FrameDeclaration{signature: actual, transport: role, languageType: language.String()} + } + if entries != nil && len(declarations) != len(entries) { + if bodies { + return nil, goAMD64FrameContext("original CALL needs its complete same-source leaf or selected Go body") + } + return nil, goAMD64FrameContext("complete selected ASM entry declaration inventory differs") + } + return declarations, nil +} + +// A bounded semantic language tree distinguishes equal-sized LLVM pointers, +// named-underlying changes and aggregate element/field models. Sizes belongs +// to the independent go/types model, not an observation of native pointee +// padding (Go1.20 StdSizes differs from its compiler). Only pointer/slice ABI0 +// slots are bound to the actual object. No pointee extent, alias or GC grant. +func goAMD64FrameLanguageType(typ types.Type, sizes types.Sizes) (string, error) { + return goAMD64FrameLanguageTypeRole(typ, sizes, false) +} + +// The SDK numerical declaration observer may retain an unsafe.Pointer only as +// an opaque language identity. The ordinary frame observer remains strict; +// no pointee, extent, numerical bytes, GC or physical frame follows from it. +func goAMD64FrameLanguageTypeRole(typ types.Type, sizes types.Sizes, opaqueUnsafePointer bool) (string, error) { + return goAMD64LanguageTypeRole(typ, sizes, opaqueUnsafePointer, false) +} + +func goAMD64NumericalFloatingScalar(typ types.Type) bool { + for depth := 0; depth <= 32 && typ != nil; depth++ { + switch concrete := typ.(type) { + case *types.Basic: + return concrete.Kind() == types.Float32 || concrete.Kind() == types.Float64 + case *types.Named: + if concrete.TypeParams().Len() != 0 { + return false + } + typ = concrete.Underlying() + default: + if !goAMD64FrameIsAlias(typ) { + return false // Never sample caller-defined types.Type callbacks. + } + rhs := reflect.ValueOf(typ).MethodByName("Rhs") + if !rhs.IsValid() || rhs.Type().NumIn() != 0 || rhs.Type().NumOut() != 1 { + return false + } + var valid bool + typ, valid = rhs.Call(nil)[0].Interface().(types.Type) + if !valid { + return false + } + } + } + return false +} + +// Only a top-level scalar (including its named/alias identity) selects this +// language role. Pointers, slices and aggregates keep the ordinary recursive +// restrictions; this cannot borrow the broader ABIInternal argument grammar. +func goAMD64NumericalScalarLanguageType(typ types.Type, sizes types.Sizes, opaqueUnsafePointer bool) (string, error) { + return goAMD64LanguageTypeForSourceRole(typ, sizes, opaqueUnsafePointer, false, goAMD64NumericalFloatingScalar(typ)) +} + +// The SDK argument role compares language identities only. It never grants +// the ordinary frame observer floating/string storage or a pointee extent. +func goAMD64LanguageTypeRole(typ types.Type, sizes types.Sizes, opaqueUnsafePointer, argumentCarriers bool) (string, error) { + return goAMD64LanguageTypeForSourceRole(typ, sizes, opaqueUnsafePointer, argumentCarriers, false) +} + +func goAMD64LanguageTypeForSourceRole(typ types.Type, sizes types.Sizes, opaqueUnsafePointer, argumentCarriers, floatingScalar bool) (string, error) { + if sizes == nil { + return "", goAMD64FrameContext("missing target language layout") + } + var output strings.Builder + active := make(map[types.Type]bool) + nodes := 0 + var appendType func(types.Type, int) error + appendType = func(typ types.Type, depth int) error { + nodes++ + if typ == nil || depth > 32 || nodes > 4096 || active[typ] || output.Len() > 64<<10 { + return goAMD64FrameContext("recursive or unbounded language type") + } + active[typ] = true + defer delete(active, typ) + // Check the language role before invoking Sizes: gc Sizes may panic + // for type parameters. An unknown or generic role is not a layout. + switch concrete := typ.(type) { + case *types.Basic: + if concrete.Info()&(types.IsInteger|types.IsBoolean) == 0 && !(opaqueUnsafePointer && concrete.Kind() == types.UnsafePointer) && + !(argumentCarriers && concrete.Info()&(types.IsFloat|types.IsComplex|types.IsString) != 0) && + !(floatingScalar && goAMD64NumericalFloatingScalar(concrete)) || + concrete.Info()&types.IsUntyped != 0 { + return goAMD64FrameContext("unsupported noninteger or untyped language type") + } + case *types.Named: + if concrete.TypeParams().Len() != 0 { + return goAMD64FrameContext("generic language type") + } + case *types.Pointer, *types.Slice, *types.Array, *types.Struct: + default: + if !goAMD64FrameIsAlias(typ) { + return goAMD64FrameContext("unknown language type role") + } + if method := reflect.ValueOf(typ).MethodByName("TypeParams"); method.IsValid() { + parameters, ok := method.Call(nil)[0].Interface().(*types.TypeParamList) + if !ok || parameters.Len() != 0 { + return goAMD64FrameContext("generic alias language type") + } + } + } + // Composite identity is written below one node at a time. Asking + // TypeString to expand an anonymous DAG defeats the node/byte bound. + spelling := "" + switch typ.(type) { + case *types.Basic, *types.Named: + spelling = types.TypeString(typ, func(pkg *types.Package) string { return pkg.Path() }) + default: + if goAMD64FrameIsAlias(typ) { + spelling = types.TypeString(typ, func(pkg *types.Package) string { return pkg.Path() }) + } + } + fmt.Fprintf(&output, "(%q:", spelling) + switch typ := typ.(type) { + case *types.Basic: + if opaqueUnsafePointer && typ.Kind() == types.UnsafePointer { + output.WriteString("opaque_unsafe_pointer") + } else if typ.Kind() == types.Bool { + output.WriteString("bool") + } else if typ.Info()&types.IsInteger != 0 { + fmt.Fprintf(&output, "integer%d", typ.Kind()) + } else if floatingScalar && goAMD64NumericalFloatingScalar(typ) { + fmt.Fprintf(&output, "floating_scalar%d", typ.Kind()) + } else if argumentCarriers && typ.Info()&(types.IsFloat|types.IsComplex|types.IsString) != 0 { + fmt.Fprintf(&output, "argument_scalar%d", typ.Kind()) + } else { + return goAMD64FrameContext("unsupported noninteger pointee/element") + } + case *types.Named: + if typ.TypeParams().Len() != 0 { + return goAMD64FrameContext("generic language type") + } + output.WriteString("named:") + if err := appendType(typ.Underlying(), depth+1); err != nil { + return err + } + case *types.Pointer: + output.WriteString("pointer:") + if err := appendType(typ.Elem(), depth+1); err != nil { + return err + } + case *types.Slice: + output.WriteString("slice:") + if err := appendType(typ.Elem(), depth+1); err != nil { + return err + } + case *types.Array: + fmt.Fprintf(&output, "array%d:", typ.Len()) + if err := appendType(typ.Elem(), depth+1); err != nil { + return err + } + // Go1.20's Sizes can wrap a large array to zero. Its recorded + // result is not evidence of a representable concrete layout. + element, alignment := sizes.Sizeof(typ.Elem()), sizes.Alignof(typ.Elem()) + const maxLayout = int64(1<<63 - 1) + if typ.Len() < 0 || element < 0 || alignment <= 0 || element > maxLayout-(alignment-1) { + return goAMD64FrameContext("array element layout is unrepresentable") + } + stride := (element + alignment - 1) / alignment * alignment + if typ.Len() > 0 && stride != 0 && typ.Len()-1 > (maxLayout-element)/stride { + return goAMD64FrameContext("array language layout overflow") + } + case *types.Struct: + if typ.NumFields() > 4096-nodes { + return goAMD64FrameContext("language field bound") + } + fields := make([]*types.Var, typ.NumFields()) + for i := range fields { + fields[i] = typ.Field(i) + } + // Validate every descendant before Sizes traverses the aggregate: + // generic/incomplete elements are not a concrete layout. + for _, field := range fields { + if err := appendType(field.Type(), depth+1); err != nil { + return err + } + } + offsets := sizes.Offsetsof(fields) + output.WriteString("struct:") + for i, field := range fields { + pkg := "" + if field.Pkg() != nil { + pkg = field.Pkg().Path() + } + width := sizes.Sizeof(field.Type()) + if offsets[i] < 0 || width < 0 || offsets[i] > (1<<63-1)-width { + return goAMD64FrameContext("struct language layout overflow") + } + fmt.Fprintf(&output, "%q/%q/%q/%t/%d:", pkg, field.Name(), typ.Tag(i), field.Embedded(), offsets[i]) + if output.Len() > 64<<10 { + return goAMD64FrameContext("language field metadata byte bound") + } + } + default: + // Alias is newer than Go1.20. Its actual spelling and the local + // TypeName.IsAlias role are retained alongside the complete RHS. + // Underlying alone would lose a named RHS's language identity. + if !goAMD64FrameIsAlias(typ) { + return goAMD64FrameContext("unknown language type role") + } + method := reflect.ValueOf(typ).MethodByName("Rhs") + if !method.IsValid() || method.Type().NumIn() != 0 || method.Type().NumOut() != 1 { + return goAMD64FrameContext("unknown alias RHS contract") + } + rhs, ok := method.Call(nil)[0].Interface().(types.Type) + if !ok { + return goAMD64FrameContext("missing alias RHS language type") + } + output.WriteString("alias:") + if err := appendType(rhs, depth+1); err != nil { + return err + } + } + size, align := sizes.Sizeof(typ), sizes.Alignof(typ) + if size < 0 || align <= 0 { + return goAMD64FrameContext("language type lacks a concrete target layout") + } + fmt.Fprintf(&output, ";model_size=%d,model_align=%d)", size, align) + return nil + } + if err := appendType(typ, 0); err != nil { + return "", err + } + if output.Len() > 64<<10 { + return "", goAMD64FrameContext("language type byte bound") + } + return output.String(), nil +} + +func goAMD64FrameIsAlias(typ types.Type) bool { + kind := reflect.TypeOf(typ) + return kind != nil && kind.Kind() == reflect.Pointer && kind.Elem().PkgPath() == "go/types" && kind.Elem().Name() == "Alias" +} + +func (source *GoAMD64FrameSource) preprocess(name string, original []byte) (string, error) { + if source.sdkFrame != nil { + return source.preprocessSDKFrame(name, original) + } + if len(source.assemblerDefines) == 0 { + return "", goAMD64FrameContext("CPP lacks its actual assembler macro registration") + } + source.origins[filepath.Join(source.request.Directory, name)] = bytes.Count(original, []byte("\n")) + 1 + // The owned asm command passes this exact basename from the canonical + // package directory; stock Go can retain that relative spelling in -S. + source.origins[name] = source.origins[filepath.Join(source.request.Directory, name)] + var expand func(string, []byte) (string, error) + expand = func(name string, data []byte) (string, error) { + return PreprocessAssemblySource(string(data), AssemblyPreprocessOptions{FileName: name, Defines: source.assemblerDefines, ReadInclude: func(parent, include string) (string, []byte, error) { + if filepath.Base(include) != include || include == "go_asm.h" { + return "", nil, goAMD64FrameContext("generated/relative CPP origin requires qualified metadata") + } + file := filepath.Join(source.request.Directory, include) + if _, err := os.Stat(file); os.IsNotExist(err) { + file = filepath.Join(source.env["GOROOT"], "pkg", "include", include) + } + data, err := source.read(file) + if err != nil { + return "", nil, err + } + source.origins[file] = bytes.Count(data, []byte("\n")) + 1 + return file, data, nil + }}) + } + return expand(name, original) +} + +var goAMD64FrameRow = regexp.MustCompile(`^\s*0x([0-9a-f]+) [0-9]+ \((.+)\)\s+(.+)$`) + +// Zero-frame ordinary entries have no automatic saved-BP slot. They remain +// in the complete source inventory without acquiring private local backing. +func goAMD64FrameNativeLocals(frame int64) int64 { + if frame == 0 { + return 0 + } + return frame + 8 +} + +func goAMD64FrameListing(listing []byte, file *File, pkg GoPackage, origins map[string]int) (map[string]goAMD64FrameFunction, error) { + return goAMD64FrameListingWithObservations(listing, file, pkg, origins, nil, nil) +} + +func goAMD64FrameListingWithNumericalObject(listing []byte, file *File, pkg GoPackage, origins map[string]int, texts []gostaticstorage.TextRecord) (map[string]goAMD64FrameFunction, error) { + return goAMD64FrameListingWithObservations(listing, file, pkg, origins, texts, nil) +} + +func goAMD64FrameListingWithWindowsUnwind(listing []byte, file *File, pkg GoPackage, origins map[string]int, unwind *gostaticstorage.WindowsAMD64AssemblyUnwind) (map[string]goAMD64FrameFunction, error) { + return goAMD64FrameListingWithObservations(listing, file, pkg, origins, nil, unwind) +} + +// These independent, target-specific observation routes retain the strict +// default parser. A numerical relocation does not authorize Windows metadata, +// and Windows unwind metadata does not authorize a numerical memory read. +func goAMD64FrameListingWithObservations(listing []byte, file *File, pkg GoPackage, origins map[string]int, texts []gostaticstorage.TextRecord, unwind *gostaticstorage.WindowsAMD64AssemblyUnwind) (map[string]goAMD64FrameFunction, error) { + if texts != nil && unwind != nil { + return nil, goAMD64FrameContext("mixed target-specific object observation roles") + } + return goAMD64FrameListingRecords(listing, file, pkg, origins, texts, unwind, nil, "", "", "") +} + +func goAMD64FrameListingWithDataObject(listing []byte, file *File, pkg GoPackage, origins map[string]int, + data *gostaticstorage.AMD64AssemblyData, unwind *gostaticstorage.WindowsAMD64AssemblyUnwind, target, directory, root string, + callees ...map[string]goAMD64FrameDeclaration) (map[string]goAMD64FrameFunction, error) { + if data == nil || unwind != nil && !reflect.DeepEqual(data.Member.Texts, unwind.Texts) { + return nil, goAMD64FrameContext("mixed DATA/Windows complete object TEXT observations differ") + } + texts, err := goAMD64FrameSourceTexts(listing, data) + if err != nil { + return nil, err + } + return goAMD64FrameListingRecords(listing, file, pkg, origins, texts, unwind, data, target, directory, root, callees...) +} + +func goAMD64FrameListingRecords(listing []byte, file *File, pkg GoPackage, origins map[string]int, texts []gostaticstorage.TextRecord, unwind *gostaticstorage.WindowsAMD64AssemblyUnwind, data *gostaticstorage.AMD64AssemblyData, splitTarget, splitDirectory, splitRoot string, + callees ...map[string]goAMD64FrameDeclaration) (map[string]goAMD64FrameFunction, error) { + if len(callees) > 1 { + return nil, goAMD64FrameContext("ambiguous ordinary Go call declarations") + } + leaves, err := goAMD64FramePrivateLeaves(file, pkg) + if err != nil { + return nil, err + } + if len(leaves) != 0 && data == nil { + return nil, goAMD64FrameContext("private leaf requires complete original source/object association") + } + names := make(map[string]string) + if data != nil { + if len(data.TextOrder) != len(file.Funcs) { + return nil, goAMD64FrameContext("complete parsed/source-object TEXT counts differ") + } + for index, fn := range file.Funcs { + names[fn.Sym] = data.TextOrder[index].Name + } + } + functions := make(map[string]goAMD64FrameFunction) + objects := make(map[string]gostaticstorage.TextRecord) + metadata := make(map[string][]byte) + metadataHeaders := make(map[string]string) + seenMetadata := make(map[string]bool) + if unwind != nil { + for _, record := range unwind.Unwind { + if _, duplicate := metadata[record.Name]; duplicate { + return nil, goAMD64FrameContext("duplicate actual Windows metadata definition") + } + metadata[record.Name] = record.Data + header := record.Name + " SSEHUNWINDINFO dupok size=12" + if unwind.ListingAlignment { + header += " align=0x4" + } + metadataHeaders[header] = record.Name + } + } + scanner := bufio.NewScanner(bytes.NewReader(listing)) + scanner.Buffer(nil, 4<<20) + name := "" + metadataName := "" + var metadataBytes []byte + var expected int64 + textIndex, relocationIndex := -1, 0 + finishRelocations := func() error { + if texts != nil && textIndex >= 0 && relocationIndex != len(texts[textIndex].Relocations) { + return goAMD64FrameContext("missing complete original object/listing relocations") + } + return nil + } + for scanner.Scan() { + line := scanner.Text() + if header, valid := parseGoAMD64NativeHeader(line); valid { + if !header.ordinaryFrameRole() { + return nil, goAMD64FrameContext("native header has no ordinary/private frame entry role") + } + if !header.NoSplit && (splitTarget == "" || texts == nil) { + return nil, goAMD64FrameContext("automatic split layout requires its complete original object") + } + if metadataName != "" { + return nil, goAMD64FrameContext("Windows listing TEXT follows its metadata inventory") + } + if err := finishRelocations(); err != nil { + return nil, err + } + if texts != nil { + textIndex++ + relocationIndex = 0 + if textIndex >= len(texts) || texts[textIndex].Name != header.Symbol { + return nil, goAMD64FrameContext("reordered or foreign complete object/listing TEXT") + } + abi := uint16(0) + if header.Static { + abi = 65535 + } + if texts[textIndex].Kind != header.Kind || texts[textIndex].ABI != abi || + header.objectFlags() != texts[textIndex].Flags&0x19 { + return nil, goAMD64FrameContext("complete object/listing TEXT kind or ABI differs") + } + if splitTarget != "" && texts[textIndex].Flags != header.objectFlags() { + return nil, goAMD64FrameContext("complete object/listing TEXT NOSPLIT flags differ") + } + objects[header.Symbol] = texts[textIndex] + } else if header.Kind != 1 { + return nil, goAMD64FrameContext("FIPS TEXT needs its complete original object kind") + } + if name != "" && int64(len(functions[name].code)) != expected { + return nil, goAMD64FrameContext("partial native symbol bytes") + } + name = header.Symbol + if _, duplicate := functions[name]; duplicate { + return nil, goAMD64FrameContext("duplicate native TEXT") + } + expected = int64(header.Size) + functions[name] = goAMD64FrameFunction{locals: int64(header.Locals), args: header.Args, split: !header.NoSplit, positions: make(map[int]string), sourcePositions: make(map[int][]goAMD64FrameSourcePosition)} + continue + } + if header, exists := metadataHeaders[line]; exists { + if seenMetadata[header] || metadata[header] == nil || + name != "" && int64(len(functions[name].code)) != expected || + metadataName != "" && !bytes.Equal(metadataBytes, metadata[metadataName]) { + return nil, goAMD64FrameContext("unbound, duplicate or incomplete Windows metadata listing") + } + metadataName, name, metadataBytes = header, "", nil + seenMetadata[metadataName] = true + continue + } + if metadataName != "" { + row, err := goAMD64WindowsUnwindByteRow(line, len(metadataBytes), metadata[metadataName]) + if err != nil { + return nil, err + } + metadataBytes = append(metadataBytes, row...) + continue + } + if name == "" { + return nil, goAMD64FrameContext("native listing contains unbound/unknown symbol") + } + fn := functions[name] + if strings.HasPrefix(strings.TrimSpace(line), "rel ") && texts != nil { + relocations := texts[textIndex].Relocations + if relocationIndex >= len(relocations) { + return nil, goAMD64FrameContext("extra actual object/listing relocation") + } + expectedLine := goAMD64FrameNumericalRelocationLine(relocations[relocationIndex]) + if splitTarget != "" { + expectedLine = "" + } + if data != nil { + expectedLine = goAMD64FrameDataRelocationLine(relocations[relocationIndex], data) + } + if fn.split && splitTarget != "" { + if generated := goAMD64FrameSplitRelocationLine(relocations[relocationIndex], splitTarget); generated != "" { + expectedLine = generated + } + } + if expectedLine == "" || strings.Join(strings.Fields(line), " ") != expectedLine { + return nil, goAMD64FrameContext("foreign, duplicate or reordered actual object/listing relocation") + } + relocationIndex++ + continue + } + if row := goAMD64FrameRow.FindStringSubmatch(line); row != nil { + pc, err := strconv.ParseInt(row[1], 16, 32) + if err != nil || pc < 0 || pc >= expected { + return nil, goAMD64FrameContext("invalid original instruction/source PC") + } + separator := strings.LastIndexByte(row[2], ':') + if separator < 0 { + return nil, goAMD64FrameContext("unbound original instruction/source path") + } + line, err := strconv.Atoi(row[2][separator+1:]) + if err != nil || line <= 0 || line > origins[row[2][:separator]] { + return nil, goAMD64FrameContext("unbound original instruction/source line: %s", row[2]) + } + fn.positions[int(pc)] += row[2] + "\t" + row[3] + "\n" + fn.sourcePositions[int(pc)] = append(fn.sourcePositions[int(pc)], + goAMD64FrameSourcePosition{row[2][:separator], line}) + } else { + pc, code, err := goAssemblyListingByteRow(line) + if err != nil || pc != len(fn.code) || int64(pc+len(code)) > expected { + return nil, goAMD64FrameContext("invalid native byte record or gap/overlap: %v", err) + } + fn.code = append(fn.code, code...) + } + functions[name] = fn + } + if err := scanner.Err(); err != nil { + return nil, err + } + if err := finishRelocations(); err != nil { + return nil, err + } + if texts != nil && textIndex+1 != len(texts) { + return nil, goAMD64FrameContext("missing complete original object/listing TEXT") + } + if len(functions) != len(file.Funcs) || + metadataName == "" && (name == "" || int64(len(functions[name].code)) != expected) || + metadataName != "" && !bytes.Equal(metadataBytes, metadata[metadataName]) || + len(seenMetadata) != len(metadata) { + return nil, goAMD64FrameContext("missing/incomplete original TEXT inventory") + } + bound := make(map[string]goAMD64FrameFunction) + for _, original := range file.Funcs { + symbol := pkg.Path + "." + strings.TrimPrefix(original.Sym, "·") + if leaves[original.Sym] { + symbol = names[original.Sym] + } + observed := symbol + fn, exists := functions[observed] + if !exists { + // Older actual Go objects retain PkgIdxSelf's empty-package + // spelling. Keep those observed bytes/symbols distinct from the + // package identity independently selected and type-checked above. + observed = `"".` + strings.TrimPrefix(original.Sym, "·") + fn, exists = functions[observed] + } + declaration, declared := pkg.Types.Scope().Lookup(strings.TrimPrefix(original.Sym, "·")).(*types.Func) + fn.sourceSymbol, fn.privateLeaf = original.Sym, leaves[original.Sym] + if text, found := objects[observed]; found { + fn.nativeABI = text.ABI + } + if fn.privateLeaf { + if !exists || declared || fn.split || fn.locals != goAMD64FrameNativeLocals(original.FrameSize) || + fn.args != 0 && fn.args != -0x80000000 { + return nil, goAMD64FrameContext("original private leaf/native frame identity differs") + } + } else if !exists || !declared || original.FrameSize < 0 || original.FrameSize%8 != 0 || fn.locals != goAMD64FrameNativeLocals(original.FrameSize) || !goAMD64FrameArgumentsMatch(original, fn.args) || strings.Contains(original.Sym, "<") { + return nil, goAMD64FrameContext("original ABI0 TEXT/declaration/frame identity mismatch") + } + var err error + if !fn.privateLeaf { + fn.signature, err = DeriveGoFuncABI0Signature(symbol, declaration, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + return nil, err + } + var end int64 + for _, slot := range append(append([]FrameSlot(nil), fn.signature.Frame.Params...), fn.signature.Frame.Results...) { + if next := slot.Offset + frameTypeSize(slot.Type, 8); next > end { + end = next + } + } + if !x86TEXTArgumentsUnspecified(original) && !goabi.MatchesABI0TextSize("amd64", fn.args, end) { + return nil, goAMD64FrameContext("original TEXT args disagree with actual declaration") + } + } + callTargets, callNames := leaves, names + if len(callees) == 1 && len(callees[0]) != 0 && original.FrameSize > 0 && !fn.privateLeaf { + callTargets = make(map[string]bool, len(leaves)+len(callees[0])) + for name := range leaves { + callTargets[name] = true + } + callNames = make(map[string]string, len(names)+len(callees[0])) + for name, target := range names { + callNames[name] = target + } + for _, instruction := range original.Instrs { + if instruction.Op == OpCALL && len(instruction.Args) == 1 && instruction.Args[0].Kind == OpSym { + symbol, offset, valid := parseSBRef(instruction.Args[0].Sym) + name := goAMD64FrameCallName(pkg.Path, symbol) + if _, found := callees[0][name]; valid && offset == 0 && found { + callTargets[symbol], callNames[symbol] = true, name + } + } + } + } + if err := goAMD64FrameSourceInstructions(original, callTargets); err != nil { + return nil, err + } + if original.FrameSize > 0 { + shape := fn + if fn.split { + shape, err = goAMD64FrameSplitProjection(fn, objects[observed], original.FrameSize, splitTarget, splitDirectory, splitRoot, data) + if err != nil { + return nil, err + } + fn.prologue = shape.prologue + } + calls, err := goAMD64FrameSourceCallPCs(shape, original, objects[observed], callTargets, callNames) + if err != nil { + return nil, err + } + if err := goAMD64FrameMachineShapeWithCalls(shape, original.FrameSize, calls); err != nil { + return nil, err + } + } else if fn.privateLeaf { + if err := goAMD64FrameMachineShapeWithCalls(fn, 0, nil); err != nil { + return nil, err + } + } else if fn.split { + return nil, goAMD64FrameContext("zero-frame split entry needs its separate native layout") + } + if text, exists := objects[observed]; exists && !bytes.Equal(text.Code, fn.code) { + return nil, goAMD64FrameContext("complete original object/listing TEXT bytes differ") + } + fn.listingSymbol, fn.objectSymbol = observed, symbol + bound[symbol] = fn + } + return bound, nil +} + +func goAMD64FrameSourceInstructions(fn Func, privateLeaves ...map[string]bool) error { + localJump := false + for _, ins := range fn.Instrs { + switch ins.Op { + case OpCALL: + if len(privateLeaves) != 1 || len(ins.Args) != 1 || ins.Args[0].Kind != OpSym { + return goAMD64FrameContext("original CALL needs its complete same-source leaf") + } + name, offset, valid := parseSBRef(ins.Args[0].Sym) + if !valid || offset != 0 || !privateLeaves[0][name] { + return goAMD64FrameContext("original CALL needs its complete same-source leaf") + } + case "BYTE", "WORD", "LONG", "QUAD", "PCALIGN", "PCALIGNMAX": + return goAMD64FrameContext("raw, calling or address-sensitive original frame requires a later source contract") + case OpJMP: + localJump = true + case OpRET: + if len(ins.Args) != 0 { + return goAMD64FrameContext("nonordinary original RET transport") + } + } + } + if localJump { + // Share the complete local-label inventory with the fixed-frame CFG. + // A named branch within this TEXT is not an external tail-call contract. + _, _, err := fixedLocalEdges(fn) + return err + } + return nil +} + +func goAMD64FrameMachineShape(fn goAMD64FrameFunction, frame int64) error { + return goAMD64FrameMachineShapeWithCalls(fn, frame, nil) +} + +func goAMD64FrameMachineShapeWithCalls(fn goAMD64FrameFunction, frame int64, calls map[int]bool) error { + var instructions []x86asm.Inst + starts := make(map[int]bool) + for pc := 0; pc < len(fn.code); { + var ins x86asm.Inst + if x86NumericalVectorPrefix(fn.code[pc:], 64) { + length, err := x86NumericalVectorBoundary(fn.code[pc:], 64) + if err != nil || length <= 0 { + end := pc + 15 + if end > len(fn.code) { + end = len(fn.code) + } + return goAMD64FrameContext("unbound original vector machine instruction at PC %d (%x): %v", pc+fn.prologue, fn.code[pc:end], err) + } + // Exact boundaries only: these cannot impersonate the scalar + // automatic prologue, epilogue, native CALL or return below. + ins.Len = length + } else if _, length, known, err := decodedX86SHAInstructionWithMemory(fn.code[pc:], 64, x86RawMemoryDecoder{physicalPC: true}); known || err != nil { + if err != nil || length <= 0 { + return goAMD64FrameContext("unbound original SHA machine instruction at PC %d: %v", pc+fn.prologue, err) + } + ins.Len = length // No scalar prologue, epilogue or branch authority. + } else if _, length, _, known, err := decodeX86RawADXWithMemory(fn.code[pc:], 0, 64, x86RawMemoryDecoder{physicalPC: true}); known || err != nil { + if err != nil || length <= 0 { + return goAMD64FrameContext("unbound original ADX machine instruction at PC %d: %v", pc+fn.prologue, err) + } + ins.Len = length // The numerical boundary decoder grants no scalar frame role. + } else { + var err error + ins, err = x86asm.Decode(fn.code[pc:], 64) + if err != nil || ins.Len == 0 || ins.Op == 0 || ins.Op == x86asm.LCALL || + ins.Op == x86asm.CALL && (!calls[pc] || ins.Len != 5) { + return goAMD64FrameContext("unknown or calling original machine instruction in %s at PC %d (%s): %v", + fn.sourceSymbol, pc+fn.prologue, fn.positions[pc], err) + } + } + if len(fn.positions[pc]) == 0 { + return goAMD64FrameContext("machine instruction lacks its original source position") + } + // A projected source body must not jump back into the discarded + // automatic stack check or forward into its runtime retry tail. + if relative, ok := ins.Args[0].(x86asm.Rel); ok && fn.split && ins.Op != x86asm.CALL { + destination := pc + ins.Len + int(relative) + if destination < 0 || destination >= len(fn.code) { + return goAMD64FrameContext("source branch enters an automatic split region") + } + } + starts[pc] = true + instructions = append(instructions, ins) + pc += ins.Len + } + for pc := range fn.positions { + if !starts[pc] { + return goAMD64FrameContext("original source position is not a machine instruction boundary") + } + } + if frame == 0 { + returns := 0 + for _, ins := range instructions { + if ins.Op == x86asm.PUSH || ins.Op == x86asm.POP { + return goAMD64FrameContext("private leaf acquired an implicit native BP/SP frame") + } + for _, arg := range ins.Args { + if reg, ok := arg.(x86asm.Reg); ok && (reg == x86asm.RSP || reg == x86asm.ESP || reg == x86asm.SP || reg == x86asm.SPB) { + return goAMD64FrameContext("private leaf machine instruction observes its physical SP") + } + if memory, ok := arg.(x86asm.Mem); ok && (memory.Base == x86asm.RSP || memory.Index == x86asm.RSP) { + return goAMD64FrameContext("private leaf machine memory observes its physical SP") + } + } + if ins.Op == x86asm.RET { + returns++ + } + } + if returns == 0 { + return goAMD64FrameContext("private leaf has no original native return") + } + return goAMD64FrameMachineBranches(instructions, starts) + } + if len(instructions) < 6 { + return goAMD64FrameContext("partial automatic BP/SP frame") + } + first, second, third := instructions[0], instructions[1], instructions[2] + push := first.Op == x86asm.PUSH && first.Args[0] == x86asm.RBP && + second.Op == x86asm.MOV && second.Args[0] == x86asm.RBP && second.Args[1] == x86asm.RSP && + goAMD64FrameStackAdjustment(third, -frame) + slot := x86asm.Mem{Base: x86asm.RSP, Scale: 1, Disp: frame} + stored := goAMD64FrameStackAdjustment(first, -frame-8) && + second.Op == x86asm.MOV && second.Args[0] == slot && second.Args[1] == x86asm.RBP && second.DataSize == 64 && + third.Op == x86asm.LEA && third.Args[0] == x86asm.RBP && third.Args[1] == slot && third.DataSize == 64 + if !push && !stored { + return goAMD64FrameContext("unrecognized automatic BP/SP prologue") + } + // Source control flow may return before the physical end of TEXT, then + // close the final block with a local backedge. No source edge may enter an + // automatic prologue or skip part of a return's automatic stack restoration. + starts[0], starts[first.Len], starts[first.Len+second.Len] = false, false, false + returns := 0 + pc := 0 + for i, ins := range instructions { + if ins.Op == x86asm.RET { + returns++ + if i < 2 { + return goAMD64FrameContext("partial original RET epilogue") + } + penultimate, previous := instructions[i-2], instructions[i-1] + pushReturn := push && previous.Op == x86asm.POP && previous.Args[0] == x86asm.RBP && + goAMD64FrameStackAdjustment(penultimate, frame) + storedReturn := stored && penultimate.Op == x86asm.MOV && penultimate.Args[0] == x86asm.RBP && penultimate.Args[1] == slot && penultimate.DataSize == 64 && + goAMD64FrameStackAdjustment(previous, frame+8) + if !pushReturn && !storedReturn { + return goAMD64FrameContext("unrecognized original RET epilogue") + } + starts[pc], starts[pc-previous.Len] = false, false + } + pc += ins.Len + } + last := instructions[len(instructions)-1] + if returns == 0 || last.Op != x86asm.RET && last.Op != x86asm.JMP { + return goAMD64FrameContext("original frame has no conventional RET") + } + return goAMD64FrameMachineBranches(instructions, starts) +} + +func goAMD64FrameMachineBranches(instructions []x86asm.Inst, starts map[int]bool) error { + pc := 0 + for _, ins := range instructions { + pc += ins.Len + if ins.Op == x86asm.CALL { + continue // Exact source/relocation target, not a local body branch. + } + relative, direct := ins.Args[0].(x86asm.Rel) + if ins.Op == x86asm.JMP && !direct || direct && !starts[pc+int(relative)] { + return goAMD64FrameContext("original branch leaves the source body or enters an automatic frame interior") + } + } + return nil +} + +// Go's ADJSP encoder chooses ADD/SUB and a signed immediate together. At +// 128 bytes the opposite operation with -128 has the shorter encoding. +// Validate the exact 64-bit stack delta, not one spelling of that delta. +func goAMD64FrameStackAdjustment(ins x86asm.Inst, delta int64) bool { + if ins.DataSize != 64 || ins.Args[0] != x86asm.RSP || ins.Args[2] != nil || ins.Args[3] != nil { + return false + } + amount, immediate := ins.Args[1].(x86asm.Imm) + return immediate && (ins.Op == x86asm.ADD && int64(amount) == delta || + ins.Op == x86asm.SUB && int64(amount) == -delta) +} + +func (source *GoAMD64FrameSource) verifyObjectListing(ctx context.Context, object string, wanted map[string]goAMD64FrameFunction) error { + data, err := source.run(ctx, "tool", "objdump", object) + if err != nil { + return err + } + return goAMD64FrameObjectBytes(data, wanted) +} + +func goAMD64FrameObjectBytes(data []byte, wanted map[string]goAMD64FrameFunction) error { + actual := make(map[string][]byte) + addresses := make(map[string]uint64) + name := "" + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + if len(fields) == 3 && fields[0] == "TEXT" && strings.HasSuffix(fields[1], "(SB)") { + name = strings.TrimSuffix(fields[1], "(SB)") + if _, duplicate := actual[name]; duplicate { + return goAMD64FrameContext("duplicate actual object symbol") + } + actual[name] = nil + continue + } + if len(fields) < 4 || name == "" || !strings.HasPrefix(fields[1], "0x") { + return goAMD64FrameContext("unknown actual object disassembly record") + } + value, err := hex.DecodeString(fields[2]) + if err != nil || len(value) == 0 { + return goAMD64FrameContext("invalid actual object bytes") + } + address, err := strconv.ParseUint(strings.TrimPrefix(fields[1], "0x"), 16, 64) + if len(actual[name]) == 0 { + addresses[name] = address + } + if err != nil || address != addresses[name]+uint64(len(actual[name])) { + return goAMD64FrameContext("actual object instruction bytes have a gap/overlap") + } + actual[name] = append(actual[name], value...) + } + if len(actual) != len(wanted) { + return goAMD64FrameContext("object/listing symbol inventories differ") + } + for _, fn := range wanted { + name := fn.objectSymbol + if fn.privateLeaf && fn.nativeABI == 65535 { + // cmd/internal/objfile.goobjName renders the actual static ABI + // bank as version 1. Do not strip arbitrary symbol suffixes. + name += "<1>" + } + if !bytes.Equal(actual[name], fn.code) { + return goAMD64FrameContext("object/listing physical bytes differ for %s", fn.objectSymbol) + } + } + return nil +} diff --git a/amd64_go_frame_source_ownership_test.go b/amd64_go_frame_source_ownership_test.go new file mode 100644 index 000000000..7e538520a --- /dev/null +++ b/amd64_go_frame_source_ownership_test.go @@ -0,0 +1,248 @@ +package plan9asm + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +func TestGoAMD64FrameSourceCloseRejectsReplacedOwnedDirectory(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + temporary := t.TempDir() + t.Setenv("TMPDIR", temporary) + request.Environment = append(request.Environment, "TMPDIR="+temporary, "GOTMPDIR="+temporary) + source := requireGoAMD64FrameCapture(t, pkg, request) + owned := source.owned + original, err := sourcepath.Lstat(owned) + if err != nil { + t.Fatal(err) + } + // Every path here belongs to t.TempDir. The replacement models a different + // owner at the same spelling; Close must not recursively remove that owner. + moved := filepath.Join(t.TempDir(), "original-artifacts") + if err := os.Rename(owned, moved); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(owned, 0700); err != nil { + t.Fatal(err) + } + replacement, err := sourcepath.Lstat(owned) + if err != nil || os.SameFile(original, replacement) { + t.Fatalf("fixture did not replace directory identity: %v", err) + } + canary := filepath.Join(owned, "replacement-owner.txt") + if err := os.WriteFile(canary, []byte("not a factory artifact\n"), 0600); err != nil { + t.Fatal(err) + } + if err := source.Close(); !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("Close did not reject changed directory identity: %v", err) + } + if _, err := os.Stat(canary); err != nil { + t.Errorf("Close removed replacement owner's file: %v", err) + } + if source.ready { + t.Error("failed cleanup retained ready observation") + } + // Restore only our known original directory for the fixture's registered + // cleanup. RemoveAll is limited to this exact t.TempDir-owned replacement. + if err := os.RemoveAll(owned); err != nil { + t.Fatal(err) + } + if err := os.Rename(moved, owned); err != nil { + t.Fatal(err) + } +} + +func TestGoAMD64FrameSourceRequestTemporaryOwnership(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + parentTemporary := t.TempDir() + requestTemporary := t.TempDir() + canonicalTemporary, err := filepath.EvalSymlinks(requestTemporary) + if err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", parentTemporary) + // Like exec.Cmd, the supplied environment's last occurrence wins. Neither + // the inherited parent TMPDIR nor the earlier supplied value is ownership. + request.Environment = append(request.Environment, + "TMPDIR="+parentTemporary, "TMPDIR="+requestTemporary, + "GOTMPDIR="+requestTemporary) + source := requireGoAMD64FrameCapture(t, pkg, request) + if filepath.Dir(source.owned) != canonicalTemporary { + t.Fatalf("factory artifacts are outside request TMPDIR: got parent %q, want %q", filepath.Dir(source.owned), canonicalTemporary) + } + if err := source.Verify(context.Background()); err != nil { + t.Fatal(err) + } + owned := source.owned + if err := source.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(owned); !os.IsNotExist(err) { + t.Fatalf("Close retained factory artifacts: %v", err) + } + for _, name := range []string{parentTemporary, requestTemporary, request.Directory} { + if _, err := os.Stat(name); err != nil { + t.Fatalf("Close removed a caller-owned directory: %v", err) + } + } +} + +func TestGoAMD64FrameSourceRepeatedEnvironmentActualValues(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + for _, key := range []string{"GOPATH", "GOMODCACHE", "GOCACHE", "GOTMPDIR"} { + var expected string + for _, item := range request.Environment { + name, value, _ := strings.Cut(item, "=") + if name == key { + expected = value + } + } + if !filepath.IsAbs(expected) { + t.Fatalf("fixture lacks owned %s", key) + } + request.Environment = append(request.Environment, key+"="+t.TempDir(), key+"="+expected) + } + // Target and routing settings are deliberately overridden, not accepted + // from a different validation view of a repeated key. + request.Environment = append(request.Environment, "GOOS=windows", "GOOS=darwin", "GOARCH=386", "GOARCH=arm64") + source := requireGoAMD64FrameCapture(t, pkg, request) + for _, key := range []string{"GOPATH", "GOMODCACHE", "GOCACHE", "GOTMPDIR"} { + var expected string + for _, item := range source.request.Environment { + name, value, _ := strings.Cut(item, "=") + if name == key { + expected = value + } + } + if source.env[key] != expected { + t.Fatalf("actual %s differs from effective supplied value: %q != %q", key, source.env[key], expected) + } + } + if source.env["GOOS"] != "linux" || source.env["GOARCH"] != "amd64" { + t.Fatalf("repeated target keys changed actual target: %v", source.env) + } +} + +func TestGoAMD64FrameSourceOwnedIdentityRejectsRedirection(t *testing.T) { + for _, kind := range []string{"equal-byte directory", "symlink", "missing"} { + t.Run(kind, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + source := requireGoAMD64FrameCapture(t, pkg, request) + owned := source.owned + moved := filepath.Join(t.TempDir(), "original-artifacts") + if err := os.Rename(owned, moved); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(owned); err != nil { + t.Error(err) + } + if err := os.Rename(moved, owned); err != nil { + t.Error(err) + } + }) + switch kind { + case "equal-byte directory": + if err := os.Mkdir(owned, 0700); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(moved) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + data, err := os.ReadFile(filepath.Join(moved, entry.Name())) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(owned, entry.Name()), data, 0600); err != nil { + t.Fatal(err) + } + } + case "symlink": + if err := os.Symlink(moved, owned); err != nil { + t.Fatal(err) + } + } + if err := source.Verify(context.Background()); !errors.Is(err, ErrProbeNeedsContext) || source.ready { + t.Fatalf("changed owned directory retained observation: %v", err) + } + if err := source.Close(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Close did not reject changed ownership: %v", err) + } + if kind != "missing" { + if _, err := os.Lstat(owned); err != nil { + t.Fatalf("Close removed a replacement directory or link: %v", err) + } + } + if _, err := os.Stat(filepath.Join(moved, request.AssemblyFile+".o")); err != nil { + t.Fatalf("Close searched for or removed moved original artifacts: %v", err) + } + }) + } +} + +func TestGoAMD64FrameSourceCanonicalRootAndExplicitToolPaths(t *testing.T) { + t.Run("resolved package root is fixed", func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + original, err := filepath.EvalSymlinks(request.Directory) + if err != nil { + t.Fatal(err) + } + alias := filepath.Join(t.TempDir(), "package-alias") + if err := os.Symlink(original, alias); err != nil { + t.Fatal(err) + } + request.Directory = alias + source := requireGoAMD64FrameCapture(t, pkg, request) + if source.request.Directory != original { + t.Fatalf("capture did not bind actual canonical root: %q", source.request.Directory) + } + if err := os.Remove(alias); err != nil { + t.Fatal(err) + } + if err := os.Symlink(t.TempDir(), alias); err != nil { + t.Fatal(err) + } + if err := source.Verify(context.Background()); err != nil { + t.Fatalf("unconsumed caller alias changed fixed canonical source: %v", err) + } + }) + t.Run("Go binary symlink is not a source identity", func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + alias := filepath.Join(t.TempDir(), "go") + if err := os.Symlink(request.GoBinary, alias); err != nil { + t.Fatal(err) + } + request.GoBinary = alias + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("indirected driver produced an observation: %v", err) + } + }) +} + +func TestGoAMD64FrameSourceRepeatedForbiddenEnvironmentStaysContext(t *testing.T) { + for _, key := range []string{"GOFLAGS", "GOEXPERIMENT", "GOPACKAGESDRIVER"} { + t.Run(key, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + // Capture conservatively rejects any requested unsupported mapping, + // including one shadowed by a later empty occurrence. + request.Environment = append(request.Environment, key+"=unbound", key+"=") + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unsupported duplicate %s mapping was accepted: %v", key, err) + } + }) + } +} diff --git a/amd64_go_frame_source_positions_test.go b/amd64_go_frame_source_positions_test.go new file mode 100644 index 000000000..3318babee --- /dev/null +++ b/amd64_go_frame_source_positions_test.go @@ -0,0 +1,62 @@ +package plan9asm + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestGoAMD64FrameSDKSourcePCPaths(t *testing.T) { + root := filepath.Join(t.TempDir(), "sdk") + file := filepath.Join(root, "src", "sample", "input.s") + directory := filepath.Dir(file) + for _, mutation := range []string{ + "original", "absolute", "ordinary source", "different root", "missing input", + "wrong line", "wrong listing", "dot segment", "parent segment", "double slash", + "foreign file", "invalid origin", + } { + t.Run(mutation, func(t *testing.T) { + text := gostaticstorage.TextRecord{Name: "sample.Value", Positions: []gostaticstorage.TextPosition{ + {Start: 0, End: 1, File: "$GOROOT/src/sample/input.s", Line: 2}, + }} + fn := goAMD64FrameFunction{code: []byte{0xc3}, sourcePositions: map[int][]goAMD64FrameSourcePosition{ + 0: {{file: file, line: 2}}, + }} + origins := map[string]int{file: 3} + capturedRoot := root + switch mutation { + case "absolute": + text.Positions[0].File = file + case "ordinary source": + capturedRoot = "" + case "different root": + capturedRoot = filepath.Join(root, "other") + case "missing input": + delete(origins, file) + case "wrong line": + text.Positions[0].Line = 4 + case "wrong listing": + fn.sourcePositions[0][0].line = 3 + case "dot segment": + text.Positions[0].File = "$GOROOT/src/sample/./input.s" + case "parent segment": + text.Positions[0].File = "$GOROOT/src/sample/../sample/input.s" + case "double slash": + text.Positions[0].File = "$GOROOT/src//sample/input.s" + case "foreign file": + text.Positions[0].File = "$GOROOT/src/sample/foreign.s" + case "invalid origin": + text.Positions[0].File = "$GOROOT/src/sample/./input.s" + origins[text.Positions[0].File] = 3 + fn.sourcePositions[0][0].file = text.Positions[0].File + } + err := goAMD64FrameSourcePCs(text, fn, directory, origins, capturedRoot) + wantPass := mutation == "original" || mutation == "absolute" + if wantPass && err != nil || !wantPass && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("captured original object/listing source paths: %v", err) + } + }) + } +} diff --git a/amd64_go_frame_source_test.go b/amd64_go_frame_source_test.go new file mode 100644 index 000000000..2a83e3916 --- /dev/null +++ b/amd64_go_frame_source_test.go @@ -0,0 +1,540 @@ +package plan9asm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +func requireGoAMD64FrameCapture(t *testing.T, pkg GoPackage, request GoAMD64FrameCaptureRequest) *GoAMD64FrameSource { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + return source +} + +func goAMD64FrameSourceFixture(t *testing.T) (GoPackage, GoAMD64FrameCaptureRequest) { + t.Helper() + // Keep AST positions, the caller's request and strict original-file reads + // bound to the same physical fixture, including aliased temporary roots. + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal("cannot resolve original frame fixture directory", err) + } + source := []byte("package framefixture\nfunc Value(a uint64) uint64\n") + files := map[string][]byte{ + "go.mod": []byte("module example.invalid/framefixture\ngo 1.20\n"), + "value.go": source, + "value_amd64.s": []byte("#include \"textflag.h\"\nTEXT ·Value(SB),NOSPLIT,$160-16\nMOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nMOVQ 16(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n"), + } + for name, data := range files { + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + t.Fatal(err) + } + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, filepath.Join(dir, "value.go"), source, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + pkg, err := (&types.Config{Sizes: sizes}).Check("example.invalid/framefixture", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + binary, err := exec.LookPath("go") + if err != nil { + t.Fatal(err) + } + binary, err = filepath.Abs(binary) + if err != nil { + t.Fatal(err) + } + binary, err = filepath.EvalSymlinks(binary) + if err != nil { + t.Fatal(err) + } + // This independent pure-Go module does not inherit the translator test + // binary's build tags (notably Windows' LLVM byollvm tag). Remove rather + // than shadow them: production capture rejects repeated forbidden flags. + var env []string + for _, entry := range os.Environ() { + if !strings.HasPrefix(entry, "GOFLAGS=") { + env = append(env, entry) + } + } + env = append(env, "GOFLAGS=") + for _, key := range []string{"GOPATH", "GOMODCACHE", "GOCACHE", "TMPDIR", "GOTMPDIR"} { + if !filepath.IsAbs(os.Getenv(key)) { + env = append(env, key+"="+t.TempDir()) + } + } + return GoPackage{Path: pkg.Path(), Types: pkg, Syntax: []*ast.File{syntax}}, GoAMD64FrameCaptureRequest{ + GoBinary: binary, Directory: dir, AssemblyFile: "value_amd64.s", Target: "linux/amd64", Environment: env, + } +} + +func TestGoAMD64FrameSourceActualCaptureAndSealedLifecycle(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + // Independent original-Go acceptance precedes the proposed token. This + // object observation is not itself a native-memory authorization. + raw, err := os.ReadFile(filepath.Join(request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + listing := x86InsertExtractGoListing(t, "amd64", strings.Replace(string(raw), "#include \"textflag.h\"", "#define NOSPLIT 4", 1)) + t.Logf("actual stock-Go original nonzero TEXT listing:\n%s", listing) + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + if len(source.functions) != 1 || source.functions["example.invalid/framefixture.Value"].locals != 168 { + t.Fatalf("missing actual Go frame witness: %+v", source.functions) + } + encoded, err := json.Marshal(source) + if err != nil { + t.Fatal(err) + } + var restored GoAMD64FrameSource + if err := json.Unmarshal(encoded, &restored); err != nil { + t.Fatal(err) + } + if !errors.Is(restored.Verify(ctx), ErrProbeNeedsContext) { + t.Fatal("JSON recreated private frame authority") + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + if !errors.Is(source.Verify(ctx), ErrProbeNeedsContext) { + t.Fatal("closed witness retained authority") + } +} + +func TestGoAMD64FrameSourceRefusesSelfReportedSelection(t *testing.T) { + for _, mutation := range []string{"wrong types", "missing sibling", "dependency", "generated header", "source changed", "same width signedness", "target", "custom flags", "conditional CPP", "raw RET", "zero frame", "pointer", "unknown tool"} { + t.Run(mutation, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + switch mutation { + case "wrong types": + pkg.Types = types.NewPackage(pkg.Path, "forged") + case "missing sibling": + if err := os.WriteFile(filepath.Join(request.Directory, "sibling_amd64.s"), []byte("TEXT ·Other(SB),4,$8-0\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + request.AssemblyFile = "sibling_amd64.s" + case "dependency": + if err := os.WriteFile(filepath.Join(request.Directory, "dep.go"), []byte("package framefixture\nimport _ \"unsafe\"\n"), 0600); err != nil { + t.Fatal(err) + } + case "generated header": + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte("#include \"go_asm.h\"\nTEXT ·Value(SB),4,$160-16\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + case "source changed": + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\nfunc Value(a uint32) uint32\n"), 0600); err != nil { + t.Fatal(err) + } + case "target": + request.Target = "linux/386" + case "same width signedness": + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\nfunc Value(a int64) int64\n"), 0600); err != nil { + t.Fatal(err) + } + case "custom flags": + request.Environment = append(request.Environment, "GOFLAGS=-tags=custom") + case "conditional CPP": + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte("#ifdef GOARCH_amd64\n#include \"go_asm.h\"\nTEXT ·Value(SB),4,$160-16\nRET\n#endif\n"), 0600); err != nil { + t.Fatal(err) + } + case "raw RET": + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte("TEXT ·Value(SB),4,$160-16\nBYTE $0xc3\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + case "zero frame": + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte("TEXT ·Value(SB),4,$0-16\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + case "pointer": + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\nfunc Value(a *uint64) *uint64\n"), 0600); err != nil { + t.Fatal(err) + } + case "unknown tool": + request.GoBinary = filepath.Join(request.Directory, "not-a-go-driver") + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + proof, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if proof != nil { + proof.Close() + t.Fatal("unsupported/incomplete source produced token") + } + if err == nil { + t.Fatal("self-reported typed selection was accepted") + } + }) + } +} + +func TestGoAMD64FrameSourceActualTargetAndFullSiblingInventory(t *testing.T) { + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + t.Run(target, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = target + data, err := os.ReadFile(filepath.Join(request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(request.Directory, "sibling_amd64.s"), bytes.ReplaceAll(data, []byte("Value"), []byte("Other")), 0600); err != nil { + t.Fatal(err) + } + declarations := []byte("package framefixture\nfunc Value(a uint64) uint64\nfunc Other(a uint64) uint64\n") + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), declarations, 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "value.go", declarations, 0) + if err != nil { + t.Fatal(err) + } + pkg.Types, err = (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check(pkg.Path, fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if target == "windows/amd64" && errors.Is(err, ErrProbeNeedsContext) && strings.Contains(err.Error(), "SSEHUNWINDINFO") { + if source != nil { + t.Fatal("unsupported generated unwind role returned a token") + } + t.Logf("actual Windows auxiliary object role remains unbound: %v", err) + return + } + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + if len(source.selection.SFiles) != 2 || len(source.functions) != 1 || source.env["GOOS"]+"/"+source.env["GOARCH"] != target { + t.Fatalf("incomplete actual target/sibling observation: %+v", source.selection) + } + if _, observed := source.functions[pkg.Path+".Other"]; observed { + t.Fatal("unrequested sibling acquired frame facts") + } + }) + } +} + +func TestGoAMD64FrameSourceInputMutationPermanentlyRevokes(t *testing.T) { + for _, kind := range []string{"Go", "ASM", "unused header", "include", "inventory", "object", "listing", "cancel"} { + t.Run(kind, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + header := filepath.Join(request.Directory, "fixture.h") + if err := os.WriteFile(header, []byte("#define FRAME 160\n"), 0600); err != nil { + t.Fatal(err) + } + if kind == "include" { + assembly := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(assembly) + if err != nil { + t.Fatal(err) + } + original = append([]byte("#include \"fixture.h\"\n"), bytes.Replace(original, []byte("$160-16"), []byte("$FRAME-16"), 1)...) + if err := os.WriteFile(assembly, original, 0600); err != nil { + t.Fatal(err) + } + } + source := requireGoAMD64FrameCapture(t, pkg, request) + if kind == "include" && source.origins[filepath.Join(source.request.Directory, "fixture.h")] == 0 { + t.Fatal("active original CPP header was not captured") + } + filename := filepath.Join(source.request.Directory, "value.go") + switch kind { + case "ASM": + filename = filepath.Join(source.request.Directory, request.AssemblyFile) + case "unused header", "include": + filename = filepath.Join(source.request.Directory, "fixture.h") + case "inventory": + filename = filepath.Join(source.request.Directory, "new.h") + case "object": + filename = filepath.Join(source.owned, request.AssemblyFile+".o") + case "listing": + filename = filepath.Join(source.owned, request.AssemblyFile+".o.listing") + } + ctx := context.Background() + if kind == "cancel" { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } else if err := os.WriteFile(filename, []byte("changed\n"), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) || source.ready { + t.Fatalf("changed input retained observation: %v", err) + } + if !errors.Is(source.Verify(context.Background()), ErrProbeNeedsContext) { + t.Fatal("revoked observation was reopened") + } + }) + } +} + +func TestGoAMD64FrameSourceObjectListingMutations(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + source := requireGoAMD64FrameCapture(t, pkg, request) + listing, err := os.ReadFile(filepath.Join(source.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(source.request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + expanded, err := source.preprocess(request.AssemblyFile, raw) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + fn := source.functions[pkg.Path+".Value"] + var pcs []int + for pc := range fn.positions { + if pc > 0 { + pcs = append(pcs, pc) + } + } + sort.Ints(pcs) + for _, change := range []string{"source path", "source line", "interior PC", "partial bytes", "frame", "symbol", "duplicate", "relocation"} { + t.Run(change, func(t *testing.T) { + mutated := append([]byte(nil), listing...) + switch change { + case "source path": + mutated = bytes.ReplaceAll(mutated, []byte(request.AssemblyFile), []byte("unbound.s")) + case "source line": + mutated = bytes.ReplaceAll(mutated, []byte(".s:2)"), []byte(".s:999999)")) + case "interior PC": + before := fmt.Sprintf("0x%04x %05d", pcs[0], pcs[0]) + after := fmt.Sprintf("0x%04x %05d", pcs[0]+1, pcs[0]+1) + mutated = bytes.Replace(mutated, []byte(before), []byte(after), 1) + case "partial bytes": + mutated = mutated[:bytes.LastIndex(mutated, []byte("\n\t0x"))+1] + case "frame": + mutated = bytes.Replace(mutated, []byte("locals=0xa8"), []byte("locals=0xa0"), 1) + case "symbol": + mutated = bytes.ReplaceAll(mutated, []byte(fn.listingSymbol), []byte("forged.Value")) + case "duplicate": + mutated = append(mutated, listing...) + case "relocation": + mutated = append(mutated, []byte("rel 0+4 t=R_CALL forged\n")...) + } + if _, err := goAMD64FrameListing(mutated, file, pkg, source.origins); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mutated %s listing accepted: %v", change, err) + } + }) + } + for _, locals := range []string{"fffffffffffffff8", "80000000", "ffffffff", "10000000000000000"} { + t.Run("foreign-locals-"+locals, func(t *testing.T) { + if bytes.Count(listing, []byte("locals=0xa8")) != 1 { + t.Fatal("actual positive-frame precondition changed") + } + mutated := bytes.Replace(listing, []byte("locals=0xa8"), []byte("locals=0x"+locals), 1) + if _, err := goAMD64FrameListing(mutated, file, pkg, source.origins); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("negative, noncanonical or oversized metadata replaced the actual positive frame", locals, err) + } + }) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + object := filepath.Join(source.owned, request.AssemblyFile+".o") + disassembly, err := source.run(ctx, "tool", "objdump", object) + if err != nil { + t.Fatal(err) + } + if err := goAMD64FrameObjectBytes(disassembly, source.functions); err != nil { + t.Fatal(err) + } + for _, change := range []string{"bytes", "symbol", "missing", "gap", "extra"} { + t.Run("object "+change, func(t *testing.T) { + mutated := append([]byte(nil), disassembly...) + switch change { + case "bytes": + rows := strings.Split(string(mutated), "\n") + fields := strings.Fields(rows[1]) + rows[1] = strings.Replace(rows[1], fields[2], "90", 1) + mutated = []byte(strings.Join(rows, "\n")) + case "symbol": + mutated = bytes.ReplaceAll(mutated, []byte(fn.objectSymbol), []byte("forged.Value")) + case "missing": + mutated = mutated[:bytes.LastIndex(mutated, []byte("\n "))+1] + case "gap": + rows := strings.Split(string(mutated), "\n") + fields := strings.Fields(rows[2]) + rows[2] = strings.Replace(rows[2], fields[1], "0x0", 1) + mutated = []byte(strings.Join(rows, "\n")) + case "extra": + mutated = append(mutated, disassembly...) + } + if err := goAMD64FrameObjectBytes(mutated, source.functions); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mutated %s object accepted: %v", change, err) + } + }) + } +} + +func TestGoAMD64FrameSourceConcurrentVerifyClose(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + source := requireGoAMD64FrameCapture(t, pkg, request) + owned := source.owned + var workers sync.WaitGroup + for i := 0; i < 3; i++ { + workers.Add(1) + go func() { + defer workers.Done() + if err := source.Verify(context.Background()); err != nil && !errors.Is(err, ErrProbeNeedsContext) { + t.Error(err) + } + }() + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + workers.Wait() + if _, err := os.Stat(owned); !os.IsNotExist(err) { + t.Fatalf("factory-owned artifacts survived Close: %v", err) + } + if _, err := os.Stat(filepath.Join(request.Directory, request.AssemblyFile)); err != nil { + t.Fatalf("Close removed original source: %v", err) + } +} + +func TestGoAMD64FrameSourceFailedSiblingCleansOnlyOwnedObjects(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + temporary := t.TempDir() + t.Setenv("TMPDIR", t.TempDir()) + request.Environment = append(request.Environment, "TMPDIR="+temporary, "GOTMPDIR="+temporary) + // The selected sibling cannot be bound to any actual declaration. Request + // it explicitly: its failure must not leak the captured object directory. + sibling := filepath.Join(request.Directory, "z_sibling_amd64.s") + if err := os.WriteFile(sibling, []byte("TEXT ·Other(SB),4,$160-0\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + request.AssemblyFile = filepath.Base(sibling) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("partial complete-file observation accepted: %v", err) + } + entries, err := os.ReadDir(temporary) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "plan9asm-go-frame-object-") { + t.Fatalf("partial capture leaked %s", entry.Name()) + } + } + if _, err := os.Stat(sibling); err != nil { + t.Fatalf("cleanup removed an original selected source: %v", err) + } +} + +func TestGoAMD64FrameSourceRejectsIndirectedOriginalAndCacheFallback(t *testing.T) { + for _, change := range []string{"source symlink", "package subtree", "implicit cache"} { + t.Run(change, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + switch change { + case "source symlink": + target := filepath.Join(t.TempDir(), "source.go") + if err := os.WriteFile(target, []byte("package framefixture\nfunc Value(a uint64) uint64\n"), 0600); err != nil { + t.Fatal(err) + } + original := filepath.Join(request.Directory, "value.go") + if err := os.Remove(original); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, original); err != nil { + t.Fatal(err) + } + case "package subtree": + // Ordinary child directories have an independent positive + // owner. An indirected child still cannot become an input. + if err := os.Symlink(t.TempDir(), filepath.Join(request.Directory, "nested")); err != nil { + t.Fatal(err) + } + case "implicit cache": + request.Environment = nil + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unsupported original source/cache origin captured: %v", err) + } + }) + } +} + +func TestGoAMD64FrameSourceRejectsSelectedSystemObjects(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + // A genuine Go object renamed .syso is still selected by the actual driver + // and packed into its export. It must not be silently treated as an ASM/H + // source role merely because the directory and file bytes were hashed. + env := append(append([]string(nil), request.Environment...), "GOOS=linux", "GOARCH=amd64", "GOTOOLCHAIN=local") + object := filepath.Join(request.Directory, "system_amd64.syso") + include := filepath.Join(filepath.Dir(filepath.Dir(request.GoBinary)), "pkg", "include") + if _, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, "tool", "asm", "-p", pkg.Path, "-I", include, "-o", object, request.AssemblyFile); err != nil { + t.Fatal(err) + } + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil { + if closeErr := source.Close(); closeErr != nil { + t.Fatal(closeErr) + } + t.Fatal("actual selected .syso role was omitted from complete observation") + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown system-object transport did not retain Context: %v", err) + } +} diff --git a/amd64_go_frame_split.go b/amd64_go_frame_split.go new file mode 100644 index 000000000..a6ceab1a2 --- /dev/null +++ b/amd64_go_frame_split.go @@ -0,0 +1,227 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strconv" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "golang.org/x/arch/x86/x86asm" +) + +func goAMD64FrameSplitListing(listing []byte) bool { + for len(listing) != 0 { + var line []byte + line, listing, _ = bytes.Cut(listing, []byte{'\n'}) + if len(line) == 0 || line[0] == '\t' || line[0] == ' ' { + continue + } + if header, valid := parseGoAMD64NativeHeader(string(line)); valid && !header.NoSplit { + return true + } + } + return false +} + +func goAMD64FrameListingWithSplitObject(listing []byte, file *File, pkg GoPackage, origins map[string]int, + texts []gostaticstorage.TextRecord, unwind *gostaticstorage.WindowsAMD64AssemblyUnwind, target, directory, root string) (map[string]goAMD64FrameFunction, error) { + if len(texts) == 0 || target != "linux/amd64" && target != "darwin/amd64" && target != "windows/amd64" { + return nil, goAMD64FrameContext("automatic split layout lacks its complete object/target") + } + return goAMD64FrameListingRecords(listing, file, pkg, origins, texts, unwind, nil, target, directory, root) +} + +// Listing identity is necessary but not sufficient: the complete relocation +// inventory must subsequently match the exact generated instruction fields. +func goAMD64FrameSplitRelocationLine(r gostaticstorage.TextRelocation, target string) string { + if r.Size != 4 || r.Add != 0 { + return "" + } + kind, name := "", r.Target.Name + switch { + case r.Type == 15 && target != "windows/amd64" && r.Target == (gostaticstorage.TextReference{}): + kind, name = "R_TLS_LE", "TLS" + case r.Type == 14 && target == "windows/amd64" && name == "runtime.tls_g" && r.Target.ABIKnown && r.Target.ABI == 0: + kind = "R_PCREL" + case r.Type == 7 && name == "runtime.morestack_noctxt" && r.Target.ABIKnown && r.Target.ABI == 0: + kind = "R_CALL" + default: + return "" + } + return fmt.Sprintf("rel %d+4 t=%s %s+0", r.Offset, kind, name) +} + +// Go's ABI0 stack check runs before any source instruction and retries the +// entry after growth. LLVM owns a different function frame: only the ordinary +// body layout is projected for BP/SP validation, never a runtime call. Original +// bytes, positions and relocations remain in the live complete source owner. +// After either entry path, private bytes are SP+[0,frame), saved BP is at frame, +// return PC at frame+8 and FP arguments at frame+16. Generated R12/R14/FLAGS +// values are not supplied to the source body's existing typed input proof. +func goAMD64FrameSplitProjection(fn goAMD64FrameFunction, text gostaticstorage.TextRecord, frame int64, + target, directory, root string, data *gostaticstorage.AMD64AssemblyData) (goAMD64FrameFunction, error) { + phase := "entry identity" + fail := func() (goAMD64FrameFunction, error) { + return goAMD64FrameFunction{}, goAMD64FrameContext("unrecognized original automatic split layout/source/relocation: %s", phase) + } + if !fn.split || text.ABI != 0 || text.Flags != fn.nativeFlags() || !bytes.Equal(fn.code, text.Code) || + frame <= 0 || frame > 4096-8 { + return fail() + } + phase = "TEXT source position" + entry := goAMD64FrameSourcePosition{} + for _, row := range strings.Split(fn.positions[0], "\n") { + position, instruction, ok := strings.Cut(row, "\t") + if !ok || !strings.HasPrefix(instruction, "TEXT\t") { + continue + } + separator := strings.LastIndexByte(position, ':') + if separator < 0 || entry.file != "" { + return fail() + } + entry.file = position[:separator] + entry.line, _ = strconv.Atoi(position[separator+1:]) + } + if entry.file == "" || entry.line <= 0 { + return fail() + } + entryPath := goAMD64FrameSourcePath(entry.file, directory, root) + if entryPath == "" { + return fail() + } + pc := 0 + read := func(op x86asm.Op, args ...x86asm.Arg) bool { + if pc >= len(fn.code) { + return false + } + ins, err := x86asm.Decode(fn.code[pc:], 64) + if err != nil || ins.Len <= 0 || ins.Op != op { + return false + } + if ins.Op == x86asm.LEA && ins.AddrSize == 64 { + // x/arch retains disp32 as unsigned, unlike disp8. Share the + // decoder's physical-field normalization before exact RSP matching. + ins = normalizeDecodedX86MemoryDisplacements(ins) + } + for index, arg := range args { + if ins.Args[index] != arg { + return false + } + } + pc += ins.Len + return true + } + // The exact ABI0 TLS materialization differs by OS. None of these reads + // is source-visible or grants the body permission to read TLS or entry R14. + phase = "generated TLS load" + tlsOffset := int64(5) + segment := x86asm.FS + if target == "darwin/amd64" { + segment = x86asm.GS + } + if target == "windows/amd64" { + tlsOffset = 3 + if !read(x86asm.MOV, x86asm.R14, x86asm.Mem{Base: x86asm.RIP}) || + !read(x86asm.MOV, x86asm.R14, x86asm.Mem{Segment: x86asm.GS, Base: x86asm.R14}) || + !read(x86asm.MOV, x86asm.R14, x86asm.Mem{Base: x86asm.R14}) { + return fail() + } + } else if target != "linux/amd64" && target != "darwin/amd64" || + !read(x86asm.MOV, x86asm.R14, x86asm.Mem{Segment: segment, Scale: 1}) { + return fail() + } + phase = "generated stack comparison" + compare := x86asm.RSP + if frame+8 > 128 { + if !read(x86asm.LEA, x86asm.R12, x86asm.Mem{Base: x86asm.RSP, Scale: 1, Disp: 128 - frame - 8}) { + return fail() + } + compare = x86asm.R12 + } + if !read(x86asm.CMP, compare, x86asm.Mem{Base: x86asm.R14, Disp: 16}) { + return fail() + } + phase = "generated slow-path branch" + if pc >= len(fn.code) { + return fail() + } + branch, err := x86asm.Decode(fn.code[pc:], 64) + jump, ok := branch.Args[0].(x86asm.Rel) + if err != nil || branch.Op != x86asm.JBE || !ok { + return fail() + } + pc += branch.Len + start, end := pc, pc+int(jump) + if end <= start || end+5 >= len(fn.code) || fn.code[end] != 0xe8 || !bytes.Equal(fn.code[end+1:end+5], make([]byte, 4)) { + return fail() + } + phase = "generated runtime retry" + pc = end + 5 + retry, err := x86asm.Decode(fn.code[pc:], 64) + relative, ok := retry.Args[0].(x86asm.Rel) + if err != nil || retry.Op != x86asm.JMP || !ok || pc+retry.Len != len(fn.code) || pc+retry.Len+int(relative) != 0 { + return fail() + } + // Generated fields take precedence over ordinary DATA references, including + // Windows' ABI0 R_PCREL runtime.tls_g. Every other field must belong wholly + // to the body and retain the same complete DATA object's identity rules. + phase = "generated/body relocation partition" + generated := 0 + var previousRelocationEnd int64 + for _, relocation := range text.Relocations { + if relocation.Size != 4 || relocation.Offset < previousRelocationEnd || relocation.Offset > int64(len(fn.code))-relocation.Size { + return fail() + } + previousRelocationEnd = relocation.Offset + relocation.Size + switch relocation.Offset { + case tlsOffset: + if relocation.Type == 7 || goAMD64FrameSplitRelocationLine(relocation, target) == "" { + return fail() + } + generated |= 1 + case int64(end + 1): + if relocation.Type != 7 || goAMD64FrameSplitRelocationLine(relocation, target) == "" { + return fail() + } + generated |= 2 + default: + if data == nil || relocation.Offset < int64(start) || relocation.Offset > int64(end)-relocation.Size || + relocation.Type != 14 && !goAMD64FrameLocalCallTarget(relocation, data.Member.Texts) || + goAMD64FrameDataRelocationLine(relocation, data) == "" { + return fail() + } + } + } + if generated != 3 { + return fail() + } + // Actual object PC intervals, not just printed instruction names, bind + // every generated byte to the original TEXT directive's source position. + phase = "generated source-PC intervals" + var previous int64 + for _, span := range text.Positions { + if span.Start != previous || span.End <= span.Start || span.End > int64(len(fn.code)) { + return fail() + } + previous = span.End + if span.Start < int64(start) || span.End > int64(end) { + if goAMD64FrameSourcePath(span.File, directory, root) != entryPath || int(span.Line) != entry.line { + return fail() + } + } + } + if previous != int64(len(fn.code)) { + return fail() + } + projected := fn + projected.code = fn.code[start:end] + projected.prologue = start + projected.positions = make(map[int]string) + for position, rows := range fn.positions { + if position >= start && position < end { + projected.positions[position-start] = rows + } + } + return projected, nil +} diff --git a/amd64_go_frame_split_data_test.go b/amd64_go_frame_split_data_test.go new file mode 100644 index 000000000..5a149e305 --- /dev/null +++ b/amd64_go_frame_split_data_test.go @@ -0,0 +1,261 @@ +//go:build go1.27 + +package plan9asm + +import ( + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "golang.org/x/arch/x86/x86asm" +) + +func fixedLocalSplitDataFixture(t *testing.T, target, triple string, frame int) (GoPackage, GoAMD64FrameCaptureRequest, Options, string) { + t.Helper() + pkg, request, _, options, original := fixedLocalDataFixture(t, + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", fixedLocalLiteralData) + assembly := strings.Replace(original, "TEXT ·Value(SB),4,$160-40", fmt.Sprintf("TEXT ·Value(SB),$%d-40", frame), 1) + if assembly == original { + t.Fatal("fixture did not remove the source NOSPLIT flag") + } + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + request.Target, options.TargetTriple = target, triple + return pkg, request, options, assembly +} + +func TestGoAMD64FrameAutomaticSplitWithDataObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + for _, frame := range []int{160, 536} { + t.Run(fmt.Sprintf("frame%d", frame), func(t *testing.T) { + pkg, request, options, assembly := fixedLocalSplitDataFixture(t, target.name, target.triple, frame) + x86LEAActualTargetGoObject(t, "amd64", target.triple, assembly) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "internal constant [32 x i8]") || + !strings.Contains(ir, "fixed_local_native") || strings.Contains(ir, "morestack") { + t.Fatal("complete DATA/split source lost its ordinary LLVM frame or private literal", err) + } + compileLLVMToObject(t, llc, target.triple, "split-data.ll", "split-data.o", ir) + compileLLVMToObject(t, llc, target.triple, "split-data-opt.ll", "split-data-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + }) + } +} + +func TestGoAMD64FrameAutomaticSplitWithSDKDataObservation(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "crypto/internal/fips140/sha256") + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal(err) + } + defer observed.Close() + pkg := sdkAMD64FrameActualPackage(t, observed.source) + source, err := CaptureGoAMD64SDKFrameSource(ctx, pkg, GoAMD64FrameCaptureRequest{ + GoBinary: request.GoBinary, Directory: request.Directory, AssemblyFile: "sha256block_amd64.s", + Target: request.Target, Environment: request.Environment, + }) + if err != nil { + for ordinal, object := range observed.source.observation.Objects.Sources { + for _, text := range object.Original.Texts { + if text.Name != request.PackagePath+".blockAVX2" || len(text.Code) < 64 || len(text.Positions) == 0 { + continue + } + t.Logf("original prefix=%x tail=%x relocations=%+v firstPC=%+v lastPC=%+v", text.Code[:64], + text.Code[len(text.Code)-16:], text.Relocations, text.Positions[0], text.Positions[len(text.Positions)-1]) + rows := strings.SplitN(string(observed.source.observation.Frames[ordinal].OriginalListing), "\n", 4) + if len(rows) > 3 { + rows = rows[:3] + } + t.Log(strings.Join(rows, "\n")) + } + } + t.Fatal("complete original SDK split/DATA observation", err) + } + defer source.Close() + fn, found := source.functions[request.PackagePath+".blockAVX2"] + if !found || !fn.split || fn.prologue <= 0 || len(source.functions) != 2 { + t.Fatal("complete SDK source lost its split frame or ordinary sibling") + } + expanded := source.sdkFrame.owner.source.observation.CPP[0].Output + data, err := source.fixedLocalDataObservation(source.request.AssemblyFile, expanded) + if err != nil { + t.Fatal(err) + } + var text gostaticstorage.TextRecord + for _, candidate := range data.Member.Texts { + if candidate.Name == request.PackagePath+".blockAVX2" { + text = candidate + } + } + if len(text.Positions) == 0 || !strings.HasPrefix(text.Positions[0].File, "$GOROOT/") { + t.Fatal("actual SDK object did not exercise captured-root source metadata") + } + tls, err := x86asm.Decode(text.Code, 64) + if err != nil { + t.Fatal(err) + } + lea, err := x86asm.Decode(text.Code[tls.Len:], 64) + if err != nil || lea.Op != x86asm.LEA || lea.Len != 8 || lea.AddrSize != 64 { + t.Fatal("actual SDK prologue lacks the original disp32 LEA", err) + } + for _, displacement := range []int32{-1 << 31, -417, -415, -129, -128, -1, 0, 127, 128, 1<<31 - 1} { + t.Run(fmt.Sprintf("wrong displacement %d", displacement), func(t *testing.T) { + changed := text + changed.Code = append([]byte(nil), text.Code...) + binary.LittleEndian.PutUint32(changed.Code[tls.Len+lea.Len-4:], uint32(displacement)) + changedFn := fn + changedFn.code = changed.Code + if _, err := goAMD64FrameSplitProjection(changedFn, changed, 536, request.Target, request.Directory, source.env["GOROOT"], data); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("wrong physical stack-check displacement was accepted", err) + } + }) + } + for _, mutation := range []string{"empty root", "foreign root", "unclean metadata", "generated line"} { + t.Run(mutation, func(t *testing.T) { + changed, root := text, source.env["GOROOT"] + changed.Positions = append([]gostaticstorage.TextPosition(nil), text.Positions...) + switch mutation { + case "empty root": + root = "" + case "foreign root": + root = t.TempDir() + case "unclean metadata": + changed.Positions[0].File = strings.Replace(changed.Positions[0].File, "$GOROOT/", "$GOROOT/../", 1) + case "generated line": + changed.Positions[0].Line++ + } + if _, err := goAMD64FrameSplitProjection(fn, changed, 536, request.Target, request.Directory, root, data); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("generated SDK region borrowed an unbound source position", err) + } + }) + } + if err := source.Verify(ctx); err != nil { + t.Fatal("live complete SDK split/DATA observation", err) + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + t.Log("complete original SHA256 source observed; this is not a whole-file translation or execution result") +} + +func TestGoAMD64FrameAutomaticSplitDataRelocationPartition(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + // Both private and ordinary references belong to this original body. + // Capturing Global's address gives it no fixed-local read permission. + assembly := fixedLocalLiteralData + `TEXT ·Value(SB),$160-16 +MOVQ a+0(FP),AX +MOVQ AX,16(SP) +MOVQ $·Global(SB),BX +MOVQ literal<>(SB),CX +MOVQ 16(SP),AX +MOVQ AX,ret+8(FP) +RET +` + pkg, request, _, _ := fixedLocalTransportFixture(t, assembly, + "var Global uint64\nfunc Value(a uint64) uint64\n", target.name, target.triple) + source := requireGoAMD64FrameCapture(t, pkg, request) + expanded, err := source.preprocess(request.AssemblyFile, []byte(assembly)) + if err != nil { + t.Fatal(err) + } + data, err := source.fixedLocalDataObservation(request.AssemblyFile, expanded) + if err != nil || len(data.Member.Texts) != 1 { + t.Fatal("missing complete original object", err) + } + text, fn := data.Member.Texts[0], source.functions[pkg.Path+".Value"] + if len(text.Relocations) != 4 || text.Relocations[1].Target.Name != pkg.Path+".Global" || text.Relocations[2].Target.ABI != 65535 { + t.Fatal("actual Go object lacks both generated and body relocation roles") + } + shape, err := goAMD64FrameSplitProjection(fn, text, 160, request.Target, request.Directory, "", data) + if err != nil { + t.Fatal("complete unmodified generated/body relocation partition", err) + } + encoded, err := json.Marshal(text) + if err != nil { + t.Fatal(err) + } + for _, mutation := range []string{ + "missing TLS", "missing morestack", "duplicate TLS", "duplicate morestack", "duplicate body", "reordered body", + "DATA in prefix", "DATA in tail", "DATA crosses body end", "body call", "private index", "private ABI", + "TLS ordinary fallback", "morestack ordinary fallback", "TLS field moved", "morestack field moved", + } { + t.Run(mutation, func(t *testing.T) { + var changed gostaticstorage.TextRecord + if err := json.Unmarshal(encoded, &changed); err != nil { + t.Fatal(err) + } + relocations := changed.Relocations + switch mutation { + case "missing TLS": + relocations = relocations[1:] + case "missing morestack": + relocations = relocations[:3] + case "duplicate TLS": + relocations = append([]gostaticstorage.TextRelocation{relocations[0]}, relocations...) + case "duplicate morestack": + relocations = append(relocations, relocations[3]) + case "duplicate body": + relocations[2] = relocations[1] + case "reordered body": + relocations[1], relocations[2] = relocations[2], relocations[1] + case "DATA in prefix": + relocations[1].Offset = int64(shape.prologue - 4) + case "DATA in tail": + relocations[2].Offset = relocations[3].Offset + 4 + case "DATA crosses body end": + relocations[2].Offset = int64(shape.prologue + len(shape.code) - 2) + case "body call": + relocations[1].Type, relocations[1].Target = relocations[3].Type, relocations[3].Target + if goAMD64FrameDataRelocationLine(relocations[1], data) == "" || + goAMD64FrameLocalCallTarget(relocations[1], data.Member.Texts) { + t.Fatal("ordinary external CALL observation did not reach the independent body-role guard") + } + case "private index": + relocations[2].Target.SymbolIndex++ + case "private ABI": + relocations[2].Target.ABI = 0 + case "TLS ordinary fallback": + relocations[0].Type, relocations[0].Target = relocations[1].Type, relocations[1].Target + if goAMD64FrameDataRelocationLine(relocations[0], data) == "" { + t.Fatal("mutation did not exercise ordinary R_PCREL fallback") + } + case "morestack ordinary fallback": + relocations[3].Type, relocations[3].Target = relocations[1].Type, relocations[1].Target + case "TLS field moved": + relocations[0].Offset++ + case "morestack field moved": + relocations[3].Offset++ + } + changed.Relocations = relocations + if _, err := goAMD64FrameSplitProjection(fn, changed, 160, request.Target, request.Directory, "", data); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed generated/body relocation roles were accepted", err) + } + }) + } + }) + } +} diff --git a/amd64_go_frame_split_test.go b/amd64_go_frame_split_test.go new file mode 100644 index 000000000..39df47b2a --- /dev/null +++ b/amd64_go_frame_split_test.go @@ -0,0 +1,232 @@ +//go:build go1.27 + +package plan9asm + +import ( + "encoding/json" + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestGoAMD64FrameAutomaticSplitLayout(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + // Go's ADJSP encoder flips ADD/SUB at the signed-byte boundary. + // Exercise both sides, with and without generated stack checks. + for _, frame := range []int{64, 120, 128, 160, 256} { + for _, nosplit := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/frame%d/nosplit%t", target.name, frame, nosplit), func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = target.name + flags := "" + if nosplit { + flags = "4," + } + assembly := fmt.Sprintf("TEXT ·Value(SB),%s$%d-16\nMOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nMOVQ 16(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n", flags, frame) + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + // The original assembler decides whether a source-level split + // flag actually generates a stack check (small leaves do not). + listing := x86InsertExtractGoListing(t, "amd64", assembly) + wantSplit := !nosplit && frame+8 >= 128 + if strings.Contains(string(listing), "CALL\truntime.morestack_noctxt") != wantSplit { + t.Fatal("independent Go prologue did not match the frame/flag case", listing) + } + name := pkg.Path + ".Value" + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup("Value").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: target.triple, + Sigs: map[string]FuncSig{name: sig}, ResolveSym: func(string) string { return name }} + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate"}, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "%fixed_local_native = alloca") || strings.Contains(ir, "morestack") { + t.Fatal("ordinary source frame did not use the LLVM function frame", err) + } + compileLLVMToObject(t, llc, options.TargetTriple, "split.ll", "split.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "split-opt.ll", "split-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + } + } +} + +func TestGoAMD64FrameAutomaticSplitRejectsChangedObservation(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + pkg, request := goAMD64FrameSourceFixture(t) + name := filepath.Join(request.Directory, request.AssemblyFile) + raw, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + raw = []byte(strings.Replace(string(raw), ",NOSPLIT,", ",", 1)) + if err := os.WriteFile(name, raw, 0600); err != nil { + t.Fatal(err) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + expanded, err := source.preprocess(request.AssemblyFile, raw) + if err != nil { + t.Fatal(err) + } + object, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o")) + if err != nil { + t.Fatal(err) + } + observed, err := source.frameDataObject(expanded, object) + if err != nil || len(observed.Member.Texts) != 1 { + t.Fatal("missing complete original split TEXT", err) + } + fn := source.functions[pkg.Path+".Value"] + text := observed.Member.Texts[0] + encoded, err := json.Marshal(text) + if err != nil { + t.Fatal(err) + } + projection, err := goAMD64FrameSplitProjection(fn, text, 160, request.Target, request.Directory, "", nil) + if err != nil || projection.prologue <= 0 || len(projection.code) >= len(fn.code) { + t.Fatal("missing generated layout projection", err) + } + listing, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + for _, mismatch := range []string{"FIPS header", "FIPS object", "missing kind"} { + t.Run(mismatch, func(t *testing.T) { + changed, header := text, listing + switch mismatch { + case "FIPS header": + header = []byte(strings.Replace(string(listing), " STEXT ", " STEXTFIPS ", 1)) + case "FIPS object": + changed.Kind = 2 + case "missing kind": + changed.Kind = 0 + } + _, err := goAMD64FrameListingWithSplitObject(header, file, pkg, source.origins, + []gostaticstorage.TextRecord{changed}, nil, request.Target, request.Directory, "") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("listing silently changed the complete object TEXT kind", err) + } + }) + } + for _, mutation := range []string{ + "TLS offset", "TLS type", "TLS target", "call offset", "call target", "call ABI", "extra relocation", + "prefix bytes", "branch destination", "retry destination", "generated source line", "empty positions", "position gap", "object bytes", "body branch to entry", "NOSPLIT flag", "extra flag", + } { + t.Run(mutation, func(t *testing.T) { + var changed gostaticstorage.TextRecord + if err := json.Unmarshal(encoded, &changed); err != nil { + t.Fatal(err) + } + switch mutation { + case "TLS offset": + changed.Relocations[0].Offset++ + case "TLS type": + changed.Relocations[0].Type = 16 + case "TLS target": + changed.Relocations[0].Target.Name = "runtime.tls_g" + case "call offset": + changed.Relocations[1].Offset++ + case "call target": + changed.Relocations[1].Target.Name = "runtime.morestack" + case "call ABI": + changed.Relocations[1].Target.ABI = 1 + case "extra relocation": + changed.Relocations = append(changed.Relocations, changed.Relocations[1]) + case "prefix bytes": + changed.Code[0] = 0x65 + case "branch destination": + changed.Code[projection.prologue-1]++ + case "retry destination": + changed.Code[len(changed.Code)-1]++ + case "generated source line": + changed.Positions[0].Line++ + case "empty positions": + changed.Positions = nil + case "position gap": + changed.Positions[0].Start++ + case "object bytes": + changed.Code = changed.Code[:len(changed.Code)-1] + case "NOSPLIT flag": + changed.Flags = 16 + case "extra flag": + changed.Flags = 1 + case "body branch to entry": + pc := int(changed.Positions[1].Start) + changed.Code[pc], changed.Code[pc+1] = 0x74, byte(-pc-2) + } + changedFn := fn + if mutation != "object bytes" { + changedFn.code = changed.Code + } + shape, err := goAMD64FrameSplitProjection(changedFn, changed, 160, request.Target, request.Directory, "", nil) + if err == nil { + err = goAMD64FrameMachineShape(shape, 160) + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed complete split observation accepted", err) + } + }) + } +} + +func TestGoAMD64FrameAutomaticSplitDoesNotAuthorizeSourceRuntime(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + for _, instruction := range []string{"CALL runtime·morestack_noctxt(SB)", "MOVQ (TLS),R14", "MOVQ 168(SP),AX"} { + t.Run(instruction, func(t *testing.T) { + pkg, request, _, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + name := filepath.Join(request.Directory, request.AssemblyFile) + raw, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + assembly := strings.Replace(string(raw), "TEXT ·Value(SB),4,$160-40", "TEXT ·Value(SB),$160-40\n"+instruction, 1) + if err := os.WriteFile(name, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + x86InsertExtractGoListing(t, "amd64", assembly) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil { + defer func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }() + expanded, readErr := source.preprocess(request.AssemblyFile, []byte(assembly)) + if readErr != nil { + t.Fatal(readErr) + } + file, parseErr := Parse(ArchAMD64, expanded) + if parseErr != nil { + t.Fatal(parseErr) + } + _, err = source.BindFixedLocal(ctx, pkg, file, options) + } + if !errors.Is(err, ErrProbeNeedsContext) || strings.Contains(err.Error(), "producer") { + t.Fatal("automatic split check authorized a source runtime or return-slot effect", err) + } + }) + } +} diff --git a/amd64_go_frame_stack_adjust_test.go b/amd64_go_frame_stack_adjust_test.go new file mode 100644 index 000000000..48c0ce853 --- /dev/null +++ b/amd64_go_frame_stack_adjust_test.go @@ -0,0 +1,62 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestGoAMD64FrameMachineStackAdjustment(t *testing.T) { + for _, frame := range []int64{64, 120, 128, 136, 256} { + t.Run(fmt.Sprint(frame), func(t *testing.T) { + assembly := fmt.Sprintf("TEXT value(SB),4,$%d-0\nNOP\nRET\n", frame) + listing := x86InsertExtractGoListing(t, "amd64", assembly) + fn := goAMD64FrameFunction{positions: make(map[int]string)} + for _, line := range strings.Split(string(listing), "\n") { + fn.code = append(fn.code, x86InsertExtractListedBytes(line)...) + } + var starts []int + for pc := 0; pc < len(fn.code); { + ins, err := x86asm.Decode(fn.code[pc:], 64) + if err != nil || ins.Len <= 0 { + t.Fatal("independent Go object cannot be decoded", err) + } + starts = append(starts, pc) + // This unit test exercises machine shape only. Full capture + // tests independently bind real listing/source positions. + fn.positions[pc] = "machine-shape unit fixture" + pc += ins.Len + } + if len(starts) != 6 { + t.Fatalf("unexpected original Go prologue/epilogue: %x", fn.code) + } + if err := goAMD64FrameMachineShape(fn, frame); err != nil { + t.Fatal("original Go stack adjustment", err) + } + for _, index := range []int{2, 3} { + for _, mutation := range []string{"width", "register", "operation", "amount"} { + t.Run(fmt.Sprintf("instruction%d/%s", index, mutation), func(t *testing.T) { + changed := fn + changed.code = append([]byte(nil), fn.code...) + pc, end := starts[index], starts[index+1] + switch mutation { + case "width": + changed.code[pc] = 0x40 // REX without W: ESP, not RSP. + case "register": + changed.code[pc+2] &^= 7 // Same operation on AX. + case "operation": + changed.code[pc+2] ^= 0x28 // Exchange ADD and SUB. + case "amount": + changed.code[end-1] ^= 1 + } + if err := goAMD64FrameMachineShape(changed, frame); err == nil { + t.Fatal("altered automatic frame adjustment accepted") + } + }) + } + } + }) + } +} diff --git a/amd64_go_frame_standard_imports.go b/amd64_go_frame_standard_imports.go new file mode 100644 index 000000000..c72ed24cb --- /dev/null +++ b/amd64_go_frame_standard_imports.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "bytes" + "context" + "encoding/json" + "io" + "path/filepath" + "reflect" + + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +// SDK imports supply actual compiler types, never caller-memory permissions. +// Exact downloaded modules have their own independent archive handoff; +// nonordinary compiler roles remain separate. +func (source *GoAMD64FrameSource) loadOriginalDeclarationImports(ctx context.Context) (map[string]goAMD64FrameSelection, error) { + if len(source.selection.Imports) == 0 { + return nil, nil + } + output, err := source.run(ctx, "list", "-deps", "-compiled", "-export", "-json", "-mod=readonly", ".") + if err != nil { + return nil, err + } + if err := source.captureDownloadedDeclarationOrigins(ctx, output); err != nil { + return nil, err + } + return source.bindOriginalDeclarationImports(output) +} + +// A caller retaining the actual command bytes may share the same complete +// graph decoder. This routine observes files/exports, not frame authority. +func (source *GoAMD64FrameSource) bindOriginalDeclarationImports(output []byte) (map[string]goAMD64FrameSelection, error) { + if len(output) > 16<<20 { + return nil, goAMD64FrameContext("declaration import graph exceeds its bound") + } + root, err := sourcepath.Resolve(source.env["GOROOT"]) + if err != nil { + return nil, err + } + cache, err := sourcepath.Resolve(source.env["GOCACHE"]) + if err != nil { + return nil, err + } + graph := make(map[string]goAMD64FrameSelection) + decoder := json.NewDecoder(bytes.NewReader(output)) + var originalFound bool + var inputBytes, inputFiles int + for { + var selected goAMD64FrameSelection + if err := decoder.Decode(&selected); err == io.EOF { + break + } else if err != nil { + return nil, err + } + if selected.ImportPath == "" || len(graph) >= 4096 || + len(selected.Error)+len(selected.DepsErrors) != 0 || selected.ForTest != "" || selected.nonordinaryRole() { + return nil, goAMD64FrameContext("incomplete or nonordinary declaration import graph") + } + if selected.ImportPath == source.selection.ImportPath { + selected.Export, err = sourcepath.Resolve(selected.Export) + if err != nil || originalFound || !reflect.DeepEqual(selected, source.selection) { + return nil, goAMD64FrameContext("declaration graph lost the actual original package") + } + originalFound = true + continue + } + if _, duplicate := graph[selected.ImportPath]; duplicate || + !filepath.IsLocal(filepath.FromSlash(selected.ImportPath)) { + return nil, goAMD64FrameContext("foreign or duplicate declaration dependency needs authenticated handoff") + } + dir, err := sourcepath.Resolve(selected.Dir) + var wanted string + var wantedErr error + if selected.Standard && selected.Module == nil { + wanted, wantedErr = sourcepath.Resolve(filepath.Join(root, "src", filepath.FromSlash(selected.ImportPath))) + } else { + wanted, wantedErr = source.downloadedDeclarationDirectory(selected) + } + if err != nil || wantedErr != nil || dir != wanted { + return nil, goAMD64FrameContext("actual declaration dependency escaped its original source origin: %v", wantedErr) + } + compiled := append([]string(nil), selected.CompiledGoFiles...) + for index, file := range compiled { + if filepath.IsAbs(file) { + relative, err := filepath.Rel(dir, file) + if err != nil || filepath.Base(relative) != relative { + return nil, goAMD64FrameContext("generated dependency compiler sources are unbound") + } + compiled[index] = relative + } + } + if selected.ImportPath == "unsafe" { + compiled = selected.GoFiles // This builtin package has no compiled export. + } + if len(selected.GoFiles) == 0 || !reflect.DeepEqual(compiled, selected.GoFiles) { + return nil, goAMD64FrameContext("declaration dependency omitted ordinary compiler sources") + } + for _, files := range [][]string{selected.GoFiles, selected.SFiles, selected.HFiles} { + for _, file := range files { + if filepath.Base(file) != file { + return nil, goAMD64FrameContext("declaration dependency source is not a package-local file") + } + data, err := source.read(filepath.Join(dir, file)) + if err != nil { + return nil, err + } + inputBytes += len(data) + inputFiles++ + if inputBytes > 64<<20 || inputFiles > 32768 { + return nil, goAMD64FrameContext("complete declaration dependency inputs exceed their bound") + } + } + } + if selected.ImportPath != "unsafe" { + selected.Export, err = sourcepath.Resolve(selected.Export) + if err != nil || !goAssemblyPathWithinRoot(cache, selected.Export) { + return nil, goAMD64FrameContext("declaration compiler export is outside its actual cache") + } + data, err := source.read(selected.Export) + if err != nil || len(data) == 0 { + return nil, goAMD64FrameContext("declaration compiler export is empty or changed: %v", err) + } + } + graph[selected.ImportPath] = selected + } + if !originalFound { + return nil, goAMD64FrameContext("declaration graph omitted its original package") + } + for _, selected := range append([]goAMD64FrameSelection{source.selection}, frameImportGraphValues(graph)...) { + for _, path := range selected.Imports { + if _, present := graph[path]; !present { + return nil, goAMD64FrameContext("declaration graph omitted an actual import") + } + } + for _, target := range selected.ImportMap { + if _, present := graph[target]; !present { + return nil, goAMD64FrameContext("declaration graph omitted an actual import mapping") + } + } + } + return graph, nil +} + +func frameImportGraphValues(graph map[string]goAMD64FrameSelection) []goAMD64FrameSelection { + values := make([]goAMD64FrameSelection, 0, len(graph)) + for _, selected := range graph { + values = append(values, selected) + } + return values +} diff --git a/amd64_go_frame_text_metadata_test.go b/amd64_go_frame_text_metadata_test.go new file mode 100644 index 000000000..d89f14103 --- /dev/null +++ b/amd64_go_frame_text_metadata_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "go/types" + "strings" + "testing" +) + +// Original Go object metadata and typed declarations are independent inputs. +func TestGoAMD64FrameTEXTArgumentMetadata(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + for _, row := range []struct { + name, declaration, body string + args []string + }{ + {"void", "func Value()", "MOVQ $7,AX\nMOVQ AX,0(SP)\n", []string{"", "-0"}}, + {"input-result", "func Value(a uint64) uint64", "MOVQ a+0(FP),AX\nMOVQ AX,0(SP)\nMOVQ 0(SP),AX\nMOVQ AX,ret+8(FP)\n", []string{"", "-16"}}, + {"result-only", "func Value() uint64", "MOVQ $7,AX\nMOVQ AX,0(SP)\nMOVQ 0(SP),AX\nMOVQ AX,ret+0(FP)\n", []string{"", "-8"}}, + {"narrow-padded", "func Value(a uint8) uint32", "MOVBLZX a+0(FP),AX\nMOVQ AX,0(SP)\nMOVQ 0(SP),AX\nMOVL AX,ret+8(FP)\n", []string{"", "-12", "-16"}}, + } { + for _, args := range row.args { + t.Run(target.name+"/"+row.name+"/24"+args, func(t *testing.T) { + assembly := "TEXT ·Value(SB),4,$24" + args + "\n" + row.body + "RET\n" + pkg, request := goAMD64FrameTypedFixture(t, row.declaration, assembly) + request.Target = target.name + listing := frameRelocationOriginal(t, pkg, request) + known := false + for _, line := range strings.Split(string(listing), "\n") { + if header, valid := parseGoAMD64NativeHeader(line); valid { + known = true + if header.ArgsKnown != (args != "") { + t.Fatal("original Go lost omitted/explicit argument metadata", header) + } + } + } + if !known { + t.Fatal("original Go TEXT header missing") + } + if row.name == "void" && args == "" && target.name != "windows/amd64" { + t.Run("known-header-cannot-replace-omission", func(t *testing.T) { + changed := bytes.Replace(listing, []byte("args=0xffffffff80000000"), []byte("args=0x0"), 1) + if bytes.Equal(changed, listing) { + t.Fatal("actual Go omitted-argument header was not changed") + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + _, err = goAMD64FrameListing(changed, file, pkg, + map[string]int{request.AssemblyFile: strings.Count(assembly, "\n") + 1}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a changed known-zero header replaced actual unknown metadata", err) + } + }) + } + name := pkg.Path + ".Value" + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup("Value").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: target.triple, + Sigs: map[string]FuncSig{name: sig}, ResolveSym: func(string) string { return name }} + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("actual typed declaration and original TEXT metadata failed", err) + } + compileLLVMToObject(t, llc, target.triple, "text-args.ll", "text-args.o", ir) + compileLLVMToObject(t, llc, target.triple, "text-args-o2.ll", "text-args-o2.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + } + } +} + +func TestGoAMD64FrameTEXTExplicitArgumentMismatch(t *testing.T) { + for _, args := range []int{0, 8, 15, 17, 24} { + t.Run(fmt.Sprint(args), func(t *testing.T) { + assembly := fmt.Sprintf("TEXT ·Value(SB),4,$24-%d\nMOVQ a+0(FP),AX\nMOVQ AX,ret+8(FP)\nRET\n", args) + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a uint64) uint64", assembly) + frameRelocationOriginal(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if owner != nil { + if closeErr := owner.Close(); closeErr != nil { + t.Error(closeErr) + } + } + if !errors.Is(err, ErrProbeNeedsContext) || owner != nil { + t.Fatal("explicit incorrect TEXT acquired a typed frame extent", err) + } + }) + } +} + +func TestGoAMD64FrameDeclaredNoFrameMetadata(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + for _, args := range []string{"", "-0"} { + t.Run(target+"/Idle0"+args, func(t *testing.T) { + assembly := "TEXT ·Idle(SB),$0" + args + "\nRET\n" + + "TEXT ·Value(SB),4,$24-16\nMOVQ a+0(FP),AX\nMOVQ AX,0(SP)\nMOVQ 0(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + pkg, request := goAMD64FrameTypedFixture(t, "func Idle()\nfunc Value(a uint64) uint64", assembly) + request.Target = target + frameRelocationOriginal(t, pkg, request) + owner := requireGoAMD64FrameCapture(t, pkg, request) + idle, found := owner.functions[pkg.Path+".Idle"] + if !found || idle.locals != 0 || idle.privateLeaf || idle.split || len(idle.signature.Frame.Params)+len(idle.signature.Frame.Results) != 0 { + t.Fatal("ordinary declared no-frame entry acquired a private or incoming-frame role") + } + if value := owner.functions[pkg.Path+".Value"]; value.locals != 32 { + t.Fatal("framed companion lost its exact independent physical frame") + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for _, name := range []string{"Idle", "Value"} { + sig, err := DeriveGoFuncABI0Signature(pkg.Path+"."+name, + pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[sig.Name] = sig + } + triple := map[string]string{ + "linux/amd64": "x86_64-unknown-linux-gnu", + "darwin/amd64": "x86_64-apple-darwin", + "windows/amd64": "x86_64-pc-windows-msvc", + }[target] + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := owner.BindFixedLocal(ctx, pkg, file, Options{ + Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return pkg.Path + "." + strings.TrimPrefix(symbol, "·") }, + }) + if err != nil { + t.Fatal("ordinary no-frame companion did not retain complete source binding", err) + } + if bound.Sigs[pkg.Path+".Idle"].x86FixedLocal != nil || bound.Sigs[pkg.Path+".Value"].x86FixedLocal == nil { + t.Fatal("ordinary no-frame entry borrowed its companion's local backing") + } + }) + } + for _, args := range []string{"", "-16"} { + t.Run(target+"/FPLeaf0"+args, func(t *testing.T) { + assembly := "TEXT ·Leaf(SB),$0" + args + "\nMOVQ a+0(FP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + + "TEXT ·Value(SB),4,$24-16\nMOVQ a+0(FP),AX\nMOVQ AX,0(SP)\nMOVQ 0(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + pkg, request := goAMD64FrameTypedFixture(t, "func Leaf(a uint64) uint64\nfunc Value(a uint64) uint64", assembly) + request.Target = target + frameRelocationOriginal(t, pkg, request) + sigs := make(map[string]FuncSig) + for _, name := range []string{"Leaf", "Value"} { + sig, err := DeriveGoFuncABI0Signature(pkg.Path+"."+name, pkg.Types.Scope().Lookup(name).(*types.Func), + "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[sig.Name] = sig + } + triple := map[string]string{ + "linux/amd64": "x86_64-unknown-linux-gnu", "darwin/amd64": "x86_64-apple-darwin", + "windows/amd64": "x86_64-pc-windows-msvc", + }[target] + fixedLocalFreshAPIs(t, pkg, request, Options{ + Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return pkg.Path + "." + strings.TrimPrefix(symbol, "·") }, + }, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("ordinary frameless FP input/result lost its typed declaration", err) + } + compileLLVMToObject(t, llc, triple, "leaf.ll", "leaf.o", ir) + compileLLVMToObject(t, llc, triple, "leaf-o2.ll", "leaf-o2.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + } +} diff --git a/amd64_go_frame_typed_source_test.go b/amd64_go_frame_typed_source_test.go new file mode 100644 index 000000000..5509ff271 --- /dev/null +++ b/amd64_go_frame_typed_source_test.go @@ -0,0 +1,490 @@ +package plan9asm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func goAMD64FrameTypedFixture(t *testing.T, declaration, assembly string) (GoPackage, GoAMD64FrameCaptureRequest) { + t.Helper() + _, request := goAMD64FrameSourceFixture(t) + source := []byte("package framefixture\n" + declaration + "\n") + filename := filepath.Join(request.Directory, "value.go") + if err := os.WriteFile(filename, source, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, filename, source, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check("example.invalid/framefixture", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{file}}, request +} + +func TestGoAMD64FrameTypedSourceSliceObservation(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a []byte, n uint64) uint64", "#include \"textflag.h\"\nTEXT ·Value(SB),NOSPLIT,$160-40\nMOVQ n+24(FP),AX\nMOVQ AX,16(SP)\nMOVQ 16(SP),AX\nMOVQ AX,ret+32(FP)\nRET\n") + source := requireGoAMD64FrameCapture(t, pkg, request) + fn := source.functions[pkg.Path+".Value"] + if fn.args != 40 || fn.locals != 168 || len(fn.signature.Frame.Params) != 4 || fn.signature.Frame.Results[0].Offset != 32 { + t.Fatalf("actual slice observation lost complete original ABI0 frame: %+v", fn) + } + if fn.transport != goAMD64FrameCompositeTransport || fn.languageType == "" { + t.Fatal("slice observation lost its nonnumeric transport role") + } + for i, slot := range fn.signature.Frame.Params[:3] { + if slot.Offset != int64(i*8) || slot.Index != 0 || slot.Field != i { + t.Fatalf("slice field %d has an invented offset/identity: %+v", i, slot) + } + } + if !errors.Is(source.numericDeclarations(pkg), ErrProbeNeedsContext) { + t.Fatal("slice observation entered the numeric-only fixed-local contract") + } +} + +func TestGoAMD64FrameTypedSourceActualLayoutMatrix(t *testing.T) { + for _, test := range []struct { + name, declaration string + args int64 + params, results []int64 + role goAMD64FrameTransport + }{ + {"integer widths", "func Value(a int8, b uint8, c int16, d uint16, e int32, f uint32, g int64, h uint64, i int, j uint, k uintptr) (r uint64)", 64, []int64{0, 1, 2, 4, 8, 12, 16, 24, 32, 40, 48}, []int64{56}, goAMD64FrameNumericTransport}, + {"named integer", "type Count uint64\nfunc Value(a Count) Count", 16, []int64{0}, []int64{8}, goAMD64FrameNumericTransport}, + {"integer alias", "type Count = uint64\nfunc Value(a Count) Count", 16, []int64{0}, []int64{8}, goAMD64FrameNumericTransport}, + {"pointer", "func Value(a *uint64, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"pointer to array", "func Value(a *[64]byte, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"nested fields", "type Packet struct { Tag byte; Samples [3]int16; Tail uint64 }\nfunc Value(a *Packet, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"nested pointer fields", "type Packet struct { P *[4]uint16; S []uint32 }\nfunc Value(a *Packet, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"named pointer", "type Pointer *uint64\nfunc Value(a Pointer, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"pointer alias", "type Pointer = *uint64\nfunc Value(a Pointer, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"slice integer", "func Value(a []int32, n uint64) uint64", 40, []int64{0, 8, 16, 24}, []int64{32}, goAMD64FrameCompositeTransport}, + {"slice pointer", "func Value(a []*uint64, n uint64) uint64", 40, []int64{0, 8, 16, 24}, []int64{32}, goAMD64FrameCompositeTransport}, + {"named slice", "type Bytes []byte\nfunc Value(a Bytes, n uint64) uint64", 40, []int64{0, 8, 16, 24}, []int64{32}, goAMD64FrameCompositeTransport}, + {"slice alias", "type Bytes = []byte\nfunc Value(a Bytes, n uint64) uint64", 40, []int64{0, 8, 16, 24}, []int64{32}, goAMD64FrameCompositeTransport}, + {"pointer result", "func Value(a *uint64, n uint64) *uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"slice result", "func Value(a []byte, n uint64) []byte", 56, []int64{0, 8, 16, 24}, []int64{32, 40, 48}, goAMD64FrameCompositeTransport}, + {"mixed results", "func Value(a *uint64, n int32) (p *uint64, count uint32)", 32, []int64{0, 8}, []int64{16, 24}, goAMD64FrameCompositeTransport}, + } { + t.Run(test.name, func(t *testing.T) { + // Object capture does not execute the body or claim initialized + // results/GC safety. The independent ABI0 offsets are the oracle. + assembly := fmt.Sprintf("TEXT ·Value(SB),4,$160-%d\nRET\n", test.args) + pkg, request := goAMD64FrameTypedFixture(t, test.declaration, assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + fn := source.functions[pkg.Path+".Value"] + if fn.args != test.args || fn.locals != 168 || fn.transport != test.role || fn.languageType == "" { + t.Fatalf("actual object/transport mismatch: %+v", fn) + } + check := func(slots []FrameSlot, offsets []int64) { + t.Helper() + if len(slots) != len(offsets) { + t.Fatalf("complete ABI0 slots differ: %+v; want %v", slots, offsets) + } + for i, slot := range slots { + if slot.Offset != offsets[i] { + t.Fatalf("ABI0 slot %d is %+v; want offset %d", i, slot, offsets[i]) + } + } + } + check(fn.signature.Frame.Params, test.params) + check(fn.signature.Frame.Results, test.results) + object := filepath.Join(source.owned, request.AssemblyFile+".o") + t.Logf("actual stock-Go observation: target=%s object_sha=%x listing_sha=%x code_bytes=%d args=%d locals=%d transport=%d", request.Target, source.inputs[object], source.inputs[object+".listing"], len(fn.code), fn.args, fn.locals, fn.transport) + err := source.numericDeclarations(pkg) + if test.role == goAMD64FrameNumericTransport && err != nil || test.role != goAMD64FrameNumericTransport && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("numeric-only consumer role mismatch: %v", err) + } + }) + } +} + +func goAMD64FrameTypedSelection(t *testing.T, pkg GoPackage, request GoAMD64FrameCaptureRequest) *GoAMD64FrameSource { + t.Helper() + // The production capture entry resolves the source root before binding + // selection paths. This internal fixture must exercise that same contract, + // including temporary-directory aliases supplied by the host OS. + canonical, err := filepath.EvalSymlinks(request.Directory) + if err != nil { + t.Fatal(err) + } + request.Directory = canonical + return &GoAMD64FrameSource{ + request: request, inputs: make(map[string][32]byte), + selection: goAMD64FrameSelection{GoFiles: []string{"value.go"}, CompiledGoFiles: []string{filepath.Join(request.Directory, "value.go")}}, + } +} + +func TestGoAMD64FrameTypedSelectionCanonicalSourceRoot(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a *uint64) uint64", "TEXT ·Value(SB),4,$160-16\nRET\n") + canonical, err := filepath.EvalSymlinks(request.Directory) + if err != nil { + t.Fatal(err) + } + source := goAMD64FrameTypedSelection(t, pkg, request) + if source.request.Directory != canonical || len(source.selection.CompiledGoFiles) != 1 || + source.selection.CompiledGoFiles[0] != filepath.Join(canonical, "value.go") { + t.Fatal("typed fixture did not bind the production entry's canonical source root") + } + if _, err := source.typedDeclarations(pkg, false); err != nil { + t.Fatalf("canonical source must reach the independent language check: %v", err) + } +} + +func TestGoAMD64FrameTypedSourceLanguageIdentityMutations(t *testing.T) { + for _, test := range []struct{ name, original, mutation string }{ + {"same pointer width", "func Value(a *uint64) uint64", "func Value(a *int64) uint64"}, + {"slice element", "func Value(a []byte) uint64", "func Value(a []uint16) uint64"}, + {"slice versus pointer", "func Value(a []byte) uint64", "func Value(a *byte) uint64"}, + {"array length", "func Value(a *[16]byte) uint64", "func Value(a *[32]byte) uint64"}, + {"named underlying", "type Count uint64\nfunc Value(a *Count) uint64", "type Count int64\nfunc Value(a *Count) uint64"}, + {"alias versus named", "type Count = uint64\nfunc Value(a *Count) uint64", "type Count uint64\nfunc Value(a *Count) uint64"}, + {"alias RHS identity", "type A uint64\ntype B uint64\ntype Choice = A\nfunc Value(a *Choice) uint64", "type A uint64\ntype B uint64\ntype Choice = B\nfunc Value(a *Choice) uint64"}, + {"field signedness", "type Packet struct { X uint64 }\nfunc Value(a *Packet) uint64", "type Packet struct { X int64 }\nfunc Value(a *Packet) uint64"}, + {"field identity", "type Packet struct { X uint64 }\nfunc Value(a *Packet) uint64", "type Packet struct { Y uint64 }\nfunc Value(a *Packet) uint64"}, + {"field order", "type Packet struct { X byte; Y uint64 }\nfunc Value(a *Packet) uint64", "type Packet struct { Y uint64; X byte }\nfunc Value(a *Packet) uint64"}, + {"field tag", "type Packet struct { X uint64 `a` }\nfunc Value(a *Packet) uint64", "type Packet struct { X uint64 `b` }\nfunc Value(a *Packet) uint64"}, + {"embedded identity", "type Count uint64\ntype Packet struct { Count }\nfunc Value(a *Packet) uint64", "type Count uint64\ntype Packet struct { Count Count }\nfunc Value(a *Packet) uint64"}, + {"parameter name", "func Value(a *uint64) uint64", "func Value(b *uint64) uint64"}, + {"result pointee", "func Value(a *uint64) *uint64", "func Value(a *uint64) *int64"}, + { + "unrelated interface does not hide signedness", + "type XOF interface { Clone() XOF }\nfunc Value(a *uint64) uint64", + "type XOF interface { Clone() XOF }\nfunc Value(a *int64) uint64", + }, + { + "unrelated recursive type does not hide extent", + "type Node struct { Next *Node }\nfunc Value(a *[16]byte) uint64", + "type Node struct { Next *Node }\nfunc Value(a *[32]byte) uint64", + }, + { + "unrelated callback does not hide alias RHS", + "type Callback func()\ntype A uint64\ntype B uint64\ntype Choice = A\nfunc Value(a *Choice) uint64", + "type Callback func()\ntype A uint64\ntype B uint64\ntype Choice = B\nfunc Value(a *Choice) uint64", + }, + } { + t.Run(test.name, func(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, test.original, "TEXT ·Value(SB),4,$160-16\nRET\n") + guard := goAMD64FrameTypedSelection(t, pkg, request) + if _, err := guard.typedDeclarations(pkg, false); err != nil { + t.Fatalf("unmutated independent typecheck failed: %v", err) + } + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\n"+test.mutation+"\n"), 0600); err != nil { + t.Fatal(err) + } + guard.inputs = make(map[string][32]byte) + if _, err := guard.typedDeclarations(pkg, false); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("stale supplied type tree accepted actual source mutation: %v", err) + } + }) + } +} + +func TestGoAMD64FrameTypedSourceUnsupportedRolesRemainContext(t *testing.T) { + for _, declaration := range []string{ + "func Value(a float32) uint64", "func Value(a complex128) uint64", + "func Value(a string) uint64", "func Value(a *float64) uint64", + "func Value(a map[uint64]uint64) uint64", "func Value(a chan uint64) uint64", + "func Value(a func(uint64) uint64) uint64", "func Value(a interface{}) uint64", + "func Value(a [2]uint64) uint64", "func Value(a struct { X uint64 }) uint64", + "type Node struct { Next *Node }\nfunc Value(a *Node) uint64", + "type Node[T any] struct { Value T }\nfunc Value(a *Node[uint64]) uint64", + "func Value[T ~uint64](a T) uint64 { return uint64(a) }", "func Value(a ...uint64) uint64", + "func Value(a *uint64) uint64 { return *a }", + } { + t.Run(declaration, func(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, declaration, "TEXT ·Value(SB),4,$160-16\nRET\n") + guard := goAMD64FrameTypedSelection(t, pkg, request) + _, err := guard.typedDeclarations(pkg, false) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown/GC/aggregate role did not retain Context: %v", err) + } + if strings.Contains(err.Error(), "input path is redirected or inaccessible") { + t.Fatalf("path rejection cannot prove an unsupported language role: %v", err) + } + }) + } +} + +func TestGoAMD64FrameTypedSourceOrdinaryPointerValueIsObservationOnly(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, + "var P *uint64\nfunc Value(a *uint64) uint64", + "TEXT ·Value(SB),4,$160-16\nRET\n") + source := requireGoAMD64FrameCapture(t, pkg, request) + if len(source.functions) != 1 || len(source.declarations) != 1 { + t.Fatal("ordinary pointer value created an additional frame capability") + } + function := source.functions[pkg.Path+".Value"] + if function.transport != goAMD64FrameCompositeTransport || function.args != 16 { + t.Fatalf("original pointer declaration lost its separate observation role: %+v", function) + } + if !errors.Is(source.numericDeclarations(pkg), ErrProbeNeedsContext) { + t.Fatal("ordinary pointer value or declaration entered numeric-only fixed-local admission") + } +} + +func TestGoAMD64FrameTypedSourceActualCaptureRejectsStalePointee(t *testing.T) { + for _, test := range []struct{ original, mutation string }{ + {"func Value(a *uint64) uint64", "func Value(a *int64) uint64"}, + {"type Packet struct { X [4]byte }\nfunc Value(a *Packet) uint64", "type Packet struct { X [8]byte }\nfunc Value(a *Packet) uint64"}, + {"type A uint64\ntype B uint64\ntype Choice = A\nfunc Value(a *Choice) uint64", "type A uint64\ntype B uint64\ntype Choice = B\nfunc Value(a *Choice) uint64"}, + } { + t.Run(test.mutation, func(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, test.original, "TEXT ·Value(SB),4,$160-16\nRET\n") + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\n"+test.mutation+"\n"), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("actual compiler export accepted stale supplied language identity: %v", err) + } + }) + } +} + +func TestGoAMD64FrameTypedSourceLanguageBounds(t *testing.T) { + sizes := types.SizesFor("gc", "amd64") + constraint := types.NewInterfaceType(nil, nil).Complete() + parameter := types.NewTypeParam(types.NewTypeName(token.NoPos, nil, "T", nil), constraint) + for _, typ := range []types.Type{nil, parameter, types.Typ[types.UntypedInt], types.Typ[types.Float64], types.NewMap(types.Typ[types.Uint64], types.Typ[types.Uint64])} { + if _, err := goAMD64FrameLanguageType(typ, sizes); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown language role did not fail closed: %v", err) + } + } + deep := types.Type(types.Typ[types.Uint64]) + for i := 0; i < 34; i++ { + deep = types.NewPointer(deep) + } + fields := make([]*types.Var, 4097) + for i := range fields { + fields[i] = types.NewVar(token.NoPos, nil, fmt.Sprintf("F%d", i), types.Typ[types.Uint64]) + } + for _, typ := range []types.Type{deep, types.NewStruct(fields, nil), types.NewArray(types.Typ[types.Uint64], 1<<62)} { + if _, err := goAMD64FrameLanguageType(typ, sizes); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unbounded or overflowing type obtained an observation: %v", err) + } + } + large := types.NewArray(types.Typ[types.Uint8], 1<<62) + fields = []*types.Var{ + types.NewVar(token.NoPos, nil, "A", large), + types.NewVar(token.NoPos, nil, "B", large), + } + if _, err := goAMD64FrameLanguageType(types.NewStruct(fields, nil), sizes); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("aggregate size overflow was recorded as a layout: %v", err) + } + if _, err := goAMD64FrameLanguageType(types.Typ[types.Uint64], nil); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing target sizes were treated as a layout") + } +} + +func TestGoAMD64FrameTypedSourceMissingSignatureRemainsContext(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a []byte) uint64", "TEXT ·Value(SB),4,$160-32\nRET\n") + pkg.Types = types.NewPackage(pkg.Path, "framefixture") + pkg.Types.Scope().Insert(types.NewFunc(token.NoPos, pkg.Types, "Value", nil)) + guard := goAMD64FrameTypedSelection(t, pkg, request) + if _, err := guard.typedDeclarations(pkg, false); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("missing supplied typed signature did not retain Context: %v", err) + } +} + +func TestGoAMD64FrameTypedSourceMutationAndCloseRevocation(t *testing.T) { + for _, change := range []string{"source", "object", "header inventory", "cancel", "close"} { + t.Run(change, func(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a []byte, n uint64) uint64", "TEXT ·Value(SB),4,$160-40\nRET\n") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + switch change { + case "source": + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\nfunc Value(a []uint16, n uint64) uint64\n"), 0600); err != nil { + t.Fatal(err) + } + case "object": + if err := os.WriteFile(filepath.Join(source.owned, request.AssemblyFile+".o"), []byte("not the original object"), 0600); err != nil { + t.Fatal(err) + } + case "header inventory": + if err := os.WriteFile(filepath.Join(request.Directory, "late.h"), []byte("#define NEW 1\n"), 0600); err != nil { + t.Fatal(err) + } + case "cancel": + cancel() + case "close": + if err := source.Close(); err != nil { + t.Fatal(err) + } + } + if err := source.Verify(ctx); err == nil { + t.Fatal("changed/closed composite source retained its observation") + } + if !errors.Is(source.Verify(context.Background()), ErrProbeNeedsContext) { + t.Fatal("failed composite Verify did not permanently revoke the token") + } + }) + } +} + +func TestGoAMD64FrameTypedSourceNoMemoryPermissionInThreeAPIs(t *testing.T) { + assembly := "TEXT ·Value(SB),4,$160-40\nPXOR X0,X0\nMOVOU X0,16(SP)\nPADDQ 16(SP),X0\nRET\n" + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a []byte, n uint64) uint64", assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + sig := source.functions[pkg.Path+".Value"].signature + if !errors.Is(source.numericDeclarations(pkg), ErrProbeNeedsContext) { + t.Fatal("composite observation entered fixed-local numeric transport") + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) } + opt := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: map[string]FuncSig{sig.Name: sig}} + for _, api := range []string{"Translate", "TranslateModuleInContext", "TranslateGoModule"} { + t.Run(api, func(t *testing.T) { + var err error + switch api { + case "Translate": + _, err = Translate(file, opt) + case "TranslateModuleInContext": + ctx := llvm.NewContext() + defer ctx.Dispose() + module, failure := TranslateModuleInContext(ctx, file, opt) + err = failure + if !module.IsNil() { + defer module.Dispose() + } + case "TranslateGoModule": + translation, failure := TranslateGoModule(pkg, []byte(assembly), GoModuleOptions{GOOS: "linux", GOARCH: "amd64", TargetTriple: opt.TargetTriple, ResolveSym: resolve}) + err = failure + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("sealed observation alone granted native local memory: %v", err) + } + }) + } + encoded, err := json.Marshal(source) + if err != nil { + t.Fatal(err) + } + var restored GoAMD64FrameSource + if err := json.Unmarshal(encoded, &restored); err != nil || !errors.Is(restored.Verify(context.Background()), ErrProbeNeedsContext) { + t.Fatalf("public JSON recreated a composite observation: %v", err) + } + if !reflect.DeepEqual(source.declarations[sig.Name].signature, sig) { + t.Fatal("object signature diverged from independently typed declaration") + } +} + +type goAMD64FrameLayoutCounter struct { + types.Sizes + structCalls int +} + +func (counter *goAMD64FrameLayoutCounter) Sizeof(typ types.Type) int64 { + if _, ok := typ.(*types.Struct); ok { + counter.structCalls++ + } + return counter.Sizes.Sizeof(typ) +} + +func TestGoAMD64FrameTypedSourceMetadataByteBoundBeforeLayout(t *testing.T) { + for _, field := range []string{"name", "tag", "package"} { + t.Run(field, func(t *testing.T) { + name, tag := "Value", "" + pkg := types.NewPackage("example.invalid/model", "model") + switch field { + case "name": + name = strings.Repeat("x", 64<<10+1) + case "tag": + tag = strings.Repeat("x", 64<<10+1) + case "package": + pkg = types.NewPackage("example.invalid/"+strings.Repeat("x", 64<<10+1), "model") + } + field := types.NewVar(token.NoPos, pkg, name, types.Typ[types.Uint64]) + structure := types.NewStruct([]*types.Var{field}, []string{tag}) + counter := &goAMD64FrameLayoutCounter{Sizes: types.SizesFor("gc", "amd64")} + if _, err := goAMD64FrameLanguageType(structure, counter); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("oversized field metadata was recorded: %v", err) + } + if counter.structCalls != 0 { + t.Fatalf("aggregate layout continued after metadata exceeded its bound: %d calls", counter.structCalls) + } + }) + } +} + +func TestGoAMD64FrameTypedSourcePointeeModelIsNotMachineExtent(t *testing.T) { + declaration := `type Packet struct { X uint64; Y byte } +type EmptyTail struct { X uint64; Y struct{} } +func Value(a *Packet, b *EmptyTail, n uint64) uint64` + pkg, request := goAMD64FrameTypedFixture(t, declaration, "TEXT ·Value(SB),4,$160-32\nRET\n") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + header := filepath.Join(source.owned, "pointee-layout.h") + archive := filepath.Join(source.owned, "pointee-layout.a") + if _, err := source.run(ctx, "tool", "compile", "-p", pkg.Path, "-asmhdr", header, "-o", archive, "value.go"); err != nil { + t.Fatal(err) + } + data, err := source.read(header) + if err != nil { + t.Fatal(err) + } + if data, err := source.read(archive); err != nil || len(data) == 0 { + t.Fatalf("actual original compiler archive missing: %v", err) + } + definitions := make(map[string]string) + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 3 && fields[0] == "#define" { + if _, duplicate := definitions[fields[1]]; duplicate { + t.Fatalf("duplicate original machine definition: %s", line) + } + definitions[fields[1]] = fields[2] + } + } + fn := source.functions[pkg.Path+".Value"] + for _, name := range []string{"Packet", "EmptyTail"} { + // This is the independently observed machine definition. The + // capture descriptor is deliberately only the go/types model. + if definitions[name+"__size"] != "16" || definitions[name+"_X"] != "0" || definitions[name+"_Y"] != "8" { + t.Fatalf("unexpected actual compiler padded layout for %s: %v", name, definitions) + } + typ := pkg.Types.Scope().Lookup(name).Type() + model := types.SizesFor("gc", "amd64").Sizeof(typ) + identity, err := goAMD64FrameLanguageType(typ, types.SizesFor("gc", "amd64")) + if err != nil || !strings.Contains(fn.languageType, identity) { + t.Fatalf("missing independent language model: %v", err) + } + t.Logf("%s: recorded go/types model=%d; independently observed stock-Go machine size=16", name, model) + } + if fn.transport != goAMD64FrameCompositeTransport || !errors.Is(source.numericDeclarations(pkg), ErrProbeNeedsContext) { + t.Fatal("language model was used as a numeric backing/pointee permission") + } + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } +} diff --git a/amd64_go_legacy_metadata_roles_test.go b/amd64_go_legacy_metadata_roles_test.go new file mode 100644 index 000000000..84bda8116 --- /dev/null +++ b/amd64_go_legacy_metadata_roles_test.go @@ -0,0 +1,68 @@ +package plan9asm + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/xgo-dev/plan9asm/internal/golegacymod" + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" +) + +func TestGoAMD64LegacyDeclarationMetadataRoles(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + metadata := filepath.Join(t.TempDir(), "legacy.mod") + member := filepath.Join(root, "go.mod") + for _, name := range []string{metadata, member} { + if err := os.WriteFile(name, []byte("module example.invalid/legacy\n"), 0600); err != nil { + t.Fatal(err) + } + } + metadata, err = filepath.EvalSymlinks(metadata) + if err != nil { + t.Fatal(err) + } + original := gooriginalsrc.Module{Root: root, ModulePath: "example.invalid/legacy", + SourceModule: "example.invalid/legacy", SourceVersion: "v1.0.0", GoMod: metadata, + GoModOrigin: golegacymod.ProxyGoModProtocol} + for _, role := range []string{"dependency", "invented member", "main"} { + t.Run(role, func(t *testing.T) { + data, err := json.Marshal(map[string]any{ + "Dir": root, "ImportPath": original.ModulePath, + "Module": map[string]any{"Path": original.ModulePath, "Version": original.SourceVersion, + "Dir": root, "GoMod": metadata}, + }) + if err != nil { + t.Fatal(err) + } + var selected goAMD64FrameSelection + if err := json.Unmarshal(data, &selected); err != nil { + t.Fatal(err) + } + // Predicate data only, deliberately not an actual whole-ZIP origin + // or frame token. The duplicate member tests source-role confusion. + source := &GoAMD64FrameSource{downloadedOrigin: &gooriginalsrc.OriginSet{}, downloadedModule: &original, + downloadedImports: map[string]gooriginalsrc.Module{original.ModulePath + "@" + original.SourceVersion: original}} + if role == "invented member" { + selected.Module.GoMod = member + } + if role == "main" { + selected.Module.Main = true + selected.Module.Version, selected.Module.GoVersion = "", "1.16" + } + _, err = source.downloadedDeclarationDirectory(selected) + if role == "dependency" && err != nil || role != "dependency" && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("legacy metadata lost its exact dependency-only origin", err) + } + if err := source.Verify(context.Background()); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("module predicate data manufactured frame authority", err) + } + }) + } +} diff --git a/amd64_go_native_header.go b/amd64_go_native_header.go new file mode 100644 index 000000000..ab76e0cf7 --- /dev/null +++ b/amd64_go_native_header.go @@ -0,0 +1,82 @@ +package plan9asm + +import ( + "regexp" + "strconv" +) + +// This is listing metadata, not native entry, local backing or ABI authority. +// Preserve absent fields and the two official writer alignment positions. +type goAMD64NativeHeader struct { + Symbol string + Kind uint8 + Static, DupOK, CFunc, NoSplit, TopFrame, ASM, Leaf bool + Size, Locals, Align uint64 + Args int64 + ArgsKnown, HasAlign, HasFuncID, LegacyAlign bool + FuncID uint8 +} + +var goAMD64NativeHeaderPattern = regexp.MustCompile(`^(\S+) (STEXT|STEXTFIPS)(?: (static))?(?: (dupok))?(?: (cfunc))?(?: (nosplit))?(?: (topframe))?(?: (asm))? size=([0-9]+)(?: align=0x([0-9a-f]+))? args=0x([0-9a-f]+) locals=0x([0-9a-f]+)(?: funcid=0x([0-9a-f]+))?(?: align=0x([0-9a-f]+))?(?: (leaf))?$`) + +func parseGoAMD64NativeHeader(line string) (goAMD64NativeHeader, bool) { + var header goAMD64NativeHeader + fields := goAMD64NativeHeaderPattern.FindStringSubmatch(line) + if fields == nil || fields[10] != "" && fields[14] != "" { + return header, false + } + size, sizeErr := strconv.ParseUint(fields[9], 10, 63) + args, argsErr := strconv.ParseUint(fields[11], 16, 64) + locals, localsErr := goAMD64FrameSignedLocals(fields[12]) + if sizeErr != nil || argsErr != nil || localsErr != nil || uint64(int64(int32(args))) != args || + int64(args) < 0 && int64(args) != -0x80000000 { + return header, false + } + header = goAMD64NativeHeader{Symbol: fields[1], Kind: 1, Static: fields[3] != "", DupOK: fields[4] != "", + CFunc: fields[5] != "", NoSplit: fields[6] != "", TopFrame: fields[7] != "", ASM: fields[8] != "", Leaf: fields[15] != "", + Size: size, Args: int64(args), Locals: uint64(locals), ArgsKnown: int64(args) != -0x80000000, + HasAlign: fields[10] != "" || fields[14] != "", HasFuncID: fields[13] != "", LegacyAlign: fields[14] != ""} + if fields[2] == "STEXTFIPS" { + header.Kind = 2 + } + if header.HasFuncID { + funcID, err := strconv.ParseUint(fields[13], 16, 8) + if err != nil { + return goAMD64NativeHeader{}, false + } + header.FuncID = uint8(funcID) + } + if header.HasAlign { + raw, bits := fields[10], 15 // Current LSym.Align is a nonnegative int16. + if header.LegacyAlign { + raw, bits = fields[14], 31 // Older FuncInfo.Align is int32. + } + align, err := strconv.ParseUint(raw, 16, bits) + if err != nil { + return goAMD64NativeHeader{}, false + } + header.Align = align + } + return header, true +} + +func (header goAMD64NativeHeader) objectFlags() uint8 { + var flags uint8 + if header.DupOK { + flags |= 0x01 + } + if header.Leaf { + flags |= 0x08 + } + if header.NoSplit { + flags |= 0x10 + } + return flags +} + +// Special runtime/linker roles remain observations, outside the existing +// ordinary ABI0 and same-source private leaf entry contracts. +func (header goAMD64NativeHeader) ordinaryFrameRole() bool { + return header.ASM && header.HasFuncID && header.FuncID == 0 && + !header.DupOK && !header.CFunc && !header.TopFrame && !header.Leaf +} diff --git a/amd64_go_native_header_grammar_go127_test.go b/amd64_go_native_header_grammar_go127_test.go new file mode 100644 index 000000000..3c5c8fd93 --- /dev/null +++ b/amd64_go_native_header_grammar_go127_test.go @@ -0,0 +1,88 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestSDKAMD64NativeHeaderActualGoAttributes(t *testing.T) { + _, profile, _, _ := numericalSBFixture(t, "amd64") + ctx, cancel := runtimeOracleProofContext(t) + defer cancel() + sdk := filepath.Dir(filepath.Dir(profile.GoBinary)) + builtin, err := os.ReadFile(filepath.Join(sdk, "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + t.Fatal(err) + } + // Decimal flags are independent official TEXT values: DUPOK=2, + // NOSPLIT=4, TOPFRAME=2048. These are metadata objects, not entries. + const source = `TEXT ·Ordinary(SB),4,$16-0 + RET +TEXT ·Duplicate(SB),6,$16-0 + RET +TEXT ·Top(SB),2052,$16-0 + RET +TEXT private<>(SB),4,$0-0 + RET +TEXT ·Combined(SB),2054,$16-0 + RET +TEXT crypto∕internal∕fips140∕headerfixture·FIPS(SB),2054,$16-0 + RET +TEXT runtime·goexit(SB),2052,$0-0 + RET +` + input := filepath.Join(profile.Directory, "header-attributes.s") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + for _, goos := range []string{"linux", "darwin", "windows"} { + t.Run(goos, func(t *testing.T) { + env := append([]string(nil), profile.Environment...) + for index, pair := range env { + if strings.HasPrefix(pair, "GOOS=") { + env[index] = "GOOS=" + goos + } + } + objectPath := filepath.Join(t.TempDir(), "header-attributes.o") + listing, err := goasmcmd.Run(ctx, profile.Directory, env, profile.GoBinary, + "tool", "asm", "-p", "headerfixture", "-S", "-o", objectPath, input) + if err != nil { + t.Fatal("actual Go assembler precondition", err) + } + object, err := os.ReadFile(objectPath) + if err != nil || len(object) == 0 { + t.Fatal("actual complete Go object precondition", err) + } + observed, err := gostaticstorage.ObserveAMD64AssemblyData([]byte(source), object, builtin, "headerfixture", goos+"/amd64") + if err != nil || len(observed.Member.Texts) != 7 || len(observed.TextOrder) != 7 { + t.Fatal("complete original source/object association precondition", err) + } + if !bytes.Contains(listing, []byte(" dupok ")) || !bytes.Contains(listing, []byte(" topframe ")) || + bytes.Count(listing, []byte(" STEXTFIPS ")) != 1 { + t.Fatalf("actual Go did not emit the independent attribute/kind priors:\n%s", listing) + } + records, err := sdkAMD64FrameNativeRecordsForSource(listing, observed.Member.Texts, observed.TextOrder) + if err != nil || len(records) != 7 { + t.Fatal("complete actual native headers lost their object associations", err) + } + for index, record := range records { + locals := uint64(24) + if index == 3 || index == 6 { + locals = 0 + } + if record.Symbol != observed.TextOrder[index].Name || record.Static != (index == 3) || + !record.NoSplit || !record.ASM || !record.ArgsKnown || record.Args != 0 || record.Locals != locals { + t.Fatal("actual header changed independent symbol/frame/argument metadata", index, record) + } + } + }) + } +} diff --git a/amd64_go_native_header_grammar_test.go b/amd64_go_native_header_grammar_test.go new file mode 100644 index 000000000..6a8764740 --- /dev/null +++ b/amd64_go_native_header_grammar_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "errors" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +// These are format/object-field observations, not captured source permissions. +func nativeHeaderFixture(header string, kind, flags uint8, abi uint16) ([]byte, []gostaticstorage.TextRecord, []gostaticstorage.ASMSourceText) { + namespace := gostaticstorage.TextSymbolNonpkgDefinition + if abi == 65535 { + namespace = gostaticstorage.TextSymbolSelf + } + text := gostaticstorage.TextRecord{Name: "fixture.Entry", ABI: abi, Namespace: namespace, + Kind: kind, Flags: flags, Size: 1, Code: []byte{0xc3}} + order := gostaticstorage.ASMSourceText{SourceOrdinal: 0, ObjectOrdinal: 0, Name: text.Name, + ABI: abi, Namespace: namespace} + return []byte(header + "\n"), []gostaticstorage.TextRecord{text}, []gostaticstorage.ASMSourceText{order} +} + +func nativeHeaderRecord(t *testing.T, header string, kind, flags uint8, abi uint16) sdkAMD64FrameNativeRecord { + t.Helper() + listing, texts, order := nativeHeaderFixture(header, kind, flags, abi) + before := append([]gostaticstorage.TextRecord(nil), texts...) + before[0].Code = append([]byte(nil), texts[0].Code...) + records, err := sdkAMD64FrameNativeRecordsForSource(listing, texts, order) + if err != nil || len(records) != 1 { + t.Fatalf("complete writer header lost its object association: records=%v error=%v header=%q", records, err, header) + } + if !reflect.DeepEqual(texts, before) || records[0].Symbol != "fixture.Entry" || records[0].Size != 1 { + t.Fatal("header parsing rewrote the original object or identity", records) + } + return records[0] +} + +const nativeHeaderCurrent = "fixture.Entry STEXT nosplit asm size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0" + +func TestSDKAMD64NativeHeaderCompleteWriterForms(t *testing.T) { + for _, row := range []struct { + name, attributes, tail string + kind, flags uint8 + abi uint16 + }{ + {"current", "nosplit asm", "size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0", 1, 0x10, 0}, + {"legacy", "nosplit asm", "size=1 args=0x0 locals=0x0 funcid=0x0 align=0x0", 1, 0x10, 0}, + {"minimal-diagnostic", "asm", "size=1 args=0x0 locals=0x0", 1, 0, 0}, + {"static", "static nosplit asm", "size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0", 1, 0x10, 65535}, + {"dupok", "dupok nosplit asm", "size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0", 1, 0x11, 0}, + {"cfunc", "cfunc nosplit", "size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0", 1, 0x10, 0}, + {"topframe", "nosplit topframe asm", "size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0", 1, 0x10, 0}, + {"leaf", "nosplit asm", "size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0 leaf", 1, 0x18, 0}, + {"all-current", "static dupok cfunc nosplit topframe asm", "size=1 align=0x0 args=0xffffffff80000000 locals=0xfffffffffffffff8 funcid=0xff leaf", 2, 0x19, 65535}, + {"all-legacy-format", "static dupok cfunc nosplit topframe asm", "size=1 args=0xffffffff80000000 locals=0xfffffffffffffff8 funcid=0xff align=0x0 leaf", 2, 0x19, 65535}, + {"current-width-boundaries", "nosplit asm", "size=1 align=0x7fff args=0x7fffffff locals=0x7fffffff funcid=0xff", 1, 0x10, 0}, + {"legacy-width-boundaries", "nosplit asm", "size=1 args=0x7fffffff locals=0x7fffffff funcid=0xff align=0x7fffffff", 1, 0x10, 0}, + } { + t.Run(row.name, func(t *testing.T) { + kind := "STEXT" + if row.kind == 2 { + kind = "STEXTFIPS" + } + header := "fixture.Entry " + kind + " " + row.attributes + " " + row.tail + record := nativeHeaderRecord(t, header, row.kind, row.flags, row.abi) + if record.Static != (row.abi == 65535) || record.NoSplit != (row.flags&0x10 != 0) || + record.ASM != strings.Contains(" "+row.attributes+" ", " asm ") { + t.Fatal("independent native attributes changed", record) + } + if strings.Contains(row.tail, "locals=0xfffffffffffffff8") && + (record.Locals != ^uint64(7) || record.ArgsKnown || record.Args != -0x80000000) { + t.Fatal("writer attributes changed signed locals or the independent argument sentinel", record) + } + if strings.Contains(row.tail, "args=0x7fffffff") { + align := uint64(0x7fff) + if row.name == "legacy-width-boundaries" { + align = 0x7fffffff + } + if !record.ArgsKnown || record.Args != 0x7fffffff || record.Locals != 0x7fffffff || + !record.HasFuncID || record.FuncID != 0xff || !record.HasAlign || record.Align != align { + t.Fatal("official writer numeric boundary metadata changed", record) + } + } + }) + } +} + +func TestSDKAMD64NativeHeaderRetainsObservedMetadata(t *testing.T) { + base := nativeHeaderRecord(t, nativeHeaderCurrent, 1, 0x10, 0) + for _, row := range []struct { + name, header string + kind, flags uint8 + }{ + {"kind", strings.Replace(nativeHeaderCurrent, "STEXT", "STEXTFIPS", 1), 2, 0x10}, + {"dupok", strings.Replace(nativeHeaderCurrent, " nosplit", " dupok nosplit", 1), 1, 0x11}, + {"cfunc", strings.Replace(nativeHeaderCurrent, " nosplit", " cfunc nosplit", 1), 1, 0x10}, + {"topframe", strings.Replace(nativeHeaderCurrent, " asm", " topframe asm", 1), 1, 0x10}, + {"leaf", nativeHeaderCurrent + " leaf", 1, 0x18}, + {"funcid", strings.Replace(nativeHeaderCurrent, "funcid=0x0", "funcid=0x1", 1), 1, 0x10}, + {"alignment", strings.Replace(nativeHeaderCurrent, "align=0x0", "align=0x4", 1), 1, 0x10}, + {"absent-funcid", strings.Replace(nativeHeaderCurrent, " funcid=0x0", "", 1), 1, 0x10}, + {"absent-alignment", strings.Replace(nativeHeaderCurrent, " align=0x0", "", 1), 1, 0x10}, + {"legacy-alignment-position", "fixture.Entry STEXT nosplit asm size=1 args=0x0 locals=0x0 funcid=0x0 align=0x0", 1, 0x10}, + } { + t.Run(row.name, func(t *testing.T) { + changed := nativeHeaderRecord(t, row.header, row.kind, row.flags, 0) + if reflect.DeepEqual(base, changed) { + t.Fatal("distinct observed native metadata was discarded", row.name, base) + } + }) + } +} + +func TestSDKAMD64NativeHeaderRejectsMalformedWriterForms(t *testing.T) { + for _, header := range []string{ + strings.Replace(nativeHeaderCurrent, " nosplit", " nosplit nosplit", 1), + strings.Replace(nativeHeaderCurrent, "nosplit asm", "asm nosplit", 1), + strings.Replace(nativeHeaderCurrent, "nosplit asm", "nosplit dupok asm", 1), + strings.Replace(nativeHeaderCurrent, "nosplit asm", "nosplit asm topframe", 1), + strings.Replace(nativeHeaderCurrent, " nosplit", " foreign nosplit", 1), + strings.Replace(nativeHeaderCurrent, " nosplit", " noframe nosplit", 1), + strings.Replace(nativeHeaderCurrent, "STEXT", "STEXTFOREIGN", 1), + nativeHeaderCurrent + " leaf leaf", + nativeHeaderCurrent + " junk", + nativeHeaderCurrent + " align=0x0", + strings.Replace(nativeHeaderCurrent, "size=1", "size=1 size=1", 1), + strings.Replace(nativeHeaderCurrent, "size=1", "size=9223372036854775808", 1), + strings.Replace(nativeHeaderCurrent, "args=0x0", "args=0x80000000", 1), + strings.Replace(nativeHeaderCurrent, "args=0x0", "args=0xffffffff", 1), + strings.Replace(nativeHeaderCurrent, "args=0x0", "args=0xffffffffffffffff", 1), + strings.Replace(nativeHeaderCurrent, "locals=0x0", "locals=0x80000000", 1), + strings.Replace(nativeHeaderCurrent, "locals=0x0", "locals=0x10000000000000000", 1), + strings.Replace(nativeHeaderCurrent, "funcid=0x0", "funcid=0x100", 1), + strings.Replace(nativeHeaderCurrent, "align=0x0", "align=0x8000", 1), + "fixture.Entry STEXT nosplit asm size=1 args=0x0 locals=0x0 funcid=0x0 align=0x80000000", + } { + t.Run(header, func(t *testing.T) { + listing, texts, order := nativeHeaderFixture(header, 1, 0x10, 0) + if _, err := sdkAMD64FrameNativeRecordsForSource(listing, texts, order); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("malformed or out-of-range header acquired metadata", header, err) + } + }) + } +} + +func TestSDKAMD64NativeHeaderObjectAttributeIdentity(t *testing.T) { + for _, row := range []struct { + name, attributes, tail string + mask uint8 + }{ + {"dupok", "dupok nosplit asm", "", 0x01}, + {"nosplit", "nosplit asm", "", 0x10}, + {"leaf", "nosplit asm", " leaf", 0x08}, + } { + t.Run(row.name, func(t *testing.T) { + header := "fixture.Entry STEXT " + row.attributes + " size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0" + row.tail + flags := uint8(0x10) | row.mask + nativeHeaderRecord(t, header, 1, flags, 0) + listing, texts, order := nativeHeaderFixture(header, 1, flags^row.mask, 0) + if _, err := sdkAMD64FrameNativeRecordsForSource(listing, texts, order); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("header borrowed a different actual object attribute", row.name, err) + } + }) + } + for _, mask := range []uint8{0x01, 0x08, 0x10} { + listing, texts, order := nativeHeaderFixture("fixture.Entry STEXT asm size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0", 1, mask, 0) + if _, err := sdkAMD64FrameNativeRecordsForSource(listing, texts, order); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unprinted object attribute was silently ignored", mask, err) + } + } +} + +func TestGoAMD64FrameSplitSelectorUsesCompleteWriterHeader(t *testing.T) { + for _, attributes := range []string{"asm", "dupok asm", "cfunc asm", "topframe asm", "static dupok cfunc topframe asm"} { + t.Run(attributes, func(t *testing.T) { + header := "fixture.Entry STEXT " + attributes + " size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0\n" + if !goAMD64FrameSplitListing([]byte(header)) { + t.Fatal("complete split header disappeared before its later role check", header) + } + nosplit := strings.Replace(header, " asm ", " nosplit asm ", 1) + if strings.Contains(attributes, "topframe") { + nosplit = strings.Replace(header, " topframe ", " nosplit topframe ", 1) + } + if goAMD64FrameSplitListing([]byte(nosplit)) { + t.Fatal("NOSPLIT metadata was interpreted as split entry permission", nosplit) + } + }) + } +} diff --git a/amd64_go_native_header_roles_test.go b/amd64_go_native_header_roles_test.go new file mode 100644 index 000000000..79c5ae007 --- /dev/null +++ b/amd64_go_native_header_roles_test.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// Keep the complete actual listing and change only one independently named +// writer role. Attribute parsing must not authorize a different entry contract. +func nativeHeaderRoleChanges() []struct{ name, before, after string } { + return []struct{ name, before, after string }{ + {"dupok", " nosplit", " dupok nosplit"}, + {"cfunc", " nosplit", " cfunc nosplit"}, + {"topframe", " asm", " topframe asm"}, + {"leaf", "", " leaf"}, + {"funcid", "funcid=0x0", "funcid=0x1"}, + {"missing-funcid", " funcid=0x0", ""}, + {"missing-asm", " asm", ""}, + } +} + +func nativeHeaderChangedRole(header, before, after string) string { + if before == "" { + return header + after + } + return strings.Replace(header, before, after, 1) +} + +func TestGoAMD64NativeHeaderSpecialRolesKeepEntryContext(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + source := requireGoAMD64FrameCapture(t, pkg, request) + listing, err := os.ReadFile(filepath.Join(source.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + expanded, err := source.preprocess(request.AssemblyFile, raw) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + if _, err := goAMD64FrameListing(listing, file, pkg, source.origins); err != nil { + t.Fatal("complete original ordinary listing positive", err) + } + header, body, found := bytes.Cut(listing, []byte{'\n'}) + if !found { + t.Fatal("actual listing lacks its complete header") + } + for _, change := range nativeHeaderRoleChanges() { + t.Run(change.name, func(t *testing.T) { + if !strings.Contains(string(header), change.before) { + t.Fatal("actual ordinary header precondition changed", string(header)) + } + changed := nativeHeaderChangedRole(string(header), change.before, change.after) + mutated := []byte(changed + "\n" + string(body)) + if _, err := goAMD64FrameListing(mutated, file, pkg, source.origins); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("observed special or unknown header role acquired ordinary entry authority", err) + } + }) + } +} + +func TestGoAMD64NativeHeaderActualSpecialRolesStayContext(t *testing.T) { + for _, row := range []struct{ name, flags string }{ + {"dupok", "6"}, + {"topframe", "2052"}, + {"wrapper-funcid", "36"}, + } { + t.Run(row.name, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + path := filepath.Join(request.Directory, request.AssemblyFile) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + // Official TEXT bits: NOSPLIT=4, DUPOK=2, WRAPPER=32, + // TOPFRAME=2048. Keep every original body instruction. + assembly := strings.Replace(string(raw), "#include \"textflag.h\"\n", "", 1) + assembly = strings.Replace(assembly, "NOSPLIT", row.flags, 1) + x86InsertExtractGoListing(t, "amd64", assembly) + if err := os.WriteFile(path, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + owner, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if owner != nil { + if closeErr := owner.Close(); closeErr != nil { + t.Error(closeErr) + } + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("actual Go special role acquired ordinary fixed-local entry authority", err) + } + }) + } +} diff --git a/amd64_go_sdk_frame_binding_go127_test.go b/amd64_go_sdk_frame_binding_go127_test.go new file mode 100644 index 000000000..87dcc75bb --- /dev/null +++ b/amd64_go_sdk_frame_binding_go127_test.go @@ -0,0 +1,174 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestGoAMD64SDKFrameReadonlyDataComposition(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "crypto/internal/fips140/sha512") + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal(err) + } + defer observed.Close() + pkg := sdkAMD64FrameActualPackage(t, observed.source) + source, err := CaptureGoAMD64SDKFrameSource(ctx, pkg, GoAMD64FrameCaptureRequest{ + GoBinary: request.GoBinary, Directory: request.Directory, AssemblyFile: "sha512block_amd64.s", + Target: request.Target, Environment: request.Environment, + }) + if err != nil { + t.Fatal("complete original frame and readonly DATA observation", err) + } + defer source.Close() + expanded := source.sdkFrame.owner.source.observation.CPP[0].Output + file, err := Parse(ArchAMD64, expanded) + if err != nil || len(file.Globl) == 0 || len(file.Funcs) != len(source.functions) { + t.Fatal("complete original source inventory", err) + } + resolve := func(symbol string) string { return pkg.Path + "." + strings.TrimPrefix(symbol, "·") } + source.mu.Lock() + data, names, err := source.fixedLocalData(file, expanded, resolve) + source.mu.Unlock() + if err != nil || len(data) != len(file.Globl) || len(names) != len(file.Globl) { + t.Fatal("readonly DATA lost its same-source extent/symbol projection", err) + } + for _, changed := range []struct{ name, expanded string }{ + {"missing_amd64.s", expanded}, + {source.request.AssemblyFile, expanded + "\n"}, + } { + if _, err := source.fixedLocalDataObservation(changed.name, changed.expanded); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unmatched source borrowed the complete SDK DATA observation", err) + } + } + if err := source.Verify(ctx); err != nil { + t.Fatal("complete live frame/DATA source", err) + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("closed frame/DATA source revived", err) + } + t.Logf("%d complete source functions and %d readonly definitions; not a full translation result", len(file.Funcs), len(data)) +} + +func TestGoAMD64SDKFrameOriginalSourceCapture(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "crypto/md5") + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal("complete original SDK source/native frame precondition", err) + } + defer observed.Close() + pkg := sdkAMD64FrameActualPackage(t, observed.source) + const assembly = "md5block_amd64.s" + source, err := CaptureGoAMD64SDKFrameSource(ctx, pkg, GoAMD64FrameCaptureRequest{ + GoBinary: request.GoBinary, Directory: request.Directory, AssemblyFile: assembly, + Target: request.Target, Environment: request.Environment, + }) + if err != nil { + t.Fatal("independent SDK-primary native frame handoff is absent", err) + } + defer source.Close() + fn, found := source.functions[request.PackagePath+".block"] + if !found || len(source.functions) != 1 || fn.locals != 16 || fn.args != 32 { + t.Fatal("complete original SDK frame/declaration/native layout", fn) + } + if err := source.Verify(ctx); err != nil { + t.Fatal("live SDK-primary native frame", err) + } + // Even a fresh outer identity cannot borrow the SDK origin's live owner. + copy := &GoAMD64FrameSource{sdkFrame: source.sdkFrame} + copy.identity = copy + if err := copy.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("copied SDK frame acquired the original native source", err) + } + if err := copy.Close(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("copied SDK frame removed original artifacts", err) + } + if err := source.Verify(ctx); err != nil { + t.Fatal("copied SDK frame revoked the authentic original", err) + } + // Capture supplies source/layout facts, not a completed body proof. Do not + // freeze an obsolete unsupported-op diagnostic as a successful observation: + // valid new typed families may subsequently make BindFixedLocal succeed. + if fn.signature.x86FixedLocal != nil { + t.Fatal("capturing a frame implicitly supplied a complete body proof") + } + original := observed.source.observation.ObjectInputs.Sources[0].Original + after, err := os.ReadFile(filepath.Join(request.Directory, assembly)) + if err != nil || !bytes.Equal(after, original) { + t.Fatal("SDK source was replaced or cropped", err) + } + if err := source.Close(); err != nil { + t.Fatal("SDK frame artifact cleanup", err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("closed SDK frame retained its native source lifetime", err) + } +} + +func TestGoAMD64SDKFrameRejectsChangedProfile(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "crypto/md5") + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal(err) + } + defer observed.Close() + pkg := sdkAMD64FrameActualPackage(t, observed.source) + source, err := CaptureGoAMD64SDKFrameSource(ctx, pkg, GoAMD64FrameCaptureRequest{GoBinary: request.GoBinary, + Directory: request.Directory, AssemblyFile: "md5block_amd64.s", Target: request.Target, Environment: request.Environment}) + if err != nil { + t.Fatal(err) + } + defer source.Close() + // The factory owns an independent environment copy; mutating the caller's + // request must neither update its sealed profile nor revoke its lifetime. + request.Environment[0] += "changed" + if err := source.Verify(ctx); err != nil { + t.Fatal("caller environment aliases the private SDK profile", err) + } + original := source.request.Target + source.request.Target = "windows/amd64" + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed target retained the original SDK frame", err) + } + source.request.Target = original + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("restoring a changed SDK profile revived its frame", err) + } +} + +func TestGoAMD64SDKFrameRejectsChangedNativeWitness(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "crypto/md5") + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal(err) + } + defer observed.Close() + pkg := sdkAMD64FrameActualPackage(t, observed.source) + source, err := CaptureGoAMD64SDKFrameSource(ctx, pkg, GoAMD64FrameCaptureRequest{GoBinary: request.GoBinary, + Directory: request.Directory, AssemblyFile: "md5block_amd64.s", Target: request.Target, Environment: request.Environment}) + if err != nil { + t.Fatal(err) + } + defer source.Close() + name := request.PackagePath + ".block" + original := source.functions[name] + changed := original + changed.locals++ + source.functions[name] = changed + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed private native frame supplied backing", err) + } + source.functions[name] = original + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("restoring native metadata revived a revoked frame", err) + } +} diff --git a/amd64_go_sdk_frame_context_go127_test.go b/amd64_go_sdk_frame_context_go127_test.go new file mode 100644 index 000000000..ee3febf7f --- /dev/null +++ b/amd64_go_sdk_frame_context_go127_test.go @@ -0,0 +1,54 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestGoAMD64SDKFrameBindContextCallbacksMayClose(t *testing.T) { + for _, method := range []string{"Err", "Deadline", "Done"} { + t.Run(method, func(t *testing.T) { + original, request := sdkASMDataRequest(t, "amd64", "crypto/md5") + observed, err := CaptureGoAMD64SDKFrameObservation(original, request) + if err != nil { + t.Fatal(err) + } + defer observed.Close() + pkg := sdkAMD64FrameActualPackage(t, observed.source) + source, err := CaptureGoAMD64SDKFrameSource(original, pkg, GoAMD64FrameCaptureRequest{ + GoBinary: request.GoBinary, Directory: request.Directory, AssemblyFile: "md5block_amd64.s", + Target: request.Target, Environment: request.Environment, + }) + if err != nil { + t.Fatal(err) + } + defer source.Close() + file, err := Parse(ArchAMD64, observed.source.observation.CPP[0].Output) + if err != nil { + t.Fatal(err) + } + ctx := goAMD64FrameCloseCallback(t, original, source, method) + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{request.PackagePath + ".block": source.functions[request.PackagePath+".block"].signature}, + ResolveSym: func(symbol string) string { + // The initial Verify has completed. The next context sample + // in the binder must still occur outside the observer lock. + ctx.armed.Store(true) + return request.PackagePath + "." + strings.TrimPrefix(symbol, "·") + }} + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if ctx.locked.Load() || !ctx.called.Load() { + t.Fatalf("SDK Bind caller %s callback ran under frame lock (Close would deadlock) or was never sampled", method) + } + if !errors.Is(err, ErrProbeNeedsContext) || len(bound.Sigs) != 0 { + t.Fatal("callback-closed SDK source published a local binding", err) + } + if err := source.Verify(original); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("SDK Bind revived its callback-closed source", err) + } + }) + } +} diff --git a/amd64_go_sdk_frame_inventory.go b/amd64_go_sdk_frame_inventory.go new file mode 100644 index 000000000..2ceccd5d0 --- /dev/null +++ b/amd64_go_sdk_frame_inventory.go @@ -0,0 +1,80 @@ +package plan9asm + +import ( + "crypto/sha256" + "path/filepath" + + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +// This distinct SDK-primary role retains every direct member, including all +// inactive candidates and child-directory identities. Child contents belong +// to separate packages/generators, not the selected compiler package. Any +// selected dependency retains its independent complete original source/export +// graph; selected relative CPP headers retain their own bounded input route. +// Root membership and child identities are rechecked before/after commands. The ordinary owned +// flat inventory keeps its 256-member bound. The SDK retains its 16 MiB byte +// budget and uses the shared original-source membership bound. +func (source *GoX86SDKASMDataSource) snapshotSDKPrimaryDirectory() error { + return captureSDKPackageDirectoryInputs(source.guard, source.retainDirectory) +} + +// SDK assembly, numerical reads and literal alignment share only this input +// inventory. It grants neither child-package source/include authority nor any +// frame, native entry or initialized-storage role. Each caller authenticates +// its own complete SDK selection and retains/rechecks every child identity. +func captureSDKPackageDirectoryInputs(guard *GoAMD64FrameSource, retainDirectory func(string) error) error { + resolved, err := sourcepath.Resolve(guard.request.Directory) + if err != nil || resolved != guard.request.Directory { + return sdkASMDataContext("SDK-primary package directory is not canonical") + } + inventory := gooriginalsrc.NewSnapshot() + if err := inventory.Directory(resolved, false); err != nil { + return sdkASMDataContext("SDK-primary complete package inventory: %v", err) + } + // Snapshot.Directory closes the complete EOF and applies the existing + // shared file/directory count budgets. Use its observed cardinality, not + // a newly guessed per-package cap, for the same direct member list. + entries, info, err := gooriginalsrc.ReadDirectory(resolved, len(inventory.View().Entries[resolved])) + if err != nil || len(entries) == 0 { + return sdkASMDataContext("SDK-primary complete package inventory changed: %v", err) + } + guard.directoryInfo = info + remaining := int64(16 << 20) + for _, entry := range entries { + guard.entries = append(guard.entries, entry.Name()) + path := filepath.Join(resolved, entry.Name()) + if entry.IsDir() { + if err := retainDirectory(path); err != nil { + return err + } + continue + } + data, err := goAMD64FrameReadBound(path, remaining) + if err != nil { + return err + } + digest := sha256.Sum256(data) + if previous, present := guard.inputs[path]; present && previous != digest { + return sdkASMDataContext("SDK-primary original inventory changed while capturing") + } + remaining -= int64(len(data)) + guard.inputs[path] = digest + } + if err := inventory.Verify(); err != nil { + return sdkASMDataContext("SDK-primary input membership changed during capture: %v", err) + } + guard.directoryInventory = inventory + return nil +} + +// The frame owner copies mutable input maps/slices, while retaining the same +// immutable directory observation through its existing live SDK source owner. +// Directory metadata does not supply frame, numerical or native-entry facts. +func (source *GoAMD64FrameSource) copySDKPrimaryInputInventory(original *GoAMD64FrameSource) { + source.inputs = cloneSDKASMDataInputs(original.inputs) + source.entries = append([]string(nil), original.entries...) + source.directoryInfo = original.directoryInfo + source.directoryInventory = original.directoryInventory +} diff --git a/amd64_go_sdk_frame_inventory_go127_test.go b/amd64_go_sdk_frame_inventory_go127_test.go new file mode 100644 index 000000000..9b807249e --- /dev/null +++ b/amd64_go_sdk_frame_inventory_go127_test.go @@ -0,0 +1,24 @@ +//go:build go1.27 + +package plan9asm + +import "testing" + +func TestSDKAMD64PrimaryInventoryActualCompleteMD5(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "crypto/md5") + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal("complete unchanged SDK compiler-selected source/native frame", err) + } + defer observed.Close() + source := observed.source + if source.guard.selection.Module != nil || source.guard.selection.ImportPath != request.PackagePath || + len(source.observation.Frames) != 1 || len(source.observation.Frames[0].Records) != 1 || + !source.observation.Frames[0].Records[0].NoSplit || source.observation.Frames[0].Records[0].Locals != 16 || + source.observation.Frames[0].Records[0].Args != 32 { + t.Fatal("SDK-primary exact selected source or native frame inventory changed") + } + if err := observed.Verify(ctx); err != nil { + t.Fatal("live complete SDK source with independent child identities", err) + } +} diff --git a/amd64_go_sdk_frame_inventory_test.go b/amd64_go_sdk_frame_inventory_test.go new file mode 100644 index 000000000..921c4ff9f --- /dev/null +++ b/amd64_go_sdk_frame_inventory_test.go @@ -0,0 +1,315 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +func TestSDKAMD64PrimaryInventorySeparatesChildSourceRoles(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "generator"), 0700); err != nil { + t.Fatal(err) + } + for path, bytes := range map[string][]byte{"source.go": []byte("package fixture\n"), "ignored.s": []byte("// inactive candidate\n"), + "generator/source.go": []byte("package separate\n")} { + if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(path)), bytes, 0600); err != nil { + t.Fatal(err) + } + } + root, err := sourcepath.Resolve(root) + if err != nil { + t.Fatal(err) + } + request := GoAMD64FrameCaptureRequest{Directory: root} + ordinary := newGoOriginalInputGuard(request) + if err := ordinary.snapshotDirectory(); err != nil { + t.Fatal("ordinary package omitted a bounded direct-child identity", err) + } + if !reflect.DeepEqual(ordinary.entries, []string{"generator", "ignored.s", "source.go"}) || len(ordinary.inputs) != 2 || + len(ordinary.childDirectories) != 1 || ordinary.childDirectories[filepath.Join(root, "generator")] == nil { + t.Fatal("ordinary inventory confused direct files with child-package contents") + } + ordinary.owned, err = sourcepath.Resolve(t.TempDir()) + if err != nil { + t.Fatal(err) + } + ordinary.ownedInfo, err = sourcepath.Lstat(ordinary.owned) + if err != nil { + t.Fatal(err) + } + source := &GoX86SDKASMDataSource{guard: newGoOriginalInputGuard(request), directories: make(map[string]os.FileInfo)} + if err := source.snapshotSDKPrimaryDirectory(); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(source.guard.entries, []string{"generator", "ignored.s", "source.go"}) || len(source.guard.inputs) != 2 || + source.directories[filepath.Join(root, "generator")] == nil { + t.Fatal("complete direct/inactive input or separate child identity was omitted") + } + if err := os.WriteFile(filepath.Join(root, "generator", "source.go"), []byte("package changedChild\n"), 0600); err != nil { + t.Fatal(err) + } + if err := ordinary.verifyInputs(); err != nil { + t.Fatal("ordinary selected package read separately owned child contents", err) + } + if err := os.Rename(filepath.Join(root, "generator"), filepath.Join(root, "original-generator")); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(root, "generator"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "generator", "source.go"), []byte("package separate\n"), 0600); err != nil { + t.Fatal(err) + } + if err := source.verifyDirectories(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("replacement child directory acquired original selected-package identity", err) + } + if err := ordinary.verifyInputs(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("replacement child directory retained the ordinary original identity", err) + } +} + +func TestSDKAMD64PrimaryInventoryRootMembershipGuard(t *testing.T) { + root, owned := t.TempDir(), t.TempDir() + if err := os.WriteFile(filepath.Join(root, "source.go"), []byte("package fixture\n"), 0600); err != nil { + t.Fatal(err) + } + root, err := sourcepath.Resolve(root) + if err != nil { + t.Fatal(err) + } + owned, err = sourcepath.Resolve(owned) + if err != nil { + t.Fatal(err) + } + source := &GoX86SDKASMDataSource{guard: newGoOriginalInputGuard(GoAMD64FrameCaptureRequest{Directory: root}), directories: make(map[string]os.FileInfo)} + if err := source.snapshotSDKPrimaryDirectory(); err != nil { + t.Fatal(err) + } + source.guard.owned, source.guard.ownedInfo = owned, nil + source.guard.ownedInfo, err = sourcepath.Lstat(owned) + if err != nil { + t.Fatal(err) + } + if err := source.guard.verifyInputs(); err != nil { + t.Fatal("unchanged complete primary membership", err) + } + if err := os.WriteFile(filepath.Join(root, "new-inactive.s"), []byte("// new inactive input\n"), 0600); err != nil { + t.Fatal(err) + } + if err := source.guard.verifyInputs(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("new unselected root member escaped the complete input guard", err) + } +} + +func TestSDKAMD64PrimaryInventoryByteBoundary(t *testing.T) { + for _, size := range []int{16 << 20, 16<<20 + 1} { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "source.go"), []byte(strings.Repeat(" ", size)), 0600); err != nil { + t.Fatal(err) + } + root, err := sourcepath.Resolve(root) + if err != nil { + t.Fatal(err) + } + source := &GoX86SDKASMDataSource{guard: newGoOriginalInputGuard(GoAMD64FrameCaptureRequest{Directory: root}), directories: make(map[string]os.FileInfo)} + err = source.snapshotSDKPrimaryDirectory() + if size == 16<<20 { + if err != nil || len(source.guard.inputs) != 1 { + t.Fatal("exact original primary byte bound was rejected", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || len(source.guard.inputs) != 0 { + t.Fatal("oversized primary source entered the unchanged byte inventory", err) + } + } +} + +func TestSDKAMD64PrimaryInventoryMemberBoundary(t *testing.T) { + for _, count := range []int{256, 257, 300, 792} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + root := t.TempDir() + for index := 0; index < count; index++ { + name, data := fmt.Sprintf("%03d.go", index), []byte("package fixture\n") + if index == 0 { + name, data = "kernel_amd64.s", []byte("TEXT ·F(SB),4,$0-0\nRET\n") + } else if index == 1 { + data = []byte("package fixture\nfunc F()\n") + } + if err := os.WriteFile(filepath.Join(root, name), data, 0600); err != nil { + t.Fatal(err) + } + } + root, err := sourcepath.Resolve(root) + if err != nil { + t.Fatal(err) + } + request := GoAMD64FrameCaptureRequest{Directory: root, AssemblyFile: "kernel_amd64.s"} + ordinary := newGoOriginalInputGuard(request) + err = ordinary.snapshotDirectory() + if count == 256 { + if err != nil { + t.Fatal("ordinary flat boundary rejected", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || len(ordinary.inputs) != 0 { + t.Fatal("SDK observation widened the ordinary flat input role", err) + } + source := &GoX86SDKASMDataSource{guard: newGoOriginalInputGuard(request), directories: make(map[string]os.FileInfo)} + if err := source.snapshotSDKPrimaryDirectory(); err != nil || len(source.guard.inputs) != count { + t.Fatal("complete SDK input inventory was rejected or truncated", err, len(source.guard.inputs), count) + } + owned, err := sourcepath.Resolve(t.TempDir()) + if err != nil { + t.Fatal(err) + } + source.guard.owned = owned + source.guard.ownedInfo, err = sourcepath.Lstat(owned) + if err != nil { + t.Fatal(err) + } + if err := source.guard.verifyInputs(); err != nil { + t.Fatal("unchanged complete SDK inventory failed verification", err) + } + if err := os.WriteFile(filepath.Join(root, "new-inactive.s"), nil, 0600); err != nil { + t.Fatal(err) + } + if err := source.guard.verifyInputs(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("larger directory became a silently accepted bounded prefix", err) + } + }) + } +} + +func TestSDKAMD64PrimaryInventoryRejectsRedirectedChildren(t *testing.T) { + for _, directory := range []bool{false, true} { + root, outside := t.TempDir(), t.TempDir() + target := outside + if !directory { + target = filepath.Join(outside, "source.go") + if err := os.WriteFile(target, []byte("package foreign\n"), 0600); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(target, filepath.Join(root, "redirected")); err != nil { + t.Fatal("actual redirected child control", err) + } + root, err := sourcepath.Resolve(root) + if err != nil { + t.Fatal(err) + } + source := &GoX86SDKASMDataSource{guard: newGoOriginalInputGuard(GoAMD64FrameCaptureRequest{Directory: root}), directories: make(map[string]os.FileInfo)} + if err := source.snapshotSDKPrimaryDirectory(); !errors.Is(err, ErrProbeNeedsContext) || len(source.guard.inputs) != 0 || len(source.directories) != 0 { + t.Fatal("redirected file/directory acquired a selected SDK-primary input role", err) + } + } +} + +func TestSDKAMD64PrimaryInventoryDoesNotAuthorizeChildInclude(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "generator"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "source.s"), []byte("#include \"generator/body.h\"\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "generator", "body.h"), []byte("TEXT ·F(SB),4,$0\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + root, err := sourcepath.Resolve(root) + if err != nil { + t.Fatal(err) + } + source := &GoX86SDKASMDataSource{request: GoX86SDKASMDataCaptureRequest{Directory: root}, + guard: newGoOriginalInputGuard(GoAMD64FrameCaptureRequest{Directory: root}), directories: make(map[string]os.FileInfo)} + source.guard.env = map[string]string{"GOOS": "linux", "GOARCH": "amd64", "GOAMD64": "v1"} + if err := source.snapshotSDKPrimaryDirectory(); err != nil { + t.Fatal(err) + } + if _, _, err := source.captureAssembly(context.Background(), 0, "source.s"); !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "relative/foreign include") { + t.Fatal("observed child identity became compiler/include source authority", err) + } +} + +// This tests the same private input transfer used by the SDK frame factory. +// It supplies no ready frame or ABI permission and runs no compiler subprocess. +func TestSDKAMD64PrimaryInventoryCloneRetainsCompleteMembership(t *testing.T) { + for _, change := range []string{"none", "add", "remove", "bytes", "replace root", "close owner", "copy inputs", "copy entries", "drop inventory"} { + t.Run(change, func(t *testing.T) { + root, err := sourcepath.Resolve(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for index := 0; index < 300; index++ { + if err := os.WriteFile(filepath.Join(root, fmt.Sprintf("%03d.go", index)), nil, 0600); err != nil { + t.Fatal(err) + } + } + request := GoAMD64FrameCaptureRequest{Directory: root} + source := &GoX86SDKASMDataSource{guard: newGoOriginalInputGuard(request), directories: make(map[string]os.FileInfo)} + if err := source.snapshotSDKPrimaryDirectory(); err != nil { + t.Fatal(err) + } + cloned := newGoOriginalInputGuard(request) + cloned.copySDKPrimaryInputInventory(source.guard) + owned, err := sourcepath.Resolve(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, guard := range []*GoAMD64FrameSource{source.guard, cloned} { + guard.owned = owned + guard.ownedInfo, err = sourcepath.Lstat(owned) + if err != nil { + t.Fatal(err) + } + if err := guard.verifyInputs(); err != nil { + t.Fatal("complete original or copied SDK inputs rejected", err) + } + } + switch change { + case "add": + err = os.WriteFile(filepath.Join(root, "added.s"), nil, 0600) + case "remove": + err = os.Remove(filepath.Join(root, "299.go")) + case "bytes": + err = os.WriteFile(filepath.Join(root, "299.go"), []byte("// changed\n"), 0600) + case "close owner": + err = source.guard.Close() + case "copy inputs": + cloned.inputs[filepath.Join(root, "299.go")] = [32]byte{} + case "copy entries": + cloned.entries[0] = "foreign.go" + case "drop inventory": + cloned.directoryInventory = nil + case "replace root": + original := root + "-original" + err = os.Rename(root, original) + if err == nil { + t.Cleanup(func() { + if err := os.RemoveAll(original); err != nil { + t.Error(err) + } + }) + err = os.Mkdir(root, 0700) + } + } + if err != nil { + t.Fatal(err) + } + for _, guard := range []*GoAMD64FrameSource{source.guard, cloned} { + err := guard.verifyInputs() + copyOnly := change == "copy inputs" || change == "copy entries" || change == "drop inventory" + changed := change != "none" && (!copyOnly || guard == cloned) + if !changed && err != nil || changed && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original/copy complete-input invalidation differs", change, err) + } + } + }) + } +} diff --git a/amd64_go_sdk_frame_native.go b/amd64_go_sdk_frame_native.go new file mode 100644 index 000000000..32bde7502 --- /dev/null +++ b/amd64_go_sdk_frame_native.go @@ -0,0 +1,157 @@ +package plan9asm + +import ( + "bufio" + "bytes" + "context" + "fmt" + "path/filepath" + "reflect" + "strconv" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +type sdkAMD64FrameNativeRecord = goAMD64NativeHeader + +type sdkAMD64FrameNativeObservation struct { + Source string + OriginalListing, MappedListing []byte + Records []sdkAMD64FrameNativeRecord +} + +// Both native listings share one explicit SDK-primary representation bound. +// Ordinary/CPU source, CPP, event and token budgets are unchanged. Charge a +// complete command result before retaining its command or frame projection. +const sdkAMD64FrameListingBytes = 16 << 20 + +func (source *GoX86SDKASMDataSource) retainSDKFrameListingBytes(bytes int) error { + if bytes < 0 || source.frameListingBytes < 0 || bytes > sdkAMD64FrameListingBytes-source.frameListingBytes { + return sdkASMDataContext("SDK-primary native listings exceed their explicit byte bound") + } + source.frameListingBytes += bytes + source.observation.FrameListingBytes = source.frameListingBytes + return nil +} + +// FuncInfo.Locals is int32, but the official listing writer prints uint64. +// Decode its canonical sign extension without interpreting it as an extent. +func goAMD64FrameSignedLocals(raw string) (int64, error) { + bits, err := strconv.ParseUint(raw, 16, 64) + if err != nil { + return 0, err + } + value := int64(int32(bits)) + if uint64(value) != bits { + return 0, fmt.Errorf("noncanonical signed int32 locals: %s", raw) + } + return value, nil +} + +func sdkAMD64FrameNativeRecords(listing []byte, texts []gostaticstorage.TextRecord) ([]sdkAMD64FrameNativeRecord, error) { + return sdkAMD64FrameNativeRecordsInOrder(listing, texts, nil) +} + +func sdkAMD64FrameNativeRecordsForSource(listing []byte, texts []gostaticstorage.TextRecord, order []gostaticstorage.ASMSourceText) ([]sdkAMD64FrameNativeRecord, error) { + if len(order) != len(texts) { + return nil, sdkASMDataContext("SDK-primary native source/object ordinal association is incomplete") + } + indices := make([]int, len(order)) + seen := make(map[int]bool, len(order)) + for ordinal, association := range order { + index := association.ObjectOrdinal + if association.SourceOrdinal != ordinal || index < 0 || index >= len(texts) || seen[index] { + return nil, sdkASMDataContext("SDK-primary native source/object ordinal association is duplicate or unbound") + } + text := texts[index] + if association.Name != text.Name || association.ABI != text.ABI || association.Namespace != text.Namespace || + association.Relative != text.Relative || association.Index != text.Index { + return nil, sdkASMDataContext("SDK-primary native source/object physical association differs") + } + indices[ordinal], seen[index] = index, true + } + return sdkAMD64FrameNativeRecordsInOrder(listing, texts, indices) +} + +// nil indices is the existing format-only diagnostic helper. Production SDK +// capture always supplies the independently source-derived association; no +// original TextRecord is reordered or rewritten to resemble source evidence. +func sdkAMD64FrameNativeRecordsInOrder(listing []byte, texts []gostaticstorage.TextRecord, indices []int) ([]sdkAMD64FrameNativeRecord, error) { + var records []sdkAMD64FrameNativeRecord + scanner := bufio.NewScanner(bytes.NewReader(listing)) + scanner.Buffer(nil, 4<<20) + for scanner.Scan() { + line := scanner.Text() + if !strings.Contains(line, " STEXT ") && !strings.Contains(line, " STEXTFIPS ") { + continue // Complete native objects validate every other record. + } + header, valid := parseGoAMD64NativeHeader(line) + if !valid || len(records) >= len(texts) { + return nil, sdkASMDataContext("SDK-primary native frame header has an unbound role or inventory") + } + index := len(records) + if indices != nil { + index = indices[index] + } + text := texts[index] + if text.Size < 0 || header.Symbol != text.Name || header.Kind != text.Kind || + header.Size != uint64(text.Size) || header.Size != uint64(len(text.Code)) || + header.objectFlags() != text.Flags&0x19 { + return nil, sdkASMDataContext("SDK-primary complete native frame/object TEXT differs") + } + if indices != nil && header.Static != (text.ABI == 65535) { + return nil, sdkASMDataContext("SDK-primary native listing static/ABI source role differs") + } + records = append(records, header) + } + if err := scanner.Err(); err != nil || len(records) != len(texts) { + return nil, sdkASMDataContext("SDK-primary native frame inventory is incomplete: %v", err) + } + return records, nil +} + +func (source *GoX86SDKASMDataSource) captureSDKFrameListings(ctx context.Context) error { + for _, role := range []string{"src/internal/abi/symtab.go", "src/cmd/asm/internal/asm/asm.go", "src/cmd/internal/obj/objfile.go"} { + if _, err := source.guard.read(filepath.Join(source.guard.env["GOROOT"], filepath.FromSlash(role))); err != nil { + return err + } + } + for ordinal, input := range source.observation.ObjectInputs.Sources { + var objects, listings [2][]byte + paths := [2]string{filepath.Join(source.request.Directory, input.Name), filepath.Join(source.guard.owned, fmt.Sprintf("expanded-%d.s", ordinal))} + for role, path := range paths { + object := filepath.Join(source.guard.owned, fmt.Sprintf("frame-%d-%d.o", ordinal, role)) + args := append(source.asmArguments(source.guard.owned), "-S", "-o", object, path) + listing, err := source.run(ctx, args...) + if err != nil { + return err + } + objects[role], err = source.guard.read(object) + if err != nil { + return err + } + if _, err := source.artifact(fmt.Sprintf("frame-%d-%d.listing", ordinal, role), listing); err != nil { + return err + } + listings[role] = listing + } + original, mapped, err := gostaticstorage.ObserveX86SDKASMDataListingObjects(source.observation.ObjectInputs, ordinal, objects[0], objects[1]) + if err != nil { + return sdkASMDataContext("SDK-primary original/mapped/archive complete frame object join: %v", err) + } + order := source.observation.Objects.Sources[ordinal].TextOrder + first, err := sdkAMD64FrameNativeRecordsForSource(listings[0], original, order) + if err != nil { + return err + } + second, err := sdkAMD64FrameNativeRecordsForSource(listings[1], mapped, order) + if err != nil || !reflect.DeepEqual(first, second) { + return sdkASMDataContext("SDK-primary original/mapped complete native frame metadata differs: %v", err) + } + source.observation.Frames = append(source.observation.Frames, sdkAMD64FrameNativeObservation{ + Source: input.Name, OriginalListing: listings[0], MappedListing: listings[1], Records: first, + }) + } + return nil +} diff --git a/amd64_go_sdk_frame_native_go127_test.go b/amd64_go_sdk_frame_native_go127_test.go new file mode 100644 index 000000000..e099fc471 --- /dev/null +++ b/amd64_go_sdk_frame_native_go127_test.go @@ -0,0 +1,204 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestSDKAMD64NativeFrameSignedLocalsActualGoObjects(t *testing.T) { + _, profile, _, _ := numericalSBFixture(t, "amd64") + ctx, cancel := runtimeOracleProofContext(t) + defer cancel() + root := filepath.Dir(filepath.Dir(profile.GoBinary)) + builtin, err := os.ReadFile(filepath.Join(root, "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + t.Fatal(err) + } + const source = "TEXT bootstrap(SB),4,$-8\nRET\nTEXT ·Ordinary(SB),4,$0-0\nRET\n" + path := filepath.Join(profile.Directory, "signed-locals.s") + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + for _, goos := range []string{"linux", "darwin", "windows"} { + t.Run(goos, func(t *testing.T) { + env := append([]string(nil), profile.Environment...) + for index, pair := range env { + if strings.HasPrefix(pair, "GOOS=") { + env[index] = "GOOS=" + goos + } + } + objectPath := filepath.Join(t.TempDir(), "signed-locals.o") + listing, err := goasmcmd.Run(ctx, profile.Directory, env, profile.GoBinary, + "tool", "asm", "-p", "framefixture", "-S", "-o", objectPath, path) + if err != nil { + t.Fatal("actual Go assembler", err) + } + object, err := os.ReadFile(objectPath) + if err != nil { + t.Fatal(err) + } + observed, err := gostaticstorage.ObserveAMD64AssemblyData([]byte(source), object, builtin, "framefixture", goos+"/amd64") + if err != nil || len(observed.Member.Texts) != 2 || len(observed.TextOrder) != 2 { + t.Fatal("complete actual source/object precondition", err) + } + if !bytes.Contains(listing, []byte("locals=0xfffffffffffffff8")) { + t.Fatalf("actual Go did not emit the independent signed-locals prior:\n%s", listing) + } + records, err := sdkAMD64FrameNativeRecordsForSource(listing, observed.Member.Texts, observed.TextOrder) + if err != nil || len(records) != 2 { + t.Fatal("actual signed-locals listing lost its complete original object association", err) + } + if records[0].Locals != ^uint64(7) || records[1].Locals != 0 || records[0].ArgsKnown || !records[1].ArgsKnown { + t.Fatal("signed locals or independent unknown-argument sentinel was rewritten", records) + } + }) + } +} + +func TestSDKAMD64NativeFrameKindsActualGoObjects(t *testing.T) { + _, profile, _, _ := numericalSBFixture(t, "amd64") + ctx, cancel := runtimeOracleProofContext(t) + defer cancel() + root := filepath.Dir(filepath.Dir(profile.GoBinary)) + builtin, err := os.ReadFile(filepath.Join(root, "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + t.Fatal(err) + } + // The encoder chooses the kinds from ordinary source symbols. This is a + // standalone compiler-format fixture, never an SDK provenance certificate. + source := []byte(`TEXT crypto∕internal∕fips140∕framefixture·First(SB),4,$16-8 + MOVQ $11, AX + RET +TEXT ·Ordinary(SB),4,$0-0 + RET +TEXT private<>(SB),4,$8 + RET +TEXT crypto∕internal∕fips140∕framefixture·Last(SB),4,$24-16 + RET +`) + path := filepath.Join(profile.Directory, "mixed.asm") + if err := os.WriteFile(path, source, 0600); err != nil { + t.Fatal(err) + } + for _, goos := range []string{"linux", "darwin", "windows"} { + t.Run(goos, func(t *testing.T) { + env := append([]string(nil), profile.Environment...) + for index, pair := range env { + if strings.HasPrefix(pair, "GOOS=") { + env[index] = "GOOS=" + goos + } + } + objectPath := filepath.Join(t.TempDir(), "mixed.o") + listing, err := goasmcmd.Run(ctx, profile.Directory, env, profile.GoBinary, + "tool", "asm", "-p", "framefixture", "-S", "-o", objectPath, path) + if err != nil { + t.Fatal("actual Go assembler", err) + } + object, err := os.ReadFile(objectPath) + if err != nil { + t.Fatal(err) + } + observed, err := gostaticstorage.ObserveAMD64AssemblyData(source, object, builtin, "framefixture", goos+"/amd64") + if err != nil || len(observed.Member.Texts) != 4 || len(observed.TextOrder) != 4 { + t.Fatal("complete actual source/object precondition", err) + } + kinds := []uint8{2, 1, 1, 2} + for ordinal, entry := range observed.TextOrder { + if observed.Member.Texts[entry.ObjectOrdinal].Kind != kinds[ordinal] { + t.Fatal("actual Go object did not emit the independent mixed-kind prior", ordinal) + } + } + if bytes.Count(listing, []byte(" STEXTFIPS ")) != 2 || bytes.Count(listing, []byte(" STEXT ")) != 2 { + t.Fatal("actual assembler listing did not preserve both TEXT kinds") + } + records, err := sdkAMD64FrameNativeRecordsForSource(listing, observed.Member.Texts, observed.TextOrder) + if err != nil || len(records) != 4 { + t.Fatal("complete actual mixed-kind listing lost native frames", err) + } + if records[0].Locals != 24 || records[0].Args != 8 || records[1].Locals != 0 || + !records[2].Static || records[2].ArgsKnown || records[2].Locals != 16 || records[3].Locals != 32 || records[3].Args != 16 { + t.Fatal("actual ordinary/FIPS/static frame metadata changed", records) + } + }) + } +} + +func TestSDKAMD64NativeFrameFIPSCompleteOriginalSource(t *testing.T) { + // Share only the stock compiler cache. Each source capture retains its own + // complete inputs, command outputs, verification and revocable lifetime. + t.Setenv("GOCACHE", t.TempDir()) + for _, pkg := range []string{ + "crypto/internal/fips140/aes/gcm", + "crypto/internal/fips140/sha256", + "crypto/internal/fips140/sha512", + "crypto/internal/fips140/nistec", + } { + t.Run(pkg, func(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", pkg) + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal("complete original SDK source/listing/object observation", err) + } + defer observed.Close() + source := observed.source + if len(source.observation.Frames) == 0 || len(source.observation.Frames) != len(source.observation.Objects.Sources) { + t.Fatal("native frame source inventory differs") + } + var frames, fips int + for ordinal, frame := range source.observation.Frames { + object := source.observation.Objects.Sources[ordinal] + if len(frame.Records) != len(object.Original.Texts) { + t.Fatal("native frame TEXT inventory differs", frame.Source) + } + frames += len(frame.Records) + for _, text := range object.Original.Texts { + if text.Kind == 2 { + fips++ + } + } + } + if fips == 0 || frames == 0 { + t.Fatal("actual original FIPS source precondition absent") + } + if err := observed.Verify(ctx); err != nil { + t.Fatal("live original source verification", err) + } + t.Logf("complete native frames=%d FIPS=%d; observation only, not translation or execution", frames, fips) + sdkASMDataRetain(t, source) + }) + } +} + +func TestSDKAMD64ReadonlyLiteralFIPSCompleteOriginalSource(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "crypto/internal/fips140/sha512") + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal("complete original SDK source/object observation", err) + } + defer observed.Close() + images, err := gostaticstorage.ReadonlyLiteralDataInitializers(observed.source.observation.Objects) + if err != nil { + t.Fatal("actual FIPS readonly literal definitions", err) + } + fips := 0 + for _, image := range images { + if image.Definition.Kind == 4 { + fips++ + } + } + if fips == 0 { + t.Fatal("actual Go object did not supply the FIPS readonly prior") + } + if err := observed.Verify(ctx); err != nil { + t.Fatal("live original source verification", err) + } + t.Logf("%d literal images, %d FIPS; source/object bytes, not program execution", len(images), fips) +} diff --git a/amd64_go_sdk_frame_native_test.go b/amd64_go_sdk_frame_native_test.go new file mode 100644 index 000000000..dc9dc9330 --- /dev/null +++ b/amd64_go_sdk_frame_native_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestSDKAMD64NativeFrameSignedLocalsRepresentation(t *testing.T) { + for _, locals := range []int32{0, 1, 8, 0x7fffffff, -1, -8, -0x80000000} { + for _, kind := range []uint8{1, 2} { + t.Run(fmt.Sprintf("locals-%d/kind-%d", locals, kind), func(t *testing.T) { + label := "STEXT" + if kind == 2 { + label = "STEXTFIPS" + } + // The official writer prints uint64(FuncInfo.Locals), whose + // source type is int32. Preserve that exact representation. + bits := uint64(int64(locals)) + listing := []byte(fmt.Sprintf("fixture.Entry %s nosplit asm size=1 args=0x0 locals=0x%x\n", label, bits)) + texts := []gostaticstorage.TextRecord{{Name: "fixture.Entry", Kind: kind, Flags: 0x10, Size: 1, Code: []byte{0xc3}}} + records, err := sdkAMD64FrameNativeRecords(listing, texts) + if err != nil || len(records) != 1 || records[0].Locals != bits { + t.Fatalf("canonical signed int32 locals lost their original metadata bits: records=%v error=%v", records, err) + } + if records[0].Symbol != texts[0].Name || !records[0].NoSplit || !records[0].ASM || !records[0].ArgsKnown { + t.Fatal("signed locals changed an independent native frame role", records[0]) + } + }) + } + } +} + +func TestSDKAMD64NativeFrameRejectsNoncanonicalLocals(t *testing.T) { + texts := []gostaticstorage.TextRecord{{Name: "fixture.Entry", Kind: 1, Flags: 0x10, Size: 1, Code: []byte{0xc3}}} + for _, locals := range []string{"80000000", "ffffffff", "100000000", "ffffffff7fffffff", "10000000000000000", "nothex", ""} { + t.Run("locals-"+locals, func(t *testing.T) { + listing := []byte("fixture.Entry STEXT nosplit asm size=1 args=0x0 locals=0x" + locals + "\n") + if _, err := sdkAMD64FrameNativeRecords(listing, texts); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("noncanonical or out-of-range signed locals acquired native metadata", locals, err) + } + }) + } +} + +func TestSDKAMD64OriginalFrameNativeCompleteMetadata(t *testing.T) { + texts := []gostaticstorage.TextRecord{{Name: "private", Kind: 1, Flags: 0x10, Size: 1, Code: []byte{0xc3}}, {Name: "sdk.Entry", Kind: 1, Size: 1, Code: []byte{0xc3}}} + listing := []byte("private STEXT static nosplit asm size=1 align=0x0 args=0xffffffff80000000 locals=0x0 funcid=0x0\n" + + "sdk.Entry STEXT asm size=1 align=0x0 args=0x61 locals=0x128 funcid=0x0\n") + records, err := sdkAMD64FrameNativeRecords(listing, texts) + if err != nil || len(records) != 2 || !records[0].Static || !records[0].NoSplit || records[0].ArgsKnown || + records[0].Args != -0x80000000 || records[1].NoSplit || !records[1].ArgsKnown || records[1].Args != 97 || records[1].Locals != 296 { + t.Fatal("private unknown args or stack-split native frame was disguised as a declared leaf", records, err) + } + for _, raw := range []string{ + "sdk.Entry STEXT asm size=1 args=0x61 locals=0x128\n", + "private STEXT static nosplit asm size=1 args=0xffffffff80000000 locals=0x0\n", + "private STEXT static nosplit asm size=2 args=0xffffffff80000000 locals=0x0\n", + "private STEXT foreign size=1 args=0x0 locals=0x0\n", + string(listing) + string(listing), + } { + if _, err := sdkAMD64FrameNativeRecords([]byte(raw), texts); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("partial, reordered, foreign or duplicate native frame inventory survived", raw, err) + } + } +} + +func TestSDKAMD64NativeFrameTextKindsKeepExactObjectIdentity(t *testing.T) { + for _, kinds := range [][]uint8{{1, 1}, {2, 2}, {1, 2}, {2, 1}} { + t.Run(fmt.Sprint(kinds), func(t *testing.T) { + var listing strings.Builder + var texts []gostaticstorage.TextRecord + for index, kind := range kinds { + name := fmt.Sprintf("fixture.Entry%d", index) + label := "STEXT" + if kind == 2 { + label = "STEXTFIPS" + } + fmt.Fprintf(&listing, "%s %s nosplit asm size=1 align=0x0 args=0x0 locals=0x0 funcid=0x0\n", name, label) + texts = append(texts, gostaticstorage.TextRecord{Name: name, Kind: kind, Flags: 0x10, Size: 1, Code: []byte{0xc3}}) + } + raw := []byte(listing.String()) + records, err := sdkAMD64FrameNativeRecords(raw, texts) + if err != nil || len(records) != len(texts) { + t.Fatal("complete ordinary/FIPS TEXT listing lost native frames", records, err) + } + for index, record := range records { + if record.Symbol != texts[index].Name || !record.NoSplit || !record.ASM || !record.ArgsKnown { + t.Fatal("native frame identity or attributes changed", record) + } + for _, wrongKind := range []uint8{0, 3 - texts[index].Kind, 3} { + changed := append([]gostaticstorage.TextRecord(nil), texts...) + changed[index].Kind = wrongKind + if _, err := sdkAMD64FrameNativeRecords(raw, changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("listing borrowed a different object TEXT/storage kind", index, wrongKind, err) + } + } + } + for _, changed := range []string{ + strings.SplitAfter(listing.String(), "\n")[0], + listing.String() + listing.String(), + strings.Replace(listing.String(), " nosplit", " foreign", 1), + strings.Replace(listing.String(), " asm", " dupok asm", 1), + strings.Replace(listing.String(), "STEXT", "STEXTFOREIGN", 1), + } { + if _, err := sdkAMD64FrameNativeRecords([]byte(changed), texts); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("incomplete, duplicate or unobserved listing acquired native frames", changed, err) + } + } + }) + } +} + +func TestSDKAMD64OriginalFrameNativeListingByteBoundary(t *testing.T) { + source := &GoX86SDKASMDataSource{} + if err := source.retainSDKFrameListingBytes(sdkAMD64FrameListingBytes); err != nil { + t.Fatal("exact SDK-primary listing byte boundary", err) + } + for _, added := range []int{1, -1} { + if err := source.retainSDKFrameListingBytes(added); !errors.Is(err, ErrProbeNeedsContext) || + source.frameListingBytes != sdkAMD64FrameListingBytes { + t.Fatal("oversized native listing changed its retained accounting", added, err) + } + } +} diff --git a/amd64_go_sdk_frame_observation.go b/amd64_go_sdk_frame_observation.go new file mode 100644 index 000000000..72dd5c415 --- /dev/null +++ b/amd64_go_sdk_frame_observation.go @@ -0,0 +1,52 @@ +package plan9asm + +import "context" + +// GoAMD64SDKFrameObservation is a query-only complete SDK-primary source, +// default/config profile and native frame observation. It retains all selected +// Go/ASM sources and native TEXTs, including private and stack-splitting bodies. +// It grants no local backing, ABI transport, caller, loop, DATA read or extent +// permission. Ordinary owned-main and numerical factories remain distinct. +type GoAMD64SDKFrameObservation struct { + identity *GoAMD64SDKFrameObservation + source *GoX86SDKASMDataSource +} + +// CaptureGoAMD64SDKFrameObservation consumes the original producer environment, +// including an actual bounded GOENV file (or off) and the observed cgo default. +// No config, proxy, feature profile or compiler input is replaced for capture. +func CaptureGoAMD64SDKFrameObservation(ctx context.Context, request GoX86SDKASMDataCaptureRequest) (*GoAMD64SDKFrameObservation, error) { + source, err := captureGoX86SDKASMDataSource(ctx, request, true) + if err != nil { + return nil, err + } + observed := &GoAMD64SDKFrameObservation{source: source} + observed.identity = observed + return observed, nil +} + +func (observed *GoAMD64SDKFrameObservation) Verify(ctx context.Context) error { + if observed == nil || observed.identity != observed || observed.source == nil { + return sdkASMDataContext("missing or copied SDK-primary frame observation") + } + return observed.source.Verify(ctx) +} + +// Readback returns diagnostics only. Serialized observations cannot construct +// a live source or a frame capability. +func (observed *GoAMD64SDKFrameObservation) Readback(ctx context.Context) ([]byte, error) { + if observed == nil || observed.identity != observed || observed.source == nil { + return nil, sdkASMDataContext("missing or copied SDK-primary frame observation") + } + return observed.source.Readback(ctx) +} + +func (observed *GoAMD64SDKFrameObservation) Close() error { + if observed == nil { + return nil + } + if observed.identity != observed || observed.source == nil { + return sdkASMDataContext("copied SDK-primary observation cannot close its original source") + } + return observed.source.Close() +} diff --git a/amd64_go_sdk_frame_observation_test.go b/amd64_go_sdk_frame_observation_test.go new file mode 100644 index 000000000..ae1ea4a4a --- /dev/null +++ b/amd64_go_sdk_frame_observation_test.go @@ -0,0 +1,24 @@ +package plan9asm + +import ( + "context" + "errors" + "testing" +) + +func TestSDKAMD64OriginalFrameObservationMissingLifetime(t *testing.T) { + for _, observed := range []*GoAMD64SDKFrameObservation{nil, {}} { + if err := observed.Verify(context.Background()); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing observation acquired a verification role", err) + } + if data, err := observed.Readback(context.Background()); !errors.Is(err, ErrProbeNeedsContext) || data != nil { + t.Fatal("missing observation published a source/frame readback", err) + } + } + if err := (*GoAMD64SDKFrameObservation)(nil).Close(); err != nil { + t.Fatal("nil Close is not harmless", err) + } + if err := (&GoAMD64SDKFrameObservation{}).Close(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized observer acquired cleanup ownership", err) + } +} diff --git a/amd64_go_sdk_frame_profile.go b/amd64_go_sdk_frame_profile.go new file mode 100644 index 000000000..66b827f63 --- /dev/null +++ b/amd64_go_sdk_frame_profile.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "crypto/sha256" + "errors" + "io" + "os" + "path/filepath" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +const sdkAMD64FrameConfigBytes = 64 << 10 + +func (source *GoX86SDKASMDataSource) captureSDKFrameConfig(path string) error { + resolved, err := sourcepath.Resolve(path) + before, statErr := sourcepath.Lstat(path) + if err != nil || !filepath.IsAbs(path) || resolved != path || statErr != nil || + !before.Mode().IsRegular() || before.Size() > sdkAMD64FrameConfigBytes { + return sdkASMDataContext("SDK-primary GOENV is not one bounded canonical original config") + } + file, err := os.Open(path) + if err != nil { + return err + } + opened, err := file.Stat() + if err != nil || !opened.Mode().IsRegular() || !os.SameFile(before, opened) || opened.Size() != before.Size() { + return errors.Join(sdkASMDataContext("SDK-primary GOENV role changed while opening"), file.Close()) + } + data, readErr := io.ReadAll(io.LimitReader(file, sdkAMD64FrameConfigBytes+1)) + closeErr := file.Close() + after, statErr := sourcepath.Lstat(path) + resolved, resolveErr := sourcepath.Resolve(path) + if err := errors.Join(readErr, closeErr); err != nil || statErr != nil || resolveErr != nil || resolved != path || + len(data) > sdkAMD64FrameConfigBytes || int64(len(data)) != opened.Size() || !after.Mode().IsRegular() || + after.Size() != opened.Size() || !os.SameFile(opened, after) { + return sdkASMDataContext("SDK-primary bounded GOENV changed while reading: %v", err) + } + source.guard.inputs[path] = sha256.Sum256(data) + source.originalConfig, source.originalConfigID = path, before + return nil +} + +// The original SDK-primary role consumes the exact producer environment. It +// never substitutes a config, proxy, cgo default, cache or CPU profile. Complete +// actual SDK selection and compiler-source/export/object joins remain required. +// This profile is private to the distinct SDK frame factory: ordinary owned +// sources, public ASM DATA and numerical producers keep their existing rules. +func sdkAMD64FrameEnvironment(request GoX86SDKASMDataCaptureRequest) (map[string]string, error) { + if request.Target != "linux/amd64" && request.Target != "darwin/amd64" && request.Target != "windows/amd64" { + return nil, sdkASMDataContext("SDK-primary frame profile requires an original AMD64 target") + } + return sdkX86OriginalSourceEnvironment(request) +} + +// Original source observation does not select a native frame/entry role. The +// AMD64 factory above retains its separate exact target check. +func sdkX86OriginalSourceEnvironment(request GoX86SDKASMDataCaptureRequest) (map[string]string, error) { + if !gostaticstorage.X86SDKTarget(request.Target) { + return nil, sdkASMDataContext("SDK-primary source profile requires an original x86 target") + } + provided := make(map[string]string) + for _, pair := range request.Environment { + key, value, valid := strings.Cut(pair, "=") + if _, duplicate := provided[key]; !valid || key == "" || duplicate || strings.ContainsRune(pair, '\x00') { + return nil, sdkASMDataContext("SDK-primary malformed or duplicate actual command environment") + } + provided[key] = value + } + goos, arch, _ := strings.Cut(request.Target, "/") + for key, value := range map[string]string{"GOOS": goos, "GOARCH": arch, "GOFLAGS": "", "GOTOOLCHAIN": "local", "GOWORK": "off"} { + if actual, present := provided[key]; !present || actual != value { + return nil, sdkASMDataContext("SDK-primary unbound actual command profile: %s", key) + } + } + if provided["CGO_ENABLED"] != "0" && provided["CGO_ENABLED"] != "1" { + return nil, sdkASMDataContext("SDK-primary needs the actual explicit compiler cgo default") + } + if provided["GOENV"] != "off" && !filepath.IsAbs(provided["GOENV"]) { + return nil, sdkASMDataContext("SDK-primary requires the actual resolved GOENV or explicit off") + } + if provided["GOEXPERIMENT"] != "" || provided["GOPACKAGESDRIVER"] != "" && provided["GOPACKAGESDRIVER"] != "off" { + return nil, sdkASMDataContext("SDK-primary custom experiment or package driver needs a separate role") + } + return provided, nil +} diff --git a/amd64_go_sdk_frame_profile_go127_test.go b/amd64_go_sdk_frame_profile_go127_test.go new file mode 100644 index 000000000..5a46451c8 --- /dev/null +++ b/amd64_go_sdk_frame_profile_go127_test.go @@ -0,0 +1,127 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +func TestSDKAMD64OriginalFrameProfileActualCompleteSource(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "vendor/golang.org/x/crypto/chacha20poly1305") + config := filepath.Join(t.TempDir(), "go-env") + // This is a separate config producer input, never a replacement SDK source. + if err := os.WriteFile(config, []byte("GOPROXY=https://original-config.invalid,direct\n"), 0600); err != nil { + t.Fatal(err) + } + config, err := sourcepath.Resolve(config) + if err != nil { + t.Fatal(err) + } + var env []string + for _, pair := range request.Environment { + key, _, _ := strings.Cut(pair, "=") + if key != "GOENV" && key != "GOPROXY" && key != "CGO_ENABLED" && key != "GOAMD64" && key != "GOEXPERIMENT" { + env = append(env, pair) + } + } + env = append(env, "GOENV="+config) + raw, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, "env", "-json", "CGO_ENABLED") + var defaults map[string]string + if err != nil || json.Unmarshal(raw, &defaults) != nil || defaults["CGO_ENABLED"] != "0" && defaults["CGO_ENABLED"] != "1" { + t.Fatal("actual target compiler cgo default", err) + } + request.Environment = append(env, "CGO_ENABLED="+defaults["CGO_ENABLED"]) + original := append([]string(nil), request.Environment...) + public, err := CaptureGoX86SDKASMDataSource(ctx, request) + if public != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("new role leaked into public DATA source", err) + } + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal("actual unchanged SDK source/profile/header/object observation", err) + } + defer observed.Close() + source := observed.source + if source.guard.env["GOENV"] != config || source.guard.env["CGO_ENABLED"] != defaults["CGO_ENABLED"] || + source.guard.env["GOPROXY"] != "https://original-config.invalid,direct" || + strings.Join(source.request.Environment, "\x00") != strings.Join(original, "\x00") || + len(source.observation.Objects.Sources) != 1 || len(source.observation.Objects.Sources[0].Original.Texts) != 3 { + t.Fatal("SDK-primary source/default profile or complete three TEXT inventory changed") + } + frames := source.observation.Frames + if len(frames) != 1 || len(frames[0].Records) != 3 || frames[0].Records[0].ArgsKnown || + frames[0].Records[0].Args != -0x80000000 || !frames[0].Records[0].Static || !frames[0].Records[0].NoSplit || + frames[0].Records[1].NoSplit || frames[0].Records[1].Args != 97 || frames[0].Records[1].Locals != 296 || + frames[0].Records[2].NoSplit || frames[0].Records[2].Args != 96 || frames[0].Records[2].Locals != 296 { + t.Fatal("complete original private helper/stack-split native metadata was omitted or relabeled", frames) + } + t.Logf("native listing bytes %d/%d; actual target CGO default %s; complete 3 TEXT frames %+v", + source.frameListingBytes, sdkAMD64FrameListingBytes, defaults["CGO_ENABLED"], frames[0].Records) + copy := *observed + if err := copy.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("copied SDK-primary observation acquired original lifetime", err) + } + if data, err := copy.Readback(ctx); !errors.Is(err, ErrProbeNeedsContext) || data != nil { + t.Fatal("copied SDK-primary observation published original source/frame readback", err) + } + if err := copy.Close(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("copied SDK-primary observation removed original artifacts", err) + } + if err := source.Verify(ctx); err != nil { + t.Fatal("live original config/profile", err) + } + before, err := os.ReadFile(config) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(config, bytes.ReplaceAll(before, []byte("original-config"), []byte("replaced-config")), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed original config retained SDK-primary source", err) + } + if _, err := source.Readback(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("failed config verification did not revoke SDK-primary source", err) + } +} + +func TestSDKAMD64OriginalFrameProfileRejectsEqualByteConfigReplacement(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "internal/cpu") + config := filepath.Join(t.TempDir(), "go-env") + bytes := []byte("GOPROXY=off\n") + if err := os.WriteFile(config, bytes, 0600); err != nil { + t.Fatal(err) + } + config, err := sourcepath.Resolve(config) + if err != nil { + t.Fatal(err) + } + for index, pair := range request.Environment { + if strings.HasPrefix(pair, "GOENV=") { + request.Environment[index] = "GOENV=" + config + } + } + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal("actual original SDK source/config", err) + } + defer observed.Close() + if err := os.Rename(config, config+".original"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(config, bytes, 0600); err != nil { + t.Fatal(err) + } + if err := observed.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("equal-byte replacement acquired lazily captured original config identity", err) + } +} diff --git a/amd64_go_sdk_frame_profile_test.go b/amd64_go_sdk_frame_profile_test.go new file mode 100644 index 000000000..ff1577bfa --- /dev/null +++ b/amd64_go_sdk_frame_profile_test.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/sourcepath" +) + +func TestSDKAMD64OriginalFrameProfilePreservesDefaults(t *testing.T) { + request := GoX86SDKASMDataCaptureRequest{Target: "linux/amd64", Environment: []string{ + "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=1", "GOENV=" + filepath.Join(t.TempDir(), "env"), + "GOFLAGS=", "GOTOOLCHAIN=local", "GOWORK=off", + }} + original := append([]string(nil), request.Environment...) + actual, err := sdkAMD64FrameEnvironment(request) + if err != nil || actual["CGO_ENABLED"] != "1" || actual["GOENV"] == "off" || + !reflect.DeepEqual(request.Environment, original) { + t.Fatal("SDK-primary profile replaced the original default/config", err) + } + if _, present := actual["GOPROXY"]; present { + t.Fatal("SDK-primary profile invented a proxy override") + } + if _, present := actual["GOAMD64"]; present { + t.Fatal("SDK-primary profile invented a CPU default override") + } + if _, err := sdkASMDataEnvironment(request); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("SDK-primary request weakened the existing public DATA profile", err) + } +} + +func TestSDKAMD64OriginalFrameConfigExactByteBoundary(t *testing.T) { + for _, size := range []int{sdkAMD64FrameConfigBytes, sdkAMD64FrameConfigBytes + 1} { + path := filepath.Join(t.TempDir(), "env") + if err := os.WriteFile(path, []byte(strings.Repeat("#", size)), 0600); err != nil { + t.Fatal(err) + } + path, err := sourcepath.Resolve(path) + if err != nil { + t.Fatal(err) + } + source := &GoX86SDKASMDataSource{guard: newGoOriginalInputGuard(GoAMD64FrameCaptureRequest{})} + err = source.captureSDKFrameConfig(path) + if size == sdkAMD64FrameConfigBytes { + if err != nil || len(source.guard.inputs) != 1 { + t.Fatal("exact original config bound was rejected", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || len(source.guard.inputs) != 0 || source.originalConfigID != nil { + t.Fatal("oversized original config was retained", err) + } + } +} + +func TestSDKAMD64OriginalFrameConfigGuardEagerIdentity(t *testing.T) { + path := filepath.Join(t.TempDir(), "env") + data := []byte("GOPROXY=off\n") + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + path, err := sourcepath.Resolve(path) + if err != nil { + t.Fatal(err) + } + source := &GoX86SDKASMDataSource{guard: newGoOriginalInputGuard(GoAMD64FrameCaptureRequest{})} + if err := source.captureSDKFrameConfig(path); err != nil { + t.Fatal(err) + } + if err := source.verifyDirectories(); err != nil { + t.Fatal("unchanged eager config identity", err) + } + // Capture retained an open-handle ID. Byte equality cannot substitute for + // that identity when the next verification observes a new file at the path. + if err := os.Rename(path, path+".original"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + if err := source.verifyDirectories(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("equal-byte replacement acquired retained original config identity", err) + } +} + +func TestSDKAMD64OriginalFrameProfileRejectsUnboundRoles(t *testing.T) { + for _, bad := range []string{"GOENV=", "GOENV=relative", "CGO_ENABLED=", "CGO_ENABLED=2", "GOFLAGS=-overlay=foreign.json", + "GOTOOLCHAIN=auto", "GOWORK=workspace", "GOEXPERIMENT=foreign", "GOPACKAGESDRIVER=foreign"} { + request := GoX86SDKASMDataCaptureRequest{Target: "linux/amd64", Environment: []string{ + "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOENV=off", "GOFLAGS=", "GOTOOLCHAIN=local", "GOWORK=off", + }} + key, _, _ := strings.Cut(bad, "=") + var env []string + for _, pair := range request.Environment { + if !strings.HasPrefix(pair, key+"=") { + env = append(env, pair) + } + } + request.Environment = append(env, bad) + if _, err := sdkAMD64FrameEnvironment(request); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("foreign or duplicate profile acquired SDK-primary source", bad, err) + } + } +} diff --git a/amd64_go_sdk_frame_source.go b/amd64_go_sdk_frame_source.go new file mode 100644 index 000000000..a64341226 --- /dev/null +++ b/amd64_go_sdk_frame_source.go @@ -0,0 +1,278 @@ +package plan9asm + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "path/filepath" + "reflect" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +type goAMD64SDKFrameOrigin struct { + identity *GoAMD64FrameSource + owner *GoAMD64SDKFrameObservation + pkg GoPackage + request GoAMD64FrameCaptureRequest +} + +// CaptureGoAMD64SDKFrameSource is a distinct actual SDK-primary typed ABI0 +// native frame factory. It authenticates the whole selected source/profile, +// compiler graph and all TEXTs, without relabeling SDK source as owned-main. +// Entries must be Go-declared with exact logical/native names, layout and +// source PCs. Same-source frame-unobserving leaf helpers retain no Go signature. +// Nonzero frames also require automatic BP/SP shapes; +// zero-frame siblings retain their ordinary ABI0 path without local backing. +// Generated split checks are observed separately from the source body. +// Native callers and ABI selectors require separate contracts. BindFixedLocal +// also accounts for the observed private leaf's automatic BP/FLAGS effects and +// proves every full-source access/initialization/CFG requirement before granting +// local bytes. +func CaptureGoAMD64SDKFrameSource(ctx context.Context, pkg GoPackage, request GoAMD64FrameCaptureRequest) (_ *GoAMD64FrameSource, result error) { + if pkg.Types == nil || pkg.Path != pkg.Types.Path() || request.NumericalReads != nil || + filepath.Base(request.AssemblyFile) != request.AssemblyFile || filepath.Ext(request.AssemblyFile) != ".s" { + return nil, goAMD64FrameContext("SDK-primary requires its exact complete package/ABI0 source role") + } + owner, err := CaptureGoAMD64SDKFrameObservation(ctx, GoX86SDKASMDataCaptureRequest{ + GoBinary: request.GoBinary, Directory: request.Directory, PackagePath: pkg.Path, + Target: request.Target, Environment: request.Environment, + }) + if err != nil { + return nil, err + } + source := newGoOriginalInputGuard(request) + source.sdkFrame = &goAMD64SDKFrameOrigin{identity: source, owner: owner, pkg: pkg, request: source.request} + source.sdkFrame.request.Environment = append([]string(nil), source.request.Environment...) + defer func() { + if result != nil { + result = errors.Join(result, source.Close()) + } + }() + data := owner.source + data.operation.Lock() + // Clone complete compiler state: the SDK owner's verification may update + // its private guard maps, and a frame must neither alias nor mutate them. + state := struct { + Selection goAMD64FrameSelection + Env map[string]string + Imports map[string]goAMD64FrameSelection + }{data.guard.selection, data.guard.env, data.guard.imports} + raw, err := json.Marshal(state) + if err == nil { + state.Selection, state.Env, state.Imports = goAMD64FrameSelection{}, nil, nil + err = json.Unmarshal(raw, &state) + // RawMessage's nil marshals as null but decodes as literal "null". + // Preserve the actual absent error projection rather than relabeling + // an independently observed source selection during this private copy. + state.Selection.Error = append(json.RawMessage(nil), data.guard.selection.Error...) + for path, copied := range state.Imports { + copied.Error = append(json.RawMessage(nil), data.guard.imports[path].Error...) + state.Imports[path] = copied + } + } + if err == nil { + source.selection, source.env, source.imports = state.Selection, state.Env, state.Imports + source.copySDKPrimaryInputInventory(data.guard) + source.owned, source.ownedInfo = data.guard.owned, data.guard.ownedInfo + source.functions, source.declarations, err = source.sdkFrameRecords() + } + data.operation.Unlock() + if err != nil { + return nil, err + } + source.ready = true + if err := source.Verify(ctx); err != nil { + return nil, err + } + return source, nil +} + +// Caller holds the SDK owner's operation lock. Full original/mapped/listing +// objects are already independently joined to the actual compiler archive. +// Reusing their native code is not permission to execute a helper or caller. +func (source *GoAMD64FrameSource) sdkFrameRecords() (map[string]goAMD64FrameFunction, map[string]goAMD64FrameDeclaration, error) { + origin := source.sdkFrame + data := origin.owner.source + declarations, err := source.typedDeclarations(origin.pkg, false) + if err != nil { + return nil, nil, err + } + functions := make(map[string]goAMD64FrameFunction) + found := false + withFrame := false + for ordinal, input := range data.observation.ObjectInputs.Sources { + found = found || input.Name == source.request.AssemblyFile + if ordinal >= len(data.observation.CPP) || ordinal >= len(data.observation.Frames) || ordinal >= len(data.observation.Objects.Sources) { + return nil, nil, goAMD64FrameContext("SDK-primary complete native frame observation is missing") + } + cpp, native := data.observation.CPP[ordinal], data.observation.Frames[ordinal] + original, err := source.read(filepath.Join(source.request.Directory, input.Name)) + if err != nil || !bytes.Equal(original, input.Original) || native.Source != input.Name { + return nil, nil, goAMD64FrameContext("SDK-primary original assembly identity changed: %v", err) + } + file, err := ParseWithDefines(ArchAMD64, cpp.Output, nil) + object := data.observation.Objects.Sources[ordinal] + texts := object.Original.Texts + if err != nil || len(file.Funcs) == 0 || len(file.UnlinkedPrelude) != 0 || len(file.Funcs) != len(native.Records) || len(file.Funcs) != len(texts) { + return nil, nil, goAMD64FrameContext("SDK-primary complete parsed/native TEXT inventory differs: %v", err) + } + leaves, err := goAMD64FramePrivateLeaves(file, origin.pkg) + if err != nil { + return nil, nil, err + } + for index, fn := range file.Funcs { + name := source.selection.ImportPath + "." + strings.TrimPrefix(fn.Sym, "·") + _, declared := declarations[name] + if index >= len(object.TextOrder) || object.TextOrder[index].SourceOrdinal != index || + object.TextOrder[index].ObjectOrdinal < 0 || object.TextOrder[index].ObjectOrdinal >= len(texts) { + return nil, nil, goAMD64FrameContext("SDK-primary complete source/TEXT association differs") + } + record, text := native.Records[index], texts[object.TextOrder[index].ObjectOrdinal] + if !record.ordinaryFrameRole() { + return nil, nil, goAMD64FrameContext("SDK-primary native header has no ordinary/private frame entry role: %s", fn.Sym) + } + if leaves[fn.Sym] { + if declared || record.Symbol != text.Name || record.Locals != uint64(goAMD64FrameNativeLocals(fn.FrameSize)) || !record.NoSplit || + record.ArgsKnown && record.Args != 0 || record.Static != strings.HasSuffix(fn.Sym, "<>") { + return nil, nil, goAMD64FrameContext("SDK-primary private leaf/native identity differs: %s", fn.Sym) + } + continue // All original bytes/PCs/relocations are still consumed below. + } + // Real -std assembler objects retain qualified ABI0 declarations + // in the non-package definition block. Its role is authenticated by + // the complete same-source/compiler archive join, not by the name. + defined := text.Namespace == gostaticstorage.TextSymbolSelf || text.Namespace == gostaticstorage.TextSymbolNonpkgDefinition + if !declared || strings.Contains(fn.Sym, "<") || record.Static || !record.ArgsKnown || + !defined || text.ABI != 0 || name != record.Symbol || name != text.Name { + return nil, nil, goAMD64FrameContext("SDK-primary frame requires an exact Go-declared logical ABI0 entry; private/native helper has no role: %s (declared=%t native=%s namespace=%d ABI=%d static=%t args-known=%t)", fn.Sym, declared, record.Symbol, text.Namespace, text.ABI, record.Static, record.ArgsKnown) + } + if fn.FrameSize < 0 || fn.FrameSize > 64<<10 || fn.FrameSize%8 != 0 || record.Locals != uint64(goAMD64FrameNativeLocals(fn.FrameSize)) || record.Args != fn.ArgSize { + return nil, nil, goAMD64FrameContext("SDK-primary original declaration/native fixed-frame layout differs: %s", fn.Sym) + } + withFrame = withFrame || fn.FrameSize > 0 + } + for _, observed := range cpp.Sources { + expected, captured := source.inputs[observed.File] + bytes, err := goAMD64FrameRead(observed.File) + if err != nil || !captured || sha256.Sum256(bytes) != expected { + return nil, nil, goAMD64FrameContext("SDK-primary native source PC lacks its original physical input: %v", err) + } + source.origins[observed.File] = strings.Count(string(bytes), "\n") + 1 + } + var observed map[string]goAMD64FrameFunction + if len(file.Data)+len(file.Globl) != 0 || bytes.Contains(native.OriginalListing, []byte("\trel ")) { + data, readErr := source.fixedLocalDataObservation(input.Name, cpp.Output) + if readErr != nil { + return nil, nil, readErr + } + listing := native.OriginalListing + if len(data.Layouts) != 0 { + listing, readErr = goAMD64FrameDataListing(listing, data) + if readErr != nil { + return nil, nil, readErr + } + } + observed, err = goAMD64FrameListingWithDataObject(listing, file, origin.pkg, source.origins, data, nil, + source.request.Target, source.request.Directory, source.env["GOROOT"]) + if err == nil { + err = goAMD64FrameDataTextJoin(observed, data, source.request.Directory, source.origins, source.env["GOROOT"]) + } + } else if goAMD64FrameSplitListing(native.OriginalListing) { + observed, err = goAMD64FrameListingWithSplitObject(native.OriginalListing, file, origin.pkg, source.origins, texts, nil, + source.request.Target, source.request.Directory, source.env["GOROOT"]) + } else { + observed, err = goAMD64FrameListingWithNumericalObject(native.OriginalListing, file, origin.pkg, source.origins, texts) + } + if err != nil { + return nil, nil, err + } + for name, fn := range observed { + declaration, declared := declarations[name] + if _, duplicate := functions[name]; duplicate || fn.privateLeaf && declared || + !fn.privateLeaf && (!declared || !reflect.DeepEqual(fn.signature, declaration.signature)) { + return nil, nil, goAMD64FrameContext("SDK-primary complete native/declaration frame identity differs") + } + fn.transport, fn.languageType = declaration.transport, declaration.languageType + functions[name] = fn + } + } + if !found { + return nil, nil, goAMD64FrameContext("SDK-primary requested assembly is absent from complete selection") + } + if !withFrame { + return nil, nil, goAMD64FrameContext("SDK-primary complete source has no nonzero native frame") + } + return functions, declarations, nil +} + +func (source *GoAMD64FrameSource) verifySDKFrame(ctx context.Context) (result error) { + origin := source.sdkFrame + if origin == nil || origin.identity != source || origin.owner == nil || origin.owner.source == nil { + return goAMD64FrameContext("copied or missing SDK-primary native frame owner") + } + defer func() { + if result != nil { + source.mu.Lock() + source.ready = false + source.mu.Unlock() + } + }() + // SDK Verify samples caller contexts outside all frame/operation locks. + if err := origin.owner.Verify(ctx); err != nil { + return err + } + data := origin.owner.source + data.operation.Lock() + source.mu.Lock() + err := func() error { + if !source.ready || !reflect.DeepEqual(source.request, origin.request) || + !reflect.DeepEqual(source.selection, data.guard.selection) || !reflect.DeepEqual(source.env, data.guard.env) || + !reflect.DeepEqual(source.imports, data.guard.imports) || !reflect.DeepEqual(source.inputs, data.guard.inputs) || + source.directoryInventory != data.guard.directoryInventory { + return goAMD64FrameContext("SDK-primary native frame source/profile/complete input identity changed") + } + if err := source.verifyInputs(); err != nil { + return err + } + functions, declarations, err := source.sdkFrameRecords() + if err != nil || !reflect.DeepEqual(functions, source.functions) || !reflect.DeepEqual(declarations, source.declarations) { + return goAMD64FrameContext("SDK-primary original typed/native frame witness changed: %v", err) + } + return nil + }() + source.mu.Unlock() + data.operation.Unlock() + if err != nil { + return err + } + return origin.owner.Verify(ctx) +} + +func (source *GoAMD64FrameSource) closeSDKFrame() error { + source.mu.Lock() + origin := source.sdkFrame + if origin == nil || origin.identity != source || origin.owner == nil { + source.mu.Unlock() + return goAMD64FrameContext("copied SDK-primary frame cannot close its original owner") + } + source.ready = false + source.mu.Unlock() + return origin.owner.Close() +} + +// Called during the existing locked full-file binder. This immutable program +// was authenticated in capture/Verify; the outer postcheck rejects revocation +// or resolver-induced source changes before publishing any signature proof. +func (source *GoAMD64FrameSource) preprocessSDKFrame(name string, original []byte) (string, error) { + data := source.sdkFrame.owner.source + for ordinal, input := range data.observation.ObjectInputs.Sources { + if input.Name == name && bytes.Equal(input.Original, original) && ordinal < len(data.observation.CPP) { + return data.observation.CPP[ordinal].Output, nil + } + } + return "", goAMD64FrameContext("SDK-primary fixed-local source lacks its complete original CPP identity") +} diff --git a/amd64_go_sdk_frame_source_go127_test.go b/amd64_go_sdk_frame_source_go127_test.go new file mode 100644 index 000000000..3ec4f5d83 --- /dev/null +++ b/amd64_go_sdk_frame_source_go127_test.go @@ -0,0 +1,81 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func sdkAMD64FrameActualPackage(t *testing.T, source *GoX86SDKASMDataSource) GoPackage { + t.Helper() + fset := token.NewFileSet() + var syntax []*ast.File + for _, name := range source.guard.selection.GoFiles { + file, err := parser.ParseFile(fset, filepath.Join(source.request.Directory, name), nil, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + syntax = append(syntax, file) + } + actualImports := importer.ForCompiler(fset, "gc", func(path string) (io.ReadCloser, error) { + if mapped, found := source.guard.selection.ImportMap[path]; found { + path = mapped + } + dependency, found := source.guard.imports[path] + if !found || dependency.Export == "" { + return nil, fmt.Errorf("missing actual SDK export: %s", path) + } + return os.Open(dependency.Export) + }) + typed, err := (&types.Config{Sizes: types.SizesFor("gc", "amd64"), Importer: actualImports, GoVersion: "go1.27"}).Check(source.request.PackagePath, fset, syntax, nil) + if err != nil { + t.Fatal("independent complete original SDK Go package", err) + } + return GoPackage{Path: typed.Path(), Types: typed, Syntax: syntax} +} + +func TestGoAMD64FrameOriginalSDKSourceObservation(t *testing.T) { + ctx, request := sdkASMDataRequest(t, "amd64", "vendor/golang.org/x/crypto/chacha20poly1305") + observed, err := CaptureGoAMD64SDKFrameObservation(ctx, request) + if err != nil { + t.Fatal("complete actual SDK source/header/CPP/native frame observation", err) + } + defer observed.Close() + data := observed.source + const assembly = "chacha20poly1305_amd64.s" + pkg := sdkAMD64FrameActualPackage(t, data) + declarations, err := data.guard.typedDeclarations(pkg, false) + if err != nil || len(declarations) != 2 || len(data.observation.Frames[0].Records) != 3 { + t.Fatal("private helper invented a Go declaration or complete Go/ASM scope was cropped", err) + } + if err := observed.Verify(ctx); err != nil { + t.Fatal("live complete SDK-primary source", err) + } + frame, err := CaptureGoAMD64FrameSource(ctx, pkg, GoAMD64FrameCaptureRequest{ + GoBinary: request.GoBinary, Directory: request.Directory, AssemblyFile: assembly, + Target: request.Target, Environment: request.Environment, + }) + if frame != nil { + frame.Close() + } + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "SDK/cache") { + t.Fatal("query-only SDK observation acquired an ordinary owned-main frame role", err) + } + input := data.observation.ObjectInputs.Sources[0] + actual, err := os.ReadFile(filepath.Join(request.Directory, assembly)) + if err != nil || !bytes.Equal(actual, input.Original) { + t.Fatal("SDK role manufactured or rewrote a source fixture", err) + } +} diff --git a/amd64_go_sdk_frame_source_order_test.go b/amd64_go_sdk_frame_source_order_test.go new file mode 100644 index 000000000..90f8f8ff6 --- /dev/null +++ b/amd64_go_sdk_frame_source_order_test.go @@ -0,0 +1,56 @@ +package plan9asm + +import ( + "bytes" + "errors" + "reflect" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestSDKAMD64NativeFrameMixedNamespaceSourceOrder(t *testing.T) { + member := gostaticstorage.ASMDataMember{Flags: 4, Texts: []gostaticstorage.TextRecord{ + {Name: "private", ABI: 65535, Namespace: gostaticstorage.TextSymbolSelf, Kind: 1, Flags: 0x10, Size: 1, Code: []byte{0xc3}}, + {Name: "foreign.Entry", ABI: 1, Namespace: gostaticstorage.TextSymbolNonpkgDefinition, Index: 10, Kind: 1, Flags: 0x10, Size: 1, Code: []byte{0xc3}}, + {Name: "fixture.Own", Namespace: gostaticstorage.TextSymbolNonpkgDefinition, Index: 11, Relative: 1, Kind: 1, Flags: 0x10, Size: 1, Code: []byte{0xc3}}, + }} + source := []byte("TEXT foreign·Entry(SB),4,$0-0\nRET\nTEXT private<>(SB),4,$0-0\nRET\nTEXT ·Own(SB),4,$0-0\nRET\n") + order, err := gostaticstorage.ASMSourceTextOrder(source, "fixture", member) + if err != nil || len(order) != 3 { + t.Fatal("complete physical source association", err) + } + listing := []byte("foreign.Entry STEXT nosplit asm size=1 args=0x0 locals=0x0\nprivate STEXT static nosplit asm size=1 args=0x0 locals=0x0\nfixture.Own STEXT nosplit asm size=1 args=0x0 locals=0x0\n") + before := append([]gostaticstorage.TextRecord(nil), member.Texts...) + records, err := sdkAMD64FrameNativeRecordsForSource(listing, member.Texts, order) + if err != nil || len(records) != 3 { + t.Fatal("source-order listing must join the unchanged physical object", err) + } + if records[0].Symbol != "foreign.Entry" || records[1].Symbol != "private" || !records[1].Static || + records[2].Symbol != "fixture.Own" || !reflect.DeepEqual(before, member.Texts) { + t.Fatal("native metadata lost source order or rewrote original evidence") + } + for _, mutate := range []func([]gostaticstorage.ASMSourceText){ + func(m []gostaticstorage.ASMSourceText) { m[0].SourceOrdinal++ }, + func(m []gostaticstorage.ASMSourceText) { m[0].ObjectOrdinal = m[1].ObjectOrdinal }, + func(m []gostaticstorage.ASMSourceText) { m[0].ObjectOrdinal = 3 }, + func(m []gostaticstorage.ASMSourceText) { m[0].Name += "foreign" }, + func(m []gostaticstorage.ASMSourceText) { m[0].ABI = 0 }, + func(m []gostaticstorage.ASMSourceText) { m[0].Namespace = gostaticstorage.TextSymbolSelf }, + func(m []gostaticstorage.ASMSourceText) { m[0].Index++ }, + func(m []gostaticstorage.ASMSourceText) { m[0].Relative++ }, + } { + changed := append([]gostaticstorage.ASMSourceText(nil), order...) + mutate(changed) + if _, err := sdkAMD64FrameNativeRecordsForSource(listing, member.Texts, changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unclosed/duplicate physical mapping became native source metadata", err) + } + } + if _, err := sdkAMD64FrameNativeRecordsForSource(listing, member.Texts, order[:2]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing source ordinal became native metadata", err) + } + wrongStatic := bytes.Replace(listing, []byte(" STEXT static"), []byte(" STEXT"), 1) + if _, err := sdkAMD64FrameNativeRecordsForSource(wrongStatic, member.Texts, order); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("static object acquired a named listing role", err) + } +} diff --git a/amd64_go_virtual_arguments.go b/amd64_go_virtual_arguments.go new file mode 100644 index 000000000..80adbfa36 --- /dev/null +++ b/amd64_go_virtual_arguments.go @@ -0,0 +1,271 @@ +package plan9asm + +import ( + "crypto/sha256" + "go/types" + "reflect" + "strings" +) + +// This call-local issuer binds LLVM values to virtual Go registers. It is not +// the stock compiler/archive issuer and supplies no native entry, stack home, +// closure, pointee extent or caller capability. Closing the GoModule call also +// closes every returned signature's private argument role. +type amd64GoVirtualArguments struct { + pkg GoPackage + sizes, frameSizes types.Sizes + syntax []string + imports map[string]*types.Package + file *File + fingerprint [32]byte + input []byte + inputSHA [32]byte + entries map[string]*amd64GoArgumentBinding + symbols map[string]string + resolve func(string) string + active bool + failure error +} + +func amd64VirtualArgumentDeclaration(pkg GoPackage, fn Func) *types.Func { + symbol := goStripABISuffix(fn.Sym) + name := goReadOnlyLocalName(pkg, symbol) + if name == "" { + name, _ = goDeclNameForSymbol(symbol, goLinknameRemoteToLocal(pkg.Syntax)) + } + declaration, _ := pkg.Types.Scope().Lookup(name).(*types.Func) + return declaration +} + +func amd64VirtualArgumentsNeeded(fn *types.Func) bool { + if fn == nil { + return false + } + sig, ok := fn.Type().(*types.Signature) + if !ok || sig.Params().Len() > 9 { + return true + } + for index := 0; index < sig.Params().Len(); index++ { + switch typ := sig.Params().At(index).Type().Underlying().(type) { + case *types.Basic: + if typ.Info()&types.IsInteger == 0 && typ.Kind() != types.UnsafePointer { + return true + } + case *types.Pointer, *types.Map, *types.Chan: + default: + return true + } + } + return false +} + +func bindAMD64GoVirtualArguments(pkg GoPackage, file *File, input []byte, inputSHA [32]byte, + opt GoModuleOptions, resolve func(string) string, +) (*amd64GoVirtualArguments, error) { + if opt.GOARCH != "amd64" || opt.NumericalReads != nil || opt.X86TLSMoves != nil || opt.ARM64ByteCarrierSource != nil { + return nil, nil + } + declarations := make(map[string]*types.Func) + for _, fn := range file.Funcs { + if strings.HasSuffix(fn.Sym, "") { + if declaration := amd64VirtualArgumentDeclaration(pkg, fn); amd64VirtualArgumentsNeeded(declaration) { + declarations[fn.Sym] = declaration + } + } + } + if len(declarations) == 0 { + return nil, nil + } + owner := &amd64GoVirtualArguments{pkg: pkg, sizes: opt.Sizes, frameSizes: opt.FrameSizes, + input: input, inputSHA: inputSHA, file: x86DirectTailSnapshot(file), resolve: resolve, active: true, + entries: make(map[string]*amd64GoArgumentBinding), symbols: make(map[string]string), imports: make(map[string]*types.Package)} + if owner.sizes == nil { + owner.sizes = types.SizesFor("gc", "amd64") + } + if owner.frameSizes == nil { + owner.frameSizes = owner.sizes + } + if !x86CallerTargetSizesValid(owner.sizes) || !x86CallerTargetSizesValid(owner.frameSizes) { + return nil, amd64GoArgumentContext("virtual arguments need the actual AMD64 target value/frame widths") + } + var err error + owner.syntax, err = goReadOnlySyntax(pkg) + if err != nil { + return nil, err + } + for path, dependency := range pkg.Imports { + actual := false + for _, imported := range pkg.Types.Imports() { + actual = actual || imported == dependency + } + if dependency == nil || dependency.Path() != path || !dependency.Complete() || !actual { + return nil, amd64GoArgumentContext("virtual declaration import is not the actual complete typed dependency") + } + owner.imports[path] = dependency + } + typed, err := goReadOnlyTypedSyntax(pkg, owner.syntax, owner.sizes, nil) + if err != nil { + return nil, err + } + owner.fingerprint, err = x86CallerSourceFingerprint(owner.file) + if err != nil { + return nil, err + } + for _, fn := range file.Funcs { + name := resolve(goTextSymbolForResolution(fn.Sym)) + owner.symbols[goTextSymbolForResolution(fn.Sym)] = name + declaration := declarations[fn.Sym] + if declaration == nil { + continue + } + actual, _ := typed.Scope().Lookup(declaration.Name()).(*types.Func) + if actual == nil || name == "" || fn.FrameSize != 0 || len(fn.X86RawText) != 0 || owner.entries[name] != nil { + return nil, amd64GoArgumentContext("virtual entry lacks one complete declaration and zero-frame source") + } + signature, err := goFuncSigForDeclaredFunc(name, declaration, "amd64", owner.sizes, owner.frameSizes, true) + if err != nil { + return nil, err + } + rechecked, err := goFuncSigForDeclaredFunc(name, actual, "amd64", owner.sizes, owner.frameSizes, true) + if err != nil || !reflect.DeepEqual(signature, rechecked) { + return nil, amd64GoArgumentContext("virtual signature differs from actual declaration syntax") + } + switch signature.Ret { + case I1, I8, I16, I32, I64: + default: + return nil, amd64GoArgumentContext("virtual parameters require their independent scalar result contract") + } + plan, err := planAMD64GoArguments(declaration.Type().(*types.Signature)) + if err != nil { + return nil, err + } + if err := plan.requireRegisters(); err != nil { + return nil, err + } + entry := &amd64GoArgumentBinding{virtual: owner, declaration: declaration, symbol: fn.Sym, signature: signature, plan: plan} + entry.identity = entry + entry.seal, err = entry.digest() + if err != nil { + return nil, err + } + owner.entries[name] = entry + } + if err := owner.verify(); err != nil { + return nil, err + } + return owner, nil +} + +// Internal lowering consumes only the private plan/source copies. This check +// observes call lifetime and permanent revocation, not a cached successful +// source query. Complete attach/audit/publication boundaries call verify. +func (owner *amd64GoVirtualArguments) live() error { + if owner == nil || !owner.active { + return amd64GoArgumentContext("virtual argument call has ended") + } + return owner.failure +} + +// Returned signatures keep only a closed tombstone, not the call's complete +// source/type graph or callbacks. Clearing private entries does not alter the +// separate public signature values or the caller-owned LLVM module. +func (owner *amd64GoVirtualArguments) close() { + if owner == nil { + return + } + owner.active = false + for _, entry := range owner.entries { + *entry = amd64GoArgumentBinding{identity: entry, virtual: owner} + } + *owner = amd64GoVirtualArguments{} +} + +func (owner *amd64GoVirtualArguments) verify() (result error) { + if err := owner.live(); err != nil { + return err + } + defer func() { + if result != nil { + owner.failure = result + } + }() + if err := owner.verifySource(); err != nil { + return err + } + for _, entry := range owner.entries { + actual, err := goFuncSigForDeclaredFunc(entry.signature.Name, entry.declaration, "amd64", owner.sizes, owner.frameSizes, true) + if err != nil || !reflect.DeepEqual(actual, entry.signature) { + return amd64GoArgumentContext("virtual declaration transport changed during consumption") + } + } + // Sizes is caller supplied and may execute code even at this final query. + // Recheck afterward using only concrete syntax/source/private-plan data. + return owner.verifySource() +} + +func (owner *amd64GoVirtualArguments) verifySource() error { + fingerprint, err := x86CallerSourceFingerprint(owner.file) + syntax, syntaxErr := goReadOnlySyntax(owner.pkg) + if err != nil || syntaxErr != nil || fingerprint != owner.fingerprint || + sha256.Sum256(owner.input) != owner.inputSHA || !reflect.DeepEqual(syntax, owner.syntax) || len(owner.imports) != len(owner.pkg.Imports) { + return amd64GoArgumentContext("virtual argument source changed during consumption") + } + for path, dependency := range owner.imports { + if owner.pkg.Imports[path] != dependency { + return amd64GoArgumentContext("virtual declaration import changed during consumption") + } + } + for _, entry := range owner.entries { + if owner.pkg.Types.Scope().Lookup(entry.declaration.Name()) != entry.declaration || !entry.matches(entry.signature) { + return amd64GoArgumentContext("virtual declaration identity changed during consumption") + } + } + return nil +} + +func (owner *amd64GoVirtualArguments) resolveSymbol(symbol string) string { + actual := owner.resolve(symbol) + if expected, known := owner.symbols[goTextSymbolForResolution(symbol)]; known && actual != expected { + owner.failure = amd64GoArgumentContext("virtual entry resolver changed during consumption") + } + return actual +} + +func (owner *amd64GoVirtualArguments) attach(signatures map[string]FuncSig) error { + if owner == nil { + return nil + } + if err := owner.verify(); err != nil { + return err + } + for name, entry := range owner.entries { + sig, present := signatures[name] + if !present || !reflect.DeepEqual(numericalSignatureShape(sig), entry.signature) || sig.amd64GoArguments != nil && sig.amd64GoArguments != entry { + return amd64GoArgumentContext("virtual entry lost its complete actual signature") + } + sig.amd64GoArguments = entry + signatures[name] = sig + } + return nil +} + +func (owner *amd64GoVirtualArguments) audit(file *File, signatures map[string]FuncSig, opt GoModuleOptions, resolve func(string) string, storage *goReadOnlyStorage) error { + if owner == nil || file.x86DirectTails != nil { + return nil // The complete direct-tail producer already audited every root. + } + for _, fn := range file.Funcs { + sig := signatures[resolve(goTextSymbolForResolution(fn.Sym))] + if sig.amd64GoArguments == nil || sig.amd64GoArguments.virtual != owner { + continue + } + c := newX86Ctx(new(strings.Builder), fn, sig, resolve, signatures, "amd64", opt.TargetTriple, false) + c.goReadOnlyStorage = storage + c.fpScalarData = x86LoweredScalarDataReads(file, resolve, signatures, "amd64") + c.fpStaticAddressValues = x86LoweredStaticAddressValues(file, resolve, signatures, "amd64") + c.fpShiftLiterals = x86LoweredShiftLiterals(file, resolve, c.fpStaticAddressValues, "amd64") + if err := c.scalarFPResultInitialized(); err != nil { + return err + } + } + return owner.verify() +} diff --git a/amd64_go_virtual_arguments_execution_test.go b/amd64_go_virtual_arguments_execution_test.go new file mode 100644 index 000000000..7bb84e3b1 --- /dev/null +++ b/amd64_go_virtual_arguments_execution_test.go @@ -0,0 +1,220 @@ +package plan9asm + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/nativeoracle" +) + +const amd64VirtualArgumentValueDeclarations = amd64VirtualArgumentsDeclarations + ` +func SliceLength(a []byte) uint64 +func SliceCapacity(a []byte) uint64 +func SecondSliceLength(a, b []byte) uint64 +func SecondSliceCapacity(a, b []byte) uint64 +func StringLength(a string) uint64 +` + +const amd64VirtualArgumentValueSource = amd64VirtualArgumentsSource + ` +TEXT ·SliceLength(SB),4,$0 + MOVQ BX, AX + RET + +TEXT ·SliceCapacity(SB),4,$0 + MOVQ CX, AX + RET + +TEXT ·SecondSliceLength(SB),4,$0 + MOVQ SI, AX + RET + +TEXT ·SecondSliceCapacity(SB),4,$0 + MOVQ R8, AX + RET + +TEXT ·StringLength(SB),4,$0 + MOVQ BX, AX + RET +` + +// The scalar C ABI shims construct the actual grouped LLVM arguments. They +// append independent callers to the unchanged translated definitions; they +// neither flatten/rewrite those definitions nor claim stock-Go native entry. +func amd64VirtualArgumentScalarShims() string { + var result strings.Builder + for _, row := range []struct { + name, entry string + arguments int + slice bool + }{ + {"virtual_compare_slices", "CompareSlices", 2, true}, + {"virtual_compare_strings", "CompareStrings", 2, false}, + {"virtual_slice_length", "SliceLength", 1, true}, + {"virtual_slice_capacity", "SliceCapacity", 1, true}, + {"virtual_second_length", "SecondSliceLength", 2, true}, + {"virtual_second_capacity", "SecondSliceCapacity", 2, true}, + {"virtual_string_length", "StringLength", 1, false}, + } { + typ, fields := "{ ptr, i64 }", 2 + if row.slice { + typ, fields = "{ ptr, i64, i64 }", 3 + } + var parameters, arguments []string + for argument := 0; argument < row.arguments; argument++ { + parameters = append(parameters, fmt.Sprintf("ptr %%p%d, i64 %%n%d", argument, argument)) + if row.slice { + parameters = append(parameters, fmt.Sprintf("i64 %%c%d", argument)) + } + arguments = append(arguments, fmt.Sprintf("%s %%a%d_%d", typ, argument, fields-1)) + } + fmt.Fprintf(&result, "\ndefine i64 @%s(%s) {\n", row.name, strings.Join(parameters, ", ")) + for argument := 0; argument < row.arguments; argument++ { + fmt.Fprintf(&result, " %%a%d_0 = insertvalue %s poison, ptr %%p%d, 0\n", argument, typ, argument) + fmt.Fprintf(&result, " %%a%d_1 = insertvalue %s %%a%d_0, i64 %%n%d, 1\n", argument, typ, argument, argument) + if row.slice { + fmt.Fprintf(&result, " %%a%d_2 = insertvalue %s %%a%d_1, i64 %%c%d, 2\n", argument, typ, argument, argument) + } + } + fmt.Fprintf(&result, " %%value = call i64 @%q(%s)\n ret i64 %%value\n}\n", + amd64VirtualArgumentsPackage+"."+row.entry, strings.Join(arguments, ", ")) + } + return result.String() +} + +func amd64VirtualArgumentValueIR(t *testing.T, triple, api string, preprocessed bool) string { + t.Helper() + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, amd64VirtualArgumentValueDeclarations) + ir, err := amd64VirtualArgumentsRequest(t, pkg, []byte(amd64VirtualArgumentValueSource), triple, api, preprocessed, nil) + if err != nil { + t.Fatal("complete original virtual argument source", err) + } + if strings.Count(ir, "define ") != 7 { + t.Fatal("the complete seven public entries were cropped or a private node became a callable entry", ir) + } + for _, name := range []string{"CompareSlices", "CompareStrings", "SliceLength", "SliceCapacity", "SecondSliceLength", "SecondSliceCapacity", "StringLength"} { + if !strings.Contains(ir, "define i64 @"+fmt.Sprintf("%q", pkg.Path+"."+name)+"(") { + t.Fatal("original grouped entry missing", name, ir) + } + } + return ir +} + +func TestGoModuleAMD64VirtualArgumentsCompleteObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("complete virtual argument objects require LLVM22 llc") + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + for _, api := range []string{"public", "owned-context"} { + t.Run(triple+"/"+api, func(t *testing.T) { + for _, preprocessed := range []bool{false, true} { + ir := amd64VirtualArgumentValueIR(t, triple, api, preprocessed) + amd64VirtualArgumentScalarShims() + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86UndefinedTrapOptimizedIR(t, input) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, input, level) + } + } + } + }) + } + } +} + +func TestGoModuleAMD64VirtualArgumentsPortableValues(t *testing.T) { + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("portable virtual argument execution requires LLVM22 llc/clang") + } + // This runs numerical/opaque-pointer transport on the actual LLVM host. + // Large descriptor lengths/capacities test word-width transport only: no + // memory is read, and they are not claims about live Go heap allocations. + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + compiler := []string{clang} + if runtime.GOOS == "darwin" { + ctx, cancel := runtimeOracleStageContext(t) + flags, err := nativeoracle.DarwinClangFlags(ctx, os.Environ()) + cancel() + if err != nil { + t.Fatal(err) + } + compiler = append(compiler, flags...) + } + const driver = `#include +#include + +extern int64_t virtual_compare_slices(const void *, uint64_t, uint64_t, + const void *, uint64_t, uint64_t); +extern int64_t virtual_compare_strings(const void *, uint64_t, + const void *, uint64_t); +extern uint64_t virtual_slice_length(const void *, uint64_t, uint64_t); +extern uint64_t virtual_slice_capacity(const void *, uint64_t, uint64_t); +extern uint64_t virtual_second_length(const void *, uint64_t, uint64_t, + const void *, uint64_t, uint64_t); +extern uint64_t virtual_second_capacity(const void *, uint64_t, uint64_t, + const void *, uint64_t, uint64_t); +extern uint64_t virtual_string_length(const void *, uint64_t); + +int main(void) { + const uint64_t values[] = { + 0, 1, 63, 64, UINT64_C(0x7fffffff), UINT64_C(0xffffffff), + UINT64_C(0x100000000), INT64_MAX + }; + volatile uint8_t canaries[128]; + for (unsigned i = 0; i < sizeof(canaries); ++i) canaries[i] = (uint8_t)(3*i+7); + const void *a = (const void *)&canaries[1]; + const void *b = (const void *)&canaries[63]; + if (virtual_compare_slices(NULL,0,0,NULL,0,0) != 0 || + virtual_compare_strings(NULL,0,NULL,0) != 0 || + virtual_slice_length(NULL,0,0) != 0 || + virtual_slice_capacity(NULL,0,0) != 0 || + virtual_string_length(NULL,0) != 0) return 3; + for (unsigned i = 0; i < sizeof(values)/sizeof(values[0]); ++i) { + for (unsigned j = 0; j < sizeof(values)/sizeof(values[0]); ++j) { + uint64_t al = values[i], bl = values[j]; + uint64_t ac = al + (al < INT64_MAX); + uint64_t bc = bl < INT64_MAX-3 ? bl+3 : bl; + int64_t want = (al > bl) - (al < bl); + if (virtual_compare_slices(a,al,ac,b,bl,bc) != want || + virtual_compare_strings(a,al,b,bl) != want || + virtual_slice_length(a,al,ac) != al || + virtual_slice_capacity(a,al,ac) != ac || + virtual_second_length(a,al,ac,b,bl,bc) != bl || + virtual_second_capacity(a,al,ac,b,bl,bc) != bc || + virtual_string_length(a,al) != al) { + fprintf(stderr,"virtual parameter mismatch i=%u j=%u\n",i,j); + return 1; + } + for (unsigned k = 0; k < sizeof(canaries); ++k) { + if (canaries[k] != (uint8_t)(3*k+7)) { + fprintf(stderr,"pointer transport wrote caller memory\n"); + return 2; + } + } + } + } + return 0; +} +` + for _, api := range []string{"public", "owned-context"} { + ir := amd64VirtualArgumentValueIR(t, triple, api, true) + amd64VirtualArgumentScalarShims() + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86UndefinedTrapOptimizedIR(t, input) + } + for _, level := range []int{0, 2} { + t.Run(fmt.Sprintf("%s/IR_O2=%t/backendO%d", api, optimized, level), func(t *testing.T) { + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, + "virtual-go-arguments", triple, input, driver, nil, level) + }) + } + } + } +} diff --git a/amd64_go_virtual_arguments_test.go b/amd64_go_virtual_arguments_test.go new file mode 100644 index 000000000..d0e399ecb --- /dev/null +++ b/amd64_go_virtual_arguments_test.go @@ -0,0 +1,429 @@ +package plan9asm + +import ( + "crypto/sha256" + "errors" + "fmt" + "go/ast" + "go/types" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +const amd64VirtualArgumentsPackage = "example.com/virtualargs" + +const amd64VirtualArgumentsDeclarations = `package virtualargs +func CompareSlices(a, b []byte) int +func CompareStrings(a, b string) int +` + +// Two independent public declarations share the complete private continuation. +// The fixture observes numerical lengths only: pointer identity transport must +// not be mistaken for permission to dereference, cast or expose those pointers. +const amd64VirtualArgumentsSource = ` +TEXT ·CompareSlices(SB),4,$0-56 + MOVQ SI, DX + MOVQ AX, SI + JMP compareLengths<>(SB) + +TEXT ·CompareStrings(SB),4,$0-40 + MOVQ AX, SI + MOVQ DI, DX + MOVQ CX, DI + JMP compareLengths<>(SB) + +TEXT compareLengths<>(SB),4,$0-0 + CMPQ BX, DX + JEQ equal + JLT less + MOVQ $1, AX + RET +less: + MOVQ $-1, AX + RET +equal: + XORQ AX, AX + RET +` + +func amd64VirtualArgumentsRequest(t *testing.T, pkg GoPackage, source []byte, triple, api string, preprocessed bool, + configure func(*GoModuleOptions), +) (string, error) { + t.Helper() + // This helper only translates supplied declarations in the current process. + // It captures no stock-Go SDK, compiler archive, source lease or native ABI. + return importedGlobalReadRequest(t, pkg, source, "amd64", triple, api, preprocessed, configure) +} + +func amd64VirtualArgumentsRequireContext(t *testing.T, ir string, err error) { + t.Helper() + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("virtual language arguments granted unrelated state: output=%t err=%v", ir != "", err) + } +} + +func TestGoModuleAMD64VirtualArgumentsCompleteContinuation(t *testing.T) { + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + for _, api := range []string{"public", "owned-context"} { + for _, preprocessed := range []bool{false, true} { + for _, linkedMain := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/preprocessed=%t/main=%t", triple, api, preprocessed, linkedMain), func(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, amd64VirtualArgumentsDeclarations) + if linkedMain { + pkg.Path = "main" // Logical linked namespace, not Types.Path(). + } + ir, err := amd64VirtualArgumentsRequest(t, pkg, []byte(amd64VirtualArgumentsSource), triple, api, preprocessed, nil) + if err != nil || ir == "" { + t.Fatal("actual loaded Go declarations did not initialize their virtual ABIInternal arguments", err) + } + for _, fragment := range []string{ + "extractvalue { ptr, i64, i64 } %arg0, 1", + "extractvalue { ptr, i64, i64 } %arg1, 1", + "extractvalue { ptr, i64 } %arg0, 1", + "extractvalue { ptr, i64 } %arg1, 1", + } { + if !strings.Contains(ir, fragment) { + t.Fatal("grouped LLVM parameters did not feed the original register leaves", fragment, ir) + } + } + for _, name := range []string{"CompareSlices", "CompareStrings"} { + found := false + for _, line := range strings.Split(ir, "\n") { + if strings.HasPrefix(line, "define i64 ") && strings.Contains(line, pkg.Path+"."+name) { + found = strings.Count(line, "{ ptr, i64") == 2 + } + } + if !found { + t.Fatal("public grouped Go signature was flattened, renamed or omitted", name, ir) + } + } + if strings.Contains(ir, " nest ") || strings.Contains(ir, " swiftself ") || strings.Contains(ir, " noalias ") { + t.Fatal("ordinary virtual parameters acquired hidden/native alias authority", ir) + } + }) + } + } + } + } +} + +func TestGoModuleAMD64VirtualArgumentsDeclaredWidths(t *testing.T) { + for _, row := range []struct{ name, parameters, body, extraction string }{ + {"slice-length", "a []byte", "MOVQ BX,AX", "extractvalue { ptr, i64, i64 } %arg0, 1"}, + {"slice-capacity", "a []byte", "MOVQ CX,AX", "extractvalue { ptr, i64, i64 } %arg0, 2"}, + {"second-slice-length", "a,b []byte", "MOVQ SI,AX", "extractvalue { ptr, i64, i64 } %arg1, 1"}, + {"second-slice-capacity", "a,b []byte", "MOVQ R8,AX", "extractvalue { ptr, i64, i64 } %arg1, 2"}, + {"string-length", "a string", "MOVQ BX,AX", "extractvalue { ptr, i64 } %arg0, 1"}, + {"mixed-string-byte", "a string,b byte", "MOVBQZX CL,AX", ""}, + {"mixed-string-bool", "a string,b bool", "MOVBQZX CL,AX", ""}, + {"mixed-string-float32", "a string,b float32", "MOVL X0,AX", ""}, + {"mixed-string-float64", "a string,b float64", "MOVQ X0,AX", ""}, + } { + t.Run(row.name, func(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, "package virtualargs\nfunc Entry("+row.parameters+") uint64\n") + source := []byte("TEXT ·Entry(SB),4,$0\n" + row.body + "\nRET\n") + ir, err := amd64VirtualArgumentsRequest(t, pkg, source, "x86_64-unknown-linux-gnu", "owned-context", true, nil) + if err != nil || ir == "" || row.extraction != "" && !strings.Contains(ir, row.extraction) { + t.Fatal("the original grouped parameter leaf was not transported at its own width", err, ir) + } + }) + } +} + +func TestGoModuleAMD64VirtualArgumentsDoNotGrantOtherState(t *testing.T) { + for _, row := range []struct{ name, parameters, body string }{ + {"unused-GP", "a string", "MOVQ R11,AX"}, + {"unused-vector", "a string", "MOVQ X0,AX"}, + {"hidden-context", "a string", "MOVQ 8(DX),AX"}, + {"stack", "a string", "MOVQ (SP),AX"}, + {"ABI0-home", "a string", "MOVQ a+0(FP),AX"}, + {"unknown-FLAGS", "a string", "SETEQ AL\nMOVBQZX AL,AX"}, + {"pointer-not-NUM", "a string", "MOVQ AX,X0\nMOVQ X0,AX"}, + {"byte-upper", "a string,b byte", "MOVQ CX,AX"}, + {"bool-upper", "a string,b bool", "MOVQ CX,AX"}, + {"float32-upper", "a string,b float32", "MOVQ X0,AX"}, + {"whole-array-spill", "a [2]uint64", "MOVQ AX,AX"}, + {"bank-overflow-spill", "a,b,c,d,e,f,g,h,i,j uint64", "MOVQ AX,AX"}, + } { + t.Run(row.name, func(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, "package virtualargs\nfunc Entry("+row.parameters+") uint64\n") + source := []byte("TEXT ·Entry(SB),4,$0\n" + row.body + "\nRET\n") + ir, err := amd64VirtualArgumentsRequest(t, pkg, source, "x86_64-unknown-linux-gnu", "owned-context", true, nil) + amd64VirtualArgumentsRequireContext(t, ir, err) + }) + } +} + +func TestGoModuleAMD64VirtualArgumentsNeedActualDeclarations(t *testing.T) { + for _, mutation := range []string{"missing-syntax", "missing-function", "wrong-parameter", "wrong-result", "other-package"} { + t.Run(mutation, func(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, "package virtualargs\nfunc Entry(a []byte) uint64\n") + declarations := map[string]string{ + "missing-function": "package virtualargs\nfunc Another(a []byte) uint64\n", + "wrong-parameter": "package virtualargs\nfunc Entry(a uint64) uint64\n", + "wrong-result": "package virtualargs\nfunc Entry(a []byte) uint32\n", + "other-package": "package foreign\nfunc Entry(a []byte) uint64\n", + } + pkg.Syntax = nil + if replacement := declarations[mutation]; replacement != "" { + pkg.Syntax = mustGoPackage(t, amd64VirtualArgumentsPackage, replacement).Syntax + } + source := []byte("TEXT ·Entry(SB),4,$0\nMOVQ BX,AX\nRET\n") + ir, err := amd64VirtualArgumentsRequest(t, pkg, source, "x86_64-unknown-linux-gnu", "owned-context", true, nil) + amd64VirtualArgumentsRequireContext(t, ir, err) + }) + } +} + +func TestGoModuleAMD64VirtualArgumentsCannotLendToRawAPI(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, "package virtualargs\nfunc Entry(a []byte) uint64\n") + // Raw APIs still accept their existing low-level register model. This + // independent scalar/SIMD observation invokes its complete byte proof, + // where a grouped shape alone cannot initialize BX as a slice length. + source := "TEXT ·Entry(SB),4,$0\nMOVQ BX,AX\nMOVQ AX,X15\nMOVQ X15,AX\nRET\n" + for _, api := range []string{"text", "module"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewPackageAPI(t, source, pkg, "amd64", "x86_64-unknown-linux-gnu", api) + amd64VirtualArgumentsRequireContext(t, ir, err) + }) + } + t.Run("returned-signatures", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + resolve := testResolveSym(pkg.Path) + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", SourcePreprocessed: true, ResolveSym: resolve, + }) + if err != nil || translated == nil { + t.Fatal("complete GoModule source failed before the borrowing control", err) + } + defer translated.Module.Dispose() + if err := llvm.VerifyModule(translated.Module, llvm.ReturnStatusAction); err != nil { + t.Fatal("invalid virtual argument module", err) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: resolve, Sigs: translated.Signatures}) + amd64VirtualArgumentsRequireContext(t, ir, err) + }) +} + +func TestGoModuleAMD64VirtualArgumentsRecheckCallbacks(t *testing.T) { + for _, mutation := range []string{"control", "syntax", "source", "resolver"} { + t.Run(mutation, func(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, "package virtualargs\nfunc Entry(a []byte) uint64\n") + source := []byte("TEXT ·Entry(SB),4,$0\nMOVQ BX,AX\nRET\n") + called := false + ir, err := amd64VirtualArgumentsRequest(t, pkg, source, "x86_64-unknown-linux-gnu", "owned-context", true, func(options *GoModuleOptions) { + resolve := options.ResolveSym + options.ResolveSym = func(symbol string) string { + if mutation == "resolver" && called { + return "foreign.Entry" + } + return resolve(symbol) + } + options.KeepFunc = func(_, _ string) bool { + called = true + if mutation == "syntax" { + ast.Inspect(pkg.Syntax[0], func(node ast.Node) bool { + if ident, ok := node.(*ast.Ident); ok && ident.Name == "byte" { + ident.Name = "uint64" + } + return true + }) + } else if mutation == "source" { + source[len(source)-1] = ' ' + } + return true // No source or public entry is filtered out. + } + }) + if !called { + t.Fatal("the real source callback was not reached") + } + if mutation == "control" { + if err != nil || ir == "" { + t.Fatal("unmodified virtual argument source failed", err) + } + } else { + amd64VirtualArgumentsRequireContext(t, ir, err) + } + }) + } +} + +type amd64VirtualArgumentCountingSizes struct { + types.Sizes + queries int + onQuery func() +} + +func (sizes *amd64VirtualArgumentCountingSizes) Sizeof(typ types.Type) int64 { + sizes.queries++ + if sizes.onQuery != nil { + sizes.onQuery() + } + return sizes.Sizes.Sizeof(typ) +} + +func (sizes *amd64VirtualArgumentCountingSizes) Alignof(typ types.Type) int64 { + sizes.queries++ + return sizes.Sizes.Alignof(typ) +} + +func (sizes *amd64VirtualArgumentCountingSizes) Offsetsof(fields []*types.Var) []int64 { + sizes.queries++ + return sizes.Sizes.Offsetsof(fields) +} + +// Internal symbol/register operations consume the private immutable plan, +// not repeated full-package declaration observations. Complete boundaries +// still query the live supplied inputs and revoke the call after a mutation. +func TestGoModuleAMD64VirtualArgumentsBoundedInternalChecks(t *testing.T) { + for _, mutation := range []string{"control", "syntax", "source"} { + t.Run(mutation, func(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, "package virtualargs\nfunc Entry(a []byte) uint64\n") + source := []byte("TEXT ·Entry(SB),4,$0\nMOVQ BX,AX\nRET\n") + file, err := Parse(ArchAMD64, string(source)) + if err != nil { + t.Fatal(err) + } + sizes := &amd64VirtualArgumentCountingSizes{Sizes: types.SizesFor("gc", "amd64")} + owner, err := bindAMD64GoVirtualArguments(pkg, file, source, sha256.Sum256(source), + GoModuleOptions{GOARCH: "amd64", Sizes: sizes}, testResolveSym(pkg.Path)) + if err != nil || owner == nil || sizes.queries == 0 { + t.Fatal("the complete declaration boundary was not observed", err, sizes.queries) + } + defer func() { owner.active = false }() + entry := owner.entries[pkg.Path+".Entry"] + if entry == nil { + t.Fatal("actual declaration lost its private parameter plan") + } + sig := entry.signature + sig.amd64GoArguments = entry + sizes.queries = 0 + for count := 0; count < 100; count++ { + if owner.resolveSymbol("·Entry") != sig.Name || !entry.valid(file.Funcs[0], sig, "amd64") { + t.Fatal("internal immutable-plan check failed", count) + } + } + if sizes.queries != 0 { + t.Fatal("internal resolution repeated whole declaration/layout observations", sizes.queries) + } + if mutation == "syntax" { + pkg.Syntax[0].Name.Name = "foreign" + } else if mutation == "source" { + source[len(source)-1] = ' ' + } + err = owner.verify() + if mutation == "control" { + if err != nil || sizes.queries == 0 { + t.Fatal("final boundary did not recheck live declarations", err, sizes.queries) + } + } else { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("final boundary accepted changed inputs", err) + } + pkg.Syntax[0].Name.Name = "virtualargs" + source[len(source)-1] = '\n' + if entry.valid(file.Funcs[0], sig, "amd64") { + t.Fatal("restoring an invalidated source revived its old call-local role") + } + } + }) + } +} + +func TestGoModuleAMD64VirtualArgumentsFinalLayoutCallbacks(t *testing.T) { + for _, mutation := range []string{"syntax", "source", "imports"} { + t.Run(mutation, func(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, "package virtualargs\nfunc Entry(a []byte) uint64\n") + pkg.Imports = make(map[string]*types.Package) + source := []byte("TEXT ·Entry(SB),4,$0\nMOVQ BX,AX\nRET\n") + file, err := Parse(ArchAMD64, string(source)) + if err != nil { + t.Fatal(err) + } + sizes := &amd64VirtualArgumentCountingSizes{Sizes: types.SizesFor("gc", "amd64")} + owner, err := bindAMD64GoVirtualArguments(pkg, file, source, sha256.Sum256(source), + GoModuleOptions{GOARCH: "amd64", Sizes: sizes}, testResolveSym(pkg.Path)) + if err != nil || owner == nil { + t.Fatal("complete declaration boundary failed", err) + } + defer func() { owner.active = false }() + called := false + sizes.onQuery = func() { + sizes.onQuery = nil // Mutate in the final layout query, not during capture. + called = true + switch mutation { + case "syntax": + pkg.Syntax[0].Name.Name = "foreign" + case "source": + source[len(source)-1] = ' ' + case "imports": + pkg.Imports["foreign"] = types.NewPackage("foreign", "foreign") + } + } + if err := owner.verify(); !called || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a final layout callback changed supplied inputs after their last observation", called, err) + } + }) + } +} + +func TestGoModuleAMD64VirtualArgumentsReleaseCallInputs(t *testing.T) { + for _, api := range []string{"public", "owned-context"} { + t.Run(api, func(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, "package virtualargs\nfunc Entry(a []byte) uint64\n") + pkg.SourceSelection = new(GoSourceSelection) // No raw caller consumes this object. + source := []byte("TEXT ·Entry(SB),4,$0\nMOVQ BX,AX\nRET\n") + options := GoModuleOptions{GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + SourcePreprocessed: true, ResolveSym: testResolveSym(pkg.Path)} + ctx := llvm.NewContext() + defer ctx.Dispose() + var translated *GoModuleTranslation + var err error + if api == "public" { + translated, err = TranslateGoModule(pkg, source, options) + } else { + translated, err = translateGoModuleInContext(ctx, pkg, source, options) + } + if err != nil || translated == nil { + t.Fatal("complete virtual argument translation failed", err) + } + defer translated.Module.Dispose() + if err := llvm.VerifyModule(translated.Module, llvm.ReturnStatusAction); err != nil { + t.Fatal("invalid returned virtual argument module", err) + } + sig := translated.Signatures[pkg.Path+".Entry"] + entry := sig.amd64GoArguments + if entry == nil || entry.virtual == nil { + t.Fatal("returned signatures lost their closed private-role tombstone") + } + owner := entry.virtual + if owner.active || !errors.Is(owner.live(), ErrProbeNeedsContext) { + t.Fatal("returned signature retained an active call-local role") + } + if !reflect.DeepEqual(owner.pkg, GoPackage{}) || owner.syntax != nil || owner.imports != nil || + owner.input != nil || owner.file != nil || owner.entries != nil || owner.symbols != nil || + owner.resolve != nil || owner.sizes != nil || owner.frameSizes != nil || entry.plan != nil || entry.declaration != nil { + t.Fatal("closed signature retained owned package/types/syntax/source/plan/callback inputs") + } + if len(sig.Args) != 1 || sig.Args[0] != "{ ptr, i64, i64 }" || sig.Ret != I64 || len(sig.Frame.Params) != 3 { + t.Fatal("releasing private call inputs changed the returned public language signature", sig) + } + file, err := Parse(ArchAMD64, string(source)) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: options.TargetTriple, + ResolveSym: options.ResolveSym, Sigs: translated.Signatures}) + amd64VirtualArgumentsRequireContext(t, ir, err) + }) + } +} diff --git a/amd64_go_virtual_bool_test.go b/amd64_go_virtual_bool_test.go new file mode 100644 index 000000000..701dfa01b --- /dev/null +++ b/amd64_go_virtual_bool_test.go @@ -0,0 +1,161 @@ +package plan9asm + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/nativeoracle" +) + +const amd64VirtualBoolDeclarations = `package virtualargs +type Flag bool +type Pair struct { First, Second bool } +func Identity(a bool) bool +func Invert(a bool) bool +func Widen(a bool) uint64 +func Second(a, b bool) bool +func Named(a Flag) Flag +func PairSecond(a Pair) bool +` + +const amd64VirtualBoolSource = ` +TEXT ·Identity(SB),4,$0 + RET +TEXT ·Invert(SB),4,$0 + XORB $1, AX + RET +TEXT ·Widen(SB),4,$0 + MOVBQZX AL, AX + RET +TEXT ·Second(SB),4,$0 + MOVB BL, AL + RET +TEXT ·Named(SB),4,$0 + RET +TEXT ·PairSecond(SB),4,$0 + MOVB BL, AL + RET +` + +// These wrappers call the unchanged grouped LLVM declarations. Their i1 +// inputs represent Go bool values, not arbitrary native bytes or a stock-Go +// register ABI. Returning i64 avoids depending on the host C bool ABI. +func amd64VirtualBoolWrappers() string { + var wrappers strings.Builder + for _, name := range []string{"Identity", "Invert", "Widen", "Second", "Named", "PairSecond"} { + fmt.Fprintf(&wrappers, "\ndefine i64 @bool_%s(i32 %%a, i32 %%b) {\n", name) + wrappers.WriteString(" %x = icmp ne i32 %a, 0\n %y = icmp ne i32 %b, 0\n") + arguments, result := "i1 %x", "i1" + switch name { + case "Second": + arguments = "i1 %x, i1 %y" + case "PairSecond": + wrappers.WriteString(" %first = insertvalue { i1, i1 } poison, i1 %x, 0\n") + wrappers.WriteString(" %pair = insertvalue { i1, i1 } %first, i1 %y, 1\n") + arguments = "{ i1, i1 } %pair" + case "Widen": + result = "i64" + } + fmt.Fprintf(&wrappers, " %%value = call %s @%q(%s)\n", result, amd64VirtualArgumentsPackage+"."+name, arguments) + if result == "i1" { + wrappers.WriteString(" %wide = zext i1 %value to i64\n ret i64 %wide\n}\n") + } else { + wrappers.WriteString(" ret i64 %value\n}\n") + } + } + return wrappers.String() +} + +func amd64VirtualBoolIR(t *testing.T, triple, api string) string { + t.Helper() + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, amd64VirtualBoolDeclarations) + ir, err := amd64VirtualArgumentsRequest(t, pkg, []byte(amd64VirtualBoolSource), triple, api, false, nil) + if err != nil || strings.Count(ir, "define ") != 6 { + t.Fatal("complete typed boolean input family was not translated", err, ir) + } + return ir + amd64VirtualBoolWrappers() +} + +func TestGoModuleAMD64VirtualBoolObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("typed boolean objects require LLVM22 llc") + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + for _, api := range []string{"public", "owned-context"} { + t.Run(triple+"/"+api, func(t *testing.T) { + x86PackedStringCompileObject(t, llc, triple, amd64VirtualBoolIR(t, triple, api), "-O2") + }) + } + } +} + +func TestGoModuleAMD64VirtualBoolPortableValues(t *testing.T) { + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("typed boolean execution requires LLVM22 llc/clang") + } + const driver = `#include +extern uint64_t bool_Identity(uint32_t,uint32_t), bool_Invert(uint32_t,uint32_t); +extern uint64_t bool_Widen(uint32_t,uint32_t), bool_Second(uint32_t,uint32_t); +extern uint64_t bool_Named(uint32_t,uint32_t), bool_PairSecond(uint32_t,uint32_t); +int main(void) { + for (uint32_t a = 0; a != 2; ++a) { + for (uint32_t b = 0; b != 2; ++b) { + if (bool_Identity(a,b) != a || bool_Invert(a,b) != 1-a || + bool_Widen(a,b) != a || bool_Second(a,b) != b || + bool_Named(a,b) != a || bool_PairSecond(a,b) != b) return 1; + } + } + return 0; +} +` + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + compiler := []string{clang} + if runtime.GOOS == "darwin" { + ctx, cancel := runtimeOracleStageContext(t) + flags, err := nativeoracle.DarwinClangFlags(ctx, os.Environ()) + cancel() + if err != nil { + t.Fatal(err) + } + compiler = append(compiler, flags...) + } + for _, api := range []string{"public", "owned-context"} { + ir := amd64VirtualBoolIR(t, triple, api) + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86UndefinedTrapOptimizedIR(t, input) + } + for _, level := range []int{0, 2} { + t.Run(fmt.Sprintf("%s/IR_O2=%t/backendO%d", api, optimized, level), func(t *testing.T) { + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, "virtual-bool", triple, input, driver, nil, level) + }) + } + } + } +} + +func TestGoModuleAMD64VirtualBoolKeepsOtherStateUnknown(t *testing.T) { + for _, row := range []struct{ name, declaration, body string }{ + {"upper-input", "func Entry(a bool) uint64", "MOVQ AX,AX"}, + {"upper-named-input", "type Flag bool\nfunc Entry(a Flag) uint64", "MOVQ AX,AX"}, + {"upper-second-input", "func Entry(a,b bool) uint64", "MOVQ BX,AX"}, + {"aggregate-result", "func Entry(a bool) (bool,bool)", "MOVB $1,AX\nMOVB $1,BX"}, + {"noncanonical-result", "func Entry(a bool) bool", "MOVB $2,AX"}, + {"unknown-register", "func Entry(a bool) bool", "MOVB DL,AL"}, + } { + for _, api := range []string{"public", "owned-context"} { + t.Run(row.name+"/"+api, func(t *testing.T) { + pkg := mustGoPackage(t, amd64VirtualArgumentsPackage, "package virtualargs\n"+row.declaration+"\n") + source := []byte("TEXT ·Entry(SB),4,$0\n" + row.body + "\nRET\n") + ir, err := amd64VirtualArgumentsRequest(t, pkg, source, "x86_64-unknown-linux-gnu", api, false, nil) + amd64VirtualArgumentsRequireContext(t, ir, err) + }) + } + } +} diff --git a/amd64_go_windows_private_leaf_go127_test.go b/amd64_go_windows_private_leaf_go127_test.go new file mode 100644 index 000000000..5380f8fb3 --- /dev/null +++ b/amd64_go_windows_private_leaf_go127_test.go @@ -0,0 +1,201 @@ +//go:build go1.27 + +package plan9asm + +import ( + "encoding/json" + "errors" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestGoAMD64WindowsPrivateLeafCompleteObjects(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, spelling := range []string{"leaf<>", "leaf"} { + t.Run(spelling, func(t *testing.T) { + assembly := fixedLocalPrivateLeafSource("CALL leaf<>(SB)", "ADDQ BX,(BP)\nMOVQ (BP),AX") + assembly = strings.ReplaceAll(assembly, "leaf<>", spelling) + pkg, request, options := fixedLocalPrivateLeafFixture(t, assembly) + request.Target = "windows/amd64" + options.TargetTriple = "x86_64-pc-windows-msvc" + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + x86CompactExpandRequireAllDefinitions(t, ir, []string{pkg.Path + ".Value"}) + compileLLVMToObject(t, llc, options.TargetTriple, "private-leaf.ll", "private-leaf.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "private-leaf-opt.ll", "private-leaf-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +func TestGoAMD64WindowsPrivateLeafSourceRoleBoundaries(t *testing.T) { + for _, spelling := range []string{"leaf<>", "leaf"} { + t.Run(spelling, func(t *testing.T) { + assembly := strings.ReplaceAll(fixedLocalPrivateLeafSource("CALL leaf<>(SB)", + "ADDQ BX,(BP)\nMOVQ (BP),AX"), "leaf<>", spelling) + pkg, request, _ := fixedLocalPrivateLeafFixture(t, assembly) + request.Target = "windows/amd64" + source := requireGoAMD64FrameCapture(t, pkg, request) + object, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o")) + if err != nil { + t.Fatal(err) + } + listing, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + data, err := source.frameDataObject(assembly, object) + if err != nil { + t.Fatal(err) + } + leaves, err := goAMD64WindowsSourceLeafTexts(file, pkg, listing, data.Member.Texts, data.TextOrder) + if err != nil || len(leaves) != 1 { + t.Fatal("complete source did not retain its one exact private leaf", err) + } + observed, err := source.observeWindowsUnwind(object, leaves) + if err != nil || len(observed.Texts) != 2 || len(observed.Unwind) != 1 { + t.Fatal("framed root and zero-frame leaf lost their distinct SEH roles", err) + } + if _, err := source.observeWindowsUnwind(object); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("default metadata observation inferred a private source role", err) + } + for _, change := range nativeHeaderRoleChanges() { + t.Run("header-"+change.name, func(t *testing.T) { + rows := strings.Split(string(listing), "\n") + changed := false + for index, row := range rows { + if strings.HasPrefix(row, leaves[0].Name+" STEXT ") && strings.Contains(row, change.before) { + rows[index] = nativeHeaderChangedRole(row, change.before, change.after) + changed = true + } + } + if !changed { + t.Fatal("actual private header precondition changed") + } + mutated := []byte(strings.Join(rows, "\n")) + copyData := *data + copyData.Member.Texts = append([]gostaticstorage.TextRecord(nil), data.Member.Texts...) + for index := range copyData.Member.Texts { + text := ©Data.Member.Texts[index] + if text.Index == leaves[0].Index { + if change.name == "dupok" { + text.Flags |= 0x01 + } else if change.name == "leaf" { + text.Flags |= 0x08 + } + } + } + // These edited observations are mutation controls, not new + // Go objects. Matching metadata still grants no private role. + if _, err := sdkAMD64FrameNativeRecordsForSource(mutated, copyData.Member.Texts, copyData.TextOrder); err != nil { + t.Fatal("role control did not reach the private entry boundary", err) + } + if _, err := goAMD64WindowsSourceLeafTexts(file, pkg, mutated, copyData.Member.Texts, copyData.TextOrder); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("special or unknown metadata acquired a Windows private leaf role", err) + } + }) + } + var root gostaticstorage.TextRecord + for _, text := range observed.Texts { + if text.Index != leaves[0].Index { + root = text + } + } + serialized, err := json.Marshal(leaves) + if err != nil { + t.Fatal(err) + } + mutations := map[string]func([]gostaticstorage.TextRecord) []gostaticstorage.TextRecord{ + "missing leaf": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { return nil }, + "duplicate leaf": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + return append(v, v[0]) + }, + "framed root is not leaf": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + return []gostaticstorage.TextRecord{root} + }, + "foreign name": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + v[0].Name += "Other" + return v + }, + "foreign ABI": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + v[0].ABI = 1 + return v + }, + "foreign index": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + v[0].Index = root.Index + return v + }, + "foreign namespace": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + v[0].Namespace = gostaticstorage.TextSymbolHashed + return v + }, + "changed bytes": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + v[0].Code[0] ^= 1 + return v + }, + "changed source PC": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + v[0].Positions[0].Line++ + return v + }, + "changed relative index": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + v[0].Relative++ + return v + }, + "borrowed root auxiliary": func(v []gostaticstorage.TextRecord) []gostaticstorage.TextRecord { + v[0].Auxiliary = root.Auxiliary + return v + }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + var changed []gostaticstorage.TextRecord + if err := json.Unmarshal(serialized, &changed); err != nil { + t.Fatal(err) + } + if _, err := source.observeWindowsUnwind(object, mutate(changed)); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed source/object leaf descriptor acquired a metadata exception", err) + } + }) + } + for name, change := range map[string][2]string{ + "nonzero helper frame": {"TEXT " + spelling + "(SB),4,$0", "TEXT " + spelling + "(SB),4,$8"}, + "helper SP read": {"ADDQ BX,(BP)", "MOVQ (SP),AX"}, + "helper stack write": {"ADDQ BX,(BP)", "PUSHQ AX"}, + "helper is not leaf": {"ADDQ BX,(BP)", "CALL " + spelling + "(SB)"}, + } { + t.Run(name, func(t *testing.T) { + changed := strings.Replace(assembly, change[0], change[1], 1) + if changed == assembly { + t.Fatal("source role control did not alter the complete source") + } + parsed, err := Parse(ArchAMD64, changed) + if err != nil { + t.Fatal(err) + } + if _, err := goAMD64WindowsSourceLeafTexts(parsed, pkg, listing, data.Member.Texts, data.TextOrder); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("non-leaf source acquired a zero-frame metadata role", err) + } + }) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if err := source.Verify(ctx); err != nil { + t.Fatal("readonly role controls changed the original capture", err) + } + }) + } +} diff --git a/amd64_go_windows_unwind.go b/amd64_go_windows_unwind.go new file mode 100644 index 000000000..00ddfd245 --- /dev/null +++ b/amd64_go_windows_unwind.go @@ -0,0 +1,240 @@ +package plan9asm + +import ( + "bytes" + "go/types" + "path/filepath" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "golang.org/x/arch/x86/x86asm" +) + +func (source *GoAMD64FrameSource) observeWindowsUnwind(object []byte, privateLeaves ...[]gostaticstorage.TextRecord) (gostaticstorage.WindowsAMD64AssemblyUnwind, error) { + var empty gostaticstorage.WindowsAMD64AssemblyUnwind + builtin, err := source.windowsUnwindBuiltin() + if err != nil { + return empty, err + } + observed, err := gostaticstorage.ObserveWindowsAMD64AssemblyUnwind(object, builtin, privateLeaves...) + if err != nil { + return empty, goAMD64FrameContext("Windows unwind complete object metadata: %v", err) + } + if observed.Version != source.env["GOVERSION"] { + return empty, goAMD64FrameContext("Windows unwind object differs from its actual producer") + } + return observed, nil +} + +func (source *GoAMD64FrameSource) windowsUnwindBuiltin() ([]byte, error) { + for _, declaration := range source.declarations { + if !fixedLocalTransportRole(declaration.transport) { + return nil, goAMD64FrameContext("Windows unwind join requires the closed typed ABI0 transport scope") + } + } + builtin, err := gostaticstorage.WindowsAMD64UnwindSources(source.env["GOVERSION"], func(name string) ([]byte, error) { + return source.read(filepath.Join(source.env["GOROOT"], filepath.FromSlash(name))) + }) + if err != nil { + return nil, goAMD64FrameContext("Windows unwind actual SDK sources: %v", err) + } + return builtin, nil +} + +// Only the complete original source classification can select leaf records. +// Keep their full raw object identity, including namespace, ABI, bytes and PCs; +// neither a static ABI number nor absent SEH metadata establishes this role. +func goAMD64WindowsSourceLeafTexts(file *File, pkg GoPackage, listing []byte, + texts []gostaticstorage.TextRecord, order []gostaticstorage.ASMSourceText) ([]gostaticstorage.TextRecord, error) { + leaves, err := goAMD64FramePrivateLeaves(file, pkg) + if err != nil { + return nil, err + } + records, err := sdkAMD64FrameNativeRecordsForSource(listing, texts, order) + if err != nil || len(records) != len(file.Funcs) { + return nil, goAMD64FrameContext("Windows private leaf lacks a complete source/object TEXT association: %v", err) + } + var result []gostaticstorage.TextRecord + for index, fn := range file.Funcs { + if !leaves[fn.Sym] && !goAMD64DeclaredNoFrameLeaf(fn, pkg) { + continue + } + text := texts[order[index].ObjectOrdinal] + expectedName := strings.TrimSuffix(fn.Sym, "<>") + if !leaves[fn.Sym] { + expectedName = pkg.Path + "." + strings.TrimPrefix(fn.Sym, "·") + } + abi, namespace := uint16(0), gostaticstorage.TextSymbolNonpkgDefinition + if strings.HasSuffix(fn.Sym, "<>") { + abi, namespace = 65535, gostaticstorage.TextSymbolSelf + } + if text.Name != expectedName || text.ABI != abi || text.Namespace != namespace || + !records[index].ordinaryFrameRole() || records[index].Locals != uint64(goAMD64FrameNativeLocals(fn.FrameSize)) { + return nil, goAMD64FrameContext("Windows private leaf changed its exact source symbol/ABI/namespace") + } + result = append(result, text) + } + return result, nil +} + +// Match the pinned x86 producer's automatic NoFrame leaf rule. This selects +// metadata observations only; ordinary FP slots still come from the declaration +// and this function never establishes a private CALL or local backing role. +func goAMD64DeclaredNoFrameLeaf(fn Func, pkg GoPackage) bool { + if fn.FrameSize != 0 || pkg.Types == nil { + return false + } + if _, declared := pkg.Types.Scope().Lookup(strings.TrimPrefix(fn.Sym, "·")).(*types.Func); !declared { + return false + } + for _, ins := range fn.Instrs { + if ins.Op == OpCALL || ins.Op == "DUFFCOPY" || ins.Op == "DUFFZERO" { + return false + } + } + return true +} + +func goAMD64WindowsUnwindByteRow(line string, offset int, expected []byte) ([]byte, error) { + pc, actual, err := goAssemblyListingByteRow(line) + if err != nil || len(expected) != 12 || offset != 0 || pc != 0 || !bytes.Equal(actual, expected) { + return nil, goAMD64FrameContext("Windows unwind listing differs from complete original binary bytes") + } + return actual, nil +} + +// This private join consumes the factory's independently read whole object. +// A public offline observation is never itself an authorization input. +func goAMD64FrameWindowsUnwindListing(listing []byte, file *File, pkg GoPackage, origins map[string]int, directory string, observed gostaticstorage.WindowsAMD64AssemblyUnwind) (map[string]goAMD64FrameFunction, error) { + functions, err := goAMD64FrameListingWithSplitObject(listing, file, pkg, origins, observed.Texts, &observed, "windows/amd64", directory, "") + if err != nil { + return nil, err + } + if err := goAMD64FrameWindowsUnwindFunctions(functions, origins, directory, observed); err != nil { + return nil, err + } + return functions, nil +} + +func goAMD64FrameWindowsUnwindFunctions(functions map[string]goAMD64FrameFunction, origins map[string]int, directory string, observed gostaticstorage.WindowsAMD64AssemblyUnwind) error { + if len(observed.Texts) != len(functions) { + return goAMD64FrameContext("Windows object/listing complete TEXT inventories differ") + } + unwind := make(map[string]gostaticstorage.WindowsAMD64UnwindRecord) + for _, record := range observed.Unwind { + unwind[record.Name] = record + } + seen := make(map[string]bool) + for _, text := range observed.Texts { + fn, exists := functions[text.Name] + abi := uint16(0) + if fn.privateLeaf { + abi = fn.nativeABI + } + if !exists || seen[text.Name] || text.ABI != abi || text.Kind != 1 || text.Flags != fn.nativeFlags() || text.Flags2 != observed.TextFlags2 || + !bytes.Equal(text.Code, fn.code) || fn.prologue < 0 || fn.prologue >= len(fn.code) { + return goAMD64FrameContext("Windows object/listing TEXT name/ABI/code role differs") + } + seen[text.Name] = true + var metadata gostaticstorage.WindowsAMD64UnwindRecord + for _, aux := range text.Auxiliary { + if aux.Type == observed.AuxType { + metadata = unwind[aux.Target.Name] + } + } + if fn.locals == 0 { + if metadata.Name != "" || fn.prologue != 0 || fn.split { + return goAMD64FrameContext("Windows private leaf acquired an automatic frame or SEH prologue") + } + if fn.privateLeaf { + if err := goAMD64FrameMachineShapeWithCalls(fn, 0, nil); err != nil { + return err + } + } + if err := goAMD64FrameSourcePCs(text, fn, directory, origins, ""); err != nil { + return err + } + continue + } + first, err := x86asm.Decode(fn.code[fn.prologue:], 64) + if err != nil || first.Op != x86asm.PUSH || first.Args[0] != x86asm.RBP || fn.prologue+first.Len != int(metadata.PushEnd) { + return goAMD64FrameContext("Windows unwind push end differs from actual automatic prologue") + } + second, err := x86asm.Decode(fn.code[fn.prologue+first.Len:], 64) + if err != nil || second.Op != x86asm.MOV || second.Args[0] != x86asm.RBP || second.Args[1] != x86asm.RSP || + fn.prologue+first.Len+second.Len != int(metadata.PrologueEnd) { + return goAMD64FrameContext("Windows unwind frame end differs from actual automatic prologue") + } + if err := goAMD64FrameSourcePCs(text, fn, directory, origins, ""); err != nil { + return err + } + } + return nil +} + +func goAMD64FrameSourcePath(name, directory, root string) string { + if strings.HasPrefix(name, "$GOROOT/") { + if root == "" { + return "" + } + actual, valid := originalGoObjectSourcePath(name, root) + if !valid { + return "" + } + return actual + } + if !filepath.IsAbs(name) { + name = filepath.Join(directory, filepath.FromSlash(name)) + } + return filepath.Clean(name) +} + +func goAMD64FrameSourcePCs(text gostaticstorage.TextRecord, fn goAMD64FrameFunction, directory string, origins map[string]int, root string) error { + normalize := func(name string) string { return goAMD64FrameSourcePath(name, directory, root) } + // Go object file tables use slash paths on Windows, while captured + // origins retain host paths and the command's relative source spelling. + // Compare their already-bound paths consistently with the listing join; + // aliases must still agree on the complete original source extent. SDK + // metadata additionally uses only the captured root's strict path spelling; + // resolving it never adds an input to this complete original inventory. + linesByPath := make(map[string]int, len(origins)) + for name, lines := range origins { + path := normalize(name) + if path == "" { + return goAMD64FrameContext("source origin has an unbound SDK path") + } + if previous, found := linesByPath[path]; found && previous != lines { + return goAMD64FrameContext("source origin aliases disagree on their original extent") + } + linesByPath[path] = lines + } + var previous int64 + for _, span := range text.Positions { + if span.File == "" || span.Line <= 0 || int(span.Line) > linesByPath[normalize(span.File)] || span.Start < 0 || + span.Start != previous || span.End <= span.Start || span.End > int64(len(fn.code)) { + return goAMD64FrameContext("object source-PC interval lacks its original file/line/extent: %s %s:%d [%d,%d)", + text.Name, span.File, span.Line, span.Start, span.End) + } + previous = span.End + } + if previous != int64(len(fn.code)) { + return goAMD64FrameContext("source-PC intervals do not cover the complete physical TEXT") + } + for pc, positions := range fn.sourcePositions { + matched := false + for _, span := range text.Positions { + if span.Start > int64(pc) || span.End <= int64(pc) { + continue + } + for _, position := range positions { + if normalize(position.file) == normalize(span.File) && position.line == int(span.Line) { + matched = true + } + } + } + if !matched { + return goAMD64FrameContext("object/listing source-PC file/line witness differs") + } + } + return nil +} diff --git a/amd64_go_windows_unwind_join_test.go b/amd64_go_windows_unwind_join_test.go new file mode 100644 index 000000000..0537ed243 --- /dev/null +++ b/amd64_go_windows_unwind_join_test.go @@ -0,0 +1,250 @@ +//go:build go1.23 + +package plan9asm + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestGoAMD64WindowsUnwindCompleteObjectListingJoinGuards(t *testing.T) { + goAMD64WindowsUnwindCompleteObjectListingJoinGuards(t) +} + +func TestGoAMD64WindowsUnwindCompleteJoinTemporaryParentAlias(t *testing.T) { + goAMD64WindowsUnwindTemporaryParentAlias(t) + // A new child owns a fresh testing.TempDir root under the explicit alias. + // The parent's root was already created while constructing that alias. + t.Run("AliasedTemporaryParent", goAMD64WindowsUnwindCompleteObjectListingJoinGuards) +} + +func goAMD64WindowsUnwindCompleteObjectListingJoinGuards(t *testing.T) { + t.Helper() + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = "windows/amd64" + source := requireGoAMD64FrameCapture(t, pkg, request) + t.Logf("ACTUAL_WINDOWS_SOURCE_CAPTURE request=%q observed=%q functions=%d", + request.Directory, source.request.Directory, len(source.functions)) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + object, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o")) + if err != nil { + t.Fatal(err) + } + listing, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + original, err := source.read(filepath.Join(request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + expanded, err := source.preprocess(request.AssemblyFile, original) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + observed, err := source.observeWindowsUnwind(object) + if err != nil { + t.Fatal(err) + } + for _, version := range []string{"go1.27.1", "go1.27.2", "go1.99.1"} { + if version == observed.Version { + continue + } + changed := bytes.Replace(object, []byte(observed.Version+" "), []byte(version+" "), 1) + if bytes.Equal(changed, object) { + t.Fatal("object version mutation did not replace its producer") + } + if _, err := source.observeWindowsUnwind(changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("object substituted another actual producer", version, err) + } + } + functions, err := goAMD64FrameWindowsUnwindListing(listing, file, pkg, source.origins, request.Directory, observed) + if err != nil { + t.Fatal("independent actual original metadata join control failed", err) + } + for _, spelling := range []string{"absolute", "slash absolute", "relative", "conflicting extent", "foreign directory"} { + t.Run("source origins/"+spelling, func(t *testing.T) { + origins := make(map[string]int) + relativeNames := 0 + for name, lines := range source.origins { + if !filepath.IsAbs(name) { + name = filepath.Join(request.Directory, name) + } + if spelling == "slash absolute" { + name = filepath.ToSlash(name) + } + if spelling == "relative" { + relative, err := filepath.Rel(request.Directory, name) + if err != nil { + // A Windows SDK can live on another volume. That + // header must remain absolute; own assembly inputs + // below still exercise actual relative spellings. + if filepath.VolumeName(request.Directory) == filepath.VolumeName(name) { + t.Fatal(err) + } + } else { + name = relative + relativeNames++ + } + } + if spelling == "foreign directory" { + name = filepath.Join(request.Directory, "foreign", filepath.Base(name)) + } + origins[name] = lines + } + if spelling == "relative" && relativeNames == 0 { + t.Fatal("relative-source control did not exercise any relative path") + } + if spelling == "conflicting extent" { + origins[request.AssemblyFile] = source.origins[request.AssemblyFile] + 1 + } + for _, text := range observed.Texts { + err := goAMD64FrameSourcePCs(text, functions[text.Name], request.Directory, origins, "") + if spelling == "conflicting extent" || spelling == "foreign directory" { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound or contradictory source origin acquired a PC witness", err) + } + } else if err != nil { + t.Fatal("actual source origin was lost through its host path spelling", err) + } + } + }) + } + serialized, err := json.Marshal(observed) + if err != nil { + t.Fatal(err) + } + // These are ordinary read-only observations, deliberately copied only for + // decoder negatives. None are accepted as a public Source/factory input. + type mutation func([]byte, *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte + metadataStart := bytes.Index(listing, []byte(observed.Unwind[0].Name+" SSEHUNWINDINFO")) + if metadataStart < 0 { + t.Fatal("original listing omitted its actual complete metadata section") + } + mutations := map[string]mutation{ + "wrong metadata name": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("go:sehuw.12."), []byte("go:sehuw.13."), 1) + }, + "wrong metadata kind": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("SSEHUNWINDINFO"), []byte("SRODATA"), 1) + }, + "wrong metadata flags": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("SSEHUNWINDINFO dupok"), []byte("SSEHUNWINDINFO"), 1) + }, + "wrong metadata alignment": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("SSEHUNWINDINFO dupok size=12"), []byte("SSEHUNWINDINFO dupok size=12 align=0x8"), 1) + }, + "other SDK alignment spelling": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + if observed.ListingAlignment { + return bytes.Replace(v, []byte("size=12 align=0x4"), []byte("size=12"), 1) + } + return bytes.Replace(v, []byte("SSEHUNWINDINFO dupok size=12"), []byte("SSEHUNWINDINFO dupok size=12 align=0x4"), 1) + }, + "wrong metadata byte": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("01 04 02 05"), []byte("02 04 02 05"), 1) + }, + "wrong metadata ASCII": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte(".......P...."), []byte("............"), 1) + }, + "missing metadata section": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { return v[:metadataStart] }, + "duplicate metadata section": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return append(v, v[metadataStart:]...) + }, + "unknown trailing record": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return append(v, []byte("UNKNOWN_NATIVE_RECORD\n")...) + }, + "truncated metadata EOF": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { return v[:len(v)-5] }, + "different actual TEXT bytes": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Code[0] ^= 1 + return v + }, + "different actual TEXT ABI": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { o.Texts[0].ABI = 1; return v }, + "extra actual TEXT": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts = append(o.Texts, o.Texts[0]) + return v + }, + "missing actual TEXT": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { o.Texts = nil; return v }, + "wrong object source line": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions[0].Line++ + return v + }, + "wrong object source file": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions[0].File = "other.s" + return v + }, + "object source-PC gap": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions[0].Start = 1 + return v + }, + "object source-PC overlap": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions = append(o.Texts[0].Positions, o.Texts[0].Positions[0]) + return v + }, + "object source-PC missing tail": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions = o.Texts[0].Positions[:len(o.Texts[0].Positions)-1] + return v + }, + } + for _, role := range []string{"wrong push end", "wrong prologue end"} { + mutations[role] = func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + metadata := &o.Unwind[0] + oldName := metadata.Name + if role == "wrong push end" { + metadata.PushEnd, metadata.Data[6] = 2, 2 + } else { + metadata.PrologueEnd, metadata.Data[1], metadata.Data[4] = 5, 5, 5 + } + metadata.Name = "go:sehuw.12." + base64.StdEncoding.EncodeToString(metadata.Data) + for index := range o.Texts[0].Auxiliary { + if o.Texts[0].Auxiliary[index].Type == o.AuxType { + o.Texts[0].Auxiliary[index].Target.Name = metadata.Name + } + } + v = bytes.ReplaceAll(v, []byte(oldName), []byte(metadata.Name)) + rowStart := bytes.Index(v[metadataStart:], []byte("\n")) + metadataStart + 1 + return append(append(v[:rowStart:rowStart], goAMD64WindowsUnwindTestByteRow(metadata.Data)...), '\n') + } + } + for name, change := range mutations { + t.Run(name, func(t *testing.T) { + var independent gostaticstorage.WindowsAMD64AssemblyUnwind + if err := json.Unmarshal(serialized, &independent); err != nil { + t.Fatal(err) + } + changed := change(append([]byte(nil), listing...), &independent) + if _, err := goAMD64FrameWindowsUnwindListing(changed, file, pkg, source.origins, request.Directory, independent); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed original Windows binary/listing join was accepted", err) + } + }) + } + if err := source.Verify(ctx); err != nil { + t.Fatal("read-only metadata negative controls changed original source/artifacts", err) + } +} + +func goAMD64WindowsUnwindTestByteRow(data []byte) []byte { + var result, ascii strings.Builder + result.WriteString("\t0x0000") + for _, value := range data { + fmt.Fprintf(&result, " %02x", value) + if value < ' ' || value > 0x7e { + value = '.' + } + ascii.WriteByte(value) + } + fmt.Fprintf(&result, " %s", ascii.String()) + return []byte(result.String()) +} diff --git a/amd64_go_windows_unwind_source_test.go b/amd64_go_windows_unwind_source_test.go new file mode 100644 index 000000000..c55a92afc --- /dev/null +++ b/amd64_go_windows_unwind_source_test.go @@ -0,0 +1,141 @@ +//go:build go1.23 + +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +func TestGoAMD64WindowsUnwindActualSourceFamily(t *testing.T) { + goAMD64WindowsUnwindActualSourceFamily(t) +} + +// A temporary-directory alias is an ordinary host configuration, not permission +// to relax the original object reader's canonical physical-path contract. +func TestGoAMD64WindowsUnwindTemporaryParentAlias(t *testing.T) { + goAMD64WindowsUnwindTemporaryParentAlias(t) + goAMD64WindowsUnwindActualSourceFamily(t) +} + +func goAMD64WindowsUnwindTemporaryParentAlias(t *testing.T) { + t.Helper() + parent, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + physical := filepath.Join(parent, "physical") + alias := filepath.Join(parent, "alias") + if err := os.Mkdir(physical, 0700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(physical, alias); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", alias) + t.Setenv("GOTMPDIR", alias) +} + +func goAMD64WindowsUnwindActualSourceFamily(t *testing.T) { + t.Helper() + for _, role := range []string{"numeric", "void", "shared numeric and void"} { + t.Run(role, func(t *testing.T) { + declaration := "func Value(a uint64) uint64" + assembly := "TEXT ·Value(SB),4,$160-16\nMOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nMOVQ 16(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + if role == "void" { + declaration = "func Value(a uint64)" + assembly = "TEXT ·Value(SB),4,$160-8\nMOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nRET\n" + } else if role == "shared numeric and void" { + declaration += "\nfunc Other(a uint64)" + assembly += "TEXT ·Other(SB),4,$160-8\nMOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nRET\n" + } + pkg, request := goAMD64FrameTypedFixture(t, declaration, assembly) + request.Target = "windows/amd64" + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + var environment []string + for _, value := range request.Environment { + key, _, _ := strings.Cut(value, "=") + if key != "GOOS" && key != "GOARCH" && key != "CGO_ENABLED" { + environment = append(environment, value) + } + } + environment = append(environment, "GOOS=windows", "GOARCH=amd64", "CGO_ENABLED=0") + request.Environment = environment + objectDirectory, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal("cannot resolve original Windows output directory", err) + } + object := filepath.Join(objectDirectory, "original-windows.o") + listing, err := goasmcmd.Run(ctx, request.Directory, environment, request.GoBinary, + "tool", "asm", "-S", "-p", pkg.Path, "-o", object, request.AssemblyFile) + if err != nil { + t.Fatal("actual original Windows assembler control failed", err) + } + info, err := os.Stat(object) + if err != nil || !info.Mode().IsRegular() || info.Size() <= 0 { + t.Fatal("actual original Windows object was not produced", err) + } + original, err := os.ReadFile(object) + if err != nil || int64(len(original)) != info.Size() || + !bytes.Contains(listing, []byte("SSEHUNWINDINFO")) || + !bytes.Contains(original, []byte("go:sehuw.12.")) { + t.Fatal("independent original Windows object precondition failed", err) + } + t.Logf("ACTUAL_WINDOWS_ORIGINAL_OUTPUT_EXISTS bytes=%d sha256=%x hex=%x\nACTUAL_WINDOWS_ORIGINAL_LISTING\n%s", + len(original), sha256.Sum256(original), original, listing) + data, err := goAMD64FrameRead(object) + if err != nil || !bytes.Equal(data, original) || !bytes.Contains(listing, []byte("SSEHUNWINDINFO")) || + !bytes.Contains(data, []byte("go:sehuw.12.")) { + t.Fatal("actual original Windows object/listing lacks its unwind metadata", err) + } + t.Logf("ACTUAL_WINDOWS_ORIGINAL_OBJECT bytes=%d sha256=%x hex=%x\nACTUAL_WINDOWS_ORIGINAL_LISTING\n%s", + len(data), sha256.Sum256(data), data, listing) + if !t.Run("PlainParserRefusesUnboundMetadata", func(t *testing.T) { + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + origins := map[string]int{request.AssemblyFile: strings.Count(assembly, "\n") + 1} + if _, err := goAMD64FrameListing(listing, file, pkg, origins); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("plain listing parser borrowed Windows binary metadata authority", err) + } + }) { + return + } + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("actual ordinary Windows numeric/void source requires its complete SEH join", err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + want := 1 + if role == "shared numeric and void" { + want = 2 + } + if len(source.functions) != want || source.Verify(ctx) != nil { + t.Fatal("complete Windows numeric/void original source did not verify") + } + copied := new(GoAMD64FrameSource) + reflect.ValueOf(copied).Elem().Set(reflect.ValueOf(source).Elem()) + if !errors.Is(copied.Verify(ctx), ErrProbeNeedsContext) || + !errors.Is(copied.Close(), ErrProbeNeedsContext) || source.Verify(ctx) != nil { + t.Fatal("Windows metadata route bypassed exact source owner identity") + } + if _, err := os.Stat(request.Directory); err != nil { + t.Fatal("source owner identity control changed the original tree", err) + } + }) + } +} diff --git a/amd64_gp_simd_native_result_test.go b/amd64_gp_simd_native_result_test.go new file mode 100644 index 000000000..47c1cc8ea --- /dev/null +++ b/amd64_gp_simd_native_result_test.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +var x86GPSIMDNativeOracle = []struct { + op, original string + width int +}{ + {"MOVL", "MOVL", 4}, + {"MOVQ", "MOVQ", 8}, + {"MOVD", "MOVD", 8}, // Go's MOVD is the qword alias. + {"VMOVD", "MOVL", 4}, + {"VMOVQ", "MOVQ", 8}, +} + +func x86GPSIMDNativeFixture(name, arch, op, address, destination string, lane int) (string, string) { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + extract := "PEXTRD" + if strings.HasPrefix(op, "V") { + extract = "VPEXTRD" + } + source := fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),DX\nMOVL $1,BX\n%s %s,%s\n%s $%d,%s,AX\nMOVL AX,ret+%d(FP)\nRET\n", + name, word+4, move, op, address, destination, extract, lane, destination, word) + return source, fmt.Sprintf("func %s(p *[8]byte) uint32\n", name) +} + +func TestX86GPSIMDNativeResultsThreeAPIFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, row := range x86GPSIMDNativeOracle { + for _, address := range []string{"(DX)", "-1(DX)(BX*1)"} { + for _, destination := range []string{"X0", "X17"} { + if destination == "X17" && (target.arch == "386" || !strings.HasPrefix(row.op, "V")) { + continue // High banks retain their independent grammar/physical-mode checks. + } + for lane := 0; lane < 4; lane++ { + asm, decl := x86GPSIMDNativeFixture(fmt.Sprintf("F%d", count), target.arch, row.op, address, destination, lane) + source.WriteString(asm) + declarations.WriteString(decl) + count++ + } + } + } + } + want := 56 + if target.arch == "386" { + want = 40 + } + if count != want { + t.Fatal("incomplete native GP/X source and zero-lane matrix", count, want) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("GP/X memory read lost its numerical/zero result bytes:", err) + } + if strings.Count(ir, "load volatile i") != count { + t.Fatal("numerical continuation lost a paired native read") + } + compileLLVMToObject(t, llc, target.triple, "gp-x-native.ll", "gp-x-native.o", ir) + }) + } + }) + } +} + +func TestX86GPSIMDNativeResultsRetainSourceBoundaries(t *testing.T) { + for _, row := range x86GPSIMDNativeOracle { + t.Run(row.op, func(t *testing.T) { + source, declaration := x86GPSIMDNativeFixture("Value", "amd64", row.op, "(DX)", "X0", 0) + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", strings.TrimSpace(declaration), + strings.ReplaceAll(strings.SplitN(source, "\n", 2)[1], "ret+8(FP)", "ret0+8(FP)")) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("standalone query acquired native memory", err) + } + } + load := row.op + " (DX),X0" + missing := strings.Replace(source, load, "CMPL BX,$0\nJE after\n"+load+"\nafter:", 1) + for name, bad := range map[string]string{ + "unknown base": strings.Replace(source, "(DX)", "(SI)", 1), + "unknown index": strings.Replace(source, "(DX)", "(DX)(SI*1)", 1), + "pointer index": strings.Replace(source, "(DX)", "(DX)(DX*1)", 1), + "virtual frame": strings.Replace(source, "(DX)", "(SP)", 1), + "unknown static": strings.Replace(source, "(DX)", "·external(SB)", 1), + "numeric base": strings.Replace(source, load, "MOVL $42,DX\n"+load, 1), + "unknown store data": strings.Replace(source, load, load+"\n"+row.op+" X1,(DX)", 1), + "missing path": missing, + } { + t.Run(name, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", bad, "", true) + if name == "numeric base" { + x86RequireNumericNativeSourceContext(t, bad, declaration, "amd64") + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, bad, declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if name == "numeric base" { + if err != nil || !strings.Contains(ir, fmt.Sprintf("load volatile i%d,", row.width*8)) { + t.Fatal("complete NUM address lost the GP/SIMD transfer width", api, err) + } + x86NumericNativeAddressObject(t, ir, "x86_64-unknown-linux-gnu") + continue + } + if name == "unknown store data" || name == "missing path" { + x86VectorScratchCodegenControl(t, bad, declaration, "amd64", "x86_64-unknown-linux-gnu", ir, err) + continue + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved native transfer boundary accepted", api, err) + } + } + }) + } + }) + } +} + +func TestX86GPSIMDNativeResultBytesAndOrigins(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86GPSIMDNativeOracle { + t.Run(arch+"/"+row.op, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, row.op+" -1(DX)(BX*1),X1") + form, known, err := x86GPSIMDTransferProperties(ins, arch) + if !known || err != nil || form.bits != row.width*8 { + t.Fatal("native memory width changed", form, err) + } + state := x86ResultState{flags: x86ByteFlagsAll} + base, _, _ := x86ResultGP(DX, 4, arch) + index, _, _ := x86ResultGP(BX, 4, arch) + state.pointerGP[base], state.gp[index] = true, 0xff + state.writeVector(1, 64, 0x0123456789ab0080, false) + before := state.clone() + want := uint64(0x0123456789abffff) + if strings.HasPrefix(row.op, "V") { + want = ^uint64(0) + } + if !x86ResultGPSIMDTransferWithNativeMemory(ins, form, &state, nil, 0, arch, true) || + state.vectorBytes(1) != want || state.flags != before.flags || + state.gp != before.gp || state.pointerGP != before.pointerGP { + t.Fatal("native transfer lost loaded/zero bytes or changed unrelated state") + } + for _, mutation := range []string{"decoded", "raw", "raw return", "address", "unbound address", "segment"} { + changed, rejected := ins, before.clone() + switch mutation { + case "decoded": + changed.x86Encoded = true + case "raw": + changed.x86SourceRaw = new(x86SourceRawInstruction) + case "raw return": + changed.x86RawNearReturn = new(x86RawNearReturnForm) + case "address": + changed.x86AddressBits = 32 + case "unbound address": + changed.x86UnboundAddressOverride = true + case "segment": + changed.x86SegmentPrefix = 0x64 + } + if x86ResultGPSIMDTransferWithNativeMemory(changed, form, &rejected, nil, 0, arch, true) || !reflect.DeepEqual(rejected, before) { + t.Fatal("unbound instruction origin acquired native result bytes", mutation) + } + } + }) + } + } +} diff --git a/amd64_gp_simd_native_runtime_unix_test.go b/amd64_gp_simd_native_runtime_unix_test.go new file mode 100644 index 000000000..fcc8c9d54 --- /dev/null +++ b/amd64_gp_simd_native_runtime_unix_test.go @@ -0,0 +1,67 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86GPSIMDNativeResultsAndFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, originals, rows strings.Builder + count := 0 + for _, row := range x86GPSIMDNativeOracle { + for discarded := 0; discarded < 2; discarded++ { + name, original := fmt.Sprintf("F%d", count), fmt.Sprintf("Original%d", count) + body := row.op + " -1(DX)(BX*1),X0\nPEXTRQ $0,X0,AX\n" + native := row.original + " (DI),X0\nMOVQ X0,AX\n" + if discarded == 1 { + body += "MOVQ $0x87654321fedcba98,AX\n" + native += "MOVQ $0x87654321fedcba98,AX\n" + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),DX\nMOVL $1,BX\n%sMOVQ AX,ret+8(FP)\nRET\n", name, body) + fmt.Fprintf(&declarations, "func %s(p *[8]byte) uint64\n", name) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+native+"RET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\nextern uint64_t %s(const uint8_t*);\n", name, original) + fmt.Fprintf(&rows, "{%s,%s,%d,%d},\n", name, original, row.width, discarded) + count++ + } + } + if count != 10 { + t.Fatal("incomplete live/dead GP/X family", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf(x86NativeScalarReadRuntimeC, prototypes.String(), originals.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text, level := ir, "-O0" + if optimized { + text, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + output := execution.compileAndRunLLVM(t, "gp-x-native", text, driver, level) + if string(output) != "10 native scalar kernels: numerical 1280, faults 120, tripwires 2; failures 0\n" { + t.Fatal("original/LLVM GP/X numerical/zero bytes or read widths differ:", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } +} diff --git a/amd64_gp_simd_transfer_boundaries_test.go b/amd64_gp_simd_transfer_boundaries_test.go new file mode 100644 index 000000000..7f28d9cc9 --- /dev/null +++ b/amd64_gp_simd_transfer_boundaries_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GPSIMDTransferRejectsCompleteSiblingInvalidFormats(t *testing.T) { + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + rows := []string{op + ".Z AX,X0", op + ".BCST (AX),X0", op + ".SAE X0,AX"} + for _, args := range []string{"Y0,X1", "Z0,X1", "K1,X1", "X0,Y1", "X0,Z1", "X0,K1", "$7,X1", "$1.25,X1", + "X32,X0", "X0,X32", "X0,K1,X1", "X0,$1", "X0,$1.25", "(AX)(SP*1),X1"} { + rows = append(rows, op+" "+args) + } + if op == "MOVL" || op == "VMOVD" { + rows = append(rows, op+" X0,X1") + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + ins := x86YMMReadSource(t, row) + if _, known, err := x86GPSIMDTransferProperties(ins, arch); !known || err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("invalid syntax became an owned valid/context form", row, arch, known, err) + } + if x86CallerVectorMoveKnown(ins) { + t.Fatal("invalid sibling acquired caller effect") + } + x86RequireZeroMaskAPIOutcome(t, arch, source, false) + } + }) + } + } +} + +func TestX86GPSIMDTransferActualCallerKeepsSafetyBoundaries(t *testing.T) { + for name, body := range map[string]string{ + "unknown GP": "VMOVD BX,X1\nVMOVSS X1,(DX)", + "partial GP": "MOVB $1,BX\nVMOVD BX,X1\nVMOVSS X1,(DX)", + "pointer bits": "VMOVQ DI,X1\nVMOVSS X1,(DX)", + "stack identity": "MOVQ SP,X1\nVMOVSS X1,(DX)", + "frame identity": "MOVQ BP,X1\nVMOVSS X1,(DX)", + "undefined X": "VMOVD X0,AX\nMOVL AX,(DX)", + "partial X for qword": "MOVSS (DI),X2\nMOVSS X2,X0\nVMOVQ X0,AX\nMOVL AX,(DX)", + "unknown memory": "VMOVD (BX),X1\nVMOVSS X1,(DX)", + "length overread": "VMOVQ 4(CX),X1\nVMOVSS X1,(DX)", + "result overread": "MOVQ (DX),X1\nVMOVSS X1,(DX)", + "input write": "VMOVUPS (DI),X0\nVMOVD X0,(DI)", + "length write": "VMOVUPS (DI),X0\nVMOVQ X0,(CX)", + "wide result": "VMOVUPS (DI),X0\nVMOVQ X0,(DX)", + "result overlap": "VMOVUPS (DI),X0\nVMOVD X0,1(DX)", + "pointer lost as GP": "VMOVUPS (DI),X0\nVMOVQ X0,DI\nVMOVD (DI),X1\nVMOVSS X1,(DX)", + "g identity overwritten": "VMOVUPS (DI),X0\nVMOVQ X0,R14\nVMOVSS X0,(DX)", + "unknown join": "CMPQ AX,$0\nJE joined\nVMOVD AX,X1\njoined:\nVMOVSS X1,(DX)", + "legacy upper unknown": "MOVL AX,X1\nVEXTRACTF128 $1,Y1,X0\nVMOVSS X0,(DX)", + "MMX native state": "MOVQ X0,M0\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound state acquired caller permission", err) + } + }) + } +} + +func TestX86GPSIMDTransferParserRejectsInvalidScale(t *testing.T) { + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + source := "TEXT f(SB),4,$0-0\n" + op + " X0,(AX)(CX*3)\nRET\n" + t.Run(op, func(t *testing.T) { + if _, err := Parse(ArchAMD64, source); err == nil || + !strings.Contains(err.Error(), `invalid x86 symbolic address register/scale group: "(CX*3)"`) { + t.Fatal("parser repaired an invalid scale token", err) + } + for _, arch := range []string{"amd64", "386"} { + goAssemblerOracleOutput(t, arch, source, "", false) + } + for _, target := range x86RawReturnTargets { + if _, err := x86ScalarFPBitViewAPI(t, strings.Replace(source, "f(SB)", "·view(SB)", 1), + "func view()", target.arch, target.triple, "Go"); err == nil || + !strings.Contains(err.Error(), `invalid x86 symbolic address register/scale group: "(CX*3)"`) { + t.Fatal("Go API repaired an invalid original scale", target.triple, err) + } + } + }) + } +} + +func TestX86GPSIMDTransferFPCompleteFieldsThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + bits := 32 + if op == "MOVQ" || op == "MOVD" || op == "VMOVQ" { + bits = 64 + } + for _, input := range []string{"uint64", fmt.Sprintf("[%d]byte", bits/8)} { + fixture := x86ScalarFPBitFixture{name: op + "/" + input, + declaration: fmt.Sprintf("func view(x %s) uint%d\n", input, bits), + body: op + " x+0(FP),X1\n" + op + " X1,ret+8(FP)"} + for _, target := range x86RawReturnTargets { + t.Run(fixture.name+"/"+target.triple, func(t *testing.T) { + source := x86ScalarFPBitAssembly(t, fixture, target.arch) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, fixture.declaration, target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("complete independent field lost its data", api, err) + } + compileLLVMToObject(t, llc, target.triple, "transfer-fp-"+api+".ll", "transfer-fp-"+api+".o", ir) + } + }) + } + } + } +} diff --git a/amd64_gp_simd_transfer_family_test.go b/amd64_gp_simd_transfer_family_test.go new file mode 100644 index 000000000..449f3d84f --- /dev/null +++ b/amd64_gp_simd_transfer_family_test.go @@ -0,0 +1,169 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// This table is an independent projection of all twelve original AVX rows, +// including their zcase/advances, W bits, transfer opcodes and memory tuples. +func x86OfficialGPSIMDTransferRows(t *testing.T) []string { + t.Helper() + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + var rows []string + counts := map[string]int{} + for _, rule := range rules { + if rule.Opcode != "VMOVD" && rule.Opcode != "VMOVQ" { + continue + } + classes := [][]string{{"Yxr", "Yml"}, {"Yml", "Yxr"}, {"YxrEvex", "Yml"}, {"Yml", "YxrEvex"}} + zCases := []string{"Zvex_r_v_rm", "Zvex_rm_v_r", "Zevex_r_v_rm", "Zevex_rm_v_r"} + offsets := []int{0, 2, 4, 7} + advances := []int{2, 2, 3, 3} + opcodes := []string{"0x7E", "0x6E", "0x7E", "0x6E"} + ws := []string{"W0", "W0", "W0", "W0"} + pps := []string{"66", "66", "66", "66"} + sources, destinations := []string{"X0", "AX", "X17", "AX"}, []string{"AX", "X1", "AX", "X17"} + table, count, tuple := "_yvmovd", 10, "evexN4" + if rule.Opcode == "VMOVQ" { + classes = [][]string{{"Yxr", "Yml"}, {"Yxr", "Yxm"}, {"Yml", "Yxr"}, {"Yxm", "Yxr"}, + {"YxrEvex", "Yml"}, {"YxrEvex", "YxmEvex"}, {"Yml", "YxrEvex"}, {"YxmEvex", "YxrEvex"}} + zCases = []string{"Zvex_r_v_rm", "Zvex_r_v_rm", "Zvex_rm_v_r", "Zvex_rm_v_r", "Zevex_r_v_rm", "Zevex_r_v_rm", "Zevex_rm_v_r", "Zevex_rm_v_r"} + offsets, advances = []int{0, 2, 4, 6, 8, 11, 14, 17}, []int{2, 2, 2, 2, 3, 3, 3, 3} + opcodes = []string{"0x7E", "0xD6", "0x6E", "0x7E", "0x7E", "0xD6", "0x6E", "0x7E"} + ws, pps = []string{"W1", "W0", "W1", "W0", "W1", "W1", "W1", "W1"}, []string{"66", "66", "66", "F3", "66", "66", "66", "F3"} + sources, destinations = []string{"X0", "X0", "AX", "X0", "X17", "X17", "AX", "X0"}, []string{"AX", "X1", "X1", "X1", "AX", "X0", "X17", "X17"} + table, count, tuple = "_yvmovq", 20, "evexN8" + } + index := rule.RowIndex + if index < 0 || index >= len(classes) || rule.Ytab != table || rule.Prefix != "Pavx" || + !reflect.DeepEqual(rule.Classes, classes[index]) || rule.ZCase != zCases[index] || + rule.EncodingOffset != offsets[index] || rule.ZOffset != advances[index] || len(rule.OpBytes) != count { + t.Fatalf("unmodeled complete GP/SIMD transfer row: %+v", rule) + } + generation := "vex" + if strings.Contains(zCases[index], "evex") { + generation = "evex" + } + terms := []string{"avxEscape", generation + "128", generation + pps[index], generation + "0F", generation + ws[index]} + if !reflect.DeepEqual(strings.Split(rule.OpBytes[offsets[index]], " | "), terms) { + t.Fatalf("unmodeled GP/SIMD encoding generation/W/prefix: %+v", rule) + } + opcodeIndex := offsets[index] + 1 + if generation == "evex" { + if rule.OpBytes[opcodeIndex] != tuple { + t.Fatalf("unmodeled transfer memory width/mask attributes: %+v", rule) + } + opcodeIndex++ + } + if rule.OpBytes[opcodeIndex] != opcodes[index] { + t.Fatalf("unmodeled complete transfer opcode: %+v", rule) + } + rows = append(rows, rule.Opcode+" "+sources[index]+","+destinations[index]) + counts[rule.Opcode]++ + } + if counts["VMOVD"] != 4 || counts["VMOVQ"] != 8 || len(rows) != 12 { + t.Fatalf("complete original transfer denominator changed: %v", counts) + } + return rows +} + +func TestX86GPSIMDTransferCompleteCallerRegistration(t *testing.T) { + rows := x86OfficialGPSIMDTransferRows(t) + for _, op := range []string{"MOVL", "MOVQ", "MOVD"} { + rows = append(rows, op+" AX,X0", op+" X0,AX", op+" (DI),X0", op+" X0,(DX)") + if op != "MOVL" { + rows = append(rows, op+" X0,X1") + } + } + for _, op := range []string{"VMOVD", "VMOVQ"} { + rows = append(rows, op+" (DI),X0", op+" X0,(DX)") + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + ins := x86YMMReadSource(t, row) + if !x86CallerVectorMoveKnown(ins) { + t.Fatal("Go-accepted whole-family transfer has no caller effect", row) + } + }) + } +} + +func TestX86GPSIMDTransferActualSelectedCallerThreeAPIs(t *testing.T) { + count := 0 + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "MOVDQ2Q", "VMOVD", "VMOVQ"} { + for _, direction := range []string{"gp-to-x", "x-to-gp", "memory-load", "memory-store", "x-to-x"} { + if direction == "x-to-x" && (op == "MOVL" || op == "VMOVD") { + continue + } + count++ + t.Run(op+"/"+direction, func(t *testing.T) { + body := "MOVQ $7,AX\n" + op + " AX,X1\nVMOVSS X1,(DX)" + switch direction { + case "x-to-gp": + body = "VMOVUPS (DI),X0\n" + op + " X0,AX\nMOVL AX,(DX)" + case "memory-load": + body = op + " (DI),X1\nVMOVSS X1,(DX)" + case "memory-store": + body = "VMOVUPS (DI),X0\n" + op + " X0,(DX)" + case "x-to-x": + body = "VMOVUPS (DI),X0\n" + op + " X0,X1\nVMOVSS X1,(DX)" + } + source := x86CallerFixtureSource + if direction == "memory-store" && op != "MOVL" && op != "VMOVD" { + source = strings.NewReplacer("var res float32", "var res uint64", ") float32 {", ") uint64 {").Replace(source) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, source, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Error("text caller consumer", err) + } else { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + compileLLVMToObject(t, llc, options.TargetTriple, "transfer-text.ll", "transfer-text.o", ir) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Error("module caller consumer", err) + } else { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + compileLLVMToObject(t, llc, options.TargetTriple, "transfer-module.ll", "transfer-module.o", module.String()) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, options.TargetTriple); err != nil { + t.Error("actual selected typed caller", err) + } + }) + } + } + // Scope denominator is independent of the typed implementation table. + if count != 28 { + t.Fatalf("caller direction denominator changed: %d", count) + } +} diff --git a/amd64_gp_simd_transfer_fault_runtime_test.go b/amd64_gp_simd_transfer_fault_runtime_test.go new file mode 100644 index 000000000..9c2882a4b --- /dev/null +++ b/amd64_gp_simd_transfer_fault_runtime_test.go @@ -0,0 +1,111 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The reference executes the actual Go object's three legacy opcodes. VEX +// translation is compared only for its equivalent unmasked read width, not +// claimed as original AVX/EVEX execution on Rosetta. +func TestCrossLinuxRuntimeMatrixX86GPSIMDTransferOriginalDeadReadFaults(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, references, pairs strings.Builder + sigs := map[string]FuncSig{} + for index, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + name := fmt.Sprint("transferfault", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-8\nMOVQ p+0(FP),AX\n%s (AX),X1\nRET\n", name, op) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + original := op + if op == "VMOVD" { + original = "MOVL" + } else if op == "VMOVQ" { + original = "MOVQ" + } + listing := x86InsertExtractGoListing(t, "amd64", "TEXT f(SB),4,$0-0\n"+original+" (DI),X1\nRET\n") + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + if len(code) < 4 || code[len(code)-1] != 0xc3 { + t.Fatal("original Go memory opcode did not retain its exact ordinary RET", op, code) + } + var operands []string + for _, value := range code[:len(code)-1] { + operands = append(operands, fmt.Sprintf("0x%02x", value)) + } + fmt.Fprintf(&references, "extern void %s(const void*);\nstatic void original%d(const void *p) {\n", name, index) + fmt.Fprintf(&references, " __asm__ volatile(\".byte %s\"::\"D\"(p):\"xmm1\",\"memory\");\n}\n", strings.Join(operands, ",")) + if index > 0 { + pairs.WriteString(",") + } + width := 8 + if op == "MOVL" || op == "VMOVD" { + width = 4 + } + fmt.Fprintf(&pairs, "{original%d,%s,%d}", index, name, width) + } + oracle := `#include +#include +#include +#include +#include +#include +#include +` + references.String() + ` +typedef void (*operation)(const void*); +struct pair {operation original,translated;unsigned width;}; +static const struct pair pairs[]={ ` + pairs.String() + ` }; +static int outcome(operation fn,const void *p) { + pid_t pid=fork(); if(pid<0) return -1; + if(!pid) { + struct rlimit limit={0,0}; if(setrlimit(RLIMIT_CORE,&limit)) _exit(120); + fn(p); _exit(0); + } + int status; if(waitpid(pid,&status,0)!=pid) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status)) return WTERMSIG(status); + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); if(page<128) return 1; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(region==MAP_FAILED || mprotect(region+page,page,PROT_NONE)) return 2; + for(unsigned op=0;op<5;op++) for(unsigned extent=0;extent<=8;extent++) { + const void *p=region+page-extent; + int original=outcome(pairs[op].original,p),translated=outcome(pairs[op].translated,p); + if(original!=translated || (extent>=pairs[op].width ? original!=0 : original!=SIGSEGV && original!=SIGBUS)) { + fprintf(stderr,"read op=%u extent=%u original=%d translated=%d\n",op,extent,original,translated); + return 3; + } + } + for(unsigned op=0;op<5;op++) { + int original=outcome(pairs[op].original,0),translated=outcome(pairs[op].translated,0); + if(original!=translated || (original!=SIGSEGV && original!=SIGBUS)) return 4; + } + return munmap(region,2*page)!=0; +} +` + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "gp-simd-original-dead-faults", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_gp_simd_transfer_forms.go b/amd64_gp_simd_transfer_forms.go new file mode 100644 index 000000000..cdbf92a22 --- /dev/null +++ b/amd64_gp_simd_transfer_forms.go @@ -0,0 +1,126 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86GPSIMDTransferSpec struct { + bits int + legacy, rawOnly bool +} + +// MOVD is Go's MOVQ alias, not Intel's 32-bit MOVD. Ordinary GP-only +// MOVL/Q/D and the separate MMX rows retain their existing scalar grammar. +var x86GPSIMDTransferSpecs = map[Op]x86GPSIMDTransferSpec{ + "MOVL": {bits: 32, legacy: true}, + "MOVQ": {bits: 64, legacy: true}, + "MOVD": {bits: 64, legacy: true}, + "MOVDQ2Q": {bits: 64, legacy: true}, + "VMOVD": {bits: 32}, + "VMOVQ": {bits: 64}, + "VMOVW": {bits: 16, rawOnly: true}, +} + +type x86GPSIMDTransferForm struct { + spec x86GPSIMDTransferSpec + bits, memory int + sourceX, destinationX, mmx bool + memoryWrite bool +} + +// A scalar transfer replaces its low X bytes, zeroes the rest of X, and +// preserves or clears wider aliases according to its original encoding. +// Numerical availability does not supply a pointer or memory access role. +func (f x86GPSIMDTransferForm) definedVectorBytes(source, previous uint64) uint64 { + low := x86CallerVectorByteMask(int64(f.bits / 8)) + upper := ^uint64(0xffff) + if f.spec.legacy { + upper &= previous + } + return upper | (0xffff &^ low) | (source & low) +} + +// Encoding/width selection grants neither initialized data nor memory backing. +// In mode32 VEX.W is ignored for GP transfers, but Q memory/X copies stay 64-bit. +func x86GPSIMDTransferProperties(ins Instr, arch string) (x86GPSIMDTransferForm, bool, error) { + base, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86GPSIMDTransferSpecs[Op(base)] + if !known { + return x86GPSIMDTransferForm{}, false, nil + } + f := x86GPSIMDTransferForm{spec: spec, bits: spec.bits, memory: -1} + hasX := false + for _, arg := range ins.Args { + hasX = hasX || isAMD64GoXRegister(arg) + } + if spec.legacy && !hasX { + return f, false, nil + } + fail := func() (x86GPSIMDTransferForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside complete Go GP/X transfer grammar: %q", arch, ins.Op, ins.Raw) + } + if arch == "" { + arch = "amd64" + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != 2 || spec.rawOnly && !ins.x86Encoded { + return fail() + } + src, dst := ins.Args[0], ins.Args[1] + f.sourceX, f.destinationX = isAMD64GoXRegister(src), isAMD64GoXRegister(dst) + for _, arg := range ins.Args { + if isAMD64GoXRegister(arg) { + index, _ := amd64ParseXReg(arg.Reg) + if spec.legacy && index >= 16 { + return fail() + } + } + } + if f.sourceX && f.destinationX { + if spec.bits != 64 { + return fail() + } + } else if f.sourceX || f.destinationX { + other, index := src, 0 + if f.sourceX { + other, index = dst, 1 + } + if isAMD64MemoryOperand(other) && x86PackedShiftMemoryValid(other, arch) { + f.memory, f.memoryWrite = index, index == 1 + } else if other.Kind == OpReg && isX86YrlRegisterForArch(other.Reg, arch) { + if spec.legacy && spec.bits == 64 && arch == "386" { + return fail() + } + if arch == "386" && !spec.legacy && f.bits > 32 { + f.bits = 32 + } + } else if f.sourceX && spec.legacy && spec.bits == 64 && isAMD64MOVDMRegister(other) { + f.mmx = true // Go's MOVDQ2Q row, never a physical caller-state proof. + } else if !f.sourceX && other.Kind == OpImm && ins.x86RIPLiteral && + !other.ImmIsFloat && other.ImmRaw == "" { + // Only the decoder's already-bound RIP data receipt supplies this + // representation. A named literal cannot acquire it. + } else { + return fail() + } + } else { + return fail() + } + return f, true, x86Mode32VectorRegisterContext(ins, arch) +} + +// Preserve the original operand class before normalization can rewrite an +// invalid register/address spelling. Direct and textual APIs share this guard. +func validateX86GPSIMDTransferFile(file *File, arch string) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + if _, known, err := x86GPSIMDTransferProperties(ins, arch); known && err != nil { + return fmt.Errorf("%s: %w", fn.Sym, err) + } + } + } + return nil +} diff --git a/amd64_gp_simd_transfer_grammar_test.go b/amd64_gp_simd_transfer_grammar_test.go new file mode 100644 index 000000000..5eca495d9 --- /dev/null +++ b/amd64_gp_simd_transfer_grammar_test.go @@ -0,0 +1,218 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86GPSIMDTransferCompleteLegacyOriginalTables(t *testing.T) { + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, table := range []struct{ op, name, prefix, bytes, rows string }{ + {"MOVB", "ymovb", "Pb", "0x88,0x8a,0xb0,0xc6,00", ` + {Zr_m,1,argList{Yrb,Ymb}},{Zm_r,1,argList{Ymb,Yrb}}, + {Zib_rp,1,argList{Yi32,Yrb}},{Zibo_m,2,argList{Yi32,Ymb}},`}, + {"MOVW", "ymovw", "Pe", "0x89,0x8b,0xb8,0xc7,00,0", ` + {Zr_m,1,argList{Yrl,Yml}},{Zm_r,1,argList{Yml,Yrl}}, + {Zil_rp,1,argList{Yi32,Yrl}},{Zilo_m,2,argList{Yi32,Yml}}, + {Zaut_r,2,argList{Yiauto,Yrl}},`}, + {"MOVL", "ymovl", "Px", "0x89,0x8b,0xb8,0xc7,00,0x6e,0x7e,Pe,0x6e,Pe,0x7e,0", ` + {Zr_m,1,argList{Yrl,Yml}},{Zm_r,1,argList{Yml,Yrl}}, + {Zil_rp,1,argList{Yi32,Yrl}},{Zilo_m,2,argList{Yi32,Yml}}, + {Zm_r_xm,1,argList{Yml,Ymr}},{Zr_m_xm,1,argList{Ymr,Yml}}, + {Zm_r_xm,2,argList{Yml,Yxr}},{Zr_m_xm,2,argList{Yxr,Yml}}, + {Zaut_r,2,argList{Yiauto,Yrl}},`}, + {"MOVQ", "ymovq", "Pw8", "0x6f,0x7f,Pf2,0xd6,Pf3,0x7e,Pe,0xd6,0x89,0x8b,0xb8,0xc7,00,0x6e,0x7e,Pe,0x6e,Pe,0x7e,0", ` + {Zm_r_xm_nr,1,argList{Ym,Ymr}},{Zr_m_xm_nr,1,argList{Ymr,Ym}}, + {Zm_r_xm_nr,2,argList{Yxr,Ymr}},{Zm_r_xm_nr,2,argList{Yxm,Yxr}}, + {Zr_m_xm_nr,2,argList{Yxr,Yxm}},{Zr_m,1,argList{Yrl,Yml}}, + {Zm_r,1,argList{Yml,Yrl}},{Ziq_rp,1,argList{Yi64,Yrl}}, + {Zilo_m,2,argList{Yi32,Yml}},{Zm_r_xm,1,argList{Ymm,Ymr}}, + {Zr_m_xm,1,argList{Ymr,Ymm}},{Zm_r_xm,2,argList{Yml,Yxr}}, + {Zr_m_xm,2,argList{Yxr,Yml}},{Zaut_r,1,argList{Yiauto,Yrl}},`}, + } { + t.Run(table.op, func(t *testing.T) { + definition := regexp.MustCompile(`(?s)var ` + table.name + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(string(source)) + entry := regexp.MustCompile(`\{A` + table.op + `, (\w+), (\w+), opBytes\{([^}]+)\}\}`).FindStringSubmatch(string(source)) + strip := func(text string) string { + text = regexp.MustCompile(`//[^\n]*`).ReplaceAllString(text, "") + return strings.Join(strings.Fields(text), "") + } + if len(definition) != 2 || len(entry) != 4 || entry[1] != table.name || entry[2] != table.prefix { + t.Fatal("complete legacy row/prefix/opcode inventory changed", entry, definition) + } + inventories := []struct{ name, rows, bytes string }{{"current", table.rows, table.bytes}} + if table.op == "MOVQ" { + // Go 1.20 has a separate signed-imm32 row before its imm64 + // row. Pin that complete historical inventory, rather than + // discarding scalar rows to make the GP/X subset look equal. + inventories = append(inventories, struct{ name, rows, bytes string }{ + "signed-imm32", + strings.Replace(table.rows, "{Ziq_rp,1,argList{Yi64,Yrl}}", + "{Zilo_m,2,argList{Ys32,Yrl}},{Ziq_rp,1,argList{Yi64,Yrl}}", 1), + "0x6f,0x7f,Pf2,0xd6,Pf3,0x7e,Pe,0xd6,0x89,0x8b,0xc7,00,0xb8,0xc7,00,0x6e,0x7e,Pe,0x6e,Pe,0x7e,0", + }) + } + for _, inventory := range inventories { + if strip(definition[1]) == strip(inventory.rows) && strip(entry[3]) == inventory.bytes { + t.Logf("%s complete %s inventory: %s", runtime.Version(), table.op, inventory.name) + return + } + } + t.Fatal("complete legacy row/prefix/opcode inventory changed", entry, definition) + }) + } +} + +func TestX86GPSIMDTransferAllRegisterAndAddressAxes(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var assembly strings.Builder + assembly.WriteString("TEXT f(SB),4,$0-0\n") + count := 0 + appendRow := func(op, src, dst string) { + row := op + " " + src + "," + dst + ins := x86YMMReadSource(t, row) + before := fmt.Sprintf("%#v", ins) + if _, known, err := x86GPSIMDTransferProperties(ins, arch); !known || err != nil { + t.Fatal("independent valid axis not consumed", row, err) + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("grammar mutated original source") + } + fmt.Fprintln(&assembly, row) + count++ + } + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + legacy := !strings.HasPrefix(op, "V") + gps := []string{"AX", "BX", "CX", "DX", "DI", "SI", "BP", "SP"} + if arch == "amd64" { + gps = append(gps, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + } + if arch == "amd64" || !legacy || op == "MOVL" { + for _, gp := range gps { + appendRow(op, gp, "X0") + appendRow(op, "X0", gp) + } + } + limit := 8 + if arch == "amd64" { + limit = 16 + if !legacy { + limit = 32 + } + } + for index := 0; index < limit; index++ { + x := fmt.Sprint("X", index) + appendRow(op, "3(AX)", x) + appendRow(op, x, "5(BX)") + if op == "MOVQ" || op == "MOVD" || op == "VMOVQ" { + appendRow(op, x, "X0") + appendRow(op, x, x) + } + } + for _, memory := range []string{"(AX)(CX*1)", "7(AX)(CX*2)", "-9(AX)(CX*4)", "11(AX)(CX*8)"} { + appendRow(op, memory, "X1") + appendRow(op, "X1", memory) + } + } + assembly.WriteString("RET\n") + goAssemblerOracleOutput(t, arch, assembly.String(), "", true) + want := 552 + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + want, triples = 216, []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + if count != want { + t.Fatalf("independent register/address denominator changed: %d, want %d", count, want) + } + for _, triple := range triples { + t.Run(triple, func(t *testing.T) { + // Keep every original uninitialized axis as a Go/public-Context + // control, separate from initialized public object owners. + ir := x86UnboundVoidScalarFormatObject(t, assembly.String(), "f", arch, triple) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, text, level) + } + } + }) + } + }) + } +} + +func TestX86GPSIMDTransferPartialByteKnowledgeAndUnaffectedState(t *testing.T) { + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + width := 4 + if op == "MOVQ" || op == "MOVD" || op == "VMOVQ" { + width = 8 + } + for _, direction := range []string{"gp-to-x", "x-to-gp", "x-to-x"} { + if direction == "x-to-x" && width == 4 { + continue + } + t.Run(op+"/"+direction, func(t *testing.T) { + for seed := 0; seed < 256; seed++ { + initial := x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, gOriginal: true, numericFlags: true, + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}}, known: map[Reg]uint8{AX: uint8(seed), BX: 0x53}, + vectors: map[Reg]uint64{"Z0": uint64(seed), "Z1": 0xa735de42c9156ef0, "Z31": 0x1927}, + masks: map[Reg]bool{"K1": true}, + } + args := "AX,X1" + if direction == "x-to-gp" { + args = "X0,AX" + } else if direction == "x-to-x" { + args = "X0,X1" + } + ins := x86YMMReadSource(t, op+" "+args) + state, want := x86CloneCallerState(initial), x86CloneCallerState(initial) + low := uint64(seed) & (uint64(1)< +#include +#include +` + declarations.String() + ` +static int check(void (*program)(uint8_t*,const uint8_t*,uint64_t),const char *name, + unsigned width,int kind,int vector) { + uint64_t seeds[]={0,UINT64_MAX,UINT64_C(0x98af3bce72510d64),UINT64_C(0x8000000000000001)}; + for(unsigned s=0;s<4;s++) { + uint8_t input[82],original[82],output[84],before[84],want[84]; + for(unsigned i=0;i<82;i++) input[i]=(uint8_t)(i*37+149); + for(unsigned i=0;i<84;i++) output[i]=(uint8_t)(i*19+71); + memcpy(original,input,sizeof(input)); memcpy(before,output,sizeof(output)); + memcpy(want,before,sizeof(want)); + uint64_t gp=UINT64_MAX; + if(kind==1) { gp=0; memcpy(&gp,input+1,width); } + else if(kind==3) memcpy(want+4,input+1,width); + else { + for(unsigned i=0;i<64;i++) { + uint8_t value=before[1+i]; + if(i>=16) { if(vector) value=0; } + else if(i>=width) value=0; + else if(kind==0) value=(uint8_t)(seeds[s]>>(i*8)); + else if(kind==2) value=input[4+i]; + else if(kind==4) value=input[1+i]; + want[1+i]=value; + } + } + memcpy(want+65,&gp,8); want[73]=1; want[74]=0; + program(output+1,input+1,seeds[s]); + if(memcmp(input,original,sizeof(input)) || memcmp(output,want,sizeof(output))) { + for(unsigned i=0;i<84;i++) if(output[i]!=want[i]) + fprintf(stderr,"%s seed=%u byte=%u got=%x want=%x\n",name,s,i,output[i],want[i]); + return 1; + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "gp-simd-transfer-complete-bytes", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_gpsimd_overwritten_store_test.go b/amd64_gpsimd_overwritten_store_test.go new file mode 100644 index 000000000..2679f1d08 --- /dev/null +++ b/amd64_gpsimd_overwritten_store_test.go @@ -0,0 +1,170 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Two different native writes to the same address must remain observable even +// when no numerical consumer reads the first one. Plan 9 MOVD is the 64-bit +// MOVQ alias; it is not Intel's 32-bit MOVD. Expected widths below are independent +// of the production GP/SIMD transfer grammar and emitter. +func TestX86GPSIMDOverwrittenNativeStoresThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + forms := []struct { + op string + bits int + }{{"MOVL", 32}, {"MOVQ", 64}, {"MOVD", 64}, {"VMOVD", 32}, {"VMOVQ", 64}} + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var assembly, declarations strings.Builder + declarations.WriteString("package stores\n") + sigs := map[string]FuncSig{} + pointerBytes, pointerMove := 8, "MOVQ" + if target.arch == "386" { + pointerBytes, pointerMove = 4, "MOVL" + } + for index, form := range forms { + for _, memory := range []bool{true, false} { + name := fmt.Sprintf("Store%d_%t", index, memory) + destination := fmt.Sprintf("sink%d<>(SB)", index) + sig := FuncSig{Name: name, Ret: Void} + frame := 0 + if memory { + frame, destination = pointerBytes, "(AX)" + sig.Args = []LLVMType{Ptr} + sig.Frame = FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}} + fmt.Fprintf(&declarations, "func %s(p *[8]byte)\n", name) + } else { + fmt.Fprintf(&declarations, "func %s()\n", name) + fmt.Fprintf(&assembly, "GLOBL %s,8,$8\n", destination) + } + sigs[name] = sig + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-%d\n", name, frame) + if memory { + fmt.Fprintf(&assembly, "%s p+0(FP),AX\n", pointerMove) + } + fmt.Fprintf(&assembly, "PXOR X0,X0\n%s X0,%s\nPCMPEQB X1,X1\n%s X1,%s\nRET\n", form.op, destination, form.op, destination) + } + } + // The original assembler independently establishes each store spelling + // and its real 386/amd64 operand acceptance before any translation. + goAssemblerOracleOutput(t, target.arch, assembly.String(), "", true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + resolve := func(name string) string { return strings.TrimPrefix(name, "·") } + options := Options{Goarch: target.arch, TargetTriple: target.triple, ResolveSym: resolve, Sigs: sigs} + textIR, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "stores", declarations.String()) + goModule, err := translateGoModuleInContext(ctx, pkg, []byte(assembly.String()), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve}) + if goModule != nil && !goModule.Module.IsNil() { + defer goModule.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + for _, api := range []struct{ name, ir string }{{"text", textIR}, {"module", module.String()}, {"go-module", goModule.Module.String()}} { + t.Run(api.name, func(t *testing.T) { + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_default_O2=%t", optimized), func(t *testing.T) { + ir := api.ir + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + for index, form := range forms { + for _, memory := range []bool{true, false} { + name := fmt.Sprintf("Store%d_%t", index, memory) + t.Run(form.op+fmt.Sprintf("/OpMem=%t", memory), func(t *testing.T) { + x86GPSIMDCheckNativeStoreEffects(t, ir, name, form.bits) + }) + } + } + // Object emission is a separate backend-O0 axis, both before + // and after the explicit LLVM IR default pass pipeline. + compileLLVMToObject(t, llc, target.triple, "native-stores.ll", "native-stores.o", ir) + }) + } + }) + } + }) + } +} + +func x86GPSIMDCheckNativeStoreEffects(t *testing.T, ir, name string, bits int) { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := parseIRModuleInContext(ctx, ir) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + function := module.NamedFunction(name) + if function.IsNil() { + t.Fatal("native store function disappeared", name) + } + stores, observable := 0, 0 + for block := function.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.IsAStoreInst().IsNil() || x86GPSIMDStoreTargetsBackingAlloca(instruction.Operand(1)) { + continue // Register banks and synthesized ABI0 frames are not native writes. + } + typ := instruction.Operand(0).Type() + if typ.TypeKind() != llvm.IntegerTypeKind { + t.Fatal("native scalar write widened to a non-integer type", instruction.String()) + } + stores++ + if typ.IntTypeWidth() != bits { + t.Errorf("native store width=%d want=%d: %s", typ.IntTypeWidth(), bits, instruction.String()) + } + if instruction.IsVolatile() { + observable++ + } + } + } + t.Logf("native stores=%d observable=%d exact-width=%d", stores, observable, bits) + if stores != 2 || observable != 2 { + t.Errorf("overwritten native store effect lost: stores=%d observable=%d want=2/2\n%s", stores, observable, function.String()) + } +} + +func x86GPSIMDStoreTargetsBackingAlloca(pointer llvm.Value) bool { + for depth := 0; depth < 16; depth++ { + if !pointer.IsAAllocaInst().IsNil() { + return true + } + if pointer.IsAGetElementPtrInst().IsNil() && pointer.IsABitCastInst().IsNil() { + return false + } + pointer = pointer.Operand(0) + } + return false +} diff --git a/amd64_helper_edge_test.go b/amd64_helper_edge_test.go index 8b3a077c9..9fc9c20c5 100644 --- a/amd64_helper_edge_test.go +++ b/amd64_helper_edge_test.go @@ -1,6 +1,8 @@ package plan9asm import ( + "errors" + "math" "strings" "testing" ) @@ -154,14 +156,29 @@ func TestAMD64CtxHelperEdges(t *testing.T) { t.Fatalf("evalFPToI64(%d) = (%q, %v)", off, got, err) } } + if got, err := c.evalFPToF32(56); err != nil || got == "" { + t.Fatalf("evalFPToF32(float) = (%q, %v)", got, err) + } + if got, err := c.evalFPToF32(32); err != nil || got == "" { + t.Fatalf("evalFPToF32(i32 bits) = (%q, %v)", got, err) + } + if _, err := c.evalFPToF32(999); err == nil { + t.Fatal("evalFPToF32(missing slot) unexpectedly succeeded") + } c.fpParams[64] = FrameSlot{Offset: 64, Type: LLVMType("v4i32"), Index: 0} if _, err := c.evalFPToI64(64); err == nil { t.Fatalf("evalFPToI64(unsupported type) unexpectedly succeeded") } + if _, err := c.evalFPToF32(64); err == nil { + t.Fatal("evalFPToF32(unsupported type) unexpectedly succeeded") + } c.fpParams[72] = FrameSlot{Offset: 72, Type: I64, Index: 99} if _, err := c.evalFPToI64(72); err == nil { t.Fatalf("evalFPToI64(invalid index) unexpectedly succeeded") } + if _, err := c.evalFPToF32(72); err == nil { + t.Fatal("evalFPToF32(invalid index) unexpectedly succeeded") + } if err := c.storeFPResult(80, I64, "11"); err != nil { t.Fatalf("storeFPResult(i64->i32) error = %v", err) @@ -285,8 +302,8 @@ func TestAMD64CtxHelperEdges(t *testing.T) { "alloca <64 x i8>", "alloca i64", "ptrtoint ptr %arg0 to i64", - "bitcast double %arg6 to i64", - "bitcast float %arg7 to i32", + "load double, ptr %fp_arg_48", + "load float, ptr %fp_arg_56", "trunc i64 11 to i32", "inttoptr i64 12 to ptr", "bitcast i64 13 to double", @@ -356,9 +373,14 @@ func TestAMD64AtomicAndBranchEdges(t *testing.T) { c.blockBase = []int{0, 1, 2} c.blockByIdx = map[int]int{0: 0, 1: 1, 2: 2} - if ok, term, err := c.lowerAtomic("LOCK", Instr{Raw: "LOCK"}); !ok || term || err != nil { + beforeStandaloneLock := b.Len() + beforeLockTemporary, beforeLockPrefix := c.tmp, c.namedAtomicLock + if ok, term, err := c.lowerAtomic("LOCK", Instr{Raw: "LOCK"}); !ok || term || !errors.Is(err, ErrProbeNeedsContext) { t.Fatalf("lowerAtomic(LOCK) = (%v, %v, %v)", ok, term, err) } + if b.Len() != beforeStandaloneLock || c.tmp != beforeLockTemporary || c.namedAtomicLock != beforeLockPrefix { + t.Fatal("standalone LOCK changed IR or temporary state without a complete adjacent source operation") + } if ok, _, err := c.lowerAtomic("CMPXCHGL", Instr{ Raw: "CMPXCHGL CX, 8(BX)", Args: []Operand{ @@ -395,14 +417,14 @@ func TestAMD64AtomicAndBranchEdges(t *testing.T) { }); !ok || err != nil { t.Fatalf("lowerAtomic(XCHGQ sym) = (%v, %v)", ok, err) } - if ok, _, err := c.lowerAtomic("ANDB", Instr{ + if ok, _, err := c.lowerScalarMemoryRMW("ANDB", Instr{ Raw: "ANDB $1, 4(BX)", Args: []Operand{ {Kind: OpImm, Imm: 1}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 4}}, }, }); !ok || err != nil { - t.Fatalf("lowerAtomic(ANDB) = (%v, %v)", ok, err) + t.Fatalf("lowerScalarMemoryRMW(ANDB) = (%v, %v)", ok, err) } if ok, term, err := c.lowerAtomic("ORQ", Instr{ Raw: "ORQ AX, BX", @@ -505,16 +527,41 @@ func TestAMD64AtomicAndBranchEdges(t *testing.T) { if err := c.tailCallAndRet(Operand{Kind: OpReg, Reg: AX}); err == nil { t.Fatalf("tailCallAndRet(non-sym) unexpectedly succeeded") } + + var b386 strings.Builder + c386 := newX86Ctx(&b386, Func{}, FuncSig{Name: "example.ptr386", Ret: Ptr}, testResolveSym("example"), map[string]FuncSig{ + "example.word386": {Name: "example.word386", Ret: I32}, + }, "386", "i386-unknown-linux-gnu", false) + if err := c386.emitEntryAllocas(); err != nil { + t.Fatal(err) + } + if err := c386.tailCallAndRet(Operand{Kind: OpSym, Sym: "word386(SB)"}); err != nil { + t.Fatalf("386 tailCallAndRet(i32 -> ptr) error = %v", err) + } + if got := b386.String(); !strings.Contains(got, "inttoptr i32") || !strings.Contains(got, "ret ptr") { + t.Fatalf("386 tailCallAndRet(i32 -> ptr) output:\n%s", got) + } if err := c.tailCallIndirectAddrAndRet("123"); err != nil { t.Fatalf("tailCallIndirectAddrAndRet() error = %v", err) } out := b.String() + if strings.Contains(out, "cmpxchg ptr") { + t.Fatal("native CMPXCHGL silently acquired software atomic alignment or LOCK semantics") + } + if strings.Contains(out, "atomicrmw and ptr") { + t.Fatal("ordinary ANDB silently acquired implicit LOCK semantics") + } + if strings.Contains(out, "lock; xadd") { + t.Fatal("bare XADD borrowed LOCK from the rejected standalone prefix") + } for _, want := range []string{ - "cmpxchg ptr", - "atomicrmw add ptr", - "atomicrmw xchg ptr", - "atomicrmw and ptr", + "asm sideeffect \"cmpxchgl $3, $2; sete $1\"", + "asm sideeffect \"xadd", + "asm sideeffect \"xchg", + "load volatile i8", + "and i8", + "store volatile i8", "icmp eq i8", "icmp slt i8", "call i64", @@ -550,19 +597,30 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { }, } c, b := newAMD64CtxWithFuncForTest(t, fn, sig, nil) - mustLower := func(op Op, ins Instr) { + mustLowerWith := func(lower func(Op, Instr) (bool, bool, error), op Op, ins Instr) { t.Helper() - if ok, term, err := c.lowerArith(op, ins); !ok || term || err != nil { - t.Fatalf("lowerArith(%s) = (%v, %v, %v)", op, ok, term, err) + if ok, term, err := lower(op, ins); !ok || term || err != nil { + t.Fatalf("lower(%s) = (%v, %v, %v)", op, ok, term, err) } } + mustLower := func(op Op, ins Instr) { + t.Helper() + mustLowerWith(c.lowerArith, op, ins) + } + if ok, term, err := c.lowerScalarShiftRotate("RCRQ", Instr{Raw: "RCRQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}); !ok || term || err != nil { + t.Fatalf("lowerScalarShiftRotate(RCRQ) = (%v, %v, %v)", ok, term, err) + } mustLower("PUSHQ", Instr{Raw: "PUSHQ $1", Args: []Operand{{Kind: OpImm, Imm: 1}}}) mustLower("POPQ", Instr{Raw: "POPQ CX", Args: []Operand{{Kind: OpReg, Reg: CX}}}) mustLower("PUSHFQ", Instr{Raw: "PUSHFQ"}) mustLower("POPFQ", Instr{Raw: "POPFQ"}) mustLower("LFENCE", Instr{Raw: "LFENCE"}) - mustLower("UNDEF", Instr{Raw: "UNDEF"}) + trap, trapOutput := newAMD64CtxWithFuncForTest(t, fn, FuncSig{Ret: Void}, nil) + if known, terminated, err := trap.lowerArith("UNDEF", Instr{Raw: "UNDEF"}); !known || !terminated || err != nil || + !strings.Contains(trapOutput.String(), "asm sideeffect \"ud2\"") { + t.Fatalf("UNDEF must terminate with its architectural fault: %t/%t/%v", known, terminated, err) + } mustLower("RDTSC", Instr{Raw: "RDTSC"}) mustLower(OpCPUID, Instr{Raw: "CPUID"}) mustLower(OpXGETBV, Instr{Raw: "XGETBV"}) @@ -572,7 +630,6 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { mustLower("STOSQ", Instr{Raw: "STOSQ"}) mustLower("NEGL", Instr{Raw: "NEGL AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) mustLower("NEGL", Instr{Raw: "NEGL 4(BX)", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 4}}}}) - mustLower("RCRQ", Instr{Raw: "RCRQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) mustLower("ADDQ", Instr{Raw: "ADDQ $2, AX", Args: []Operand{{Kind: OpImm, Imm: 2}, {Kind: OpReg, Reg: AX}}}) mustLower("SUBQ", Instr{Raw: "SUBQ CX, 8(BX)", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}}) mustLower("XORQ", Instr{Raw: "XORQ $3, AX", Args: []Operand{{Kind: OpImm, Imm: 3}, {Kind: OpReg, Reg: AX}}}) @@ -581,8 +638,8 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { mustLower("ADCQ", Instr{Raw: "ADCQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) mustLower("ADCB", Instr{Raw: "ADCB $1, AL", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AL}}}) mustLower("SBBQ", Instr{Raw: "SBBQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("ADCXQ", Instr{Raw: "ADCXQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("ADOXQ", Instr{Raw: "ADOXQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLower("ADCXQ", Instr{Raw: "ADCXQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLower("ADOXQ", Instr{Raw: "ADOXQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) mustLower("ADDL", Instr{Raw: "ADDL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) mustLower("ADDB", Instr{Raw: "ADDB CL, DX", Args: []Operand{{Kind: OpReg, Reg: CL}, {Kind: OpReg, Reg: DX}}}) mustLower("SUBL", Instr{Raw: "SUBL CX, 12(BX)", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 12}}}}) @@ -592,55 +649,65 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { mustLower("XORB", Instr{Raw: "XORB $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) mustLower("ANDB", Instr{Raw: "ANDB $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) mustLower("ORB", Instr{Raw: "ORB $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("INCQ", Instr{Raw: "INCQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) - mustLower("DECQ", Instr{Raw: "DECQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) - mustLower("DECQ", Instr{Raw: "DECQ 96(AX)", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX, Off: 96}}}}) - mustLower("INCL", Instr{Raw: "INCL AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) - mustLower("DECL", Instr{Raw: "DECL AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) - mustLower("DECL", Instr{Raw: "DECL 12(BX)", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 12}}}}) + mustLowerWith(c.lowerScalarIncDec, "INCQ", Instr{Raw: "INCQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarIncDec, "DECQ", Instr{Raw: "DECQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarIncDec, "DECQ", Instr{Raw: "DECQ 96(AX)", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX, Off: 96}}}}) + mustLowerWith(c.lowerScalarIncDec, "INCL", Instr{Raw: "INCL AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarIncDec, "DECL", Instr{Raw: "DECL AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarIncDec, "DECL", Instr{Raw: "DECL 12(BX)", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 12}}}}) mustLower("LEAQ", Instr{Raw: "LEAQ 8(BX)(CX*2), DI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Index: CX, Scale: 2, Off: 8}}, {Kind: OpReg, Reg: DI}}}) mustLower("LEAL", Instr{Raw: "LEAL 4(BX), SI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 4}}, {Kind: OpReg, Reg: SI}}}) mustLower("LEAQ", Instr{Raw: "LEAQ ret+16(FP), R8", Args: []Operand{{Kind: OpFP, FPOffset: 16}, {Kind: OpReg, Reg: Reg("R8")}}}) - mustLower("LEAQ", Instr{Raw: "LEAQ $ret+16(FP), R9", Args: []Operand{{Kind: OpFPAddr, FPOffset: 16}, {Kind: OpReg, Reg: Reg("R9")}}}) + // TYPE_ADDR is not Go's LEA Ym source; retain the former false-positive + // spelling as a rejection, independently checked by the full LEA oracle. + if _, _, err := c.lowerArith("LEAQ", Instr{Raw: "LEAQ $ret+16(FP), R9", Args: []Operand{{Kind: OpFPAddr, FPOffset: 16}, {Kind: OpReg, Reg: Reg("R9")}}}); err == nil { + t.Fatal("LEAQ accepted a TYPE_ADDR operand outside Go's Ym/Yrl row") + } mustLower("LEAQ", Instr{Raw: "LEAQ global<>(SB), AX", Args: []Operand{{Kind: OpSym, Sym: "global<>(SB)"}, {Kind: OpReg, Reg: AX}}}) mustLower("POPCNTL", Instr{Raw: "POPCNTL AX, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}) mustLower("POPCNTQ", Instr{Raw: "POPCNTQ AX, CX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: CX}}}) mustLower("TZCNTQ", Instr{Raw: "TZCNTQ BX, DX", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: DX}}}) + // This is a fragment-shape test, not a module/native-entry certificate. + // Supply the compiler-local bridge name and its explicit target; complete + // public-API tests independently require the emitted no-red-zone helper. + oldScanNamespace, oldScanTarget := c.bitScanNS, c.targetTriple + c.bitScanNS, c.targetTriple = "__test_bitscan_", "x86_64-unknown-linux-gnu" mustLower("BSFQ", Instr{Raw: "BSFQ AX, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}) mustLower("BSRQ", Instr{Raw: "BSRQ AX, CX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: CX}}}) mustLower("BSWAPQ", Instr{Raw: "BSWAPQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) mustLower("BSFL", Instr{Raw: "BSFL AX, DX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: DX}}}) mustLower("BSRL", Instr{Raw: "BSRL AX, DI", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: DI}}}) - mustLower("SETEQ", Instr{Raw: "SETEQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) - mustLower("SETGT", Instr{Raw: "SETGT ret+8(FP)", Args: []Operand{{Kind: OpFP, FPOffset: 8}}}) - mustLower("SETGE", Instr{Raw: "SETGE AH", Args: []Operand{{Kind: OpReg, Reg: AH}}}) - mustLower("SETHI", Instr{Raw: "SETHI BX", Args: []Operand{{Kind: OpReg, Reg: BX}}}) - mustLower("SETCS", Instr{Raw: "SETCS AL", Args: []Operand{{Kind: OpReg, Reg: AL}}}) - mustLower("SETCS", Instr{Raw: "SETCS AH", Args: []Operand{{Kind: OpReg, Reg: AH}}}) - mustLower("CMOVQEQ", Instr{Raw: "CMOVQEQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) - mustLower("CMOVQNE", Instr{Raw: "CMOVQNE CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) - mustLower("CMOVQCS", Instr{Raw: "CMOVQCS CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) - mustLower("CMOVQCC", Instr{Raw: "CMOVQCC CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) - mustLower("CMOVQGT", Instr{Raw: "CMOVQGT CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + c.bitScanNS, c.targetTriple = oldScanNamespace, oldScanTarget + mustLowerWith(c.lowerConditionalSet, "SETEQ", Instr{Raw: "SETEQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerConditionalSet, "SETGT", Instr{Raw: "SETGT ret+8(FP)", Args: []Operand{{Kind: OpFP, FPOffset: 8}}}) + mustLowerWith(c.lowerConditionalSet, "SETGE", Instr{Raw: "SETGE AH", Args: []Operand{{Kind: OpReg, Reg: AH}}}) + mustLowerWith(c.lowerConditionalSet, "SETHI", Instr{Raw: "SETHI BX", Args: []Operand{{Kind: OpReg, Reg: BX}}}) + mustLowerWith(c.lowerConditionalSet, "SETCS", Instr{Raw: "SETCS AL", Args: []Operand{{Kind: OpReg, Reg: AL}}}) + mustLowerWith(c.lowerConditionalSet, "SETCS", Instr{Raw: "SETCS AH", Args: []Operand{{Kind: OpReg, Reg: AH}}}) + mustLowerWith(c.lowerConditionalMove, "CMOVQEQ", Instr{Raw: "CMOVQEQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerConditionalMove, "CMOVQNE", Instr{Raw: "CMOVQNE CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerConditionalMove, "CMOVQCS", Instr{Raw: "CMOVQCS CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerConditionalMove, "CMOVQCC", Instr{Raw: "CMOVQCC CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerConditionalMove, "CMOVQGT", Instr{Raw: "CMOVQGT CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) mustLower("ANDNL", Instr{Raw: "ANDNL AX, BX, CX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: CX}}}) mustLower("ANDNQ", Instr{Raw: "ANDNQ AX, BX, DX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: DX}}}) mustLower("BEXTRQ", Instr{Raw: "BEXTRQ CX, AX, DX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: DX}}}) mustLower("BZHIQ", Instr{Raw: "BZHIQ DI, AX, CX", Args: []Operand{{Kind: OpReg, Reg: DI}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: CX}}}) - mustLower("SHRQ", Instr{Raw: "SHRQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("SHLQ", Instr{Raw: "SHLQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "SHRQ", Instr{Raw: "SHRQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "SHLQ", Instr{Raw: "SHLQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) mustLower("SHLXQ", Instr{Raw: "SHLXQ DI, CX, AX", Args: []Operand{{Kind: OpReg, Reg: DI}, {Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) - mustLower("SARQ", Instr{Raw: "SARQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("SARL", Instr{Raw: "SARL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("SHLL", Instr{Raw: "SHLL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("SHLB", Instr{Raw: "SHLB $2, R10", Args: []Operand{{Kind: OpImm, Imm: 2}, {Kind: OpReg, Reg: Reg("R10")}}}) - mustLower("SHRL", Instr{Raw: "SHRL CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "SARQ", Instr{Raw: "SARQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "SARL", Instr{Raw: "SARL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "SHLL", Instr{Raw: "SHLL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "SHLB", Instr{Raw: "SHLB $2, R10", Args: []Operand{{Kind: OpImm, Imm: 2}, {Kind: OpReg, Reg: Reg("R10")}}}) + mustLowerWith(c.lowerScalarShiftRotate, "SHRL", Instr{Raw: "SHRL CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) mustLower("SHRXQ", Instr{Raw: "SHRXQ DI, AX, CX", Args: []Operand{{Kind: OpReg, Reg: DI}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: CX}}}) - mustLower("SALQ", Instr{Raw: "SALQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("SALL", Instr{Raw: "SALL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("ROLL", Instr{Raw: "ROLL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("ROLQ", Instr{Raw: "ROLQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) - mustLower("RORQ", Instr{Raw: "RORQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("RORL", Instr{Raw: "RORL CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "SALQ", Instr{Raw: "SALQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "SALL", Instr{Raw: "SALL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "ROLL", Instr{Raw: "ROLL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "ROLQ", Instr{Raw: "ROLQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "RORQ", Instr{Raw: "RORQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLowerWith(c.lowerScalarShiftRotate, "RORL", Instr{Raw: "RORL CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) mustLower("RORXL", Instr{Raw: "RORXL $1, AX, BX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}) mustLower("RORXQ", Instr{Raw: "RORXQ $1, AX, CX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: CX}}}) mustLower("NOTL", Instr{Raw: "NOTL AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) @@ -648,9 +715,54 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { mustLower("NOTQ", Instr{Raw: "NOTQ 20(BX)", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 20}}}}) mustLower("BSWAPL", Instr{Raw: "BSWAPL AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) mustLower("MULQ", Instr{Raw: "MULQ CX", Args: []Operand{{Kind: OpReg, Reg: CX}}}) + // MULX consumes a complete source proof. This broad fragment context has + // Void plus synthetic result slots, so give only MULX its own real source + // and matching signature. Do not lend that proof to the remaining table. + mulxSource, err := Parse(ArchAMD64, `TEXT mulxfragment(SB),4,$0-0 +MOVQ $7,BX +MOVQ $3,DX +MULXQ BX, CX, DI +RET +`) + if err != nil { + t.Fatal(err) + } + mulxContext, mulxOutput := newAMD64CtxWithFuncForTest(t, mulxSource.Funcs[0], + FuncSig{Name: "example.mulxfragment", Ret: Void}, nil) + if err := mulxContext.storeReg(BX, "7"); err != nil { + t.Fatal(err) + } + if err := mulxContext.storeReg(DX, "3"); err != nil { + t.Fatal(err) + } + fragmentContext := c + c = mulxContext mustLower("MULXQ", Instr{Raw: "MULXQ BX, CX, DI", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: DI}}}) + c = fragmentContext + if !strings.Contains(mulxOutput.String(), "mul i128") || !strings.Contains(mulxOutput.String(), "lshr i128") { + t.Fatal("complete MULX fragment lost either product half") + } mustLower("MULL", Instr{Raw: "MULL CX", Args: []Operand{{Kind: OpReg, Reg: CX}}}) + // This direct emitter fragment has no public entry/NUM certificate. Its + // trapping DIV still requires an explicit physical instruction target; + // missing target must fail without emitting a partial native operation. + beforeDivideBytes, beforeDivideTemporary := b.Len(), c.tmp + divide := Instr{Raw: "DIVL CX", Args: []Operand{{Kind: OpReg, Reg: CX}}} + known, terminated, err := c.lowerArith("DIVL", divide) + if !known || terminated || !errors.Is(err, ErrProbeNeedsContext) || + b.Len() != beforeDivideBytes || c.tmp != beforeDivideTemporary { + t.Fatal("untargeted direct division acquired a native operation or partial IR", known, terminated, err) + } + oldDivideTarget := c.targetTriple + c.targetTriple = "x86_64-unknown-linux-gnu" mustLower("DIVL", Instr{Raw: "DIVL CX", Args: []Operand{{Kind: OpReg, Reg: CX}}}) + divideIR := b.String()[beforeDivideBytes:] + for _, forbidden := range []string{"udiv ", "urem ", "sdiv ", "srem "} { + if strings.Contains(divideIR, forbidden) { + t.Fatal("native division was replaced by LLVM poison-producing arithmetic", forbidden) + } + } + c.targetTriple = oldDivideTarget mustLower("IMULQ", Instr{Raw: "IMULQ CX", Args: []Operand{{Kind: OpReg, Reg: CX}}}) mustLower("IMULQ", Instr{Raw: "IMULQ CX, DX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: DX}}}) mustLower("IMUL3Q", Instr{Raw: "IMUL3Q $3, CX, DI", Args: []Operand{{Kind: OpImm, Imm: 3}, {Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: DI}}}) @@ -663,12 +775,11 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { "call i32 @llvm.ctpop.i32", "call i64 @llvm.ctpop.i64", "call i64 @llvm.cttz.i64", - "call i32 @llvm.ctlz.i32", + "@__test_bitscan_amd64_bsrl(", "call i64 @llvm.bswap.i64", "call i32 @llvm.bswap.i32", "mul i128", - "udiv i64", - "urem i64", + `asm sideeffect "divl ${4:k}", "=&{ax},=&{dx},0,1,r,~{flags}"`, "select i1", "ptrtoint ptr @\"example.global\" to i64", "store i8", @@ -715,8 +826,8 @@ func TestAMD64EcosystemScalarValidationCoverage(t *testing.T) { {Raw: string(op) + " X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}, {Raw: string(op) + " AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, } { - if ok, term, err := c.lowerArith(op, ins); ok || term || err != nil { - t.Fatalf("lowerArith(%s) invalid form = (%v, %v, %v), want unsupported", op, ok, term, err) + if ok, term, err := c.lowerArith(op, ins); !ok || term || err == nil { + t.Fatalf("lowerArith(%s) invalid form = (%v, %v, %v), want recognized validation error", op, ok, term, err) } } } @@ -748,7 +859,7 @@ func TestAMD64SetCSUsesCarryFlag(t *testing.T) { {Raw: "SETCS AL", Args: []Operand{{Kind: OpReg, Reg: AL}}}, {Raw: "SETCS AH", Args: []Operand{{Kind: OpReg, Reg: AH}}}, } { - if ok, term, err := c.lowerArith("SETCS", ins); !ok || term || err != nil { + if ok, term, err := c.lowerConditionalSet("SETCS", ins); !ok || term || err != nil { t.Fatalf("lowerArith(%s) = (%v, %v, %v)", ins.Raw, ok, term, err) } } @@ -775,7 +886,7 @@ func TestAMD64SetGEUsesSignedFlag(t *testing.T) { b.WriteString(" store i1 false, ptr " + c.flagsSltSlot + "\n") ins := Instr{Raw: "SETGE AH", Args: []Operand{{Kind: OpReg, Reg: AH}}} - if ok, term, err := c.lowerArith("SETGE", ins); !ok || term || err != nil { + if ok, term, err := c.lowerConditionalSet("SETGE", ins); !ok || term || err != nil { t.Fatalf("lowerArith(%s) = (%v, %v, %v)", ins.Raw, ok, term, err) } @@ -814,7 +925,7 @@ func TestAMD64ADCBUsesCarryFlag(t *testing.T) { "store i1 true, ptr %flags_cf", "load i1, ptr %flags_cf", "zext i1 %", - "add i8", + "add i16", "zext i8", "icmp ugt i16", "shl i64", @@ -826,6 +937,24 @@ func TestAMD64ADCBUsesCarryFlag(t *testing.T) { } } +// Exercise the actual typed owners before the residual vector dispatcher, in +// the same order as lowerInstr. The old fixtures remain emitter-only checks. +func lowerAMD64PackedArithmeticOrVectorForTest(c *amd64Ctx, op Op, ins Instr) (bool, bool, error) { + if known, terminated, err := c.lowerPackedWordMultiply(op, ins); known { + return known, terminated, err + } + if known, terminated, err := c.lowerPackedIntegerAdd(op, ins); known { + return known, terminated, err + } + if known, terminated, err := c.lowerPackedIntegerSubtract(op, ins); known { + return known, terminated, err + } + if known, terminated, err := c.lowerPackedMoveMask(op, ins); known { + return known, terminated, err + } + return c.lowerVec(op, ins) +} + func TestAMD64VectorCoverage(t *testing.T) { fn := Func{ Instrs: []Instr{ @@ -844,17 +973,31 @@ func TestAMD64VectorCoverage(t *testing.T) { }, } c, b := newAMD64CtxWithFuncForTest(t, fn, FuncSig{Name: "example.vec", Ret: Void}, nil) + // Emitter-only native vector formats still need their actual x86 backend. + // Complete initialized public programs have separate object owners. + c.targetTriple = "x86_64-unknown-linux-gnu" mustLower := func(op Op, ins Instr) { t.Helper() - if ok, term, err := c.lowerVec(op, ins); !ok || term || err != nil { - t.Fatalf("lowerVec(%s) = (%v, %v, %v)", op, ok, term, err) + if ok, term, err := lowerAMD64PackedArithmeticOrVectorForTest(c, op, ins); !ok || term || err != nil { + t.Fatalf("typed vector lowering(%s) = (%v, %v, %v)", op, ok, term, err) + } + } + mustLowerTransfer := func(op Op, ins Instr) { + t.Helper() + ins.Op = op + if ok, term, err := c.lowerVectorScalarIntegerMove(op, ins); !ok || term || err != nil { + t.Fatalf("lowerVectorScalarIntegerMove(%s) = (%v, %v, %v)", op, ok, term, err) } } - mustLower("MOVL", Instr{Raw: "MOVL $1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}}}) - mustLower("MOVD", Instr{Raw: "MOVD AX, X1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X1")}}}) - mustLower("MOVQ", Instr{Raw: "MOVQ BX, X2", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: Reg("X2")}}}) - mustLower("MOVQ", Instr{Raw: "MOVQ X2, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X2")}, {Kind: OpReg, Reg: AX}}}) + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\nMOVL $1,AX\nMOVL AX,X0\nMOVD AX,X1\nMOVQ BX,X2\nMOVQ X2,AX\nRET\n", "", true) + if err := c.storeReg(AX, "1"); err != nil { + t.Fatal(err) + } + mustLowerTransfer("MOVL", Instr{Raw: "MOVL AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}) + mustLowerTransfer("MOVD", Instr{Raw: "MOVD AX, X1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X1")}}}) + mustLowerTransfer("MOVQ", Instr{Raw: "MOVQ BX, X2", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: Reg("X2")}}}) + mustLowerTransfer("MOVQ", Instr{Raw: "MOVQ X2, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X2")}, {Kind: OpReg, Reg: AX}}}) mustLower("KXORQ", Instr{Raw: "KXORQ K1, K2, K3", Args: []Operand{{Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("K2")}, {Kind: OpReg, Reg: Reg("K3")}}}) mustLower("KMOVB", Instr{Raw: "KMOVB AX, K1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("K1")}}}) mustLower("KMOVW", Instr{Raw: "KMOVW K1, 8(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("K1")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}}) @@ -923,7 +1066,9 @@ func TestAMD64VectorCoverage(t *testing.T) { mustLower("MOVOU", Instr{Raw: "MOVOU X0, 112(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 112}}}}) mustLower("PXOR", Instr{Raw: "PXOR X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) mustLower("PAND", Instr{Raw: "PAND X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) - mustLower("PANDN", Instr{Raw: "PANDN X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) + if ok, term, err := c.lowerPackedAndNot("PANDN", Instr{Raw: "PANDN X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}); !ok || term || err != nil { + t.Fatalf("lowerPackedAndNot(PANDN) = (%v, %v, %v)", ok, term, err) + } mustLower("PADDL", Instr{Raw: "PADDL X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) mustLower("PADDQ", Instr{Raw: "PADDQ X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) mustLower("PSUBL", Instr{Raw: "PSUBL X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) @@ -966,9 +1111,6 @@ func TestAMD64VectorCoverage(t *testing.T) { if got := llvmShiftLeftBytesMask(3); !strings.Contains(got, "i32 16") { t.Fatalf("llvmShiftLeftBytesMask(3) = %q", got) } - if got := llvmAlignRightBytesMask(20); !strings.Contains(got, "i32 20") { - t.Fatalf("llvmAlignRightBytesMask(20) = %q", got) - } if got := llvmAllOnesI8Vec(4); got != "" { t.Fatalf("llvmAllOnesI8Vec(4) = %q", got) } @@ -1085,17 +1227,18 @@ func TestAMD64TranslateReturnCoverage(t *testing.T) { } func TestAMD64FPMovCoverage(t *testing.T) { - fn := Func{ - Instrs: []Instr{ - {Op: "MOVQ", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}, - {Op: "MOVQ", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}, - {Op: "MOVQ", Args: []Operand{{Kind: OpReg, Reg: Reg("X2")}, {Kind: OpReg, Reg: Reg("X3")}}}, - }, - } + fixture, err := Parse(ArchAMD64, "TEXT example.fpmov(SB),$0-65\n"+ + "MOVQ arg+0(FP),AX\nMOVQ AX,BX\nPXOR X0,X0\nPXOR X2,X2\nMOVQ X0,X1\nMOVQ X2,X3\n"+ + "MOVSD input+8(FP),X0\nMOVSD X0,ret+40(FP)\nMOVQ AX,ret+48(FP)\n"+ + "MOVW AX,ret+56(FP)\nMOVB AX,ret+64(FP)\nRET\n") + if err != nil { + t.Fatal(err) + } + fn := fixture.Funcs[0] sig := FuncSig{ Name: "example.fpmov", Args: []LLVMType{I64, LLVMType("double"), I64, I64}, - Ret: Void, + Ret: LLVMType("{ double, i64, i16, i8 }"), Frame: FrameLayout{ Params: []FrameSlot{ {Offset: 0, Type: I64, Index: 0, Field: -1}, @@ -1112,6 +1255,7 @@ func TestAMD64FPMovCoverage(t *testing.T) { }, } c, b := newAMD64CtxWithFuncForTest(t, fn, sig, nil) + c.goarch, c.targetTriple = "amd64", "x86_64-unknown-linux-gnu" check := func(kind string, ins Instr, ok bool, err error) { t.Helper() if err != nil { @@ -1152,7 +1296,7 @@ func TestAMD64FPMovCoverage(t *testing.T) { {Op: "ANDNPD", Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}, Raw: "ANDNPD X1, X0"}, {Op: "ORPD", Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}, Raw: "ORPD X1, X0"}, {Op: "XORPS", Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}, Raw: "XORPS X1, X0"}, - {Op: "MOVSD", Args: []Operand{{Kind: OpImm, Imm: 7}, {Kind: OpReg, Reg: "X0"}}, Raw: "MOVSD $7, X0"}, + {Op: "MOVSD", Args: []Operand{{Kind: OpImm, Imm: int64(math.Float64bits(7)), ImmIsFloat: true}, {Kind: OpReg, Reg: "X0"}}, Raw: "MOVSD $7.0, X0"}, {Op: "MOVSD", Args: []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 16}}}, Raw: "MOVSD X0, 16(BX)"}, {Op: "MOVSD", Args: []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpSym, Sym: "example.f64(SB)"}}, Raw: "MOVSD X0, example.f64(SB)"}, {Op: "MOVSD", Args: []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpFP, FPOffset: 40}}, Raw: "MOVSD X0, ret+40(FP)"}, @@ -1174,14 +1318,27 @@ func TestAMD64FPMovCoverage(t *testing.T) { {Op: "VFMADD213SD", Args: []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X2"}}, Raw: "VFMADD213SD X0, X1, X2"}, {Op: "VFNMADD231SD", Args: []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X2"}}, Raw: "VFNMADD231SD X0, X1, X2"}, } { + before := b.Len() ok, _, err := c.lowerFP(ins.Op, ins) check("lowerFP", ins, ok, err) + if ins.Op == "VFMADD213SD" || ins.Op == "VFNMADD231SD" { + // This fragment has an explicit native target. Preserve its + // original fused opcode/MXCSR effects rather than demanding the + // separate portable numerical lowerer's fneg/llvm.fma sequence. + fragment := b.String()[before:] + native := `asm sideeffect "` + strings.ToLower(string(ins.Op)) + " " + if strings.Count(fragment, native) != 1 || !strings.Contains(fragment, "~{fpsr}") || + !strings.Contains(fragment, "~{memory}") || !strings.Contains(fragment, ",0,") || + strings.Contains(fragment, "fneg double") || strings.Contains(fragment, "@llvm.fma.") { + t.Fatal("native scalar FMA lost its exact fused opcode or floating effects", ins.Raw) + } + } } c.setCmpFlags("1", "2") for _, ins := range []Instr{ {Op: "CMOVQLT", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: DX}}, Raw: "CMOVQLT CX, DX"}, - {Op: "MOVLQSX", Args: []Operand{{Kind: OpImm, Imm: 21}, {Kind: OpReg, Reg: AX}}, Raw: "MOVLQSX $21, AX"}, + {Op: "MOVLQSX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}, Raw: "MOVLQSX AX, AX"}, {Op: "MOVLQSX", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: CX}}, Raw: "MOVLQSX BX, CX"}, {Op: "MOVLQSX", Args: []Operand{{Kind: OpFP, FPOffset: 16}, {Kind: OpReg, Reg: DX}}, Raw: "MOVLQSX arg+16(FP), DX"}, {Op: "MOVLQSX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: SI, Off: 4}}, {Kind: OpReg, Reg: SI}}, Raw: "MOVLQSX 4(SI), SI"}, @@ -1245,27 +1402,29 @@ func TestAMD64FPMovCoverage(t *testing.T) { out := b.String() for _, want := range []string{ - "fadd double", - "fsub double", - "fmul double", - "fdiv double", - "fneg double", - "fcmp ueq double", + "call double @llvm.experimental.constrained.fadd.f64", + "call double @__plan9asm_add_sub_special_f64", + "call double @llvm.experimental.constrained.fsub.f64", + "call double @llvm.experimental.constrained.fmul.f64", + "call double @llvm.experimental.constrained.fdiv.f64", + `call { i8, i8, i8 } asm sideeffect "comisd $3, $4; setb $0; sete $1; setp $2"`, "fcmp ord double", - "call double @llvm.sqrt.f64", - "call double @llvm.rint.f64", - "sitofp i64", - "sitofp i32", - "fptosi double", + `asm sideeffect "sqrtsd $2, $0"`, + `call i32 asm sideeffect "cvtsd2sil $1, $0"`, + "@llvm.experimental.constrained.sitofp.f64.i64", + "@llvm.experimental.constrained.sitofp.f64.i32", + `call i64 asm sideeffect "cvttsd2siq $1, $0"`, "select i1", "bitcast i64", "bitcast double", "store double", "store i64", - "store i32", + "store volatile i32", "store i16", "store i8", - `load <16 x i8>, ptr @"example.vec"`, + // Native MOVAPD must retain its aligned opcode and dead-read fault; + // an ordinary LLVM load is no longer the correct lowering contract. + `call <4 x float> asm sideeffect "movapd ($1), $0", "=x,r,~{memory}"(ptr @"example.vec")`, } { if !strings.Contains(out, want) { t.Fatalf("missing %q in output:\n%s", want, out) @@ -1277,10 +1436,10 @@ func TestAMD64CmpBtCoverage(t *testing.T) { sig := FuncSig{ Name: "example.cmpbt", Args: []LLVMType{I64}, - Ret: Void, + Ret: I64, Frame: FrameLayout{ - Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0}}, - Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0}}, + Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, }, } c, b := newAMD64CtxWithFuncForTest(t, Func{}, sig, nil) @@ -1307,13 +1466,13 @@ func TestAMD64CmpBtCoverage(t *testing.T) { } } - check("CMPB", Instr{Raw: "CMPB $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + check("CMPB", Instr{Raw: "CMPB AX, $1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpImm, Imm: 1}}}) check("CMPW", Instr{Raw: "CMPW 4(BX), CX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 4}}, {Kind: OpReg, Reg: CX}}}) check("CMPL", Instr{Raw: "CMPL arg+0(FP), DI", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpReg, Reg: DI}}}) check("CMPQ", Instr{Raw: "CMPQ example.global(SB), SI", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: SI}}}) check("TESTB", Instr{Raw: "TESTB AX, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}) check("TESTW", Instr{Raw: "TESTW $7, arg+0(FP)", Args: []Operand{{Kind: OpImm, Imm: 7}, {Kind: OpFP, FPOffset: 0}}}) - check("TESTL", Instr{Raw: "TESTL $const, 8(BX)", Args: []Operand{{Kind: OpSym, Sym: "$const"}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}}) + check("TESTL", Instr{Raw: "TESTL $7, 8(BX)", Args: []Operand{{Kind: OpImm, Imm: 7}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}}) check("TESTQ", Instr{Raw: "TESTQ example.global(SB), DI", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: DI}}}) check("BTQ", Instr{Raw: "BTQ $3, AX", Args: []Operand{{Kind: OpImm, Imm: 3}, {Kind: OpReg, Reg: AX}}}) check("BTSQ", Instr{Raw: "BTSQ DX, AX", Args: []Operand{{Kind: OpReg, Reg: DX}, {Kind: OpReg, Reg: AX}}}) @@ -1329,7 +1488,7 @@ func TestAMD64CmpBtCoverage(t *testing.T) { if _, _, err := c.lowerCmpBt("TESTQ", Instr{Raw: "TESTQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}); err == nil { t.Fatalf("short TESTQ unexpectedly succeeded") } - if _, _, err := c.lowerCmpBt("BTQ", Instr{Raw: "BTQ AX, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}); err == nil { + if _, _, err := c.lowerCmpBt("BTQ", Instr{Raw: "BTQ (AX), BX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX}}, {Kind: OpReg, Reg: BX}}}); err == nil { t.Fatalf("bad BTQ unexpectedly succeeded") } if _, _, err := c.lowerCmpBt("BTSQ", Instr{Raw: "BTSQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}); err == nil { @@ -1338,8 +1497,8 @@ func TestAMD64CmpBtCoverage(t *testing.T) { if _, _, err := c.lowerCmpBt("BTRQ", Instr{Raw: "BTRQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}); err == nil { t.Fatalf("short BTRQ unexpectedly succeeded") } - if got, err := c.evalIntSized(Operand{Kind: OpSym, Sym: "$const"}, I32); err != nil || got != "0" { - t.Fatalf("evalIntSized($const) = (%q, %v)", got, err) + if _, err := c.evalIntSized(Operand{Kind: OpSym, Sym: "$const"}, I32); err == nil { + t.Fatal("evalIntSized($const) invented a value for an unresolved constant") } if _, err := c.evalIntSized(Operand{Kind: OpSym, Sym: "bad"}, I32); err == nil { t.Fatalf("evalIntSized(bad sym) unexpectedly succeeded") @@ -1358,7 +1517,7 @@ func TestAMD64CmpBtCoverage(t *testing.T) { "store i1 false, ptr %flags_cf", "lshr i64", "shl i64 1", - "load i64, ptr @\"example.global\"", + "load volatile i64, ptr @\"example.global\"", } { if !strings.Contains(out, want) { t.Fatalf("missing %q in output:\n%s", want, out) @@ -1370,13 +1529,20 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { sig := FuncSig{ Name: "example.mov", Args: []LLVMType{I64}, - Ret: Void, + Ret: LLVMType("{ i64, i16, i8 }"), Frame: FrameLayout{ - Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0}}, - Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0}, {Offset: 16, Type: I16, Index: 1}, {Offset: 24, Type: I8, Index: 2}}, + Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}, {Offset: 16, Type: I16, Index: 1, Field: -1}, {Offset: 24, Type: I8, Index: 2, Field: -1}}, }, } - c, b := newAMD64CtxWithFuncForTest(t, Func{}, sig, nil) + // Fragment-level emitter assertions still need a complete source when a + // partial logical FP view asks for the independent whole-byte proof. + fixture, err := Parse(ArchAMD64, "TEXT example.mov(SB),$0-25\nMOVQ arg+0(FP),AX\nMOVQ AX,ret+8(FP)\nMOVW AX,ret+16(FP)\nMOVB AX,ret+24(FP)\nRET\n") + if err != nil { + t.Fatal(err) + } + fn := fixture.Funcs[0] + c, b := newAMD64CtxWithFuncForTest(t, fn, sig, nil) for _, tc := range []struct { r Reg v string @@ -1403,7 +1569,7 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { } } - checkMov("MOVLQSX", Instr{Raw: "MOVLQSX $7, AX", Args: []Operand{{Kind: OpImm, Imm: 7}, {Kind: OpReg, Reg: AX}}}) + checkMov("MOVLQSX", Instr{Raw: "MOVLQSX AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}) checkMov("MOVLQSX", Instr{Raw: "MOVLQSX BX, CX", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: CX}}}) checkMov("MOVLQSX", Instr{Raw: "MOVLQSX arg+0(FP), DX", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpReg, Reg: DX}}}) checkMov("MOVLQSX", Instr{Raw: "MOVLQSX 8(BX), SI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: SI}}}) @@ -1456,9 +1622,9 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { t.Fatalf("MOVQ yreg dst = (%v, %v, %v)", ok, term, err) } - cBadReg, _ := newAMD64CtxWithFuncForTest(t, Func{}, sig, nil) - if ok, term, err := cBadReg.lowerMov("MOVB", Instr{Raw: "MOVB BAD, ret+24(FP)", Args: []Operand{{Kind: OpReg, Reg: Reg("BAD")}, {Kind: OpFP, FPOffset: 24}}}); !ok || term || err != nil { - t.Fatalf("MOVB missing src reg = (%v, %v, %v)", ok, term, err) + cBadReg, _ := newAMD64CtxWithFuncForTest(t, fn, sig, nil) + if ok, term, err := cBadReg.lowerMov("MOVB", Instr{Raw: "MOVB BAD, ret+24(FP)", Args: []Operand{{Kind: OpReg, Reg: Reg("BAD")}, {Kind: OpFP, FPOffset: 24}}}); !ok || term || err == nil { + t.Fatalf("MOVB invalid src reg = (%v, %v, %v), want handled error", ok, term, err) } if ok, term, err := cBadReg.lowerMov("CMOVQLT", Instr{Raw: "CMOVQLT BAD, BX", Args: []Operand{{Kind: OpReg, Reg: Reg("BAD")}, {Kind: OpReg, Reg: BX}}}); !ok || term || err != nil { t.Fatalf("CMOVQLT missing regs = (%v, %v, %v)", ok, term, err) @@ -1470,21 +1636,21 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { t.Fatalf("MOVL missing reg = (%v, %v, %v)", ok, term, err) } - cBadMem, _ := newAMD64CtxWithFuncForTest(t, Func{}, sig, nil) - if ok, term, err := cBadMem.lowerMov("MOVLQSX", Instr{Raw: "MOVLQSX 8(BAD), SI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}, {Kind: OpReg, Reg: SI}}}); !ok || term || err != nil { + cBadMem, _ := newAMD64CtxWithFuncForTest(t, fn, sig, nil) + if ok, term, err := cBadMem.lowerMov("MOVLQSX", Instr{Raw: "MOVLQSX 8(BAD), SI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}, {Kind: OpReg, Reg: SI}}}); !ok || term || err == nil { t.Fatalf("MOVLQSX bad mem = (%v, %v, %v)", ok, term, err) } if ok, term, err := cBadMem.lowerMov("MOVW", Instr{Raw: "MOVW arg+0(FP), 8(BAD)", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}}}); !ok || term || err != nil { t.Fatalf("MOVW bad dst mem = (%v, %v, %v)", ok, term, err) } - if ok, term, err := cBadMem.lowerMov("MOVQ", Instr{Raw: "MOVQ 8(BAD), AX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}, {Kind: OpReg, Reg: AX}}}); !ok || term || err != nil { + if ok, term, err := cBadMem.lowerMov("MOVQ", Instr{Raw: "MOVQ 8(BAD), AX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("MOVQ bad src mem = (%v, %v, %v)", ok, term, err) } if ok, term, err := cBadMem.lowerMov("MOVL", Instr{Raw: "MOVL 24(BAD), example.dest(SB)", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 24}}, {Kind: OpSym, Sym: "example.dest(SB)"}}}); !ok || term || err != nil { t.Fatalf("MOVL bad src mem = (%v, %v, %v)", ok, term, err) } - if ok, term, err := c.lowerMov("MOVLQSX", Instr{Raw: "MOVLQSX broken, AX", Args: []Operand{{Kind: OpSym, Sym: "broken"}, {Kind: OpReg, Reg: AX}}}); !ok || term || err != nil { + if ok, term, err := c.lowerMov("MOVLQSX", Instr{Raw: "MOVLQSX broken, AX", Args: []Operand{{Kind: OpSym, Sym: "broken"}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("MOVLQSX bad sym = (%v, %v, %v)", ok, term, err) } if ok, term, err := c.lowerMov("MOVB", Instr{Raw: "MOVB broken, AX", Args: []Operand{{Kind: OpSym, Sym: "broken"}, {Kind: OpReg, Reg: AX}}}); !ok || term || err != nil { @@ -1504,6 +1670,7 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { } sysc, sysb := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Name: "example.sys", Ret: Void}, nil) + sysc.targetTriple = "x86_64-unknown-linux-gnu" // SYSCALL has a Linux kernel ABI, not an ambient hosted convention. for _, tc := range []struct { r Reg v string @@ -1543,7 +1710,7 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { if ok, term, err := sysc.lowerSyscall("BAD", Instr{}); ok || term || err != nil { t.Fatalf("lowerSyscall(BAD) = (%v, %v, %v)", ok, term, err) } - if _, _, err := sysc.lowerCrc32("CRC32B", Instr{Raw: "CRC32B AX, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}); err == nil { + if _, _, err := sysc.lowerCrc32("CRC32B", Instr{Raw: "CRC32B $1, BX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: BX}}}); err == nil { t.Fatalf("bad CRC32B unexpectedly succeeded") } if ok, term, err := sysc.lowerCrc32("BAD", Instr{}); ok || term || err != nil { @@ -1558,8 +1725,8 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { "zext i8", "store i16", "store i32", - "@syscall(i64", - "@cliteErrno()", + "asm sideeffect \"syscall\"", + "={ax},0,{di},{si},{dx},{r10},{r8},{r9},~{rcx},~{r11}", "@llvm.x86.sse42.crc32.32.8", "@llvm.x86.sse42.crc32.32.16", "@llvm.x86.sse42.crc32.32.32", @@ -1570,6 +1737,9 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { t.Fatalf("missing %q in output:\n%s", want, out) } } + if strings.Contains(sysb.String(), "@syscall(") || strings.Contains(sysb.String(), "@cliteErrno(") { + t.Fatal("raw Linux instruction used a hosted errno-changing wrapper", sysb.String()) + } } func TestAMD64BranchCoverageDeep(t *testing.T) { @@ -1654,7 +1824,7 @@ func TestAMD64BranchCoverageDeep(t *testing.T) { "or i1", "call i64 @\"example.tail\"", "ret i64", - "br label %V1", + "br label %" + amd64LLVMBlockName("V1"), } { if !strings.Contains(out, want) { t.Fatalf("missing %q in output:\n%s", want, out) @@ -1753,10 +1923,10 @@ func TestAMD64FPExtraCoverage(t *testing.T) { if _, _, err := c.lowerFP("MOVSD", Instr{Raw: "MOVSD X0, label", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpIdent, Ident: "label"}}}); err == nil { t.Fatalf("MOVSD bad dst unexpectedly succeeded") } - if ok, term, err := c.lowerFP("ADDSD", Instr{Raw: "ADDSD X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}); ok || term || err != nil { + if ok, term, err := c.lowerFP("ADDSD", Instr{Raw: "ADDSD X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("ADDSD non-X dst = (%v, %v, %v)", ok, term, err) } - if ok, term, err := c.lowerFP("SQRTSD", Instr{Raw: "SQRTSD X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}); ok || term || err != nil { + if ok, term, err := c.lowerFP("SQRTSD", Instr{Raw: "SQRTSD X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("SQRTSD non-X dst = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerFP("COMISD", Instr{Raw: "COMISD X0", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}}}); err == nil { @@ -1768,10 +1938,10 @@ func TestAMD64FPExtraCoverage(t *testing.T) { if _, _, err := c.lowerFP("CMPSD", Instr{Raw: "CMPSD X0, X1, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}, {Kind: OpReg, Reg: AX}}}); err == nil { t.Fatalf("CMPSD reg predicate unexpectedly succeeded") } - if ok, term, err := c.lowerFP("CVTSQ2SD", Instr{Raw: "CVTSQ2SD AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}); ok || term || err != nil { + if ok, term, err := c.lowerFP("CVTSQ2SD", Instr{Raw: "CVTSQ2SD AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("CVTSQ2SD non-X dst = (%v, %v, %v)", ok, term, err) } - if ok, term, err := c.lowerFP("CVTSL2SD", Instr{Raw: "CVTSL2SD AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}); ok || term || err != nil { + if ok, term, err := c.lowerFP("CVTSL2SD", Instr{Raw: "CVTSL2SD AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("CVTSL2SD non-X dst = (%v, %v, %v)", ok, term, err) } @@ -1785,8 +1955,9 @@ func TestAMD64FPExtraCoverage(t *testing.T) { "fcmp uno double", "fcmp une double", "fcmp ugt double", - "fadd double", - "fmul double", + "call double @llvm.experimental.constrained.fadd.f64", + "call double @__plan9asm_add_sub_special_f64", + "call double @llvm.fma.f64", "fneg double", } { if !strings.Contains(out, want) { @@ -1813,7 +1984,7 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { Args: []LLVMType{I64}, Ret: Void, Frame: FrameLayout{ - Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0}}, + Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}, }, } c, b := newAMD64CtxWithFuncForTest(t, fn, sig, nil) @@ -1845,6 +2016,13 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { t.Fatalf("lowerVec(%s %q) = (%v, %v, %v)", op, ins.Raw, ok, term, err) } } + checkTransfer := func(op Op, ins Instr) { + t.Helper() + ins.Op = op + if ok, term, err := c.lowerVectorScalarIntegerMove(op, ins); !ok || term || err != nil { + t.Fatalf("lowerVectorScalarIntegerMove(%s %q) = (%v, %v, %v)", op, ins.Raw, ok, term, err) + } + } check("VZEROUPPER", Instr{Raw: "VZEROUPPER"}) check("VZEROALL", Instr{Raw: "VZEROALL"}) @@ -1859,14 +2037,14 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { check("MOVUPS", Instr{Raw: "MOVUPS 8(BX), X0", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: Reg("X0")}}}) check("MOVAPS", Instr{Raw: "MOVAPS X0, 16(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 16}}}}) check("MOVO", Instr{Raw: "MOVO X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) - check("MOVQ", Instr{Raw: "MOVQ arg+0(FP), X0", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpReg, Reg: Reg("X0")}}}) - check("MOVQ", Instr{Raw: "MOVQ 24(BX), X1", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 24}}, {Kind: OpReg, Reg: Reg("X1")}}}) - check("MOVQ", Instr{Raw: "MOVQ example.global(SB), X2", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: Reg("X2")}}}) - check("MOVQ", Instr{Raw: "MOVQ X0, 32(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 32}}}}) + checkTransfer("MOVQ", Instr{Raw: "MOVQ arg+0(FP), X0", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpReg, Reg: Reg("X0")}}}) + checkTransfer("MOVQ", Instr{Raw: "MOVQ 24(BX), X1", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 24}}, {Kind: OpReg, Reg: Reg("X1")}}}) + checkTransfer("MOVQ", Instr{Raw: "MOVQ example.global(SB), X2", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: Reg("X2")}}}) + checkTransfer("MOVQ", Instr{Raw: "MOVQ X0, 32(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 32}}}}) check("MOVOU", Instr{Raw: "MOVOU example.global(SB), X0", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: Reg("X0")}}}) check("MOVOA", Instr{Raw: "MOVOA X0, example.out(SB)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpSym, Sym: "example.out(SB)"}}}) check("KMOVB", Instr{Raw: "KMOVB 8(BX), K1", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: Reg("K1")}}}) - check("KMOVQ", Instr{Raw: "KMOVQ 8(BX), AX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: AX}}}) + check("KMOVD", Instr{Raw: "KMOVD K1, K7", Args: []Operand{{Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("K7")}}}) check("VMOVDQU64", Instr{Raw: "VMOVDQU64 example.zin(SB), Z0", Args: []Operand{{Kind: OpSym, Sym: "example.zin(SB)"}, {Kind: OpReg, Reg: Reg("Z0")}}}) check("VMOVDQU64", Instr{Raw: "VMOVDQU64 Z0, example.zout(SB)", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpSym, Sym: "example.zout(SB)"}}}) check("VMOVDQU", Instr{Raw: "VMOVDQU example.yin(SB), Y0", Args: []Operand{{Kind: OpSym, Sym: "example.yin(SB)"}, {Kind: OpReg, Reg: Reg("Y0")}}}) @@ -1875,16 +2053,16 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { check("VMOVDQU", Instr{Raw: "VMOVDQU X0, example.xout(SB)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpSym, Sym: "example.xout(SB)"}}}) check("PEXTRB", Instr{Raw: "PEXTRB $1, X1, example.byteout(SB)", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X1")}, {Kind: OpSym, Sym: "example.byteout(SB)"}}}) - if _, _, err := c.lowerVec("MOVL", Instr{Raw: "MOVL label, X0", Args: []Operand{{Kind: OpIdent, Ident: "label"}, {Kind: OpReg, Reg: Reg("X0")}}}); err == nil { + if _, _, err := c.lowerVectorScalarIntegerMove("MOVL", Instr{Op: "MOVL", Raw: "MOVL label, X0", Args: []Operand{{Kind: OpIdent, Ident: "label"}, {Kind: OpReg, Reg: Reg("X0")}}}); err == nil { t.Fatalf("MOVL bad src unexpectedly succeeded") } - if _, _, err := c.lowerVec("MOVQ", Instr{Raw: "MOVQ label, X0", Args: []Operand{{Kind: OpIdent, Ident: "label"}, {Kind: OpReg, Reg: Reg("X0")}}}); err == nil { + if _, _, err := c.lowerVectorScalarIntegerMove("MOVQ", Instr{Op: "MOVQ", Raw: "MOVQ label, X0", Args: []Operand{{Kind: OpIdent, Ident: "label"}, {Kind: OpReg, Reg: Reg("X0")}}}); err == nil { t.Fatalf("MOVQ bad src unexpectedly succeeded") } - if _, _, err := c.lowerVec("MOVQ", Instr{Raw: "MOVQ X0, label", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpIdent, Ident: "label"}}}); err == nil { + if _, _, err := c.lowerVectorScalarIntegerMove("MOVQ", Instr{Op: "MOVQ", Raw: "MOVQ X0, label", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpIdent, Ident: "label"}}}); err == nil { t.Fatalf("MOVQ bad dst unexpectedly succeeded") } - if ok, term, err := c.lowerVec("KXORQ", Instr{Raw: "KXORQ AX, K1, K2", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("K2")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("KXORQ", Instr{Raw: "KXORQ AX, K1, K2", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("K2")}}}); !ok || term || err == nil { t.Fatalf("KXORQ non-k src = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("KMOVB", Instr{Raw: "KMOVB AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}); err == nil { @@ -1902,19 +2080,19 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { if _, _, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, 7", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpImm, Imm: 7}}}); err == nil { t.Fatalf("VPERMB non-reg dst unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); !ok || term || err == nil { t.Fatalf("VPERMB wrong dst reg = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, 7, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpImm, Imm: 7}, {Kind: OpReg, Reg: Reg("Z2")}}}); err == nil { t.Fatalf("VPERMB non-reg mask unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, AX, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z2")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, AX, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z2")}}}); !ok || term || err == nil { t.Fatalf("VPERMB wrong mask reg = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("VGF2P8AFFINEQB", Instr{Raw: "VGF2P8AFFINEQB Z0, Z1, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Z2")}}}); err == nil { t.Fatalf("VGF2P8AFFINEQB short form unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VGF2P8AFFINEQB", Instr{Raw: "VGF2P8AFFINEQB $1, Z0, Z1, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VGF2P8AFFINEQB", Instr{Raw: "VGF2P8AFFINEQB $1, Z0, Z1, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); !ok || term || err == nil { t.Fatalf("VGF2P8AFFINEQB wrong dst reg = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}}}); err == nil { @@ -1923,13 +2101,13 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { if _, _, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, 7", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpImm, Imm: 7}}}); err == nil { t.Fatalf("VPERMI2B non-reg dst unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); !ok || term || err == nil { t.Fatalf("VPERMI2B wrong dst reg = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, 7, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpImm, Imm: 7}, {Kind: OpReg, Reg: Reg("Z2")}}}); err == nil { t.Fatalf("VPERMI2B non-reg mask unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, AX, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z2")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, AX, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z2")}}}); !ok || term || err == nil { t.Fatalf("VPERMI2B wrong mask reg = (%v, %v, %v)", ok, term, err) } for _, tc := range []struct { @@ -1938,53 +2116,60 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { want string wantHandled bool }{ - {"VPOPCNTB", Instr{Raw: "VPOPCNTB Z0, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", false}, + {"VPOPCNTB", Instr{Raw: "VPOPCNTB Z0, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", true}, {"VPCMPUQ", Instr{Raw: "VPCMPUQ $3, Z0, Z1, K1", Args: []Operand{{Kind: OpImm, Imm: 3}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("K1")}}}, "bad imm", true}, {"VPCMPUQ", Instr{Raw: "VPCMPUQ $1, Z0, Z1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}}}, "wrong dst reg", false}, - {"VPCOMPRESSQ", Instr{Raw: "VPCOMPRESSQ Z0, AX, Z1", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z1")}}}, "wrong mask", false}, - {"VPCOMPRESSQ", Instr{Raw: "VPCOMPRESSQ Z0, K1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", false}, + {"VPCOMPRESSQ", Instr{Raw: "VPCOMPRESSQ Z0, AX, Z1", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z1")}}}, "wrong mask", true}, + {"VPCOMPRESSQ", Instr{Raw: "VPCOMPRESSQ Z0, K1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", true}, {"VPXORQ", Instr{Raw: "VPXORQ Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong z dst", true}, - {"VPSHUFB", Instr{Raw: "VPSHUFB Y0, Y1, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: AX}}}, "wrong dst", false}, - {"VPSHUFD", Instr{Raw: "VPSHUFD $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", false}, + {"VPSHUFB", Instr{Raw: "VPSHUFB Y0, Y1, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: AX}}}, "wrong dst", true}, + {"VPSHUFD", Instr{Raw: "VPSHUFD $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", true}, {"VPSLLD", Instr{Raw: "VPSLLD Y0, Y1", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}}}, "short form", true}, - {"VPERM2I128", Instr{Raw: "VPERM2I128 $1, Y0, Y1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst", false}, - {"VINSERTI128", Instr{Raw: "VINSERTI128 $1, X0, Y0, X1", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "wrong dst", false}, + {"VPERM2I128", Instr{Raw: "VPERM2I128 $1, Y0, Y1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst", true}, + {"VINSERTI128", Instr{Raw: "VINSERTI128 $1, X0, Y0, X1", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "wrong dst", true}, {"VMOVNTDQ", Instr{Raw: "VMOVNTDQ Y0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: AX}}}, "bad dst", true}, {"AESKEYGENASSIST", Instr{Raw: "AESKEYGENASSIST X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "missing imm", true}, - {"VPTEST", Instr{Raw: "VPTEST X0, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", false}, - {"PCMPESTRI", Instr{Raw: "PCMPESTRI $1, 8(BX), X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: Reg("X0")}}}, "unsupported imm", true}, + {"VPTEST", Instr{Raw: "VPTEST X0, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", true}, + {"PCMPESTRI", Instr{Raw: "PCMPESTRI $-1, 8(BX), X0", Args: []Operand{{Kind: OpImm, Imm: -1}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, {"PCMPESTRI", Instr{Raw: "PCMPESTRI $0x0c, AX, X0", Args: []Operand{{Kind: OpImm, Imm: 0x0c}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "bad mem operand", true}, {"VPBLENDD", Instr{Raw: "VPBLENDD $1, Y0, Y1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst", false}, {"VPBROADCASTB", Instr{Raw: "VPBROADCASTB Y0, Y1", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}}}, "wrong src", false}, - {"VPSRLDQ", Instr{Raw: "VPSRLDQ $2, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 2}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", false}, + {"VPSRLDQ", Instr{Raw: "VPSRLDQ $2, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 2}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", true}, {"PUNPCKLBW", Instr{Raw: "PUNPCKLBW Y0, X0", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", false}, - {"PSHUFHW", Instr{Raw: "PSHUFHW $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", false}, + {"PSHUFHW", Instr{Raw: "PSHUFHW $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", true}, {"SHUFPS", Instr{Raw: "SHUFPS $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", false}, {"MOVOU", Instr{Raw: "MOVOU AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", true}, {"MOVOU", Instr{Raw: "MOVOU X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}, "wrong dst class", false}, {"PXOR", Instr{Raw: "PXOR X0", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}}}, "short form", true}, - {"PADDL", Instr{Raw: "PADDL X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}, "wrong dst class", false}, + {"PADDL", Instr{Raw: "PADDL X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}, "wrong dst class", true}, {"PSLLL", Instr{Raw: "PSLLL AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "non-imm shift", true}, {"PCMPEQL", Instr{Raw: "PCMPEQL Y0, X0", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", false}, - {"VPCLMULQDQ", Instr{Raw: "VPCLMULQDQ $1, Z0, Z1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst class", false}, - {"VPTERNLOGD", Instr{Raw: "VPTERNLOGD $0x95, Z0, Z1, Z2", Args: []Operand{{Kind: OpImm, Imm: 0x95}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Z2")}}}, "unsupported imm", true}, - {"VEXTRACTF32X4", Instr{Raw: "VEXTRACTF32X4 $1, X0, X1", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "wrong src class", false}, + {"VPCLMULQDQ", Instr{Raw: "VPCLMULQDQ $1, Z0, Z1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst class", true}, + {"VPTERNLOGD", Instr{Raw: "VPTERNLOGD $0x100, Z0, Z1, Z2", Args: []Operand{{Kind: OpImm, Imm: 0x100}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Z2")}}}, "out-of-range imm", true}, + {"VEXTRACTF32X4", Instr{Raw: "VEXTRACTF32X4 $1, X0, X1", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "wrong src class", true}, {"PCLMULQDQ", Instr{Raw: "PCLMULQDQ AX, X0, X1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "missing imm", true}, {"PCMPEQB", Instr{Raw: "PCMPEQB Y0, X0", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", false}, - {"PMOVMSKB", Instr{Raw: "PMOVMSKB Y0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: AX}}}, "wrong src class", false}, + {"PMOVMSKB", Instr{Raw: "PMOVMSKB Y0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: AX}}}, "wrong src class", true}, {"PSHUFB", Instr{Raw: "PSHUFB Y0, X0", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong mask class", true}, {"PINSRQ", Instr{Raw: "PINSRQ AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "short form", true}, - {"PINSRD", Instr{Raw: "PINSRD $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", false}, - {"PINSRW", Instr{Raw: "PINSRW $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", false}, - {"PINSRB", Instr{Raw: "PINSRB $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", false}, + {"PINSRD", Instr{Raw: "PINSRD $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", true}, + {"PINSRW", Instr{Raw: "PINSRW $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", true}, + {"PINSRB", Instr{Raw: "PINSRB $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", true}, {"PEXTRB", Instr{Raw: "PEXTRB AX, X0, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: BX}}}, "missing imm", true}, - {"PALIGNR", Instr{Raw: "PALIGNR $1, Y0, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", false}, - {"PSRLDQ", Instr{Raw: "PSRLDQ $17, X0", Args: []Operand{{Kind: OpImm, Imm: 17}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, - {"PSLLDQ", Instr{Raw: "PSLLDQ $17, X0", Args: []Operand{{Kind: OpImm, Imm: 17}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, + {"PALIGNR", Instr{Raw: "PALIGNR $1, Y0, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", true}, + {"PSRLDQ", Instr{Raw: "PSRLDQ $128, X0", Args: []Operand{{Kind: OpImm, Imm: 128}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, + {"PSLLDQ", Instr{Raw: "PSLLDQ $128, X0", Args: []Operand{{Kind: OpImm, Imm: 128}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, {"PSRLQ", Instr{Raw: "PSRLQ AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "non-imm shift", true}, {"PEXTRD", Instr{Raw: "PEXTRD $1, X0, label", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpIdent, Ident: "label"}}}, "bad dst kind", true}, } { - ok, term, err := c.lowerVec(tc.op, tc.ins) + before := b.String() + ok, term, err := lowerAMD64PackedArithmeticOrVectorForTest(c, tc.op, tc.ins) + if _, typed := x86InsertExtractSpecs[tc.op]; typed { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+tc.ins.Raw+"\nRET\n", "", false) + if !ok || term || err == nil || b.String() != before { + t.Fatalf("typed insert/extract rejection emitted IR: %s: (%v, %v, %v)", tc.ins.Raw, ok, term, err) + } + } if tc.wantHandled { if !ok || err == nil { t.Fatalf("%s %s = (%v, %v, %v), want handled error", tc.op, tc.want, ok, term, err) @@ -1998,8 +2183,8 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { out := b.String() for _, want := range []string{ - "load i64, ptr @\"example.global\"", - "load i8, ptr", + "load volatile i64, ptr @\"example.global\"", + "load volatile i8, ptr", "and i64", "store i64", "zext i8", @@ -2012,6 +2197,40 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { } } +func TestAMD64RejectedLegacyVectorDestinationDoesNotMutateIR(t *testing.T) { + for _, test := range []struct { + op Op + handled bool + }{ + {op: "PSUBUSB", handled: true}, + {op: "PABSD"}, + {op: "PTEST", handled: true}, + } { + t.Run(string(test.op), func(t *testing.T) { + ins := Instr{ + Op: test.op, Raw: string(test.op) + " X0, AX", + Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}, + } + c, b := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + beforeIR, beforeTmp := b.String(), c.tmp + ok, term, err := lowerAMD64PackedArithmeticOrVectorForTest(c, test.op, ins) + if test.handled { + if !ok || term || err == nil { + t.Fatalf("lowerVec(%s) = (%v, %v, %v), want handled rejection", test.op, ok, term, err) + } + } else if ok || term || err != nil { + t.Fatalf("lowerVec(%s) = (%v, %v, %v), want unhandled rejection", test.op, ok, term, err) + } + if got := b.String(); got != beforeIR { + t.Fatalf("lowerVec(%s) emitted IR before rejecting destination:\n%s", test.op, strings.TrimPrefix(got, beforeIR)) + } + if c.tmp != beforeTmp { + t.Fatalf("lowerVec(%s) advanced temporary counter from %d to %d", test.op, beforeTmp, c.tmp) + } + }) + } +} + func TestAMD64CtxAliasAndFPFallbackCoverage(t *testing.T) { sig := FuncSig{ Name: "example.ctxfallback", @@ -2055,7 +2274,7 @@ func TestAMD64CtxAliasAndFPFallbackCoverage(t *testing.T) { t.Fatalf("storeRegSized(%s, %s) error = %v", tc.r, tc.ty, err) } } - for _, r := range []Reg{AL, AH, BL, BH, CL, CH, DL, DH} { + for _, r := range []Reg{AL, AH, BL, BH, CL, CH, DL, DH, BPB, SIB, DIB, R8B, R9B, R10B, R11B, R12B, R13B, R14B, R15B} { if got, err := c.loadReg(r); err != nil || got == "" { t.Fatalf("loadReg(%s) = (%q, %v)", r, got, err) } @@ -2075,6 +2294,17 @@ func TestAMD64CtxAliasAndFPFallbackCoverage(t *testing.T) { {CH, "35"}, {DL, "20"}, {DH, "36"}, + {BPB, "22"}, + {SIB, "23"}, + {DIB, "24"}, + {R8B, "25"}, + {R9B, "26"}, + {R10B, "27"}, + {R11B, "28"}, + {R12B, "29"}, + {R13B, "30"}, + {R14B, "31"}, + {R15B, "32"}, } { if err := c.storeReg(tc.r, tc.v); err != nil { t.Fatalf("storeReg(%s) error = %v", tc.r, err) diff --git a/amd64_horizontal_float_arithmetic_type_test.go b/amd64_horizontal_float_arithmetic_type_test.go new file mode 100644 index 000000000..30adcddad --- /dev/null +++ b/amd64_horizontal_float_arithmetic_type_test.go @@ -0,0 +1,42 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The arithmetic carrier must retain the same lane type as the horizontal +// shuffles. These are the complete register rows of Go's yxm/_yvaddsubpd +// tables, independently enumerated rather than derived from the emitter. +func TestX86HorizontalFloatArithmeticPreservesTypedLaneWidth(t *testing.T) { + for _, opcode := range []string{ + "HADDPS", "HADDPD", "HSUBPS", "HSUBPD", + "VHADDPS", "VHADDPD", "VHSUBPS", "VHSUBPD", + } { + banks := []string{"X"} + vector := strings.HasPrefix(opcode, "V") + if vector { + banks = append(banks, "Y") + } + for _, bank := range banks { + t.Run(opcode+"/"+bank, func(t *testing.T) { + row := fmt.Sprintf("%s %s0,%s1", opcode, bank, bank) + if vector { + row += "," + bank + "2" + } + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + ir := x86UnboundVoidScalarFormatObject(t, source, "f", "amd64", "x86_64-unknown-linux-gnu") + width := 16 + if bank == "Y" { + width = 32 + } + call := fmt.Sprintf("call <%d x float> asm sideeffect", width/4) + if strings.Count(ir, call) != 1 || !strings.Contains(ir, strings.ToLower(opcode)+" ") || !strings.Contains(ir, "~{fpsr}") { + t.Fatalf("%s lost its complete typed arithmetic carrier %q", row, call) + } + }) + } + } +} diff --git a/amd64_horizontal_float_forms.go b/amd64_horizontal_float_forms.go new file mode 100644 index 000000000..152ba508f --- /dev/null +++ b/amd64_horizontal_float_forms.go @@ -0,0 +1,74 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86HorizontalFloatForm struct { + spec amd64HorizontalFloatingSpec + first, second, destination, memory int + byteWidth int +} + +// Each V-prefixed opcode has exactly the two _yvaddsubpd VEX rows, not +// EVEX rows. Sharing that ytab does not confer VADDSUB's alternating effect. +func x86HorizontalFloatProperties(ins Instr, goarch string) (x86HorizontalFloatForm, bool, error) { + base, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := amd64HorizontalFloatingSpecs[Op(base)] + if !recognized { + return x86HorizontalFloatForm{}, false, nil + } + form := x86HorizontalFloatForm{spec: spec, first: 0, second: 1, destination: len(ins.Args) - 1, memory: -1, byteWidth: 16} + fail := func() (x86HorizontalFloatForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go horizontal floating grammar: %q", goarch, ins.Op, ins.Raw) + } + if dotted || goarch != "386" && goarch != "amd64" || !spec.vector && len(ins.Args) != 2 || spec.vector && len(ins.Args) != 3 { + return fail() + } + if spec.vector { + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if form.byteWidth != 16 && form.byteWidth != 32 { + return fail() + } + } + register := func(operand Operand) bool { + index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + return operand.Kind == OpReg && valid && (goarch == "amd64" && index < 16 || goarch == "386" && index < 8) + } + if goarch == "386" && spec.vector { + for _, operand := range ins.Args { + index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + if operand.Kind == OpReg && valid && index >= 8 && index < 16 { + // Go's named VEX ytab can emit these encodings in mode32, + // but that does not establish extra physical SIMD registers. + return form, true, fmt.Errorf("%w: 386 horizontal VEX register %s has no proved physical mode32 binding", ErrProbeNeedsContext, operand.Reg) + } + } + } + if !register(ins.Args[form.second]) || !register(ins.Args[form.destination]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) { + if first.Kind == OpSym { + symbol := strings.TrimSpace(first.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if first.Kind == OpMem { + displacement := first.Mem.Off + if displacement != int64(int32(displacement)) && (goarch != "386" || displacement != int64(uint32(displacement))) || + !x86MemoryRegistersValidForArch(first.Mem, goarch) || first.Mem.Index != "" && + (first.Mem.Index == SP || first.Mem.Scale != 1 && first.Mem.Scale != 2 && first.Mem.Scale != 4 && first.Mem.Scale != 8) { + return fail() + } + } + form.memory = 0 + } else if !register(first) { + return fail() + } + return form, true, nil +} diff --git a/amd64_horizontal_float_forms_test.go b/amd64_horizontal_float_forms_test.go new file mode 100644 index 000000000..995fce932 --- /dev/null +++ b/amd64_horizontal_float_forms_test.go @@ -0,0 +1,268 @@ +package plan9asm + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86HorizontalFloatEveryGoOpcodeAttribute(t *testing.T) { + if len(amd64HorizontalFloatingSpecs) != 8 { + t.Fatal("incomplete horizontal floating family") + } + if !goToolchainAtLeast(runtime.Version(), 1, 27) { + return // older source builds still execute grammar, object and bit tests + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, op := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD", "VHADDPS", "VHADDPD", "VHSUBPS", "VHSUBPD"} { + bits, prefix, opcode := 32, "vexF2", "0x7C" + if strings.HasSuffix(op, "PD") { + bits, prefix = 64, "vex66" + } + if strings.Contains(op, "SUB") { + opcode = "0x7D" + } + spec := amd64HorizontalFloatingSpecs[Op(op)] + if spec.laneBits != bits || spec.subtract != strings.Contains(op, "SUB") || spec.vector != strings.HasPrefix(op, "V") { + t.Fatalf("%s independent opcode attributes differ: %+v", op, spec) + } + if !spec.vector { + if !strings.Contains(string(legacy), "{A"+op+", yxm,") { + t.Fatalf("Go legacy yxm changed: %s", op) + } + continue + } + start := strings.Index(string(avx), "{as: A"+op+",") + if start < 0 { + t.Fatalf("Go omits %s", op) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(row, "ytab: _yvaddsubpd,") || strings.Contains(row, "evex") || strings.Contains(row, "Bcst") || strings.Contains(row, "Sae") { + t.Fatalf("Go VEX-only grammar changed: %s", row) + } + for _, width := range []string{"vex128", "vex256"} { + if !strings.Contains(row, "avxEscape | "+width+" | "+prefix+" | vex0F | vexW0, "+opcode) { + t.Fatalf("%s complete encoding attributes changed: %s", op, row) + } + } + } +} + +func TestX86HorizontalFloatDispatchIdentityBeforeEmission(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + op := file.Funcs[0].Instrs[1].Op + for _, identity := range []Op{"", op, Op(strings.ToLower(string(op))), "UNKNOWN", "VHADDPS.Z"} { + ins := file.Funcs[0].Instrs[1] + ins.Op = identity + original := ins + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerHorizontalFloating(op, ins) + conflict := identity == "UNKNOWN" || identity == "VHADDPS.Z" + if !known || term || (err != nil) != conflict || conflict && output.String() != before || !reflect.DeepEqual(ins, original) { + t.Fatalf("%s/%s source identity: %t/%t/%v", op, identity, known, term, err) + } + } + } +} + +func TestX86HorizontalFloatMalformedFormsBeforeEmission(t *testing.T) { + for _, row := range []string{ + "HADDPS X0,Y1", "HSUBPD X0,X16", "HADDPD.SAE X0,X1", "HSUBPS $data(SB),X1", + "VHADDPS X0,X1", "VHADDPD X0,Y1,Y2", "VHSUBPS Z0,Z1,Z2", "VHSUBPD X0,X1,X16", + "VHADDPS X0,X1,K1,X2", "VHADDPD.Z X0,X1,X2", "VHSUBPS.BCST (AX),X1,X2", + "VHSUBPD.RN_SAE Y0,Y1,Y2", "VHADDPS. X0,X1,X2", "VHADDPD (AX)(SP*2),X1,X2", + "VHSUBPS 2147483648(AX),X1,X2", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerHorizontalFloating(ins.Op, ins) + if !known || term || err == nil || output.String() != before { + t.Fatalf("invalid form emitted IR: %t/%t/%v", known, term, err) + } + }) + } +} + +func TestX86HorizontalFloatGoFormatsFiveLLVMTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + for index, row := range x86HorizontalFloatFormats() { + name := fmt.Sprintf("horizontal%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + triples := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + file, signatures := x86UnboundVoidFormatSources(t, source.String(), arch, triple) + ir := x86ScalarFormatObjectUnit(t, file, signatures, arch, triple, 0) + compileLLVMToObject(t, llc, triple, "horizontal-formats.ll", "horizontal-formats.o", ir) + } + } +} + +func TestX86HorizontalFloatLegacyMode32ClassesMatchGo(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + // NOSPLIT is a CPP macro, not an assembler constant without its header. + goAssemblerOracleOutput(t, arch, + "TEXT horizontal_floating_complete_forms(SB),NOSPLIT,$0-0\nHADDPS X0,X1\nRET\n", "", false) + } + // Keep the former complete-form fixture's rejected source, rather than + // treating mode32 X15 as a translator-supported ordinary register. + for _, opcode := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD"} { + for _, instruction := range []string{opcode + " X15,X1", opcode + " 8(BX),X15"} { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + instruction + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", false) + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}, + }); err == nil { + t.Fatal("translator accepted the original Go-rejected legacy X15 form") + } + }) + } + } +} + +func TestX86HorizontalFloat386VEXHighRegistersKeepPhysicalContext(t *testing.T) { + // Execute each compatibility Go assembler rather than trusting the current + // optab-only oracle. Actual objects are syntax evidence, not mode32 runtime. + const source = "TEXT f(SB),4,$0-0\nVHADDPS X8,X9,X10\nVHSUBPD Y8,Y9,Y10\nRET\n" + dir := t.TempDir() + asm, object := filepath.Join(dir, "high.s"), filepath.Join(dir, "high.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "oracle", "-o", object, asm) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=386") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), true, "386"); failure != nil { + t.Fatalf("actual named high-register fixture: %v\n%s", failure, output) + } + if !bytes.Contains(output, []byte("c4 41 33 7c d0 c4 41 35 7d d0 c3")) { + t.Fatalf("actual Go mode32 source bytes changed:\n%s", output) + } + for _, row := range []struct{ code, op, bank string }{{"c441337cd0", "VHADDPS", "X"}, {"c441357dd0", "VHSUBPD", "Y"}} { + code, err := hex.DecodeString(row.code) + if err != nil { + t.Fatal(err) + } + ins, length, recognized, err := decodedX86VEXHorizontalFloatInstruction(code, 64) + if !recognized || err != nil || length != len(code) || string(ins.Op) != row.op || + ins.Args[0].Reg != Reg(row.bank+"8") || ins.Args[1].Reg != Reg(row.bank+"9") || ins.Args[2].Reg != Reg(row.bank+"10") { + t.Fatalf("source bytes lost physical register identities: %+v/%d/%t/%v", ins, length, recognized, err) + } + if _, _, known, err := decodedX86VEXHorizontalFloatInstruction(code, 32); !known || err == nil { + t.Fatal("mode32 decoder invented high physical SIMD registers") + } + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": {Ret: Void}}}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("named Go object laundered as physical mode32 success: %v", err) + } + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\nHADDPS X8,X9\nRET\n", "", false) +} + +func TestX86HorizontalFloatCompleteTypedFPSpanBeforeEmission(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + if !strings.Contains(row, "(BX)") { + continue + } + width := 16 + if strings.Contains(row, "Y") { + width = 32 + } + row = strings.Replace(row, "(BX)", "a+0(FP)", 1) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-32\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, missing := range []int{-1, 0, width/8 - 1} { + sig := FuncSig{Name: "f", Ret: Void} + for index := 0; index < width/8; index++ { + if index == missing { + continue + } + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(index * 8), Type: I64, Index: len(sig.Args), Field: -1}) + sig.Args = append(sig.Args, I64) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + ins := file.Funcs[0].Instrs[1] + before := output.String() + known, term, err := ctx.lowerHorizontalFloating(ins.Op, ins) + needsContext := missing >= 0 || !strings.HasPrefix(row, "V") + if !known || term || (err != nil) != needsContext || needsContext && output.String() != before { + t.Fatalf("%s missing typed slot %d: %t/%t/%v", row, missing, known, term, err) + } + } + } +} + +func TestX86HorizontalFloatPairsUseSingleStrictVectorOperation(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ir := x86ScalarFormatObjectUnit(t, file, map[string]FuncSig{"f": {Ret: Void}}, "amd64", "x86_64-unknown-linux-gnu", 0) + // One original native reduction owns its joint exception, pair NaN + // priority and ambient MXCSR semantics. No scalar reconstruction or + // pure arithmetic carrier can replace it, including a dead output. + // The common prelude still declares unrelated constrained intrinsics. + // Inspect this original instruction's body, not global declarations. + _, body, found := strings.Cut(ir, "define void @InstructionFormat0(") + body, _, ended := strings.Cut(body, "\n}") + opcode := `asm sideeffect "` + strings.ToLower(strings.Fields(row)[0]) + ` ` + if got := strings.Count(body, opcode); !found || !ended || got != 1 || + strings.Count(body, "asm sideeffect") != 1 || !strings.Contains(body, "~{fpsr}") || + !strings.Contains(body, "~{memory}") || strings.Contains(body, "@llvm.experimental.constrained") { + t.Fatalf("%s lost its one native strict horizontal operation (%d):\n%s", row, got, ir) + } + } +} diff --git a/amd64_horizontal_float_go_oracle_test.go b/amd64_horizontal_float_go_oracle_test.go new file mode 100644 index 000000000..397b8c5b3 --- /dev/null +++ b/amd64_horizontal_float_go_oracle_test.go @@ -0,0 +1,64 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86HorizontalFloatOriginalGoBitOracle(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle +import ("encoding/binary";"testing") +func TestOriginalHorizontalGolden(t *testing.T) { +`) + for index, opcode := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD"} { + name, bits, move := fmt.Sprintf("horizontal%d", index), 32, "MOVUPS" + if strings.HasSuffix(opcode, "PD") { + bits, move = 64, "MOVUPD" + } + fmt.Fprintf(&declarations, "func %s(out *[20]byte,a,b *[16]byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\nSTMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\n%s (BX),X0\n%s (CX),X1\n%s X0,X1\nMOVUPS X1,(AX)\nSTMXCSR 16(AX)\nLDMXCSR saved-4(SP)\nRET\n", name, move, move, opcode) + // Different qNaN/sNaN payloads in both halves establish horizontal + // pair priority independently of the production binary helper. + left, right, quiet := uint64(0x7f812345), uint64(0xffc45678), uint64(0x7fc12345) + if bits == 64 { + left, right, quiet = 0x7ff0123456789abc, 0xfff8fedcba987654, 0x7ff8123456789abc + } + fmt.Fprintf(&tests, `{ + var a,b [16]byte;var out [20]byte + for lane:=0;lane<16/%[2]d;lane++ { + value:=uint64(0x%[3]x);if lane%%2!=0 {value=0x%[4]x} + if %[5]d==32 {binary.LittleEndian.PutUint32(a[lane*4:],uint32(value));binary.LittleEndian.PutUint32(b[lane*4:],uint32(value))} else {binary.LittleEndian.PutUint64(a[lane*8:],value);binary.LittleEndian.PutUint64(b[lane*8:],value)} + } + %[1]s(&out,&a,&b,0x1f80) + for lane:=0;lane<16/%[2]d;lane++ { + var got uint64;if %[5]d==32 {got=uint64(binary.LittleEndian.Uint32(out[lane*4:]))}else{got=binary.LittleEndian.Uint64(out[lane*8:])} + if got!=0x%[6]x {t.Fatalf("%[1]s pair priority lane=%%d got=%%x",lane,got)} + } + if flags:=binary.LittleEndian.Uint32(out[16:]);flags!=0x1f81 {t.Fatalf("%[1]s sNaN flags=%%x",flags)} +} +`, name, bits/8, left, right, bits, quiet) + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go horizontal bit/flags oracle: %v\n%s", err, output) + } +} diff --git a/amd64_horizontal_float_numerical_test.go b/amd64_horizontal_float_numerical_test.go new file mode 100644 index 000000000..c66db7827 --- /dev/null +++ b/amd64_horizontal_float_numerical_test.go @@ -0,0 +1,133 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// This list follows Go's four yxm and four _yvaddsubpd opcode rows. It is +// deliberately independent of the production parser and emitter. +func x86HorizontalFloatFormats() []string { + var rows []string + for _, op := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD"} { + rows = append(rows, op+" X0,X1", op+" (BX),X1") + } + for _, op := range []string{"VHADDPS", "VHADDPD", "VHSUBPS", "VHSUBPD"} { + for _, bank := range []string{"X", "Y"} { + rows = append(rows, fmt.Sprintf("%s %s0,%s1,%s2", op, bank, bank, bank), fmt.Sprintf("%s (BX),%s1,%s2", op, bank, bank)) + } + } + return rows +} + +func x86HorizontalFloatSemanticsFormats() []string { + rows := x86HorizontalFloatFormats() + for _, row := range x86HorizontalFloatFormats() { + if !strings.HasPrefix(row, "V") || strings.Contains(row, "(BX)") { + continue + } + for _, destination := range []string{"0", "1"} { + rows = append(rows, strings.TrimSuffix(row, "2")+destination) + } + } + return rows +} + +// A real SSE3 horizontal instruction supplies the entire 128-bit result, +// not a scalar ADD/SUB reconstruction. VEX Y results concatenate two such +// independent blocks. MXCSR is observed before restoring the caller state. +const x86HorizontalFloatSSEOracle = `#include +#include +#include +typedef uint64_t vector128 __attribute__((vector_size(16))); +static unsigned last_horizontal_csr; +static void horizontal_result(void *out,const void *second,const void *first,unsigned bits,int sub,unsigned csr) { + unsigned saved;vector128 a,b;memcpy(&a,second,16);memcpy(&b,first,16); + __asm__ volatile("stmxcsr %0":"=m"(saved));__asm__ volatile("ldmxcsr %0"::"m"(csr)); + if(bits==32) { + if(sub)__asm__ volatile("hsubps %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("haddps %1,%0":"+x"(a):"x"(b)); + } else { + if(sub)__asm__ volatile("hsubpd %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("haddpd %1,%0":"+x"(a):"x"(b)); + } + memcpy(out,&a,16);__asm__ volatile("stmxcsr %0":"=m"(last_horizontal_csr)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); +} +` + +func TestCrossLinuxRuntimeMatrixX86HorizontalFloatCompletePhysicalSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for index, row := range x86HorizontalFloatSemanticsFormats() { + op := strings.Fields(row)[0] + bits, width, dest, legacy := 32, 16, 2, !strings.HasPrefix(op, "V") + if strings.HasSuffix(op, "PD") { + bits = 64 + } + if strings.Contains(row, "Y") { + width = 32 + } + if legacy { + dest = 1 + } else { + dest = int(row[len(row)-1] - '0') + } + name := fmt.Sprintf("horizontal%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nRET\n", name, row, dest) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*);\n", name) + fmt.Fprintf(&checks, ` + for(unsigned vector=0;vector<20;vector++)for(unsigned mode=0;mode<4;mode++)for(unsigned control=0;control<6;control++) { + _Alignas(16) uint8_t aStorage[96],bStorage[96];uint8_t out[67],expected[64],beforeA[64],beforeB[64]; + unsigned offset=%[7]d?17:16;uint8_t *a=aStorage+offset,*b=bStorage+offset;memset(aStorage,0xa7,96);memset(bStorage,0xb9,96);memset(out,0xc3,67); + for(unsigned lane=0;lane<64/%[2]d;lane++) { + uint64_t av=edges%[3]d[(vector+lane)%%20],bv=edges%[3]d[(vector*3+lane*7)%%20]; + memcpy(a+lane*%[2]d,&av,%[2]d);memcpy(b+lane*%[2]d,&bv,%[2]d); + } + memcpy(beforeA,a,64);memcpy(beforeB,b,64); + unsigned csr=controls[control]|(mode<<13),observed,flags=0; + __asm__ volatile("ldmxcsr %%0"::"m"(csr));%[1]s(out+1,a,b); + __asm__ volatile("stmxcsr %%0":"=m"(observed)); + if(%[4]d)memcpy(expected,beforeB,64);else memset(expected,0,64); + for(unsigned block=0;block<%[5]d;block+=16) { + horizontal_result(expected+block,beforeB+block,beforeA+block,%[3]d,%[6]d,csr);flags|=last_horizontal_csr&63; + } + for(unsigned byte=0;byte<64;byte++)if(out[1+byte]!=expected[byte]) { + fprintf(stderr,"%[1]s vector/mode/csr=%%u/%%u/%%x byte=%%u got=%%x want=%%x\n",vector,mode,csr,byte,out[1+byte],expected[byte]);return 1; + } + if(observed!=(csr|flags)){fprintf(stderr,"%[1]s MXCSR=%%x want=%%x\n",observed,csr|flags);return 2;} + if(out[0]!=0xc3 || out[65]!=1 || out[66]!=0xc3)return 3; + if(memcmp(a,beforeA,64) || memcmp(b,beforeB,64))return 5; + for(unsigned byte=0;byte 1 || rule.Ytab != "_yvaddsubpd" || rule.Prefix != "Pavx" || rule.ZCase != "Zvex_rm_v_r" || rule.ZOffset != 2 || rule.EncodingOffset != 2*index || !reflect.DeepEqual(rule.Classes, classes[index]) || !reflect.DeepEqual(rule.OpBytes, bytes) { + t.Fatalf("unmodeled complete horizontal VEX row: %+v", rule) + } + bank := []string{"X", "Y"}[index] + rows = append(rows, rule.Opcode+" "+bank+"0,"+bank+"1,"+bank+"7", rule.Opcode+" -7(BX)(CX*4),"+bank+"1,"+bank+"7") + counts[rule.Opcode]++ + } + for _, op := range x86HorizontalIntegerOpcodes { + if counts["V"+op] != 2 { + t.Fatalf("missing complete original encoding rows for %s: %v", op, counts) + } + bits := 16 + if op == "PHADDD" || op == "PHSUBD" { + bits = 32 + } + for _, name := range []string{op, "V" + op} { + spec := amd64HorizontalIntegerSpecs[Op(name)] + if spec.laneBits != bits || spec.subtract != strings.Contains(name, "SUB") || spec.saturate != strings.HasSuffix(name, "SW") || spec.vector != strings.HasPrefix(name, "V") || spec.allowMMX != (name == "PHADDD") { + t.Fatalf("independent horizontal arithmetic attributes differ: %s %+v", name, spec) + } + } + } + return rows +} + +func TestX86HorizontalIntegerCompleteEncoderRowsThreeAPIsFiveTargets(t *testing.T) { + rows := x86OfficialHorizontalIntegerRows(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + source := "TEXT horizontal(SB),4,$0-0\n" + strings.Join(rows, "\n") + "\nRET\n" + if arch == "386" { + source = strings.Replace(source, "\nRET\n", "\nVPHSUBD 2147483648(AX),X1,X2\nRET\n", 1) + } + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"horizontal": {Name: "horizontal", Ret: Void}}} + if ir, err := Translate(file, options); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized original formats acquired numerical/MMX inputs", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("module API invented original inputs", err) + } + translated, err := translateGoModuleInContext(ctx, x86SADGoPackage(t, arch, "()"), []byte(strings.Replace(source, "horizontal(SB)", "·sad(SB)", 1)), GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go API invented original inputs", err) + } + ir := x86InstructionFormatObjectUnit(t, file, options.Sigs, arch, triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerHorizontalInteger(ins.Op, ins) + }) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, text, level) + } + } + }) + } +} + +func TestX86HorizontalIntegerGrammarRejectsBeforeEveryConsumer(t *testing.T) { + for _, row := range []string{ + "PHADDW M0,M1", "PHADDSW X0,M1", "PHSUBD Y0,Y1", "PHADDD.Z X0,X1", "PHADDW X0,X16", + "VPHADDD X0,X1", "VPHADDW X0,Y1,Y2", "VPHSUBD Z0,Z1,Z2", "VPHSUBSW X0,X1,K1,X2", "VPHSUBW.Z X0,X1,X2", + "VPHADDD.BCST (AX),X1,X2", "VPHADDW.SAE X0,X1,X2", "VPHSUBW.RN_SAE Y0,Y1,Y2", "VPHADDD (AX)(SP*2),X1,X2", "VPHSUBD 4294967296(AX),X1,X2", "PHADDW $table(SB),X1", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT sad(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerHorizontalInteger(ins.Op, ins) + if !known || term || err == nil || output.String() != before || x86CallerVectorMoveKnown(ins) { + t.Fatal("invalid horizontal grammar emitted or registered", known, term, err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0)}} + state = x86CloneCallerState(state) + original := x86CloneCallerState(state) + if x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, original) { + t.Fatal("invalid grammar mutated caller state") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + goAssemblerOracleOutput(t, arch, source, "", false) + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"sad": {Ret: Void}}} + if ir, err := Translate(file, options); err == nil || ir != "" { + t.Fatal("invalid text API form accepted") + } + llvmCtx := llvm.NewContext() + defer llvmCtx.Dispose() + module, err := TranslateModuleInContext(llvmCtx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err == nil { + t.Fatal("invalid module API form accepted") + } + translated, err := translateGoModuleInContext(llvmCtx, x86SADGoPackage(t, arch, "()"), []byte(strings.Replace(source, "sad(SB)", "·sad(SB)", 1)), GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err == nil { + t.Fatal("invalid Go API form accepted") + } + }) + } + }) + } +} + +func TestX86HorizontalIntegerDispatchAndCompleteFPSpanBeforeEmission(t *testing.T) { + for _, row := range x86OfficialHorizontalIntegerRows(t) { + ins := x86YMMReadSource(t, row) + for _, identity := range []Op{"", ins.Op, Op(strings.ToLower(string(ins.Op))), "UNKNOWN", "VPHADDD.Z"} { + local := ins + local.Op = identity + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + before := output.String() + _, _, err := ctx.lowerHorizontalInteger(ins.Op, local) + if identity == "UNKNOWN" || identity == "VPHADDD.Z" { + if err == nil || output.String() != before { + t.Fatal("dispatch conflict emitted", row, identity) + } + } else if err != nil { + t.Fatal(row, identity, err) + } + } + form, _, err := x86HorizontalIntegerProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + features := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", Func{Instrs: []Instr{ins}}) + wantFeatures := "" + if form.memory >= 0 && !form.spec.vector && !form.mmx { + wantFeatures = "+sse2,+ssse3" + } + if features != wantFeatures { + t.Fatalf("%s native carrier features %s != %s", row, features, wantFeatures) + } + if !strings.Contains(row, "(AX)") && !strings.Contains(row, "(BX)") { + continue + } + row = regexp.MustCompile(`(?:-7\(BX\)\(CX\*4\)|\(AX\))`).ReplaceAllString(row, "a+0(FP)") + ins = x86YMMReadSource(t, row) + form, _, err = x86HorizontalIntegerProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for hole := -1; hole < form.byteWidth; hole++ { + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-32\n"+row+"\nRET\n") + sig := FuncSig{Ret: Void} + for at := 0; at < form.byteWidth; at++ { + if at == hole { + continue + } + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(at), Type: I8, Index: len(sig.Args), Field: -1}) + sig.Args = append(sig.Args, I8) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + _, _, err := ctx.lowerHorizontalInteger(ins.Op, ins) + wantContext := !form.spec.vector && !form.mmx + if (err != nil) != (hole >= 0 || wantContext) || err != nil && output.String() != before || wantContext && hole < 0 && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s byte hole %d: %v", row, hole, err) + } + } + } +} + +func TestX86HorizontalIntegerEveryInputByteAndAliasEffect(t *testing.T) { + for _, op := range x86HorizontalIntegerOpcodes { + for _, bank := range []string{"legacy", "X", "Y"} { + for destination := 0; destination < 3; destination++ { + row := op + " X0,X1" + if bank != "legacy" { + row = fmt.Sprintf("V%s %s0,%s1,%s%d", op, bank, bank, bank, destination) + } + ins := x86YMMReadSource(t, row) + form, known, err := x86HorizontalIntegerProperties(ins, "amd64") + if !known || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatal("missing whole-family registration", row, err) + } + for operand := 0; operand < 2; operand++ { + for hole := -1; hole < form.byteWidth; hole++ { + state := x86SourceStackState{bp: x86SourceStackAddress{original: true}, gOriginal: true, numericFlags: true, + known: map[Reg]uint8{AX: 0xff}, masks: map[Reg]bool{"K1": true}, + vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": 0x55, "Z31": 0x1234}} + if hole >= 0 { + state.vectors[x86CallerVectorRoot(ins.Args[operand].Reg)] &^= uint64(1) << uint(hole) + } + state = x86CloneCallerState(state) + before := x86CloneCallerState(state) + accepted := x86CallerVectorMoveStep(&ins, &state, nil) + if accepted != (hole < 0) || !accepted && !reflect.DeepEqual(state, before) { + t.Fatalf("%s input %d byte hole %d invented input/changed state", row, operand, hole) + } + if accepted { + want := x86CloneCallerState(before) + root := x86CallerVectorRoot(ins.Args[form.destination].Reg) + want.vectors[root] = ^uint64(0) + if !form.spec.vector { + want.vectors[root] = before.vectors[root] | x86CallerVectorByteMask(int64(form.byteWidth)) + } + if !reflect.DeepEqual(state, want) { + t.Fatal("horizontal effect altered unrelated state", row) + } + } + } + } + for _, upper := range []uint64{0, 0xff00000000000000, ^x86CallerVectorByteMask(int64(form.byteWidth))} { + mask := x86CallerVectorByteMask(int64(form.byteWidth)) + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": mask | upper, "Z1": mask | upper, "Z2": upper}} + if !x86CallerVectorMoveStep(&ins, &state, nil) { + t.Fatal("initialized complete input failed", row) + } + want := ^uint64(0) + if !form.spec.vector { + want = mask | upper + } + if state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)] != want { + t.Fatal("legacy preservation/VEX upper clearing diverged", row) + } + } + } + } + } +} diff --git a/amd64_horizontal_integer_forms.go b/amd64_horizontal_integer_forms.go new file mode 100644 index 000000000..cbc96184b --- /dev/null +++ b/amd64_horizontal_integer_forms.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64HorizontalIntegerSpec struct { + laneBits int + subtract bool + saturate bool + vector bool + allowMMX bool + // VEX.66.0F38 opcode, shared by raw decoding and typed lowering. + opcode byte +} + +// The complete PHADD/PHSUB family has one exceptional legacy MMX row: +// PHADDD's ymmxmm0f38. Its siblings use yxm_q4; VEX uses X/Y only. +var amd64HorizontalIntegerSpecs = map[Op]amd64HorizontalIntegerSpec{ + "PHADDD": {laneBits: 32, allowMMX: true}, + "PHADDSW": {laneBits: 16, saturate: true}, + "PHADDW": {laneBits: 16}, + "PHSUBD": {laneBits: 32, subtract: true}, + "PHSUBSW": {laneBits: 16, subtract: true, saturate: true}, + "PHSUBW": {laneBits: 16, subtract: true}, + "VPHADDD": {laneBits: 32, vector: true, opcode: 0x02}, + "VPHADDSW": {laneBits: 16, saturate: true, vector: true, opcode: 0x03}, + "VPHADDW": {laneBits: 16, vector: true, opcode: 0x01}, + "VPHSUBD": {laneBits: 32, subtract: true, vector: true, opcode: 0x06}, + "VPHSUBSW": {laneBits: 16, subtract: true, saturate: true, vector: true, opcode: 0x07}, + "VPHSUBW": {laneBits: 16, subtract: true, vector: true, opcode: 0x05}, +} + +type x86HorizontalIntegerForm struct { + spec amd64HorizontalIntegerSpec + first, second, destination, memory int + byteWidth int + mmx bool +} + +// Forms describe ordered bytes and encoding only; they confer no caller, +// stack, alignment, MMX state or address permission. +func x86HorizontalIntegerProperties(ins Instr, arch string) (x86HorizontalIntegerForm, bool, error) { + base, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := amd64HorizontalIntegerSpecs[Op(base)] + if !known { + return x86HorizontalIntegerForm{}, false, nil + } + form := x86HorizontalIntegerForm{spec: spec, first: 0, second: 1, destination: len(ins.Args) - 1, memory: -1, byteWidth: 16} + fail := func() (x86HorizontalIntegerForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go horizontal integer grammar: %q", arch, ins.Op, ins.Raw) + } + if dotted || arch != "amd64" && arch != "386" || !spec.vector && len(ins.Args) != 2 || spec.vector && len(ins.Args) != 3 { + return fail() + } + if spec.allowMMX { + legacy, _, err := x86LegacyMMXReadProperties(ins, arch) + if err != nil { + return form, true, err + } + if err := x86LegacyMMXReadEmitterContext(ins, arch); err != nil { + return form, true, err + } + form.mmx, form.byteWidth = legacy.mmx, legacy.byteWidth + if legacy.memoryBytes != 0 { + form.memory = 0 + } + return form, true, nil + } + if spec.vector { + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if form.byteWidth != 16 && form.byteWidth != 32 { + return fail() + } + } + register := func(operand Operand) bool { + index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + return operand.Kind == OpReg && valid && index < 16 && (spec.vector || arch == "amd64" || index < 8) + } + if !register(ins.Args[form.second]) || !register(ins.Args[form.destination]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) && x86PackedShiftMemoryValid(first, arch) { + form.memory = form.first + } else if !register(first) { + return fail() + } + if spec.vector { + if err := x86Mode32VectorRegisterContext(ins, arch); err != nil { + return form, true, err + } + } + return form, true, nil +} + +func x86CallerHorizontalIntegerStep(ins *Instr, form x86HorizontalIntegerForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx { + return false + } + required := x86CallerVectorByteMask(int64(form.byteWidth)) + // Require all potential input bytes before certifying memory or writing + // aliases. Horizontal subtract with equal operands is not a zero idiom. + for _, index := range []int{form.first, form.second} { + if index != form.memory && state.vectors[x86CallerVectorRoot(ins.Args[index].Reg)]&required != required { + return false + } + } + if form.memory >= 0 { + // No current caller producer supplies the actual legacy m128 alignment. + if !form.spec.vector || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.byteWidth), false) { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + known := ^uint64(0) + if !form.spec.vector { + known = state.vectors[destination] | required + } + state.vectors[destination] = known + return true +} diff --git a/amd64_horizontal_integer_full_alias_runtime_test.go b/amd64_horizontal_integer_full_alias_runtime_test.go new file mode 100644 index 000000000..59874989d --- /dev/null +++ b/amd64_horizontal_integer_full_alias_runtime_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// All three physical vector-number views are observed after the instruction. +// The scalar reference does not read the production grammar or lane emitter. +// This executes portable LLVM, not original AVX/AVX512 on the host. +func TestCrossLinuxRuntimeMatrixX86HorizontalIntegerFullAliasSentinels(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, prototypes, checks strings.Builder + declarations.WriteString("package narroworacle\n") + count := 0 + for _, entry := range []struct { + legacy, vex string + lane, subtract, saturate int + }{ + {"PHADDW", "VPHADDW", 2, 0, 0}, + {"PHADDD", "VPHADDD", 4, 0, 0}, + {"PHADDSW", "VPHADDSW", 2, 0, 1}, + {"PHSUBW", "VPHSUBW", 2, 1, 0}, + {"PHSUBD", "VPHSUBD", 4, 1, 0}, + {"PHSUBSW", "VPHSUBSW", 2, 1, 1}, + } { + for _, legacy := range []bool{true, false} { + widths := []int{16} + if !legacy { + widths = append(widths, 32) + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y"}[width] + for source := 0; source < 3; source++ { + // Distinct registers, both inputs the destructive source, + // and full-width memory each have independent witnesses. + first := bank + "0" + if source == 1 { + first = bank + "1" + } else if source == 2 { + first = "(CX)" + } + for destination := 0; destination < 3; destination++ { + if legacy && destination != 1 { + continue + } + op := entry.vex + args := first + "," + bank + "1" + if legacy { + op = entry.legacy + } else { + args += fmt.Sprintf(",%s%d", bank, destination) + } + name := fmt.Sprintf("H%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[256]byte,a,b,old *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-44\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nVMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nLDMXCSR csr+40(FP)\nPUSHQ flags+32(FP)\nPOPFQ\n%s %s\nVMOVUPS Z0,(AX)\nVMOVUPS Z1,64(AX)\nVMOVUPS Z2,128(AX)\nVMOVUPS Y%d,192(AX)\nVMOVUPS X%d,224(AX)\nPUSHFQ\nPOPQ R9\nMOVQ R9,240(AX)\nSTMXCSR 248(AX)\nRET\n", name, op, args, destination, destination) + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%q,%d,%d,%d,%d,%d,%d,%d))return 1;\n", name, op+" "+args, width, entry.lane, entry.subtract, entry.saturate, x86FloatAddSubBoolInt(legacy), source, destination) + } + } + } + } + } + if count != 126 { + t.Fatal("independent legacy/VEX source and alias universe changed", count) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + reference := x86HorizontalIntegerFullAliasC + prototypes.String() + "int main(void){\n" + checks.String() + fmt.Sprintf("printf(\"horizontal full-alias kernels %d cases %d failures 0\\n\");return 0;}\n", count, count*97*8) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_default_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + for _, backend := range []string{"-O0", "-O2"} { + t.Run("backend"+backend, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "horizontal-full-alias", text, reference, backend) + want := fmt.Sprintf("horizontal full-alias kernels %d cases %d failures 0\n", count, count*97*8) + if string(output) != want { + t.Fatal("full-byte alias oracle did not complete", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } + t.Log("126 independently Go-accepted legacy/VEX kernels; 97 boundary/random samples and 8 six-flag seeds per axis. All64 bytes of Z0/Z1/Z2 plus destination X/Y views, untouched sources, input/output guards and complete MXCSR; portable LLVM/Rosetta is not original AVX execution.") +} + +const x86HorizontalIntegerFullAliasC = ` +#include +#include +#include +#include +typedef void(*Program)(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t); +struct Input {_Alignas(64) uint8_t before[64],value[64],after[64];}; +struct Output {_Alignas(64) uint8_t before[64],value[256],after[64];}; +static const uint32_t edges16[]={0,1,2,0x7ffe,0x7fff,0x8000,0x8001,0xfffe,0xffff}; +static const uint32_t edges32[]={0,1,2,0x7ffffffe,0x7fffffff,0x80000000,0x80000001,0xfffffffe,0xffffffff}; +static const uint64_t flagSeeds[]={0,0x8d5,1,4,0x10,0x40,0x80,0x800}; +static int64_t readLane(const uint8_t *p,int bytes) { + uint64_t value=0;for(int i=0;i=81) {x=UINT32_C(0x9e3779b9)*(unsigned)(sample*17+lane*31);y=UINT32_C(0x85ebca6b)*(unsigned)(sample*43+lane*7);} + for(int i=0;i>(i*8));b[lane*bytes+i]=(uint8_t)(y>>(i*8));} + } +} +static int check(Program program,const char *name,int width,int lane,int sub,int sat,int legacy,int source,int destination) { + for(int sample=0;sample<97;sample++) for(unsigned state=0;state32767)value=32767;} + for(int i=0;i>(8*i)); + } + if(!legacy)memset(result+width,0,64-width); + uint64_t seed=flagSeeds[state],flags=0;uint32_t csr=0; + uint32_t profile=(uint32_t)(0x1f80|((sample%4)<<13)|(sample%64)),savedCSR=_mm_getcsr(); + _mm_setcsr(profile);program(out.value,a.value,b.value,old.value,seed,profile); + uint32_t afterCSR=_mm_getcsr();_mm_setcsr(savedCSR); + memcpy(&flags,out.value+240,8);memcpy(&csr,out.value+248,4); + if((flags&0x8d5)!=seed||csr!=profile||afterCSR!=profile||memcmp(&a,&savedA,sizeof(a))||memcmp(&b,&savedB,sizeof(b))||memcmp(&old,&savedOld,sizeof(old))) { + fprintf(stderr,"%s sample%d state%u flags/CSR/input guard mismatch\n",name,sample,state);return 1; + } + for(int i=0;i<192;i++)if(out.value[i]!=want[i]) { + fprintf(stderr,"%s sample%d state%u physical Z%d byte%d got%x want%x\n",name,sample,state,i/64,i%64,out.value[i],want[i]);return 2; + } + for(int i=0;i<32;i++)if(out.value[192+i]!=result[i]) { + fprintf(stderr,"%s sample%d state%u Y%d byte%d got%x want%x\n",name,sample,state,destination,i,out.value[192+i],result[i]);return 3; + } + for(int i=0;i<16;i++)if(out.value[224+i]!=result[i]) { + fprintf(stderr,"%s sample%d state%u X%d byte%d got%x want%x\n",name,sample,state,destination,i,out.value[224+i],result[i]);return 4; + } + for(int i=0;i<64;i++)if(out.before[i]!=0xc3||out.after[i]!=0xc3)return 5; + for(int i=252;i<256;i++)if(out.value[i]!=0xc3)return 6; + } + return 0; +} +` diff --git a/amd64_horizontal_integer_memory_runtime_test.go b/amd64_horizontal_integer_memory_runtime_test.go new file mode 100644 index 000000000..a9010ddef --- /dev/null +++ b/amd64_horizontal_integer_memory_runtime_test.go @@ -0,0 +1,205 @@ +//go:build unix + +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86HorizontalIntegerOriginalNativeMemory(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, originals, checks strings.Builder + sigs := map[string]FuncSig{} + // This independent list fixes the original SSSE3 bytes and arithmetic; + // neither native oracle nor expected values consume the production spec. + for index, entry := range []struct { + op string + opcode byte + lane, subtract, clamp int + }{ + {"PHADDW", 1, 2, 0, 0}, {"PHADDD", 2, 4, 0, 0}, {"PHADDSW", 3, 2, 0, 1}, + {"PHSUBW", 5, 2, 1, 0}, {"PHSUBD", 6, 4, 1, 0}, {"PHSUBSW", 7, 2, 1, 1}, + } { + code := assembleX87ControlBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+entry.op+" (SI),X1\nRET\n") + if !bytes.Equal(code, []byte{0x66, 0x0f, 0x38, entry.opcode, 0x0e, 0xc3}) { + t.Fatalf("original horizontal m128 bytes changed: %s %x", entry.op, code) + } + for _, dead := range []bool{false, true} { + name := fmt.Sprintf("horizontalmemory%d_%t", index, dead) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-16\nMOVQ out+0(FP),AX\nMOVQ input+8(FP),BX\nMOVOU (AX),X1\nCMPQ BX,BX\nSTC\n%s (BX),X1\n", name, entry.op) + tail := "" + if !dead { + source.WriteString("MOVOU X1,16(AX)\n") + tail = "; movdqu %%xmm1,16(%%rdi)" + } + source.WriteString("SETCS 32(AX)\nSETNE 33(AX)\nSTMXCSR 36(AX)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*);\n", name) + fmt.Fprintf(&originals, `static void original_%s(uint8_t *out,const uint8_t *input) { + __asm__ volatile("movdqu (%%%%rdi),%%%%xmm1; cmpq %%%%rsi,%%%%rsi; stc; .byte 0x66,0x0f,0x38,0x%02x,0x0e%s; setc 32(%%%%rdi); setnz 33(%%%%rdi); stmxcsr 36(%%%%rdi)" :: + "D"(out),"S"(input) : "xmm1","cc","memory"); +} +`, name, entry.opcode, tail) + fmt.Fprintf(&checks, ` + if(compare(original_%[1]s,%[1]s,region,0,%[2]d,%[3]d,%[4]d,%[5]d) || + compare(original_%[1]s,%[1]s,boundary-16,0,%[2]d,%[3]d,%[4]d,%[5]d) || + compare(original_%[1]s,%[1]s,region+1,alignmentFault,%[2]d,%[3]d,%[4]d,%[5]d) || + compare(original_%[1]s,%[1]s,boundary-15,1,%[2]d,%[3]d,%[4]d,%[5]d) || + compare(original_%[1]s,%[1]s,0,1,%[2]d,%[3]d,%[4]d,%[5]d)) return 1; +`, name, entry.lane, entry.subtract, entry.clamp, x86FloatAddSubBoolInt(dead)) + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + oracle := `#include +#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + originals.String() + ` +typedef void (*memoryfn)(uint8_t*,const uint8_t*); +static int64_t laneValue(const uint8_t *p,int lane) { + if(lane==2) {int16_t v;memcpy(&v,p,2);return v;} + int32_t v;memcpy(&v,p,4);return v; +} +static int observe(memoryfn fn,const uint8_t *input,int lane,int sub,int clamp,int dead) { + pid_t child=fork(); + if(child<0) return -1; + if(child==0) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(2); + uint8_t out[48],expected[48];unsigned csr,after; + for(unsigned i=0;i32767)value=32767;} + uint64_t bits=(uint64_t)value; + for(int byte=0;byte>(8*byte)); + } + } + expected[33]=1;expected[34]=0;memcpy(expected+37,&csr,4); + __asm__ volatile("stmxcsr %0":"=m"(after)); + if(memcmp(out,expected,sizeof(out)) || after!=csr) _exit(3); + _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status)) return WEXITSTATUS(status); + if(WIFSIGNALED(status)) return 128+WTERMSIG(status); + return -1; +} +static int compare(memoryfn reference,memoryfn translated,const uint8_t *input,int fault,int lane,int sub,int clamp,int dead) { + int native=observe(reference,input,lane,sub,clamp,dead),got=observe(translated,input,lane,sub,clamp,dead); + int memoryFault=native==128+SIGSEGV || native==128+SIGBUS; + int valid=fault==1 ? memoryFault : fault==-1 ? native==0 || memoryFault : native==0; + if(native<0 || native!=got || !valid) { + fprintf(stderr,"horizontal native=%d LLVM=%d input=%p fault=%d lane=%d sub=%d clamp=%d dead=%d\n",native,got,input,fault,lane,sub,clamp,dead); + return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<64) return 1; + uint8_t *region=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(region==MAP_FAILED || mprotect(region+page,(size_t)page,PROT_NONE)) return 1; + for(long i=0;i +#include +#include +#include +#include +#include +#include +` + declarations.String() + ` +static int check(void (*program)(const uint8_t*),const uint8_t *input,int fault) { + pid_t child=fork();if(child<0)return 1; + if(child==0) {struct rlimit limit={0,0};if(setrlimit(RLIMIT_CORE,&limit))_exit(2);program(input);_exit(0);} + int status;if(waitpid(child,&status,0)!=child)return 1; + int got=WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS); + if(fault ? !got : !WIFEXITED(status) || WEXITSTATUS(status)!=0) { + fprintf(stderr,"horizontal dead read input=%p fault=%d status=%x\n",input,fault,status);return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE);if(page<=0)return 1; + uint8_t *memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED || mprotect(memory+page,(size_t)page,PROT_NONE))return 1; + uint8_t *boundary=memory+page; +` + checks.String() + `return munmap(memory,(size_t)page*2)!=0;} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "horizontal-vex-dead-full-read", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_ifma_forms.go b/amd64_ifma_forms.go new file mode 100644 index 000000000..ad931af34 --- /dev/null +++ b/amd64_ifma_forms.go @@ -0,0 +1,72 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86IFMAForm struct { + x86PackedBinaryForm + spec amd64IFMASpec +} + +func x86IFMAProperties(op Op, ins Instr, arch string) (x86IFMAForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64IFMASpecs[Op(base)] + if !known { + return x86IFMAForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86IFMAForm{}, true, fmt.Errorf("IFMA dispatch/source opcode mismatch") + } + local := ins + local.Op = op + physical := ins.x86Encoded || ins.x86SourceRaw != nil + form, err := x86PackedBinaryOperandGrammar(local, arch, spec.x86PackedBinarySpec, physical) + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && form.memory >= 0 { + source := ins.Args[form.memory] + if source.Kind == OpMem { + if !x86ConditionalMoveOffsetKnown(source.Mem) || source.Mem.IndexExt != "" || + source.Mem.Segment != "" && source.Mem.Segment != FS && source.Mem.Segment != GS { + err = fmt.Errorf("IFMA source has an unresolved or foreign address") + } else { + err = x86SymbolIndexedMemoryError(source.Mem, arch) + } + } + if source.Kind == OpSym { + if address, parseErr := parseInt(strings.TrimSpace(source.Sym)); parseErr == nil && + address != int64(int32(address)) && !(arch == "386" && address == int64(uint32(address))) { + err = fmt.Errorf("IFMA absolute source exceeds Go's disp32 extent") + } + } + if err == nil && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: IFMA needs its original default address mode", ErrProbeNeedsContext) + } + } + return x86IFMAForm{form, spec}, true, err +} + +func x86TypedIFMAByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86IFMAProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.byteWidth) + firstWidth := width + if form.memory >= 0 { + firstWidth = int64(form.memoryBytes) + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: firstWidth, read: true}, + {index: form.second, operand: ins.Args[form.second], width: width, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: width, read: true, write: true}, + }} + if form.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return effect, true, nil +} diff --git a/amd64_ifma_forms_test.go b/amd64_ifma_forms_test.go new file mode 100644 index 000000000..98dacd747 --- /dev/null +++ b/amd64_ifma_forms_test.go @@ -0,0 +1,154 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86IFMACompleteIndependentGoRows(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + opcodes := map[string]string{"VPMADD52HUQ": "0xB5", "VPMADD52LUQ": "0xB4"} + counts := map[string]int{} + for _, rule := range rules { + opcode, wanted := opcodes[rule.Opcode] + if !wanted { + continue + } + index := counts[rule.Opcode] + counts[rule.Opcode]++ + classes := []string{ + "YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", + "YymEvex,YyrEvex,YyrEvex", "YymEvex,YyrEvex,Yknot0,YyrEvex", + "Yzm,Yzr,Yzr", "Yzm,Yzr,Yknot0,Yzr", + } + var bytes []string + for _, width := range []int{128, 256, 512} { + bytes = append(bytes, fmt.Sprintf("avxEscape | evex%d | evex66 | evex0F38 | evexW1", width), + fmt.Sprintf("evexN%d | evexBcstN8 | evexZeroingEnabled", width/8), opcode) + } + zcase, offset := "Zevex_rm_v_r", 0 + if index%2 != 0 { + zcase, offset = "Zevex_rm_v_k_r", 3 + } + if index >= 6 || rule.RowIndex != index || rule.Ytab != "_yvblendmpd" || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != offset || rule.EncodingOffset != index/2*3 || + strings.Join(rule.Classes, ",") != classes[index] || !reflect.DeepEqual(rule.OpBytes, bytes) { + t.Fatal("independent complete IFMA row/tuple attributes changed", rule) + } + } + for name := range opcodes { + if counts[name] != 6 { + t.Fatal("complete IFMA family row denominator changed", counts) + } + } + if len(amd64IFMASpecs) != 2 { + t.Fatal("unaccounted IFMA operation") + } +} + +func TestX86IFMAEveryNumericalSourceAndAccumulatorByte(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, name := range []string{"VPMADD52HUQ", "VPMADD52LUQ"} { + for _, bank := range []string{"X", "Y", "Z"} { + ins := x86ScalarLaneInstruction(t, name+" "+bank+"0,"+bank+"1,"+bank+"2") + width := amd64VectorByteWidth(ins.Args[2].Reg) + for input := 0; input < 3; input++ { + for missing := -1; missing < width; missing++ { + state := x86ResultState{flags: x86ByteFlagsAll, direction: x86ResultDirectionEither} + for register := 0; register < 3; register++ { + state.writeVector(register, 64, ^uint64(0), false) + } + state.pointerGP[6], state.gp[3], state.masks[1] = true, 0x55, true + if missing >= 0 { + state.writeVector(input, 64, state.vectorBytes(input)&^(uint64(1)<= 0 && (input == 2 || missing%8 != 7) { + defined &^= uint64(0xff) << uint(missing/8*8) + } + want.writeVector(2, width, defined, false) + accepted, known := x86ResultIFMAStep(ins, &state, nil, arch) + if !known || !accepted || !reflect.DeepEqual(state, want) { + t.Fatal("IFMA invented numerical bytes, lost meaningful subwords or changed independent state", + arch, name, bank, input, missing) + } + } + } + } + } + } +} + +func TestX86IFMAZeroingStillReadsTheActualAccumulator(t *testing.T) { + for _, name := range []string{"VPMADD52HUQ", "VPMADD52LUQ"} { + for _, bank := range []string{"X", "Y", "Z"} { + ins := x86ScalarLaneInstruction(t, name+".Z "+bank+"0,"+bank+"1,K1,"+bank+"2") + state := x86ResultState{flags: x86ByteFlagsAll} + state.writeVector(0, 64, ^uint64(0), false) + state.writeVector(1, 64, ^uint64(0), false) + state.masks[1] = true + accepted, known := x86ResultIFMAStep(ins, &state, nil, "amd64") + width := amd64VectorByteWidth(ins.Args[3].Reg) + if !known || !accepted || state.vectorBytes(2)&x86CallerVectorByteMask(int64(width)) != 0 || + state.flags != x86ByteFlagsAll { + t.Fatal("EVEX.z supplied the missing arithmetic accumulator or changed FLAGS", name, bank) + } + } + } +} + +func TestX86IFMAEncodedUpperWritesCannotPreserveStaleAliases(t *testing.T) { + for _, target := range x86RawReturnTargets { + var source, declarations strings.Builder + var names []string + for _, name := range []string{"VPMADD52HUQ", "VPMADD52LUQ"} { + for _, bank := range []string{"X", "Y"} { + for _, raw := range []bool{false, true} { + for _, mode := range []string{"plain", "merge", "zero"} { + if target.arch == "386" && !raw && mode != "plain" { + continue // Named frontend rejects these; raw physical rows are retained. + } + op, mask := name, "" + if mode != "plain" { + mask = "K1," + } + if mode == "zero" { + op += ".Z" + } + instruction := op + " " + bank + "0," + bank + "1," + mask + bank + "2" + if raw { + encoderArch := target.arch + if mode != "plain" { + encoderArch = "amd64" + } + instruction = x86RawNumericalInstruction(t, encoderArch, instruction) + } + function := fmt.Sprint("Upper", len(names)) + names = append(names, function) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-4\nVPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\n"+ + "MOVL $-1,AX\nVPBROADCASTD AX,Z2\nMOVL $0,AX\nKMOVD AX,K1\n%s\n"+ + "VEXTRACTI32X4 $3,Z2,X3\nPEXTRD $0,X3,AX\nMOVL AX,ret+0(FP)\nRET\n", function, instruction) + fmt.Fprintf(&declarations, "func %s()uint32\n", function) + } + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("complete IFMA upper-alias source lost its initialized inputs", target, api, err) + } + x86VectorUpperRequireZeroReturns(t, ir, names) + } + } +} diff --git a/amd64_immediate_blend_forms.go b/amd64_immediate_blend_forms.go new file mode 100644 index 000000000..2a5b87f6b --- /dev/null +++ b/amd64_immediate_blend_forms.go @@ -0,0 +1,82 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ImmediateBlendForm struct { + source, base, destination int + byteWidth, laneBytes int + selector uint8 + legacy, memory bool +} + +// Go's yxshuf and _yvblendpd supply eleven rows, not an EVEX family. +// Physical mode32 VEX has its own decoder role; its four operands do not +// become legal genuine named Go386 syntax. +func x86ImmediateBlendProperties(op Op, ins Instr, arch string) (x86ImmediateBlendForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + bits, vector, known := amd64ImmediatePackedBlendProperties(name) + f := x86ImmediateBlendForm{source: 1, base: 2, destination: 2, laneBytes: bits / 8, legacy: !vector} + if !known { + return f, false, nil + } + fail := func() (x86ImmediateBlendForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside complete Go immediate-blend grammar: %q", arch, op, ins.Raw) + } + if arch == "" { + arch = "amd64" + } + if arch != "amd64" && arch != "386" || dotted || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + if vector { + f.destination = 3 + } + if len(ins.Args) != f.destination+1 || arch == "386" && vector && !ins.x86Encoded { + return fail() + } + immediate, integer := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !integer || immediate.Imm < 0 || immediate.Imm > 255 { + return fail() + } + f.selector = uint8(immediate.Imm) + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + if f.byteWidth != 16 && (f.legacy || f.byteWidth != 32) { + return fail() + } + register := func(operand Operand) bool { + if !amd64VEXVectorRegister(operand, f.byteWidth) { + return false + } + index, _ := amd64VectorRegisterIndex(operand.Reg, f.byteWidth) + return arch != "386" || index < 8 + } + if !register(ins.Args[f.destination]) || !register(ins.Args[f.base]) { + return fail() + } + source := ins.Args[f.source] + f.memory = isAMD64MemoryOperand(source) + if !register(source) && (!f.memory || !x86PackedShiftMemoryValid(source, arch)) { + return fail() + } + return f, true, nil +} + +func (f x86ImmediateBlendForm) selectsFirst(output int) bool { + return f.selector>>uint(output/f.laneBytes%8)&1 != 0 +} + +func x86TypedImmediateBlendByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + f, known, err := x86ImmediateBlendProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + return x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: f.source, operand: ins.Args[f.source], width: int64(f.byteWidth), read: true}, + {index: f.base, operand: ins.Args[f.base], width: int64(f.byteWidth), read: true}, + {index: f.destination, operand: ins.Args[f.destination], width: int64(f.byteWidth), write: true}, + }}, true, nil +} diff --git a/amd64_immediate_packed_compare_forms.go b/amd64_immediate_packed_compare_forms.go new file mode 100644 index 000000000..03304dd78 --- /dev/null +++ b/amd64_immediate_packed_compare_forms.go @@ -0,0 +1,52 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ImmediatePackedCompareForm struct { + x86PackedIntegerCompareForm + unsigned bool + predicate uint8 +} + +// All eight _yvpcmpb opcodes use the same six ordered Go rows. Operand +// properties are shared with lowering and numerical effects; no source or +// memory authority follows from matching this grammar. +func x86ImmediatePackedCompareProperties(op Op, ins Instr, arch string) (x86ImmediatePackedCompareForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64ImmediatePackedCompareSpecs[base] + if !known { + return x86ImmediatePackedCompareForm{}, false, nil + } + form := x86ImmediatePackedCompareForm{unsigned: spec.unsigned} + form.spec.laneBits, form.maskResult = spec.laneBits, true + fail := func() (x86ImmediatePackedCompareForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go immediate packed compare grammar: %q", arch, op, ins.Raw) + } + // The named Go 386 frontend cannot represent these four/five-operand + // instructions. It does not borrow the distinct physical raw EVEX role. + physicalMode32 := arch == "386" && (ins.x86Encoded || ins.x86SourceRaw != nil) + if arch != "amd64" && !physicalMode32 || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || + len(ins.Args) != 4 && len(ins.Args) != 5 { + return fail() + } + immediate := ins.Args[0] + if immediate.Kind != OpImm || immediate.ImmRaw != "" || immediate.ImmIsFloat || immediate.Imm < 0 || immediate.Imm > 255 { + return fail() + } + binary, valid := x86PackedMaskCompareProperties(ins, arch, 1, spec.laneBits, dotted, suffix) + if !valid { + return fail() + } + if err := x86Mode32VectorRegisterContext(ins, arch); err != nil { + return form, true, err + } + form.x86PackedBinaryForm = binary + form.source, form.predicate = ins.Args[form.first], uint8(immediate.Imm) + if form.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + return form, true, fmt.Errorf("%w: immediate packed compare has no bound address-size override", ErrProbeNeedsContext) + } + return form, true, nil +} diff --git a/amd64_immediate_packed_compare_forms_test.go b/amd64_immediate_packed_compare_forms_test.go new file mode 100644 index 000000000..6e0f66dc2 --- /dev/null +++ b/amd64_immediate_packed_compare_forms_test.go @@ -0,0 +1,69 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86ImmediatePackedCompareCompleteGoRowsAndAttributes(t *testing.T) { + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + actual := strings.Join(strings.Fields(string(source)), "") + rows := []string{ + "{zcase:Zevex_i_rm_v_r,zoffset:0,args:argList{Yu8,YxmEvex,YxrEvex,Yk}}", + "{zcase:Zevex_i_rm_v_k_r,zoffset:3,args:argList{Yu8,YxmEvex,YxrEvex,Yknot0,Yk}}", + "{zcase:Zevex_i_rm_v_r,zoffset:0,args:argList{Yu8,YymEvex,YyrEvex,Yk}}", + "{zcase:Zevex_i_rm_v_k_r,zoffset:3,args:argList{Yu8,YymEvex,YyrEvex,Yknot0,Yk}}", + "{zcase:Zevex_i_rm_v_r,zoffset:0,args:argList{Yu8,Yzm,Yzr,Yk}}", + "{zcase:Zevex_i_rm_v_k_r,zoffset:3,args:argList{Yu8,Yzm,Yzr,Yknot0,Yk}}", + } + if !strings.Contains(actual, "var_yvpcmpb=[]ytab{"+strings.Join(rows, ",")+",}") { + t.Fatal("actual SDK complete ordered immediate comparison rows changed") + } + seen := map[string]bool{} + for _, match := range regexp.MustCompile(`\{as:A(VPCMPU?[BWDQ]),ytab:_yvpcmpb,`).FindAllStringSubmatch(actual, -1) { + if seen[match[1]] { + t.Fatal("duplicate actual SDK immediate comparison member", match[1]) + } + seen[match[1]] = true + } + if len(seen) != 8 || len(amd64ImmediatePackedCompareSpecs) != 8 { + t.Fatal("complete immediate comparison family changed", seen) + } + for _, op := range x86ImmediateCompareResultMembers { + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + unsigned := strings.HasPrefix(op, "VPCMPU") + w, broadcast := "W0", "" + if bits == 16 || bits == 64 { + w = "W1" + } + if bits >= 32 { + broadcast = fmt.Sprintf("|evexBcstN%d", bits/8) + } + opcode := 0x3f + if bits >= 32 { + opcode = 0x1f + } + if unsigned { + opcode-- + } + var attributes strings.Builder + for _, width := range []int{128, 256, 512} { + fmt.Fprintf(&attributes, "avxEscape|evex%d|evex66|evex0F3A|evex%s,evexN%d%s,0x%X,", width, w, width/8, broadcast, opcode) + } + if !strings.Contains(actual, "{as:A"+op+",ytab:_yvpcmpb,prefix:Pavx,op:opBytes{"+attributes.String()+"}},") { + t.Fatal("actual SDK immediate comparison encoding attributes changed", op) + } + spec, present := amd64ImmediatePackedCompareSpecs[op] + if !seen[op] || !present || spec.laneBits != bits || spec.unsigned != unsigned { + t.Fatal("typed spec differs from complete independent Go member", op, spec) + } + } +} diff --git a/amd64_immediate_shuffle_class_boundaries_test.go b/amd64_immediate_shuffle_class_boundaries_test.go new file mode 100644 index 000000000..6c1ea3b42 --- /dev/null +++ b/amd64_immediate_shuffle_class_boundaries_test.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86ImmediateShuffleRejectsNonIntegerClassesEveryConsumer(t *testing.T) { + for _, op := range []string{"PSHUFW", "PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for _, literal := range []string{"$0.0", "$1.0", "$UNKNOWN"} { + bank := "X" + if op == "PSHUFW" { + bank = "M" + } + row := fmt.Sprintf("%s %s,%s0,%s1", op, literal, bank, bank) + t.Run(row, func(t *testing.T) { + assembly := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + for _, arch := range []string{"amd64", "386"} { + goAssemblerOracleOutput(t, arch, assembly, "", false) + ins := x86YMMReadSource(t, row) + if _, known, err := x86ImmediateShuffleProperties(ins, arch); !known || err == nil { + t.Error("non-integer class acquired integer grammar", arch, literal, err) + } + var output strings.Builder + ctx := &amd64Ctx{goarch: arch, b: &output} + if known, _, err := ctx.lowerPackedWordShuffle(ins.Op, ins); !known || err == nil || output.Len() != 0 { + t.Error("non-integer class emitted IR", arch, err, output.String()) + } + state := x86CloneCallerState(x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0)}}) + before := x86CloneCallerState(state) + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, before) { + t.Error("non-integer class acquired caller effects or changed state", arch) + } + result := x86ResultState{} + result.writeVector(0, 64, ^uint64(0), false) + result.writeVector(1, 64, ^uint64(0), false) + beforeResult := result.clone() + if x86ResultTransfer(ins, &result, nil, 0, arch) || !reflect.DeepEqual(result, beforeResult) { + t.Error("non-integer class acquired result effects or changed state", arch) + } + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}} + if _, err := Translate(file, options); err == nil { + t.Error("text API accepted non-integer immediate") + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err == nil { + t.Error("module API accepted non-integer immediate") + } + pkg := mustGoPackage(t, "example.invalid/shuffle", "package shuffle\nfunc f()\n") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: options.TargetTriple, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err == nil { + t.Error("actual typed declaration API accepted non-integer immediate") + } + }) + } + } +} + +func TestX86ImmediateShuffleGoMode32ConstantClassification(t *testing.T) { + // Go's oclass truncates TYPE_CONST to int32 only in I386 mode. This is + // source classification, not permission to invent an encoded raw operand. + values := []int64{4294967296, -4294967296, 4294967168, 4294967423, 4294967424, 4294967551, 4294967552, -4294967297} + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + for _, op := range []string{"PSHUFW", "PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + banks := []string{"X"} + if op == "PSHUFW" { + banks = []string{"M"} + } else if strings.HasPrefix(op, "V") { + banks = append(banks, "Z") + } + for _, bank := range banks { + for _, value := range values { + classified := value + if arch == "386" { + classified = int64(int32(value)) + } + accepted := classified >= 0 && classified <= 255 + if op == "PSHUFW" { + accepted = classified >= -128 && classified <= 127 + } else if strings.HasPrefix(op, "V") && bank == "X" { + accepted = classified >= -128 && classified <= 255 + } + row := fmt.Sprintf("%s $%d,%s0,%s1", op, value, bank, bank) + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", accepted) + ins := x86YMMReadSource(t, row) + before := fmt.Sprintf("%#v", ins) + form, known, err := x86ImmediateShuffleProperties(ins, arch) + if !known || accepted != (err == nil) || accepted && form.immediate != uint8(classified) { + t.Errorf("Go %s accepted=%v for %s, grammar=%+v error=%v", arch, accepted, row, form, err) + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("classification changed original constant/source") + } + } + } + } + }) + } +} + +func TestX86ImmediateShuffleRawMMXCompleteImmediateBits(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + var expected []byte + for value := 0; value < 256; value++ { + code := []byte{0x0f, 0x70, 0xc8, byte(value)} + fmt.Fprintf(&source, "PSHUFW $%d,M0,M1\n", int8(value)) + expected = append(expected, code...) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil || len(decoded.Instrs) != 1 { + t.Fatalf("physical imm8 %x lost instruction identity: %+v %v", code, decoded, err) + } + form, known, err := x86ImmediateShuffleProperties(decoded.Instrs[0], arch) + if !known || err != nil || form.immediate != uint8(value) || !form.spec.mmx { + t.Errorf("raw PSHUFW imm8=%d lost Go Yi8 spelling: %+v %v", value, form, err) + } + } + source.WriteString("RET\n") + expected = append(expected, 0xc3) + if got := assembleX87ControlBytes(t, arch, source.String()); !bytes.HasPrefix(got, expected) { + t.Fatal("independent Go signed source did not encode the original physical imm8 bytes") + } + }) + } +} diff --git a/amd64_immediate_shuffle_effects_test.go b/amd64_immediate_shuffle_effects_test.go new file mode 100644 index 000000000..ed6850195 --- /dev/null +++ b/amd64_immediate_shuffle_effects_test.go @@ -0,0 +1,239 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86ImmediateShuffleCompleteLegacyOriginalRows(t *testing.T) { + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + pattern := regexp.MustCompile(`\{A(PSHUFW|PSHUFL|PSHUFD|PSHUFHW|PSHUFLW), (\w+), (\w+), opBytes\{([^}]+)\}\}`) + rows := pattern.FindAllStringSubmatch(string(source), -1) + if len(rows) != 5 { + t.Fatalf("complete legacy original shuffle denominator changed: %d", len(rows)) + } + for _, row := range rows { + prefix := map[string]string{"PSHUFW": "Pm", "PSHUFL": "Pq", "PSHUFD": "Pq", "PSHUFHW": "Pf3", "PSHUFLW": "Pf2"}[row[1]] + table := "yxshuf" + classes := "{Zibm_r,2,argList{Yu8,Yxm,Yxr}}," + if row[1] == "PSHUFW" { + table, classes = "ymshuf", "{Zibm_r,2,argList{Yi8,Ymm,Ymr}}," + } + definition := regexp.MustCompile(`(?s)var ` + table + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(string(source)) + if len(definition) != 2 || strings.Join(strings.Fields(definition[1]), "") != classes || + row[2] != table || row[3] != prefix || row[4] != "0x70, 00" && row[4] != "0x70, 0" { + t.Fatal("unmodeled complete legacy prefix/operands/opcode", row) + } + } +} + +func TestX86ImmediateShuffleByteKnowledgeAndUnaffectedState(t *testing.T) { + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + widths := []int{16} + if strings.HasPrefix(op, "V") { + widths = []int{16, 32, 64} + } + for _, width := range widths { + for _, mode := range []string{"plain", "merge", "zero"} { + if !strings.HasPrefix(op, "V") && mode != "plain" { + continue + } + for _, alias := range []bool{false, true} { + for _, immediate := range []int{0, 27, 228, 255} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + destination := 1 + if alias { + destination = 0 + } + row := op + if mode == "zero" { + row += ".Z" + } + row += fmt.Sprintf(" $%d,%s0,", immediate, bank) + if mode != "plain" { + row += "K1," + } + row += fmt.Sprint(bank, destination) + t.Run(row, func(t *testing.T) { + ins := x86YMMReadSource(t, row) + for seed := 0; seed < 32; seed++ { + initial := x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, gOriginal: true, numericFlags: true, + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}}, known: map[Reg]uint8{AX: 0xff}, + vectors: map[Reg]uint64{"Z0": uint64(seed)*0x1937a73b15d94ca7 ^ 0x31c097642eaf859b, + "Z1": uint64(seed)*0xf03755e989ab7361 ^ 0xae78f9234cbad619, "Z31": 0x55}, + masks: map[Reg]bool{"K1": true, "K0": false}, + } + state, want := x86CloneCallerState(initial), x86CloneCallerState(initial) + var expected uint64 + lane := 4 + if strings.Contains(op, "HW") || strings.Contains(op, "LW") { + lane = 2 + } + old := initial.vectors[Reg(fmt.Sprint("Z", destination))] + for output := 0; output < 64; output++ { + known := old>>uint(output)&1 != 0 + if output >= width { + known = strings.HasPrefix(op, "V") || known + } else { + at := output + if lane == 4 { + at = output/16*16 + (immediate>>(2*(output%16/4))&3)*4 + output%4 + } else if word := output % 16 / 2; word < 4 && strings.Contains(op, "LW") || word >= 4 && strings.Contains(op, "HW") { + half := 0 + if strings.Contains(op, "HW") { + half = 8 + } + at = output/16*16 + half + (immediate>>(2*(word%4))&3)*2 + output%2 + } + known = initial.vectors["Z0"]>>uint(at)&1 != 0 + if mode == "merge" { + known = known && old>>uint(output)&1 != 0 + } + } + if known { + expected |= uint64(1) << uint(output) + } + } + want.vectors[Reg(fmt.Sprint("Z", destination))] = expected + // The independent result observer must consume the same + // family, not reject it after correct caller/lowering work. + result := x86ResultState{flags: 0x3f, fp: []bool{true, false}, auto: []bool{false, true}} + result.gp[0], result.pointerGP[7], result.masks[1] = 0xff, true, true + for _, index := range []int{0, 1, 31} { + result.writeVector(index, 64, initial.vectors[Reg(fmt.Sprint("Z", index))], false) + } + wantResult := result.clone() + wantResult.writeVector(destination, 64, expected, false) + if !x86ResultTransfer(ins, &result, nil, 0, "amd64") || !reflect.DeepEqual(result, wantResult) { + t.Fatalf("%s seed=%d lost result bytes or unrelated state: got=%+v want=%+v", row, seed, result, wantResult) + } + if !x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, want) { + t.Fatalf("%s seed=%d changed byte dependencies or unrelated state: got=%+v want=%+v", row, seed, state, want) + } + state = x86CloneCallerState(initial) + state.masks["K1"] = false + before := x86CloneCallerState(state) + if mode != "plain" && (x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, before)) { + t.Fatal("unknown mask acquired data or mutated state") + } + result.masks[1] = false + beforeResult := result.clone() + if mode != "plain" && (x86ResultTransfer(ins, &result, nil, 0, "amd64") || !reflect.DeepEqual(result, beforeResult)) { + t.Fatal("unknown result mask acquired data or mutated state") + } + } + }) + } + } + } + } + } +} + +func TestX86ImmediateShuffleActualCallerThreeAPIs(t *testing.T) { + var body strings.Builder + rows := x86OfficialImmediateShuffleRows(t) + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + rows = append(rows, op+" $255,X0,X1") + } + for _, row := range rows { + destination := strings.Split(row, ",") + vector := destination[len(destination)-1] + index := vector[1:] + fmt.Fprintf(&body, "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z%s\nKMOVQ AX,K1\n%s\nVMOVSS X%s,(DX)\n", index, row, index) + } + // Broadcast, full tuple, merge and zeroing use actual caller input regions. + for _, row := range []string{"VPSHUFD.BCST.Z $255,(DI),K1,Z1", "VPSHUFHW.Z $0,(DI),K1,Z1", "VPSHUFLW $27,(DI),K1,Y1"} { + fmt.Fprintf(&body, "VMOVUPS (SI),Z1\nKMOVQ AX,K1\n%s\nVMOVSS X1,(DX)\n", row) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body.String(), 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "immediate-shuffle-text.ll", "immediate-shuffle-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "immediate-shuffle-module.ll", "immediate-shuffle-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("actual selected caller failed", err) + } + }) + } +} + +func TestX86ImmediateShufflePartialSourceActualCaller(t *testing.T) { + for _, test := range []struct { + name, body string + accepted bool + }{ + {"only selected dword", "MOVSS (DI),X0\nVPSHUFD $0,X0,Y1", false}, + {"selected low dword", "MOVSS (DI),X2\nMOVSS X2,X0\nVPSHUFD $0,X0,X1", true}, + {"selected unknown dword", "MOVSS (DI),X2\nMOVSS X2,X0\nVPSHUFD $255,X0,X1", false}, + {"unused upper words", "MOVSS (DI),X2\nMOVSS X2,X0\nVPSHUFLW $0,X0,X1", true}, + {"whole unknown vector", "VPSHUFD $0,X0,X1", false}, + {"unknown mask", "VMOVUPS (DI),Z0\nVPSHUFD.Z $0,X0,K1,X1", false}, + {"merge unknown destination", "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVPSHUFD $0,X0,K1,X1", false}, + {"zeroing no old destination", "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVPSHUFD.Z $0,X0,K1,X1", true}, + {"legacy alignment unbound", "PSHUFL $0,(DI),X1", false}, + } { + t.Run(test.name, func(t *testing.T) { + // The width-mismatch case is a syntax negative, not caller Context. + if test.name == "only selected dword" { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+test.body+"\nRET\n", "", false) + if _, known, err := x86ImmediateShuffleProperties(x86YMMReadSource(t, "VPSHUFD $0,X0,Y1"), "amd64"); !known || err == nil { + t.Fatal("width mismatch acquired shuffle grammar") + } + return + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", test.body+"\nVMOVSS X1,(DX)", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("partial caller accepted=%v got %v", test.accepted, err) + } + }) + } + + ins := x86YMMReadSource(t, "vpshufd $27,X0,X1") + var output strings.Builder + ctx := &amd64Ctx{goarch: "amd64", b: &output} + if known, _, err := ctx.lowerPackedWordShuffle("VPSHUFHW", ins); !known || err == nil || output.Len() != 0 { + t.Fatal("contradictory dispatch consumed source or emitted IR") + } +} diff --git a/amd64_immediate_shuffle_family_test.go b/amd64_immediate_shuffle_family_test.go new file mode 100644 index 000000000..0e71bb8e4 --- /dev/null +++ b/amd64_immediate_shuffle_family_test.go @@ -0,0 +1,268 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// Read every original encoding row and its opcode-specific attributes. Sharing +// _yvpshufd does not give the word shuffles dword broadcast permissions. +func x86OfficialImmediateShuffleRows(t *testing.T) []string { + t.Helper() + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + var rows []string + counts := map[string]int{} + for _, rule := range rules { + pp := map[string]string{"VPSHUFD": "66", "VPSHUFHW": "F3", "VPSHUFLW": "F2"}[rule.Opcode] + if pp == "" { + continue + } + index := rule.RowIndex + classes := [][]string{ + {"Yu8", "Yxm", "Yxr"}, {"Yi8", "Yxm", "Yxr"}, + {"Yu8", "Yym", "Yyr"}, {"Yi8", "Yym", "Yyr"}, + {"Yu8", "YxmEvex", "YxrEvex"}, {"Yu8", "YxmEvex", "Yknot0", "YxrEvex"}, + {"Yu8", "YymEvex", "YyrEvex"}, {"Yu8", "YymEvex", "Yknot0", "YyrEvex"}, + {"Yu8", "Yzm", "Yzr"}, {"Yu8", "Yzm", "Yknot0", "Yzr"}, + } + zcase := "Zvex_i_rm_r" + if index >= 4 { + zcase = "Zevex_i_rm_r" + if index%2 == 1 { + zcase = "Zevex_i_rm_k_r" + } + } + offsets := []int{0, 0, 2, 2, 4, 4, 7, 7, 10, 10} + advances := []int{0, 2, 0, 2, 0, 3, 0, 3, 0, 3} + if index < 0 || index >= len(classes) || rule.Ytab != "_yvpshufd" || rule.Prefix != "Pavx" || + !reflect.DeepEqual(rule.Classes, classes[index]) || rule.ZCase != zcase || + rule.EncodingOffset != offsets[index] || rule.ZOffset != advances[index] || len(rule.OpBytes) != 13 { + t.Fatalf("unmodeled original shuffle row: %+v", rule) + } + generation, bits := "vex", "128" + if index == 2 || index == 3 || index == 6 || index == 7 { + bits = "256" + } else if index >= 8 { + bits = "512" + } + if index >= 4 { + generation = "evex" + } + terms := []string{"avxEscape", generation + bits, generation + pp, generation + "0F", generation + "W0"} + if !reflect.DeepEqual(strings.Split(rule.OpBytes[rule.EncodingOffset], " | "), terms) { + t.Fatalf("unmodeled original shuffle prefix: %+v", rule) + } + opcodeOffset := rule.EncodingOffset + 1 + if generation == "evex" { + attributes := "evexN" + map[string]string{"128": "16", "256": "32", "512": "64"}[bits] + if rule.Opcode == "VPSHUFD" { + attributes += " | evexBcstN4" + } + attributes += " | evexZeroingEnabled" + if rule.OpBytes[opcodeOffset] != attributes { + t.Fatalf("unmodeled original shuffle attributes: %+v", rule) + } + opcodeOffset++ + } + if rule.OpBytes[opcodeOffset] != "0x70" { + t.Fatalf("unmodeled original shuffle opcode: %+v", rule) + } + bank := map[string]string{"128": "X", "256": "Y", "512": "Z"}[bits] + immediate := "255" + if rule.Classes[0] == "Yi8" { + immediate = "-128" + } + args := "$" + immediate + "," + bank + "0," + if len(rule.Classes) == 4 { + args += "K1," + } + // High banks force the unmasked X/Y EVEX row instead of VEX selection. + destination := "1" + if generation == "evex" && bits != "512" { + destination = "17" + } + rows = append(rows, rule.Opcode+" "+args+bank+destination) + counts[rule.Opcode]++ + } + for _, op := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + if counts[op] != 10 { + t.Fatalf("%s complete original-row denominator changed: %d", op, counts[op]) + } + } + if len(rows) != 30 { + t.Fatalf("complete original shuffle denominator changed: %d", len(rows)) + } + return rows +} + +func TestX86ImmediateShuffleOfficialCallerRegistration(t *testing.T) { + rows := x86OfficialImmediateShuffleRows(t) + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + rows = append(rows, op+" $255,X0,X1") + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !x86CallerVectorMoveKnown(file.Funcs[0].Instrs[1]) { + t.Fatal("Go-accepted original shuffle row has no caller effect") + } + }) + } +} + +func TestX86ImmediateShuffleMemoryShapeIsPhysicalTuple(t *testing.T) { + for _, op := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for bank, width := range map[string]int{"X": 16, "Y": 32, "Z": 64} { + for _, suffix := range []string{"", ".Z"} { + row := fmt.Sprintf("%s%s $0,(AX),K1,%s1", op, suffix, bank) + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + bytes, write, valid := x86InstructionMemoryShape(file.Funcs[0].Instrs[1], 1, "amd64") + if !valid || write || bytes != int64(width) { + t.Fatalf("physical tuple read got %d/%v/%v, want %d/false/true", bytes, write, valid, width) + } + }) + } + } + } +} + +func TestCrossLinuxRuntimeMatrixX86ImmediateShuffleCompleteBytes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + widths := []int{16} + if strings.HasPrefix(op, "V") { + widths = []int{16, 32, 64} + } + for _, width := range widths { + for _, mode := range []int{0, 1, 2} { + if !strings.HasPrefix(op, "V") && mode != 0 { + continue + } + broadcasts := []bool{false} + if op == "VPSHUFD" { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("immshuffle%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + row := op + if broadcast { + row += ".BCST" + } + if mode == 2 { + row += ".Z" + } + input := bank + "0" + if broadcast { + input = "(BX)" + } + row += " $27," + input + "," + if mode != 0 { + row += "K1," + } + row += bank + "1" + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ in+8(FP),BX\nMOVQ mask+16(FP),DX\n", name) + fmt.Fprintf(&source, "VMOVUPS (BX),Z0\nVMOVUPS (AX),Z1\nKMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s\nVMOVUPS Z1,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", row) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}} + kind := 0 + if strings.Contains(op, "HW") { + kind = 2 + } else if strings.Contains(op, "LW") { + kind = 1 + } + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,\"%s\",%d,%d,%d,%d,%d)) return 1;\n", name, name, width, kind, mode, + x86FloatAddSubBoolInt(strings.HasPrefix(op, "V")), x86FloatAddSubBoolInt(broadcast)) + } + } + } + } + if count != 40 { + t.Fatalf("independent complete runtime denominator changed: %d", count) + } + oracle := `#include +#include +#include +` + declarations.String() + ` +static int check(void (*program)(uint8_t*,const uint8_t*,uint64_t),const char *name, + unsigned width,int kind,int mode,int vector,int broadcast) { + const uint64_t masks[]={0,1,UINT64_MAX,UINT64_C(0x55aa55aa55aa55aa)}; + for(unsigned m=0;m<4;m++) { + uint8_t input[66],original[66],output[68],before[68]; + for(unsigned i=0;i<66;i++) input[i]=(uint8_t)(i*29+197); + for(unsigned i=0;i<68;i++) output[i]=(uint8_t)(i*13+71); + memcpy(original,input,sizeof(input)); memcpy(before,output,sizeof(output)); + program(output+1,input+1,masks[m]); + if(output[0]!=before[0] || output[67]!=before[67] || + output[65]!=1 || output[66]!=0 || memcmp(input,original,sizeof(input))) return 1; + unsigned lane=kind ? 2 : 4; + for(unsigned i=0;i<64;i++) { + uint8_t want=before[1+i]; + if(i>=width) { if(vector) want=0; } + else if(mode && !(masks[m]>>(i/lane)&1)) { if(mode==2) want=0; } + else { + unsigned at=i; + if(broadcast) at=i%4; + else if(kind==0) at=(i/16)*16+((27>>(2*((i%16)/4)))&3)*4+i%4; + else { + unsigned word=(i%16)/2; + if((kind==1 && word<4) || (kind==2 && word>=4)) + at=(i/16)*16+(kind==2 ? 8 : 0)+((27>>(2*(word%4)))&3)*2+i%2; + } + want=input[1+at]; + } + if(output[1+i]!=want) { + fprintf(stderr,"%s mask=%llx byte=%u got=%x want=%x\n",name,(unsigned long long)masks[m],i,output[1+i],want); + return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "immediate-shuffle-complete-bytes", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_immediate_shuffle_fault_test.go b/amd64_immediate_shuffle_fault_test.go new file mode 100644 index 000000000..569cbed92 --- /dev/null +++ b/amd64_immediate_shuffle_fault_test.go @@ -0,0 +1,260 @@ +//go:build unix + +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ImmediateShuffleLegacyNativeMemory(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, references, checks strings.Builder + sigs := map[string]FuncSig{} + for index, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + // The C oracle executes bytes independently observed from original Go, + // with fixed RSI/RDI inputs. It does not reuse the production grammar. + original := "TEXT f(SB),4,$0-0\n" + op + " $27,(SI),X1\nRET\n" + listing := x86InsertExtractGoListing(t, "amd64", original) + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + prefixByte := byte(0x66) + if op == "PSHUFHW" { + prefixByte = 0xf3 + } else if op == "PSHUFLW" { + prefixByte = 0xf2 + } + if !bytes.Equal(code, []byte{prefixByte, 0x0f, 0x70, 0x0e, 27, 0xc3}) { + t.Fatalf("independent original Go m128 opcode changed: %s %x", op, code) + } + var literals []string + for _, value := range code[:5] { + literals = append(literals, fmt.Sprintf("0x%02x", value)) + } + kind := 0 + if op == "PSHUFHW" { + kind = 2 + } else if op == "PSHUFLW" { + kind = 1 + } + for _, dead := range []bool{false, true} { + name := fmt.Sprintf("legacymemory%d_%t", index, dead) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-16\nMOVQ out+0(FP),AX\nMOVQ input+8(FP),BX\nCMPQ BX,BX\nSTC\n%s $27,(BX),X1\n", name, op) + tail := "" + if !dead { + source.WriteString("MOVOU X1,(AX)\nSETCS 16(AX)\nSETNE 17(AX)\n") + tail = "; movdqu %%xmm1,(%%rdi); setc 16(%%rdi); setnz 17(%%rdi)" + } + source.WriteString("RET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*);\n", name) + fmt.Fprintf(&references, `static void original_%s(uint8_t *out,const uint8_t *input) { + __asm__ volatile("cmpq %%%%rsi,%%%%rsi; stc; .byte %s%s" :: + "D"(out), "S"(input) : "xmm1", "cc", "memory"); +} +`, name, strings.Join(literals, ","), tail) + fmt.Fprintf(&checks, ` + if(compare(original_%[1]s,%[1]s,region,0,%[2]d,%[3]d) || + compare(original_%[1]s,%[1]s,boundary-16,0,%[2]d,%[3]d) || + compare(original_%[1]s,%[1]s,region+1,-1,%[2]d,%[3]d) || + compare(original_%[1]s,%[1]s,boundary-15,1,%[2]d,%[3]d) || + compare(original_%[1]s,%[1]s,0,1,%[2]d,%[3]d)) return 1; +`, name, kind, x86FloatAddSubBoolInt(dead)) + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + oracle := `#include +#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + references.String() + ` +typedef void (*memoryfn)(uint8_t*,const uint8_t*); +static int observe(memoryfn fn,const uint8_t *input,int kind,int dead) { + pid_t child=fork(); + if(child<0) return -1; + if(child==0) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(2); + uint8_t result[20], before[16]; + memset(result,0xc3,sizeof(result)); + // Copy only for a successful readable input; boundary and NULL probes + // must fault in the tested opcode, not in the independent reference. + if(input && !((uintptr_t)input & 15)) memcpy(before,input,16); + fn(result+1,input); + if(result[0]!=0xc3 || result[19]!=0xc3) _exit(3); + if(dead) { + for(unsigned i=1;i<19;i++) if(result[i]!=0xc3) _exit(4); + } else { + if(result[17]!=1 || result[18]!=0) _exit(5); + for(unsigned i=0;i<16;i++) { + unsigned at=i; + if(kind==0) at=((27>>(2*(i/4)))&3)*4+i%4; + else { + unsigned word=i/2; + if((kind==1 && word<4) || (kind==2 && word>=4)) + at=(kind==2 ? 8 : 0)+((27>>(2*(word%4)))&3)*2+i%2; + } + if(result[i+1]!=input[at]) _exit(6); + } + if(!((uintptr_t)input & 15) && memcmp(before,input,16)) _exit(7); + } + _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status)) return WEXITSTATUS(status); + if(WIFSIGNALED(status)) return 128+WTERMSIG(status); + return -1; +} +static int compare(memoryfn reference,memoryfn translated,const uint8_t *input, + int fault,int kind,int dead) { + int native=observe(reference,input,kind,dead); + int got=observe(translated,input,kind,dead); + int memoryFault=native==128+SIGSEGV || native==128+SIGBUS; + int valid=fault==1 ? memoryFault : fault==-1 ? native==0 || memoryFault : native==0; + if(native<0 || native!=got || !valid) { + fprintf(stderr,"legacy shuffle input=%p native=%d LLVM=%d requiredFault=%d\n",input,native,got,fault); + return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<64) return 1; + uint8_t *region=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(region==MAP_FAILED || mprotect(region+page,(size_t)page,PROT_NONE)) return 1; + for(long i=0;i +#include +#include +#include +#include +#include +#include +` + declarations.String() + ` +static int check(void (*program)(const uint8_t*),const uint8_t *input,int fault) { + pid_t child=fork(); + if(child<0) return 1; + if(child==0) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(2); + program(input); + _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return 1; + int got=WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS); + if(fault ? !got : !WIFEXITED(status) || WEXITSTATUS(status)!=0) { + fprintf(stderr,"shuffle physical-read input=%p fault=%d status=%x\n",input,fault,status); + return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<=0) return 1; + uint8_t *memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED || mprotect(memory+page,(size_t)page,PROT_NONE)) return 1; + uint8_t *boundary=memory+page; +` + checks.String() + ` + return munmap(memory,(size_t)page*2)!=0; +} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "immediate-shuffle-native-reads", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_immediate_shuffle_forms.go b/amd64_immediate_shuffle_forms.go new file mode 100644 index 000000000..ad80a8873 --- /dev/null +++ b/amd64_immediate_shuffle_forms.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ImmediateShuffleSpec struct { + laneBytes int + high, legacy, mmx bool +} + +// Go's ymshuf/yxshuf and complete _yvpshufd rows share the four-element +// selector grammar. Only the dword EVEX spelling permits m32 broadcast. +var x86ImmediateShuffleSpecs = map[Op]x86ImmediateShuffleSpec{ + "PSHUFW": {2, false, true, true}, + "PSHUFL": {4, false, true, false}, + "PSHUFD": {4, false, true, false}, + "PSHUFHW": {2, true, true, false}, + "PSHUFLW": {2, false, true, false}, + "VPSHUFD": {4, false, false, false}, + "VPSHUFHW": {2, true, false, false}, + "VPSHUFLW": {2, false, false, false}, +} + +type x86ImmediateShuffleForm struct { + spec x86ImmediateShuffleSpec + source Operand + destination, mask, memory int + byteWidth, memoryBytes int + immediate uint8 + broadcast, zeroing bool +} + +// Immediate shuffles read one complete source, including the word half left +// in place. Unlike PSHUFB, the old destination is never an input unless it +// aliases that source. Memory extent/alignment remain separate obligations. +func x86TypedImmediateShuffleByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + local := ins + local.Op = op + form, known, err := x86ImmediateShuffleProperties(local, arch) + if known && ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86TypedByteEffect{}, true, fmt.Errorf("immediate-shuffle dispatch/source mismatch: %s/%s", op, ins.Op) + } + if !known || err != nil || !form.spec.legacy || form.spec.mmx { + return x86TypedByteEffect{}, known && (err != nil || form.spec.legacy && !form.spec.mmx), err + } + return x86ImmediateShuffleDependencyEffect(ins, form), true, nil +} + +// The admitted encoding chooses upper writes and memory alignment; both use +// one source-only dependency shape. This descriptor does not admit a form. +func x86ImmediateShuffleDependencyEffect(ins Instr, form x86ImmediateShuffleForm) x86TypedByteEffect { + vector := x86TypedVectorEffect{x86TypedVectorUpperPreserve, 16} + if !form.spec.legacy { + vector = x86TypedVectorEffect{x86TypedVectorUpperZero, 1} + } + return x86TypedByteEffect{ + vector: vector, + operands: []x86TypedByteOperand{ + {index: 1, operand: form.source, width: int64(form.byteWidth), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), write: true}, + }, + } +} + +// This supplies encoding, native access extent and byte selectors, not source +// values, original pointer/alignment backing or permission to borrow an ABI. +func x86ImmediateShuffleProperties(ins Instr, arch string) (x86ImmediateShuffleForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86ImmediateShuffleSpecs[Op(base)] + if !known { + return x86ImmediateShuffleForm{}, false, nil + } + f := x86ImmediateShuffleForm{spec: spec, destination: len(ins.Args) - 1, mask: -1, memory: -1} + fail := func() (x86ImmediateShuffleForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside Go immediate-shuffle grammar: %q", arch, ins.Op, ins.Raw) + } + if arch == "" { + arch = "amd64" + } + if arch != "amd64" && arch != "386" || len(ins.Args) != 3 && len(ins.Args) != 4 { + return fail() + } + immediate, integer := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !integer { + return fail() + } + if spec.legacy { + if dotted || len(ins.Args) != 3 || spec.mmx && !amd64SignedImmediate(immediate, 8) || + !spec.mmx && !amd64UnsignedImmediate(immediate, 8) { + return fail() + } + f.byteWidth = 16 + if spec.mmx { + f.byteWidth = 8 + } + } else { + switch suffix { + case "": + if dotted { + return fail() + } + case "Z": + f.zeroing = true + case "BCST": + f.broadcast = true + case "BCST.Z": + f.broadcast, f.zeroing = true, true + default: + return fail() + } + if immediate.Imm < -128 || immediate.Imm > 255 || f.broadcast && spec.laneBytes != 4 { + return fail() + } + if len(ins.Args) == 4 { + if arch == "386" && !ins.x86Encoded || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + f.mask = 2 + } else if f.zeroing { + return fail() + } + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + } + register := func(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + if spec.mmx { + _, valid := amd64ParseMReg(operand.Reg) + return valid + } + if spec.legacy { + index, valid := amd64ParseXReg(operand.Reg) + return valid && (arch == "amd64" && index < 16 || arch == "386" && index < 8) + } + return amd64GoPackedVectorMoveRegister(operand, f.byteWidth, arch) + } + if !register(ins.Args[f.destination]) { + return fail() + } + local, err := x86LegacyMMXReadInstruction(ins, arch) + if err != nil { + return f, true, err + } + f.source = local.Args[1] + if isAMD64MemoryOperand(f.source) && x86PackedShiftMemoryValid(f.source, arch) { + f.memory, f.memoryBytes = 1, f.byteWidth + if f.broadcast { + f.memoryBytes = 4 + } + } else if f.broadcast || !register(f.source) { + return fail() + } + if !spec.legacy && immediate.Imm < 0 && + (f.broadcast || amd64PackedWordShuffleRequiresEVEX(f.source, ins.Args[f.destination], f.byteWidth, f.mask >= 0)) { + return fail() + } + f.immediate = uint8(immediate.Imm) + return f, true, x86Mode32VectorRegisterContext(ins, arch) +} + +func x86ImmediateShuffleConstant(operand Operand, arch string, encoded bool) (Operand, bool) { + if operand.Kind != OpImm || operand.ImmIsFloat || operand.ImmRaw != "" { + return operand, false + } + // Go's I386 oclass classifies TYPE_CONST after signed int32 truncation; + // retain the original operand, but use its classified value for every row. + if arch == "386" && !encoded { + operand.Imm = int64(int32(operand.Imm)) + } + return operand, true +} + +// Every output byte depends on exactly one source byte; the word half not +// shuffled still comes from the source, never the old destination. +func (f x86ImmediateShuffleForm) sourceByte(destination int) int { + if f.broadcast { + return destination % 4 + } + groupBytes := 16 + if f.spec.mmx { + groupBytes = 8 + } + lane := destination % groupBytes / f.spec.laneBytes + if f.spec.laneBytes == 2 && !f.spec.mmx && (lane < 4) == f.spec.high { + return destination + } + half := 0 + if f.spec.high { + half = 8 + } + return destination/groupBytes*groupBytes + half + + int(f.immediate>>(2*(lane%4))&3)*f.spec.laneBytes + destination%f.spec.laneBytes +} diff --git a/amd64_immediate_shuffle_grammar_test.go b/amd64_immediate_shuffle_grammar_test.go new file mode 100644 index 000000000..6a9203142 --- /dev/null +++ b/amd64_immediate_shuffle_grammar_test.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86ImmediateShuffleCompleteImmediateDomains(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT probe(SB),4,$0-0\n") + count := 0 + for _, op := range []string{"PSHUFW", "PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for value := 0; value < 256; value++ { + immediate, bank := value, "X" + if op == "PSHUFW" { + immediate, bank = value-128, "M" + } + row := fmt.Sprintf("%s $%d,%s0,%s1", op, immediate, bank, bank) + fmt.Fprintln(&source, row) + ins := x86YMMReadSource(t, row) + before := fmt.Sprintf("%#v", ins) + if _, known, err := x86ImmediateShuffleProperties(ins, arch); !known || err != nil { + t.Fatalf("complete imm8 domain missing %s: %v", row, err) + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("pure grammar changed original source") + } + count++ + } + } + source.WriteString("RET\n") + goAssemblerOracleOutput(t, arch, source.String(), "", true) + if count != 2048 { + t.Fatal("complete independent immediate denominator changed", count) + } + }) + } +} + +func TestX86ImmediateShuffleRejectsSiblingInvalidFormats(t *testing.T) { + var rows []string + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + for _, args := range []string{"$-1,X0,X1", "$256,X0,X1", "$0,X16,X1", "$0,X0,Y1", "$0,AX,X1", "$0,X0,(AX)", "$0,X0,K1,X1"} { + rows = append(rows, op+" "+args) + } + rows = append(rows, op+".Z $0,X0,X1") + } + for _, args := range []string{"$128,M0,M1", "$-129,M0,M1", "$0,X0,M1", "$0,M0,X1", "$0,M8,M1", "$0,M0,(AX)"} { + rows = append(rows, "PSHUFW "+args) + } + rows = append(rows, "PSHUFW.Z $0,M0,M1") + for _, op := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for _, args := range []string{"$-129,X0,X1", "$256,X0,X1", "$-1,Z0,Z1", "$-1,X16,X1", "$-1,X0,K1,X1", "$0,X0,Y1", "$0,AX,X1", "$0,X0,(AX)", "$0,X0,K0,X1", "$0,X0,K8,X1", "$0,X0,X32"} { + rows = append(rows, op+" "+args) + } + rows = append(rows, op+".Z $0,X0,X1", op+".BCST $0,X0,X1", op+".SAE $0,(AX),X1") + if op != "VPSHUFD" { + rows = append(rows, op+".BCST $0,(AX),X1") + } + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + masklessZero := row == "VPSHUFD.Z $0,X0,X1" || row == "VPSHUFHW.Z $0,X0,X1" || row == "VPSHUFLW.Z $0,X0,X1" + historicalReserved := masklessZero && !goToolchainAtLeast(runtime.Version(), 1, 21) + goAssemblerOracleOutput(t, arch, source, "", historicalReserved) + if masklessZero { + // Before Go 1.21 the encoder emitted reserved EVEX.z/aaa=0. + // Observe its exact original bytes; neither named nor raw + // translation may acquire semantics from that old acceptance. + pp := byte(1) + if strings.HasPrefix(row, "VPSHUFHW") { + pp = 2 + } else if strings.HasPrefix(row, "VPSHUFLW") { + pp = 3 + } + code := []byte{0x62, 0xf1, 0x7c | pp, 0x88, 0x70, 0xc8, 0} + if historicalReserved { + listing := x86InsertExtractGoListing(t, arch, source) + var observed []byte + for _, line := range strings.Split(string(listing), "\n") { + observed = append(observed, x86InsertExtractListedBytes(line)...) + } + if !bytes.Equal(observed, append(append([]byte(nil), code...), 0xc3)) { + t.Fatalf("historical reserved shuffle bytes changed: %x", observed) + } + t.Logf("%s/%s historical reserved z=1/aaa=0 observation; not a legal form", runtime.Version(), arch) + } + mode := 64 + if arch == "386" { + mode = 32 + } + if _, _, known, err := decodedX86PackedShuffleInstruction(code, mode); !known || err == nil { + t.Fatal("reserved raw zeroing acquired shuffle semantics", err) + } + x86RequireZeroMaskAPIOutcome(t, arch, source, false) + x86RequireZeroMaskAPIOutcome(t, arch, x86RawINSERTPSSource("f", code), false) + } + ins := x86YMMReadSource(t, row) + if _, known, err := x86ImmediateShuffleProperties(ins, arch); !known || err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go-rejected syntax became a valid/context form", row, arch, err) + } + } + }) + } +} + +func TestX86ImmediateShuffleMode32PhysicalAndMMXBoundaries(t *testing.T) { + for _, op := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for _, row := range []string{op + " $0,X16,X31", op + " $0,Y16,Y31"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + if _, known, err := x86ImmediateShuffleProperties(x86YMMReadSource(t, row), "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("source-only high bank acquired physical binding", row, err) + } + } + for _, row := range []string{op + " $0,Z8,Z0", op + " $0,X0,K1,X1"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + if _, known, err := x86ImmediateShuffleProperties(x86YMMReadSource(t, row), "386"); !known || err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go-rejected mode32 form acquired physical binding", row, err) + } + } + } + for _, arch := range []string{"amd64", "386"} { + ins := x86YMMReadSource(t, "PSHUFW $27,AX,M1") + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+ins.Raw+"\nRET\n", "", true) + form, known, err := x86ImmediateShuffleProperties(ins, arch) + if !known || err != nil || form.source.Reg != "M0" || !form.spec.mmx { + t.Fatal("original Go Ymm spelling lost its actual MMX bank", arch, form, err) + } + if !errors.Is(x86LegacyMMXReadEmitterContext(ins, arch), ErrProbeNeedsContext) || x86CallerVectorMoveKnown(ins) { + t.Fatal("GP source spelling became initialized MMX data or caller permission") + } + } +} + +func TestX86ImmediateShuffleNativeFeatureScope(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []string{"PSHUFL $0,(AX),X1", "PSHUFD $0,(AX),X1", "PSHUFHW $0,(AX),X1", "PSHUFLW $0,(AX),X1"} { + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, Func{Instrs: []Instr{x86YMMReadSource(t, row)}}); got != "+sse2" { + t.Fatal("validated native legacy m128 needs its private SSE2 carrier", arch, row, got) + } + } + for _, row := range []string{"PSHUFL $-1,(AX),X1", "PSHUFL $0,X0,X1", "VPSHUFD $0,(AX),X1", "PSHUFW $0,(AX),M1"} { + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, Func{Instrs: []Instr{x86YMMReadSource(t, row)}}); got != "" { + t.Fatal("non-native/malformed form acquired features", arch, row, got) + } + } + } +} + +func TestX86ImmediateShuffleNativeMemoryCannotBorrowVirtualBacking(t *testing.T) { + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + for _, operand := range []string{"a+0(FP)", "(SP)", "(BP)", "(AX)(BP*2)"} { + row := op + " $0," + operand + ",X1" + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-16\n"+row+"\nRET\n", "", true) + ins := x86YMMReadSource(t, row) + var output strings.Builder + ctx := &amd64Ctx{goarch: "amd64", b: &output} + if known, _, err := ctx.lowerPackedWordShuffle(ins.Op, ins); !known || !errors.Is(err, ErrProbeNeedsContext) || output.Len() != 0 { + t.Fatal("virtual/native frame acquired backing before IR", row, err, output.String()) + } + }) + } + for _, bits := range []int{32, 64} { + ins := x86YMMReadSource(t, op+" $0,(AX),X1") + ins.x86AddressBits = bits + var output strings.Builder + ctx := &amd64Ctx{goarch: "amd64", b: &output} + if known, _, err := ctx.lowerPackedWordShuffle(ins.Op, ins); !known || !errors.Is(err, ErrProbeNeedsContext) || output.Len() != 0 { + t.Fatal("unbound address-size identity acquired native backing", bits, err) + } + } + } +} diff --git a/amd64_immediate_shuffle_objects_test.go b/amd64_immediate_shuffle_objects_test.go new file mode 100644 index 000000000..d9468e415 --- /dev/null +++ b/amd64_immediate_shuffle_objects_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86ImmediateShuffleOfficialRowsThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + rows := x86OfficialImmediateShuffleRows(t) + var selected []string + for _, row := range rows { + if target.arch == "386" { + if strings.Count(row, ",") == 3 { + // This is an actual frontend refusal, not a skipped test. + goAssemblerOracleOutput(t, target.arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + continue + } + row = strings.NewReplacer("X17", "X1", "Y17", "Y1").Replace(row) + } + selected = append(selected, row) + args := strings.Split(strings.Fields(row)[1], ",") + args[1] = "8(AX)" + selected = append(selected, strings.Fields(row)[0]+" "+strings.Join(args, ",")) + } + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + selected = append(selected, op+" $255,X0,X1", op+" $0,16(AX),X1") + } + selected = append(selected, "PSHUFW $-128,M0,M1", "PSHUFW $127,8(AX),M1") + if target.arch == "386" { + for _, op := range []string{"PSHUFW", "PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + bank, values := "X", []int{0, 27, 255} + if op == "PSHUFW" { + bank, values = "M", []int{0, -1, 127} + } else if strings.HasPrefix(op, "V") { + values = []int{0, -1, 255} + } + for _, value := range values { + selected = append(selected, fmt.Sprintf("%s $%d,%s0,%s1", op, int64(4294967296)+int64(value), bank, bank)) + } + } + } + for _, bank := range []string{"X", "Y", "Z"} { + selected = append(selected, "VPSHUFD.BCST $0,8(AX),"+bank+"1") + if target.arch == "amd64" { + selected = append(selected, "VPSHUFD.BCST.Z $255,8(AX),K1,"+bank+"1") + } + } + want := 76 + if target.arch == "386" { + want = 79 + } + if len(selected) != want { + t.Fatal("official row/operand variation denominator changed", len(selected), want) + } + var source strings.Builder + for index, row := range selected { + name := fmt.Sprintf("F%d", index) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\n%s\nRET\n", name, row) + } + // Preserve every original uninitialized row, including the unclosed + // MMX sources. Syntax objects cannot supply numerical or x87 state. + file, sigs := x86UnboundVoidFormatSources(t, source.String(), target.arch, target.triple) + before := fmt.Sprintf("%#v %#v", file, sigs) + ir := x86ScalarFormatObjectUnit(t, file, sigs, target.arch, target.triple, 0) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + if before != fmt.Sprintf("%#v %#v", file, sigs) { + t.Fatal("object verification mutated original input or signatures") + } + t.Logf("%d original Go forms, three public Context checks and emitter-only LLVM22 objects; initialized caller/MMX owners are separate", len(selected)) + }) + } +} diff --git a/amd64_immediate_shuffle_result.go b/amd64_immediate_shuffle_result.go new file mode 100644 index 000000000..b48fde068 --- /dev/null +++ b/amd64_immediate_shuffle_result.go @@ -0,0 +1,46 @@ +package plan9asm + +func x86ResultImmediateShuffleStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + return x86ResultImmediateShuffleStepWithRead(ins, state, owned, arch, false) +} + +func x86ResultImmediateShuffleStepWithRead(ins Instr, state *x86ResultState, owned []bool, arch string, nativeRead bool) (bool, bool) { + form, known, err := x86ImmediateShuffleProperties(ins, arch) + if !known || err != nil || form.spec.mmx { + return false, known // Physical MMX does not alias the SIMD result bank. + } + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false, true + } + } + var source uint64 + if form.memory >= 0 { + // A selected byte or zero mask cannot shrink the native memory tuple. + // Legacy m128 also needs native alignment/backing; a logical FP/AUTO + // allocation is not that witness. The paired original legacy opcode + // retains alignment faults; continued execution asserts no alignment. + if _, static := x86StaticSBReadAddress(form.source, arch); form.spec.legacy && !static && + (!nativeRead || !x86OrdinaryNativeMemoryOperand(form.source, arch)) { + return false, true + } + if !x86ResultReadWithNativeContinuation(ins, form.source, int64(form.memoryBytes), state, owned, arch, nativeRead) { + return false, true + } + source = x86CallerVectorByteMask(int64(form.memoryBytes)) + } else { + index, _ := amd64VectorRegisterIndex(form.source.Reg, form.byteWidth) + source = state.vectorBytes(index) + } + var defined uint64 + for output := 0; output < form.byteWidth; output++ { + defined |= (source >> uint(form.sourceByte(output)) & 1) << uint(output) + } + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + if form.mask >= 0 && !form.zeroing { + defined &= state.vectorBytes(destination) + } + state.writeVector(destination, form.byteWidth, defined, form.spec.legacy) + return true, true +} diff --git a/amd64_immediate_shuffle_result_test.go b/amd64_immediate_shuffle_result_test.go new file mode 100644 index 000000000..58e100569 --- /dev/null +++ b/amd64_immediate_shuffle_result_test.go @@ -0,0 +1,139 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86ImmediateShuffleResultMemoryExtent(t *testing.T) { + for _, row := range []struct { + instruction string + bytes int + }{ + {"VPSHUFD $0,value+0(FP),X1", 16}, + {"VPSHUFHW.Z $0,value+0(FP),K1,Y1", 32}, + {"VPSHUFLW.Z $0,value+0(FP),K1,Z1", 64}, + {"VPSHUFD.BCST.Z $0,value+0(FP),K1,Z1", 4}, + } { + t.Run(row.instruction, func(t *testing.T) { + ins := x86YMMReadSource(t, row.instruction) + for missing := -1; missing < 64; missing++ { + state := x86ResultState{fp: make([]bool, 64)} + owned := make([]bool, 64) + for index := range owned { + owned[index], state.fp[index] = true, index != missing + } + state.masks[1] = true + accepted := x86ResultTransfer(ins, &state, owned, 64, "amd64") + if want := missing < 0 || missing >= row.bytes; accepted != want { + t.Fatalf("full original tuple bytes=%d missing=%d accepted=%v", row.bytes, missing, accepted) + } + } + }) + } + for _, row := range []string{ + "PSHUFHW $0,value+0(FP),X1", "VPSHUFD $0,(AX),X1", + "VPSHUFD $0,unknown<>(SB),X1", "PSHUFW $0,M0,M1", + } { + state := x86ResultState{fp: make([]bool, 16)} + owned := make([]bool, 16) + for index := range owned { + owned[index], state.fp[index] = true, true + } + state.pointerGP[0] = true + if x86ResultTransfer(x86YMMReadSource(t, row), &state, owned, 16, "amd64") { + t.Fatal("logical storage, pointer or SIMD state acquired native backing/MMX authority", row) + } + } +} + +func amd64ImmediateShuffleResultMatrix() (source, declarations, harness string) { + var asm, decl, prototypes, checks strings.Builder + index := 0 + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + width, start := 4, 0 + if strings.HasSuffix(op, "HW") || strings.HasSuffix(op, "LW") { + width = 2 + } + if strings.HasSuffix(op, "HW") { + start = 8 + } + for _, immediate := range []int{0, 27, 228, 255} { + for _, destination := range []string{"X0", "X1"} { + name := fmt.Sprint("shuffle", index) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$0-24\nMOVQ AX,X0\nPINSRQ $1,BX,X0\n%s $%d,X0,%s\nPEXTRQ $1,%s,CX\nMOVQ %s,AX\nXORQ CX,AX\nRET\n", + name, op, immediate, destination, destination, destination) + fmt.Fprintf(&decl, "func %s(a,b uint64) uint64\n", name) + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint64_t,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d,%d,%d)){fprintf(stderr,\"%s failed\\n\");return 1;}\n", name, width, start, immediate, name) + index++ + } + } + } + const reference = ` +static int check(uint64_t (*fn)(uint64_t,uint64_t), unsigned width, + unsigned start, unsigned imm) { + const uint64_t values[]={0,UINT64_MAX,UINT64_C(0x1020304050607080),UINT64_C(0xfedcba9876543210)}; + for(unsigned a=0;a<4;a++) { + for(unsigned b=0;b<4;b++) { + uint8_t input[16],output[16]; + uint64_t low,high; + memcpy(input,&values[a],8); + memcpy(input+8,&values[b],8); + memcpy(output,input,16); + for(unsigned lane=0;lane<4;lane++) { + unsigned selected=(imm>>(2*lane))&3; + memcpy(output+start+width*lane,input+start+width*selected,width); + } + memcpy(&low,output,8); + memcpy(&high,output+8,8); + if(fn(values[a],values[b])!=(low^high)) return 1; + } + } + return 0; +} +` + return asm.String(), decl.String(), "#include \n#include \n#include \n" + + prototypes.String() + reference + "int main(void){\n" + checks.String() + "return 0; }\n" +} + +func TestAMD64ImmediateShuffleRegisterResultsObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + source, declarations, _ := amd64ImmediateShuffleResultMatrix() + requireAMD64RegisterResultGoObject(t, source) + for _, target := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + for _, api := range []string{"text", "module", "Go"} { + t.Run(target+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", target, api) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target, ir, "shuffle-result") + }) + } + } +} + +func TestCrossLinuxRuntimeMatrixAMD64ImmediateShuffleRegisterResults(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + source, declarations, harness := amd64ImmediateShuffleResultMatrix() + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimize := range []bool{false, true} { + text := ir + if optimize { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "shuffle-results", triple, text, harness, prefix) + } + }) + } +} diff --git a/amd64_implicit_divide.go b/amd64_implicit_divide.go new file mode 100644 index 000000000..fffeecd78 --- /dev/null +++ b/amd64_implicit_divide.go @@ -0,0 +1,222 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ImplicitDivideSpec struct { + bits int + signed bool +} + +// Complete Go ydivb/Ymb and ydivl/Yml rows. Division has trapping semantics, +// not multiply's ordinary numerical transfer or a generic LLVM div/rem rule. +var x86ImplicitDivideSpecs = map[Op]x86ImplicitDivideSpec{ + "DIVB": {8, false}, "DIVW": {16, false}, "DIVL": {32, false}, "DIVQ": {64, false}, + "IDIVB": {8, true}, "IDIVW": {16, true}, "IDIVL": {32, true}, "IDIVQ": {64, true}, +} + +type x86ImplicitDivideForm struct { + spec x86ImplicitDivideSpec + source Operand +} + +func x86ImplicitDivideProperties(op Op, ins Instr, arch string) (x86ImplicitDivideForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86ImplicitDivideSpecs[Op(name)] + form := x86ImplicitDivideForm{spec: spec} + if !known { + return form, false, nil + } + fail := func() (x86ImplicitDivideForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside Go implicit division source grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || arch == "386" && spec.bits == 64 || dotted || len(ins.Args) != 1 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) || + !isX86ScalarMultiplyDivideSource(ins.Args[0], spec.bits, arch) { + return fail() + } + form.source = ins.Args[0] + if form.source.Kind == OpReg { + return form, true, nil + } + if projected, segment, err := x86NamedSegmentMemorySource(ins, form.source, arch); segment { + if err != nil { + return form, true, err + } + form.source = projected + } + if form.source.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(form.source.Mem, arch); err != nil { + return form, true, err + } + } + if !x86ScalarCompareMemoryValid(form.source, arch) { + return fail() + } + return form, true, nil +} + +func (c *amd64Ctx) lowerImplicitDivide(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86ImplicitDivideProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || + !x86ResultNamedPredicateInstruction(ins) || ins.x86NumericalRead != nil { + return true, false, fmt.Errorf("%w: division needs its named x86 target and original address size", ErrProbeNeedsContext) + } + if form.source.Kind == OpReg { + if x86LogicalSourceFrameRegister(form.source.Reg, c.goarch) { + return true, false, fmt.Errorf("%w: division cannot consume an unbound physical SP/BP source", ErrProbeNeedsContext) + } + } else if form.source.Kind != OpFP && !c.nativeMemoryOperand(ins, 0) { + return true, false, fmt.Errorf("%w: division needs owned logical FP bytes or original native memory backing", ErrProbeNeedsContext) + } + return true, false, c.emitNativeImplicitDivide(form) +} + +// The original opcode owns zero-divisor and quotient-overflow #DE as well as +// exact memory fault priority. Pure LLVM div/rem can become poison or truncate +// an overflowing quotient; llvm.trap would raise the wrong native exception. +// Early-clobber outputs keep the divisor/address distinct from the tied old +// AX/DX inputs, including AL/AH aliases. No result is read before the call. +func (c *amd64Ctx) emitNativeImplicitDivide(form x86ImplicitDivideForm) error { + width := amd64IntegerTypeForBits(form.spec.bits) + accumulatorType := width + if form.spec.bits == 8 { + accumulatorType = I16 + } + accumulator, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, accumulatorType) + if err != nil { + return err + } + name := "div" + if form.spec.signed { + name = "idiv" + } + name += map[int]string{8: "b", 16: "w", 32: "l", 64: "q"}[form.spec.bits] + outputType, constraints, arguments := string(accumulatorType), "=&{ax},0", string(accumulatorType)+" "+accumulator + operandIndex := 2 + if form.spec.bits != 8 { + high, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, width) + if err != nil { + return err + } + outputType = fmt.Sprintf("{ %s, %s }", width, width) + constraints, arguments = "=&{ax},=&{dx},0,1", fmt.Sprintf("%s %s, %s %s", width, accumulator, width, high) + operandIndex = 4 + } + assembly, memoryClobber := "", "" + if form.source.Kind == OpReg || form.source.Kind == OpFP { + value, err := c.implicitDivideRegisterValue(form.source, width) + if err != nil { + return err + } + constraint := "r" + if form.spec.bits == 8 { + constraint = "q" + } + constraints += "," + constraint + arguments += fmt.Sprintf(", %s %s", width, value) + modifier := map[int]string{8: "b", 16: "w", 32: "k", 64: "q"}[form.spec.bits] + assembly = fmt.Sprintf("%s ${%d:%s}", name, operandIndex, modifier) + } else { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.source) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + constraints += ",r" + arguments += fmt.Sprintf(", %s %s", pointerType, pointer) + assembly, memoryClobber = fmt.Sprintf("%s %s($%d)", name, segment, operandIndex), ",~{memory}" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, outputType, assembly, + constraints+",~{flags}"+memoryClobber, arguments) + if form.spec.bits == 8 { + return c.storeRegSized(AX, I16, "%"+result) + } + for index, register := range []Reg{AX, DX} { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, %d\n", value, outputType, result, index) + if err := c.storeRegSized(register, width, "%"+value); err != nil { + return err + } + } + return nil +} + +// Logical FP is a collection of separately owned scalar fields, not native +// contiguous stack backing. Gather its exact initialized bits into the same +// register-source opcode used above. The opcode, rather than LLVM arithmetic, +// still owns zero-divisor and quotient-overflow #DE, including dead results. +func (c *amd64Ctx) implicitDivideRegisterValue(source Operand, width LLVMType) (string, error) { + if source.Kind != OpFP { + return c.evalIntSized(source, width) + } + value, handled, err := c.loadIntegerFPBytesMode(source.FPOffset, width, true) + if err != nil { + return "", err + } + if !handled { + return "", fmt.Errorf("%w: division divisor lacks exact owned logical FP bytes", ErrProbeNeedsContext) + } + return value, nil +} + +// This descriptor is deliberately not in x86TypedInstructionByteEffect. +// Only the paired native lowerer supplies a numerical continuation. The two +// post-transfer footprint consumers may still revoke exactly the GP outputs. +func x86ImplicitDivideWrites(form x86ImplicitDivideForm) []x86TypedByteOperand { + if form.spec.bits == 8 { + return []x86TypedByteOperand{{operand: Operand{Kind: OpReg, Reg: AX}, width: 2, write: true}} + } + width := int64(form.spec.bits / 8) + return []x86TypedByteOperand{ + {operand: Operand{Kind: OpReg, Reg: AX}, width: width, write: true}, + {operand: Operand{Kind: OpReg, Reg: DX}, width: width, write: true}, + } +} + +func x86LoweredImplicitDivideStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, known, err := x86ImplicitDivideProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + if !x86ResultNamedPredicateInstruction(ins) || ins.x86NumericalRead != nil || + form.source.Kind == OpReg && x86LogicalSourceFrameRegister(form.source.Reg, arch) { + return false, true + } + if form.source.Kind != OpReg && form.source.Kind != OpFP && !x86RegisterAddressedNativeMemoryOperand(form.source, arch) { + return false, true + } + width, accumulatorWidth := int64(form.spec.bits/8), int64(form.spec.bits/8) + if form.spec.bits == 8 { + accumulatorWidth = 2 + } + if !x86ResultReadWithNativeContinuation(ins, form.source, width, state, owned, arch, true) || + !x86ResultRead(Operand{Kind: OpReg, Reg: AX}, accumulatorWidth, state, owned, arch) || + form.spec.bits != 8 && !x86ResultRead(Operand{Kind: OpReg, Reg: DX}, width, state, owned, arch) { + return false, true + } + for _, output := range x86ImplicitDivideWrites(form) { + if !x86ResultStore(output.operand, output.width, state, owned, 0, arch) { + return false, true + } + index, mask, _ := x86ResultGP(output.operand.Reg, output.width, arch) + state.canonicalGP[index] &^= mask + if output.width == 4 && arch == "amd64" { + state.canonicalGP[index] |= 0xf0 + } + state.callerAddressGP[index] = false + } + state.flags = 0 // All six status flags are architecturally undefined; DF/ID stay unchanged. + return true, true +} diff --git a/amd64_implicit_mask_move_forms.go b/amd64_implicit_mask_move_forms.go new file mode 100644 index 000000000..ef89050ee --- /dev/null +++ b/amd64_implicit_mask_move_forms.go @@ -0,0 +1,75 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func x86ImplicitMaskMoveProperties(op Op, ins Instr, arch string) (amd64ImplicitMaskMoveSpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64ImplicitMaskMoveSpecs[Op(name)] + if !known { + return spec, false, nil + } + if arch == "" { + arch = "amd64" + } + fail := func() (amd64ImplicitMaskMoveSpec, bool, error) { + return spec, true, fmt.Errorf("%s %s outside complete Go implicit mask-store grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != 2 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + for _, operand := range ins.Args { + if operand.Kind != OpReg { + return fail() + } + if spec.byteWidth == 8 { + if _, valid := amd64ParseMReg(operand.Reg); !valid { + return fail() + } + continue + } + index, valid := amd64ParseXReg(operand.Reg) + if !valid || index >= 16 || arch == "386" && index >= 8 { + return fail() + } + } + return spec, true, nil +} + +func x86TypedImplicitMaskMoveByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + spec, known, err := x86ImplicitMaskMoveProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(spec.byteWidth) + return x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: width, read: true}, + {index: 1, operand: ins.Args[1], width: width, read: true}, + {index: -1, operand: Operand{Kind: OpMem, Mem: MemRef{Base: DI}}, width: width, write: true}, + }}, true, nil +} + +// MASKMOVQ belongs exclusively to its complete physical MMX/EMMS region. +// SIMD stores also need the actual paired opcode and incoming DI identity; +// an encoding, numerical address or zero mask provides neither storage nor +// initialized bytes. A masked write establishes no new memory facts. +func x86ResultImplicitMaskMoveStep(ins Instr, state *x86ResultState, arch string, native bool) (bool, bool) { + spec, known, err := x86ImplicitMaskMoveProperties(ins.Op, ins, arch) + if !known || err != nil || !x86ResultNamedPredicateInstruction(ins) { + return false, known + } + if !native || spec.byteWidth == 8 { + return false, true + } + required := x86CallerVectorByteMask(int64(spec.byteWidth)) + for _, operand := range ins.Args { + index, _ := amd64ParseXReg(operand.Reg) + if state.vectorBytes(index)&required != required { + return false, true + } + } + return x86ResultIncomingPointerAddressIdentity(Operand{Kind: OpMem, Mem: MemRef{Base: DI}}, state, arch), true +} diff --git a/amd64_indexed_permute_forms.go b/amd64_indexed_permute_forms.go new file mode 100644 index 000000000..16fb70225 --- /dev/null +++ b/amd64_indexed_permute_forms.go @@ -0,0 +1,41 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// _yvblendmpd is the existing complete EVEX binary grammar. The mode adds +// old-destination dependencies, not another register/memory/suffix parser. +func x86IndexedPermuteProperties(op Op, ins Instr, arch string) (x86PackedBinaryForm, amd64IndexedPermuteSpec, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64IndexedPermuteSpecs[Op(base)] + if !known { + return x86PackedBinaryForm{}, spec, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86PackedBinaryForm{}, spec, true, fmt.Errorf("indexed permute dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local, contextual := x86VectorValueMemoryInput(ins, arch) + local.Op = op + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, zeroing: true} + if spec.broadcast { + grammar.broadcastBytes = spec.laneBits / 8 + } + form, err := x86PackedBinaryProperties(local, arch, grammar) + if err == nil && contextual { + err = x86VectorValueMemoryContext(ins) + } + return form, spec, true, err +} + +func x86ResultIndexedPermuteStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, spec, known, err := x86IndexedPermuteProperties(ins.Op, ins, arch) + if !known { + return false, false + } + if err != nil { + return false, true + } + return x86ResultFixedVectorInputs(ins, form, spec.mode != amd64IndexedPermuteSingle, state, owned, arch), true +} diff --git a/amd64_insert_extract_forms.go b/amd64_insert_extract_forms.go new file mode 100644 index 000000000..9d83c3835 --- /dev/null +++ b/amd64_insert_extract_forms.go @@ -0,0 +1,220 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86InsertExtractSpec struct { + insert, scalar, vex, legacy bool + chunk, laneBits int + allowY, allowZ bool +} + +var x86InsertExtractSpecs = map[Op]x86InsertExtractSpec{ + "EXTRACTPS": {scalar: true, legacy: true, chunk: 4, laneBits: 32}, + "INSERTPS": {insert: true, scalar: true, legacy: true, chunk: 4, laneBits: 32}, + "VEXTRACTF128": {vex: true, chunk: 16, laneBits: 8, allowY: true}, + "VEXTRACTI128": {vex: true, chunk: 16, laneBits: 8, allowY: true}, + "VEXTRACTF32X4": {chunk: 16, laneBits: 32, allowY: true, allowZ: true}, + "VEXTRACTI32X4": {chunk: 16, laneBits: 32, allowY: true, allowZ: true}, + "VEXTRACTF64X2": {chunk: 16, laneBits: 64, allowY: true, allowZ: true}, + "VEXTRACTI64X2": {chunk: 16, laneBits: 64, allowY: true, allowZ: true}, + "VEXTRACTF32X8": {chunk: 32, laneBits: 32, allowZ: true}, + "VEXTRACTI32X8": {chunk: 32, laneBits: 32, allowZ: true}, + "VEXTRACTF64X4": {chunk: 32, laneBits: 64, allowZ: true}, + "VEXTRACTI64X4": {chunk: 32, laneBits: 64, allowZ: true}, + "VINSERTF128": {insert: true, vex: true, chunk: 16, laneBits: 8, allowY: true}, + "VINSERTI128": {insert: true, vex: true, chunk: 16, laneBits: 8, allowY: true}, + "VINSERTF32X4": {insert: true, chunk: 16, laneBits: 32, allowY: true, allowZ: true}, + "VINSERTI32X4": {insert: true, chunk: 16, laneBits: 32, allowY: true, allowZ: true}, + "VINSERTF64X2": {insert: true, chunk: 16, laneBits: 64, allowY: true, allowZ: true}, + "VINSERTI64X2": {insert: true, chunk: 16, laneBits: 64, allowY: true, allowZ: true}, + "VINSERTF32X8": {insert: true, chunk: 32, laneBits: 32, allowZ: true}, + "VINSERTI32X8": {insert: true, chunk: 32, laneBits: 32, allowZ: true}, + "VINSERTF64X4": {insert: true, chunk: 32, laneBits: 64, allowZ: true}, + "VINSERTI64X4": {insert: true, chunk: 32, laneBits: 64, allowZ: true}, + "VEXTRACTPS": {scalar: true, chunk: 4, laneBits: 32}, + "VINSERTPS": {insert: true, scalar: true, chunk: 4, laneBits: 32}, +} + +type x86InsertExtractForm struct { + spec x86InsertExtractSpec + source, base, destination, mask, memory int + width, output, offset int + immediate byte + zeroing, memoryWrite bool +} + +func (form x86InsertExtractForm) undefinedOpcode() bool { + // Go emits EVEX.z for a memory destination, but the ISA makes that + // encoding terminal #UD rather than a zeroing store. + return form.zeroing && form.memoryWrite +} + +// Block transfers read the inserted/selected chunk and, for insertion, only +// the retained base bytes. This describes numerical inputs, not memory extent. +func (form x86InsertExtractForm) blockInputBytes() (source, base uint64) { + chunk := x86CallerVectorByteMask(int64(form.spec.chunk)) + selected := chunk << uint(form.offset) + if form.spec.insert { + return chunk, x86CallerVectorByteMask(int64(form.width)) &^ selected + } + return selected, 0 +} + +func x86InsertExtractProperties(ins Instr, goarch string) (x86InsertExtractForm, bool, error) { + op, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86InsertExtractSpecs[Op(op)] + if !recognized { + return x86InsertExtractForm{}, false, nil + } + f := x86InsertExtractForm{spec: spec, source: 1, base: -1, destination: len(ins.Args) - 1, mask: -1, memory: -1} + fail := func() (x86InsertExtractForm, bool, error) { + return f, true, fmt.Errorf("%s %s operands/suffix outside Go insert/extract grammar: %q", goarch, ins.Op, ins.Raw) + } + count := 3 + if spec.insert { + count, f.base = 4, 2 + if spec.legacy { + count = 3 + } + } + // Go's 386 frontend cannot represent a four-operand named insertion. + // A validated physical VINSERTPS has its independent mode32 register and + // encoding checks; a textual Raw annotation cannot create that identity. + physicalScalarInsert := spec.insert && spec.scalar && ins.x86Encoded + if goarch != "amd64" && goarch != "386" || spec.insert && !spec.legacy && goarch == "386" && !physicalScalarInsert || len(ins.Args) != count && len(ins.Args) != count+1 || + len(ins.Args) == count+1 && (spec.vex || spec.scalar) || dotted && (suffix != "Z" || spec.vex || spec.scalar) || + ins.Args[0].Kind != OpImm || ins.Args[0].ImmRaw != "" || ins.Args[0].ImmIsFloat { + return fail() + } + if len(ins.Args) == count+1 { + f.mask = count - 1 + if !amd64NonzeroKOperand(ins.Args[f.mask]) { + return fail() + } + } + if goarch == "386" && f.mask >= 0 { + return fail() + } + f.zeroing = dotted + if f.zeroing && f.mask < 0 { + return fail() + } + minimum := int64(0) + if !spec.insert && spec.vex { + minimum = -128 + } + if spec.scalar && !spec.insert && !spec.legacy { + index, valid := amd64VectorRegisterIndex(ins.Args[1].Reg, 16) + if valid && index < 16 { + minimum = -128 + } + } + maximum := int64(255) + if spec.legacy && !spec.insert { + maximum = 3 + } + if ins.Args[0].Imm < minimum || ins.Args[0].Imm > maximum { + return fail() + } + f.immediate = byte(ins.Args[0].Imm) + f.width, f.output = 16, spec.chunk + large := ins.Args[f.source] + if spec.insert { + large = ins.Args[f.base] + } + if !spec.scalar { + f.width = amd64VectorByteWidth(large.Reg) + if large.Kind != OpReg || !(f.width == 32 && spec.allowY || f.width == 64 && spec.allowZ) { + return fail() + } + f.offset = (int(f.immediate) & (f.width/spec.chunk - 1)) * spec.chunk + } else { + f.offset = int(f.immediate&3) * 4 + } + if spec.insert { + f.output = f.width + } + highMode32 := Reg("") + register := func(arg Operand, width int) bool { + index, valid := amd64VectorRegisterIndex(arg.Reg, width) + limit := 32 + if spec.vex || spec.legacy { + limit = 16 + } + if goarch == "386" && spec.legacy { + limit = 8 + } + if arg.Kind != OpReg || !valid || index >= limit { + return false + } + if goarch == "386" && index >= 8 { + highMode32 = arg.Reg + } + return true + } + if !register(large, f.width) { + return fail() + } + if spec.insert && !register(ins.Args[f.destination], f.output) { + return fail() + } + transfer := ins.Args[f.destination] + if spec.insert { + transfer = ins.Args[f.source] + } + if isAMD64MemoryOperand(transfer) { + f.memory, f.memoryWrite = f.destination, !spec.insert + if spec.insert { + f.memory = f.source + } + if transfer.Kind == OpSym { + _, absolute := parseInt(strings.TrimSpace(transfer.Sym)) + if strings.HasPrefix(strings.TrimSpace(transfer.Sym), "$") || !strings.HasSuffix(transfer.Sym, "(SB)") && absolute != nil { + return fail() + } + } + if transfer.Kind == OpMem && (!x86MemoryRegistersValidForArch(transfer.Mem, goarch) || + transfer.Mem.Index != "" && (transfer.Mem.Index == SP || transfer.Mem.Scale != 1 && transfer.Mem.Scale != 2 && transfer.Mem.Scale != 4 && transfer.Mem.Scale != 8) || + transfer.Mem.Off != int64(int32(transfer.Mem.Off)) && (goarch != "386" || transfer.Mem.Off != int64(uint32(transfer.Mem.Off)))) { + return fail() + } + } else if !spec.insert && spec.scalar { + if transfer.Kind != OpReg || !isX86YrlRegisterForArch(transfer.Reg, goarch) { + return fail() + } + } else { + width := spec.chunk + if spec.scalar { + width = 16 + } + if !register(transfer, width) { + return fail() + } + } + if highMode32 != "" { + // Go can encode named VEX/EVEX high SIMD registers in mode32. + // Its syntax/object acceptance does not supply physical register + // identities which the independent mode32 raw decoder rejects. + return f, true, fmt.Errorf("%w: 386 insert/extract register %s has no proved physical mode32 binding", ErrProbeNeedsContext, highMode32) + } + return f, true, nil +} + +func (c *amd64Ctx) bindInsertExtractForm(op Op, ins Instr) (Instr, x86InsertExtractForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86InsertExtractSpecs[Op(base)]; !known { + return ins, x86InsertExtractForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(op), string(ins.Op)) { + return ins, x86InsertExtractForm{}, true, fmt.Errorf("insert/extract dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + f, known, err := x86InsertExtractProperties(ins, c.goarch) + if err == nil && !f.undefinedOpcode() && f.memory >= 0 && ins.Args[f.memory].Kind == OpFP { + err = c.validateFPVectorSpan(ins.Args[f.memory].FPOffset, f.spec.chunk) + } + return ins, f, known, err +} diff --git a/amd64_insert_extract_forms_test.go b/amd64_insert_extract_forms_test.go new file mode 100644 index 000000000..e8d1c504f --- /dev/null +++ b/amd64_insert_extract_forms_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +// Independent full opcode/ytab axes, not a rendering of the production map. +func x86InsertExtractFormats() []string { + var rows []string + for _, action := range []string{"EXTRACT", "INSERT"} { + for _, kind := range []string{"F", "I"} { + for _, size := range []string{"128", "32X4", "64X2", "32X8", "64X4"} { + op, small := "V"+action+kind+size, "X" + widths := []string{"Y", "Z"} + if size == "128" { + widths = []string{"Y"} + } + if strings.HasSuffix(size, "X8") || strings.HasSuffix(size, "X4") && strings.HasPrefix(size, "64") { + small, widths = "Y", []string{"Z"} + } + for _, wide := range widths { + for _, memory := range []bool{false, true} { + for _, masking := range []string{"", "merge", "zero"} { + if size == "128" && masking != "" || action == "EXTRACT" && memory && masking == "zero" { + continue + } + mnemonic, mask := op, "" + if masking != "" { + mask = ",K1" + } + if masking == "zero" { + mnemonic += ".Z" + } + transfer := small + "0" + if memory { + transfer = "(BX)" + } + if action == "INSERT" { + rows = append(rows, fmt.Sprintf("%s $255,%s,%s1%s,%s2", mnemonic, transfer, wide, mask, wide)) + } else { + rows = append(rows, fmt.Sprintf("%s $255,%s1%s,%s", mnemonic, wide, mask, transfer)) + } + } + } + } + } + } + } + return append(rows, "VEXTRACTPS $255,X1,AX", "VEXTRACTPS $255,X1,(BX)", "VINSERTPS $255,X0,X1,X2", "VINSERTPS $255,(BX),X1,X2", + "EXTRACTPS $3,X1,AX", "EXTRACTPS $3,X1,(BX)", "INSERTPS $255,X0,X1", "INSERTPS $255,(BX),X1") +} + +func TestX86InsertExtractEveryGoOpcodeAttribute(t *testing.T) { + if len(x86InsertExtractSpecs) != 24 { + t.Fatal("incomplete family") + } + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + expected := map[string]string{ + "VEXTRACTF128": "vexW0, 0x19", "VEXTRACTI128": "vexW0, 0x39", + "VEXTRACTF32X4": "evexW0, evexN16", "VEXTRACTI32X4": "evexW0, evexN16", + "VEXTRACTF64X2": "evexW1, evexN16", "VEXTRACTI64X2": "evexW1, evexN16", + "VEXTRACTF32X8": "evexW0, evexN32", "VEXTRACTI32X8": "evexW0, evexN32", + "VEXTRACTF64X4": "evexW1, evexN32", "VEXTRACTI64X4": "evexW1, evexN32", + "VINSERTF128": "vexW0, 0x18", "VINSERTI128": "vexW0, 0x38", + "VINSERTF32X4": "evexW0, evexN16", "VINSERTI32X4": "evexW0, evexN16", + "VINSERTF64X2": "evexW1, evexN16", "VINSERTI64X2": "evexW1, evexN16", + "VINSERTF32X8": "evexW0, evexN32", "VINSERTI32X8": "evexW0, evexN32", + "VINSERTF64X4": "evexW1, evexN32", "VINSERTI64X4": "evexW1, evexN32", + "VEXTRACTPS": "evexW0, evexN4", "VINSERTPS": "evexW0, evexN4", + } + var observed []string + for _, row := range strings.Split(string(data), "{as: A") { + op := strings.SplitN(row, ",", 2)[0] + if !strings.HasPrefix(op, "VEXTRACT") && !strings.HasPrefix(op, "VINSERT") { + continue + } + observed = append(observed, op) + body := strings.SplitN(row, "\n\t}},", 2)[0] + spec, exists := x86InsertExtractSpecs[Op(op)] + if !exists { + t.Fatalf("unregistered actual Go opcode: %s", op) + } + if strings.Contains(body, "Bcst") || strings.Contains(body, "Sae") || + strings.Contains(body, "evexZeroingEnabled") != (!spec.scalar && !spec.vex) { + t.Fatalf("per-opcode flags changed: %s", body) + } + if spec.vex && strings.Contains(body, "evex") || !spec.vex && !strings.Contains(body, "evex") { + t.Fatalf("encoding identity changed: %s", body) + } + if attribute, exists := expected[op]; !exists || !strings.Contains(body, attribute) || + !strings.Contains(body, "0F3A") || spec.allowY != strings.Contains(body, "256") || spec.allowZ != strings.Contains(body, "512") { + t.Fatalf("per-opcode W/tuple/native width attributes changed: %s", body) + } + } + sort.Strings(observed) + if len(observed) != 22 { + t.Fatalf("actual Go family: %v", observed) + } + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{AEXTRACTPS, yextractps, Pq, opBytes{0x3a, 0x17, 0}}", + "{AINSERTPS, yxshuf, Pq, opBytes{0x3a, 0x21, 0}}", + "var yextractps = []ytab{\n\t{Zibr_m, 2, argList{Yu2, Yxr, Yml}}", + "var yxshuf = []ytab{\n\t{Zibm_r, 2, argList{Yu8, Yxm, Yxr}}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatalf("actual Go legacy PS row changed: %s", row) + } + } +} + +func TestX86InsertExtractFormatsAndFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + for index, row := range x86InsertExtractFormats() { + if arch == "386" && (strings.HasPrefix(row, "VINSERT") || strings.Contains(row, "K1")) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, known, err := x86InsertExtractProperties(file.Funcs[0].Instrs[1], arch) + if !known || err != nil { + t.Fatalf("%s/%s: %v", arch, row, err) + } + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + name := fmt.Sprintf("transfer%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + } + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + t.Run(triple, func(t *testing.T) { + // Preserve every original uninitialized row as Go/public-Context + // and emitter-object coverage. Complete initialized programs have + // separate public object and executed-byte owners. + file, signatures := x86UnboundVoidFormatSources(t, source.String(), arch, triple) + ir := x86InstructionFormatObjectUnit(t, file, signatures, arch, triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerInsertExtractFamily(ins.Op, ins) + }) + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + }) + } + } +} + +func TestX86InsertExtractDispatchIdentityAndGrammarBeforeEmission(t *testing.T) { + for _, row := range x86InsertExtractFormats() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", ins.Op, Op(strings.ToLower(string(ins.Op))), "UNKNOWN", "VEXTRACTPS.Z"} { + copy := ins + copy.Op = identity + original := copy + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerInsertExtractFamily(ins.Op, copy) + failure := identity == "UNKNOWN" || identity == "VEXTRACTPS.Z" + if !known || term || (err != nil) != failure || failure && output.String() != before || !reflect.DeepEqual(copy, original) { + t.Fatalf("%s/%s: %t/%t/%v", row, identity, known, term, err) + } + } + } + for _, row := range []string{ + "VEXTRACTF128 $256,Y0,X1", "VEXTRACTF128 $-129,Y0,X1", "VEXTRACTF128 $0,Y16,X1", + "VEXTRACTF64X4 $-1,Z0,Y1", "VEXTRACTF32X4.Z $0,Z0,X1", "VEXTRACTI32X4 $0,Z0,K0,X1", + "VEXTRACTPS $-1,X31,AX", "VEXTRACTPS $0,X0,X1", "VINSERTPS $-1,X0,X1,X2", + "VINSERTI128 $0,X0,Y0,Y16", "VINSERTF32X8.BCST $0,(AX),Z0,Z1", + "VINSERTF64X2.RN_SAE $0,X0,Z0,Z1", "VINSERTI64X4.Z $0,Y0,Z0,Z1", + "VINSERTPS $0,(AX)(SP*2),X0,X1", "VINSERTPS $0,2147483648(AX),X0,X1", + "VEXTRACTF32X4 $0,Z0,$data(SB)", + "VEXTRACTF128 $selector,Y0,X1", "VINSERTPS $1.5,X0,X1,X2", + "INSERTPS $selector,X0,X1", "EXTRACTPS $1.5,X0,AX", "EXTRACTPS $4,X0,AX", + "INSERTPS $0,X16,X0", "EXTRACTPS $0,X16,AX", "INSERTPS.Z $0,X0,X1", + } { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + // Go 1.20 emitted the reserved EVEX.z=1/aaa=0 combination for + // this unchanged negative. Current Go rejects it. Syntax acceptance + // cannot authorize that physical invalid-opcode form. + reservedZeroMask := row == "VEXTRACTF32X4.Z $0,Z0,X1" || row == "VINSERTI64X4.Z $0,Y0,Z0,Z1" + if reservedZeroMask { + x86InsertExtractReservedZeroMaskGoObservation(t, source) + } else { + goAssemblerOracleOutput(t, "amd64", source, "", false) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, _, err := ctx.lowerInsertExtractFamily(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]) + if !known || err == nil || output.String() != before { + t.Fatalf("malformed source emitted IR: %s: %v", row, err) + } + } +} + +func TestX86InsertExtractCompleteTypedFPSpanBeforeEmission(t *testing.T) { + for _, row := range x86InsertExtractFormats() { + if !strings.Contains(row, "(BX)") { + continue + } + row = strings.Replace(row, "(BX)", "a+0(FP)", 1) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-32\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86InsertExtractProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + width := form.spec.chunk + if form.spec.scalar && !form.spec.insert { + // The PS writer now consumes the full source-byte proof too. + // Keep this test about complete spans, not unknown X input data. + file.Funcs[0].Instrs = append([]Instr{ + file.Funcs[0].Instrs[0], x86ScalarLaneInstruction(t, "PXOR X1,X1"), + }, file.Funcs[0].Instrs[1:]...) + } + for _, missing := range []int{-1, 0, width - 1} { + sig := FuncSig{Name: "f", Ret: Void} + for index := 0; index < width; index++ { + if index != missing { + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(index), Type: I8, Index: len(sig.Args), Field: -1}) + sig.Args = append(sig.Args, I8) + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + known, term, err := ctx.lowerInsertExtractFamily(ins.Op, ins) + // Native E6NF requires a physical contiguous destination. Typed + // individual ABI0 shadow cells do not supply it, even if every + // slot is bound. This remains Context, not fabricated memory. + failure := missing >= 0 || form.memoryWrite && form.mask >= 0 + if !known || term || (err != nil) != failure || failure && output.String() != before { + t.Fatalf("%s missing FP byte %d: %t/%t/%v", row, missing, known, term, err) + } + } + } +} diff --git a/amd64_insert_extract_go_oracle_test.go b/amd64_insert_extract_go_oracle_test.go new file mode 100644 index 000000000..a7fd6d38b --- /dev/null +++ b/amd64_insert_extract_go_oracle_test.go @@ -0,0 +1,274 @@ +package plan9asm + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func x86InsertExtractGoListing(t *testing.T, arch, source string) []byte { + t.Helper() + dir := t.TempDir() + asm, object := filepath.Join(dir, "original.s"), filepath.Join(dir, "original.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "oracle", "-o", object, asm) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH="+arch) + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), true, arch); failure != nil { + t.Fatalf("actual original assembler listing: %v\n%s", failure, output) + } + return output +} + +// Parse only concrete byte columns, not assembly mnemonic text. Go's listing +// starts its columns with 0xNNNN; llvm-objdump uses NNNN:. Both are independent +// physical object observations, rather than a rendering of the typed table. +func x86InsertExtractListedBytes(line string) []byte { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") && !strings.HasSuffix(fields[0], ":") { + return nil + } + if strings.HasPrefix(fields[0], "0x") { + _, code, err := goAssemblyListingByteRow(line) + if err != nil { + return nil + } + return code + } + var result []byte + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := hex.DecodeString(field) + if err != nil { + break + } + result = append(result, value[0]) + } + return result +} + +func TestX86InsertExtract386HighRegistersKeepPhysicalContext(t *testing.T) { + const source = "TEXT f(SB),4,$0-0\nVEXTRACTF128 $1,Y8,X9\nVEXTRACTPS $3,X8,AX\nRET\n" + listing := x86InsertExtractGoListing(t, "386", source) + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + if !bytes.Equal(code, []byte{0xc4, 0x43, 0x7d, 0x19, 0xc1, 1, 0xc4, 0x63, 0x79, 0x17, 0xc0, 3, 0xc3}) { + t.Fatalf("actual Go named mode32 bytes changed: %x", code) + } + for _, op := range []Op{"VEXTRACTF128", "VEXTRACTPS"} { + decode := decodedX86VEXTRACT128Instruction + if op == "VEXTRACTPS" { + decode = decodedX86PackedScalarExtractInstruction + } + ins, size, recognized, err := decode(code, 64) + if !recognized || err != nil || ins.Op != op || ins.Args[1].Reg != map[Op]Reg{"VEXTRACTF128": "Y8", "VEXTRACTPS": "X8"}[op] { + t.Fatalf("actual Go high-register bytes %x lost source identity: %+v/%v", code, ins, err) + } + if _, _, recognized, err := decode(code[:size], 32); !recognized || err == nil { + t.Fatal("mode32 decoder invented high physical SIMD registers") + } + t.Logf("actual Go accepted %s bytes %x; mode32 physical binding remains Context", op, code[:size]) + code = code[size:] + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": {Ret: Void}}}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go's mode32 accepted bytes acquired ordinary physical meaning: %v", err) + } +} + +func TestX86InsertExtractNativeMaskedStorePhysicalObjects(t *testing.T) { + llc, objdump := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || objdump == "" { + t.Fatal("LLVM22 llc and llvm-objdump are required") + } + var source strings.Builder + sigs := map[string]FuncSig{} + want := map[string]int{} + count := 0 + for _, row := range x86InsertExtractFormats() { + if !strings.HasPrefix(row, "VEXTRACT") || !strings.Contains(row, "K1,(BX)") { + continue + } + listing := x86InsertExtractGoListing(t, "amd64", "TEXT f(SB),4,$0-0\n"+strings.Replace(row, "1,K1", "0,K1", 1)+"\nRET\n") + matched := false + for _, line := range strings.Split(string(listing), "\n") { + code := x86InsertExtractListedBytes(line) + ins, _, recognized, err := decodedX86VEXTRACT128Instruction(code, 64) + if recognized { + if err != nil || ins.Args[0].Imm != 255 || ins.Args[2].Reg != "K1" || ins.Args[3].Kind != OpMem { + t.Fatalf("Go masked store identity: %+v/%v", ins, err) + } + matched = true + want[string(ins.Op)+":"+string(ins.Args[1].Reg)]++ + } + } + if !matched { + t.Fatalf("Go listing has no concrete E6NF bytes:\n%s", listing) + } + name := fmt.Sprintf("store%d", count) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-16\nMOVQ destination+0(FP),BX\nMOVQ mask+8(FP),AX\nKMOVQ AX,K1\nVPXORD Z1,Z1,Z1\n%s\nRET\n", name, row) + sigs[name] = FuncSig{Ret: Void, Args: []LLVMType{Ptr, I64}, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}}} + count++ + } + if count != 12 { + t.Fatalf("incomplete EVEX masked memory width/opcode family: %d", count) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if strings.Count(ir, "~{zmm0},~{k1},~{memory}") != count || strings.Contains(ir, "target-features") { + t.Fatal("native instruction bridge lost explicit private clobbers or enabled global ISA") + } + dir := t.TempDir() + input, object := filepath.Join(dir, "store.ll"), filepath.Join(dir, "store.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, llc, "-O2", "-mtriple="+triple, "-filetype=obj", input, "-o", object) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("LLVM22 native E6NF object: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + command = exec.CommandContext(ctx, objdump, "-d", object) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("LLVM22 native E6NF disassembly: %v\n%s", err, output) + } + observed := 0 + got := map[string]int{} + for _, line := range strings.Split(string(output), "\n") { + code := x86InsertExtractListedBytes(line) + ins, size, recognized, err := decodedX86VEXTRACT128Instruction(code, 64) + if !recognized { + continue + } + if err != nil || size != len(code) || ins.Args[0].Imm != 255 || ins.Args[1].Reg != "Y0" && ins.Args[1].Reg != "Z0" || ins.Args[2].Reg != "K1" || ins.Args[3].Kind != OpMem { + t.Fatalf("LLVM native write-only instruction identity %s: %+v/%v", line, ins, err) + } + observed++ + got[string(ins.Op)+":"+string(ins.Args[1].Reg)]++ + } + if observed != count || !reflect.DeepEqual(got, want) { + t.Fatalf("%s: native masked stores observed %d want %d\n%s", triple, observed, count, output) + } + } +} + +func x86InsertExtractReservedZeroMaskGoObservation(t *testing.T, source string, expectedCode ...byte) { + t.Helper() + dir := t.TempDir() + asm, object := filepath.Join(dir, "reserved.s"), filepath.Join(dir, "reserved.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-o", object, asm) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), err == nil, "amd64"); failure != nil { + t.Fatalf("reserved zero-mask source observation: %v\n%s", failure, output) + } + if err != nil { + return // actual concrete source rejection; never an unknown tool failure + } + code := []byte{0x62, 0xf3, 0x7d, 0xc8, 0x19, 0xc1, 0} + decode := decodedX86VEXTRACT128Instruction + if strings.Contains(source, "VINSERT") { + code = []byte{0x62, 0xf3, 0xfd, 0xc8, 0x3a, 0xc8, 0} + decode = decodedX86EVEXPackedInsertInstruction + } + if len(expectedCode) != 0 { + code = expectedCode + } + var listing strings.Builder + for _, value := range code { + fmt.Fprintf(&listing, "%02x ", value) + } + if !bytes.Contains(output, []byte(strings.TrimSpace(listing.String()))) { + t.Fatalf("accepted old Go reserved bytes changed:\n%s", output) + } + if _, _, recognized, err := decode(code, 64); !recognized || err == nil { + t.Fatal("reserved EVEX.z with aaa=0 acquired valid physical meaning") + } +} + +func TestCrossLinuxRuntimeMatrixX86InsertExtractOriginalGoPSOracle(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package oracle\n") + checks.WriteString(`package oracle +import ("encoding/binary";"testing") +func TestOriginalPSBits(t *testing.T) { + a:=[4]uint32{0x7f812345,0x80000000,0xffc45678,0x01234567} + b:=[4]uint32{0x89abcdef,0x7fc11111,0x7f800000,1} +`) + for index, immediate := range []int{0, 16, 32, 48, 64, 128, 192, 1, 2, 4, 8, 15, 255} { + name := fmt.Sprintf("insert%d", index) + fmt.Fprintf(&declarations, "func %s(out,a,b *[4]uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVUPS (BX),X0\nMOVUPS (CX),X1\nINSERTPS $%d,X0,X1\nMOVUPS X1,(AX)\nRET\n", name, immediate) + fmt.Fprintf(&checks, "{var out [4]uint32;want:=b;want[%d]=a[%d];for lane:=0;lane<4;lane++{if %d&(1<>4&3, immediate>>6&3, immediate, name, name+" got=%x want=%x") + } + for lane := 0; lane < 4; lane++ { + name := fmt.Sprintf("extract%d", lane) + fmt.Fprintf(&declarations, "func %s(out *[8]byte,a *[4]uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-16\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVUPS (BX),X0\nEXTRACTPS $%d,X0,CX\nMOVQ CX,(AX)\nRET\n", name, lane) + fmt.Fprintf(&checks, "{var out [8]byte;%s(&out,&a);if binary.LittleEndian.Uint64(out[:])!=uint64(a[%d]){t.Fatalf(%q,out)}}\n", name, lane, name+" zero-extension got=%x") + } + declarations.WriteString("func fpFields(a,b,c,d byte) uint32\nfunc fpStores(a,b,c,d byte) uint32\n") + // ABI0 aligns the result area to the AMD64 stack word independently of + // the four byte parameters; the actual Go wrapper reads ret+8(FP). + assembly.WriteString("TEXT ·fpFields(SB),4,$0-12\nPXOR X0,X0\nINSERTPS $0,a+0(FP),X0\nEXTRACTPS $0,X0,ret+8(FP)\nRET\n" + + "TEXT ·fpStores(SB),4,$0-12\nMOVL $0x87654321,CX\nMOVD CX,X1\nEXTRACTPS $0,X1,a+0(FP)\nMOVBLZX a+0(FP),AX\nMOVBLZX b+1(FP),BX\nSHLL $8,BX\nORL BX,AX\nMOVBLZX c+2(FP),BX\nSHLL $16,BX\nORL BX,AX\nMOVBLZX d+3(FP),BX\nSHLL $24,BX\nORL BX,AX\nMOVL AX,ret+8(FP)\nRET\n") + checks.WriteString(`for i:=0;i<256;i++ { + a,b,c,d:=byte(i),byte(i*37),byte(i*59),byte(i*71) + want:=uint32(a)|uint32(b)<<8|uint32(c)<<16|uint32(d)<<24 + if got:=fpFields(a,b,c,d);got!=want{t.Fatalf("original Go FP fields got=%x want=%x",got,want)} + if got:=fpStores(a,b,c,d);got!=0x87654321{t.Fatalf("original Go FP stores got=%x",got)} +} +`) + checks.WriteString("}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + dir := t.TempDir() + for file, contents := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": checks.String()} { + if err := os.WriteFile(filepath.Join(dir, file), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=asmdecl", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go PS original bit-transfer oracle: %v\n%s", err, output) + } +} diff --git a/amd64_insert_extract_native_runtime_linux_amd64_test.go b/amd64_insert_extract_native_runtime_linux_amd64_test.go new file mode 100644 index 000000000..28a564f5c --- /dev/null +++ b/amd64_insert_extract_native_runtime_linux_amd64_test.go @@ -0,0 +1,9 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86InsertExtractMaskedNativeMemory(t *testing.T) { + x86InsertExtractMaskedMemoryRuntime(t) +} diff --git a/amd64_insert_extract_native_runtime_test.go b/amd64_insert_extract_native_runtime_test.go new file mode 100644 index 000000000..433ccb756 --- /dev/null +++ b/amd64_insert_extract_native_runtime_test.go @@ -0,0 +1,303 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// This is exactly the subset deliberately excluded by the portable byte +// oracle: E6NF masked memory destinations execute a physical native opcode. +func x86InsertExtractMaskedMemoryRows() []string { + var rows []string + for _, row := range x86InsertExtractFormats() { + if strings.HasPrefix(row, "VEXTRACT") && strings.Contains(row, "(BX)") && strings.Contains(row, "K1") { + rows = append(rows, row) + } + } + return rows +} + +func x86InsertExtractMaskedMemoryAssembly() (string, map[string]FuncSig) { + var assembly strings.Builder + signatures := map[string]FuncSig{} + for index, row := range x86InsertExtractMaskedMemoryRows() { + name := fmt.Sprintf("masked%d", index) + row = strings.Replace(row, "(BX)", "(AX)", 1) + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),R9\nKMOVQ R9,K1\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nSTC\n%s\nSETCS 64(AX)\nRET\n", name, row) + signatures[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + } + return assembly.String(), signatures +} + +func x86InsertExtractMaskedMemoryShape(row string) (chunk, width, lane int) { + chunk, width, lane = 16, 32, 4 + if strings.Contains(row, "32X8") || strings.Contains(row, "64X4") { + chunk = 32 + } + if strings.Contains(row, ",Z") { + width = 64 + } + if strings.Contains(row, "64X") { + lane = 8 + } + return +} + +func TestX86InsertExtractMaskedMemoryRequiredInventory(t *testing.T) { + rows := x86InsertExtractMaskedMemoryRows() + if len(rows) != 12 || len(x86InsertExtractFormats()) != 148 { + t.Fatalf("changed complete/native fixture denominator: %d/%d", len(x86InsertExtractFormats()), len(rows)) + } + file, err := parser.ParseFile(token.NewFileSet(), "amd64_insert_extract_native_runtime_linux_amd64_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + var body *ast.BlockStmt + for _, declaration := range file.Decls { + if fn, ok := declaration.(*ast.FuncDecl); ok && fn.Name.Name == "TestCrossLinuxRuntimeMatrixX86InsertExtractMaskedNativeMemory" { + body = fn.Body + } + } + if body == nil || len(body.List) != 1 { + t.Fatal("all masked-memory extract cases require the native execution consumer") + } + statement, ok := body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("native extract counterpart cannot return or skip") + } + call, ok := statement.X.(*ast.CallExpr) + if !ok || len(call.Args) != 1 { + t.Fatal("native extract counterpart does not consume its full fixture") + } + callee, ok := call.Fun.(*ast.Ident) + if !ok || callee.Name != "x86InsertExtractMaskedMemoryRuntime" { + t.Fatal("native extract counterpart substituted another consumer") + } + assembly, signatures := x86InsertExtractMaskedMemoryAssembly() + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + parsed, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + ir, err := Translate(parsed, Options{Goarch: "amd64", TargetTriple: triple, Sigs: signatures}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "masked-extract.ll", "masked-extract.o", ir) + } + // The complete original Go oracle must compile even on a portable host; + // only its required native counterpart may execute these AVX512 bytes. + dir := x86InsertExtractMaskedOriginalPackage(t) + command := exec.Command("go", "test", "-p=2", "-c", "-o", filepath.Join(dir, "original"), ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + x86FloatCompareCaptureCommand(t, command) + t.Log("148 original formats preserved: all 12 excluded native memory-mask cases now have stock Go objects, 3 LLVM target objects and a mandatory Linux AMD64 runtime counterpart (not executed here)") +} + +// Intel Vol2 KMOVQ r64,k requires AVX512BW; the extract forms additionally +// require F/DQ/VL. Checking F alone cannot authorize this existing bridge. +const x86InsertExtractNativePreflightC = ` +#undef main +__attribute__((target("avx512f,avx512dq,avx512bw,avx512vl"))) +static int original_extract(void) { + uint8_t input[64],output[34]; + for (unsigned i=0;i<64;i++) input[i]=(uint8_t)(i*13+0x27); + for (unsigned i=0;i<34;i++) output[i]=0xc3; + uint64_t mask=1; + __asm__ volatile("vmovdqu64 (%0),%%zmm0; kmovq %1,%%k1; " + "vextracti32x8 $1,%%zmm0,(%2)%{%%k1%}" + :: "r"(input),"r"(mask),"r"(output+1) : "zmm0","k1","memory"); + for (unsigned i=0;i<34;i++) { + uint8_t want=i>=1 && i<=4 ? input[31+i] : 0xc3; + if (output[i]!=want) return 0; + } + return 1; +} +static int extract_required_state(unsigned dq,unsigned bw) { return dq && bw; } +int main(void) { + int failure=compare_preflight_main(); + if (failure) return failure; + unsigned a,b,c,d,dq=0,bw=0; + if (__get_cpuid_count(7,0,&a,&b,&c,&d)) { dq=(b>>17)&1; bw=(b>>30)&1; } + printf("masked extract AVX512DQ=%u AVX512BW=%u\n",dq,bw); + fflush(stdout); + if (!extract_required_state(dq,bw)) return 81; + return original_extract() ? 0 : 82; +} +` + +func x86InsertExtractMaskedMemoryRuntime(t *testing.T) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + dir := t.TempDir() + source, binary := filepath.Join(dir, "extract-preflight.c"), filepath.Join(dir, "extract-preflight") + program := "#define REQUIRED_AXIS 3\n#define main compare_preflight_main\n" + x86FloatCompareOpcodePreflightC + x86InsertExtractNativePreflightC + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, binary) + t.Logf("original native extract preflight: %s", execution.run(t, binary)) + original := x86InsertExtractMaskedOriginalPackage(t) + command := exec.Command("go", "test", "-p=2", "-count=1", "-run", "^TestOriginalMaskedExtract$", "-v", ".") + command.Dir = original + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + t.Logf("original Go native masked extract: %s", x86FloatCompareCaptureCommand(t, command)) + assembly, signatures := x86InsertExtractMaskedMemoryAssembly() + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: signatures}) + if err != nil { + t.Fatal(err) + } + var declarations, calls strings.Builder + for index, row := range x86InsertExtractMaskedMemoryRows() { + chunk, width, lane := x86InsertExtractMaskedMemoryShape(row) + fmt.Fprintf(&declarations, "extern void masked%d(void*,const void*,const void*,uint64_t);\n", index) + fmt.Fprintf(&calls, " if (check(masked%d,%d,%d,%d)) return %d;\n", index, chunk, width, lane, index+1) + } + execution.compileAndRunLLVM(t, "masked-native-extract", ir, x86InsertExtractMaskedMemoryC+declarations.String()+"int main(void) {\n"+calls.String()+" return 0;\n}\n", "-O2") + t.Log("all 12 native masked-memory extract programs x40 byte/MXCSR/CF/input-output-canary cases executed by original Go and translated LLVM") +} + +func TestX86InsertExtractNativeOpcodePreflightContract(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, binary := filepath.Join(dir, "extract-capabilities.c"), filepath.Join(dir, "extract-capabilities") + // Compile the actual original-opcode preflight, but execute only its exact + // extra capability predicate here. Synthetic bits never permit ISA execution. + program := "#define REQUIRED_AXIS 3\n#define main compare_preflight_main\n" + x86FloatCompareOpcodePreflightC + + strings.Replace(x86InsertExtractNativePreflightC, "int main(void)", "int actual_extract_main(void)", 1) + ` +int main(void) { + if (!extract_required_state(1,1) || extract_required_state(0,1) || + extract_required_state(1,0) || extract_required_state(0,0)) return 1; + return 0; +} +` + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, binary) + execution.run(t, binary) +} + +func x86InsertExtractMaskedOriginalPackage(t *testing.T) string { + t.Helper() + dir := t.TempDir() + assembly, _ := x86InsertExtractMaskedMemoryAssembly() + assembly = strings.ReplaceAll(assembly, "TEXT masked", "TEXT ·masked") + assembly += "TEXT ·setCSR(SB),4,$0-4\nLDMXCSR csr+0(FP)\nRET\nTEXT ·getCSR(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\n" + var declarations, cases strings.Builder + for index, row := range x86InsertExtractMaskedMemoryRows() { + chunk, width, lane := x86InsertExtractMaskedMemoryShape(row) + fmt.Fprintf(&declarations, "func masked%d(out *[65]byte,a,b *[64]byte,mask uint64)\n", index) + fmt.Fprintf(&cases, " {masked%d,%d,%d,%d},\n", index, chunk, width, lane) + } + for name, contents := range map[string]string{ + "go.mod": "module nativemaskedextract\n\ngo 1.20\n", "original_amd64.s": assembly, + "original_test.go": "package original\nimport (\"testing\"; \"runtime\")\n" + declarations.String() + "func setCSR(csr uint32)\nfunc getCSR() uint32\n" + strings.Replace(x86InsertExtractMaskedMemoryGo, "REQUIRED_CASES", cases.String(), 1), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + return dir +} + +const x86InsertExtractMaskedMemoryGo = ` +func TestOriginalMaskedExtract(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + defer setCSR(getCSR()) + cases:=[]struct{fn func(*[65]byte,*[64]byte,*[64]byte,uint64); chunk,width,lane int}{ +REQUIRED_CASES + } + masks:=[]uint64{0,1,0xaaaaaaaaaaaaaaaa,0xffffffffffffffff,0xffff000000000000} + for index,fixture:=range cases { + for vector:=0;vector<8;vector++ { + for _,mask:=range masks { + var a,b,out [67]byte + for i:=range a { + a[i]=0xa7 + b[i]=0xb9 + out[i]=0xc3 + } + for i:=0;i<64;i++ { + a[i+1]=byte((i*31+vector*17)^0x81) + b[i+1]=byte((i*13+vector*29)^0x27) + } + aCopy,bCopy:=a,b + want:=out + offset:=(255&(fixture.width/fixture.chunk-1))*fixture.chunk + for i:=0;i +#include +#include +typedef void(*transfer)(void*,const void*,const void*,uint64_t); +static int check(transfer fn,unsigned chunk,unsigned width,unsigned laneBytes) { + const uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX,UINT64_C(0xffff000000000000)}; + for (unsigned vector=0;vector<8;vector++) { + for (unsigned m=0;m<5;m++) { + uint8_t as[67],bs[67],out[67],want[67],aCopy[67],bCopy[67]; + uint8_t *a=as+1,*b=bs+1; + memset(as,0xa7,67); memset(bs,0xb9,67); memset(out,0xc3,67); + for (unsigned i=0;i<64;i++) { + a[i]=(i*31+vector*17)^0x81; + b[i]=(i*13+vector*29)^0x27; + } + memcpy(aCopy,as,67); memcpy(bCopy,bs,67); memcpy(want,out,67); + unsigned offset=(255&(width/chunk-1))*chunk; + for (unsigned i=0;i +#include +extern uint32_t fields(uint8_t,uint8_t,uint8_t,uint8_t); +extern uint32_t stores(uint8_t,uint8_t,uint8_t,uint8_t); +int main(void) { + for(unsigned i=0;i<256;i++) { + uint8_t a=i,b=i*37,c=i*59,d=i*71; + uint32_t want=(uint32_t)a|((uint32_t)b<<8)|((uint32_t)c<<16)|((uint32_t)d<<24); + uint32_t got=fields(a,b,c,d); + if(got!=want){fprintf(stderr,"independent FP fields got=%x want=%x\n",got,want);return 1;} + got=stores(a,b,c,d); + if(got!=UINT32_C(0x87654321)){fprintf(stderr,"independent FP stores got=%x\n",got);return 2;} + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "insert-extract-independent-fp", triple, ir, reference, prefix) +} + +func x86InsertExtractNumericalRows() []string { + rows := x86InsertExtractFormats() + // Actual source/destination aliases plus every imm8 selector bit for the + // PS forms. Chunk immediates ignore their high bits, independently tested. + for _, row := range x86InsertExtractFormats() { + if strings.Contains(row, "(BX)") || strings.Contains(row, "K1") || strings.Contains(row, "PS ") { + continue + } + if strings.HasPrefix(row, "VINSERT") { + rows = append(rows, strings.TrimSuffix(row, "2")+"1", strings.TrimSuffix(row, "2")+"0") + } else { + rows = append(rows, strings.TrimSuffix(row, "0")+"1") + } + rows = append(rows, strings.Replace(row, "$255", "$0", 1)) + } + for _, imm := range []int{0, 16, 32, 48, 64, 128, 192, 1, 2, 4, 8, 15, 255} { + rows = append(rows, fmt.Sprintf("VINSERTPS $%d,X0,X1,X2", imm), fmt.Sprintf("VINSERTPS $%d,(BX),X1,X1", imm), fmt.Sprintf("VEXTRACTPS $%d,X1,R8", imm)) + rows = append(rows, fmt.Sprintf("INSERTPS $%d,X0,X1", imm), fmt.Sprintf("INSERTPS $%d,(BX),X1", imm)) + } + for lane := 0; lane < 4; lane++ { + rows = append(rows, fmt.Sprintf("EXTRACTPS $%d,X1,R8", lane)) + } + rows = append(rows, "INSERTPS $128,X1,X1", "INSERTPS $192,X1,X0", "INSERTPS $64,X0,X0") + return rows +} + +func x86InsertExtractRuntimeCheck(name string, parameters ...int) string { + var call strings.Builder + fmt.Fprintf(&call, "if(check(%s", name) + for _, parameter := range parameters { + fmt.Fprintf(&call, ",%d", parameter) + } + // Preserve the case identity in text; a process exit status must never + // wrap a large generated index to zero on Unix. + fmt.Fprintf(&call, ")) { fprintf(stderr, %q); return 1; }\n", name+" numerical/canary failure\n") + return call.String() +} + +func TestCrossLinuxRuntimeMatrixX86InsertExtractByteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + rows := x86InsertExtractNumericalRows() + for index, row := range rows { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + op := strings.Fields(row)[0] + insert, scalar, memory, masked := strings.Contains(op, "INSERT"), strings.HasSuffix(op, "PS"), strings.Contains(row, "(BX)"), strings.Contains(row, "K1") + if !insert && memory && masked { + continue // physical native E6NF objects are checked separately; AVX512 execution is not available here + } + chunk, width, laneBytes := 16, 32, 4 + if strings.Contains(op, "X8") || strings.Contains(op, "64X4") { + chunk = 32 + } + if strings.Contains(op, "64X") { + laneBytes = 8 + } + if strings.Contains(row, ",Z") { + width = 64 + } + if scalar { + chunk, width = 4, 16 + } + destination := ins.Args[len(ins.Args)-1] + store := "" + if destination.Kind == OpReg { + if scalar && !insert { + // AX is the output address, so choose a non-address GP result. + row = strings.TrimSuffix(row, string(destination.Reg)) + "R8" + store = "MOVQ R8,(AX)\n" + } else { + store = fmt.Sprintf("VMOVUPS %s,(AX)\n", x86CallerVectorRoot(destination.Reg)) + } + } else { + row = strings.Replace(row, "(BX)", "(AX)", 1) + } + name := fmt.Sprintf("transfer%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),R9\nKMOVQ R9,K1\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nSTC\n%s\n%sSETCS 64(AX)\nRET\n", name, row, store) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*,uint64_t);\n", name) + checks.WriteString(x86InsertExtractRuntimeCheck(name, + x86FloatAddSubBoolInt(insert), x86FloatAddSubBoolInt(scalar), chunk, width, laneBytes, int(ins.Args[0].Imm), + x86FloatAddSubBoolInt(memory), x86FloatAddSubBoolInt(masked), x86FloatAddSubBoolInt(strings.Contains(op, ".Z")), x86FloatAddSubBoolInt(destination.Kind == OpReg), x86FloatAddSubBoolInt(!strings.HasPrefix(op, "V")), + x86FloatAddSubBoolInt(ins.Args[1].Reg == "X1"), x86FloatAddSubBoolInt(!strings.HasPrefix(op, "V") && destination.Reg == "X0"))) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const reference = `#include +#include +#include +typedef void(*transfer)(void*,const void*,const void*,uint64_t); +__attribute__((noinline)) static int check(transfer fn,int insert,int scalar,int chunk,int width,int laneBytes,unsigned imm,int memory,int masked,int zero,int reg,int legacy,int sourceB,int baseA) { + const uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_C(0xffffffffffffffff),UINT64_C(0xffff000000000000)}; + for(unsigned vector=0;vector<8;vector++)for(unsigned m=0;m<5;m++) { + uint8_t as[67],bs[67],out[67],want[64],aCopy[67],bCopy[67];uint8_t *a=as+1,*b=bs+1; + memset(as,0xa7,67);memset(bs,0xb9,67);memset(out,0xc3,67); + for(unsigned i=0;i<64;i++){a[i]=(i*31+vector*17)^0x81;b[i]=(i*13+vector*29)^0x27;} + memcpy(aCopy,as,67);memcpy(bCopy,bs,67);memset(want,reg&&!(scalar&&!insert)?0:0xc3,64); + const uint8_t *base=baseA?a:b,*insertSource=sourceB?b:a; + if(legacy&&insert)memcpy(want,base,64); + unsigned offset=(imm&(width/chunk-1))*chunk; + if(insert) { + memcpy(want,base,width); + if(scalar){offset=((imm>>4)&3)*4;memcpy(want+offset,insertSource+(memory?0:((imm>>6)&3)*4),4); + for(unsigned lane=0;lane<4;lane++)if(imm&(1u< +static void transfer255(void) {} +static int check(void (*fn)(void), ...) { return 1; } +int main(void) { +` + x86InsertExtractRuntimeCheck("transfer255", make([]int, 13)...) + "return 0;\n}\n" + requireX86GeneratedOracleFailure(t, program, "transfer255 numerical/canary failure") +} diff --git a/amd64_insert_extract_sdk_source_test.go b/amd64_insert_extract_sdk_source_test.go new file mode 100644 index 000000000..c37f87a87 --- /dev/null +++ b/amd64_insert_extract_sdk_source_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + "time" +) + +func TestX86InsertExtractActualSDKTablesAreMandatory(t *testing.T) { + const rootName = "TestX86InsertExtractEveryGoOpcodeAttribute" + for _, name := range []string{"avx_optabs.go", "asm6.go"} { + t.Run(name, func(t *testing.T) { + // Execute the actual attribute root from this SDK's test binary. + // A partial fixture GOROOT supplies no replacement encoder evidence. + fixture := t.TempDir() + directory := filepath.Join(fixture, "src", "cmd", "internal", "obj", "x86") + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + if name == "asm6.go" { + original, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86", "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, "avx_optabs.go"), original, 0600); err != nil { + t.Fatal(err) + } + } + missing := filepath.Join(directory, name) + if _, err := os.ReadFile(missing); !os.IsNotExist(err) { + t.Fatalf("missing-table fixture is not absent: %v", err) + } + + stage, cancelStage := runtimeOracleStageContext(t) + defer cancelStage() + ctx, cancel := context.WithTimeout(stage, 30*time.Second) + defer cancel() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^"+rootName+"$", "-test.count=1", "-test.timeout=20s") + for _, item := range os.Environ() { + if !strings.HasPrefix(item, "GOROOT=") { + command.Env = append(command.Env, item) + } + } + command.Env = append(command.Env, "GOROOT="+fixture) + output, err := runtimeOracleCombinedOutput(ctx, command) + exit, plainExit := err.(*exec.ExitError) + if ctx.Err() != nil || !plainExit || exit.ProcessState == nil || + !exit.ProcessState.Exited() || exit.ExitCode() != 1 { + t.Fatalf("actual SDK-table root did not fail with plain exit 1: %v (context %v)\n%s", err, ctx.Err(), output) + } + + // Bind the complete failure to this one root, its real source file, + // and the exact missing table. Panics, cleanup errors, extra failures, + // unrelated diagnostics and a PASS from version-return are not evidence. + _, missingErr := os.ReadFile(missing) + if !os.IsNotExist(missingErr) { + t.Fatalf("missing-table fixture changed during the child: %v", missingErr) + } + if !x86InsertExtractMissingTableOutput(output, rootName, missingErr.Error(), testing.CoverMode() != "") { + t.Fatalf("unbound actual SDK-table failure (want %s):\n%s", missing, output) + } + t.Logf("actual %s child: plain exit 1, exact %s diagnostic\n%s", runtime.Version(), name, output) + }) + } +} + +func x86InsertExtractMissingTableOutput(output []byte, root, diagnostic string, covered bool) bool { + pattern := fmt.Sprintf(`\A--- FAIL: %s \([0-9]+\.[0-9]+s\)\n amd64_insert_extract_forms_test\.go:[0-9]+: %s\nFAIL\n`, + regexp.QuoteMeta(root), regexp.QuoteMeta(diagnostic)) + if covered { + // The child is this same instrumented test binary. Require Go's one + // coverage trailer only in that role; never strip arbitrary output. + // Go 1.20's coverage writer also prints this exact package prefix. + pattern += `(?:\tgithub\.com/xgo-dev/plan9asm\t)?coverage: (?:(?:[0-9]|[1-9][0-9])\.[0-9]|100\.0)% of statements\n` + } + return regexp.MustCompile(pattern + `\z`).Match(output) +} + +func TestX86InsertExtractMissingTableOutputGrammar(t *testing.T) { + const root, diagnostic = "TestActualTable", "open exact/asm6.go: missing table" + base := "--- FAIL: " + root + " (0.01s)\n amd64_insert_extract_forms_test.go:66: " + diagnostic + "\nFAIL\n" + const trailer = "coverage: 0.2% of statements\n" + for _, covered := range []bool{false, true} { + valid := base + if covered { + valid += trailer + } + if !x86InsertExtractMissingTableOutput([]byte(valid), root, diagnostic, covered) { + t.Fatalf("exact original failure rejected (coverage=%v)", covered) + } + for _, invalid := range []string{ + valid + "PASS\n", "panic: unrelated\n" + valid, valid + trailer, + strings.Replace(valid, "exact/asm6.go", "other/asm6.go", 1), + strings.Replace(valid, root, "TestAnotherRoot", 1), + } { + if x86InsertExtractMissingTableOutput([]byte(invalid), root, diagnostic, covered) { + t.Fatalf("unbound child failure accepted (coverage=%v): %q", covered, invalid) + } + } + } + if x86InsertExtractMissingTableOutput([]byte(base), root, diagnostic, true) || + x86InsertExtractMissingTableOutput([]byte(base+trailer), root, diagnostic, false) { + t.Fatal("child borrowed the wrong coverage instrumentation role") + } + if !x86InsertExtractMissingTableOutput([]byte(base+"\tgithub.com/xgo-dev/plan9asm\t"+trailer), root, diagnostic, true) || + x86InsertExtractMissingTableOutput([]byte(base+"\tother/package\t"+trailer), root, diagnostic, true) { + t.Fatal("historical coverage trailer lost its exact package identity") + } +} diff --git a/amd64_integer_lane_mask_test.go b/amd64_integer_lane_mask_test.go new file mode 100644 index 000000000..111b274dd --- /dev/null +++ b/amd64_integer_lane_mask_test.go @@ -0,0 +1,185 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Include narrow conversion results as well as whole X/Y/Z carriers. This +// independent helper domain is a superset of the current typed callers, not an +// inventory or execution certificate for the instructions which call it. +func x86IntegerLaneMaskShapes() []struct{ lanes, bits int } { + var shapes []struct{ lanes, bits int } + for _, bits := range []int{8, 16, 32, 64} { + for lanes := 1; lanes <= 64 && lanes*bits <= 512; lanes *= 2 { + shapes = append(shapes, struct{ lanes, bits int }{lanes, bits}) + } + } + return shapes +} + +func TestX86IntegerLaneMaskBoundedVectorIR(t *testing.T) { + for _, shape := range x86IntegerLaneMaskShapes() { + for _, zeroing := range []bool{false, true} { + name := fmt.Sprintf("%dxi%d/zeroing=%t", shape.lanes, shape.bits, zeroing) + t.Run(name, func(t *testing.T) { + var body strings.Builder + ctx := &amd64Ctx{b: &body} + result := amd64ApplyIntegerLaneMask(ctx, shape.lanes, shape.bits, + "%computed", "%unavailableOld", "%mask", zeroing) + ir := body.String() + if got := strings.Count(ir, " = "); got != 5 { + t.Fatalf("mask helper emitted %d instructions; want five independent of lane count:\n%s", got, ir) + } + for _, forbidden := range []string{"extractelement", "select i1 ", "bitcast i64", "lshr i64"} { + if strings.Contains(ir, forbidden) { + t.Fatalf("lane mask expanded scalar operations or used endian-dependent packing %q:\n%s", forbidden, ir) + } + } + for _, want := range []string{ + fmt.Sprintf("lshr <%d x i64>", shape.lanes), + fmt.Sprintf("trunc <%d x i64>", shape.lanes), + fmt.Sprintf("select <%d x i1>", shape.lanes), + } { + if !strings.Contains(ir, want) { + t.Fatalf("mask predicate lacks %q:\n%s", want, ir) + } + } + if zeroing && strings.Contains(ir, "%unavailableOld") { + t.Fatal("zeroing read the unavailable old destination") + } + if !zeroing && strings.Count(ir, "%unavailableOld") != 1 { + t.Fatal("merge must select the old vector exactly once") + } + if !strings.Contains(ir, result+" = select ") { + t.Fatal("helper did not return the selected vector") + } + }) + } + } +} + +func TestX86IntegerLaneMaskEmptyShape(t *testing.T) { + for _, lanes := range []int{-1, 0} { + for _, zeroing := range []bool{false, true} { + var body strings.Builder + ctx := &amd64Ctx{b: &body} + want := "%old" + if zeroing { + want = "zeroinitializer" + } + got := amd64ApplyIntegerLaneMask(ctx, lanes, 8, "%computed", "%old", "%mask", zeroing) + if got != want || body.Len() != 0 { + t.Fatalf("empty shape lanes=%d zeroing=%t emitted %q, %q", lanes, zeroing, got, body.String()) + } + } + } +} + +// Exercise the helper's emitted LLVM with an independent byte-selection C +// oracle. No native ISA attributes or scalar-mask bitcasts are needed. Physical +// x86 instruction behavior and active-lane memory faults retain separate owners. +func TestX86IntegerLaneMaskPortableSemantics(t *testing.T) { + execution := x86ScalarCountHostExecution(t) + var module, prototypes, checks strings.Builder + count := 0 + for _, shape := range x86IntegerLaneMaskShapes() { + for _, zeroing := range []bool{false, true} { + name := fmt.Sprintf("laneMask%d", count) + count++ + vec := fmt.Sprintf("<%d x i%d>", shape.lanes, shape.bits) + fmt.Fprintf(&module, "define void @%s(ptr %%out, ptr %%input, ptr %%prior, i64 %%mask) {\n", name) + fmt.Fprintf(&module, " %%computed = load %s, ptr %%input, align 1\n", vec) + if !zeroing { + fmt.Fprintf(&module, " %%old = load %s, ptr %%prior, align 1\n", vec) + } + ctx := &amd64Ctx{b: &module} + result := amd64ApplyIntegerLaneMask(ctx, shape.lanes, shape.bits, "%computed", "%old", "%mask", zeroing) + fmt.Fprintf(&module, " store %s %s, ptr %%out, align 1\n ret void\n}\n", vec, result) + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, " if (check(%s, %q, %d, %d, %t)) return 1;\n", + name, name, shape.lanes, shape.bits/8, zeroing) + } + } + if count != 44 { + t.Fatal("incomplete independent lane-mask shape/mode union", count) + } + driver := strings.NewReplacer("PROTOTYPES", prototypes.String(), "CHECKS", checks.String()).Replace(x86IntegerLaneMaskDriver) + ir := module.String() + for _, triple := range []string{"x86_64-unknown-linux-gnu", "i386-unknown-linux-gnu"} { + t.Run("baseline-object/"+triple, func(t *testing.T) { + x86ResultInitializationObject(t, execution.llc, triple, ir, "lane-mask-baseline") + }) + } + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, ir) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR-O2=%t/%s", optimized, level), func(t *testing.T) { + execution.compileAndRunLLVM(t, "integer-lane-mask", text, driver, level) + }) + } + } +} + +const x86IntegerLaneMaskDriver = ` +#include +#include +#include +#include +PROTOTYPES + +static int check(void (*fn)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t), + const char *name, unsigned lanes, unsigned bytes, bool zeroing) { + uint64_t masks[69] = {0, UINT64_MAX, UINT64_C(0xaaaaaaaaaaaaaaaa), UINT64_C(0x5555555555555555)}; + for (unsigned bit = 0; bit < 64; ++bit) { + masks[4+bit] = UINT64_C(1) << bit; + } + masks[68] = lanes == 64 ? 0 : UINT64_MAX << lanes; + for (unsigned sample = 0; sample < 8; ++sample) { + uint8_t input[66], prior[66], originalInput[66], originalPrior[66], output[66]; + for (unsigned i = 0; i < 66; ++i) { + input[i] = (uint8_t)(sample*71 + i*29 + 1); + prior[i] = (uint8_t)(sample*13 - i*47 + 128); + } + memcpy(originalInput, input, sizeof(input)); + memcpy(originalPrior, prior, sizeof(prior)); + for (unsigned m = 0; m < 69; ++m) { + memset(output, 0xa7, sizeof(output)); + fn(output+1, input+1, zeroing ? NULL : prior+1, masks[m]); + for (unsigned offset = 0; offset < 64; ++offset) { + uint8_t want = 0xa7; + if (offset < lanes*bytes) { + unsigned active = (unsigned)((masks[m] >> (offset/bytes)) & 1); + if (active) { + want = input[offset+1]; + } else if (zeroing) { + want = 0; + } else { + want = prior[offset+1]; + } + } + if (output[offset+1] != want) { + fprintf(stderr, "%s sample=%u mask=%llx byte=%u got=%x want=%x\n", + name, sample, (unsigned long long)masks[m], offset, output[offset+1], want); + return 1; + } + } + if (output[0] != 0xa7 || output[65] != 0xa7 || + memcmp(input, originalInput, sizeof(input)) || memcmp(prior, originalPrior, sizeof(prior))) { + fprintf(stderr, "%s changed inputs or output canaries\n", name); + return 1; + } + } + } + return 0; +} + +int main(void) { +CHECKS + return 0; +} +` diff --git a/amd64_integer_minmax_forms.go b/amd64_integer_minmax_forms.go new file mode 100644 index 000000000..10c139bea --- /dev/null +++ b/amd64_integer_minmax_forms.go @@ -0,0 +1,75 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86IntegerMinMaxForm struct { + x86PackedBinaryForm + spec amd64PackedIntegerMinMaxSpec +} + +func x86IntegerMinMaxProperties(op Op, ins Instr, arch string) (x86IntegerMinMaxForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64PackedIntegerMinMaxSpecs[base] + legacy := false + if !known && (strings.HasPrefix(base, "PMIN") || strings.HasPrefix(base, "PMAX")) { + // The twelve legacy B/W/D names have the same operation as their + // V siblings, but Go's yxm/yxm_q4 rows admit only X, not MMX/Q. + spec, known = amd64PackedIntegerMinMaxSpecs["V"+base] + known = known && spec.laneBits <= 32 + legacy = known + } + form := x86IntegerMinMaxForm{spec: spec} + if !known { + return form, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return form, true, fmt.Errorf("min/max dispatch/source opcode mismatch") + } + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true} + if legacy { + grammar = x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP} + } else if spec.laneBits == 64 { + grammar.encoding = x86PackedLogicalEVEX + } + if spec.broadcast && !legacy { + grammar.broadcastBytes = spec.laneBits / 8 + } + local := ins + local.Op = op + // Physical mode32 EVEX bytes have mask operands even though Go's named + // frontend caps instruction arity at three. Preserve that decoder-only + // shape; whole-function replay and numerical inputs remain separate checks. + physical := !legacy && ins.x86Encoded && x86RawOrdinaryVectorSource(ins, arch) + var err error + form.x86PackedBinaryForm, err = x86ConflictFunnelOperandGrammar(local, arch, grammar, physical) + return form, true, err +} + +func x86TypedIntegerMinMaxByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86IntegerMinMaxProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width, firstWidth := int64(form.byteWidth), int64(form.byteWidth) + if form.memory >= 0 { + firstWidth = int64(form.memoryBytes) + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: firstWidth, read: true}, + {index: form.second, operand: ins.Args[form.second], width: width, read: true, write: form.legacy}, + }} + if !form.legacy { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: form.destination, operand: ins.Args[form.destination], width: width, + read: form.mask >= 0 && !form.zeroing, write: true, + }) + } + if form.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return effect, true, nil +} diff --git a/amd64_integer_splat_test.go b/amd64_integer_splat_test.go new file mode 100644 index 000000000..b45f539c3 --- /dev/null +++ b/amd64_integer_splat_test.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestLLVMIntegerSplatsPreserveExactText(t *testing.T) { + for _, test := range []struct { + name string + bits int + value int64 + element string + }{ + {"zero", 8, 0, "i8 0"}, + {"boolean", 1, 1, "i1 1"}, + {"negative-byte", 8, -128, "i8 -128"}, + {"negative-saturation", 32, -32768, "i32 -32768"}, + {"positive-saturation", 32, 32767, "i32 32767"}, + {"all-ones", 64, -1, "i64 -1"}, + {"minimum", 64, -1 << 63, "i64 -9223372036854775808"}, + {"maximum", 64, 1<<63 - 1, "i64 9223372036854775807"}, + {"wide-carrier", 128, -1 << 63, "i128 -9223372036854775808"}, + } { + t.Run("signed/"+test.name, func(t *testing.T) { + checkIntegerSplatText(t, test.element, func(lanes int) string { + return llvmSplatSignedInteger(lanes, test.bits, test.value) + }) + }) + } + for _, test := range []struct { + name string + bits int + value uint64 + element string + }{ + {"zero", 8, 0, "i8 0"}, + {"boolean", 1, 1, "i1 1"}, + {"byte-mask", 8, 255, "i8 255"}, + {"word-mask", 16, 65535, "i16 65535"}, + {"dword-mask", 32, 4294967295, "i32 4294967295"}, + {"high-bit", 64, 1 << 63, "i64 9223372036854775808"}, + {"maximum", 64, ^uint64(0), "i64 18446744073709551615"}, + {"wide-carrier", 128, ^uint64(0), "i128 18446744073709551615"}, + } { + t.Run("unsigned/"+test.name, func(t *testing.T) { + checkIntegerSplatText(t, test.element, func(lanes int) string { + return llvmSplatInteger(lanes, test.bits, test.value) + }) + }) + } +} + +// The decimal spellings above are independent literal expectations. This +// oracle constructs a slice and joins it; it does not reuse the formatter or +// production builder, and includes the existing empty result for lanes <= 0. +func checkIntegerSplatText(t *testing.T, element string, splat func(int) string) { + t.Helper() + for _, lanes := range []int{-2, 0, 1, 2, 4, 16, 64, 256} { + t.Run(fmt.Sprint(lanes), func(t *testing.T) { + var elements []string + for lane := 0; lane < lanes; lane++ { + elements = append(elements, element) + } + want := "<" + strings.Join(elements, ", ") + ">" + if got := splat(lanes); got != want { + t.Fatalf("splat(%d): got %q, want %q", lanes, got, want) + } + }) + } +} + +func TestLLVMIntegerSplatsHaveBoundedAllocationGrowth(t *testing.T) { + for _, test := range integerSplatAllocationCases() { + t.Run(test.name, func(t *testing.T) { + var result string + small := testing.AllocsPerRun(20, func() { + result = test.splat(1) + }) + large := testing.AllocsPerRun(20, func() { + result = test.splat(256) + }) + if strings.Count(result, ", ") != 255 { + t.Fatal("allocation check did not produce all 256 lanes") + } + // Formatting and output storage may allocate a small constant + // amount. Neither per-lane boxing nor repeated builder growth is + // part of the contract; retain room for SDK formatting overhead. + if large > small+1 || large > 8 { + t.Fatalf("allocations grow with lane count: 1 lane=%g, 256 lanes=%g", small, large) + } + }) + } +} + +func integerSplatAllocationCases() []struct { + name string + splat func(int) string +} { + return []struct { + name string + splat func(int) string + }{ + {"signed-min", func(lanes int) string { return llvmSplatSignedInteger(lanes, 64, -1<<63) }}, + {"signed-saturation", func(lanes int) string { return llvmSplatSignedInteger(lanes, 32, -32768) }}, + {"unsigned-max", func(lanes int) string { return llvmSplatInteger(lanes, 64, ^uint64(0)) }}, + {"unsigned-byte", func(lanes int) string { return llvmSplatInteger(lanes, 8, 255) }}, + } +} + +var integerSplatBenchmarkSink string + +func BenchmarkLLVMIntegerSplats(b *testing.B) { + for _, test := range integerSplatAllocationCases() { + b.Run(test.name, func(b *testing.B) { + for _, lanes := range []int{1, 16, 64, 256} { + b.Run(fmt.Sprint(lanes), func(b *testing.B) { + b.SetBytes(int64(len(test.splat(lanes)))) + b.ReportAllocs() + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + integerSplatBenchmarkSink = test.splat(lanes) + } + }) + } + }) + } +} diff --git a/amd64_lea_byte_effects.go b/amd64_lea_byte_effects.go new file mode 100644 index 000000000..a6b2b3f55 --- /dev/null +++ b/amd64_lea_byte_effects.go @@ -0,0 +1,76 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// This describes only the pure numerical effective-address subset of Go's +// complete ym_rl grammar. The register dependencies are read before the +// destination is overwritten; no memory is read and FLAGS are unchanged. +// Modulo-W/L/Q arithmetic needs only those low input bytes. A W write keeps +// the other bytes, while L clears upper32 through the existing write consumer. +// Actual initialization and pointer exclusion remain consumers' obligations. +func x86TypedLEAByteEffect(op Op, original Instr, arch string) (x86TypedByteEffect, bool, error) { + effect, known, err := x86NamedLEAByteEffect(op, original, arch) + if !known || err != nil { + return effect, known, err + } + // Ordinary source consumers leave physical frame registers to their + // independently bound stack route. This is a role constraint, not part + // of the complete encoding grammar used by fixed-frame consumers. + for _, operand := range effect.operands { + if x86LogicalSourceFrameRegister(operand.operand.Reg, arch) { + return x86TypedByteEffect{}, false, nil + } + } + return effect, true, nil +} + +func x86NamedLEAByteEffect(op Op, original Instr, arch string) (x86TypedByteEffect, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + if name != "LEAW" && name != "LEAL" && name != "LEAQ" { + return x86TypedByteEffect{}, false, nil + } + // A decoded physical instruction belongs to its existing separate effect + // consumers. Do not intercept it with this named-source registration. + if original.x86Encoded || original.x86SourceRaw != nil || original.x86RawNearReturn != nil || + original.x86AddressBits != 0 || original.x86UnboundAddressOverride || original.x86SegmentPrefix != 0 { + return x86TypedByteEffect{}, false, nil + } + fail := func() (x86TypedByteEffect, bool, error) { + return x86TypedByteEffect{}, true, fmt.Errorf("%w: %s %s needs a named Go effective-address form: %q", ErrProbeNeedsContext, arch, op, original.Raw) + } + if arch != "amd64" && arch != "386" || dotted || + original.Op != "" && !strings.EqualFold(string(original.Op), name) { + return fail() + } + ins := original + ins.Op = Op(name) + ins = x86LEASourceInstruction(ins) + width, err := x86LEAOperandWidth(ins, arch) + if err != nil { + return x86TypedByteEffect{}, true, err + } + if ins.Args[0].Kind != OpMem { + return fail() + } + memory := ins.Args[0].Mem + if memory.Sym != "" || memory.OffRaw != "" || memory.Segment != "" || memory.IndexExt != "" || + memory.Index == "" && memory.Scale != 0 { + return fail() + } + effect := x86TypedByteEffect{} + for _, register := range []Reg{memory.Base, memory.Index} { + if register == "" { + continue + } + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: 0, operand: Operand{Kind: OpReg, Reg: register}, width: width, read: true, + }) + } + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: 1, operand: ins.Args[1], width: width, write: true, + }) + return effect, true, nil +} diff --git a/amd64_lea_dispatch_test.go b/amd64_lea_dispatch_test.go new file mode 100644 index 000000000..8c160d7e8 --- /dev/null +++ b/amd64_lea_dispatch_test.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86LEADispatchBindsCompleteWidths(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, op := range []Op{"LEAW", "LEAL", "LEAQ"} { + for _, instructionOp := range []Op{"", op, Op(strings.ToLower(string(op)))} { + t.Run(fmt.Sprintf("%s/%s/op=%q", arch, op, instructionOp), func(t *testing.T) { + instruction := Instr{Op: instructionOp, Args: []Operand{ + {Kind: OpMem, Mem: MemRef{Base: BX, Index: CX, Scale: 2, Off: 8}}, + {Kind: OpReg, Reg: AX}, + }} + registered := instruction + registered.Op = op + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{registered}}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := output.String() + ok, terminated, err := ctx.lowerArith(op, instruction) + if arch == "386" && op == "LEAQ" { + if !ok || terminated || err == nil || output.String() != before { + t.Fatalf("386 LEAQ must reject before emission: (%v, %v, %v)", ok, terminated, err) + } + return + } + if !ok || terminated || err != nil { + t.Fatalf("LEA dispatch = (%v, %v, %v)", ok, terminated, err) + } + if instruction.Op != instructionOp { + t.Fatal("dispatch mutated the caller's original instruction identity") + } + if op == "LEAW" && !strings.Contains(output.String()[len(before):], "to i16") || + op == "LEAL" && !strings.Contains(output.String()[len(before):], "to i32") { + t.Fatal("LEA used a different destination width than the dispatch") + } + }) + } + } + } +} + +func TestX86LEADispatchRejectsContradictoryIdentityBeforeEmission(t *testing.T) { + for _, dispatch := range []Op{"LEAW", "LEAL", "LEAQ"} { + for _, source := range []Op{"LEAW", "LEAL", "LEAQ", "LEAL.Z", "MOVQ", "UNKNOWN"} { + if source == dispatch { + continue + } + t.Run(fmt.Sprintf("%s/source=%s", dispatch, source), func(t *testing.T) { + instruction := Instr{Op: source, Args: []Operand{ + {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, + {Kind: OpReg, Reg: AX}, + }} + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + before := output.String() + ok, terminated, err := ctx.lowerArith(dispatch, instruction) + if !ok || terminated || err == nil { + t.Fatalf("accepted an unknown/contradictory LEA identity: (%v, %v, %v)", ok, terminated, err) + } + if output.String() != before { + t.Fatal("contradictory LEA emitted address loads/truncation/stores before rejection") + } + }) + } + } +} + +func TestX86LEADispatchUnknownOpcodeCannotConsumeKnownInstruction(t *testing.T) { + for _, dispatch := range []Op{"UNKNOWN", "LEAL.Z", "leal"} { + for _, source := range []Op{"", "LEAW", "LEAL", "LEAQ", "UNKNOWN"} { + t.Run(fmt.Sprintf("%s/source=%q", dispatch, source), func(t *testing.T) { + instruction := Instr{Op: source, Args: []Operand{ + {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, + {Kind: OpReg, Reg: AX}, + }} + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + before := output.String() + ok, terminated, _ := ctx.lowerArith(dispatch, instruction) + if ok || terminated || output.String() != before { + t.Fatal("unknown arithmetic dispatch was treated as an emitted LEA") + } + }) + } + } +} diff --git a/amd64_lea_effects.go b/amd64_lea_effects.go new file mode 100644 index 000000000..91780bbcb --- /dev/null +++ b/amd64_lea_effects.go @@ -0,0 +1,206 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" +) + +// Go 1.27's ALEAW/ALEAL/ALEAQ rows all use ym_rl (Ym, Yrl). Address +// calculation and destination width are independent: W preserves the upper +// register, L zero-extends, and Q is available only in 64-bit mode. +func x86LEAWidth(op Op, arch string) int64 { + switch op { + case "LEAW": + return 2 + case "LEAL": + return 4 + case "LEAQ": + if arch != "386" { + return 8 + } + } + return 0 +} + +func x86LEAOperandWidth(ins Instr, arch string) (int64, error) { + ins = x86LEASourceInstruction(ins) + width := x86LEAWidth(ins.Op, arch) + if width == 0 { + return 0, fmt.Errorf("%s %s has no Go LEA width: %q", arch, ins.Op, ins.Raw) + } + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[1].Reg, arch) { + return 0, fmt.Errorf("%s %s expects Ym address and Yrl destination: %q", arch, ins.Op, ins.Raw) + } + if bits := ins.x86AddressBits; bits != 0 && (ins.Args[0].Kind != OpMem || + !(arch == "386" && bits == 16 || arch != "386" && bits == 32)) { + return 0, fmt.Errorf("%w: %s %s has an unbound physical address size %d", ErrProbeNeedsContext, arch, ins.Op, bits) + } + switch ins.Args[0].Kind { + case OpMem: + memory := ins.Args[0].Mem + if err := x86SymbolIndexedMemoryError(memory, arch); err != nil { + return 0, err + } + if arch == "386" && strings.HasSuffix(memory.Sym, "(SB)") { + memory.Base = "" // Go's named-memory encoder discards Reg. + } + if !x86MemoryRegistersValidForArch(memory, arch) { + return 0, fmt.Errorf("%s %s source uses an out-of-range address register: %q", arch, ins.Op, ins.Raw) + } + if memory.Index != "" && (memory.Index == SP || + memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8) { + return 0, fmt.Errorf("%s %s source has an unencodable SIB index/scale: %q", arch, ins.Op, ins.Raw) + } + // asm6.go:asmandsz truncates an AMD64 LEAL displacement to int32. + // Other AMD64 widths require signed int32; 386 also accepts uint32. + signed32 := memory.Off == int64(int32(memory.Off)) + unsigned32 := memory.Off == int64(uint32(memory.Off)) + if !(arch != "386" && width == 4) && !signed32 && !(arch == "386" && unsigned32) { + return 0, fmt.Errorf("%s %s displacement is outside Go's encoded address range: %q", arch, ins.Op, ins.Raw) + } + case OpSym: + if !strings.HasSuffix(strings.TrimSpace(ins.Args[0].Sym), "(SB)") || strings.HasPrefix(strings.TrimSpace(ins.Args[0].Sym), "$") { + return 0, fmt.Errorf("%s %s source is not a Go SB memory expression: %q", arch, ins.Op, ins.Raw) + } + fallthrough + case OpFP: + // Bound FP storage and symbol identity remain the actual lowerer's + // responsibility; this row does not establish pointer provenance. + default: + return 0, fmt.Errorf("%s %s expects Ym address and Yrl destination: %q", arch, ins.Op, ins.Raw) + } + return width, nil +} + +// A bare integer expression is Go's TYPE_MEM/NAME_NONE displacement, not a +// symbol or a dereference for LEA. Normalize only this local operand copy; +// the original opcode, Raw and source/certificate identity remain unchanged. +// Unknown expressions, address immediates and physical decoder roles retain +// their separate grammar instead of acquiring a guessed zero displacement. +func x86LEASourceInstruction(ins Instr) Instr { + if len(ins.Args) != 2 || ins.Args[0].Kind != OpSym || ins.x86Encoded || ins.x86SourceRaw != nil || + ins.x86RawNearReturn != nil || ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 { + return ins + } + if ins.Op != "LEAW" && ins.Op != "LEAL" && ins.Op != "LEAQ" { + return ins + } + value, resolved := parseImmExpr(strings.TrimSpace(ins.Args[0].Sym)) + if !resolved { + return ins + } + ins.Args = append([]Operand(nil), ins.Args...) + ins.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Off: int64(value)}} + return ins +} + +// Named TYPE_ADDR PARAM/AUTO moves form addresses, never DATA reads. Go's +// progedit rewrites L/Q before table selection; W's Zaut_r emits word LEA. +// This local view keeps the actual Op/Raw untouched for source replay. It +// supplies syntax/width only, not a frame, initialized data or numerical bits. +func x86NamedFrameMoveAddressInstruction(ins Instr, arch string) (Instr, bool, error) { + if len(ins.Args) != 2 || ins.Args[0].Kind != OpFPAddr && ins.Args[0].Kind != OpSym { + return ins, false, nil + } + op := x86SourceMoveInstruction(ins, arch).Op + if (op != "MOVW" && op != "MOVL" && op != "MOVQ") || !x86ResultNamedPredicateInstruction(ins) { + return ins, false, nil + } + var address Operand + switch source := ins.Args[0]; source.Kind { + case OpFPAddr: + if !validPPIdentifier(source.FPName) || x86SymbolGoRegisterName(source.FPName) || + source.FPName == "FP" || source.FPName == "SB" || source.FPName == "PC" || arch == "amd64" && source.FPName == "g" { + return ins, true, fmt.Errorf("%w: named MOV FP address requires an original Go symbol", ErrProbeNeedsContext) + } + address = source + address.Kind = OpFP + case OpSym: + body := strings.TrimSpace(source.Sym) + if !strings.HasPrefix(body, "$") { + return ins, false, nil + } + body = strings.TrimSpace(strings.TrimPrefix(body, "$")) + literal := x86ResultAUTOLiteral.FindStringSubmatch(body) + if literal == nil { + return ins, false, nil // SB and unnamed addresses retain their own consumers. + } + prefix := strings.TrimSpace(literal[1]) + _, named := parseNamedStackConstantOffset(prefix) + if validPPIdentifier(prefix) { + named = true + } + if !named { + return ins, false, nil // MOVW NAME_NONE falls through to TYPE_CONST, not Zaut_r. + } + var err error + address, err = parseOperandForArch(ArchAMD64, body) + if err != nil || address.Kind != OpMem || address.Mem.Base != SP { + return ins, true, fmt.Errorf("named MOV frame address has no complete AUTO operand") + } + default: + return ins, false, nil + } + // A local opcode view cannot repair stale parsed operands or discard the + // descriptive Go name. Replay the complete original two-operand spelling. + rawOp, rest := splitOpcode(ins.Raw) + literals := splitTopLevelCSV(rest) + if !strings.EqualFold(rawOp, string(ins.Op)) || len(literals) != len(ins.Args) { + return ins, true, fmt.Errorf("%w: named MOV frame address lacks its original spelling", ErrProbeNeedsContext) + } + for index, literal := range literals { + parsed, err := parseOperandForArch(Arch(arch), literal) + if err != nil || !reflect.DeepEqual(parsed, ins.Args[index]) { + return ins, true, fmt.Errorf("%w: named MOV frame address differs from its original operands", ErrProbeNeedsContext) + } + } + view := ins + view.Op = Op("LEA" + string(op)[3:]) + view.Args = append([]Operand(nil), ins.Args...) + view.Args[0] = address + _, err := x86LEAOperandWidth(view, arch) + return view, true, err +} + +func x86CallerLEAStep(ins *Instr, state *x86SourceStackState) bool { + form := x86LEASourceInstruction(*ins) + width, err := x86LEAOperandWidth(form, "amd64") + if err != nil || form.Args[0].Kind != OpMem || ins.x86AddressBits != 0 { + return false + } + memory := form.Args[0].Mem + destination, full := x86CallerGP(ins.Args[1].Reg) + if !full || destination != ins.Args[1].Reg || destination == SP || destination == BP || + memory.Sym != "" || memory.Segment != "" { + return false + } + base, _ := x86CallerGP(memory.Base) + if pointer, present := state.callers[base]; present { + // Only the full, independently bound caller address survives. A + // narrowed address must not become numerical initialization; a + // variable offset cannot borrow this constant-offset certificate. + if width != 8 || memory.Base != base || memory.Index != "" || + memory.Off < -x86SourceStackLimit || memory.Off > x86SourceStackLimit { + return false + } + var valid bool + pointer, valid = x86CallerPointerShift(pointer, memory.Off) + if !valid { + return false + } + state.callers[destination] = pointer + delete(state.known, destination) + return true + } + // Arithmetic modulo the result width depends only on those low input + // bytes. Multiplication/addition carry cannot invent any higher input + // dependency. NumericRead still rejects *all* pointer-derived inputs, + // including an index or a source/destination alias. + for _, register := range []Reg{memory.Base, memory.Index} { + if register != "" && !x86CallerNumericRead(Operand{Kind: OpReg, Reg: register}, width, state) { + return false + } + } + return x86CallerNumericWrite(destination, width, state) +} diff --git a/amd64_lea_go_oracle_receipt_test.go b/amd64_lea_go_oracle_receipt_test.go new file mode 100644 index 000000000..37e9b9ae9 --- /dev/null +++ b/amd64_lea_go_oracle_receipt_test.go @@ -0,0 +1,204 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "go/ast" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// A test-only observation of this invocation, not a frame or caller capability. +type x86LEAGoObjectReceipt struct { + Target, GoVersion string + Source, Driver, Object [sha256.Size]byte +} + +// These existing LEAW/L/Q source forms also belong to the historical SDK. +// The generic current-table helper remains unchanged for other families. +func requireLEANativeGoObject(t *testing.T, arch, source string, wantSuccess bool) { + t.Helper() + if wantSuccess { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + x86LEAActualTargetGoObject(t, arch, triple, source) + } else { + x86LEAActualSDKDriver(t) + goAssemblerOracleOutput(t, arch, source, "", false) + } +} + +func x86LEAActualSDKDriver(t *testing.T) (string, [sha256.Size]byte) { + t.Helper() + driver, err := exec.LookPath("go") + if err != nil { + t.Fatal(err) + } + driver, err = filepath.EvalSymlinks(driver) + if err != nil { + t.Fatal(err) + } + root, err := filepath.EvalSymlinks(runtime.GOROOT()) + if err != nil || driver != x86LEASDKDriverPath(root, runtime.GOOS) { + t.Fatalf("LEA oracle must invoke this SDK's driver: %s: %v", driver, err) + } + driverBytes, err := os.ReadFile(driver) + if err != nil { + t.Fatal(err) + } + return driver, sha256.Sum256(driverBytes) +} + +func x86LEASDKDriverPath(root, hostGOOS string) string { + filename := "go" + if hostGOOS == "windows" { + filename = "go.exe" + } + return filepath.Join(root, "bin", filename) +} + +func TestX86LEAActualSDKDriverPathUsesHostToolSuffix(t *testing.T) { + root := filepath.Join("selected-sdk", "root") + for _, test := range []struct { + goos, filename string + }{ + {"windows", "go.exe"}, + {"darwin", "go"}, + {"linux", "go"}, + } { + t.Run(test.goos, func(t *testing.T) { + got := x86LEASDKDriverPath(root, test.goos) + want := filepath.Join(root, "bin", test.filename) + if got != want { + t.Fatalf("actual SDK driver for host %s = %q, want %q", test.goos, got, want) + } + }) + } +} + +func x86LEAReadObjectReceipt(t *testing.T, arch, goos, source, input, driver string, driverHash [sha256.Size]byte, object string) *x86LEAGoObjectReceipt { + t.Helper() + info, err := os.Lstat(object) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatalf("actual LEA oracle lacks a regular nonempty object: %v", err) + } + data, err := os.ReadFile(object) + if err != nil { + t.Fatal(err) + } + header := strings.Fields(string(bytes.SplitN(data, []byte("\n"), 2)[0])) + if len(header) < 5 || header[0] != "go" || header[1] != "object" || + header[2] != goos || header[3] != arch || header[4] != runtime.Version() { + t.Fatalf("LEA object does not bind the actual target/SDK: %q", header) + } + inputBytes, err := os.ReadFile(input) + if err != nil || !bytes.Equal(inputBytes, []byte(source)) { + t.Fatalf("actual LEA source changed during the tool invocation: %v", err) + } + driverBytes, err := os.ReadFile(driver) + if err != nil || sha256.Sum256(driverBytes) != driverHash { + t.Fatalf("actual LEA driver changed during the tool invocation: %v", err) + } + receipt := &x86LEAGoObjectReceipt{ + Target: goos + "/" + arch, GoVersion: header[4], + Source: sha256.Sum256(inputBytes), Driver: driverHash, + Object: sha256.Sum256(data), + } + t.Logf("actual LEA source/object receipt: SDK=%s target=%s source=%x driver=%x object=%x", + receipt.GoVersion, receipt.Target, receipt.Source, receipt.Driver, receipt.Object) + return receipt +} + +func TestX86LEAActualGoOracleCannotSilentlyReturnBeforeTool(t *testing.T) { + for name, filename := range map[string]string{ + "x86LEAActualTargetGoObject": "amd64_go_caller_lea_test.go", + "requireLEANativeGoObject": "amd64_lea_go_oracle_receipt_test.go", + } { + file, err := parser.ParseFile(token.NewFileSet(), filename, nil, 0) + if err != nil { + t.Fatal(err) + } + ast.Inspect(file, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if ok { + if identifier, ok := call.Fun.(*ast.Ident); ok && identifier.Name == "requireX86GoAssemblerResult" { + t.Errorf("%s uses a current-table helper instead of its actual applicable LEA tool oracle", filename) + } + } + return true + }) + found := false + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if !ok || function.Name.Name != name { + continue + } + found = true + ast.Inspect(function.Body, func(node ast.Node) bool { + if statement, ok := node.(*ast.ReturnStmt); ok { + missing := len(statement.Results) == 0 + if len(statement.Results) == 1 { + value, ok := statement.Results[0].(*ast.Ident) + missing = ok && value.Name == "nil" + } + if missing { + t.Errorf("%s has a silent return instead of its actual tool receipt", name) + } + } + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if identifier, ok := call.Fun.(*ast.Ident); ok && + (identifier.Name == "goToolchainAtLeast" || identifier.Name == "currentGoMinorAtLeast") { + t.Errorf("%s silently gates its actual tool invocation by build version", name) + } + if selector, ok := call.Fun.(*ast.SelectorExpr); ok && + (selector.Sel.Name == "Skip" || selector.Sel.Name == "Skipf" || selector.Sel.Name == "SkipNow") { + t.Errorf("%s cannot skip an applicable LEA source oracle", name) + } + return true + }) + } + if !found { + t.Fatalf("actual LEA oracle %s is missing", name) + } + } +} + +func TestX86LEAActualGoOracleRequiresToolAndObjectReceipt(t *testing.T) { + targets := map[string]string{ + "x86_64-apple-darwin": "darwin/amd64", + "x86_64-unknown-linux-gnu": "linux/amd64", + "x86_64-pc-windows-msvc": "windows/amd64", + "i386-unknown-linux-gnu": "linux/386", + "i686-pc-windows-msvc": "windows/386", + } + for _, target := range x86RawReturnTargets { + for _, suffix := range []string{"W", "L", "Q"} { + if x86LEAWidth(Op("LEA"+suffix), target.arch) == 0 { + continue // No LEAQ physical encoding in 386 mode. + } + t.Run(target.triple+"/"+suffix, func(t *testing.T) { + source := "TEXT lea(SB),4,$0-0\nLEA" + suffix + " 19(AX)(BX*4),AX\nRET\n" + receipt := x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + if receipt == nil { + t.Fatal("actual LEA oracle omitted its selected SDK tool/object invocation") + } + zero := [sha256.Size]byte{} + if receipt.GoVersion != runtime.Version() || receipt.Source != sha256.Sum256([]byte(source)) || + receipt.Driver == zero || receipt.Object == zero || receipt.Target != targets[target.triple] { + t.Fatalf("actual LEA receipt is incomplete or misbound: %+v", receipt) + } + }) + } + } +} diff --git a/amd64_legacy_alternating_round_memory.go b/amd64_legacy_alternating_round_memory.go new file mode 100644 index 000000000..9925939c3 --- /dev/null +++ b/amd64_legacy_alternating_round_memory.go @@ -0,0 +1,50 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// A paired legacy whole opcode supplies numerical bytes only on continuation. +// It grants no pointee extent, stack backing, source or raw-entry permission. +func x86LegacyAlternatingRoundMemorySource(ins Instr, source Operand, arch string) bool { + return x86ResultNamedPredicateInstruction(ins) && + (source.Kind == OpMem || source.Kind == OpSym) && x86OrdinaryNativeMemoryOperand(source, arch) +} + +// Both families consume their already normalized form. A ROUND immediate +// distinguishes its unary packed result from ADDSUB's old-destination input; +// scalar ROUND reads only copied upper bytes, not its overwritten low lane. +func (c *amd64Ctx) emitLegacyAlternatingRoundMemory(ins Instr, form x86FloatBinaryForm, immediate *uint8) error { + source := ins.Args[form.first] + if !form.legacy || form.byteWidth != 16 || !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || + !x86ResultNamedPredicateInstruction(ins) || !c.nativeMemoryOperand(ins, form.first) { + return fmt.Errorf("%w: %s needs its original ordinary legacy memory source", ErrProbeNeedsContext, ins.Op) + } + typ := amd64FMA3LLVMType(128/form.bits, form.bits) + input, inputType, constraint, template, err := c.nativeFloatingCompareSource(source, + x86FloatCompareForm{x86FloatBinaryForm: form}, typ, "") + if err != nil { + return err + } + args, constraints := inputType+" "+input, "=x,"+constraint + if immediate == nil || form.scalar { + old, err := c.loadX(ins.Args[form.destination].Reg) + if err != nil { + return err + } + carrier := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", carrier, old, typ) + args, constraints = typ+" %"+carrier+", "+args, "=x,0,"+constraint + template = strings.ReplaceAll(template, "$1", "$2") + } + assembly := strings.ToLower(string(ins.Op)) + " " + if immediate != nil { + assembly += fmt.Sprintf("$$%d, ", *immediate) + } + assembly += template + ", $0" + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, typ, assembly, constraints+",~{memory},~{fpsr}", args) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytes, typ, result) + return c.storeX(ins.Args[form.destination].Reg, "%"+bytes) +} diff --git a/amd64_legacy_float_memory_context_test.go b/amd64_legacy_float_memory_context_test.go new file mode 100644 index 000000000..14a972625 --- /dev/null +++ b/amd64_legacy_float_memory_context_test.go @@ -0,0 +1,220 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86LegacyPackedFloatingNativeMemoryUnboundSourcesBeforeEmission(t *testing.T) { + for _, op := range []Op{"ADDPS", "ADDPD", "SUBPS", "SUBPD", "MULPS", "MULPD", "DIVPS", "DIVPD"} { + for _, operand := range []Operand{ + {Kind: OpMem, Mem: MemRef{Base: SP, Off: 16}}, + {Kind: OpMem, Mem: MemRef{Base: BP, Off: -32}}, + {Kind: OpMem, Mem: MemRef{Base: AX, Index: BP, Scale: 1}}, + {Kind: OpFP, FPOffset: 0}, + } { + t.Run(string(op)+"/"+operand.String(), func(t *testing.T) { + ins := Instr{Op: op, Args: []Operand{operand, {Kind: OpReg, Reg: "X0"}}} + c, ir := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + before := ir.String() + form := x86LegacyPackedFloatContextForm(t, ins) + if err := c.lowerLegacyPackedFloatingMemory(ins, form); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unbound native memory source accepted: %v", err) + } + if ir.String() != before { + t.Fatal("unbound source rejection emitted IR side effects") + } + }) + } + for _, addressBits := range []int{16, 32} { + ins := Instr{ + Op: op, Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX}}, {Kind: OpReg, Reg: "X0"}}, + x86AddressBits: addressBits, + } + c, ir := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + before := ir.String() + if err := c.lowerLegacyPackedFloatingMemory(ins, x86LegacyPackedFloatContextForm(t, ins)); !errors.Is(err, ErrProbeNeedsContext) || ir.String() != before { + t.Fatalf("address-size %d bypassed native source binding or emitted IR: %v", addressBits, err) + } + } + } +} + +func x86LegacyPackedFloatContextForm(t *testing.T, ins Instr) x86FloatBinaryForm { + t.Helper() + addsub, known, err := x86FloatAddSubProperties(ins, "amd64") + if known { + if err != nil { + t.Fatal(err) + } + return addsub.x86FloatBinaryForm + } + muldiv, known, err := x86FloatMulDivProperties(ins, "amd64") + if !known || err != nil { + t.Fatalf("missing complete typed legacy form: known=%t err=%v", known, err) + } + return muldiv.x86FloatBinaryForm +} + +func TestX86LegacyPackedFloatingNativeMemoryUnboundSourcesFiveTargets(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, op := range []string{"ADDPS", "ADDPD", "SUBPS", "SUBPD", "MULPS", "MULPD", "DIVPS", "DIVPD"} { + for _, operand := range []string{"16(SP)", "-32(BP)", "(AX)(BP*1)"} { + t.Run(target.triple+"/"+op+"/"+operand, func(t *testing.T) { + source, sig, pkg := x86OrdinaryBPNativeFixture(t, target.arch, "Value", op+" "+operand+",X0", operand, 0) + x86InsertExtractGoListing(t, target.arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + t.Run("Translate", func(t *testing.T) { + ir, err := Translate(file, opt) + x86RequireOrdinaryBPNativeOutcome(t, target.arch, operand, ir, err) + }) + t.Run("TranslateModuleInContext", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + ir := "" + if !module.IsNil() { + ir = module.String() + } + x86RequireOrdinaryBPNativeOutcome(t, target.arch, operand, ir, err) + }) + t.Run("TranslateGoModule", func(t *testing.T) { + translation, err := TranslateGoModule(pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, + ResolveSym: opt.ResolveSym, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + ir := "" + if translation != nil && !translation.Module.IsNil() { + ir = translation.Module.String() + } + x86RequireOrdinaryBPNativeOutcome(t, target.arch, operand, ir, err) + }) + }) + } + } + } +} + +func TestX86LegacyPackedFloatingNativeMemoryUnboundAddressSizeActualGo(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, row := range []struct { + opcode string + prefix string + machine int + }{ + {"ADDPS", "", 0x58}, {"ADDPD", "BYTE $0x66\n", 0x58}, + {"SUBPS", "", 0x5c}, {"SUBPD", "BYTE $0x66\n", 0x5c}, + {"MULPS", "", 0x59}, {"MULPD", "BYTE $0x66\n", 0x59}, + {"DIVPS", "", 0x5e}, {"DIVPD", "BYTE $0x66\n", 0x5e}, + } { + t.Run(target.triple+"/"+row.opcode, func(t *testing.T) { + body := row.prefix + fmt.Sprintf("BYTE $0x67\nBYTE $0x0f\nBYTE $%d\nBYTE $0x03\nRET", row.machine) + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, body) + x86InsertExtractGoListing(t, target.arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + wantBits := 32 + if target.arch == "386" { + wantBits = 16 + } + if len(decoded.Instrs) != 3 || decoded.Instrs[0].Op != "TEXT" || + string(decoded.Instrs[1].Op) != row.opcode || decoded.Instrs[1].x86AddressBits != wantBits || + decoded.Instrs[1].x86SourceRaw == nil || decoded.Instrs[2].Op != "RET" { + t.Fatalf("physical address-size/opcode identity lost: %+v", decoded.Instrs) + } + t.Run("Translate", func(t *testing.T) { + _, err := Translate(file, opt) + x86LegacyFloatMemoryRequireContext(t, err) + }) + t.Run("TranslateModuleInContext", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + x86LegacyFloatMemoryRequireContext(t, err) + }) + t.Run("TranslateGoModule", func(t *testing.T) { + translation, err := TranslateGoModule(pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: opt.ResolveSym, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + x86LegacyFloatMemoryRequireContext(t, err) + }) + }) + } + } +} + +func TestX86LegacyPackedFloatingNativeMemoryUnboundAddressSizeProducerIdentity(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for index, opcode := range x86LegacyFloatMemoryOpcodes() { + for _, input := range []struct { + name, body string + raw bool + }{ + {"native_memory", "BYTE $0x0f\nBYTE $@OP\nBYTE $0x03", true}, + {"register_override", "BYTE $0x67\nBYTE $0x0f\nBYTE $@OP\nBYTE $0xc3", true}, + {"forged_raw_comment", opcode + " (AX),X0 // decoded from BYTE $0x67", false}, + } { + t.Run(arch+"/"+opcode+"/"+input.name, func(t *testing.T) { + body := input.body + if input.raw { + body = strings.ReplaceAll(body, "@OP", fmt.Sprint([]int{0x58, 0x5c, 0x59, 0x5e}[index/2])) + if index%2 != 0 { + body = "BYTE $0x66\n" + body + } + } + source := "TEXT f(SB),4,$0-0\n" + body + "\nRET\n" + x86InsertExtractGoListing(t, arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 3 || string(decoded.Instrs[1].Op) != opcode { + t.Fatalf("lost complete default/register form: %+v", decoded.Instrs) + } + instruction := decoded.Instrs[1] + if instruction.x86AddressBits != 0 || (instruction.x86SourceRaw != nil) != input.raw { + t.Fatalf("default/register/forged-comment form acquired an override or raw origin: %+v", instruction) + } + }) + } + } + } +} + +func x86LegacyFloatMemoryRequireContext(t *testing.T, err error) { + t.Helper() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("actual Go accepted source acquired native virtual-stack permission: %v", err) + } +} diff --git a/amd64_legacy_float_memory_runtime_test.go b/amd64_legacy_float_memory_runtime_test.go new file mode 100644 index 000000000..d742b1e4a --- /dev/null +++ b/amd64_legacy_float_memory_runtime_test.go @@ -0,0 +1,139 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86LegacyFloatMemoryProgram(t *testing.T, triple string, dead bool) string { + t.Helper() + var source strings.Builder + sigs := map[string]FuncSig{} + for i, opcode := range x86LegacyFloatMemoryOpcodes() { + name := fmt.Sprintf("memory%d", i) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nVMOVUPS (BX),Z0\n%s (CX),X0\n", name, opcode) + if !dead { + source.WriteString("VMOVUPS Z0,(AX)\n") + } + source.WriteString("RET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + return ir +} + +// The independent reference executes each original m128 opcode, rather than +// MOVDQA or a register-only arithmetic substitute. The original physical +// operation supplies the expected typed status, without guessing a vendor, +// emulator, signal mapping, or machine-control setting. The required native +// CI counterpart has the same test root; Rosetta is not native hardware proof. +func TestCrossLinuxRuntimeMatrixX86LegacyPackedFloatingNativeMemory(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var references, table strings.Builder + for i, opcode := range x86LegacyFloatMemoryOpcodes() { + fmt.Fprintf(&references, `extern void memory%d(void*,const void*,const void*); +static __attribute__((noinline)) void reference%d(void *out,const void *a,const void *b) { + uint8_t low[16]; + __asm__ volatile("movups (%%1),%%%%xmm0; %s (%%2),%%%%xmm0; movups %%%%xmm0,(%%0)"::"r"(low),"r"(a),"r"(b):"xmm0","memory"); + memmove(out,a,64);memcpy(out,low,16); +} +`, i, i, strings.ToLower(opcode)) + if i > 0 { + table.WriteString(",") + } + fmt.Fprintf(&table, "{reference%d,memory%d,%d}", i, i, 32+32*(i%2)) + } + main := ` +#include +#include +#include +#include +#include +#include +#include +#include +#include +` + references.String() + ` +typedef void (*operation)(void*,const void*,const void*); +struct pair {operation original,translated;int bits;}; +static const struct pair pairs[]={ ` + table.String() + ` }; +struct outcome {int kind,exit_code,signal,core;}; +static struct outcome fault(operation fn,const void *b) { + struct outcome result={0,0,0,0};pid_t pid=fork();if(pid<0)return result; + if(!pid){struct rlimit limit={0,0};if(setrlimit(RLIMIT_CORE,&limit))_exit(120); + uint8_t a[64] __attribute__((aligned(16)))={0},out[64];unsigned csr=0x1f80; + __asm__ volatile("ldmxcsr %0"::"m"(csr));fn(out,a,b);_exit(0);} + int status;if(waitpid(pid,&status,0)!=pid)return result; + if(WIFSIGNALED(status)){result.kind=2;result.signal=WTERMSIG(status);result.core=!!WCOREDUMP(status);} + else if(WIFEXITED(status)){result.kind=1;result.exit_code=WEXITSTATUS(status);} + return result; +} +static int same_status(struct outcome original,struct outcome translated) { + if(!original.kind||!translated.kind||original.exit_code||translated.exit_code)return 0; + return original.kind==translated.kind&&original.exit_code==translated.exit_code&&original.signal==translated.signal&&original.core==translated.core; +} +int main(void) { + const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0}; + const uint64_t edges32[]={0,0x80000000,1,0x00800000,0x3f800001,0xbf800001,0x7f800000,0xff800000,0x7fc12345,0x7f812345}; + const uint64_t edges64[]={0,0x8000000000000000ULL,1,0x0010000000000000ULL,0x3ff0000000000001ULL,0xbff0000000000001ULL,0x7ff0000000000000ULL,0xfff0000000000000ULL,0x7ff8123456789abcULL,0x7ff0123456789abcULL}; + uint8_t bStorage[96] __attribute__((aligned(16)))={0}; + for(int op=0;op<8;op++) { + for(int offset=0;offset<16;offset++) { + struct outcome original=fault(pairs[op].original,bStorage+offset),translated=fault(pairs[op].translated,bStorage+offset); + if(!same_status(original,translated)){fprintf(stderr,"opcode=%d mod16=%d original=%d/%d/%d/%d translated=%d/%d/%d/%d\n",op,offset,original.kind,original.exit_code,original.signal,original.core,translated.kind,translated.exit_code,translated.signal,translated.core);return 1;} + } +#if !DEAD_RESULT + for(int round=0;round<4;round++)for(int control=0;control<6;control++)for(int edge=0;edge<10;edge++)for(int alias=0;alias<3;alias++) { + uint8_t a[64] __attribute__((aligned(16))),b[64] __attribute__((aligned(16))),original[66],translated[66]; + for(int j=0;j<64;j++){a[j]=(uint8_t)(j*13+edge);b[j]=0xb7;} + const uint64_t *edges=pairs[op].bits==32?edges32:edges64;int bytes=pairs[op].bits/8; + for(int lane=0;lane<16/bytes;lane++){memcpy(a+lane*bytes,&edges[(edge+lane)%10],bytes);memcpy(b+lane*bytes,&edges[(edge+lane+3)%10],bytes);} + memset(original,0xc3,66);memset(translated,0xc3,66); + unsigned csr=controls[control]|(round<<13),expected,actual; + uint8_t originalA[64] __attribute__((aligned(16))),originalB[64] __attribute__((aligned(16))),translatedA[64] __attribute__((aligned(16))),translatedB[64] __attribute__((aligned(16))); + memcpy(originalA,a,64);memcpy(originalB,b,64);memcpy(translatedA,a,64);memcpy(translatedB,b,64); + void *originalOut=alias==1?originalA:alias==2?originalB:original+1,*translatedOut=alias==1?translatedA:alias==2?translatedB:translated+1; + __asm__ volatile("ldmxcsr %0"::"m"(csr));pairs[op].original(originalOut,originalA,originalB);__asm__ volatile("stmxcsr %0":"=m"(expected)); + __asm__ volatile("ldmxcsr %0"::"m"(csr));pairs[op].translated(translatedOut,translatedA,translatedB);__asm__ volatile("stmxcsr %0":"=m"(actual)); + if(alias){memcpy(original+1,originalOut,64);memcpy(translated+1,translatedOut,64);} + if(memcmp(original,translated,66)||expected!=actual){fprintf(stderr,"opcode=%d round=%d control=%d edge=%d CSR=%x/%x\n",op,round,control,edge,expected,actual);return 3;} + if(memcmp(originalA,translatedA,64)||memcmp(originalB,translatedB,64)||(!alias&&(memcmp(translatedA,a,64)||memcmp(translatedB,b,64))))return 8; + } +#endif + } + long page=sysconf(_SC_PAGESIZE);if(page<128)return 4; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(region==MAP_FAILED||mprotect(region+page,page,PROT_NONE))return 5; + for(int op=0;op<8;op++)for(int extent=0;extent<=16;extent++) { + const void *pointer=region+page-extent;struct outcome original=fault(pairs[op].original,pointer),translated=fault(pairs[op].translated,pointer); + if(!same_status(original,translated)){fprintf(stderr,"guard opcode=%d extent=%d original=%d/%d/%d/%d translated=%d/%d/%d/%d\n",op,extent,original.kind,original.exit_code,original.signal,original.core,translated.kind,translated.exit_code,translated.signal,translated.core);return 6;} + } + if(munmap(region,2*page))return 7; + unsigned csr=0x1f80;__asm__ volatile("ldmxcsr %0"::"m"(csr));return 0; +} +` + for _, dead := range []bool{false, true} { + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("dead=%t/O2=%t", dead, optimized), func(t *testing.T) { + ir := x86LegacyFloatMemoryProgram(t, triple, dead) + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy-float-memory", triple, ir, fmt.Sprintf("#define DEAD_RESULT %d\n", x86FloatAddSubBoolInt(dead))+main, prefix) + }) + } + } +} diff --git a/amd64_legacy_float_memory_test.go b/amd64_legacy_float_memory_test.go new file mode 100644 index 000000000..61aee3629 --- /dev/null +++ b/amd64_legacy_float_memory_test.go @@ -0,0 +1,242 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func x86LegacyFloatMemoryOpcodes() []string { + return []string{"ADDPS", "ADDPD", "SUBPS", "SUBPD", "MULPS", "MULPD", "DIVPS", "DIVPD"} +} + +func TestX86LegacyPackedFloatingMemoryActualGoOpcodeBytes(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for i, opcode := range x86LegacyFloatMemoryOpcodes() { + listing := x86InsertExtractGoListing(t, arch, "TEXT f(SB),4,$0-0\n"+opcode+" (AX),X1\nRET\n") + var actual []byte + for _, line := range strings.Split(string(listing), "\n") { + actual = append(actual, x86InsertExtractListedBytes(line)...) + } + want := []byte{0x0f, []byte{0x58, 0x5c, 0x59, 0x5e}[i/2], 0x08, 0xc3} + if i%2 != 0 { + want = append([]byte{0x66}, want...) + } + if !bytes.Equal(actual, want) { + t.Fatalf("%s/%s actual Go bytes=%x independent SSE=%x", arch, opcode, actual, want) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryRetainsOriginalOpcode(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + for _, arch := range []string{"386", "amd64"} { + for _, source := range []string{"(AX)", "16(AX)(BX*2)", "data(SB)", "(FS)", "(GS)"} { + t.Run(opcode+"/"+arch+"/"+source, func(t *testing.T) { + assembly := "TEXT f(SB),4,$0-0\n" + opcode + " " + source + ",X1\nRET\n" + requireX86GoAssemblerResult(t, arch, assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: testTargetTriple("linux", arch), + Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "asm sideeffect \""+strings.ToLower(opcode)+" ") || strings.Contains(ir, "movdqa") { + t.Fatal("native m128 source lost its exact original observable opcode") + } + }) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryFiveTargetsKeepNativeOpcodes(t *testing.T) { + llc, objdump := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || objdump == "" { + t.Fatal("LLVM22 llc and llvm-objdump required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + var source strings.Builder + sigs := map[string]FuncSig{} + for i, opcode := range x86LegacyFloatMemoryOpcodes() { + for j, operand := range []string{"(AX)", "16(AX)(BX*2)", "data(SB)", "(FS)", "(GS)"} { + name := fmt.Sprintf("memory_%d_%d", i, j) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s %s,X1\nRET\n", name, opcode, operand) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/O2=%t", triple, optimized), func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + dir := t.TempDir() + input, object := filepath.Join(dir, "memory.ll"), filepath.Join(dir, "memory.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + level := "-O0" + if optimized { + level = "-O2" + } + ctx, cancel := runtimeOracleStageContext(t) + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, llc, level, "-mtriple="+triple, "-filetype=obj", input, "-o", object)) + cancel() + if err != nil { + t.Fatalf("native memory object: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + output, err = runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, objdump, "-d", object)) + cancel() + if err != nil { + t.Fatalf("native memory disassembly: %v\n%s", err, output) + } + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + pattern := regexp.MustCompile(`(?m)\b` + strings.ToLower(opcode) + `\s+[^\n]*\([^\n]*\),\s*%xmm`) + if count := len(pattern.FindAll(output, -1)); count != 5 { + t.Fatalf("%s count=%d, original memory opcode lost:\n%s", opcode, count, output) + } + } + if !strings.Contains(string(output), "%fs:") || !strings.Contains(string(output), "%gs:") { + t.Fatal("native FS/GS source prefixes lost") + } + }) + } + } +} + +func TestX86LegacyPackedFloatingMemoryCallerPermissionsAndNativeWidth(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + for _, operand := range []string{"(DI)", "16(DI)", "(BX)", "(SP)", "a+0(FP)", "(R14)", "(CX)"} { + body := fmt.Sprintf("VMOVUPS (SI),Z1\n%s %s,X1\nVMOVSS X1,(DX)", opcode, operand) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + want := operand == "(DI)" || operand == "16(DI)" + if want && err != nil || !want && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s/%s actual source permission=%v want admitted=%t", opcode, operand, err, want) + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+opcode+" (DI),X1\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, extent := range []int64{15, 16} { + ins := file.Funcs[0].Instrs[1] + proof := &x86GoCallerRegionProof{arguments: []x86CallerArgumentRegion{{kind: x86CallerFreshResult, width: extent}}} + state := x86SourceStackState{callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}}, vectors: map[Reg]uint64{"Z1": ^uint64(0)}} + accepted := x86CallerVectorMoveStep(&ins, &state, proof) + if accepted != (extent == 16) { + t.Fatalf("%s certified a partial m128 source extent=%d accepted=%t", opcode, extent, accepted) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryIdentityAndMalformedBeforeEmission(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + valid, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+opcode+" (AX),X1\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, sourceOp := range []Op{"", Op(opcode), Op(strings.ToLower(opcode)), "UNKNOWN", "ADDPS.Z", "DIVPS"} { + ins := valid.Funcs[0].Instrs[1] + ins.Op = sourceOp + ctx, output := newAMD64CtxWithFuncForTest(t, valid.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + var known, term bool + if strings.HasPrefix(opcode, "ADD") || strings.HasPrefix(opcode, "SUB") { + known, term, err = ctx.lowerFloatAddSub(Op(opcode), ins) + } else { + known, term, err = ctx.lowerFloatMulDiv(Op(opcode), ins) + } + conflict := sourceOp != "" && !strings.EqualFold(string(sourceOp), opcode) + if !known || term || (err != nil) != conflict || conflict && output.Len() != before || ins.Op != sourceOp { + t.Fatalf("%s/%s changed identity or emitted before rejection: %t/%t/%v", opcode, sourceOp, known, term, err) + } + } + for _, row := range []string{opcode + " (AX),Y1", opcode + " (AX),X16", opcode + ".Z (AX),X1", opcode + " $data(SB),X1", opcode + " (AX),X1,X2"} { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + continue + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + ins := file.Funcs[0].Instrs[1] + var known bool + var term bool + if strings.HasPrefix(opcode, "ADD") || strings.HasPrefix(opcode, "SUB") { + known, term, err = ctx.lowerFloatAddSub(ins.Op, ins) + } else { + known, term, err = ctx.lowerFloatMulDiv(ins.Op, ins) + } + if !known || term || err == nil || before != output.Len() { + t.Fatalf("malformed %s emitted native IR: %t/%t/%v", row, known, term, err) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryMode32FeaturesAreFormBound(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + for _, row := range []string{opcode + " (AX),X1", opcode + " (AX),X8", opcode + ".Z (AX),X1", opcode + " value+0(FP),X1"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + bits := x86LegacyPackedFloatingNativeMemoryBits(file.Funcs[0].Instrs[1], "386") + want := 0 + if row == opcode+" (AX),X1" { + want = 32 + if strings.HasSuffix(opcode, "PD") { + want = 64 + } + } + if bits != want { + t.Fatalf("unvalidated source form acquired SSE transport: %s %d/%d", row, bits, want) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryVirtualFPIsNotNativeStorage(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + assembly := fmt.Sprintf("TEXT f(SB),4,$0-16\n%s source+0(FP),X1\nRET\n", opcode) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + sig := FuncSig{Name: "f", Args: []LLVMType{I64, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}, + }}} + _, err = Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{"f": sig}}) + if err == nil || !strings.Contains(err.Error(), "contiguous original FP") { + t.Fatalf("%s invented native m128 alignment/backing from independent virtual fields: %v", opcode, err) + } + } +} diff --git a/amd64_legacy_integer_arithmetic_effects.go b/amd64_legacy_integer_arithmetic_effects.go new file mode 100644 index 000000000..1a80a50c1 --- /dev/null +++ b/amd64_legacy_integer_arithmetic_effects.go @@ -0,0 +1,46 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86LegacyIntegerArithmeticForm struct { + x86PackedBinaryForm + source, destination Operand +} + +// Reuse the arithmetic specifications and Go operand rows already owned by +// lowering. This descriptor adds no frame, source, caller or initialized bank. +func x86LegacyIntegerArithmeticProperties(op Op, ins Instr, arch string) (x86LegacyIntegerArithmeticForm, bool, error) { + spec, known := x86PackedArithmeticSpecForOp(op, x86PackedArithmeticAdd|x86PackedArithmeticSubtract) + if !known || spec.encoding != x86PackedLogicalLegacyFP && spec.encoding != x86PackedLogicalLegacyInteger { + return x86LegacyIntegerArithmeticForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86LegacyIntegerArithmeticForm{}, true, fmt.Errorf("legacy integer arithmetic dispatch/source mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + encoding := spec.encoding + if encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(local, arch) + if err != nil { + return x86LegacyIntegerArithmeticForm{}, true, err + } + } + form, err := x86PackedBinaryProperties(local, arch, x86PackedBinarySpec{encoding: encoding}) + result := x86LegacyIntegerArithmeticForm{x86PackedBinaryForm: form} + if err == nil { + result.source, result.destination = local.Args[0], local.Args[1] + } + return result, true, err +} + +func x86LegacyIntegerArithmeticEmitterContext(form x86LegacyIntegerArithmeticForm, ins Instr) error { + if form.mmx && ins.Args[0].Kind == OpReg && ins.Args[0].Reg != form.source.Reg { + return fmt.Errorf("%w: legacy integer arithmetic GP spelling encodes an MMX source without its physical bank transport", ErrProbeNeedsContext) + } + return nil +} diff --git a/amd64_legacy_integer_arithmetic_forms_test.go b/amd64_legacy_integer_arithmetic_forms_test.go new file mode 100644 index 000000000..91d7b3247 --- /dev/null +++ b/amd64_legacy_integer_arithmetic_forms_test.go @@ -0,0 +1,286 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86LegacyIntegerArithmeticTestOpcodes = []byte{ + 0xfc, 0xfd, 0xfe, 0xfe, 0xd4, 0xec, 0xed, 0xdc, 0xdd, + 0xf8, 0xf9, 0xfa, 0xfb, 0xe8, 0xe9, 0xd8, 0xd9, +} + +func TestX86LegacyIntegerArithmeticGrammarEffectsAndNativeWidths(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, op := range x86LegacyIntegerArithmeticTestNames { + for _, operands := range []string{"X0,X1", "X0,X0", "(AX),X1", "8(AX)(CX*4),X1", "a+0(FP),X1", "value-32(SP),X1", "·data(SB),X1", "(FS),X1", "(GS),X1"} { + t.Run(arch+"/"+op+" "+operands, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$64-64\n"+op+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, recognized, err := x86LegacyIntegerArithmeticProperties(ins.Op, ins, arch) + if !recognized || err != nil || form.byteWidth != 16 || form.mmx { + t.Fatal("full accepted XMM row lost", form, err) + } + effect, recognized, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !recognized || err != nil || len(effect.operands) != 2 || effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatal("legacy arithmetic lost its two inputs or unchanged flags", effect, err) + } + for index, role := range effect.operands { + if role.index != index || role.width != 16 || !role.read || role.write != (index == 1) || !reflect.DeepEqual(role.operand, ins.Args[index]) { + t.Fatalf("wrong ordered input/output bytes: %#v", role) + } + width, write, known := x86InstructionMemoryShape(ins, index, arch) + if known != (form.memory == index) || known && (width != 16 || write) { + t.Fatal("descriptor and native memory reader differ", width, write, known) + } + if ins.Args[index].Kind == OpFP { + width, write, known = X86FPAccessBytes(ins, index, arch) + if !known || width != 16 || write { + t.Fatal("FP observer does not preserve full native read", width, write, known) + } + } + } + if strings.HasPrefix(op, "PADDS") || strings.HasPrefix(op, "PADDUS") { + if x86CallerVectorMoveKnown(ins) || x86SourceStackInstructionKnown(ins, "MOVQ", "Q", arch) { + t.Fatal("pure descriptor granted unproved saturating caller transport") + } + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("shape observer changed original source") + } + }) + } + } + } +} + +func TestX86LegacyIntegerArithmeticActualGoBytesAndMMXBank(t *testing.T) { + for opIndex, op := range x86LegacyIntegerArithmeticTestNames { + for _, arch := range []string{"amd64", "386"} { + for _, operands := range []struct { + text string + modrm byte + }{ + {"X1,X0", 0xc1}, {"X0,X7", 0xf8}, {"(AX),X1", 0x08}, + } { + t.Run(arch+"/"+op+" "+operands.text, func(t *testing.T) { + source := "TEXT probe(SB),4,$0-0\n" + op + " " + operands.text + "\nRET\n" + want := []byte{0x66, 0x0f, x86LegacyIntegerArithmeticTestOpcodes[opIndex], operands.modrm, 0xc3} + if got := x86PackedShiftActualGoBytes(t, arch, source); !bytes.Equal(got, want) { + t.Fatalf("original source width/opcode differs: %x want %x", got, want) + } + }) + } + } + if !strings.HasPrefix(op, "PADD") || op == "PADDQ" { + continue + } + for index, spelling := range []string{"M0", "M1", "M2", "M3", "M4", "M5", "M6", "M7", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + t.Run("mmx/"+op+" "+spelling, func(t *testing.T) { + source := "TEXT probe(SB),4,$0-0\n" + op + " " + spelling + ",M7\nRET\n" + want := []byte{0x0f, x86LegacyIntegerArithmeticTestOpcodes[opIndex], 0xf8 | byte(index&7), 0xc3} + if index >= 16 { + want = append([]byte{0x41}, want...) + } + if got := x86PackedShiftActualGoBytes(t, "amd64", source); !bytes.Equal(got, want) { + t.Fatalf("Go's GP spelling must name the physical MMX bank: %x want %x", got, want) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + effect, recognized, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64") + if !recognized || err != nil || effect.operands[0].width != 8 || effect.operands[0].operand.Reg != Reg(fmt.Sprintf("M%d", index&7)) { + t.Fatal("pure effects confused a numeric GP with a physical MMX input", effect, err) + } + state := x86ResultState{flags: x86ByteFlagsAll} + for i := range state.gp { + state.gp[i] = 0xff + } + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("initialized GP bytes fabricated MMX input/output permission") + } + }) + } + } +} + +func TestX86ABI0ResultLegacyIntegerArithmeticPartialBytesAndFlags(t *testing.T) { + for _, op := range x86LegacyIntegerArithmeticTestNames { + ins := Instr{Op: Op(op), Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}} + for source := 0; source < 2; source++ { + for missing := 0; missing < 16; missing++ { + t.Run(fmt.Sprintf("%s/input%d/byte%d", op, source, missing), func(t *testing.T) { + state := x86ResultState{flags: x86ByteFlagsAll} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + state.xmm[1-source] &^= uint16(1) << uint(missing) + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("read-modify-write borrowed one missing input byte") + } + }) + } + } + for flags := uint8(0); flags <= x86ByteFlagsAll; flags++ { + state := x86ResultState{flags: flags} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + if !x86ResultTransfer(ins, &state, nil, 0, "amd64") || state.flags != flags || state.xmm[0] != 0xffff || state.xmm[1] != 0xffff { + t.Fatal("legacy lane arithmetic altered flags or byte aliases", op, flags, state) + } + } + body := "MOVSS a+0(FP),X1\nMOVL $1,AX\nCMPL AX,$0\nJE partial\nMOVSD a+0(FP),X0\nJMP joined\npartial:\nMOVSS X1,X0\njoined:\n" + op + " X1,X0\nMOVSD X0,ret0+8(FP)\nRET" + file, sig, _ := x86ResultAUTOFixture(t, "amd64", "func Value(a uint64) uint64", 64, body) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("partial destination path became initialized", op, err) + } + } +} + +func TestX86LegacyIntegerArithmeticThreeAPIFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + for _, op := range x86LegacyIntegerArithmeticTestNames { + t.Run(target.triple+"/"+op, func(t *testing.T) { + body := "MOVSD a+0(FP),X0\nMOVSD c+16(FP),X1\n" + op + " X1,X0\n" + op + " X0,X0\nMOVSD X0,ret0+32(FP)\nRET" + pkg, file, sig, source := x86ResultInitializationFixture(t, target.arch, "func Value(a,b,c,d uint64) uint64", body) + goAssemblerOracleOutput(t, target.arch, source, "", true) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, target.arch); err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Value": sig}, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + before := fmt.Sprintf("%#v %#v", file, sig) + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "text") + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, module.String(), "module") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: options.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, translated.Module.String(), "typed-go") + if before != fmt.Sprintf("%#v %#v", file, sig) { + t.Fatal("API changed original inputs") + } + }) + } + } +} + +func TestX86LegacyIntegerArithmeticMalformedFormsAndIdentity(t *testing.T) { + for _, op := range x86LegacyIntegerArithmeticTestNames { + for _, operands := range []string{"$1,X0", "X0,(AX)", "Y0,Y1", "Z0,Z1", "X16,X0", "(AX),(BX)", "X0,K1,X1"} { + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\n"+op+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + if _, recognized, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64"); !recognized || err == nil { + t.Fatal("malformed row acquired effects", ins, err) + } + } + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\n"+op+" X0,X1\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, identity := range []Op{"", Op(op), Op(strings.ToLower(op))} { + ins := file.Funcs[0].Instrs[1] + ins.Op = identity + if _, recognized, err := x86LegacyIntegerArithmeticProperties(Op(op), ins, "amd64"); !recognized || err != nil { + t.Fatal("matching dispatch lost compatibility", identity, err) + } + } + ins := file.Funcs[0].Instrs[1] + ins.Op = "MOVQ" + if _, recognized, err := x86TypedInstructionByteEffect(Op(op), ins, "amd64"); !recognized || err == nil { + t.Fatal("contradictory dispatch obtained effects", err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + var known bool + if strings.HasPrefix(op, "PADD") { + known, _, err = ctx.lowerPackedIntegerAdd(Op(op), ins) + } else { + known, _, err = ctx.lowerPackedIntegerSubtract(Op(op), ins) + } + if !known || err == nil || output.String() != before { + t.Fatal("contradictory dispatch changed IR before rejection", op, known, err) + } + for _, operands := range []string{"X8,X0", "X0,X15"} { + source := "TEXT probe(SB),4,$0-0\n" + op + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + if _, recognized, err := x86TypedInstructionByteEffect(ins.Op, ins, "386"); !recognized || err == nil { + t.Fatal("mode32 acquired an unavailable physical X bank", op, err) + } + } + } +} + +func TestX86ABI0ResultLegacyIntegerArithmeticDoesNotBorrowPermission(t *testing.T) { + for _, op := range x86LegacyIntegerArithmeticTestNames { + for name, body := range map[string]string{ + "source unknown": "MOVSD a+0(FP),X0\n" + op + " X2,X0\nMOVSD X0,ret0+32(FP)\nRET", + "old destination unknown": "MOVSD a+0(FP),X1\n" + op + " X1,X0\nMOVSD X0,ret0+32(FP)\nRET", + "native pointer": "MOVSD a+0(FP),X0\nMOVQ a+0(FP),AX\n" + op + " (AX),X0\nMOVSD X0,ret0+32(FP)\nRET", + "AUTO partial": "MOVSD a+0(FP),X0\nMOVQ $1,value-32(SP)\n" + op + " value-32(SP),X0\nMOVSD X0,ret0+32(FP)\nRET", + "FP gap or result": "MOVSD a+0(FP),X0\n" + op + " b+28(FP),X0\nMOVSD X0,ret0+32(FP)\nRET", + "call kills facts": "MOVSD a+0(FP),X0\nMOVSD c+16(FP),X1\nCALL ·Other(SB)\n" + op + " X1,X0\nMOVSD X0,ret0+32(FP)\nRET", + "dead unbound call": "MOVSD a+0(FP),X0\nMOVSD c+16(FP),X1\n" + op + " X1,X0\nMOVSD X0,ret0+32(FP)\nRET\nCALL ·Unknown(SB)\nRET", + "escape logical AUTO": "MOVSD a+0(FP),X0\nMOVSD c+16(FP),X1\nLEAQ value-32(SP),AX\n" + op + " X1,X0\nMOVSD X0,ret0+32(FP)\nRET", + } { + t.Run(op+"/"+name, func(t *testing.T) { + file, sig, _ := x86ResultAUTOFixture(t, "amd64", "func Value(a,b,c,d uint64) uint64", 64, body) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("pure arithmetic effects fabricated initialization or backing", err) + } + }) + } + t.Run(op+"/dead modeled timestamp", func(t *testing.T) { + // RDTSC has a complete implicit-output effect now. Retain its + // positive source-wide control instead of calling it unknown. + body := "MOVSD a+0(FP),X0\nMOVSD c+16(FP),X1\n" + op + + " X1,X0\nMOVSD X0,ret0+32(FP)\nRET\nRDTSC\nRET" + file, sig, _ := x86ResultAUTOFixture(t, "amd64", "func Value(a,b,c,d uint64) uint64", 64, body) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); err != nil { + t.Fatal("modeled dead timestamp lost its closed source effect", err) + } + }) + } +} diff --git a/amd64_legacy_integer_arithmetic_runtime_test.go b/amd64_legacy_integer_arithmetic_runtime_test.go new file mode 100644 index 000000000..1989326c5 --- /dev/null +++ b/amd64_legacy_integer_arithmetic_runtime_test.go @@ -0,0 +1,422 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Independent mathematical attributes, not emitted from the production specs. +var x86LegacyIntegerArithmeticTestBytes = []int{1, 2, 4, 4, 8, 1, 2, 1, 2, 1, 2, 4, 8, 1, 2, 1, 2} +var x86LegacyIntegerArithmeticTestModes = []int{0, 0, 0, 0, 0, 1, 1, 2, 2, 0, 0, 0, 0, 1, 1, 2, 2} + +const x86LegacyIntegerArithmeticGoMath = ` +func lane(value []byte) uint64 { + var result uint64 + for i,b := range value { result |= uint64(b)<max-b { return max } + return a+b + } + sign := int64(1)< sign-1 { result=sign-1 } + return uint64(result) +} +func fill(value []byte,n,sample,side int) { + max := ^uint64(0) + if n<8 { max=(uint64(1)<>1,(max>>1)+1,max-1,2,(max>>1)-1} + for offset:=0;offset<16;offset+=n { + number:=uint64(sample*71+offset*37+side*19)*0x11223344556677 + if sample<64 { number=choices[(sample/(1+side*7)+offset/n)%len(choices)] } + for j:=0;j>uint(j*8)) } + } +} +` + +const x86LegacyIntegerArithmeticCMath = ` +static uint64_t lane(const uint8_t *value,int n) { + uint64_t result=0; + for(int i=0;imax-b?max:a+b; + } + int64_t sign=INT64_C(1)<<(n*8-1),left=(int64_t)a,right=(int64_t)b; + if(left&sign) left-=sign*2; + if(right&sign) right-=sign*2; + int64_t result=subtract?right-left:right+left; + if(result< -sign) result= -sign; + if(result>sign-1) result=sign-1; + return (uint64_t)result; +} +static void fill(uint8_t *value,int n,int sample,int side) { + uint64_t max=n==8?UINT64_MAX:(UINT64_C(1)<<(n*8))-1; + uint64_t choices[]={0,1,max,max>>1,(max>>1)+1,max-1,2,(max>>1)-1}; + for(int offset=0;offset<16;offset+=n) { + uint64_t number=(uint64_t)(sample*71+offset*37+side*19)*UINT64_C(0x11223344556677); + if(sample<64) number=choices[(sample/(1+side*7)+offset/n)%8]; + for(int j=0;j>(j*8)); + } +} +` + +func TestCrossLinuxRuntimeMatrixX86LegacyIntegerArithmeticOriginalMathFlagsAndCanaries(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, goTable, cTable, cDeclarations strings.Builder + declarations.WriteString("package packedarithmetic\nfunc ReadCSR() uint32\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\n") + for index, op := range x86LegacyIntegerArithmeticTestNames { + for _, kind := range []string{"Register", "Alias", "Memory"} { + name := op + kind + source := "X1" + if kind == "Alias" { + source = "X0" + } else if kind == "Memory" { + source = "(AX)" + } + fmt.Fprintf(&declarations, "func %s(out,x,y *[16]byte,flags *uint64,seed uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-40\nMOVQ out+0(FP),DX\nMOVQ x+8(FP),AX\nMOVQ y+16(FP),CX\nMOVOU (AX),X1\nMOVOU (CX),X0\nPUSHQ seed+32(FP)\nPOPFQ\n%s %s,X0\nMOVOU X0,(DX)\nPUSHFQ\nPOPQ BX\nMOVQ flags+24(FP),SI\nMOVQ BX,(SI)\nRET\n", name, op, source) + fmt.Fprintf(&goTable, "{%q,%s,%d,%d,%t,%t},\n", name, name, + x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestModes[index], index >= 9, kind == "Alias") + fmt.Fprintf(&cDeclarations, "extern void %s(uint8_t*,uint8_t*,uint8_t*,uint64_t*,uint64_t);\n", name) + fmt.Fprintf(&cTable, "{%q,%s,%d,%d,%d,%d},\n", name, name, + x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestModes[index], x86FloatAddSubBoolInt(index >= 9), x86FloatAddSubBoolInt(kind == "Alias")) + } + } + goTest := `package packedarithmetic +import ( + "testing" + "unsafe" +) +` + x86LegacyIntegerArithmeticGoMath + ` +func aligned(value *[80]byte) (*[16]byte,int) { + offset:=16+int((-uintptr(unsafe.Pointer(&value[16])))&15) + return (*[16]byte)(unsafe.Pointer(&value[offset])),offset +} + +func TestOriginal(t *testing.T) { + kernels:=[]struct { + name string + call func(*[16]byte,*[16]byte,*[16]byte,*uint64,uint64) + n,mode int + subtract,alias bool + }{ +` + goTable.String() + ` } + for _,kernel:=range kernels { + for sample:=0;sample<257;sample++ { + var x,y,out [80]byte + xp,_:=aligned(&x);yp,_:=aligned(&y);op,offset:=aligned(&out) + for i:=range x { x[i]=0xb9;y[i]=0xa7 } + fill(xp[:],kernel.n,sample,0);fill(yp[:],kernel.n,sample,1) + originalX,originalY:=x,y + for _,seed:=range []uint64{0,0x8d5} { + for i:=range out { out[i]=0xa5 } + box:=struct{before,value,after uint64}{0x1122334455667788,0,0x8877665544332211} + csr:=ReadCSR() + kernel.call(op,xp,yp,&box.value,seed) + if box.value&0x8d5!=seed||box.before!=0x1122334455667788||box.after!=0x8877665544332211||ReadCSR()!=csr||x!=originalX||y!=originalY { + t.Fatalf("%s state/canary sample%d flags%x",kernel.name,sample,box.value) + } + var want [16]byte + for i:=0;i<16;i+=kernel.n { + a,b:=lane(xp[i:i+kernel.n]),lane(yp[i:i+kernel.n]) + if kernel.alias { a=b } + value:=math(a,b,kernel.n,kernel.mode,kernel.subtract) + for j:=0;j>uint(j*8)) } + } + for i,value:=range out { + expected:=byte(0xa5) + if i>=offset&&i +#include +#include +#include +struct Aligned { _Alignas(16) uint8_t before[16],value[16],after[16]; }; +` + cDeclarations.String() + x86LegacyIntegerArithmeticCMath + ` +static const struct Kernel { + const char *name; + void (*call)(uint8_t*,uint8_t*,uint8_t*,uint64_t*,uint64_t); + int n,mode,subtract,alias; +} kernels[]={ +` + cTable.String() + `}; +int main(void) { + for(unsigned k=0;kn,sample,0);fill(y.value,kernel->n,sample,1); + struct Aligned originalX=x,originalY=y; + for(int status=0;status<2;status++) { + uint64_t seed=status?UINT64_C(0x8d5):0; + memset(&out,0xa5,sizeof(out)); + struct {uint64_t before,value,after;} flags={UINT64_C(0x1122334455667788),0,UINT64_C(0x8877665544332211)}; + uint32_t csr=_mm_getcsr(); + kernel->call(out.value,x.value,y.value,&flags.value,seed); + if((flags.value&UINT64_C(0x8d5))!=seed||flags.before!=UINT64_C(0x1122334455667788)||flags.after!=UINT64_C(0x8877665544332211)||_mm_getcsr()!=csr||memcmp(&x,&originalX,sizeof(x))||memcmp(&y,&originalY,sizeof(y))) { + fprintf(stderr,"%s state/input/canary sample%d flags%llx\n",kernel->name,sample,(unsigned long long)flags.value);return 1; + } + for(int i=0;i<16;i+=kernel->n) { + uint64_t a=lane(x.value+i,kernel->n),b=lane(y.value+i,kernel->n); + if(kernel->alias) a=b; + uint64_t expected=math(a,b,kernel->n,kernel->mode,kernel->subtract); + for(int j=0;jn;j++) if(out.value[i+j]!=(uint8_t)(expected>>(j*8))) { + fprintf(stderr,"%s sample%d byte%d got%x want%x\n",kernel->name,sample,i+j,out.value[i+j],(unsigned)(uint8_t)(expected>>(j*8)));return 2; + } + } + for(int i=0;i<16;i++) if(out.before[i]!=0xa5||out.after[i]!=0xa5) return 3; + } + } + } + return 0; +} +` + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module packedarithmetic\n\ngo 1.20\n", "arithmetic.go": declarations.String(), + "arithmetic_amd64.s": assembly.String(), "arithmetic_test.go": goTest, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + binary := filepath.Join(dir, "original.test") + command := exec.Command("go", "test", "-p=2", "-vet=off", "-c", "-o", binary, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + x86FloatCompareCaptureCommand(t, command) + output := execution.run(t, binary, "-test.run=^TestOriginal$", "-test.count=1", "-test.v") + if !strings.Contains(string(output), "--- PASS: TestOriginal") || !strings.Contains(string(output), "PASS\n") { + t.Fatalf("original Go legacy arithmetic lacks successful terminal: %s", output) + } + pkg := mustGoPackage(t, "example.invalid/packedarithmetic", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }}) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "legacy-integer-arithmetic", ir, main, level) + }) + } + if !t.Failed() { + t.Log("51 original-Go/LLVM XMM kernels ×257 initialized boundary/random samples ×2 full-six-flag seeds, complete bytes/inputs/MXCSR/canaries; Darwin/Rosetta is not native AMD64/386 fault proof.") + } +} + +func TestCrossLinuxRuntimeMatrixX86LegacyIntegerArithmeticOriginalMMXBank(t *testing.T) { + // Go's GP-looking source is an MMX ModRM spelling, not an ABI GP input. + // This original-Go-only oracle does not initialize the LLVM/MMX observer. + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, table strings.Builder + declarations.WriteString("package packedmmx\nfunc ReadCSR() uint32\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\n") + spellings := []string{"M0", "M1", "M2", "M3", "M4", "M5", "M6", "M7", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + for opIndex, op := range x86LegacyIntegerArithmeticTestNames { + if !strings.HasPrefix(op, "PADD") || op == "PADDQ" { + continue + } + for index, spelling := range spellings { + name := fmt.Sprintf("%sBank%d", op, index) + fmt.Fprintf(&declarations, "func %s(a,b,c,d,e,f,v,h uint64,flags *uint64,seed uint64) uint64\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-88\n", name) + for bank, parameter := range []string{"a", "b", "c", "d", "e", "f", "v", "h"} { + fmt.Fprintf(&assembly, "MOVQ %s+%d(FP),M%d\n", parameter, bank*8, bank) + } + // Do not change native SP/BP/g. Other GP values deliberately do not + // match any initialized MMX value. + for _, reg := range []string{"AX", "CX", "DX", "BX", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R15"} { + fmt.Fprintf(&assembly, "MOVQ $0x5a5a5a5a,%s\n", reg) + } + fmt.Fprintf(&assembly, "PUSHQ seed+72(FP)\nPOPFQ\n%s %s,M7\nMOVQ M7,ret0+80(FP)\nEMMS\nPUSHFQ\nPOPQ AX\nMOVQ flags+64(FP),DX\nMOVQ AX,(DX)\nRET\n", op, spelling) + fmt.Fprintf(&table, "{%q,%s,%d,%d,%d},\n", name, name, index&7, + x86LegacyIntegerArithmeticTestBytes[opIndex], x86LegacyIntegerArithmeticTestModes[opIndex]) + } + } + goTest := `package packedmmx +import "testing" +` + x86LegacyIntegerArithmeticGoMath + ` +func TestOriginal(t *testing.T) { + kernels:=[]struct { + name string + call func(uint64,uint64,uint64,uint64,uint64,uint64,uint64,uint64,*uint64,uint64) uint64 + bank,n,mode int + }{ +` + table.String() + ` } + for _,kernel:=range kernels { + for sample:=0;sample<33;sample++ { + var values [8]uint64 + for bank:=range values { + var bytes [16]byte + fill(bytes[:],kernel.n,sample,bank) + values[bank]=lane(bytes[:8]) + } + var want uint64 + for offset:=0;offset<8;offset+=kernel.n { + mask:=(uint64(1)<>uint(offset*8))&mask + b:=(values[7]>>uint(offset*8))&mask + value:=math(a,b,kernel.n,kernel.mode,false)&mask + want|=value<\n#include \n#include \n" + x86LegacyIntegerArithmeticCMath) + constant := func(multiplier, addend int) string { + var values []string + for index := 0; index < 64; index++ { + values = append(values, fmt.Sprintf("i8 %d", byte(index*multiplier+addend))) + } + return "<" + strings.Join(values, ",") + ">" + } + for index, op := range x86LegacyIntegerArithmeticTestNames { + profiles[op+"Shadow"] = "" + ins := Instr{Op: Op(op), Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}} + fn := Func{Instrs: []Instr{{Op: "VMOVUPS", Args: []Operand{{Kind: OpReg, Reg: "Z0"}, {Kind: OpReg, Reg: "Z1"}}}, ins}} + ctx, output := newAMD64CtxWithFuncForTest(t, fn, FuncSig{Ret: Void}, nil) + ctx.goarch = "amd64" + if err := ctx.storeZ("Z0", constant(37, 11)); err != nil { + t.Fatal(err) + } + if err := ctx.storeZ("Z1", constant(19, 71)); err != nil { + t.Fatal(err) + } + known, term, err := ctx.lowerPackedIntegerAdd(ins.Op, ins) + if !known { + known, term, err = ctx.lowerPackedIntegerSubtract(ins.Op, ins) + } + if !known || term || err != nil { + t.Fatal("legacy lowerer did not consume typed X bank", known, term, err) + } + first, err := ctx.loadZ("Z0") + if err != nil { + t.Fatal(err) + } + second, err := ctx.loadZ("Z1") + if err != nil { + t.Fatal(err) + } + fmt.Fprintf(&ir, "define void @%sShadow(ptr %%out) {\n%s store <64 x i8> %s, ptr %%out, align 1\n %%next = getelementptr i8, ptr %%out, i64 64\n store <64 x i8> %s, ptr %%next, align 1\n ret void\n}\n", op, output.String(), first, second) + fmt.Fprintf(&reference, "extern void %sShadow(uint8_t*);\n", op) + fmt.Fprintf(&checks, ` { + memset(out,0xa5,sizeof(out)); + %sShadow(out+16); + uint8_t a[64],b[64]; + for(int i=0;i<64;i++){a[i]=(uint8_t)(i*19+71);b[i]=(uint8_t)(i*37+11);} + for(int offset=0;offset<16;offset+=%d){ + uint64_t value=math(lane(a+offset,%d),lane(b+offset,%d),%d,%d,%d); + for(int j=0;j<%d;j++)b[offset+j]=(uint8_t)(value>>(j*8)); + } + if(memcmp(out+16,b,64)||memcmp(out+80,a,64)){fprintf(stderr,"%s alias\n");return 1;} + for(int i=0;i<16;i++)if(out[i]!=0xa5||out[144+i]!=0xa5)return 2; + } +`, op, x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestModes[index], x86FloatAddSubBoolInt(index >= 9), x86LegacyIntegerArithmeticTestBytes[index], op) + } + reference.WriteString("int main(void){uint8_t out[160];\n" + checks.String() + "return 0;}\n") + // The private shadow has no ISA attributes. Use the same strict body guard + // as public portable oracles so new native effects cannot silently enter it. + text, err := x86PackedShiftPortableFeatureProjection(ir.String(), profiles) + if err != nil { + t.Fatal(err) + } + t.Run("amd64-object", func(t *testing.T) { + const triple = "x86_64-unknown-linux-gnu" + crossIR := strings.Replace(text, "target triple = \""+execution.triple+"\"\n", + "target triple = \""+triple+"\"\n", 1) + x86ResultInitializationObject(t, execution.llc, triple, crossIR, "legacy-integer-arithmetic-alias") + }) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "legacy-integer-arithmetic-alias", text, reference.String(), level) + }) + } + t.Log("17 historical software-shadow operations retain all 128 alias bytes and canaries; host execution and AMD64 object, not public/native opcode execution") +} diff --git a/amd64_legacy_logical_memory_linux_amd64_test.go b/amd64_legacy_logical_memory_linux_amd64_test.go new file mode 100644 index 000000000..df2003b39 --- /dev/null +++ b/amd64_legacy_logical_memory_linux_amd64_test.go @@ -0,0 +1,9 @@ +package plan9asm + +import "testing" + +// Required by the cross-runtime CI prefix. The native Linux axis additionally +// checks the original aligned-m128 contract, which Rosetta does not reproduce. +func TestCrossLinuxRuntimeMatrixX86LegacyLogicalMemoryNativeFaults(t *testing.T) { + x86LegacyLogicalMemoryRuntime(t, true) +} diff --git a/amd64_legacy_logical_memory_runtime_unix_test.go b/amd64_legacy_logical_memory_runtime_unix_test.go new file mode 100644 index 000000000..581ae33a7 --- /dev/null +++ b/amd64_legacy_logical_memory_runtime_unix_test.go @@ -0,0 +1,143 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Execute only first-party, initialized storage. The original reference uses +// actual stock-Go opcode bytes, not a compiler intrinsic or reconstructed math. +// Rosetta is a diagnostic transport; required Linux/amd64 executes this root. +func TestCrossLinuxRuntimeMatrixX86LegacyLogicalMemoryOriginalFaultsAndBytes(t *testing.T) { + x86LegacyLogicalMemoryRuntime(t, false) +} + +func x86LegacyLogicalMemoryRuntime(t *testing.T, nativeAlignment bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, kernels, checks strings.Builder + signatures := map[string]FuncSig{} + for index, name := range x86LegacyLogicalMemoryNames { + code := x86LegacyLogicalOriginalBytes(t, "amd64", name) + literal := make([]string, len(code)) + for i, value := range code { + literal[i] = fmt.Sprintf("0x%02x", value) + } + fmt.Fprintf(&kernels, ` +static __attribute__((noinline)) void original%d(uint8_t *out,const uint8_t *a,const uint8_t *b) { + __asm__ volatile("movups (%%1),%%%%xmm0\n\tstc\n\t.byte %s\n\tmovups %%%%xmm0,(%%2)\n\tsetc 16(%%2)" + : : "a"(a),"r"(b),"r"(out) : "xmm0","memory","cc"); +} +`, index, strings.Join(literal, ",")) + for _, dead := range []bool{false, true} { + fn := fmt.Sprintf("logical%d_%t", index, dead) + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVUPS (CX),X0\nSTC\n%s (BX),X0\n", fn, name) + if !dead { + assembly.WriteString("MOVUPS X0,(AX)\n") + } + assembly.WriteString("SETCS 16(AX)\nRET\n") + signatures[fn] = FuncSig{Name: fn, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*);\n", fn) + } + mode := []int{0, 0, 1, 1, 2, 2, 3, 3, 0, 1, 2, 3}[index] + fmt.Fprintf(&checks, `{ + if(check_values(original%[1]d,logical%[1]d_false,%[2]d)) return %[3]d; + for(int kind=%[5]d;kind<3;++kind) { + int original_fault=observe_fault(original%[1]d,kind); + if(original_fault!=SIGSEGV && original_fault!=SIGBUS) { fprintf(stderr,"%[4]s original fault kind%%d=%%d\n",kind,original_fault); return %[3]d; } + if(observe_fault(logical%[1]d_false,kind)!=original_fault || observe_fault(logical%[1]d_true,kind)!=original_fault) return %[3]d; + } +} +`, index, mode, index+10, name, map[bool]int{false: 1, true: 0}[nativeAlignment]) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: signatures}) + if err != nil { + t.Fatal(err) + } + const helper = ` +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef void (*kernel)(uint8_t*,const uint8_t*,const uint8_t*); +static int observe_fault(kernel call,int kind) { + _Alignas(16) uint8_t a[48],b[48],out[48]; + memset(a,0x35,sizeof(a)); memset(b,0x71,sizeof(b)); memset(out,0xa7,sizeof(out)); + long page=sysconf(_SC_PAGESIZE); + if(page<=0) return -3; + uint8_t *area=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_ANON|MAP_PRIVATE,-1,0); + if(area==MAP_FAILED) return -4; + if(mprotect(area+page,(size_t)page,PROT_NONE)) { munmap(area,(size_t)page*2); return -5; } + pid_t child=fork(); if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + const uint8_t *source=kind==0?a+1:kind==1?0:area+page; + call(out+1,source,b+1); + _exit(91); + } + int status=0; + if(waitpid(child,&status,0)!=child) return -2; + if(munmap(area,(size_t)page*2)) return -6; + return WIFSIGNALED(status)?WTERMSIG(status):-(WEXITSTATUS(status)+100); +} +static int check_values(kernel original,kernel translated,int mode) { + for(int sample=0;sample<257;++sample) for(int rc=0;rc<4;++rc) { + _Alignas(16) uint8_t a[48],b[48],go[48],llvm[48]; + memset(a,0xa7,sizeof(a)); memset(b,0xb9,sizeof(b)); + memset(go,0xc3,sizeof(go)); memset(llvm,0xc3,sizeof(llvm)); + for(int i=0;i<16;++i) { a[i+16]=(uint8_t)(sample*71+i*19); b[i+1]=(uint8_t)(sample*31-i*59); } + uint8_t original_a[48],original_b[48]; memcpy(original_a,a,48); memcpy(original_b,b,48); + uint32_t saved=_mm_getcsr(),csr=0x1f80|(rc<<13)|(sample&63); + _mm_setcsr(csr); original(go+1,a+16,b+1); uint32_t go_csr=_mm_getcsr(); + _mm_setcsr(csr); translated(llvm+1,a+16,b+1); uint32_t llvm_csr=_mm_getcsr(); _mm_setcsr(saved); + if(go_csr!=csr || llvm_csr!=csr || memcmp(a,original_a,48) || memcmp(b,original_b,48) || memcmp(go,llvm,48)) return 1; + if(go[17]!=1) return 2; + for(int i=0;i<48;++i) if((i<1 || i>17) && go[i]!=0xc3) return 2; + for(int i=0;i<16;++i) { + uint8_t x=a[i+16],y=b[i+1],want; + /* Two PS/PD encodings per operation; the last four are integer aliases. */ + if(mode==0) want=x&y; + else if(mode==1) want=x&(uint8_t)~y; + else if(mode==2) want=x|y; + else if(mode==3) want=x^y; + else return 3; + if(go[i+1]!=want) return 4; + } + } + return 0; +} +` + driver := helper + kernels.String() + declarations.String() + "int main(void) {\n" + checks.String() + "return 0;\n}\n" + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "legacy-logical-native", ir, driver, level) + }) + } + t.Logf("12 original memory opcodes: 257×4 initialized byte/CF/MXCSR/canary inputs; live/dead NULL+guard-page faults at O0/O2; native alignment=%t; host=%s/%s", nativeAlignment, runtime.GOOS, runtime.GOARCH) +} + +func TestX86LegacyLogicalMemoryFaultChildDisablesCore(t *testing.T) { + if err := runtimeOracleFaultCorePolicy(runtimeOracleFaultC(t, "amd64_legacy_logical_memory_runtime_unix_test.go")); err != nil { + t.Fatal(err) + } +} diff --git a/amd64_legacy_logical_memory_test.go b/amd64_legacy_logical_memory_test.go new file mode 100644 index 000000000..f0d5fe2f1 --- /dev/null +++ b/amd64_legacy_logical_memory_test.go @@ -0,0 +1,312 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func x86LegacyLogicalOriginalBytes(t *testing.T, arch, name string) []byte { + t.Helper() + listing := x86InsertExtractGoListing(t, arch, "TEXT original(SB),4,$0-0\n"+name+" (AX),X0\nRET\n") + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + // Independent Intel encoding, not obtained from the translator's spec. + opcode := map[string]byte{ + "ANDPS": 0x54, "ANDPD": 0x54, "ANDNPS": 0x55, "ANDNPD": 0x55, + "ORPS": 0x56, "ORPD": 0x56, "XORPS": 0x57, "XORPD": 0x57, + "PAND": 0xdb, "PANDN": 0xdf, "POR": 0xeb, "PXOR": 0xef, + }[name] + want := []byte{0x0f, opcode, 0x00, 0xc3} + if strings.HasSuffix(name, "PD") || strings.HasPrefix(name, "P") { + want = append([]byte{0x66}, want...) + } + if !bytes.Equal(code, want) { + t.Fatalf("%s/%s actual Go memory opcode=%x independent encoding=%x", arch, name, code, want) + } + t.Logf("%s/%s original complete opcode+RET=%x", arch, name, code) + return append([]byte(nil), code[:len(code)-1]...) +} + +func TestX86LegacyLogicalMemoryOriginalGoBytes(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, name := range x86LegacyLogicalMemoryNames { + t.Run(arch+"/"+name, func(t *testing.T) { x86LegacyLogicalOriginalBytes(t, arch, name) }) + } + } +} + +func TestX86LegacyLogicalMemoryRequiredNativeSelection(t *testing.T) { + const file = "amd64_legacy_logical_memory_linux_amd64_test.go" + for _, target := range []struct{ os, arch string }{ + {"linux", "amd64"}, {"linux", "arm64"}, {"linux", "386"}, + {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, + } { + context := build.Default + context.GOOS, context.GOARCH = target.os, target.arch + selected, err := context.MatchFile(".", file) + if err != nil || selected != (target.os == "linux" && target.arch == "amd64") { + t.Fatalf("stock Go required native selection %s/%s: %t/%v", target.os, target.arch, selected, err) + } + } + syntax, err := parser.ParseFile(token.NewFileSet(), file, nil, 0) + if err != nil { + t.Fatal(err) + } + if len(syntax.Decls) != 2 { + t.Fatal("native wrapper must contain only its import and required root") + } + function, ok := syntax.Decls[1].(*ast.FuncDecl) + if !ok || function.Name.Name != "TestCrossLinuxRuntimeMatrixX86LegacyLogicalMemoryNativeFaults" || len(function.Body.List) != 1 { + t.Fatal("native root registration changed or became optional") + } + statement, ok := function.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("native root bypassed its mandatory helper") + } + call, ok := statement.X.(*ast.CallExpr) + if !ok || len(call.Args) != 2 { + t.Fatal("native root must call the full fault helper") + } + callee, ok := call.Fun.(*ast.Ident) + if !ok || callee.Name != "x86LegacyLogicalMemoryRuntime" { + t.Fatal("native root changed helper") + } + argument, ok := call.Args[1].(*ast.Ident) + if !ok || argument.Name != "true" { + t.Fatal("native root omitted its alignment axis") + } + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil || !strings.Contains(string(workflow), "-ci-selector '^(TestCrossLinuxRuntimeMatrix|") { + t.Fatalf("required cross-runtime prefix missing: %v", err) + } +} + +func TestX86LegacyLogicalMemoryUnboundAddressSizeBeforeEmission(t *testing.T) { + for _, name := range x86LegacyLogicalMemoryNames { + file, err := Parse(ArchAMD64, "TEXT view(SB),4,$0-0\n"+name+" (AX),X0\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, bits := range []int{32, 64} { + ins := file.Funcs[0].Instrs[1] + ins.x86AddressBits = bits + var output strings.Builder + context := &amd64Ctx{goarch: "amd64", b: &output} + known, terminal, err := context.lowerPackedLogicalFamily(ins.Op, ins) + if !known || terminal || !errors.Is(err, ErrProbeNeedsContext) || output.Len() != 0 { + t.Fatalf("%s unbound address-size%d emitted IR: %t/%t/%v", name, bits, known, terminal, err) + } + } + } +} + +// Independent legacy namespace: these twelve Go names all have an m128 +// source whose original SSE instruction requires aligned native storage. +var x86LegacyLogicalMemoryNames = []string{ + "ANDPS", "ANDPD", "ANDNPS", "ANDNPD", "ORPS", "ORPD", + "XORPS", "XORPD", "PAND", "PANDN", "POR", "PXOR", +} + +var x86LegacyLogicalMemoryTargets = []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, +} + +func TestX86LegacyLogicalMemoryFPNeedsNativeBacking(t *testing.T) { + for _, target := range x86LegacyLogicalMemoryTargets { + for _, name := range x86LegacyLogicalMemoryNames { + source := "TEXT ·view(SB),4,$0-16\nMOVOU a+0(FP),X0\n" + name + " a+0(FP),X0\nRET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+name+"/"+api, func(t *testing.T) { + _, err := x86ScalarFPBitViewAPI(t, source, "func view(a,b uint64)", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "original native storage") { + t.Fatalf("independent FP bytes fabricated native m128 backing/alignment: %v", err) + } + }) + } + } + } +} + +func TestX86LegacyLogicalMemoryVirtualStackNeedsCertificate(t *testing.T) { + for _, target := range x86LegacyLogicalMemoryTargets { + for _, name := range x86LegacyLogicalMemoryNames { + for _, address := range []string{"16(SP)", "16(BP)", "16(AX)(BP*1)"} { + source := x86OrdinaryBPNativeSource(target.arch, "view", name+" "+address+",X0", address, 0) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+name+"/"+address+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *byte)", target.arch, target.triple, api) + x86RequireOrdinaryBPNativeOutcome(t, target.arch, address, ir, err) + }) + } + } + } + } +} + +func TestX86LegacyLogicalMemoryNativeOpcodeObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86LegacyLogicalMemoryTargets { + var source, ordinary, declaration strings.Builder + source.WriteString("GLOBL ·data(SB),8,$16\n") + ordinary.WriteString("GLOBL ·data(SB),8,$16\n") + declaration.WriteString("var data [16]byte\n") + for _, name := range x86LegacyLogicalMemoryNames { + move, size := "MOVQ", 8 + if target.arch == "386" { + move, size = "MOVL", 4 + } + for i, address := range []string{"(AX)", "16(AX)(CX*2)", "·data(SB)", "(FS)", "(GS)"} { + fn := fmt.Sprintf("%s%d", strings.ToLower(name), i) + body := fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n"+ + "%s p+0(FP),AX\n%s index+%d(FP),CX\nPXOR X0,X0\n%s %s,X0\nRET\n", + fn, size*2, move, move, size, name, address) + source.WriteString(body) + if i < 3 { + ordinary.WriteString(body) + } + fmt.Fprintf(&declaration, "func %s(p *byte,index uintptr)\n", fn) + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "logical", "package logical\n"+declaration.String()) + resolve := func(name string) string { return strings.TrimPrefix(name, "·") } + signatures, err := goSigsForAsmFile(pkg, file, resolve, target.arch, nil, nil, nil) + if err != nil || len(file.Funcs) != 60 { + t.Fatal("original logical-memory format inventory changed", err) + } + // Original FS/GS operands have no Go TLS/segment contract. Keep all + // sixty formats as emitter objects and public Context, not public success. + formatIR := x86ScalarFormatObjectUnit(t, file, signatures, target.arch, target.triple, 0) + for _, name := range x86LegacyLogicalMemoryNames { + for _, segment := range []string{"fs", "gs"} { + want := "asm sideeffect \"" + strings.ToLower(name) + " %" + segment + ":($2), $0\"" + if !strings.Contains(formatIR, want) { + t.Fatal("original native segment format lost its prefix", name, segment) + } + } + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, formatIR, level) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, ordinary.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declaration.String(), target.arch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original segmented source acquired an unbound TLS contract", api, err) + } + ir, err = x86ScalarFPBitViewAPI(t, ordinary.String(), declaration.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + for _, name := range x86LegacyLogicalMemoryNames { + want := "asm sideeffect \"" + strings.ToLower(name) + " ($2), $0\", \"=x,0,r,~{memory}\"" + if !strings.Contains(ir, want) { + t.Fatalf("dead result lost original single native %s memory effect", name) + } + } + compileLLVMToObject(t, llc, target.triple, "logical-native.ll", "logical-native.o", ir) + }) + } + } +} + +func TestX86LegacyLogicalMemoryPSRetainsSSE1Carrier(t *testing.T) { + var source strings.Builder + for _, name := range []string{"ANDPS", "ANDNPS", "ORPS", "XORPS"} { + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\nMOVUPS (AX),X0\n%s (AX),X0\nMOVUPS X0,(BX)\nRET\n", strings.ToLower(name), name) + } + file, signatures := x86UnboundVoidFormatSources(t, source.String(), "386", "i386-unknown-linux-gnu") + ir := x86ScalarFormatObjectUnit(t, file, signatures, "386", "i386-unknown-linux-gnu", 0) + x86LegacyLogicalSSE1Objects(t, ir) +} + +func TestX86LegacyLogicalMemoryPSInitializedSSE1Objects(t *testing.T) { + var source, declaration strings.Builder + for _, name := range []string{"ANDPS", "ANDNPS", "ORPS", "XORPS"} { + fn := strings.ToLower(name) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-8\n"+ + "MOVL p+0(FP),AX\nMOVL out+4(FP),BX\nMOVUPS (AX),X0\n"+ + "%s (AX),X0\nMOVUPS X0,(BX)\nRET\n", fn, name) + fmt.Fprintf(&declaration, "func %s(p,out *byte)\n", fn) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + x86LEAActualTargetGoObject(t, "386", triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(triple+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declaration.String(), "386", triple, api) + if err != nil { + t.Fatal("initialized SSE1 native-memory program", err) + } + x86LegacyLogicalSSE1Objects(t, ir) + }) + } + } +} + +func x86LegacyLogicalSSE1Objects(t *testing.T, ir string) { + t.Helper() + llc, objdump := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || objdump == "" { + t.Fatal("LLVM22 llc and objdump required") + } + if !strings.Contains(ir, "call <4 x float> asm sideeffect") || strings.Contains(ir, "+sse2") { + t.Fatal("PS native carrier silently requires SSE2") + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + directory := t.TempDir() + input, object := filepath.Join(directory, "sse1.ll"), filepath.Join(directory, "sse1.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, llc, level, "-filetype=obj", input, "-o", object)) + cancel() + if err != nil { + t.Fatalf("SSE1 object: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + output, err = runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, objdump, "-d", object)) + cancel() + if err != nil { + t.Fatalf("SSE1 disassembly: %v\n%s", err, output) + } + for _, name := range []string{"andps", "andnps", "orps", "xorps"} { + if !strings.Contains(string(output), name) { + t.Fatalf("SSE1 object lost %s", name) + } + } + for _, forbidden := range []string{"movdqa", "movdqu", "pand", "pxor", "por", "movapsd", "movq\t%xmm"} { + if strings.Contains(string(output), forbidden) { + t.Fatalf("SSE1 source introduced %s:\n%s", forbidden, output) + } + } + }) + } +} diff --git a/amd64_legacy_mmx_bank.go b/amd64_legacy_mmx_bank.go new file mode 100644 index 000000000..30d108a22 --- /dev/null +++ b/amd64_legacy_mmx_bank.go @@ -0,0 +1,233 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86LegacyMMXTable uint8 + +const ( + x86LegacyMMXBinary x86LegacyMMXTable = iota + x86LegacyMMXHorizontal + x86LegacyMMXShuffle + x86LegacyMMXCount +) + +type x86LegacyMMXReadSpec struct { + table x86LegacyMMXTable + opcode, immediateOpcode, opcodeDigit byte +} + +// This is the complete integer MMX-read class in Go's ymm, ymmxmm0f38, +// ymshuf and yps tables. MOVQ, PINSR, PEXTR and floating conversions have +// different ordered operand roles and must not use this bank selector. +var x86LegacyMMXReadSpecs = map[Op]x86LegacyMMXReadSpec{ + "PACKSSLW": {opcode: 0x6b}, + "PACKSSWB": {opcode: 0x63}, + "PACKUSWB": {opcode: 0x67}, + "PADDB": {opcode: 0xfc}, + "PADDL": {opcode: 0xfe}, + "PADDW": {opcode: 0xfd}, + "PADDSB": {opcode: 0xec}, + "PADDSW": {opcode: 0xed}, + "PADDUSB": {opcode: 0xdc}, + "PADDUSW": {opcode: 0xdd}, + "PAND": {opcode: 0xdb}, + "PANDN": {opcode: 0xdf}, + "POR": {opcode: 0xeb}, + "PXOR": {opcode: 0xef}, + "PAVGB": {opcode: 0xe0}, + "PAVGW": {opcode: 0xe3}, + "PCMPEQB": {opcode: 0x74}, + "PCMPEQW": {opcode: 0x75}, + "PCMPEQL": {opcode: 0x76}, + "PCMPGTB": {opcode: 0x64}, + "PCMPGTW": {opcode: 0x65}, + "PCMPGTL": {opcode: 0x66}, + "PMADDWL": {opcode: 0xf5}, + "PMULLW": {opcode: 0xd5}, + "PMULHW": {opcode: 0xe5}, + "PMULHUW": {opcode: 0xe4}, + "PMULULQ": {opcode: 0xf4}, + "PUNPCKHBW": {opcode: 0x68}, + "PUNPCKHWL": {opcode: 0x69}, + "PUNPCKHLQ": {opcode: 0x6a}, + "PUNPCKLBW": {opcode: 0x60}, + "PUNPCKLWL": {opcode: 0x61}, + "PUNPCKLLQ": {opcode: 0x62}, + "PHADDD": {table: x86LegacyMMXHorizontal, opcode: 0x02}, + "PSHUFW": {table: x86LegacyMMXShuffle, opcode: 0x70}, + "PSLLW": {x86LegacyMMXCount, 0xf1, 0x71, 6}, + "PSLLL": {x86LegacyMMXCount, 0xf2, 0x72, 6}, + "PSLLQ": {x86LegacyMMXCount, 0xf3, 0x73, 6}, + "PSRLW": {x86LegacyMMXCount, 0xd1, 0x71, 2}, + "PSRLL": {x86LegacyMMXCount, 0xd2, 0x72, 2}, + "PSRLQ": {x86LegacyMMXCount, 0xd3, 0x73, 2}, + "PSRAW": {x86LegacyMMXCount, 0xe1, 0x71, 4}, + "PSRAL": {x86LegacyMMXCount, 0xe2, 0x72, 4}, +} + +type x86LegacyMMXEncodingRow struct { + classes []string + zcase string + zoffset, byteOffset int +} + +func (s x86LegacyMMXReadSpec) encoderRows() (string, string, []byte, []x86LegacyMMXEncodingRow) { + switch s.table { + case x86LegacyMMXBinary: + return "ymm", "Py1", []byte{s.opcode, 0x66, s.opcode}, []x86LegacyMMXEncodingRow{ + {[]string{"Ymm", "Ymr"}, "Zm_r_xm", 1, 0}, + {[]string{"Yxm", "Yxr"}, "Zm_r_xm", 2, 1}, + } + case x86LegacyMMXHorizontal: + // Go advances z by zoffset+xo; the initial 0x0f makes xo=1. + return "ymmxmm0f38", "Px", []byte{0x0f, 0x38, s.opcode, 0, 0x66, 0x0f, 0x38, s.opcode, 0}, []x86LegacyMMXEncodingRow{ + {[]string{"Ymm", "Ymr"}, "Zlitm_r", 3, 0}, + {[]string{"Yxm", "Yxr"}, "Zlitm_r", 5, 4}, + } + case x86LegacyMMXShuffle: + return "ymshuf", "Pm", []byte{s.opcode, 0}, []x86LegacyMMXEncodingRow{ + {[]string{"Yi8", "Ymm", "Ymr"}, "Zibm_r", 2, 0}, + } + default: + return "yps", "Py3", []byte{s.opcode, s.immediateOpcode, s.opcodeDigit, + 0x66, s.opcode, 0x66, s.immediateOpcode, s.opcodeDigit}, []x86LegacyMMXEncodingRow{ + {[]string{"Ymm", "Ymr"}, "Zm_r_xm", 1, 0}, + {[]string{"Yi8", "Ymr"}, "Zibo_m_xm", 2, 1}, + {[]string{"Yxm", "Yxr"}, "Zm_r_xm", 2, 3}, + {[]string{"Yi8", "Yxr"}, "Zibo_m_xm", 3, 5}, + } + } +} + +type x86LegacyMMXReadForm struct { + canonical Op + spec x86LegacyMMXReadSpec + row int + sourceIndex, destination int + source Operand + byteWidth, memoryBytes int + mmx, gpSpelling bool +} + +func x86LegacyMMXGPBank(reg Reg, arch string) (Reg, bool) { + for index, register := range [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI, + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + if reg == register && (arch == "amd64" || arch == "386" && index < 8) { + return [...]Reg{"M0", "M1", "M2", "M3", "M4", "M5", "M6", "M7"}[index&7], true + } + } + return "", false +} + +// The selected Go row records physical source identity only. It does not +// define any M register, authorize a native address, model x87 state or +// supply an incoming/caller contract. Raw physical instructions never turn +// a GP source into MMX through Go's source-only ycover compatibility. +func x86LegacyMMXReadProperties(ins Instr, arch string) (x86LegacyMMXReadForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + if name == "PADDD" { + name = "PADDL" + } + spec, known := x86LegacyMMXReadSpecs[Op(name)] + if !known { + return x86LegacyMMXReadForm{}, false, nil + } + form := x86LegacyMMXReadForm{canonical: Op(name), spec: spec, sourceIndex: 0, + destination: len(ins.Args) - 1, byteWidth: 16} + fail := func() (x86LegacyMMXReadForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go integer Ymm read rows: %q", arch, ins.Op, ins.Raw) + } + want := 2 + if spec.table == x86LegacyMMXShuffle { + want, form.sourceIndex = 3, 1 + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != want || ins.Args[form.destination].Kind != OpReg { + return fail() + } + destination := ins.Args[form.destination] + _, form.mmx = amd64ParseMReg(destination.Reg) + if form.mmx { + form.byteWidth = 8 + if arch == "386" && !ins.x86Encoded && (spec.table == x86LegacyMMXBinary || spec.table == x86LegacyMMXCount) { + return fail() + } + } else { + index, valid := amd64ParseXReg(destination.Reg) + if !valid || index >= 16 || arch == "386" && index >= 8 || spec.table == x86LegacyMMXShuffle { + return fail() + } + } + form.source = ins.Args[form.sourceIndex] + immediate := spec.table == x86LegacyMMXCount && form.source.Kind == OpImm + if spec.table == x86LegacyMMXShuffle || immediate { + operand, integer := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !integer || !amd64SignedImmediate(operand, 8) { + return fail() + } + } + switch { + case immediate: + // An immediate count has no source-bank read. + case form.source.Kind == OpReg && form.mmx: + if _, valid := amd64ParseMReg(form.source.Reg); !valid { + register, valid := x86LegacyMMXGPBank(form.source.Reg, arch) + if !valid || ins.x86Encoded { + return fail() + } + form.source.Reg, form.gpSpelling = register, true + } + case form.source.Kind == OpReg: + index, valid := amd64ParseXReg(form.source.Reg) + if !valid || index >= 16 || arch == "386" && index >= 8 { + return fail() + } + case isAMD64MemoryOperand(form.source) && x86PackedShiftMemoryValid(form.source, arch): + form.memoryBytes = form.byteWidth + if form.mmx && (name == "PUNPCKLBW" || name == "PUNPCKLWL" || name == "PUNPCKLLQ") { + form.memoryBytes = 4 + } + default: + return fail() + } + if spec.table == x86LegacyMMXCount { + if form.mmx { + if immediate { + form.row = 1 + } + } else { + form.row = 2 + if immediate { + form.row = 3 + } + } + } else if !form.mmx { + form.row = 1 + } + return form, true, nil +} + +func x86LegacyMMXReadInstruction(ins Instr, arch string) (Instr, error) { + form, known, err := x86LegacyMMXReadProperties(ins, arch) + if !known || err != nil { + return ins, err + } + local := ins + local.Args = append([]Operand(nil), ins.Args...) + local.Args[form.sourceIndex] = form.source + return local, nil +} + +func x86LegacyMMXReadEmitterContext(ins Instr, arch string) error { + form, known, err := x86LegacyMMXReadProperties(ins, arch) + if !known || err != nil { + return err + } + if form.gpSpelling { + return fmt.Errorf("%w: %s source %s encodes %s without a proved initialized physical MMX bank transport", + ErrProbeNeedsContext, ins.Op, ins.Args[form.sourceIndex].Reg, form.source.Reg) + } + return nil +} diff --git a/amd64_legacy_mmx_bank_test.go b/amd64_legacy_mmx_bank_test.go new file mode 100644 index 000000000..7bf5b6bde --- /dev/null +++ b/amd64_legacy_mmx_bank_test.go @@ -0,0 +1,549 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +// This census is independent of the implementation. Ordered Go tables, +// opcode prefixes and all opBytes remain available to each comparison. +type x86YMMOriginalRule struct { + name, table, prefix, opBytes string +} + +func x86YMMOriginalRules(t *testing.T) []x86YMMOriginalRule { + t.Helper() + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + var rules []x86YMMOriginalRule + pattern := regexp.MustCompile(`\{A([A-Z0-9]+), (ymm|ymmxmm0f38|ymshuf|yps), (\w+), opBytes\{([^}]+)\}\}`) + for _, row := range pattern.FindAllStringSubmatch(string(source), -1) { + rules = append(rules, x86YMMOriginalRule{row[1], row[2], row[3], row[4]}) + } + if len(rules) != 43 { + t.Fatalf("complete integer MMX-read rule census changed: %d", len(rules)) + } + return rules +} + +func x86YMMReadSource(t *testing.T, row string) Instr { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + return file.Funcs[0].Instrs[1] +} + +func TestX86YMMExistingDescriptorsOriginalBank(t *testing.T) { + var names []string + for _, rule := range x86YMMOriginalRules(t) { + if strings.HasPrefix(rule.name, "PUNPCK") || rule.name == "PAND" || rule.name == "PANDN" || + rule.name == "POR" || rule.name == "PXOR" || rule.table == "yps" && !strings.HasPrefix(rule.name, "PSRA") { + names = append(names, rule.name) + } + } + sort.Strings(names) + if len(names) != 16 { + t.Fatal("logical/unpack/uniform logical-shift descriptor class changed", names) + } + for _, name := range names { + for index, source := range []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + t.Run(name+"/"+source, func(t *testing.T) { + ins := x86YMMReadSource(t, name+" "+source+",M1") + before := fmt.Sprintf("%#v", ins) + var known bool + var err error + var selected Operand + switch { + case strings.HasPrefix(name, "PUNPCK"): + var form x86PackedUnpackForm + form, known, err = x86PackedUnpackProperties(ins, "amd64") + selected = form.source + case strings.HasPrefix(name, "PS"): + var form x86PackedShiftForm + form, known, err = x86PackedShiftProperties(ins.Op, ins, "amd64") + selected = form.countOperand + default: + var form x86PackedLogicalForm + form, known, err = x86PackedLogicalProperties(ins, "amd64") + selected = form.source + } + if !known || err != nil { + t.Errorf("actual Go MMX-read spelling is absent from pure descriptor: %t/%v", known, err) + } + if err == nil && (selected.Kind != OpReg || selected.Reg != Reg(fmt.Sprintf("M%d", index&7))) { + t.Errorf("family descriptor retained the source GP bank: %+v", selected) + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("pure descriptor mutated original source") + } + }) + } + } +} + +func TestX86YMMOriginalGoIntegerReadBanks(t *testing.T) { + registers := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15", "M0", "M7"} + var rows []string + var expected []x86asm.Reg + for _, rule := range x86YMMOriginalRules(t) { + for index, source := range registers { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + rows = append(rows, rule.name+" "+prefix+source+",M1") + bank := index & 7 + if source == "M0" { + bank = 0 + } else if source == "M7" { + bank = 7 + } + expected = append(expected, x86asm.M0+x86asm.Reg(bank)) + } + } + // Alias has independent original Go evidence, not a prefix-name inference. + rows = append(rows, "PADDD AX,M1") + expected = append(expected, x86asm.M0) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+strings.Join(rows, "\n")+"\nRET\n") + t.Logf("complete original Go bytes=%x SHA256=%x", code, sha256.Sum256(code)) + for index, row := range rows { + physical, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal("original object cannot be decoded", row, err) + } + if physical.Args[0] != x86asm.M1 || physical.Args[1] != expected[index] { + t.Fatalf("original Go selected different physical register bank: %s bytes=%x operands=%v expected M1/%s", + row, code[:physical.Len], physical.Args, expected[index]) + } + decoded, err := decodeX86RawDirectiveGroup(code[:physical.Len], 64, 0, row, map[string]bool{}) + if err != nil || len(decoded) != 1 { + t.Fatal("actual physical Go instruction did not decode as one typed row", row, decoded, err) + } + form, known, err := x86LegacyMMXReadProperties(decoded[0], "amd64") + if !known || err != nil || form.gpSpelling || form.source.Reg != Reg(expected[index].String()) { + t.Fatal("raw decoder changed the actual MMX bank into a source spelling", row, decoded, form, known, err) + } + code = code[physical.Len:] + } + if len(code) == 0 || code[0] != 0xc3 || !bytes.Equal(bytes.Trim(code[1:], "\x00"), nil) { + t.Fatalf("original complete native instruction sequence has an unexpected tail: %x", code) + } + t.Logf("actual Go integer MMX-read rows=%d, physical M input independent of GP spelling", len(rows)) +} + +func TestX86YMMSelectorReusableDescriptorBanks(t *testing.T) { + for _, rule := range x86YMMOriginalRules(t) { + for index, register := range []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + row := rule.name + " " + register + ",M1" + if rule.table == "ymshuf" { + row = rule.name + " $27," + register + ",M1" + } + ins := x86YMMReadSource(t, row) + form, known, err := x86LegacyMMXReadProperties(ins, "amd64") + if !known || err != nil || !form.gpSpelling || !form.mmx || form.row != 0 || + form.source.Reg != Reg(fmt.Sprintf("M%d", index&7)) { + t.Fatal("complete selector lost physical source bank", row, form, known, err) + } + effect, known, err := x86PackedBinaryByteEffect(ins.Op, ins, "amd64") + if known { + if err != nil || len(effect.operands) != 2 || effect.operands[0].operand.Reg != form.source.Reg || + effect.operands[0].width != 8 || !effect.operands[0].read || effect.operands[0].write { + t.Fatal("existing byte descriptor does not share encoded bank selection", row, effect, err) + } + } + if strings.HasPrefix(rule.name, "PACK") { + pack, known, err := x86PackedNarrowProperties(ins, "amd64") + if !known || err != nil || pack.source.Reg != form.source.Reg { + t.Fatal("pack descriptor does not share encoded bank selection", row, pack, known, err) + } + } + } + } +} + +func TestX86YMMSelectorContextIsBeforeIRAndKeepsInput(t *testing.T) { + for _, rule := range x86YMMOriginalRules(t) { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + ins := x86YMMReadSource(t, rule.name+" "+prefix+"AX,M1") + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before, original := output.String(), fmt.Sprintf("%#v", ins) + terminated, err := ctx.lowerInstr(0, 0, ins, nil, nil) + if terminated || !errors.Is(err, ErrProbeNeedsContext) || before != output.String() || original != fmt.Sprintf("%#v", ins) { + t.Fatal("source-bank Context was late or mutated the source/IR", rule.name, terminated, err) + } + } +} + +func TestX86YMMSelectorCompleteOriginalOrderedRules(t *testing.T) { + bytesSource, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + source := string(bytesSource) + if len(x86LegacyMMXReadSpecs) != 43 { + t.Fatal("selector class does not have exactly 43 canonical names") + } + for _, rule := range x86YMMOriginalRules(t) { + t.Run(rule.name, func(t *testing.T) { + spec, found := x86LegacyMMXReadSpecs[Op(rule.name)] + if !found { + t.Fatal("complete original integer read class lacks a selector") + } + table, prefix, opcodeBytes, rows := spec.encoderRows() + var originalBytes []byte + for _, token := range strings.Split(rule.opBytes, ",") { + token = strings.TrimSpace(token) + if token == "Pe" { + token = "0x66" + } + value, err := strconv.ParseUint(token, 0, 8) + if err != nil { + t.Fatal("unknown original opBytes expression", token, err) + } + originalBytes = append(originalBytes, byte(value)) + } + if table != rule.table || prefix != rule.prefix || !bytes.Equal(opcodeBytes, originalBytes) { + t.Fatalf("selected encoding metadata differs from original Go: %s/%s %x; original=%+v %x", + table, prefix, opcodeBytes, rule, originalBytes) + } + block := regexp.MustCompile(`(?s)var ` + table + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(source) + if len(block) != 2 { + t.Fatal("original ordered table was not found", table) + } + pattern := regexp.MustCompile(`\{(Z\w+), (\d+), argList\{([^}]+)\}\}`) + originalRows := pattern.FindAllStringSubmatch(block[1], -1) + if len(rows) != len(originalRows) { + t.Fatal("original ordered table row count changed") + } + byteOffset, xo := 0, 0 + if originalBytes[0] == 0x0f { + xo = 1 + } + for index, original := range originalRows { + zoffset, _ := strconv.Atoi(original[2]) + classes := strings.Split(strings.ReplaceAll(original[3], " ", ""), ",") + row := rows[index] + if row.zcase != original[1] || row.zoffset != zoffset || row.byteOffset != byteOffset || + !reflect.DeepEqual(row.classes, classes) { + t.Fatalf("ordered row %d differs: %+v original=%v offset=%d", index, row, original, byteOffset) + } + byteOffset += zoffset + xo + } + }) + } +} + +func TestX86YMMSelectorCompleteRolesAndImmutability(t *testing.T) { + for _, rule := range x86YMMOriginalRules(t) { + for destination := 0; destination < 8; destination++ { + for source := 0; source < 8; source++ { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + ins := x86YMMReadSource(t, fmt.Sprintf("%s %sM%d,M%d", rule.name, prefix, source, destination)) + before := fmt.Sprintf("%#v", ins) + form, known, err := x86LegacyMMXReadProperties(ins, "amd64") + if !known || err != nil || !form.mmx || form.gpSpelling || form.row != 0 || + form.byteWidth != 8 || form.memoryBytes != 0 || form.source.Reg != Reg(fmt.Sprintf("M%d", source)) { + t.Fatal("literal MMX source/destination role changed", ins, form, known, err) + } + local, err := x86LegacyMMXReadInstruction(ins, "amd64") + if err != nil || before != fmt.Sprintf("%#v", ins) { + t.Fatal("selector mutated original", err) + } + local.Args[form.sourceIndex].Reg = "M6" + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("normalized operand slice aliases original") + } + } + } + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + for _, bank := range []string{"M", "X"} { + if bank == "X" && rule.table == "ymshuf" { + continue + } + ins := x86YMMReadSource(t, rule.name+" "+prefix+"(AX),"+bank+"1") + form, known, err := x86LegacyMMXReadProperties(ins, "amd64") + want := 16 + if bank == "M" { + want = 8 + if strings.HasPrefix(rule.name, "PUNPCKL") { + want = 4 + } + } + if !known || err != nil || form.memoryBytes != want || form.gpSpelling { + t.Fatal("complete native read shape changed", ins, form, known, err) + } + if rule.table == "yps" { + for _, count := range []int{-128, 0, 127} { + ins = x86YMMReadSource(t, fmt.Sprintf("%s $%d,%s1", rule.name, count, bank)) + form, known, err = x86LegacyMMXReadProperties(ins, "amd64") + if !known || err != nil || form.row != 1 && form.row != 3 || form.memoryBytes != 0 || form.gpSpelling { + t.Fatal("immediate row borrowed a bank read", ins, form, known, err) + } + } + } + } + } +} + +func TestX86YMMSelectorActualGoModeAndRoleBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, rule := range x86YMMOriginalRules(t) { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + row := rule.name + " " + prefix + "AX,M1" + source := "TEXT probe(SB),4,$0-0\n" + row + "\nRET\n" + want := arch == "amd64" || rule.table == "ymshuf" || rule.table == "ymmxmm0f38" + goAssemblerOracleOutput(t, arch, source, "", want) + form, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), arch) + if !known || (err == nil) != want || want && (form.source.Reg != "M0" || !form.gpSpelling) { + t.Fatal("actual named mode admission differs from selector", arch, row, form, known, err) + } + if want { + code := x86PackedShiftActualGoBytes(t, arch, source) + mode := 64 + if arch == "386" { + mode = 32 + } + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Args[0] != x86asm.M1 || physical.Args[1] != x86asm.M0 { + t.Fatal("actual accepted mode did not select physical MMX", arch, row, physical, err) + } + } + } + } + for _, row := range []string{"PAND M0,X1", "PAND X0,M1", "PAND AX,X1", "PAND M0,AX", + "PAND R8,M1", "PSHUFW $128,M0,M1", "PSHUFW $-129,M0,M1", "PSRLW $128,X1"} { + arch := "amd64" + if row == "PAND R8,M1" { + arch = "386" + } + goAssemblerOracleOutput(t, arch, "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n", "", false) + _, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), arch) + if !known || err == nil { + t.Fatal("original Go rejected row obtained a pure bank form", arch, row, known, err) + } + } +} + +func TestX86YMMSelectorActualMode32GPSpellings(t *testing.T) { + for _, name := range []string{"PHADDD", "PSHUFW"} { + for index, register := range []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + prefix := "" + if name == "PSHUFW" { + prefix = "$27," + } + row := name + " " + prefix + register + ",M1" + source := "TEXT probe(SB),4,$0-0\n" + row + "\nRET\n" + want := index < 8 + goAssemblerOracleOutput(t, "386", source, "", want) + form, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), "386") + if !known || (err == nil) != want || want && form.source.Reg != Reg(fmt.Sprintf("M%d", index)) { + t.Fatal("mode32 original GP class does not match selected MMX bank", row, form, known, err) + } + } + } +} + +func TestX86YMMSelectorExcludesRealGPRolesAndPhysicalForgery(t *testing.T) { + for _, row := range []string{"MOVQ AX,M1", "PINSRB $0,AX,X1", "PINSRD $0,AX,X1", "PINSRQ $0,AX,X1", + "PEXTRB $0,X1,AX", "PEXTRD $0,X1,AX", "PEXTRQ $0,X1,AX", "CVTPL2PS M0,X1", "CVTPL2PD M0,X1"} { + _, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), "amd64") + if known || err != nil { + t.Fatal("different ordered GP/x87 role was claimed by integer read selector", row, known, err) + } + } + for _, rule := range x86YMMOriginalRules(t) { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + ins := x86YMMReadSource(t, rule.name+" "+prefix+"AX,M1") + ins.x86Encoded = true + if _, known, err := x86LegacyMMXReadProperties(ins, "amd64"); !known || err == nil { + t.Fatal("physical decoder identity acquired source-only GP compatibility", rule.name) + } + ins.Args[len(ins.Args)-2].Reg = "M0" + if form, known, err := x86LegacyMMXReadProperties(ins, "386"); !known || err != nil || form.gpSpelling { + t.Fatal("actual physical MMX input was changed into a source alias", rule.name, form, known, err) + } + } + for _, row := range []string{"PAND.BAD AX,M1", "PAND AX,M8", "PSHUFW $0,AX,X1"} { + if _, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), "amd64"); !known || err == nil { + t.Fatal("malformed selector identity was accepted", row) + } + } +} + +func TestX86YMMSelectorDoesNotGrantEmitterOrCallerAuthority(t *testing.T) { + for _, rule := range x86YMMOriginalRules(t) { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + row := rule.name + " " + prefix + "AX,M1" + ins := x86YMMReadSource(t, row) + if !errors.Is(x86LegacyMMXReadEmitterContext(ins, "amd64"), ErrProbeNeedsContext) || x86CallerVectorMoveKnown(ins) { + t.Fatal("bank identity alone acquired transport or caller permission", row) + } + ins.Args[len(ins.Args)-2].Reg = "M0" + state := x86SourceStackState{vectors: map[Reg]uint64{}, masks: map[Reg]bool{}} + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, &x86GoCallerRegionProof{}) { + t.Fatal("undefined MMX input acquired a caller value", rule.name) + } + pkg := mustGoPackage(t, "probe", "package probe\nfunc probe()\n") + source := "TEXT ·probe(SB),4,$0-0\n" + row + "\nRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{"probe": {Ret: Void}}, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("text API granted unproved MMX source transport", row, triple, err) + } + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("module API granted unproved MMX source transport", row, triple, err) + } + result, err := translateGoModuleInContext(context, pkg, []byte(source), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: triple, ResolveSym: options.ResolveSym}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go API granted unproved MMX source transport", row, triple, err) + } + }() + } + } +} + +func TestX86YMMSelectorFiveTargetRegisteredXMMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc is required") + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", + "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + source := "TEXT ·probe(SB),4,$0-16\nMOVOU value+0(FP),X0\nMOVOU value+0(FP),X1\n" + count := 0 + for _, rule := range x86YMMOriginalRules(t) { + if rule.table == "ymshuf" { + continue // This original row is MMX-only, not an XMM grant. + } + source += rule.name + " X0,X1\n" + count++ + } + source += "RET\n" + if count != 42 { + t.Fatal("complete original XMM subdomain changed", count) + } + goAssemblerOracleOutput(t, arch, source, "", true) + pkg := mustGoPackage(t, "probe", "package probe\nfunc probe(value [16]byte)\n") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + sig, err := DeriveGoFuncABI0Signature("probe", pkg.Types.Scope().Lookup("probe").(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"probe": sig}, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(triple, err) + } + x86ResultInitializationObject(t, llc, triple, ir, "ymm-selector-text") + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(triple, err) + } + x86ResultInitializationObject(t, llc, triple, module.String(), "ymm-selector-module") + result, err := translateGoModuleInContext(context, pkg, []byte(source), GoModuleOptions{ + GOARCH: arch, TargetTriple: triple, ResolveSym: options.ResolveSym}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatal(triple, err) + } + x86ResultInitializationObject(t, llc, triple, result.Module.String(), "ymm-selector-go") + }() + } +} + +func TestX86YMMOriginalGoOrderedGPControls(t *testing.T) { + for _, row := range []string{"MOVQ AX,M1", "PINSRB $0,AX,X1", "PINSRD $0,AX,X1", "PINSRQ $0,AX,X1", + "PEXTRB $0,X1,AX", "PEXTRD $0,X1,AX", "PEXTRQ $0,X1,AX"} { + t.Run(row, func(t *testing.T) { + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n") + physical, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + for _, arg := range physical.Args { + if arg == x86asm.M0 { + t.Fatal("actual transfer/insertion/extraction GP role was changed to MMX", physical) + } + } + if reflect.DeepEqual(physical.Args, x86asm.Args{}) { + t.Fatal("control has no actual decoded operands") + } + }) + } +} diff --git a/amd64_legacy_mmx_mode32_test.go b/amd64_legacy_mmx_mode32_test.go new file mode 100644 index 000000000..d57e394d0 --- /dev/null +++ b/amd64_legacy_mmx_mode32_test.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86YMMMode32LegacyShiftActualGoRows(t *testing.T) { + var names []string + for _, rule := range x86YMMOriginalRules(t) { + if rule.table == "yps" { + names = append(names, rule.name) + } + } + if len(names) != 8 { + t.Fatal("complete original yps class changed", names) + } + + for _, arch := range []string{"amd64", "386"} { + for _, name := range names { + for _, bank := range []string{"M", "X"} { + for _, source := range []string{"$3", bank + "0", "8(AX)"} { + row := fmt.Sprintf("%s %s,%s1", name, source, bank) + t.Run(arch+"/"+row, func(t *testing.T) { + text := "TEXT probe(SB),4,$0-0\n" + row + "\nRET\n" + want := arch == "amd64" || bank == "X" + goAssemblerOracleOutput(t, arch, text, "", want) + t.Logf("original source SHA256=%x accepted=%t", sha256.Sum256([]byte(text)), want) + if want { + code := x86PackedShiftActualGoBytes(t, arch, text) + mode := 64 + if arch == "386" { + mode = 32 + } + physical, err := x86asm.Decode(code, mode) + destination := x86asm.M1 + if bank == "X" { + destination = x86asm.X1 + } + if err != nil || physical.Args[0] != destination { + t.Fatal("original Go selected a different physical destination", row, code, physical, err) + } + t.Logf("original Go bytes=%x SHA256=%x physical=%v", code, sha256.Sum256(code), physical) + } + + form, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), arch) + if !known || (err == nil) != want { + t.Fatal("named Go mode admission differs from pure selector", row, form, known, err) + } + if !want { + file, err := Parse(ArchAMD64, text) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"probe": {Name: "probe", Ret: Void}}, + }); err == nil { + t.Fatal("Go-rejected named mode32 MMX row produced LLVM IR", row) + } + } + }) + } + } + } + } +} diff --git a/amd64_legacy_move_native_linux_amd64_test.go b/amd64_legacy_move_native_linux_amd64_test.go new file mode 100644 index 000000000..2a593e373 --- /dev/null +++ b/amd64_legacy_move_native_linux_amd64_test.go @@ -0,0 +1,9 @@ +package plan9asm + +import "testing" + +// Unlike Rosetta diagnostics, this required native CI axis also compares the +// original aligned-SSE faults for every nonzero misalignment. +func TestCrossLinuxRuntimeMatrixX86LegacyMoveNativeAlignment(t *testing.T) { + x86LegacyMoveNativeRuntime(t, true) +} diff --git a/amd64_legacy_move_native_runtime_unix_test.go b/amd64_legacy_move_native_runtime_unix_test.go new file mode 100644 index 000000000..ed8a32d3e --- /dev/null +++ b/amd64_legacy_move_native_runtime_unix_test.go @@ -0,0 +1,201 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86LegacyMoveNativeBytesAndFaults(t *testing.T) { + x86LegacyMoveNativeRuntime(t, false) +} + +func x86LegacyMoveNativeRuntime(t *testing.T, nativeAlignment bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + boolInt := func(value bool) int { + if value { + return 1 + } + return 0 + } + var source, declarations, originals, pairs strings.Builder + sigs := map[string]FuncSig{} + for index, row := range x86LegacyMoveNativeOracle { + for _, store := range []bool{false, true} { + for _, dead := range []bool{false, true} { + name := fmt.Sprintf("move%d_%t_%t", index, store, dead) + source.WriteString(x86LegacyMoveNativeSource("amd64", name, row.name, store, dead)) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,uint8_t*);\n", name) + operands := "(DI),X1" + before, after := "stc\\n\\t", "movups %%xmm1,(%1)\\n\\t" + if store { + operands = "X1,(DI)" + before, after = "movups (%1),%%xmm1\\n\\tstc\\n\\t", "" + } else if dead { + after = "" + } + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row.name+" "+operands+"\nRET\n") + if len(code) < 4 || code[len(code)-1] != 0xc3 { + t.Fatal("missing original Go opcode and ordinary RET", code) + } + var bytes []string + for _, value := range code[:len(code)-1] { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + fmt.Fprintf(&originals, `static void original_%s(uint8_t *p,uint8_t *out) { + __asm__ volatile("%s.byte %s\n\t%ssetc 16(%%1)" + : : "D"(p),"r"(out) : "xmm1","memory","cc"); +} +`, name, before, strings.Join(bytes, ","), after) + fmt.Fprintf(&pairs, "{original_%s,%s,%d,%d,%d,16,0},\n", name, name, + boolInt(row.aligned), boolInt(store), boolInt(dead)) + } + } + for lane := 0; lane < 4; lane++ { + name := fmt.Sprintf("move_result%d_%d", index, lane) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-12\nMOVQ p+0(FP),AX\n%s (AX),X1\nPEXTRD $%d,X1,CX\nMOVL CX,ret+8(FP)\nRET\n", name, row.name, lane) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: I32, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I32, Index: 0, Field: -1}}, + }} + fmt.Fprintf(&declarations, `extern uint32_t %s(uint8_t*); +static void translated_%s(uint8_t *p,uint8_t *out) { + uint32_t value=%s(p); memcpy(out,&value,4); out[16]=1; +} +`, name, name, name) + code := x86PackedShiftActualGoBytes(t, "amd64", fmt.Sprintf("TEXT f(SB),4,$0-0\n%s (DI),X1\nPEXTRD $%d,X1,AX\nRET\n", row.name, lane)) + if len(code) < 8 || code[len(code)-1] != 0xc3 { + t.Fatal("missing original Go load/extract/RET", code) + } + var bytes []string + for _, value := range code[:len(code)-1] { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + fmt.Fprintf(&originals, `static void original_%s(uint8_t *p,uint8_t *out) { + uint32_t value; + __asm__ volatile(".byte %s" : "=a"(value) : "D"(p) : "xmm1","memory"); + memcpy(out,&value,4); out[16]=1; +} +`, name, strings.Join(bytes, ",")) + fmt.Fprintf(&pairs, "{original_%s,translated_%s,%d,0,0,4,%d},\n", name, name, boolInt(row.aligned), lane*4) + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const helper = ` +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef void (*operation)(uint8_t*,uint8_t*); +struct pair { operation original,translated; int aligned,store,dead,width,offset; }; +static int outcome(operation fn,uint8_t *p) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + uint8_t data[32]; memset(data,0x53,sizeof(data)); + fn(p,data); _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + return WIFSIGNALED(status)?WTERMSIG(status):-2; +} +static int check_bytes(struct pair pair,int offset) { + for(int sample=0;sample<257;++sample) for(int rc=0;rc<4;++rc) { + _Alignas(16) uint8_t a[64],b[64],x[64],y[64]; + for(int i=0;i<64;++i) { + a[i]=b[i]=(uint8_t)(sample*71+i*19); + x[i]=y[i]=(uint8_t)(sample*31-i*59); + } + uint8_t before_a[64],before_x[64]; + memcpy(before_a,a,64); memcpy(before_x,x,64); + uint32_t saved=_mm_getcsr(),csr=0x1f80|(rc<<13)|(sample&63); + _mm_setcsr(csr); pair.original(a+offset,x+1); uint32_t original_csr=_mm_getcsr(); + _mm_setcsr(csr); pair.translated(b+offset,y+1); uint32_t translated_csr=_mm_getcsr(); + _mm_setcsr(saved); + if(original_csr!=csr || translated_csr!=csr || memcmp(a,b,64) || memcmp(x,y,64)) return 1; + for(int i=0;i<64;++i) { + uint8_t want_a=pair.store && i>=offset && i=1 && i<1+pair.width?before_a[offset+pair.offset+i-1]:before_x[i]; + if(i==17) want_x=1; + if(a[i]!=want_a || x[i]!=want_x) return 2; + } + } + return 0; +} +` + main := fmt.Sprintf("#define CHECK_ALIGNMENT %d\n", boolInt(nativeAlignment)) + helper + declarations.String() + originals.String() + "static const struct pair pairs[]={\n" + pairs.String() + `}; +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<128) return 1; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(region==MAP_FAILED || mprotect(region+page,page,PROT_NONE)) return 2; + for(unsigned op=0;op +#include +extern void observe(uint32_t*); +int main(void) { + uint32_t out[6] = {77,0,0,0,0,88}; + observe(out+1); + for(unsigned i=1;i<=4;i++) if(out[i]!=i) { + fprintf(stderr,"result field %u: got %u, want %u\n",i-1,out[i],i); + return 1; + } + return out[0]!=77 || out[5]!=88; +} +` + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "packed-fp-results", ir, main, level) + }) + } + }) + } + }) + } +} diff --git a/amd64_legacy_packed_move_runtime_test.go b/amd64_legacy_packed_move_runtime_test.go new file mode 100644 index 000000000..8dad27d8f --- /dev/null +++ b/amd64_legacy_packed_move_runtime_test.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86LegacyPackedMoveOriginalBytesAndCanaries(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, goTest, reference, checks strings.Builder + declarations.WriteString("package packedmove\nfunc ReadCSR() uint32\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\n") + goTest.WriteString(`package packedmove +import ( + "testing" + "unsafe" +) +func aligned(value *[80]byte) (*[16]byte,int) { + offset:=16+int((-uintptr(unsafe.Pointer(&value[16])))&15) + return (*[16]byte)(unsafe.Pointer(&value[offset])),offset +} +func TestOriginal(t *testing.T) { + for sample:=0;sample<257;sample++ { for _,seed:=range []uint64{0,0x8d5} { + var x,y,out [80]byte + xp,_:=aligned(&x);yp,_:=aligned(&y);op,offset:=aligned(&out) + for i:=range x { x[i]=byte(sample*71+i*37);y[i]=byte(sample*19+i*11);out[i]=0xa5 } + originalX,originalY:=x,y +`) + reference.WriteString(`#include +#include +#include +#include +struct Aligned { _Alignas(16) uint8_t before[16],value[16],after[16]; }; +`) + for _, op := range x86LegacyPackedMoveTestNames { + for _, kind := range []string{"Register", "Alias", "Load", "Store"} { + name := op + kind + operation, input := op+" X1,X0\nMOVOU X0,(DX)", "xp" + switch kind { + case "Alias": + operation, input = op+" X0,X0\nMOVOU X0,(DX)", "yp" + case "Load": + operation = op + " (AX),X0\nMOVOU X0,(DX)" + case "Store": + operation = op + " X1,(DX)" + } + fmt.Fprintf(&declarations, "func %s(out,x,y *[16]byte,flags *uint64,seed uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-40\nMOVQ out+0(FP),DX\nMOVQ x+8(FP),AX\nMOVQ y+16(FP),CX\nMOVOU (AX),X1\nMOVOU (CX),X0\nPUSHQ seed+32(FP)\nPOPFQ\n%s\nPUSHFQ\nPOPQ BX\nMOVQ flags+24(FP),SI\nMOVQ BX,(SI)\nRET\n", name, operation) + fmt.Fprintf(&goTest, `{ + for i:=range out { out[i]=0xa5 } + box:=struct{before,value,after uint64}{0x1122334455667788,0,0x8877665544332211} + csr:=ReadCSR() + %s(op,xp,yp,&box.value,seed) + if box.value&0x8d5!=seed||box.before!=0x1122334455667788||box.after!=0x8877665544332211||ReadCSR()!=csr||x!=originalX||y!=originalY { t.Fatalf("%s state/canary sample%%d flags%%x",sample,box.value) } + for i,value:=range out { + want:=byte(0xa5) + if i>=offset&&i\n#include \n#include \n") + constant := func(multiplier, addend int) string { + var values []string + for index := 0; index < 64; index++ { + values = append(values, fmt.Sprintf("i8 %d", byte(index*multiplier+addend))) + } + return "<" + strings.Join(values, ",") + ">" + } + for _, op := range x86LegacyPackedMoveTestNames { + ins := Instr{Op: Op(op), Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}} + fn := Func{Instrs: []Instr{ + {Op: "VMOVUPS", Args: []Operand{{Kind: OpReg, Reg: "Z0"}, {Kind: OpReg, Reg: "Z1"}}}, ins, + }} + ctx, output := newAMD64CtxWithFuncForTest(t, fn, FuncSig{Ret: Void}, nil) + if err := ctx.storeZ("Z0", constant(37, 11)); err != nil { + t.Fatal(err) + } + if err := ctx.storeZ("Z1", constant(19, 71)); err != nil { + t.Fatal(err) + } + known, term, err := ctx.lowerFP(ins.Op, ins) + if !known { + known, term, err = ctx.lowerVec(ins.Op, ins) + } + if !known || term || err != nil { + t.Fatal("legacy lowerer did not consume its typed X-bank shape", known, term, err) + } + first, err := ctx.loadZ("Z0") + if err != nil { + t.Fatal(err) + } + second, err := ctx.loadZ("Z1") + if err != nil { + t.Fatal(err) + } + fmt.Fprintf(&ir, "define void @%sShadow(ptr %%out) {\n%s store <64 x i8> %s, ptr %%out, align 1\n %%next = getelementptr i8, ptr %%out, i64 64\n store <64 x i8> %s, ptr %%next, align 1\n ret void\n}\n", op, output.String(), first, second) + fmt.Fprintf(&reference, "extern void %sShadow(uint8_t*);\n", op) + fmt.Fprintf(&checks, "{memset(out,0xa5,sizeof(out));%sShadow(out+16);for(int i=0;i<64;i++){uint8_t want=(uint8_t)(i<16?i*19+71:i*37+11);if(out[16+i]!=want||out[80+i]!=(uint8_t)(i*19+71)){fprintf(stderr,\"%s shadow byte%%d\\n\",i);return 1;}}for(int i=0;i<16;i++)if(out[i]!=0xa5||out[144+i]!=0xa5)return 2;}\n", op, op) + } + reference.WriteString("int main(void){uint8_t out[160];\n" + checks.String() + "return 0;}\n") + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "legacy-packed-move-alias", ir.String(), reference.String(), level) + }) + } +} diff --git a/amd64_legacy_ps_insert_extract_test.go b/amd64_legacy_ps_insert_extract_test.go new file mode 100644 index 000000000..fba8a59f5 --- /dev/null +++ b/amd64_legacy_ps_insert_extract_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86LegacyPSIndependentFPFields(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + const source = "TEXT fields(SB),4,$0-12\nPXOR X0,X0\nINSERTPS $0,a+0(FP),X0\nEXTRACTPS $0,X0,ret+8(FP)\nRET\n" + + "TEXT stores(SB),4,$0-12\nMOVL $0x87654321,CX\nMOVD CX,X1\nEXTRACTPS $0,X1,a+0(FP)\nMOVBLZX a+0(FP),AX\nMOVBLZX b+1(FP),BX\nSHLL $8,BX\nORL BX,AX\nMOVBLZX c+2(FP),BX\nSHLL $16,BX\nORL BX,AX\nMOVBLZX d+3(FP),BX\nSHLL $24,BX\nORL BX,AX\nMOVL AX,ret+8(FP)\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + pkg := mustGoPackage(t, "oracle", "package oracle\nfunc fields(a,b,c,d byte) uint32\nfunc stores(a,b,c,d byte) uint32\n") + sigs := map[string]FuncSig{} + for _, name := range []string{"fields", "stores"} { + decl := pkg.Types.Scope().Lookup(name).(*types.Func) + sig, err := DeriveGoFuncABI0Signature(name, decl, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + if len(sig.Frame.Params) != 4 || len(sig.Frame.Results) != 1 || sig.Frame.Results[0].Offset != 8 { + t.Fatalf("actual AMD64 ABI0 four-byte fields/result layout changed: %+v", sig.Frame) + } + sigs[name] = sig + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const reference = `#include +#include +extern uint32_t fields(uint8_t,uint8_t,uint8_t,uint8_t); +extern uint32_t stores(uint8_t,uint8_t,uint8_t,uint8_t); +int main(void) { + unsigned failures=0; + for(unsigned i=0;i<256;i++) { + uint8_t a=i,b=i*37,c=i*59,d=i*71; + uint32_t want=(uint32_t)a|((uint32_t)b<<8)|((uint32_t)c<<16)|((uint32_t)d<<24); + uint32_t got=fields(a,b,c,d); + if(got!=want){if(!(failures&1))fprintf(stderr,"legacy FP fields got=%x want=%x\n",got,want);failures|=1;} + got=stores(a,b,c,d); + if(got!=UINT32_C(0x87654321)){if(!(failures&2))fprintf(stderr,"legacy FP stores got=%x\n",got);failures|=2;} + } + return failures; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy-ps-independent-fp", triple, ir, reference, prefix) +} + +func TestX86LegacyPSInsertExtractSourceCallerReachesObjects(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),Z0\nVMOVUPS (SI),X1\nINSERTPS $64,X1,X0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),X0\nEXTRACTPS $3,X0,AX\nMOVL AX,(DX)", + } { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("%s: %v", body, err) + } + } +} + +func TestX86LegacyPSInsertedByteKnowledgePreservesUpper(t *testing.T) { + for _, missing := range []int{-1, 0, 12, 16, 31, 63} { + const row = "INSERTPS $0,X0,X1" + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + f, recognized, err := x86InsertExtractProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatalf("legacy shared grammar missing: %t/%v", recognized, err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 15, "Z1": ^uint64(0)}, masks: map[Reg]bool{}} + if missing >= 0 { + state.vectors["Z1"] &^= uint64(1) << uint(missing) + } + if !x86CallerInsertExtractStep(&ins, f, &state, nil) { + t.Fatal("defined inserted bytes rejected") + } + want := ^uint64(0) + if missing >= 4 { + want &^= uint64(1) << uint(missing) + } + if got := state.vectors["Z1"]; got != want { + t.Fatal(fmt.Sprintf("legacy upper knowledge %d: got%x want%x", missing, got, want)) + } + } +} + +func TestX86LegacyPSCallerUnknownInputsAndPermissions(t *testing.T) { + for name, body := range map[string]string{ + "unknown inserted lane": "VMOVUPS (DI),X1\nINSERTPS $64,X0,X1\nVMOVSS X1,(DX)", + "unknown retained base": "VMOVUPS (DI),X0\nINSERTPS $16,X0,X1\nVMOVSS X1,(DX)", + "unknown extract lane": "EXTRACTPS $0,X0,AX\nMOVL AX,(DX)", + "unknown memory even all clear": "INSERTPS $15,(BX),X0\nVMOVSS X0,(DX)", + "pointer FP source": "VMOVUPS (DI),X1\nINSERTPS $0,a+0(FP),X1\nVMOVSS X1,(DX)", + "source SP": "VMOVUPS (DI),X1\nINSERTPS $0,(SP),X1\nVMOVSS X1,(DX)", + "input write": "VMOVUPS (DI),X0\nEXTRACTPS $0,X0,(DI)", + "g write": "VMOVUPS (DI),X0\nEXTRACTPS $0,X0,R14", + "missing CFG source": "CMPQ AX,$0\nJE join\nVMOVUPS (DI),X0\njoin:\nEXTRACTPS $0,X0,AX\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("legacy unknown source acquired caller permission: %v", err) + } + }) + } +} diff --git a/amd64_legacy_scalar_move_alias_runtime_test.go b/amd64_legacy_scalar_move_alias_runtime_test.go new file mode 100644 index 000000000..52044a034 --- /dev/null +++ b/amd64_legacy_scalar_move_alias_runtime_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +type x86LegacyScalarAliasForm struct { + name, move string + width int + register bool + self bool +} + +func x86LegacyScalarAliasForms() []x86LegacyScalarAliasForm { + return []x86LegacyScalarAliasForm{ + {name: "loadSS", move: "MOVSS", width: 4}, + {name: "copySS", move: "MOVSS", width: 4, register: true}, + {name: "selfSS", move: "MOVSS", width: 4, register: true, self: true}, + {name: "loadSD", move: "MOVSD", width: 8}, + {name: "copySD", move: "MOVSD", width: 8, register: true}, + {name: "selfSD", move: "MOVSD", width: 8, register: true, self: true}, + } +} + +func x86LegacyScalarAliasAssembly(wide int, goarch string) string { + var source strings.Builder + word, pointerMove := 8, "MOVQ" + if goarch == "386" { + word, pointerMove = 4, "MOVL" + } + register := "Y0" + if wide == 64 { + register = "Z0" + } + for _, form := range x86LegacyScalarAliasForms() { + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s input+0(FP),AX\n%s output+%d(FP),BX\nVMOVUPS (AX),%s\n", + form.name, word*2, pointerMove, pointerMove, word, register) + switch { + case form.self: + fmt.Fprintf(&source, "%s X0,X0\n", form.move) + case form.register: + fmt.Fprintf(&source, "%s 64(AX),X1\n%s X1,X0\n", form.move, form.move) + default: + fmt.Fprintf(&source, "%s 64(AX),X0\n", form.move) + } + // A store must not alter the register or any overlapping view. + fmt.Fprintf(&source, "%s X0,128(BX)\nMOVUPS X0,(BX)\nVMOVUPS Y0,16(BX)\n", form.move) + if wide == 64 { + source.WriteString("VMOVUPS Z0,48(BX)\n") + } + source.WriteString("RET\n") + } + return source.String() +} + +func x86LegacyScalarAliasIR(t *testing.T, wide int, goarch, triple string) string { + t.Helper() + assembly := x86LegacyScalarAliasAssembly(wide, goarch) + requireX86GoAssemblerResult(t, goarch, assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + word := int64(8) + if goarch == "386" { + word = 4 + } + for _, form := range x86LegacyScalarAliasForms() { + symbol := "scalar." + form.name + sigs[symbol] = FuncSig{ + Name: symbol, Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: word, Type: Ptr, Index: 1, Field: -1}, + }}, + } + } + ir, err := Translate(file, Options{ + Goarch: goarch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "scalar.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + return ir +} + +func TestX86LegacyScalarMoveAliasFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, wide := range []int{32, 64} { + for _, target := range []struct{ goarch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i386-pc-windows-msvc"}, + } { + t.Run(fmt.Sprintf("%d/%s", wide, target.triple), func(t *testing.T) { + ir := x86LegacyScalarAliasIR(t, wide, target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "legacy-alias.ll", "legacy-alias.o", ir) + }) + } + } +} + +func x86LegacyScalarAliasActualGoOracle(t *testing.T, wide int) { + t.Helper() + var declarations, tests strings.Builder + declarations.WriteString("package scalar\nimport \"unsafe\"\n") + tests.WriteString("package scalar\nimport (\"testing\"; \"unsafe\")\nfunc TestFullViews(t *testing.T) {\n") + for _, form := range x86LegacyScalarAliasForms() { + fmt.Fprintf(&declarations, "func %s(input, output unsafe.Pointer)\n", form.name) + fmt.Fprintf(&tests, ` + for iteration := 0; iteration < 64; iteration++ { + var input [80]byte + var output [160]byte + for index := range input { + input[index] = byte(index*17 + iteration) + } + for index := range output { + output[index] = byte(index*7 + iteration) + } + beforeInput, want := input, output + for _, view := range [][2]int{{0,16},{16,32},{48,%[4]d}} { + if view[0] == 48 && %[4]d != 64 { + continue + } + for lane := 0; lane < view[1]; lane++ { + value := input[lane] + if !%[3]t && lane < %[2]d { + value = input[64+lane] + } else if !%[3]t && !%[5]t && lane < 16 { + value = 0 + } + want[16+view[0]+lane] = value + } + } + for lane := 0; lane < %[2]d; lane++ { + value := input[64+lane] + if %[3]t { + value = input[lane] + } + want[144+lane] = value + } + %[1]s(unsafe.Pointer(&input[0]), unsafe.Pointer(&output[16])) + if output != want || input != beforeInput { + t.Fatalf("%[1]s iteration=%%d got=%%x want=%%x inputChanged=%%t", iteration, output, want, input != beforeInput) + } + } +`, form.name, form.width, form.self, wide, form.register) + } + tests.WriteString("}\n") + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module scalar\n\ngo 1.20\n", "scalar.go": declarations.String(), + "scalar_amd64.s": x86LegacyScalarAliasAssembly(wide, "amd64"), "scalar_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + command.WaitDelay = 2 * time.Second + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("actual Go %d-byte legacy alias oracle: %v\n%s", wide, err, output) + } +} + +func TestCrossLinuxRuntimeMatrixX86LegacyScalarMoveAliasViews(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the Linux/amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + // This original Go counterpart requires AVX, not AVX512. ZMM encodings + // below are independently Go-assembled and portable LLVM/C checked; that + // does not claim that the host executes those original AVX512 instructions. + x86LegacyScalarAliasActualGoOracle(t, 32) + for _, wide := range []int{32, 64} { + var source strings.Builder + source.WriteString(`#include +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range x86LegacyScalarAliasForms() { + fmt.Fprintf(&source, "extern void %s(void*,void*) __asm__(LABEL(\"scalar.%s\"));\n", form.name, form.name) + } + source.WriteString("int main(void) {\n") + for _, form := range x86LegacyScalarAliasForms() { + register, self := 0, 0 + if form.register { + register = 1 + } + if form.self { + self = 1 + } + fmt.Fprintf(&source, ` + for (unsigned iteration = 0; iteration < 64; iteration++) { + uint8_t input[80], output[160], before_input[80], want[160]; + for (unsigned index = 0; index < sizeof(input); index++) { + input[index] = index*17 + iteration; + } + for (unsigned index = 0; index < sizeof(output); index++) { + output[index] = index*7 + iteration; + } + memcpy(before_input, input, sizeof(input)); + memcpy(want, output, sizeof(output)); + const unsigned offsets[] = {0,16,48}, widths[] = {16,32,%[4]d}; + for (unsigned view = 0; view < (%[4]d == 64 ? 3 : 2); view++) { + for (unsigned lane = 0; lane < widths[view]; lane++) { + uint8_t value = input[lane]; + if (!%[3]d && lane < %[2]d) { + value = input[64+lane]; + } else if (!%[3]d && !%[5]d && lane < 16) { + value = 0; + } + want[16+offsets[view]+lane] = value; + } + } + for (unsigned lane = 0; lane < %[2]d; lane++) { + want[144+lane] = input[(%[3]d ? 0 : 64)+lane]; + } + %[1]s(input, output + 16); + for (unsigned index = 0; index < sizeof(output); index++) { + if (output[index] != want[index]) { + fprintf(stderr, "%[1]s wide=%[4]d iteration=%%u byte=%%u got=%%02x want=%%02x\n", iteration, index, output[index], want[index]); + return 1; + } + } + if (memcmp(input, before_input, sizeof(input))) { + return 2; + } + } +`, form.name, form.width, self, wide, register) + } + source.WriteString("\treturn 0;\n}\n") + triple := testTargetTriple(runtime.GOOS, "amd64") + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, fmt.Sprintf("legacy_alias_%d", wide), triple, + x86LegacyScalarAliasIR(t, wide, "amd64", triple), source.String(), runner) + } +} diff --git a/amd64_legacy_scalar_move_native.go b/amd64_legacy_scalar_move_native.go new file mode 100644 index 000000000..89454a8c1 --- /dev/null +++ b/amd64_legacy_scalar_move_native.go @@ -0,0 +1,76 @@ +package plan9asm + +import "fmt" + +// Only the complete legacy MOVSS/MOVSD source grammar selects this loader. +// FP fields, constants and register copies retain their existing bit-view +// contracts. Native memory is an exact unaligned, mandatory m32/m64 read. +func (c *amd64Ctx) loadLegacyScalarMoveSource(ins Instr, bits int) (string, error) { + form, known, err := amd64ScalarVectorMoveProperties(ins, c.goarch) + if err != nil { + return "", err + } + if !known || !form.spec.legacy || form.spec.bits != bits { + return "", fmt.Errorf("%s does not have the required legacy scalar move source", ins.Op) + } + source := ins.Args[form.low] + if form.memory != 0 || source.Kind != OpMem && source.Kind != OpSym { + if bits == 32 { + return c.evalF32(source) + } + return c.evalF64(source) + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return "", fmt.Errorf("%w: %s needs its original native address-size binding", ErrProbeNeedsContext, ins.Op) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + loaded, scalar := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %s, align 1\n", loaded, bits, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast i%d %%%s to %s\n", scalar, bits, loaded, amd64FloatingScalarType(bits)) + return "%" + scalar, nil +} + +// The public observer keeps logical-only storage. Only the paired legacy +// volatile emitter can continue an actual incoming-pointer m32/m64 access. +// Register copies preserve old high X bytes; memory copies zero those bytes. +// Both preserve every bit above 127, with no new storage or pointer facts. +func x86ResultLegacyScalarMoveStep(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string, native bool) (bool, bool) { + form, recognized, err := amd64ScalarVectorMoveProperties(ins, arch) + if !recognized || !form.spec.legacy { + return false, false + } + if err != nil || form.mask >= 0 || len(ins.Args) != 2 { + return false, true + } + width := int64(form.spec.bits / 8) + mask := uint16(1)<= 16 && index < 32 { + lane := index - 16 + want = 0 + if lane < %[2]d { + want = input[16+lane] + } else if %[3]t { + want = input[lane] + } + } + if output[index] != want { + t.Fatalf("%[1]s iteration=%%d byte=%%d got=%%#x want=%%#x", iteration, index, output[index], want) + } + } + if input != beforeInput { + t.Fatal("%[1]s changed input") + } + } +`, form.name, form.width, form.register) + } + tests.WriteString("}\n") + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module scalar\n\ngo 1.20\n", "scalar.go": declarations.String(), + "scalar_amd64.s": x86LegacyScalarMoveUpperAssembly("amd64"), "scalar_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + command.WaitDelay = 2 * time.Second + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("actual Go full-byte scalar move oracle: %v\n%s", err, output) + } +} + +func TestCrossLinuxRuntimeMatrixX86LegacyScalarMoveUpperBytes(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the Linux/amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + x86LegacyScalarMoveActualGoUpperOracle(t) + var source strings.Builder + source.WriteString(`#include +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range x86LegacyScalarMoveUpperForms() { + fmt.Fprintf(&source, "extern void %s(void*,void*) __asm__(LABEL(\"scalar.%s\"));\n", form.name, form.name) + } + source.WriteString("int main(void) {\n") + for _, form := range x86LegacyScalarMoveUpperForms() { + register := 0 + if form.register { + register = 1 + } + fmt.Fprintf(&source, ` + for (unsigned iteration = 0; iteration < 128; iteration++) { + uint8_t input[32], output[48], before_input[32], before_output[48]; + for (unsigned index = 0; index < sizeof(input); index++) { + input[index] = index*17 + iteration; + } + for (unsigned index = 0; index < sizeof(output); index++) { + output[index] = index*7 + iteration; + } + memcpy(before_input, input, sizeof(input)); + memcpy(before_output, output, sizeof(output)); + %[1]s(input, output + 16); + for (unsigned index = 0; index < sizeof(output); index++) { + uint8_t want = before_output[index]; + if (index >= 16 && index < 32) { + unsigned lane = index - 16; + want = lane < %[2]d ? input[16+lane] : (%[3]d ? input[lane] : 0); + } + if (output[index] != want) { + fprintf(stderr, "%[1]s iteration=%%u byte=%%u got=%%02x want=%%02x\n", iteration, index, output[index], want); + return 1; + } + } + if (memcmp(input, before_input, sizeof(input))) { + return 2; + } + } +`, form.name, form.width, register) + } + source.WriteString("\treturn 0;\n}\n") + triple := testTargetTriple(runtime.GOOS, "amd64") + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_upper_bytes", triple, + x86LegacyScalarMoveUpperIR(t, "amd64", triple), source.String(), runner) +} diff --git a/amd64_local_label_collision_test.go b/amd64_local_label_collision_test.go new file mode 100644 index 000000000..b27dff95a --- /dev/null +++ b/amd64_local_label_collision_test.go @@ -0,0 +1,54 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestX86LocalLabelsDoNotCollideWithIRValues(t *testing.T) { + for _, target := range []struct { + arch, triple, mov string + }{ + {"amd64", "x86_64-apple-darwin", "MOVQ"}, + {"amd64", "x86_64-unknown-linux-gnu", "MOVQ"}, + {"amd64", "x86_64-pc-windows-msvc", "MOVQ"}, + {"386", "i386-unknown-linux-gnu", "MOVL"}, + {"386", "i686-pc-windows-msvc", "MOVL"}, + } { + t.Run(target.triple, func(t *testing.T) { + source := fmt.Sprintf(`TEXT collision(SB),4,$0-0 +PXOR X1,X1 +JMP x1 +x1: +%s X1,AX +JMP t1 +t1: +JMP reg_AX +reg_AX: +JMP bb_foo +bb_foo: +RET +`, target.mov) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + ir, err := Translate(file, Options{ + Goarch: target.arch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"collision": { + Name: "collision", Ret: Void, + }}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "local-label.ll", "local-label.o", ir) + }) + } +} diff --git a/amd64_lock_scalar_rmw_backend_test.go b/amd64_lock_scalar_rmw_backend_test.go new file mode 100644 index 000000000..bab48bdf5 --- /dev/null +++ b/amd64_lock_scalar_rmw_backend_test.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86LockScalarRMWNativeBackendFollowsActualCalls(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + native, otherMode := testTargetTriple("linux", arch), "i386-unknown-linux-gnu" + if arch == "386" { + otherMode = "x86_64-unknown-linux-gnu" + } + for _, row := range []struct{ instruction, mnemonic string }{ + {"ADCL CX,1(SI)", "adcl"}, + {"ADDB $1,1(SI)", "addb"}, + {"NOTL 1(SI)", "notl"}, + {"BTCL $1,1(SI)", "btcl"}, + } { + source := x86NamedLockOracleSource(arch, "CMPL AX,AX\nLOCK\n"+row.instruction) + x86LockScalarOriginalInstruction(t, arch, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+row.instruction+"/"+api+"/native", func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, native) + if err != nil { + t.Fatal("complete native LOCK request rejected", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := parseIRModuleInContext(ctx, ir) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + caller := module.NamedFunction("locked") + if caller.IsNil() { + t.Fatal("public LOCK source body is absent") + } + calls := 0 + for block := caller.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.IsACallInst().IsNil() { + continue + } + helper := instruction.CalledValue() + if !strings.HasPrefix(helper.Name(), "__plan9asm_lock_rmw_") { + continue + } + calls++ + if helper.FirstUse().IsNil() || helper.FirstBasicBlock().IsNil() { + t.Fatal("caller did not reference a defined native helper") + } + nativeUse := false + for body := helper.FirstBasicBlock(); !body.IsNil(); body = llvm.NextBasicBlock(body) { + for emitted := body.FirstInstruction(); !emitted.IsNil(); emitted = llvm.NextInstruction(emitted) { + for index := 0; index < emitted.OperandsCount(); index++ { + assembly := emitted.Operand(index).IsAInlineAsm() + if assembly.IsNil() { + continue + } + info := assembly.InlineAsmInfo() + if info.HasSideEffects && strings.Contains(info.Assembly, "lock; "+row.mnemonic) { + nativeUse = true + } + } + } + } + if !nativeUse { + t.Fatal("actually called helper lost its native atomic instruction") + } + } + } + if calls != 1 { + t.Fatal("source body must call exactly one complete LOCK helper", calls) + } + }) + for _, triple := range []string{"", otherMode, "aarch64-unknown-linux-gnu", "wasm32-unknown-unknown"} { + t.Run(arch+"/"+row.instruction+"/"+api+"/"+triple, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, triple) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "inline assembly") { + t.Fatal("complete native LOCK escaped an absent or foreign backend", ir != "", err) + } + }) + } + } + } + } +} + +func TestX86LockScalarRMWPortableNonLOCKDoesNotAcquireHelpers(t *testing.T) { + source := "TEXT ·plain(SB),4,$0-0\nMOVL $1,AX\nMOVL $2,BX\nADDL AX,BX\nRET\n" + for _, arch := range []string{"amd64", "386"} { + goAssemblerOracleOutput(t, arch, source, "", true) + for _, triple := range []string{"", "aarch64-unknown-linux-gnu", "wasm32-unknown-unknown"} { + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+triple+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func plain()", arch, triple, api) + if err != nil || ir == "" || strings.Contains(ir, "__plan9asm_lock_rmw_") { + t.Fatal("portable unprefixed scalar operation acquired a native helper or backend", err) + } + }) + } + } + } +} diff --git a/amd64_lock_scalar_rmw_bank_test.go b/amd64_lock_scalar_rmw_bank_test.go new file mode 100644 index 000000000..58a650140 --- /dev/null +++ b/amd64_lock_scalar_rmw_bank_test.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86LockScalarRMWByteSourcePhysicalBank(t *testing.T) { + for _, stem := range []string{"ADC", "ADD", "AND", "OR", "SBB", "SUB", "XOR"} { + for _, source := range []string{"AH", "CH", "DH", "BH"} { + for _, address := range []struct { + name, setup, memory string + rex bool + }{ + {"SI", "", "(SI)", false}, + {"R8", "MOVQ SI,R8\n", "(R8)", true}, + {"R15", "MOVQ SI,R15\n", "(R15)", true}, + {"R8-index", "MOVQ $0,R8\n", "(SI)(R8*1)", true}, + {"R15-index", "MOVQ $0,R15\n", "(SI)(R15*1)", true}, + } { + t.Run(stem+"/"+source+"/"+address.name, func(t *testing.T) { + assembly := x86NamedLockOracleSource("amd64", address.setup+ + "CLC\nLOCK\n"+stem+"B "+source+","+address.memory) + code := x86LockScalarOriginalInstruction(t, "amd64", assembly) + physical, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go bytes=%x operands=%s", code, x86asm.GoSyntax(physical, 0, nil)) + want := map[string]x86asm.Reg{"AH": x86asm.AH, "CH": x86asm.CH, + "DH": x86asm.DH, "BH": x86asm.BH}[source] + if address.rex { + want = map[string]x86asm.Reg{"AH": x86asm.SPB, "CH": x86asm.BPB, + "DH": x86asm.SIB, "BH": x86asm.DIB}[source] + } + if physical.Args[1] != want { + t.Fatalf("actual source physical bank=%v, want %v", physical.Args[1], want) + } + triple := "x86_64-unknown-linux-gnu" + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, assembly, + "package ordering\nfunc locked(p *[16]byte)\n", "amd64", triple) + if !address.rex { + if err != nil || !strings.Contains(ir, "lock; "+strings.ToLower(stem)+"b") { + t.Fatal("native high byte without REX lost its actual source bank", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("source high-byte spelling borrowed a different REX physical bank", err) + } + }) + } + }) + } + } + } +} diff --git a/amd64_lock_scalar_rmw_boundaries_test.go b/amd64_lock_scalar_rmw_boundaries_test.go new file mode 100644 index 000000000..5432cc8f8 --- /dev/null +++ b/amd64_lock_scalar_rmw_boundaries_test.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86LockScalarRMWTypedRolesAgreeWithoutMemoryPermission(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86LockScalarOracleForms(arch) { + t.Run(arch+"/"+row.text, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·f(SB),4,$0-0\n"+row.text+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if len(file.Funcs[0].Instrs) != 3 || file.Funcs[0].Instrs[0].Op != OpTEXT { + t.Fatal("complete fixture lost its TEXT/RMW/RET roles") + } + ins := file.Funcs[0].Instrs[1] + form, known, err := x86LockScalarRMWProperties(ins.Op, ins, arch) + if err != nil || !known { + t.Fatal("complete accepted grammar row lost its typed form", err) + } + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if err != nil || !known || len(effect.operands) != len(ins.Args) { + t.Fatal("accepted lowering row lacks its complete ordered byte effect", err) + } + for index, operand := range effect.operands { + if operand.index != index || operand.width != int64(form.spec.bits/8) || !operand.read || + operand.write != (index == len(ins.Args)-1) { + t.Fatal("read/write/native width role changed", effect) + } + } + stem := row.op[:len(row.op)-1] + defined, killed, read := x86ByteFlagsAll, uint8(0), uint8(0) + switch stem { + case "ADC", "SBB": + read = x86ByteFlagCF + case "AND", "OR", "XOR": + defined &^= x86ByteFlagAF + killed = x86ByteFlagAF + case "INC", "DEC": + defined &^= x86ByteFlagCF + case "NOT": + defined = 0 + case "BTC", "BTR", "BTS": + defined = x86ByteFlagCF + killed = x86ByteFlagOF | x86ByteFlagSF | x86ByteFlagAF | x86ByteFlagPF + } + bitRegister := strings.HasPrefix(stem, "BT") && ins.Args[0].Kind == OpReg + if effect.flagsRead != read || effect.flagsDefined != defined || effect.flagsKilled != killed || + effect.dynamicMemory != bitRegister { + t.Fatal("defined/preserved/killed flags or full bit-string memory role changed", effect) + } + for _, address := range []string{"p+0(FP)", "local-16(SP)", "0(SP)", "0(BP)", "1(SI)(BP*1)"} { + body := strings.Replace(row.text, "1(SI)", address, 1) + source := strings.Replace(x86NamedLockOracleSource(arch, "CLC\nLOCK\n"+body), "$0-", "$32-", 1) + x86LockScalarOriginalInstruction(t, arch, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("pure effect borrowed virtual FP/AUTO native backing", api, address, err) + } + } + } + }) + } + } +} + +func TestX86LockScalarRMWNativeBoundaryCannotBorrowVirtualOrMixedProof(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86LockScalarOracleForms(arch) { + t.Run(arch+"/"+row.text, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·f(SB),4,$32-0\n"+row.text+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, known, err := x86LockScalarRMWProperties(ins.Op, ins, arch) + if err != nil || !known { + t.Fatal(err) + } + // These are consumer-input mutations, not additional Go syntax + // acceptance claims. In particular, SP as an index is invalid. + for _, memory := range []MemRef{ + {Base: SP}, {Base: BP}, {Base: SI, Index: SP, Scale: 1}, + {Base: SI, Index: BP, Scale: 1}, {Base: SP, Sym: "local", Off: -16}, + } { + changed := form + changed.destination = Operand{Kind: OpMem, Mem: memory} + if err := x86LockScalarNativeMemoryBound(changed, ins, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native form boundary borrowed virtual frame backing", memory, err) + } + } + ins.x86UnboundAddressOverride = true + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch, ctx.namedAtomicLock, ctx.lockScalarNS = arch, true, "__bound_" + before, temporary := output.String(), ctx.tmp + known, terminated, err := ctx.lowerLockScalarRMW(ins.Op, ins) + if !known || terminated || !errors.Is(err, ErrProbeNeedsContext) || + before != output.String() || temporary != ctx.tmp { + t.Fatal("mixed unbound native marker borrowed another form's bridge", err) + } + }) + } + } +} + +func TestX86LockScalarRMWMalformedSourceAndDispatchRejectBeforeIR(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, body := range []string{ + "ADDL CX", "ADDL (SI),(DI)", "ADDL X0,(SI)", "ADDL CX,$1", + "ADDL $1.0,(SI)", "ADDL.BAD CX,(SI)", "BTCL $128,(SI)", + "BTCL $-129,(SI)", "INCL $1", "NOTL (SI),(DI)", + "ADDL CX,(SI)(SP*1)", "ADDL CX,(SI)(DI*3)", + } { + t.Run(arch+"/"+body, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, "CLC\nLOCK\n"+body) + goAssemblerOracleOutput(t, arch, source, "", false) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + if err == nil || ir != "" { + t.Fatal("malformed complete source emitted IR", api, err) + } + } + }) + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.namedAtomicLock, ctx.lockScalarNS = true, "__bound_" + before, temporary := output.String(), ctx.tmp + ins := Instr{Op: "SUBL", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpMem, Mem: MemRef{Base: SI}}}} + known, terminated, err := ctx.lowerLockScalarRMW("ADDL", ins) + if !known || terminated || err == nil || before != output.String() || temporary != ctx.tmp { + t.Fatal("contradictory dispatch mutated IR or temporary state", known, terminated, err) + } +} + +func TestX86LockScalarRMWCarryDoesNotAuthorizeFrameReads(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, address := range []string{"0(SP)", "0(BP)", "missing+32(FP)"} { + t.Run(arch+"/"+address, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, "CMPL "+address+",AX\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("CF observation borrowed an unproved stack/frame read", api, err) + } + } + }) + } + } +} diff --git a/amd64_lock_scalar_rmw_carry.go b/amd64_lock_scalar_rmw_carry.go new file mode 100644 index 000000000..35ed7cfca --- /dev/null +++ b/amd64_lock_scalar_rmw_carry.go @@ -0,0 +1,358 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" +) + +// This domain proves only the source of the CF value consumed by a named +// locked ADC/SBB. It grants no register, memory, frame or caller permission. +// GP byte facts prevent a default-zero alloca or an empty flags stack from +// becoming an incoming carry. The ordinary consumers still check operands. +type x86LockCarryState struct { + carry bool + gp [16]uint8 + stack []x86LockCarryCell + stackUnbound bool +} + +type x86LockCarryCell struct { + bits int + valueKnown bool + carryKnown bool +} + +func (state x86LockCarryState) clone() x86LockCarryState { + state.stack = append([]x86LockCarryCell(nil), state.stack...) + return state +} + +func (state x86LockCarryState) intersect(other x86LockCarryState) x86LockCarryState { + state = state.clone() + state.carry = state.carry && other.carry + state.stackUnbound = state.stackUnbound || other.stackUnbound + for index := range state.gp { + state.gp[index] &= other.gp[index] + } + if len(state.stack) != len(other.stack) { + state.stack = nil + state.stackUnbound = true + } else { + for index := range state.stack { + if state.stack[index].bits != other.stack[index].bits { + state.stack = nil + state.stackUnbound = true + break + } + state.stack[index].valueKnown = state.stack[index].valueKnown && other.stack[index].valueKnown + state.stack[index].carryKnown = state.stack[index].carryKnown && other.stack[index].carryKnown + } + } + if state.stackUnbound { + state.stack = nil + } + return state +} + +func (c *amd64Ctx) lockCarryRead(operand Operand, width int64, state x86LockCarryState) bool { + if operand.Kind == OpFP { + for _, slot := range c.sig.Frame.Params { + bytes := x86FrameScalarBytes(slot.Type, c.goarch) + if bytes > 0 && operand.FPOffset >= slot.Offset && + operand.FPOffset-slot.Offset <= bytes-width && width > 0 { + return true // Actual typed argument bits, not pointee/backing facts. + } + } + return false + } + if operand.Kind == OpMem || operand.Kind == OpSym { + // A native instruction reads actual memory bits; this does not prove + // its address or authorize the read. The final memory guards do that. + return operand.Kind == OpSym && !strings.HasPrefix(operand.Sym, "$") || + operand.Kind == OpMem && operand.Mem.Sym == "" && operand.Mem.Base != SP && operand.Mem.Base != BP + } + return x86ResultRead(operand, width, &x86ResultState{gp: state.gp}, nil, c.goarch) +} + +func (c *amd64Ctx) lockCarryWrite(operand Operand, width int64, known bool, state *x86LockCarryState) { + if operand.Kind == OpMem || operand.Kind == OpSym || operand.Kind == OpFP { + // No numerical/address fact here establishes nonaliasing with an + // outstanding source PUSH. Even an independently allowed native store + // cannot preserve a stack cell without a separate exact alias proof. + state.stack = nil + return + } + if operand.Kind != OpReg { + return + } + if base, valid := x86CallerGP(operand.Reg); valid && base == SP { + // Explicit SP replacement/adjustment is not an ordinary balanced + // PUSH/POP. This domain cannot recover its original physical identity, + // even if a later instruction defines the pushed GP value again. + state.stack = nil + state.stackUnbound = true + } + index, mask, valid := x86ResultGP(operand.Reg, width, c.goarch) + if !valid { + state.gp = [16]uint8{} + return + } + state.gp[index] &^= mask + if known { + state.gp[index] |= mask + } + if width == 4 || width == 8 { + state.gp[index] = 0 + if known { + state.gp[index] = 0xff + } + } +} + +func (c *amd64Ctx) lockCarryTransfer(ins Instr, state x86LockCarryState) x86LockCarryState { + ins = x86SourceMoveInstruction(ins, c.goarch) // Local numerical dispatch, never source/certificate identity. + op := Op(strings.ToUpper(string(ins.Op))) + if op == OpTEXT || op == OpRET || op == "LOCK" && len(ins.Args) == 0 || + op == OpJMP || isAMD64ConditionalBranch(op) || op == "LOOP" { + return state + } + if (op == "CLC" || op == "STC" || op == "CMC") && len(ins.Args) == 0 { + state.carry = op != "CMC" || state.carry + return state + } + if width := x86RawReturnScalarMoveBytes(op); width != 0 && + op == ins.Op && len(ins.Args) == 2 && !(width == 8 && c.goarch == "386") { + known := c.lockCarryRead(ins.Args[0], width, state) + c.lockCarryWrite(ins.Args[1], width, known, &state) + return state + } + if spec, found := amd64StackWidthSpecs[op]; found { + validMode := spec.bits == 16 || spec.bits == 32 && c.goarch == "386" || spec.bits == 64 && c.goarch == "amd64" + validCount := len(ins.Args) == 0 + if spec.mode == amd64StackPushValue || spec.mode == amd64StackPopValue { + validCount = len(ins.Args) == 1 + } + if !validMode || !validCount { + return x86LockCarryState{stackUnbound: true} + } + if spec.mode == amd64StackPushValue || spec.mode == amd64StackPushFlags { + if len(state.stack) >= 64 { + return x86LockCarryState{stackUnbound: true} + } + cell := x86LockCarryCell{bits: spec.bits, carryKnown: state.carry} + if spec.mode == amd64StackPushValue { + cell.valueKnown = c.lockCarryRead(ins.Args[0], int64(spec.bits/8), state) + cell.carryKnown = cell.valueKnown + } + if state.stackUnbound { + cell.valueKnown, cell.carryKnown = false, false + } + // PUSHF captures a flags word, but this domain proves only its CF. + // POP into GP must not promote unknown OF or other bits to numeric + // byte facts. A direct POPF may independently consume known CF. + state.stack = append(state.stack, cell) + return state + } + cell := x86LockCarryCell{} + if len(state.stack) != 0 { + cell = state.stack[len(state.stack)-1] + state.stack = state.stack[:len(state.stack)-1] + if cell.bits != spec.bits { + cell = x86LockCarryCell{} + } + } + if spec.mode == amd64StackPopFlags { + state.carry = cell.carryKnown + } else { + c.lockCarryWrite(ins.Args[0], int64(spec.bits/8), cell.valueKnown, &state) + } + return state + } + if effect, recognized, err := x86TypedInstructionByteEffect(op, ins, c.goarch); recognized { + if err != nil { + return x86LockCarryState{stackUnbound: true} + } + known := effect.flagsRead&x86ByteFlagCF == 0 || state.carry + for _, operand := range effect.operands { + if operand.read { + known = known && c.lockCarryRead(operand.operand, operand.width, state) + } + } + for _, operand := range effect.operands { + if operand.write { + c.lockCarryWrite(operand.operand, operand.width, known, &state) + } + } + if effect.flagsKilled&x86ByteFlagCF != 0 { + state.carry = false + } + if effect.flagsDefined&x86ByteFlagCF != 0 { + state.carry = known + } + return state + } + if (op == "CMPB" || op == "CMPW" || op == "CMPL" || op == "CMPQ" || + op == "TESTB" || op == "TESTW" || op == "TESTL" || op == "TESTQ") && len(ins.Args) == 2 { + bits := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[string(op)[len(op)-1]] + valid := validateAMD64ScalarAddSubOperands(c.goarch, string(op), bits, ins.Args[0], ins.Args[1], ins) == nil + if strings.HasPrefix(string(op), "CMP") { + valid = valid || validateAMD64ScalarAddSubOperands(c.goarch, string(op), bits, ins.Args[1], ins.Args[0], ins) == nil + } + if !(bits == 64 && c.goarch == "386") && valid { + state.carry = c.lockCarryRead(ins.Args[0], int64(bits/8), state) && c.lockCarryRead(ins.Args[1], int64(bits/8), state) + return state + } + } + if op == "STMXCSR" && len(ins.Args) == 1 && isAMD64MemoryOperand(ins.Args[0]) { + c.lockCarryWrite(ins.Args[0], 4, false, &state) + return state + } + if op == "NOP" && len(ins.Args) == 0 { + return state + } + // Calls, native/raw effects and unregistered forms have no incoming flags + // preservation contract. A later explicit definition may start a new fact. + return x86LockCarryState{stackUnbound: true} +} + +func (c *amd64Ctx) verifyNamedLockCarry() error { + var required [][2]int + for block, body := range c.blocks { + for index := 1; index < len(body.instrs); index++ { + previous, ins := body.instrs[index-1], body.instrs[index] + if previous.Op != "LOCK" || len(previous.Args) != 0 { + continue + } + form, known, err := x86LockScalarRMWProperties(ins.Op, ins, c.goarch) + if known && err == nil && (form.spec.stem == "ADC" || form.spec.stem == "SBB") { + required = append(required, [2]int{block, index}) + } + } + } + if len(required) == 0 { + return nil // Unrelated translations retain their existing contracts. + } + fail := func(reason string) error { + return fmt.Errorf("%w: named LOCK incoming carry is not source-defined: %s", ErrProbeNeedsContext, reason) + } + if len(c.blocks) > 4096 { + return fail("source CFG exceeds the bounded proof domain") + } + steps := 0 + states := map[int]x86LockCarryState{0: {}} + queue := []int{0} + inputs := make(map[[2]int]bool) + for visits := 0; len(queue) != 0; visits++ { + if visits >= 32768 { + return fail("source CFG fixed-point budget exhausted") + } + block := queue[0] + queue = queue[1:] + state := states[block].clone() + for index, ins := range c.blocks[block].instrs { + steps++ + if steps > 262144 { + return fail("combined source instruction work budget exhausted") + } + inputs[[2]int{block, index}] = state.carry + state = c.lockCarryTransfer(ins, state) + } + successors, valid := c.lockCarrySuccessors(block) + if !valid { + return fail("indirect, external or unresolved source control flow") + } + for _, target := range successors { + if previous, found := states[target]; found { + stateAtTarget := previous.intersect(state) + if reflect.DeepEqual(previous, stateAtTarget) { + continue + } + states[target] = stateAtTarget + } else { + states[target] = state.clone() + } + queue = append(queue, target) + } + } + for _, location := range required { + if !inputs[location] { + return fail("a complete ADC/SBB has an unknown predecessor CF value") + } + } + return nil +} + +func (c *amd64Ctx) lockCarrySuccessors(block int) ([]int, bool) { + return c.sourceBlockSuccessors(block, nil) +} + +// This decodes only ordinary source control flow. It supplies no LOCK, carry, +// memory or entry authority. A complete label index avoids repeated scans for +// consumers whose analysis must remain linear in source nodes and edges. +func (c *amd64Ctx) sourceBlockSuccessors(block int, labels map[string]int) ([]int, bool) { + body := c.blocks[block].instrs + if len(body) == 0 { + if block+1 < len(c.blocks) { + return []int{block + 1}, true + } + return nil, true + } + ins := body[len(body)-1] + op := Op(strings.ToUpper(string(ins.Op))) + if op == OpRET { + return nil, len(ins.Args) == 0 + } + if op != OpJMP && op != "LOOP" && !isAMD64ConditionalBranch(op) { + if block+1 < len(c.blocks) { + return []int{block + 1}, true + } + return nil, false + } + args := ins.Args + if op != OpJMP && op != "LOOP" && !isAMD64CounterZeroBranch(op) && + len(args) == 2 && args[0].Kind == OpImm && (args[0].Imm == 0 || args[0].Imm == 1) { + args = args[1:] + } + if len(args) != 1 { + return nil, false + } + target, found := -1, false + if args[0].Kind == OpMem && args[0].Mem.Base == PC && args[0].Mem.OffRaw == "" { + base := int64(c.blockBase[block] + len(body) - 1) + if args[0].Mem.Off > 1<<63-1-base { + return nil, false + } + pc := base + args[0].Mem.Off + if pc >= 0 && pc <= int64(^uint(0)>>1) { + target, found = c.blockByIdx[int(pc)] + } + } else { + label := "" + switch args[0].Kind { + case OpLabel, OpSym: + label = args[0].Sym + case OpIdent: + label = args[0].Ident + case OpReg: + label = string(args[0].Reg) + } + if labels != nil { + target, found = labels[label] + } else { + for index, candidate := range c.blocks { + if candidate.name == label { + target, found = index, true + break + } + } + } + } + if !found || target < 0 || target >= len(c.blocks) { + return nil, false + } + if op == OpJMP { + return []int{target}, true + } + return []int{target, block + 1}, block+1 < len(c.blocks) +} diff --git a/amd64_lock_scalar_rmw_carry_alias_test.go b/amd64_lock_scalar_rmw_carry_alias_test.go new file mode 100644 index 000000000..27616f720 --- /dev/null +++ b/amd64_lock_scalar_rmw_carry_alias_test.go @@ -0,0 +1,82 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86LockScalarRMWCarryStackAliasesAndIdentityStayContext(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word := "Q" + if arch == "386" { + word = "L" + } + for _, width := range []string{"W", word} { + push, pop := "PUSH"+width+" DX\n", "\nPOPF"+width + for _, control := range []struct{ name, body string }{ + {"explicit stack store", push + "MOV" + width + " BP,0(SP)" + pop}, + {"derived stack store", push + "MOV" + word + " SP,BX\nMOV" + width + " BP,(BX)" + pop}, + {"unknown vector store", push + "MOVOU X0,0(SP)" + pop}, + {"CSR stack store", push + "STMXCSR 0(SP)" + pop}, + {"SP replacement before push", "MOV" + word + " $0,SP\nMOVL $5,DX\n" + push + "POPF" + width}, + {"POP SP before new push", "PUSH" + word + " BP\nPOP" + word + " SP\nMOVL $5,DX\n" + push + "POPF" + width}, + {"LEA SP before push", "LEA" + word + " 0(SP),SP\nMOVL $5,DX\n" + push + "POPF" + width}, + {"balanced SP adjustment", "SUB" + word + " $16,SP\nMOVL $5,DX\n" + push + "POPF" + width + "\nADD" + word + " $16,SP"}, + } { + t.Run(arch+"/"+width+"/"+control.name, func(t *testing.T) { + // These accepted objects are unsafe/unknown-input tripwires. + // They are never executed as numerical or caller evidence. + source := x86NamedLockOracleSource(arch, control.body+"\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + if err := ctx.verifyNamedLockCarry(); !errors.Is(err, ErrProbeNeedsContext) || + output.String() != before || ctx.tmp != temporary { + t.Error("pure CF domain retained stale stack bits or regained replaced SP identity", err) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("a stale stack value or replaced SP authorized carry", err) + } + }) + } + }) + } + } + } +} + +func TestX86LockScalarRMWCarryInvalidStackDoesNotInvalidateExplicitCF(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word := "Q" + if arch == "386" { + word = "L" + } + for _, seed := range []string{"CLC", "STC"} { + source := x86NamedLockOracleSource(arch, "MOV"+word+" $0,SP\n"+seed+"\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + if err := ctx.verifyNamedLockCarry(); err != nil || output.String() != before || ctx.tmp != temporary { + t.Fatal("explicit CF fact was lost or granted an IR/memory permission", err) + } + if strings.Contains(output.String(), "lock;") { + t.Fatal("a CF-only observation emitted an atomic object") + } + } + } +} diff --git a/amd64_lock_scalar_rmw_carry_stack_test.go b/amd64_lock_scalar_rmw_carry_stack_test.go new file mode 100644 index 000000000..dc9f7bfec --- /dev/null +++ b/amd64_lock_scalar_rmw_carry_stack_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86LockScalarRMWCarryStackDoesNotInventWholeFlagsWord(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word, counter := "Q", "JCXZQ" + if arch == "386" { + word, counter = "L", "JCXZL" + } + for _, suffix := range []string{"W", word} { + compare := "AND" + suffix + " $2048,AX\nCMP" + suffix + " AX,$1" + for _, control := range []struct { + name, body string + known bool + }{ + {"unknown OF through flags word", "CLC\nPUSHF" + suffix + "\nPOP" + suffix + " AX\n" + compare, false}, + {"CF-only flags roundtrip", "CLC\nPUSHF" + suffix + "\nPOPF" + suffix, true}, + {"defined whole numerical word", "PUSH" + suffix + " DX\nPOP" + suffix + " AX\n" + compare, true}, + {"mixed flags and numerical join", "CLC\n" + counter + " flags\nPUSH" + suffix + " DX\nPOP" + suffix + " AX\nJMP compare\nflags:\nPUSHF" + suffix + "\nPOP" + suffix + " AX\ncompare:\n" + compare, false}, + {"partial flags word through GP", "CLC\nPUSHF" + suffix + "\nPOP" + suffix + " AX\nPUSH" + suffix + " AX\nPOPF" + suffix, false}, + } { + t.Run(arch+"/"+suffix+"/"+control.name, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, control.body+"\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + triple := testTargetTriple("linux", arch) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, triple) + if control.known { + if err != nil || !strings.Contains(ir, "lock; adcl") { + t.Fatal("known numerical word or CF roundtrip was lost", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("CF-only PUSHF falsely defined unknown GP/OF bits", err) + } + }) + } + }) + } + } + } +} diff --git a/amd64_lock_scalar_rmw_carry_test.go b/amd64_lock_scalar_rmw_carry_test.go new file mode 100644 index 000000000..dcfabe3c3 --- /dev/null +++ b/amd64_lock_scalar_rmw_carry_test.go @@ -0,0 +1,139 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86LockScalarRMWUnknownIncomingCarryNeedsContext(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + widths := "BWL" + if arch == "amd64" { + widths += "Q" + } + for _, stem := range []string{"ADC", "SBB"} { + for _, width := range widths { + register := "CX" + if width == 'B' { + register = "CL" + } + body := "LOCK\n\t" + stem + string(width) + " " + register + ",(SI)" + t.Run(arch+"/"+stem+string(width), func(t *testing.T) { + source := x86NamedLockOracleSource(arch, body) + machine := x86LockScalarOriginalInstruction(t, arch, source) + t.Logf("actual Go accepts original instruction=%x; incoming CF remains undefined by the source", machine) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + // The unchanged source initializes its GP operands but + // supplies no CF producer. The public error phase/text + // is not part of this missing-input contract. + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("native carry-dependent memory effect borrowed an undefined entry flag: %v", err) + } + }) + } + }) + } + } + } +} + +func TestX86LockScalarRMWCarryCFGAndStackSources(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + suffix, counter := "Q", "JCXZQ" + if arch == "386" { + suffix, counter = "L", "JCXZL" + } + for _, control := range []struct { + name, body string + known bool + }{ + {"CLC", "CLC", true}, + {"STC CMC", "STC\nCMC", true}, + {"initialized CMP", "CMPL AX,$0", true}, + {"undefined CMC", "CMC", false}, + {"undefined compare", "CMPL BP,$0", false}, + {"explicit stack seed", "PUSH" + suffix + " DX\nPOPF" + suffix, true}, + {"unknown stack seed", "PUSH" + suffix + " BP\nPOPF" + suffix, false}, + {"unseeded join", counter + " join\nCLC\njoin:", false}, + {"seeded join", "STC\n" + counter + " join\nCLC\njoin:", true}, + {"call clobber", "STC\nCALL ·other(SB)", false}, + {"undefined divide flags", "STC\nDIVL CX", false}, + {"loop retains seed", "STC\nloop:\nNOP\nLOOP loop", true}, + } { + t.Run(arch+"/"+control.name, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, + control.body+"\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + carryErr := ctx.verifyNamedLockCarry() + if output.String() != before || ctx.tmp != temporary { + t.Fatal("carry-only proof changed emitted code") + } + if control.known { + if carryErr != nil { + t.Fatal("defined source carry was lost", carryErr) + } + } else if !errors.Is(carryErr, ErrProbeNeedsContext) { + t.Fatal("independent carry proof accepted an undefined source", carryErr) + } + // The public proof may reject an earlier undefined stack value + // before reaching LOCK. Its phase is not the carry-only contract. + triple := testTargetTriple("linux", arch) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\nfunc other()\n", arch, triple) + if control.known { + if err != nil || !strings.Contains(ir, "lock; adcl") { + t.Fatal("independent defined carry was not retained", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("unclosed carry acquired a numerical value", err) + } + }) + } + }) + } + } +} + +func TestX86LockScalarRMWCarryBudgetAndInputsRemainUnchanged(t *testing.T) { + file, err := Parse(ArchAMD64, x86NamedLockOracleSource("amd64", "CLC\nLOCK\nADCL CX,(SI)")) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before, temporary := output.String(), ctx.tmp + source := fmt.Sprintf("%#v", file) + for count := 0; count != 2; count++ { + if err := ctx.verifyNamedLockCarry(); err != nil || before != output.String() || temporary != ctx.tmp || source != fmt.Sprintf("%#v", file) { + t.Fatal("CF proof modified IR, temporary state or input source", err) + } + } + ctx.blocks = make([]amd64Block, 4097) + ctx.blocks[0].instrs = []Instr{{Op: "LOCK"}, {Op: "ADCL", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpMem, Mem: MemRef{Base: SI}}}}} + if err := ctx.verifyNamedLockCarry(); !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "bounded") { + t.Fatal("oversized CF domain did not fail closed", err) + } + ctx.blocks = nil + // Synthetic empty/mismatched cells are consumer tripwires, not Go source + // acceptance evidence: stock Go rejects a bare POPF as an unbalanced pop. + if state := ctx.lockCarryTransfer(Instr{Op: "POPFQ"}, x86LockCarryState{carry: true}); state.carry { + t.Fatal("synthetic empty stack supplied carry") + } + state := x86LockCarryState{carry: true, stack: []x86LockCarryCell{{bits: 16, valueKnown: true, carryKnown: true}}} + if state = ctx.lockCarryTransfer(Instr{Op: "POPFQ"}, state); state.carry { + t.Fatal("synthetic mismatched-width stack cell supplied carry") + } +} diff --git a/amd64_lock_scalar_rmw_dispatch_test.go b/amd64_lock_scalar_rmw_dispatch_test.go new file mode 100644 index 000000000..131210031 --- /dev/null +++ b/amd64_lock_scalar_rmw_dispatch_test.go @@ -0,0 +1,73 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// A correct but uncalled helper is not instruction coverage. Independently +// follow every public source body through dispatch and both IR levels. +func TestX86LockScalarRMWPublicBodiesCallCompleteNativeHelpers(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86OrderingHintTargets { + t.Run(target.triple, func(t *testing.T) { + rows := x86LockScalarOracleForms(target.arch) + want := 90 + if target.arch == "386" { + want = 66 + } + if len(rows) != want { + t.Fatal("incomplete independent scalar LOCK family", len(rows), want) + } + var source, declarations strings.Builder + declarations.WriteString("package ordering\n") + names := make([]string, len(rows)) + helpers := make([]string, len(rows)) + for index, row := range rows { + names[index] = fmt.Sprintf("locked%d", index) + original := x86NamedLockOracleSource(target.arch, "CMPL AX,AX\nLOCK\n"+row.text) + source.WriteString(strings.Replace(original, "·locked", "·"+names[index], 1)) + fmt.Fprintf(&declarations, "func %s(p *[16]byte)\n", names[index]) + bits := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[row.op[len(row.op)-1]] + stem := strings.ToLower(row.op[:len(row.op)-1]) + helpers[index] = fmt.Sprintf("__plan9asm_lock_rmw_0_%s_%s%d_native", target.arch, stem, bits) + if strings.Contains(row.text, "$1") { + helpers[index] += "_imm_0000000000000001" + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + for _, level := range []int{0, 2} { + t.Run(fmt.Sprintf("IR%d", level), func(t *testing.T) { + text := ir + if level == 2 { + text = x86MulDivRoundOptimize(t, text) + } + bodies := x86StringMemoryFunctions(t, text, names) + for index, name := range names { + t.Run(rows[index].text, func(t *testing.T) { + body := bodies[name] + if strings.Count(body, "@"+helpers[index]+"(") != 1 { + t.Fatal("public body bypassed its complete native LOCK helper", name, helpers[index], body) + } + }) + } + for _, backend := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, backend) + } + }) + } + }) + } + }) + } +} diff --git a/amd64_lock_scalar_rmw_encoder_test.go b/amd64_lock_scalar_rmw_encoder_test.go new file mode 100644 index 000000000..7d0b989d6 --- /dev/null +++ b/amd64_lock_scalar_rmw_encoder_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +// This independent carrier binds every ordered row, operand class, zcase, +// offset, prefix and opcode byte. It does not establish an address permission +// or assert which overlapping row an actual source instruction selects. +func x86LockScalarEncoderCarrier(source []byte) error { + file, err := parser.ParseFile(token.NewFileSet(), "asm6.go", source, 0) + if err != nil { + return err + } + variables := make(map[string]*ast.CompositeLit) + for _, declaration := range file.Decls { + general, ok := declaration.(*ast.GenDecl) + if !ok || general.Tok != token.VAR { + continue + } + for _, spec := range general.Specs { + value := spec.(*ast.ValueSpec) + for index, name := range value.Names { + if index < len(value.Values) { + if literal, ok := value.Values[index].(*ast.CompositeLit); ok { + variables[name.Name] = literal + } + } + } + } + } + canonical := func(node ast.Node) string { + var output bytes.Buffer + if err := format.Node(&output, token.NewFileSet(), node); err != nil { + return "" + } + return output.String() + } + parse := func(source string) (ast.Expr, error) { + return parser.ParseExpr(source) + } + tables := map[string]string{ + "yxorb": `[]ytab{ + {Zib_, 1, argList{Yi32, Yal}}, + {Zibo_m, 2, argList{Yi32, Ymb}}, + {Zr_m, 1, argList{Yrb, Ymb}}, + {Zm_r, 1, argList{Ymb, Yrb}}, + }`, + "yaddl": `[]ytab{ + {Zibo_m, 2, argList{Yi8, Yml}}, + {Zil_, 1, argList{Yi32, Yax}}, + {Zilo_m, 2, argList{Yi32, Yml}}, + {Zr_m, 1, argList{Yrl, Yml}}, + {Zm_r, 1, argList{Yml, Yrl}}, + }`, + "yincl": `[]ytab{{Z_rp, 1, argList{Yrl}}, {Zo_m, 2, argList{Yml}}}`, + "yincq": `[]ytab{{Zo_m, 2, argList{Yml}}}`, + "yscond": `[]ytab{{Zo_m, 2, argList{Ymb}}}`, + "ybtl": `[]ytab{{Zibo_m, 2, argList{Yi8, Yml}}, {Zr_m, 1, argList{Yrl, Yml}}}`, + } + for name, text := range tables { + want, err := parse(text) + if err != nil { + return err + } + actual := variables[name] + if actual == nil || canonical(actual) != canonical(want) { + return fmt.Errorf("complete ordered Go encoder table changed: %s", name) + } + } + optab := variables["optab"] + if optab == nil { + return fmt.Errorf("Go encoder has no complete optab") + } + pattern := regexp.MustCompile(`^A(?:ADC|ADD|AND|OR|SBB|SUB|XOR|INC|DEC|NEG|NOT|BTC|BTR|BTS)[BWLQ]$`) + rows := make(map[string]string) + for _, entry := range optab.Elts { + row, ok := entry.(*ast.CompositeLit) + if !ok || len(row.Elts) == 0 { + continue + } + name := canonical(row.Elts[0]) + if !pattern.MatchString(name) { + continue + } + if _, exists := rows[name]; exists { + return fmt.Errorf("duplicate encoder row: %s", name) + } + rows[name] = canonical(row) + } + if len(rows) != 53 { + return fmt.Errorf("complete opcode carrier has %d rows, want 53", len(rows)) + } + // The native byte formulas below are independent of the typed production + // spec. Group numbers are written in the actual Go table's octal spelling. + for _, stem := range []string{"ADC", "ADD", "AND", "OR", "SBB", "SUB", "XOR", "INC", "DEC", "NEG", "NOT", "BTC", "BTR", "BTS"} { + for _, width := range "BWLQ" { + if strings.HasPrefix(stem, "BT") && width == 'B' { + continue + } + name := "A" + stem + string(width) + table, prefix := "yaddl", map[rune]string{'B': "Pb", 'W': "Pe", 'L': "Px", 'Q': "Pw"}[width] + encoding := "" + switch stem { + case "INC", "DEC": + group, short := 0, 0x40 + if stem == "DEC" { + group, short = 1, 0x48 + } + table = "yincq" + encoding = fmt.Sprintf("0xff, %02o", group) + if width == 'B' { + table, encoding = "yscond", fmt.Sprintf("0xfe, %02o", group) + } else if width == 'L' { + table, prefix, encoding = "yincl", "Px1", fmt.Sprintf("0x%02x, 0xff, %02o", short, group) + } + case "NEG", "NOT": + group, opcode := 3, 0xf7 + if stem == "NOT" { + group = 2 + } + if width == 'B' { + opcode = 0xf6 + } + table, encoding = "yscond", fmt.Sprintf("0x%02x, %02o", opcode, group) + case "BTC", "BTR", "BTS": + group, opcode := map[string]int{"BTC": 7, "BTR": 6, "BTS": 5}[stem], map[string]int{"BTC": 0xbb, "BTR": 0xb3, "BTS": 0xab}[stem] + table, encoding = "ybtl", fmt.Sprintf("0xba, %02o, 0x%02x", group, opcode) + prefix = "Pm" + if width == 'W' { + prefix = "Pq" + } else if width == 'Q' { + prefix, encoding = "Pw", fmt.Sprintf("0x0f, 0xba, %02o, 0x0f, 0x%02x", group, opcode) + } + default: + group := map[string]int{"ADD": 0, "OR": 1, "ADC": 2, "SBB": 3, "AND": 4, "SUB": 5, "XOR": 6}[stem] + base := group * 8 + encoding = fmt.Sprintf("0x83, %02o, 0x%02x, 0x81, %02o, 0x%02x, 0x%02x", group, base+5, group, base+1, base+3) + if width == 'B' { + table, encoding = "yxorb", fmt.Sprintf("0x%02x, 0x80, %02o, 0x%02x, 0x%02x", base+4, group, base, base+2) + } + } + want, err := parse("[]Optab{{" + name + ", " + table + ", " + prefix + ", opBytes{" + encoding + "}}}") + if err != nil { + return err + } + if rows[name] != canonical(want.(*ast.CompositeLit).Elts[0]) { + return fmt.Errorf("opcode-specific complete encoder attributes changed: %s", name) + } + } + } + return nil +} + +func TestX86LockScalarRMWEncoderCarrierRejectsRuleMutation(t *testing.T) { + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if err := x86LockScalarEncoderCarrier(source); err != nil { + t.Fatal(err) + } + for _, mutation := range []struct{ name, from, to string }{ + {"class", "{Zib_, 1, argList{Yi32, Yal}}", "{Zib_, 1, argList{Yi8, Yal}}"}, + {"operand-count", "{Zib_, 1, argList{Yi32, Yal}}", "{Zib_, 1, argList{Yi32, Yal, Ymb}}"}, + {"zcase", "{Zib_, 1, argList{Yi32, Yal}}", "{Zil_, 1, argList{Yi32, Yal}}"}, + {"zoffset", "{Zib_, 1, argList{Yi32, Yal}}", "{Zib_, 2, argList{Yi32, Yal}}"}, + {"row-order", "{Zib_, 1, argList{Yi32, Yal}},\n\t{Zibo_m, 2, argList{Yi32, Ymb}}", "{Zibo_m, 2, argList{Yi32, Ymb}},\n\t{Zib_, 1, argList{Yi32, Yal}}"}, + {"extra-row", "var yincq = []ytab{", "var yincq = []ytab{\n{Zo_m, 2, argList{Ymb}},"}, + {"prefix", "{AADCB, yxorb, Pb,", "{AADCB, yxorb, Px,"}, + {"opcode-bytes", "{AADCB, yxorb, Pb, opBytes{0x14,", "{AADCB, yxorb, Pb, opBytes{0x15,"}, + {"table-binding", "{AADCB, yxorb, Pb,", "{AADCB, yaddl, Pb,"}, + } { + t.Run(mutation.name, func(t *testing.T) { + if !strings.Contains(string(source), mutation.from) { + t.Fatal("mutation no longer binds its actual Go source") + } + changed := strings.Replace(string(source), mutation.from, mutation.to, 1) + if err := x86LockScalarEncoderCarrier([]byte(changed)); err == nil { + t.Fatal("full Go encoder carrier admitted a changed rule") + } + }) + } +} diff --git a/amd64_lock_scalar_rmw_forms.go b/amd64_lock_scalar_rmw_forms.go new file mode 100644 index 000000000..015cf77b1 --- /dev/null +++ b/amd64_lock_scalar_rmw_forms.go @@ -0,0 +1,217 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// This is an operand/effect record, not an atomic, pointer or frame capability. +// LOCK requires the independent complete source-adjacency consumer. +type x86LockScalarRMWSpec struct { + stem string + bits int + unary bool + bitString bool + carryInput bool + flagsDefined uint8 + flagsKilled uint8 +} + +type x86LockScalarRMWForm struct { + spec x86LockScalarRMWSpec + source Operand + destination Operand +} + +// Numerical output bytes are independent of FLAGS preservation. INC/DEC do +// not read CF to compute data, while ADC/SBB do. Arithmetic carry/borrow flows +// only toward higher bytes; the four logical operations are byte-local. +func (s x86LockScalarRMWSpec) numericalByteInputs(output int) (mask uint8, carry bool) { + if output < 0 || output >= s.bits/8 { + return 0, false + } + switch s.stem { + case "AND", "OR", "XOR", "NOT": + return 1 << uint(output), false + case "ADC", "SBB": + return uint8((uint16(1) << uint(output+1)) - 1), true + case "ADD", "SUB", "INC", "DEC", "NEG": + return uint8((uint16(1) << uint(output+1)) - 1), false + default: + return 0, false // Bit-string indexes retain their separate dependencies. + } +} + +var x86LockScalarRMWSpecs = func() map[Op]x86LockScalarRMWSpec { + result := make(map[Op]x86LockScalarRMWSpec) + for _, stem := range []string{"ADC", "ADD", "AND", "OR", "SBB", "SUB", "XOR", "INC", "DEC", "NEG", "NOT", "BTC", "BTR", "BTS"} { + for index, suffix := range "BWLQ" { + spec := x86LockScalarRMWSpec{stem: stem, bits: []int{8, 16, 32, 64}[index], flagsDefined: x86ByteFlagsAll} + switch stem { + case "ADC", "SBB": + spec.carryInput = true + case "AND", "OR", "XOR": + spec.flagsDefined &^= x86ByteFlagAF + spec.flagsKilled = x86ByteFlagAF + case "INC", "DEC": + spec.unary, spec.carryInput = true, true + spec.flagsDefined &^= x86ByteFlagCF + case "NEG": + spec.unary = true + case "NOT": + spec.unary, spec.flagsDefined = true, 0 + case "BTC", "BTR", "BTS": + if suffix == 'B' { + continue + } + spec.bitString = true + spec.flagsDefined = x86ByteFlagCF + spec.flagsKilled = x86ByteFlagOF | x86ByteFlagSF | x86ByteFlagAF | x86ByteFlagPF + } + result[Op(stem+string(suffix))] = spec + } + } + return result +}() + +func x86LockScalarRMWProperties(op Op, ins Instr, arch string) (x86LockScalarRMWForm, bool, error) { + name := strings.ToUpper(string(op)) + base, _, _ := strings.Cut(name, ".") + spec, recognized := x86LockScalarRMWSpecs[Op(base)] + if !recognized { + return x86LockScalarRMWForm{}, false, nil + } + if spec.bitString { + form, _, err := x86BitTestProperties(op, ins, arch) + return x86LockScalarRMWForm{spec: spec, source: form.index, destination: form.destination}, true, err + } + fail := func(reason string) (x86LockScalarRMWForm, bool, error) { + return x86LockScalarRMWForm{}, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + if name != base || ins.Op != "" && !strings.EqualFold(string(ins.Op), name) || + arch != "amd64" && arch != "386" || arch == "386" && spec.bits == 64 { + return fail("has an invalid suffix, dispatch identity or machine mode") + } + count := 2 + if spec.unary { + count = 1 + } + if len(ins.Args) != count { + return fail("has an incomplete operand list") + } + form := x86LockScalarRMWForm{spec: spec, destination: ins.Args[count-1]} + if projected, segment, err := x86NamedSegmentMemorySource(ins, form.destination, arch); segment { + if err != nil { + return form, true, err + } + form.destination = projected + } + if form.destination.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(form.destination.Sym), "$") { + return fail("requires writable memory, not a symbol address literal") + } + if !spec.unary { + form.source = ins.Args[0] + if projected, segment, err := x86NamedSegmentMemorySource(ins, form.source, arch); segment { + if err != nil { + return form, true, err + } + form.source = projected + } + } + if spec.unary { + if form.destination.Kind == OpReg { + allowed := isX86YrlRegisterForArch(form.destination.Reg, arch) + if spec.stem == "NEG" || spec.stem == "NOT" || spec.bits == 8 { + allowed = isGoYmbRegisterForArch(form.destination.Reg, arch) + } + if !allowed { + return fail("destination is outside its complete Go register class") + } + if spec.stem == "NEG" || spec.stem == "NOT" { + form.destination.Reg = amd64YmbEffectiveRegister(form.destination.Reg, spec.bits) + } + } else if !isAMD64MemoryOperand(form.destination) { + return fail("destination is outside the Go memory class") + } + } else { + if err := validateAMD64ScalarAddSubOperands(arch, base, spec.bits, form.source, form.destination, ins); err != nil { + return x86LockScalarRMWForm{}, true, err + } + if form.source.Kind == OpImm { + form.source.Imm = amd64ScalarAddSubImmediateInt64(form.source.Imm, spec.bits) + } + } + needsTLS := false + for _, operand := range []Operand{form.source, form.destination} { + if operand.Kind == OpImm && (operand.ImmRaw != "" || operand.ImmIsFloat) { + return fail("requires a resolved integer immediate") + } + if operand.Kind != OpMem { + continue + } + if x86SourceTLSMemoryRole(operand, arch) { + needsTLS = true + continue + } + memory := operand.Mem + if err := x86SymbolIndexedMemoryError(memory, arch); err != nil { + return x86LockScalarRMWForm{}, true, err + } + if arch == "386" && strings.HasSuffix(memory.Sym, "(SB)") { + memory.Base = "" // Only the actual Go ignored annotation is discarded. + } + if !x86MemoryRegistersValidForArch(memory, arch) || memory.OffRaw != "" || + memory.Index == SP || memory.Index != "" && memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8 || + memory.Off != int64(int32(memory.Off)) && !(arch == "386" && memory.Off == int64(uint32(memory.Off))) { + return fail("has an unencodable or unresolved native memory operand") + } + } + if needsTLS { + // Validate the complete operand row before preserving TLS's source + // role. It is not an ordinary SIB address or a native memory grant. + return form, true, fmt.Errorf("%w: %s %s TLS memory needs its actual Go OS/arch/shared address contract: %q", + ErrProbeNeedsContext, arch, op, ins.Raw) + } + if arch == "amd64" && spec.bits == 8 && x86CompareExchangeHasREXByteView(x86CompareExchangeForm{ + source: form.source, destination: form.destination, + }) { + // Go emits these original Ymb spellings with their raw ModRM number. + // A REX prefix selects SP/BP/SI/DI's low byte instead of AH/CH/DH/BH. + // Recording the physical view does not bind a source/native input. + for _, operand := range []*Operand{&form.source, &form.destination} { + if operand.Kind == OpReg && (operand.Reg == AH || operand.Reg == CH || operand.Reg == DH || operand.Reg == BH) { + operand.Reg = amd64YmbEffectiveRegister(operand.Reg, 16) + } + } + } + return form, true, nil +} + +func x86LockScalarNativeMemoryBound(form x86LockScalarRMWForm, ins Instr, arch string) error { + if form.destination.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(form.destination.Sym), "$") { + return fmt.Errorf("%s LOCK RMW address literal is not writable native memory", arch) + } + if form.destination.Kind != OpMem && form.destination.Kind != OpSym || ins.x86UnboundAddressOverride { + return fmt.Errorf("%w: %s LOCK RMW needs its bound native memory destination, not a virtual FP/AUTO span", ErrProbeNeedsContext, arch) + } + if form.destination.Kind == OpMem { + memory := form.destination.Mem + if memory.Base == SP || memory.Base == BP || memory.Index == SP || memory.Index == BP { + // Numeric source slots and LLVM's private SP allocation do not + // establish the original instruction's physical native address. + return fmt.Errorf("%w: %s LOCK RMW needs independent native SP/BP backing", ErrProbeNeedsContext, arch) + } + } + if arch == "amd64" && form.spec.bits == 8 && form.source.Kind == OpReg && + len(ins.Args) == 2 && form.source.Reg != ins.Args[0].Reg { + return fmt.Errorf("%w: amd64 LOCK byte source needs its original Go REX physical register view", ErrProbeNeedsContext) + } + if arch == "386" && form.spec.bits == 8 && form.source.Kind == OpReg { + switch form.source.Reg { + case AX, BX, CX, DX, AL, AH, BL, BH, CL, CH, DL, DH: + default: + return fmt.Errorf("%w: 386 LOCK byte source requires Go's original scratch-register expansion", ErrProbeNeedsContext) + } + } + return nil +} diff --git a/amd64_lock_scalar_rmw_forms_test.go b/amd64_lock_scalar_rmw_forms_test.go new file mode 100644 index 000000000..f32842ebf --- /dev/null +++ b/amd64_lock_scalar_rmw_forms_test.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +type x86LockScalarOracleForm struct { + op string + text string +} + +// This inventory is independent of the production dispatch. Intel's complete +// LOCK list contains these fourteen stems plus the five existing exchange and +// compare-exchange stems. Go's yxorb/yaddl/yscond/yincl/yincq/ybtl ordered tables +// determine their immediate, register and memory operand forms. +func x86LockScalarOracleForms(arch string) []x86LockScalarOracleForm { + widths := "BWL" + if arch == "amd64" { + widths += "Q" + } + var forms []x86LockScalarOracleForm + for _, stem := range []string{"ADC", "ADD", "AND", "OR", "SBB", "SUB", "XOR"} { + for _, width := range widths { + op, register := stem+string(width), "CX" + if width == 'B' { + register = "CL" + } + for _, source := range []string{register, "$1"} { + forms = append(forms, x86LockScalarOracleForm{op, op + " " + source + ",1(SI)"}) + } + } + } + for _, stem := range []string{"INC", "DEC", "NEG", "NOT"} { + for _, width := range widths { + op := stem + string(width) + forms = append(forms, x86LockScalarOracleForm{op, op + " 1(SI)"}) + } + } + for _, stem := range []string{"BTC", "BTR", "BTS"} { + for _, width := range strings.TrimPrefix(widths, "B") { + op := stem + string(width) + for _, source := range []string{"CX", "$1"} { + forms = append(forms, x86LockScalarOracleForm{op, op + " " + source + ",1(SI)"}) + } + } + } + return forms +} + +func TestX86LockScalarRMWCompleteGoTables(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if err := x86LockScalarEncoderCarrier(data); err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`(?m)^\s*\{A((?:ADC|ADD|AND|OR|SBB|SUB|XOR|INC|DEC|NEG|NOT|BTC|BTR|BTS)[BWLQ]),\s*(\w+),`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 53 { + t.Fatalf("complete scalar LOCK opcode inventory changed: %d", len(rows)) + } + seen := make(map[string]bool) + for _, row := range rows { + op, table := row[1], row[2] + if seen[op] { + t.Fatal("duplicate Go opcode registration", op) + } + seen[op] = true + width, stem := op[len(op)-1], op[:len(op)-1] + want := "yaddl" + switch stem { + case "BTC", "BTR", "BTS": + want = "ybtl" + case "INC", "DEC": + want = "yincq" + if width == 'B' { + want = "yscond" + } else if width == 'L' { + want = "yincl" + } + case "NEG", "NOT": + want = "yscond" + default: + if width == 'B' { + want = "yxorb" + } + } + if table != want { + t.Fatalf("Go ordered-table role changed: %s/%s, want %s", op, table, want) + } + } + for _, form := range x86LockScalarOracleForms("amd64") { + if !seen[form.op] { + t.Fatal("independent syntax inventory is absent from actual Go tables", form.op) + } + } +} + +func x86LockScalarOriginalInstruction(t *testing.T, arch, source string) []byte { + t.Helper() + code := x86OrderingHintGoBytes(t, arch, source) + mode := 64 + if arch == "386" { + mode = 32 + } + for offset := 0; offset < len(code); { + instruction, err := x86asm.Decode(code[offset:], mode) + if err != nil || instruction.Len <= 0 { + t.Fatalf("actual original Go bytes cannot be decoded at %d: %v", offset, err) + } + if code[offset] == 0xf0 { + return append([]byte(nil), code[offset:offset+instruction.Len]...) + } + offset += instruction.Len + } + t.Fatalf("actual complete Go object lacks its LOCK-prefixed instruction: %x", code) + return nil +} + +func TestX86LockScalarRMWWholeSourceAdmission(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + forms := x86LockScalarOracleForms(arch) + want := 90 + if arch == "386" { + want = 66 + } + if len(forms) != want { + t.Fatal("complete accepted operand-form denominator changed", arch, len(forms)) + } + for _, form := range forms { + t.Run(arch+"/"+form.text, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, "CMPL AX,AX\n\tLOCK\n\t"+form.text) + machine := x86LockScalarOriginalInstruction(t, arch, source) + t.Logf("actual original LOCK object bytes=%x", machine) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + if err != nil || !strings.Contains(ir, "lock; "+strings.ToLower(form.op)) { + t.Fatalf("actual Go accepted whole scalar LOCK RMW lacks its ordered atomic effect: %v", err) + } + }) + } + }) + } + } +} + +func TestX86LockScalarRMWRawWholeInstructionRetainsContext(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, form := range x86LockScalarOracleForms(arch) { + t.Run(arch+"/"+form.text, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, "CMPL AX,AX\n\tLOCK\n\t"+form.text) + machine := x86LockScalarOriginalInstruction(t, arch, source) + var body strings.Builder + for _, value := range machine { + fmt.Fprintf(&body, "BYTE $%d\n", value) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, + x86NamedLockOracleSource(arch, "CMPL AX,AX\n\t"+body.String()), + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("raw LOCK lost its unresolved whole physical-prefix contract", err) + } + }) + } + }) + } + } +} diff --git a/amd64_lock_scalar_rmw_linux_amd64_test.go b/amd64_lock_scalar_rmw_linux_amd64_test.go new file mode 100644 index 000000000..8767104de --- /dev/null +++ b/amd64_lock_scalar_rmw_linux_amd64_test.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// These are required native execution axes. Darwin/Rosetta and QEMU numerical +// observations do not establish native Linux arithmetic or atomic semantics. +func TestCrossLinuxRuntimeMatrixX86LockScalarRMWNativeAMD64(t *testing.T) { + x87IA32RequireNativeHost(t) + TestCrossLinuxRuntimeMatrixX86LockScalarRMWOriginalAndLLVM(t) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixX86LockScalarRMWNative386(t *testing.T) { + x87IA32RequireNativeHost(t) + llc := findLLVM22Tool("llc") + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if llc == "" || err != nil { + t.Fatal("required LLVM22 and IA32 static compiler unavailable", err) + } + declarations, assembly, driver, reference, count := x86LockScalarRuntimeFixture("386") + if count != 129 { + t.Fatal("complete IA32 kernel denominator changed", count) + } + want := fmt.Sprintf("lock observations %d\n", count*292*2*8) + dir := t.TempDir() + original := x86OrderingHintBuildOriginal(t, dir, "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != want { + t.Fatalf("actual original Go native IA32 full arithmetic/status/canary oracle: %q", output) + } + mode := "i386-unknown-linux-gnu" + ir, err := x86OrderingHintTranslateAPI(t, "Go", assembly, declarations, "386", mode) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := filepath.Join(dir, "lock"+level+".ll") + object := filepath.Join(dir, "lock"+level+".o") + main := filepath.Join(dir, "main.c") + executable := filepath.Join(dir, "lock"+level) + for name, contents := range map[string]string{input: ir, main: reference} { + if err := os.WriteFile(name, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, + "-mtriple="+mode, "-filetype=obj", input, "-o", object)) + x86FloatCompareCaptureCommand(t, exec.Command(compiler, "-static", "-no-pie", + object, main, "-O2", "-o", executable)) + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != want { + t.Fatalf("native LLVM IA32 full-width/flags/MXCSR/canary oracle: %q", output) + } + }) + } + x87IA32RequireNativeHost(t) +} diff --git a/amd64_lock_scalar_rmw_namespace_test.go b/amd64_lock_scalar_rmw_namespace_test.go new file mode 100644 index 000000000..fe217310d --- /dev/null +++ b/amd64_lock_scalar_rmw_namespace_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86LockScalarRMWNoCandidateDoesNotResolveLocals(t *testing.T) { + for _, body := range []string{ + "MOVQ value+0(FP),AX\nloop:\nRET", + "LOCK\nADDL AX,BX\nRET", + } { + file, err := Parse(ArchAMD64, "TEXT ·f(SB),4,$0-8\n"+body+"\n") + if err != nil { + t.Fatal(err) + } + var calls []string + namespace := x86LockScalarNamespace(file, "amd64", func(symbol string) string { + calls = append(calls, symbol) + return strings.TrimPrefix(symbol, "·") + }, map[string]FuncSig{"f": {Name: "f", Ret: Void}}) + if namespace != "" || len(calls) != 0 { + t.Fatalf("no legal helper must not allocate names or call the user resolver: namespace=%q calls=%v", namespace, calls) + } + } +} + +func TestX86LockScalarRMWNamespaceUsesFinalGlobalsOnly(t *testing.T) { + file, err := Parse(ArchAMD64, `TEXT ·f(SB),4,$0-8 + MOVQ value+0(FP),SI + MOVL $1,AX + LOCK + ADDL AX,(SI) +done: + RET +DATA ·table+0(SB)/8,$·imported(SB) +GLOBL ·table(SB),8,$8 +`) + if err != nil { + t.Fatal(err) + } + final := map[string]string{ + "·f": "__plan9asm_lock_rmw_0_amd64_add32_native", + "·table": "__plan9asm_lock_rmw_1_amd64_add32_native", + "·imported": "imported", + } + var unknown []string + resolve := func(symbol string) string { + if name, ok := final[symbol]; ok { + return name + } + unknown = append(unknown, symbol) + return symbol + } + sigs := map[string]FuncSig{ + final["·f"]: {Name: final["·f"], Ret: Void}, + "imported": {Name: "__plan9asm_lock_rmw_2_amd64_add32_native", Ret: Void}, + } + if namespace := x86LockScalarNamespace(file, "amd64", resolve, sigs); namespace != "__plan9asm_lock_rmw_3_" || len(unknown) != 0 { + t.Fatalf("helper namespace must bind final function/data/import names, not FP or local labels: namespace=%q unknown=%v", namespace, unknown) + } +} + +func TestX86LockScalarRMWUnrelatedTranslationKeepsResolverDomain(t *testing.T) { + file, err := Parse(ArchAMD64, `TEXT ·f(SB),4,$0-8 + MOVQ value+0(FP),AX +done: + RET +`) + if err != nil { + t.Fatal(err) + } + var calls, unknown []string + resolve := func(symbol string) string { + calls = append(calls, symbol) + if symbol != "·f" { + unknown = append(unknown, symbol) + } + return strings.TrimPrefix(symbol, "·") + } + sig := FuncSig{Name: "f", Args: []LLVMType{I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}}} + ir, err := Translate(file, Options{Goarch: "amd64", ResolveSym: resolve, Sigs: map[string]FuncSig{"f": sig}}) + if err != nil || len(unknown) != 0 || len(calls) != 6 || strings.Contains(ir, "__plan9asm_lock_rmw_") { + t.Fatalf("a no-LOCK translation changed its existing resolver domain/count: error=%v calls=%v", err, calls) + } +} diff --git a/amd64_lock_scalar_rmw_native.go b/amd64_lock_scalar_rmw_native.go new file mode 100644 index 000000000..054ee6b21 --- /dev/null +++ b/amd64_lock_scalar_rmw_native.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "fmt" + "sort" + "strings" +) + +// A namespace is a compiler name allocation, never a source certificate. +func x86LockScalarNamespace(file *File, arch string, resolve func(string) string, sigs map[string]FuncSig) string { + if len(x86LockScalarRMWHelperForms(file, arch)) == 0 { + return "" + } + return x86CompilerHelperNamespace(file, resolve, sigs, "lock_rmw") +} + +func x86LockScalarHelperName(namespace string, form x86LockScalarRMWForm, arch string) string { + segment := "native" + if form.destination.Kind == OpMem && form.destination.Mem.Segment != "" { + segment = strings.ToLower(string(form.destination.Mem.Segment)) + } + name := namespace + arch + "_" + strings.ToLower(form.spec.stem) + fmt.Sprint(form.spec.bits) + "_" + segment + if form.source.Kind == OpImm { + name += fmt.Sprintf("_imm_%016x", uint64(form.source.Imm)) + } + return name +} + +func x86LockScalarPointerType(form x86LockScalarRMWForm) string { + if form.destination.Kind == OpMem { + switch form.destination.Mem.Segment { + case FS: + return "ptr addrspace(257)" + case GS: + return "ptr addrspace(256)" + } + } + return "ptr" +} + +func x86LockScalarRMWHelperForms(file *File, arch string) map[string]x86LockScalarRMWForm { + forms := make(map[string]x86LockScalarRMWForm) + if file == nil || file.Arch != ArchAMD64 || arch != "amd64" && arch != "386" { + return forms + } + for _, fn := range file.Funcs { + for _, block := range amd64SplitBlocks(fn) { + for index := 1; index < len(block.instrs); index++ { + prefix, body := block.instrs[index-1], block.instrs[index] + if !strings.EqualFold(string(prefix.Op), "LOCK") || len(prefix.Args) != 0 { + continue + } + form, recognized, err := x86LockScalarRMWProperties(body.Op, body, arch) + if !recognized || err != nil || x86LockScalarNativeMemoryBound(form, body, arch) != nil { + continue // The ordinary final consumer retains the actual error. + } + forms[x86LockScalarHelperName("", form, arch)] = form + } + } + } + return forms +} + +func emitX86LockScalarRMWHelpers(b *strings.Builder, file *File, arch, namespace string) { + if namespace == "" { + return + } + forms := x86LockScalarRMWHelperForms(file, arch) + names := make([]string, 0, len(forms)) + for name := range forms { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + emitX86LockScalarRMWHelper(b, namespace+name, forms[name], arch) + } +} + +func emitX86LockScalarRMWHelper(b *strings.Builder, name string, form x86LockScalarRMWForm, arch string) { + spec := form.spec + typ, word := amd64IntegerTypeForBits(spec.bits), I64 + push, pop, carryTest := "pushfq", "popq", "btq" + if arch == "386" { + word, push, pop, carryTest = I32, "pushfl", "popl", "btl" + } + pointer := x86LockScalarPointerType(form) + fmt.Fprintf(b, "define internal %s @%s(%s %%memory, %s %%source, %s %%carry) noinline noredzone {\nentry:\n", + word, name, pointer, typ, word) + width := map[int]string{8: "b", 16: "w", 32: "l", 64: "q"}[spec.bits] + segment := "" + if form.destination.Kind == OpMem && form.destination.Mem.Segment != "" { + segment = "%" + strings.ToLower(string(form.destination.Mem.Segment)) + ":" + } + // The only native stack use is balanced within this compiler-owned helper. + // noinline+noredzone prevents a later inliner/red-zone allocation from + // invalidating it. No source SP/FP allocation or extra LAHF ISA is borrowed. + if spec.stem == "NOT" { + fmt.Fprintf(b, " call void asm sideeffect \"lock; not%s %s$0\", \"=*m,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %%memory, %s elementtype(%s) %%memory)\n", + width, segment, pointer, typ, pointer, typ) + fmt.Fprintf(b, " ret %s 0\n}\n\n", word) + return + } + opcode := "lock; " + strings.ToLower(spec.stem) + width + " " + if spec.unary { + opcode += segment + "$1" + } else { + opcode += "$2, " + segment + "$1" + } + if spec.carryInput { + opcode = carryTest + " $$0, $3; " + opcode + } + opcode += "; " + push + "; " + pop + " $0" + register := "r" + if spec.bits == 8 { + register = "q" + } + source := "%source" + if form.source.Kind == OpImm { + register, source = "i", fmt.Sprint(form.source.Imm) + } + // Keep fixed operand numbering even for unary forms. The unused source is + // a compile-time zero, not an invented machine input. Output flags are + // written after every input is consumed; indirect *m is read/write. + fmt.Fprintf(b, " %%flags = call %s asm sideeffect \"%s\", \"=r,=*m,%s,r,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %%memory, %s %s, %s %%carry, %s elementtype(%s) %%memory)\n", + word, opcode, register, pointer, typ, typ, source, word, pointer, typ) + fmt.Fprintf(b, " ret %s %%flags\n}\n\n", word) +} + +func (c *amd64Ctx) lowerLockScalarRMW(op Op, ins Instr) (bool, bool, error) { + if !c.namedAtomicLock { + return false, false, nil + } + form, recognized, err := x86LockScalarRMWProperties(op, ins, c.goarch) + if !recognized { + return false, false, nil + } + if err != nil { + return true, false, err + } + if err := x86LockScalarNativeMemoryBound(form, ins, c.goarch); err != nil { + return true, false, err + } + if c.lockScalarNS == "" { + return true, false, fmt.Errorf("%w: scalar LOCK has no compiler-owned complete bridge", ErrProbeNeedsContext) + } + pointer, pointerType, err := c.compareExchangePointer(form.destination) + if err != nil { + return true, false, err + } + if pointerType != x86LockScalarPointerType(form) { + return true, false, fmt.Errorf("%w: scalar LOCK native address space changed", ErrProbeNeedsContext) + } + typ, word := amd64IntegerTypeForBits(form.spec.bits), I64 + if c.goarch == "386" { + word = I32 + } + source := "0" + if !form.spec.unary { + if form.source.Kind == OpImm { + source = fmt.Sprint(form.source.Imm) + } else if source, err = c.evalIntSized(form.source, typ); err != nil { + return true, false, err + } + } + carry := "0" + if form.spec.carryInput { + flag := c.loadFlag(c.flagsCFSlot) + name := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", name, flag, word) + carry = "%" + name + } + flags := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, %s %s)\n", flags, word, + x86LockScalarHelperName(c.lockScalarNS, form, c.goarch), pointerType, pointer, typ, source, word, carry) + for _, flag := range []struct { + mask uint8 + bit int + slot string + }{ + {x86ByteFlagCF, 0, c.flagsCFSlot}, + {x86ByteFlagPF, 2, c.flagsPFSlot}, + {x86ByteFlagAF, 4, c.flagsAFSlot}, + {x86ByteFlagZF, 6, c.flagsZSlot}, + {x86ByteFlagSF, 7, c.flagsSltSlot}, + {x86ByteFlagOF, 11, c.flagsOFSlot}, + } { + if form.spec.flagsDefined&flag.mask == 0 { + continue // Preserved/undefined bits are not forged into defined facts. + } + shift, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %d\n", shift, word, flags, flag.bit) + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to i1\n", value, word, shift) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", value, flag.slot) + } + return true, false, nil +} diff --git a/amd64_lock_scalar_rmw_objects_test.go b/amd64_lock_scalar_rmw_objects_test.go new file mode 100644 index 000000000..0707b8c55 --- /dev/null +++ b/amd64_lock_scalar_rmw_objects_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86LockScalarRMWThreeAPIsFiveTargetsNativeObjects(t *testing.T) { + x86LockScalarRMWNativeObjectMatrix(t, false) +} + +func TestX86LockScalarRMWIndexedSegmentsThreeAPIsFiveTargetsObjects(t *testing.T) { + x86LockScalarRMWNativeObjectMatrix(t, true) +} + +func x86LockScalarRMWNativeObjectMatrix(t *testing.T, indexed bool) { + t.Helper() + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 compiler and disassembler are required") + } + for _, target := range x86OrderingHintTargets { + var source, declarations strings.Builder + declarations.WriteString("package ordering\n") + type expectedInstruction struct { + op string + bits int + segment x86asm.Reg + } + want := make(map[string]expectedInstruction) + for index, row := range x86LockScalarOracleForms(target.arch) { + for _, segment := range []string{"native", "FS", "GS"} { + memory := "1(SI)" + if segment != "native" { + memory = "1(" + segment + ")" + if indexed { + memory = "1(AX)(" + segment + "*4)" + } + } + body := strings.Replace(row.text, "1(SI)", memory, 1) + name := fmt.Sprintf("rmw%d_%s", index, strings.ToLower(segment)) + original := strings.Replace(x86NamedLockOracleSource(target.arch, + "CMPL AX,AX\n\tLOCK\n\t"+body), "·locked", "·"+name, 1) + x86LockScalarOriginalInstruction(t, target.arch, original) + source.WriteString(original) + fmt.Fprintf(&declarations, "func %s(p *[16]byte)\n", name) + + bits := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[row.op[len(row.op)-1]] + stem := strings.ToLower(row.op[:len(row.op)-1]) + helper := "__plan9asm_lock_rmw_0_" + target.arch + "_" + + stem + fmt.Sprint(bits) + "_" + strings.ToLower(segment) + if strings.Contains(row.text, "$1") { + helper += "_imm_0000000000000001" + } + physicalSegment := x86asm.Reg(0) + if segment == "FS" { + physicalSegment = x86asm.FS + } else if segment == "GS" { + physicalSegment = x86asm.GS + } + want[helper] = expectedInstruction{strings.ToUpper(stem), bits, physicalSegment} + } + } + + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), + declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + if strings.Contains(ir, "atomicrmw ") || strings.Contains(ir, " = cmpxchg ") { + t.Fatal("source RMW acquired a guessed LLVM atomic alignment or software replacement") + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + dir := t.TempDir() + input, object := filepath.Join(dir, "lock.ll"), filepath.Join(dir, "lock.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, + "-mtriple="+target.triple, "-filetype=obj", input, "-o", object)) + listing := x86FloatCompareCaptureCommand(t, + exec.Command(disassembler, "--disassemble", object)) + sections := regexp.MustCompile(`(?m)^[0-9a-f]+ <([^>]+)>:\n`).FindAllSubmatchIndex(listing, -1) + seen := make(map[string]bool) + for index, section := range sections { + name := string(listing[section[2]:section[3]]) + if strings.HasPrefix(name, "___plan9asm_lock_rmw_") { + name = strings.TrimPrefix(name, "_") // Mach-O/COFF leading underscore. + } + expected, known := want[name] + if !known { + continue + } + if seen[name] { + t.Fatal("duplicate compiler helper", name) + } + end := len(listing) + if index+1 < len(sections) { + end = sections[index+1][0] + } + var code []byte + for _, line := range strings.Split(string(listing[section[1]:end]), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasSuffix(fields[0], ":") { + continue + } + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := hex.DecodeString(field) + if err != nil { + break + } + code = append(code, value[0]) + } + } + mode := 64 + if target.arch == "386" { + mode = 32 + } + count := 0 + for len(code) != 0 { + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len <= 0 || physical.Len > len(code) { + t.Fatalf("incomplete physical helper %s: %x %v", name, code, err) + } + machine := code[:physical.Len] + code = code[physical.Len:] + locked, segmentPrefixes := false, 0 + for _, prefix := range physical.Prefix { + if byte(prefix) == 0xf0 { + locked = true + } + if byte(prefix) == 0x64 || byte(prefix) == 0x65 { + segmentPrefixes++ + } + } + if !locked { + continue + } + count++ + var memory x86asm.Mem + foundMemory := false + for _, argument := range physical.Args { + if value, ok := argument.(x86asm.Mem); ok { + memory, foundMemory = value, true + } + } + wantSegments := 0 + if expected.segment != 0 { + wantSegments = 1 + } + if physical.Op.String() != expected.op || !foundMemory || + physical.MemBytes*8 != expected.bits || + memory.Segment != expected.segment || segmentPrefixes != wantSegments || + physical.AddrSize != mode { + t.Fatalf("native width/segment/address/prefix identity changed: %s bytes=%x decoded=%s memBytes=%d segmentPrefixes=%d expected=%+v", + name, machine, x86asm.GoSyntax(physical, 0, nil), physical.MemBytes, segmentPrefixes, expected) + } + } + if count != 1 { + t.Fatalf("%s helper has %d complete native RMW instructions", name, count) + } + seen[name] = true + } + if len(seen) != len(want) { + t.Fatalf("emitted %d/%d complete native helpers\n%s", len(seen), len(want), listing) + } + t.Logf("actual %s object independently decoded %d whole native RMW helpers", target.triple, len(seen)) + }) + } + }) + } + } +} diff --git a/amd64_lock_scalar_rmw_runtime_test.go b/amd64_lock_scalar_rmw_runtime_test.go new file mode 100644 index 000000000..8d346dafb --- /dev/null +++ b/amd64_lock_scalar_rmw_runtime_test.go @@ -0,0 +1,476 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +type x86LockScalarRuntimeCase struct { + name, stem, body string + width int + immediate bool + literal uint64 +} + +func x86ScalarRuntimeStatusMask(stem string) int { + switch stem { + case "BTC", "BTR", "BTS": + return 0x41 // CF is defined; ZF is preserved. Other status bits are undefined. + case "AND", "OR", "XOR": + return 0x8c5 // AF is undefined, not a numerical zero. + default: + return 0x8d5 + } +} + +func x86LockScalarRuntimeCases(arch string) []x86LockScalarRuntimeCase { + var rows []x86LockScalarRuntimeCase + for _, form := range x86LockScalarOracleForms(arch) { + width := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[form.op[len(form.op)-1]] + stem := form.op[:len(form.op)-1] + immediate := strings.Contains(form.text, "$1") + rows = append(rows, x86LockScalarRuntimeCase{ + name: fmt.Sprintf("sample%d", len(rows)), stem: stem, + body: strings.ReplaceAll(strings.Replace(form.text, "1(SI)", "(SI)", 1), "CL,", "CX,"), + width: width, immediate: immediate, literal: 1, + }) + if !immediate || stem == "BTC" || stem == "BTR" || stem == "BTS" { + continue + } + // Independent Yi32 boundaries and equal-bit-pattern spellings. Native Q + // sign-extends imm32; the smaller widths retain their low encoded bits. + for _, literal := range []uint64{0x7fffffff, 0x80000000, 0xffffffff} { + rows = append(rows, x86LockScalarRuntimeCase{ + name: fmt.Sprintf("sample%d", len(rows)), stem: stem, + body: strings.Replace(strings.Replace(form.text, "1(SI)", "(SI)", 1), + "$1,", fmt.Sprintf("$%d,", literal), 1), + width: width, immediate: true, literal: literal, + }) + } + } + return rows +} + +func x86LockScalarRuntimeFixture(arch string) (declarations, assembly, driver, reference string, count int) { + rows := x86LockScalarRuntimeCases(arch) + word, suffix, slot := "uint64", "Q", 8 + if arch == "386" { + word, suffix, slot = "uint32", "L", 4 + } + var decl, source, goRows, prototypes, cRows strings.Builder + decl.WriteString("package main\n") + for _, row := range rows { + fmt.Fprintf(&decl, "func %s(memory *[128]byte, output *[8]uint64, offset %s, value %s, seed %s)\n", + row.name, word, word, word) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n", row.name, slot*5) + fmt.Fprintf(&source, "MOV%s memory+0(FP),SI\nMOV%s output+%d(FP),DI\nMOV%s offset+%d(FP),AX\n", + suffix, suffix, slot, suffix, slot*2) + fmt.Fprintf(&source, "ADD%s AX,SI\nMOV%s value+%d(FP),CX\nMOV%s seed+%d(FP),DX\n", + suffix, suffix, slot*3, suffix, slot*4) + fmt.Fprintf(&source, "PUSH%s DX\nPOPF%s\nSTMXCSR 24(DI)\nLOCK\n%s\nPUSHF%s\nPOP%s AX\n", + suffix, suffix, row.body, suffix, suffix) + // Refine the original saved word before observing it as numerical + // output. A mask in the C/Go reference cannot define missing source bits. + fmt.Fprintf(&source, "AND%s $%d,AX\nMOV%s AX,0(DI)\nMOV%s CX,8(DI)\nMOV%s DX,16(DI)\nSTMXCSR 32(DI)\nRET\n", + suffix, x86ScalarRuntimeStatusMask(row.stem), suffix, suffix, suffix) + fmt.Fprintf(&goRows, "{%s,%q,%d,%t,%d},\n", row.name, row.stem, row.width, row.immediate, row.literal) + cWord := "uint64_t" + if arch == "386" { + cWord = "uint32_t" + } + fmt.Fprintf(&prototypes, "extern void %s(uint8_t *, uint64_t *, %s, %s, %s);\n", row.name, cWord, cWord, cWord) + imm := 0 + if row.immediate { + imm = 1 + } + fmt.Fprintf(&cRows, "{%s,%q,%d,%d,UINT64_C(%d)},\n", row.name, row.stem, row.width, imm, row.literal) + } + driver = strings.NewReplacer("@WORD@", word, "@ROWS@", goRows.String()).Replace(x86LockScalarOriginalReference) + cWord := "uint64_t" + if arch == "386" { + cWord = "uint32_t" + } + reference = strings.NewReplacer("@WORD@", cWord, "@PROTOTYPES@", prototypes.String(), "@ROWS@", cRows.String()).Replace(x86LockScalarIndependentC) + return decl.String(), source.String(), driver, reference, len(rows) +} + +func TestCrossLinuxRuntimeMatrixX86LockScalarRMWOriginalAndLLVM(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, assembly, driver, reference, count := x86LockScalarRuntimeFixture("amd64") + if count != 174 { + t.Fatal("independent complete runtime kernel denominator changed", count) + } + original := x86OrderingHintBuildOriginal(t, t.TempDir(), runtime.GOOS, "amd64", declarations, assembly, driver) + want := fmt.Sprintf("lock observations %d\n", count*292*2*8) + if output := execution.run(t, original); string(output) != want { + t.Fatalf("actual original Go initialized-memory/defined-flags/canary oracle: %q, want %q", output, want) + } + t.Logf("actual original Go: %d kernels / %d complete initialized-memory observations", count, count*292*2*8) + ir := x86NarrowOracleIR(t, assembly, declarations, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + if output := execution.compileAndRunLLVM(t, "lock-rmw", ir, reference, level); string(output) != want { + t.Fatalf("independent integer arithmetic/bit-string/memory/flags/canary oracle: %q", output) + } + t.Logf("%d complete observations: all initialized memory bytes, defined/preserved status bits, source registers, MXCSR and canaries", count*292*2*8) + }) + } +} + +func TestX86LockScalarRMWOriginal386StaticELFCompileOnly(t *testing.T) { + declarations, assembly, driver, _, count := x86LockScalarRuntimeFixture("386") + if count != 129 { + t.Fatal("complete mode-32 runtime kernel denominator changed", count) + } + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go %d-kernel static IA32 ELF: %d bytes SHA256=%x; execution is a distinct required native axis", + count, sealed.size, sealed.hash) +} + +const x86LockScalarOriginalReference = `package main + +import ( + "fmt" + "math/bits" + "runtime" +) + +type word = @WORD@ +type kernel struct { + fn func(*[128]byte, *[8]uint64, word, word, word) + op string + width int + immediate bool + literal uint64 +} +var kernels = []kernel{ + @ROWS@ +} + +func load(p *[128]byte, offset, width int) uint64 { + var value uint64 + for i := 0; i < width/8; i++ { + value |= uint64(p[offset+i]) << (8*i) + } + return value +} + +func store(p *[128]byte, offset, width int, value uint64) { + for i := 0; i < width/8; i++ { + p[offset+i] = byte(value >> (8*i)) + } +} + +func reference(k kernel, memory *[128]byte, offset int, b, seed uint64) (uint64, uint64) { + mask := ^uint64(0) + if k.width < 64 { + mask = (uint64(1) << k.width) - 1 + } + if k.immediate { + b = uint64(int64(int32(k.literal))) + } + b &= mask + if k.op == "BTC" || k.op == "BTR" || k.op == "BTS" { + index := int64(b) + if k.width == 16 { + index = int64(int16(b)) + } else if k.width == 32 { + index = int64(int32(b)) + } + if k.immediate { + index = int64(k.literal % uint64(k.width)) + } + quotient, bit := index/int64(k.width), index%int64(k.width) + if bit < 0 { + quotient-- + bit += int64(k.width) + } + offset += int(quotient) * (k.width / 8) + a := load(memory, offset, k.width) + cf := (a >> uint(bit)) & 1 + switch k.op { + case "BTC": + a ^= uint64(1) << uint(bit) + case "BTR": + a &^= uint64(1) << uint(bit) + case "BTS": + a |= uint64(1) << uint(bit) + } + store(memory, offset, k.width, a) + return cf | seed&64, 65 // Other status bits are architecturally undefined. + } + a := load(memory, offset, k.width) + if k.op == "NOT" { + store(memory, offset, k.width, ^a) + return seed & 0x8d5, 0x8d5 + } + if k.op == "AND" || k.op == "OR" || k.op == "XOR" { + switch k.op { + case "AND": + a &= b + case "OR": + a |= b + case "XOR": + a ^= b + } + a &= mask + store(memory, offset, k.width, a) + flags := uint64(0) + if bits.OnesCount8(uint8(a))%2 == 0 { flags |= 4 } + if a == 0 { flags |= 64 } + if a>>(k.width-1) != 0 { flags |= 128 } + return flags, 0x8c5 // AF is undefined, not preserved or forced to zero. + } + carry := uint64(0) + if k.op == "ADC" || k.op == "SBB" { + carry = seed & 1 + } + add := k.op == "ADD" || k.op == "ADC" || k.op == "INC" + if k.op == "INC" || k.op == "DEC" { b = 1 } + if k.op == "NEG" { b, a = a, 0 } + result, cf := bits.Sub64(a, b, carry) + af := (a & 15) < (b & 15) + carry + if add { + result, cf = bits.Add64(a, b, carry) + if k.width < 64 && result > mask { cf = 1 } + af = (a & 15) + (b & 15) + carry > 15 + } + result &= mask + sign := uint64(1) << (k.width - 1) + overflow := (a^b)&(a^result)&sign != 0 + if add { overflow = ^(a^b)&(a^result)&sign != 0 } + if k.op == "INC" || k.op == "DEC" { cf = seed & 1 } + flags := cf + if bits.OnesCount8(uint8(result))%2 == 0 { flags |= 4 } + if af { flags |= 16 } + if result == 0 { flags |= 64 } + if result&sign != 0 { flags |= 128 } + if overflow { flags |= 2048 } + store(memory, offset, k.width, result) + return flags, 0x8d5 +} + +func main() { + runtime.LockOSThread() + edges := []uint64{0,1,15,16,17,127,128,255,256,32767,32768,65535, + 0x7fffffff,0x80000000,0xffffffff,0x7fffffffffffffff,0x8000000000000000,^uint64(0)} + const sentinel = uint64(0xa7a7a7a7a7a7a7a7) + observations := 0 + var memory, expected [128]byte + var out [8]uint64 + for _, k := range kernels { + for sample := 0; sample < 292; sample++ { + a, b := uint64(sample/16), uint64(sample%16) + if sample >= 256 { + a = edges[(sample-256)/2] + b = edges[((sample-256)/2 + 1 + 6*(sample%2))%len(edges)] + } + if (k.op == "BTC" || k.op == "BTR" || k.op == "BTS") && !k.immediate { + index := []int{-2*k.width-1,-k.width,-1,0,1,k.width-1,k.width,k.width+1,2*k.width} + b = uint64(int64(index[sample%len(index)])) + } + b = uint64(word(b)) + for _, seed := range []uint64{0x8c7, 0x12} { + for unaligned := 0; unaligned < 8; unaligned++ { + for i := range memory { memory[i] = byte(i*73+19) ^ byte(sample*29) } + offset := 64 + unaligned + store(&memory, offset, k.width, a) + expected = memory + for i := range out { out[i] = sentinel } + flags, valid := reference(k, &expected, offset, b, seed) + k.fn(&memory, &out, word(offset), word(b), word(seed)) + wordMask := uint64(^word(0)) + if memory != expected || out[0]&valid != flags || + out[1]&wordMask != b || out[2]&wordMask != seed || + uint32(out[3]) != uint32(out[4]) { + panic(fmt.Sprintf("%s/%d imm=%t literal=%x sample=%d unaligned=%d b=%x seed=%x flags=%x/%x mask=%x out=%x", + k.op,k.width,k.immediate,k.literal,sample,unaligned,b,seed,out[0],flags,valid,out)) + } + for i := 0; i < 3; i++ { + if out[i]&^wordMask != sentinel&^wordMask { panic("upper output canary") } + } + if out[3]>>32 != sentinel>>32 || out[4]>>32 != sentinel>>32 || + out[5] != sentinel || out[6] != sentinel || out[7] != sentinel { + panic("output extent canary") + } + observations++ + } + } + } + } + fmt.Printf("lock observations %d\n", observations) +} +` + +// Shared independent arithmetic only. Each execution driver separately owns +// its memory, FLAGS and platform-specific observations. +const x86ScalarMemoryIndependentC = ` +#include +#include +#include + +#if defined(__i386__) +// The largest IA32 operation is 32 bits; 64-bit intermediates close its carry +// and signed-overflow bounds without relying on unsupported int128 on i386. +typedef uint64_t wide_unsigned; +typedef int64_t wide_signed; +#else +typedef __uint128_t wide_unsigned; +typedef __int128 wide_signed; +#endif + +@PROTOTYPES@ +typedef @WORD@ word; +typedef void (*function)(uint8_t *, uint64_t *, word, word, word); +struct kernel { + function fn; + const char *op; + unsigned width, immediate; + uint64_t literal; +}; +static const struct kernel kernels[] = { + @ROWS@ +}; + +static uint64_t load(const uint8_t *memory, int offset, unsigned width) { + uint64_t value = 0; + for (unsigned i = 0; i < width / 8; i++) { + value |= (uint64_t)memory[offset + i] << (8 * i); + } + return value; +} +static void store(uint8_t *memory, int offset, unsigned width, uint64_t value) { + for (unsigned i = 0; i < width / 8; i++) { + memory[offset + i] = value >> (8 * i); + } +} +static uint64_t reference(const struct kernel *k, uint8_t *memory, + int offset, uint64_t b, uint64_t seed, uint64_t *valid) { + uint64_t mask = k->width == 64 ? UINT64_MAX : (UINT64_C(1) << k->width) - 1; + if (k->immediate) { + b = (uint64_t)(int64_t)(int32_t)k->literal; + } + b &= mask; + if (!strcmp(k->op, "BTC") || !strcmp(k->op, "BTR") || !strcmp(k->op, "BTS")) { + int64_t index = (int64_t)b; + if (k->width == 16) { index = (int16_t)b; } + if (k->width == 32) { index = (int32_t)b; } + if (k->immediate) { index = k->literal % k->width; } + int64_t quotient = index / (int64_t)k->width; + int64_t bit = index % (int64_t)k->width; + if (bit < 0) { + quotient--; + bit += k->width; + } + offset += quotient * (k->width / 8); + uint64_t a = load(memory, offset, k->width), cf = (a >> bit) & 1; + uint64_t selected = UINT64_C(1) << bit; + if (!strcmp(k->op, "BTC")) { a ^= selected; } + if (!strcmp(k->op, "BTR")) { a &= ~selected; } + if (!strcmp(k->op, "BTS")) { a |= selected; } + store(memory, offset, k->width, a); + *valid = 65; + return cf | (seed & 64); + } + uint64_t a = load(memory, offset, k->width), result; + *valid = 0x8d5; + if (!strcmp(k->op, "NOT")) { + store(memory, offset, k->width, ~a); + return seed & *valid; + } + unsigned cf = 0, af = 0, of = 0; + if (!strcmp(k->op, "AND") || !strcmp(k->op, "OR") || !strcmp(k->op, "XOR")) { + result = !strcmp(k->op, "AND") ? a & b : !strcmp(k->op, "OR") ? a | b : a ^ b; + *valid = 0x8c5; + } else { + unsigned carry = !strcmp(k->op, "ADC") || !strcmp(k->op, "SBB") ? seed & 1 : 0; + unsigned add = !strcmp(k->op, "ADD") || !strcmp(k->op, "ADC") || !strcmp(k->op, "INC"); + if (!strcmp(k->op, "INC") || !strcmp(k->op, "DEC")) { b = 1; } + if (!strcmp(k->op, "NEG")) { b = a; a = 0; } + wide_unsigned wide = add ? (wide_unsigned)a + b + carry : (wide_unsigned)a - b - carry; + result = wide & mask; + cf = add ? wide > mask : (wide_unsigned)a < (wide_unsigned)b + carry; + af = add ? (a & 15) + (b & 15) + carry > 15 : (a & 15) < (b & 15) + carry; + wide_signed sign = (wide_signed)1 << (k->width - 1); + wide_signed sa = (a & (uint64_t)sign) ? (wide_signed)a - 2 * sign : a; + wide_signed sb = (b & (uint64_t)sign) ? (wide_signed)b - 2 * sign : b; + wide_signed signed_wide = add ? sa + sb + carry : sa - sb - carry; + of = signed_wide < -sign || signed_wide >= sign; + if (!strcmp(k->op, "INC") || !strcmp(k->op, "DEC")) { cf = seed & 1; } + } + result &= mask; + unsigned ones = 0; + for (unsigned bit = 0; bit < 8; bit++) { ones += (result >> bit) & 1; } + uint64_t flags = cf | ((ones % 2 == 0) << 2) | (af << 4) | + ((result == 0) << 6) | ((result >> (k->width - 1)) << 7) | (of << 11); + store(memory, offset, k->width, result); + return flags; +} + +` + +const x86LockScalarIndependentC = x86ScalarMemoryIndependentC + `int main(void) { + static const uint64_t edges[] = {0,1,15,16,17,127,128,255,256,32767,32768,65535, + 0x7fffffff,0x80000000,0xffffffff,0x7fffffffffffffff,0x8000000000000000,UINT64_MAX}; + const uint64_t sentinel = UINT64_C(0xa7a7a7a7a7a7a7a7); + unsigned observations = 0; + for (unsigned kernel = 0; kernel < sizeof(kernels) / sizeof(kernels[0]); kernel++) { + const struct kernel *k = &kernels[kernel]; + for (unsigned sample = 0; sample < 292; sample++) { + uint64_t a = sample / 16, b = sample % 16; + if (sample >= 256) { + a = edges[(sample - 256) / 2]; + b = edges[((sample - 256) / 2 + 1 + 6 * (sample % 2)) % 18]; + } + if ((!strcmp(k->op, "BTC") || !strcmp(k->op, "BTR") || !strcmp(k->op, "BTS")) && !k->immediate) { + const int index[] = {-2*(int)k->width-1,-(int)k->width,-1,0,1, + (int)k->width-1,(int)k->width,(int)k->width+1,2*(int)k->width}; + b = (uint64_t)(int64_t)index[sample % 9]; + } + b = (word)b; + for (unsigned s = 0; s < 2; s++) { + uint64_t seed = s ? 0x12 : 0x8c7; + for (unsigned unaligned = 0; unaligned < 8; unaligned++) { + uint8_t memory[128], expected[128]; + uint64_t out[8]; + for (unsigned i = 0; i < 128; i++) { memory[i] = (i*73+19) ^ (sample*29); } + int offset = 64 + unaligned; + store(memory, offset, k->width, a); + memcpy(expected, memory, 128); + for (unsigned i = 0; i < 8; i++) { out[i] = sentinel; } + uint64_t valid, flags = reference(k, expected, offset, b, seed, &valid); + unsigned before_csr, after_csr; + __asm__ volatile("stmxcsr %0" : "=m"(before_csr)); + k->fn(memory, out, offset, b, seed); + __asm__ volatile("stmxcsr %0" : "=m"(after_csr)); + uint64_t word_mask = (word)~(word)0; + if (memcmp(memory, expected, 128) || (out[0] & valid) != flags || + (out[1] & word_mask) != b || (out[2] & word_mask) != seed || + (uint32_t)out[3] != before_csr || (uint32_t)out[4] != before_csr || + after_csr != before_csr) { + fprintf(stderr, "%s/%u imm=%u literal=%llx sample=%u unaligned=%u b=%llx seed=%llx flags=%llx/%llx mask=%llx\n", + k->op,k->width,k->immediate,(unsigned long long)k->literal,sample,unaligned, + (unsigned long long)b,(unsigned long long)seed,(unsigned long long)out[0], + (unsigned long long)flags,(unsigned long long)valid); + return 1; + } + for (unsigned i = 0; i < 3; i++) { + if ((out[i] & ~word_mask) != (sentinel & ~word_mask)) { return 2; } + } + if (out[3] >> 32 != sentinel >> 32 || out[4] >> 32 != sentinel >> 32 || + out[5] != sentinel || out[6] != sentinel || out[7] != sentinel) { return 3; } + observations++; + } + } + } + } + printf("lock observations %u\n", observations); + return 0; +} +` diff --git a/amd64_lock_scalar_rmw_sdk_test.go b/amd64_lock_scalar_rmw_sdk_test.go new file mode 100644 index 000000000..bdbaa70f2 --- /dev/null +++ b/amd64_lock_scalar_rmw_sdk_test.go @@ -0,0 +1,75 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86LockScalarRMWActualSDKOr8(t *testing.T) { + var data []byte + var sourcePath string + for _, relative := range []string{ + "src/internal/runtime/atomic/atomic_amd64.s", + "src/runtime/internal/atomic/atomic_amd64.s", + } { + path := filepath.Join(runtime.GOROOT(), filepath.FromSlash(relative)) + actual, err := os.ReadFile(path) + if os.IsNotExist(err) { + continue // Actual SDK source layout, not an encoder-version guess. + } + if err != nil || sourcePath != "" { + t.Fatal("actual SDK atomic source is missing or ambiguous", err) + } + sourcePath, data = path, actual + } + start := strings.Index(string(data), "TEXT ·Or8(SB)") + if sourcePath == "" || start < 0 { + t.Fatal("actual SDK lacks the discovering Or8 source") + } + end := strings.Index(string(data[start:]), "\nTEXT ") + if end < 0 { + t.Fatal("SDK Or8 has no complete next-TEXT boundary") + } + body := string(data[start:start+end]) + "\n" + header, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "pkg/include/textflag.h")) + if err != nil { + t.Fatal(err) + } + // The original body is unchanged. The actual stock header, not a manual + // NOSPLIT substitution, supplies the same Go assembler definitions. + original := string(header) + "\n" + body + machine := x86LockScalarOriginalInstruction(t, "amd64", original) + t.Logf("actual %s Or8 source SHA256=%x header SHA256=%x whole native instruction=%x", + runtime.Version(), sha256.Sum256(data), sha256.Sum256(header), machine) + expanded, err := PreprocessAssemblySource(original, AssemblyPreprocessOptions{}) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required for the actual SDK sample") + } + for _, target := range x86OrderingHintTargets { + if target.arch != "amd64" { + continue // This original SDK file is explicitly AMD64. + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, expanded, + "package ordering\nfunc Or8(ptr *uint8, val uint8)\n", "amd64", target.triple) + if err != nil || !strings.Contains(ir, "lock; orb") { + t.Fatal("actual SDK Or8 retains its whole-source atomic effect", err) + } + compileLLVMToObject(t, llc, target.triple, "or8.ll", fmt.Sprintf("or8-%s.o", api), ir) + }) + } + } + if after, err := os.ReadFile(sourcePath); err != nil || sha256.Sum256(after) != sha256.Sum256(data) { + t.Fatal("actual SDK source changed during sample proof", err) + } +} diff --git a/amd64_lock_scalar_rmw_symbol_test.go b/amd64_lock_scalar_rmw_symbol_test.go new file mode 100644 index 000000000..31b5cf66d --- /dev/null +++ b/amd64_lock_scalar_rmw_symbol_test.go @@ -0,0 +1,210 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86LockScalarRMWAddressLiteralIsNotMemory(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86LockScalarOracleForms(arch) { + body := strings.Replace(row.text, "1(SI)", "$·cell(SB)", 1) + t.Run(arch+"/"+body, func(t *testing.T) { + source := "GLOBL ·cell(SB),8,$16\n" + + x86NamedLockOracleSource(arch, "CLC\nLOCK\n"+body) + // An emitted address literal is Go TYPE_ADDR, not writable + // memory. This is actual assembler rejection, not execution. + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[len(file.Funcs[0].Instrs)-2] + form, recognized, err := x86LockScalarRMWProperties(ins.Op, ins, arch) + if !recognized || err == nil { + t.Error("typed memory grammar accepted an address literal", form, err) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + if err == nil || ir != "" { + t.Error("address literal became a locked memory object", api, err) + } + } + ordinary := strings.Replace(source, "$·cell(SB)", "·cell(SB)", 1) + x86LockScalarOriginalInstruction(t, arch, ordinary) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, ordinary, + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + // This exact original source declares RODATA. Go format + // acceptance does not make its storage writable. + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Error("original readonly SB source acquired a write permission", api, err) + } + } + }) + } + } +} + +func TestX86StaticBitStringLockedAndVolatileObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86OrderingHintTargets { + for _, locked := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/locked=%t", target.triple, locked), func(t *testing.T) { + var source, declarations strings.Builder + source.WriteString("DATA ·cell+0(SB)/8,$0x1234\nDATA ·cell+8(SB)/8,$0x5678\nGLOBL ·cell(SB),16,$16\n") + declarations.WriteString("package ordering\n") + var names []string + for _, row := range x86LockScalarOracleForms(target.arch) { + if !strings.HasPrefix(row.op, "BT") { + continue + } + name := fmt.Sprintf("staticBit%d", len(names)) + names = append(names, name) + body := strings.Replace(row.text, "1(SI)", "·cell(SB)", 1) + if locked { + body = "LOCK\n" + body + } + original := x86NamedLockOracleSource(target.arch, "CLC\n"+body) + original = strings.Replace(original, "·locked", "·"+name, 1) + // The register-indexed rows consume an actual unbounded Go + // numerical input. No constant or fixed-span range is assumed. + if target.arch == "amd64" { + original = strings.Replace(original, "$0-8", "$0-16", 1) + original = strings.Replace(original, "MOVL $4,CX", "MOVQ index+8(FP),CX", 1) + fmt.Fprintf(&declarations, "func %s(p *[16]byte, index uint64)\n", name) + } else { + original = strings.Replace(original, "$0-4", "$0-8", 1) + original = strings.Replace(original, "MOVL $4,CX", "MOVL index+4(FP),CX", 1) + fmt.Fprintf(&declarations, "func %s(p *[16]byte, index uint32)\n", name) + } + source.WriteString(original) + } + want := 18 + if target.arch == "386" { + want = 12 + } + if len(names) != want || len(names) > 32 { + t.Fatal("complete original bit-string object batch changed", len(names), want) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal("complete native static bit-string source", err) + } + if !locked { + for _, name := range names { + body := x86StringMemoryFunction(t, ir, name) + if !strings.Contains(body, "load volatile") || !strings.Contains(body, "store volatile") { + t.Fatal("unlocked bit-string lost an actual native memory effect", name) + } + } + } + for _, level := range []int{0, 2} { + text := ir + if level == 2 { + text = x86MulDivRoundOptimize(t, ir) + } + x86PackedStringCompileObject(t, llc, target.triple, text, fmt.Sprintf("-O%d", level)) + } + }) + } + }) + } + } +} + +// Keep the original readonly/address-literal source above. These are new +// positive TDD controls, not evidence that the current consumer supports them. +// Partitioning preserves all original forms, including signed dynamic indexes. +func TestX86LockScalarRMWMutableSBOriginalForms(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + forms := x86LockScalarOracleForms(arch) + groups := make(map[string][]x86LockScalarOracleForm) + for _, row := range forms { + partition := "scalar" + if strings.HasPrefix(row.op, "BTC") || strings.HasPrefix(row.op, "BTR") || strings.HasPrefix(row.op, "BTS") { + operands := strings.SplitN(row.text, " ", 2) + if len(operands) != 2 { + t.Fatal("original bit-string row lost its operands", row.text) + } + index := strings.SplitN(operands[1], ",", 2)[0] + switch index { + case "$1": + partition = "bit-immediate" + case "CX": + partition = "bit-register" + default: + t.Fatal("original bit-string index inventory changed", row.text) + } + } + groups[partition] = append(groups[partition], row) + } + wantAll, wantScalar, wantBits := 90, 72, 9 + if arch == "386" { + wantAll, wantScalar, wantBits = 66, 54, 6 + } + if len(forms) != wantAll || len(groups["scalar"]) != wantScalar || + len(groups["bit-immediate"]) != wantBits || len(groups["bit-register"]) != wantBits { + t.Fatal("complete original scalar/bit-string partition changed", arch, + len(forms), len(groups["scalar"]), len(groups["bit-immediate"]), len(groups["bit-register"])) + } + for _, partition := range []string{"scalar", "bit-immediate", "bit-register"} { + t.Run(arch+"/"+partition, func(t *testing.T) { + for _, row := range groups[partition] { + body := strings.Replace(row.text, "1(SI)", "·cell(SB)", 1) + if body == row.text { + t.Fatal("original memory operand was not replaced", row.text) + } + t.Run(body, func(t *testing.T) { + // DATA covers the actual complete 16-byte mutable object. + // This does not prove a register-indexed bit string's range. + source := "DATA ·cell+0(SB)/8,$0x1234\n" + + "DATA ·cell+8(SB)/8,$0x5678\n" + + "GLOBL ·cell(SB),16,$16\n" + + x86NamedLockOracleSource(arch, "CLC\nLOCK\n"+body) + listing, err := x86ScalarLiteralGoListing(t, arch, source) + if err != nil { + t.Fatal("complete mutable original Go object", err) + } + x86GPSegmentListingBytes(t, listing, []string{"locked"}) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + if err != nil { + t.Error("complete mutable original SB row lacks its atomic effect", api, err) + continue + } + // A module containing an uncalled helper is not coverage. + // Follow this exact public body to its actual native helper. + caller := x86StringMemoryFunction(t, ir, "locked") + const prefix = "@__plan9asm_lock_rmw_" + if strings.Count(caller, prefix) != 1 { + t.Error("mutable original SB body bypassed its complete helper", api) + continue + } + call := caller[strings.Index(caller, prefix)+1:] + end := strings.Index(call, "(") + if end < 0 { + t.Fatal("called native helper lacks its argument list", api) + } + helper := x86StringMemoryFunction(t, ir, call[:end]) + if !strings.Contains(helper, "asm sideeffect") || + !strings.Contains(helper, "lock; "+strings.ToLower(row.op)) { + t.Error("actually called helper lost the exact native SB effect", api) + } + } + }) + } + }) + } + } +} diff --git a/amd64_logical_auto_storage.go b/amd64_logical_auto_storage.go new file mode 100644 index 000000000..202165318 --- /dev/null +++ b/amd64_logical_auto_storage.go @@ -0,0 +1,61 @@ +package plan9asm + +import "fmt" + +// Ordinary logical AUTO bytes use immutable entry SP plus the frame extent, +// not addresses below the mutable physical/private SP. Private PUSH storage +// must not overwrite them. This consumes an existing complete source proof; +// independently bound native/caller storage keeps its own address projection. +func (c *amd64Ctx) prepareLogicalAUTOStorage() error { + fn := c.fpScalarSource + if !c.fpScalarChecked || c.fpScalarCheckErr != nil || c.sourceStack != nil || c.fixedLocal != nil || + fn.x86SourceStack != nil || fn.x86FixedLocal != nil || fn.x86DirectTailRoot != nil || + fn.x86ContinuationFlow != nil || fn.x86BoolCalls != nil { + return nil + } + if _, scopedControl := x86X87ScopedControlScratchExtent(fn, c.goarch); scopedControl { + if c.frameSize != 0 { + return fmt.Errorf("%w: scoped x87 scratch lost its original zero frame", ErrProbeNeedsContext) + } + // Keep actual negative SP offsets in the existing bounded private + // stack object. Do not manufacture positive logical AUTO storage. + return nil + } + stack, err := x86ResultPrivateFlagsSource(fn, c.goarch) + if err != nil { + return err + } + c.privateFlagsStack = stack != nil + extent, err := x86ResultAUTOExtentWithPrivateStack(fn, c.goarch, c.privateFlagsStack) + if err != nil { + return err + } + if extent != 0 && int64(extent) != c.frameSize { + return fmt.Errorf("%w: logical AUTO emitter lost its original frame extent", ErrProbeNeedsContext) + } + c.logicalAUTOExtent = extent + return nil +} + +func (c *amd64Ctx) logicalAUTOAddress(mem MemRef) (string, bool, error) { + if c.logicalAUTOExtent == 0 || mem.Base != SP { + return "", false, nil + } + if c.privateFlagsStack && mem.Off >= 0 && mem.Sym == "" && mem.OffRaw == "" && + mem.Index == "" && mem.IndexExt == "" && mem.Segment == "" && mem.Scale == 0 { + return "", false, nil // Reaching PUSH depth, rather than the immutable AUTO base, owns this access. + } + if _, static := x86SBMemoryAddressRole(Operand{Kind: OpMem, Mem: mem}, c.goarch); static { + return "", false, nil + } + offset, valid := x86ResultAUTOOffset(Operand{Kind: OpMem, Mem: mem}, 1, c.logicalAUTOExtent) + if !valid || c.logicalAUTOBase == "" { + return "", true, fmt.Errorf("%w: logical AUTO address no longer matches its complete source", ErrProbeNeedsContext) + } + if offset == 0 { + return c.logicalAUTOBase, true, nil + } + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %d\n", address, c.logicalAUTOBase, offset) + return "%" + address, true, nil +} diff --git a/amd64_lower_abs_sign.go b/amd64_lower_abs_sign.go new file mode 100644 index 000000000..04e16bbb6 --- /dev/null +++ b/amd64_lower_abs_sign.go @@ -0,0 +1,137 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// The complete shared grammar retains each source/data/mask role. Legacy +// Type4 keeps its original aligned m128 instruction; VEX is unaligned and +// EVEX E4 suppresses only inactive lanes. These access rules are distinct. +func (c *amd64Ctx) lowerPackedAbsSign(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86PackedAbsSignProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := c.validateConflictFunnelMemory(ins, form.x86PackedBinaryForm); err != nil { + return true, false, err + } + if form.legacy && form.memory >= 0 { + return true, false, c.lowerLegacyAbsSignMemory(ins, form) + } + mask := "" + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + bits, width := form.spec.laneBits, form.byteWidth + first, err := c.loadX86PackedShiftLanes(ins.Args[form.first], width, bits, form.broadcast, mask) + if err != nil { + return true, false, err + } + result := "" + if form.spec.sign { + data, err := c.loadPackedCompareLanes(ins.Args[form.second], width, bits, false) + if err != nil { + return true, false, err + } + result = c.emitPackedSign(first, data, width, bits) + } else { + result = c.emitPackedAbs(first, width, bits) + } + lanes := width * 8 / bits + destination := ins.Args[form.destination] + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + oldBytes, err := c.loadPackedCompareBytes(destination, width) + if err != nil { + return true, false, err + } + old = c.bitcastVectorBytesToIntegerLanes(width, lanes, bits, oldBytes) + } + result = amd64ApplyIntegerLaneMask(c, lanes, bits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, bits, result, width) + if form.legacy { + return true, false, c.storeLegacyXViews(destination.Reg, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, width, "%"+out) +} + +// Intel Type4's legacy SSE m128 alignment exception applies to all six +// PABS[BWD]/PSIGN[BWD] names, unlike their VEX counterparts. A volatile +// align-1 read followed by arithmetic loses that exception, and align16 is +// merely an LLVM precondition, not a runtime check. Keep the actual opcode +// even for a dead result. Virtual FP bytes are not original native storage. +func (c *amd64Ctx) lowerLegacyAbsSignMemory(ins Instr, form x86PackedAbsSignForm) error { + source := ins.Args[form.first] + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || source.Kind == OpFP { + return fmt.Errorf("%w: legacy ABS/sign m128 needs its original native x86 storage and alignment", ErrProbeNeedsContext) + } + bits := form.spec.laneBits + mnemonic := strings.ToLower(string(ins.Op)) + var result string + var err error + if form.spec.sign { + data, failure := c.loadPackedCompareLanes(ins.Args[form.second], 16, bits, false) + if failure != nil { + return failure + } + result, err = c.emitPackedBinaryNativeMemory(source, bits, data, mnemonic) + } else { + // Unary ABS has no old-destination input. Do not manufacture one just + // to reuse the tied binary carrier used by PSIGN and other families. + pointer, pointerType, _, failure := c.packedVectorMemoryPointer(source) + if failure != nil { + return failure + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + call, lanes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(%s %s)\n", + call, mnemonic+" "+segment+"($1), $0", "=x,r,~{memory}", pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <%d x i%d>\n", lanes, call, 128/bits, bits) + result = "%" + lanes + } + if err != nil { + return err + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <16 x i8>\n", bytes, 128/bits, bits, result) + return c.storeLegacyXViews(ins.Args[form.destination].Reg, "%"+bytes) +} + +func (c *amd64Ctx) emitPackedAbs(values string, byteWidth, laneBits int) string { + lanes := byteWidth * 8 / laneBits + negative := c.newTmp() + negated := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i%d> %s, zeroinitializer\n", negative, lanes, laneBits, values) + // No nsw: ABS of the most negative signed value wraps to itself. + fmt.Fprintf(c.b, " %%%s = sub <%d x i%d> zeroinitializer, %s\n", negated, lanes, laneBits, values) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %%%s, <%d x i%d> %s\n", result, lanes, negative, lanes, laneBits, negated, lanes, laneBits, values) + return "%" + result +} + +func (c *amd64Ctx) emitPackedSign(control, data string, width, bits int) string { + lanes := width * 8 / bits + positive := c.newTmp() + negative := c.newTmp() + negated := c.newTmp() + nonpositive := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt <%d x i%d> %s, zeroinitializer\n", positive, lanes, bits, control) + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i%d> %s, zeroinitializer\n", negative, lanes, bits, control) + fmt.Fprintf(c.b, " %%%s = sub <%d x i%d> zeroinitializer, %s\n", negated, lanes, bits, data) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %%%s, <%d x i%d> zeroinitializer\n", nonpositive, lanes, negative, lanes, bits, negated, lanes, bits) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %s, <%d x i%d> %%%s\n", result, lanes, positive, lanes, bits, data, lanes, bits, nonpositive) + return "%" + result +} diff --git a/amd64_lower_abs_sign_test.go b/amd64_lower_abs_sign_test.go new file mode 100644 index 000000000..4801a8e4e --- /dev/null +++ b/amd64_lower_abs_sign_test.go @@ -0,0 +1,233 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedAbsSignCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses yxm_q4 for the legacy PABS*/PSIGN* rows, + // _yvmovddup for VPABSB/W/D, _yvexpandpd for VPABSQ, and + // _yvaddsubpd for the VEX-only VPSIGN* rows. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedabssignforms(SB),NOSPLIT,$0-0\n") + for index, suffix := range []string{"B", "W", "D"} { + fmt.Fprintf(&source, "\tPABS%s X0, X1\n", suffix) + fmt.Fprintf(&source, "\tPABS%s %d(AX), X2\n", suffix, 16*index) + fmt.Fprintf(&source, "\tPSIGN%s X3, X4\n", suffix) + fmt.Fprintf(&source, "\tPSIGN%s %d(AX), X5\n", suffix, 48+16*index) + fmt.Fprintf(&source, "\tVPABS%s X6, X7\n", suffix) + fmt.Fprintf(&source, "\tVPABS%s %d(AX), Y1\n", suffix, 96+32*index) + fmt.Fprintf(&source, "\tVPABS%s X20, X21\n", suffix) + fmt.Fprintf(&source, "\tVPABS%s X8, K1, X9\n", suffix) + fmt.Fprintf(&source, "\tVPABS%s.Z %d(AX), K2, Y3\n", suffix, 192+32*index) + fmt.Fprintf(&source, "\tVPABS%s Z4, K3, Z5\n", suffix) + fmt.Fprintf(&source, "\tVPSIGN%s X8, X9, X10\n", suffix) + fmt.Fprintf(&source, "\tVPSIGN%s %d(AX), Y11, Y12\n", suffix, 288+32*index) + } + source.WriteString(` VPABSD.BCST 384(AX), Y11 + VPABSD.BCST.Z 388(AX), K4, Z6 + VPABSQ X0, X1 + VPABSQ 400(AX), Y2 + VPABSQ Z3, K5, Z4 + VPABSQ.Z 464(AX), K6, Z5 + VPABSQ.BCST 472(AX), X6 + VPABSQ.BCST.Z 480(AX), K7, Y7 + RET +`) + physical := x86PhysicalMode32ObjectSource(t, source.String(), target.goarch) + ll := x86UnboundVoidScalarFormatObject(t, physical, "packedabssignforms", target.goarch, target.triple) + for _, want := range []string{"icmp slt <", "icmp sgt <", "sub <", "select <"} { + if !strings.Contains(ll, want) { + t.Fatalf("packed abs/sign IR is missing %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-abs-sign-"+target.name+".ll", "packed-abs-sign-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedAbsSignRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PABSB Y0, X1", + "PABSQ X0, X1", + "PABSD.Z X0, X1", + "VPABSB X0, Y1", + "VPABSW Y0, X1", + "VPABSD Z0, Y1", + "VPABSQ X0, Y1", + "VPABSQ X0, K0, X1", + "VPABSB.Z X0, X1", + "VPABSW.BCST (AX), X1", + "VPABSQ.BCST X0, X1", + "PSIGNB Y0, X1", + "PSIGNQ X0, X1", + "VPSIGNB X0, X1", + "VPSIGNW Z0, Z1, Z2", + "VPSIGND X0, K1, X1, X2", + "VPSIGNB.Z X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedAbsSignRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PABSB X8, X0", + "PABSW X0, X8", + "PSIGND X8, X0", + } { + assertX86PackedAbsSignRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86PackedAbsSignRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed abs/sign tables", instruction) + } +} + +func qualifyAMD64PackedAbsSignRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT packedabssign(SB),NOSPLIT,$0-40 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ control+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), SI + KMOVQ SI, K1 + + MOVOU (BX), X0 + PABSB X0, X0 + MOVOU X0, 0(AX) + + MOVOU (BX), X0 + PSIGNW (CX), X0 + MOVOU X0, 16(AX) + + VMOVDQU (DX), Y1 + VPABSD.BCST (BX), K1, Y1 + VMOVDQU Y1, 32(AX) + + VMOVDQU (BX), Y0 + VPSIGND (CX), Y0, Y2 + VMOVDQU Y2, 64(AX) + + VMOVDQU64 (BX), Z0 + VPABSQ.Z Z0, K1, Z1 + VMOVDQU64 Z1, 96(AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedabssign": { + Name: "packedabssign", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +extern void packedabssign(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static uint8_t abs8(uint8_t v) { return (v & 0x80) ? (uint8_t)(0-v) : v; } +static int16_t sign16(int16_t value, int16_t control) { + return control > 0 ? value : control < 0 ? (int16_t)(0-(uint16_t)value) : 0; +} +static int32_t sign32(int32_t value, int32_t control) { + return control > 0 ? value : control < 0 ? (int32_t)(0-(uint32_t)value) : 0; +} +static int32_t abs32(int32_t value) { return value < 0 ? (int32_t)(0-(uint32_t)value) : value; } +static int64_t abs64(int64_t value) { return value < 0 ? (int64_t)(0-(uint64_t)value) : value; } +int main(void) { + union { uint8_t b[64]; int16_t w[32]; int32_t d[16]; int64_t q[8]; } source, control, old; + uint8_t out[160]; + for (int i = 0; i < 64; i++) { + source.b[i] = (uint8_t)(i * 29 + 0x80); + control.b[i] = (uint8_t)(i * 17 - 71); + old.b[i] = (uint8_t)(0xf0 - i); + } + control.w[0] = 1; control.w[1] = 0; control.w[2] = -1; control.w[3] = INT16_MIN; + control.d[0] = 1; control.d[1] = 0; control.d[2] = -1; control.d[3] = INT32_MIN; + memset(out, 0xcc, sizeof(out)); + const uint64_t mask = 0xa5; + packedabssign(out, source.b, control.b, old.b, mask); + for (int i = 0; i < 16; i++) if (out[i] != abs8(source.b[i])) return 10+i; + const int16_t *got16 = (const int16_t *)(out+16); + for (int i = 0; i < 8; i++) if (got16[i] != sign16(source.w[i], control.w[i])) return 40+i; + const int32_t *gotAbs32 = (const int32_t *)(out+32); + int32_t broadcast = abs32(source.d[0]); + for (int i = 0; i < 8; i++) { + int32_t want = (mask >> i) & 1 ? broadcast : old.d[i]; + if (gotAbs32[i] != want) return 60+i; + } + const int32_t *gotSign32 = (const int32_t *)(out+64); + for (int i = 0; i < 8; i++) if (gotSign32[i] != sign32(source.d[i], control.d[i])) return 80+i; + const int64_t *gotAbs64 = (const int64_t *)(out+96); + for (int i = 0; i < 8; i++) { + int64_t want = (mask >> i) & 1 ? abs64(source.q[i]) : 0; + if (gotAbs64[i] != want) return 100+i; + } + return 0; +} +` + x86OriginalFloatingRuntimeRequireCPU(t, "packed-abs-sign", clang, triple, runPrefix) + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_abs_sign", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_accumulator_sign_extend.go b/amd64_lower_accumulator_sign_extend.go new file mode 100644 index 000000000..c27ff4c66 --- /dev/null +++ b/amd64_lower_accumulator_sign_extend.go @@ -0,0 +1,27 @@ +package plan9asm + +import "fmt" + +// lowerAccumulatorSignExtend implements the complete Go 1.27 ynone family +// sharing opcodes 98/99: CBW/CWDE/CDQE extend into AX/EAX/RAX, while +// CWD/CDQ/CQO produce the sign half in DX/EDX/RDX. None of them modify flags. +func (c *amd64Ctx) lowerAccumulatorSignExtend(op Op, ins Instr) (ok bool, terminated bool, err error) { + spec, known, err := x86AccumulatorSignExtendProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + sourceType := amd64IntegerTypeForBits(int(spec.sourceBytes * 8)) + targetType := amd64IntegerTypeForBits(int(spec.destinationBytes * 8)) + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, sourceType) + if err != nil { + return true, false, err + } + result := c.newTmp() + if spec.destination == AX { + fmt.Fprintf(c.b, " %%%s = sext %s %s to %s\n", result, sourceType, value, targetType) + } else { + bits, _ := amd64IntegerTypeBits(sourceType) + fmt.Fprintf(c.b, " %%%s = ashr %s %s, %d\n", result, sourceType, value, bits-1) + } + return true, false, c.storeRegSized(spec.destination, targetType, "%"+result) +} diff --git a/amd64_lower_accumulator_sign_extend_test.go b/amd64_lower_accumulator_sign_extend_test.go new file mode 100644 index 000000000..2f97e7e17 --- /dev/null +++ b/amd64_lower_accumulator_sign_extend_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func x86AccumulatorSignExtendCompleteFormsSource(goarch string) string { + ops := []string{"CBW", "CWDE", "CWD", "CDQ"} + if goarch == "amd64" { + ops = append(ops, "CDQE", "CQO") + } + var source strings.Builder + source.WriteString("TEXT signextendforms(SB),$0-0\n") + for _, op := range ops { + source.WriteString("\t" + op + "\n") + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86AccumulatorSignExtendCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86AccumulatorSignExtendCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + ir := x86UnboundVoidScalarFormatObject(t, source, "signextendforms", target.goarch, target.triple) + for _, want := range []string{"sext i8", "sext i16", "ashr i16", "ashr i32"} { + if !strings.Contains(ir, want) { + t.Fatalf("accumulator sign-extension lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" { + for _, want := range []string{"sext i32", "ashr i64"} { + if !strings.Contains(ir, want) { + t.Fatalf("amd64 accumulator sign-extension lowering omitted %q:\n%s", want, ir) + } + } + } + compileLLVMToObject(t, llc, target.triple, "accumulator-sign-extend-"+target.name+".ll", "accumulator-sign-extend-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86AccumulatorSignExtendRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "CBW AX"}, + {goarch: "amd64", instruction: "CWDE $1"}, + {goarch: "amd64", instruction: "CDQ AX, DX"}, + {goarch: "amd64", instruction: "CWD.Z"}, + {goarch: "386", instruction: "CDQE"}, + {goarch: "386", instruction: "CQO"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ynone tables", test.instruction) + } + }) + } +} + +func TestAMD64AccumulatorSignExtendRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT signextendsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + + MOVQ $0x1122334455660080, AX + CBW + MOVQ AX, 0(DI) + + MOVQ $0x1122334455667788, DX + MOVW $0x8001, AX + CWD + MOVQ DX, 8(DI) + + MOVQ $0x1122334455668001, AX + CWDE + MOVQ AX, 16(DI) + + MOVL $0x80000001, AX + MOVQ $0x1122334455667788, DX + CDQ + MOVQ DX, 24(DI) + + MOVL $0x80000001, AX + CDQE + MOVQ AX, 32(DI) + + MOVQ $0x8000000000000001, AX + CQO + MOVQ DX, 40(DI) + + SETCS 48(DI) + SETOS 49(DI) + SETEQ 50(DI) + SETMI 51(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "signextendsemantics": { + Name: "signextendsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void signextendsemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[56] = {0}; + signextendsemantics(out); + if (load64(out+0) != UINT64_C(0x112233445566ff80)) return 10; + if (load64(out+8) != UINT64_C(0x112233445566ffff)) return 11; + if (load64(out+16) != UINT64_C(0x00000000ffff8001)) return 12; + if (load64(out+24) != UINT64_C(0x00000000ffffffff)) return 13; + if (load64(out+32) != UINT64_C(0xffffffff80000001)) return 14; + if (load64(out+40) != UINT64_C(0xffffffffffffffff)) return 15; + if (out[48] != 1 || out[49] != 1 || out[50] != 0 || out[51] != 1) return 16; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "accumulator_sign_extend_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_adc_sbb.go b/amd64_lower_adc_sbb.go new file mode 100644 index 000000000..72105d8fb --- /dev/null +++ b/amd64_lower_adc_sbb.go @@ -0,0 +1,152 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerScalarADCSBB implements every operand row in Go 1.27's yxorb (B) +// and yaddl (W/L/Q) tables for add-with-carry and subtract-with-borrow. +func (c *amd64Ctx) lowerScalarADCSBB(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + add, bits, recognized := amd64ScalarADCSBBProperties(baseOp) + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects source and destination: %q", c.goarch, baseOp, ins.Raw) + } + if err := validateAMD64ScalarAddSubOperands(c.goarch, baseOp, bits, ins.Args[0], ins.Args[1], ins); err != nil { + return true, false, err + } + + source, destination := ins.Args[0], ins.Args[1] + typ := amd64IntegerTypeForBits(bits) + var sourceValue string + if source.Kind == OpImm { + sourceValue = amd64ScalarAddSubImmediate(source.Imm, bits) + } else { + sourceValue, destination, err = c.evalScalarArithmeticSource(ins, typ) + if err != nil { + return true, false, err + } + } + destinationValue, store, err := c.loadScalarRMWDestination(op, ins, destination, typ) + if err != nil { + return true, false, err + } + + wideType := LLVMType(fmt.Sprintf("i%d", bits*2)) + widen := func(value string) string { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", wide, typ, value, wideType) + return "%" + wide + } + wideDestination := widen(destinationValue) + wideSource := widen(sourceValue) + carry := c.loadFlag(c.flagsCFSlot) + wideCarryName := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", wideCarryName, carry, wideType) + wideCarry := "%" + wideCarryName + + partial := c.newTmp() + wideResult := c.newTmp() + var carryOut string + if add { + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", partial, wideType, wideDestination, wideSource) + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %s\n", wideResult, wideType, partial, wideCarry) + carryOut = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %s\n", carryOut, wideType, wideResult, amd64UnsignedMaximum(bits)) + } else { + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", partial, wideType, wideSource, wideCarry) + fmt.Fprintf(c.b, " %%%s = sub %s %s, %%%s\n", wideResult, wideType, wideDestination, partial) + carryOut = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %%%s\n", carryOut, wideType, wideDestination, partial) + } + + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", result, wideType, wideResult, typ) + resultValue := "%" + result + if err := store(resultValue); err != nil { + return true, false, err + } + c.setScalarADCSBBFlags(typ, add, destinationValue, sourceValue, resultValue, "%"+carryOut) + return true, false, nil +} + +func amd64ScalarADCSBBProperties(op string) (add bool, bits int, ok bool) { + switch op { + case "ADCB": + return true, 8, true + case "ADCW": + return true, 16, true + case "ADCL": + return true, 32, true + case "ADCQ": + return true, 64, true + case "SBBB": + return false, 8, true + case "SBBW": + return false, 16, true + case "SBBL": + return false, 32, true + case "SBBQ": + return false, 64, true + default: + return false, 0, false + } +} + +func amd64UnsignedMaximum(bits int) string { + switch bits { + case 8: + return "255" + case 16: + return "65535" + case 32: + return "4294967295" + case 64: + return "18446744073709551615" + default: + panic(fmt.Sprintf("unsupported x86 integer width %d", bits)) + } +} + +func (c *amd64Ctx) setScalarADCSBBFlags(typ LLVMType, add bool, destination, source, result, carry string) { + c.setAuxiliaryCarryFlag(typ, destination, source, result) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", carry, c.flagsCFSlot) + + xorOperands := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorOperands, typ, destination, source) + overflowInput := "%" + xorOperands + if add { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, -1\n", inverted, typ, overflowInput) + overflowInput = "%" + inverted + } + xorResult := c.newTmp() + overflowBits := c.newTmp() + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorResult, typ, destination, result) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", overflowBits, typ, overflowInput, xorResult) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", overflow, typ, overflowBits) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) + + zero := c.newTmp() + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, 0\n", sign, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + c.setParityFlagSized(typ, result) +} diff --git a/amd64_lower_adc_sbb_test.go b/amd64_lower_adc_sbb_test.go new file mode 100644 index 000000000..19c3e5c1b --- /dev/null +++ b/amd64_lower_adc_sbb_test.go @@ -0,0 +1,176 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86ADCSBBCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT adcsbbforms(SB),NOSPLIT,$32-0\n") + for _, stem := range []string{"ADC", "SBB"} { + for _, width := range []string{"B", "W", "L", "Q"} { + if target.goarch == "386" && width == "Q" { + continue + } + op := stem + width + srcReg, dstReg := "BX", "AX" + if width == "B" { + srcReg, dstReg = "BL", "AL" + } + fmt.Fprintf(&source, "\t%s $1, %s\n", op, dstReg) + fmt.Fprintf(&source, "\t%s %s, %s\n", op, srcReg, dstReg) + fmt.Fprintf(&source, "\t%s 8(BX), %s\n", op, dstReg) + fmt.Fprintf(&source, "\t%s $1, 16(BX)\n", op) + fmt.Fprintf(&source, "\t%s %s, 24(BX)\n", op, srcReg) + if target.goarch == "386" && width != "B" { + fmt.Fprintf(&source, "\t%s $0, SP\n", op) + } + if target.goarch == "amd64" && width == "Q" { + fmt.Fprintf(&source, "\t%s $4294967295, AX\n", op) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, "#include \"textflag.h\"\n"+source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func adcsbbforms()", target.goarch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original ADC/SBB forms invented carry, numerical/native inputs or SP", api, err) + } + } + ll := x86ScalarFormatObjectUnit(t, file, map[string]FuncSig{"adcsbbforms": {Name: "adcsbbforms", Ret: Void}}, + target.goarch, target.triple, 32) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, ir := range []string{ll, x86DwordProductIROptimize(t, ll)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + } + }) + } +} + +func TestTranslateX86ADCSBBRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + tests := []struct { + goarch string + triple string + instruction string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ADCB SP, AL"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SBBB AL, SP"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ADCQ 0(BX), 8(CX)"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SBBL X0, AX"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ADCW.Z AX, BX"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ADCQ $4294967296, AX"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "ADCQ AX, BX"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "SBBQ $1, 0(BX)"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "ADCL R8, AX"}, + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+test.instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ADC/SBB tables for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64ADCSBBMemoryRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT adcsbbmemory(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0xffffffffffffffff, 0(DI) + CLC + ADCQ $1, 0(DI) + SETCS 32(DI) + SBBQ $0, 0(DI) + SETCS 33(DI) + MOVL $0x7fffffff, 8(DI) + STC + ADCL $0, 8(DI) + SETOS 34(DI) + MOVW $0, 16(DI) + STC + SBBW $0, 16(DI) + SETCS 35(DI) + MOVB $0xff, 24(DI) + CLC + ADCB $1, 24(DI) + SETCS 36(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "adcsbbmemory": {Name: "adcsbbmemory", Args: []LLVMType{Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void adcsbbmemory(uint8_t *); +int main(void) { + uint8_t out[40] = {0}; + adcsbbmemory(out); + if (*(uint64_t *)(out+0) != UINT64_MAX) return 10; + if (*(uint32_t *)(out+8) != 0x80000000U) return 11; + if (*(uint16_t *)(out+16) != 0xffffU) return 12; + if (out[24] != 0) return 13; + for (int i = 32; i <= 36; i++) if (out[i] != 1) return 20+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "adc_sbb_memory", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_addsub.go b/amd64_lower_addsub.go new file mode 100644 index 000000000..2295e6d7f --- /dev/null +++ b/amd64_lower_addsub.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerScalarAddSub implements all operand rows in Go 1.27's yxorb (B) and +// yaddl (W/L/Q) tables for ADD and SUB. +func (c *amd64Ctx) lowerScalarAddSub(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + add, bits, recognized := amd64ScalarAddSubProperties(baseOp) + if !recognized { + return false, false, nil + } + if err := validateAMD64ScalarAddSubInstruction(c.goarch, ins); err != nil { + return true, false, err + } + source, destination := ins.Args[0], ins.Args[1] + typ := amd64IntegerTypeForBits(bits) + var sourceValue string + if source.Kind == OpImm { + sourceValue = amd64ScalarAddSubImmediate(source.Imm, bits) + } else { + sourceValue, destination, err = c.evalScalarArithmeticSource(ins, typ) + if err != nil { + return true, false, err + } + } + destinationValue, store, err := c.loadScalarRMWDestination(op, ins, destination, typ) + if err != nil { + return true, false, err + } + result := c.newTmp() + llvmOp := "sub" + if add { + llvmOp = "add" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", result, llvmOp, typ, destinationValue, sourceValue) + resultValue := "%" + result + if err := store(resultValue); err != nil { + return true, false, err + } + c.setScalarAddSubFlags(typ, add, destinationValue, sourceValue, resultValue) + return true, false, nil +} + +// validateAMD64ScalarAddSubInstruction is deliberately independent of a +// lowering context so the same Go-assembler table is enforced before either +// the direct/linear path or the CFG path is selected. +func validateAMD64ScalarAddSubInstruction(goarch string, ins Instr) error { + rawOp := strings.ToUpper(string(ins.Op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + _, bits, recognized := amd64ScalarAddSubProperties(baseOp) + if !recognized { + return nil + } + if rawOp != baseOp { + return fmt.Errorf("%s %s does not accept instruction suffixes: %q", goarch, baseOp, ins.Raw) + } + if goarch == "386" && bits == 64 { + return fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 2 { + return fmt.Errorf("%s %s expects source and destination: %q", goarch, baseOp, ins.Raw) + } + return validateAMD64ScalarAddSubOperands(goarch, baseOp, bits, ins.Args[0], ins.Args[1], ins) +} + +func validateAMD64ScalarAddSubFunction(goarch string, fn Func) error { + for _, ins := range fn.Instrs { + if err := validateAMD64ScalarAddSubInstruction(goarch, ins); err != nil { + return err + } + } + return nil +} + +func amd64ScalarAddSubProperties(op string) (add bool, bits int, ok bool) { + switch op { + case "ADDB": + return true, 8, true + case "ADDW": + return true, 16, true + case "ADDL": + return true, 32, true + case "ADDQ": + return true, 64, true + case "SUBB": + return false, 8, true + case "SUBW": + return false, 16, true + case "SUBL": + return false, 32, true + case "SUBQ": + return false, 64, true + default: + return false, 0, false + } +} + +func validateAMD64ScalarAddSubOperands(goarch, baseOp string, bits int, source, destination Operand, ins Instr) error { + registerAllowed := func(reg Reg) bool { + if bits == 8 { + // Although SP is in the wider general-register class, Go's + // ADD/SUB byte tables do not accept it as a direct byte operand. + return reg != SP && isGoYmbRegisterForArch(reg, goarch) + } + return isX86YrlRegisterForArch(reg, goarch) + } + destinationRegister := destination.Kind == OpReg && registerAllowed(destination.Reg) + destinationMemory := isAMD64MemoryOperand(destination) + if !destinationRegister && !destinationMemory { + return fmt.Errorf("%s %s destination is outside Go 1.27's Ymb/Yml classes: %q", goarch, baseOp, ins.Raw) + } + switch source.Kind { + case OpImm: + if goarch != "386" && (source.Imm < -2147483648 || source.Imm > 4294967295) { + return fmt.Errorf("amd64 %s immediate is outside Go 1.27's Yi32 class: %q", baseOp, ins.Raw) + } + case OpReg: + if !registerAllowed(source.Reg) { + return fmt.Errorf("%s %s source is outside Go 1.27's Yrb/Yrl classes: %q", goarch, baseOp, ins.Raw) + } + case OpSym: + if x86AbsoluteRelocationSource(source, goarch, bits) { + return nil // Yi32 address, not a TYPE_MEM read; retain the source operand. + } + fallthrough + default: + if !isAMD64MemoryOperand(source) || !destinationRegister { + return fmt.Errorf("%s %s operands do not match Go 1.27's register/memory rows: %q", goarch, baseOp, ins.Raw) + } + } + return nil +} + +// The assembler emits only the low operand-width bits. Q-width immediates are +// encoded as imm32 and sign-extended by the CPU. +func amd64ScalarAddSubImmediate(immediate int64, bits int) string { + return fmt.Sprintf("%d", amd64ScalarAddSubImmediateInt64(immediate, bits)) +} + +func amd64ScalarAddSubImmediateInt64(immediate int64, bits int) int64 { + var normalized int64 + switch bits { + case 8: + normalized = int64(int8(uint8(immediate))) + case 16: + normalized = int64(int16(uint16(immediate))) + case 32, 64: + normalized = int64(int32(uint32(immediate))) + } + return normalized +} + +func (c *amd64Ctx) setScalarAddSubFlags(typ LLVMType, add bool, destination, source, result string) { + c.setAuxiliaryCarryFlag(typ, destination, source, result) + carry := c.newTmp() + if add { + fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %s\n", carry, typ, result, destination) + } else { + fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %s\n", carry, typ, destination, source) + } + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + + xorOperands := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorOperands, typ, destination, source) + overflowInput := "%" + xorOperands + if add { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, -1\n", inverted, typ, overflowInput) + overflowInput = "%" + inverted + } + xorResult := c.newTmp() + overflowBits := c.newTmp() + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorResult, typ, destination, result) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", overflowBits, typ, overflowInput, xorResult) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", overflow, typ, overflowBits) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) + + zero := c.newTmp() + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, 0\n", sign, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + c.setParityFlagSized(typ, result) +} diff --git a/amd64_lower_addsub_float.go b/amd64_lower_addsub_float.go new file mode 100644 index 000000000..af56215ee --- /dev/null +++ b/amd64_lower_addsub_float.go @@ -0,0 +1,56 @@ +package plan9asm + +import "fmt" + +// lowerAlternatingFloatingAddSubtract implements the complete Go 1.27 yxm +// and _yvaddsubpd families. Even lanes subtract the Plan 9 first source from +// the destination/base source; odd lanes add it. +func (c *amd64Ctx) lowerAlternatingFloatingAddSubtract(op Op, ins Instr) (ok bool, terminated bool, err error) { + local := ins + local.Op = op + form, known, err := x86AlternatingFloatProperties(local, c.goarch) + if !known || err != nil { + return known, false, err + } + if form.legacy && form.memory >= 0 && ins.Args[form.first].Kind != OpFP { + return true, false, c.emitLegacyAlternatingRoundMemory(local, form, nil) + } + first, err := c.loadAlternatingRoundFloat(ins.Args[form.first], form.byteWidth, form.bits, !form.legacy) + if err != nil { + return true, false, err + } + second, err := c.loadHorizontalFloatingOperand(ins.Args[form.second], form.byteWidth, form.bits) + if err != nil { + return true, false, err + } + result := c.emitAlternatingFloatingAddSubtract(form.byteWidth, form.bits, first, second) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, amd64FMA3LLVMType(form.byteWidth*8/form.bits, form.bits), result, form.byteWidth) + if form.legacy { + return true, false, c.storeX(ins.Args[form.destination].Reg, "%"+out) + } + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitAlternatingFloatingAddSubtract(byteWidth, laneBits int, first, second string) string { + lanes := byteWidth * 8 / laneBits + typeName := amd64FMA3LLVMType(lanes, laneBits) + elementType := amd64FMA3LLVMType(1, laneBits) + result := "poison" + for lane := 0; lane < lanes; lane++ { + left := c.newTmp() + right := c.newTmp() + value := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", left, typeName, second, lane) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", right, typeName, first, lane) + operation := "fsub" + if lane%2 == 1 { + operation = "fadd" + } + fmt.Fprintf(c.b, " %%%s = %s %s %%%s, %%%s\n", value, operation, elementType, left, right) + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, typeName, result, elementType, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_addsub_float_test.go b/amd64_lower_addsub_float_test.go new file mode 100644 index 000000000..f77113b3d --- /dev/null +++ b/amd64_lower_addsub_float_test.go @@ -0,0 +1,81 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86AlternatingFloatingAddSubtractCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT alternatingfloataddsubforms(SB),NOSPLIT,$0-0\n") + for _, element := range []string{"PS", "PD"} { + op := "ADDSUB" + element + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + vop := "V" + op + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", vop) + fmt.Fprintf(&src, "\t%s (AX), X14, X15\n", vop) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", vop) + fmt.Fprintf(&src, "\t%s (AX), Y14, Y15\n", vop) + } + src.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, src.String(), "alternatingfloataddsubforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "alternating-float-addsub-"+target.name+".ll", "alternating-float-addsub-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86AlternatingFloatingAddSubtractRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, op := range []string{"ADDSUBPS", "ADDSUBPD", "VADDSUBPS", "VADDSUBPD"} { + forms := []string{"X0", "Y0, Y1", "X0, X1, X2"} + valid := "X0, X1" + if strings.HasPrefix(op, "V") { + forms = []string{"X0", "X0, X1", "X0, Y1, Y2", "Z0, Z1, Z2", "X0, X1, K1, X2"} + valid = "X0, X1, X2" + } + for _, operands := range forms { + instruction := op + " " + operands + t.Run(strings.NewReplacer(" ", "_", ",", "_").Replace(instruction), func(t *testing.T) { + assertX86AlternatingAddSubRejected(t, instruction) + }) + } + for _, suffix := range []string{".Z", ".BCST", ".SAE", ".RN_SAE"} { + instruction := op + suffix + " " + valid + t.Run(strings.NewReplacer(" ", "_", ".", "_").Replace(instruction), func(t *testing.T) { + assertX86AlternatingAddSubRejected(t, instruction) + }) + } + } +} + +func assertX86AlternatingAddSubRejected(t *testing.T, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yxm/_yvaddsubpd tables", instruction) + } +} diff --git a/amd64_lower_addvec.go b/amd64_lower_addvec.go new file mode 100644 index 000000000..c49f9fcf7 --- /dev/null +++ b/amd64_lower_addvec.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "fmt" + "strconv" +) + +type amd64PackedAddMode = x86PackedArithmeticMode + +const ( + amd64PackedAddWrap = x86PackedWrap + amd64PackedAddSignedSaturating = x86PackedSignedSaturating + amd64PackedAddUnsignedSaturating = x86PackedUnsignedSaturating +) + +// lowerPackedIntegerAdd implements the complete Go 1.27 packed-integer add +// family: the legacy ymm/yxm tables and the _yvandnpd VEX/EVEX table. +func (c *amd64Ctx) lowerPackedIntegerAdd(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerPackedIntegerAddForm(op, ins) +} + +func (c *amd64Ctx) loadLegacyMMXPackedSource(src Operand) (string, error) { + if src.Kind == OpReg { + if _, ok := amd64ParseMReg(src.Reg); !ok { + return "", fmt.Errorf("expected MMX register or memory, got %s", src.String()) + } + return c.loadReg(src.Reg) + } + if !isAMD64MemoryOperand(src) { + return "", fmt.Errorf("expected MMX register or memory, got %s", src.String()) + } + return c.evalIntSized(src, I64) +} + +func (c *amd64Ctx) bitcastI64ToIntegerLanes(lanes, laneBits int, value string) string { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to <%d x i%d>\n", cast, value, lanes, laneBits) + return "%" + cast +} + +func (c *amd64Ctx) emitPackedIntegerAdd(lanes, laneBits int, first, second string, mode amd64PackedAddMode) string { + vecType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + if mode == amd64PackedAddWrap { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", result, vecType, second, first) + return "%" + result + } + + wideBits := laneBits * 2 + wideType := fmt.Sprintf("<%d x i%d>", lanes, wideBits) + firstWide := c.newTmp() + secondWide := c.newTmp() + extend := "sext" + if mode == amd64PackedAddUnsignedSaturating { + extend = "zext" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", firstWide, extend, vecType, first, wideType) + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", secondWide, extend, vecType, second, wideType) + sum := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %%%s\n", sum, wideType, secondWide, firstWide) + + max := int64((uint64(1) << laneBits) - 1) + predicate := "ugt" + if mode == amd64PackedAddSignedSaturating { + max = int64((uint64(1) << (laneBits - 1)) - 1) + predicate = "sgt" + } + above := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s %s %%%s, %s\n", above, predicate, wideType, sum, llvmSplatSignedInteger(lanes, wideBits, max)) + capped := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %%%s\n", capped, lanes, above, wideType, llvmSplatSignedInteger(lanes, wideBits, max), wideType, sum) + clamped := "%" + capped + if mode == amd64PackedAddSignedSaturating { + min := -int64(uint64(1) << (laneBits - 1)) + below := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, %s\n", below, wideType, clamped, llvmSplatSignedInteger(lanes, wideBits, min)) + bounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", bounded, lanes, below, wideType, llvmSplatSignedInteger(lanes, wideBits, min), wideType, clamped) + clamped = "%" + bounded + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", result, wideType, clamped, vecType) + return "%" + result +} + +func llvmSplatSignedInteger(lanes, bits int, value int64) string { + return llvmSplatIntegerText(lanes, bits, strconv.FormatInt(value, 10)) +} diff --git a/amd64_lower_adx.go b/amd64_lower_adx.go new file mode 100644 index 000000000..eea63cb40 --- /dev/null +++ b/amd64_lower_adx.go @@ -0,0 +1,67 @@ +package plan9asm + +import "fmt" + +type amd64ADXCarry uint8 + +const ( + amd64ADXCarryCF amd64ADXCarry = iota + amd64ADXCarryOF +) + +type amd64ADXSpec struct { + bits int + prefix byte + carry amd64ADXCarry +} + +// Go 1.27 asm6.go uses yml_rl for every ADX form. One spec binds the +// register/memory grammar, encoded mandatory prefix, width and carry chain. +var amd64ADXSpecs = map[Op]amd64ADXSpec{ + "ADCXL": {bits: 32, prefix: 0x66, carry: amd64ADXCarryCF}, + "ADCXQ": {bits: 64, prefix: 0x66, carry: amd64ADXCarryCF}, + "ADOXL": {bits: 32, prefix: 0xf3, carry: amd64ADXCarryOF}, + "ADOXQ": {bits: 64, prefix: 0xf3, carry: amd64ADXCarryOF}, +} + +// lowerADX implements ADCXL/Q and ADOXL/Q. All four opcodes use Go 1.27's +// yml_rl row: register/memory source and register destination. ADCX consumes +// and updates CF; ADOX independently consumes and updates OF. +func (c *amd64Ctx) lowerADX(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86ADXProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + bits, flagSlot := form.spec.bits, c.flagsCFSlot + if form.spec.carry == amd64ADXCarryOF { + flagSlot = c.flagsOFSlot + } + source, destination := form.source, form.destination + typ := amd64IntegerTypeForBits(bits) + sourceValue, err := c.evalSizedNativeGPSource(source, typ, ins.x86AddressBits) + if err != nil { + return true, false, err + } + destinationValue, err := c.evalIntSized(destination, typ) + if err != nil { + return true, false, err + } + carryIn := c.loadFlag(flagSlot) + carryValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", carryValue, carryIn, typ) + sum := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", sum, typ, destinationValue, sourceValue) + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %%%s\n", result, typ, sum, carryValue) + carryFromOperands := c.newTmp() + carryFromInput := c.newTmp() + carryOut := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %s\n", carryFromOperands, typ, sum, destinationValue) + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %%%s\n", carryFromInput, typ, result, sum) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", carryOut, carryFromOperands, carryFromInput) + if err := c.storeRegSized(destination.Reg, typ, "%"+result); err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carryOut, flagSlot) + return true, false, nil +} diff --git a/amd64_lower_adx_test.go b/amd64_lower_adx_test.go new file mode 100644 index 000000000..dd9aad2db --- /dev/null +++ b/amd64_lower_adx_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86ADXCompleteFormsSource(goarch string) string { + var source strings.Builder + symbol := "adxdata" + size := 8 + if goarch == "386" { + symbol = "adxdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $1\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT adxforms(SB),$0-0\n") + widths := []string{"L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, stem := range []string{"ADCX", "ADOX"} { + for _, width := range widths { + op := stem + width + fmt.Fprintf(&source, "\t%s AX, BX\n", op) + fmt.Fprintf(&source, "\t%s 8(BX), CX\n", op) + fmt.Fprintf(&source, "\t%s %s(SB), DX\n", op, symbol) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12\n", op) + } else { + fmt.Fprintf(&source, "\t%s AX, SP\n", op) + } + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86ADXCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86ADXCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + ll := x86UnboundVoidScalarFormatObject(t, source, "adxforms", target.goarch, target.triple) + for _, want := range []string{"add i32", "icmp ult i32", `"target-features"="+adx"`} { + if !strings.Contains(ll, want) { + t.Fatalf("ADX lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "add i64") { + t.Fatalf("ADX Q-width lowering omitted add i64:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "adx-"+target.name+".ll", "adx-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ADXRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "ADCXQ $1, AX"}, + {goarch: "amd64", instruction: "ADOXL AX, 0(BX)"}, + {goarch: "amd64", instruction: "ADCXL X0, AX"}, + {goarch: "amd64", instruction: "ADOXQ AX, X0"}, + {goarch: "amd64", instruction: "ADCXQ AX"}, + {goarch: "amd64", instruction: "ADOXL.Z AX, BX"}, + {goarch: "386", instruction: "ADCXL R8, AX"}, + {goarch: "386", instruction: "ADOXQ AX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ADX yml_rl table", test.instruction) + } + }) + } +} + +func TestAMD64ADXRuntimeSemanticsAndIndependentFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86ADXOriginalRuntimeSource + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + mainC := x86ADXOriginalRuntimeDriver + rawSource := x86ADXOriginalRawSource(source) + for _, form := range []struct{ name, source string }{{"text", source}, {"raw", rawSource}} { + t.Run(form.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, form.source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "adxsemantics": { + Name: "adxsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "adx_semantics_"+form.name, triple, ir, mainC, runPrefix) + }) + } +} diff --git a/amd64_lower_aes.go b/amd64_lower_aes.go new file mode 100644 index 000000000..bf70d9cb9 --- /dev/null +++ b/amd64_lower_aes.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64AESRoundIntrinsics = map[Op]string{ + "AESENC": "@llvm.x86.aesni.aesenc", + "AESENCLAST": "@llvm.x86.aesni.aesenclast", + "AESDEC": "@llvm.x86.aesni.aesdec", + "AESDECLAST": "@llvm.x86.aesni.aesdeclast", +} + +// lowerAES implements the complete Go 1.27 AES/VAES family. Wide VAES round +// instructions apply the corresponding 128-bit AES operation independently +// to every lane. +func (c *amd64Ctx) lowerAES(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86AESProperties(rawOp, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + source, destination := f.sourceOperand, ins.Args[f.destination] + for _, slot := range c.fpResults { + if c.fixedLocal != nil { + break + } // its independent whole-source proof owns native local reads/results + if slot.Type == I8 || slot.Type == I16 || slot.Type == I32 || slot.Type == I64 || slot.Type == LLVMType("float") || slot.Type == LLVMType("double") { + // An intervening X→GP move, unrelated FP store or RET fallback + // cannot bypass the whole original-source must-initialized proof. + if err := c.scalarFPResultInitialized(); err != nil { + return true, false, err + } + break + } + } + if f.memory >= 0 { + // Native stack bytes need their independent exact backing certificate. + // Type 4 retains a 16-byte alignment exception for legacy SSE. A + // logical FP value is not the original physical address and cannot + // preserve that exception. VAES has no corresponding alignment fault. + if !f.spec.vector && source.Kind == OpFP { + return true, false, fmt.Errorf("%w: legacy AES FP source needs its original native storage and alignment", ErrProbeNeedsContext) + } + if source.Kind == OpMem && + !c.nativeMemoryOperand(ins, f.memory) && + !x86CallerMemoryCertified(ins, f.memory) && !c.fixedLocalMemoryCertified(ins, f.memory) { + return true, false, fmt.Errorf("%w: AES native stack source has no original storage certificate", ErrProbeNeedsContext) + } + if source.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(source.FPOffset, f.byteWidth); err != nil { + return true, false, err + } + readResult := false + for offset := source.FPOffset; offset < source.FPOffset+int64(f.byteWidth); offset++ { + slot, result, _ := c.fpVectorSlot(offset) + if slot.Type != I8 && slot.Type != I16 && slot.Type != I32 && slot.Type != I64 && slot.Type != LLVMType("float") && slot.Type != LLVMType("double") { + return true, false, fmt.Errorf("%w: AES FP byte is not declared numerical storage", ErrProbeNeedsContext) + } + readResult = readResult || result + } + if readResult { + if err := c.scalarFPResultInitialized(); err != nil { + return true, false, err + } + } + } + } + var value string + if f.memory >= 0 && !f.spec.vector && source.Kind != OpFP { + value, err = c.emitAESNativeMemory(ins, f) + } else { + // VAES has no fault-suppression rows. Even a dead key-assist result + // reads all m128 bytes, not only its numerical dwords 1 and 3. + var key string + key, err = c.loadX86PackedShiftBytes(source, f.byteWidth, true) + if err == nil && f.spec.operation < x86AESInverseMix { + var state string + state, err = c.loadPackedCompareBytes(ins.Args[f.state], f.byteWidth) + if err == nil { + legacyOp := Op(strings.TrimPrefix(strings.ToUpper(string(rawOp)), "V")) + value = c.emitAESRound(f.byteWidth, state, key, amd64AESRoundIntrinsics[legacyOp]) + } + } else if err == nil { + carrier, call, bytes := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", carrier, key) + if f.spec.operation == x86AESInverseMix { + fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64> %%%s)\n", call, carrier) + } else { + fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %%%s, i8 %d)\n", call, carrier, uint8(ins.Args[0].Imm)) + } + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bytes, call) + value = "%" + bytes + } + } + if err != nil { + return true, false, err + } + if !f.spec.vector { + return true, false, c.storeLegacyPackedLogical(destination, value) + } + return true, false, c.storePackedMoveOperand(destination, f.byteWidth, value) +} + +// Retain the original legacy opcode and its complete memory envelope even +// when the result is dead, including legacy SSE's Type 4 alignment exception. +func (c *amd64Ctx) emitAESNativeMemory(ins Instr, f x86AESForm) (string, error) { + source := f.sourceOperand + if f.spec.operation < x86AESInverseMix { + data, err := c.loadPackedCompareLanes(ins.Args[f.state], 16, 64, false) + if err != nil { + return "", err + } + result, err := c.emitPackedBinaryNativeMemory(source, 64, data, strings.ToLower(string(ins.Op))) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %s to <16 x i8>\n", value, result) + return "%" + value, nil + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + assembly := strings.ToLower(string(ins.Op)) + " " + segment + "($1), $0" + if f.spec.operation == x86AESKeyAssist { + assembly = fmt.Sprintf("aeskeygenassist $$%d, %s($1), $0", uint8(ins.Args[0].Imm), segment) + } + call, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <4 x i32> asm sideeffect %q, \"=x,r,~{memory}\"(%s %s)\n", call, assembly, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", value, call) + return "%" + value, nil +} + +func (c *amd64Ctx) emitAESRound(byteWidth int, stateBytes, keyBytes, intrinsic string) string { + qwords := byteWidth / 8 + state := c.newTmp() + key := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i64>\n", state, byteWidth, stateBytes, qwords) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i64>\n", key, byteWidth, keyBytes, qwords) + result := "zeroinitializer" + for lane := 0; lane < byteWidth/16; lane++ { + laneState := c.newTmp() + laneKey := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i64> %%%s, <%d x i64> poison, <2 x i32> \n", laneState, qwords, state, qwords, lane*2, lane*2+1) + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i64> %%%s, <%d x i64> poison, <2 x i32> \n", laneKey, qwords, key, qwords, lane*2, lane*2+1) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <2 x i64> %s(<2 x i64> %%%s, <2 x i64> %%%s)\n", call, intrinsic, laneState, laneKey) + for word := 0; word < 2; word++ { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 %d\n", extracted, call, word) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, qwords, result, extracted, lane*2+word) + result = "%" + inserted + } + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", bytes, qwords, result, byteWidth) + return "%" + bytes +} diff --git a/amd64_lower_aes_test.go b/amd64_lower_aes_test.go new file mode 100644 index 000000000..24ff01270 --- /dev/null +++ b/amd64_lower_aes_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86AESCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + zLast := 31 + if target.goarch == "386" { + legacyLast = 7 + zLast = 7 + } + var source strings.Builder + source.WriteString("TEXT aesforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"AESENC", "AESENCLAST", "AESDEC", "AESDECLAST"} { + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + vop := "V" + op + source.WriteString("\t" + vop + " X0, X1, X2\n") + source.WriteString("\t" + vop + " 16(AX), X1, X2\n") + source.WriteString("\t" + vop + " Y0, Y1, Y2\n") + source.WriteString("\t" + vop + " 32(AX), Y1, Y2\n") + if target.goarch == "amd64" { + source.WriteString("\t" + vop + " X16, X30, X31\n") + } + fmt.Fprintf(&source, "\t%s Z0, Z1, Z%d\n", vop, zLast) + } + fmt.Fprintf(&source, "\tAESIMC X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tAESIMC 8(AX), X%d\n", legacyLast) + fmt.Fprintf(&source, "\tVAESIMC X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tVAESIMC 8(AX), X%d\n", legacyLast) + fmt.Fprintf(&source, "\tAESKEYGENASSIST $0, X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tAESKEYGENASSIST $255, 8(AX), X%d\n", legacyLast) + fmt.Fprintf(&source, "\tVAESKEYGENASSIST $-128, X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tVAESKEYGENASSIST $255, 8(AX), X%d\n", legacyLast) + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"aesforms": {Name: "aesforms", Ret: Void}}, + }) + if ll != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original AES formats invented vector or native-pointer inputs", err) + } + x86LEAActualTargetGoObject(t, target.goarch, target.triple, "#include \"textflag.h\"\n"+source.String()) + ll = x86InstructionFormatObjectUnit(t, file, + map[string]FuncSig{"aesforms": {Name: "aesforms", Ret: Void}}, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerAES(ins.Op, ins) }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "aes-"+target.name+".ll", "aes-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86AESRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "AESENC Y0, X1", + "AESENC X0, X16", + "AESENC.Z X0, X1", + "AESIMC X0, (AX)", + "AESKEYGENASSIST $-1, X0, X1", + "AESKEYGENASSIST $256, X0, X1", + "VAESENC X0, Y1, Y2", + "VAESENC X0, X1", + "VAESENC X0, X1, K1, X2", + "VAESENC.Z X0, X1, X2", + "VAESENC X0, X1, (AX)", + "VAESIMC Y0, Y1", + "VAESIMC X16, X1", + "VAESKEYGENASSIST $-129, X0, X1", + "VAESKEYGENASSIST $256, X0, X1", + "VAESKEYGENASSIST $1, Y0, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86AESRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "AESENC X8, X0", + "AESIMC X0, X8", + "AESKEYGENASSIST $1, X8, X0", + "VAESIMC X8, X0", + "VAESKEYGENASSIST $1, X0, X8", + "VAESENC Z8, Z0, Z1", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86AESRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86AESRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's AES/VAES tables for %s", instruction, goarch) + } +} diff --git a/amd64_lower_align_elements.go b/amd64_lower_align_elements.go new file mode 100644 index 000000000..1f5bdadbe --- /dev/null +++ b/amd64_lower_align_elements.go @@ -0,0 +1,17 @@ +package plan9asm + +import "strings" + +// Preserve the original family registry for source-op extraction. +var amd64VectorAlignSpecs = map[Op]int{ + "VALIGND": 32, + "VALIGNQ": 64, +} + +func (c *amd64Ctx) lowerVectorAlign(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64VectorAlignSpecs[Op(base)]; !known { + return false, false, nil + } + return c.lowerPackedAlignFamily(op, ins) +} diff --git a/amd64_lower_align_elements_test.go b/amd64_lower_align_elements_test.go new file mode 100644 index 000000000..3ffe244f8 --- /dev/null +++ b/amd64_lower_align_elements_test.go @@ -0,0 +1,200 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateAMD64VectorAlignCompleteGo127Forms(t *testing.T) { + // VALIGND/Q share Go 1.27's six-row _yvalignd table: X/Y/Z, + // each with an unmasked and K1-K7 masked row. Their EVEX attributes add + // D/Q scalar broadcast and zeroing. + var source strings.Builder + source.WriteString("TEXT vector_align_forms(SB),$0-0\n") + for _, op := range []string{"VALIGND", "VALIGNQ"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s $0, %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s $255, 8(BX), %s3, %s4\n", op, width, width) + fmt.Fprintf(&source, "\t%s $1, %s5, %s6, K1, %s7\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z $2, 16(BX), %s8, K2, %s9\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST $3, 24(BX), %s10, %s11\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z $4, 32(BX), %s12, K3, %s13\n", op, width, width) + } + fmt.Fprintf(&source, "\t%s $5, X29, X30, X31\n", op) + fmt.Fprintf(&source, "\t%s $6, Y29, Y30, Y31\n", op) + fmt.Fprintf(&source, "\t%s $7, Z29, Z30, Z31\n", op) + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + file, signatures := x86UnboundVoidFormatSources(t, source.String(), "amd64", target.triple) + ir := x86InstructionFormatObjectUnit(t, file, signatures, "amd64", target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerVectorAlign(ins.Op, ins) }) + compileLLVMToObject(t, llc, target.triple, "vector-align-"+target.name+".ll", "vector-align-"+target.name+".o", ir) + }) + } +} + +func TestTranslateAMD64VectorAlignRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VALIGND X0, X1, X2", + "VALIGND $0, X0, X1", + "VALIGNQ $0, X0, X1, X2, X3, X4", + "VALIGND $-1, X0, X1, X2", + "VALIGNQ $256, X0, X1, X2", + "VALIGND AX, X0, X1, X2", + "VALIGND $0, Y0, X1, X2", + "VALIGNQ $0, X0, Y1, X2", + "VALIGND $0, X0, X1, Y2", + "VALIGNQ $0, X0, 8(BX), X2", + "VALIGND $0, X0, X1, 8(BX)", + "VALIGND $0, X0, X1, K0, X2", + "VALIGNQ $0, X0, X1, K8, X2", + "VALIGND.Z $0, X0, X1, X2", + "VALIGNQ.BCST $0, X0, X1, X2", + "VALIGND.BCST.Z $0, 8(BX), X1, X2", + "VALIGNQ.Z.BCST $0, 8(BX), X1, K1, X2", + "VALIGND.SAE $0, X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VALIGND/Q table", instruction) + } + }) + } +} + +func TestTranslate386VectorAlignMatchesGoAssemblerOperandLimit(t *testing.T) { + for _, instruction := range []string{ + "VALIGND $0, X0, X1, X2", + "VALIGNQ $255, 8(BX), Y1, Y2", + "VALIGND.Z $1, Z0, Z1, K1, Z2", + "VALIGNQ.BCST.Z $2, 16(BX), X1, K7, X2", + } { + t.Run(strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted 386 %q although Go 1.27 rejects its operand count", instruction) + } + } + }) + } +} + +func TestAMD64VectorAlignRuntimeSemantics(t *testing.T) { + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable integer oracle requires its native 64-bit host counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT vectorAlignSemantics(SB),NOSPLIT,$0-40 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + MOVQ scalar+24(FP), DX + VMOVDQU32 (BX), X0 + VMOVDQU32 (CX), X1 + VALIGND $1, X0, X1, X2 + VMOVDQU32 X2, 0(AX) + VALIGND $255, X0, X1, X3 + VMOVDQU32 X3, 16(AX) + VALIGNQ $1, X0, X1, X4 + VMOVDQU32 X4, 32(AX) + KMOVQ mask+32(FP), K1 + VMOVDQU32 X0, X5 + VALIGND $1, X0, X1, K1, X5 + VMOVDQU32 X5, 48(AX) + VALIGND.Z $1, X0, X1, K1, X6 + VMOVDQU32 X6, 64(AX) + VALIGND.BCST $1, (DX), X1, X7 + VMOVDQU32 X7, 80(AX) + RET +` + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", "#include \"textflag.h\"\n"+source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "vectorAlignSemantics": { + Name: "vectorAlignSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, + Ret: Void, + Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void vectorAlignSemantics(uint32_t *, const uint32_t *, const uint32_t *, const uint32_t *, uint64_t); +int main(void) { + uint32_t first[4] = {10, 11, 12, 13}; + uint32_t second[4] = {20, 21, 22, 23}; + uint32_t scalar = 99; + uint32_t out[24] = {0}; + const uint32_t want[24] = { + 11, 12, 13, 20, + 13, 20, 21, 22, + 12, 13, 20, 21, + 11, 11, 13, 13, + 11, 0, 13, 0, + 99, 99, 99, 20, + }; + vectorAlignSemantics(out, first, second, &scalar, 5); + for (int i = 0; i < 24; i++) if (out[i] != want[i]) return i + 1; + return 0; +} +` + x86ConflictFunnelPortableRuntime(t, llc, clang, "vector_align", triple, ir, mainC) +} diff --git a/amd64_lower_align_right.go b/amd64_lower_align_right.go new file mode 100644 index 000000000..ef430a292 --- /dev/null +++ b/amd64_lower_align_right.go @@ -0,0 +1,19 @@ +package plan9asm + +import "strings" + +// Keep both original dispatcher entries on the complete shared grammar. +func (c *amd64Ctx) lowerPackedAlignRight(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if base != "PALIGNR" && base != "VPALIGNR" { + return false, false, nil + } + return c.lowerPackedAlignFamily(op, ins) +} + +func amd64UnsignedImmediate(operand Operand, bits int) bool { + if operand.Kind != OpImm || operand.ImmIsFloat || operand.ImmRaw != "" || bits <= 0 || bits >= 63 { + return false + } + return operand.Imm >= 0 && operand.Imm < int64(uint64(1)< +extern void alignrightlegacy(uint8_t *, const uint8_t *, const uint8_t *); +extern void alignrightvector(uint8_t *, const uint8_t *, const uint8_t *); +int main(void) { + uint8_t low[32], high[32], out[32]; + for (int i = 0; i < 32; i++) { low[i] = (uint8_t)i; high[i] = (uint8_t)(100+i); out[i] = 0; } + alignrightlegacy(out, low, high); + for (int i = 0; i < 8; i++) if (out[i] != low[i+8]) return 10+i; + for (int i = 8; i < 16; i++) if (out[i] != high[i-8]) return 30+i; + for (int i = 0; i < 32; i++) out[i] = 0; + alignrightvector(out, low, high); + for (int lane = 0; lane < 2; lane++) { + int base = lane*16; + for (int i = 0; i < 8; i++) if (out[base+i] != low[base+i+8]) return 60+base+i; + for (int i = 8; i < 16; i++) if (out[base+i] != high[base+i-8]) return 100+base+i; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_align_right", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_andn.go b/amd64_lower_andn.go new file mode 100644 index 000000000..d6be17817 --- /dev/null +++ b/amd64_lower_andn.go @@ -0,0 +1,33 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerANDN(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86TernaryGPProperties(op, ins, c.goarch) + if !known || form.spec.operation != x86TernaryANDN { + return false, false, nil + } + if err != nil { + return true, false, err + } + typ := amd64IntegerTypeForBits(form.spec.bits) + first, err := c.evalSizedNativeGPSource(form.first, typ, ins.x86AddressBits) + if err != nil { + return true, false, err + } + second, err := c.evalIntSized(form.second, typ) + if err != nil { + return true, false, err + } + inverted, result := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, -1\n", inverted, typ, second) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", result, typ, inverted, first) + if typ == I64 { + c.setZSFlagsFromI64("%" + result) + } else { + c.setZSFlagsFromI32("%" + result) + } + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + return true, false, c.storeRegSized(form.destination.Reg, typ, "%"+result) +} diff --git a/amd64_lower_andn_test.go b/amd64_lower_andn_test.go new file mode 100644 index 000000000..6d23acae1 --- /dev/null +++ b/amd64_lower_andn_test.go @@ -0,0 +1,199 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86RawANDNReportedEncoding(t *testing.T) { + const source = ` +TEXT rawANDN(SB),$0-0 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x38 + BYTE $0xf2 + BYTE $0xe2 + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-apple-darwin"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source, target.goarch, target.triple) + if !strings.Contains(ir, `"target-features"="+bmi"`) { + t.Fatalf("raw ANDN did not infer BMI1 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-andn.ll", "raw-andn.o", ir) + }) + } +} + +func TestX86ANDNCompleteGo127FormsAcrossTargets(t *testing.T) { + registers386 := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} + registersAMD64 := append(append([]string{}, registers386...), "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + targets := []struct { + name string + goarch string + triple string + registers []string + }{ + {name: "386-linux", goarch: "386", triple: "i386-unknown-linux-gnu", registers: registers386}, + {name: "386-windows", goarch: "386", triple: "i686-pc-windows-msvc", registers: registers386}, + {name: "amd64-darwin", goarch: "amd64", triple: "x86_64-apple-darwin", registers: registersAMD64}, + {name: "amd64-linux", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registers: registersAMD64}, + {name: "amd64-windows", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registers: registersAMD64}, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT andnall(SB),$0-0\n") + for _, width := range []string{"L", "Q"} { + for index, first := range target.registers { + second := target.registers[(index+3)%len(target.registers)] + destination := target.registers[(index+5)%len(target.registers)] + fmt.Fprintf(&source, "\tANDN%s %s, %s, %s\n", width, first, second, destination) + } + fmt.Fprintf(&source, "\tANDN%s 8(AX), CX, DX\n", width) + fmt.Fprintf(&source, "\tANDN%s 16(AX)(CX*4), DX, BX\n", width) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "andnall", target.goarch, target.triple) + if !strings.Contains(ir, `"target-features"="+bmi"`) { + t.Fatalf("ANDN family did not infer BMI1 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "andn-family.ll", "andn-family.o", ir) + }) + } +} + +func TestX86RawANDNCompleteRegisterFields(t *testing.T) { + for _, mode := range []int{32, 64} { + registerCount := 8 + if mode == 64 { + registerCount = 16 + } + for _, width64 := range []bool{false, true} { + for first := 0; first < registerCount; first++ { + for second := 0; second < registerCount; second++ { + for destination := 0; destination < registerCount; destination++ { + p0 := byte((^destination>>3)&1)<<7 | 1<<6 | byte((^first>>3)&1)<<5 | 2 + p1 := byte((^second)&15) << 3 + if width64 { + p1 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0xc4, p0, p1, 0xf2, modRM} + instruction, length, ok, err := decodedX86ANDNInstruction(encoding, mode) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("mode=%d width64=%v first=%d second=%d destination=%d: ok=%v length=%d err=%v", mode, width64, first, second, destination, ok, length, err) + } + wantOp := Op("ANDNL") + if width64 && mode == 64 { + wantOp = "ANDNQ" + } + wantFirst, _ := decodedX86GeneralRegister(first) + wantSecond, _ := decodedX86GeneralRegister(second) + wantDestination, _ := decodedX86GeneralRegister(destination) + if instruction.Op != wantOp || len(instruction.Args) != 3 || instruction.Args[0].Reg != wantFirst || instruction.Args[1].Reg != wantSecond || instruction.Args[2].Reg != wantDestination { + t.Fatalf("encoding %x decoded as %#v", encoding, instruction) + } + } + } + } + } + } +} + +func TestX86RawANDNMemoryForms(t *testing.T) { + for _, test := range []struct { + name string + encoding []byte + }{ + {name: "direct", encoding: []byte{0xc4, 0xe2, 0x30, 0xf2, 0x13}}, + {name: "extended base", encoding: []byte{0xc4, 0xc2, 0x30, 0xf2, 0x13}}, + {name: "sib disp8", encoding: []byte{0xc4, 0x82, 0x88, 0xf2, 0x54, 0x88, 0x10}}, + {name: "base less", encoding: []byte{0xc4, 0xe2, 0x30, 0xf2, 0x14, 0x25, 0x78, 0x56, 0x34, 0x12}}, + {name: "segment", encoding: []byte{0x64, 0xc4, 0xe2, 0x30, 0xf2, 0x54, 0x48, 0x08}}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86ANDNInstruction(test.encoding, 64) + if !ok || err != nil || length != len(test.encoding) || len(instruction.Args) != 3 || instruction.Args[0].Kind != OpMem { + t.Fatalf("encoding %x: instruction=%#v ok=%v length=%d err=%v", test.encoding, instruction, ok, length, err) + } + }) + } +} + +func TestX86RawANDNRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + }{ + {name: "vex l", mode: 64, encoding: []byte{0xc4, 0xe2, 0x34, 0xf2, 0xd2}}, + {name: "vex pp", mode: 64, encoding: []byte{0xc4, 0xe2, 0x31, 0xf2, 0xd2}}, + {name: "missing modrm", mode: 64, encoding: []byte{0xc4, 0xe2, 0x30, 0xf2}}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe2, 0x30, 0xf2, 0xd2}}, + {name: "rip relative", mode: 64, encoding: []byte{0xc4, 0xe2, 0x30, 0xf2, 0x15, 0, 0, 0, 0}}, + {name: "extended first 386", mode: 32, encoding: []byte{0xc4, 0xc2, 0x70, 0xf2, 0xc0}}, + {name: "extended second 386", mode: 32, encoding: []byte{0xc4, 0xe2, 0x38, 0xf2, 0xc0}}, + {name: "extended destination 386", mode: 32, encoding: []byte{0xc4, 0x62, 0x70, 0xf2, 0xc0}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86ANDNInstruction(test.encoding, test.mode) + if !ok || err == nil { + t.Fatalf("encoding %x: ok=%v err=%v, want recognized rejection", test.encoding, ok, err) + } + }) + } +} + +func TestX86ANDNRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86ANDNOriginalRuntimeSource + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "andnsemantics": {Name: "andnsemantics", Args: []LLVMType{Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := x86ANDNOriginalRuntimeDriver + compileAndRunRuntimeTestForTarget(t, llc, clang, "andn_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_approximate_math.go b/amd64_lower_approximate_math.go new file mode 100644 index 000000000..a469dc0df --- /dev/null +++ b/amd64_lower_approximate_math.go @@ -0,0 +1,54 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ApproximateMathMode uint8 + +const ( + amd64ApproximateExp2 amd64ApproximateMathMode = iota + amd64ApproximateReciprocal28 + amd64ApproximateReciprocalSqrt28 +) + +type amd64ApproximateMathSpec struct { + laneBits int + scalar bool + mode amd64ApproximateMathMode +} + +// amd64ApproximateMathSpecs models the complete AVX512ER family exposed by +// Go 1.27's _yvexp2pd and _yvgetexpsd tables. The type axes describe the +// mathematical operation, lane type, and packed/scalar operand shape; the +// parser below derives every legal mask, zero, broadcast, and SAE form. +var amd64ApproximateMathSpecs = map[Op]amd64ApproximateMathSpec{ + "VEXP2PS": {laneBits: 32, mode: amd64ApproximateExp2}, + "VEXP2PD": {laneBits: 64, mode: amd64ApproximateExp2}, + "VRCP28PS": {laneBits: 32, mode: amd64ApproximateReciprocal28}, + "VRCP28PD": {laneBits: 64, mode: amd64ApproximateReciprocal28}, + "VRCP28SS": {laneBits: 32, scalar: true, mode: amd64ApproximateReciprocal28}, + "VRCP28SD": {laneBits: 64, scalar: true, mode: amd64ApproximateReciprocal28}, + "VRSQRT28PS": {laneBits: 32, mode: amd64ApproximateReciprocalSqrt28}, + "VRSQRT28PD": {laneBits: 64, mode: amd64ApproximateReciprocalSqrt28}, + "VRSQRT28SS": {laneBits: 32, scalar: true, mode: amd64ApproximateReciprocalSqrt28}, + "VRSQRT28SD": {laneBits: 64, scalar: true, mode: amd64ApproximateReciprocalSqrt28}, +} + +// All ten EVEX approximation members share the existing typed native helper. +// The original instruction owns SAE/MXCSR, masking and the actual memory access; +// source data is never eagerly loaded before a masked native instruction. +func (c *amd64Ctx) lowerApproximateMath(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64ApproximateMathSpecs[Op(base)]; !known { + return false, false, nil + } + return c.lowerNativeFloatingHelper(op, ins) +} + +func (c *amd64Ctx) approximateMathMask32(mask string) string { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", truncated, mask) + return "%" + truncated +} diff --git a/amd64_lower_approximate_math_test.go b/amd64_lower_approximate_math_test.go new file mode 100644 index 000000000..b395d9426 --- /dev/null +++ b/amd64_lower_approximate_math_test.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +type approximateMathTestSpec struct { + op string + laneBits int + scalar bool + mode amd64ApproximateMathMode +} + +func go127ApproximateMathSpecs() []approximateMathTestSpec { + return []approximateMathTestSpec{ + {op: "VEXP2PS", laneBits: 32, mode: amd64ApproximateExp2}, + {op: "VEXP2PD", laneBits: 64, mode: amd64ApproximateExp2}, + {op: "VRCP28PS", laneBits: 32, mode: amd64ApproximateReciprocal28}, + {op: "VRCP28PD", laneBits: 64, mode: amd64ApproximateReciprocal28}, + {op: "VRCP28SS", laneBits: 32, scalar: true, mode: amd64ApproximateReciprocal28}, + {op: "VRCP28SD", laneBits: 64, scalar: true, mode: amd64ApproximateReciprocal28}, + {op: "VRSQRT28PS", laneBits: 32, mode: amd64ApproximateReciprocalSqrt28}, + {op: "VRSQRT28PD", laneBits: 64, mode: amd64ApproximateReciprocalSqrt28}, + {op: "VRSQRT28SS", laneBits: 32, scalar: true, mode: amd64ApproximateReciprocalSqrt28}, + {op: "VRSQRT28SD", laneBits: 64, scalar: true, mode: amd64ApproximateReciprocalSqrt28}, + } +} + +func TestAMD64ApproximateMathGrammarCoversCompleteGoFamily(t *testing.T) { + expected := go127ApproximateMathSpecs() + if len(amd64ApproximateMathSpecs) != len(expected) { + t.Fatalf("approximate-math grammar has %d entries, want %d", len(amd64ApproximateMathSpecs), len(expected)) + } + for _, want := range expected { + got, ok := amd64ApproximateMathSpecs[Op(want.op)] + if !ok { + t.Errorf("approximate-math grammar omitted %s", want.op) + continue + } + if got.laneBits != want.laneBits || got.scalar != want.scalar || got.mode != want.mode { + t.Errorf("approximate-math grammar %s = %+v, want laneBits=%d scalar=%v mode=%d", want.op, got, want.laneBits, want.scalar, want.mode) + } + } +} + +func TestTranslateX86ApproximateMathCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + vectorDestination := 23 + if target.goarch == "386" { + vectorDestination = 7 + } + var source strings.Builder + source.WriteString("TEXT approximatemathforms(SB),$0-0\n") + for index, spec := range go127ApproximateMathSpecs() { + if spec.scalar { + fmt.Fprintf(&source, "\t%s X1, X2, X23\n", spec.op) + fmt.Fprintf(&source, "\t%s %d(AX), X2, X23\n", spec.op, index*8) + fmt.Fprintf(&source, "\t%s.SAE X1, X2, X23\n", spec.op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X2, K1, X23\n", spec.op) + fmt.Fprintf(&source, "\t%s.Z %d(AX), X2, K2, X23\n", spec.op, index*8) + fmt.Fprintf(&source, "\t%s.SAE.Z X1, X2, K3, X23\n", spec.op) + } + continue + } + fmt.Fprintf(&source, "\t%s Z1, Z%d\n", spec.op, vectorDestination) + fmt.Fprintf(&source, "\t%s %d(AX), Z%d\n", spec.op, index*8, vectorDestination) + fmt.Fprintf(&source, "\t%s Z1, K1, Z%d\n", spec.op, vectorDestination) + fmt.Fprintf(&source, "\t%s.Z %d(AX), K2, Z%d\n", spec.op, index*8, vectorDestination) + fmt.Fprintf(&source, "\t%s.BCST %d(AX), Z%d\n", spec.op, index*8, vectorDestination) + fmt.Fprintf(&source, "\t%s.BCST.Z %d(AX), K3, Z%d\n", spec.op, index*8, vectorDestination) + fmt.Fprintf(&source, "\t%s.SAE Z1, Z%d\n", spec.op, vectorDestination) + fmt.Fprintf(&source, "\t%s.SAE.Z Z1, K4, Z%d\n", spec.op, vectorDestination) + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + // Keep the complete original uninitialized source and its actual + // Go acceptance. Neither the grammar nor a private register slot + // supplies the missing vector, mask or native address inputs. + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func approximatemathforms()", + target.goarch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original unbound approximation formats invented their inputs", api, err) + } + } + counterpart := source.String() + if target.goarch == "386" { + // Go's original high-X syntax above remains a separate source + // observation. The initialized physical counterpart uses X7. + counterpart = strings.ReplaceAll(counterpart, "X23", "X7") + } + initialized, declaration, count := x86Reciprocal14InitializeOriginalShapes(t, counterpart, target.goarch) + want := 72 + if target.goarch == "386" { + want = 60 + } + if count != want { + t.Fatal("original approximation shape counterpart omitted an instruction", count, want) + } + x86LEAActualTargetGoObject(t, target.goarch, target.triple, initialized) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, initialized, declaration, target.goarch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("complete initialized original approximation shapes rejected", api, err) + } + for _, mnemonic := range []string{"vexp2ps", "vexp2pd", "vrcp28ps", "vrcp28pd", "vrcp28ss", "vrcp28sd", "vrsqrt28ps", "vrsqrt28pd", "vrsqrt28ss", "vrsqrt28sd"} { + if !strings.Contains(ir, mnemonic) { + t.Fatalf("translation omitted inline %s:\n%s", mnemonic, ir) + } + } + for _, fragment := range []string{"kmovw", "{%k1}", "{sae}", `"target-features"="+avx512f"`} { + if !strings.Contains(ir, fragment) { + t.Fatalf("translation omitted %q:\n%s", fragment, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "approximate-math-"+target.name+"-"+api+".ll", "approximate-math-"+target.name+"-"+api+".o", ir) + }) + } + }) + } +} + +func TestTranslateX86ApproximateMathRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VEXP2PS X0, X1"}, + {goarch: "amd64", instruction: "VEXP2PD Y0, Y1"}, + {goarch: "amd64", instruction: "VRCP28PS Z0, K0, Z1"}, + {goarch: "amd64", instruction: "VRSQRT28PD.Z Z0, Z1"}, + {goarch: "amd64", instruction: "VEXP2PS.BCST Z0, Z1"}, + {goarch: "amd64", instruction: "VRCP28PD.SAE 0(AX), Z1"}, + {goarch: "amd64", instruction: "VRCP28SS X0, X1"}, + {goarch: "amd64", instruction: "VRSQRT28SD Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRCP28SS.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VRSQRT28SD.SAE 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VEXP2PS.RN_SAE Z0, Z1"}, + {goarch: "386", instruction: "VEXP2PS Z0, Z8"}, + {goarch: "386", instruction: "VRCP28SS X0, X1, K1, X2"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's approximate-math tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_arith.go b/amd64_lower_arith.go index 2a25025ce..06f5511c0 100644 --- a/amd64_lower_arith.go +++ b/amd64_lower_arith.go @@ -24,6 +24,17 @@ func (c *amd64Ctx) loadIntDestination(dst Operand, ty LLVMType) (string, func(st fmt.Fprintf(c.b, " store %s %s, %s %s, align 1\n", ty, out, ptrType, p) return nil }, nil + case OpFP: + value, err := c.evalFPToI64(dst.FPOffset) + if err != nil { + return "", nil, err + } + if ty != I64 { + value = c.truncI64(value, ty) + } + return value, func(out string) error { + return c.storeFPResult(dst.FPOffset, ty, out) + }, nil case OpSym: p, err := c.ptrFromSB(dst.Sym) if err != nil { @@ -45,22 +56,65 @@ type x86FlagSlot struct { bit uint } -func (c *amd64Ctx) x86FlagSlots() []x86FlagSlot { - flags := []x86FlagSlot{ +type x86StatusFlagMask uint16 + +const ( + x86StatusCF x86StatusFlagMask = 1 << 0 + x86StatusPF x86StatusFlagMask = 1 << 2 + x86StatusAF x86StatusFlagMask = 1 << 4 + x86StatusZF x86StatusFlagMask = 1 << 6 + x86StatusSF x86StatusFlagMask = 1 << 7 + x86StatusOF x86StatusFlagMask = 1 << 11 + x86StatusAll = x86StatusCF | x86StatusPF | x86StatusAF | x86StatusZF | x86StatusSF | x86StatusOF +) + +func (c *amd64Ctx) x86StatusFlagSlots() []x86FlagSlot { + return []x86FlagSlot{ {c.flagsCFSlot, 0}, + {c.flagsPFSlot, 2}, + {c.flagsAFSlot, 4}, {c.flagsZSlot, 6}, {c.flagsSltSlot, 7}, + {c.flagsOFSlot, 11}, } +} + +func (c *amd64Ctx) x86FlagSlots() []x86FlagSlot { + flags := c.x86StatusFlagSlots() if c.directionSlot != "" { flags = append(flags, x86FlagSlot{c.directionSlot, 10}) } - flags = append(flags, x86FlagSlot{c.flagsOFSlot, 11}) if c.flagsIDSlot != "" { flags = append(flags, x86FlagSlot{c.flagsIDSlot, 21}) } return flags } +// Clear exactly the ISA-defined status bits. Reusing the complete slot model +// prevents a consumer's hand-written subset from silently omitting AF. +func (c *amd64Ctx) clearX86StatusFlags(mask x86StatusFlagMask) { + for _, flag := range c.x86StatusFlagSlots() { + if mask&(1<= uint(operandBits) { + continue + } shifted := packed if flag.bit != 0 { shift := c.newTmp() @@ -100,68 +159,118 @@ func (c *amd64Ctx) truncI64(v string, ty LLVMType) string { return "%" + t } -func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err error) { - switch op { - case "PUSHL": - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64 PUSHL requires GOARCH=386") - } - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 PUSHL expects src: %q", ins.Raw) - } - v32, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - return true, false, c.pushI32(v32) - case "POPL": - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64 POPL requires GOARCH=386") - } - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 POPL expects dst: %q", ins.Raw) - } - v32, err := c.popI32() - if err != nil { - return true, false, err - } - switch ins.Args[0].Kind { - case OpReg: - return true, false, c.storeRegSized(ins.Args[0].Reg, I32, v32) - case OpMem: - p, ptrType, err := c.ptrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i32 %s, %s %s, align 1\n", v32, ptrType, p) - return true, false, nil +func isGoYmbRegisterForArch(r Reg, goarch string) bool { + if goarch == "386" { + // Go classifies SP/BP/SI/DI as Yrl32 in 386 mode, so they do not + // directly cover Ymb. Its assembler nevertheless accepts BP/SI/DI + // and synthesizes an equivalent sequence through BX; only a direct + // SP spelling remains invalid. + switch r { + case AX, BX, CX, DX, BP, SI, DI, AL, AH, BL, BH, CL, CH, DL, DH: + return true default: - return true, false, fmt.Errorf("amd64 POPL expects reg/mem dst: %q", ins.Raw) - } - case "PUSHFL": - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64 PUSHFL requires GOARCH=386") + return false } - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("amd64 PUSHFL takes no operands: %q", ins.Raw) - } - flags := c.truncI64(c.packX86Flags(), I32) - return true, false, c.pushI32(flags) - case "POPFL": - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64 POPFL requires GOARCH=386") - } - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("amd64 POPFL takes no operands: %q", ins.Raw) - } - flags32, err := c.popI32() - if err != nil { - return true, false, err + } + if isX86YrlRegisterForArch(r, goarch) { + return true + } + switch r { + case AL, AH, BL, BH, CL, CH, DL, DH, + BPB, SIB, DIB, R8B, R9B, R10B, R11B, R12B, R13B, R14B, R15B: + return true + default: + return false + } +} + +// amd64YmbEffectiveRegister mirrors the raw ModRM numbering used when Go's +// broad Ymb table accepts a byte-register spelling for a wider NEG/NOT opcode. +func amd64YmbEffectiveRegister(r Reg, bits int) Reg { + if bits == 8 { + return r + } + switch r { + case AL: + return AX + case CL: + return CX + case DL: + return DX + case BL: + return BX + case AH: + return SP + case CH: + return BP + case DH: + return SI + case BH: + return DI + default: + if base, _, ok := amd64ByteAlias(r); ok { + return base } - flags64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", flags64, flags32) - c.unpackX86Flags("%" + flags64) - return true, false, nil + return r + } +} + +func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err error) { + if known, terminated, err := c.lowerByteSwap(op, ins); known { + return true, terminated, err + } + if known, terminated, err := c.lowerScalarBitScan(op, ins); known { + return true, terminated, err + } + if ok, terminated, err := c.lowerX86CPUQuery(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerX86OrderingHint(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerStackWidth(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerAccumulatorSignExtend(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerBZHI(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerBEXTR(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerADX(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerMULX(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerANDN(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerScalarBitCount(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerBMI2ShiftRotate(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerScalarUnaryYmb(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerLowestSetBitManipulation(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerHardwareRandom(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerScalarADCSBB(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerScalarMultiplyDivide(op, ins); ok { + return ok, terminated, err + } + switch op { case "PUSHAL": if c.goarch != "386" { return true, false, fmt.Errorf("amd64 PUSHAL requires GOARCH=386") @@ -223,140 +332,35 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e } } return true, false, nil - case "PUSHQ": - // Stack-manipulation appears in syscall asm stubs (e.g. preserve return - // address register around SYSCALL). Lower to the local virtual stack. - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 PUSHQ expects src: %q", ins.Raw) + case "PREFETCHNTA", "PREFETCHT0", "PREFETCHT1", "PREFETCHT2", + "PREFETCHRST2", "PREFETCHIT0", "PREFETCHIT1": + // Go 1.27's shared yprefetch table has exactly one memory operand for + // each named locality hint. The raw-only 0F 18 /4, /6 and /7 forms + // share that one-memory-operand grammar. Cache hints have no + // SSA-visible result. + if len(ins.Args) != 1 || !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s expects one memory operand: %q", c.goarch, op, ins.Raw) } - v, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - c.pushI64(v) - return true, false, nil - case "POPQ": - if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 POPQ expects dstReg: %q", ins.Raw) - } - v := c.popI64() - if err := c.storeReg(ins.Args[0].Reg, v); err != nil { - return true, false, err - } - return true, false, nil - case "PUSHFQ": - // Keep the historical amd64 stack-shape model. The 32-bit PUSHFL path - // above carries flags because 386 runtime asm consumes them. - c.pushI64("0") - return true, false, nil - case "POPFQ": - _ = c.popI64() - return true, false, nil - case "LFENCE", "MFENCE", "SFENCE", "PAUSE": - if c.goarch == "386" { - fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", strings.ToLower(string(op)), "~{memory}") - } - return true, false, nil - case "PREFETCHNTA", "EMMS": - // Prefetch does not change SSA-visible values. MMX registers are modeled - // as ordinary integers, so there is no hardware x87 tag state to clear. return true, false, nil + case "EMMS": + // A closed native MMX region owns its actual EMMS before generic + // dispatch. Never turn an unbound physical tag-state operation into + // a successful logical-register no-op. + return true, false, fmt.Errorf("%w: EMMS needs its source-bound native MMX/x87 continuation", ErrProbeNeedsContext) case "UNDEF": - if c.goarch == "386" { - c.emitX86Trap() - return true, true, nil - } - // Preserve the historical amd64 approximation. - return true, false, nil - case "RDTSC": - if c.goarch == "386" { - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("386 RDTSC takes no operands: %q", ins.Raw) - } - return true, false, c.lower386Timestamp(false) - } - if err := c.storeReg(AX, "0"); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "0"); err != nil { - return true, false, err - } - return true, false, nil - case OpCPUID: - eax64, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - ecx64, err := c.loadReg(CX) - if err != nil { - return true, false, err - } - eax32 := c.newTmp() - ecx32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", eax32, eax64) - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", ecx32, ecx64) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call { i32, i32, i32, i32 } asm sideeffect \"cpuid\", \"={ax},={bx},={cx},={dx},{ax},{cx},~{dirflag},~{fpsr},~{flags}\"(i32 %%%s, i32 %%%s)\n", call, eax32, ecx32) - storeOut := func(idx int, reg Reg) error { - part := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue { i32, i32, i32, i32 } %%%s, %d\n", part, call, idx) - wide := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, part) - return c.storeReg(reg, "%"+wide) - } - if err := storeOut(0, AX); err != nil { - return true, false, err - } - if err := storeOut(1, BX); err != nil { - return true, false, err - } - if err := storeOut(2, CX); err != nil { - return true, false, err - } - if err := storeOut(3, DX); err != nil { - return true, false, err - } - return true, false, nil - case OpXGETBV: - ecx64, err := c.loadReg(CX) - if err != nil { - return true, false, err - } - ecx32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", ecx32, ecx64) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call { i32, i32 } asm sideeffect \"xgetbv\", \"={ax},={dx},{cx},~{dirflag},~{fpsr},~{flags}\"(i32 %%%s)\n", call, ecx32) - storeOut := func(idx int, reg Reg) error { - part := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue { i32, i32 } %%%s, %d\n", part, call, idx) - wide := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, part) - return c.storeReg(reg, "%"+wide) - } - if err := storeOut(0, AX); err != nil { - return true, false, err - } - if err := storeOut(1, DX); err != nil { - return true, false, err - } - return true, false, nil - case "RDTSCP": - if c.goarch == "386" { - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("386 RDTSCP takes no operands: %q", ins.Raw) - } - return true, false, c.lower386Timestamp(true) - } - if err := c.storeReg(AX, "0"); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "0"); err != nil { - return true, false, err + return c.lowerX86UndefinedTrap(op, ins) + case "RDTSC", "RDTSCP": + return c.lowerX86Timestamp(op, ins) + case "RDPID": + if strings.ToUpper(string(ins.Op)) != string(op) || len(ins.Args) != 1 || ins.Args[0].Kind != OpReg || !isAMD64YrlRegister(ins.Args[0].Reg) { + return true, false, fmt.Errorf("%s RDPID expects one general register with no suffix: %q", c.goarch, ins.Raw) } - if err := c.storeReg(CX, "0"); err != nil { - return true, false, err + if c.goarch == "386" && amd64IsExtendedGPRegister(ins.Args[0].Reg) { + return true, false, fmt.Errorf("386 RDPID destination is outside the 386 register file: %q", ins.Raw) } - return true, false, nil + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.rdpid()\n", value) + return true, false, c.storeRegSized(ins.Args[0].Reg, I32, "%"+value) case "MOVSB": si, err := c.loadReg(SI) if err != nil { @@ -424,101 +428,25 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e return true, false, err } return true, false, nil - case "NEGL": - // 32-bit negate with x86 semantics: write back low 32 bits and zero-extend. - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 NEGL expects dst: %q", ins.Raw) - } - switch ins.Args[0].Kind { - case OpReg: - dv, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - t32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", t32, dv) - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 0, %%%s\n", neg, t32) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", cf, t32) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, neg) - if err := c.storeReg(ins.Args[0].Reg, "%"+z); err != nil { - return true, false, err - } - c.setZSFlagsFromI32("%" + neg) - return true, false, nil - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", ld, p) - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 0, %%%s\n", neg, ld) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", cf, ld) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - fmt.Fprintf(c.b, " store i32 %%%s, ptr %s, align 1\n", neg, p) - c.setZSFlagsFromI32("%" + neg) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 NEGL expects reg/mem dst: %q", ins.Raw) - } - case "RCRQ": - // Rotate through carry right (count=1) used by runtime time division path. - if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm { - return true, false, fmt.Errorf("amd64 RCRQ expects $count, dstReg: %q", ins.Raw) - } - if ins.Args[0].Imm != 1 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 RCRQ currently supports $1, reg: %q", ins.Raw) - } - dv, err := c.loadReg(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - oldCF := c.loadFlag(c.flagsCFSlot) - lsb := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 1\n", lsb, dv) - newCF := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", newCF, lsb) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", newCF, c.flagsCFSlot) - shr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, 1\n", shr, dv) - cf64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i1 %s to i64\n", cf64, oldCF) - cfhi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, 63\n", cfhi, cf64) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", out, shr, cfhi) - if err := c.storeReg(ins.Args[1].Reg, "%"+out); err != nil { - return true, false, err - } - c.setZSFlagsFromI64("%" + out) - return true, false, nil - case "ADDQ", "SUBQ", "XORQ", "ANDQ", "ORQ": if len(ins.Args) != 2 { return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) } - src, err := c.evalI64(ins.Args[0]) + src, destination, err := c.evalScalarArithmeticSource(ins, I64) if err != nil { return true, false, err } loadDst := func() (string, func(string) error, error) { - switch ins.Args[1].Kind { + switch destination.Kind { case OpReg: - dst := ins.Args[1].Reg + dst := destination.Reg dv, err := c.loadReg(dst) if err != nil { return "", nil, err } return dv, func(v string) error { return c.storeReg(dst, v) }, nil case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) + addr, err := c.addrFromMem(destination.Mem) if err != nil { return "", nil, err } @@ -566,244 +494,25 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e default: fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) } - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) - c.setZSFlagsFromI64(r) - return true, false, nil - - case "ADCL", "SBBL": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) - } - src, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - dst, storeDst, err := c.loadIntDestination(ins.Args[1], I32) - if err != nil { - return true, false, err - } - carry := c.loadFlag(c.flagsCFSlot) - carry64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i1 %s to i64\n", carry64, carry) - src64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", src64, src) - dst64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", dst64, dst) - result64 := c.newTmp() - if op == "ADCL" { - partial := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", partial, dst64, src64) - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", result64, partial, carry64) - } else { - partial := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", partial, src64, carry64) - fmt.Fprintf(c.b, " %%%s = sub i64 %%%s, %%%s\n", result64, dst64, partial) - } - result := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", result, result64) - if err := storeDst("%" + result); err != nil { - return true, false, err - } - newCarry := c.newTmp() - if op == "ADCL" { - fmt.Fprintf(c.b, " %%%s = icmp ugt i64 %%%s, 4294967295\n", newCarry, result64) + if op == "ADDQ" || op == "SUBQ" { + xorOperands := c.newTmp() + xorResult := c.newTmp() + overflowBits := c.newTmp() + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", xorOperands, dv, src) + if op == "ADDQ" { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %%%s, -1\n", inverted, xorOperands) + xorOperands = inverted + } + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", xorResult, dv, r) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %%%s\n", overflowBits, xorOperands, xorResult) + fmt.Fprintf(c.b, " %%%s = icmp slt i64 %%%s, 0\n", overflow, overflowBits) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) } else { - subtrahend := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", subtrahend, src64, carry64) - fmt.Fprintf(c.b, " %%%s = icmp ult i64 %%%s, %%%s\n", newCarry, dst64, subtrahend) - } - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", newCarry, c.flagsCFSlot) - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) - c.setZSFlagsFromI32("%" + result) - return true, false, nil - - case "ADCQ", "SBBQ": - // Add/subtract with carry/borrow: src, dstReg. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - dst := ins.Args[1].Reg - dv, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - cfIn := c.loadFlag(c.flagsCFSlot) - cf64t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i1 %s to i64\n", cf64t, cfIn) - cf64 := "%" + cf64t - - dv128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", dv128, dv) - src128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", src128, src) - cf128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", cf128, cf64) - - if op == "ADCQ" { - sum := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %s, %s\n", sum, dv, src) - res := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %s\n", res, sum, cf64) - out := "%" + res - if err := c.storeReg(dst, out); err != nil { - return true, false, err - } - - total1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i128 %%%s, %%%s\n", total1, dv128, src128) - total2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i128 %%%s, %%%s\n", total2, total1, cf128) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ugt i128 %%%s, 18446744073709551615\n", cf, total2) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) - c.setZSFlagsFromI64(out) - return true, false, nil - } - - subtr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i128 %%%s, %%%s\n", subtr, src128, cf128) - borrow := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i128 %%%s, %%%s\n", borrow, dv128, subtr) - res := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 %s, %s\n", res, dv, src) - res2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 %%%s, %s\n", res2, res, cf64) - out := "%" + res2 - if err := c.storeReg(dst, out); err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", borrow, c.flagsCFSlot) - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) - c.setZSFlagsFromI64(out) - return true, false, nil - - case "ADCB": - // 8-bit add with carry: src, dstReg/mem. - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) - } - var d8 string - var storeDst func(string) error - switch ins.Args[1].Kind { - case OpReg: - dst := ins.Args[1].Reg - dv64, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", t, dv64) - d8 = "%" + t - storeDst = func(v8 string) error { - return c.storeRegSized(dst, I8, v8) - } - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i8, ptr %s, align 1\n", ld, p) - d8 = "%" + ld - storeDst = func(v8 string) error { - fmt.Fprintf(c.b, " store i8 %s, ptr %s, align 1\n", v8, p) - return nil - } - default: - return true, false, fmt.Errorf("amd64 %s expects reg/mem dst: %q", op, ins.Raw) - } - - s8, err := c.evalIntSized(ins.Args[0], I8) - if err != nil { - return true, false, err - } - cfIn := c.loadFlag(c.flagsCFSlot) - cf8t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i1 %s to i8\n", cf8t, cfIn) - cf8 := "%" + cf8t - - sum := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i8 %s, %s\n", sum, d8, s8) - res := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i8 %%%s, %s\n", res, sum, cf8) - out8 := "%" + res - if err := storeDst(out8); err != nil { - return true, false, err - } - - d16 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i8 %s to i16\n", d16, d8) - s16 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i8 %s to i16\n", s16, s8) - cf16 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i8 %s to i16\n", cf16, cf8) - total1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i16 %%%s, %%%s\n", total1, d16, s16) - total2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i16 %%%s, %%%s\n", total2, total1, cf16) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ugt i16 %%%s, 255\n", cf, total2) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) - zf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i8 %%%s, 0\n", zf, res) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zf, c.flagsZSlot) - sf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp slt i8 %%%s, 0\n", sf, res) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sf, c.flagsSltSlot) - return true, false, nil - - case "ADCXQ", "ADOXQ": - // BMI2 add-with-carry chains. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - dst := ins.Args[1].Reg - dv, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - carryIn := c.loadFlag(c.flagsCFSlot) - flagOut := c.flagsCFSlot - if op == "ADOXQ" { - carryIn = c.loadFlag(c.flagsOFSlot) - flagOut = c.flagsOFSlot - } - cf64t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i1 %s to i64\n", cf64t, carryIn) - cf64 := "%" + cf64t - sum := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %s, %s\n", sum, dv, src) - res := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %s\n", res, sum, cf64) - out := "%" + res - if err := c.storeReg(dst, out); err != nil { - return true, false, err } - dv128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", dv128, dv) - src128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", src128, src) - cf128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", cf128, cf64) - total1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i128 %%%s, %%%s\n", total1, dv128, src128) - total2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i128 %%%s, %%%s\n", total2, total1, cf128) - carry := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ugt i128 %%%s, 18446744073709551615\n", carry, total2) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, flagOut) - // ADCX/ADOX do not define ZF/SF in the same way as ADD; keep current bits. + c.setZSFlagsFromI64(r) return true, false, nil case "ADDL", "SUBL", "XORL", "ANDL", "ORL": @@ -811,11 +520,11 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e if len(ins.Args) != 2 { return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) } - s32, err := c.evalIntSized(ins.Args[0], I32) + s32, destination, err := c.evalScalarArithmeticSource(ins, I32) if err != nil { return true, false, err } - dtr, storeDst, err := c.loadIntDestination(ins.Args[1], I32) + dtr, storeDst, err := c.loadIntDestination(destination, I32) if err != nil { return true, false, err } @@ -857,11 +566,11 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e if len(ins.Args) != 2 { return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) } - d8, storeDst, err := c.loadIntDestination(ins.Args[1], I8) + s8, destination, err := c.evalScalarArithmeticSource(ins, I8) if err != nil { return true, false, err } - s8, err := c.evalIntSized(ins.Args[0], I8) + d8, storeDst, err := c.loadIntDestination(destination, I8) if err != nil { return true, false, err } @@ -923,17 +632,18 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e sf := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp slt i8 %%%s, 0\n", sf, x) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sf, c.flagsSltSlot) + c.setParityFlagSized(I8, "%"+x) return true, false, nil case "ANDW", "ORW", "XORW", "ADDW", "SUBW": if len(ins.Args) != 2 { return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) } - src, err := c.evalIntSized(ins.Args[0], I16) + src, destination, err := c.evalScalarArithmeticSource(ins, I16) if err != nil { return true, false, err } - dst, storeDst, err := c.loadIntDestination(ins.Args[1], I16) + dst, storeDst, err := c.loadIntDestination(destination, I16) if err != nil { return true, false, err } @@ -977,120 +687,37 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) } fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + c.setParityFlagSized(I16, "%"+out) return true, false, nil - case "INCQ", "DECQ": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 %s expects dst: %q", op, ins.Raw) + case "LEAW", "LEAL", "LEAQ": + // All three widths use Go 1.27's sole ym_rl row: an addressable + // memory expression and an in-range general-register destination. + if ins.Op != "" && Op(strings.ToUpper(string(ins.Op))) != op { + return true, false, fmt.Errorf("%s LEA dispatch %q disagrees with instruction %q", c.goarch, op, ins.Op) } - var v string - var storeDst func(string) error - switch ins.Args[0].Kind { - case OpReg: - r := ins.Args[0].Reg - dv, err := c.loadReg(r) - if err != nil { - return true, false, err - } - v = dv - storeDst = func(out string) error { return c.storeReg(r, out) } - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - v = "%" + ld - storeDst = func(out string) error { - fmt.Fprintf(c.b, " store i64 %s, ptr %s, align 1\n", out, p) - return nil - } - default: - return true, false, fmt.Errorf("amd64 %s expects reg/mem dst: %q", op, ins.Raw) - } - t := c.newTmp() - if op == "INCQ" { - fmt.Fprintf(c.b, " %%%s = add i64 %s, 1\n", t, v) - } else { - fmt.Fprintf(c.b, " %%%s = sub i64 %s, 1\n", t, v) - } - out := "%" + t - if err := storeDst(out); err != nil { - return true, false, err - } - c.setZSFlagsFromI64(out) - return true, false, nil - - case "INCL", "DECL": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 %s expects dst: %q", op, ins.Raw) - } - var v64 string - var storeDst func(string) error - switch ins.Args[0].Kind { - case OpReg: - r := ins.Args[0].Reg - dv, err := c.loadReg(r) - if err != nil { - return true, false, err - } - v64 = dv - storeDst = func(out32 string) error { - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", z, out32) - return c.storeReg(r, "%"+z) - } - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", ld, p) - v64 = "%" + ld - storeDst = func(out32 string) error { - fmt.Fprintf(c.b, " store i32 %s, ptr %s, align 1\n", out32, p) - return nil - } - default: - return true, false, fmt.Errorf("amd64 %s expects reg/mem dst: %q", op, ins.Raw) - } - tr := c.newTmp() - if ins.Args[0].Kind == OpMem { - fmt.Fprintf(c.b, " %%%s = add i32 0, %s\n", tr, v64) - } else { - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) - } - x := c.newTmp() - if op == "INCL" { - fmt.Fprintf(c.b, " %%%s = add i32 %%%s, 1\n", x, tr) - } else { - fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, 1\n", x, tr) - } - if err := storeDst("%" + x); err != nil { - return true, false, err - } - c.setZSFlagsFromI32("%" + x) - return true, false, nil - - case "LEAQ", "LEAL": - // LEA{Q,L} srcAddr, dstReg - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects srcAddr, dstReg: %q", op, ins.Raw) + // Internal helpers may omit Instr.Op. Bind only this local copy so + // grammar validation and destination emission use one identity; the + // original source/caller instruction remains independently required. + ins.Op = op + ins = x86LEASourceInstruction(ins) + if _, err := x86LEAOperandWidth(ins, c.goarch); err != nil { + return true, false, err } dst := ins.Args[1].Reg storeLEA := func(addr string) error { - if op == "LEAL" { + switch op { + case "LEAW": + t := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i16\n", t, addr) + return c.storeRegSized(dst, I16, "%"+t) + case "LEAL": t := c.newTmp() fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", t, addr) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, t) - return c.storeReg(dst, "%"+z) + return c.storeRegSized(dst, I32, "%"+t) + default: + return c.storeRegSized(dst, I64, addr) } - return c.storeReg(dst, addr) } switch ins.Args[0].Kind { case OpMem: @@ -1098,46 +725,24 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e if err != nil { return true, false, err } + if bits := ins.x86AddressBits; bits != 0 { + // Address size is independent of the result register width. + // Narrow the complete sum first, then zero-extend that effective + // offset before W/L/Q destination preservation/extension. + typ := I32 + if bits == 16 { + typ = I16 + } + low := c.truncI64(addr, typ) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, typ, low) + addr = "%" + wide + } return true, false, storeLEA(addr) case OpFP: - if c.classicFrame != "" { - c.markFPResultAddrTaken(ins.Args[0].FPOffset) - addr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", addr, c.classicFramePtr(ins.Args[0].FPOffset)) - return true, false, storeLEA("%" + addr) - } - // LEA of a return slot, e.g. "LEAQ ret+32(FP), R8". - alloca, _, ok := c.fpResultAlloca(ins.Args[0].FPOffset) - if ok { - c.markFPResultAddrTaken(ins.Args[0].FPOffset) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", t, alloca) - return true, false, storeLEA("%" + t) - } - // Fallback: treat FP slot value as pointer-like integer address. - v, err := c.evalFPToI64(ins.Args[0].FPOffset) + v, err := c.boundFPAddress(ins.Args[0].FPOffset) if err != nil { - v = "0" - } - return true, false, storeLEA(v) - case OpFPAddr: - if c.classicFrame != "" { - c.markFPResultAddrTaken(ins.Args[0].FPOffset) - addr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", addr, c.classicFramePtr(ins.Args[0].FPOffset)) - return true, false, storeLEA("%" + addr) - } - // Address of a return slot alloca. - alloca, _, ok := c.fpResultAlloca(ins.Args[0].FPOffset) - if ok { - c.markFPResultAddrTaken(ins.Args[0].FPOffset) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", t, alloca) - return true, false, storeLEA("%" + t) - } - v, err := c.evalFPToI64(ins.Args[0].FPOffset) - if err != nil { - v = "0" + return true, false, err } return true, false, storeLEA(v) case OpSym: @@ -1152,1312 +757,6 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e return true, false, fmt.Errorf("amd64 %s unsupported src: %q", op, ins.Raw) } - case "POPCNTL", "POPCNTQ": - // POPCNT{L,Q} src, dstReg accepts the same register-or-memory source - // family as the Go x86 assembler. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) - } - validFullReg := func(r Reg) bool { - if _, _, ok := amd64ByteAlias(r); ok || r == FS || r == GS { - return false - } - _, ok := amd64FullRegBase(r) - return ok - } - if !validFullReg(ins.Args[1].Reg) { - return false, false, nil - } - src := ins.Args[0] - switch src.Kind { - case OpReg: - if !validFullReg(src.Reg) { - return false, false, nil - } - case OpMem, OpFP: - case OpSym: - if strings.HasPrefix(strings.TrimSpace(src.Sym), "$") { - return false, false, nil - } - default: - return false, false, nil - } - if op == "POPCNTL" { - srcv, err := c.evalIntSized(src, I32) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.ctpop.i32(i32 %s)\n", call, srcv) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+z) - } - srcv, err := c.evalIntSized(src, I64) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @llvm.ctpop.i64(i64 %s)\n", call, srcv) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+call) - - case "TZCNTQ": - // TZCNTQ srcReg, dstReg. - if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 TZCNTQ expects srcReg, dstReg: %q", ins.Raw) - } - srcv, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @llvm.cttz.i64(i64 %s, i1 false)\n", call, srcv) - if err := c.storeReg(ins.Args[1].Reg, "%"+call); err != nil { - return true, false, err - } - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %s, 0\n", cf, srcv) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - c.setZSFlagsFromI64("%" + call) - return true, false, nil - - case "BSFW", "BSRW": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) - } - src, err := c.evalIntSized(ins.Args[0], I16) - if err != nil { - return true, false, err - } - zf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i16 %s, 0\n", zf, src) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zf, c.flagsZSlot) - ext := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i16 %s to i32\n", ext, src) - var result string - if op == "BSFW" { - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.cttz.i32(i32 %%%s, i1 false)\n", call, ext) - result = "%" + call - } else { - clz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.ctlz.i32(i32 %%%s, i1 false)\n", clz, ext) - sub := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 31, %%%s\n", sub, clz) - result = "%" + sub - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i32 %s to i16\n", tr, result) - return true, false, c.storeRegSized(ins.Args[1].Reg, I16, "%"+tr) - - case "BSFQ", "BSRQ", "BSWAPQ", "BSFL", "BSRL": - // Bit scans accept register or memory sources. BSWAPQ is register-only. - sv := "" - dst := Reg("") - switch len(ins.Args) { - case 1: - if op != "BSWAPQ" || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects reg or srcReg,dstReg: %q", op, ins.Raw) - } - dst = ins.Args[0].Reg - var err error - sv, err = c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - case 2: - if op == "BSWAPQ" || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) - } - dst = ins.Args[1].Reg - srcType := I64 - if op == "BSFL" || op == "BSRL" { - srcType = I32 - } - var err error - sv, err = c.evalIntSized(ins.Args[0], srcType) - if err != nil { - return true, false, err - } - default: - return true, false, fmt.Errorf("amd64 %s expects 1 or 2 operands: %q", op, ins.Raw) - } - switch op { - case "BSFQ": - // ZF is set when src == 0. - zf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %s, 0\n", zf, sv) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zf, c.flagsZSlot) - // dst = cttz(src). Use non-poison form for src==0. - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @llvm.cttz.i64(i64 %s, i1 false)\n", call, sv) - return true, false, c.storeReg(dst, "%"+call) - case "BSRQ": - // ZF is set when src == 0. - zf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %s, 0\n", zf, sv) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zf, c.flagsZSlot) - // dst = 63 - ctlz(src). Use non-poison form for src==0. - clz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @llvm.ctlz.i64(i64 %s, i1 false)\n", clz, sv) - sub := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 63, %%%s\n", sub, clz) - return true, false, c.storeReg(dst, "%"+sub) - case "BSWAPQ": - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @llvm.bswap.i64(i64 %s)\n", call, sv) - return true, false, c.storeReg(dst, "%"+call) - case "BSFL": - // ZF is set when the 32-bit source is zero. - zf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 0\n", zf, sv) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zf, c.flagsZSlot) - // dst = zext(cttz(src)). Use non-poison form for src==0. - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.cttz.i32(i32 %s, i1 false)\n", call, sv) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(dst, "%"+z) - case "BSRL": - // ZF is set when the 32-bit source is zero. - zf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 0\n", zf, sv) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zf, c.flagsZSlot) - // dst = zext(31 - ctlz(src)). Use non-poison form for src==0. - clz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.ctlz.i32(i32 %s, i1 false)\n", clz, sv) - sub := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 31, %%%s\n", sub, clz) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, sub) - return true, false, c.storeReg(dst, "%"+z) - } - return true, false, fmt.Errorf("amd64: unsupported bit op %s", op) - - case "SETEQ", "SETLT", "SETGT", "SETGE", "SETHI", "SETCS": - // SETcc dst: set byte based on flags. - // Register, memory, and FP result destinations all occur in Go's x86 - // assembly sources. - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 %s expects one destination: %q", op, ins.Raw) - } - cond := "" - switch op { - case "SETEQ": - cond = c.loadFlag(c.flagsZSlot) - case "SETLT": - cond = c.loadFlag(c.flagsSltSlot) - case "SETGT": - // signed > - slt := c.loadFlag(c.flagsSltSlot) - z := c.loadFlag(c.flagsZSlot) - t1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t1, slt, z) - t2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) - cond = "%" + t2 - case "SETGE": - slt := c.loadFlag(c.flagsSltSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, slt) - cond = "%" + t - case "SETHI": - // unsigned > - cf := c.loadFlag(c.flagsCFSlot) - z := c.loadFlag(c.flagsZSlot) - t1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t1, cf, z) - t2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) - cond = "%" + t2 - case "SETCS": - cond = c.loadFlag(c.flagsCFSlot) - } - switch ins.Args[0].Kind { - case OpReg: - sel := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %s, i8 1, i8 0\n", sel, cond) - return true, false, c.storeRegSized(ins.Args[0].Reg, I8, "%"+sel) - case OpFP: - return true, false, c.storeFPResult(ins.Args[0].FPOffset, I1, cond) - case OpMem: - p, ptrType, err := c.ptrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - sel := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %s, i8 1, i8 0\n", sel, cond) - fmt.Fprintf(c.b, " store i8 %%%s, %s %s, align 1\n", sel, ptrType, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 %s expects reg, mem, or fp destination: %q", op, ins.Raw) - } - - case "CMOVQEQ", "CMOVQNE", "CMOVQCS", "CMOVQCC", "CMOVQGT": - // Conditional move: src, dstReg - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - dst := ins.Args[1].Reg - cur, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - cond := "" - switch op { - case "CMOVQEQ": - cond = c.loadFlag(c.flagsZSlot) - case "CMOVQNE": - z := c.loadFlag(c.flagsZSlot) - nz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", nz, z) - cond = "%" + nz - case "CMOVQCS": - cond = c.loadFlag(c.flagsCFSlot) - case "CMOVQCC": - cf := c.loadFlag(c.flagsCFSlot) - nc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", nc, cf) - cond = "%" + nc - case "CMOVQGT": - slt := c.loadFlag(c.flagsSltSlot) - z := c.loadFlag(c.flagsZSlot) - t1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t1, slt, z) - t2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) - cond = "%" + t2 - } - sel := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %s, i64 %s, i64 %s\n", sel, cond, src, cur) - return true, false, c.storeReg(dst, "%"+sel) - - case "ANDNL", "ANDNQ": - // BMI1 ANDN: dst = ~src2 & src1 - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src1, src2, dstReg: %q", op, ins.Raw) - } - src1, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - src2, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - dst := ins.Args[2].Reg - if op == "ANDNQ" { - n := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %s, -1\n", n, src2) - a := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %s\n", a, n, src1) - return true, false, c.storeReg(dst, "%"+a) - } - s1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", s1, src1) - s2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", s2, src2) - n := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i32 %%%s, -1\n", n, s2) - a := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i32 %%%s, %%%s\n", a, n, s1) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, a) - return true, false, c.storeReg(dst, "%"+z) - - case "BEXTRQ": - // BMI1 bit field extract: control, src, dst. - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 BEXTRQ expects control, src, dstReg: %q", ins.Raw) - } - ctrl, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - src, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - start := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 255\n", start, ctrl) - lenShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, 8\n", lenShift, ctrl) - length := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 255\n", length, lenShift) - startOK := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i64 %%%s, 64\n", startOK, start) - safeStart := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 63\n", safeStart, startOK, start) - shifted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %%%s\n", shifted, src, safeStart) - rawLen := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 0\n", rawLen, startOK, length) - remain := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 64, %%%s\n", remain, safeStart) - useRawLen := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i64 %%%s, %%%s\n", useRawLen, rawLen, remain) - effLen := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 %%%s\n", effLen, useRawLen, rawLen, remain) - isFull := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, 64\n", isFull, effLen) - safeLen := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 63\n", safeLen, effLen) - one := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 1, %%%s\n", one, safeLen) - maskTmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, -1\n", maskTmp, one) - mask := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 -1, i64 %%%s\n", mask, isFull, maskTmp) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %%%s\n", out, shifted, mask) - return true, false, c.storeReg(ins.Args[2].Reg, "%"+out) - - case "BZHIQ": - // BMI2 zero high bits: index, src, dst. - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 BZHIQ expects index, src, dstReg: %q", ins.Raw) - } - idx, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - src, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - valid := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, 64\n", valid, idx) - isZero := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %s, 0\n", isZero, idx) - safeIdx := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 63\n", safeIdx, idx) - one := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 1, %%%s\n", one, safeIdx) - maskTmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, -1\n", maskTmp, one) - maskOrZero := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 0, i64 %%%s\n", maskOrZero, isZero, maskTmp) - mask := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 -1\n", mask, valid, maskOrZero) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, %%%s\n", out, src, mask) - return true, false, c.storeReg(ins.Args[2].Reg, "%"+out) - - case "SHRQ", "SHLQ", "SARQ", "SHLL", "SHRL", "SARL", "SALQ", "SALL": - // Shift ops: - // - 2-operand: amt, dst (in-place) - // - 3-operand SHL/SHR: amt, src, dst (x86 SHLD/SHRD) - if len(ins.Args) == 3 { - switch op { - case "SHLL", "SHLQ", "SHRL", "SHRQ": - return true, false, c.lowerDoubleShift(op, ins) - default: - return true, false, fmt.Errorf("amd64 %s has no three-operand form: %q", op, ins.Raw) - } - } - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects amt,dst: %q", op, ins.Raw) - } - if ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s destination must be reg: %q", op, ins.Raw) - } - dst := ins.Args[1].Reg - dv, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - amtMask := int64(63) - valTy := I64 - if op == "SHLL" || op == "SHRL" || op == "SARL" || op == "SALL" { - amtMask = 31 - valTy = I32 - } - amtI64 := "" - switch ins.Args[0].Kind { - case OpImm: - amtI64 = fmt.Sprintf("%d", ins.Args[0].Imm&amtMask) - case OpReg: - av, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - m := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", m, av, amtMask) - amtI64 = "%" + m - default: - return true, false, fmt.Errorf("amd64 %s unsupported shift amt: %q", op, ins.Raw) - } - - if valTy == I64 { - t := c.newTmp() - switch op { - case "SHRQ": - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %s\n", t, dv, amtI64) - case "SHLQ", "SALQ": - fmt.Fprintf(c.b, " %%%s = shl i64 %s, %s\n", t, dv, amtI64) - case "SARQ": - fmt.Fprintf(c.b, " %%%s = ashr i64 %s, %s\n", t, dv, amtI64) - } - return true, false, c.storeReg(dst, "%"+t) - } - - // 32-bit shifts: operate on low 32, zero-extend to 64. - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, dv) - amt32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", amt32, amtI64) - sh := c.newTmp() - if op == "SHLL" || op == "SALL" { - fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, %%%s\n", sh, tr, amt32) - } else if op == "SARL" { - fmt.Fprintf(c.b, " %%%s = ashr i32 %%%s, %%%s\n", sh, tr, amt32) - } else { - fmt.Fprintf(c.b, " %%%s = lshr i32 %%%s, %%%s\n", sh, tr, amt32) - } - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, sh) - return true, false, c.storeReg(dst, "%"+z) - - case "SHLW", "SHRW", "SARW", "SALW": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects amt, dstReg: %q", op, ins.Raw) - } - dst := ins.Args[1].Reg - dv, err := c.evalIntSized(ins.Args[1], I16) - if err != nil { - return true, false, err - } - var amt string - switch ins.Args[0].Kind { - case OpImm: - amt = fmt.Sprintf("%d", ins.Args[0].Imm&31) - case OpReg: - av, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - masked := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 31\n", masked, av) - amt = "%" + masked - default: - return true, false, fmt.Errorf("amd64 %s unsupported shift amt: %q", op, ins.Raw) - } - inRange := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, 16\n", inRange, amt) - safeAmt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %s, i64 15\n", safeAmt, inRange, amt) - amt16 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i16\n", amt16, safeAmt) - shifted := c.newTmp() - switch op { - case "SHLW", "SALW": - fmt.Fprintf(c.b, " %%%s = shl i16 %s, %%%s\n", shifted, dv, amt16) - case "SARW": - fmt.Fprintf(c.b, " %%%s = ashr i16 %s, %%%s\n", shifted, dv, amt16) - case "SHRW": - fmt.Fprintf(c.b, " %%%s = lshr i16 %s, %%%s\n", shifted, dv, amt16) - } - result := "%" + shifted - if op != "SARW" { - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %%%s, i16 0\n", out, inRange, shifted) - result = "%" + out - } - return true, false, c.storeRegSized(dst, I16, result) - - case "SHLB": - // 8-bit logical left shift: amt, dstReg. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 SHLB expects amt, dstReg: %q", ins.Raw) - } - dst := ins.Args[1].Reg - dv64, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - d8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", d8, dv64) - var amt string - switch ins.Args[0].Kind { - case OpImm: - amt = fmt.Sprintf("%d", ins.Args[0].Imm&31) - case OpReg: - av, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - m := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 31\n", m, av) - amt = "%" + m - default: - return true, false, fmt.Errorf("amd64 SHLB unsupported shift amt: %q", ins.Raw) - } - inRange := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, 8\n", inRange, amt) - safeAmt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %s, i64 7\n", safeAmt, inRange, amt) - amt8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i8\n", amt8, safeAmt) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i8 %%%s, %%%s\n", sh, d8, amt8) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i8 %%%s, i8 0\n", out, inRange, sh) - return true, false, c.storeRegSized(dst, I8, "%"+out) - - case "SHLXQ", "SHRXQ": - // BMI2 variable shifts: amt, src, dst. - if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects amt, srcReg, dstReg: %q", op, ins.Raw) - } - src, err := c.loadReg(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - var amt string - switch ins.Args[0].Kind { - case OpImm: - amt = fmt.Sprintf("%d", ins.Args[0].Imm&63) - case OpReg: - av, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - m := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 63\n", m, av) - amt = "%" + m - default: - return true, false, fmt.Errorf("amd64 %s unsupported shift amt: %q", op, ins.Raw) - } - t := c.newTmp() - if op == "SHLXQ" { - fmt.Fprintf(c.b, " %%%s = shl i64 %s, %s\n", t, src, amt) - } else { - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %s\n", t, src, amt) - } - return true, false, c.storeReg(ins.Args[2].Reg, "%"+t) - - case "ROLL": - // 32-bit rotate-left: count, dstReg. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 ROLL expects count, dstReg: %q", ins.Raw) - } - dst := ins.Args[1].Reg - dv64, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - dv32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", dv32, dv64) - - var cnt32 string - switch ins.Args[0].Kind { - case OpImm: - cnt32 = fmt.Sprintf("%d", uint32(ins.Args[0].Imm)) - case OpReg: - cv64, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, cv64) - cnt32 = "%" + tr - default: - return true, false, fmt.Errorf("amd64 ROLL unsupported count: %q", ins.Raw) - } - - cm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i32 %s, 31\n", cm, cnt32) - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 32, %%%s\n", neg, cm) - nm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 31\n", nm, neg) - lhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, %%%s\n", lhs, dv32, cm) - rhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i32 %%%s, %%%s\n", rhs, dv32, nm) - rot := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i32 %%%s, %%%s\n", rot, lhs, rhs) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, rot) - return true, false, c.storeReg(dst, "%"+z) - - case "ROLQ": - // 64-bit rotate-left: count, dstReg. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 ROLQ expects count, dstReg: %q", ins.Raw) - } - dst := ins.Args[1].Reg - dv, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - var cnt string - switch ins.Args[0].Kind { - case OpImm: - cnt = fmt.Sprintf("%d", uint64(ins.Args[0].Imm)&63) - case OpReg: - cv, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - m := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 63\n", m, cv) - cnt = "%" + m - default: - return true, false, fmt.Errorf("amd64 ROLQ unsupported count: %q", ins.Raw) - } - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 64, %s\n", neg, cnt) - nm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 63\n", nm, neg) - lhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %s, %s\n", lhs, dv, cnt) - rhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %%%s\n", rhs, dv, nm) - rot := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", rot, lhs, rhs) - return true, false, c.storeReg(dst, "%"+rot) - - case "RORQ": - // 64-bit rotate-right: count, dstReg. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 RORQ expects count, dstReg: %q", ins.Raw) - } - dst := ins.Args[1].Reg - dv, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - var cnt string - switch ins.Args[0].Kind { - case OpImm: - cnt = fmt.Sprintf("%d", uint64(ins.Args[0].Imm)&63) - case OpReg: - cv, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - m := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 63\n", m, cv) - cnt = "%" + m - default: - return true, false, fmt.Errorf("amd64 RORQ unsupported count: %q", ins.Raw) - } - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 64, %s\n", neg, cnt) - nm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 63\n", nm, neg) - lhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %s\n", lhs, dv, cnt) - rhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %s, %%%s\n", rhs, dv, nm) - rot := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", rot, lhs, rhs) - return true, false, c.storeReg(dst, "%"+rot) - - case "RORL": - // 32-bit rotate-right: count, dstReg. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 RORL expects count, dstReg: %q", ins.Raw) - } - dst := ins.Args[1].Reg - dv64, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - dv32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", dv32, dv64) - var cnt32 string - switch ins.Args[0].Kind { - case OpImm: - cnt32 = fmt.Sprintf("%d", uint32(ins.Args[0].Imm)&31) - case OpReg: - cv64, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, cv64) - cm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 31\n", cm, tr) - cnt32 = "%" + cm - default: - return true, false, fmt.Errorf("amd64 RORL unsupported count: %q", ins.Raw) - } - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 32, %s\n", neg, cnt32) - nm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 31\n", nm, neg) - lhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i32 %%%s, %s\n", lhs, dv32, cnt32) - rhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, %%%s\n", rhs, dv32, nm) - rot := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i32 %%%s, %%%s\n", rot, lhs, rhs) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, rot) - return true, false, c.storeReg(dst, "%"+z) - - case "RORXL", "RORXQ": - // BMI2 rotate-right without flags: - // - RORXL $imm, srcReg, dstReg (32-bit) - // - RORXQ $imm, srcReg, dstReg (64-bit) - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, srcReg, dstReg: %q", op, ins.Raw) - } - src, err := c.loadReg(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - dst := ins.Args[2].Reg - if op == "RORXQ" { - n := uint64(ins.Args[0].Imm) & 63 - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 64, %d\n", neg, n) - nm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 63\n", nm, neg) - lhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", lhs, src, n) - rhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %s, %%%s\n", rhs, src, nm) - rot := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", rot, lhs, rhs) - return true, false, c.storeReg(dst, "%"+rot) - } - n := uint32(ins.Args[0].Imm) & 31 - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, src) - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 32, %d\n", neg, n) - nm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 31\n", nm, neg) - lhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i32 %%%s, %d\n", lhs, tr, n) - rhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, %%%s\n", rhs, tr, nm) - rot := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i32 %%%s, %%%s\n", rot, lhs, rhs) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, rot) - return true, false, c.storeReg(dst, "%"+z) - - case "NOTB": - // NOT does not modify flags. For a register operand, only the selected - // low byte is changed and the remaining register bits are preserved. - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 NOTB expects one operand: %q", ins.Raw) - } - switch ins.Args[0].Kind { - case OpReg: - r := ins.Args[0].Reg - v64, err := c.loadReg(r) - if err != nil { - return true, false, err - } - v8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", v8, v64) - not := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i8 %%%s, -1\n", not, v8) - return true, false, c.storeRegSized(r, I8, "%"+not) - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - load := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i8, ptr %s, align 1\n", load, p) - not := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i8 %%%s, -1\n", not, load) - fmt.Fprintf(c.b, " store i8 %%%s, ptr %s, align 1\n", not, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 NOTB expects reg or mem: %q", ins.Raw) - } - - case "NOTL": - // 32-bit bitwise NOT, result zero-extended to 64-bit. - if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 NOTL expects reg: %q", ins.Raw) - } - r := ins.Args[0].Reg - v64, err := c.loadReg(r) - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) - x := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i32 %%%s, -1\n", x, tr) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, x) - return true, false, c.storeReg(r, "%"+z) - - case "NOTQ": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 NOTQ expects one operand: %q", ins.Raw) - } - switch ins.Args[0].Kind { - case OpReg: - r := ins.Args[0].Reg - v, err := c.loadReg(r) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %s, -1\n", t, v) - return true, false, c.storeReg(r, "%"+t) - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %%%s, -1\n", t, ld) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s, align 1\n", t, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 NOTQ expects reg or mem: %q", ins.Raw) - } - - case "BSWAPL": - // BSWAPL reg: byte swap low 32 bits and zero-extend result to i64. - if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 BSWAPL expects reg: %q", ins.Raw) - } - r := ins.Args[0].Reg - v64, err := c.loadReg(r) - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) - bswap := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.bswap.i32(i32 %%%s)\n", bswap, tr) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, bswap) - return true, false, c.storeReg(r, "%"+z) - - case "MULQ": - // MULQ src: RDX:RAX = RAX * src (unsigned). - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 MULQ expects src: %q", ins.Raw) - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - ax, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - a128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", a128, ax) - b128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", b128, src) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", p, a128, b128) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", lo, p) - hiShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i128 %%%s, 64\n", hiShift, p) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", hi, hiShift) - if err := c.storeReg(AX, "%"+lo); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "%"+hi); err != nil { - return true, false, err - } - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", cf, hi) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - return true, false, nil - - case "MULXQ": - // BMI2 MULXQ src, loDst, hiDst: {hi,lo} = RDX * src (unsigned). - if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 MULXQ expects src, loDst, hiDst: %q", ins.Raw) - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - dx, err := c.loadReg(DX) - if err != nil { - return true, false, err - } - a128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", a128, dx) - b128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", b128, src) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", p, a128, b128) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", lo, p) - hiShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i128 %%%s, 64\n", hiShift, p) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", hi, hiShift) - if err := c.storeReg(ins.Args[1].Reg, "%"+lo); err != nil { - return true, false, err - } - if err := c.storeReg(ins.Args[2].Reg, "%"+hi); err != nil { - return true, false, err - } - return true, false, nil - - case "MULL": - // MULL src: EDX:EAX = EAX * src (unsigned 32-bit) - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 MULL expects src: %q", ins.Raw) - } - src32, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - ax64, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - ax32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", ax32, ax64) - az := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", az, ax32) - bz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", bz, src32) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, %%%s\n", p, az, bz) - lo32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", lo32, p) - hiShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %%%s, 32\n", hiShift, p) - hi32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", hi32, hiShift) - lo64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", lo64, lo32) - hi64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", hi64, hi32) - if err := c.storeReg(AX, "%"+lo64); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "%"+hi64); err != nil { - return true, false, err - } - return true, false, nil - - case "DIVL": - // DIVL src: unsigned divide EDX:EAX by src; quotient->EAX remainder->EDX. - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 DIVL expects src: %q", ins.Raw) - } - src32, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - ax64, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - dx64, err := c.loadReg(DX) - if err != nil { - return true, false, err - } - ax32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", ax32, ax64) - dx32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", dx32, dx64) - az := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", az, ax32) - dz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", dz, dx32) - divisor := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", divisor, src32) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, 32\n", hi, dz) - dividend := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", dividend, hi, az) - q := c.newTmp() - fmt.Fprintf(c.b, " %%%s = udiv i64 %%%s, %%%s\n", q, dividend, divisor) - r := c.newTmp() - fmt.Fprintf(c.b, " %%%s = urem i64 %%%s, %%%s\n", r, dividend, divisor) - q32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", q32, q) - r32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", r32, r) - q64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", q64, q32) - r64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", r64, r32) - if err := c.storeReg(AX, "%"+q64); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "%"+r64); err != nil { - return true, false, err - } - return true, false, nil - - case "IMULL", "IMUL3L": - // Signed 32-bit multiply. The one-operand form writes DX:AX; the two- - // and three-operand forms write the low word to their explicit dst. - switch len(ins.Args) { - case 1: - src, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - ax, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) - if err != nil { - return true, false, err - } - a64 := c.newTmp() - b64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i32 %s to i64\n", a64, ax) - fmt.Fprintf(c.b, " %%%s = sext i32 %s to i64\n", b64, src) - product := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, %%%s\n", product, a64, b64) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", lo, product) - hiShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ashr i64 %%%s, 32\n", hiShift, product) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", hi, hiShift) - if err := c.storeRegSized(AX, I32, "%"+lo); err != nil { - return true, false, err - } - return true, false, c.storeRegSized(DX, I32, "%"+hi) - case 2: - if ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 IMULL expects src, dstReg: %q", ins.Raw) - } - src, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - dst, err := c.evalIntSized(ins.Args[1], I32) - if err != nil { - return true, false, err - } - product := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i32 %s, %s\n", product, dst, src) - return true, false, c.storeRegSized(ins.Args[1].Reg, I32, "%"+product) - case 3: - if op != "IMUL3L" || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects imm, src, dstReg: %q", op, ins.Raw) - } - a, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - bv, err := c.evalIntSized(ins.Args[1], I32) - if err != nil { - return true, false, err - } - product := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i32 %s, %s\n", product, bv, a) - return true, false, c.storeRegSized(ins.Args[2].Reg, I32, "%"+product) - default: - return true, false, fmt.Errorf("amd64 %s unsupported operand count: %q", op, ins.Raw) - } - - case "IMULQ", "IMUL3Q": - // IMULQ src -> RDX:RAX = signed RAX*src - // IMULQ src, dst -> dst = signed(dst*src) - // IMUL3Q imm,src,dst -> dst = signed(src*imm) - switch len(ins.Args) { - case 1: - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - ax, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - a128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", a128, ax) - b128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", b128, src) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", p, a128, b128) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", lo, p) - hiShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ashr i128 %%%s, 64\n", hiShift, p) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", hi, hiShift) - if err := c.storeReg(AX, "%"+lo); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "%"+hi); err != nil { - return true, false, err - } - return true, false, nil - case 2: - if ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 IMULQ expects src, dstReg: %q", ins.Raw) - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - dst := ins.Args[1].Reg - dv, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - a128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", a128, dv) - b128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", b128, src) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", p, a128, b128) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", lo, p) - return true, false, c.storeReg(dst, "%"+lo) - case 3: - if op != "IMUL3Q" || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects imm, src, dstReg: %q", op, ins.Raw) - } - imm, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - src, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - a128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", a128, src) - b128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", b128, imm) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", p, a128, b128) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", lo, p) - return true, false, c.storeReg(ins.Args[2].Reg, "%"+lo) - default: - return true, false, fmt.Errorf("amd64 %s expects 1/2/3 operands: %q", op, ins.Raw) - } - - case "NEGQ": - // NEGQ reg - if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 NEGQ expects reg: %q", ins.Raw) - } - r := ins.Args[0].Reg - v, err := c.loadReg(r) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 0, %s\n", t, v) - out := "%" + t - if err := c.storeReg(r, out); err != nil { - return true, false, err - } - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %s, 0\n", cf, v) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - c.setZSFlagsFromI64(out) - return true, false, nil } return false, false, nil } - -func (c *amd64Ctx) lowerDoubleShift(op Op, ins Instr) error { - if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg { - return fmt.Errorf("amd64 %s expects amt, srcReg, dst: %q", op, ins.Raw) - } - ty := I64 - mask := int64(63) - left := op == "SHLQ" - if op == "SHLL" || op == "SHRL" { - ty = I32 - mask = 31 - left = op == "SHLL" - } - - src, err := c.evalIntSized(ins.Args[1], ty) - if err != nil { - return err - } - dst, storeDst, err := c.loadIntDestination(ins.Args[2], ty) - if err != nil { - return err - } - - count64 := "" - switch ins.Args[0].Kind { - case OpImm: - count64 = fmt.Sprintf("%d", ins.Args[0].Imm&mask) - case OpReg: - value, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return err - } - masked := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", masked, value, mask) - count64 = "%" + masked - default: - return fmt.Errorf("amd64 %s unsupported shift amount: %q", op, ins.Raw) - } - - count := count64 - if ty == I32 { - truncated := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", truncated, count64) - count = "%" + truncated - } - inverseNeg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", inverseNeg, ty, count) - inverse := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", inverse, ty, inverseNeg, mask) - zero := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", zero, ty, count) - - dstPart := c.newTmp() - srcPart := c.newTmp() - if left { - fmt.Fprintf(c.b, " %%%s = shl %s %s, %s\n", dstPart, ty, dst, count) - fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", srcPart, ty, src, inverse) - } else { - fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", dstPart, ty, dst, count) - fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", srcPart, ty, src, inverse) - } - merged := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", merged, ty, dstPart, srcPart) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %s, %s %%%s\n", out, zero, ty, dst, ty, merged) - return storeDst("%" + out) -} - -func (c *amd64Ctx) lower386Timestamp(serialized bool) error { - asm := "rdtsc" - resultType := "{ i32, i32 }" - constraints := "={ax},={dx},~{dirflag},~{fpsr},~{flags},~{memory}" - regs := []Reg{AX, DX} - if serialized { - asm = "rdtscp" - resultType = "{ i32, i32, i32 }" - constraints = "={ax},={dx},={cx},~{dirflag},~{fpsr},~{flags},~{memory}" - regs = append(regs, CX) - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", call, resultType, asm, constraints) - for i, reg := range regs { - value := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, %d\n", value, resultType, call, i) - if err := c.storeRegSized(reg, I32, "%"+value); err != nil { - return err - } - } - return nil -} diff --git a/amd64_lower_arithmetic_shift.go b/amd64_lower_arithmetic_shift.go new file mode 100644 index 000000000..9f8dd4655 --- /dev/null +++ b/amd64_lower_arithmetic_shift.go @@ -0,0 +1,34 @@ +package plan9asm + +import "fmt" + +// The complete uniform arithmetic family shares yps/_yvpslld operand shapes +// with logical shifts. VPSRAQ has only the unsigned-immediate EVEX rows. +// These descriptors grant no register input, native backing or caller role. +var amd64PackedArithmeticShiftSpecs = map[Op]amd64PackedLogicalShiftSpec{ + "PSRAW": {laneBits: 16}, + "PSRAL": {laneBits: 32}, + "VPSRAW": {laneBits: 16, vector: true}, + "VPSRAD": {laneBits: 32, vector: true}, + "VPSRAQ": {laneBits: 64, vector: true}, +} + +func (c *amd64Ctx) lowerPackedArithmeticRightShift(op Op, ins Instr) (bool, bool, error) { + return c.lowerX86PackedShift(op, ins) +} + +func (c *amd64Ctx) emitPackedArithmeticRightShift(lanes, laneBits int, data, count64 string) string { + valid, safe := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, %d\n", valid, count64, laneBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %s, i64 %d\n", safe, valid, count64, laneBits-1) + count := "%" + safe + if laneBits < 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", narrow, count, laneBits) + count = "%" + narrow + } + counts := amd64SplatInteger(c, lanes, laneBits, count) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ashr <%d x i%d> %s, %s\n", result, lanes, laneBits, data, counts) + return "%" + result +} diff --git a/amd64_lower_arithmetic_shift_test.go b/amd64_lower_arithmetic_shift_test.go new file mode 100644 index 000000000..5c03f3859 --- /dev/null +++ b/amd64_lower_arithmetic_shift_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedArithmeticRightShiftCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, zLast := 15, 22 + if target.goarch == "386" { + legacyLast, zLast = 7, 7 + } + var source strings.Builder + source.WriteString("TEXT packedarithmeticrightshiftforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSRAW", "PSRAL"} { + // Go's Py3 MMX rows are named-source mode64 only. Their + // complete mode32 rejection controls are independently + // exercised by TestX86YMMMode32LegacyShiftActualGoRows. + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, M1\n", op) + fmt.Fprintf(&source, "\t%s M0, M1\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M1\n", op) + } + fmt.Fprintf(&source, "\t%s $-1, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + } + for _, op := range []string{"VPSRAW", "VPSRAD", "VPSRAQ"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s $3, %s20, %s21\n", op, width, width) + fmt.Fprintf(&source, "\t%s $3, 8(AX), %s21\n", op, width) + fmt.Fprintf(&source, "\t%s X0, %s20, %s21\n", op, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s20, %s21\n", op, width, width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, %s20, K1, %s22\n", op, width, width) + fmt.Fprintf(&source, "\t%s.Z X0, %s20, K7, %s22\n", op, width, width) + } + } + fmt.Fprintf(&source, "\t%s $3, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s $3, 8(AX), Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s X0, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, Z20, K2, Z22\n", op) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z20, K6, Z22\n", op) + } + if op != "VPSRAW" { + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), X21\n", op) + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), Y21\n", op) + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z $3, 8(AX), K5, Z22\n", op) + } + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + options := Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedarithmeticrightshiftforms": {Name: "packedarithmeticrightshiftforms", Ret: Void}, + }, + } + goAssemblerOracleOutput(t, target.goarch, strings.ReplaceAll(source.String(), "NOSPLIT", "4"), "", true) + for _, api := range []string{"text", "module", "Go"} { + original := strings.Replace(source.String(), "packedarithmeticrightshiftforms(SB)", "·view(SB)", 1) + if ir, err := x86ScalarFPBitViewAPI(t, original, "func view()", target.goarch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized original formats acquired MMX/numerical/native inputs", api, err) + } + } + if target.goarch == "386" { + // Go emits high-bank EVEX spellings in mode32, but those + // physical registers do not exist. Retain the original Go + // objects/Context above; LLVM format objects use this explicit + // low-bank counterpart, not an invented high-bank permission. + counterpart := strings.NewReplacer("X20", "X5", "X21", "X6", "Y20", "Y5", "Y21", "Y6").Replace(source.String()) + goAssemblerOracleOutput(t, target.goarch, strings.ReplaceAll(counterpart, "NOSPLIT", "4"), "", true) + file, err = Parse(ArchAMD64, counterpart) + if err != nil { + t.Fatal(err) + } + } + ll := x86InstructionFormatObjectUnit(t, file, options.Sigs, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerPackedArithmeticRightShift(ins.Op, ins) + }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-arithmetic-right-shift-"+target.name+".ll", "packed-arithmetic-right-shift-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedArithmeticRightShiftRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSRAW.Z $3, X0", + "PSRAL $3", + "PSRAW M0, X1", + "PSRAL X0, M1", + "PSRAQ $3, X0", + "PSRAW $3, Y0", + "VPSRAW R8, X1, X2", + "VPSRAD Y0, X1, X2", + "VPSRAQ X0, 8(AX), X2", + "VPSRAW.BCST $3, 8(AX), X2", + "VPSRAD.BCST X0, X1, X2", + "VPSRAQ.BCST $3, X1, X2", + "VPSRAW.Z $3, X1, X2", + "VPSRAD $3, X1, K0, X2", + "VPSRAQ $3, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedArithmeticRightShiftRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PSRAW X8, X0", + "PSRAL X0, X8", + "VPSRAW $3, Z8, Z1", + "VPSRAD X0, Z1, Z8", + "VPSRAQ $3, X1, K1, X2", + "VPSRAW.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedArithmeticRightShiftRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedArithmeticRightShiftRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed arithmetic-right-shift forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_atomic.go b/amd64_lower_atomic.go index c2783260b..e2bb314e5 100644 --- a/amd64_lower_atomic.go +++ b/amd64_lower_atomic.go @@ -3,283 +3,23 @@ package plan9asm import "fmt" func (c *amd64Ctx) lowerAtomic(op Op, ins Instr) (ok bool, terminated bool, err error) { + if known, terminated, err := c.lowerLockScalarRMW(op, ins); known { + return known, terminated, err + } + if spec, recognized := amd64ExchangeSpecs[Op(normalizeInstructionOpcode(op))]; recognized { + return c.lowerExchange(op, spec, ins) + } + if _, _, recognized := amd64CompareExchangeProperties(normalizeInstructionOpcode(op)); recognized { + return c.lowerCompareExchange(op, ins) + } switch op { case "LOCK": - // LOCK is a prefix in Plan 9 syntax. Our lowering emits atomic IR for - // the following memory RMW instruction, so the prefix itself is a no-op. - return true, false, nil - - case "CMPXCHG8B": - if len(ins.Args) != 1 || ins.Args[0].Kind != OpMem { - return true, false, fmt.Errorf("amd64 CMPXCHG8B expects mem: %q", ins.Raw) - } - word := func(r Reg) (string, error) { - return c.evalIntSized(Operand{Kind: OpReg, Reg: r}, I32) - } - ax, err := word(AX) - if err != nil { - return true, false, err - } - dx, err := word(DX) - if err != nil { - return true, false, err - } - bx, err := word(BX) - if err != nil { - return true, false, err - } - cx, err := word(CX) - if err != nil { - return true, false, err - } - pair := func(lo, hi string) string { - lo64 := c.newTmp() - hi64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", lo64, lo) - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", hi64, hi) - shifted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, 32\n", shifted, hi64) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", out, shifted, lo64) - return "%" + out - } - expected := pair(ax, dx) - desired := pair(bx, cx) - ptr, ptrType, err := c.ptrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - if ptrType != "ptr" { - return true, false, fmt.Errorf("amd64 CMPXCHG8B does not support segment-relative memory: %q", ins.Raw) - } - result := c.newTmp() - fmt.Fprintf(c.b, " %%%s = cmpxchg ptr %s, i64 %s, i64 %s seq_cst seq_cst, align 8\n", result, ptr, expected, desired) - old := c.newTmp() - success := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i1 } %%%s, 0\n", old, result) - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i1 } %%%s, 1\n", success, result) - oldLo := c.newTmp() - oldHiShift := c.newTmp() - oldHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", oldLo, old) - fmt.Fprintf(c.b, " %%%s = lshr i64 %%%s, 32\n", oldHiShift, old) - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", oldHi, oldHiShift) - if err := c.storeRegSized(AX, I32, "%"+oldLo); err != nil { - return true, false, err - } - if err := c.storeRegSized(DX, I32, "%"+oldHi); err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", success, c.flagsZSlot) - return true, false, nil - - case "CMPXCHGL", "CMPXCHGQ": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpMem { - return true, false, fmt.Errorf("amd64 %s expects src, mem: %q", op, ins.Raw) - } - ty := I32 - align := 4 - if op == "CMPXCHGQ" { - ty = I64 - align = 8 - } - - expAX, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - exp, err := c.amd64AtomicTruncFromI64(expAX, ty) - if err != nil { - return true, false, err - } - newv64, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - newv, err := c.amd64AtomicTruncFromI64(newv64, ty) - if err != nil { - return true, false, err - } - ptr, err := c.amd64AtomicPtrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - cx := c.newTmp() - fmt.Fprintf(c.b, " %%%s = cmpxchg ptr %s, %s %s, %s %s seq_cst seq_cst, align %d\n", cx, ptr, ty, exp, ty, newv, align) - old := c.newTmp() - okv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue {%s, i1} %%%s, 0\n", old, ty, cx) - fmt.Fprintf(c.b, " %%%s = extractvalue {%s, i1} %%%s, 1\n", okv, ty, cx) - old64, err := c.amd64AtomicExtendToI64("%"+old, ty) - if err != nil { - return true, false, err - } - if err := c.storeReg(AX, old64); err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", okv, c.flagsZSlot) - return true, false, nil - - case "XADDL", "XADDQ": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpMem { - return true, false, fmt.Errorf("amd64 %s expects srcReg, mem: %q", op, ins.Raw) - } - srcReg := ins.Args[0] - if srcReg.Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects register src: %q", op, ins.Raw) - } - ty := I32 - if op == "XADDQ" { - ty = I64 - } - src64, err := c.loadReg(srcReg.Reg) - if err != nil { - return true, false, err - } - src, err := c.amd64AtomicTruncFromI64(src64, ty) - if err != nil { - return true, false, err - } - ptr, err := c.amd64AtomicPtrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - old := c.newTmp() - fmt.Fprintf(c.b, " %%%s = atomicrmw add ptr %s, %s %s seq_cst\n", old, ptr, ty, src) - old64, err := c.amd64AtomicExtendToI64("%"+old, ty) - if err != nil { - return true, false, err - } - return true, false, c.storeReg(srcReg.Reg, old64) - - case "XCHGB", "XCHGL", "XCHGQ": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects srcReg, dst: %q", op, ins.Raw) - } - srcReg := ins.Args[0] - if srcReg.Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects register src: %q", op, ins.Raw) - } - ty := I8 - switch op { - case "XCHGL": - ty = I32 - case "XCHGQ": - ty = I64 - } - src64, err := c.loadReg(srcReg.Reg) - if err != nil { - return true, false, err - } - if ins.Args[1].Kind == OpReg { - dstReg := ins.Args[1].Reg - dst64, err := c.loadReg(dstReg) - if err != nil { - return true, false, err - } - switch ty { - case I64: - if err := c.storeReg(srcReg.Reg, dst64); err != nil { - return true, false, err - } - return true, false, c.storeReg(dstReg, src64) - case I32: - s32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", s32, src64) - d32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", d32, dst64) - sz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", sz, s32) - dz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", dz, d32) - if err := c.storeReg(srcReg.Reg, "%"+dz); err != nil { - return true, false, err - } - return true, false, c.storeReg(dstReg, "%"+sz) - default: - s8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", s8, src64) - d8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", d8, dst64) - sz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", sz, s8) - dz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", dz, d8) - if err := c.storeReg(srcReg.Reg, "%"+dz); err != nil { - return true, false, err - } - return true, false, c.storeReg(dstReg, "%"+sz) - } - } - if ins.Args[1].Kind != OpMem && ins.Args[1].Kind != OpSym { - return true, false, fmt.Errorf("amd64 %s expects srcReg, mem/reg: %q", op, ins.Raw) - } - src, err := c.amd64AtomicTruncFromI64(src64, ty) - if err != nil { - return true, false, err - } - ptr := "" - if ins.Args[1].Kind == OpMem { - ptr, err = c.amd64AtomicPtrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - } else { - ptr, err = c.ptrFromSB(ins.Args[1].Sym) - if err != nil { - return true, false, err - } - } - old := c.newTmp() - fmt.Fprintf(c.b, " %%%s = atomicrmw xchg ptr %s, %s %s seq_cst\n", old, ptr, ty, src) - old64, err := c.amd64AtomicExtendToI64("%"+old, ty) - if err != nil { - return true, false, err - } - return true, false, c.storeReg(srcReg.Reg, old64) - - case "ORB", "ANDB", "ORL", "ANDL", "ORQ", "ANDQ": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpMem { - return false, false, nil - } - ty := I8 - switch op { - case "ORL", "ANDL": - ty = I32 - case "ORQ", "ANDQ": - ty = I64 - } - src64, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - src, err := c.amd64AtomicTruncFromI64(src64, ty) - if err != nil { - return true, false, err - } - ptr, err := c.amd64AtomicPtrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - rmw := "or" - if op == "ANDB" || op == "ANDL" || op == "ANDQ" { - rmw = "and" + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%w: LOCK operands cannot substitute for a complete following atomic instruction: %q", ErrProbeNeedsContext, ins.Raw) } - tmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = atomicrmw %s ptr %s, %s %s seq_cst\n", tmp, rmw, ptr, ty, src) - // atomicrmw returns the old value, while x86 logical instructions set - // flags from the new value. LOCK does not alter that flag behavior. - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = %s %s %%%s, %s\n", out, rmw, ty, tmp, src) - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) - zf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, 0\n", zf, ty, out) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zf, c.flagsZSlot) - sf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", sf, ty, out) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sf, c.flagsSltSlot) - return true, false, nil + // Only the whole-source consumer can bind this prefix to an adjacent + // complete native operation. An isolated helper has no such body. + return true, false, fmt.Errorf("%w: standalone LOCK has no source-adjacent complete atomic operation", ErrProbeNeedsContext) } return false, false, nil } diff --git a/amd64_lower_average.go b/amd64_lower_average.go new file mode 100644 index 000000000..5a49eb389 --- /dev/null +++ b/amd64_lower_average.go @@ -0,0 +1,80 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerPackedUnsignedAverage(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedPhysical(op, ins, x86PackedAverage) +} + +func (c *amd64Ctx) lowerPackedAverageForm(op Op, ins Instr, form x86PackedPhysicalForm) (bool, bool, error) { + ins.Op = op + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + first, second, destination := form.source, ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + // A virtual FP/SP/BP view is not the source's native aligned m128. + // Only the existing exact fixed-local certificate supplies that backing. + virtual := first.Kind == OpFP + if first.Kind == OpMem { + virtual = !c.nativeMemoryOperand(ins, form.first) && !c.fixedLocalMemoryCertified(ins, form.memory) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed average m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + bits := form.spec.outputBits + lanes := form.byteWidth * 8 / bits + if form.mmx { + var a string + var err error + if first.Kind == OpReg { + a, err = c.loadReg(first.Reg) + } else { + // Exact m64 bytes, including independent logical FP fields. Native + // reads stay observable without inventing an x87/tag-state bridge. + a, err = c.loadPackedUnpackMMXSource(first, 8) + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + value := c.emitPackedUnsignedAverage(lanes, bits, c.bitcastI64ToIntegerLanes(lanes, bits, a), c.bitcastI64ToIntegerLanes(lanes, bits, b)) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", out, lanes, bits, value) + return true, false, c.storeReg(destination.Reg, "%"+out) + } + mnemonic := "pavgb" + if bits == 16 { + mnemonic = "pavgw" + } + return true, false, c.emitPackedBinaryE4(ins, form.x86PackedBinaryForm, form.spec.memoryBits, + mnemonic, c.emitPackedUnsignedAverage) +} + +func (c *amd64Ctx) emitPackedUnsignedAverage(lanes, laneBits int, first, second string) string { + narrowType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + wideBits := laneBits * 2 + wideType := fmt.Sprintf("<%d x i%d>", lanes, wideBits) + firstWide := c.newTmp() + secondWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", firstWide, narrowType, first, wideType) + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", secondWide, narrowType, second, wideType) + sum := c.newTmp() + rounded := c.newTmp() + average := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %%%s\n", sum, wideType, firstWide, secondWide) + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %s\n", rounded, wideType, sum, llvmSplatSignedInteger(lanes, wideBits, 1)) + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %s\n", average, wideType, rounded, llvmSplatSignedInteger(lanes, wideBits, 1)) + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", result, wideType, average, narrowType) + return "%" + result +} diff --git a/amd64_lower_average_test.go b/amd64_lower_average_test.go new file mode 100644 index 000000000..7b9a3e8eb --- /dev/null +++ b/amd64_lower_average_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedUnsignedAverageCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, zLast := 15, 22 + if target.goarch == "386" { + legacyLast, zLast = 7, 7 + } + var source strings.Builder + source.WriteString("TEXT packedunsignedaverageforms(SB),4,$0-0\n") + for _, stem := range []string{"PAVGB", "PAVGW"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M1\n", stem) + fmt.Fprintf(&source, "\t%s 8(AX), M2\n", stem) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", stem, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", stem, legacyLast) + vector := "V" + stem + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", vector, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s6, %s7\n", vector, width, width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", vector, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 8(AX), %s21, K7, %s22\n", vector, width, width) + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, Z%d\n", vector, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", vector, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K2, Z22\n", vector) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z21, K6, Z22\n", vector) + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + goAssemblerOracleOutput(t, target.goarch, source.String(), "", true) + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedunsignedaverageforms": {Name: "packedunsignedaverageforms", Ret: Void}, + }, + }) + if ll != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized original average formats acquired numerical or memory inputs", err) + } + ll = x86AverageFormatObjectUnit(t, file, map[string]FuncSig{ + "packedunsignedaverageforms": {Name: "packedunsignedaverageforms", Ret: Void}, + }, target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-unsigned-average-"+target.name+".ll", "packed-unsigned-average-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedUnsignedAverageRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PAVGB.Z X0, X1", + "PAVGW X0", + "PAVGB M0, X1", + "PAVGW X0, M1", + "PAVGB Y0, Y1", + "PAVGW X0, (AX)", + "VPAVGB X0, X1", + "VPAVGW X0, Y1, Y2", + "VPAVGB X0, X1, (AX)", + "VPAVGW.BCST 8(AX), X1, X2", + "VPAVGB.Z X0, X1, X2", + "VPAVGW X0, X1, K0, X2", + "VPAVGB X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedUnsignedAverageRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PAVGB X8, X0", + "PAVGW X0, X8", + "VPAVGB Z8, Z1, Z2", + "VPAVGW Z0, Z1, Z8", + "VPAVGB X0, X1, K1, X2", + "VPAVGW.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedUnsignedAverageRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedUnsignedAverageRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed unsigned-average forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_bextr.go b/amd64_lower_bextr.go new file mode 100644 index 000000000..55fd91049 --- /dev/null +++ b/amd64_lower_bextr.go @@ -0,0 +1,88 @@ +package plan9asm + +import "fmt" + +// lowerBEXTR implements Go 1.27's complete _ybextrl operand row for BEXTRL/Q: +// register control, register/memory source, and register destination. +func (c *amd64Ctx) lowerBEXTR(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86BEXTRProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + bits := form.bits + typ := amd64IntegerTypeForBits(bits) + controlValue, err := c.evalIntSized(form.control, typ) + if err != nil { + return true, false, err + } + sourceValue, err := c.evalScalarBitFieldSource(ins, form, typ) + if err != nil { + return true, false, err + } + start := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, 255\n", start, typ, controlValue) + lengthShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, 8\n", lengthShift, typ, controlValue) + length := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 255\n", length, typ, lengthShift) + startOK := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %d\n", startOK, typ, start, bits) + safeStart := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s %d\n", safeStart, startOK, typ, start, typ, bits-1) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", shifted, typ, sourceValue, safeStart) + rawLength := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s 0\n", rawLength, startOK, typ, length, typ) + remaining := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %d, %%%s\n", remaining, typ, bits, safeStart) + useRawLength := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %%%s\n", useRawLength, typ, rawLength, remaining) + effectiveLength := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s %%%s\n", effectiveLength, useRawLength, typ, rawLength, typ, remaining) + isFullWidth := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %d\n", isFullWidth, typ, effectiveLength, bits) + safeLength := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", safeLength, typ, effectiveLength, bits-1) + one := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s 1, %%%s\n", one, typ, safeLength) + partialMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %%%s, -1\n", partialMask, typ, one) + mask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s -1, %s %%%s\n", mask, isFullWidth, typ, typ, partialMask) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", result, typ, shifted, mask) + if err := c.storeRegSized(form.destination.Reg, typ, "%"+result); err != nil { + return true, false, err + } + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + return true, false, nil +} + +func (c *amd64Ctx) evalScalarBitFieldSource(ins Instr, form x86ScalarBitFieldForm, typ LLVMType) (string, error) { + if form.source.Kind == OpFP && x86ResultNamedPredicateInstruction(ins) { + if value, handled, err := c.loadIntegerFPBytesMode(form.source.FPOffset, typ, true); handled || err != nil { + return value, err + } + } + if isAMD64MemoryOperand(form.source) && form.source.Kind != OpFP && + (c.nativeMemoryOperand(ins, 1) || + x86CallerMemoryCertified(ins, 1) || c.fixedLocalMemoryCertified(ins, 1)) { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return "", fmt.Errorf("%w: %s needs its original native address size", ErrProbeNeedsContext, ins.Op) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.source) + if err != nil { + return "", err + } + value := c.newTmp() + // Preserve the full physical read even for zero controls/dead outputs. + // Alignment, bounds and pointee validity are not optimizer assumptions. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", value, typ, pointerType, pointer) + return "%" + value, nil + } + return c.evalIntSized(form.source, typ) +} diff --git a/amd64_lower_bextr_test.go b/amd64_lower_bextr_test.go new file mode 100644 index 000000000..95e41ee64 --- /dev/null +++ b/amd64_lower_bextr_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86BEXTRCompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA bextrdata+0(SB)/8, $1\n") + source.WriteString("GLOBL bextrdata(SB), $8\n") + source.WriteString("TEXT bextrforms(SB),$0-0\n") + for _, width := range []string{"L", "Q"} { + op := "BEXTR" + width + fmt.Fprintf(&source, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&source, "\t%s DX, 8(BX), SI\n", op) + fmt.Fprintf(&source, "\t%s CX, bextrdata(SB), DX\n", op) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12, R13\n", op) + } else { + fmt.Fprintf(&source, "\t%s AX, BX, SP\n", op) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86BEXTRCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86BEXTRCompleteFormsSource(target.goarch) + // Preserve every original row, including unknown inputs and the + // physical SP destination. A syntax object is not an ABI proof. + ir := x86UnboundVoidScalarFormatObject(t, source, "bextrforms", target.goarch, target.triple) + wantIR := []string{"lshr i32", `"target-features"="+bmi"`} + if target.goarch == "amd64" { + wantIR = append(wantIR, "lshr i64") + } else if strings.Contains(ir, "lshr i64") { + t.Fatal("mode32 Q spelling used a fictitious 64-bit extraction") + } + for _, want := range wantIR { + if !strings.Contains(ir, want) { + t.Fatalf("BEXTR lowering omitted %q:\n%s", want, ir) + } + } + if strings.Contains(ir, "+bmi2") { + t.Fatalf("BEXTR is BMI1 but inferred BMI2:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "bextr-"+target.name+".ll", "bextr-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86BEXTRRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "BEXTRL $1, AX, BX"}, + {goarch: "amd64", instruction: "BEXTRQ 0(AX), BX, CX"}, + {goarch: "amd64", instruction: "BEXTRQ AX, $1, CX"}, + {goarch: "amd64", instruction: "BEXTRL AX, BX, 0(CX)"}, + {goarch: "amd64", instruction: "BEXTRQ X0, AX, BX"}, + {goarch: "amd64", instruction: "BEXTRQ AX, X0, BX"}, + {goarch: "amd64", instruction: "BEXTRQ AX, BX, X0"}, + {goarch: "amd64", instruction: "BEXTRQ AX, BX"}, + {goarch: "amd64", instruction: "BEXTRQ.Z AX, BX, CX"}, + {goarch: "386", instruction: "BEXTRL R8, AX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _ybextrl table", test.instruction) + } + }) + } +} + +func TestAMD64BEXTRRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = x86BEXTROriginalRuntimeSource + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "bextrsemantics": { + Name: "bextrsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "bextr_semantics", triple, ir, x86BEXTROriginalRuntimeDriver, runPrefix) +} + +const x86BEXTROriginalRuntimeSource = `TEXT bextrsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVL $0x0804, AX + MOVL $0x1234, BX + BEXTRL AX, BX, CX + MOVQ CX, 0(DI) + SETCS 8(DI) + SETOS 9(DI) + SETEQ 10(DI) + + MOVL $0x0120, AX + MOVL $-1, BX + BEXTRL AX, BX, CX + MOVQ CX, 16(DI) + SETEQ 24(DI) + + MOVQ $0x083c, AX + MOVQ $-1, 32(DI) + BEXTRQ AX, 32(DI), CX + MOVQ CX, 40(DI) + + MOVQ $0x0404, AX + MOVQ $0xf0, BX + BEXTRQ AX, BX, AX + MOVQ AX, 48(DI) + RET +` + +const x86BEXTROriginalRuntimeDriver = ` +#include +#include +extern void bextrsemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[56] = {0}; + bextrsemantics(out); + if (load64(out + 0) != UINT64_C(0x23)) return 10; + if (out[8] != 0 || out[9] != 0 || out[10] != 0) return 11; + if (load64(out + 16) != 0 || out[24] != 1) return 12; + if (load64(out + 40) != UINT64_C(0xf)) return 13; + if (load64(out + 48) != UINT64_C(0xf)) return 14; + return 0; +} +` diff --git a/amd64_lower_bf16_convert.go b/amd64_lower_bf16_convert.go new file mode 100644 index 000000000..da42e06a2 --- /dev/null +++ b/amd64_lower_bf16_convert.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64BF16ConvertSourceBytes = map[Op]int{ + "VCVTNEPS2BF16X": 16, + "VCVTNEPS2BF16Y": 32, + "VCVTNEPS2BF16": 64, +} + +func (c *amd64Ctx) lowerBF16Convert(op Op, ins Instr) (ok bool, terminated bool, err error) { + base := Op(strings.SplitN(string(op), ".", 2)[0]) + sourceBytes := amd64BF16ConvertSourceBytes[base] + if sourceBytes == 0 { + return false, false, nil + } + if !ins.x86Encoded { + return true, false, fmt.Errorf("%s has no named Go 1.27 encoder form: %q", base, ins.Raw) + } + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s expects source, [K mask,] destination: %q", base, ins.Raw) + } + broadcast := strings.Contains(string(op), ".BCST") + zeroing := strings.HasSuffix(string(op), ".Z") + masked := len(ins.Args) == 3 + if zeroing && !masked || broadcast && !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s has invalid mask or broadcast operands: %q", base, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s requires K1-K7 for masking: %q", base, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + destinationBytes := sourceBytes / 2 + if destinationBytes < 16 { + destinationBytes = 16 + } + if destination.Kind != OpReg || amd64VectorByteWidth(destination.Reg) != destinationBytes { + return true, false, fmt.Errorf("%s requires a %d-byte destination register: %q", base, destinationBytes, ins.Raw) + } + source := ins.Args[0] + if source.Kind == OpReg && amd64VectorByteWidth(source.Reg) != sourceBytes || + source.Kind != OpReg && !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s requires a %d-byte vector or memory source: %q", base, sourceBytes, ins.Raw) + } + + lanes := sourceBytes / 4 + mask := "" + if masked { + mask, err = c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + } + inputBits, err := c.loadMaskedPackedCompareLanes(source, sourceBytes, 32, broadcast, mask) + if err != nil { + return true, false, err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x float>\n", input, lanes, inputBits, lanes) + resultLanes := lanes + if resultLanes < 8 { + resultLanes = 8 + } + intrinsic := map[int]string{ + 16: "llvm.x86.vcvtneps2bf16128", + 32: "llvm.x86.vcvtneps2bf16256", + 64: "llvm.x86.avx512bf16.cvtneps2bf16.512", + }[sourceBytes] + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <%d x bfloat> @%s(<%d x float> %%%s)\n", + converted, resultLanes, intrinsic, lanes, input) + computed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x bfloat> %%%s to <%d x i16>\n", + computed, resultLanes, converted, resultLanes) + result := "%" + computed + if lanes < resultLanes { + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i16> %s, <%d x i16> poison, <%d x i32> <", + low, resultLanes, result, resultLanes, lanes) + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + c.b.WriteString(", ") + } + fmt.Fprintf(c.b, "i32 %d", lane) + } + c.b.WriteString(">\n") + result = "%" + low + } + if masked { + old, loadErr := c.loadPackedExtendInputs(destination, destinationBytes, lanes, 16) + if loadErr != nil { + return true, false, loadErr + } + result = amd64ApplyIntegerLaneMask(c, lanes, 16, result, old, mask, zeroing) + } + outputBytes := lanes * 2 + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x i8>\n", + bytesValue, lanes, result, outputBytes) + return true, false, c.storePackedHalfResult(destination, destinationBytes, outputBytes, "%"+bytesValue) +} diff --git a/amd64_lower_binary_float.go b/amd64_lower_binary_float.go new file mode 100644 index 000000000..7e9db9138 --- /dev/null +++ b/amd64_lower_binary_float.go @@ -0,0 +1,322 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64BinaryFloatingMode uint8 + +const ( + amd64BinaryFloatingAdd amd64BinaryFloatingMode = iota + amd64BinaryFloatingSub + amd64BinaryFloatingMul + amd64BinaryFloatingDiv + amd64BinaryFloatingMax + amd64BinaryFloatingMin + amd64BinaryFloatingSqrt + amd64BinaryFloatingScale +) + +type amd64BinaryFloatingSpec struct { + laneBits int + mode amd64BinaryFloatingMode + scalar bool + sae bool + rawOnly bool +} + +// The residual catalog owns raw FP16 and SCALEF. Named f32/f64 arithmetic +// and SQRT dispatch through their complete typed source grammars first. +var amd64BinaryFloatingSpecs = map[Op]amd64BinaryFloatingSpec{ + "VADDPH": {laneBits: 16, mode: amd64BinaryFloatingAdd, rawOnly: true}, + "VSUBPH": {laneBits: 16, mode: amd64BinaryFloatingSub, rawOnly: true}, + "VMULPH": {laneBits: 16, mode: amd64BinaryFloatingMul, rawOnly: true}, + "VDIVPH": {laneBits: 16, mode: amd64BinaryFloatingDiv, rawOnly: true}, + "VMINPH": {laneBits: 16, mode: amd64BinaryFloatingMin, sae: true, rawOnly: true}, + "VMAXPH": {laneBits: 16, mode: amd64BinaryFloatingMax, sae: true, rawOnly: true}, + "VADDSH": {laneBits: 16, mode: amd64BinaryFloatingAdd, scalar: true, rawOnly: true}, + "VSUBSH": {laneBits: 16, mode: amd64BinaryFloatingSub, scalar: true, rawOnly: true}, + "VMULSH": {laneBits: 16, mode: amd64BinaryFloatingMul, scalar: true, rawOnly: true}, + "VDIVSH": {laneBits: 16, mode: amd64BinaryFloatingDiv, scalar: true, rawOnly: true}, + "VMINSH": {laneBits: 16, mode: amd64BinaryFloatingMin, scalar: true, sae: true, rawOnly: true}, + "VMAXSH": {laneBits: 16, mode: amd64BinaryFloatingMax, scalar: true, sae: true, rawOnly: true}, + "VSQRTSH": {laneBits: 16, mode: amd64BinaryFloatingSqrt, scalar: true, rawOnly: true}, + "VSCALEFPS": {laneBits: 32, mode: amd64BinaryFloatingScale}, + "VSCALEFPD": {laneBits: 64, mode: amd64BinaryFloatingScale}, + "VSCALEFSS": {laneBits: 32, mode: amd64BinaryFloatingScale, scalar: true}, + "VSCALEFSD": {laneBits: 64, mode: amd64BinaryFloatingScale, scalar: true}, +} + +type amd64BinaryFloatingSuffix struct { + broadcast bool + zeroing bool + rounding string + sae bool +} + +func parseAMD64BinaryFloatingSuffix(suffix string) (amd64BinaryFloatingSuffix, bool) { + switch suffix { + case "": + return amd64BinaryFloatingSuffix{}, true + case "Z": + return amd64BinaryFloatingSuffix{zeroing: true}, true + case "BCST": + return amd64BinaryFloatingSuffix{broadcast: true}, true + case "BCST.Z": + return amd64BinaryFloatingSuffix{broadcast: true, zeroing: true}, true + case "SAE": + return amd64BinaryFloatingSuffix{sae: true}, true + case "SAE.Z": + return amd64BinaryFloatingSuffix{sae: true, zeroing: true}, true + case "RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE": + return amd64BinaryFloatingSuffix{rounding: suffix}, true + case "RN_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", "RZ_SAE.Z": + return amd64BinaryFloatingSuffix{rounding: strings.TrimSuffix(suffix, ".Z"), zeroing: true}, true + default: + return amd64BinaryFloatingSuffix{}, false + } +} + +func (c *amd64Ctx) lowerBinaryFloating(op Op, ins Instr) (ok bool, terminated bool, err error) { + if known, term, err := c.lowerFP16Binary(op, ins); known { + return known, term, err + } + if known, term, err := c.lowerSqrt(op, ins); known { + return known, term, err + } + if ok, term, err := c.lowerFloatArithmetic(op, ins, ""); ok { + return ok, term, err + } + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64BinaryFloatingSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if spec.mode == amd64BinaryFloatingScale && x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return c.lowerNativeFloatingHelper(op, ins) + } + if spec.rawOnly && !ins.x86Encoded { + return true, false, fmt.Errorf("%s has no named Go assembler form: %q", baseOp, ins.Raw) + } + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix { + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's binary floating optab: %q", c.goarch, baseOp, ins.Raw) + } + if spec.scalar && properties.broadcast { + return true, false, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + if spec.sae { + if properties.rounding != "" { + return true, false, fmt.Errorf("%s %s supports SAE, not explicit rounding: %q", c.goarch, baseOp, ins.Raw) + } + } else if properties.sae { + return true, false, fmt.Errorf("%s %s supports explicit rounding, not SAE: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects src1, src2, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked && !spec.rawOnly { + return true, false, fmt.Errorf("386 %s mask forms exceed Go 1.27's assembler operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if spec.scalar { + if byteWidth != 16 { + return true, false, fmt.Errorf("%s %s scalar destination must be an X register: %q", c.goarch, baseOp, ins.Raw) + } + } else if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s packed destination must be an X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoEVEXVectorRegister(destination, byteWidth) || !c.isGoEVEXVectorRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("%s %s second source and destination must be same-width Go 1.27 vector registers: %q", c.goarch, baseOp, ins.Raw) + } + if (properties.rounding != "" || properties.sae) && !spec.scalar && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s packed rounding/SAE is available only for Z registers: %q", c.goarch, baseOp, ins.Raw) + } + if properties.broadcast { + if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s.BCST requires a memory first source: %q", c.goarch, baseOp, ins.Raw) + } + } else if ins.Args[0].Kind == OpReg { + if !c.isGoEVEXVectorRegister(ins.Args[0], byteWidth) { + return true, false, fmt.Errorf("%s %s first source must match the destination width and register class: %q", c.goarch, baseOp, ins.Raw) + } + } else if properties.rounding != "" || properties.sae { + return true, false, fmt.Errorf("%s %s explicit rounding/SAE requires a register first source: %q", c.goarch, baseOp, ins.Raw) + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s first source must be a vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + var mask string + if masked { + if !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + loadedMask, loadErr := c.loadK(ins.Args[2].Reg) + if loadErr != nil { + return true, false, loadErr + } + mask = loadedMask + } + if spec.scalar { + return c.lowerScalarBinaryFloating(spec, properties, ins, destination, mask) + } + return c.lowerPackedBinaryFloating(spec, properties, ins, destination, byteWidth, mask) +} + +func (c *amd64Ctx) lowerPackedBinaryFloating(spec amd64BinaryFloatingSpec, properties amd64BinaryFloatingSuffix, ins Instr, destination Operand, byteWidth int, mask string) (bool, bool, error) { + if spec.mode == amd64BinaryFloatingScale { + return c.lowerPackedScaleF(spec, properties, ins.Args[0], ins.Args[1], destination, byteWidth, mask) + } + // The typed f32/f64 owners return before this fallback. Its remaining + // arithmetic consumers are raw FP16, including portable target lowering. + lanes := byteWidth * 8 / spec.laneBits + llvmType := amd64FMA3LLVMType(lanes, spec.laneBits) + load := func(operand Operand, allowBroadcast bool) (string, error) { + sourceMask := "" + if allowBroadcast { + sourceMask = mask + } + bits, err := c.loadMaskedPackedCompareLanes(operand, byteWidth, spec.laneBits, allowBroadcast && properties.broadcast, sourceMask) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", value, lanes, spec.laneBits, bits, llvmType) + return "%" + value, nil + } + first, err := load(ins.Args[0], true) + if err != nil { + return true, false, err + } + second, err := load(ins.Args[1], false) + if err != nil { + return true, false, err + } + computed := c.emitBinaryFloating(spec, llvmType, lanes, second, first, properties.rounding) + if mask != "" { + computedBits := c.bitcastFloatingToIntegerLanes(llvmType, lanes, spec.laneBits, computed) + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + oldBits := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + masked := amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computedBits, oldBits, mask, properties.zeroing) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", back, lanes, spec.laneBits, masked, llvmType) + computed = "%" + back + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, llvmType, computed, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarBinaryFloating(spec amd64BinaryFloatingSpec, properties amd64BinaryFloatingSuffix, ins Instr, destination Operand, mask string) (bool, bool, error) { + if spec.mode == amd64BinaryFloatingScale { + return c.lowerScalarScaleF(spec, properties, ins.Args[0], ins.Args[1], destination, mask) + } + // SCALEF has returned; only the portable raw half-precision path remains. + var firstBits, secondBits string + var err error + if ins.Args[0].Kind == OpReg { + firstBits, err = c.loadXLowInteger(ins.Args[0].Reg, 16) + } else if mask != "" { + firstBits, err = c.loadVectorScalarMemory(ins.Args[0], 16, mask) + } else { + firstBits, err = c.evalIntSized(ins.Args[0], "i16") + } + if err == nil { + secondBits, err = c.loadXLowInteger(ins.Args[1].Reg, 16) + } + if err != nil { + return true, false, err + } + firstTemp, secondTemp := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i16 %s to half\n", firstTemp, firstBits) + fmt.Fprintf(c.b, " %%%s = bitcast i16 %s to half\n", secondTemp, secondBits) + first, second := "%"+firstTemp, "%"+secondTemp + llvmType := amd64FMA3LLVMType(1, spec.laneBits) + computed := c.emitBinaryFloating(spec, llvmType, 1, second, first, properties.rounding) + computedBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", computedBits, llvmType, computed, spec.laneBits) + secondBytes, err := c.loadX(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, secondBytes) + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.laneBits, "%"+computedBits, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitBinaryFloating(spec amd64BinaryFloatingSpec, llvmType string, lanes int, left, right, rounding string) string { + result := c.newTmp() + switch spec.mode { + case amd64BinaryFloatingSqrt: + if rounding == "" { + fmt.Fprintf(c.b, " %%%s = call %s @llvm.sqrt.%s(%s %s)\n", result, llvmType, amd64BinaryFloatingIntrinsicSuffix(lanes, spec.laneBits), llvmType, right) + } else { + intrinsic := fmt.Sprintf("llvm.experimental.constrained.sqrt.%s", amd64BinaryFloatingIntrinsicSuffix(lanes, spec.laneBits)) + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, metadata !\"%s\", metadata !\"fpexcept.ignore\")\n", result, llvmType, intrinsic, llvmType, right, amd64FMA3RoundingMetadata(rounding)) + } + case amd64BinaryFloatingMax, amd64BinaryFloatingMin: + predicate := "ogt" + if spec.mode == amd64BinaryFloatingMin { + predicate = "olt" + } + comparison := c.newTmp() + predicateType := "i1" + if lanes > 1 { + predicateType = fmt.Sprintf("<%d x i1>", lanes) + } + fmt.Fprintf(c.b, " %%%s = fcmp %s %s %s, %s\n", comparison, predicate, llvmType, left, right) + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", result, predicateType, comparison, llvmType, left, llvmType, right) + default: + operation := "fadd" + switch spec.mode { + case amd64BinaryFloatingSub: + operation = "fsub" + case amd64BinaryFloatingMul: + operation = "fmul" + case amd64BinaryFloatingDiv: + operation = "fdiv" + } + if rounding == "" { + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", result, operation, llvmType, left, right) + } else { + intrinsic := fmt.Sprintf("llvm.experimental.constrained.%s.%s", operation, amd64BinaryFloatingIntrinsicSuffix(lanes, spec.laneBits)) + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, metadata !\"%s\", metadata !\"fpexcept.ignore\")\n", result, llvmType, intrinsic, llvmType, left, llvmType, right, amd64FMA3RoundingMetadata(rounding)) + } + } + return "%" + result +} + +func (c *amd64Ctx) bitcastFloatingToIntegerLanes(llvmType string, lanes, laneBits int, value string) string { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i%d>\n", result, llvmType, value, lanes, laneBits) + return "%" + result +} + +func amd64BinaryFloatingIntrinsicSuffix(lanes, laneBits int) string { + suffix := "f32" + if laneBits == 16 { + suffix = "f16" + } else if laneBits == 64 { + suffix = "f64" + } + if lanes > 1 { + return fmt.Sprintf("v%d%s", lanes, suffix) + } + return suffix +} diff --git a/amd64_lower_bittest.go b/amd64_lower_bittest.go new file mode 100644 index 000000000..ca04dd1ba --- /dev/null +++ b/amd64_lower_bittest.go @@ -0,0 +1,201 @@ +package plan9asm + +import "fmt" + +// lowerBitTestFamily implements the complete Go 1.27 ybtl table. BTW/L/Q, +// BTCW/L/Q, BTRW/L/Q, and BTSW/L/Q all accept exactly either a signed imm8 or +// a Yrl register bit index followed by a Yml register/memory destination. +func (c *amd64Ctx) lowerBitTestFamily(op Op, ins Instr) (ok bool, terminated bool, err error) { + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86BitTestProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + if form.destination.Kind != OpReg && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + return true, false, fmt.Errorf("%w: bit string needs its original native address size", ErrProbeNeedsContext) + } + stem, bits := form.stem, form.bits + typ := amd64IntegerTypeForBits(bits) + bitIndex, memoryWordOffset, registerIndex, err := c.bitTestIndex(form.index, bits, typ) + if err != nil { + return true, false, err + } + value, store, err := c.loadBitTestDestination(form.destination, typ, memoryWordOffset, registerIndex) + if err != nil { + return true, false, err + } + + shifted := c.newTmp() + bit := c.newTmp() + carry := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", shifted, typ, value, bitIndex) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 1\n", bit, typ, shifted) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, 0\n", carry, typ, bit) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + if stem == "BT" { + return true, false, nil + } + + bitMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s 1, %s\n", bitMask, typ, bitIndex) + result := c.newTmp() + switch stem { + case "BTC": + fmt.Fprintf(c.b, " %%%s = xor %s %s, %%%s\n", result, typ, value, bitMask) + case "BTR": + inverseMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %%%s, -1\n", inverseMask, typ, bitMask) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", result, typ, value, inverseMask) + case "BTS": + fmt.Fprintf(c.b, " %%%s = or %s %s, %%%s\n", result, typ, value, bitMask) + } + return true, false, store("%" + result) +} + +func amd64BitTestProperties(op string) (stem string, bits int, ok bool) { + switch op { + case "BTW": + return "BT", 16, true + case "BTL": + return "BT", 32, true + case "BTQ": + return "BT", 64, true + case "BTCW": + return "BTC", 16, true + case "BTCL": + return "BTC", 32, true + case "BTCQ": + return "BTC", 64, true + case "BTRW": + return "BTR", 16, true + case "BTRL": + return "BTR", 32, true + case "BTRQ": + return "BTR", 64, true + case "BTSW": + return "BTS", 16, true + case "BTSL": + return "BTS", 32, true + case "BTSQ": + return "BTS", 64, true + } + return "", 0, false +} + +// bitTestIndex returns the within-word index and, for a register index, the +// signed byte offset used when a memory destination is treated as an unbounded +// bit string. The index register has the same width as the instruction. +// Intel SDM Vol.2A, Bit(BitBase, BitOffset) and BT/BTC/BTR/BTS: +// https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf +// Immediate high bits are ignored by the CPU; unlike some assemblers, actual +// Go does not fold them into a new displacement (tested for every imm8). +func (c *amd64Ctx) bitTestIndex(index Operand, bits int, typ LLVMType) (bitIndex, memoryWordOffset string, registerIndex bool, err error) { + if index.Kind == OpImm { + return fmt.Sprintf("%d", index.Imm&int64(bits-1)), "", false, nil + } + value, err := c.evalIntSized(index, typ) + if err != nil { + return "", "", false, err + } + withinWord := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %d\n", withinWord, typ, value, bits-1) + bitIndex = "%" + withinWord + + signedIndex := value + if bits != 64 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext %s %s to i64\n", extended, typ, value) + signedIndex = "%" + extended + } + wordIndex := c.newTmp() + byteOffset := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ashr i64 %s, %d\n", wordIndex, signedIndex, bitTestLog2(bits)) + fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, %d\n", byteOffset, wordIndex, bits/8) + return bitIndex, "%" + byteOffset, true, nil +} + +func bitTestLog2(bits int) int { + switch bits { + case 16: + return 4 + case 32: + return 5 + default: + return 6 + } +} + +func (c *amd64Ctx) loadBitTestDestination(destination Operand, typ LLVMType, memoryWordOffset string, registerIndex bool) (string, func(string) error, error) { + if destination.Kind == OpReg { + return c.loadIntDestination(destination, typ) + } + if destination.Kind == OpFP { + if registerIndex { + return "", nil, fmt.Errorf("%w: a register-indexed FP bit string has no fixed logical byte span", ErrProbeNeedsContext) + } + return c.loadBitTestFPDestination(destination, typ) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(destination) + if err != nil { + return "", nil, err + } + if registerIndex { + adjusted := c.newTmp() + // Signed floor division selects the complete containing word. Keep + // the original pointer address space and its target-width wrapping. + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %s\n", adjusted, pointerType, pointer, memoryWordOffset) + pointer = "%" + adjusted + } + loaded := c.newTmp() + // Intel SDM Vol.2A BT/BTC/BTR/BTS: even a dead CF retains the complete + // native read/fault; modifying forms also retain their write, including + // when the selected bit already has the requested value. This is not LOCK. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", loaded, typ, pointerType, pointer) + return "%" + loaded, func(value string) error { + fmt.Fprintf(c.b, " store volatile %s %s, %s %s, align 1\n", typ, value, pointerType, pointer) + return nil + }, nil +} + +func (c *amd64Ctx) loadBitTestFPDestination(destination Operand, typ LLVMType) (string, func(string) error, error) { + bits, _ := amd64IntegerTypeBits(typ) + width := bits / 8 + result, err := c.bitTestFPSpan(destination.FPOffset, width) + if err != nil { + return "", nil, err + } + if result { + if err := c.scalarFPResultInitialized(); err != nil { + return "", nil, err + } + } + bytes, err := c.loadX86PackedShiftBytes(destination, width, true) + if err != nil { + return "", nil, err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", value, width, bytes, typ) + return "%" + value, func(value string) error { + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytes, typ, value, width) + return c.storeFPVectorBytes(destination.FPOffset, width, "%"+bytes, "") + }, nil +} + +func (c *amd64Ctx) bitTestFPSpan(off int64, width int) (bool, error) { + if err := c.validateX86PackedShiftFPSpan(off, width); err != nil { + return false, err + } + result := false + for offset := off; offset < off+int64(width); offset++ { + slot, output, _ := c.fpVectorSlot(offset) + if slot.Type == Ptr || slot.Type == I1 { + return false, fmt.Errorf("%w: FP bit string needs declared numerical bytes", ErrProbeNeedsContext) + } + result = result || output + } + return result, nil +} diff --git a/amd64_lower_blend_imm.go b/amd64_lower_blend_imm.go new file mode 100644 index 000000000..ca0938fcc --- /dev/null +++ b/amd64_lower_blend_imm.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerImmediatePackedBlend implements the complete Go 1.27 immediate +// packed-blend family. The legacy instructions use yxshuf, while all four +// V-prefixed instructions use _yvblendpd and therefore have VEX X/Y forms +// only (no EVEX Z, mask, zeroing, broadcast, or SAE forms). +func (c *amd64Ctx) lowerImmediatePackedBlend(op Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86ImmediateBlendProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if f.legacy && f.memory { + return true, false, c.lowerLegacyImmediateBlendMemory(ins, f) + } + first, err := c.loadX86PackedShiftBytes(ins.Args[f.source], f.byteWidth, true) + if err != nil { + return true, false, err + } + base, err := c.loadPackedCompareBytes(ins.Args[f.base], f.byteWidth) + if err != nil { + return true, false, err + } + result := c.emitImmediatePackedBlend(f.byteWidth, f.laneBytes*8, f.selector, first, base) + if f.legacy { + return true, false, c.storeLegacyXViews(ins.Args[f.destination].Reg, result) + } + return true, false, c.storePackedMoveOperand(ins.Args[f.destination], f.byteWidth, result) +} + +func amd64ImmediatePackedBlendProperties(op string) (laneBits int, vector, ok bool) { + switch op { + case "PBLENDW": + return 16, false, true + case "BLENDPS": + return 32, false, true + case "BLENDPD": + return 64, false, true + case "VPBLENDW": + return 16, true, true + case "VPBLENDD", "VBLENDPS": + return 32, true, true + case "VBLENDPD": + return 64, true, true + default: + return 0, false, false + } +} + +// Legacy Type4 m128 retains its original alignment/fault behavior, including +// imm8=0 and dead results. Logical FP bytes are not native aligned backing. +func (c *amd64Ctx) lowerLegacyImmediateBlendMemory(ins Instr, f x86ImmediateBlendForm) error { + source := ins.Args[f.source] + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || source.Kind == OpFP { + return fmt.Errorf("%w: legacy immediate blend needs original native m128 storage", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + base, err := c.loadPackedCompareBytes(ins.Args[f.base], 16) + if err != nil { + return err + } + carrier, result, bytes := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x float>\n", carrier, base) + assembly := fmt.Sprintf("%s $$%d, %s($2), $0", strings.ToLower(string(ins.Op)), f.selector, segment) + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(<4 x float> %%%s, %s %s)\n", + result, assembly, "=x,0,r,~{memory}", carrier, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <16 x i8>\n", bytes, result) + return c.storeLegacyXViews(ins.Args[f.destination].Reg, "%"+bytes) +} + +// emitImmediatePackedBlend selects the first (r/m) source when the immediate +// bit is one and the second source when it is zero. VPBLENDW repeats the eight +// immediate bits independently in each 128-bit lane of a Y register. +func (c *amd64Ctx) emitImmediatePackedBlend(byteWidth, laneBits int, imm uint8, firstBytes, secondBytes string) string { + lanes := byteWidth * 8 / laneBits + first := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, firstBytes) + second := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, secondBytes) + mask := make([]string, 0, lanes) + for lane := 0; lane < lanes; lane++ { + if imm&(1< %s, <%d x i%d> %s, <%d x i32> <%s>\n", + shuffled, lanes, laneBits, first, lanes, laneBits, second, lanes, strings.Join(mask, ", ")) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <%d x i8>\n", out, lanes, laneBits, shuffled, byteWidth) + return "%" + out +} diff --git a/amd64_lower_blend_mask.go b/amd64_lower_blend_mask.go new file mode 100644 index 000000000..167ff5a90 --- /dev/null +++ b/amd64_lower_blend_mask.go @@ -0,0 +1,60 @@ +package plan9asm + +import "fmt" + +type amd64MaskBlendSpec struct { + laneBits int + broadcast bool +} + +// amd64MaskBlendSpecs is the complete shared _yvblendmpd grammar for the +// floating-point and integer mask-blend families. +var amd64MaskBlendSpecs = map[Op]amd64MaskBlendSpec{ + "VBLENDMPS": {laneBits: 32, broadcast: true}, + "VBLENDMPD": {laneBits: 64, broadcast: true}, + "VPBLENDMB": {laneBits: 8}, + "VPBLENDMW": {laneBits: 16}, + "VPBLENDMD": {laneBits: 32, broadcast: true}, + "VPBLENDMQ": {laneBits: 64, broadcast: true}, +} + +// lowerMaskBlend implements Go 1.27's complete _yvblendmpd operand table for +// VBLENDM{PS,PD} and VPBLENDM{B,W,D,Q}. A set K bit selects the first Plan 9 +// (Intel r/m) source; an unset bit selects the second source, or zero for .Z. +func (c *amd64Ctx) lowerMaskBlend(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86MaskBlendProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := c.validateConflictFunnelMemory(ins, form.x86PackedBinaryForm); err != nil { + return true, false, err + } + mask := "" + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + // E4 suppresses only inactive memory elements. The shared volatile reader + // retains active faults even when all result bytes are discarded after O2. + first, err := c.loadX86PackedShiftLanes(ins.Args[form.first], form.byteWidth, form.spec.laneBits, form.broadcast, mask) + if err != nil { + return true, false, err + } + result := first + lanes := form.byteWidth * 8 / form.spec.laneBits + if form.mask >= 0 { + second := "zeroinitializer" + if !form.zeroing { + second, err = c.loadPackedCompareLanes(ins.Args[form.second], form.byteWidth, form.spec.laneBits, false) + if err != nil { + return true, false, err + } + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.spec.laneBits, first, second, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.spec.laneBits, result, form.byteWidth) + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, "%"+out) +} diff --git a/amd64_lower_blend_mask_test.go b/amd64_lower_blend_mask_test.go new file mode 100644 index 000000000..d8ec2557f --- /dev/null +++ b/amd64_lower_blend_mask_test.go @@ -0,0 +1,234 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64MaskBlendGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64MaskBlendSpec{ + "VBLENDMPS": {laneBits: 32, broadcast: true}, + "VBLENDMPD": {laneBits: 64, broadcast: true}, + "VPBLENDMB": {laneBits: 8}, + "VPBLENDMW": {laneBits: 16}, + "VPBLENDMD": {laneBits: 32, broadcast: true}, + "VPBLENDMQ": {laneBits: 64, broadcast: true}, + } + if len(amd64MaskBlendSpecs) != len(expected) { + t.Fatalf("mask-blend grammar has %d entries, want %d", len(amd64MaskBlendSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64MaskBlendSpecs[op]; !ok { + t.Errorf("mask-blend grammar omitted %s", op) + } else if got != want { + t.Errorf("mask-blend grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86MaskBlendCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 shares _yvblendmpd across VBLENDM{PS,PD} and + // VPBLENDM{B,W,D,Q}: matching X/Y/Z sources and destination, optional + // K1-K7 selection mask, and .Z. PS/PD/D/Q additionally enable + // scalar-memory broadcast on the first Plan 9 source. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT maskblendforms(SB),$0-0\n") + for _, family := range []struct { + op string + broadcast bool + }{ + {op: "VBLENDMPS", broadcast: true}, + {op: "VBLENDMPD", broadcast: true}, + {op: "VPBLENDMB"}, + {op: "VPBLENDMW"}, + {op: "VPBLENDMD", broadcast: true}, + {op: "VPBLENDMQ", broadcast: true}, + } { + op := family.op + fmt.Fprintf(&source, "\t%s X1, X20, X21\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s Z1, Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X20, K1, X21\n", op) + fmt.Fprintf(&source, "\t%s.Z Y1, Y20, K2, Y21\n", op) + fmt.Fprintf(&source, "\t%s.Z 40(AX), Z20, K3, Z21\n", op) + } + if family.broadcast { + fmt.Fprintf(&source, "\t%s.BCST 104(AX), X20, X21\n", op) + fmt.Fprintf(&source, "\t%s.BCST 112(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s.BCST 120(AX), Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z 128(AX), Z20, K4, Z21\n", op) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, signatures := x86UnboundVoidFormatSource(t, source.String(), + "maskblendforms", target.goarch, target.triple) + ll := x86InstructionFormatObjectUnit(t, file, signatures, target.goarch, + target.triple, 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + // Go accepts EVEX high banks in named mode32 fixtures; + // this private emitter view does not bind public registers. + arch := c.goarch + if arch == "386" && x86Mode32VectorRegisterContext(ins, arch) != nil { + c.goarch = "amd64" + defer func() { c.goarch = arch }() + } + return c.lowerMaskBlend(ins.Op, ins) + }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "mask-blend-"+target.name+".ll", "mask-blend-"+target.name+".o", ll) + }) + } +} + +func TestAMD64FloatingMaskBlendRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT floatingmaskblendsemantics(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VBLENDMPS Z0, Z1, K1, Z2 + VMOVDQU64 Z2, 0(AX) + VBLENDMPD.Z Z0, Z1, K1, Z2 + VMOVDQU64 Z2, 64(AX) + VBLENDMPS.BCST 0(BX), Z1, K1, Z2 + VMOVDQU64 Z2, 128(AX) + VBLENDMPD.BCST.Z 0(BX), Z1, K1, Z2 + VMOVDQU64 Z2, 192(AX) + STC + VBLENDMPS Z0, Z1, K1, Z2 + SETCS 256(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"floatingmaskblendsemantics": { + Name: "floatingmaskblendsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void floatingmaskblendsemantics(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +int main(void) { + uint32_t first32[16], second32[16]; uint8_t out[257] = {0}; + const uint64_t mask = UINT64_C(0xa55a); + for (int i=0;i<16;i++) { first32[i]=UINT32_C(0x7f000000)+i; second32[i]=UINT32_C(0x3f000000)+i; } + floatingmaskblendsemantics(out,(const uint8_t *)first32,(const uint8_t *)second32,mask); + uint32_t *ps=(uint32_t *)(out+0), *psbcst=(uint32_t *)(out+128); + for (int i=0;i<16;i++) { + uint32_t want=(mask>>i)&1 ? first32[i] : second32[i]; if (ps[i]!=want) return 10+i; + want=(mask>>i)&1 ? first32[0] : second32[i]; if (psbcst[i]!=want) return 30+i; + } + uint64_t first64[8], second64[8], pdzero[8], pdbczero[8]; + memcpy(first64,first32,64); memcpy(second64,second32,64); + memcpy(pdzero,out+64,64); memcpy(pdbczero,out+192,64); + for (int i=0;i<8;i++) { + uint64_t want=(mask>>i)&1 ? first64[i] : 0; if (pdzero[i]!=want) return 50+i; + want=(mask>>i)&1 ? first64[0] : 0; if (pdbczero[i]!=want) return 70+i; + } + if (out[256]!=1) return 90; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "floating_mask_blend_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateX86MaskBlendRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VBLENDMPS X0, Y1, Y2", + "VBLENDMPD X0, X1, K0, X2", + "VBLENDMPS.BCST X0, X1, X2", + "VBLENDMPD.RN_SAE X0, X1, X2", + "VPBLENDMQ X0, X1", + "VPBLENDMQ X0, Y1, Y2", + "VPBLENDMQ X0, (AX), X2", + "VPBLENDMQ X0, X1, K0, X2", + "VPBLENDMQ X0, X1, AX", + "VPBLENDMQ.Z X0, X1, X2", + "VPBLENDMQ.BCST X0, X1, X2", + "VPBLENDMB.BCST (AX), X1, X2", + "VPBLENDMW.BCST (AX), X1, X2", + "VPBLENDMQ.Z.BCST (AX), X1, K1, X2", + "VPBLENDMQ.RN_SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86MaskBlendRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\tVPBLENDMQ X0, X1, K1, X2\n\tRET\n", false) + assertX86MaskBlendRejected(t, "386", "i386-unknown-linux-gnu", "VPBLENDMQ X0, X1, K1, X2") + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\tVPBLENDMQ Z0, Z1, Z8\n\tRET\n", false) + assertX86MaskBlendRejected(t, "386", "i386-unknown-linux-gnu", "VPBLENDMQ Z0, Z1, Z8") +} + +func assertX86MaskBlendRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPBLENDM forms", instruction) + } +} diff --git a/amd64_lower_blend_variable.go b/amd64_lower_blend_variable.go new file mode 100644 index 000000000..d7b298ea5 --- /dev/null +++ b/amd64_lower_blend_variable.go @@ -0,0 +1,68 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64VariableBlendSpec struct { + laneBits int + vector bool + opcode int +} + +// Go's yblendvpd and _yvblendvpd differ only in implicit/explicit mask, +// destructive/non-destructive destination and the available vector widths. +// The VEX raw decoder consumes the same opcode table as named lowering. +var amd64VariableBlendSpecs = map[Op]amd64VariableBlendSpec{ + "BLENDVPS": {laneBits: 32, opcode: 0x14}, + "BLENDVPD": {laneBits: 64, opcode: 0x15}, + "PBLENDVB": {laneBits: 8, opcode: 0x10}, + "VBLENDVPS": {laneBits: 32, vector: true, opcode: 0x4a}, + "VBLENDVPD": {laneBits: 64, vector: true, opcode: 0x4b}, + "VPBLENDVB": {laneBits: 8, vector: true, opcode: 0x4c}, +} + +// lowerVariableBlend implements Go 1.27's complete yblendvpd and _yvblendvpd +// families. Each opcode tests the high bit of its mask lane: false selects the +// base/destination lane and true selects the register-or-memory source lane. +func (c *amd64Ctx) lowerVariableBlend(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, recognized, err := x86BlendMinposProperties(op, ins, c.goarch) + if !recognized || form.minimum { + return false, false, nil + } + if err != nil { + return true, false, err + } + laneBits, byteWidth := form.laneBits, form.byteWidth + maskArg, source := ins.Args[form.mask], ins.Args[form.source] + base, destination := ins.Args[form.base], ins.Args[form.destination] + if form.legacy && form.memory { + return true, false, c.lowerLegacyBlendMinposMemory(ins, form) + } + maskBytes, err := c.loadPackedCompareBytes(maskArg, byteWidth) + if err != nil { + return true, false, err + } + sourceBytes, err := c.loadX86PackedShiftBytes(source, byteWidth, true) + if err != nil { + return true, false, err + } + baseBytes, err := c.loadPackedCompareBytes(base, byteWidth) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / laneBits + maskLanes := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, maskBytes) + sourceLanes := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, sourceBytes) + baseLanes := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, baseBytes) + selectedMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i%d> %s, zeroinitializer\n", selectedMask, lanes, laneBits, maskLanes) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %s, <%d x i%d> %s\n", result, lanes, selectedMask, lanes, laneBits, sourceLanes, lanes, laneBits, baseLanes) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + if !form.legacy { + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) + } + return true, false, c.storeLegacyXViews(destination.Reg, "%"+out) +} diff --git a/amd64_lower_blend_variable_test.go b/amd64_lower_blend_variable_test.go new file mode 100644 index 000000000..b97d5046d --- /dev/null +++ b/amd64_lower_blend_variable_test.go @@ -0,0 +1,97 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86VariableBlendCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's yblendvpd and _yvblendvpd tables are shared by the + // PS (dword lanes), PD (qword lanes), and PBLENDVB (byte lanes) variants. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + var src strings.Builder + src.WriteString("TEXT variableblendforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"BLENDVPS", "BLENDVPD", "PBLENDVB"} { + fmt.Fprintf(&src, "\t%s X0, (AX), X1\n", op) + fmt.Fprintf(&src, "\t%s X0, X2, X%d\n", op, legacyLast) + } + for _, op := range []string{"VBLENDVPS", "VBLENDVPD", "VPBLENDVB"} { + fmt.Fprintf(&src, "\t%s X0, (AX), X1, X2\n", op) + fmt.Fprintf(&src, "\t%s X12, X13, X14, X15\n", op) + fmt.Fprintf(&src, "\t%s Y0, 32(AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Y12, Y13, Y14, Y15\n", op) + } + src.WriteString("\tRET\n") + var ll string + if target.goarch == "386" { + ll = x86BlendMinposOriginal386NamedFormats(t, src.String(), target.triple) + } else { + ll = x86UnboundBlendMinposFormatObject(t, src.String(), target.goarch, target.triple) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "variable-blend-"+target.name+".ll", "variable-blend-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86VariableBlendRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "BLENDVPS X1, X2, X3", + "BLENDVPD X0, Y1, X2", + "PBLENDVB X0, X1, Y2", + "BLENDVPS.Z X0, X1, X2", + "VBLENDVPS X0, X1, X2", + "VBLENDVPD X0, X1, X2, X3, X4", + "VPBLENDVB (AX), X1, X2, X3", + "VBLENDVPS X0, Y1, X2, X3", + "VBLENDVPD Y0, Y1, X2, Y3", + "VPBLENDVB Y0, Y1, Y2, X3", + "VBLENDVPS.Z X0, X1, X2, X3", + "VBLENDVPD X16, X1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86VariableBlendRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "BLENDVPS X0, X8, X1", + "BLENDVPD X0, X1, X8", + "PBLENDVB X8, X1, X2", + } { + assertX86VariableBlendRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86VariableBlendRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's variable blend tables", instruction) + } +} diff --git a/amd64_lower_bls.go b/amd64_lower_bls.go new file mode 100644 index 000000000..17fb6b9b5 --- /dev/null +++ b/amd64_lower_bls.go @@ -0,0 +1,52 @@ +package plan9asm + +import "fmt" + +// lowerLowestSetBitManipulation implements Go 1.27's complete _yblsil table: +// BLSI, BLSMSK, and BLSR at L/Q widths, with a Yml register-or-memory source +// and a Yrl destination register. +func (c *amd64Ctx) lowerLowestSetBitManipulation(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86BLSProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + typ := amd64IntegerTypeForBits(form.spec.bits) + value, err := c.evalBLSSource(ins, form, typ) + if err != nil { + return true, false, err + } + minusOne := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %s, 1\n", minusOne, typ, value) + result := c.newTmp() + carry := "" + switch form.spec.kind { + case x86BLSIsolate: + negative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", negative, typ, value) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", result, typ, value, negative) + carry = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne %s %s, 0\n", carry, typ, value) + case x86BLSMask: + fmt.Fprintf(c.b, " %%%s = xor %s %s, %%%s\n", result, typ, value, minusOne) + carry = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", carry, typ, value) + case x86BLSReset: + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", result, typ, value, minusOne) + carry = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", carry, typ, value) + } + + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + if form.spec.kind == x86BLSMask { + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsZSlot) + } else { + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + } + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", sign, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + return true, false, c.storeRegSized(form.destination.Reg, typ, "%"+result) +} diff --git a/amd64_lower_bls_test.go b/amd64_lower_bls_test.go new file mode 100644 index 000000000..fcefdd328 --- /dev/null +++ b/amd64_lower_bls_test.go @@ -0,0 +1,293 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86RawBLSMSKReportedEncoding(t *testing.T) { + const source = `TEXT rawBLSMSK(SB), NOSPLIT, $0-0 + BYTE $0xc4 + BYTE $0xe2 + BYTE $0x88 + BYTE $0xf3 + BYTE $0xd2 + RET +` + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + if !strings.Contains(ir, `"target-features"="+bmi"`) { + t.Fatalf("raw BLSMSK did not infer BMI1 target feature:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-blsmsk-reported.ll", "raw-blsmsk-reported.o", ir) +} + +func TestTranslateX86BLSCompleteGoAssemblerForms(t *testing.T) { + registers386 := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} + registersAMD64 := append(append([]string{}, registers386...), "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + targets := []struct { + name string + goarch string + triple string + widths []string + registers []string + }{ + {name: "386-linux", goarch: "386", triple: "i386-unknown-linux-gnu", widths: []string{"L", "Q"}, registers: registers386}, + {name: "386-windows", goarch: "386", triple: "i686-pc-windows-msvc", widths: []string{"L", "Q"}, registers: registers386}, + {name: "amd64-darwin", goarch: "amd64", triple: "x86_64-apple-darwin", widths: []string{"L", "Q"}, registers: registersAMD64}, + {name: "amd64-linux", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", widths: []string{"L", "Q"}, registers: registersAMD64}, + {name: "amd64-windows", goarch: "amd64", triple: "x86_64-pc-windows-msvc", widths: []string{"L", "Q"}, registers: registersAMD64}, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT blsforms(SB),$0-0\n") + for _, family := range []string{"BLSI", "BLSMSK", "BLSR"} { + for _, width := range target.widths { + op := family + width + for index, register := range target.registers { + other := target.registers[(index+5)%len(target.registers)] + fmt.Fprintf(&source, "\t%s %s, %s\n", op, register, other) + fmt.Fprintf(&source, "\t%s %s, %s\n", op, other, register) + } + fmt.Fprintf(&source, "\t%s 8(AX), DX\n", op) + fmt.Fprintf(&source, "\t%s 16(AX)(CX*4), BX\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "blsforms", target.goarch, target.triple) + if !strings.Contains(ir, `"target-features"="+bmi"`) { + t.Fatalf("BLS family did not infer BMI1 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "bls-family.ll", "bls-family.o", ir) + }) + } +} + +func TestTranslateX86BLSRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "386", instruction: "BLSIL R8, AX"}, + {goarch: "386", instruction: "BLSIQ R8, AX"}, + {goarch: "386", instruction: "BLSIL AX, R8"}, + {goarch: "386", instruction: "BLSIQ AX, R8"}, + {goarch: "amd64", instruction: "BLSIW AX, DX"}, + {goarch: "amd64", instruction: "BLSIL AL, DX"}, + {goarch: "amd64", instruction: "BLSIL X0, DX"}, + {goarch: "amd64", instruction: "BLSIL $1, DX"}, + {goarch: "amd64", instruction: "BLSIL AX, 0(DX)"}, + {goarch: "amd64", instruction: "BLSIL AX"}, + {goarch: "amd64", instruction: "BLSIL AX, DX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: test.goarch, TargetTriple: triple, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yblsil table", test.instruction) + } + }) + } +} + +func TestX86RawBLSCompleteRegisterFields(t *testing.T) { + families := []struct { + digit byte + name string + }{ + {digit: 1, name: "BLSR"}, + {digit: 2, name: "BLSMSK"}, + {digit: 3, name: "BLSI"}, + } + for _, mode := range []int{32, 64} { + registerCount := 8 + widths := []bool{false, true} + if mode == 64 { + registerCount = 16 + } + for _, family := range families { + for _, width64 := range widths { + for destinationNumber := 0; destinationNumber < registerCount; destinationNumber++ { + for sourceNumber := 0; sourceNumber < registerCount; sourceNumber++ { + vex1 := byte(0xe2) + if sourceNumber >= 8 { + vex1 &^= 0x20 + } + vex2 := byte((^destinationNumber & 15) << 3) + if width64 { + vex2 |= 0x80 + } + modRM := byte(0xc0 | int(family.digit)<<3 | sourceNumber&7) + code := []byte{0xc4, vex1, vex2, 0xf3, modRM} + instruction, length, ok, err := decodedX86BLSInstruction(code, mode) + if err != nil || !ok || length != len(code) { + t.Fatalf("mode=%d %x: instruction=%#v length=%d ok=%v err=%v", mode, code, instruction, length, ok, err) + } + width := "L" + if width64 && mode == 64 { + width = "Q" + } + source, _ := decodedX86GeneralRegister(sourceNumber) + destination, _ := decodedX86GeneralRegister(destinationNumber) + if instruction.Op != Op(family.name+width) || len(instruction.Args) != 2 || instruction.Args[0].Reg != source || instruction.Args[1].Reg != destination { + t.Fatalf("mode=%d %x decoded as %#v, want %s%s %s, %s", mode, code, instruction, family.name, width, source, destination) + } + } + } + } + } + } +} + +func TestX86RawBLSCompleteMemoryFormsAndTargets(t *testing.T) { + targets := []struct { + name string + goarch string + triple string + encodings [][]byte + }{ + { + name: "386-linux", goarch: "386", triple: "i386-unknown-linux-gnu", + encodings: [][]byte{ + {0xc4, 0xe2, 0x70, 0xf3, 0x18}, // BLSIL 0(AX), CX + {0xc4, 0xe2, 0x68, 0xf3, 0x54, 0x48, 0x08}, // BLSMSKL 8(AX)(CX*2), DX + {0xc4, 0xe2, 0x60, 0xf3, 0x0d, 0x78, 0x56, 0x34, 0x12}, // BLSRL 0x12345678, BX + }, + }, + { + name: "386-windows", goarch: "386", triple: "i686-pc-windows-msvc", + encodings: [][]byte{ + {0xc4, 0xe2, 0x70, 0xf3, 0x18}, + {0xc4, 0xe2, 0x68, 0xf3, 0x54, 0x48, 0x08}, + {0xc4, 0xe2, 0x60, 0xf3, 0x0d, 0x78, 0x56, 0x34, 0x12}, + }, + }, + { + name: "amd64-darwin", goarch: "amd64", triple: "x86_64-apple-darwin", + encodings: [][]byte{ + {0xc4, 0xe2, 0x88, 0xf3, 0xd2}, // reported BLSMSKQ DX, R14 + {0xc4, 0x82, 0x88, 0xf3, 0x5c, 0x88, 0x10}, // BLSIQ 16(R8)(R9*4), R14 + {0x64, 0xc4, 0xe2, 0x80, 0xf3, 0x4c, 0x48, 0x08}, // BLSRQ FS:8(AX)(CX*2), R15 + {0xc4, 0xe2, 0x80, 0xf3, 0x1c, 0x25, 0x78, 0x56, 0x34, 0x12}, // BLSIQ 0x12345678, R15 + }, + }, + { + name: "amd64-linux", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", + encodings: [][]byte{ + {0xc4, 0xe2, 0x88, 0xf3, 0xd2}, + {0xc4, 0x82, 0x88, 0xf3, 0x5c, 0x88, 0x10}, + {0x64, 0xc4, 0xe2, 0x80, 0xf3, 0x4c, 0x48, 0x08}, + {0xc4, 0xe2, 0x80, 0xf3, 0x1c, 0x25, 0x78, 0x56, 0x34, 0x12}, + }, + }, + { + name: "amd64-windows", goarch: "amd64", triple: "x86_64-pc-windows-msvc", + encodings: [][]byte{ + {0xc4, 0xe2, 0x88, 0xf3, 0xd2}, + {0xc4, 0x82, 0x88, 0xf3, 0x5c, 0x88, 0x10}, + {0x64, 0xc4, 0xe2, 0x80, 0xf3, 0x4c, 0x48, 0x08}, + {0xc4, 0xe2, 0x80, 0xf3, 0x1c, 0x25, 0x78, 0x56, 0x34, 0x12}, + }, + }, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + for index, encoding := range target.encodings { + name := fmt.Sprintf("rawBLS_%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + source.WriteString("\tRET\n") + } + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-bls.ll", "raw-bls.o", ir) + }) + } +} + +func TestX86RawBLSRejectsInvalidOrLayoutDependentForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + }{ + {name: "vex_l", mode: 64, encoding: []byte{0xc4, 0xe2, 0x8c, 0xf3, 0xd2}}, + {name: "vex_prefix", mode: 64, encoding: []byte{0xc4, 0xe2, 0x89, 0xf3, 0xd2}}, + {name: "invalid_digit", mode: 64, encoding: []byte{0xc4, 0xe2, 0x88, 0xf3, 0xc2}}, + {name: "extended_destination_386", mode: 32, encoding: []byte{0xc4, 0xe2, 0x38, 0xf3, 0xd2}}, + {name: "address_size_override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe2, 0x88, 0xf3, 0x54, 0x48, 0x08}}, + {name: "rip_relative", mode: 64, encoding: []byte{0xc4, 0xe2, 0x88, 0xf3, 0x15, 0, 0, 0, 0}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86BLSInstruction(test.encoding, test.mode) + if test.name == "vex_l" || test.name == "vex_prefix" { + if ok { + t.Fatalf("reserved VEX encoding %x was recognized: %v", test.encoding, err) + } + return + } + if !ok || err == nil { + t.Fatalf("encoding %x: ok=%v err=%v, want recognized rejection", test.encoding, ok, err) + } + }) + } +} + +func TestAMD64BLSRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = x86BLSOriginalRuntimeSource + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "blssemantics": {Name: "blssemantics", Args: []LLVMType{Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = x86BLSOriginalRuntimeDriver + compileAndRunRuntimeTestForTarget(t, llc, clang, "bls_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_bmi2_shift.go b/amd64_lower_bmi2_shift.go new file mode 100644 index 000000000..8aa294aed --- /dev/null +++ b/amd64_lower_bmi2_shift.go @@ -0,0 +1,62 @@ +package plan9asm + +import "fmt" + +type amd64BMI2ShiftSpec struct { + bits int + prefix int + kind string + immediate bool +} + +var amd64BMI2ShiftSpecs = map[Op]amd64BMI2ShiftSpec{ + "SARXL": {bits: 32, prefix: 2, kind: "ashr"}, + "SARXQ": {bits: 64, prefix: 2, kind: "ashr"}, + "SHLXL": {bits: 32, prefix: 1, kind: "shl"}, + "SHLXQ": {bits: 64, prefix: 1, kind: "shl"}, + "SHRXL": {bits: 32, prefix: 3, kind: "lshr"}, + "SHRXQ": {bits: 64, prefix: 3, kind: "lshr"}, + "RORXL": {bits: 32, prefix: 3, kind: "rotate", immediate: true}, + "RORXQ": {bits: 64, prefix: 3, kind: "rotate", immediate: true}, +} + +// All eight L/Q shifts and rotates consume the shared complete _ybextrl or +// _yrorxl grammar, exact source width and count mask. None changes FLAGS. +func (c *amd64Ctx) lowerBMI2ShiftRotate(op Op, ins Instr) (ok bool, terminated bool, err error) { + spec, supported := amd64BMI2ShiftSpecs[op] + if !supported { + return false, false, nil + } + form, _, err := x86ScalarShiftProperties(op, ins, c.goarch) + if err != nil { + return true, false, err + } + typ := amd64IntegerTypeForBits(form.bits) + countValue, err := c.evalIntSized(form.count, typ) + if err != nil { + return true, false, err + } + sourceValue, _, err := c.loadScalarShiftOperand(ins, 1, typ) + if err != nil { + return true, false, err + } + maskedCount := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %d\n", maskedCount, typ, countValue, form.bits-1) + result := c.newTmp() + if spec.immediate { + // Both shifts stay in range at count zero; LLVM must never see poison. + negative, reverse := c.newTmp(), c.newTmp() + low, high := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s 0, %%%s\n", negative, typ, maskedCount) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", reverse, typ, negative, form.bits-1) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", low, typ, sourceValue, maskedCount) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", high, typ, sourceValue, reverse) + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", result, typ, low, high) + } else { + fmt.Fprintf(c.b, " %%%s = %s %s %s, %%%s\n", result, spec.kind, typ, sourceValue, maskedCount) + } + if err := c.storeRegSized(form.destination.Reg, typ, "%"+result); err != nil { + return true, false, err + } + return true, false, nil +} diff --git a/amd64_lower_bmi2_shift_test.go b/amd64_lower_bmi2_shift_test.go new file mode 100644 index 000000000..4496a3f84 --- /dev/null +++ b/amd64_lower_bmi2_shift_test.go @@ -0,0 +1,192 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86BMI2VariableShiftFormsSource(goarch string) string { + var source strings.Builder + symbol := "bmi2shiftdata" + size := 8 + if goarch == "386" { + symbol = "bmi2shiftdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $1\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT bmi2shiftforms(SB),$0-0\n") + for _, stem := range []string{"SARX", "SHLX", "SHRX"} { + for _, width := range []string{"L", "Q"} { + op := stem + width + fmt.Fprintf(&source, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&source, "\t%s DX, 8(BX), SI\n", op) + fmt.Fprintf(&source, "\t%s CX, %s(SB), DX\n", op, symbol) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12, R13\n", op) + } + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86BMI2VariableShiftCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86BMI2VariableShiftFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + ll := x86UnboundVoidScalarFormatObject(t, source, "bmi2shiftforms", target.goarch, target.triple) + operations := []string{"ashr i32", "shl i32", "lshr i32", `"target-features"="+bmi2"`} + if target.goarch == "amd64" { + operations = append(operations, "ashr i64", "shl i64", "lshr i64") + } else if strings.Contains(ll, "ashr i64") || strings.Contains(ll, "shl i64") || strings.Contains(ll, "lshr i64") { + t.Fatal("386 BMI2 shift used 64-bit arithmetic") + } + for _, want := range operations { + if !strings.Contains(ll, want) { + t.Fatalf("BMI2 variable-shift lowering omitted %q:\n%s", want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "bmi2-shift-"+target.name+".ll", "bmi2-shift-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86BMI2VariableShiftRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "SHLXQ $1, AX, BX"}, + {goarch: "amd64", instruction: "SHRXQ 0(BX), AX, CX"}, + {goarch: "amd64", instruction: "SARXL AX, BX, 0(CX)"}, + {goarch: "amd64", instruction: "SHLXL X0, AX, BX"}, + {goarch: "amd64", instruction: "SHRXQ AX, X0, BX"}, + {goarch: "amd64", instruction: "SARXQ AX, BX"}, + {goarch: "amd64", instruction: "SHLXQ.Z AX, BX, CX"}, + {goarch: "386", instruction: "SHLXL R8, AX, BX"}, + {goarch: "386", instruction: "SHRXQ AX, R8, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _ybextrl table", test.instruction) + } + }) + } +} + +func TestAMD64BMI2VariableShiftRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT bmi2shiftsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $36, AX + MOVQ $0x12345678, BX + SHLXL AX, BX, CX + MOVQ CX, 0(DI) + MOVQ $-81985529216486896, 8(DI) + MOVQ $72, AX + SHRXQ AX, 8(DI), CX + MOVQ CX, 8(DI) + MOVQ $0x80000000, BX + SARXL AX, BX, CX + MOVQ CX, 16(DI) + MOVQ $-9223372036854775808, BX + SARXQ AX, BX, CX + MOVQ CX, 24(DI) + MOVQ $1, AX + MOVQ $3, BX + CMPQ BX, BX + STC + SHLXQ AX, BX, AX + MOVQ AX, 32(DI) + SETCS 40(DI) + SETEQ 41(DI) + RET +` + // Preserve the original literal above as an independent Go-rejected + // counterexample. Only the unencodable imm64 memory store is repaired. + repaired := x86BMI2VariableShiftRepairedRuntimeSource(t) + if repaired == source { + t.Fatal("BMI2 fixture repair did not change its invalid store") + } + file, err := Parse(ArchAMD64, repaired) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "bmi2shiftsemantics": { + Name: "bmi2shiftsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void bmi2shiftsemantics(uint8_t *); +int main(void) { + uint8_t out[48] = {0}; + bmi2shiftsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0x23456780)) return 10; + if (*(uint64_t *)(out+8) != UINT64_C(0x00fedcba98765432)) return 11; + if (*(uint64_t *)(out+16) != UINT64_C(0xff800000)) return 12; + if (*(uint64_t *)(out+24) != UINT64_C(0xff80000000000000)) return 13; + if (*(uint64_t *)(out+32) != 6) return 14; + if (out[40] != 1 || out[41] != 1) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "bmi2_shift_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_bound.go b/amd64_lower_bound.go new file mode 100644 index 000000000..bd5f55a4c --- /dev/null +++ b/amd64_lower_bound.go @@ -0,0 +1,63 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64BoundOperandBits = map[Op]int{ + "BOUNDW": 16, + "BOUNDL": 32, +} + +// lowerBound implements Go 1.27's complete yrl_m family. BOUND is valid only +// in 16/32-bit x86 modes, so plan9asm preserves the exact faulting instruction +// as constrained inline assembly rather than approximating its exception. +func (c *amd64Ctx) lowerBound(op Op, ins Instr) (ok bool, terminated bool, err error) { + raw := strings.ToUpper(string(op)) + base := raw + if dot := strings.IndexByte(raw, '.'); dot >= 0 { + base = raw[:dot] + } + bits, recognized := amd64BoundOperandBits[Op(base)] + if !recognized { + return false, false, nil + } + if raw != base { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, base, ins.Raw) + } + if c.goarch != "386" { + return true, false, fmt.Errorf("%s %s is unavailable outside 32-bit x86 mode: %q", c.goarch, base, ins.Raw) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("386 %s expects GP register and bounds memory: %q", base, ins.Raw) + } + if ins.Args[0].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[0].Reg, c.goarch) { + return true, false, fmt.Errorf("386 %s source is outside Go 1.27's Yrl class: %q", base, ins.Raw) + } + if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("386 %s second operand must be Go 1.27 Ym memory: %q", base, ins.Raw) + } + typeName := I32 + if bits == 16 { + typeName = I16 + } + value, err := c.evalIntSized(ins.Args[0], typeName) + if err != nil { + return true, false, err + } + pointer, pointerType, err := c.x86DescriptorMemoryPointer(ins.Args[1]) + if err != nil { + return true, false, err + } + // LLVM 22's integrated assembler no longer recognizes the BOUND mnemonic, + // even for an i386 triple. Constrain the value and bounds pointer to AX/CX + // and emit 62 /r with ModRM 00:000:001. The 66 prefix selects BOUNDW. + encoding := ".byte 0x62, 0x01" + if bits == 16 { + encoding = ".byte 0x66, 0x62, 0x01" + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s, %s %s)\n", + encoding, "{ax},{cx},~{memory},~{dirflag},~{fpsr},~{flags}", typeName, value, pointerType, pointer) + return true, false, nil +} diff --git a/amd64_lower_bound_test.go b/amd64_lower_bound_test.go new file mode 100644 index 000000000..329d1d487 --- /dev/null +++ b/amd64_lower_bound_test.go @@ -0,0 +1,79 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslate386BoundCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + triple string + }{ + {name: "linux-386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := "TEXT boundforms(SB),NOSPLIT,$16-0\n" + for _, op := range []string{"BOUNDW", "BOUNDL"} { + for _, reg := range []string{"AX", "BX", "CX", "DX", "SP", "BP", "SI", "DI"} { + source += "\t" + op + " " + reg + ", 0(SP)\n" + } + source += "\t" + op + " AX, 8(BX)\n" + } + source += "\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "386", + Sigs: map[string]FuncSig{"boundforms": {Name: "boundforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{".byte 0x66, 0x62, 0x01", ".byte 0x62, 0x01", `"{ax},{cx},~{memory},~{dirflag},~{fpsr},~{flags}"`} { + if !strings.Contains(ll, want) { + t.Fatalf("translated BOUND family missing %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "bound-"+target.name+".ll", "bound-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86BoundRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + tests := []struct { + goarch string + triple string + instruction string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "BOUNDW AX, 0(BX)"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "BOUNDL AX, 0(BX)"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "BOUNDL X0, 0(BX)"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "BOUNDW $1, 0(BX)"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "BOUNDL AX, BX"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "BOUNDL.Z AX, 0(BX)"}, + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+test.instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's BOUND tables for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_branch.go b/amd64_lower_branch.go index 345ec225a..f54ebfef1 100644 --- a/amd64_lower_branch.go +++ b/amd64_lower_branch.go @@ -8,6 +8,16 @@ import ( func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64EmitBr, emitCondBr amd64EmitCondBr) (ok bool, terminated bool, err error) { switch op { case "CALL": + if c.directTail.privateCallTransition(bi) { + if ii != 0 || c.directTail.instructions[bi].Op != OpCALL { + return true, false, x86DirectTailContext("CALL lost its original source point") + } + if err := c.lowerPrivateFrameTransition(bi); err != nil { + return true, false, err + } + emitBr(c.blocks[c.directTail.edges[bi][0]].name) + return true, true, nil + } if len(ins.Args) != 1 { return true, false, fmt.Errorf("amd64 CALL expects 1 operand: %q", ins.Raw) } @@ -34,6 +44,16 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi } return true, false, nil case OpSym: + if strings.HasPrefix(strings.TrimSpace(ins.Args[0].Sym), "*") { + addr, err := c.loadIndirectSymbolAddr(ins.Args[0].Sym) + if err != nil { + return true, false, err + } + if err := c.callIndirectAddr(addr); err != nil { + return true, false, err + } + return true, false, nil + } if err := c.callSym(ins.Args[0]); err != nil { return true, false, err } @@ -42,16 +62,21 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi return true, false, fmt.Errorf("amd64 CALL expects reg or symbol(SB) target: %q", ins.Raw) } - case "JMP", - "JE", "JEQ", "JZ", "JNE", "JNZ", - "JL", "JLT", "JLE", "JG", "JGT", "JGE", "JS", "JNS", - "JA", "JHI", "JAE", "JHS", "JB", "JLO", "JBE", "JLS", "JNA", - "JC", "JNC", "JCC": - // ok default: - return false, false, nil + if op != "JMP" && op != "LOOP" && !isAMD64ConditionalBranch(op) { + return false, false, nil + } } - if len(ins.Args) != 1 { + counter, counterKnown, counterErr := x86CounterControlProperties(op, ins, c.goarch) + if counterKnown && counterErr != nil { + return true, false, counterErr + } + // The Go assembler accepts a legacy x86 branch-hint operand such as + // JEQ $1, target. It is not part of the encoded instruction on current + // toolchains. Share its complete resolved-integer grammar with the source + // CFG observers; floating/unresolved tokens cannot become zero hints. + args := x86DirectTailBranchArgs(ins) + if len(args) != 1 { return true, false, fmt.Errorf("amd64 %s expects 1 operand: %q", op, ins.Raw) } target := "" @@ -65,65 +90,99 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi } return false } - switch ins.Args[0].Kind { - case OpIdent: - target = ins.Args[0].Ident - case OpReg: - // Labels like V1 may be tokenized as registers. Prefer block labels; - // otherwise treat JMP reg as an indirect tail jump. - name := string(ins.Args[0].Reg) - if knownBlock(name) { - target = name - break - } - if op == "JMP" { - addr, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err + if c.directTail != nil { + if bi >= 0 && bi < len(c.directTail.externalTails) && c.directTail.externalTails[bi] && ii == 0 { + if op != OpJMP || len(args) != 1 || args[0].Kind != OpSym { + return true, false, x86DirectTailContext("typed terminal lost its original source branch") } - if err := c.tailCallIndirectAddrAndRet(addr); err != nil { + if err := c.tailCallAndRet(args[0]); err != nil { return true, false, err } return true, true, nil } - return true, false, fmt.Errorf("amd64 %s invalid register target: %q", op, ins.Raw) - case OpSym: - s := strings.TrimSpace(ins.Args[0].Sym) - // Treat JMP foo(SB) as a tailcall to another TEXT (common in stdlib asm). - if op == "JMP" && strings.HasSuffix(s, "(SB)") { - if err := c.tailCallAndRet(ins.Args[0]); err != nil { - return true, false, err - } - return true, true, nil + if bi < 0 || bi >= len(c.directTail.edges) || ii != 0 || len(c.directTail.edges[bi]) == 0 { + return true, false, x86DirectTailContext("branch lost its original source node") } - // Local labels sometimes appear with "<>" or (SB) suffix. - s = strings.TrimSuffix(strings.TrimSuffix(s, "(SB)"), "<>") - target = s - case OpMem: - // PC-relative branches like "JEQ 2(PC)" are used as a compact way to skip - // the next instruction. Handle them by mapping the offset to our block - // sequence (blocks are split at terminators, so the pattern works well). - if strings.EqualFold(string(ins.Args[0].Mem.Base), "PC") { - pcRel = true - pcOff = ins.Args[0].Mem.Off - break - } - if op == "JMP" { - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err + edges := c.directTail.edges[bi] + target = c.blocks[edges[len(edges)-1]].name + } else { + switch args[0].Kind { + case OpIdent: + target = args[0].Ident + case OpReg: + // Labels like V1 may be tokenized as registers. Prefer block labels; + // otherwise treat JMP reg as an indirect tail jump. + name := string(args[0].Reg) + if knownBlock(name) { + target = name + break } - p := c.ptrFromAddrI64(addr) - fnptr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", fnptr, p) - if err := c.tailCallIndirectAddrAndRet("%" + fnptr); err != nil { - return true, false, err + if op == "JMP" { + addr, err := c.loadReg(args[0].Reg) + if err != nil { + return true, false, err + } + if err := c.tailCallIndirectAddrAndRet(addr); err != nil { + return true, false, err + } + return true, true, nil } - return true, true, nil + return true, false, fmt.Errorf("amd64 %s invalid register target: %q", op, ins.Raw) + case OpSym: + s := strings.TrimSpace(args[0].Sym) + if op == "JMP" && strings.HasPrefix(s, "*") { + addr, err := c.loadIndirectSymbolAddr(s) + if err != nil { + return true, false, err + } + if err := c.tailCallIndirectAddrAndRet(addr); err != nil { + return true, false, err + } + return true, true, nil + } + // Treat JMP foo(SB) as a tailcall to another TEXT (common in stdlib asm). + if op == "JMP" && strings.HasSuffix(s, "(SB)") { + if err := c.tailCallAndRet(args[0]); err != nil { + return true, false, err + } + return true, true, nil + } + // Local labels sometimes appear with "<>" or (SB) suffix. + s = strings.TrimSuffix(strings.TrimSuffix(s, "(SB)"), "<>") + target = s + case OpMem: + // PC-relative branches like "JEQ 2(PC)" are used as a compact way to skip + // the next instruction. Handle them by mapping the offset to our block + // sequence (blocks are split at terminators, so the pattern works well). + if strings.EqualFold(string(args[0].Mem.Base), "PC") { + pcRel = true + pcOff = args[0].Mem.Off + break + } + if op == "JMP" { + addr, err := c.addrFromMem(args[0].Mem) + if err != nil { + return true, false, err + } + p := c.ptrFromAddrI64(addr) + fnptr := c.newTmp() + if c.goarch == "386" { + fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", fnptr, p) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, fnptr) + fnptr = wide + } else { + fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", fnptr, p) + } + if err := c.tailCallIndirectAddrAndRet("%" + fnptr); err != nil { + return true, false, err + } + return true, true, nil + } + fallthrough + default: + return true, false, fmt.Errorf("amd64 %s invalid target: %q", op, ins.Raw) } - fallthrough - default: - return true, false, fmt.Errorf("amd64 %s invalid target: %q", op, ins.Raw) } if !pcRel && target == "" { return true, false, fmt.Errorf("amd64 %s empty target: %q", op, ins.Raw) @@ -149,70 +208,17 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi } fall := c.blocks[bi+1].name cond := "" - switch op { - case "JE", "JEQ", "JZ": - cond = c.loadFlag(c.flagsZSlot) - case "JNE", "JNZ": - z := c.loadFlag(c.flagsZSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, z) - cond = "%" + t - case "JL", "JLT": - cond = c.loadFlag(c.flagsSltSlot) - case "JGE": - slt := c.loadFlag(c.flagsSltSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, slt) - cond = "%" + t - case "JS": - cond = c.loadFlag(c.flagsSltSlot) - case "JNS": - slt := c.loadFlag(c.flagsSltSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, slt) - cond = "%" + t - case "JLE": - slt := c.loadFlag(c.flagsSltSlot) - z := c.loadFlag(c.flagsZSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t, slt, z) - cond = "%" + t - case "JG", "JGT": - slt := c.loadFlag(c.flagsSltSlot) - z := c.loadFlag(c.flagsZSlot) - t1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t1, slt, z) - t2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) - cond = "%" + t2 - case "JB", "JLO", "JC": - cond = c.loadFlag(c.flagsCFSlot) - case "JNC", "JCC": - cf := c.loadFlag(c.flagsCFSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, cf) - cond = "%" + t - case "JAE", "JHS": - cf := c.loadFlag(c.flagsCFSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, cf) - cond = "%" + t - case "JBE", "JLS", "JNA": - cf := c.loadFlag(c.flagsCFSlot) - z := c.loadFlag(c.flagsZSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t, cf, z) - cond = "%" + t - case "JA", "JHI": - cf := c.loadFlag(c.flagsCFSlot) - z := c.loadFlag(c.flagsZSlot) - t1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t1, cf, z) - t2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) - cond = "%" + t2 - default: - return true, false, fmt.Errorf("amd64: unsupported branch %s", op) + if counterKnown { + cond, err = c.counterControlCondition(counter) + } else { + condition := x86BranchCondition(op) + if condition == "" { + return true, false, fmt.Errorf("amd64: unsupported branch %s", op) + } + cond, err = c.x86Condition(condition) + } + if err != nil { + return true, false, err } if err := emitCondBr(cond, target, fall); err != nil { @@ -221,9 +227,42 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi return true, true, nil } +func (c *amd64Ctx) loadIndirectSymbolAddr(sym string) (string, error) { + sym = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(sym), "*")) + if !strings.HasSuffix(sym, "(SB)") { + return "", fmt.Errorf("amd64 indirect branch expects register, memory, or *symbol(SB), got %q", sym) + } + ptr, err := c.ptrFromSB(sym) + if err != nil { + return "", err + } + value := c.newTmp() + if c.goarch == "386" { + fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", value, ptr) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, value) + return "%" + wide, nil + } + fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", value, ptr) + return "%" + value, nil +} + func (c *amd64Ctx) callIndirectAddr(addr string) error { fptr := c.newTmp() fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", fptr, addr) + if sig := c.indirectABI0; sig != nil { + args, err := c.abi0CallArgs("indirect callback", *sig) + if err != nil { + return err + } + if sig.Ret == Void { + fmt.Fprintf(c.b, " call void %%%s(%s)\n", fptr, strings.Join(args, ", ")) + return nil + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s %%%s(%s)\n", result, sig.Ret, fptr, strings.Join(args, ", ")) + return c.storeABI0CallResult("indirect callback", *sig, "%"+result) + } di, _ := c.loadReg(DI) si, _ := c.loadReg(SI) dx, _ := c.loadReg(DX) @@ -237,6 +276,15 @@ func (c *amd64Ctx) callIndirectAddr(addr string) error { } func (c *amd64Ctx) tailCallIndirectAddrAndRet(addr string) error { + if len(c.indirectLabels) != 0 { + ptr := c.ptrFromAddrI64(addr) + labels := make([]string, len(c.indirectLabels)) + for i, label := range c.indirectLabels { + labels[i] = "label %" + amd64LLVMBlockName(label) + } + fmt.Fprintf(c.b, " indirectbr ptr %s, [%s]\n", ptr, strings.Join(labels, ", ")) + return nil + } if err := c.callIndirectAddr(addr); err != nil { return err } @@ -294,11 +342,12 @@ func (c *amd64Ctx) callSym(symOp Operand) error { if !strings.HasSuffix(s, "(SB)") { return fmt.Errorf("amd64 call expects (SB) symbol, got %q", s) } + internalABI := strings.HasSuffix(strings.TrimSuffix(s, "(SB)"), "") s = strings.TrimSuffix(s, "(SB)") callee := c.resolve(s) // Syscall stubs invoke runtime entersyscall/exitsyscall around SYSCALL. // llgo runtime does not require these scheduler hooks at this layer. - if callee == "runtime.entersyscall" || callee == "runtime.exitsyscall" { + if arm64ElidesSchedulerHook(callee) { return nil } @@ -310,6 +359,25 @@ func (c *amd64Ctx) callSym(symOp Operand) error { } callee = funcSigSymbol(callee, csig) + stackABI := !internalABI && len(csig.ArgRegs) == 0 && + (len(csig.Frame.Params) != 0 || len(csig.Frame.Results) != 0) + if stackABI { + args, err := c.abi0CallArgs(callee, csig) + if err != nil { + return err + } + if csig.Ret == Void { + fmt.Fprintf(c.b, " call void %s(%s)\n", llvmGlobal(callee), strings.Join(args, ", ")) + return nil + } + t := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s %s(%s)\n", t, csig.Ret, llvmGlobal(callee), strings.Join(args, ", ")) + if len(csig.Frame.Results) != 0 { + return c.storeABI0CallResult(callee, csig, "%"+t) + } + return fmt.Errorf("amd64 call %q: ABI0 result %s has no frame slot", callee, csig.Ret) + } + args := make([]string, 0, len(csig.Args)) goABI := []Reg{AX, BX, CX, DI, SI, Reg("R8"), Reg("R9"), Reg("R10"), Reg("R11")} regCursor := 0 @@ -418,12 +486,13 @@ func (c *amd64Ctx) tailCallAndRet(symOp Operand) error { // If ArgRegs is empty, default to register-based passing (ABIInternal-ish) // because most intra-asm tailcalls depend on explicit register setup. // - // Exception: for tailcalls to Go functions with an identical signature, + // Exception: for tailcalls to Go functions with identical argument types, // use the current function's LLVM args. This matches stdlib patterns like // "JMP ·countGeneric(SB)" that happen before any register shuffling and - // are stack-ABI tailcalls in the original asm. + // are stack-ABI tailcalls in the original asm. The return types may differ + // while using ABI-equivalent physical result slots. useLLVMArgs := false - if len(csig.ArgRegs) == 0 && len(csig.Args) == len(c.sig.Args) && csig.Ret == c.sig.Ret { + if len(csig.ArgRegs) == 0 && len(csig.Args) == len(c.sig.Args) { same := true for j := 0; j < len(csig.Args); j++ { if csig.Args[j] != c.sig.Args[j] { @@ -527,8 +596,22 @@ func (c *amd64Ctx) tailCallAndRet(symOp Operand) error { return nil } if c.sig.Ret != csig.Ret { + if adapted, ok, err := c.adaptTailCallAggregateReturn("%"+t, csig.Ret, c.sig.Ret); err != nil { + return fmt.Errorf("amd64 tailcall return type mismatch for %q: %w", callee, err) + } else if ok { + fmt.Fprintf(c.b, " ret %s %s\n", c.sig.Ret, adapted) + return nil + } conv := c.newTmp() switch { + case c.goarch == "386" && csig.Ret == I32 && c.sig.Ret == Ptr: + fmt.Fprintf(c.b, " %%%s = inttoptr i32 %%%s to ptr\n", conv, t) + fmt.Fprintf(c.b, " ret ptr %%%s\n", conv) + return nil + case c.goarch == "386" && csig.Ret == Ptr && c.sig.Ret == I32: + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %%%s to i32\n", conv, t) + fmt.Fprintf(c.b, " ret i32 %%%s\n", conv) + return nil case csig.Ret == I64 && (c.sig.Ret == I1 || c.sig.Ret == I8 || c.sig.Ret == I16 || c.sig.Ret == I32): fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to %s\n", conv, t, c.sig.Ret) fmt.Fprintf(c.b, " ret %s %%%s\n", c.sig.Ret, conv) @@ -552,3 +635,47 @@ func (c *amd64Ctx) tailCallAndRet(symOp Operand) error { fmt.Fprintf(c.b, " ret %s %%%s\n", c.sig.Ret, t) return nil } + +func (c *amd64Ctx) adaptTailCallAggregateReturn(value string, fromTy, toTy LLVMType) (string, bool, error) { + fromFields, fromAggregate := parseLiteralStructFields(fromTy) + toFields, toAggregate := parseLiteralStructFields(toTy) + if !fromAggregate || !toAggregate { + return "", false, nil + } + if len(fromFields) != len(toFields) { + return "", false, fmt.Errorf("caller %s and callee %s use different aggregate field counts", toTy, fromTy) + } + + wordTy := I64 + if c.goarch == "386" { + wordTy = I32 + } + for i := range fromFields { + from, to := fromFields[i], toFields[i] + if from == to || (from == wordTy && to == Ptr) || (from == Ptr && to == wordTy) { + continue + } + return "", false, fmt.Errorf("caller field %d has type %s but callee field has ABI-incompatible type %s", i, to, from) + } + + result := "undef" + for i := range fromFields { + from, to := fromFields[i], toFields[i] + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", extracted, fromTy, value, i) + field := "%" + extracted + if from != to { + converted := c.newTmp() + if from == wordTy && to == Ptr { + fmt.Fprintf(c.b, " %%%s = inttoptr %s %s to ptr\n", converted, wordTy, field) + } else { + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to %s\n", converted, field, wordTy) + } + field = "%" + converted + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertvalue %s %s, %s %s, %d\n", inserted, toTy, result, to, field, i) + result = "%" + inserted + } + return result, true, nil +} diff --git a/amd64_lower_branch_test.go b/amd64_lower_branch_test.go index da0c48225..ebf12e273 100644 --- a/amd64_lower_branch_test.go +++ b/amd64_lower_branch_test.go @@ -1,6 +1,3 @@ -//go:build !llgo -// +build !llgo - package plan9asm import ( diff --git a/amd64_lower_broadcast.go b/amd64_lower_broadcast.go new file mode 100644 index 000000000..4bf364f2a --- /dev/null +++ b/amd64_lower_broadcast.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerPackedScalarBroadcast implements the complete Go 1.27 +// _yvpbroadcastb operand table shared by VPBROADCASTB/W/D/Q. +func (c *amd64Ctx) lowerPackedScalarBroadcast(op Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86BroadcastProperties(op, ins, c.goarch) + if !known || f.kind != x86BroadcastInteger { + return false, false, nil + } + if err != nil { + return true, false, err + } + baseOp, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + laneBits, byteWidth := f.laneBits, f.byteWidth + dstArg := ins.Args[f.destination] + masked, zeroing := f.mask >= 0, f.zeroing + var mask string + if masked { + mask, err = c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + } + + load := func() (string, error) { + return c.loadPackedBroadcastScalar(ins.Args[0], laneBits, ins.x86Encoded) + } + var scalar string + if masked && ins.Args[0].Kind != OpReg { + // Any active destination lane enables the one scalar read. Irrelevant + // high K bits must not turn a fully masked memory source into a load. + lanes := byteWidth * 8 / laneBits + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)< %s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(dstArg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) loadPackedBroadcastScalar(src Operand, laneBits int, rawEncoded bool) (string, error) { + if src.Kind == OpImm && rawEncoded { + value := src.Imm + switch laneBits { + case 8: + value = int64(int8(value)) + case 16: + value = int64(int16(value)) + case 32: + value = int64(int32(value)) + } + return fmt.Sprintf("%d", value), nil + } + if src.Kind == OpReg { + if isAMD64XReg(src.Reg) { + bytesValue, err := c.loadX(src.Reg) + if err != nil { + return "", err + } + lanes := 128 / laneBits + laneVector := c.bitcastVectorBytesToIntegerLanes(16, lanes, laneBits, bytesValue) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 0\n", low, lanes, laneBits, laneVector) + return "%" + low, nil + } + if !isAMD64YrlRegister(src.Reg) { + return "", fmt.Errorf("expected an X register, GP register, or memory, got %s", src.String()) + } + return c.evalIntSized(src, amd64IntegerTypeForBits(laneBits)) + } + if !isAMD64MemoryOperand(src) { + return "", fmt.Errorf("expected an X register, GP register, or memory, got %s", src.String()) + } + bytes, err := c.loadX86PackedShiftBytes(src, laneBits/8, true) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", value, laneBits/8, bytes, laneBits) + return "%" + value, nil +} + +// isAMD64YrlRegister mirrors Go's amd64 Yrl register class. Byte aliases, +// vector, mask, MMX, and x87 registers are deliberately excluded. +func isAMD64YrlRegister(r Reg) bool { + switch strings.ToUpper(string(r)) { + case "AX", "BX", "CX", "DX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15": + return true + default: + return false + } +} diff --git a/amd64_lower_broadcast_block.go b/amd64_lower_broadcast_block.go new file mode 100644 index 000000000..89b6f01b3 --- /dev/null +++ b/amd64_lower_broadcast_block.go @@ -0,0 +1,126 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedBlockBroadcastSpec struct { + sourceBytes int + laneBits int + allowXSource bool + vexMemoryOnly bool + allowDestinations map[int]bool +} + +var amd64PackedBlockBroadcastSpecs = map[string]amd64PackedBlockBroadcastSpec{ + "VBROADCASTF128": {sourceBytes: 16, laneBits: 64, vexMemoryOnly: true, allowDestinations: map[int]bool{32: true}}, + "VBROADCASTI128": {sourceBytes: 16, laneBits: 64, vexMemoryOnly: true, allowDestinations: map[int]bool{32: true}}, + "VBROADCASTF32X2": {sourceBytes: 8, laneBits: 32, allowXSource: true, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTI32X2": {sourceBytes: 8, laneBits: 32, allowXSource: true, allowDestinations: map[int]bool{16: true, 32: true, 64: true}}, + "VBROADCASTF32X4": {sourceBytes: 16, laneBits: 32, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTF64X2": {sourceBytes: 16, laneBits: 64, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTI32X4": {sourceBytes: 16, laneBits: 32, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTI64X2": {sourceBytes: 16, laneBits: 64, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTF32X8": {sourceBytes: 32, laneBits: 32, allowDestinations: map[int]bool{64: true}}, + "VBROADCASTF64X4": {sourceBytes: 32, laneBits: 64, allowDestinations: map[int]bool{64: true}}, + "VBROADCASTI32X8": {sourceBytes: 32, laneBits: 32, allowDestinations: map[int]bool{64: true}}, + "VBROADCASTI64X4": {sourceBytes: 32, laneBits: 64, allowDestinations: map[int]bool{64: true}}, +} + +// lowerPackedBlockBroadcast implements all Go 1.27 EVEX packed-block +// broadcast tables. The X2 forms copy the low 64 source bits; the X4 and X8 +// forms accept only 128- and 256-bit memory sources respectively. +func (c *amd64Ctx) lowerPackedBlockBroadcast(op Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86BroadcastProperties(op, ins, c.goarch) + if !known || f.kind != x86BroadcastBlock { + return false, false, nil + } + if err != nil { + return true, false, err + } + baseOp, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec := amd64PackedBlockBroadcastSpecs[baseOp] + source, destination := ins.Args[0], ins.Args[f.destination] + destinationBytes := f.byteWidth + masked, zeroing := f.mask >= 0, f.zeroing + mask := "" + if masked { + mask, err = c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + } + + var block string + if mask != "" && source.Kind != OpReg { + // E6 suppresses each source element unless at least one repeated + // destination lane selects it. Fold the destination mask onto the + // source tuple before reusing the exact guarded volatile lane reader. + sourceLanes := spec.sourceBytes * 8 / spec.laneBits + folded := mask + for shift := sourceLanes; shift < destinationBytes*8/spec.laneBits; shift += sourceLanes { + high, combined := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", high, mask, shift) + fmt.Fprintf(c.b, " %%%s = or i64 %s, %%%s\n", combined, folded, high) + folded = "%" + combined + } + lanes, readErr := c.loadX86PackedShiftLanes(source, spec.sourceBytes, spec.laneBits, false, folded) + err = readErr + if err == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", + value, sourceLanes, spec.laneBits, lanes, spec.sourceBytes) + block = "%" + value + } + } else { + block, err = c.loadPackedBlockBroadcastSource(source, spec.sourceBytes) + } + if err != nil { + return true, false, err + } + broadcast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> %s, <%d x i32> %s\n", + broadcast, spec.sourceBytes, block, spec.sourceBytes, block, destinationBytes, llvmRepeatI8Mask(spec.sourceBytes, destinationBytes)) + result := "%" + broadcast + + if masked { + lanes := destinationBytes * 8 / spec.laneBits + computed := c.bitcastVectorBytesToIntegerLanes(destinationBytes, lanes, spec.laneBits, result) + oldBytes, err := c.loadPackedCompareBytes(destination, destinationBytes) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(destinationBytes, lanes, spec.laneBits, oldBytes) + maskedResult := amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computed, old, mask, zeroing) + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", cast, lanes, spec.laneBits, maskedResult, destinationBytes) + result = "%" + cast + } + return true, false, c.storePackedMoveOperand(destination, destinationBytes, result) +} + +func amd64PackedBlockXSourceHint(allowed bool) string { + if allowed { + return " or X-register source" + } + return "" +} + +func (c *amd64Ctx) loadPackedBlockBroadcastSource(source Operand, sourceBytes int) (string, error) { + if source.Kind == OpReg { + value, err := c.loadX(source.Reg) + if err != nil { + return "", err + } + words := c.newTmp() + low := c.newTmp() + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", words, value) + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", low, words) + fmt.Fprintf(c.b, " %%%s = bitcast i64 %%%s to <8 x i8>\n", bytesValue, low) + return "%" + bytesValue, nil + } + + return c.loadX86PackedShiftBytes(source, sourceBytes, true) +} diff --git a/amd64_lower_broadcast_block_test.go b/amd64_lower_broadcast_block_test.go new file mode 100644 index 000000000..a8c019ae7 --- /dev/null +++ b/amd64_lower_broadcast_block_test.go @@ -0,0 +1,203 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedBlockBroadcastCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 exposes the complete VEX/EVEX packed-block broadcast family as: + // + // VBROADCASTF128 m128 -> Y + // VBROADCASTI128 m128 -> Y + // VBROADCASTF32X2 X/m64 -> Y/Z + // VBROADCASTI32X2 X/m64 -> X/Y/Z + // VBROADCASTF32X4 m128 -> Y/Z + // VBROADCASTF64X2 m128 -> Y/Z + // VBROADCASTI32X4 m128 -> Y/Z + // VBROADCASTI64X2 m128 -> Y/Z + // VBROADCASTF32X8 m256 -> Z + // VBROADCASTF64X4 m256 -> Z + // VBROADCASTI32X8 m256 -> Z + // VBROADCASTI64X4 m256 -> Z + // + // Every form also accepts K1-K7 merge masking and .Z zero masking. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastY := 15 + lastZ := 21 + if target.goarch == "386" { + lastY, lastZ = 7, 7 + } + var src strings.Builder + src.WriteString("TEXT packedblockbroadcastforms(SB),$0-0\n") + fmt.Fprintf(&src, "\tVBROADCASTF128 0(AX), Y%d\n", lastY) + fmt.Fprintf(&src, "\tVBROADCASTI128 16(AX), Y%d\n", lastY) + src.WriteString("\tVBROADCASTF32X2 X20, Y21\n") + src.WriteString("\tVBROADCASTF32X2 8(AX), K1, Y20\n") + fmt.Fprintf(&src, "\tVBROADCASTF32X2.Z X20, K2, Z%d\n", lastZ) + src.WriteString("\tVBROADCASTI32X2 X20, X21\n") + src.WriteString("\tVBROADCASTI32X2 16(AX), K3, Y20\n") + fmt.Fprintf(&src, "\tVBROADCASTI32X2.Z X20, K4, Z%d\n", lastZ) + for _, op := range []string{ + "VBROADCASTF32X4", "VBROADCASTF64X2", + "VBROADCASTI32X4", "VBROADCASTI64X2", + } { + fmt.Fprintf(&src, "\t%s 24(AX), Y20\n", op) + fmt.Fprintf(&src, "\t%s 40(AX), K5, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z 56(AX), K6, Z%d\n", op, lastZ) + } + for _, op := range []string{ + "VBROADCASTF32X8", "VBROADCASTF64X4", + "VBROADCASTI32X8", "VBROADCASTI64X4", + } { + fmt.Fprintf(&src, "\t%s 72(AX), Z%d\n", op, lastZ) + fmt.Fprintf(&src, "\t%s.Z 104(AX), K7, Z%d\n", op, lastZ) + } + src.WriteString("\tRET\n") + source := src.String() + if target.goarch == "386" { + // High Go spellings have no physical mode32 binding. The + // independent authority controls retain their Context cases. + source = strings.NewReplacer("X20", "X3", "X21", "X4", "Y20", "Y3", "Y21", "Y4").Replace(source) + } + ll := x86UnboundVoidScalarFormatObject(t, source, "packedblockbroadcastforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-block-broadcast-"+target.name+".ll", "packed-block-broadcast-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedBlockBroadcastRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VBROADCASTF128 (AX), Y16", + "VBROADCASTI128 (AX), Y16", + "VBROADCASTF128 X0, Y1", + "VBROADCASTI128 X0, Y1", + "VBROADCASTF128 (AX), X1", + "VBROADCASTI128 (AX), Z1", + "VBROADCASTF128 (AX), K1, Y1", + "VBROADCASTI128.Z (AX), K1, Y1", + "VBROADCASTF32X2 X0, X1", + "VBROADCASTF32X2 Y0, Y1", + "VBROADCASTI32X2 Y0, Y1", + "VBROADCASTF32X4 X0, Y1", + "VBROADCASTF64X2 X0, Y1", + "VBROADCASTI32X4 X0, Y1", + "VBROADCASTI64X2 X0, Y1", + "VBROADCASTF32X4 (AX), X1", + "VBROADCASTF32X8 (AX), Y1", + "VBROADCASTF64X4 Y0, Z1", + "VBROADCASTI32X8 Y0, Z1", + "VBROADCASTI64X4 (AX), Y1", + "VBROADCASTI32X4 (AX), K0, Y1", + "VBROADCASTI32X4.Z (AX), Y1", + "VBROADCASTI32X4.BCST (AX), Y1", + "VBROADCASTI32X4 (AX), K1, K2, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86PackedBlockBroadcastRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VBROADCASTF128 (AX), Y16", + "VBROADCASTI128 (AX), Y16", + "VBROADCASTI32X2 X20, Z8", + "VBROADCASTI32X4 (AX), Z20", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86PackedBlockBroadcastRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86PackedBlockBroadcastRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed-block broadcast forms", instruction) + } +} + +func TestAMD64LegacyBlockBroadcastRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT legacyblockbroadcast(SB),$0-16 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + VBROADCASTF128 (BX), Y0 + VMOVDQU Y0, 0(AX) + VBROADCASTI128 (BX), Y1 + VMOVDQU Y1, 32(AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "legacyblockbroadcast": {Name: "legacyblockbroadcast", Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +extern void legacyblockbroadcast(uint8_t *, const uint8_t *); +int main(void) { + const uint8_t source[16] = {0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0xaa,0xbb,0xcc,0xdd,0xee,0xff}; + uint8_t out[64] = {0}; + legacyblockbroadcast(out, source); + for (int block = 0; block < 2; block++) { + if (memcmp(out + 32*block, source, 16) || memcmp(out + 32*block + 16, source, 16)) return block + 1; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy_block_broadcast", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_broadcast_float.go b/amd64_lower_broadcast_float.go new file mode 100644 index 000000000..5b15db794 --- /dev/null +++ b/amd64_lower_broadcast_float.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "fmt" +) + +// lowerScalarFloatBroadcast implements all Go 1.27 _yvbroadcastss and +// _yvbroadcastsd operand forms, including EVEX mask merge and zeroing. +func (c *amd64Ctx) lowerScalarFloatBroadcast(op Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86BroadcastProperties(op, ins, c.goarch) + if !known || f.kind != x86BroadcastFloat { + return false, false, nil + } + if err != nil { + return true, false, err + } + laneBits, byteWidth := f.laneBits, f.byteWidth + source, dst := ins.Args[0], ins.Args[f.destination] + masked, zeroing := f.mask >= 0, f.zeroing + var mask string + if masked { + mask, err = c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + } + + var scalar string + if source.Kind == OpReg { + scalar, err = c.loadXLowInteger(source.Reg, laneBits) + } else if source.Kind == OpImm { + scalar, err = c.evalIntSized(source, amd64IntegerTypeForBits(laneBits)) + } else { + if masked { + // Broadcasting reads the scalar only if any destination lane is + // active. High K bits outside this vector's lane count do not + // enable the load; checking only mask bit zero is also incorrect. + lanes := byteWidth * 8 / laneBits + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)< %s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(dst, byteWidth, "%"+out) +} diff --git a/amd64_lower_broadcast_mask.go b/amd64_lower_broadcast_mask.go new file mode 100644 index 000000000..5fbdec76a --- /dev/null +++ b/amd64_lower_broadcast_mask.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64MaskBroadcastSpec struct { + sourceBits int + laneBits int +} + +// amd64MaskBroadcastSpecs is the complete Go 1.27 _yvpbroadcastmb2q grammar. +// The low byte/word of a K register is zero-extended and broadcast into every +// qword/dword lane of an EVEX X, Y, or Z destination. +var amd64MaskBroadcastSpecs = map[Op]amd64MaskBroadcastSpec{ + "VPBROADCASTMB2Q": {sourceBits: 8, laneBits: 64}, + "VPBROADCASTMW2D": {sourceBits: 16, laneBits: 32}, +} + +func (c *amd64Ctx) lowerMaskRegisterBroadcast(op Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86BroadcastProperties(op, ins, c.goarch) + if !known || f.kind != x86BroadcastMask { + return false, false, nil + } + if err != nil { + return true, false, err + } + spec := amd64MaskBroadcastSpecs[Op(strings.ToUpper(string(op)))] + byteWidth := f.byteWidth + mask, err := c.loadK(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", narrow, mask, spec.sourceBits) + scalar := "%" + narrow + if spec.sourceBits != spec.laneBits { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %s to i%d\n", wide, spec.sourceBits, scalar, spec.laneBits) + scalar = "%" + wide + } + lanes := byteWidth * 8 / spec.laneBits + result := amd64SplatInteger(c, lanes, spec.laneBits, scalar) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(ins.Args[1], byteWidth, "%"+out) +} diff --git a/amd64_lower_broadcast_mask_test.go b/amd64_lower_broadcast_mask_test.go new file mode 100644 index 000000000..0cdde793d --- /dev/null +++ b/amd64_lower_broadcast_mask_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64MaskBroadcastGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64MaskBroadcastSpec{ + "VPBROADCASTMB2Q": {sourceBits: 8, laneBits: 64}, + "VPBROADCASTMW2D": {sourceBits: 16, laneBits: 32}, + } + if len(amd64MaskBroadcastSpecs) != len(expected) { + t.Fatalf("mask-broadcast grammar has %d entries, want %d", len(amd64MaskBroadcastSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64MaskBroadcastSpecs[op]; !ok { + t.Errorf("mask-broadcast grammar omitted %s", op) + } else if got != want { + t.Errorf("mask-broadcast grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86MaskBroadcastCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + xyLast, zLast := 31, 31 + if target.goarch == "386" { + xyLast, zLast = 7, 7 + } + var source strings.Builder + source.WriteString("TEXT maskbroadcastforms(SB),$0-0\n") + for _, op := range []string{"VPBROADCASTMB2Q", "VPBROADCASTMW2D"} { + fmt.Fprintf(&source, "\t%s K0, X%d\n", op, xyLast) + fmt.Fprintf(&source, "\t%s K7, Y%d\n", op, xyLast) + fmt.Fprintf(&source, "\t%s K1, Z%d\n", op, zLast) + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "maskbroadcastforms", target.goarch, target.triple) + for _, want := range []string{"<2 x i64>", "<8 x i32>", "<8 x i64>", "<16 x i32>"} { + if !strings.Contains(ir, want) { + t.Fatalf("mask broadcast lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "mask-broadcast-"+target.name+".ll", "mask-broadcast-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86MaskBroadcastRejectsFormsOutsideGoTable(t *testing.T) { + for _, instruction := range []string{ + "VPBROADCASTMB2Q AX, X0", + "VPBROADCASTMW2D K8, X0", + "VPBROADCASTMB2Q K1, K2", + "VPBROADCASTMW2D.Z K1, X0", + "VPBROADCASTMB2Q K1, K2, X0", + } { + t.Run(strings.NewReplacer(" ", "_", ",", "").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvpbroadcastmb2q table", instruction) + } + }) + } +} + +func TestAMD64MaskBroadcastRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT maskbroadcastsemantics(SB),$0-16 + MOVQ out+0(FP), CX + MOVQ mask+8(FP), AX + KMOVQ AX, K1 + STC + VPBROADCASTMB2Q K1, X0 + MOVUPS X0, 0(CX) + VPBROADCASTMW2D K1, X1 + MOVUPS X1, 16(CX) + SETCS 32(CX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"maskbroadcastsemantics": { + Name: "maskbroadcastsemantics", Args: []LLVMType{Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void maskbroadcastsemantics(uint8_t *, uint64_t); +int main(void) { + uint8_t out[33] = {0}; + uint64_t q[2]; uint32_t d[4]; + maskbroadcastsemantics(out, UINT64_C(0x1234beefa5)); + memcpy(q, out, 16); memcpy(d, out + 16, 16); + if (q[0] != UINT64_C(0xa5) || q[1] != UINT64_C(0xa5)) return 10; + for (int i = 0; i < 4; i++) if (d[i] != UINT32_C(0xefa5)) return 11 + i; + if (out[32] != 1) return 20; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask_broadcast_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_byte_shift.go b/amd64_lower_byte_shift.go new file mode 100644 index 000000000..7f235c2bf --- /dev/null +++ b/amd64_lower_byte_shift.go @@ -0,0 +1,61 @@ +package plan9asm + +import ( + "fmt" +) + +// lowerPackedByteShift implements Go 1.27's PSLLO/PSRLO aliases and +// PSLLDQ/PSRLDQ vector family. Every operation shifts bytes independently +// within each 128-bit lane and fills vacated bytes with zero. +func (c *amd64Ctx) lowerPackedByteShift(op Op, ins Instr) (bool, bool, error) { + return c.lowerX86PackedShift(op, ins) +} + +func amd64SignedImmediate(operand Operand, bits int) bool { + if operand.Kind != OpImm || operand.ImmIsFloat || operand.ImmRaw != "" || bits <= 0 || bits >= 63 { + return false + } + limit := int64(1) << (bits - 1) + return operand.Imm >= -limit && operand.Imm < limit +} + +func amd64VectorByteShiftImmediate(operand Operand) bool { + return operand.Kind == OpImm && !operand.ImmIsFloat && operand.ImmRaw == "" && operand.Imm >= -128 && operand.Imm <= 255 +} + +func amd64PackedByteShiftRequiresEVEX(source, destination Operand, byteWidth int) bool { + if byteWidth == 64 || isAMD64MemoryOperand(source) { + return true + } + for _, operand := range []Operand{source, destination} { + index, ok := amd64VectorRegisterIndex(operand.Reg, byteWidth) + if !ok || index >= 16 { + return true + } + } + return false +} + +func (c *amd64Ctx) emitPackedByteShift(byteWidth int, source string, count uint8, left bool) string { + if count >= 16 { + return "zeroinitializer" + } + result := "zeroinitializer" + for laneBase := 0; laneBase < byteWidth; laneBase += 16 { + for destinationByte := 0; destinationByte < 16; destinationByte++ { + sourceByte := destinationByte + int(count) + if left { + sourceByte = destinationByte - int(count) + } + if sourceByte < 0 || sourceByte >= 16 { + continue + } + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", extracted, byteWidth, source, laneBase+sourceByte) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i8> %s, i8 %%%s, i32 %d\n", inserted, byteWidth, result, extracted, laneBase+destinationByte) + result = "%" + inserted + } + } + return result +} diff --git a/amd64_lower_byte_shift_test.go b/amd64_lower_byte_shift_test.go new file mode 100644 index 000000000..958b0b34f --- /dev/null +++ b/amd64_lower_byte_shift_test.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedByteShiftCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + zSource, zDestination := 16, 31 + if target.goarch == "386" { + legacyLast = 7 + zSource, zDestination = 6, 7 + } + var source strings.Builder + source.WriteString("TEXT packedbyteshiftforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSLLO", "PSLLDQ", "PSRLO", "PSRLDQ"} { + fmt.Fprintf(&source, "\t%s $-128, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s $127, X%d\n", op, legacyLast) + } + for _, op := range []string{"VPSLLDQ", "VPSRLDQ"} { + source.WriteString("\t" + op + " $-128, X0, X1\n") + source.WriteString("\t" + op + " $255, Y0, Y1\n") + source.WriteString("\t" + op + " $1, 8(AX), X2\n") + source.WriteString("\t" + op + " $2, 16(AX), Y2\n") + fmt.Fprintf(&source, "\t%s $3, X%d, X%d\n", op, zSource, zDestination) + fmt.Fprintf(&source, "\t%s $4, Z%d, Z%d\n", op, zSource, zDestination) + } + source.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, source.String(), "packedbyteshiftforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-byte-shift-"+target.name+".ll", "packed-byte-shift-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedByteShiftRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSLLO $-129, X0", + "PSLLDQ $128, X0", + "PSRLO $1, X16", + "PSRLDQ $1, Y0", + "PSLLDQ.Z $1, X0", + "VPSLLDQ $-129, X0, X1", + "VPSRLDQ $256, X0, X1", + "VPSLLDQ $-1, 8(AX), X1", + "VPSRLDQ $-1, X16, X1", + "VPSLLDQ $-1, Z0, Z1", + "VPSRLDQ $1, X0, Y1", + "VPSLLDQ $1, X0, K1, X1", + "VPSRLDQ.Z $1, X0, X1", + "VPSLLDQ $1, X0, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedByteShiftRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PSLLO $1, X8", + "PSRLO $1, X15", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedByteShiftRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedByteShiftRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed byte-shift tables for %s", instruction, goarch) + } +} + +func TestAMD64PackedByteShiftRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT packedbyteshift(SB),NOSPLIT,$0-16 + MOVQ out+0(FP), AX + MOVQ in+8(FP), BX + VMOVDQU (BX), Y0 + VPSLLDQ $3, Y0, Y1 + VMOVDQU Y1, 0(AX) + VPSRLDQ $5, Y0, Y1 + VMOVDQU Y1, 32(AX) + VPSLLDQ $-1, Y0, Y1 + VMOVDQU Y1, 64(AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedbyteshift": {Name: "packedbyteshift", Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void packedbyteshift(uint8_t *, const uint8_t *); +int main(void) { + uint8_t in[32], out[96]; + for (int i = 0; i < 32; i++) in[i] = (uint8_t)(i+1); + for (int i = 0; i < 96; i++) out[i] = 0xff; + packedbyteshift(out, in); + for (int lane = 0; lane < 2; lane++) for (int i = 0; i < 16; i++) { + int base = lane*16; + uint8_t left = i < 3 ? 0 : in[base+i-3]; + uint8_t right = i+5 >= 16 ? 0 : in[base+i+5]; + if (out[base+i] != left) return 10+base+i; + if (out[32+base+i] != right) return 50+base+i; + if (out[64+base+i] != 0) return 90+base+i; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_byte_shift", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_bzhi.go b/amd64_lower_bzhi.go new file mode 100644 index 000000000..318593e13 --- /dev/null +++ b/amd64_lower_bzhi.go @@ -0,0 +1,51 @@ +package plan9asm + +import "fmt" + +// lowerBZHI implements Go 1.27's complete _ybextrl operand row for BZHIL/Q: +// register index, register/memory source, and register destination. +func (c *amd64Ctx) lowerBZHI(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86ScalarBitFieldFamilyProperties(op, ins, c.goarch, x86BitFieldZeroHigh) + if !known || err != nil { + return known, false, err + } + + bits := form.bits + typ := amd64IntegerTypeForBits(bits) + indexValue, err := c.evalIntSized(form.control, typ) + if err != nil { + return true, false, err + } + sourceValue, err := c.evalScalarBitFieldSource(ins, form, typ) + if err != nil { + return true, false, err + } + indexLowByte := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, 255\n", indexLowByte, typ, indexValue) + indexInRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %d\n", indexInRange, typ, indexLowByte, bits) + safeIndex := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", safeIndex, typ, indexLowByte, bits-1) + one := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s 1, %%%s\n", one, typ, safeIndex) + partialMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %%%s, -1\n", partialMask, typ, one) + mask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s -1\n", mask, indexInRange, typ, partialMask, typ) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", result, typ, sourceValue, mask) + if err := c.storeRegSized(form.destination.Reg, typ, "%"+result); err != nil { + return true, false, err + } + carry := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", carry, indexInRange) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", sign, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + return true, false, nil +} diff --git a/amd64_lower_bzhi_test.go b/amd64_lower_bzhi_test.go new file mode 100644 index 000000000..cc53e5d48 --- /dev/null +++ b/amd64_lower_bzhi_test.go @@ -0,0 +1,201 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86BZHICompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA bzhidata+0(SB)/8, $1\n") + source.WriteString("GLOBL bzhidata(SB), $8\n") + source.WriteString("TEXT bzhiforms(SB),$0-0\n") + for _, width := range []string{"L", "Q"} { + op := "BZHI" + width + fmt.Fprintf(&source, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&source, "\t%s DX, 8(BX), SI\n", op) + fmt.Fprintf(&source, "\t%s CX, bzhidata(SB), DX\n", op) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12, R13\n", op) + } else { + fmt.Fprintf(&source, "\t%s AX, BX, SP\n", op) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86BZHICompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86BZHICompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + // Preserve every original format, including its unbound registers + // and mode32 SP destination. These are not complete source entries. + ir := x86UnboundVoidScalarFormatObject(t, source, "bzhiforms", target.goarch, target.triple) + wants := []string{"shl i32", `"target-features"="+bmi2"`} + if target.goarch == "amd64" { + wants = append(wants, "shl i64") + } else if strings.Contains(ir, "shl i64") { + t.Fatal("mode32 Q spelling used a fictitious 64-bit zero-high operation") + } + for _, want := range wants { + if !strings.Contains(ir, want) { + t.Fatalf("BZHI lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "bzhi-"+target.name+".ll", "bzhi-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86BZHIRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "BZHIL $1, AX, BX"}, + {goarch: "amd64", instruction: "BZHIQ 0(AX), BX, CX"}, + {goarch: "amd64", instruction: "BZHIQ AX, $1, CX"}, + {goarch: "amd64", instruction: "BZHIL AX, BX, 0(CX)"}, + {goarch: "amd64", instruction: "BZHIQ X0, AX, BX"}, + {goarch: "amd64", instruction: "BZHIQ AX, X0, BX"}, + {goarch: "amd64", instruction: "BZHIQ AX, BX, X0"}, + {goarch: "amd64", instruction: "BZHIQ AX, BX"}, + {goarch: "amd64", instruction: "BZHIQ.Z AX, BX, CX"}, + {goarch: "386", instruction: "BZHIL R8, AX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _ybextrl table", test.instruction) + } + }) + } +} + +func TestAMD64BZHIRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT bzhisemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + + MOVL $0x7fffffff, R8 + ADDL $1, R8 + MOVL $8, AX + MOVL $0x1234, BX + BZHIL AX, BX, CX + MOVQ CX, 0(DI) + SETCS 8(DI) + SETOS 9(DI) + SETEQ 10(DI) + SETMI 11(DI) + + MOVL $31, AX + MOVL $0x80000000, BX + BZHIL AX, BX, CX + MOVQ CX, 16(DI) + SETEQ 24(DI) + + MOVL $32, AX + MOVL $0x80000000, BX + BZHIL AX, BX, CX + MOVQ CX, 32(DI) + SETCS 40(DI) + SETMI 41(DI) + + MOVQ $0x1000000000000004, AX + MOVQ $-1, 48(DI) + BZHIQ AX, 48(DI), CX + MOVQ CX, 56(DI) + + MOVQ $64, AX + MOVQ $0x1234, BX + BZHIQ AX, BX, AX + MOVQ AX, 64(DI) + SETCS 72(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "bzhisemantics": { + Name: "bzhisemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void bzhisemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[80] = {0}; + bzhisemantics(out); + if (load64(out + 0) != UINT64_C(0x34)) return 10; + if (out[8] != 0 || out[9] != 0 || out[10] != 0 || out[11] != 0) return 11; + if (load64(out + 16) != 0 || out[24] != 1) return 12; + if (load64(out + 32) != UINT64_C(0x80000000) || out[40] != 1 || out[41] != 1) return 13; + if (load64(out + 56) != UINT64_C(0xf)) return 14; + if (load64(out + 64) != UINT64_C(0x1234) || out[72] != 1) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "bzhi_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_cache_line.go b/amd64_lower_cache_line.go new file mode 100644 index 000000000..57dbc2054 --- /dev/null +++ b/amd64_lower_cache_line.go @@ -0,0 +1,42 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var x86CacheLineWritebackOps = map[string]struct{}{ + "CLFLUSH": {}, + "CLFLUSHOPT": {}, + "CLWB": {}, +} + +// lowerCacheLineWriteback covers Go 1.27's complete x86 yclflush operand +// table. All three instructions accept exactly one Ym memory operand on both +// 386 and amd64. Preserve the architectural cache operation as inline +// assembly: replacing it with an ordinary load would lose ordering and +// persistence semantics. +func (c *amd64Ctx) lowerCacheLineWriteback(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if _, ok := x86CacheLineWritebackOps[baseOp]; !ok { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 1 || !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s expects one Go 1.27 Ym memory operand: %q", c.goarch, baseOp, ins.Raw) + } + + ptr, ptrType, err := c.x86DescriptorMemoryPointer(ins.Args[0]) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype(i8) %s)\n", + strings.ToLower(baseOp)+" $0", "*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, ptr) + return true, false, nil +} diff --git a/amd64_lower_cache_line_test.go b/amd64_lower_cache_line_test.go new file mode 100644 index 000000000..52f173bdb --- /dev/null +++ b/amd64_lower_cache_line_test.go @@ -0,0 +1,85 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86CacheLineWritebackCompleteGoAssemblerForms(t *testing.T) { + const source = ` +TEXT cachelinewriteback(SB),$16-0 + CLFLUSH (AX) + CLFLUSHOPT 8(BX) + CLWB local+0(SP) + RET +` + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "386", triple: "i686-pc-windows-msvc"}, + {goarch: "amd64", triple: "x86_64-apple-darwin"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.goarch+"/"+target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + TargetTriple: target.triple, Goarch: target.goarch, + Sigs: map[string]FuncSig{"cachelinewriteback": {Name: "cachelinewriteback", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, mnemonic := range []string{"clflush $0", "clflushopt $0", "clwb $0"} { + if !strings.Contains(ir, mnemonic) { + t.Fatalf("cache-line lowering omitted %q:\n%s", mnemonic, ir) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "x86-cache-line-writeback.ll", "x86-cache-line-writeback.o", ir) + }) + } +} + +func TestTranslateX86CacheLineWritebackRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + triple := "i386-unknown-linux-gnu" + if goarch == "amd64" { + triple = "x86_64-unknown-linux-gnu" + } + for _, instruction := range []string{ + "CLFLUSH", + "CLFLUSH AX", + "CLFLUSH $1", + "CLFLUSH (AX), (BX)", + "CLFLUSH.X (AX)", + "CLFLUSHOPT AX", + "CLWB AX", + } { + t.Run(goarch+"/"+strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + TargetTriple: triple, Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + if err == nil { + t.Fatalf("Translate accepted form rejected by Go 1.27: %s", instruction) + } + }) + } + } +} diff --git a/amd64_lower_call_abi0_test.go b/amd64_lower_call_abi0_test.go new file mode 100644 index 000000000..8346f9523 --- /dev/null +++ b/amd64_lower_call_abi0_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86ABI0CallUsesOutgoingStackFrame(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + mov string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", mov: "MOVQ"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", mov: "MOVL"}, + } { + t.Run(target.goarch, func(t *testing.T) { + source := "TEXT ·abi0Caller(SB),$48-0\n" + + "\t" + target.mov + " $1, AX\n" + + "\t" + target.mov + " AX, 0(SP)\n" + + "\t" + target.mov + " $2, AX\n" + + "\t" + target.mov + " AX, 8(SP)\n" + + "\t" + target.mov + " $3, AX\n" + + "\t" + target.mov + " AX, 16(SP)\n" + + "\tMOVL $0x3fc00000, 24(SP)\n" + + "\t" + target.mov + " $4, AX\n" + + "\t" + target.mov + " AX, 32(SP)\n" + + "\tCALL ·abi0Callee(SB)\n" + + "\t" + target.mov + " 40(SP), AX\n" + + "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + word := I64 + if target.goarch == "386" { + word = I32 + } + aggregate := LLVMType("{ " + string(Ptr) + ", " + string(word) + ", " + string(word) + " }") + callee := FuncSig{ + Name: "abi0Callee", + Args: []LLVMType{aggregate, LLVMType("float"), LLVMType("double")}, + Ret: word, + Frame: FrameLayout{ + Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: 0}, + {Offset: 8, Type: word, Index: 0, Field: 1}, + {Offset: 16, Type: word, Index: 0, Field: 2}, + {Offset: 24, Type: LLVMType("float"), Index: 1, Field: -1}, + {Offset: 32, Type: LLVMType("double"), Index: 2, Field: -1}, + }, + Results: []FrameSlot{{Offset: 40, Type: word, Index: 0, Field: -1}}, + }, + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + ResolveSym: func(sym string) string { return strings.TrimPrefix(sym, "·") }, + Sigs: map[string]FuncSig{ + "abi0Caller": {Name: "abi0Caller", Ret: Void}, + "abi0Callee": callee, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"load ptr", "load float", "load double", "call " + string(word) + " @abi0Callee", "store " + string(word)} { + if !strings.Contains(ll, want) { + t.Fatalf("ABI0 call omitted %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "abi0-call-"+target.goarch+".ll", "abi0-call-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86IndirectBranchCompatibilitySyntax(t *testing.T) { + const source = ` +TEXT ·indirectCallRegister(SB),$0-0 + CALL *AX + RET + +TEXT ·indirectCallSymbol(SB),$0-0 + CALL *·pointer(SB) + RET + +TEXT ·indirectJumpRegister(SB),$0-0 + JMP *AX +` + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + ResolveSym: func(sym string) string { return strings.TrimPrefix(sym, "·") }, + Sigs: map[string]FuncSig{ + "indirectCallRegister": {Name: "indirectCallRegister", Ret: Void}, + "indirectCallSymbol": {Name: "indirectCallSymbol", Ret: Void}, + "indirectJumpRegister": {Name: "indirectJumpRegister", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + if got := strings.Count(ll, "call i64 %"); got != 3 { + t.Fatalf("indirect CALL/JMP count = %d, want 3:\n%s", got, ll) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "indirect-branch-"+target.goarch+".ll", "indirect-branch-"+target.goarch+".o", ll) + }) + } +} diff --git a/amd64_lower_carry_flag_test.go b/amd64_lower_carry_flag_test.go new file mode 100644 index 000000000..186758247 --- /dev/null +++ b/amd64_lower_carry_flag_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func TestTranslateX86CarryFlagControlCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT carryflagforms(SB),NOSPLIT,$0-0\n\tCLC\n\tSTC\n\tCMC\n\tRET\n") + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"carryflagforms": {Name: "carryflagforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"store i1 false, ptr %flags_cf", "store i1 true, ptr %flags_cf", "xor i1"} { + if !strings.Contains(ll, want) { + t.Fatalf("carry flag lowering missing %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "carry-flag-"+target.name+".ll", "carry-flag-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86CarryFlagControlRejectsOperands(t *testing.T) { + for _, instruction := range []string{"CLC AX", "STC $1", "CMC 0(BX)", "CLC.Z"} { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's carry-control ynone table", instruction) + } + }) + } +} + +func TestAMD64CarryFlagControlRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT carryflagsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + CLC + MOVQ $0, AX + ADCQ $0, AX + MOVQ AX, 0(DI) + STC + MOVQ $0, AX + ADCQ $0, AX + MOVQ AX, 8(DI) + CLC + CMC + MOVQ $0, AX + ADCQ $0, AX + MOVQ AX, 16(DI) + STC + CMC + MOVQ $0, AX + ADCQ $0, AX + MOVQ AX, 24(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "carryflagsemantics": {Name: "carryflagsemantics", Args: []LLVMType{Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void carryflagsemantics(uint64_t *); +int main(void) { + uint64_t out[4] = {9, 9, 9, 9}; + const uint64_t want[4] = {0, 1, 1, 0}; + carryflagsemantics(out); + for (int i = 0; i < 4; i++) if (out[i] != want[i]) return 10+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "carry_flag_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_carry_rotate_test.go b/amd64_lower_carry_rotate_test.go new file mode 100644 index 000000000..b0daf861a --- /dev/null +++ b/amd64_lower_carry_rotate_test.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86CarryRotateCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + widths := []string{"B", "W", "L"} + if target.goarch == "amd64" { + widths = append(widths, "Q") + } + var source strings.Builder + source.WriteString("TEXT carryrotateforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"RCL", "RCR"} { + for _, width := range widths { + op := stem + width + reg := "AX" + if width == "B" { + reg = "AH" + } + fmt.Fprintf(&source, "\t%s $1, %s\n", op, reg) + fmt.Fprintf(&source, "\t%s $255, 8(BX)\n", op) + fmt.Fprintf(&source, "\t%s CL, %s\n", op, reg) + fmt.Fprintf(&source, "\t%s CX, 16(BX)\n", op) + } + } + source.WriteString("\tRET\n") + // Counts, CF, data and native addresses are intentionally absent. + // Preserve every original Go row and private emitter object, not + // a public numerical or caller-memory contract. + ll := x86UnboundVoidScalarFormatObject(t, source.String(), "carryrotateforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "carry-rotate-"+target.name+".ll", "carry-rotate-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86CarryRotateRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "RCLB $-1, AL", + "RCRW $256, AX", + "RCLL AX, BX", + "RCRQ (AX), BX", + "RCLQ $1, X0", + "RCRB.Z $1, AL", + "RCLL $1, AX, BX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86CarryRotateRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "RCLQ $1, AX", + "RCRQ CL, (BX)", + "RCLB $1, SP", + "RCRL $1, R8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86CarryRotateRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86CarryRotateRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's RCL/RCR tables for %s", instruction, goarch) + } +} diff --git a/amd64_lower_clmul.go b/amd64_lower_clmul.go new file mode 100644 index 000000000..4781c6126 --- /dev/null +++ b/amd64_lower_clmul.go @@ -0,0 +1,90 @@ +package plan9asm + +import "fmt" + +// lowerPackedCarrylessMultiply implements Go 1.27's complete legacy and +// vector PCLMULQDQ tables. VPCLMULQDQ applies one independent 64x64->128 +// carryless multiplication to every 128-bit lane. +func (c *amd64Ctx) lowerPackedCarrylessMultiply(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86CLMULProperties(rawOp, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if f.legacy && f.memory >= 0 { + return true, false, c.lowerLegacyCLMULMemory(ins, f) + } + second, err := c.loadX86PackedShiftBytes(ins.Args[f.source], f.byteWidth, true) + if err != nil { + return true, false, err + } + first, err := c.loadPackedCompareBytes(ins.Args[f.second], f.byteWidth) + if err != nil { + return true, false, err + } + result := c.emitPackedCarrylessMultiply(f.byteWidth, first, second, f.immediate) + if f.legacy { + return true, false, c.storeX(ins.Args[f.destination].Reg, result) + } + return true, false, c.storePackedMoveOperand(ins.Args[f.destination], f.byteWidth, result) +} + +// The legacy m128 opcode, not its selected numerical qword, determines the +// native memory/alignment/fault effect. Retain the whole original instruction +// even when its product is dead; virtual FP/SP/BP storage is not that address. +func (c *amd64Ctx) lowerLegacyCLMULMemory(ins Instr, f x86CLMULForm) error { + source := ins.Args[f.source] + virtual := source.Kind == OpFP + if source.Kind == OpMem && !x86CallerMemoryCertified(ins, f.source) && !c.fixedLocalMemoryCertified(ins, f.source) { + virtual = !c.nativeMemoryOperand(ins, f.source) + } + if virtual { + return fmt.Errorf("%w: legacy carryless multiply m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return err + } + first, err := c.loadPackedCompareLanes(ins.Args[f.second], 16, 64, false) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + assembly := fmt.Sprintf("pclmulqdq $$%d, %s($2), $0", f.immediate, segment) + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <2 x i64> asm sideeffect %q, %q(<2 x i64> %s, %s %s)\n", + result, assembly, "=x,0,r,~{memory}", first, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bytes, result) + return c.storeX(ins.Args[f.destination].Reg, "%"+bytes) +} + +func (c *amd64Ctx) emitPackedCarrylessMultiply(byteWidth int, firstBytes, secondBytes string, immediate uint8) string { + qwords := byteWidth / 8 + first := c.newTmp() + second := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i64>\n", first, byteWidth, firstBytes, qwords) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i64>\n", second, byteWidth, secondBytes, qwords) + result := "zeroinitializer" + for lane := 0; lane < byteWidth/16; lane++ { + firstLane := c.newTmp() + secondLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i64> %%%s, <%d x i64> poison, <2 x i32> \n", firstLane, qwords, first, qwords, lane*2, lane*2+1) + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i64> %%%s, <%d x i64> poison, <2 x i32> \n", secondLane, qwords, second, qwords, lane*2, lane*2+1) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %%%s, <2 x i64> %%%s, i8 %d)\n", product, firstLane, secondLane, immediate) + for word := 0; word < 2; word++ { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 %d\n", extracted, product, word) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, qwords, result, extracted, lane*2+word) + result = "%" + inserted + } + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", bytes, qwords, result, byteWidth) + return "%" + bytes +} diff --git a/amd64_lower_clmul_test.go b/amd64_lower_clmul_test.go new file mode 100644 index 000000000..b92772964 --- /dev/null +++ b/amd64_lower_clmul_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedCarrylessMultiplyCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + fmt.Fprintf(&source, "TEXT carrylessmultiplyforms(SB),NOSPLIT,$0-0\n\tPCLMULQDQ $0, X0, X%d\n\tPCLMULQDQ $255, 8(AX), X%d\n", last, last) + if target.goarch == "amd64" { + source.WriteString("\tVPCLMULQDQ $0, X0, X1, X2\n") + source.WriteString("\tVPCLMULQDQ $1, 8(AX), X1, X2\n") + source.WriteString("\tVPCLMULQDQ $16, Y0, Y1, Y2\n") + source.WriteString("\tVPCLMULQDQ $17, 8(AX), Y1, Y2\n") + source.WriteString("\tVPCLMULQDQ $127, Z0, Z1, Z2\n") + source.WriteString("\tVPCLMULQDQ $128, 8(AX), Z1, Z2\n") + source.WriteString("\tVPCLMULQDQ $255, X16, X31, X0\n") + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "carrylessmultiplyforms": {Name: "carrylessmultiplyforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-carryless-multiply-"+target.name+".ll", "packed-carryless-multiply-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedCarrylessMultiplyRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PCLMULQDQ $-1, X0, X1", + "PCLMULQDQ $256, X0, X1", + "PCLMULQDQ $1, X16, X1", + "PCLMULQDQ $1, Y0, Y1", + "PCLMULQDQ.Z $1, X0, X1", + "PCLMULQDQ $1, X0, (AX)", + "VPCLMULQDQ $-1, X0, X1, X2", + "VPCLMULQDQ $256, X0, X1, X2", + "VPCLMULQDQ $1, X0, Y1, Y2", + "VPCLMULQDQ $1, X0, X1, K1, X2", + "VPCLMULQDQ.Z $1, X0, X1, X2", + "VPCLMULQDQ $1, X0, (AX), X2", + "VPCLMULQDQ $1, X0, X1, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedCarrylessMultiplyRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PCLMULQDQ $1, X8, X0", + "PCLMULQDQ $1, X0, X8", + "VPCLMULQDQ $1, X0, X1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedCarrylessMultiplyRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedCarrylessMultiplyRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's PCLMULQDQ/VPCLMULQDQ tables for %s", instruction, goarch) + } +} + +func TestAMD64PackedCarrylessMultiplyRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT carrylessmultiply(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), AX + MOVQ a+8(FP), BX + MOVQ b+16(FP), CX + VMOVDQU (BX), Y0 + VMOVDQU (CX), Y1 + VPCLMULQDQ $0x10, Y0, Y1, Y2 + VMOVDQU Y2, (AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "carrylessmultiply": {Name: "carrylessmultiply", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void carrylessmultiply(uint64_t *, const uint64_t *, const uint64_t *); +static void clmul(uint64_t a, uint64_t b, uint64_t *lo, uint64_t *hi) { + uint64_t l = 0, h = 0; + for (unsigned i = 0; i < 64; i++) if ((b >> i) & 1) { + l ^= a << i; + if (i) h ^= a >> (64-i); + } + *lo = l; *hi = h; +} +int main(void) { + uint64_t a[4] = {0x123456789abcdef0ULL, 0xfedcba9876543210ULL, 0x1111222233334444ULL, 0x5555666677778888ULL}; + uint64_t b[4] = {0x0123456789abcdefULL, 0x0f1e2d3c4b5a6978ULL, 0x9999aaaabbbbccccULL, 0xddddeeeeffff0001ULL}; + uint64_t out[4] = {0}, want[4] = {0}; + clmul(b[0], a[1], &want[0], &want[1]); + clmul(b[2], a[3], &want[2], &want[3]); + carrylessmultiply(out, a, b); + for (int i = 0; i < 4; i++) if (out[i] != want[i]) return 10+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_carryless_multiply", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_cmov.go b/amd64_lower_cmov.go new file mode 100644 index 000000000..a2e62bb9f --- /dev/null +++ b/amd64_lower_cmov.go @@ -0,0 +1,137 @@ +package plan9asm + +import "fmt" + +// lowerConditionalMove implements the complete Go 1.27 yml_rl conditional +// move table: every x86 condition code at W/L/Q width, with a register or +// memory source and a GP-register destination. +func (c *amd64Ctx) lowerConditionalMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86ConditionalMoveProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + ty := amd64IntegerTypeForBits(form.bits) + src, err := c.loadConditionalMoveSource(form, ins) + if err != nil { + return true, false, err + } + cur, err := c.evalIntSized(form.destination, ty) + if err != nil { + return true, false, err + } + condition, err := c.x86Condition(form.condition) + if err != nil { + return true, false, err + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, %s %s, %s %s\n", selected, condition, ty, src, ty, cur) + // Select only the operand width, then apply the architectural writeback. + // In particular a false L predicate still zeroes GP bits 63:32. + return true, false, c.storeRegSized(form.destination.Reg, ty, "%"+selected) +} + +func (c *amd64Ctx) loadConditionalMoveSource(form x86ConditionalMoveForm, ins Instr) (string, error) { + source := form.source + if source.Kind == OpReg { + return c.evalIntSized(source, amd64IntegerTypeForBits(form.bits)) + } + if source.Kind == OpMem && !c.nativeMemoryOperand(ins, 0) && + !x86CallerMemoryCertified(ins, 0) && !c.fixedLocalMemoryCertified(ins, 0) { + return "", fmt.Errorf("%w: CMOV native stack source lacks its original storage certificate", ErrProbeNeedsContext) + } + if source.Kind == OpFP { + // Only existing declared numerical ABI0 fields supply logical bytes. + // Neither pointer fields, holes nor an unknown native frame are inputs. + if err := c.validateX86PackedShiftFPSpan(source.FPOffset, form.bits/8); err != nil { + return "", err + } + result := false + for offset := source.FPOffset; offset < source.FPOffset+int64(form.bits/8); offset++ { + slot, output, _ := c.fpVectorSlot(offset) + if slot.Type != I8 && slot.Type != I16 && slot.Type != I32 && slot.Type != I64 && slot.Type != LLVMType("float") && slot.Type != LLVMType("double") { + return "", fmt.Errorf("%w: CMOV FP byte is not declared numerical storage", ErrProbeNeedsContext) + } + result = result || output + } + if result { + if err := c.scalarFPResultInitialized(); err != nil { + return "", err + } + } + } + // The shared FP loader bitcasts floating slots before extracting bytes; + // NaN payloads and signed zero are not floating-to-integer conversions. + // Intel CMOVcc reads the complete memory source unconditionally, even for + // a false predicate or dead result. Preserve the original width and fault. + bytes, err := c.loadX86PackedShiftBytes(source, form.bits/8, true) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", value, form.bits/8, bytes, form.bits) + if source.Kind != OpFP || c.classicFrame != "" { + // LLVM22's x86 select expansion can sink even a volatile load into + // one branch. An unconditional register use keeps the actual source + // read before that expansion without emitting an instruction or + // granting a new memory/FP permission. A false CMOV must still fault. + fmt.Fprintf(c.b, " call void asm sideeffect \"\", \"r\"(i%d %%%s)\n", form.bits, value) + } + return "%" + value, nil +} + +func (c *amd64Ctx) x86Condition(code string) (string, error) { + invert := func(value string) string { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", out, value) + return "%" + out + } + combine := func(instruction, left, right string) string { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i1 %s, %s\n", out, instruction, left, right) + return "%" + out + } + + switch code { + case "CC": + return invert(c.loadFlag(c.flagsCFSlot)), nil + case "CS": + return c.loadFlag(c.flagsCFSlot), nil + case "EQ": + return c.loadFlag(c.flagsZSlot), nil + case "GE", "GT", "LE", "LT": + signedLess := combine("xor", c.loadFlag(c.flagsSltSlot), c.loadFlag(c.flagsOFSlot)) + switch code { + case "LT": + return signedLess, nil + case "GE": + return invert(signedLess), nil + case "LE": + return combine("or", c.loadFlag(c.flagsZSlot), signedLess), nil + case "GT": + return invert(combine("or", c.loadFlag(c.flagsZSlot), signedLess)), nil + } + case "HI": + return combine("and", invert(c.loadFlag(c.flagsCFSlot)), invert(c.loadFlag(c.flagsZSlot))), nil + case "LS": + return combine("or", c.loadFlag(c.flagsCFSlot), c.loadFlag(c.flagsZSlot)), nil + case "MI": + return c.loadFlag(c.flagsSltSlot), nil + case "NE": + return invert(c.loadFlag(c.flagsZSlot)), nil + case "OC": + return invert(c.loadFlag(c.flagsOFSlot)), nil + case "OS": + return c.loadFlag(c.flagsOFSlot), nil + case "PC": + return invert(c.loadFlag(c.flagsPFSlot)), nil + case "PL": + return invert(c.loadFlag(c.flagsSltSlot)), nil + case "PS": + return c.loadFlag(c.flagsPFSlot), nil + } + return "", fmt.Errorf("unsupported x86 condition code %q", code) +} diff --git a/amd64_lower_cmpbt.go b/amd64_lower_cmpbt.go index 3b9369859..e82f6bb64 100644 --- a/amd64_lower_cmpbt.go +++ b/amd64_lower_cmpbt.go @@ -6,179 +6,87 @@ import ( ) func (c *amd64Ctx) lowerCmpBt(op Op, ins Instr) (ok bool, terminated bool, err error) { - switch op { - case "CMPB", "CMPW", "CMPL", "CMPQ": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 CMPQ expects 2 operands: %q", ins.Raw) - } - ty := I64 - switch op { - case "CMPB": - ty = I8 - case "CMPW": - ty = I16 - case "CMPL": - ty = I32 - case "CMPQ": - ty = I64 - } - a, err := c.evalIntSized(ins.Args[0], ty) - if err != nil { - return true, false, err - } - b, err := c.evalIntSized(ins.Args[1], ty) - if err != nil { - return true, false, err - } - c.setCmpFlagsSized(ty, a, b) - return true, false, nil - - case "TESTB", "TESTW", "TESTL", "TESTQ": - // TEST* a,b sets flags based on (a & b) without storing the result. - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects 2 operands: %q", op, ins.Raw) - } - ty := I64 - switch op { - case "TESTB": - ty = I8 - case "TESTW": - ty = I16 - case "TESTL": - ty = I32 - case "TESTQ": - ty = I64 - } - a, err := c.evalIntSized(ins.Args[0], ty) - if err != nil { - return true, false, err - } - b, err := c.evalIntSized(ins.Args[1], ty) - if err != nil { - return true, false, err - } - c.setTestFlagsSized(ty, a, b) - return true, false, nil - - case "BTQ": - // BTQ $imm, reg (sets CF to selected bit) - if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 BTQ expects $imm, reg: %q", ins.Raw) - } - v, err := c.loadReg(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - amt := ins.Args[0].Imm & 63 - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", sh, v, amt) - and := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 1\n", and, sh) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", cf, and) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - return true, false, nil - - case "BTSQ", "BTRQ": - // BTSQ/BTRQ src, dstReg|dstMem: CF = old bit, then set or - // clear the selected destination bit. - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) + if ok, terminated, err := c.lowerBitTestFamily(op, ins); ok { + return ok, terminated, err + } + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86ScalarCompareProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + for index, operand := range form.args { + if operand.Kind == OpReg && operand.Reg == SP && ins.Args[index].Reg != SP { + return true, false, fmt.Errorf("%w: Go REX byte view needs the original physical SP", ErrProbeNeedsContext) } - var amt string - var registerIndex string - switch ins.Args[0].Kind { - case OpImm: - amt = fmt.Sprintf("%d", ins.Args[0].Imm&63) - case OpReg: - av, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err + } + native := form.memory >= 0 && (c.nativeMemoryOperand(ins, form.memory) || + x86CallerMemoryCertified(ins, form.memory) || c.fixedLocalMemoryCertified(ins, form.memory)) + if native && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + return true, false, fmt.Errorf("%w: scalar comparison needs its original native address size", ErrProbeNeedsContext) + } + typ := LLVMType(fmt.Sprintf("i%d", form.bits)) + var values [2]string + for index, operand := range form.args { + if index == form.memory && native { + if source := c.goReadOnlyStorage; source != nil { + address, named := x86StaticSBSourceAddress(operand, arch) + if named && source.object(address.symbol).variable != nil && + !source.allows(operand, int64(form.bits/8), arch) { + return true, false, goReadOnlyContext("comparison exceeds the declared numerical span") + } } - m := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 63\n", m, av) - amt = "%" + m - registerIndex = av - default: - return true, false, fmt.Errorf("amd64 %s expects imm/reg bit index: %q", op, ins.Raw) - } - - var dst string - var storeDst func(string) error - switch ins.Args[1].Kind { - case OpReg: - dv, err := c.loadReg(ins.Args[1].Reg) - if err != nil { - return true, false, err + pointer, pointerType, _, pointerErr := c.packedVectorMemoryPointer(operand) + if pointerErr != nil { + return true, false, pointerErr } - dst = dv - storeDst = func(v string) error { return c.storeReg(ins.Args[1].Reg, v) } - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) + loaded := c.newTmp() + // A compare whose flags are dead must still perform the exact + // native read (and fault). No backing/alignment assumption follows. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", loaded, typ, pointerType, pointer) + values[index] = "%" + loaded + } else { + values[index], err = c.evalIntSized(operand, typ) if err != nil { return true, false, err } - // A register bit index on a memory operand selects a bit in an - // unbounded bit string, not merely within the first qword. Adjust - // the address by floor(index/64)*8 and retain index&63 within it. - if registerIndex != "" { - word := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ashr i64 %s, 6\n", word, registerIndex) - byteOff := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, 3\n", byteOff, word) - adjusted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %s, %%%s\n", adjusted, addr, byteOff) - addr = "%" + adjusted - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - dst = "%" + ld - storeDst = func(v string) error { - fmt.Fprintf(c.b, " store i64 %s, ptr %s, align 1\n", v, p) - return nil - } - default: - return true, false, fmt.Errorf("amd64 %s expects reg/mem dst: %q", op, ins.Raw) - } - - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %s\n", sh, dst, amt) - and := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 1\n", and, sh) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", cf, and) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - one := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 1, %s\n", one, amt) - out := c.newTmp() - if op == "BTSQ" { - fmt.Fprintf(c.b, " %%%s = or i64 %s, %%%s\n", out, dst, one) - } else { - mask := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %%%s, -1\n", mask, one) - fmt.Fprintf(c.b, " %%%s = and i64 %s, %%%s\n", out, dst, mask) - } - if err := storeDst("%" + out); err != nil { - return true, false, err } - return true, false, nil } - return false, false, nil + if form.test { + c.setTestFlagsSized(typ, values[0], values[1]) + } else { + c.setCmpFlagsSized(typ, values[0], values[1]) + } + return true, false, nil } func (c *amd64Ctx) setCmpFlagsSized(ty LLVMType, a, b string) { // Unlike most Plan 9 x86 instructions, CMP spells its operands in // comparison order: CMP a,b sets flags for a-b. + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %s, %s\n", result, ty, a, b) + c.setAuxiliaryCarryFlag(ty, a, b, "%"+result) zt := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, %s\n", zt, ty, a, b) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zt, c.flagsZSlot) - slt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, %s\n", slt, ty, a, b) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", slt, c.flagsSltSlot) + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", sign, ty, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) ult := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %s\n", ult, ty, a, b) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", ult, c.flagsCFSlot) + xorOperands := c.newTmp() + xorResult := c.newTmp() + overflowBits := c.newTmp() + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorOperands, ty, a, b) + fmt.Fprintf(c.b, " %%%s = xor %s %s, %%%s\n", xorResult, ty, a, result) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", overflowBits, ty, xorOperands, xorResult) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", overflow, ty, overflowBits) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) + c.setParityFlagSized(ty, "%"+result) } func (c *amd64Ctx) setTestFlagsSized(ty LLVMType, a, b string) { @@ -192,9 +100,23 @@ func (c *amd64Ctx) setTestFlagsSized(ty LLVMType, a, b string) { fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", slt, ty, and) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", slt, c.flagsSltSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + c.setParityFlagSized(ty, "%"+and) } func (c *amd64Ctx) evalIntSized(op Operand, ty LLVMType) (string, error) { + if err := unresolvedSymbolicImmediateErrorForArch(op, ArchAMD64); err != nil { + return "", err + } + if read := c.goReadOnlyOperand; read != nil && ty == LLVMType(fmt.Sprintf("i%d", read.width*8)) && read.allows(op, read.width) { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(op) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", value, ty, pointerType, pointer) + return "%" + value, nil + } switch op.Kind { case OpImm: // LLVM will interpret the literal in the destination integer width. @@ -233,6 +155,18 @@ func (c *amd64Ctx) evalIntSized(op Operand, ty LLVMType) (string, error) { s := strings.TrimSpace(op.Sym) if strings.HasPrefix(s, "$") { address := strings.TrimSpace(strings.TrimPrefix(s, "$")) + if memory, matched, err := parseX86SymbolIndexedMemory(address); matched { + if err != nil { + return "", err + } + wide, err := c.addrFromSymbolIndexedAddress(memory) + if err != nil || ty == I64 { + return wide, err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", value, wide, ty) + return "%" + value, nil + } if strings.HasSuffix(address, "(SB)") { p, err := c.ptrFromSB(address) if err != nil { @@ -247,10 +181,14 @@ func (c *amd64Ctx) evalIntSized(op Operand, ty LLVMType) (string, error) { fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to %s\n", value, wide, ty) return "%" + value, nil } - // Macro-style symbolic immediates (e.g. $const_avxSupported) may - // survive preprocessing when include constants are unavailable. - // Keep translation progressing with a conservative zero value. - return "0", nil + return "", fmt.Errorf("unresolved symbolic immediate %q", op.Sym) + } + // Go's x86 syntax treats a bare integer as absolute memory. Keep + // this distinct from the '$' immediate spelling above. + if address, parseErr := parseInt(s); parseErr == nil { + t := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", t, ty, c.ptrFromAddrI64(fmt.Sprintf("%d", address))) + return "%" + t, nil } if strings.HasSuffix(s, "(SB)") { p, err := c.ptrFromSB(s) diff --git a/amd64_lower_cmps_test.go b/amd64_lower_cmps_test.go new file mode 100644 index 000000000..ae3fb32f2 --- /dev/null +++ b/amd64_lower_cmps_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86CMPSCompleteFormsSource(goarch string) string { + widths := []string{"B", "W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + var source strings.Builder + source.WriteString("TEXT cmpsforms(SB),$0-0\n") + for _, width := range widths { + op := "CMPS" + width + fmt.Fprintf(&source, "\t%s\n", op) + fmt.Fprintf(&source, "\tCLD; REP; %s\n", op) + fmt.Fprintf(&source, "\tSTD; REPN; %s\n", op) + source.WriteString("\tCLD\n") + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86CMPSCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86CMPSCompleteFormsSource(target.goarch) + ir := x86UnboundVoidScalarFormatObject(t, source, "cmpsforms", target.goarch, target.triple) + for _, width := range []string{"b", "w", "l"} { + if want := "@__plan9asm_rep_cmps" + width; !strings.Contains(ir, want) { + t.Fatalf("CMPS lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "@__plan9asm_rep_cmpsq") { + t.Fatalf("CMPSQ lowering omitted qword helper:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "cmps-"+target.name+".ll", "cmps-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86CMPSRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "CMPSB AX"}, + {goarch: "amd64", instruction: "CMPSW AX, BX"}, + {goarch: "amd64", instruction: "CMPSL.Z"}, + {goarch: "386", instruction: "CMPSQ"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's CMPS ynone table", test.instruction) + } + }) + } +} + +func TestAMD64CMPSRuntimeSemanticsPrefixesDirectionAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT cmpssemantics(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), R8 + MOVQ left+8(FP), R9 + MOVQ right+16(FP), R10 + + MOVQ R9, SI + MOVQ R10, DI + MOVQ $4, CX + CLD + REP; CMPSB + MOVQ SI, 0(R8) + MOVQ DI, 8(R8) + MOVQ CX, 16(R8) + SETCS 24(R8) + SETEQ 25(R8) + SETMI 26(R8) + SETOS 27(R8) + SETPS 28(R8) + + MOVQ R9, SI + ADDQ $8, SI + MOVQ R10, DI + ADDQ $8, DI + MOVQ $3, CX + REPN; CMPSW + MOVQ SI, 32(R8) + MOVQ DI, 40(R8) + MOVQ CX, 48(R8) + SETCS 56(R8) + SETEQ 57(R8) + + MOVQ R9, SI + ADDQ $28, SI + MOVQ R10, DI + ADDQ $28, DI + STD + CMPSL + CLD + MOVQ SI, 64(R8) + MOVQ DI, 72(R8) + SETCS 80(R8) + SETEQ 81(R8) + SETMI 82(R8) + SETOS 83(R8) + + MOVQ R9, SI + ADDQ $40, SI + MOVQ R10, DI + ADDQ $40, DI + CMPSQ + MOVQ SI, 88(R8) + MOVQ DI, 96(R8) + SETEQ 104(R8) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "cmpssemantics": { + Name: "cmpssemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void cmpssemantics(uint8_t *, uint8_t *, uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t left[56] = {1,2,9,4}; + uint8_t right[56] = {1,2,3,4}; + const uint16_t leftw[3] = {5,7,8}, rightw[3] = {6,7,9}; + memcpy(left + 8, leftw, sizeof(leftw)); + memcpy(right + 8, rightw, sizeof(rightw)); + const uint32_t leftl[2] = {0,1}, rightl[2] = {0,2}; + memcpy(left + 24, leftl, sizeof(leftl)); + memcpy(right + 24, rightl, sizeof(rightl)); + const uint64_t q = UINT64_C(0x0123456789abcdef); + memcpy(left + 40, &q, sizeof(q)); + memcpy(right + 40, &q, sizeof(q)); + uint8_t out[112] = {0}; + cmpssemantics(out, left, right); + if (load64(out+0) != (uintptr_t)(left+3)) return 10; + if (load64(out+8) != (uintptr_t)(right+3)) return 11; + if (load64(out+16) != 1) return 12; + if (out[24] != 0 || out[25] != 0 || out[26] != 0 || out[27] != 0 || out[28] != 1) return 11; + if (load64(out+32) != (uintptr_t)(left+12) || load64(out+40) != (uintptr_t)(right+12) || load64(out+48) != 1) return 12; + if (out[56] != 0 || out[57] != 1) return 13; + if (load64(out+64) != (uintptr_t)(left+24) || load64(out+72) != (uintptr_t)(right+24)) return 14; + if (out[80] != 1 || out[81] != 0 || out[82] != 1 || out[83] != 0) return 15; + if (load64(out+88) != (uintptr_t)(left+48) || load64(out+96) != (uintptr_t)(right+48) || out[104] != 1) return 16; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "cmps_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_cmpxchg.go b/amd64_lower_cmpxchg.go new file mode 100644 index 000000000..3cf59ce4b --- /dev/null +++ b/amd64_lower_cmpxchg.go @@ -0,0 +1,266 @@ +package plan9asm + +import "fmt" + +// lowerCompareExchange implements every operand row in Go 1.27's yrb_mb, +// yrl_ml, and yscond tables for the CMPXCHG family. +func (c *amd64Ctx) lowerCompareExchange(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, recognized, err := x86CompareExchangeProperties(op, ins, c.goarch) + if !recognized { + return false, false, nil + } + if err != nil { + return true, false, err + } + if form.spec.pair { + return c.lowerPairCompareExchange(string(op), form.spec.bits, ins) + } + return c.lowerScalarCompareExchange(string(op), form.spec.bits, ins) +} + +func amd64CompareExchangeProperties(op string) (kind string, bits int, ok bool) { + spec, recognized := x86CompareExchangeSpecs[Op(op)] + if !recognized { + return "", 0, false + } + kind = "scalar" + if spec.pair { + kind = "pair" + } + return kind, spec.bits, true +} + +func (c *amd64Ctx) lowerScalarCompareExchange(baseOp string, bits int, ins Instr) (bool, bool, error) { + form, recognized, err := x86CompareExchangeProperties(Op(baseOp), ins, c.goarch) + if err != nil { + return true, false, err + } + if !recognized || form.spec.pair || form.spec.bits != bits { + return true, false, fmt.Errorf("%s %s has no matching scalar compare-exchange width", c.goarch, baseOp) + } + source, destination := form.source, form.destination + if err := c.validateCompareExchangeFP(destination, bits); err != nil { + return true, false, err + } + + typ := amd64IntegerTypeForBits(bits) + accumulator, err := c.loadReg(AX) + if err != nil { + return true, false, err + } + expected, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, typ) + if err != nil { + return true, false, err + } + desired, err := c.evalIntSized(source, typ) + if err != nil { + return true, false, err + } + + var old, success string + if destination.Kind == OpReg { + old, err = c.evalIntSized(destination, typ) + if err != nil { + return true, false, err + } + equal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, %s\n", equal, typ, expected, old) + success = "%" + equal + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, %s %s, %s %s\n", selected, success, typ, desired, typ, old) + if err := c.storeRegSized(destination.Reg, typ, "%"+selected); err != nil { + return true, false, err + } + } else { + ptr, ptrType, err := c.compareExchangePointer(destination) + if err != nil { + return true, false, err + } + old, success = c.emitScalarCompareExchangeNative(bits, destination, ptr, ptrType, expected, desired) + } + + // On failure the accumulator receives the destination. On success it is not + // written at all. In particular, CMPXCHGL zero-extends EAX only on failure; + // a successful compare must preserve the original high half of RAX. + if destination.Kind != OpReg || !amd64CompareExchangeDestinationIsAccumulator(destination.Reg, bits) { + if err := c.storeScalarCompareExchangeAccumulator(accumulator, expected, old, success, typ); err != nil { + return true, false, err + } + } + difference := c.newTmp() + // CMPXCHG compares the accumulator with the destination, so its arithmetic + // flags come from accumulator - destination (Intel's temporary result). + fmt.Fprintf(c.b, " %%%s = sub %s %s, %s\n", difference, typ, expected, old) + c.setScalarAddSubFlags(typ, false, expected, old, "%"+difference) + // Keep the equality result authoritative even for unusual aliasing forms. + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", success, c.flagsZSlot) + return true, false, nil +} + +func (c *amd64Ctx) storeScalarCompareExchangeAccumulator(accumulator, expected, failureValue, success string, typ LLVMType) error { + if typ == I32 { + failure64 := c.newTmp() + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", failure64, failureValue) + fmt.Fprintf(c.b, " %%%s = select i1 %s, i64 %s, i64 %%%s\n", selected, success, accumulator, failure64) + return c.storeReg(AX, "%"+selected) + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, %s %s, %s %s\n", selected, success, typ, expected, typ, failureValue) + return c.storeRegSized(AX, typ, "%"+selected) +} + +func amd64CompareExchangeDestinationIsAccumulator(reg Reg, bits int) bool { + if bits == 8 { + return reg == AX || reg == AL + } + return reg == AX +} + +func (c *amd64Ctx) lowerPairCompareExchange(baseOp string, bits int, ins Instr) (bool, bool, error) { + form, recognized, err := x86CompareExchangeProperties(Op(baseOp), ins, c.goarch) + if err != nil { + return true, false, err + } + if !recognized || !form.spec.pair || form.spec.bits != bits { + return true, false, fmt.Errorf("%s %s has no matching pair compare-exchange width", c.goarch, baseOp) + } + destination := form.destination + if err := c.validateCompareExchangeFP(destination, bits); err != nil { + return true, false, err + } + + ax, err := c.loadReg(AX) + if err != nil { + return true, false, err + } + dx, err := c.loadReg(DX) + if err != nil { + return true, false, err + } + bx, err := c.loadReg(BX) + if err != nil { + return true, false, err + } + cx, err := c.loadReg(CX) + if err != nil { + return true, false, err + } + ptr, ptrType, err := c.compareExchangePointer(destination) + if err != nil { + return true, false, err + } + old, success := c.emitPairCompareExchangeNative(bits, destination, ptr, ptrType, ax, dx, bx, cx) + + oldLow, oldHigh := c.compareExchangeSplitPair(old, bits) + if err := c.storeCompareExchangePairAccumulator(AX, ax, oldLow, success, bits); err != nil { + return true, false, err + } + if err := c.storeCompareExchangePairAccumulator(DX, dx, oldHigh, success, bits); err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", success, c.flagsZSlot) + return true, false, nil +} + +func (c *amd64Ctx) validateCompareExchangeFP(destination Operand, bits int) error { + if destination.Kind != OpFP { + return nil + } + context := func(reason string) error { + return fmt.Errorf("%w: %s CMPXCHG FP native write at +%d(FP): %s", ErrProbeNeedsContext, c.goarch, destination.FPOffset, reason) + } + if bits == 128 { + return context("logical FP storage has no original CMPXCHG16B native address/alignment identity") + } + width := bits / 8 + if err := c.validateFPVectorSpan(destination.FPOffset, width); err != nil { + return context(err.Error()) + } + if c.classicFrame != "" { + return nil + } + slot, _, ok := c.fpVectorSlot(destination.FPOffset) + if !ok || slot.Offset != destination.FPOffset || x86FrameTypeSize(slot.Type) < int64(width) { + return context("native access is not an existing complete contiguous shadow slot") + } + return nil +} + +func (c *amd64Ctx) compareExchangeJoinPair(low, high string, bits int) string { + partType := LLVMType("i32") + pairType := LLVMType("i64") + shift := 32 + if bits == 128 { + partType = I64 + pairType = LLVMType("i128") + shift = 64 + } + lowPart, highPart := low, high + if partType == I32 { + lowPart = c.truncI64(low, I32) + highPart = c.truncI64(high, I32) + } + lowWide := c.newTmp() + highWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", lowWide, partType, lowPart, pairType) + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", highWide, partType, highPart, pairType) + shifted := c.newTmp() + joined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %d\n", shifted, pairType, highWide, shift) + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", joined, pairType, shifted, lowWide) + return "%" + joined +} + +func (c *amd64Ctx) compareExchangeSplitPair(pair string, bits int) (low, high string) { + pairType := LLVMType("i64") + partType := LLVMType("i32") + shift := 32 + if bits == 128 { + pairType = LLVMType("i128") + partType = I64 + shift = 64 + } + lowName := c.newTmp() + highShift := c.newTmp() + highName := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", lowName, pairType, pair, partType) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %d\n", highShift, pairType, pair, shift) + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", highName, pairType, highShift, partType) + return "%" + lowName, "%" + highName +} + +func (c *amd64Ctx) storeCompareExchangePairAccumulator(reg Reg, original, failurePart, success string, bits int) error { + failure64 := failurePart + if bits == 64 { + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", widened, failurePart) + failure64 = "%" + widened + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i64 %s, i64 %s\n", selected, success, original, failure64) + return c.storeReg(reg, "%"+selected) +} + +func (c *amd64Ctx) compareExchangePointer(destination Operand) (ptr, ptrType string, err error) { + switch destination.Kind { + case OpMem: + return c.ptrFromMem(destination.Mem) + case OpSym: + ptr, err := c.ptrFromSB(destination.Sym) + return ptr, "ptr", err + case OpFP: + if c.classicFrame != "" { + return c.classicFramePtr(destination.FPOffset), "ptr", nil + } + if ptr := c.fpParamAlloca[destination.FPOffset]; ptr != "" { + return ptr, "ptr", nil + } + if ptr, _, ok := c.fpResultAlloca(destination.FPOffset); ok { + return ptr, "ptr", nil + } + return "", "", fmt.Errorf("CMPXCHG FP operand has no mutable slot at +%d(FP)", destination.FPOffset) + default: + return "", "", fmt.Errorf("CMPXCHG expected memory destination, got %s", destination.String()) + } +} diff --git a/amd64_lower_compare_float.go b/amd64_lower_compare_float.go new file mode 100644 index 000000000..e5aad5888 --- /dev/null +++ b/amd64_lower_compare_float.go @@ -0,0 +1,179 @@ +package plan9asm + +import ( + "fmt" + "strconv" + "strings" +) + +// lowerFloatingCompare implements the complete Go 1.27 floating-point +// immediate-compare family: legacy CMP{PD,PS,SD,SS}, the VEX vector-result +// forms, and the EVEX K-result forms of VCMP{PD,PS,SD,SS}. +func (c *amd64Ctx) lowerFloatingCompare(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86FloatCompareSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("floating compare dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, known, err := x86FloatCompareProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return true, false, c.emitNativeFloatingCompare(ins, form) +} + +func amd64FloatingCompareProperties(op string) (elemBits int, packed, vector, ok bool) { + switch op { + case "CMPPD": + return 64, true, false, true + case "CMPPS": + return 32, true, false, true + case "CMPSD": + return 64, false, false, true + case "CMPSS": + return 32, false, false, true + case "VCMPPD": + return 64, true, true, true + case "VCMPPS": + return 32, true, true, true + case "VCMPSD": + return 64, false, true, true + case "VCMPSS": + return 32, false, true, true + default: + return 0, false, false, false + } +} + +func amd64LegacyFloatingCompareImmediate(arg Operand) (int64, bool) { + if arg.Kind == OpImm { + return arg.Imm, true + } + // The pre-Go-1.5 x86 assembler wrote CMP{PD,PS,SD,SS} predicates as a + // bare decimal operand. Keep this compatibility confined to the legacy + // compare family's predicate position instead of accepting bare integers + // as immediates throughout the parser. + var text string + switch arg.Kind { + case OpIdent: + text = arg.Ident + case OpSym: + text = arg.Sym + default: + return 0, false + } + value, err := strconv.ParseInt(strings.TrimSpace(text), 10, 64) + return value, err == nil +} + +func (c *amd64Ctx) isGoLegacyXReg(r Reg) bool { + index, ok := amd64ParseXReg(r) + if !ok { + return false + } + if c.goarch == "386" { + return index < 8 + } + return index < 16 +} + +func dstArgOperand(r Reg) Operand { + return Operand{Kind: OpReg, Reg: r} +} + +func amd64VEXVectorRegister(op Operand, byteWidth int) bool { + if op.Kind != OpReg || amd64VectorByteWidth(op.Reg) != byteWidth { + return false + } + index, ok := amd64VectorRegisterIndex(op.Reg, byteWidth) + return ok && index < 16 +} + +func amd64EVEXVectorRegister(op Operand, byteWidth int) bool { + if op.Kind != OpReg || amd64VectorByteWidth(op.Reg) != byteWidth { + return false + } + index, ok := amd64VectorRegisterIndex(op.Reg, byteWidth) + return ok && index < 32 +} + +func amd64VectorRegisterIndex(r Reg, byteWidth int) (int, bool) { + switch byteWidth { + case 16: + return amd64ParseXReg(r) + case 32: + return amd64ParseYReg(r) + case 64: + return amd64ParseZReg(r) + default: + return 0, false + } +} + +func amd64FloatingComparePredicate(immediate uint8) string { + return [...]string{ + "oeq", "olt", "ole", "uno", "une", "uge", "ugt", "ord", + "ueq", "ult", "ule", "false", "one", "oge", "ogt", "true", + "oeq", "olt", "ole", "uno", "une", "uge", "ugt", "ord", + "ueq", "ult", "ule", "false", "one", "oge", "ogt", "true", + }[immediate&31] +} + +func (c *amd64Ctx) loadFloatingVectorOperand(op Operand, byteWidth, lanes, elemBits int) (string, error) { + bytesValue, err := c.loadPackedCompareBytes(op, byteWidth) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", value, byteWidth, bytesValue, amd64FloatingVectorType(lanes, elemBits)) + return "%" + value, nil +} + +func (c *amd64Ctx) loadFloatingScalarOperand(op Operand, elemBits int) (string, error) { + if elemBits == 64 { + return c.evalF64(op) + } + return c.evalF32(op) +} + +func (c *amd64Ctx) splatFloatingScalar(value string, lanes, elemBits int) string { + vectorType := amd64FloatingVectorType(lanes, elemBits) + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %s, i32 %d\n", inserted, vectorType, result, amd64FloatingScalarType(elemBits), value, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) packFloatingCompareMask(compared string, lanes int) string { + if lanes == 1 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to i64\n", wide, compared) + return "%" + wide + } + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %s to %s\n", packed, lanes, compared, packedType) + if lanes == 64 { + return "%" + packed + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %%%s to i64\n", wide, packedType, packed) + return "%" + wide +} + +func amd64FloatingScalarType(elemBits int) string { + if elemBits == 64 { + return "double" + } + return "float" +} + +func amd64FloatingVectorType(lanes, elemBits int) string { + return fmt.Sprintf("<%d x %s>", lanes, amd64FloatingScalarType(elemBits)) +} diff --git a/amd64_lower_compare_float_historical_test.go b/amd64_lower_compare_float_historical_test.go new file mode 100644 index 000000000..644717991 --- /dev/null +++ b/amd64_lower_compare_float_historical_test.go @@ -0,0 +1,39 @@ +package plan9asm + +import "testing" + +func TestTranslateAMD64LegacyFloatingCompareHistoricalBarePredicate(t *testing.T) { + source := "TEXT historicalcompare(SB),NOSPLIT,$0-0\n" + + "\tCMPPD X1, X0, 1\n" + + "\tCMPPS 8(AX), X0, 2\n" + + "\tCMPSD X1, X0, 5\n" + + "\tCMPSS 8(AX), X0, -1\n" + + "\tRET\n" + ll := x86UnboundVoidScalarFormatObject(t, source, "historicalcompare", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "historical-floating-compare.ll", "historical-floating-compare.o", ll) +} + +func TestTranslateAMD64LegacyFloatingCompareRejectsInvalidBarePredicate(t *testing.T) { + for _, instruction := range []string{ + "CMPPD X1, X0, predicate", + "CMPPS X1, X0, 128", + "CMPSD X1, X0, -129", + "CMPSS X1, X0, 1+1", + } { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + continue + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted invalid historical predicate %q", instruction) + } + } +} diff --git a/amd64_lower_compare_greater_test.go b/amd64_lower_compare_greater_test.go new file mode 100644 index 000000000..647b04cc3 --- /dev/null +++ b/amd64_lower_compare_greater_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedSignedGreaterThanCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, vectorLeft, vectorRight, zLast := 15, 20, 21, 22 + if target.goarch == "386" { + legacyLast, vectorLeft, vectorRight, zLast = 7, 6, 7, 7 + } + var source strings.Builder + source.WriteString("TEXT packedsignedgreaterthanforms(SB),4,$0-0\n") + for _, op := range []string{"PCMPGTB", "PCMPGTW", "PCMPGTL"} { + // Py1's MMX encoder row is amd64-only. A frontend-accepted + // high vector spelling is likewise not a physical 386 bank. + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M1\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M2\n", op) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + } + fmt.Fprintf(&source, "\tPCMPGTQ X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tPCMPGTQ 8(AX), X%d\n", legacyLast) + for _, op := range []string{"VPCMPGTB", "VPCMPGTW", "VPCMPGTD", "VPCMPGTQ"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s1, %s2\n", op, width, width) + fmt.Fprintf(&source, "\t%s %s%d, %s%d, K3\n", op, width, vectorLeft, width, vectorRight) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, K3\n", op, width, width) + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, K3\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, K0\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K2, K3\n", op) + } else { + fmt.Fprintf(&source, "\t%s Z0, Z6, K%d\n", op, zLast) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s.BCST 8(AX), %s%d, K3\n", op, width, vectorRight) + } + } + } + source.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, source.String(), "packedsignedgreaterthanforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-signed-greater-than-"+target.name+".ll", "packed-signed-greater-than-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedSignedGreaterThanRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PCMPGTB.Z X0, X1", + "PCMPGTW X0", + "PCMPGTL M0, X1", + "PCMPGTQ M0, M1", + "PCMPGTB Y0, Y1", + "PCMPGTW X0, (AX)", + "VPCMPGTB X0, X1", + "VPCMPGTW X0, Y1, Y2", + "VPCMPGTD Z0, Z1, Z2", + "VPCMPGTQ X0, X1, K0, K2", + "VPCMPGTB.Z X0, X1, K2", + "VPCMPGTW.BCST 8(AX), X1, K2", + "VPCMPGTD.BCST X0, X1, K2", + "VPCMPGTQ.BCST 8(AX), X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedSignedGreaterThanRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PCMPGTW X8, X0", + "PCMPGTL X0, X8", + "VPCMPGTB X16, X1, X2", + "VPCMPGTW X0, X1, X16", + "VPCMPGTD Z8, Z1, K2", + "VPCMPGTQ Z0, Z1, K1, K2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedSignedGreaterThanRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } + for _, op := range []string{"PCMPGTB", "PCMPGTW", "PCMPGTL"} { + for _, operands := range []string{"M0, M1", "8(AX), M2"} { + instruction := op + " " + operands + goAssemblerOracleOutput(t, "386", "TEXT bad(SB),4,$0-0\n"+instruction+"\nRET\n", "", false) + assertX86PackedSignedGreaterThanRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } + } +} + +func assertX86PackedSignedGreaterThanRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed signed greater-than forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_compare_legacy_equal_test.go b/amd64_lower_compare_legacy_equal_test.go new file mode 100644 index 000000000..f9238a184 --- /dev/null +++ b/amd64_lower_compare_legacy_equal_test.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86LegacyPackedEqualityActualGoEncoderRows(t *testing.T) { + body, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + actual := strings.Join(strings.Fields(string(body)), "") + for _, definition := range []string{ + "var ymm = []ytab{ {Zm_r_xm, 1, argList{Ymm, Ymr}}, {Zm_r_xm, 2, argList{Yxm, Yxr}}, }", + "var yxm_q4 = []ytab{ {Zm_r, 1, argList{Yxm, Yxr}}, }", + "{APCMPEQB, ymm, Py1, opBytes{0x74, Pe, 0x74}}", + "{APCMPEQW, ymm, Py1, opBytes{0x75, Pe, 0x75}}", + "{APCMPEQL, ymm, Py1, opBytes{0x76, Pe, 0x76}}", + "{APCMPEQQ, yxm_q4, Pq4, opBytes{0x29}}", + } { + if !strings.Contains(actual, strings.Join(strings.Fields(definition), "")) { + t.Fatalf("selected Go SDK packed-equality encoding row changed: %s", definition) + } + } +} + +func TestTranslateX86LegacyPackedEqualityCompleteGoAssemblerForms(t *testing.T) { + // PCMPEQ{B,W,L} use Go's ymm table (MMX/memory -> MMX and X/memory + // -> X). Py1 admits the MMX row only in amd64; 386 must use X0-X7. + // PCMPEQQ uses yxm_q4, which has only the X/memory -> X row. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastX := 15 + if target.goarch == "386" { + lastX = 7 + } + var source strings.Builder + source.WriteString("TEXT legacypackedequalityforms(SB),4,$0-0\n") + for _, op := range []string{"PCMPEQB", "PCMPEQW", "PCMPEQL"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M1\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M2\n", op) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, lastX) + fmt.Fprintf(&source, "\t%s 16(AX), X%d\n", op, lastX) + } + fmt.Fprintf(&source, "\tPCMPEQQ X0, X%d\n", lastX) + fmt.Fprintf(&source, "\tPCMPEQQ 24(AX), X%d\n", lastX) + source.WriteString("\tRET\n") + ll := x86UnboundVoidScalarFormatObject(t, source.String(), "legacypackedequalityforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "legacy-packed-equality-"+target.name+".ll", "legacy-packed-equality-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86LegacyPackedEqualityRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PCMPEQQ M0, M1", + "PCMPEQQ X0, M1", + "PCMPEQW M0, X1", + "PCMPEQL X0, M1", + "PCMPEQB Y0, X1", + "PCMPEQQ X0, Y1", + "PCMPEQQ.Z X0, X1", + "PCMPEQQ X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86LegacyPackedEqualityRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86LegacyPackedEqualityRejected(t, "386", "i386-unknown-linux-gnu", "PCMPEQQ X8, X0") + assertX86LegacyPackedEqualityRejected(t, "386", "i386-unknown-linux-gnu", "PCMPEQQ X0, X8") + for _, op := range []string{"PCMPEQB", "PCMPEQW", "PCMPEQL"} { + for _, operands := range []string{"M0, M1", "8(AX), M2"} { + assertX86LegacyPackedEqualityRejected(t, "386", "i386-unknown-linux-gnu", op+" "+operands) + } + } +} + +func assertX86LegacyPackedEqualityRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + source := "TEXT bad(SB),4,$0-0\n\t" + instruction + "\n\tRET\n" + goAssemblerOracleOutput(t, goarch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil || ir != "" { + t.Fatalf("Translate accepted %q outside Go 1.27's legacy packed equality forms", instruction) + } +} diff --git a/amd64_lower_compare_mask.go b/amd64_lower_compare_mask.go new file mode 100644 index 000000000..9f218496a --- /dev/null +++ b/amd64_lower_compare_mask.go @@ -0,0 +1,100 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ImmediatePackedCompareSpec struct { + laneBits int + unsigned bool +} + +var amd64ImmediatePackedCompareSpecs = map[string]amd64ImmediatePackedCompareSpec{ + "VPCMPB": {laneBits: 8}, + "VPCMPW": {laneBits: 16}, + "VPCMPD": {laneBits: 32}, + "VPCMPQ": {laneBits: 64}, + "VPCMPUB": {laneBits: 8, unsigned: true}, + "VPCMPUW": {laneBits: 16, unsigned: true}, + "VPCMPUD": {laneBits: 32, unsigned: true}, + "VPCMPUQ": {laneBits: 64, unsigned: true}, +} + +// lowerImmediatePackedCompare implements the complete Go 1.27 _yvpcmpb +// operand table and all eight signed/unsigned element-width mnemonics. +func (c *amd64Ctx) lowerImmediatePackedCompare(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86ImmediatePackedCompareProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + writeMask := "" + if form.mask >= 0 { + writeMask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + // Use the existing E4 lane reader: active native reads remain volatile + // even for constant predicates/dead K outputs, and actual inactive mask + // lanes alone suppress their accesses. FP spans stay independently checked. + if form.source.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(form.source.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + firstValue, err := c.loadX86PackedShiftLanes(form.source, form.byteWidth, form.spec.laneBits, form.broadcast, writeMask) + if err != nil { + return true, false, err + } + secondValue, err := c.loadPackedCompareLanes(ins.Args[form.second], form.byteWidth, form.spec.laneBits, false) + if err != nil { + return true, false, err + } + lanes := form.byteWidth * 8 / form.spec.laneBits + compared := c.emitImmediatePackedComparePredicate(lanes, form.spec.laneBits, secondValue, firstValue, form.predicate, form.unsigned) + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %s to %s\n", packed, lanes, compared, packedType) + result := "%" + packed + if lanes < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + if writeMask != "" { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, result, writeMask) + result = "%" + masked + } + return true, false, c.storeK(ins.Args[form.destination].Reg, result) +} + +func (c *amd64Ctx) emitImmediatePackedComparePredicate(lanes, laneBits int, left, right string, immediate uint8, unsigned bool) string { + switch immediate & 7 { + case 3: + return "zeroinitializer" + case 7: + return llvmAllTrueI1VectorValue(lanes) + } + predicates := [8]string{"eq", "slt", "sle", "", "ne", "sge", "sgt", ""} + if unsigned { + predicates = [8]string{"eq", "ult", "ule", "", "ne", "uge", "ugt", ""} + } + compared := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s <%d x i%d> %s, %s\n", compared, predicates[immediate&7], lanes, laneBits, left, right) + return "%" + compared +} + +func llvmAllTrueI1VectorValue(lanes int) string { + var value strings.Builder + value.WriteByte('<') + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + value.WriteString(", ") + } + value.WriteString("i1 true") + } + value.WriteByte('>') + return value.String() +} diff --git a/amd64_lower_compare_mask_test.go b/amd64_lower_compare_mask_test.go new file mode 100644 index 000000000..cc6bf783b --- /dev/null +++ b/amd64_lower_compare_mask_test.go @@ -0,0 +1,93 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86ImmediatePackedCompareCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 gives VPCMP{B,W,D,Q,UB,UW,UD,UQ} the same _yvpcmpb + // table: unsigned-byte immediate, X/Y/Z r/m first source, matching vector + // second source, optional K1-K7 write mask, and K destination. D/Q forms + // additionally enable scalar-memory broadcast. + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT immediatepackedcompareforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{ + "VPCMPB", "VPCMPW", "VPCMPD", "VPCMPQ", + "VPCMPUB", "VPCMPUW", "VPCMPUD", "VPCMPUQ", + } { + fmt.Fprintf(&source, "\t%s $0, X1, X20, K0\n", op) + fmt.Fprintf(&source, "\t%s $1, 8(AX), X20, K1, K2\n", op) + fmt.Fprintf(&source, "\t%s $2, Y1, Y20, K3\n", op) + fmt.Fprintf(&source, "\t%s $3, 40(AX), Y20, K4, K5\n", op) + fmt.Fprintf(&source, "\t%s $7, Z1, Z20, K6\n", op) + fmt.Fprintf(&source, "\t%s $255, 104(AX), Z20, K7, K0\n", op) + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + fmt.Fprintf(&source, "\t%s.BCST $4, 168(AX), X20, K1\n", op) + fmt.Fprintf(&source, "\t%s.BCST $5, 176(AX), Y20, K2, K3\n", op) + fmt.Fprintf(&source, "\t%s.BCST $6, 184(AX), Z20, K4\n", op) + } + } + source.WriteString("\tRET\n") + + // Retain every original row as a Go object and public Context + // control. Emitter-only objects grant no unknown input; complete + // initialized public objects have their separate native-read owner. + ll := x86UnboundVoidScalarFormatObject(t, source.String(), + "immediatepackedcompareforms", "amd64", target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "immediate-packed-compare-"+target.name+".ll", "immediate-packed-compare-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ImmediatePackedCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPCMPUD X0, X1, K1", + "VPCMPUD $256, X0, X1, K1", + "VPCMPUD $1, X0, Y1, K1", + "VPCMPUD $1, X0, (AX), K1", + "VPCMPUD $1, X0, X1, K0, K1", + "VPCMPUD $1, X0, X1, AX", + "VPCMPUD.Z $1, X0, X1, K1", + "VPCMPUB.BCST $1, (AX), X1, K1", + "VPCMPUW.BCST $1, (AX), X1, K1", + "VPCMPUD.BCST $1, X0, X1, K1", + "VPCMPUQ $1, X32, X1, K1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ImmediatePackedCompareRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + // cmd/asm's 386 frontend rejects this table's four- and five-operand + // spellings before its shared x86 optab is reached. + assertX86ImmediatePackedCompareRejected(t, "386", "i386-unknown-linux-gnu", "VPCMPUD $1, Y2, Y0, K2") +} + +func assertX86ImmediatePackedCompareRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's immediate packed compare forms", instruction) + } +} diff --git a/amd64_lower_compare_vec.go b/amd64_lower_compare_vec.go new file mode 100644 index 000000000..833258656 --- /dev/null +++ b/amd64_lower_compare_vec.go @@ -0,0 +1,354 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedIntegerCompareSpec struct { + laneBits int + greater bool + mapNumber, opcode int +} + +// Every Go _yvpcmpeqb entry: X/Y VEX vector results, X/Y/Z EVEX K +// results, optional K1-7 mask, and memory-only D/Q broadcast. Raw decode +// consumes the same lane and encoding axes as the named lowerer. +var amd64PackedIntegerCompareSpecs = map[Op]amd64PackedIntegerCompareSpec{ + "VPCMPEQB": {8, false, 1, 0x74}, + "VPCMPEQW": {16, false, 1, 0x75}, + "VPCMPEQD": {32, false, 1, 0x76}, + "VPCMPEQQ": {64, false, 2, 0x29}, + "VPCMPGTB": {8, true, 1, 0x64}, + "VPCMPGTW": {16, true, 1, 0x65}, + "VPCMPGTD": {32, true, 1, 0x66}, + "VPCMPGTQ": {64, true, 2, 0x37}, +} + +// lowerPackedIntegerCompare implements the legacy PCMPGT{B,W,L,Q} family and +// the complete Go 1.27 _yvpcmpeqb operand family shared by VPCMPEQ{B,W,D,Q} +// and VPCMPGT{B,W,D,Q}. +// VEX forms write X/Y vectors. EVEX forms compare X/Y/Z lanes into a K +// register, optionally gated by a non-K0 mask; D/Q memory operands can be +// broadcast for the EVEX forms. +func (c *amd64Ctx) lowerPackedIntegerCompare(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86PackedIntegerCompareProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + ins.Op = op + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if form.legacy { + local := ins + local.Args = append([]Operand(nil), ins.Args...) + local.Args[form.first] = form.source + if form.memory >= 0 && !form.mmx { + virtual := form.source.Kind == OpFP + if form.source.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed integer compare m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + return c.lowerLegacyPackedIntegerCompare(base, form.spec.laneBits, form.spec.mmx, local) + } + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.maskResult { + return true, false, c.emitPackedIntegerMaskCompare(base, form.spec.laneBits, form.byteWidth, + form.source, ins.Args[form.second], ins.Args[form.destination].Reg, mask, form.broadcast) + } + return true, false, c.emitPackedIntegerVectorCompare(base, form.spec.laneBits, form.byteWidth, + form.source, ins.Args[form.second], ins.Args[form.destination].Reg) +} + +func (c *amd64Ctx) lowerLegacyPackedIntegerCompare(baseOp string, laneBits int, allowMMX bool, ins Instr) (bool, bool, error) { + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("%s %s expects packed source, packed destination: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[1].Reg + if _, mmx := amd64ParseMReg(destination); mmx { + if !allowMMX { + return true, false, fmt.Errorf("%s %s has no MMX form in Go 1.27's yxm_q4 table: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpReg { + if _, ok := amd64ParseMReg(ins.Args[0].Reg); !ok { + return true, false, fmt.Errorf("%s %s MMX form requires an MMX source: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s MMX form requires MMX or memory source: %q", c.goarch, baseOp, ins.Raw) + } + firstBits, err := c.loadLegacyMMXPackedSource(ins.Args[0]) + if err != nil { + return true, false, err + } + secondBits, err := c.loadReg(destination) + if err != nil { + return true, false, err + } + lanes := 64 / laneBits + first := c.bitcastI64ToIntegerLanes(lanes, laneBits, firstBits) + second := c.bitcastI64ToIntegerLanes(lanes, laneBits, secondBits) + result := c.emitLegacyPackedIntegerCompare(baseOp, lanes, laneBits, first, second) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", bits, lanes, laneBits, result) + return true, false, c.storeReg(destination, "%"+bits) + } + + if !c.isGoLegacyXReg(destination) { + return true, false, fmt.Errorf("%s %s destination must be an in-range MMX or X register: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpReg && !c.isGoLegacyXReg(ins.Args[0].Reg) { + return true, false, fmt.Errorf("%s %s XMM form requires an in-range X source: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpReg && !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s XMM form requires X or memory source: %q", c.goarch, baseOp, ins.Raw) + } + secondBytes, err := c.loadX(destination) + if err != nil { + return true, false, err + } + lanes := 128 / laneBits + second := c.bitcastVectorBytesToIntegerLanes(16, lanes, laneBits, secondBytes) + var result string + if isAMD64MemoryOperand(ins.Args[0]) { + mnemonic := strings.ToLower(baseOp) + if laneBits == 32 { + mnemonic = strings.TrimSuffix(mnemonic, "l") + "d" + } + result, err = c.emitPackedBinaryNativeMemory(ins.Args[0], laneBits, second, mnemonic) + } else { + firstBytes, loadErr := c.loadXVecOperand(ins.Args[0]) + if loadErr != nil { + return true, false, loadErr + } + first := c.bitcastVectorBytesToIntegerLanes(16, lanes, laneBits, firstBytes) + result = c.emitLegacyPackedIntegerCompare(baseOp, lanes, laneBits, first, second) + } + if err != nil { + return true, false, err + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <16 x i8>\n", out, lanes, laneBits, result) + return true, false, c.storeLegacyPackedLogical(ins.Args[1], "%"+out) +} + +func (c *amd64Ctx) emitLegacyPackedIntegerCompare(baseOp string, lanes, laneBits int, first, second string) string { + compared := c.newTmp() + result := c.newTmp() + predicate := "eq" + if strings.HasPrefix(baseOp, "PCMPGT") { + predicate = "sgt" + } + fmt.Fprintf(c.b, " %%%s = icmp %s <%d x i%d> %s, %s\n", compared, predicate, lanes, laneBits, second, first) + fmt.Fprintf(c.b, " %%%s = sext <%d x i1> %%%s to <%d x i%d>\n", result, lanes, compared, lanes, laneBits) + return "%" + result +} + +func (c *amd64Ctx) isGoVEXVectorRegister(arg Operand, byteWidth int, allowMemory bool) bool { + if allowMemory && isAMD64MemoryOperand(arg) { + return true + } + if !amd64VEXVectorRegister(arg, byteWidth) { + return false + } + if c.goarch == "386" { + index, _ := amd64VectorRegisterIndex(arg.Reg, byteWidth) + return index < 8 + } + return true +} + +func amd64PackedCompareWidth(op Operand) (int, bool) { + if op.Kind != OpReg { + return 0, false + } + switch { + case isAMD64XReg(op.Reg): + return 16, true + case isAMD64YReg(op.Reg): + return 32, true + case isAMD64ZReg(op.Reg): + return 64, true + default: + return 0, false + } +} + +func (c *amd64Ctx) loadPackedCompareBytes(op Operand, byteWidth int) (string, error) { + if op.Kind == OpReg { + width, ok := amd64PackedCompareWidth(op) + if !ok || width != byteWidth { + return "", fmt.Errorf("expected %d-byte vector register, got %s", byteWidth, op.String()) + } + switch byteWidth { + case 16: + return c.loadX(op.Reg) + case 32: + return c.loadY(op.Reg) + case 64: + return c.loadZ(op.Reg) + } + } + if op.Kind == OpFP { + return c.loadFPVectorBytes(op.FPOffset, byteWidth) + } + if op.Kind == OpSym { + // As in Go's x86 assembler, an unprefixed integer denotes an + // absolute memory address rather than an immediate value. + if address, parseErr := parseInt(strings.TrimSpace(op.Sym)); parseErr == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load <%d x i8>, ptr %s, align 1\n", value, byteWidth, c.ptrFromAddrI64(fmt.Sprintf("%d", address))) + return "%" + value, nil + } + } + if !isAMD64MemoryOperand(op) { + return "", fmt.Errorf("expected vector register or memory, got %s", op.String()) + } + switch byteWidth { + case 16: + return c.loadXVecOperand(op) + case 32: + return c.loadYVecOperand(op) + case 64: + return c.loadZVecOperand(op) + default: + return "", fmt.Errorf("unsupported packed compare width %d", byteWidth) + } +} + +func (c *amd64Ctx) loadPackedCompareLanes(op Operand, byteWidth, laneBits int, broadcast bool) (string, error) { + lanes := byteWidth * 8 / laneBits + if broadcast { + value, err := c.evalIntSized(op, amd64IntegerTypeForBits(laneBits)) + if err != nil { + return "", err + } + return amd64SplatInteger(c, lanes, laneBits, value), nil + } + bytesValue, err := c.loadPackedCompareBytes(op, byteWidth) + if err != nil { + return "", err + } + lanesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i%d>\n", lanesValue, byteWidth, bytesValue, lanes, laneBits) + return "%" + lanesValue, nil +} + +func (c *amd64Ctx) comparePackedIntegerLanes(baseOp string, laneBits, byteWidth int, first, second Operand, broadcast bool, mask string) (string, error) { + // E4/E4.nb permits only inactive native lanes to suppress their accesses. + // Keep active reads observable even when the compare result is dead. + // Independent virtual FP fields still require the full checked byte span. + if first.Kind == OpFP { + width := byteWidth + if broadcast { + width = laneBits / 8 + } + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, width); err != nil { + return "", err + } + } + firstValue, err := c.loadX86PackedShiftLanes(first, byteWidth, laneBits, broadcast, mask) + if err != nil { + return "", err + } + secondValue, err := c.loadPackedCompareLanes(second, byteWidth, laneBits, false) + if err != nil { + return "", err + } + predicate := "eq" + if amd64PackedIntegerCompareSpecs[Op(baseOp)].greater { + predicate = "sgt" + } + lanes := byteWidth * 8 / laneBits + compared := c.newTmp() + // Go/Plan 9 lists the Intel r/m operand first: dst = second cmp first. + fmt.Fprintf(c.b, " %%%s = icmp %s <%d x i%d> %s, %s\n", compared, predicate, lanes, laneBits, secondValue, firstValue) + return "%" + compared, nil +} + +func (c *amd64Ctx) emitPackedIntegerVectorCompare(baseOp string, laneBits, byteWidth int, first, second Operand, dst Reg) error { + compared, err := c.comparePackedIntegerLanes(baseOp, laneBits, byteWidth, first, second, false, "") + if err != nil { + return err + } + lanes := byteWidth * 8 / laneBits + allBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext <%d x i1> %s to <%d x i%d>\n", allBits, lanes, compared, lanes, laneBits) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <%d x i8>\n", bytesValue, lanes, laneBits, allBits, byteWidth) + return c.storePackedMoveOperand(Operand{Kind: OpReg, Reg: dst}, byteWidth, "%"+bytesValue) +} + +func (c *amd64Ctx) emitPackedIntegerMaskCompare(baseOp string, laneBits, byteWidth int, first, second Operand, dst Reg, writeMask string, broadcast bool) error { + compared, err := c.comparePackedIntegerLanes(baseOp, laneBits, byteWidth, first, second, broadcast, writeMask) + if err != nil { + return err + } + lanes := byteWidth * 8 / laneBits + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %s to %s\n", packed, lanes, compared, packedType) + result := "%" + packed + if lanes < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + if writeMask != "" { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, result, writeMask) + result = "%" + masked + } + return c.storeK(dst, result) +} + +func (c *amd64Ctx) loadMaskedPackedCompareLanes(source Operand, byteWidth, laneBits int, broadcast bool, mask string) (string, error) { + // FP describes independent checked virtual slots, not one contiguous + // native allocation. Keep per-slot loads; never form a vector pointer + // from only the first slot, even when its address happens to be available. + if source.Kind == OpFP && !broadcast { + for offset := source.FPOffset; offset < source.FPOffset+int64(byteWidth); offset += 8 { + _, parameter := c.fpParam(offset) + _, _, result := c.fpResultAlloca(offset) + if !parameter && !result { + return "", fmt.Errorf("%s packed compare uses an undeclared FP vector slot at +%d(FP)", c.goarch, offset) + } + } + } + if mask == "" || source.Kind == OpReg || source.Kind == OpFP { + return c.loadPackedCompareLanes(source, byteWidth, laneBits, broadcast) + } + lanes := byteWidth * 8 / laneBits + if broadcast { + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)<", lanes, laneBits) + fmt.Fprintf(c.b, " %%%s = call %s @llvm.masked.load.v%di%d%s(%s align 1 %s, <%d x i1> %s, %s zeroinitializer)\n", value, vectorType, lanes, laneBits, suffix, pointerType, pointer, lanes, predicate, vectorType) + return "%" + value, nil +} diff --git a/amd64_lower_compress.go b/amd64_lower_compress.go new file mode 100644 index 000000000..2bc4f343b --- /dev/null +++ b/amd64_lower_compress.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64PackedCompressLaneBits = map[Op]int{ + "VCOMPRESSPS": 32, + "VCOMPRESSPD": 64, + "VPCOMPRESSB": 8, + "VPCOMPRESSW": 16, + "VPCOMPRESSD": 32, + "VPCOMPRESSQ": 64, +} + +// lowerPackedCompress implements the complete Go 1.27 _yvcompresspd table. +// Masked register forms compact selected source lanes and merge/zero only the +// tail. Masked memory forms write selected lanes contiguously and leave every +// byte after the compacted tail untouched. Go's accepted EVEX.z memory spelling +// instead raises #UD before any mask, source or address access. +func (c *amd64Ctx) lowerPackedCompress(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86PackedCompactProperties(op, ins, c.goarch) + if !known || form.expand { + return false, false, nil + } + if err != nil { + return true, false, err + } + if form.undefinedOpcode() { + return c.lowerSyscall("UD2", Instr{Op: "UD2"}) + } + if form.memory >= 0 && c.nativeMemoryOperand(ins, form.memory) { + return true, false, c.lowerNativeCompactMemory(ins, form) + } + laneBits, byteWidth := form.laneBits, form.byteWidth + source, destination := ins.Args[0], ins.Args[form.destination] + + sourceBytes, err := c.loadPackedCompareBytes(source, byteWidth) + if err != nil { + return true, false, err + } + if form.mask < 0 { + return true, false, c.storePackedMoveOperand(destination, byteWidth, sourceBytes) + } + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / laneBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + sourceValue := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, sourceBytes) + maskVector := amd64IntegerMaskVector(c, lanes, mask) + if destination.Kind != OpReg && destination.Kind != OpFP { + pointer, pointerType, intrinsicSuffix, err := c.packedVectorMemoryPointer(destination) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void @llvm.masked.compressstore.v%di%d%s(%s %s, %s %s, <%d x i1> %s)\n", + lanes, laneBits, intrinsicSuffix, vectorType, sourceValue, pointerType, pointer, lanes, maskVector) + return true, false, nil + } + passthrough := "zeroinitializer" + if !form.zeroing { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + passthrough = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, oldBytes) + } + compressed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.experimental.vector.compress.v%di%d(%s %s, <%d x i1> %s, %s %s)\n", + compressed, vectorType, lanes, laneBits, vectorType, sourceValue, lanes, maskVector, vectorType, passthrough) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", out, vectorType, compressed, byteWidth) + // FP results are separate scalar fields, not one contiguous native alloca. + // The old complete span supplies the untouched compacted tail above. + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func amd64IntegerMaskVector(c *amd64Ctx, lanes int, mask string) string { + packed := mask + if lanes < 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", narrow, mask, lanes) + packed = "%" + narrow + } + vector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %s to <%d x i1>\n", vector, lanes, packed, lanes) + return "%" + vector +} + +func (c *amd64Ctx) packedVectorMemoryPointer(operand Operand) (ptr, ptrType, intrinsicSuffix string, err error) { + switch operand.Kind { + case OpMem: + ptr, ptrType, err = c.ptrFromMem(operand.Mem) + if err != nil { + return "", "", "", err + } + switch ptrType { + case "ptr": + return ptr, ptrType, "", nil + case "ptr addrspace(256)": + return ptr, ptrType, ".p256", nil + case "ptr addrspace(257)": + return ptr, ptrType, ".p257", nil + default: + return "", "", "", fmt.Errorf("unsupported packed-vector pointer type %s", ptrType) + } + case OpSym: + if address, parseErr := parseInt(strings.TrimSpace(operand.Sym)); parseErr == nil { + return c.ptrFromAddrI64(fmt.Sprintf("%d", address)), "ptr", "", nil + } + ptr, err = c.ptrFromSB(operand.Sym) + return ptr, "ptr", "", err + case OpFP: + if c.classicFrame != "" { + return c.classicFramePtr(operand.FPOffset), "ptr", "", nil + } + if ptr = c.fpParamAlloca[operand.FPOffset]; ptr != "" { + return ptr, "ptr", "", nil + } + if ptr, _, ok := c.fpResultAlloca(operand.FPOffset); ok { + return ptr, "ptr", "", nil + } + return "", "", "", fmt.Errorf("%s %s FP operand has no addressable slot", c.goarch, operand.String()) + default: + return "", "", "", fmt.Errorf("%s expected a packed-vector memory operand, got %s", c.goarch, operand.String()) + } +} diff --git a/amd64_lower_compress_test.go b/amd64_lower_compress_test.go new file mode 100644 index 000000000..88fac4826 --- /dev/null +++ b/amd64_lower_compress_test.go @@ -0,0 +1,227 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedCompressCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedcompressforms(SB),$0-0\n") + for _, op := range []string{"VCOMPRESSPS", "VCOMPRESSPD", "VPCOMPRESSB", "VPCOMPRESSW", "VPCOMPRESSD", "VPCOMPRESSQ"} { + for _, width := range []string{"X", "Y", "Z"} { + sourceReg, destinationReg := 20, 22 + if target.goarch == "386" && width == "Z" { + sourceReg, destinationReg = 6, 7 + } + fmt.Fprintf(&source, "\t%s %s%d, %s%d\n", op, width, sourceReg, width, destinationReg) + fmt.Fprintf(&source, "\t%s %s%d, 8(AX)\n", op, width, sourceReg) + fmt.Fprintf(&source, "\t%s %s%d, K1, %s%d\n", op, width, sourceReg, width, destinationReg) + fmt.Fprintf(&source, "\t%s.Z %s%d, K7, %s%d\n", op, width, sourceReg, width, destinationReg) + fmt.Fprintf(&source, "\t%s %s%d, K2, 8(AX)\n", op, width, sourceReg) + fmt.Fprintf(&source, "\t%s.Z %s%d, K3, 8(AX)\n", op, width, sourceReg) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + options := Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedcompressforms": {Name: "packedcompressforms", Ret: Void}, + }, + } + for _, api := range []string{"text", "module", "owned-module"} { + if ir, err := x86X87UnboundOriginalAPI(t, file, options, api); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("original format-only source acquired undefined vector/mask/address inputs", api, err) + } + } + ll := x86ZeroArgumentLowererObjectUnit(t, file, target.goarch, target.triple, "PackedCompressEmitterObjectUnit", + func(consumer *amd64Ctx, instruction Instr) (bool, bool, error) { + return consumer.lowerVec(instruction.Op, instruction) + }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-compress-"+target.name+".ll", "packed-compress-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedCompressRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VCOMPRESSPS 8(AX), X1", + "VCOMPRESSPD X0, Y1", + "VCOMPRESSPS X0, K0, X1", + "VCOMPRESSPD.Z Z0, Z1", + "VPCOMPRESSB 8(AX), X1", + "VPCOMPRESSW X0, Y1", + "VPCOMPRESSD X0, K0, X1", + "VPCOMPRESSQ.Z Z0, Z1", + "VPCOMPRESSB.BCST X0, X1", + "VPCOMPRESSW X0, AX", + "VPCOMPRESSD AX, X1", + "VPCOMPRESSQ Z0, K1, K2, Z1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + if currentGoMinorAtLeast(27) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } + assertX86PackedCompressRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VPCOMPRESSB Z8, Z0", + "VPCOMPRESSW Z0, Z8", + "VPCOMPRESSD Z8, K1, 8(AX)", + "VPCOMPRESSQ Z0, K1, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + if currentGoMinorAtLeast(27) { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } + assertX86PackedCompressRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedCompressRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvcompresspd forms for %s", instruction, goarch) + } +} + +func TestAMD64PackedCompressRuntimeSemantics(t *testing.T) { + // This oracle executes portable LLVM semantics on the actual host. It + // does not execute AVX512 opcodes or certify original-Go native execution. + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT packedcompress(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 (BX), Z0 + VMOVDQU64 (CX), Z1 + VPCOMPRESSB Z0, K1, Z1 + VMOVDQU64 Z1, 0(AX) + VMOVDQU64 (CX), Z1 + VPCOMPRESSW.Z Z0, K1, Z1 + VMOVDQU64 Z1, 64(AX) + VMOVDQU64 (CX), Z1 + VPCOMPRESSD Z0, K1, Z1 + VMOVDQU64 Z1, 128(AX) + VMOVDQU64 (CX), Z1 + VPCOMPRESSQ.Z Z0, K1, Z1 + VMOVDQU64 Z1, 192(AX) + VPCOMPRESSB Z0, K1, 256(AX) + VPCOMPRESSW Z0, K1, 320(AX) + VPCOMPRESSD Z0, K1, 384(AX) + VPCOMPRESSQ Z0, K1, 448(AX) + VMOVDQU64 (CX), Z1 + VCOMPRESSPS Z0, K1, Z1 + VMOVDQU64 Z1, 512(AX) + VMOVDQU64 (CX), Z1 + VCOMPRESSPD.Z Z0, K1, Z1 + VMOVDQU64 Z1, 576(AX) + VCOMPRESSPS Z0, K1, 640(AX) + VCOMPRESSPD Z0, K1, 704(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedcompress": {Name: "packedcompress", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +extern void packedcompress(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static int check_group(const uint8_t *got, const uint8_t *source, const uint8_t *old, int lane_bytes, uint64_t mask, int zero_tail, int memory) { + int lanes = 64 / lane_bytes, packed = 0; + for (int lane = 0; lane < lanes; lane++) if ((mask >> lane) & 1) { + if (memcmp(got + packed*lane_bytes, source + lane*lane_bytes, lane_bytes)) return 10 + lane; + packed++; + } + for (int lane = packed; lane < lanes; lane++) { + const uint8_t *want = old + lane*lane_bytes; + uint8_t zeros[8] = {0}, untouched[8]; memset(untouched, 0xcc, sizeof(untouched)); + if (memory) want = untouched; else if (zero_tail) want = zeros; + if (memcmp(got + lane*lane_bytes, want, lane_bytes)) return 80 + lane; + } + return 0; +} +int main(void) { + uint8_t source[64], old[64], out[768]; + const uint64_t mask = 0xa55aa55aa55aa55aULL; + for (int i = 0; i < 64; i++) { source[i] = (uint8_t)(i * 29 + 7); old[i] = (uint8_t)(0xf0 - i); } + memset(out, 0xcc, sizeof(out)); + packedcompress(out, source, old, mask); + int r; + if ((r = check_group(out+0, source, old, 1, mask, 0, 0))) return 100+r; + if ((r = check_group(out+64, source, old, 2, mask, 1, 0))) return 200+r; + if ((r = check_group(out+128, source, old, 4, mask, 0, 0))) return 300+r; + if ((r = check_group(out+192, source, old, 8, mask, 1, 0))) return 400+r; + if ((r = check_group(out+256, source, old, 1, mask, 0, 1))) return 500+r; + if ((r = check_group(out+320, source, old, 2, mask, 0, 1))) return 600+r; + if ((r = check_group(out+384, source, old, 4, mask, 0, 1))) return 700+r; + if ((r = check_group(out+448, source, old, 8, mask, 0, 1))) return 800+r; + if ((r = check_group(out+512, source, old, 4, mask, 0, 0))) return 900+r; + if ((r = check_group(out+576, source, old, 8, mask, 1, 0))) return 1000+r; + if ((r = check_group(out+640, source, old, 4, mask, 0, 1))) return 1100+r; + if ((r = check_group(out+704, source, old, 8, mask, 0, 1))) return 1200+r; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_compress", triple, ll, mainC, nil) +} diff --git a/amd64_lower_conflict.go b/amd64_lower_conflict.go new file mode 100644 index 000000000..6bfd1e65a --- /dev/null +++ b/amd64_lower_conflict.go @@ -0,0 +1,85 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64PackedConflictSpec struct { + laneBits int +} + +// amd64PackedConflictSpecs is the complete Go 1.27 VPCONFLICTD/Q grammar. +// Both instructions use _yvexpandpd: EVEX X/Y/Z register-or-memory sources, +// optional K1-K7 merge/.Z masking, and scalar-memory broadcast. +var amd64PackedConflictSpecs = map[Op]amd64PackedConflictSpec{ + "VPCONFLICTD": {laneBits: 32}, + "VPCONFLICTQ": {laneBits: 64}, +} + +func (c *amd64Ctx) lowerPackedConflict(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86PackedConflictProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := c.validateConflictFunnelMemory(ins, form.x86PackedBinaryForm); err != nil { + return true, false, err + } + spec, byteWidth := form.spec, form.byteWidth + destination := ins.Args[form.destination] + mask := "" + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + lanes := byteWidth * 8 / spec.laneBits + // Intel's current E4NF class retains the entire source tuple (or the + // independent m32/m64 broadcast), including zero masks and dead results. + values, err := c.loadX86PackedShiftLanes(ins.Args[form.first], byteWidth, spec.laneBits, form.broadcast, "") + if err != nil { + return true, false, err + } + computed := c.emitPackedConflict(values, lanes, spec.laneBits) + result := computed + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + } + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computed, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedConflict(values string, lanes, laneBits int) string { + elements := make([]string, lanes) + for lane := 0; lane < lanes; lane++ { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", extracted, lanes, laneBits, values, lane) + elements[lane] = "%" + extracted + } + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + conflicts := "0" + for prior := 0; prior < lane; prior++ { + equal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %s, %s\n", equal, laneBits, elements[lane], elements[prior]) + bit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d 0\n", bit, equal, laneBits, uint64(1)< %s, i%d %s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, conflicts, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_conflict_test.go b/amd64_lower_conflict_test.go new file mode 100644 index 000000000..584439a8c --- /dev/null +++ b/amd64_lower_conflict_test.go @@ -0,0 +1,210 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedConflictGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64PackedConflictSpec{ + "VPCONFLICTD": {laneBits: 32}, + "VPCONFLICTQ": {laneBits: 64}, + } + if len(amd64PackedConflictSpecs) != len(expected) { + t.Fatalf("packed-conflict grammar has %d entries, want %d", len(amd64PackedConflictSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64PackedConflictSpecs[op]; !ok { + t.Errorf("packed-conflict grammar omitted %s", op) + } else if got != want { + t.Errorf("packed-conflict grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86PackedConflictCompleteGoFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedconflictforms(SB),$0-0\n") + for _, op := range []string{"VPCONFLICTD", "VPCONFLICTQ"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s0, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s 8(R9), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 16(R9), K7, %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST 24(R9), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(R9), K2, %s%d\n", op, width, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, signatures := x86UnboundVoidFormatSources(t, source.String(), target.goarch, target.triple) + if target.goarch == "386" { + // Original accepted X31/Y31 and R9 source spellings do not + // establish physical mode32 banks or an incoming address. + // The independent initialized low-bank family owns objects. + return + } + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerPackedConflict(ins.Op, ins) }) + for _, want := range []string{"icmp eq", "select i1", "<16 x i32>", "<8 x i64>"} { + if !strings.Contains(ir, want) { + t.Fatalf("packed-conflict lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "packed-conflict-"+target.name+".ll", "packed-conflict-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedConflictRejectsFormsOutsideGoTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPCONFLICTD X0, Y1"}, + {goarch: "amd64", instruction: "VPCONFLICTQ Y0, Z1"}, + {goarch: "amd64", instruction: "VPCONFLICTD X0, K0, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTQ.Z X0, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTD.BCST X0, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTQ.Z.BCST 0(AX), K1, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTD.RN_SAE X0, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTQ X0, 0(AX)"}, + {goarch: "amd64", instruction: "VPCONFLICTD X0, K1, K2, X1"}, + {goarch: "386", instruction: "VPCONFLICTD Z8, Z0"}, + {goarch: "386", instruction: "VPCONFLICTQ Z0, K1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvexpandpd conflict table", test.instruction) + } + }) + } +} + +func TestAMD64PackedConflictRuntimeSemantics(t *testing.T) { + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable integer oracle requires its native 64-bit host counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT packedconflictsemantics(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + STC + VPCONFLICTD Z0, Z2 + VMOVDQU64 Z2, 0(AX) + VPCONFLICTQ Z0, Z2 + VMOVDQU64 Z2, 64(AX) + VPCONFLICTD Z0, K1, Z1 + VMOVDQU64 Z1, 128(AX) + VMOVDQU64 0(CX), Z1 + VPCONFLICTQ.Z Z0, K1, Z1 + VMOVDQU64 Z1, 192(AX) + VPCONFLICTD.BCST 0(BX), Z2 + VMOVDQU64 Z2, 256(AX) + VPCONFLICTQ.BCST 0(BX), Z2 + VMOVDQU64 Z2, 320(AX) + SETCS 384(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"packedconflictsemantics": { + Name: "packedconflictsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void packedconflictsemantics(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +int main(void) { + uint32_t source32[16], old32[16]; uint8_t out[385] = {0}; + for (int i = 0; i < 16; i++) { source32[i] = (uint32_t)(i % 4); old32[i] = 0x90000000u + (uint32_t)i; } + const uint64_t mask = UINT64_C(0xa55a); + packedconflictsemantics(out, (const uint8_t *)source32, (const uint8_t *)old32, mask); + uint32_t *plain32 = (uint32_t *)(out + 0), *merge32 = (uint32_t *)(out + 128), *broadcast32 = (uint32_t *)(out + 256); + for (int i = 0; i < 16; i++) { + uint32_t conflict = 0; + for (int j = 0; j < i; j++) if (source32[j] == source32[i]) conflict |= UINT32_C(1) << j; + if (plain32[i] != conflict) return 10 + i; + uint32_t merged = (mask >> i) & 1 ? conflict : old32[i]; + if (merge32[i] != merged) return 30 + i; + uint32_t broadcast = i == 0 ? 0 : ((UINT32_C(1) << i) - 1); + if (broadcast32[i] != broadcast) return 50 + i; + } + uint64_t source64[8], old64[8], plain64[8], zero64[8], broadcast64[8]; + memcpy(source64, source32, 64); memcpy(old64, old32, 64); + memcpy(plain64, out + 64, 64); memcpy(zero64, out + 192, 64); memcpy(broadcast64, out + 320, 64); + for (int i = 0; i < 8; i++) { + uint64_t conflict = 0; + for (int j = 0; j < i; j++) if (source64[j] == source64[i]) conflict |= UINT64_C(1) << j; + if (plain64[i] != conflict) return 70 + i; + uint64_t masked = (mask >> i) & 1 ? conflict : 0; + if (zero64[i] != masked) return 80 + i; + uint64_t broadcast = i == 0 ? 0 : ((UINT64_C(1) << i) - 1); + if (broadcast64[i] != broadcast) return 90 + i; + } + if (out[384] != 1) return 100; + return 0; +} +` + x86ConflictFunnelPortableRuntime(t, llc, clang, "packed_conflict_semantics", triple, ir, mainC) +} diff --git a/amd64_lower_convert.go b/amd64_lower_convert.go new file mode 100644 index 000000000..e72459aff --- /dev/null +++ b/amd64_lower_convert.go @@ -0,0 +1,31 @@ +package plan9asm + +import ( + "strconv" + "strings" +) + +func amd64ParseMReg(r Reg) (idx int, ok bool) { + s := strings.ToUpper(strings.TrimSpace(string(r))) + if !strings.HasPrefix(s, "M") { + return 0, false + } + n, err := strconv.Atoi(strings.TrimPrefix(s, "M")) + if err != nil || n < 0 || n > 7 { + return 0, false + } + return n, true +} + +// Legacy named entries use the complete shared typed/native family. +func (c *amd64Ctx) lowerPackedFloatToDword(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedNumericConversion(op, ins) +} + +func (c *amd64Ctx) lowerPackedSingleToDouble(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if base != "CVTPS2PD" && base != "VCVTPS2PD" { + return false, false, nil + } + return c.lowerPackedPrecisionConvert(op, ins) +} diff --git a/amd64_lower_convert_double_dword.go b/amd64_lower_convert_double_dword.go new file mode 100644 index 000000000..8c859a82c --- /dev/null +++ b/amd64_lower_convert_double_dword.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedDoubleDwordMode uint8 + +const ( + amd64PackedDwordToDouble amd64PackedDoubleDwordMode = iota + amd64PackedDoubleToDword +) + +type amd64PackedDoubleDwordSpec struct { + mode amd64PackedDoubleDwordMode + legacy bool + truncate bool + sourceBytes int + destBytes int + resultLanes int + explicitMode string +} + +// This table is both the translation dispatch and the source of truth used by +// cmd/plan9asmll's supported-opcode extractor. +var amd64PackedDoubleDwordOps = map[string]amd64PackedDoubleDwordSpec{ + "CVTPD2PL": {mode: amd64PackedDoubleToDword, legacy: true, sourceBytes: 16, destBytes: 16, resultLanes: 2}, + "CVTTPD2PL": {mode: amd64PackedDoubleToDword, legacy: true, truncate: true, sourceBytes: 16, destBytes: 16, resultLanes: 2}, + "VCVTDQ2PD": {mode: amd64PackedDwordToDouble}, + "VCVTPD2DQ": {mode: amd64PackedDoubleToDword, sourceBytes: 64, destBytes: 32, resultLanes: 8, explicitMode: "rounding"}, + "VCVTPD2DQX": {mode: amd64PackedDoubleToDword, sourceBytes: 16, destBytes: 16, resultLanes: 2}, + "VCVTPD2DQY": {mode: amd64PackedDoubleToDword, sourceBytes: 32, destBytes: 16, resultLanes: 4}, + "VCVTTPD2DQ": {mode: amd64PackedDoubleToDword, truncate: true, sourceBytes: 64, destBytes: 32, resultLanes: 8, explicitMode: "sae"}, + "VCVTTPD2DQX": {mode: amd64PackedDoubleToDword, truncate: true, sourceBytes: 16, destBytes: 16, resultLanes: 2}, + "VCVTTPD2DQY": {mode: amd64PackedDoubleToDword, truncate: true, sourceBytes: 32, destBytes: 16, resultLanes: 4}, +} + +type amd64PackedDoubleDwordSuffix struct { + broadcast bool + rounding string + sae bool + zeroing bool +} + +func parseAMD64PackedDoubleDwordSuffix(rawOp, baseOp string) (amd64PackedDoubleDwordSuffix, error) { + var result amd64PackedDoubleDwordSuffix + if rawOp == baseOp { + return result, nil + } + suffixes := strings.Split(strings.TrimPrefix(rawOp, baseOp+"."), ".") + for index, suffix := range suffixes { + switch suffix { + case "BCST": + if result.broadcast || result.rounding != "" || result.sae || result.zeroing || index != 0 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .BCST suffix", baseOp) + } + result.broadcast = true + case "RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE": + if result.broadcast || result.rounding != "" || result.sae || result.zeroing || index != 0 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .%s suffix", baseOp, suffix) + } + result.rounding = suffix + case "SAE": + if result.broadcast || result.rounding != "" || result.sae || result.zeroing || index != 0 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .SAE suffix", baseOp) + } + result.sae = true + case "Z": + if result.zeroing || index != len(suffixes)-1 { + return result, fmt.Errorf("amd64 %s has invalid or misplaced .Z suffix", baseOp) + } + result.zeroing = true + default: + return result, fmt.Errorf("amd64 %s has unsupported suffix .%s", baseOp, suffix) + } + } + return result, nil +} + +// Dword-to-double retains its reviewed source/caller grammar; the remaining +// eight names use the complete packed numeric conversion family. +func (c *amd64Ctx) lowerPackedDoubleDword(op Op, ins Instr) (bool, bool, error) { + if _, known, _ := x86SignedPackedFloatProperties(ins, c.goarch); known { + return c.lowerSignedPackedIntegerFloat(op, ins) + } + return c.lowerPackedNumericConversion(op, ins) +} + +func (c *amd64Ctx) lowIntegerLanes(value string, physicalLanes, resultLanes, laneBits int) string { + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %s, <%d x i%d> poison, <%d x i32> <", low, physicalLanes, laneBits, value, physicalLanes, laneBits, resultLanes) + for lane := 0; lane < resultLanes; lane++ { + if lane != 0 { + c.b.WriteString(", ") + } + fmt.Fprintf(c.b, "i32 %d", lane) + } + c.b.WriteString(">\n") + return "%" + low +} diff --git a/amd64_lower_convert_double_dword_test.go b/amd64_lower_convert_double_dword_test.go new file mode 100644 index 000000000..56278766c --- /dev/null +++ b/amd64_lower_convert_double_dword_test.go @@ -0,0 +1,296 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedDoubleDwordCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + high := 20 + conversionHigh, conversionBroadcast := 20, 9 + zBroadcast, zRound, zRoundMask := 8, 9, 11 + if target.goarch == "386" { + high = 4 + zBroadcast, zRound, zRoundMask = 0, 1, 3 + conversionHigh, conversionBroadcast = 4, 6 + } + source := fmt.Sprintf(` +TEXT packeddoubledwordforms(SB),$0-0 + CVTPD2PL X0, X1 + CVTTPD2PL 16(AX), X2 + VCVTDQ2PD X0, X1 + VCVTDQ2PD 32(AX), Y2 + VCVTDQ2PD Y3, Z4 + VCVTDQ2PD X%d, K1, X%d + VCVTDQ2PD.Z 64(AX), K2, Y7 + VCVTDQ2PD.BCST 80(AX), Z%d + VCVTDQ2PD.BCST.Z 84(AX), K3, X%d + VCVTPD2DQX X0, X1 + VCVTPD2DQX 88(AX), X2 + VCVTPD2DQX X%d, K1, X%d + VCVTPD2DQX 96(AX), K1, X2 + VCVTPD2DQX.Z 104(AX), K2, X3 + VCVTPD2DQX.BCST 112(AX), X4 + VCVTPD2DQX.BCST.Z 120(AX), K3, X5 + VCVTPD2DQY Y%d, X%d + VCVTPD2DQY 128(AX), X4 + VCVTPD2DQY Y3, K1, X4 + VCVTPD2DQY.Z 128(AX), K2, X5 + VCVTPD2DQY.BCST 136(AX), X6 + VCVTPD2DQY.BCST.Z 144(AX), K3, X7 + VCVTPD2DQ Z6, Y7 + VCVTPD2DQ 160(AX), K1, Y8 + VCVTPD2DQ.Z 168(AX), K2, Y9 + VCVTPD2DQ.BCST 176(AX), Y10 + VCVTPD2DQ.BCST.Z 184(AX), K3, Y11 + VCVTPD2DQ.RN_SAE Z%d, Y10 + VCVTPD2DQ.RD_SAE.Z Z%d, K3, Y12 + VCVTPD2DQ.RU_SAE Z%d, Y13 + VCVTPD2DQ.RZ_SAE.Z Z%d, K4, Y14 + VCVTTPD2DQX X0, X1 + VCVTTPD2DQX 216(AX), X2 + VCVTTPD2DQX X%d, K1, X%d + VCVTTPD2DQX 224(AX), K1, X2 + VCVTTPD2DQX.Z 232(AX), K2, X3 + VCVTTPD2DQX.BCST 240(AX), X4 + VCVTTPD2DQX.BCST.Z 248(AX), K3, X5 + VCVTTPD2DQY Y%d, X%d + VCVTTPD2DQY 256(AX), X4 + VCVTTPD2DQY Y3, K1, X4 + VCVTTPD2DQY.Z 256(AX), K2, X5 + VCVTTPD2DQY.BCST 264(AX), X6 + VCVTTPD2DQY.BCST.Z 272(AX), K3, X7 + VCVTTPD2DQ Z6, Y7 + VCVTTPD2DQ 288(AX), K1, Y8 + VCVTTPD2DQ.Z 296(AX), K2, Y9 + VCVTTPD2DQ.BCST 304(AX), Y10 + VCVTTPD2DQ.BCST.Z 312(AX), K3, Y11 + VCVTTPD2DQ.SAE Z%d, Y10 + VCVTTPD2DQ.SAE.Z Z%d, K3, Y12 + RET +`, conversionHigh, conversionHigh+1, zBroadcast, conversionBroadcast, + high, high+1, high, high+1, zRound, zRoundMask, zRound, zRoundMask, + high, high+1, high, high+1, zRound, zRoundMask) + if target.goarch == "386" { + for n := 8; n <= 14; n++ { + source = strings.ReplaceAll(source, fmt.Sprintf("Y%d", n), fmt.Sprintf("Y%d", n%8)) + } + } + ll := x86UnboundVoidScalarFormatObject(t, source, "packeddoubledwordforms", target.goarch, target.triple) + for _, want := range []string{"vcvtdq2pd", "cvtpd2dq", "vcvttpd2dq"} { + if !strings.Contains(ll, want) { + t.Fatalf("packed double/dword IR is missing %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-double-dword-"+target.name+".ll", "packed-double-dword-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedDoubleDwordRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "CVTPD2PL Y0, X1", + "CVTTPD2PL X0, Y1", + "CVTTPD2PL.Z X0, X1", + "VCVTDQ2PD Y0, X1", + "VCVTDQ2PD X0, Z1", + "VCVTDQ2PD.BCST X0, X1", + "VCVTDQ2PD.RN_SAE Y0, Z1", + "VCVTPD2DQ X0, X1", + "VCVTPD2DQY X0, X1", + "VCVTPD2DQX Y0, X1", + "VCVTPD2DQY.BCST X0, X1", + "VCVTPD2DQ.RN_SAE 0(AX), Y1", + "VCVTTPD2DQX.SAE X0, X1", + "VCVTTPD2DQ.SAE 0(AX), Y1", + "VCVTTPD2DQ.Z Z0, Y1", + "VCVTTPD2DQ Z0, K0, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedDoubleDwordRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86PackedDoubleDwordRejected(t, "386", "i386-unknown-linux-gnu", "VCVTPD2DQ Z8, Y1") +} + +func assertX86PackedDoubleDwordRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed double/dword tables", instruction) + } +} + +func TestAMD64PackedDoubleDwordRuntimeSemantics(t *testing.T) { + amd64PackedDoubleDwordRuntime(t, x86FloatCompareRuntimeTools(t, x86FloatCompareSSE), false) +} + +func x86PackedDoubleDwordOriginalAssembly() string { + return ` +TEXT packeddoubledword(SB),NOSPLIT,$0-40 + MOVQ out+0(FP), AX + MOVQ doubles+8(FP), BX + MOVQ dwords+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), SI + KMOVQ SI, K1 + + VMOVDQU64 (DX), Z1 + VCVTDQ2PD (CX), K1, Z1 + VMOVDQU64 Z1, 0(AX) + + VCVTTPD2DQY (BX), X2 + VMOVDQU X2, 64(AX) + + VMOVDQU64 (BX), Z0 + VCVTPD2DQ.RD_SAE.Z Z0, K1, Y3 + VMOVDQU Y3, 80(AX) + + MOVOU (BX), X4 + CVTPD2PL X4, X0 + MOVQ X0, 112(AX) + VZEROUPPER + RET +` +} + +func x86PackedDoubleDwordRawAssembly(source string) string { + return strings.NewReplacer( + "TEXT packeddoubledword(SB),NOSPLIT,$0-40", "TEXT packeddoubledword(SB),4,$0-40", + "VCVTDQ2PD (CX), K1, Z1", "BYTE $0x62; BYTE $0xf1; BYTE $0x7e; BYTE $0x49; BYTE $0xe6; BYTE $0x09", + "VCVTTPD2DQY (BX), X2", "BYTE $0xc5; BYTE $0xfd; BYTE $0xe6; BYTE $0x13", + "VCVTPD2DQ.RD_SAE.Z Z0, K1, Y3", "BYTE $0x62; BYTE $0xf1; BYTE $0xff; BYTE $0xb9; BYTE $0xe6; BYTE $0xd8", + "CVTPD2PL X4, X0", "BYTE $0xf2; BYTE $0x0f; BYTE $0xe6; BYTE $0xc4", + ).Replace(source) +} + +func x86PackedDoubleDwordSSEAssembly(t *testing.T) string { + t.Helper() + source := x86PackedDoubleDwordOriginalAssembly() + preamble := strings.Index(source, "\tKMOVQ SI, K1") + legacy := strings.Index(source, "\tMOVOU (BX), X4") + wideExit := strings.Index(source, "\tVZEROUPPER") + if preamble < 0 || legacy <= preamble || wideExit <= legacy { + t.Fatal("original complete double/dword fixture lost its explicit SSE block") + } + return source[:preamble] + source[legacy:wideExit] + "\tRET\n" +} + +func amd64PackedDoubleDwordRuntime(t *testing.T, execution x86FloatCompareExecution, nativeWide bool) { + t.Helper() + source := x86PackedDoubleDwordOriginalAssembly() + if !nativeWide { + source = x86PackedDoubleDwordSSEAssembly(t) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }} + triple := execution.triple + translationOptions := Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packeddoubledword": { + Name: "packeddoubledword", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame, + }, + }, + } + ll, err := Translate(file, translationOptions) + if err != nil { + t.Fatal(err) + } + // The entire original named and raw functions still reach Go and LLVM + // objects in the portable lane. Only their execution requires native ISA. + for _, complete := range []string{x86PackedDoubleDwordOriginalAssembly(), x86PackedDoubleDwordRawAssembly(x86PackedDoubleDwordOriginalAssembly())} { + goAssemblerOracleOutput(t, "amd64", x86PackedDoubleDwordGoAssemblerSource(complete), "", true) + parsed, err := Parse(ArchAMD64, complete) + if err != nil { + t.Fatal(err) + } + objectIR, err := Translate(parsed, translationOptions) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, execution.llc, triple, "complete-double-dword.ll", "complete-double-dword.o", objectIR) + } + mainC := fmt.Sprintf("#define NATIVE_WIDE %d\n", x86FloatAddSubBoolInt(nativeWide)) + ` +#include +#include +#include +#include +extern void packeddoubledword(uint8_t *, const double *, const int32_t *, const double *, uint64_t); +int main(void) { + const double doubles[8] = {1.75, -2.75, 2147483648.0, NAN, -2147483648.0, -2147483649.0, 3.0, -0.0}; + const int32_t dwords[8] = {0, 1, -1, INT32_MAX, INT32_MIN, 123456789, -987654321, 42}; + const double old[8] = {100.5, 101.5, 102.5, 103.5, 104.5, 105.5, 106.5, 107.5}; + const uint64_t mask = 0xa5; + uint8_t out[120]; memset(out, 0xcc, sizeof(out)); + packeddoubledword(out, doubles, dwords, old, mask); + +#if NATIVE_WIDE + const double *widened = (const double *)(out+0); + for (int i = 0; i < 8; i++) { + double want = (mask >> i) & 1 ? (double)dwords[i] : old[i]; + if (widened[i] != want) return 10+i; + } + const int32_t *truncated = (const int32_t *)(out+64); + const int32_t wantTruncated[4] = {1, -2, INT32_MIN, INT32_MIN}; + for (int i = 0; i < 4; i++) if (truncated[i] != wantTruncated[i]) return 30+i; + + const int32_t *roundedDown = (const int32_t *)(out+80); + const int32_t wantRoundedDown[8] = {1, 0, INT32_MIN, 0, 0, INT32_MIN, 0, 0}; + for (int i = 0; i < 8; i++) if (roundedDown[i] != wantRoundedDown[i]) return 50+i; +#endif + + const int32_t *legacy = (const int32_t *)(out+112); + if (legacy[0] != 2 || legacy[1] != -3) return 70; + return 0; +} +` + execution.compileAndRunLLVM(t, "packed_double_dword", ll, mainC, "-O2") + + rawSource := x86PackedDoubleDwordRawAssembly(source) + goAssemblerOracleOutput(t, "amd64", x86PackedDoubleDwordGoAssemblerSource(rawSource), "", true) + rawFile, err := Parse(ArchAMD64, rawSource) + if err != nil { + t.Fatal(err) + } + rawIR, err := Translate(rawFile, translationOptions) + if err != nil { + t.Fatal(err) + } + execution.compileAndRunLLVM(t, "raw_packed_double_dword", rawIR, mainC, "-O2") + t.Logf("complete named/raw objects; SSE legacy output executed; full 120-byte native-wide assertions=%v", nativeWide) +} diff --git a/amd64_lower_convert_fp16.go b/amd64_lower_convert_fp16.go new file mode 100644 index 000000000..6738236e8 --- /dev/null +++ b/amd64_lower_convert_fp16.go @@ -0,0 +1,108 @@ +package plan9asm + +import "fmt" + +// FP16 map5/map6 conversions are both load forms. The lane types determine +// access and result widths; encoded VL distinguishes the two X destinations. +func (c *amd64Ctx) lowerRawFP16Conversion(ins Instr) (bool, bool, error) { + f, known, err := x86FP16ConversionProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, c.emitNativeFP16Conversion(ins, f) + } + // Preserve the existing cross-target numerical implementation. It is not + // an x86 MXCSR/exception execution oracle; matching x86 targets use the + // original opcode above, even when the numerical result is dead. + spec, properties := f.spec, f.controls + if spec.scalar { + return c.lowerRawScalarHalfConversion(ins, f) + } + source, destination := ins.Args[0], ins.Args[f.destination] + lanes, inputBytes, outputBytes := f.lanes, f.inputBytes, f.outputBytes + sourceRegisterBytes, destinationRegisterBytes := f.sourceBytes, f.byteWidth + masked := f.mask >= 0 + + mask := "" + if masked { + mask, err = c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + } + var input string + if source.Kind == OpReg || !masked && !properties.broadcast { + input, err = c.loadPackedExtendInputs(source, sourceRegisterBytes, lanes, spec.inputBits) + } else { + input, err = c.loadMaskedPackedCompareLanes(source, inputBytes, spec.inputBits, properties.broadcast, mask) + } + if err != nil { + return true, false, err + } + result := c.emitHalfConversion(lanes, spec, input, properties.rounding) + if masked { + old, err := c.loadPackedExtendInputs(destination, destinationRegisterBytes, lanes, spec.outputBits) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, spec.outputBits, result, old, mask, properties.zeroing) + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, spec.outputBits, result, outputBytes) + return true, false, c.storePackedHalfResult(destination, destinationRegisterBytes, outputBytes, "%"+bytes) +} + +func (c *amd64Ctx) emitHalfConversion(lanes int, spec amd64HalfConversionSpec, input, rounding string) string { + inputType := amd64FMA3LLVMType(1, spec.inputBits) + outputType := amd64FMA3LLVMType(1, spec.outputBits) + conversion := "fpext" + if spec.outputBits == 16 { + conversion = "fptrunc" + } + floats, converted, bits := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x %s>\n", floats, lanes, spec.inputBits, input, lanes, inputType) + fmt.Fprintf(c.b, " %%%s = %s <%d x %s> %%%s to <%d x %s>\n", converted, conversion, lanes, inputType, floats, lanes, outputType) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x %s> %%%s to <%d x i%d>\n", bits, lanes, outputType, converted, lanes, spec.outputBits) + result := "%" + bits + if spec.outputBits == 16 { + result = c.adjustFP16NarrowRounding(lanes, inputType, "%"+floats, "%"+converted, result, rounding) + } + return result +} + +// Directed rounding differs from nearest-even by at most one half ULP. +// Comparing the exactly widened result lets integer correction preserve +// signed zero, subnormals, NaNs and directed overflow without changing the +// host FP environment. LLVM 22's AArch64 constrained fptrunc does not honor +// these static rounding modes for half, so it cannot serve as this oracle. +func (c *amd64Ctx) adjustFP16NarrowRounding(lanes int, inputType, original, halves, bits, rounding string) string { + if rounding == "" || rounding == "RN_SAE" { + return bits + } + widened, below, above, negative := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext <%d x half> %s to <%d x %s>\n", widened, lanes, halves, lanes, inputType) + fmt.Fprintf(c.b, " %%%s = fcmp olt <%d x %s> %%%s, %s\n", below, lanes, inputType, widened, original) + fmt.Fprintf(c.b, " %%%s = fcmp ogt <%d x %s> %%%s, %s\n", above, lanes, inputType, widened, original) + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i16> %s, zeroinitializer\n", negative, lanes, bits) + one := amd64SplatInteger(c, lanes, 16, "1") + plus, minus := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = add <%d x i16> %s, %s\n", plus, lanes, bits, one) + fmt.Fprintf(c.b, " %%%s = sub <%d x i16> %s, %s\n", minus, lanes, bits, one) + condition, adjusted := "%"+above, "%"+minus + choice := c.newTmp() + switch rounding { + case "RD_SAE": + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i16> %%%s, <%d x i16> %%%s\n", choice, lanes, negative, lanes, plus, lanes, minus) + adjusted = "%" + choice + case "RU_SAE": + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i16> %%%s, <%d x i16> %%%s\n", choice, lanes, negative, lanes, minus, lanes, plus) + condition, adjusted = "%"+below, "%"+choice + case "RZ_SAE": + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i1> %%%s, <%d x i1> %%%s\n", choice, lanes, negative, lanes, below, lanes, above) + condition = "%" + choice + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %s, <%d x i16> %s, <%d x i16> %s\n", result, lanes, condition, lanes, adjusted, lanes, bits) + return "%" + result +} diff --git a/amd64_lower_convert_fp16_scalar.go b/amd64_lower_convert_fp16_scalar.go new file mode 100644 index 000000000..53dd94300 --- /dev/null +++ b/amd64_lower_convert_fp16_scalar.go @@ -0,0 +1,33 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerRawScalarHalfConversion(ins Instr, f x86FP16ConversionForm) (bool, bool, error) { + spec, properties := f.spec, f.controls + source, upper, destination := ins.Args[0], ins.Args[f.upper], ins.Args[f.destination] + mask := "" + var err error + if f.mask >= 0 { + mask, err = c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + } + value, err := c.loadFMA3Scalar(source, spec.inputBits, mask) + if err != nil { + return true, false, err + } + inputType := amd64FMA3LLVMType(1, spec.inputBits) + vector, bits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x %s> poison, %s %s, i32 0\n", vector, inputType, inputType, value) + fmt.Fprintf(c.b, " %%%s = bitcast <1 x %s> %%%s to <1 x i%d>\n", bits, inputType, vector, spec.inputBits) + result := c.emitHalfConversion(1, spec, "%"+bits, properties.rounding) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.outputBits, result) + upperBytes, err := c.loadX(upper.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.outputBits, spec.outputBits, upperBytes) + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.outputBits, "%"+low, base, mask, properties.zeroing) +} diff --git a/amd64_lower_convert_half.go b/amd64_lower_convert_half.go new file mode 100644 index 000000000..435b5de0b --- /dev/null +++ b/amd64_lower_convert_half.go @@ -0,0 +1,103 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64HalfConversionSuffix struct { + sae bool + zeroing bool +} + +type amd64HalfConversionSpec struct { + inputBits, outputBits int + rawOnly bool + scalar bool +} + +var amd64HalfConversionSpecs = map[string]amd64HalfConversionSpec{ + "VCVTPH2PS": {inputBits: 16, outputBits: 32}, + "VCVTPS2PH": {inputBits: 32, outputBits: 16}, + "VCVTPH2PSX": {inputBits: 16, outputBits: 32, rawOnly: true}, + "VCVTPS2PHX": {inputBits: 32, outputBits: 16, rawOnly: true}, + "VCVTSH2SS": {inputBits: 16, outputBits: 32, rawOnly: true, scalar: true}, + "VCVTSS2SH": {inputBits: 32, outputBits: 16, rawOnly: true, scalar: true}, + "VCVTSH2SD": {inputBits: 16, outputBits: 64, rawOnly: true, scalar: true}, + "VCVTSD2SH": {inputBits: 64, outputBits: 16, rawOnly: true, scalar: true}, +} + +func parseAMD64HalfConversionSuffix(rawOp, baseOp string) (amd64HalfConversionSuffix, error) { + var result amd64HalfConversionSuffix + if rawOp == baseOp { + return result, nil + } + suffixes := strings.Split(strings.TrimPrefix(rawOp, baseOp+"."), ".") + for index, suffix := range suffixes { + switch suffix { + case "SAE": + if result.sae || index != 0 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .SAE suffix", baseOp) + } + result.sae = true + case "Z": + if result.zeroing || index != len(suffixes)-1 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .Z suffix", baseOp) + } + result.zeroing = true + default: + return result, fmt.Errorf("amd64 %s has unsupported suffix .%s", baseOp, suffix) + } + } + return result, nil +} + +// lowerPackedHalfConversion implements every Go 1.27 _yvcvtph2ps and +// _yvcvtps2ph row, including VEX signed immediates, EVEX masks, zeroing, SAE, +// exact-width memory accesses, and the distinct X/Y/Z narrowing widths. +func (c *amd64Ctx) lowerPackedHalfConversion(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, supported := amd64HalfConversionSpecs[baseOp] + if !supported { + return false, false, nil + } + if spec.rawOnly { + if !strings.EqualFold(string(op), string(ins.Op)) { + return true, false, fmt.Errorf("raw FP16 conversion opcode identity mismatch: %s versus %s", op, ins.Op) + } + return c.lowerRawFP16Conversion(ins) + } + return c.lowerNamedHalfConversion(op, ins) +} + +func amd64HalfConversionHighRegister(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + width := amd64VectorByteWidth(operand.Reg) + index, ok := amd64VectorRegisterIndex(operand.Reg, width) + return ok && index >= 16 +} + +func (c *amd64Ctx) storePackedHalfResult(destination Operand, registerBytes, outputBytes int, value string) error { + if destination.Kind != OpReg { + return c.storeVectorBytesOperand(destination, outputBytes, value) + } + if outputBytes == registerBytes { + return c.storePackedMoveOperand(destination, registerBytes, value) + } + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> <", widened, outputBytes, value, outputBytes, registerBytes) + for lane := 0; lane < registerBytes; lane++ { + if lane > 0 { + c.b.WriteString(", ") + } + fmt.Fprintf(c.b, "i32 %d", lane) + } + c.b.WriteString(">\n") + return c.storePackedMoveOperand(destination, registerBytes, "%"+widened) +} diff --git a/amd64_lower_convert_half_test.go b/amd64_lower_convert_half_test.go new file mode 100644 index 000000000..3730563e2 --- /dev/null +++ b/amd64_lower_convert_half_test.go @@ -0,0 +1,125 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedHalfConversionCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 defines the whole F16C/AVX-512 FP16 conversion pair in + // _yvcvtph2ps and _yvcvtps2ph: + // + // VCVTPH2PS X/m -> X or Y; Y/m -> Z + // VCVTPS2PH imm8, X -> X/m; Y -> X/m; Z -> Y/m + // + // EVEX forms additionally accept K1-K7 and register-destination zeroing. + // The VEX VCVTPS2PH rows accept both signed and unsigned imm8 spellings. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString(`TEXT halfconversionforms(SB),NOSPLIT,$0-0 + VCVTPH2PS X0, X1 + VCVTPH2PS 8(AX), X2 + VCVTPH2PS X20, X21 + VCVTPH2PS X3, K1, X4 + VCVTPH2PS.Z 16(AX), K2, X5 + VCVTPH2PS X6, Y7 + VCVTPH2PS 32(AX), Y8 + VCVTPH2PS X20, K3, Y21 + VCVTPH2PS.Z 48(AX), K4, Y9 + VCVTPH2PS Y10, Z1 + VCVTPH2PS 64(AX), Z2 + VCVTPH2PS Y20, K5, Z3 + VCVTPH2PS.Z 96(AX), K6, Z4 + VCVTPH2PS.SAE Y10, Z5 + VCVTPH2PS.SAE.Z Y11, K7, Z6 + VCVTPS2PH $0, X0, X1 + VCVTPS2PH $-1, X2, 128(AX) + VCVTPS2PH $255, Y3, X4 + VCVTPS2PH $-128, Y5, 160(AX) + VCVTPS2PH $7, Z1, Y2 + VCVTPS2PH.SAE $8, Z3, Y4 +`) + // Go 1.27's 386 assembler accepts the EVEX VCVTPH2PS mask + // rows, but its operand parser does not expose the four-operand + // masked VCVTPS2PH rows. Keep this difference visible here. + if target.goarch == "amd64" { + source.WriteString(` VCVTPS2PH $126, X7, K5, X17 + VCVTPS2PH.Z $126, X7, K5, X17 + VCVTPS2PH $126, X7, K5, 192(AX) + VCVTPS2PH $94, Y1, K7, X15 + VCVTPS2PH.Z $94, Y1, K7, X15 + VCVTPS2PH $94, Y1, K7, 224(AX) + VCVTPS2PH $121, Z5, K7, Y28 + VCVTPS2PH.SAE.Z $13, Z2, K6, Y13 + VCVTPS2PH $121, Z5, K7, 256(AX) +`) + } + source.WriteString("\tRET\n") + ll := x86NamedHalfOriginalFormatObject(t, source.String(), target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-half-conversion-"+target.name+".ll", "packed-half-conversion-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedHalfConversionRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VCVTPH2PS X0", + "VCVTPH2PS X0, X1, X2", + "VCVTPH2PS Y0, X1", + "VCVTPH2PS X0, Z1", + "VCVTPH2PS X0, K0, X1", + "VCVTPH2PS.Z X0, X1", + "VCVTPH2PS.BCST (AX), X1", + "VCVTPS2PH X0, X1", + "VCVTPS2PH $256, X0, X1", + "VCVTPS2PH $0, Z0, X1", + "VCVTPS2PH $0, X0, Y1", + "VCVTPS2PH $0, X0, K0, X1", + "VCVTPS2PH.Z $0, X0, X1", + "VCVTPS2PH.BCST $0, X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedHalfConversionRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VCVTPS2PH $-1, X20, X21", + "VCVTPS2PH $-1, Y20, X21", + "VCVTPS2PH $0, X0, K1, X1", + "VCVTPS2PH $0, Y0, K1, X1", + "VCVTPS2PH $0, Z0, K1, Y1", + } { + assertX86PackedHalfConversionRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86PackedHalfConversionRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, fmt.Sprintf("TEXT bad(SB),NOSPLIT,$0-0\n\t%s\n\tRET\n", instruction)) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VCVTPH2PS/VCVTPS2PH tables", instruction) + } +} diff --git a/amd64_lower_convert_integer_float.go b/amd64_lower_convert_integer_float.go new file mode 100644 index 000000000..a75e4167f --- /dev/null +++ b/amd64_lower_convert_integer_float.go @@ -0,0 +1,39 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerLegacyIntegerToFloat implements Go 1.27's complete legacy integer to +// floating-point conversion group: yxcvlf/yxcvqf scalar conversions and the +// two yxcvm2 packed conversions. +func (c *amd64Ctx) lowerLegacyIntegerToFloat(op Op, ins Instr) (ok bool, terminated bool, err error) { + if _, known, _ := x86SignedPackedFloatProperties(ins, c.goarch); known { + return c.lowerSignedPackedIntegerFloat(op, ins) + } + rawOp := strings.ToUpper(string(op)) + if strings.Contains(rawOp, ".") { + base := strings.SplitN(rawOp, ".", 2)[0] + switch base { + case "CVTSL2SS", "CVTSL2SD", "CVTSQ2SS", "CVTSQ2SD", "CVTPL2PS", "CVTPL2PD": + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", base, ins.Raw) + } + } + switch rawOp { + case "CVTSL2SS", "CVTSL2SD", "CVTSQ2SS", "CVTSQ2SD": + return c.lowerLegacyScalarIntegerToFloat(rawOp, ins) + case "CVTPL2PS", "CVTPL2PD": + return c.lowerLegacyPackedIntegerToFloat(rawOp, ins) + default: + return false, false, nil + } +} + +func (c *amd64Ctx) lowerLegacyScalarIntegerToFloat(op string, ins Instr) (bool, bool, error) { + return c.lowerSignedIntegerScalarFloat(Op(op), ins) +} + +func (c *amd64Ctx) lowerLegacyPackedIntegerToFloat(op string, ins Instr) (bool, bool, error) { + return c.lowerSignedPackedIntegerFloat(Op(op), ins) +} diff --git a/amd64_lower_convert_narrow.go b/amd64_lower_convert_narrow.go new file mode 100644 index 000000000..ded5f1767 --- /dev/null +++ b/amd64_lower_convert_narrow.go @@ -0,0 +1,11 @@ +package plan9asm + +import "strings" + +func (c *amd64Ctx) lowerPackedDoubleToSingle(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if base != "CVTPD2PS" && base != "VCVTPD2PSX" && base != "VCVTPD2PSY" && base != "VCVTPD2PS" { + return false, false, nil + } + return c.lowerPackedPrecisionConvert(op, ins) +} diff --git a/amd64_lower_convert_narrow_test.go b/amd64_lower_convert_narrow_test.go new file mode 100644 index 000000000..7ffcb3f88 --- /dev/null +++ b/amd64_lower_convert_narrow_test.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedDoubleToSingleCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses four related tables for this family: + // CVTPD2PS: X/m128 -> X + // VCVTPD2PSX: X/m128 -> X + // VCVTPD2PSY: Y/m256 -> X + // VCVTPD2PS: Z/m512 -> Y + // The EVEX forms accept scalar-double broadcast and K1-K7 masking. Only + // the 512-bit source form accepts explicit rounding. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + // Go permits EVEX X/Y registers through 31 in 386 mode, while a + // Z source (the Yzm class) remains limited to Z0-Z7 there. + lastZ := 21 + if target.goarch == "386" { + lastZ = 7 + } + var src strings.Builder + src.WriteString("TEXT packeddoubletosingleforms(SB),NOSPLIT,$0-0\n") + src.WriteString("\tCVTPD2PS X0, X1\n") + src.WriteString("\tCVTPD2PS 8(AX), X2\n") + src.WriteString("\tVCVTPD2PSX X20, X0\n") + src.WriteString("\tVCVTPD2PSX (AX), X1\n") + src.WriteString("\tVCVTPD2PSX X20, K1, X21\n") + src.WriteString("\tVCVTPD2PSX.Z 16(AX), K2, X3\n") + src.WriteString("\tVCVTPD2PSX.BCST 24(AX), X4\n") + src.WriteString("\tVCVTPD2PSX.BCST.Z 32(AX), K3, X5\n") + src.WriteString("\tVCVTPD2PSY Y20, X0\n") + src.WriteString("\tVCVTPD2PSY (AX), X1\n") + src.WriteString("\tVCVTPD2PSY Y20, K4, X21\n") + src.WriteString("\tVCVTPD2PSY.Z 40(AX), K5, X3\n") + src.WriteString("\tVCVTPD2PSY.BCST 48(AX), X4\n") + src.WriteString("\tVCVTPD2PSY.BCST.Z 56(AX), K6, X5\n") + fmt.Fprintf(&src, "\tVCVTPD2PS Z%d, Y0\n", lastZ) + src.WriteString("\tVCVTPD2PS (AX), Y1\n") + fmt.Fprintf(&src, "\tVCVTPD2PS Z%d, K1, Y21\n", lastZ) + src.WriteString("\tVCVTPD2PS.Z 64(AX), K2, Y3\n") + src.WriteString("\tVCVTPD2PS.BCST 72(AX), Y4\n") + src.WriteString("\tVCVTPD2PS.BCST.Z 80(AX), K3, Y5\n") + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&src, "\tVCVTPD2PS.%s Z%d, Y6\n", rounding, lastZ) + } + fmt.Fprintf(&src, "\tVCVTPD2PS.RN_SAE.Z Z%d, K7, Y6\n", lastZ) + src.WriteString("\tRET\n") + + // Preserve the entire original table and independent Go object. + // Missing inputs and mode32 high banks are public Context, not + // initialized programs. All bound rows retain private LLVM objects. + ll := x86PackedPrecisionOriginalFormatObject(t, src.String(), "packeddoubletosingleforms", + target.goarch, target.triple, false, 25, 5) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-double-to-single-"+target.name+".ll", "packed-double-to-single-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedDoubleToSingleRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "CVTPD2PS X0", + "CVTPD2PS Y0, X1", + "CVTPD2PS X0, (AX)", + "CVTPD2PS.Z X0, X1", + "VCVTPD2PSX Y0, X1", + "VCVTPD2PSX X0, Y1", + "VCVTPD2PSY X0, X1", + "VCVTPD2PSY Y0, Y1", + "VCVTPD2PS Z0, X1", + "VCVTPD2PS Y0, Y1", + "VCVTPD2PS Z0, Z1", + "VCVTPD2PSX X0, K0, X1", + "VCVTPD2PSY.Z Y0, X1", + "VCVTPD2PS.Z Z0, Y1", + "VCVTPD2PSX.BCST X0, X1", + "VCVTPD2PSY.BCST Y0, X1", + "VCVTPD2PS.BCST Z0, Y1", + "VCVTPD2PSX.RN_SAE X0, X1", + "VCVTPD2PSY.RD_SAE Y0, X1", + "VCVTPD2PS.RU_SAE (AX), Y1", + "VCVTPD2PS.SAE Z0, Y1", + "VCVTPD2PS.BCST.RN_SAE (AX), Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedDoubleToSingleRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + + for _, instruction := range []string{ + "CVTPD2PS X8, X0", + "CVTPD2PS X0, X8", + "VCVTPD2PS Z8, Y0", + "VCVTPD2PS Z20, K1, Y0", + } { + assertX86PackedDoubleToSingleRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86PackedDoubleToSingleRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed double-to-single forms", instruction) + } +} diff --git a/amd64_lower_convert_samewidth.go b/amd64_lower_convert_samewidth.go new file mode 100644 index 000000000..c333884c7 --- /dev/null +++ b/amd64_lower_convert_samewidth.go @@ -0,0 +1,156 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SameWidthConversionMode uint8 + +const ( + amd64SameWidthIntToFloat amd64SameWidthConversionMode = iota + amd64SameWidthFloatToInt + amd64SameWidthFloatToIntTruncate + amd64SameWidthSqrt +) + +type amd64SameWidthConversionSpec struct { + laneBits int + mode amd64SameWidthConversionMode + sae bool + rawOnly bool +} + +var amd64SameWidthConversionSpecs = map[Op]amd64SameWidthConversionSpec{ + "VCVTDQ2PS": {laneBits: 32, mode: amd64SameWidthIntToFloat}, + "VCVTPS2DQ": {laneBits: 32, mode: amd64SameWidthFloatToInt}, + "VCVTTPS2DQ": {laneBits: 32, mode: amd64SameWidthFloatToIntTruncate, sae: true}, + "VSQRTPD": {laneBits: 64, mode: amd64SameWidthSqrt}, + "VSQRTPS": {laneBits: 32, mode: amd64SameWidthSqrt}, + "VSQRTPH": {laneBits: 16, mode: amd64SameWidthSqrt, rawOnly: true}, +} + +// lowerSameWidthPackedConversion implements all five opcodes sharing Go +// 1.27's _yvcvtdq2ps table. Although the table is shared, VCVTTPS2DQ enables +// SAE while the other four opcodes enable explicit rounding on Z forms. +func (c *amd64Ctx) lowerSameWidthPackedConversion(op Op, ins Instr) (ok bool, terminated bool, err error) { + if known, term, err := c.lowerSqrt(op, ins); known { + return known, term, err + } + if handled, terminated, err := c.lowerPackedNumericConversion(op, ins); handled { + return handled, terminated, err + } + if _, known, _ := x86SignedPackedFloatProperties(ins, c.goarch); known { + return c.lowerSignedPackedIntegerFloat(op, ins) + } + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64SameWidthConversionSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if spec.mode == amd64SameWidthIntToFloat { + return c.lowerSignedPackedIntegerFloat(op, ins) + } + if spec.rawOnly && !ins.x86Encoded { + return true, false, fmt.Errorf("%s is absent from Go's named assembler table; raw encoding required", baseOp) + } + + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix { + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's _yvcvtdq2ps encodings: %q", c.goarch, baseOp, ins.Raw) + } + if spec.sae { + if properties.rounding != "" { + return true, false, fmt.Errorf("%s %s supports SAE, not explicit rounding: %q", c.goarch, baseOp, ins.Raw) + } + } else if properties.sae { + return true, false, fmt.Errorf("%s %s supports explicit rounding, not SAE: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects source, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's vector-register class: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + explicitControl := properties.rounding != "" || properties.sae + if explicitControl && (byteWidth != 64 || source.Kind != OpReg) { + return true, false, fmt.Errorf("%s %s explicit rounding/SAE requires a Z register source: %q", c.goarch, baseOp, ins.Raw) + } + if properties.broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := byteWidth * 8 / spec.laneBits + mask := "" + if masked { + mask, err = c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + } + sourceBits, err := c.loadMaskedPackedCompareLanes(source, byteWidth, spec.laneBits, properties.broadcast, mask) + if err != nil { + return true, false, err + } + computedBits := c.emitSameWidthPackedConversion(spec, sourceBits, byteWidth, properties) + if masked { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + computedBits = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computedBits, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, computedBits, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitSameWidthPackedConversion(spec amd64SameWidthConversionSpec, sourceBits string, byteWidth int, properties amd64BinaryFloatingSuffix) string { + lanes := byteWidth * 8 / spec.laneBits + integerType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + floatType := amd64FMA3LLVMType(lanes, spec.laneBits) + switch spec.mode { + case amd64SameWidthSqrt: + floating := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", floating, integerType, sourceBits, floatType) + computed := c.newTmp() + if properties.rounding != "" { + name := fmt.Sprintf("llvm.experimental.constrained.sqrt.v%df%d", lanes, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %%%s, metadata !\"%s\", metadata !\"fpexcept.ignore\")\n", computed, floatType, name, floatType, floating, amd64FMA3RoundingMetadata(properties.rounding)) + } else { + name := fmt.Sprintf("llvm.sqrt.v%df%d", lanes, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %%%s)\n", computed, floatType, name, floatType, floating) + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to %s\n", bits, floatType, computed, integerType) + return "%" + bits + } + panic("unreachable same-width packed conversion mode") +} diff --git a/amd64_lower_convert_samewidth_test.go b/amd64_lower_convert_samewidth_test.go new file mode 100644 index 000000000..cf233a4e1 --- /dev/null +++ b/amd64_lower_convert_samewidth_test.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86SameWidthPackedConversionCompleteGoAssemblerForms(t *testing.T) { + // These five opcodes share Go 1.27's _yvcvtdq2ps table. All accept X/Y/Z + // register or memory sources, scalar-memory broadcast, and K1-K7 masks. + // The Z forms of all but VCVTTPS2DQ accept explicit rounding; the + // truncating conversion accepts SAE instead. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + zLast := 21 + if target.goarch == "386" { + zLast = 7 + } + var src strings.Builder + src.WriteString("TEXT samewidthpackedconversionforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VCVTDQ2PS", "VCVTPS2DQ", "VCVTTPS2DQ", "VSQRTPD", "VSQRTPS"} { + high := 20 + if strings.HasPrefix(op, "VCVT") && target.goarch == "386" { + // The exact native conversion cannot borrow the wider + // software bank used by the other conversion families. + high = 4 + } + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\t%s %s0, %s1\n", op, width, width) + fmt.Fprintf(&src, "\t%s (AX), %s2\n", op, width) + fmt.Fprintf(&src, "\t%s %s%d, %s%d\n", op, width, high, width, high+1) + fmt.Fprintf(&src, "\t%s %s%d, K1, %s%d\n", op, width, high, width, high+1) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, %s%d\n", op, width, high+2) + fmt.Fprintf(&src, "\t%s.BCST 8(AX), %s%d\n", op, width, high+2) + fmt.Fprintf(&src, "\t%s.BCST.Z 8(AX), K3, %s%d\n", op, width, high+3) + } + fmt.Fprintf(&src, "\t%s Z0, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s (AX), Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s Z6, K4, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.Z (AX), K5, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.BCST 8(AX), Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.BCST.Z 8(AX), K6, Z%d\n", op, zLast) + if op == "VCVTTPS2DQ" { + fmt.Fprintf(&src, "\t%s.SAE Z6, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.SAE.Z Z6, K7, Z%d\n", op, zLast) + } else { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&src, "\t%s.%s Z6, Z%d\n", op, rounding, zLast) + } + fmt.Fprintf(&src, "\t%s.RN_SAE.Z Z6, K7, Z%d\n", op, zLast) + } + } + src.WriteString("\tRET\n") + + ll := x86SqrtOriginalFormatObjects(t, src.String(), "samewidthpackedconversionforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "same-width-packed-conversion-"+target.name+".ll", "same-width-packed-conversion-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86SameWidthPackedConversionRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VCVTDQ2PS X0", + "VCVTPS2DQ X0, Y1", + "VCVTTPS2DQ X0, (AX)", + "VSQRTPD X0, K0, X1", + "VSQRTPS.Z X0, X1", + "VCVTDQ2PS.BCST X0, X1", + "VCVTPS2DQ.SAE Z0, Z1", + "VCVTTPS2DQ.RN_SAE Z0, Z1", + "VSQRTPD.RN_SAE X0, X1", + "VSQRTPS.RN_SAE (AX), Z1", + "VCVTDQ2PS.BCST.RN_SAE (AX), Z1", + "VCVTTPS2DQ.BCST.SAE (AX), Z1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86SameWidthPackedConversionRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + + for _, instruction := range []string{ + "VCVTDQ2PS Z8, Z0", + "VCVTPS2DQ Z0, Z8", + "VCVTTPS2DQ.SAE Z8, Z0", + "VSQRTPD.RN_SAE Z0, Z8", + "VSQRTPS Z0, K1, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86SameWidthPackedConversionRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86SameWidthPackedConversionRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvcvtdq2ps forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_counter_zero_branch_test.go b/amd64_lower_counter_zero_branch_test.go new file mode 100644 index 000000000..072056ca1 --- /dev/null +++ b/amd64_lower_counter_zero_branch_test.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86CounterZeroBranchCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := "TEXT counterzerobranchforms(SB),NOSPLIT,$0-0\n" + + "\tJCXZW width16\n" + + "width16:\n" + + "\tJCXZL width32\n" + + "width32:\n" + + "\tJCXZQ width64\n" + + "width64:\n" + + "\tRET\n" + // Preserve the exact original source as a Go-format and missing-CX + // control; a bare register read does not establish its input bytes. + x86UnboundVoidFormatSources(t, source, target.goarch, target.triple) + move := "MOVQ" + if target.goarch == "386" { + move = "MOVL" + } + initialized := strings.Replace(source, "\tJCXZW", "\t"+move+" $1,CX\n\tJCXZW", 1) + x86LEAActualTargetGoObject(t, target.goarch, target.triple, "#include \"textflag.h\"\n"+initialized) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, api := range []string{"text", "module", "Go"} { + ll, err := x86ScalarFPBitViewAPI(t, initialized, "func counterzerobranchforms()", target.goarch, target.triple, api) + if err != nil { + t.Fatal("same original branches with actual numerical CX", api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ll, level) + } + } + }) + } +} + +func TestTranslateX86CounterZeroBranchMatchesGoAssemblerAddressWidths(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + widths map[string]string + }{ + { + name: "amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", + widths: map[string]string{"JCXZW": "i64", "JCXZL": "i32", "JCXZQ": "i64"}, + }, + { + name: "386", goarch: "386", triple: "i386-unknown-linux-gnu", + widths: map[string]string{"JCXZW": "i32", "JCXZL": "i16", "JCXZQ": "i32"}, + }, + } { + for op, width := range target.widths { + t.Run(target.name+"_"+op, func(t *testing.T) { + source := fmt.Sprintf("TEXT counterzero(SB),NOSPLIT,$0-0\n\t%s zero\n\tRET\nzero:\n\tRET\n", op) + x86UnboundVoidFormatSources(t, source, target.goarch, target.triple) + move := "MOVQ" + if target.goarch == "386" { + move = "MOVL" + } + initialized := strings.Replace(source, "\t"+op, "\t"+move+" $1,CX\n\t"+op, 1) + for _, api := range []string{"text", "module", "Go"} { + ll, err := x86ScalarFPBitViewAPI(t, initialized, "func counterzero()", target.goarch, target.triple, api) + if err != nil { + t.Fatal("same original branch with actual numerical CX", api, err) + } + if !strings.Contains(ll, "icmp eq "+width) { + t.Fatalf("%s/%s/%s did not test Go's %s counter width:\n%s", target.goarch, op, api, width, ll) + } + } + }) + } + } +} + +func TestTranslateX86CounterZeroBranchRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "JCXZW", + "JCXZL $1, done", + "JCXZQ done, other", + "JCXZQ AX", + "JCXZQ.Z done", + } { + t.Run(strings.NewReplacer(" ", "_", ",", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),NOSPLIT,$0-0\n\t" + instruction + "\ndone:\nother:\n\tRET\n" + file, err := Parse(ArchAMD64, source) + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yloop table", instruction) + } + }) + } +} diff --git a/amd64_lower_crc32.go b/amd64_lower_crc32.go index 054f910af..476eb80a2 100644 --- a/amd64_lower_crc32.go +++ b/amd64_lower_crc32.go @@ -3,69 +3,60 @@ package plan9asm import "fmt" func (c *amd64Ctx) lowerCrc32(op Op, ins Instr) (ok bool, terminated bool, err error) { - switch op { - case "CRC32B", "CRC32W", "CRC32L", "CRC32Q": - // ok - default: - return false, false, nil + form, known, err := x86CRC32Properties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err } - if len(ins.Args) != 2 || ins.Args[0].Kind != OpMem || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects mem, reg: %q", op, ins.Raw) + if form.source.Kind == OpReg && (form.source.Reg == SP || form.source.Reg == BP) && + form.source.Reg != ins.Args[0].Reg { + return true, false, fmt.Errorf("%w: Go CRC32 REX byte view needs its original physical SP/BP input", ErrProbeNeedsContext) } - src := ins.Args[0].Mem - dst := ins.Args[1].Reg - - addr, err := c.addrFromMem(src) + sourceType := amd64IntegerTypeForBits(form.bits) + intrinsic := fmt.Sprintf("llvm.x86.sse42.crc32.32.%d", form.bits) + resultType := I32 + if form.bits == 64 { + resultType = I64 + intrinsic = "llvm.x86.sse42.crc32.64.64" + } + sourceValue, err := c.evalCRC32Source(ins, form, sourceType) if err != nil { return true, false, err } - p := c.ptrFromAddrI64(addr) - - dv, err := c.loadReg(dst) + crcValue, err := c.evalIntSized(form.destination, I32) if err != nil { return true, false, err } + if resultType == I64 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", extended, crcValue) + crcValue = "%" + extended // Unknown old high32 never enter the intrinsic. + } + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s)\n", call, resultType, intrinsic, resultType, crcValue, sourceType, sourceValue) + return true, false, c.storeRegSized(form.destination.Reg, resultType, "%"+call) +} - switch op { - case "CRC32Q": - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @llvm.x86.sse42.crc32.64.64(i64 %s, i64 %%%s)\n", call, dv, ld) - return true, false, c.storeReg(dst, "%"+call) - - case "CRC32L": - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", ld, p) - crc32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", crc32, dv) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse42.crc32.32.32(i32 %%%s, i32 %%%s)\n", call, crc32, ld) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(dst, "%"+z) - - case "CRC32W": - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i16, ptr %s, align 1\n", ld, p) - crc32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", crc32, dv) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse42.crc32.32.16(i32 %%%s, i16 %%%s)\n", call, crc32, ld) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(dst, "%"+z) - - case "CRC32B": - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i8, ptr %s, align 1\n", ld, p) - crc32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", crc32, dv) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse42.crc32.32.8(i32 %%%s, i8 %%%s)\n", call, crc32, ld) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(dst, "%"+z) +func (c *amd64Ctx) evalCRC32Source(ins Instr, form x86CRC32Form, typ LLVMType) (string, error) { + if form.source.Kind == OpFP && x86ResultNamedPredicateInstruction(ins) { + if value, handled, err := c.loadIntegerFPBytesMode(form.source.FPOffset, typ, true); handled || err != nil { + return value, err + } + } + if isAMD64MemoryOperand(form.source) && form.source.Kind != OpFP && + (c.nativeMemoryOperand(ins, 0) || + x86CallerMemoryCertified(ins, 0) || c.fixedLocalMemoryCertified(ins, 0)) { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return "", fmt.Errorf("%w: CRC32 needs its original native address size", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.source) + if err != nil { + return "", err + } + value := c.newTmp() + // Even a zero accumulator or dead result retains the complete native + // source read. No backing, extent or alignment follows from continuing. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", value, typ, pointerType, pointer) + return "%" + value, nil } - return true, false, fmt.Errorf("amd64: unknown CRC32 op %s", op) + return c.evalIntSized(form.source, typ) } diff --git a/amd64_lower_crc32_complete_test.go b/amd64_lower_crc32_complete_test.go new file mode 100644 index 000000000..90bae49cb --- /dev/null +++ b/amd64_lower_crc32_complete_test.go @@ -0,0 +1,253 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86CRC32CompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA crc32data+0(SB)/8, $1\n") + source.WriteString("GLOBL crc32data(SB), $8\n") + source.WriteString("TEXT crc32forms(SB),$0-0\n") + for _, width := range []string{"B", "W", "L"} { + op := "CRC32" + width + sourceRegister := "AX" + if width == "B" { + sourceRegister = "AL" + } + fmt.Fprintf(&source, "\t%s %s, BX\n", op, sourceRegister) + fmt.Fprintf(&source, "\t%s 8(BX), CX\n", op) + fmt.Fprintf(&source, "\t%s crc32data(SB), DX\n", op) + if width == "B" { + fmt.Fprintf(&source, "\t%s AX, DI\n", op) + } + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12\n", op) + } else { + fmt.Fprintf(&source, "\t%s %s, SP\n", op, sourceRegister) + } + } + if goarch == "amd64" { + for _, form := range []string{ + "CRC32Q AX, BX", + "CRC32Q 8(BX), CX", + "CRC32Q crc32data(SB), DX", + "CRC32Q R11, R12", + } { + fmt.Fprintf(&source, "\t%s\n", form) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86CRC32CompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86CRC32CompleteFormsSource(target.goarch) + // Preserve the complete original mixed bank/memory source. + // Its unknown accumulators and physical SP cannot supply a public + // entry. Initialized public objects have their separate CRC32 owner. + ir := x86UnboundVoidScalarFormatObject(t, source, "crc32forms", target.goarch, target.triple) + for _, want := range []string{ + "@llvm.x86.sse42.crc32.32.8", + "@llvm.x86.sse42.crc32.32.16", + "@llvm.x86.sse42.crc32.32.32", + `"target-features"="+crc32,+sse4.2"`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("CRC32 lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "@llvm.x86.sse42.crc32.64.64") { + t.Fatalf("CRC32Q lowering omitted i64 intrinsic:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "crc32-"+target.name+".ll", "crc32-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86CRC32RejectsFormsOutsideGo127Optabs(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "CRC32B $1, AX"}, + {goarch: "amd64", instruction: "CRC32W AL, AX"}, + {goarch: "amd64", instruction: "CRC32L X0, AX"}, + {goarch: "amd64", instruction: "CRC32Q AX, 0(BX)"}, + {goarch: "amd64", instruction: "CRC32L AX, X0"}, + {goarch: "amd64", instruction: "CRC32L AX, AL"}, + {goarch: "amd64", instruction: "CRC32L AX"}, + {goarch: "amd64", instruction: "CRC32L.Z AX, BX"}, + {goarch: "386", instruction: "CRC32Q AX, BX"}, + {goarch: "386", instruction: "CRC32L R8, AX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ycrc32b/ycrc32l tables", test.instruction) + } + }) + } +} + +func x86CRC32RuntimeSemanticsSource() string { + return `TEXT crc32semantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + + MOVL $0x12345678, BX + MOVL $0xab00, AX + CRC32B AH, BX + MOVQ BX, 0(DI) + + MOVL $0x23456789, BX + MOVL $0xcdef, AX + CRC32W AX, BX + MOVQ BX, 8(DI) + + MOVL $0x3456789a, BX + MOVL $0x89abcdef, 40(DI) + CRC32L 40(DI), BX + MOVQ BX, 16(DI) + + MOVL $0x456789ab, BX + MOVQ $0x0123456789abcdef, CX + MOVQ CX, 48(DI) + CRC32Q 48(DI), BX + MOVQ BX, 24(DI) + + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVL $0, BX + CRC32B AL, BX + SETCS 32(DI) + SETOS 33(DI) + SETEQ 34(DI) + SETMI 35(DI) + RET +` +} + +func TestX86CRC32RuntimeFixtureUsesOriginalGoEncoding(t *testing.T) { + source := x86CRC32RuntimeSemanticsSource() + goAssemblerOracleOutput(t, "amd64", "#include \"textflag.h\"\n"+source, "", true) + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue // The original complete fixture includes CRC32Q, not a Go386 GP row. + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func crc32semantics(out *byte)", "amd64", target.triple, api) + if err != nil || ir == "" { + t.Fatal("complete original CRC32 numerical/flags fixture rejected", err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + }) + } + } +} + +func TestAMD64CRC32RuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86CRC32RuntimeSemanticsSource() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "crc32semantics": { + Name: "crc32semantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void crc32semantics(uint8_t *); +static uint32_t crc32c(uint32_t crc, uint64_t value, unsigned bytes) { + for (unsigned i = 0; i < bytes; i++) { + crc ^= (uint8_t)value; + value >>= 8; + for (unsigned bit = 0; bit < 8; bit++) + crc = (crc >> 1) ^ (UINT32_C(0x82f63b78) & (uint32_t)-(int32_t)(crc & 1)); + } + return crc; +} +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[64] = {0}; + crc32semantics(out); + if (load64(out + 0) != crc32c(UINT32_C(0x12345678), UINT64_C(0xab), 1)) return 10; + if (load64(out + 8) != crc32c(UINT32_C(0x23456789), UINT64_C(0xcdef), 2)) return 11; + if (load64(out + 16) != crc32c(UINT32_C(0x3456789a), UINT64_C(0x89abcdef), 4)) return 12; + if (load64(out + 24) != crc32c(UINT32_C(0x456789ab), UINT64_C(0x0123456789abcdef), 8)) return 13; + if (out[32] != 1 || out[33] != 1 || out[34] != 0 || out[35] != 1) return 14; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "crc32_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_descriptor.go b/amd64_lower_descriptor.go new file mode 100644 index 000000000..5d8cf6f98 --- /dev/null +++ b/amd64_lower_descriptor.go @@ -0,0 +1,166 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var x86DescriptorOps = map[string]struct{}{ + "LGDT": {}, "LIDT": {}, "SGDT": {}, "SIDT": {}, + "LLDT": {}, "LTR": {}, "LMSW": {}, + "SLDTW": {}, "SLDTL": {}, "SLDTQ": {}, + "SMSWW": {}, "SMSWL": {}, "SMSWQ": {}, + "STRW": {}, "STRL": {}, "STRQ": {}, +} + +// lowerDescriptorTable covers Go 1.27's complete x86 descriptor-table and +// system-selector family. These instructions are privileged or may be blocked +// by UMIP, so the lowering preserves them as LLVM inline assembly rather than +// approximating their machine-state effects. +func (c *amd64Ctx) lowerDescriptorTable(op Op, ins Instr) (ok bool, terminated bool, err error) { + if ok, terminated, err := c.lowerSegmentQuery(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerFarPointer(op, ins); ok { + return ok, terminated, err + } + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if _, ok := x86DescriptorOps[baseOp]; !ok { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 1 { + return true, false, fmt.Errorf("%s %s expects one operand: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && strings.HasSuffix(baseOp, "Q") { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + + operand := ins.Args[0] + switch baseOp { + case "LGDT", "LIDT": + if !isAMD64MemoryOperand(operand) { + return true, false, fmt.Errorf("%s %s expects Go 1.27 Ym memory: %q", c.goarch, baseOp, ins.Raw) + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(operand) + if err != nil { + return true, false, err + } + bytes := 10 + if c.goarch == "386" { + bytes = 6 + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype([%d x i8]) %s)\n", + strings.ToLower(baseOp)+" $0", "*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, bytes, ptr) + return true, false, nil + + case "SGDT", "SIDT": + if !isAMD64MemoryOperand(operand) { + return true, false, fmt.Errorf("%s %s expects Go 1.27 Ym memory: %q", c.goarch, baseOp, ins.Raw) + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(operand) + if err != nil { + return true, false, err + } + bytes := 10 + if c.goarch == "386" { + bytes = 6 + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype([%d x i8]) %s)\n", + strings.ToLower(baseOp)+" $0", "=*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, bytes, ptr) + return true, false, nil + + case "LLDT", "LTR", "LMSW": + if operand.Kind != OpReg && !isAMD64MemoryOperand(operand) { + return true, false, fmt.Errorf("%s %s expects Go 1.27 Yml register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if operand.Kind == OpReg { + if !isX86YrlRegisterForArch(operand.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s register is outside Go 1.27's Yml class: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.evalIntSized(operand, I16) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(i16 %s)\n", + strings.ToLower(baseOp)+" $0", "r,~{memory},~{dirflag},~{fpsr},~{flags}", value) + return true, false, nil + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(operand) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype(i16) %s)\n", + strings.ToLower(baseOp)+" $0", "*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, ptr) + return true, false, nil + + case "SLDTW", "SLDTL", "SLDTQ", "SMSWW", "SMSWL", "SMSWQ", "STRW", "STRL", "STRQ": + return true, false, c.lowerDescriptorResult(baseOp, operand, ins) + default: + panic("descriptor opcode precheck drift") + } +} + +func (c *amd64Ctx) lowerDescriptorResult(op string, destination Operand, ins Instr) error { + if destination.Kind != OpReg && !isAMD64MemoryOperand(destination) { + return fmt.Errorf("%s %s expects Go 1.27 Yml register or memory: %q", c.goarch, op, ins.Raw) + } + if destination.Kind == OpReg && !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return fmt.Errorf("%s %s register is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + typ := I16 + switch op[len(op)-1] { + case 'L': + typ = I32 + case 'Q': + typ = I64 + } + asm := strings.ToLower(op) + " $0" + if destination.Kind == OpReg { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", + result, typ, asm, "=r,~{memory},~{dirflag},~{fpsr},~{flags}") + return c.storeRegSized(destination.Reg, typ, "%"+result) + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(destination) + if err != nil { + return err + } + // Intel defines every memory destination as m16 even when Go selects the + // L/Q spelling (the operand-size prefix only affects a register target). + // LLVM's integrated assembler consequently rejects sldtl/sldtq, smswl/q, + // and strl/q with memory, so emit the architectural unsuffixed m16 form. + memoryAsm := strings.ToLower(op[:len(op)-1]) + " $0" + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype(%s) %s)\n", + memoryAsm, "=*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, I16, ptr) + return nil +} + +func (c *amd64Ctx) x86DescriptorMemoryPointer(operand Operand) (ptr, ptrType string, err error) { + switch operand.Kind { + case OpMem: + return c.ptrFromMem(operand.Mem) + case OpSym: + ptr, err := c.ptrFromSB(operand.Sym) + return ptr, "ptr", err + case OpFP: + if c.classicFrame != "" { + return c.classicFramePtr(operand.FPOffset), "ptr", nil + } + if ptr := c.fpParamAlloca[operand.FPOffset]; ptr != "" { + return ptr, "ptr", nil + } + if ptr, _, ok := c.fpResultAlloca(operand.FPOffset); ok { + return ptr, "ptr", nil + } + return "", "", fmt.Errorf("%s descriptor memory operand has no mutable FP slot at +%d(FP)", c.goarch, operand.FPOffset) + default: + return "", "", fmt.Errorf("%s descriptor instruction expected memory, got %s", c.goarch, operand.String()) + } +} diff --git a/amd64_lower_dot_float.go b/amd64_lower_dot_float.go new file mode 100644 index 000000000..84f5172e2 --- /dev/null +++ b/amd64_lower_dot_float.go @@ -0,0 +1,34 @@ +package plan9asm + +type amd64PackedFloatingDotForm uint8 + +const ( + // amd64PackedFloatingDotLegacy is Go's yxshuf table: imm8, X/m128, X. + amd64PackedFloatingDotLegacy amd64PackedFloatingDotForm = iota + // amd64PackedFloatingDotVEX128 is Go's _yvdppd table: imm8, X/m128, + // X, X. The Go 386 frontend cannot represent the four operands. + amd64PackedFloatingDotVEX128 + // amd64PackedFloatingDotVEX128Or256 is Go's _yvblendpd table, adding + // the corresponding Y/m256, Y, Y row. + amd64PackedFloatingDotVEX128Or256 +) + +type amd64PackedFloatingDotSpec struct { + form amd64PackedFloatingDotForm + laneBits int +} + +// amd64PackedFloatingDotSpecs is the complete Go 1.27 grammar for the SSE4.1 +// and AVX floating dot-product family. The form class captures operand count, +// legal vector widths, and the 386 frontend restriction; laneBits selects the +// hardware intrinsic and immediate-bit interpretation. +var amd64PackedFloatingDotSpecs = map[Op]amd64PackedFloatingDotSpec{ + "DPPD": {form: amd64PackedFloatingDotLegacy, laneBits: 64}, + "DPPS": {form: amd64PackedFloatingDotLegacy, laneBits: 32}, + "VDPPD": {form: amd64PackedFloatingDotVEX128, laneBits: 64}, + "VDPPS": {form: amd64PackedFloatingDotVEX128Or256, laneBits: 32}, +} + +func (c *amd64Ctx) lowerPackedFloatingDot(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerNativePackedReduction(op, ins) +} diff --git a/amd64_lower_dot_float_test.go b/amd64_lower_dot_float_test.go new file mode 100644 index 000000000..130ca6719 --- /dev/null +++ b/amd64_lower_dot_float_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedFloatingDotGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64PackedFloatingDotSpec{ + "DPPD": {form: amd64PackedFloatingDotLegacy, laneBits: 64}, + "DPPS": {form: amd64PackedFloatingDotLegacy, laneBits: 32}, + "VDPPD": {form: amd64PackedFloatingDotVEX128, laneBits: 64}, + "VDPPS": {form: amd64PackedFloatingDotVEX128Or256, laneBits: 32}, + } + if len(amd64PackedFloatingDotSpecs) != len(expected) { + t.Fatalf("floating dot grammar has %d entries, want %d", len(amd64PackedFloatingDotSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64PackedFloatingDotSpecs[op]; !ok { + t.Errorf("floating dot grammar omitted %s", op) + } else if got != want { + t.Errorf("floating dot grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86PackedFloatingDotCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + var sourceBuilder strings.Builder + sourceBuilder.WriteString("TEXT floatdotforms(SB),$0-0\n") + fmt.Fprintf(&sourceBuilder, "\tDPPD $0x31, X0, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tDPPD $0x31, 0(AX), X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tDPPS $0xf1, X0, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tDPPS $0xf1, 16(AX), X%d\n", last) + if target.goarch == "amd64" { + fmt.Fprintf(&sourceBuilder, "\tVDPPD $0x31, X0, X1, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPD $0x31, 32(AX), X1, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPS $0xf1, X0, X1, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPS $0xf1, 48(AX), X1, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPS $0xf1, Y0, Y1, Y%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPS $0xf1, 64(AX), Y1, Y%d\n", last) + } + sourceBuilder.WriteString("\tRET\n") + source := sourceBuilder.String() + requireX86GoAssemblerResult(t, target.goarch, source, true) + ir := x86UnboundVoidScalarFormatObject(t, source, "floatdotforms", target.goarch, target.triple) + for _, want := range []string{ + "dppd ", "dpps ", "asm sideeffect", "~{fpsr}", + } { + if !strings.Contains(ir, want) { + t.Fatalf("packed floating dot lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && (!strings.Contains(ir, "call <8 x float> asm sideeffect") || !strings.Contains(ir, "vdpps ")) { + t.Fatalf("packed floating dot lowering omitted Y-width original opcode:\n%s", ir) + } + wantFeatures := `"target-features"="+sse4.1"` + if target.goarch == "amd64" { + wantFeatures = `"target-features"="+avx,+sse4.1"` + } + if !strings.Contains(ir, wantFeatures) { + t.Fatalf("packed floating dot lowering omitted %s:\n%s", wantFeatures, ir) + } + compileLLVMToObject(t, llc, target.triple, "float-dot-"+target.name+".ll", "float-dot-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedFloatingDotRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "DPPD $-1, X0, X1"}, + {goarch: "amd64", instruction: "DPPD $256, X0, X1"}, + {goarch: "amd64", instruction: "DPPS.Z $1, X0, X1"}, + {goarch: "amd64", instruction: "DPPD $1, Y0, Y1"}, + {goarch: "amd64", instruction: "VDPPD $1, Y0, Y1, Y2"}, + {goarch: "amd64", instruction: "VDPPS $1, Z0, Z1, Z2"}, + {goarch: "amd64", instruction: "VDPPS $1, X0, Y1, Y2"}, + {goarch: "386", instruction: "DPPD $1, X0, X8"}, + {goarch: "386", instruction: "VDPPS $1, Y0, Y1, Y2"}, + {goarch: "386", instruction: "VDPPS $1, Y0, Y1, Y8"}, + {goarch: "386", instruction: "VDPPS $1, 8(R9), Y1, Y2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside the Go table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64PackedFloatingDotRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT floatdotsemantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ a+8(FP), CX + MOVQ b+16(FP), DX + MOVUPS 0(CX), X0 + MOVUPS 0(DX), X1 + STC + DPPS $0xf1, X0, X1 + MOVUPS X1, 0(AX) + SETCS 16(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "floatdotsemantics": { + Name: "floatdotsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void floatdotsemantics(uint8_t *, const float *, const float *); +int main(void) { + uint8_t out[17] = {0}; + const float a[4] = {1.0f, 2.0f, 3.0f, 4.0f}; + const float b[4] = {5.0f, 6.0f, 7.0f, 8.0f}; + float got[4]; + floatdotsemantics(out, a, b); + memcpy(got, out, 16); + if (got[0] != 70.0f || got[1] != 0.0f || got[2] != 0.0f || got[3] != 0.0f) return 10; + if (out[16] != 1) return 11; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_float_dot_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_dot_product.go b/amd64_lower_dot_product.go new file mode 100644 index 000000000..619151280 --- /dev/null +++ b/amd64_lower_dot_product.go @@ -0,0 +1,185 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedDotProductSpec struct { + inputBits int + opcode byte + saturate bool + bfloat bool +} + +var amd64PackedDotProductSpecs = map[Op]amd64PackedDotProductSpec{ + "VPDPBUSD": {inputBits: 8, opcode: 0x50}, + "VPDPBUSDS": {inputBits: 8, opcode: 0x51, saturate: true}, + "VPDPWSSD": {inputBits: 16, opcode: 0x52}, + "VPDPWSSDS": {inputBits: 16, opcode: 0x53, saturate: true}, + "VDPBF16PS": {inputBits: 16, bfloat: true}, +} + +// lowerPackedDotProduct implements the four VNNI dot-product opcodes sharing +// Go 1.27's _yvblendmpd table. Plan 9 lists the signed r/m source first and +// the encoded V source second; VPDPBUSD treats that second source as unsigned. +func (c *amd64Ctx) lowerPackedDotProduct(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64PackedDotProductSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if !spec.bfloat { + return c.lowerIntegerPackedDotProduct(op, ins) + } + if spec.bfloat && !ins.x86Encoded { + return true, false, fmt.Errorf("%s has no named Go encoder form; raw encoding required", baseOp) + } + + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.sae || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's _yvblendmpd encodings: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects source2, source1, [K mask,] accumulator: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked && !ins.x86Encoded { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7 as its third operand: %q", baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s accumulator must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || + !c.isGoPackedVectorMoveRegister(destination, byteWidth) || + !c.isGoPackedVectorMoveRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("%s %s source1 and accumulator must be matching Go vector registers: %q", c.goarch, baseOp, ins.Raw) + } + rmSource := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(rmSource) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source2: %q", c.goarch, baseOp, ins.Raw) + } + } else if rmSource.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(rmSource, byteWidth) { + return true, false, fmt.Errorf("%s %s source2 register must match the accumulator width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(rmSource) { + return true, false, fmt.Errorf("%s %s source2 must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + dwordLanes := byteWidth / 4 + inputLanes := byteWidth * 8 / spec.inputBits + mask := "" + if masked { + mask, err = c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + } + // One mask bit controls a complete output dword and its input pair/group. + rmDwords, err := c.loadMaskedPackedCompareLanes(rmSource, byteWidth, 32, properties.broadcast, mask) + if err != nil { + return true, false, err + } + rmValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x i%d>\n", rmValue, dwordLanes, rmDwords, inputLanes, spec.inputBits) + vBytes, err := c.loadPackedCompareBytes(ins.Args[1], byteWidth) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + rm := "%" + rmValue + v := c.bitcastVectorBytesToIntegerLanes(byteWidth, inputLanes, spec.inputBits, vBytes) + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, dwordLanes, 32, oldBytes) + computed := c.emitPackedDotProduct(spec, dwordLanes, rm, v, old) + if masked { + computed = amd64ApplyI32LaneMask(c, dwordLanes, computed, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x i8>\n", out, dwordLanes, computed, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +// LLVM's dedicated intrinsic preserves the instruction's RNE, DAZ/FTZ and +// NaN-priority semantics independently of MXCSR. Generic fmul/fadd cannot. +func (c *amd64Ctx) emitBF16DotProduct(lanes int, rm, v, accumulator string) string { + left, right, old := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x bfloat>\n", left, lanes*2, v, lanes*2) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x bfloat>\n", right, lanes*2, rm, lanes*2) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x float>\n", old, lanes, accumulator, lanes) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <%d x float> @llvm.x86.avx512bf16.dpbf16ps.%d(<%d x float> %%%s, <%d x bfloat> %%%s, <%d x bfloat> %%%s)\n", + result, lanes, lanes*32, lanes, old, lanes*2, left, lanes*2, right) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x float> %%%s to <%d x i32>\n", bits, lanes, result, lanes) + return "%" + bits +} + +func (c *amd64Ctx) emitPackedDotProduct(spec amd64PackedDotProductSpec, dwordLanes int, rm, v, accumulator string) string { + if spec.bfloat { + return c.emitBF16DotProduct(dwordLanes, rm, v, accumulator) + } + inputsPerDword := 32 / spec.inputBits + result := "poison" + for lane := 0; lane < dwordLanes; lane++ { + accumulatorLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i32> %s, i32 %d\n", accumulatorLane, dwordLanes, accumulator, lane) + wideAccumulator := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i32 %%%s to i64\n", wideAccumulator, accumulatorLane) + total := "%" + wideAccumulator + for item := 0; item < inputsPerDword; item++ { + inputLane := lane*inputsPerDword + item + rmLane := c.newTmp() + vLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", rmLane, dwordLanes*inputsPerDword, spec.inputBits, rm, inputLane) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", vLane, dwordLanes*inputsPerDword, spec.inputBits, v, inputLane) + wideRM := c.newTmp() + wideV := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i%d %%%s to i64\n", wideRM, spec.inputBits, rmLane) + if spec.inputBits == 8 { + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", wideV, vLane) + } else { + fmt.Fprintf(c.b, " %%%s = sext i16 %%%s to i64\n", wideV, vLane) + } + product := c.newTmp() + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, %%%s\n", product, wideRM, wideV) + fmt.Fprintf(c.b, " %%%s = add i64 %s, %%%s\n", next, total, product) + total = "%" + next + } + if spec.saturate { + above := c.newTmp() + below := c.newTmp() + capped := c.newTmp() + bounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt i64 %s, 2147483647\n", above, total) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 2147483647, i64 %s\n", capped, above, total) + fmt.Fprintf(c.b, " %%%s = icmp slt i64 %%%s, -2147483648\n", below, capped) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 -2147483648, i64 %%%s\n", bounded, below, capped) + total = "%" + bounded + } + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", narrow, total) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i32> %s, i32 %%%s, i32 %d\n", inserted, dwordLanes, result, narrow, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_dot_product_test.go b/amd64_lower_dot_product_test.go new file mode 100644 index 000000000..230fe5b93 --- /dev/null +++ b/amd64_lower_dot_product_test.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedDotProductCompleteGoAssemblerForms(t *testing.T) { + // These four opcodes share Go 1.27's six-row _yvblendmpd table. Each + // width accepts register/memory source 2, register source 1, and an + // accumulator destination; amd64 additionally accepts K1-K7 merge/.Z. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + zLast := 22 + if target.goarch == "386" { + zLast = 7 + } + var source strings.Builder + source.WriteString("TEXT packeddotproductforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPDPBUSD", "VPDPBUSDS", "VPDPWSSD", "VPDPWSSDS"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s20, %s21\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST 12(AX), %s20, %s22\n", op, width, width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 8(AX), %s21, K7, %s22\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z 12(AX), %s21, K2, %s22\n", op, width, width) + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s.BCST 12(AX), Z6, Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K3, Z22\n", op) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z21, K4, Z22\n", op) + fmt.Fprintf(&source, "\t%s.BCST.Z 12(AX), Z21, K5, Z22\n", op) + } + } + source.WriteString("\tRET\n") + + // The original complete source has no data, accumulator, mask + // or pointer inputs. Preserve its Go and private emitter objects + // plus all three public Context controls; initialized public + // counterparts have separate complete-family owners. + ll := x86UnboundVoidScalarFormatObject(t, source.String(), "packeddotproductforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-dot-product-"+target.name+".ll", "packed-dot-product-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedDotProductRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPDPBUSD X0, X1", + "VPDPBUSDS X0, X1, (AX)", + "VPDPWSSD X0, Y1, X2", + "VPDPWSSDS Z0, Z1, Y2", + "VPDPBUSD X0, X1, K0, X2", + "VPDPBUSD X0, K1, X1, X2", + "VPDPBUSD.Z X0, X1, X2", + "VPDPBUSDS.SAE X0, X1, X2", + "VPDPWSSD.BCST X0, X1, X2", + "VPDPWSSDS.BCST 8(AX), Y1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedDotProductRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VPDPBUSD Z8, Z0, Z1", + "VPDPBUSDS Z0, Z8, Z1", + "VPDPWSSD Z0, Z1, Z8", + "VPDPWSSDS X0, X1, K1, X2", + "VPDPBUSD.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedDotProductRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedDotProductRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvblendmpd forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_duplicate.go b/amd64_lower_duplicate.go new file mode 100644 index 000000000..d51cc3902 --- /dev/null +++ b/amd64_lower_duplicate.go @@ -0,0 +1,120 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerDuplicateMove(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86DuplicateMoveProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + ins.Op = op + return true, false, c.emitDuplicateMove(ins, form) +} + +func (c *amd64Ctx) emitDuplicateMove(ins Instr, form x86DuplicateMoveForm) error { + destination := ins.Args[form.destination] + if form.legacy && form.spec.laneBytes == 4 && form.memory >= 0 { + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || + form.source.Kind == OpFP || form.source.Kind == OpMem && + !c.nativeMemoryOperand(ins, form.memory) && + !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + return fmt.Errorf("%w: legacy duplicate m128 needs its original native alignment and storage", ErrProbeNeedsContext) + } + return c.emitLegacyDuplicateMemory(ins.Op, form, destination) + } + laneBits := form.spec.laneBytes * 8 + lanes, sourceBytes := form.byteWidth/form.spec.laneBytes, form.byteWidth + if form.memory >= 0 { + sourceBytes = form.memoryBytes + } + bytes, err := c.loadDuplicateMoveSource(form) + if err != nil { + return err + } + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i%d>\n", cast, sourceBytes, bytes, + sourceBytes/form.spec.laneBytes, laneBits) + // X-width m64 DDUP's source has one lane; every other source is full + // width. Register and memory forms share exactly the same lane selector. + indices := make([]int, lanes) + for lane := range indices { + indices[lane] = form.sourceByte(lane*form.spec.laneBytes) / form.spec.laneBytes + } + shuffle := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %%%s, <%d x i%d> poison, <%d x i32> %s\n", + shuffle, sourceBytes/form.spec.laneBytes, laneBits, cast, + sourceBytes/form.spec.laneBytes, laneBits, lanes, llvmI32Mask(indices)) + result := "%" + shuffle + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, laneBits, false) + if err != nil { + return err + } + } + result = amd64ApplyIntegerLaneMask(c, lanes, laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, laneBits, result, form.byteWidth) + if form.legacy { + return c.storeLegacyPackedLogical(destination, "%"+out) + } + return c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) loadDuplicateMoveSource(form x86DuplicateMoveForm) (string, error) { + if form.source.Kind == OpReg { + return c.loadPackedCompareBytes(form.source, form.byteWidth) + } + if form.source.Kind == OpFP { + return c.loadFPVectorBytes(form.source.FPOffset, form.memoryBytes) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.source) + if err != nil { + return "", err + } + value := c.newTmp() + // E4NF.nb and E5NF retain the complete access even for a dead result or + // zero mask. No alignment or initialized-pointee assumption is introduced. + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, %s %s, align 1\n", value, form.memoryBytes, pointerType, pointer) + return "%" + value, nil +} + +func (c *amd64Ctx) emitLegacyDuplicateMemory(op Op, form x86DuplicateMoveForm, destination Operand) error { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.source) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + loaded, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(%s %s)\n", loaded, + strings.ToLower(string(op))+" "+segment+"($1), $0", "=x,r,~{memory}", pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <16 x i8>\n", bytes, loaded) + return c.storeLegacyPackedLogical(destination, "%"+bytes) +} + +func llvmI32Mask(indices []int) string { + var b strings.Builder + b.WriteByte('<') + for i, index := range indices { + if i != 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "i32 %d", index) + } + b.WriteByte('>') + return b.String() +} diff --git a/amd64_lower_enqueue.go b/amd64_lower_enqueue.go new file mode 100644 index 000000000..8e1c839f3 --- /dev/null +++ b/amd64_lower_enqueue.go @@ -0,0 +1,64 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var x86EnqueueOps = map[string]byte{ + "ENQCMD": 0xf2, + "ENQCMDS": 0xf3, +} + +// Keep the 64-byte device enqueue operation in hardware. An ordinary memory +// copy cannot implement PASID handling, atomic submission, or retry status. +func (c *amd64Ctx) lowerEnqueue(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86EnqueueProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + source := form.source + assemblySource := "$1" + if source.Kind == OpMem && source.Mem.Segment != "" { + switch source.Mem.Segment { + case FS, GS: + // LLVM inline-asm memory operands do not preserve the pointer's + // segment address space. Spell the prefix explicitly and pass + // its offset, not an ordinary unsegmented effective address. + assemblySource = "%" + strings.ToLower(string(source.Mem.Segment)) + ":$1" + source.Mem.Segment = "" + default: + return true, false, fmt.Errorf("unsupported enqueue source segment %s", source.Mem.Segment) + } + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(source) + if err != nil { + return true, false, err + } + wordType := amd64IntegerTypeForBits(form.wordBits) + destination, err := c.evalIntSized(form.destination, wordType) + if err != nil { + return true, false, err + } + if bits := form.addressBits; bits != form.wordBits { + // Truncate the effective offset before hardware applies any explicit + // source-segment override. + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint %s %s to i%d\n", address, ptrType, ptr, bits) + wrapped := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr i%d %%%s to %s\n", wrapped, bits, address, ptrType) + ptr = "%" + wrapped + low, extended := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to i%d\n", low, wordType, destination, bits) + fmt.Fprintf(c.b, " %%%s = zext i%d %%%s to %s\n", extended, bits, low, wordType) + destination = "%" + extended + } + + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i8 asm sideeffect %q, %q(%s elementtype([64 x i8]) %s, %s %s)\n", + status, form.native+" "+assemblySource+", $2; sete $0", + "=q,*m,r,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, ptr, wordType, destination) + c.storeSegmentQueryZF("%" + status) + c.clearX86StatusFlags(x86StatusAll &^ x86StatusZF) + return true, false, nil +} diff --git a/amd64_lower_exchange.go b/amd64_lower_exchange.go new file mode 100644 index 000000000..61e07bd26 --- /dev/null +++ b/amd64_lower_exchange.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ExchangeKind uint8 + +const ( + amd64ExchangeSwap amd64ExchangeKind = iota + amd64ExchangeAdd +) + +type amd64ExchangeSpec struct { + kind amd64ExchangeKind + bits int +} + +// The complete Go yrb_mb/yrl_ml (XADD) and yml_mb/yxchg (XCHG) families. +// Width selects register classes; kind selects operand symmetry and flags. +var amd64ExchangeSpecs = map[Op]amd64ExchangeSpec{ + "XADDB": {amd64ExchangeAdd, 8}, + "XADDW": {amd64ExchangeAdd, 16}, + "XADDL": {amd64ExchangeAdd, 32}, + "XADDQ": {amd64ExchangeAdd, 64}, + "XCHGB": {amd64ExchangeSwap, 8}, + "XCHGW": {amd64ExchangeSwap, 16}, + "XCHGL": {amd64ExchangeSwap, 32}, + "XCHGQ": {amd64ExchangeSwap, 64}, +} + +type amd64ExchangeForm struct { + register Operand + other Operand + nop bool +} + +// The swap consumers use the same complete Go grammar as the emitter. Width +// and operand symmetry come from the existing exchange family, not a second +// opcode directory in each provenance or byte observer. +func x86SwapProperties(ins Instr, arch string) (amd64ExchangeForm, int64, bool, error) { + name := strings.ToUpper(string(ins.Op)) + base, _, _ := strings.Cut(name, ".") + spec, known := amd64ExchangeSpecs[Op(base)] + if !known || spec.kind != amd64ExchangeSwap { + return amd64ExchangeForm{}, 0, false, nil + } + form, err := parseAMD64ExchangeForm(arch, Op(name), spec, ins) + return form, int64(spec.bits / 8), true, err +} + +func parseAMD64ExchangeForm(goarch string, op Op, spec amd64ExchangeSpec, ins Instr) (amd64ExchangeForm, error) { + invalid := func(reason string) (amd64ExchangeForm, error) { + return amd64ExchangeForm{}, fmt.Errorf("%s %s %s: %q", goarch, op, reason, ins.Raw) + } + if strings.Contains(string(op), ".") { + return invalid("does not accept instruction suffixes") + } + if registered, ok := amd64ExchangeSpecs[op]; !ok || registered != spec || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return invalid("does not have its registered source identity and width") + } + if goarch != "amd64" && goarch != "386" { + return invalid("requires a registered x86 mode") + } + if goarch == "386" && spec.bits == 64 { + return invalid("is illegal in 32-bit mode") + } + if len(ins.Args) != 2 { + return invalid("expects two operands") + } + first, second := ins.Args[0], ins.Args[1] + if spec.kind == amd64ExchangeSwap && isAMD64MemoryOperand(first) { + first, second = second, first + } + if projected, segment, err := x86NamedSegmentMemorySource(ins, second, goarch); segment { + if err != nil { + return amd64ExchangeForm{}, err + } + second = projected + } + registerAllowed := func(reg Reg) bool { + if spec.bits == 8 { + return isGoYmbRegisterForArch(reg, goarch) + } + return isX86YrlRegisterForArch(reg, goarch) + } + if first.Kind != OpReg || !registerAllowed(first.Reg) { + return invalid("requires a source in Go's byte/general-register class") + } + if second.Kind == OpReg { + if !registerAllowed(second.Reg) { + return invalid("destination is outside Go's byte/general-register class") + } + } else if !isAMD64MemoryOperand(second) { + return invalid("expects a register or memory destination") + } else if second.Kind == OpMem && !x86MemoryRegistersValidForArch(second.Mem, goarch) { + return invalid("uses an out-of-range address register") + } + form := amd64ExchangeForm{register: first, other: second} + if goarch == "amd64" && spec.bits == 8 && x86CompareExchangeHasREXByteView(x86CompareExchangeForm{ + source: first, destination: second, + }) { + for _, operand := range []Operand{first, second} { + if operand.Kind == OpReg && (operand.Reg == AH || operand.Reg == CH || operand.Reg == DH || operand.Reg == BH) { + return amd64ExchangeForm{}, fmt.Errorf("%w: %s %s Go's REX encoding changes a high-byte spelling to an unbound low-byte physical view: %q", ErrProbeNeedsContext, goarch, op, ins.Raw) + } + } + } + if spec.kind == amd64ExchangeSwap { + // Go's accumulator/accumulator W/L/Q rows emit opcode 90 (with + // their width prefix), a NOP. In particular, EAX/EAX does not clear + // the upper RAX bytes as a genuine ModRM r32 exchange would. + form.nop = spec.bits != 8 && first.Kind == OpReg && second.Kind == OpReg && first.Reg == AX && second.Reg == AX + } + return form, nil +} + +func (c *amd64Ctx) lowerExchange(op Op, spec amd64ExchangeSpec, ins Instr) (bool, bool, error) { + form, err := parseAMD64ExchangeForm(c.goarch, op, spec, ins) + if err != nil { + return true, false, err + } + if form.nop { + return true, false, nil + } + typ := amd64IntegerTypeForBits(spec.bits) + value, err := c.evalIntSized(form.register, typ) + if err != nil { + return true, false, err + } + var old string + if form.other.Kind == OpReg { + old, err = c.evalIntSized(form.other, typ) + } else { + old, err = c.emitExchangeMemory(spec, form.other, value) + } + if err != nil { + return true, false, err + } + result := value + if spec.kind == amd64ExchangeAdd { + sum := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", sum, typ, old, value) + result = "%" + sum + } + // Read both operands before writeback. Source-first order preserves identical + // register XADD and overlapping AH/AL exchanges, including partial writes. + if err := c.storeRegSized(form.register.Reg, typ, old); err != nil { + return true, false, err + } + if form.other.Kind == OpReg { + if err := c.storeRegSized(form.other.Reg, typ, result); err != nil { + return true, false, err + } + } + if spec.kind == amd64ExchangeAdd { + c.setScalarAddSubFlags(typ, true, old, value, result) + } + return true, false, nil +} + +func (c *amd64Ctx) emitExchangeMemory(spec amd64ExchangeSpec, memory Operand, value string) (string, error) { + ptr, ptrType, err := c.compareExchangePointer(memory) + if err != nil { + return "", err + } + typ := amd64IntegerTypeForBits(spec.bits) + width := map[int]string{8: "b", 16: "w", 32: "l", 64: "q"}[spec.bits] + mnemonic := "xchg" + width // A memory XCHG is implicitly locked. + if spec.kind == amd64ExchangeAdd { + mnemonic = "xadd" + width + if c.namedAtomicLock { + mnemonic = "lock; " + mnemonic + } + } + registerClass := "r" + if spec.bits == 8 { + registerClass = "q" + } + segment := "" + if memory.Kind == OpMem && memory.Mem.Segment != "" { + // LLVM's inline-asm memory operand does not print the address space's + // FS/GS override; preserve it explicitly in the instruction template. + segment = "%" + strings.ToLower(string(memory.Mem.Segment)) + ":" + } + // Native x86 exchanges support unaligned memory. LLVM atomicrmw align 1 + // can introduce libatomic calls, while claiming natural alignment is false. + // Explicit read/write memory constraints and the tied register model the + // native operation without imposing an alignment precondition. Keep memory + // clobbering for synchronization and preserve segment-relative pointer types. + old := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect \"%s $0, %s$1\", \"=%s,=*m,0,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %s, %s %s, %s elementtype(%s) %s)\n", + old, typ, mnemonic, segment, registerClass, ptrType, typ, ptr, typ, value, ptrType, typ, ptr) + return "%" + old, nil +} diff --git a/amd64_lower_expand.go b/amd64_lower_expand.go new file mode 100644 index 000000000..70c9743ec --- /dev/null +++ b/amd64_lower_expand.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64PackedExpandSpec struct { + laneBits int +} + +// amd64PackedExpandSpecs is the complete Go 1.27 _yvexpandpd grammar for the +// packed floating-point and integer expand families. None of these optabs has +// an EVEX broadcast attribute. +var amd64PackedExpandSpecs = map[Op]amd64PackedExpandSpec{ + "VEXPANDPD": {laneBits: 64}, + "VEXPANDPS": {laneBits: 32}, + "VPEXPANDB": {laneBits: 8}, + "VPEXPANDW": {laneBits: 16}, + "VPEXPANDD": {laneBits: 32}, + "VPEXPANDQ": {laneBits: 64}, +} + +// lowerPackedExpand implements all six _yvexpandpd rows for every instruction +// above: X/Y/Z register or memory sources and the corresponding K1-K7 merge or +// .Z forms. Logical FP fields are reconstructed independently; only native +// memory uses a pointer to the compact source array. +func (c *amd64Ctx) lowerPackedExpand(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86PackedCompactProperties(op, ins, c.goarch) + if !known || !form.expand { + return false, false, nil + } + if err != nil { + return true, false, err + } + if form.memory >= 0 && c.nativeMemoryOperand(ins, form.memory) { + return true, false, c.lowerNativeCompactMemory(ins, form) + } + byteWidth, laneBits := form.byteWidth, form.laneBits + source, destination := ins.Args[0], ins.Args[form.destination] + if form.mask < 0 { + sourceBytes, err := c.loadPackedCompareBytes(source, byteWidth) + if err != nil { + return true, false, err + } + return true, false, c.storePackedMoveOperand(destination, byteWidth, sourceBytes) + } + + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / laneBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + maskVector := amd64IntegerMaskVector(c, lanes, mask) + passthrough := "zeroinitializer" + if !form.zeroing { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + passthrough = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, oldBytes) + } + + result := "" + if isAMD64MemoryOperand(source) && source.Kind != OpFP { + pointer, pointerType, intrinsicSuffix, err := c.packedVectorMemoryPointer(source) + if err != nil { + return true, false, err + } + expanded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.masked.expandload.v%di%d%s(%s %s, <%d x i1> %s, %s %s)\n", + expanded, vectorType, lanes, laneBits, intrinsicSuffix, pointerType, pointer, lanes, maskVector, vectorType, passthrough) + result = "%" + expanded + } else { + sourceBytes, err := c.loadPackedCompareBytes(source, byteWidth) + if err != nil { + return true, false, err + } + sourceLanes := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, sourceBytes) + result = c.expandPackedRegisterLanes(lanes, laneBits, sourceLanes, passthrough, mask, maskVector) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, vectorType, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +// expandPackedRegisterLanes is the register-source inverse of vector +// compression. For destination lane i, the compact source index is the number +// of set mask bits below i. Building each lane from the original source before +// the final store keeps source/destination aliasing safe. +func (c *amd64Ctx) expandPackedRegisterLanes(lanes, laneBits int, source, passthrough, mask, maskVector string) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + result := passthrough + for lane := 0; lane < lanes; lane++ { + active := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i1> %s, i32 %d\n", active, lanes, maskVector, lane) + sourceIndex := "0" + if lane != 0 { + below := c.newTmp() + count := c.newTmp() + index := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", below, mask, (uint64(1)< +#include +extern void packedexpandsemantics(uint8_t *, const uint8_t *, const uint8_t *, uint64_t, const uint8_t *); +static int check(const uint8_t *got, const uint8_t *source, const uint8_t *old, + int lane_bytes, uint64_t mask, int zeroing) { + int lanes = 64 / lane_bytes, packed = 0; + for (int lane = 0; lane < lanes; lane++) { + const uint8_t *want = zeroing ? (const uint8_t[8]){0} : old + lane*lane_bytes; + if ((mask >> lane) & 1) want = source + packed++*lane_bytes; + if (memcmp(got + lane*lane_bytes, want, lane_bytes)) return 10 + lane; + } + return 0; +} +int main(void) { + uint8_t source[64], old[64], out[513]; + const uint64_t mask = UINT64_C(0xa55aa55aa55aa55a); + for (int i = 0; i < 64; i++) { source[i] = (uint8_t)(i*37 + 9); old[i] = (uint8_t)(0xe0 - i); } + memset(out, 0xcc, sizeof(out)); + packedexpandsemantics(out, source, old, mask, 0); + int r; + if ((r = check(out+0, source, old, 1, mask, 0))) return 100+r; + if ((r = check(out+64, source, old, 2, mask, 1))) return 200+r; + if ((r = check(out+128, source, old, 4, mask, 0))) return 300+r; + if ((r = check(out+192, source, old, 8, mask, 1))) return 400+r; + if ((r = check(out+256, source, old, 4, mask, 0))) return 500+r; + if ((r = check(out+320, source, old, 8, mask, 1))) return 600+r; + if ((r = check(out+384, source, source, 4, mask, 0))) return 700+r; + if (memcmp(out+448, old, 64)) return 800; + if (out[512] != 1) return 801; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_expand_semantics", triple, ir, mainC, nil) +} diff --git a/amd64_lower_explicit_mask_move.go b/amd64_lower_explicit_mask_move.go new file mode 100644 index 000000000..9397d3850 --- /dev/null +++ b/amd64_lower_explicit_mask_move.go @@ -0,0 +1,163 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerExplicitMaskMove(op Op, ins Instr) (bool, bool, error) { + f, known, err := x86ExplicitMaskMoveProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := x86Mode32VectorRegisterContext(ins, c.goarch); err != nil { + return true, false, err + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + !x86ResultNamedPredicateInstruction(ins) && !x86RawExplicitMaskMoveSource(ins, c.goarch) && + ins.x86NumericalRead == nil && ins.x86SDKLiteral == nil { + return true, false, fmt.Errorf("%w: explicit mask move needs its exact physical source", ErrProbeNeedsContext) + } + if ins.Args[f.memory].Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(ins.Args[f.memory].FPOffset, f.byteWidth); err != nil { + return true, false, err + } + } else if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) && + (c.nativeMemoryOperand(ins, f.memory) || + x86CallerMemoryCertified(ins, f.memory) || c.fixedLocalMemoryCertified(ins, f.memory)) { + return true, false, c.lowerNativeExplicitMaskMove(op, ins, f) + } + return true, false, c.lowerPortableExplicitMaskMove(ins, f) +} + +// Type 6 masks suppress only the corresponding lane's memory faults. Preserve +// the whole original VEX operation even with a dead output. The pointer stays +// opaque: no alignment, extent, initialization or pointee promise is asserted. +// Intel SDM Vol.2C, VMASKMOV/VPMASKMOV: no SIMD FP exceptions and no AC#. +// https://cdrdv2-public.intel.com/835757/325383-sdm-vol-2abcd.pdf +func (c *amd64Ctx) lowerNativeExplicitMaskMove(op Op, ins Instr, f x86ExplicitMaskMoveForm) error { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(ins.Args[f.memory]) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + typ := fmt.Sprintf("<%d x float>", f.byteWidth/4) // Bit carrier, not FP arithmetic. + vector := func(operand Operand) (string, error) { + bytes, err := c.loadPackedCompareBytes(operand, f.byteWidth) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", value, f.byteWidth, bytes, typ) + return "%" + value, nil + } + mask, err := vector(ins.Args[1]) + if err != nil { + return err + } + mnemonic := strings.ToLower(string(op)) + if !f.load { + data, err := vector(ins.Args[0]) + if err != nil { + return err + } + assembly := mnemonic + " $0, $1, " + segment + "($2)" + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s, %s %s, %s %s)\n", + assembly, "x,x,r,~{memory}", typ, data, typ, mask, pointerType, pointer) + return nil + } + loaded, bytes := c.newTmp(), c.newTmp() + assembly := mnemonic + " " + segment + "($2), $1, $0" + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %s, %s %s)\n", + loaded, typ, assembly, "=x,x,r,~{memory}", typ, mask, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, typ, loaded, f.byteWidth) + return c.storePackedMoveOperand(ins.Args[2], f.byteWidth, "%"+bytes) +} + +// The other-target/value-field path has guarded volatile lane accesses. This +// is portable access/value evidence, not execution of the original x86 opcode. +// An inactive lane must not even eagerly read the old memory contents. +func (c *amd64Ctx) lowerPortableExplicitMaskMove(ins Instr, f x86ExplicitMaskMoveForm) error { + bits, width := f.spec.laneBits, f.byteWidth + lanes := width * 8 / bits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, bits) + mask, err := c.loadPackedCompareLanes(ins.Args[1], width, bits, false) + if err != nil { + return err + } + value := "zeroinitializer" + if !f.load { + value, err = c.loadPackedCompareLanes(ins.Args[0], width, bits, false) + if err != nil { + return err + } + } + memory := ins.Args[f.memory] + pointer, pointerType := "", "" + if memory.Kind != OpFP { + pointer, pointerType, _, err = c.packedVectorMemoryPointer(memory) + if err != nil { + return err + } + } + for lane := 0; lane < lanes; lane++ { + element, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", element, vectorType, mask, lane) + fmt.Fprintf(c.b, " %%%s = icmp slt i%d %%%s, 0\n", active, bits, element) + field := memory + address := "" + if memory.Kind == OpFP { + field.FPOffset += int64(lane * bits / 8) + } else { + address = c.newTmp() + // No inbounds: masked-off null and page-boundary lanes need no allocation. + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %d\n", address, pointerType, pointer, lane*bits/8) + } + if f.load { + loaded, err := c.loadX86ScalarIf(bits, "%"+active, func() (string, error) { + loaded := c.newTmp() + if memory.Kind == OpFP { + bytes, err := c.loadX86PackedShiftBytes(field, bits/8, true) + if err != nil { + return "", err + } + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", loaded, bits/8, bytes, bits) + } else { + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %%%s, align 1\n", loaded, bits, pointerType, address) + } + return "%" + loaded, nil + }) + if err != nil { + return err + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, value, bits, loaded, lane) + value = "%" + inserted + continue + } + data, store, join := c.newTmp(), c.newTmp()+"_mask_store", c.newTmp()+"_mask_join" + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", data, vectorType, value, lane) + fmt.Fprintf(c.b, " br i1 %%%s, label %%%s, label %%%s\n%s:\n", active, store, join, store) + if memory.Kind == OpFP { + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %%%s to <%d x i8>\n", bytes, bits, data, bits/8) + if err := c.storeVectorBytesOperand(field, bits/8, "%"+bytes); err != nil { + return err + } + } else { + fmt.Fprintf(c.b, " store volatile i%d %%%s, %s %%%s, align 1\n", bits, data, pointerType, address) + } + fmt.Fprintf(c.b, " br label %%%s\n%s:\n", join, join) + } + if !f.load { + return nil + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytes, vectorType, value, width) + return c.storePackedMoveOperand(ins.Args[2], width, "%"+bytes) +} diff --git a/amd64_lower_explicit_mask_move_test.go b/amd64_lower_explicit_mask_move_test.go new file mode 100644 index 000000000..ff689bad7 --- /dev/null +++ b/amd64_lower_explicit_mask_move_test.go @@ -0,0 +1,227 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64ExplicitMaskMoveGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64ExplicitMaskMoveSpec{ + "VMASKMOVPS": {laneBits: 32}, + "VMASKMOVPD": {laneBits: 64}, + "VPMASKMOVD": {laneBits: 32}, + "VPMASKMOVQ": {laneBits: 64}, + } + if len(amd64ExplicitMaskMoveSpecs) != len(expected) { + t.Fatalf("explicit-mask-move grammar has %d entries, want %d", len(amd64ExplicitMaskMoveSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64ExplicitMaskMoveSpecs[op]; !ok { + t.Errorf("explicit-mask-move grammar omitted %s", op) + } else if got != want { + t.Errorf("explicit-mask-move grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86ExplicitMaskMoveCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT explicitmaskmoveforms(SB),$0-0\n") + for _, op := range []string{"VMASKMOVPS", "VMASKMOVPD", "VPMASKMOVD", "VPMASKMOVQ"} { + for _, width := range []string{"X", "Y"} { + last := 15 + fmt.Fprintf(&source, "\t%s 8(AX), %s1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s %s%d, %s1, 16(AX)\n", op, width, last, width) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + signatures := map[string]FuncSig{"explicitmaskmoveforms": {Name: "explicitmaskmoveforms", Ret: Void}} + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: signatures, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound original formats acquired mask/data/address inputs", err) + } + if target.goarch == "386" { + // Keep every original accepted high-bank named row. Its mode32 + // physical-bank boundary is not an object success; independently + // initialized low-bank owners cover all sixteen rows on both + // 386 targets without changing this original source. + for _, ins := range file.Funcs[0].Instrs { + if _, known, _ := x86ExplicitMaskMoveProperties(ins.Op, ins, target.goarch); known && + !errors.Is(x86Mode32VectorRegisterContext(ins, target.goarch), ErrProbeNeedsContext) { + t.Fatal("original accepted high-bank named row lost its physical mode32 boundary") + } + } + return + } + // Direct units preserve every original format and exact native + // opcode, but supply neither public input nor execution evidence. + ir = x86ScalarFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0) + for _, op := range []string{"vmaskmovps ", "vmaskmovpd ", "vpmaskmovd ", "vpmaskmovq "} { + if !strings.Contains(ir, op) || !strings.Contains(ir, "asm sideeffect") { + t.Fatal("original direct format unit lost its complete Type6 native opcode", op) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "explicit-mask-move-"+target.name+".ll", "explicit-mask-move-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ExplicitMaskMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPMASKMOVD (AX), X0"}, + {goarch: "amd64", instruction: "VPMASKMOVD X0, X1, X2"}, + {goarch: "amd64", instruction: "VPMASKMOVD (AX), X0, (BX)"}, + {goarch: "amd64", instruction: "VPMASKMOVD X0, (AX), X1"}, + {goarch: "amd64", instruction: "VPMASKMOVD (AX), X0, Y1"}, + {goarch: "amd64", instruction: "VPMASKMOVD (AX), Z0, Z1"}, + {goarch: "amd64", instruction: "VPMASKMOVD.Z (AX), X0, X1"}, + {goarch: "amd64", instruction: "VMASKMOVPD (AX), X0, AX"}, + {goarch: "386", instruction: "VPMASKMOVQ (AX), X16, X0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's explicit-mask-move table", test.instruction) + } + }) + } +} + +func TestAMD64ExplicitMaskMoveRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT explicitmaskmovesemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ loadmem+8(FP), BX + MOVQ data+16(FP), CX + MOVQ mask+24(FP), DX + MOVQ storemem+32(FP), R8 + VMOVDQU 0(CX), Y0 + VMOVDQU 0(DX), Y1 + STC + VPMASKMOVD 0(BX), Y1, Y2 + VMOVDQU Y2, 0(AX) + VPMASKMOVD Y0, Y1, 0(R8) + VPMASKMOVQ 0(BX), Y1, Y3 + VMOVDQU Y3, 32(AX) + VPMASKMOVQ Y0, Y1, 32(R8) + VMASKMOVPS 0(BX), Y1, Y4 + VMOVDQU Y4, 64(AX) + VMASKMOVPS Y0, Y1, 64(R8) + VMASKMOVPD 0(BX), Y1, Y5 + VMOVDQU Y5, 96(AX) + VMASKMOVPD Y0, Y1, 96(R8) + SETCS 128(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"explicitmaskmovesemantics": { + Name: "explicitmaskmovesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void explicitmaskmovesemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint8_t *); +static uint32_t u32(const uint8_t *p) { uint32_t v; memcpy(&v,p,4); return v; } +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint32_t load32[8], data32[8], mask32[8]; + uint8_t out[129]={0}, store[128]; + for (int i=0;i<8;i++) { + load32[i]=UINT32_C(0x10203040)+(uint32_t)i; + data32[i]=UINT32_C(0xa0b0c000)+(uint32_t)i; + mask32[i]=(i==1||i==2||i==5)?UINT32_C(0x80000000):UINT32_C(0x7fffffff); + } + memset(store,0x5a,sizeof(store)); + explicitmaskmovesemantics(out,(uint8_t*)load32,(uint8_t*)data32,(uint8_t*)mask32,store); + for (int i=0;i<8;i++) { + int enabled=(int32_t)mask32[i]<0; + if (u32(out+4*i)!=(enabled?load32[i]:0)) return 10; + if (u32(store+4*i)!=(enabled?data32[i]:UINT32_C(0x5a5a5a5a))) return 11; + if (u32(out+64+4*i)!=(enabled?load32[i]:0)) return 12; + if (u32(store+64+4*i)!=(enabled?data32[i]:UINT32_C(0x5a5a5a5a))) return 13; + } + for (int i=0;i<4;i++) { + int enabled=(int64_t)u64((uint8_t*)mask32+8*i)<0; + if (u64(out+32+8*i)!=(enabled?u64((uint8_t*)load32+8*i):0)) return 20; + if (u64(store+32+8*i)!=(enabled?u64((uint8_t*)data32+8*i):UINT64_C(0x5a5a5a5a5a5a5a5a))) return 21; + if (u64(out+96+8*i)!=(enabled?u64((uint8_t*)load32+8*i):0)) return 22; + if (u64(store+96+8*i)!=(enabled?u64((uint8_t*)data32+8*i):UINT64_C(0x5a5a5a5a5a5a5a5a))) return 23; + } + if (out[128]!=1) return 30; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "explicit_mask_move_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_extract.go b/amd64_lower_extract.go new file mode 100644 index 000000000..e48dae899 --- /dev/null +++ b/amd64_lower_extract.go @@ -0,0 +1,45 @@ +package plan9asm + +import "fmt" + +// lowerVectorLaneExtract implements all Go 1.27 forms in _yvextractf128, +// _yvextractf32x4, and _yvextractf32x8 for their F/I lane-extract mnemonics. +func (c *amd64Ctx) lowerVectorLaneExtract(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerInsertExtractFamily(op, ins) +} + +func (c *amd64Ctx) storeVectorBytesOperand(dst Operand, byteWidth int, value string) error { + return c.storeVectorBytesOperandWithMetadata(dst, byteWidth, value, "") +} + +func (c *amd64Ctx) storeVectorBytesOperandWithMetadata(dst Operand, byteWidth int, value, metadata string) error { + if dst.Kind == OpReg { + if metadata != "" { + return fmt.Errorf("vector register destination cannot carry store metadata: %s", dst.String()) + } + if !amd64VectorRegisterHasWidth(dst, byteWidth) { + return fmt.Errorf("expected %d-byte vector register, got %s", byteWidth, dst.String()) + } + return c.storeVectorBytes(dst.Reg, byteWidth, value) + } + switch dst.Kind { + case OpMem: + addr, err := c.addrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <%d x i8> %s, ptr %s, align 1%s\n", byteWidth, value, c.ptrFromAddrI64(addr), metadata) + return nil + case OpSym: + p, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <%d x i8> %s, ptr %s, align 1%s\n", byteWidth, value, p, metadata) + return nil + case OpFP: + return c.storeFPVectorBytes(dst.FPOffset, byteWidth, value, metadata) + default: + return fmt.Errorf("expected vector register or memory destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_extract_scalar.go b/amd64_lower_extract_scalar.go new file mode 100644 index 000000000..af7d9471d --- /dev/null +++ b/amd64_lower_extract_scalar.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedScalarExtractSpec struct { + bits int + vector bool + maxImmediate int64 + allowMMX bool +} + +var amd64PackedScalarExtractSpecs = map[Op]amd64PackedScalarExtractSpec{ + "EXTRACTPS": {bits: 32, maxImmediate: 3}, + "VEXTRACTPS": {bits: 32, vector: true, maxImmediate: 255}, + "PEXTRB": {bits: 8, maxImmediate: 255, allowMMX: true}, + "PEXTRW": {bits: 16, maxImmediate: 255}, + "PEXTRD": {bits: 32, maxImmediate: 255, allowMMX: true}, + "PEXTRQ": {bits: 64, maxImmediate: 255, allowMMX: true}, + "VPEXTRB": {bits: 8, vector: true, maxImmediate: 255}, + "VPEXTRW": {bits: 16, vector: true, maxImmediate: 255}, + "VPEXTRD": {bits: 32, vector: true, maxImmediate: 255}, + "VPEXTRQ": {bits: 64, vector: true, maxImmediate: 255}, +} + +func (c *amd64Ctx) lowerPackedScalarExtract(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64PackedScalarExtractSpecs[Op(base)]; !known { + return false, false, nil + } + var f x86ScalarLaneForm + if base == "EXTRACTPS" || base == "VEXTRACTPS" { + var ps x86InsertExtractForm + ins, ps, _, err = c.bindInsertExtractForm(op, ins) + f = x86ScalarLaneForm{bits: 32, gpBits: 32, source: ps.source, destination: ps.destination, offset: ps.offset, memory: ps.memory} + if len(ins.Args) == 3 && ins.Args[2].Kind == OpReg { + f.transferReg = ins.Args[2].Reg + } + } else { + ins, f, _, err = c.bindScalarLaneForm(op, ins) + } + if err != nil { + return true, false, err + } + destination := ins.Args[f.destination] + if (base == "EXTRACTPS" || base == "VEXTRACTPS") && destination.Kind == OpFP { + // Complete destination fields never initialize unknown source lanes. + // Check before emitting any IR, including the source-register load. + if _, err := c.scalarFPSpan(destination.FPOffset, 32); err != nil { + return true, false, err + } + if err := c.scalarFPResultInitialized(); err != nil { + return true, false, err + } + } + + source, err := c.loadX(ins.Args[f.source].Reg) + if err != nil { + return true, false, err + } + lanes := 128 / f.bits + laneVector, extracted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <%d x i%d>\n", laneVector, source, lanes, f.bits) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %%%s, i32 %d\n", extracted, lanes, f.bits, laneVector, f.offset/(f.bits/8)) + value := "%" + extracted + if destination.Kind == OpReg { + destination.Reg = f.transferReg + if f.gpBits > f.bits { + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %s to i%d\n", widened, f.bits, value, f.gpBits) + value = "%" + widened + } else if f.gpBits < f.bits { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i%d %s to i%d\n", truncated, f.bits, value, f.gpBits) + value = "%" + truncated + } + return true, false, c.storeRegSized(destination.Reg, amd64IntegerTypeForBits(f.gpBits), value) + } + if (base == "EXTRACTPS" || base == "VEXTRACTPS") && destination.Kind == OpFP { + // Parameter copies and result fields share the existing logical byte + // writer. Neither source syntax nor a complete destination makes an + // unknown selected X lane initialized, or a pointer/padding numeric. + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %s to <4 x i8>\n", bytes, value) + return true, false, c.storeFPVectorBytes(destination.FPOffset, 4, "%"+bytes, "") + } + return true, false, c.storePackedScalarExtractMemory(destination, amd64IntegerTypeForBits(f.bits), value) +} + +func amd64MMXEncodingGP(index int) Reg { + return [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI}[index] +} + +func (c *amd64Ctx) storePackedScalarExtractMemory(destination Operand, typ LLVMType, value string) error { + bits := map[LLVMType]int{I8: 8, I16: 16, I32: 32, I64: 64}[typ] + if destination.Kind == OpFP { + return c.storeScalarLaneFPBits(destination.FPOffset, bits, value) + } + return c.storeVMOVIntegerScalar(destination, bits, value) +} diff --git a/amd64_lower_far_pointer.go b/amd64_lower_far_pointer.go new file mode 100644 index 000000000..95b71a520 --- /dev/null +++ b/amd64_lower_far_pointer.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FarPointerSpec struct { + segment Reg + bits int +} + +// amd64FarPointerSpecs is the complete Go 1.27 LFS/LGS/LSS grammar. Segment +// selection and offset width are data; memory shape, destination class, and +// architecture availability are parsed once for all nine opcodes. +var amd64FarPointerSpecs = map[Op]amd64FarPointerSpec{ + "LFSW": {segment: FS, bits: 16}, + "LFSL": {segment: FS, bits: 32}, + "LFSQ": {segment: FS, bits: 64}, + "LGSW": {segment: GS, bits: 16}, + "LGSL": {segment: GS, bits: 32}, + "LGSQ": {segment: GS, bits: 64}, + "LSSW": {segment: SS, bits: 16}, + "LSSL": {segment: SS, bits: 32}, + "LSSQ": {segment: SS, bits: 64}, +} + +type amd64FarPointerForm struct { + memory Operand + destination Reg +} + +func (c *amd64Ctx) lowerFarPointer(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64FarPointerSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.bits == 64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + form, err := c.parseFarPointerForm(baseOp, ins) + if err != nil { + return true, false, err + } + return c.emitFarPointer(spec, form) +} + +func (c *amd64Ctx) parseFarPointerForm(op string, ins Instr) (amd64FarPointerForm, error) { + var form amd64FarPointerForm + if len(ins.Args) != 2 { + return form, fmt.Errorf("%s %s expects far-pointer memory and GP destination: %q", c.goarch, op, ins.Raw) + } + memory, destination := ins.Args[0], ins.Args[1] + if !isAMD64MemoryOperand(memory) { + return form, fmt.Errorf("%s %s source must be Go 1.27 Ym memory: %q", c.goarch, op, ins.Raw) + } + if memory.Kind == OpMem && !x86MemoryRegistersValidForArch(memory.Mem, c.goarch) { + return form, fmt.Errorf("%s %s source address uses an out-of-range register: %q", c.goarch, op, ins.Raw) + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return form, fmt.Errorf("%s %s destination must be a Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + return amd64FarPointerForm{memory: memory, destination: destination.Reg}, nil +} + +func (c *amd64Ctx) emitFarPointer(spec amd64FarPointerSpec, form amd64FarPointerForm) (bool, bool, error) { + pointer, pointerType, err := c.x86DescriptorMemoryPointer(form.memory) + if err != nil { + return true, false, err + } + typ := amd64IntegerTypeForBits(spec.bits) + memoryBytes := spec.bits/8 + 2 + width := map[int]string{16: "w", 32: "l", 64: "q"}[spec.bits] + result := c.newTmp() + assembly := "l" + strings.ToLower(string(spec.segment)) + width + " $1, $0" + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s elementtype([%d x i8]) %s)\n", + result, typ, assembly, "=r,*m,~{memory},~{dirflag},~{fpsr},~{flags}", pointerType, memoryBytes, pointer) + if err := c.storeRegSized(form.destination, typ, "%"+result); err != nil { + return true, false, err + } + return true, false, nil +} diff --git a/amd64_lower_far_pointer_test.go b/amd64_lower_far_pointer_test.go new file mode 100644 index 000000000..45e8b3f64 --- /dev/null +++ b/amd64_lower_far_pointer_test.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "reflect" + "sort" + "strings" + "testing" +) + +func TestAMD64FarPointerGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64FarPointerSpecs)) + for op := range amd64FarPointerSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{"LFSL", "LFSQ", "LFSW", "LGSL", "LGSQ", "LGSW", "LSSL", "LSSQ", "LSSW"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("far-pointer grammar opcodes = %v, want %v", got, want) + } +} + +func TestTranslateX86FarPointerCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + destination := "DI" + base := "BX" + if target.goarch == "amd64" { + destination = "R12" + base = "R13" + } + source := "TEXT farpointerforms(SB),$0-0\n" + for _, family := range []string{"LFS", "LGS", "LSS"} { + source += "\t" + family + "W 0(BX), AX\n" + source += "\t" + family + "W 2(" + base + "), " + destination + "\n" + source += "\t" + family + "L 4(BX), DX\n" + if target.goarch == "amd64" { + source += "\t" + family + "Q 8(" + base + "), " + destination + "\n" + } else { + source += "\t" + family + "L 8(BX), SP\n" + } + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"farpointerforms": {Name: "farpointerforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "far-pointer-"+target.name+".ll", "far-pointer-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FarPointerRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LFSW AX, DX"}, + {goarch: "amd64", instruction: "LFSL (BX), (CX)"}, + {goarch: "amd64", instruction: "LGSQ (BX), X0"}, + {goarch: "amd64", instruction: "LSSW.Z (BX), DX"}, + {goarch: "386", instruction: "LFSQ (BX), AX"}, + {goarch: "386", instruction: "LGSL (R11), AX"}, + {goarch: "386", instruction: "LSSL (BX), R11"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's far-pointer tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_far_return.go b/amd64_lower_far_return.go new file mode 100644 index 000000000..205ebc36c --- /dev/null +++ b/amd64_lower_far_return.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FarReturnKind uint8 + +const ( + amd64InterruptReturn amd64FarReturnKind = iota + amd64FarReturn +) + +type amd64FarReturnSpec struct { + kind amd64FarReturnKind + bits int + allowImmediate bool +} + +// amd64FarReturnSpecs is the complete Go 1.27 interrupt/far-return grammar. +// Every RETF width accepts both the operand-free CB encoding and Go's Yi32 +// immediate row, whose value is truncated to the architectural imm16. Q-width +// rows are absent from the 386 encoder even though W/L rows are shared. +var amd64FarReturnSpecs = map[Op]amd64FarReturnSpec{ + "IRETW": {kind: amd64InterruptReturn, bits: 16}, + "IRETL": {kind: amd64InterruptReturn, bits: 32}, + "IRETQ": {kind: amd64InterruptReturn, bits: 64}, + "RETFW": {kind: amd64FarReturn, bits: 16, allowImmediate: true}, + "RETFL": {kind: amd64FarReturn, bits: 32, allowImmediate: true}, + "RETFQ": {kind: amd64FarReturn, bits: 64, allowImmediate: true}, +} + +type amd64FarReturnForm struct { + hasImmediate bool + immediate uint16 +} + +func (c *amd64Ctx) lowerFarReturn(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64FarReturnSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.bits == 64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + form, err := c.parseFarReturnForm(baseOp, spec, ins) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", + spec.encoding(form), "~{memory},~{dirflag},~{fpsr},~{flags}") + c.b.WriteString(" unreachable\n") + return true, true, nil +} + +func (c *amd64Ctx) parseFarReturnForm(op string, spec amd64FarReturnSpec, ins Instr) (amd64FarReturnForm, error) { + var form amd64FarReturnForm + if len(ins.Args) == 0 { + return form, nil + } + if !spec.allowImmediate || len(ins.Args) != 1 || ins.Args[0].Kind != OpImm || + ins.Args[0].ImmIsFloat || ins.Args[0].ImmRaw != "" { + return form, fmt.Errorf("%s %s expects no operands or one Go 1.27 Yi32 immediate: %q", c.goarch, op, ins.Raw) + } + value := ins.Args[0].Imm + if c.goarch != "386" && (value < -2147483648 || value > 4294967295) { + return form, fmt.Errorf("amd64 %s immediate is outside Go 1.27's Yi32 class: %q", op, ins.Raw) + } + form.hasImmediate = true + form.immediate = uint16(value) + return form, nil +} + +func (spec amd64FarReturnSpec) encoding(form amd64FarReturnForm) string { + bytes := make([]byte, 0, 4) + switch spec.bits { + case 16: + bytes = append(bytes, 0x66) + case 32: + case 64: + bytes = append(bytes, 0x48) + default: + panic("unknown amd64 far-return width") + } + if spec.kind == amd64InterruptReturn { + bytes = append(bytes, 0xcf) + } else if form.hasImmediate { + bytes = append(bytes, 0xca, byte(form.immediate), byte(form.immediate>>8)) + } else { + bytes = append(bytes, 0xcb) + } + parts := make([]string, len(bytes)) + for index, value := range bytes { + parts[index] = fmt.Sprintf("0x%02x", value) + } + return ".byte " + strings.Join(parts, ", ") +} diff --git a/amd64_lower_far_return_test.go b/amd64_lower_far_return_test.go new file mode 100644 index 000000000..6a8033f67 --- /dev/null +++ b/amd64_lower_far_return_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestAMD64FarReturnGrammarIsComplete(t *testing.T) { + want := map[Op]amd64FarReturnSpec{ + "IRETW": {kind: amd64InterruptReturn, bits: 16}, + "IRETL": {kind: amd64InterruptReturn, bits: 32}, + "IRETQ": {kind: amd64InterruptReturn, bits: 64}, + "RETFW": {kind: amd64FarReturn, bits: 16, allowImmediate: true}, + "RETFL": {kind: amd64FarReturn, bits: 32, allowImmediate: true}, + "RETFQ": {kind: amd64FarReturn, bits: 64, allowImmediate: true}, + } + if !reflect.DeepEqual(amd64FarReturnSpecs, want) { + t.Fatalf("far-return grammar = %+v, want %+v", amd64FarReturnSpecs, want) + } +} + +func TestAMD64FarReturnSplitsFollowingInstructions(t *testing.T) { + source := "TEXT splitfarreturn(SB),$0-0\n\tIRETL\n\tMOVL $1, AX\n\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"splitfarreturn": {Name: "splitfarreturn", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, amd64LLVMBlockName("anon_1")+":") || !strings.Contains(ir, "zext i32 1 to i64") { + t.Fatalf("instruction after IRETL was not retained in its own block:\n%s", ir) + } +} + +func TestTranslateX86FarReturnCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + forms := []struct { + op string + arg string + }{ + {op: "IRETW"}, + {op: "IRETL"}, + {op: "RETFW"}, + {op: "RETFW", arg: "$4"}, + {op: "RETFL"}, + {op: "RETFL", arg: "$4294967295"}, + } + if target.goarch == "386" { + forms = append(forms, struct { + op string + arg string + }{op: "RETFL", arg: "$4294967296"}) + } else { + forms = append(forms, + struct { + op string + arg string + }{op: "IRETQ"}, + struct { + op string + arg string + }{op: "RETFQ"}, + struct { + op string + arg string + }{op: "RETFQ", arg: "$4294967295"}, + ) + } + var source strings.Builder + sigs := make(map[string]FuncSig, len(forms)) + for index, form := range forms { + name := fmt.Sprintf("farreturnform%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n\t%s", name, form.op) + if form.arg != "" { + fmt.Fprintf(&source, " %s", form.arg) + } + source.WriteByte('\n') + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `.byte 0x66, 0xcf`, + `.byte 0xcf`, + `.byte 0x66, 0xcb`, + `.byte 0x66, 0xca, 0x04, 0x00`, + `.byte 0xcb`, + `.byte 0xca, 0xff, 0xff`, + } { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if target.goarch == "386" { + if !strings.Contains(ir, `.byte 0xca, 0x00, 0x00`) { + t.Errorf("386 IR is missing truncated Yi32 compatibility encoding:\n%s", ir) + } + } else { + for _, want := range []string{`.byte 0x48, 0xcf`, `.byte 0x48, 0xcb`, `.byte 0x48, 0xca, 0xff, 0xff`} { + if !strings.Contains(ir, want) { + t.Errorf("amd64 IR is missing %s:\n%s", want, ir) + } + } + } + if got := strings.Count(ir, "unreachable"); got != len(forms) { + t.Errorf("unreachable count = %d, want %d", got, len(forms)) + } + compileLLVMToObject(t, llc, target.triple, "far-return-"+target.name+".ll", "far-return-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FarReturnRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "IRETW AX"}, + {goarch: "amd64", instruction: "IRETL.P"}, + {goarch: "amd64", instruction: "IRETQ $4"}, + {goarch: "amd64", instruction: "RETFW AX"}, + {goarch: "amd64", instruction: "RETFL $-2147483649"}, + {goarch: "amd64", instruction: "RETFQ $4294967296"}, + {goarch: "amd64", instruction: "RETFL $1, $2"}, + {goarch: "386", instruction: "IRETQ"}, + {goarch: "386", instruction: "RETFQ"}, + {goarch: "386", instruction: "RETFQ $4"}, + } { + t.Run(test.goarch+"_"+strings.ReplaceAll(test.instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's far-return tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_fixed_system.go b/amd64_lower_fixed_system.go new file mode 100644 index 000000000..afa579ef7 --- /dev/null +++ b/amd64_lower_fixed_system.go @@ -0,0 +1,71 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FixedSystemEffect uint8 + +const ( + amd64FixedSystemContinue amd64FixedSystemEffect = iota + amd64FixedSystemTrap + amd64FixedSystemControlTransfer +) + +type amd64FixedSystemSpec struct { + encoding string + effect amd64FixedSystemEffect + rawOnly bool +} + +// amd64FixedSystemSpecs is the complete Go 1.27 operand-free system/state +// family that has no modeled GP-register data flow, plus raw-only ENDBR32. +// Exact bytes keep Go's +// accepted 386 compatibility rows (notably ENDBR64 and SWAPGS) object-level +// compatible even when LLVM's mnemonic matcher applies architectural mode +// restrictions. Effects remain typed so traps and non-returning RSM cannot +// accidentally acquire ordinary fallthrough. +var amd64FixedSystemSpecs = map[Op]amd64FixedSystemSpec{ + "CLAC": {encoding: ".byte 0x0f, 0x01, 0xca"}, + "CLI": {encoding: ".byte 0xfa"}, + "CLTS": {encoding: ".byte 0x0f, 0x06"}, + "ENDBR32": {encoding: ".byte 0xf3, 0x0f, 0x1e, 0xfb", rawOnly: true}, + "ENDBR64": {encoding: ".byte 0xf3, 0x0f, 0x1e, 0xfa"}, + "ICEBP": {encoding: ".byte 0xf1", effect: amd64FixedSystemTrap}, + "INVD": {encoding: ".byte 0x0f, 0x08"}, + "RSM": {encoding: ".byte 0x0f, 0xaa", effect: amd64FixedSystemControlTransfer}, + "STAC": {encoding: ".byte 0x0f, 0x01, 0xcb"}, + "STI": {encoding: ".byte 0xfb"}, + "SWAPGS": {encoding: ".byte 0x0f, 0x01, 0xf8"}, + "UD1": {encoding: ".byte 0x0f, 0xb9, 0x00", effect: amd64FixedSystemTrap}, + "WBINVD": {encoding: ".byte 0x0f, 0x09"}, +} + +func (c *amd64Ctx) lowerFixedSystem(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64FixedSystemSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if spec.rawOnly && !ins.x86Encoded { + return true, false, fmt.Errorf("%s %s has no Go assembler mnemonic; use its raw encoding: %q", c.goarch, baseOp, ins.Raw) + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", + spec.encoding, "~{memory},~{dirflag},~{fpsr},~{flags}") + if spec.effect == amd64FixedSystemContinue { + return true, false, nil + } + c.b.WriteString(" unreachable\n") + return true, true, nil +} diff --git a/amd64_lower_fixed_system_test.go b/amd64_lower_fixed_system_test.go new file mode 100644 index 000000000..0598f9072 --- /dev/null +++ b/amd64_lower_fixed_system_test.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64FixedSystemGrammarIsComplete(t *testing.T) { + want := map[Op]amd64FixedSystemSpec{ + "CLAC": {encoding: ".byte 0x0f, 0x01, 0xca"}, + "CLI": {encoding: ".byte 0xfa"}, + "CLTS": {encoding: ".byte 0x0f, 0x06"}, + "ENDBR32": {encoding: ".byte 0xf3, 0x0f, 0x1e, 0xfb", rawOnly: true}, + "ENDBR64": {encoding: ".byte 0xf3, 0x0f, 0x1e, 0xfa"}, + "ICEBP": {encoding: ".byte 0xf1", effect: amd64FixedSystemTrap}, + "INVD": {encoding: ".byte 0x0f, 0x08"}, + "RSM": {encoding: ".byte 0x0f, 0xaa", effect: amd64FixedSystemControlTransfer}, + "STAC": {encoding: ".byte 0x0f, 0x01, 0xcb"}, + "STI": {encoding: ".byte 0xfb"}, + "SWAPGS": {encoding: ".byte 0x0f, 0x01, 0xf8"}, + "UD1": {encoding: ".byte 0x0f, 0xb9, 0x00", effect: amd64FixedSystemTrap}, + "WBINVD": {encoding: ".byte 0x0f, 0x09"}, + } + if !reflect.DeepEqual(amd64FixedSystemSpecs, want) { + t.Fatalf("fixed-system grammar = %+v, want %+v", amd64FixedSystemSpecs, want) + } +} + +func TestTranslateX86FixedSystemCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT fixedsystemforms(SB),$0-0 + CLAC + CLI + CLTS + ENDBR64 + INVD + STAC + STI + SWAPGS + WBINVD + RET +TEXT fixedsystemicebp(SB),$0-0 + ICEBP +TEXT fixedsystemrsm(SB),$0-0 + RSM +TEXT fixedsystemud1(SB),$0-0 + UD1 +` + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "fixedsystemforms": {Name: "fixedsystemforms", Ret: Void}, + "fixedsystemicebp": {Name: "fixedsystemicebp", Ret: Void}, + "fixedsystemrsm": {Name: "fixedsystemrsm", Ret: Void}, + "fixedsystemud1": {Name: "fixedsystemud1", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `.byte 0x0f, 0x01, 0xca`, + `.byte 0xfa`, + `.byte 0x0f, 0x06`, + `.byte 0xf3, 0x0f, 0x1e, 0xfa`, + `.byte 0x0f, 0x08`, + `.byte 0x0f, 0x01, 0xcb`, + `.byte 0xfb`, + `.byte 0x0f, 0x01, 0xf8`, + `.byte 0x0f, 0x09`, + `.byte 0xf1`, + `.byte 0x0f, 0xaa`, + `.byte 0x0f, 0xb9, 0x00`, + `unreachable`, + } { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if got := strings.Count(ir, "unreachable"); got != 3 { + t.Errorf("unreachable count = %d, want 3", got) + } + compileLLVMToObject(t, llc, target.triple, "fixed-system-"+target.name+".ll", "fixed-system-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FixedSystemRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "CLAC AX"}, + {goarch: "amd64", instruction: "CLI.P"}, + {goarch: "amd64", instruction: "CLTS AX"}, + {goarch: "amd64", instruction: "ENDBR64 AX"}, + {goarch: "amd64", instruction: "ICEBP AX"}, + {goarch: "amd64", instruction: "INVD AX"}, + {goarch: "amd64", instruction: "RSM AX"}, + {goarch: "amd64", instruction: "STAC AX"}, + {goarch: "amd64", instruction: "STI AX"}, + {goarch: "amd64", instruction: "SWAPGS AX"}, + {goarch: "amd64", instruction: "UD1 AX"}, + {goarch: "amd64", instruction: "WBINVD AX"}, + {goarch: "386", instruction: "ENDBR64.P"}, + {goarch: "386", instruction: "SWAPGS AX"}, + } { + t.Run(test.goarch+"_"+strings.ReplaceAll(test.instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's fixed-system tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_fixupimm.go b/amd64_lower_fixupimm.go new file mode 100644 index 000000000..06a082dac --- /dev/null +++ b/amd64_lower_fixupimm.go @@ -0,0 +1,208 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FixupImmediateSpec struct { + laneBits int + scalar bool +} + +var amd64FixupImmediateSpecs = map[Op]amd64FixupImmediateSpec{ + "VFIXUPIMMPS": {laneBits: 32}, + "VFIXUPIMMPD": {laneBits: 64}, + "VFIXUPIMMSS": {laneBits: 32, scalar: true}, + "VFIXUPIMMSD": {laneBits: 64, scalar: true}, +} + +func (c *amd64Ctx) lowerFixupImmediate(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64FixupImmediateSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return c.lowerNativeFloatingHelper(op, ins) + } + shape := amd64UnaryImmediateFloatingShape{laneBits: spec.laneBits, scalar: spec.scalar, twoSource: true} + form, err := c.parseUnaryImmediateFloatingForm(baseOp, suffix, shape, ins) + if err != nil { + return true, false, err + } + daz := c.loadMXCSRDAZ() + if spec.scalar { + return c.lowerScalarFixupImmediate(spec, form, daz) + } + return c.lowerPackedFixupImmediate(spec, form, daz) +} + +func (c *amd64Ctx) lowerPackedFixupImmediate(spec amd64FixupImmediateSpec, form amd64UnaryImmediateFloatingForm, daz string) (bool, bool, error) { + lanes := form.byteWidth * 8 / spec.laneBits + table, err := c.loadPackedCompareLanes(form.source, form.byteWidth, spec.laneBits, form.properties.broadcast) + if err != nil { + return true, false, err + } + classified, err := c.loadPackedCompareLanes(form.passthrough, form.byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(form.destination, form.byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, spec.laneBits, oldBytes) + result := c.emitFixupImmediateBits(lanes, spec.laneBits, classified, table, old, daz) + if form.mask != "" { + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, form.mask, form.properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, form.byteWidth) + return true, false, c.storePackedMoveOperand(form.destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarFixupImmediate(spec amd64FixupImmediateSpec, form amd64UnaryImmediateFloatingForm, daz string) (bool, bool, error) { + tableBits, err := c.loadFloatingScalarBits(form.source, spec.laneBits) + if err != nil { + return true, false, err + } + classifiedBytes, err := c.loadX(form.passthrough.Reg) + if err != nil { + return true, false, err + } + classifiedVector := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, classifiedBytes) + classifiedLow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 0\n", classifiedLow, 128/spec.laneBits, spec.laneBits, classifiedVector) + oldLow, err := c.loadXLowInteger(form.destination.Reg, spec.laneBits) + if err != nil { + return true, false, err + } + tableVector, classifiedOne, oldVector := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", tableVector, spec.laneBits, spec.laneBits, tableBits) + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %%%s, i32 0\n", classifiedOne, spec.laneBits, spec.laneBits, classifiedLow) + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", oldVector, spec.laneBits, spec.laneBits, oldLow) + result := c.emitFixupImmediateBits(1, spec.laneBits, "%"+classifiedOne, "%"+tableVector, "%"+oldVector, daz) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + return true, false, c.storeScalarMoveRegister(form.destination.Reg, spec.laneBits, "%"+low, classifiedVector, form.mask, form.properties.zeroing) +} + +// emitFixupImmediateBits implements the data-result portion of Intel's +// FIXUPIMM_SP/FIXUPIMM_DP grammar. imm8 controls only MXCSR exception +// reporting; plan9asm does not currently expose floating exception state. +func (c *amd64Ctx) emitFixupImmediateBits(lanes, laneBits int, classified, table, old, daz string) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + positiveOne := uint64(0x3f800000) + constants := [16]uint64{ + 0, 0, 0, 0xffc00000, + 0xff800000, 0x7f800000, 0, 0x80000000, + 0, 0xbf800000, 0x3f800000, 0x3f000000, + 0x42b40000, 0x3fc90fdb, 0x7f7fffff, 0xff7fffff, + } + if laneBits == 64 { + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + positiveOne = uint64(0x3ff0000000000000) + constants = [16]uint64{ + 0, 0, 0, 0xfff8000000000000, + 0xfff0000000000000, 0x7ff0000000000000, 0, 0x8000000000000000, + 0, 0xbff0000000000000, 0x3ff0000000000000, 0x3fe0000000000000, + 0x4056800000000000, 0x3ff921fb54442d18, 0x7fefffffffffffff, 0xffefffffffffffff, + } + } + result := "poison" + for lane := 0; lane < lanes; lane++ { + raw, tableLane, oldLane := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", raw, vectorType, classified, lane) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", tableLane, vectorType, table, lane) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", oldLane, vectorType, old, lane) + rawExponent, exponentZero, dazZero := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", rawExponent, laneBits, raw, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, rawExponent) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %s\n", dazZero, exponentZero, daz) + effective := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 0, i%d %%%s\n", effective, dazZero, laneBits, laneBits, raw) + + exponent, mantissa, sign, absolute := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", exponent, laneBits, effective, exponentMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", mantissa, laneBits, effective, mantissaMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", sign, laneBits, effective, signBit) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", absolute, laneBits, effective, signBit-1) + exponentOnes, mantissaZero, mantissaNonzero, negative, zero := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", exponentOnes, laneBits, exponent, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", mantissaZero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", mantissaNonzero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", negative, laneBits, sign) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", zero, laneBits, absolute) + isNaN, isInfinity, quietSet := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, exponentOnes, mantissaNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, exponentOnes, mantissaZero) + quietValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", quietValue, laneBits, mantissa, quietBit) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", quietSet, laneBits, quietValue) + notQuiet, signalingNaN, quietNaN := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notQuiet, quietSet) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", signalingNaN, isNaN, notQuiet) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", quietNaN, isNaN, quietSet) + negativeInfinity, positiveInfinity, notNegative := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", negativeInfinity, isInfinity, negative) + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notNegative, negative) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", positiveInfinity, isInfinity, notNegative) + positiveOneValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", positiveOneValue, laneBits, effective, positiveOne) + + token := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 6, i%d 7\n", token, negative, laneBits, laneBits) + for _, selection := range []struct { + predicate string + value int + }{ + {predicate: positiveInfinity, value: 5}, + {predicate: negativeInfinity, value: 4}, + {predicate: positiveOneValue, value: 3}, + {predicate: zero, value: 2}, + {predicate: signalingNaN, value: 1}, + {predicate: quietNaN, value: 0}, + } { + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %%%s\n", selected, selection.predicate, laneBits, selection.value, laneBits, token) + token = selected + } + shift, shiftedTable, response := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, 2\n", shift, laneBits, token) + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %%%s\n", shiftedTable, laneBits, tableLane, shift) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, 15\n", response, laneBits, shiftedTable) + + quieted, signedInfinity := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", quieted, laneBits, effective, exponentMask|quietBit) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", signedInfinity, laneBits, sign, exponentMask) + responseValues := [16]string{ + "%" + oldLane, "%" + effective, "%" + quieted, + fmt.Sprintf("%d", constants[3]), fmt.Sprintf("%d", constants[4]), fmt.Sprintf("%d", constants[5]), "%" + signedInfinity, + fmt.Sprintf("%d", constants[7]), fmt.Sprintf("%d", constants[8]), fmt.Sprintf("%d", constants[9]), fmt.Sprintf("%d", constants[10]), + fmt.Sprintf("%d", constants[11]), fmt.Sprintf("%d", constants[12]), fmt.Sprintf("%d", constants[13]), fmt.Sprintf("%d", constants[14]), fmt.Sprintf("%d", constants[15]), + } + laneResult := responseValues[0] + for responseValue := 1; responseValue < len(responseValues); responseValue++ { + matches, selected := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", matches, laneBits, response, responseValue) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %s, i%d %s\n", selected, matches, laneBits, responseValues[responseValue], laneBits, laneResult) + laneResult = "%" + selected + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, result, laneBits, laneResult, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_fixupimm_test.go b/amd64_lower_fixupimm_test.go new file mode 100644 index 000000000..86b4a5bdb --- /dev/null +++ b/amd64_lower_fixupimm_test.go @@ -0,0 +1,244 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type fixupImmediateTestSpec struct { + op string + laneBits int + scalar bool +} + +func go127FixupImmediateSpecs() []fixupImmediateTestSpec { + return []fixupImmediateTestSpec{ + {op: "VFIXUPIMMPS", laneBits: 32}, + {op: "VFIXUPIMMPD", laneBits: 64}, + {op: "VFIXUPIMMSS", laneBits: 32, scalar: true}, + {op: "VFIXUPIMMSD", laneBits: 64, scalar: true}, + } +} + +func TestAMD64FixupImmediateGrammarCoversCompleteGoFamily(t *testing.T) { + expected := go127FixupImmediateSpecs() + if len(amd64FixupImmediateSpecs) != len(expected) { + t.Fatalf("FIXUPIMM grammar has %d entries, want %d", len(amd64FixupImmediateSpecs), len(expected)) + } + for _, want := range expected { + got, ok := amd64FixupImmediateSpecs[Op(want.op)] + if !ok { + t.Errorf("FIXUPIMM grammar omitted %s", want.op) + continue + } + if got.laneBits != want.laneBits || got.scalar != want.scalar { + t.Errorf("FIXUPIMM grammar %s = %+v, want laneBits=%d scalar=%v", want.op, got, want.laneBits, want.scalar) + } + } +} + +func TestTranslateX86FixupImmediateCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT fixupimmforms(SB),$0-0\n") + for index, spec := range go127FixupImmediateSpecs() { + if spec.scalar { + fmt.Fprintf(&source, "\t%s $%d, X0, X1, X23\n", spec.op, index) + fmt.Fprintf(&source, "\t%s $%d, %d(AX), X1, X23\n", spec.op, index+16, index*8) + fmt.Fprintf(&source, "\t%s $%d, X2, X1, K1, X23\n", spec.op, index+32) + fmt.Fprintf(&source, "\t%s.Z $%d, %d(AX), X1, K2, X23\n", spec.op, index+48, index*8) + fmt.Fprintf(&source, "\t%s.SAE $%d, X3, X1, X23\n", spec.op, index+64) + fmt.Fprintf(&source, "\t%s.SAE.Z $%d, X4, X1, K3, X23\n", spec.op, index+80) + continue + } + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s $%d, %s0, %s1, %s23\n", spec.op, index, width, width, width) + fmt.Fprintf(&source, "\t%s $%d, %d(AX), %s1, %s23\n", spec.op, index+16, index*8, width, width) + fmt.Fprintf(&source, "\t%s.BCST $%d, %d(AX), %s1, %s23\n", spec.op, index+32, index*8, width, width) + fmt.Fprintf(&source, "\t%s $%d, %s2, %s1, K1, %s23\n", spec.op, index+48, width, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z $%d, %d(AX), %s1, K2, %s23\n", spec.op, index+64, index*8, width, width) + } + fmt.Fprintf(&source, "\t%s.SAE $%d, Z3, Z1, Z23\n", spec.op, index+80) + fmt.Fprintf(&source, "\t%s.SAE.Z $%d, Z4, Z1, K3, Z23\n", spec.op, index+96) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + ir := x86FloatingHelperOriginalFormatObject(t, source.String(), "fixupimmforms", "amd64", target.triple) + compileLLVMToObject(t, llc, target.triple, "fixupimm-"+target.name+".ll", "fixupimm-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FixupImmediateRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VFIXUPIMMPS $-1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS $256, X0, X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS.Z $1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS $1, X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS.SAE $1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS.SAE $1, 0(AX), Z1, Z2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS $1, X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMSS.BCST $1, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMSS.SAE $1, 0(AX), X1, X2"}, + {goarch: "386", instruction: "VFIXUPIMMPS $1, X0, X1, X2"}, + {goarch: "386", instruction: "VFIXUPIMMSS $1, X0, X1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's FIXUPIMM tables", test.instruction) + } + }) + } +} + +func qualifyAMD64FixupImmediateRuntimeSemantics(t *testing.T) { + execution := x86FloatingHelperRuntimeTools(t) + const source = `TEXT fixupimmsemantics(SB),$0-64 + MOVQ out+0(FP), AX + MOVQ src32+8(FP), BX + MOVQ table32+16(FP), CX + MOVQ old32+24(FP), DX + MOVQ src64+32(FP), R8 + MOVQ table64+40(FP), R9 + MOVQ old64+48(FP), R10 + MOVQ mask+56(FP), R11 + KMOVQ R11, K1 + VMOVDQU32 0(BX), Z1 + VMOVDQU32 0(CX), Z0 + VMOVDQU32 0(DX), Z2 + STC + VFIXUPIMMPS $0, Z0, Z1, Z2 + VMOVDQU32 Z2, 0(AX) + VMOVDQU32 0(DX), Z2 + VFIXUPIMMPS $0, Z0, Z1, K1, Z2 + VMOVDQU32 Z2, 64(AX) + VFIXUPIMMPS.Z $0, Z0, Z1, K1, Z3 + VMOVDQU32 Z3, 128(AX) + VMOVDQU64 0(R8), Z5 + VMOVDQU64 0(R9), Z4 + VMOVDQU64 0(R10), Z6 + VFIXUPIMMPD $0, Z4, Z5, Z6 + VMOVDQU64 Z6, 192(AX) + VMOVDQU64 0(R8), X5 + VMOVDQU64 0(R9), X4 + VMOVDQU64 0(R10), X6 + VFIXUPIMMSD $0, X4, X5, X6 + VMOVDQU64 X6, 256(AX) + VMOVDQU32 0(BX), X1 + VMOVDQU32 0(CX), X0 + VMOVDQU32 0(DX), X2 + VFIXUPIMMSS $0, X0, X1, X2 + VMOVDQU32 X2, 272(AX) + SETCS 288(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + runtimeSource := x86FixupRuntimeSource(t, source) + requireX86GoAssemblerResult(t, "amd64", runtimeSource, true) + file, err := Parse(ArchAMD64, runtimeSource) + if err != nil { + t.Fatal(err) + } + triple := execution.triple + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"fixupimmsemantics": { + Name: "fixupimmsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: I64, Index: 7, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void fixupimmsemantics(uint8_t *, const uint32_t *, const uint32_t *, const uint32_t *, const uint64_t *, const uint64_t *, const uint64_t *, uint64_t); +static unsigned token32(uint32_t x, int daz) { + if (daz && (x & UINT32_C(0x7f800000)) == 0) x = 0; + uint32_t a=x&UINT32_C(0x7fffffff),m=x&UINT32_C(0x007fffff),e=x&UINT32_C(0x7f800000); + if(e==UINT32_C(0x7f800000)&&m) return m&UINT32_C(0x00400000)?0:1; + if(a==0)return 2;if(x==UINT32_C(0x3f800000))return 3; + if(a==UINT32_C(0x7f800000))return x>>31?4:5;return x>>31?6:7; +} +static unsigned token64(uint64_t x, int daz) { + if (daz && (x & UINT64_C(0x7ff0000000000000)) == 0) x = 0; + uint64_t a=x&UINT64_C(0x7fffffffffffffff),m=x&UINT64_C(0x000fffffffffffff),e=x&UINT64_C(0x7ff0000000000000); + if(e==UINT64_C(0x7ff0000000000000)&&m) return m&UINT64_C(0x0008000000000000)?0:1; + if(a==0)return 2;if(x==UINT64_C(0x3ff0000000000000))return 3; + if(a==UINT64_C(0x7ff0000000000000))return x>>63?4:5;return x>>63?6:7; +} +static uint32_t effective32(uint32_t x,int daz){return daz&&(x&UINT32_C(0x7f800000))==0?0:x;} +static uint64_t effective64(uint64_t x,int daz){return daz&&(x&UINT64_C(0x7ff0000000000000))==0?0:x;} +static uint32_t fix32(uint32_t old,uint32_t x,uint32_t table,int daz){ + x=effective32(x,daz);unsigned r=(table>>(4*token32(x,0)))&15; + static const uint32_t c[16]={0,0,0,UINT32_C(0xffc00000),UINT32_C(0xff800000),UINT32_C(0x7f800000),0,UINT32_C(0x80000000),0,UINT32_C(0xbf800000),UINT32_C(0x3f800000),UINT32_C(0x3f000000),UINT32_C(0x42b40000),UINT32_C(0x3fc90fdb),UINT32_C(0x7f7fffff),UINT32_C(0xff7fffff)}; + if(r==0)return old;if(r==1)return x;if(r==2)return x|UINT32_C(0x7fc00000);if(r==6)return (x&UINT32_C(0x80000000))|UINT32_C(0x7f800000);return c[r]; +} +static uint64_t fix64(uint64_t old,uint64_t x,uint64_t table,int daz){ + x=effective64(x,daz);unsigned r=(table>>(4*token64(x,0)))&15; + static const uint64_t c[16]={0,0,0,UINT64_C(0xfff8000000000000),UINT64_C(0xfff0000000000000),UINT64_C(0x7ff0000000000000),0,UINT64_C(0x8000000000000000),0,UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0x3fe0000000000000),UINT64_C(0x4056800000000000),UINT64_C(0x3ff921fb54442d18),UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff)}; + if(r==0)return old;if(r==1)return x;if(r==2)return x|UINT64_C(0x7ff8000000000000);if(r==6)return (x&UINT64_C(0x8000000000000000))|UINT64_C(0x7ff0000000000000);return c[r]; +} +int main(void){ + const uint32_t src32[16]={UINT32_C(0x7fc12345),1,UINT32_C(0xc0200001),0,UINT32_C(0xff800000),UINT32_C(0x7f800000),UINT32_C(0xc0000000),UINT32_C(0x40000000),UINT32_C(0x80000000),UINT32_C(0x3f800000),UINT32_C(0x7f812345),UINT32_C(0x80000001),UINT32_C(0x40400000),UINT32_C(0xc0800000),UINT32_C(0x3f400000),UINT32_C(0xbf400000)}; + const uint64_t src64[8]={UINT64_C(0x7ff8123456789abc),1,UINT64_C(0xc004000000000001),0,UINT64_C(0xfff0000000000000),UINT64_C(0x7ff0000000000000),UINT64_C(0xc000000000000000),UINT64_C(0x4000000000000000)}; + uint32_t old32[16],table32[16],got32[48],want32[48];uint64_t old64[8],table64[8],got64[10],want64[10];uint8_t out[289]; + const uint64_t mask=UINT64_C(0xa55a);unsigned original=_mm_getcsr(); + for(int i=0;i<16;i++)old32[i]=UINT32_C(0x41000000)+i;for(int i=0;i<8;i++)old64[i]=UINT64_C(0x4020000000000000)+i; + for(int daz=0;daz<2;daz++){ + _mm_setcsr((original&~64u)|(daz?64u:0));memset(out,0,sizeof(out)); + for(int i=0;i<16;i++)table32[i]=(uint32_t)i<<(4*token32(src32[i],daz)); + for(int i=0;i<8;i++)table64[i]=(uint64_t)(i+8)<<(4*token64(src64[i],daz)); + fixupimmsemantics(out,src32,table32,old32,src64,table64,old64,mask);memcpy(got32,out,192);memcpy(got64,out+192,80); + for(int i=0;i<16;i++){uint32_t v=fix32(old32[i],src32[i],table32[i],daz);want32[i]=v;want32[16+i]=(mask>>i)&1?v:old32[i];want32[32+i]=(mask>>i)&1?v:0;} + for(int i=0;i<8;i++)want64[i]=fix64(old64[i],src64[i],table64[i],daz);want64[8]=fix64(old64[0],src64[0],table64[0],daz);want64[9]=src64[1]; + for(int i=0;i<48;i++)if(got32[i]!=want32[i])return 10+daz; + for(int i=0;i<10;i++)if(got64[i]!=want64[i])return 20+daz; + uint32_t scalar32[4];memcpy(scalar32,out+272,16);if(scalar32[0]!=fix32(old32[0],src32[0],table32[0],daz)||scalar32[1]!=src32[1]||scalar32[2]!=src32[2]||scalar32[3]!=src32[3])return 30+daz; + if(out[288]!=1)return 40+daz; + } + _mm_setcsr(original);return 0; +} +` + x86FloatingHelperRunNativePipelines(t, execution, "fixupimm_semantics", ir, mainC) +} diff --git a/amd64_lower_float_add_sub.go b/amd64_lower_float_add_sub.go new file mode 100644 index 000000000..048464c34 --- /dev/null +++ b/amd64_lower_float_add_sub.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "fmt" +) + +func (c *amd64Ctx) lowerFloatAddSub(op Op, ins Instr) (bool, bool, error) { + return c.lowerFloatArithmetic(op, ins, x86FloatAddSubtract) +} + +func (c *amd64Ctx) lowerFloatAddSubForm(ins Instr, form x86FloatAddSubForm) (bool, bool, error) { + return c.lowerFloatBinaryOperands(ins, form.x86FloatBinaryForm, func(lanes int, left, right, mask string) string { + return c.emitFloatAddSub(form, lanes, left, right, mask) + }) +} + +// Shared yxm/VEX/EVEX source loading and alias writes. Each family supplies +// its arithmetic emitter, so a common shape grants no numerical behavior. +func (c *amd64Ctx) lowerFloatBinaryOperands(ins Instr, form x86FloatBinaryForm, emit func(int, string, string, string) string) (bool, bool, error) { + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.scalar && form.memory >= 0 { + return true, false, c.lowerLegacyPackedFloatingMemory(ins, form) + } + if first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + var mask string + var err error + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + bits := form.bits + if form.scalar { + var a string + var err error + if form.literal { + if bits == 32 { + a, err = c.evalF32(first) + } else { + a, err = c.evalF64(first) + } + } else { + a, err = c.loadFloatBinaryScalar(first, bits, mask) + } + if err != nil { + return true, false, err + } + b, err := c.loadFMA3Scalar(second, bits, "") + if err != nil { + return true, false, err + } + value := emit(1, b, a, mask) + integer := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", integer, amd64FMA3LLVMType(1, bits), value, bits) + upper, err := c.loadX(second.Reg) + if err != nil { + return true, false, err + } + lanes := 128 / bits + base := c.bitcastVectorBytesToIntegerLanes(16, lanes, bits, upper) + if !form.legacy { + return true, false, c.storeScalarMoveRegister(destination.Reg, bits, "%"+integer, base, mask, form.zeroing) + } + merged := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 0\n", merged, lanes, bits, base, bits, integer) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <16 x i8>\n", bytes, lanes, bits, merged) + return true, false, c.storeLegacyPackedLogical(destination, "%"+bytes) + } + lanes := form.byteWidth * 8 / bits + a, err := c.loadFloatBinaryPacked(first, form.byteWidth, bits, form.broadcast, mask) + if err != nil { + return true, false, err + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + floatType := amd64FMA3LLVMType(lanes, bits) + aFloat, bFloat := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", aFloat, lanes, bits, a, floatType) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", bFloat, lanes, bits, b, floatType) + value := emit(lanes, "%"+bFloat, "%"+aFloat, mask) + result := c.bitcastFloatingToIntegerLanes(floatType, lanes, bits, value) + if form.mask >= 0 { + old, err := c.loadPackedCompareLanes(destination, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, bits, result, old, mask, form.zeroing) + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, bits, result, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+bytes) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+bytes) +} + +func (c *amd64Ctx) emitFloatAddSub(form x86FloatAddSubForm, lanes int, left, right, mask string) string { + if form.rounding != "" { + return c.emitRoundedFloatAddSub(lanes, form.spec.bits, left, right, (form.spec.mode == amd64BinaryFloatingSub), form.rounding) + } + // Do not grant fast-math, reassociation or an ambient-rounding assumption. + // Dynamic strict operations retain x86 rounding/DAZ/FTZ/exception behavior. + operation := "fadd" + if form.spec.mode == amd64BinaryFloatingSub { + operation = "fsub" + } + return c.emitStrictFloatBinary(form.x86FloatBinaryForm, operation, lanes, left, right, mask, func(lanes int, left, right, value string) string { + return c.emitFloatAddSubSpecial(form.spec.bits, lanes, left, right, (form.spec.mode == amd64BinaryFloatingSub), value) + }) +} + +// Strict operations must remain observable under middle-end optimization. +// The correction callback handles each family's native NaN result priority +// without replacing the actual hardware rounding or exception side effects. +func (c *amd64Ctx) emitStrictFloatBinary(form x86FloatBinaryForm, operation string, lanes int, left, right, mask string, correct func(int, string, string, string) string) string { + typ := amd64FMA3LLVMType(lanes, form.bits) + if mask != "" { + // EVEX suppresses arithmetic exceptions as well as memory faults for + // inactive lanes. Selecting after an eager strict fadd is not enough. + scalarType := amd64FMA3LLVMType(1, form.bits) + result := "poison" + for lane := 0; lane < lanes; lane++ { + values := []string{left, right} + if lanes > 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, typ, value, lane) + values[index] = "%" + extracted + } + } + active := amd64MaskBitI1(c, mask, lane) + integer, _ := c.loadX86ScalarIf(form.bits, active, func() (string, error) { + value, bits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.experimental.constrained.%s.f%d(%s %s, %s %s, metadata !\"round.dynamic\", metadata !\"fpexcept.strict\")\n", value, scalarType, operation, form.bits, scalarType, values[0], scalarType, values[1]) + corrected := correct(1, values[0], values[1], "%"+value) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", bits, scalarType, corrected, form.bits) + return "%" + bits, nil + }) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %s to %s\n", value, form.bits, integer, scalarType) + if lanes == 1 { + return "%" + value + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, typ, result, scalarType, value, lane) + result = "%" + inserted + } + return result + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.experimental.constrained.%s.%s(%s %s, %s %s, metadata !\"round.dynamic\", metadata !\"fpexcept.strict\")\n", result, typ, operation, amd64BinaryFloatingIntrinsicSuffix(lanes, form.bits), typ, left, typ, right) + return correct(lanes, left, right, "%"+result) +} + +// LLVM's commutative add legalization can swap NaN operands even for a +// constrained operation. Preserve its real hardware exception/rounding +// effects, but restore x86 binary source-order payload/indefinite results. +func (c *amd64Ctx) emitFloatAddSubSpecial(bits, lanes int, left, right string, subtract bool, hardware string) string { + scalarType, typ := amd64FMA3LLVMType(1, bits), amd64FMA3LLVMType(lanes, bits) + result := "poison" + for lane := 0; lane < lanes; lane++ { + values := []string{left, right, hardware} + if lanes > 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, typ, value, lane) + values[index] = "%" + extracted + } + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @__plan9asm_add_sub_special_f%d(%s %s, %s %s, i1 %t, %s %s)\n", value, scalarType, bits, scalarType, values[0], scalarType, values[1], subtract, scalarType, values[2]) + if lanes == 1 { + return "%" + value + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, typ, result, scalarType, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_float_add_sub_round.go b/amd64_lower_float_add_sub_round.go new file mode 100644 index 000000000..8eca3ad81 --- /dev/null +++ b/amd64_lower_float_add_sub_round.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Classify binary x86 special values using integer operations only. This is +// also used after a real strict operation without changing its MXCSR effects. +func emitX86FloatAddSubSpecialBits(e *amd64FMA3IntegerEmitter, bits int) (special, result, rightSigned string) { + fraction, exponent := 23, uint64(0x7f800000) + if bits == 64 { + fraction, exponent = 52, 0x7ff0000000000000 + } + typ := amd64FMA3LLVMType(1, bits) + left := e.value("bitcast %s %%left to i%d", typ, bits) + right := e.value("bitcast %s %%right to i%d", typ, bits) + sign := uint64(1) << (bits - 1) + leftAbs := e.value("and i%d %s, %d", bits, left, sign-1) + rightAbs := e.value("and i%d %s, %d", bits, right, sign-1) + leftNaN := e.value("icmp ugt i%d %s, %d", bits, leftAbs, exponent) + rightNaN := e.value("icmp ugt i%d %s, %d", bits, rightAbs, exponent) + leftInf := e.value("icmp eq i%d %s, %d", bits, leftAbs, exponent) + rightInf := e.value("icmp eq i%d %s, %d", bits, rightAbs, exponent) + special = e.value("or i1 %s, %s", leftNaN, rightNaN) + special = e.value("or i1 %s, %s", special, leftInf) + special = e.value("or i1 %s, %s", special, rightInf) + negated := e.value("xor i%d %s, %d", bits, right, sign) + rightSigned = e.value("select i1 %%subtract, i%d %s, i%d %s", bits, negated, bits, right) + quiet := uint64(1) << (fraction - 1) + leftQuiet := e.value("or i%d %s, %d", bits, left, quiet) + rightQuiet := e.value("or i%d %s, %d", bits, right, quiet) + // SUB never toggles a NaN's sign/payload, only its finite/inf operand sign. + nan := e.value("select i1 %s, i%d %s, i%d %s", leftNaN, bits, leftQuiet, bits, rightQuiet) + anyNaN := e.value("or i1 %s, %s", leftNaN, rightNaN) + signs := e.value("xor i%d %s, %s", bits, left, rightSigned) + opposite := e.value("icmp slt i%d %s, 0", bits, signs) + bothInf := e.value("and i1 %s, %s", leftInf, rightInf) + invalid := e.value("and i1 %s, %s", bothInf, opposite) + infinity := e.value("select i1 %s, i%d %s, i%d %s", leftInf, bits, left, bits, rightSigned) + infinity = e.value("select i1 %s, i%d %d, i%d %s", invalid, bits, sign|exponent|quiet, bits, infinity) + result = e.value("select i1 %s, i%d %s, i%d %s", anyNaN, bits, nan, bits, infinity) + return +} + +func emitX86FloatAddSubSpecialHelper(b *strings.Builder, bits int) { + typ := amd64FMA3LLVMType(1, bits) + fmt.Fprintf(b, "define internal %s @__plan9asm_add_sub_special_f%d(%s %%left, %s %%right, i1 %%subtract, %s %%hardware) {\nentry:\n", typ, bits, typ, typ, typ) + e := amd64FMA3IntegerEmitter{b: b} + special, result, _ := emitX86FloatAddSubSpecialBits(&e, bits) + floatResult := e.value("bitcast i%d %s to %s", bits, result, typ) + selected := e.value("select i1 %s, %s %s, %s %%hardware", special, typ, floatResult, typ) + fmt.Fprintf(b, " ret %s %s\n}\n\n", typ, selected) +} + +// Embedded rounding suppresses exceptions. Only finite binary arithmetic +// uses the reviewed exact accumulator (a*1+b); special results remain ADD/SUB +// semantics rather than FMA's operand priority. +func emitX86FloatAddSubRoundingHelper(b *strings.Builder, bits int) { + typ := amd64FMA3LLVMType(1, bits) + fmt.Fprintf(b, "define internal %s @__plan9asm_add_sub_round_f%d(%s %%left, %s %%right, i1 %%subtract, i32 %%mode, i32 %%mxcsr) {\nentry:\n", typ, bits, typ, typ) + e := amd64FMA3IntegerEmitter{b: b} + special, result, rightSigned := emitX86FloatAddSubSpecialBits(&e, bits) + fmt.Fprintf(b, " br i1 %s, label %%special, label %%finite\nspecial:\n", special) + floatResult := e.value("bitcast i%d %s to %s", bits, result, typ) + fmt.Fprintf(b, " ret %s %s\nfinite:\n", typ, floatResult) + rightFloat := e.value("bitcast i%d %s to %s", bits, rightSigned, typ) + finite := e.value("call %s @__plan9asm_fma3_round_f%d(%s %%left, %s 1.0, %s %s, i32 %%mode, i32 %%mxcsr)", typ, bits, typ, typ, typ, rightFloat) + fmt.Fprintf(b, " ret %s %s\n}\n\n", typ, finite) +} + +func (c *amd64Ctx) emitRoundedFloatAddSub(lanes, bits int, left, right string, subtract bool, rounding string) string { + typ, vectorType := amd64FMA3LLVMType(1, bits), amd64FMA3LLVMType(lanes, bits) + mode, mxcsr := c.packedNumericRoundingMode(rounding), c.loadMXCSR() + result := "poison" + for lane := 0; lane < lanes; lane++ { + values := []string{left, right} + if lanes > 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, vectorType, value, lane) + values[index] = "%" + extracted + } + } + computed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @__plan9asm_add_sub_round_f%d(%s %s, %s %s, i1 %t, i32 %s, i32 %s)\n", computed, typ, bits, typ, values[0], typ, values[1], subtract, mode, mxcsr) + if lanes == 1 { + return "%" + computed + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, vectorType, result, typ, computed, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_float_min_max.go b/amd64_lower_float_min_max.go new file mode 100644 index 000000000..2f4aaf5cc --- /dev/null +++ b/amd64_lower_float_min_max.go @@ -0,0 +1,53 @@ +package plan9asm + +import ( + "fmt" +) + +func (c *amd64Ctx) lowerFloatMinMax(op Op, ins Instr) (bool, bool, error) { + return c.lowerFloatArithmetic(op, ins, x86FloatMinimumMaximum) +} + +func (c *amd64Ctx) lowerFloatMinMaxForm(ins Instr, f x86FloatMinMaxForm) (bool, bool, error) { + if (ins.x86Encoded || ins.x86SourceRaw != nil) && !x86RawFloatMinMaxEncodingSource(ins, c.goarch) { + return true, false, fmt.Errorf("%w: MIN/MAX original physical source identity changed", ErrProbeNeedsContext) + } + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + if f.legacy && !f.scalar && f.memory >= 0 { + return true, false, c.lowerLegacyPackedFloatingMemory(ins, f.x86FloatBinaryForm) + } + lanes := f.byteWidth * 8 / f.bits + logicalFPBits := 0 + if f.scalar { + lanes = 1 + logicalFPBits = f.bits + } + evex := x86FloatMinMaxNativeEVEX(ins, f) + return true, false, c.emitNativeFloatingOperands(ins, x86NativeFloatingOperands{ + byteWidth: f.byteWidth, lanes: lanes, memoryBytes: f.memoryBytes, + second: f.second, destination: f.destination, mask: f.mask, + controls: amd64FMA3Suffix{broadcast: f.broadcast, zeroing: f.zeroing}, + sae: f.sae, legacy: f.legacy, logicalFPBits: logicalFPBits, + lowBank: !evex, forceEVEX: evex && f.byteWidth < 64 && f.mask < 0 && !f.sae && !f.broadcast, + }) + } + // Portable numerical values do not certify x86 MXCSR or native exceptions. + return c.lowerFloatBinaryOperands(ins, f.x86FloatBinaryForm, + func(lanes int, left, right, _ string) string { + typ := amd64FMA3LLVMType(lanes, f.bits) + predicate := "ogt" + if f.spec.mode == amd64BinaryFloatingMin { + predicate = "olt" + } + condition, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = fcmp %s %s %s, %s\n", condition, predicate, typ, left, right) + predicateType := "i1" + if lanes > 1 { + predicateType = fmt.Sprintf("<%d x i1>", lanes) + } + // Native MIN/MAX chooses the first explicit Go source for equal + // values and unordered pairs. Keep its original NaN/zero bits. + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", value, predicateType, condition, typ, left, typ, right) + return "%" + value + }) +} diff --git a/amd64_lower_float_movemask_test.go b/amd64_lower_float_movemask_test.go new file mode 100644 index 000000000..995424748 --- /dev/null +++ b/amd64_lower_float_movemask_test.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86FloatingMoveMaskCompleteGoAssemblerForms(t *testing.T) { + // MOVMSK{PS,PD} uses Go's legacy yxrrl table (X -> Yrl), while the + // VMOVMSK forms use _yvmovmskpd (X/Y -> Yrl). Neither table accepts + // memory, EVEX registers, masks, or opcode suffixes. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastLegacyX := 15 + if target.goarch == "386" { + lastLegacyX = 7 + } + source := fmt.Sprintf(` +TEXT floatingmovemaskforms(SB),NOSPLIT,$0-0 + MOVMSKPS X%d, AX + MOVMSKPD X%d, BX + VMOVMSKPS X15, CX + VMOVMSKPS Y15, DX + VMOVMSKPD X15, SI + VMOVMSKPD Y15, DI + RET +`, lastLegacyX, lastLegacyX) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"floatingmovemaskforms": {Name: "floatingmovemaskforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "floating-movemask-"+target.name+".ll", "floating-movemask-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86FloatingMoveMaskRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "MOVMSKPS Y0, AX", + "MOVMSKPD (AX), BX", + "MOVMSKPS X16, AX", + "VMOVMSKPS Z0, AX", + "VMOVMSKPD X16, AX", + "VMOVMSKPS Y16, AX", + "VMOVMSKPD (AX), BX", + "VMOVMSKPS X0, X1", + "VMOVMSKPD X0, K1, AX", + "MOVMSKPS.Z X0, AX", + "VMOVMSKPD.BCST Y0, AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86FloatingMoveMaskRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86FloatingMoveMaskRejected(t, "386", "i386-unknown-linux-gnu", "MOVMSKPS X8, AX") +} + +func assertX86FloatingMoveMaskRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's floating movemask forms", instruction) + } +} diff --git a/amd64_lower_float_mul_div.go b/amd64_lower_float_mul_div.go new file mode 100644 index 000000000..2b08e2e98 --- /dev/null +++ b/amd64_lower_float_mul_div.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerFloatMulDiv(op Op, ins Instr) (bool, bool, error) { + return c.lowerFloatArithmetic(op, ins, x86FloatMultiplyDivide) +} + +func (c *amd64Ctx) lowerFloatMulDivForm(ins Instr, form x86FloatMulDivForm) (bool, bool, error) { + return c.lowerFloatBinaryOperands(ins, form.x86FloatBinaryForm, func(lanes int, left, right, mask string) string { + if form.rounding != "" { + return c.emitRoundedFloatMulDiv(form, lanes, left, right) + } + operation := "fmul" + if form.spec.mode == amd64BinaryFloatingDiv { + operation = "fdiv" + } + return c.emitStrictFloatBinary(form.x86FloatBinaryForm, operation, lanes, left, right, mask, func(lanes int, left, right, value string) string { + return c.emitFloatBinaryNaNPriority(form.bits, lanes, left, right, value) + }) + }) +} + +// LLVM may commute strict multiplication during legalization. Restore x86's +// first-source NaN payload without executing another floating operation or +// changing the real strict operation's sticky/rounding/DAZ/FTZ effects. +func (c *amd64Ctx) emitFloatBinaryNaNPriority(bits, lanes int, left, right, hardware string) string { + typ := amd64FMA3LLVMType(lanes, bits) + integerType := fmt.Sprintf("<%d x i%d>", lanes, bits) + if lanes == 1 { + integerType = fmt.Sprintf("i%d", bits) + } + exponent, quiet := uint64(0x7f800000), uint64(0x400000) + if bits == 64 { + exponent, quiet = 0x7ff0000000000000, 0x8000000000000 + } + constant := func(value uint64) string { + if lanes == 1 { + return fmt.Sprint(value) + } + values := make([]string, lanes) + for lane := range values { + values[lane] = fmt.Sprintf("i%d %d", bits, value) + } + return "<" + strings.Join(values, ", ") + ">" + } + values, nan, quieted := [2]string{left, right}, [2]string{}, [2]string{} + for index, value := range values { + raw, magnitude, isNaN, withQuiet := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", raw, typ, value, integerType) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", magnitude, integerType, raw, constant(uint64(1)<<(bits-1)-1)) + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %s\n", isNaN, integerType, magnitude, constant(exponent)) + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", withQuiet, integerType, raw, constant(quiet)) + nan[index], quieted[index] = "%"+isNaN, "%"+withQuiet + } + predicateType := "i1" + if lanes > 1 { + predicateType = fmt.Sprintf("<%d x i1>", lanes) + } + anyNaN, chosen, selected, rawHardware := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %s, %s\n", anyNaN, predicateType, nan[0], nan[1]) + fmt.Fprintf(c.b, " %%%s = select %s %s, %s %s, %s %s\n", chosen, predicateType, nan[0], integerType, quieted[0], integerType, quieted[1]) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", rawHardware, typ, hardware, integerType) + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %%%s, %s %%%s\n", selected, predicateType, anyNaN, integerType, chosen, integerType, rawHardware) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to %s\n", result, integerType, selected, typ) + return "%" + result +} + +func (c *amd64Ctx) emitRoundedFloatMulDiv(form x86FloatMulDivForm, lanes int, left, right string) string { + typ, vectorType := amd64FMA3LLVMType(1, form.bits), amd64FMA3LLVMType(lanes, form.bits) + mode, mxcsr := c.packedNumericRoundingMode(form.rounding), c.loadMXCSR() + result := "poison" + for lane := 0; lane < lanes; lane++ { + values := [2]string{left, right} + if lanes > 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, vectorType, value, lane) + values[index] = "%" + extracted + } + } + for index, floating := range values { + raw := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", raw, typ, floating, form.bits) + values[index] = "%" + raw + } + raw, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i%d @__plan9asm_mul_div_round_f%d(i%d %s, i%d %s, i1 %t, i32 %s, i32 %s)\n", raw, form.bits, form.bits, form.bits, values[0], form.bits, values[1], (form.spec.mode == amd64BinaryFloatingDiv), mode, mxcsr) + fmt.Fprintf(c.b, " %%%s = bitcast i%d %%%s to %s\n", value, form.bits, raw, typ) + if lanes == 1 { + return "%" + value + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, vectorType, result, typ, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_fma3.go b/amd64_lower_fma3.go new file mode 100644 index 000000000..9f620ed3e --- /dev/null +++ b/amd64_lower_fma3.go @@ -0,0 +1,432 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FMA3Mode uint8 + +const ( + amd64FMA3Add amd64FMA3Mode = iota + amd64FMA3Sub + amd64FMA3NegAdd + amd64FMA3NegSub + amd64FMA3AddSub + amd64FMA3SubAdd +) + +type amd64FMA3Spec struct { + laneBits int + order int + mode amd64FMA3Mode + scalar bool + rawOnly bool +} + +// amd64FMA3Specs is the complete Go 1.27 FMA3 family. The 36 packed +// instructions use _yvaddpd and the 24 scalar instructions use _yvaddsd. +// The additional 30 MAP6 FP16 forms are accepted only from raw bytes. +var amd64FMA3Specs = map[Op]amd64FMA3Spec{ + "VFMADD132PS": {laneBits: 32, order: 132, mode: amd64FMA3Add}, + "VFMADD132PD": {laneBits: 64, order: 132, mode: amd64FMA3Add}, + "VFMADD132SS": {laneBits: 32, order: 132, mode: amd64FMA3Add, scalar: true}, + "VFMADD132SD": {laneBits: 64, order: 132, mode: amd64FMA3Add, scalar: true}, + "VFMADD213PS": {laneBits: 32, order: 213, mode: amd64FMA3Add}, + "VFMADD213PD": {laneBits: 64, order: 213, mode: amd64FMA3Add}, + "VFMADD213SS": {laneBits: 32, order: 213, mode: amd64FMA3Add, scalar: true}, + "VFMADD213SD": {laneBits: 64, order: 213, mode: amd64FMA3Add, scalar: true}, + "VFMADD231PS": {laneBits: 32, order: 231, mode: amd64FMA3Add}, + "VFMADD231PD": {laneBits: 64, order: 231, mode: amd64FMA3Add}, + "VFMADD231SS": {laneBits: 32, order: 231, mode: amd64FMA3Add, scalar: true}, + "VFMADD231SD": {laneBits: 64, order: 231, mode: amd64FMA3Add, scalar: true}, + "VFMSUB132PS": {laneBits: 32, order: 132, mode: amd64FMA3Sub}, + "VFMSUB132PD": {laneBits: 64, order: 132, mode: amd64FMA3Sub}, + "VFMSUB132SS": {laneBits: 32, order: 132, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB132SD": {laneBits: 64, order: 132, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB213PS": {laneBits: 32, order: 213, mode: amd64FMA3Sub}, + "VFMSUB213PD": {laneBits: 64, order: 213, mode: amd64FMA3Sub}, + "VFMSUB213SS": {laneBits: 32, order: 213, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB213SD": {laneBits: 64, order: 213, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB231PS": {laneBits: 32, order: 231, mode: amd64FMA3Sub}, + "VFMSUB231PD": {laneBits: 64, order: 231, mode: amd64FMA3Sub}, + "VFMSUB231SS": {laneBits: 32, order: 231, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB231SD": {laneBits: 64, order: 231, mode: amd64FMA3Sub, scalar: true}, + "VFNMADD132PS": {laneBits: 32, order: 132, mode: amd64FMA3NegAdd}, + "VFNMADD132PD": {laneBits: 64, order: 132, mode: amd64FMA3NegAdd}, + "VFNMADD132SS": {laneBits: 32, order: 132, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD132SD": {laneBits: 64, order: 132, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD213PS": {laneBits: 32, order: 213, mode: amd64FMA3NegAdd}, + "VFNMADD213PD": {laneBits: 64, order: 213, mode: amd64FMA3NegAdd}, + "VFNMADD213SS": {laneBits: 32, order: 213, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD213SD": {laneBits: 64, order: 213, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD231PS": {laneBits: 32, order: 231, mode: amd64FMA3NegAdd}, + "VFNMADD231PD": {laneBits: 64, order: 231, mode: amd64FMA3NegAdd}, + "VFNMADD231SS": {laneBits: 32, order: 231, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD231SD": {laneBits: 64, order: 231, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMSUB132PS": {laneBits: 32, order: 132, mode: amd64FMA3NegSub}, + "VFNMSUB132PD": {laneBits: 64, order: 132, mode: amd64FMA3NegSub}, + "VFNMSUB132SS": {laneBits: 32, order: 132, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB132SD": {laneBits: 64, order: 132, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB213PS": {laneBits: 32, order: 213, mode: amd64FMA3NegSub}, + "VFNMSUB213PD": {laneBits: 64, order: 213, mode: amd64FMA3NegSub}, + "VFNMSUB213SS": {laneBits: 32, order: 213, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB213SD": {laneBits: 64, order: 213, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB231PS": {laneBits: 32, order: 231, mode: amd64FMA3NegSub}, + "VFNMSUB231PD": {laneBits: 64, order: 231, mode: amd64FMA3NegSub}, + "VFNMSUB231SS": {laneBits: 32, order: 231, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB231SD": {laneBits: 64, order: 231, mode: amd64FMA3NegSub, scalar: true}, + "VFMADDSUB132PS": {laneBits: 32, order: 132, mode: amd64FMA3AddSub}, + "VFMADDSUB132PD": {laneBits: 64, order: 132, mode: amd64FMA3AddSub}, + "VFMADDSUB213PS": {laneBits: 32, order: 213, mode: amd64FMA3AddSub}, + "VFMADDSUB213PD": {laneBits: 64, order: 213, mode: amd64FMA3AddSub}, + "VFMADDSUB231PS": {laneBits: 32, order: 231, mode: amd64FMA3AddSub}, + "VFMADDSUB231PD": {laneBits: 64, order: 231, mode: amd64FMA3AddSub}, + "VFMSUBADD132PS": {laneBits: 32, order: 132, mode: amd64FMA3SubAdd}, + "VFMSUBADD132PD": {laneBits: 64, order: 132, mode: amd64FMA3SubAdd}, + "VFMSUBADD213PS": {laneBits: 32, order: 213, mode: amd64FMA3SubAdd}, + "VFMSUBADD213PD": {laneBits: 64, order: 213, mode: amd64FMA3SubAdd}, + "VFMSUBADD231PS": {laneBits: 32, order: 231, mode: amd64FMA3SubAdd}, + "VFMSUBADD231PD": {laneBits: 64, order: 231, mode: amd64FMA3SubAdd}, + "VFMADD132PH": {laneBits: 16, order: 132, mode: amd64FMA3Add, rawOnly: true}, + "VFMADD132SH": {laneBits: 16, order: 132, mode: amd64FMA3Add, scalar: true, rawOnly: true}, + "VFMSUB132PH": {laneBits: 16, order: 132, mode: amd64FMA3Sub, rawOnly: true}, + "VFMSUB132SH": {laneBits: 16, order: 132, mode: amd64FMA3Sub, scalar: true, rawOnly: true}, + "VFNMADD132PH": {laneBits: 16, order: 132, mode: amd64FMA3NegAdd, rawOnly: true}, + "VFNMADD132SH": {laneBits: 16, order: 132, mode: amd64FMA3NegAdd, scalar: true, rawOnly: true}, + "VFNMSUB132PH": {laneBits: 16, order: 132, mode: amd64FMA3NegSub, rawOnly: true}, + "VFNMSUB132SH": {laneBits: 16, order: 132, mode: amd64FMA3NegSub, scalar: true, rawOnly: true}, + "VFMADDSUB132PH": {laneBits: 16, order: 132, mode: amd64FMA3AddSub, rawOnly: true}, + "VFMSUBADD132PH": {laneBits: 16, order: 132, mode: amd64FMA3SubAdd, rawOnly: true}, + "VFMADD213PH": {laneBits: 16, order: 213, mode: amd64FMA3Add, rawOnly: true}, + "VFMADD213SH": {laneBits: 16, order: 213, mode: amd64FMA3Add, scalar: true, rawOnly: true}, + "VFMSUB213PH": {laneBits: 16, order: 213, mode: amd64FMA3Sub, rawOnly: true}, + "VFMSUB213SH": {laneBits: 16, order: 213, mode: amd64FMA3Sub, scalar: true, rawOnly: true}, + "VFNMADD213PH": {laneBits: 16, order: 213, mode: amd64FMA3NegAdd, rawOnly: true}, + "VFNMADD213SH": {laneBits: 16, order: 213, mode: amd64FMA3NegAdd, scalar: true, rawOnly: true}, + "VFNMSUB213PH": {laneBits: 16, order: 213, mode: amd64FMA3NegSub, rawOnly: true}, + "VFNMSUB213SH": {laneBits: 16, order: 213, mode: amd64FMA3NegSub, scalar: true, rawOnly: true}, + "VFMADDSUB213PH": {laneBits: 16, order: 213, mode: amd64FMA3AddSub, rawOnly: true}, + "VFMSUBADD213PH": {laneBits: 16, order: 213, mode: amd64FMA3SubAdd, rawOnly: true}, + "VFMADD231PH": {laneBits: 16, order: 231, mode: amd64FMA3Add, rawOnly: true}, + "VFMADD231SH": {laneBits: 16, order: 231, mode: amd64FMA3Add, scalar: true, rawOnly: true}, + "VFMSUB231PH": {laneBits: 16, order: 231, mode: amd64FMA3Sub, rawOnly: true}, + "VFMSUB231SH": {laneBits: 16, order: 231, mode: amd64FMA3Sub, scalar: true, rawOnly: true}, + "VFNMADD231PH": {laneBits: 16, order: 231, mode: amd64FMA3NegAdd, rawOnly: true}, + "VFNMADD231SH": {laneBits: 16, order: 231, mode: amd64FMA3NegAdd, scalar: true, rawOnly: true}, + "VFNMSUB231PH": {laneBits: 16, order: 231, mode: amd64FMA3NegSub, rawOnly: true}, + "VFNMSUB231SH": {laneBits: 16, order: 231, mode: amd64FMA3NegSub, scalar: true, rawOnly: true}, + "VFMADDSUB231PH": {laneBits: 16, order: 231, mode: amd64FMA3AddSub, rawOnly: true}, + "VFMSUBADD231PH": {laneBits: 16, order: 231, mode: amd64FMA3SubAdd, rawOnly: true}, +} + +type amd64FMA3Suffix struct { + broadcast bool + zeroing bool + rounding string +} + +func parseAMD64FMA3Suffix(suffix string) (amd64FMA3Suffix, bool) { + switch suffix { + case "": + return amd64FMA3Suffix{}, true + case "Z": + return amd64FMA3Suffix{zeroing: true}, true + case "BCST": + return amd64FMA3Suffix{broadcast: true}, true + case "BCST.Z": + return amd64FMA3Suffix{broadcast: true, zeroing: true}, true + case "RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE": + return amd64FMA3Suffix{rounding: suffix}, true + case "RN_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", "RZ_SAE.Z": + return amd64FMA3Suffix{zeroing: true, rounding: strings.TrimSuffix(suffix, ".Z")}, true + default: + return amd64FMA3Suffix{}, false + } +} + +func (c *amd64Ctx) lowerFMA3(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := amd64FMA3Specs[Op(base)]; !recognized { + return false, false, nil + } + // Bind only a local copy for legacy helpers. A dispatch/source conflict + // must be rejected before reading a register or emitting any LLVM IR. + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("FMA3 dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := amd64FMA3Properties(ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + spec, properties := form.spec, form.suffix + dstArg, byteWidth := ins.Args[form.destination], form.byteWidth + if spec.rawOnly && x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + lanes := byteWidth * 8 / spec.laneBits + if spec.scalar { + lanes = 1 + } + return true, false, c.emitNativeFP16ThreeOperand(ins, x86NativeFP16ThreeOperand{ + byteWidth: byteWidth, lanes: lanes, memoryBytes: form.memoryBytes, + second: 1, destination: form.destination, mask: form.mask, + controls: properties, readDestination: true, + }) + } + named := x86ResultNamedPredicateInstruction(ins) && ins.x86FMA3Encoding == x86RawFMA3Absent + raw := !spec.rawOnly && (x86RawFMA3Source(ins, c.goarch) || x86RawFMA3LiteralForm(ins, c.goarch)) + if !spec.rawOnly && (named || raw) && x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + lanes, logicalFPBits := byteWidth*8/spec.laneBits, 0 + if spec.scalar { + lanes, logicalFPBits = 1, spec.laneBits + } + evex := x86FMA3NativeEVEX(ins, form) + return true, false, c.emitNativeFloatingOperands(ins, x86NativeFloatingOperands{ + byteWidth: byteWidth, lanes: lanes, memoryBytes: form.memoryBytes, + second: 1, destination: form.destination, mask: form.mask, + controls: properties, readDestination: true, logicalFPBits: logicalFPBits, + lowBank: !evex, forceEVEX: evex && byteWidth < 64 && form.mask < 0 && + !properties.broadcast && properties.rounding == "", + }) + } + if !spec.rawOnly && !named { + return true, false, fmt.Errorf("%w: %s requires its exact original FMA source and native x86 target", ErrProbeNeedsContext, ins.Op) + } + + // Portable numerical arithmetic is not an x86 MXCSR/native-exception + // certificate. Raw F32/F64 still requires its independent source owner. + var mask string + if form.mask >= 0 { + loadedMask, loadErr := c.loadK(ins.Args[form.mask].Reg) + if loadErr != nil { + return true, false, loadErr + } + mask = loadedMask + } + if spec.scalar { + return c.lowerScalarFMA3(spec, properties, ins, dstArg, mask) + } + return c.lowerPackedFMA3(spec, properties, ins, dstArg, byteWidth, mask) +} + +func (c *amd64Ctx) lowerPackedFMA3(spec amd64FMA3Spec, properties amd64FMA3Suffix, ins Instr, dstArg Operand, byteWidth int, mask string) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + llvmType := amd64FMA3LLVMType(lanes, spec.laneBits) + loadLanes := c.loadMaskedPackedCompareLanes + if !spec.rawOnly && x86ResultNamedPredicateInstruction(ins) && + (ins.Args[0].Kind == OpMem || ins.Args[0].Kind == OpSym) { + // This portable continuation still owes the original native access, + // including dead results and per-lane/broadcast mask suppression. + // Logical FP slots and independently owned raw forms stay separate. + loadLanes = c.loadX86PackedShiftLanes + } + load := func(arg Operand, allowBroadcast bool) (string, error) { + sourceMask := "" + if allowBroadcast { + sourceMask = mask + } + bits, err := loadLanes(arg, byteWidth, spec.laneBits, allowBroadcast && properties.broadcast, sourceMask) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", value, lanes, spec.laneBits, bits, llvmType) + return "%" + value, nil + } + first, err := load(ins.Args[0], true) + if err != nil { + return true, false, err + } + second, err := load(ins.Args[1], false) + if err != nil { + return true, false, err + } + old, err := load(dstArg, false) + if err != nil { + return true, false, err + } + computed := c.emitFMA3(spec, llvmType, lanes, first, second, old, properties.rounding) + if mask != "" { + computedBits := c.newTmp() + oldBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i%d>\n", computedBits, llvmType, computed, lanes, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i%d>\n", oldBits, llvmType, old, lanes, spec.laneBits) + masked := amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, "%"+computedBits, "%"+oldBits, mask, properties.zeroing) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", back, lanes, spec.laneBits, masked, llvmType) + computed = "%" + back + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, llvmType, computed, byteWidth) + return true, false, c.storePackedMoveOperand(dstArg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarFMA3(spec amd64FMA3Spec, properties amd64FMA3Suffix, ins Instr, dstArg Operand, mask string) (bool, bool, error) { + loadFirst := c.loadFMA3Scalar + if !spec.rawOnly && x86ResultNamedPredicateInstruction(ins) && + (ins.Args[0].Kind == OpMem || ins.Args[0].Kind == OpSym) { + loadFirst = c.loadFMA3ObservableScalar + } + first, err := loadFirst(ins.Args[0], spec.laneBits, mask) + if err != nil { + return true, false, err + } + second, err := c.loadFMA3Scalar(ins.Args[1], spec.laneBits, "") + if err != nil { + return true, false, err + } + old, err := c.loadFMA3Scalar(dstArg, spec.laneBits, "") + if err != nil { + return true, false, err + } + llvmType := amd64FMA3LLVMType(1, spec.laneBits) + computed := c.emitFMA3(spec, llvmType, 1, first, second, old, properties.rounding) + destinationBytes, err := c.loadX(dstArg.Reg) + if err != nil { + return true, false, err + } + lanes := 128 / spec.laneBits + base := c.bitcastVectorBytesToIntegerLanes(16, lanes, spec.laneBits, destinationBytes) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", bits, llvmType, computed, spec.laneBits) + return true, false, c.storeScalarMoveRegister(dstArg.Reg, spec.laneBits, "%"+bits, base, mask, properties.zeroing) +} + +// Only the already validated named FMA memory continuation uses this reader. +// One guarded volatile lane preserves its exact m32/m64 access independently +// of the numerical result. It does not certify x86 MXCSR or FP exceptions. +func (c *amd64Ctx) loadFMA3ObservableScalar(source Operand, laneBits int, mask string) (string, error) { + bits, err := c.loadX86PackedShiftLanes(source, laneBits/8, laneBits, false, mask) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <1 x i%d> %s to %s\n", value, laneBits, bits, amd64FMA3LLVMType(1, laneBits)) + return "%" + value, nil +} + +func (c *amd64Ctx) loadFMA3Scalar(arg Operand, laneBits int, mask string) (string, error) { + var bits string + var err error + if arg.Kind == OpReg { + bits, err = c.loadXLowInteger(arg.Reg, laneBits) + } else if mask != "" { + bits, err = c.loadVectorScalarMemory(arg, laneBits, mask) + } else { + bits, err = c.evalIntSized(arg, amd64IntegerTypeForBits(laneBits)) + } + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %s to %s\n", value, laneBits, bits, amd64FMA3LLVMType(1, laneBits)) + return "%" + value, nil +} + +func amd64FMA3LLVMType(lanes, laneBits int) string { + element := "float" + if laneBits == 16 { + element = "half" + } else if laneBits == 64 { + element = "double" + } + if lanes == 1 { + return element + } + return fmt.Sprintf("<%d x %s>", lanes, element) +} + +func (c *amd64Ctx) splatFMA3Scalar(lanes, laneBits int, scalar string) string { + typeName := amd64FMA3LLVMType(lanes, laneBits) + element := amd64FMA3LLVMType(1, laneBits) + seed := c.newTmp() + splat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s poison, %s %s, i32 0\n", seed, typeName, element, scalar) + fmt.Fprintf(c.b, " %%%s = shufflevector %s %%%s, %s poison, <%d x i32> zeroinitializer\n", splat, typeName, seed, typeName, lanes) + return "%" + splat +} + +func (c *amd64Ctx) emitFMA3(spec amd64FMA3Spec, llvmType string, lanes int, first, second, old, rounding string) string { + a, b, addend := old, first, second + switch spec.order { + case 213: + a, b, addend = second, old, first + case 231: + a, b, addend = second, first, old + } + if spec.mode == amd64FMA3NegAdd || spec.mode == amd64FMA3NegSub { + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg %s %s\n", negated, llvmType, a) + a = "%" + negated + } + switch spec.mode { + case amd64FMA3Sub, amd64FMA3NegSub: + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg %s %s\n", negated, llvmType, addend) + addend = "%" + negated + case amd64FMA3AddSub, amd64FMA3SubAdd: + addend = c.alternateFMA3Addend(llvmType, lanes, addend, spec.mode == amd64FMA3AddSub) + } + if rounding != "" && !spec.rawOnly { + return c.emitRoundedFMA3(lanes, spec.laneBits, a, b, addend, rounding) + } + result := c.newTmp() + intrinsic := amd64FMA3IntrinsicName(lanes, spec.laneBits, rounding != "") + if rounding == "" { + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, %s %s)\n", result, llvmType, intrinsic, llvmType, a, llvmType, b, llvmType, addend) + } else { + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, %s %s, metadata !\"%s\", metadata !\"fpexcept.ignore\")\n", result, llvmType, intrinsic, llvmType, a, llvmType, b, llvmType, addend, amd64FMA3RoundingMetadata(rounding)) + } + return "%" + result +} + +func (c *amd64Ctx) alternateFMA3Addend(llvmType string, lanes int, addend string, negateEven bool) string { + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg %s %s\n", negated, llvmType, addend) + var mask strings.Builder + mask.WriteByte('<') + for lane := 0; lane < lanes; lane++ { + if lane > 0 { + mask.WriteString(", ") + } + selected := lane%2 == 0 + if !negateEven { + selected = !selected + } + fmt.Fprintf(&mask, "i1 %t", selected) + } + mask.WriteByte('>') + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %s, %s %%%s, %s %s\n", result, lanes, mask.String(), llvmType, negated, llvmType, addend) + return "%" + result +} + +func amd64FMA3IntrinsicName(lanes, laneBits int, constrained bool) string { + prefix := "llvm.fma." + if constrained { + prefix = "llvm.experimental.constrained.fma." + } + suffix := "f32" + if laneBits == 16 { + suffix = "f16" + } else if laneBits == 64 { + suffix = "f64" + } + if lanes > 1 { + suffix = fmt.Sprintf("v%d%s", lanes, suffix) + } + return prefix + suffix +} + +func amd64FMA3RoundingMetadata(rounding string) string { + switch rounding { + case "RD_SAE": + return "round.downward" + case "RU_SAE": + return "round.upward" + case "RZ_SAE": + return "round.towardzero" + default: + return "round.tonearest" + } +} diff --git a/amd64_lower_fma3_round.go b/amd64_lower_fma3_round.go new file mode 100644 index 000000000..1498a0083 --- /dev/null +++ b/amd64_lower_fma3_round.go @@ -0,0 +1,292 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// LLVM 22 can lower fixed-rounding constrained FMA to an ambient-rounding +// libcall. Accumulate the exact product and addend with guard/sticky bits +// instead. The product needs at most 106 bits; a 128-bit accumulator retains +// every cancellation bit and enough low bits for one final IEEE rounding. +type amd64FMA3IntegerEmitter struct { + b *strings.Builder + next int +} + +func (e *amd64FMA3IntegerEmitter) value(format string, args ...any) string { + name := fmt.Sprintf("%%fma%d", e.next) + e.next++ + fmt.Fprintf(e.b, " %s = %s\n", name, fmt.Sprintf(format, args...)) + return name +} + +type amd64FMA3IntegerInput struct { + negative, magnitude, exponent, finite, floating string +} + +func (e *amd64FMA3IntegerEmitter) input(bits int, value, daz string) amd64FMA3IntegerInput { + fractionBits, bias := 23, 127 + if bits == 64 { + fractionBits, bias = 52, 1023 + } + fractionMask := uint64(1)< 0 || len(mulDivNeeded) > 0 { + b.WriteString("declare i128 @llvm.ctlz.i128(i128, i1)\n") + } + for _, bits := range []int{32, 64} { + if mulDivNeeded[bits] { + emitX86FloatMulDivRoundingHelper(b, bits) + } + if addSubSpecial[bits] { + emitX86FloatAddSubSpecialHelper(b, bits) + } + if needed[bits] { + emitAMD64FMA3RoundingHelper(b, bits) + if addSubNeeded[bits] { + emitX86FloatAddSubRoundingHelper(b, bits) + } + } + } +} + +func emitAMD64FMA3RoundingHelper(b *strings.Builder, bits int) { + fractionBits, bias := 23, 127 + if bits == 64 { + fractionBits, bias = 52, 1023 + } + floatType := amd64FMA3LLVMType(1, bits) + name := fmt.Sprintf("__plan9asm_fma3_round_f%d", bits) + fmt.Fprintf(b, "define internal %s @%s(%s %%a, %s %%b, %s %%c, i32 %%mode, i32 %%mxcsr) {\nentry:\n", floatType, name, floatType, floatType, floatType) + e := amd64FMA3IntegerEmitter{b: b} + daz := e.value("and i32 %%mxcsr, 64") + daz = e.value("icmp ne i32 %s, 0", daz) + a, multiplier, addend := e.input(bits, "%a", daz), e.input(bits, "%b", daz), e.input(bits, "%c", daz) + finite := e.value("and i1 %s, %s", a.finite, multiplier.finite) + finite = e.value("and i1 %s, %s", finite, addend.finite) + fmt.Fprintf(b, " br i1 %s, label %%finite, label %%special\nspecial:\n", finite) + // Preserve the existing target's NaN/infinity behavior without executing + // a floating operation on the finite software path or changing host fenv. + fmt.Fprintf(b, " %%special_result = call %s @llvm.fma.f%d(%s %s, %s %s, %s %s)\n ret %s %%special_result\nfinite:\n", floatType, bits, floatType, a.floating, floatType, multiplier.floating, floatType, addend.floating, floatType) + product := e.value("mul i128 %s, %s", a.magnitude, multiplier.magnitude) + productExponent := e.value("add i32 %s, %s", a.exponent, multiplier.exponent) + productSign := e.value("xor i1 %s, %s", a.negative, multiplier.negative) + productTop := e.value("add i32 %s, %d", productExponent, 2*fractionBits+1) + addendTop := e.value("add i32 %s, %d", addend.exponent, fractionBits) + largerTop := e.value("icmp sgt i32 %s, %s", productTop, addendTop) + top := e.value("select i1 %s, i32 %s, i32 %s", largerTop, productTop, addendTop) + // A zero contribution has no exponent. In particular, a huge operand + // multiplied by zero must not erase a tiny nonzero addend into sticky. + originalTop := top + productZero := e.value("icmp eq i128 %s, 0", product) + addendZero := e.value("icmp eq i128 %s, 0", addend.magnitude) + top = e.value("select i1 %s, i32 %s, i32 %s", productZero, addendTop, top) + top = e.value("select i1 %s, i32 %s, i32 %s", addendZero, productTop, top) + bothZero := e.value("and i1 %s, %s", productZero, addendZero) + top = e.value("select i1 %s, i32 %s, i32 %s", bothZero, originalTop, top) + unit := e.value("sub i32 %s, 126", top) + product = e.align(product, productExponent, unit) + add := e.align(addend.magnitude, addend.exponent, unit) + sameSign := e.value("icmp eq i1 %s, %s", productSign, addend.negative) + productLarger := e.value("icmp uge i128 %s, %s", product, add) + sum := e.value("add i128 %s, %s", product, add) + difference := e.value("sub i128 %s, %s", product, add) + reverseDifference := e.value("sub i128 %s, %s", add, product) + difference = e.value("select i1 %s, i128 %s, i128 %s", productLarger, difference, reverseDifference) + magnitude := e.value("select i1 %s, i128 %s, i128 %s", sameSign, sum, difference) + negative := e.value("select i1 %s, i1 %s, i1 %s", productLarger, productSign, addend.negative) + zero := e.value("icmp eq i128 %s, 0", magnitude) + downward := e.value("icmp eq i32 %%mode, 1") + zeroSign := e.value("select i1 %s, i1 %s, i1 %s", sameSign, productSign, downward) + negative = e.value("select i1 %s, i1 %s, i1 %s", zero, zeroSign, negative) + leading := e.value("call i128 @llvm.ctlz.i128(i128 %s, i1 false)", magnitude) + leading = e.value("trunc i128 %s to i32", leading) + highest := e.value("sub i32 127, %s", leading) + normalShift := e.value("sub i32 %s, %d", highest, fractionBits) + subnormalShift := e.value("sub i32 %d, %s", 1-bias-fractionBits, unit) + isSubnormal := e.value("icmp sgt i32 %s, %s", subnormalShift, normalShift) + shift := e.value("select i1 %s, i32 %s, i32 %s", isSubnormal, subnormalShift, normalShift) + positiveShift := e.value("icmp sgt i32 %s, 0", shift) + far := e.value("icmp sge i32 %s, 128", shift) + safeRight := e.value("select i1 %s, i32 %s, i32 0", positiveShift, shift) + safeRight = e.value("select i1 %s, i32 127, i32 %s", far, safeRight) + safeRight = e.value("zext i32 %s to i128", safeRight) + right := e.value("lshr i128 %s, %s", magnitude, safeRight) + right = e.value("select i1 %s, i128 0, i128 %s", far, right) + leftCount := e.value("sub i32 0, %s", shift) + leftCount = e.value("select i1 %s, i32 0, i32 %s", positiveShift, leftCount) + leftCount = e.value("zext i32 %s to i128", leftCount) + left := e.value("shl i128 %s, %s", magnitude, leftCount) + unrounded := e.value("select i1 %s, i128 %s, i128 %s", positiveShift, right, left) + mask := e.value("shl i128 1, %s", safeRight) + mask = e.value("sub i128 %s, 1", mask) + remainder := e.value("and i128 %s, %s", magnitude, mask) + remainder = e.value("select i1 %s, i128 %s, i128 %s", far, magnitude, remainder) + inexact := e.value("icmp ne i128 %s, 0", remainder) + halfCount := e.value("sub i32 %s, 1", shift) + halfCount = e.value("select i1 %s, i32 %s, i32 0", positiveShift, halfCount) + halfFar := e.value("icmp ugt i32 %s, 127", halfCount) + halfCount = e.value("select i1 %s, i32 127, i32 %s", halfFar, halfCount) + halfCount = e.value("zext i32 %s to i128", halfCount) + half := e.value("shl i128 1, %s", halfCount) + above := e.value("icmp ugt i128 %s, %s", remainder, half) + tied := e.value("icmp eq i128 %s, %s", remainder, half) + odd := e.value("and i128 %s, 1", unrounded) + odd = e.value("icmp ne i128 %s, 0", odd) + tied = e.value("and i1 %s, %s", tied, odd) + nearest := e.value("or i1 %s, %s", above, tied) + validHalf := e.value("xor i1 %s, true", halfFar) + validHalf = e.value("and i1 %s, %s", validHalf, positiveShift) + nearest = e.value("and i1 %s, %s", nearest, validHalf) + upward := e.value("icmp eq i32 %%mode, 2") + positive := e.value("xor i1 %s, true", negative) + directed := e.value("and i1 %s, %s", downward, negative) + upwardPositive := e.value("and i1 %s, %s", upward, positive) + directed = e.value("or i1 %s, %s", directed, upwardPositive) + awayFromZero := directed + directed = e.value("and i1 %s, %s", directed, inexact) + isNearest := e.value("icmp eq i32 %%mode, 0") + increment := e.value("select i1 %s, i1 %s, i1 %s", isNearest, nearest, directed) + increment = e.value("zext i1 %s to i128", increment) + rounded := e.value("add i128 %s, %s", unrounded, increment) + carry := e.value("icmp uge i128 %s, %d", rounded, uint64(1)<<(fractionBits+1)) + carried := e.value("lshr i128 %s, 1", rounded) + rounded = e.value("select i1 %s, i128 %s, i128 %s", carry, carried, rounded) + carry32 := e.value("zext i1 %s to i32", carry) + exponent := e.value("add i32 %s, %s", unit, shift) + exponent = e.value("add i32 %s, %d", exponent, fractionBits+bias) + exponent = e.value("add i32 %s, %s", exponent, carry32) + stillSubnormal := e.value("icmp ult i128 %s, %d", rounded, uint64(1)< 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, vectorType, value, lane) + values[index] = "%" + extracted + } + } + computed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @__plan9asm_fma3_round_f%d(%s %s, %s %s, %s %s, i32 %s, i32 %s)\n", computed, floatType, bits, floatType, values[0], floatType, values[1], floatType, values[2], mode, mxcsr) + if lanes == 1 { + return "%" + computed + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, vectorType, result, floatType, computed, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_four_source.go b/amd64_lower_four_source.go new file mode 100644 index 000000000..865f3d443 --- /dev/null +++ b/amd64_lower_four_source.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerFourSource(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86FourSourceProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || form.memory.Kind == OpFP { + return true, false, fmt.Errorf("%w: %s needs original x86 four-source memory and exception behavior", ErrProbeNeedsContext, op) + } + if err := c.validateConflictFunnelMemory(ins, x86PackedBinaryForm{memory: 0, memoryBytes: 16}); err != nil { + return true, false, err + } + return true, false, c.emitNativeFourSource(op, ins, form) +} + +// Snapshot the complete logical source block and old accumulator before the +// write. A private fixed 0..3 block plus accumulator4 is only an inline-asm +// transport: source/destination aliasing in the original register file remains +// modeled by the caller's byte values and the final single writeback. +// +// Keep one original opcode rather than four LLVM FMA/dot expressions. Each +// floating iteration has ambient MXCSR rounding and ordered exceptions, and +// the original mask suppresses either the whole m128 access or none of it. +// Even a dead result retains those exact native effects. +func (c *amd64Ctx) emitNativeFourSource(op Op, ins Instr, form amd64FourSourceForm) error { + width := form.byteWidth() + typ := amd64FloatingVectorType(width/4, 32) // Bit-preserving transport, including integer dot products. + load := func(operand Operand) (string, error) { + return c.loadFloatingVectorOperand(operand, width, width/4, 32) + } + old, err := load(form.destination) + if err != nil { + return err + } + bank := "zmm" + if form.spec.scalar { + bank = "xmm" + } + constraints := "={" + bank + "4},0" + arguments := typ + " " + old + for position := 0; position < 4; position++ { + source, err := load(form.source(position)) + if err != nil { + return err + } + constraints += fmt.Sprintf(",{%s%d}", bank, position) + arguments += ", " + typ + " " + source + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.memory) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + constraints += ",r" + arguments += ", " + pointerType + " " + pointer + mnemonic, _, _ := strings.Cut(strings.ToLower(string(op)), ".") + assembly := fmt.Sprintf("%s %s($6), %%%s0, %%%s4", mnemonic, segment, bank, bank) + if form.maskIndex >= 0 { + mask, err := c.loadK(ins.Args[form.maskIndex].Reg) + if err != nil { + return err + } + assembly = "kmovw ${7:k}, %k1; " + assembly + " {%k1}" + if form.zeroing { + assembly += " {z}" + } + constraints += ",r,~{k1}" + arguments += ", i32 " + c.approximateMathMask32(mask) + } + constraints += ",~{memory}" + if form.spec.mode == amd64FourSourceFMA { + constraints += ",~{fpsr}" + } + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, typ, assembly, constraints, arguments) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, typ, result, width) + return c.storePackedMoveOperand(form.destination, width, "%"+bytes) +} diff --git a/amd64_lower_four_source_test.go b/amd64_lower_four_source_test.go new file mode 100644 index 000000000..162ef946d --- /dev/null +++ b/amd64_lower_four_source_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "sort" + "strings" + "testing" +) + +func TestAMD64FourSourceGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64FourSourceSpecs)) + for op := range amd64FourSourceSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{"V4FMADDPS", "V4FMADDSS", "V4FNMADDPS", "V4FNMADDSS", "VP4DPWSSD", "VP4DPWSSDS"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("four-source grammar opcodes = %v, want %v", got, want) + } +} + +func TestTranslateX86FourSourceCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT foursourceforms(SB),$0-0 + V4FMADDPS (BX), [Z0-Z3], Z4 + V4FNMADDPS 32(BX), [Z0-Z3], Z5 + V4FMADDSS 64(BX), [X0-X3], X4 + V4FNMADDSS 96(BX), [X0-X3], X5 + VP4DPWSSD 128(BX), [Z0-Z3], Z6 + VP4DPWSSDS 160(BX), [Z0-Z3], Z7 +` + if target.goarch == "amd64" { + source += ` V4FMADDPS 8(BX), [Z2-Z5], K7, Z4 + V4FMADDPS.Z 16(BX), [Z2-Z5], K1, Z4 + V4FNMADDPS 40(BX), [Z2-Z5], K7, Z5 + V4FNMADDPS.Z 48(BX), [Z2-Z5], K2, Z5 + V4FMADDSS 72(BX), [X2-X5], K7, X4 + V4FMADDSS.Z 80(BX), [X2-X5], K3, X4 + V4FNMADDSS 104(BX), [X2-X5], K7, X5 + V4FNMADDSS.Z 112(BX), [X2-X5], K4, X5 + VP4DPWSSD 136(BX), [Z2-Z5], K7, Z6 + VP4DPWSSD.Z 144(BX), [Z2-Z5], K5, Z6 + VP4DPWSSDS 168(BX), [Z2-Z5], K7, Z7 + VP4DPWSSDS.Z 176(BX), [Z2-Z5], K6, Z7 +` + } + source += "\tRET\n" + // Preserve this entire original format and actual Go object. + // It does not initialize BX, sources, accumulator or K inputs; + // the three public APIs must retain Context, not fake success. + ir := x86UnboundVoidScalarFormatObject(t, source, "foursourceforms", target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "four-source-"+target.name+".ll", "four-source-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FourSourceRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "V4FMADDPS Z0, [Z0-Z3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS (AX), [Z0-Z2], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS (AX), [Z0,Z1,Z2,Z3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS (AX), [X0-X3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS (AX), [Z0-Z3], K0, Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS.Z (AX), [Z0-Z3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS.BCST (AX), [Z0-Z3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDSS (AX), [X0-X3], Z4"}, + {goarch: "amd64", instruction: "VP4DPWSSD (AX), [Z0-Z3], X4"}, + {goarch: "386", instruction: "VP4DPWSSD (AX), [Z6-Z9], Z4"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's four-source tables", test.instruction) + } + }) + } +} + +func TestAMD64FourSourceRuntimeSemantics(t *testing.T) { + const source = `TEXT foursourcesemantics(SB),NOSPLIT,$0-24 + MOVQ in+0(FP), AX + MOVQ out+8(FP), BX + MOVQ mask+16(FP), CX + VBROADCASTSS 0(AX), Z0 + VBROADCASTSS 16(AX), Z1 + VBROADCASTSS 32(AX), Z2 + VBROADCASTSS 48(AX), Z3 + VBROADCASTSS 64(AX), Z4 + V4FMADDPS 128(AX), [Z2-Z5], Z4 + VMOVUPS Z4, 0(BX) + VBROADCASTSS 64(AX), Z5 + KMOVQ CX, K1 + V4FNMADDPS.Z 128(AX), [Z0-Z3], K1, Z5 + VMOVUPS Z5, 64(BX) + MOVUPS 0(AX), X0 + MOVUPS 16(AX), X1 + MOVUPS 32(AX), X2 + MOVUPS 48(AX), X3 + MOVUPS 64(AX), X6 + V4FMADDSS 128(AX), [X0-X3], X6 + MOVUPS X6, 128(BX) + VMOVDQU64 192(AX), Z0 + VMOVDQU64 256(AX), Z1 + VMOVDQU64 320(AX), Z2 + VMOVDQU64 384(AX), Z3 + VMOVDQU64 448(AX), Z7 + VP4DPWSSD 144(AX), [Z0-Z3], Z7 + VMOVDQU64 Z7, 144(BX) + VMOVDQU64 448(AX), Z7 + VP4DPWSSDS 144(AX), [Z0-Z3], Z7 + VMOVDQU64 Z7, 208(BX) + RET + +TEXT foursourcefaultsuppression(SB),NOSPLIT,$0-24 + MOVQ memory+0(FP), AX + MOVQ out+8(FP), BX + MOVQ mask+16(FP), CX + KMOVQ CX, K1 + V4FMADDPS.Z (AX), [Z0-Z3], K1, Z4 + VMOVUPS Z4, 0(BX) + V4FMADDSS.Z (AX), [X0-X3], K1, X4 + MOVUPS X4, 64(BX) + VP4DPWSSD.Z (AX), [Z0-Z3], K1, Z5 + VMOVUPS Z5, 80(BX) + RET +` + const mainC = ` +#include +#include +#include +extern void foursourcesemantics(void *, void *, uint64_t); +extern void foursourcefaultsuppression(void *, void *, uint64_t); +static void putf(uint8_t *p, float v) { memcpy(p, &v, 4); } +static float getf(const uint8_t *p) { float v; memcpy(&v, p, 4); return v; } +static void put16(uint8_t *p, int16_t v) { memcpy(p, &v, 2); } +static void put32(uint8_t *p, int32_t v) { memcpy(p, &v, 4); } +static int32_t get32(const uint8_t *p) { int32_t v; memcpy(&v, p, 4); return v; } +int main(void) { + uint8_t in[512] = {0}, out[272] = {0}, faultOut[144] = {1}; + for (int r=0;r<4;r++) for (int lane=0;lane<4;lane++) putf(in+r*16+lane*4, (float)(r+1)); + for (int lane=0;lane<4;lane++) putf(in+64+lane*4, 10.0f+(float)lane); + putf(in+128, 1); putf(in+132, 2); putf(in+136, 3); putf(in+140, 4); + for (int r=0;r<4;r++) for (int lane=0;lane<32;lane++) put16(in+192+r*64+lane*2, (int16_t)(r+1)); + put16(in+144, 1); put16(in+146, 2); put16(in+148, 3); put16(in+150, 4); + put16(in+152, 5); put16(in+154, 6); put16(in+156, 7); put16(in+158, 8); + for (int lane=0;lane<16;lane++) put32(in+448+lane*4, lane==0 ? INT_MAX-5 : lane); + foursourcesemantics(in, out, 5); + foursourcefaultsuppression(0, faultOut, 0); + if (getf(out) != 40 || getf(out+4) != 40 || getf(out+8) != 40 || getf(out+12) != 40) return 10; + if (getf(out+64) != -20) return 21; + if (getf(out+68) != 0) return 22; + if (getf(out+72) != -20) return 23; + if (getf(out+76) != 0) return 24; + if (getf(out+128) != 40 || getf(out+132) != 11 || getf(out+136) != 12 || getf(out+140) != 13) return 12; + if (get32(out+144) != INT_MIN+104 || get32(out+148) != 111) return 13; + if (get32(out+208) != INT_MAX || get32(out+212) != 111) return 14; + for (int i=0;i<144;i++) if (faultOut[i] != 0) return 15; + return 0; +} +` + x86FourSourceOriginalMathCompanion(t, source, mainC) +} diff --git a/amd64_lower_fp.go b/amd64_lower_fp.go index 58f7fdf0c..492fb955c 100644 --- a/amd64_lower_fp.go +++ b/amd64_lower_fp.go @@ -2,164 +2,236 @@ package plan9asm import ( "fmt" + "math" "strconv" "strings" ) func (c *amd64Ctx) lowerFP(op Op, ins Instr) (ok bool, terminated bool, err error) { - switch op { - case "MOVSD", "MOVAPD", "ANDPD", "ANDNPD", "ORPD", "XORPS", - "ADDSD", "SUBSD", "MULSD", "DIVSD", "MAXSD", "MINSD", "SQRTSD", - "COMISD", "CMPSD", "VADDSD", "VFMADD213SD", "VFNMADD231SD", - "CVTSD2SL", "CVTSL2SD", "CVTSQ2SD", "CVTTSD2SQ": - // handled below - default: - return false, false, nil + if known, term, err := c.lowerSqrt(op, ins); known { + return known, term, err } - - switch op { - case "MOVAPD", "ANDPD", "ANDNPD", "ORPD", "XORPS": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, Xdst: %q", op, ins.Raw) + if known, term, err := c.lowerLegacyPackedMove(op, ins); known { + return known, term, err + } + if ok, term, err := c.lowerSignedIntegerScalarFloat(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFloatArithmetic(op, ins, ""); ok { + return ok, term, err + } + if spec, recognized := amd64ScalarVectorMoveSpecs[op]; recognized && spec.legacy { + if _, _, err := amd64ScalarVectorMoveProperties(ins, c.goarch); err != nil { + return true, false, err } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil + } + if ok, term, err := c.lowerApproximateMath(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFixupImmediate(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerScaledRound(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerGetMant(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerGetExp(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFPClass(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFloatingRange(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerReciprocal14(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerReciprocalEstimate(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerRound(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerScalarFloatToInteger(op, ins); ok { + return ok, term, err + } + if raw := strings.ToUpper(string(op)); raw != "" { + base := raw + if dot := strings.IndexByte(raw, '.'); dot >= 0 { + base = raw[:dot] } - dst := ins.Args[1].Reg - srcv, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err + if _, recognized := amd64HorizontalFloatingSpecs[Op(base)]; recognized { + return c.lowerHorizontalFloating(op, ins) } - if op == "MOVAPD" { - return true, false, c.storeX(dst, srcv) + if _, recognized := amd64BinaryFloatingSpecs[Op(base)]; recognized { + return c.lowerBinaryFloating(op, ins) } - dstv, err := c.loadX(dst) - if err != nil { - return true, false, err + if _, recognized := amd64FMA3Specs[Op(base)]; recognized { + return c.lowerFMA3(op, ins) } - t := c.newTmp() - switch op { - case "ANDPD": - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %s, %s\n", t, dstv, srcv) - case "ORPD": - fmt.Fprintf(c.b, " %%%s = or <16 x i8> %s, %s\n", t, dstv, srcv) - case "XORPS": - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", t, dstv, srcv) - case "ANDNPD": - notv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", notv, dstv, llvmAllOnesI8Vec(16)) - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %%%s, %s\n", t, notv, srcv) - } - return true, false, c.storeX(dst, "%"+t) + if _, recognized := x86PackedLogicalSpecs[Op(base)]; recognized { + return c.lowerPackedFloatingLogical(op, ins) + } + if _, known := x86ScalarPrecisionSpecs[Op(base)]; known { + return c.lowerScalarPrecisionConvert(op, ins) + } + } + switch op { + case "MOVSS", "MOVSD", "MOVAPD", "MOVUPD", + "MOVLHPS", "MOVHLPS", "MOVLPS", "MOVHPS", "MOVLPD", "MOVHPD", + "VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD", "SHUFPD", "UNPCKLPS", "UNPCKHPD", + "COMISS", "UCOMISS", "COMISD", "UCOMISD", "CMPSD", + "CVTSL2SD", "CVTSQ2SD": + // handled below + default: + return false, false, nil + } - case "MOVSD": + switch op { + case "MOVSS": if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 MOVSD expects src, dst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 MOVSS expects src, dst: %q", ins.Raw) } - v, err := c.evalF64(ins.Args[0]) + v, err := c.loadLegacyScalarMoveSource(ins, 32) if err != nil { return true, false, err } switch ins.Args[1].Kind { case OpReg: if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return true, false, fmt.Errorf("amd64 MOVSD expects X-reg destination: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 MOVSS expects X-reg destination: %q", ins.Raw) } - return true, false, c.storeXLowF64(ins.Args[1].Reg, v) + return true, false, c.storeLegacyScalarMoveRegister(ins.Args[1].Reg, v, 32, ins.Args[0].Kind == OpReg) case OpFP: - return true, false, c.storeFPResult(ins.Args[1].FPOffset, LLVMType("double"), v) + return true, false, c.storeScalarFPFloat(ins.Args[1].FPOffset, 32, v) case OpMem: addr, err := c.addrFromMem(ins.Args[1].Mem) if err != nil { return true, false, err } - p := c.ptrFromAddrI64(addr) - fmt.Fprintf(c.b, " store double %s, ptr %s, align 1\n", v, p) + fmt.Fprintf(c.b, " store volatile float %s, ptr %s, align 1\n", v, c.ptrFromAddrI64(addr)) return true, false, nil case OpSym: p, err := c.ptrFromSB(ins.Args[1].Sym) if err != nil { return true, false, err } - fmt.Fprintf(c.b, " store double %s, ptr %s, align 1\n", v, p) + fmt.Fprintf(c.b, " store float %s, ptr %s, align 1\n", v, p) return true, false, nil default: - return true, false, fmt.Errorf("amd64 MOVSD unsupported destination: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 MOVSS unsupported destination: %q", ins.Raw) } - case "ADDSD", "SUBSD", "MULSD", "DIVSD", "MAXSD", "MINSD": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, Xdst: %q", op, ins.Raw) + case "MOVLHPS", "MOVHLPS": + if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil + src, err := c.loadXAsI64x2(ins.Args[0].Reg) + if err != nil { + return true, false, err } - src, err := c.evalF64(ins.Args[0]) + dst, err := c.loadXAsI64x2(ins.Args[1].Reg) if err != nil { return true, false, err } - dst, err := c.loadXLowF64(ins.Args[1].Reg) + mask := "" + if op == "MOVHLPS" { + mask = "" + } + shuffled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <2 x i32> %s\n", shuffled, dst, src, mask) + return true, false, c.storeXFromI64x2(ins.Args[1].Reg, "%"+shuffled) + + case "MOVLPS", "MOVHPS", "MOVLPD", "MOVHPD", "VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD": + return c.lowerMOVHalf(op, ins) + + case "SHUFPD": + if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("amd64 SHUFPD expects $imm, Xsrc, Xdst: %q", ins.Raw) + } + src, err := c.loadXAsI64x2(ins.Args[1].Reg) if err != nil { return true, false, err } - t := c.newTmp() - switch op { - case "ADDSD": - fmt.Fprintf(c.b, " %%%s = fadd double %s, %s\n", t, dst, src) - case "SUBSD": - fmt.Fprintf(c.b, " %%%s = fsub double %s, %s\n", t, dst, src) - case "MULSD": - fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", t, dst, src) - case "DIVSD": - fmt.Fprintf(c.b, " %%%s = fdiv double %s, %s\n", t, dst, src) - case "MAXSD": - cmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp ogt double %s, %s\n", cmp, dst, src) - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, double %s, double %s\n", t, cmp, dst, src) - case "MINSD": - cmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp olt double %s, %s\n", cmp, dst, src) - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, double %s, double %s\n", t, cmp, dst, src) + dst, err := c.loadXAsI64x2(ins.Args[2].Reg) + if err != nil { + return true, false, err } - return true, false, c.storeXLowF64(ins.Args[1].Reg, "%"+t) + lo := ins.Args[0].Imm & 1 + hi := 2 + ((ins.Args[0].Imm >> 1) & 1) + shuffled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <2 x i32> \n", shuffled, dst, src, lo, hi) + return true, false, c.storeXFromI64x2(ins.Args[2].Reg, "%"+shuffled) - case "SQRTSD": + case "UNPCKLPS", "UNPCKHPD": if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 SQRTSD expects src, Xdst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 %s expects src, Xdst: %q", op, ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil } - src, err := c.evalF64(ins.Args[0]) + vectorType := "<4 x float>" + maskType := "<4 x i32>" + mask := "" + if op == "UNPCKHPD" { + vectorType = "<2 x double>" + maskType = "<2 x i32>" + mask = "" + } + src, err := c.loadXTypedVectorOperand(ins.Args[0], vectorType) if err != nil { return true, false, err } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call double @llvm.sqrt.f64(double %s)\n", t, src) - return true, false, c.storeXLowF64(ins.Args[1].Reg, "%"+t) + dst, err := c.loadXTypedVectorOperand(ins.Args[1], vectorType) + if err != nil { + return true, false, err + } + shuffled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector %s %s, %s %s, %s %s\n", shuffled, vectorType, dst, vectorType, src, maskType, mask) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytesValue, vectorType, shuffled) + return true, false, c.storeX(ins.Args[1].Reg, "%"+bytesValue) - case "COMISD": + case "MOVSD": if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 COMISD expects src, dst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 MOVSD expects src, dst: %q", ins.Raw) } - src, err := c.evalF64(ins.Args[0]) + v, err := c.loadLegacyScalarMoveSource(ins, 64) if err != nil { return true, false, err } - dst, err := c.evalF64(ins.Args[1]) - if err != nil { - return true, false, err + switch ins.Args[1].Kind { + case OpReg: + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + return true, false, fmt.Errorf("amd64 MOVSD expects X-reg destination: %q", ins.Raw) + } + return true, false, c.storeLegacyScalarMoveRegister(ins.Args[1].Reg, v, 64, ins.Args[0].Kind == OpReg) + case OpFP: + return true, false, c.storeScalarFPFloat(ins.Args[1].FPOffset, 64, v) + case OpMem: + addr, err := c.addrFromMem(ins.Args[1].Mem) + if err != nil { + return true, false, err + } + p := c.ptrFromAddrI64(addr) + fmt.Fprintf(c.b, " store volatile double %s, ptr %s, align 1\n", v, p) + return true, false, nil + case OpSym: + p, err := c.ptrFromSB(ins.Args[1].Sym) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store double %s, ptr %s, align 1\n", v, p) + return true, false, nil + default: + return true, false, fmt.Errorf("amd64 MOVSD unsupported destination: %q", ins.Raw) } - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp ueq double %s, %s\n", z, dst, src) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", z, c.flagsZSlot) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp ult double %s, %s\n", cf, dst, src) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - lt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp olt double %s, %s\n", lt, dst, src) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", lt, c.flagsSltSlot) - return true, false, nil + + case "COMISS", "UCOMISS", "COMISD", "UCOMISD": + return c.lowerVectorScalarFlagCompare(op, ins) case "CMPSD": if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg { @@ -213,136 +285,6 @@ func (c *amd64Ctx) lowerFP(op Op, ins Instr) (ok bool, terminated bool, err erro fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 -1, i64 0\n", mask, cmp) return true, false, c.storeXLowI64(ins.Args[1].Reg, "%"+mask) - case "CVTTSD2SQ": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 CVTTSD2SQ expects src, dstReg: %q", ins.Raw) - } - src, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fptosi double %s to i64\n", t, src) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+t) - - case "CVTSQ2SD": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 CVTSQ2SD expects src, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sitofp i64 %s to double\n", t, src) - return true, false, c.storeXLowF64(ins.Args[1].Reg, "%"+t) - - case "CVTSD2SL": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 CVTSD2SL expects src, dstReg: %q", ins.Raw) - } - src, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - round := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call double @llvm.rint.f64(double %s)\n", round, src) - i32v := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fptosi double %%%s to i32\n", i32v, round) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, i32v) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+z) - - case "CVTSL2SD": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 CVTSL2SD expects srcReg, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - i32v := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", i32v, src) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sitofp i32 %%%s to double\n", t, i32v) - return true, false, c.storeXLowF64(ins.Args[1].Reg, "%"+t) - - case "VADDSD": - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VADDSD expects src1, src2, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - s1, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - s2, err := c.evalF64(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fadd double %s, %s\n", t, s2, s1) - return true, false, c.storeXLowF64(ins.Args[2].Reg, "%"+t) - - case "VFMADD213SD": - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VFMADD213SD expects src1, src2, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - s1, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - s2, err := c.evalF64(ins.Args[1]) - if err != nil { - return true, false, err - } - dst, err := c.loadXLowF64(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - mul := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", mul, dst, s2) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fadd double %%%s, %s\n", t, mul, s1) - return true, false, c.storeXLowF64(ins.Args[2].Reg, "%"+t) - - case "VFNMADD231SD": - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VFNMADD231SD expects src1, src2, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - s1, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - s2, err := c.evalF64(ins.Args[1]) - if err != nil { - return true, false, err - } - dst, err := c.loadXLowF64(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - mul := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", mul, s1, s2) - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fneg double %%%s\n", neg, mul) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fadd double %%%s, %s\n", t, neg, dst) - return true, false, c.storeXLowF64(ins.Args[2].Reg, "%"+t) } return false, false, nil } @@ -371,11 +313,43 @@ func (c *amd64Ctx) loadXVecOperand(op Operand) (string, error) { t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", t, p) return "%" + t, nil + case OpFP: + // ABI0 fields need not be two contiguous uint64 values. Assemble + // every declared byte, including adjacent float32 and narrow fields. + return c.loadFPVectorBytes(op.FPOffset, 16) default: return "", fmt.Errorf("amd64: unsupported X-vector operand %s", op.String()) } } +func (c *amd64Ctx) storeXVecOperand(op Operand, value string) error { + switch op.Kind { + case OpReg: + if !isAMD64XReg(op.Reg) { + return fmt.Errorf("expected X register, got %s", op.String()) + } + return c.storeX(op.Reg, value) + case OpMem: + addr, err := c.addrFromMem(op.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s, align 1\n", value, c.ptrFromAddrI64(addr)) + return nil + case OpSym: + p, err := c.ptrFromSB(op.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s, align 1\n", value, p) + return nil + case OpFP: + return c.storeFPVectorBytes(op.FPOffset, 16, value, "") + default: + return fmt.Errorf("unsupported X-vector destination %s", op.String()) + } +} + func (c *amd64Ctx) loadYVecOperand(op Operand) (string, error) { switch op.Kind { case OpReg: @@ -456,6 +430,20 @@ func (c *amd64Ctx) loadXLowF64(r Reg) (string, error) { return "%" + t, nil } +func (c *amd64Ctx) loadXLowF32(r Reg) (string, error) { + xv, err := c.loadX(r) + if err != nil { + return "", err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", words, xv) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <4 x i32> %%%s, i32 0\n", low, words) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", value, low) + return "%" + value, nil +} + func (c *amd64Ctx) storeXLowI64(r Reg, low string) error { cur, err := c.loadX(r) if err != nil { @@ -480,6 +468,231 @@ func (c *amd64Ctx) storeXLowF64(r Reg, v string) error { return c.storeXLowI64(r, "%"+t) } +// Legacy scalar moves preserve bits above 127 in every physical alias. The +// source class separately determines whether the upper XMM lanes survive. +// Do not use this helper for VEX/EVEX writes, which zero their wider views. +func (c *amd64Ctx) storeLegacyScalarMoveRegister(r Reg, value string, bits int, preserve bool) error { + var store func(Reg, string) error + switch bits { + case 32: + store = c.storeXLowF32ClearingUpper + if preserve { + store = c.storeXLowF32 + } + case 64: + store = c.storeXLowF64ClearingUpper + if preserve { + store = c.storeXLowF64 + } + default: + return fmt.Errorf("invalid legacy scalar move width %d", bits) + } + if err := store(r, value); err != nil { + return err + } + xmm, err := c.loadX(r) + if err != nil { + return err + } + return c.storeLegacyXViews(r, xmm) +} + +func (c *amd64Ctx) storeXLowF64ClearingUpper(r Reg, v string) error { + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", bits, v) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> zeroinitializer, i64 %%%s, i32 0\n", inserted, bits) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", back, inserted) + return c.storeX(r, "%"+back) +} + +func (c *amd64Ctx) storeXLowF32(r Reg, v string) error { + cur, err := c.loadX(r) + if err != nil { + return err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", words, cur) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", bits, v) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> %%%s, i32 %%%s, i32 0\n", inserted, words, bits) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", back, inserted) + return c.storeX(r, "%"+back) +} + +func (c *amd64Ctx) storeXLowF32ClearingUpper(r Reg, v string) error { + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", bits, v) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> zeroinitializer, i32 %%%s, i32 0\n", inserted, bits) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", back, inserted) + return c.storeX(r, "%"+back) +} + +func (c *amd64Ctx) loadXTypedVectorOperand(op Operand, vectorType string) (string, error) { + bytesValue, err := c.loadXVecOperand(op) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", value, bytesValue, vectorType) + return "%" + value, nil +} + +func (c *amd64Ctx) loadXAsI64x2(r Reg) (string, error) { + if _, ok := amd64ParseXReg(r); !ok { + return "", fmt.Errorf("amd64: expected X register, got %s", r) + } + bytesValue, err := c.loadX(r) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", value, bytesValue) + return "%" + value, nil +} + +func (c *amd64Ctx) storeXFromI64x2(r Reg, value string) error { + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %s to <16 x i8>\n", bytesValue, value) + return c.storeX(r, "%"+bytesValue) +} + +type amd64MoveHalfSpec struct { + high bool + vector bool + registerOnly bool +} + +// The memory halves use yxmov/_yvmovhpd. The explicit register-only aliases +// add yxr/_yvmovhlps while sharing the selected source/base/output halves. +// VEX/EVEX memory halves still accept only m64 loads or stores. +var amd64MoveHalfSpecs = map[Op]amd64MoveHalfSpec{ + "MOVHLPS": {registerOnly: true}, + "MOVLHPS": {high: true, registerOnly: true}, + "VMOVHLPS": {vector: true, registerOnly: true}, + "VMOVLHPS": {high: true, vector: true, registerOnly: true}, + "MOVLPS": {}, + "MOVHPS": {high: true}, + "MOVLPD": {}, + "MOVHPD": {high: true}, + "VMOVLPS": {vector: true}, + "VMOVHPS": {high: true, vector: true}, + "VMOVLPD": {vector: true}, + "VMOVHPD": {high: true, vector: true}, +} + +func (c *amd64Ctx) lowerMOVHalf(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86MoveHalfProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if form.undocumentedPDRegister { + return true, false, fmt.Errorf("%w: %s's Go register encoding lacks documented Intel PD semantics", ErrProbeNeedsContext, op) + } + return true, false, c.emitMoveHalf(form, ins) +} + +func (c *amd64Ctx) emitMoveHalf(form x86MoveHalfForm, ins Instr) error { + if form.store { + source, err := c.loadXAsI64x2(ins.Args[form.source].Reg) + if err != nil { + return err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %s, i32 %d\n", bits, source, form.sourceLane) + return c.storePackedScalarExtractMemory(ins.Args[form.destination], I64, "%"+bits) + } + var bits string + if form.memory >= 0 { + var err error + if source := ins.Args[form.source]; source.Kind == OpFP { + bits, err = c.loadScalarLaneFPBits(source.FPOffset, 64) + } else { + bits, err = c.evalSizedNativeGPSource(source, I64, ins.x86AddressBits) + } + if err != nil { + return err + } + } else { + source, err := c.loadXAsI64x2(ins.Args[form.source].Reg) + if err != nil { + return err + } + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %s, i32 %d\n", extracted, source, form.sourceLane) + bits = "%" + extracted + } + base, err := c.loadXAsI64x2(ins.Args[form.base].Reg) + if err != nil { + return err + } + inserted, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> %s, i64 %s, i32 %d\n", inserted, base, bits, form.outputLane) + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bytes, inserted) + if form.spec.vector { + return c.storePackedMoveOperand(ins.Args[form.destination], 16, "%"+bytes) + } + return c.storeLegacyXViews(ins.Args[form.destination].Reg, "%"+bytes) +} + +func (c *amd64Ctx) evalF32(op Operand) (string, error) { + switch op.Kind { + case OpReg: + if _, ok := amd64ParseXReg(op.Reg); !ok { + return "", fmt.Errorf("amd64: expected X register for f32 operand, got %s", op.String()) + } + return c.loadXLowF32(op.Reg) + case OpImm: + bits := c.newTmp() + if op.ImmIsFloat { + // The parser retains the original float64 constant. Go's + // Float32Sym rounds its numeric value, not its low 32 bits. + rounded := math.Float32bits(float32(math.Float64frombits(uint64(op.Imm)))) + fmt.Fprintf(c.b, " %%%s = bitcast i32 %d to float\n", bits, rounded) + return "%" + bits, nil + } + fmt.Fprintf(c.b, " %%%s = trunc i64 %d to i32\n", bits, op.Imm) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", value, bits) + return "%" + value, nil + case OpFP: + return c.evalFPToF32(op.FPOffset) + case OpMem: + p, ptrType, err := c.ptrFromMem(op.Mem) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load float, %s %s, align 1\n", value, ptrType, p) + return "%" + value, nil + case OpSym: + p, err := c.ptrFromSB(op.Sym) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load float, ptr %s, align 1\n", value, p) + return "%" + value, nil + default: + return "", fmt.Errorf("amd64: unsupported f32 operand %s", op.String()) + } +} + +func (c *amd64Ctx) evalFPToF32(off int64) (string, error) { + bits, err := c.loadScalarFPBits(off, 32) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %s to float\n", value, bits) + return "%" + value, nil +} + func (c *amd64Ctx) evalF64(op Operand) (string, error) { switch op.Kind { case OpReg: @@ -515,34 +728,11 @@ func (c *amd64Ctx) evalF64(op Operand) (string, error) { } func (c *amd64Ctx) evalFPToF64(off int64) (string, error) { - slot, ok := c.fpParam(off) - if !ok { - return "", fmt.Errorf("unsupported FP read slot for f64: +%d(FP)", off) - } - idx := slot.Index - if idx < 0 || idx >= len(c.sig.Args) { - return "", fmt.Errorf("FP read slot for f64: invalid arg index %d at +%d(FP)", idx, off) - } - arg := fmt.Sprintf("%%arg%d", idx) - ty := slot.Type - if c.classicFrame != "" { - loaded := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", loaded, ty, c.classicFramePtr(off)) - arg = "%" + loaded - } else if slot.Field >= 0 { - aggTy := c.sig.Args[idx] - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", t, aggTy, arg, slot.Field) - arg = "%" + t - } - switch ty { - case LLVMType("double"): - return arg, nil - case I64: - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to double\n", t, arg) - return "%" + t, nil - default: - return "", fmt.Errorf("FP read unsupported type %q for f64 at +%d(FP)", ty, off) + bits, err := c.loadScalarFPBits(off, 64) + if err != nil { + return "", err } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to double\n", value, bits) + return "%" + value, nil } diff --git a/amd64_lower_fpclass.go b/amd64_lower_fpclass.go new file mode 100644 index 000000000..dc12c9514 --- /dev/null +++ b/amd64_lower_fpclass.go @@ -0,0 +1,207 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FPClassSpec struct { + laneBits int + bytes int + scalar bool +} + +// amd64FPClassSpecs mirrors the width-specialized opcode names in Go 1.27's +// _yvfpclasspdx/_yvfpclasspdy/_yvfpclasspdz tables. Classification itself is +// shared across every packed and scalar form. +var amd64FPClassSpecs = map[Op]amd64FPClassSpec{ + "VFPCLASSPDX": {laneBits: 64, bytes: 16}, + "VFPCLASSPDY": {laneBits: 64, bytes: 32}, + "VFPCLASSPDZ": {laneBits: 64, bytes: 64}, + "VFPCLASSPSX": {laneBits: 32, bytes: 16}, + "VFPCLASSPSY": {laneBits: 32, bytes: 32}, + "VFPCLASSPSZ": {laneBits: 32, bytes: 64}, + "VFPCLASSSD": {laneBits: 64, bytes: 16, scalar: true}, + "VFPCLASSSS": {laneBits: 32, bytes: 16, scalar: true}, +} + +func (c *amd64Ctx) lowerFPClass(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64FPClassSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return c.lowerNativeFloatingHelper(op, ins) + } + broadcast := suffix == "BCST" + if suffix != "" && !broadcast { + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's FPCLASS optab: %q", c.goarch, baseOp, ins.Raw) + } + if spec.scalar && broadcast { + return true, false, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects immediate, source, [K mask,] K destination: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("%s %s immediate must be an unsigned byte: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("%s %s input mask must be K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if !amd64IsKOperand(destination) { + return true, false, fmt.Errorf("%s %s destination must be K0-K7: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[1] + if broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, spec.bytes) { + return true, false, fmt.Errorf("%s %s source register does not match its opcode width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be its named vector width or memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := spec.bytes * 8 / spec.laneBits + var bits string + if spec.scalar { + scalarBits, loadErr := c.loadFloatingScalarBits(source, spec.laneBits) + if loadErr != nil { + return true, false, loadErr + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", inserted, spec.laneBits, spec.laneBits, scalarBits) + bits = "%" + inserted + lanes = 1 + } else { + bits, err = c.loadPackedCompareLanes(source, spec.bytes, spec.laneBits, broadcast) + if err != nil { + return true, false, err + } + } + classified := c.emitFPClassPredicate(lanes, spec.laneBits, bits, uint8(ins.Args[0].Imm)) + var result string + if lanes == 1 { + low := c.newTmp() + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i1> %s, i32 0\n", low, classified) + fmt.Fprintf(c.b, " %%%s = zext i1 %%%s to i64\n", wide, low) + result = "%" + wide + } else { + result = c.packFloatingCompareMask(classified, lanes) + } + if masked { + writeMask, loadErr := c.loadK(ins.Args[2].Reg) + if loadErr != nil { + return true, false, loadErr + } + maskedResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", maskedResult, result, writeMask) + result = "%" + maskedResult + } + return true, false, c.storeK(destination.Reg, result) +} + +func (c *amd64Ctx) emitFPClassPredicate(lanes, laneBits int, bits string, immediate uint8) string { + integerType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + predicateType := fmt.Sprintf("<%d x i1>", lanes) + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + if laneBits == 64 { + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + } + exponent, mantissa, quiet := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", exponent, integerType, bits, llvmSplatInteger(lanes, laneBits, exponentMask)) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", mantissa, integerType, bits, llvmSplatInteger(lanes, laneBits, mantissaMask)) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", quiet, integerType, bits, llvmSplatInteger(lanes, laneBits, quietBit)) + exponentOnes, exponentNotOnes := c.newTmp(), c.newTmp() + exponentZero, exponentNotZero := c.newTmp(), c.newTmp() + mantissaZero, mantissaNonzero := c.newTmp(), c.newTmp() + quietSet, quietClear := c.newTmp(), c.newTmp() + negative, nonnegative := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %s\n", exponentOnes, integerType, exponent, llvmSplatInteger(lanes, laneBits, exponentMask)) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, %s\n", exponentNotOnes, integerType, exponent, llvmSplatInteger(lanes, laneBits, exponentMask)) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", exponentZero, integerType, exponent) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", exponentNotZero, integerType, exponent) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", mantissaZero, integerType, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", mantissaNonzero, integerType, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", quietSet, integerType, quiet) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", quietClear, integerType, quiet) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, zeroinitializer\n", negative, integerType, bits) + fmt.Fprintf(c.b, " %%%s = icmp sge %s %s, zeroinitializer\n", nonnegative, integerType, bits) + daz := c.loadMXCSRDAZ() + dazClear := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i1 %s, false\n", dazClear, daz) + dazSetVector := amd64SplatI1(c, lanes, daz) + dazClearVector := amd64SplatI1(c, lanes, "%"+dazClear) + and := func(first, second string) string { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", value, predicateType, first, second) + return "%" + value + } + categories := [8]string{} + nan := and("%"+exponentOnes, "%"+mantissaNonzero) + categories[0] = and(nan, "%"+quietSet) + effectiveMantissaZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", effectiveMantissaZero, predicateType, mantissaZero, dazSetVector) + zero := and("%"+exponentZero, "%"+effectiveMantissaZero) + categories[1] = and(zero, "%"+nonnegative) + categories[2] = and(zero, "%"+negative) + infinity := and("%"+exponentOnes, "%"+mantissaZero) + categories[3] = and(infinity, "%"+nonnegative) + categories[4] = and(infinity, "%"+negative) + effectiveMantissaNonzero := and("%"+mantissaNonzero, dazClearVector) + categories[5] = and("%"+exponentZero, effectiveMantissaNonzero) + negativeFinite := and("%"+negative, "%"+exponentNotOnes) + effectiveNonzero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", effectiveNonzero, predicateType, exponentNotZero, effectiveMantissaNonzero) + categories[6] = and(negativeFinite, "%"+effectiveNonzero) + categories[7] = and(nan, "%"+quietClear) + result := "zeroinitializer" + for category, predicate := range categories { + if immediate&(1< +#include +#include +extern void fpclasssemantics(uint8_t *, const uint32_t *, const uint64_t *, uint64_t); +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + const uint32_t source32[16]={ + 0x7fc12345,0x00000000,0x80000000,0x7f800000, + 0xff800000,0x00000001,0xbf800000,0x7f812345, + 0x3f800000,0x40000000,0x00800000,0x7f7fffff, + 0x3f000000,0x40400000,0x3f800001,0x3f800002}; + const uint64_t source64[8]={ + UINT64_C(0x7ff8123456789abc),UINT64_C(0),UINT64_C(0x8000000000000000),UINT64_C(0x7ff0000000000000), + UINT64_C(0xfff0000000000000),UINT64_C(1),UINT64_C(0xbff0000000000000),UINT64_C(0x7ff0123456789abc)}; + uint8_t out[273]={0}, outDaz[273]={0}; + fpclasssemantics(out,source32,source64,UINT64_C(0xa5)); + unsigned oldCsr=_mm_getcsr(); + _mm_setcsr(oldCsr|UINT32_C(0x40)); + fpclasssemantics(outDaz,source32,source64,UINT64_C(0xa5)); + _mm_setcsr(oldCsr); + for (int category=0;category<8;category++) { + uint64_t bit=UINT64_C(1)<= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + lanes := byteWidth * 8 / spec.laneBits + // Every VBMI2 member is E4: retain exact volatile active-lane reads, + // including discarded data/counts, and suppress only masked-off lanes. + first, err := c.loadX86PackedShiftLanes(ins.Args[form.first], byteWidth, spec.laneBits, form.broadcast, mask) + if err != nil { + return true, false, err + } + second, err := c.loadPackedCompareLanes(ins.Args[form.second], byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + old := "zeroinitializer" + if spec.variable || form.mask >= 0 && !form.zeroing { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + } + + count := first + leftInput, rightInput := old, second + if !spec.variable { + count = amd64SplatInteger(c, lanes, spec.laneBits, fmt.Sprintf("%d", uint8(ins.Args[0].Imm))) + leftInput, rightInput = second, first + } + intrinsic := "fshl" + firstArgument, secondArgument := leftInput, rightInput + if !spec.left { + intrinsic = "fshr" + firstArgument, secondArgument = rightInput, leftInput + } + vectorType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + computedName := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.%s.v%di%d(%s %s, %s %s, %s %s)\n", + computedName, vectorType, intrinsic, lanes, spec.laneBits, + vectorType, firstArgument, vectorType, secondArgument, vectorType, count) + computed := "%" + computedName + if form.mask >= 0 { + computed = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computed, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, vectorType, computed, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} diff --git a/amd64_lower_funnel_shift_test.go b/amd64_lower_funnel_shift_test.go new file mode 100644 index 000000000..b5c2ec36d --- /dev/null +++ b/amd64_lower_funnel_shift_test.go @@ -0,0 +1,268 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86FunnelShiftCompleteGo127Forms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT funnel_shift_forms(SB),$0-0\n") + if target.goarch == "amd64" { + for _, op := range []string{"VPSHLDW", "VPSHLDD", "VPSHLDQ", "VPSHRDW", "VPSHRDD", "VPSHRDQ"} { + broadcast := !strings.HasSuffix(op, "W") + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s $0, %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s $255, 8(BX), %s3, %s4\n", op, width, width) + fmt.Fprintf(&source, "\t%s $1, %s20, %s21, K1, %s22\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z $2, 16(BX), %s23, K7, %s24\n", op, width, width) + if broadcast { + fmt.Fprintf(&source, "\t%s.BCST $3, 24(BX), %s25, %s26\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z $4, 32(BX), %s27, K2, %s28\n", op, width, width) + } + } + } + } + for _, op := range []string{"VPSHLDVW", "VPSHLDVD", "VPSHLDVQ", "VPSHRDVW", "VPSHRDVD", "VPSHRDVQ"} { + broadcast := !strings.HasSuffix(op, "VW") + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s 8(BX), %s3, %s4\n", op, width, width) + if broadcast { + fmt.Fprintf(&source, "\t%s.BCST 16(BX), %s5, %s6\n", op, width, width) + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 24(BX), %s23, K7, %s24\n", op, width, width) + if broadcast { + fmt.Fprintf(&source, "\t%s.BCST.Z 32(BX), %s25, K2, %s26\n", op, width, width) + } + } + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, signatures := x86UnboundVoidFormatSources(t, source.String(), target.goarch, target.triple) + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerFunnelShift(ins.Op, ins) }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "funnel-shift-"+target.name+".ll", "funnel-shift-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FunnelShiftRejectsFormsOutsideGo127Tables(t *testing.T) { + for _, instruction := range []string{ + "VPSHRDQ $0, X0, X1", + "VPSHLDW $-1, X0, X1, X2", + "VPSHRDD $256, X0, X1, X2", + "VPSHLDQ AX, X0, X1, X2", + "VPSHRDW $0, Y0, X1, X2", + "VPSHLDVD Y0, X1, X2", + "VPSHRDVQ X0, Y1, X2", + "VPSHLDVW X0, X1, Y2", + "VPSHRDQ $0, X0, 8(BX), X2", + "VPSHLDVD X0, 8(BX), X2", + "VPSHRDVQ X0, X1, 8(BX)", + "VPSHLDQ $0, X0, X1, K0, X2", + "VPSHRDVD X0, K1, X1, X2", + "VPSHLDW.Z $0, X0, X1, X2", + "VPSHRDVW.Z X0, X1, X2", + "VPSHLDW.BCST $0, 8(BX), X1, X2", + "VPSHRDVW.BCST 8(BX), X1, X2", + "VPSHLDQ.BCST X0, X1, X2", + "VPSHRDVD.BCST X0, X1, X2", + "VPSHLDQ.Z.BCST $0, 8(BX), X1, K1, X2", + "VPSHRDVQ.SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's funnel-shift tables", instruction) + } + }) + } + for _, instruction := range []string{ + "VPSHRDQ $0, X0, X1, X2", + "VPSHLDW $0, 8(BX), Y1, Y2", + "VPSHRDVD X0, X1, K1, X2", + "VPSHLDVQ.Z 8(BX), Y1, K7, Y2", + "VPSHRDVD.BCST.Z 16(BX), Z1, K2, Z2", + "VPSHLDVW Z8, Z0, Z1", + "VPSHRDVQ Z0, Z8, Z1", + "VPSHLDVD Z0, Z1, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted 386 %q outside Go 1.27's frontend boundary", instruction) + } + }) + } +} + +func TestAMD64FunnelShiftRuntimeSemantics(t *testing.T) { + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable integer oracle requires its native 64-bit host counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT funnelShiftSemantics(SB),$0-64 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + MOVQ old+24(FP), DX + MOVQ counts+32(FP), SI + MOVQ scalarCount+40(FP), R8 + MOVQ scalarData+48(FP), R9 + VMOVDQU64 (BX), X0 + VMOVDQU64 (CX), X1 + VMOVDQU64 (SI), X3 + VPSHLDW $4, X0, X1, X4 + VMOVDQU64 X4, 0(AX) + VPSHRDW $4, X0, X1, X5 + VMOVDQU64 X5, 16(AX) + VPSHLDD $8, X0, X1, X6 + VMOVDQU64 X6, 32(AX) + VPSHRDD $8, X0, X1, X7 + VMOVDQU64 X7, 48(AX) + VPSHLDQ $13, X0, X1, X8 + VMOVDQU64 X8, 64(AX) + VPSHRDQ $32, X0, X1, X9 + VMOVDQU64 X9, 80(AX) + VMOVDQU64 (DX), X10 + VPSHLDVW X3, X1, X10 + VMOVDQU64 X10, 96(AX) + VMOVDQU64 (DX), X11 + VPSHRDVD X3, X1, X11 + VMOVDQU64 X11, 112(AX) + VMOVDQU64 (DX), X12 + VPSHLDVQ X3, X1, X12 + VMOVDQU64 X12, 128(AX) + KMOVQ mask+56(FP), K1 + VMOVDQU64 (DX), X13 + VPSHRDQ $32, X0, X1, K1, X13 + VMOVDQU64 X13, 144(AX) + VMOVDQU64 (DX), X14 + VPSHRDQ.Z $32, X0, X1, K1, X14 + VMOVDQU64 X14, 160(AX) + VMOVDQU64 (DX), X15 + VPSHLDVD.BCST (R8), X1, X15 + VMOVDQU64 X15, 176(AX) + VPSHRDD.BCST $8, (R9), X1, X16 + VMOVDQU64 X16, 192(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: I64, Index: 7, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "funnelShiftSemantics": { + Name: "funnelShiftSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, + Ret: Void, + Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +typedef union { uint8_t b[16]; uint16_t w[8]; uint32_t d[4]; uint64_t q[2]; } V; +extern void funnelShiftSemantics(V *, const V *, const V *, const V *, const V *, const uint32_t *, const uint32_t *, uint64_t); +static uint16_t shl16(uint16_t a, uint16_t b, unsigned c) { c &= 15; return c ? (uint16_t)((a << c) | (b >> (16-c))) : a; } +static uint16_t shr16(uint16_t a, uint16_t b, unsigned c) { c &= 15; return c ? (uint16_t)((a >> c) | (b << (16-c))) : a; } +static uint32_t shl32(uint32_t a, uint32_t b, unsigned c) { c &= 31; return c ? (a << c) | (b >> (32-c)) : a; } +static uint32_t shr32(uint32_t a, uint32_t b, unsigned c) { c &= 31; return c ? (a >> c) | (b << (32-c)) : a; } +static uint64_t shl64(uint64_t a, uint64_t b, unsigned c) { c &= 63; return c ? (a << c) | (b >> (64-c)) : a; } +static uint64_t shr64(uint64_t a, uint64_t b, unsigned c) { c &= 63; return c ? (a >> c) | (b << (64-c)) : a; } +int main(void) { + V first = {.q = {UINT64_C(0x0123456789abcdef), UINT64_C(0xfedcba9876543210)}}; + V second = {.q = {UINT64_C(0x8877665544332211), UINT64_C(0x1020304050607080)}}; + V old = {.q = {UINT64_C(0x0f1e2d3c4b5a6978), UINT64_C(0x8796a5b4c3d2e1f0)}}; + V counts = {.q = {UINT64_C(0x001f000800030000), UINT64_C(0x0041002000110004)}}; + uint32_t scalarCount = 37; + uint32_t scalarData = UINT32_C(0xa1b2c3d4); + V out[13] = {0}, want[13] = {0}; + for (int i = 0; i < 8; i++) { + want[0].w[i] = shl16(second.w[i], first.w[i], 4); + want[1].w[i] = shr16(second.w[i], first.w[i], 4); + want[6].w[i] = shl16(old.w[i], second.w[i], counts.w[i]); + } + for (int i = 0; i < 4; i++) { + want[2].d[i] = shl32(second.d[i], first.d[i], 8); + want[3].d[i] = shr32(second.d[i], first.d[i], 8); + want[7].d[i] = shr32(old.d[i], second.d[i], counts.d[i]); + want[11].d[i] = shl32(old.d[i], second.d[i], scalarCount); + want[12].d[i] = shr32(second.d[i], scalarData, 8); + } + for (int i = 0; i < 2; i++) { + want[4].q[i] = shl64(second.q[i], first.q[i], 13); + want[5].q[i] = shr64(second.q[i], first.q[i], 32); + want[8].q[i] = shl64(old.q[i], second.q[i], counts.q[i]); + } + want[9].q[0] = want[5].q[0]; want[9].q[1] = old.q[1]; + want[10].q[0] = want[5].q[0]; want[10].q[1] = 0; + funnelShiftSemantics(out, &first, &second, &old, &counts, &scalarCount, &scalarData, 1); + return memcmp(out, want, sizeof(out)) == 0 ? 0 : 1; +} +` + x86ConflictFunnelPortableRuntime(t, llc, clang, "funnel_shift", triple, ir, mainC) +} diff --git a/amd64_lower_gather.go b/amd64_lower_gather.go new file mode 100644 index 000000000..b9a0cb0cc --- /dev/null +++ b/amd64_lower_gather.go @@ -0,0 +1,293 @@ +package plan9asm + +import "fmt" + +type amd64GatherTable uint8 + +const ( + amd64GatherDPS amd64GatherTable = iota + amd64GatherDPD + amd64GatherQPS +) + +type amd64GatherSpec struct { + table amd64GatherTable + elemBits int + indexBits int +} + +// amd64GatherSpecs is the complete Go 1.27 data-gather family. The eight +// spellings share _yvgatherdps, _yvgatherdpd, and _yvgatherqps. +var amd64GatherSpecs = map[Op]amd64GatherSpec{ + "VGATHERDPS": {table: amd64GatherDPS, elemBits: 32, indexBits: 32}, + "VGATHERQPD": {table: amd64GatherDPS, elemBits: 64, indexBits: 64}, + "VPGATHERDD": {table: amd64GatherDPS, elemBits: 32, indexBits: 32}, + "VPGATHERQQ": {table: amd64GatherDPS, elemBits: 64, indexBits: 64}, + "VGATHERDPD": {table: amd64GatherDPD, elemBits: 64, indexBits: 32}, + "VPGATHERDQ": {table: amd64GatherDPD, elemBits: 64, indexBits: 32}, + "VGATHERQPS": {table: amd64GatherQPS, elemBits: 32, indexBits: 64}, + "VPGATHERQD": {table: amd64GatherQPS, elemBits: 32, indexBits: 64}, +} + +type amd64VSIBPrefetchDirection uint8 + +const ( + amd64VSIBPrefetchGather amd64VSIBPrefetchDirection = iota + amd64VSIBPrefetchScatter +) + +type amd64VSIBPrefetchSpec struct { + direction amd64VSIBPrefetchDirection + hint int + elemBits int + indexBits int + indexBytes int +} + +// amd64VSIBPrefetchSpecs is the complete Go 1.27 gather/scatter prefetch +// grammar. Direction, cache hint, element width, and VSIB index width are data; +// all sixteen opcodes share one exact operand validator. These instructions +// are architectural hints, so lowering has no observable IR side effect. +var amd64VSIBPrefetchSpecs = map[Op]amd64VSIBPrefetchSpec{ + "VGATHERPF0DPD": {direction: amd64VSIBPrefetchGather, hint: 0, elemBits: 64, indexBits: 32, indexBytes: 32}, + "VGATHERPF0DPS": {direction: amd64VSIBPrefetchGather, hint: 0, elemBits: 32, indexBits: 32, indexBytes: 64}, + "VGATHERPF0QPD": {direction: amd64VSIBPrefetchGather, hint: 0, elemBits: 64, indexBits: 64, indexBytes: 64}, + "VGATHERPF0QPS": {direction: amd64VSIBPrefetchGather, hint: 0, elemBits: 32, indexBits: 64, indexBytes: 64}, + "VGATHERPF1DPD": {direction: amd64VSIBPrefetchGather, hint: 1, elemBits: 64, indexBits: 32, indexBytes: 32}, + "VGATHERPF1DPS": {direction: amd64VSIBPrefetchGather, hint: 1, elemBits: 32, indexBits: 32, indexBytes: 64}, + "VGATHERPF1QPD": {direction: amd64VSIBPrefetchGather, hint: 1, elemBits: 64, indexBits: 64, indexBytes: 64}, + "VGATHERPF1QPS": {direction: amd64VSIBPrefetchGather, hint: 1, elemBits: 32, indexBits: 64, indexBytes: 64}, + "VSCATTERPF0DPD": {direction: amd64VSIBPrefetchScatter, hint: 0, elemBits: 64, indexBits: 32, indexBytes: 32}, + "VSCATTERPF0DPS": {direction: amd64VSIBPrefetchScatter, hint: 0, elemBits: 32, indexBits: 32, indexBytes: 64}, + "VSCATTERPF0QPD": {direction: amd64VSIBPrefetchScatter, hint: 0, elemBits: 64, indexBits: 64, indexBytes: 64}, + "VSCATTERPF0QPS": {direction: amd64VSIBPrefetchScatter, hint: 0, elemBits: 32, indexBits: 64, indexBytes: 64}, + "VSCATTERPF1DPD": {direction: amd64VSIBPrefetchScatter, hint: 1, elemBits: 64, indexBits: 32, indexBytes: 32}, + "VSCATTERPF1DPS": {direction: amd64VSIBPrefetchScatter, hint: 1, elemBits: 32, indexBits: 32, indexBytes: 64}, + "VSCATTERPF1QPD": {direction: amd64VSIBPrefetchScatter, hint: 1, elemBits: 64, indexBits: 64, indexBytes: 64}, + "VSCATTERPF1QPS": {direction: amd64VSIBPrefetchScatter, hint: 1, elemBits: 32, indexBits: 64, indexBytes: 64}, +} + +func (c *amd64Ctx) lowerGather(op Op, ins Instr) error { + local := ins + local.Op = op + form, known, err := x86VSIBProperties(local, c.goarch) + if err != nil { + return err + } + if !known || form.scatter && !form.prefetch { + return fmt.Errorf("amd64 %s is not a gather or VSIB prefetch: %q", op, ins.Raw) + } + if form.prefetch { + return nil + } + return c.emitGather(form.spec, form.memory.Mem, form.mask, form.data.Reg, form.dataBytes, form.indexBytes, form.evex) +} + +func amd64GatherTableAllows(table amd64GatherTable, evex bool, destinationBytes, indexBytes int) bool { + allowed := false + switch table { + case amd64GatherDPS: + allowed = (destinationBytes == 16 && indexBytes == 16) || + (destinationBytes == 32 && indexBytes == 32) || + (destinationBytes == 64 && indexBytes == 64) + case amd64GatherDPD: + allowed = (destinationBytes == 16 && indexBytes == 16) || + (destinationBytes == 32 && indexBytes == 16) || + (destinationBytes == 64 && indexBytes == 32) + case amd64GatherQPS: + allowed = (destinationBytes == 16 && (indexBytes == 16 || indexBytes == 32)) || + (destinationBytes == 32 && indexBytes == 64) + } + if !allowed { + return false + } + if !evex { + return destinationBytes != 64 + } + return true +} + +func amd64GatherRegisterIndex(register Reg, byteWidth int, evex bool, goarch string, index bool) (int, bool) { + var registerIndex int + var ok bool + switch byteWidth { + case 16: + registerIndex, ok = amd64ParseXReg(register) + case 32: + registerIndex, ok = amd64ParseYReg(register) + case 64: + registerIndex, ok = amd64ParseZReg(register) + } + if !ok { + return 0, false + } + limit := 15 + if evex { + limit = 31 + } + if goarch == "386" && (index || byteWidth == 64) { + limit = 7 + } + return registerIndex, registerIndex <= limit +} + +func (c *amd64Ctx) emitGather(spec amd64GatherSpec, memory MemRef, mask Operand, destination Reg, destinationBytes, indexBytes int, evex bool) error { + destinationValue, err := c.loadGatherVector(destination, destinationBytes) + if err != nil { + return err + } + destinationLanes := destinationBytes * 8 / spec.elemBits + destinationType := fmt.Sprintf("<%d x i%d>", destinationLanes, spec.elemBits) + destinationValues := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", destinationValues, destinationBytes, destinationValue, destinationType) + result := "%" + destinationValues + + indexValue, err := c.loadGatherVector(memory.Index, indexBytes) + if err != nil { + return err + } + indexLanes := indexBytes * 8 / spec.indexBits + indexType := fmt.Sprintf("<%d x i%d>", indexLanes, spec.indexBits) + indexValues := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", indexValues, indexBytes, indexValue, indexType) + activeLanes := destinationLanes + if indexLanes < activeLanes { + activeLanes = indexLanes + // QPS/QD with an X index has two active dword lanes and architecturally + // zeroes the upper two lanes of the X destination. + for lane := activeLanes; lane < destinationLanes; lane++ { + zeroed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d 0, i32 %d\n", zeroed, destinationType, result, spec.elemBits, lane) + result = "%" + zeroed + } + } + + var maskValues string + var kMask string + if evex { + kMask, err = c.loadK(mask.Reg) + } else { + maskBytes, err := c.loadGatherVector(mask.Reg, destinationBytes) + if err != nil { + return err + } + maskValues = c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", maskValues, destinationBytes, maskBytes, destinationType) + maskValues = "%" + maskValues + } + + base, err := c.gatherBaseAddress(memory) + if err != nil { + return err + } + for lane := 0; lane < activeLanes; lane++ { + indexLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", indexLane, indexType, indexValues, lane) + wideIndex := "%" + indexLane + if spec.indexBits == 32 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i32 %%%s to i64\n", extended, indexLane) + wideIndex = "%" + extended + } + if memory.Scale != 0 && memory.Scale != 1 { + scaled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i64 %s, %d\n", scaled, wideIndex, memory.Scale) + wideIndex = "%" + scaled + } + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %s\n", address, base, wideIndex) + pointer, pointerType := c.gatherPointer(memory, "%"+address) + + condition := "" + if evex { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", shifted, kMask, lane) + bit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 1\n", bit, shifted) + set := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", set, bit) + condition = "%" + set + } else { + maskLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", maskLane, destinationType, maskValues, lane) + set := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i%d %%%s, 0\n", set, spec.elemBits, maskLane) + condition = "%" + set + } + + oldLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", oldLane, destinationType, result, lane) + loadLabel := c.newTmp() + "_gather_load" + skipLabel := c.newTmp() + "_gather_skip" + joinLabel := c.newTmp() + "_gather_join" + loaded := c.newTmp() + selected := c.newTmp() + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n", condition, loadLabel, skipLabel) + fmt.Fprintf(c.b, "%s:\n", loadLabel) + // An active architectural access still faults when its result is dead. + // Keep the access inside its lane predicate so inactive lanes stay suppressed. + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %s, align 1\n", loaded, spec.elemBits, pointerType, pointer) + fmt.Fprintf(c.b, " br label %%%s\n", joinLabel) + fmt.Fprintf(c.b, "%s:\n", skipLabel) + fmt.Fprintf(c.b, " br label %%%s\n", joinLabel) + fmt.Fprintf(c.b, "%s:\n", joinLabel) + fmt.Fprintf(c.b, " %%%s = phi i%d [ %%%s, %%%s ], [ %%%s, %%%s ]\n", selected, spec.elemBits, loaded, loadLabel, oldLane, skipLabel) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %%%s, i32 %d\n", inserted, destinationType, result, spec.elemBits, selected, lane) + result = "%" + inserted + } + + bytesResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytesResult, destinationType, result, destinationBytes) + if err := c.storePackedMoveOperand(Operand{Kind: OpReg, Reg: destination}, destinationBytes, "%"+bytesResult); err != nil { + return err + } + if evex { + return c.storeK(mask.Reg, "0") + } + return c.storePackedMoveOperand(mask, destinationBytes, "zeroinitializer") +} + +func (c *amd64Ctx) loadGatherVector(register Reg, byteWidth int) (string, error) { + switch byteWidth { + case 16: + return c.loadX(register) + case 32: + return c.loadY(register) + case 64: + return c.loadZ(register) + default: + return "", fmt.Errorf("unsupported gather vector width %d", byteWidth) + } +} + +func (c *amd64Ctx) gatherBaseAddress(memory MemRef) (string, error) { + base := "0" + if memory.Base != "" { + value, err := c.loadReg(memory.Base) + if err != nil { + return "", err + } + base = value + } + if memory.Off != 0 { + adjusted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %d\n", adjusted, base, memory.Off) + base = "%" + adjusted + } + return base, nil +} + +func (c *amd64Ctx) gatherPointer(memory MemRef, address string) (string, string) { + if memory.Segment == "" { + return c.ptrFromAddrI64(address), "ptr" + } + addressSpace := 257 + if memory.Segment == GS { + addressSpace = 256 + } + pointerType := fmt.Sprintf("ptr addrspace(%d)", addressSpace) + pointer := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to %s\n", pointer, address, pointerType) + return "%" + pointer, pointerType +} diff --git a/amd64_lower_getexp.go b/amd64_lower_getexp.go new file mode 100644 index 000000000..5b8a156e9 --- /dev/null +++ b/amd64_lower_getexp.go @@ -0,0 +1,201 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64GetExpSpec struct { + laneBits int + scalar bool +} + +var amd64GetExpSpecs = map[Op]amd64GetExpSpec{ + "VGETEXPPS": {laneBits: 32}, + "VGETEXPPD": {laneBits: 64}, + "VGETEXPSS": {laneBits: 32, scalar: true}, + "VGETEXPSD": {laneBits: 64, scalar: true}, +} + +func (c *amd64Ctx) lowerGetExp(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64GetExpSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return c.lowerNativeFloatingHelper(op, ins) + } + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's GETEXP optab: %q", c.goarch, baseOp, ins.Raw) + } + if spec.scalar && properties.broadcast { + return true, false, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + wantArgs := 2 + if spec.scalar { + wantArgs = 3 + } + if len(ins.Args) != wantArgs && len(ins.Args) != wantArgs+1 { + return true, false, fmt.Errorf("%s %s has the wrong operand count for its packed/scalar grammar: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == wantArgs+1 + if c.goarch == "386" && spec.scalar && masked { + return true, false, fmt.Errorf("386 %s mask form exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + maskIndex := len(ins.Args) - 2 + if masked && !amd64NonzeroKOperand(ins.Args[maskIndex]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if spec.scalar { + if byteWidth != 16 || !c.isGoEVEXVectorRegister(destination, 16) || !c.isGoEVEXVectorRegister(ins.Args[1], 16) { + return true, false, fmt.Errorf("%s %s scalar pass-through and destination must be X registers: %q", c.goarch, baseOp, ins.Raw) + } + } else if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s packed destination must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if properties.sae { + if source.Kind != OpReg || !spec.scalar && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s.SAE requires a register source and scalar or Z form: %q", c.goarch, baseOp, ins.Raw) + } + } + + mask := "" + if masked { + mask, err = c.loadK(ins.Args[maskIndex].Reg) + if err != nil { + return true, false, err + } + } + daz := c.loadMXCSRDAZ() + if spec.scalar { + return c.lowerScalarGetExp(spec, properties, source, ins.Args[1], destination, mask, daz) + } + return c.lowerPackedGetExp(spec, properties, source, destination, byteWidth, mask, daz) +} + +func (c *amd64Ctx) lowerPackedGetExp(spec amd64GetExpSpec, properties amd64BinaryFloatingSuffix, source, destination Operand, byteWidth int, mask, daz string) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + input, err := c.loadPackedCompareLanes(source, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + result := c.emitGetExpBits(lanes, spec.laneBits, input, daz) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarGetExp(spec amd64GetExpSpec, properties amd64BinaryFloatingSuffix, source, passthrough, destination Operand, mask, daz string) (bool, bool, error) { + sourceBits, err := c.loadFloatingScalarBits(source, spec.laneBits) + if err != nil { + return true, false, err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", input, spec.laneBits, spec.laneBits, sourceBits) + result := c.emitGetExpBits(1, spec.laneBits, "%"+input, daz) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + passthroughBytes, err := c.loadX(passthrough.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, passthroughBytes) + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitGetExpBits(lanes, laneBits int, input, daz string) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + floatType := amd64FloatingScalarType(laneBits) + mantissaBits, bias, denormalOffset := 23, 127, 149 + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + if laneBits == 64 { + mantissaBits, bias, denormalOffset = 52, 1023, 1074 + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + } + positiveInfinity := exponentMask + negativeInfinity := signBit | exponentMask + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + bits, exponent, mantissa := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", bits, vectorType, input, lane) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", exponent, laneBits, bits, exponentMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", mantissa, laneBits, bits, mantissaMask) + exponentZero, exponentOnes := c.newTmp(), c.newTmp() + mantissaZero, mantissaNonzero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, exponent) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", exponentOnes, laneBits, exponent, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", mantissaZero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", mantissaNonzero, laneBits, mantissa) + isNaN, isInfinity, isDenormal := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, exponentOnes, mantissaNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, exponentOnes, mantissaZero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isDenormal, exponentZero, mantissaNonzero) + zeroOrDAZ, isEffectiveZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %s\n", zeroOrDAZ, mantissaZero, daz) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isEffectiveZero, exponentZero, zeroOrDAZ) + + normalField, normalExponent := c.newTmp(), c.newTmp() + leadingZeros, highestBit, denormalExponent := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %d\n", normalField, laneBits, exponent, mantissaBits) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, %d\n", normalExponent, laneBits, normalField, bias) + fmt.Fprintf(c.b, " %%%s = call i%d @llvm.ctlz.i%d(i%d %%%s, i1 false)\n", leadingZeros, laneBits, laneBits, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = sub i%d %d, %%%s\n", highestBit, laneBits, laneBits-1, leadingZeros) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, %d\n", denormalExponent, laneBits, highestBit, denormalOffset) + exponentValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", exponentValue, isDenormal, laneBits, denormalExponent, laneBits, normalExponent) + exponentFloat, exponentBits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sitofp i%d %%%s to %s\n", exponentFloat, laneBits, exponentValue, floatType) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to i%d\n", exponentBits, floatType, exponentFloat, laneBits) + withInfinity, withZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %%%s\n", withInfinity, isInfinity, laneBits, positiveInfinity, laneBits, exponentBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %%%s\n", withZero, isEffectiveZero, laneBits, negativeInfinity, laneBits, withInfinity) + quietNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", quietNaN, laneBits, bits, quietBit) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", selected, isNaN, laneBits, quietNaN, laneBits, withZero) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %%%s, i32 %d\n", inserted, vectorType, result, laneBits, selected, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_getexp_test.go b/amd64_lower_getexp_test.go new file mode 100644 index 000000000..f3fc2d4f5 --- /dev/null +++ b/amd64_lower_getexp_test.go @@ -0,0 +1,254 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestAMD64GetExpGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64GetExpSpec{ + "VGETEXPPS": {laneBits: 32}, + "VGETEXPPD": {laneBits: 64}, + "VGETEXPSS": {laneBits: 32, scalar: true}, + "VGETEXPSD": {laneBits: 64, scalar: true}, + } + if len(amd64GetExpSpecs) != len(expected) { + t.Fatalf("GETEXP grammar has %d entries, want %d", len(amd64GetExpSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64GetExpSpecs[op]; !ok { + t.Errorf("GETEXP grammar omitted %s", op) + } else if got != want { + t.Errorf("GETEXP grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86GetExpCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT getexpforms(SB),$0-0\n") + for _, op := range []string{"VGETEXPPS", "VGETEXPPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST 16(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 24(AX), K2, %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(AX), K3, %s%d\n", op, width, last) + if width == "Z" { + fmt.Fprintf(&source, "\t%s.SAE Z1, Z%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE.Z Z1, K4, Z%d\n", op, last) + } + } + } + for _, op := range []string{"VGETEXPSS", "VGETEXPSD"} { + last := 31 + fmt.Fprintf(&source, "\t%s X1, X2, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s 40(AX), X2, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE X1, X2, X%d\n", op, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X2, K5, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.Z 48(AX), X2, K6, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE.Z X1, X2, K7, X%d\n", op, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86FloatingHelperOriginalFormatObject(t, source.String(), "getexpforms", target.goarch, target.triple) + if ir == "" { + return // Go/public-Context coverage, not an LLVM object claim. + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "getexp-"+target.name+".ll", "getexp-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86GetExpRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VGETEXPPS X0"}, + {goarch: "amd64", instruction: "VGETEXPPS X0, Y1"}, + {goarch: "amd64", instruction: "VGETEXPPS.BCST X0, X1"}, + {goarch: "amd64", instruction: "VGETEXPPS.SAE X0, X1"}, + {goarch: "amd64", instruction: "VGETEXPPS.SAE 0(AX), Z1"}, + {goarch: "amd64", instruction: "VGETEXPPS.RN_SAE Z0, Z1"}, + {goarch: "amd64", instruction: "VGETEXPPS.Z X0, X1"}, + {goarch: "amd64", instruction: "VGETEXPPS X0, K0, X1"}, + {goarch: "amd64", instruction: "VGETEXPSS Y0, X1, X2"}, + {goarch: "amd64", instruction: "VGETEXPSS.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VGETEXPSD X0, X1, Y2"}, + {goarch: "386", instruction: "VGETEXPSS X0, X1, K1, X2"}, + {goarch: "386", instruction: "VGETEXPPS Z8, Z0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's GETEXP table", test.instruction) + } + }) + } +} + +func qualifyAMD64GetExpRuntimeSemantics(t *testing.T) { + execution := x86FloatingHelperRuntimeTools(t) + const source = `TEXT getexpsemantics(SB),$0-56 + MOVQ out+0(FP), AX + MOVQ source32+8(FP), BX + MOVQ source64+16(FP), CX + MOVQ old32+24(FP), DX + MOVQ old64+32(FP), R8 + MOVQ upper+40(FP), R9 + MOVQ mask+48(FP), R10 + KMOVQ R10, K1 + STC + VGETEXPPS 0(BX), Z0 + VMOVDQU32 Z0, 0(AX) + VMOVDQU32 0(DX), Z1 + VGETEXPPS 0(BX), K1, Z1 + VMOVDQU32 Z1, 64(AX) + VGETEXPPS.Z 0(BX), K1, Z1 + VMOVDQU32 Z1, 128(AX) + VGETEXPPS.BCST 0(BX), Z1 + VMOVDQU32 Z1, 192(AX) + VGETEXPPD 0(CX), Z2 + VMOVDQU64 Z2, 256(AX) + VMOVDQU64 0(R8), Z3 + VGETEXPPD 0(CX), K1, Z3 + VMOVDQU64 Z3, 320(AX) + VGETEXPPD.Z 0(CX), K1, Z3 + VMOVDQU64 Z3, 384(AX) + VGETEXPPD.BCST 0(CX), Z3 + VMOVDQU64 Z3, 448(AX) + VMOVDQU64 0(R9), X4 + VGETEXPSS 40(BX), X4, X5 + VMOVDQU64 X5, 512(AX) + VGETEXPSD 40(CX), X4, X6 + VMOVDQU64 X6, 528(AX) + SETCS 544(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := execution.triple + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"getexpsemantics": { + Name: "getexpsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: I64, Index: 6, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void getexpsemantics(uint8_t *, const uint32_t *, const uint64_t *, const uint32_t *, const uint64_t *, const uint64_t *, uint64_t); +static uint32_t f32bits(float f) { uint32_t u; memcpy(&u,&f,4); return u; } +static uint64_t f64bits(double f) { uint64_t u; memcpy(&u,&f,8); return u; } +static uint32_t getexp32(uint32_t u, int daz) { + uint32_t exp=u&UINT32_C(0x7f800000), mant=u&UINT32_C(0x007fffff); + if (exp==UINT32_C(0x7f800000)) return mant?(u|UINT32_C(0x00400000)):UINT32_C(0x7f800000); + if (exp==0 && (mant==0||daz)) return UINT32_C(0xff800000); + int e; + if (exp) e=(int)(exp>>23)-127; + else { int p=31-__builtin_clz(mant); e=p-149; } + return f32bits((float)e); +} +static uint64_t getexp64(uint64_t u, int daz) { + uint64_t exp=u&UINT64_C(0x7ff0000000000000), mant=u&UINT64_C(0x000fffffffffffff); + if (exp==UINT64_C(0x7ff0000000000000)) return mant?(u|UINT64_C(0x0008000000000000)):UINT64_C(0x7ff0000000000000); + if (exp==0 && (mant==0||daz)) return UINT64_C(0xfff0000000000000); + int e; + if (exp) e=(int)(exp>>52)-1023; + else { int p=63-__builtin_clzll(mant); e=p-1074; } + return f64bits((double)e); +} +int main(void) { + const uint32_t source32[16]={ + 0x3f800000,0x40000000,0x3f400000,0xc1000000,0,0x80000000,0x7f800000,0xff800000, + 0x7fc12345,0x7f812345,1,0x007fffff,0x80000001,0x00800000,0x7f7fffff,0x3e800000}; + const uint64_t source64[8]={ + UINT64_C(0x3ff0000000000000),UINT64_C(0x4000000000000000),UINT64_C(0x3fe8000000000000),UINT64_C(0xc020000000000000), + 0,1,UINT64_C(0x000fffffffffffff),UINT64_C(0x8000000000000001)}; + uint32_t old32[16]; uint64_t old64[8]; + const uint64_t upper[2]={UINT64_C(0x1122334455667788),UINT64_C(0x99aabbccddeeff00)}; + uint8_t out[545]={0}, outDaz[545]={0}; const uint64_t mask=UINT64_C(0xa55a); + for (int i=0;i<16;i++) old32[i]=UINT32_C(0x41000000)+(uint32_t)i; + for (int i=0;i<8;i++) old64[i]=UINT64_C(0x4020000000000000)+(uint64_t)i; + getexpsemantics(out,source32,source64,old32,old64,upper,mask); + unsigned csr=_mm_getcsr(); _mm_setcsr(csr|UINT32_C(0x40)); + getexpsemantics(outDaz,source32,source64,old32,old64,upper,mask); _mm_setcsr(csr); + for (int i=0;i<16;i++) { + uint32_t want=getexp32(source32[i],0), got; memcpy(&got,out+4*i,4); if (got!=want) return 10; + memcpy(&got,out+64+4*i,4); if (got!=(((mask>>i)&1)?want:old32[i])) return 11; + memcpy(&got,out+128+4*i,4); if (got!=(((mask>>i)&1)?want:0)) return 12; + memcpy(&got,out+192+4*i,4); if (got!=getexp32(source32[0],0)) return 13; + } + for (int i=0;i<8;i++) { + uint64_t want=getexp64(source64[i],0), got; memcpy(&got,out+256+8*i,8); if (got!=want) return 20; + memcpy(&got,out+320+8*i,8); if (got!=(((mask>>i)&1)?want:old64[i])) return 21; + memcpy(&got,out+384+8*i,8); if (got!=(((mask>>i)&1)?want:0)) return 22; + memcpy(&got,out+448+8*i,8); if (got!=getexp64(source64[0],0)) return 23; + } + uint32_t scalar32; memcpy(&scalar32,out+512,4); if (scalar32!=getexp32(source32[10],0)) return 30; + if (memcmp(out+516,(uint8_t*)upper+4,12)) return 31; + uint64_t scalar64; memcpy(&scalar64,out+528,8); if (scalar64!=getexp64(source64[5],0)) return 32; + if (memcmp(out+536,(uint8_t*)upper+8,8)) return 33; + if (out[544]!=1) return 34; + uint32_t daz32; memcpy(&daz32,outDaz+40,4); if (daz32!=UINT32_C(0xff800000)) return 40; + uint64_t daz64; memcpy(&daz64,outDaz+296,8); if (daz64!=UINT64_C(0xfff0000000000000)) return 41; + memcpy(&daz32,outDaz+512,4); if (daz32!=UINT32_C(0xff800000)) return 42; + memcpy(&daz64,outDaz+528,8); if (daz64!=UINT64_C(0xfff0000000000000)) return 43; + return 0; +} +` + x86FloatingHelperRunNativePipelines(t, execution, "getexp_semantics", ir, mainC) +} diff --git a/amd64_lower_getmant.go b/amd64_lower_getmant.go new file mode 100644 index 000000000..4a68c58e5 --- /dev/null +++ b/amd64_lower_getmant.go @@ -0,0 +1,193 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64GetMantSpec = amd64UnaryImmediateFloatingShape + +var amd64GetMantSpecs = map[Op]amd64GetMantSpec{ + "VGETMANTPS": {laneBits: 32}, + "VGETMANTPD": {laneBits: 64}, + "VGETMANTSS": {laneBits: 32, scalar: true}, + "VGETMANTSD": {laneBits: 64, scalar: true}, +} + +func (c *amd64Ctx) lowerGetMant(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64GetMantSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return c.lowerNativeFloatingHelper(op, ins) + } + form, err := c.parseUnaryImmediateFloatingForm(baseOp, suffix, spec, ins) + if err != nil { + return true, false, err + } + daz := c.loadMXCSRDAZ() + if spec.scalar { + return c.lowerScalarGetMant(spec, form.properties, form.source, form.passthrough, form.destination, form.mask, daz, form.immediate) + } + return c.lowerPackedGetMant(spec, form.properties, form.source, form.destination, form.byteWidth, form.mask, daz, form.immediate) +} + +func (c *amd64Ctx) lowerPackedGetMant(spec amd64GetMantSpec, properties amd64BinaryFloatingSuffix, source, destination Operand, byteWidth int, mask, daz string, immediate uint8) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + input, err := c.loadPackedCompareLanes(source, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + result := c.emitGetMantBits(lanes, spec.laneBits, input, daz, immediate) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarGetMant(spec amd64GetMantSpec, properties amd64BinaryFloatingSuffix, source, passthrough, destination Operand, mask, daz string, immediate uint8) (bool, bool, error) { + sourceBits, err := c.loadFloatingScalarBits(source, spec.laneBits) + if err != nil { + return true, false, err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", input, spec.laneBits, spec.laneBits, sourceBits) + result := c.emitGetMantBits(1, spec.laneBits, "%"+input, daz, immediate) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + passthroughBytes, err := c.loadX(passthrough.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, passthroughBytes) + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitGetMantBits(lanes, laneBits int, input, daz string, immediate uint8) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + mantissaBits, bias := 23, 127 + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + indefinite := uint64(0xffc00000) + if laneBits == 64 { + mantissaBits, bias = 52, 1023 + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + indefinite = uint64(0xfff8000000000000) + } + interval := immediate & 3 + signControl := immediate >> 2 & 3 + positiveOne := uint64(bias) << mantissaBits + signedOne := positiveOne + if signControl&1 == 0 { + signedOne |= signBit + } + dazClear := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i1 %s, false\n", dazClear, daz) + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + bits, exponent, mantissa, sign := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", bits, vectorType, input, lane) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", exponent, laneBits, bits, exponentMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", mantissa, laneBits, bits, mantissaMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", sign, laneBits, bits, signBit) + exponentZero, exponentOnes := c.newTmp(), c.newTmp() + mantissaZero, mantissaNonzero, negative := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, exponent) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", exponentOnes, laneBits, exponent, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", mantissaZero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", mantissaNonzero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", negative, laneBits, sign) + isNaN, isInfinity, isDenormal := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, exponentOnes, mantissaNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, exponentOnes, mantissaZero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isDenormal, exponentZero, mantissaNonzero) + activeDenormal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", activeDenormal, isDenormal, dazClear) + zeroOrDAZ, effectiveZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %s\n", zeroOrDAZ, mantissaZero, daz) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", effectiveZero, exponentZero, zeroOrDAZ) + + leadingZeros, shiftCount, shifted, normalizedFraction := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i%d @llvm.ctlz.i%d(i%d %%%s, i1 false)\n", leadingZeros, laneBits, laneBits, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, %d\n", shiftCount, laneBits, leadingZeros, laneBits-mantissaBits-1) + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, %%%s\n", shifted, laneBits, mantissa, shiftCount) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", normalizedFraction, laneBits, shifted, mantissaMask) + fraction := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", fraction, activeDenormal, laneBits, normalizedFraction, laneBits, mantissa) + exponentField, unbiasedNormal, unbiasedDenormal, unbiased := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %d\n", exponentField, laneBits, exponent, mantissaBits) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, %d\n", unbiasedNormal, laneBits, exponentField, bias) + fmt.Fprintf(c.b, " %%%s = sub i%d 0, %%%s\n", unbiasedDenormal, laneBits, shiftCount) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", unbiased, activeDenormal, laneBits, unbiasedDenormal, laneBits, unbiasedNormal) + oddValue, odd := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, 1\n", oddValue, laneBits, unbiased) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", odd, laneBits, oddValue) + fractionTop, topSet := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", fractionTop, laneBits, fraction, quietBit) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", topSet, laneBits, fractionTop) + normalizedExponent := fmt.Sprintf("%d", bias) + switch interval { + case 1: + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %d\n", selected, odd, laneBits, bias-1, laneBits, bias) + normalizedExponent = "%" + selected + case 2: + normalizedExponent = fmt.Sprintf("%d", bias-1) + case 3: + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %d\n", selected, topSet, laneBits, bias-1, laneBits, bias) + normalizedExponent = "%" + selected + } + exponentBits, magnitude := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d %s, %d\n", exponentBits, laneBits, normalizedExponent, mantissaBits) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", magnitude, laneBits, exponentBits, fraction) + computed := "%" + magnitude + if signControl&1 == 0 { + withSign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", withSign, laneBits, magnitude, sign) + computed = "%" + withSign + } + zeroOrInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", zeroOrInfinity, effectiveZero, isInfinity) + specialOne := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %d\n", specialOne, negative, laneBits, signedOne, laneBits, positiveOne) + withSpecial := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %s\n", withSpecial, zeroOrInfinity, laneBits, specialOne, laneBits, computed) + selected := "%" + withSpecial + if signControl&2 != 0 { + notZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notZero, effectiveZero) + nonzeroNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", nonzeroNegative, negative, notZero) + indefiniteResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %s\n", indefiniteResult, nonzeroNegative, laneBits, indefinite, laneBits, selected) + selected = "%" + indefiniteResult + } + quietNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", quietNaN, laneBits, bits, quietBit) + final := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %s\n", final, isNaN, laneBits, quietNaN, laneBits, selected) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %%%s, i32 %d\n", inserted, vectorType, result, laneBits, final, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_getmant_test.go b/amd64_lower_getmant_test.go new file mode 100644 index 000000000..b92f01772 --- /dev/null +++ b/amd64_lower_getmant_test.go @@ -0,0 +1,286 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestAMD64GetMantGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64GetMantSpec{ + "VGETMANTPS": {laneBits: 32}, + "VGETMANTPD": {laneBits: 64}, + "VGETMANTSS": {laneBits: 32, scalar: true}, + "VGETMANTSD": {laneBits: 64, scalar: true}, + } + if len(amd64GetMantSpecs) != len(expected) { + t.Fatalf("GETMANT grammar has %d entries, want %d", len(amd64GetMantSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64GetMantSpecs[op]; !ok { + t.Errorf("GETMANT grammar omitted %s", op) + } else if got != want { + t.Errorf("GETMANT grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86GetMantCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT getmantforms(SB),$0-0\n") + for _, op := range []string{"VGETMANTPS", "VGETMANTPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $0, %s1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s $255, 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST $1, 16(AX), %s%d\n", op, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $2, %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z $3, 24(AX), K2, %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z $4, 32(AX), K3, %s%d\n", op, width, last) + } + if width == "Z" { + fmt.Fprintf(&source, "\t%s.SAE $5, Z1, Z%d\n", op, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.SAE.Z $6, Z1, K4, Z%d\n", op, last) + } + } + } + } + if target.goarch == "amd64" { + for _, op := range []string{"VGETMANTSS", "VGETMANTSD"} { + fmt.Fprintf(&source, "\t%s $0, X1, X2, X31\n", op) + fmt.Fprintf(&source, "\t%s $255, 40(AX), X2, X31\n", op) + fmt.Fprintf(&source, "\t%s.SAE $1, X1, X2, X31\n", op) + fmt.Fprintf(&source, "\t%s $2, X1, X2, K5, X31\n", op) + fmt.Fprintf(&source, "\t%s.Z $3, 48(AX), X2, K6, X31\n", op) + fmt.Fprintf(&source, "\t%s.SAE.Z $4, X1, X2, K7, X31\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86FloatingHelperOriginalFormatObject(t, source.String(), "getmantforms", target.goarch, target.triple) + if ir == "" { + return // Go/public-Context coverage, not an LLVM object claim. + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "getmant-"+target.name+".ll", "getmant-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86GetMantRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VGETMANTPS $-1, X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS $256, X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS $0, X0, Y1"}, + {goarch: "amd64", instruction: "VGETMANTPS.BCST $0, X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS.SAE $0, Z0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS.SAE $0, 0(AX), Z1"}, + {goarch: "amd64", instruction: "VGETMANTPS.RN_SAE $0, Z0, Z1"}, + {goarch: "amd64", instruction: "VGETMANTPS.Z $0, X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS $0, X0, K0, X1"}, + {goarch: "amd64", instruction: "VGETMANTSS $0, Y0, X1, X2"}, + {goarch: "amd64", instruction: "VGETMANTSS.BCST $0, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VGETMANTSD $0, X0, X1, Y2"}, + {goarch: "386", instruction: "VGETMANTSS $0, X0, X1, X2"}, + {goarch: "386", instruction: "VGETMANTPS $0, X0, K1, X2"}, + {goarch: "386", instruction: "VGETMANTPS $0, Z8, Z0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's GETMANT table", test.instruction) + } + }) + } +} + +func qualifyAMD64GetMantRuntimeSemantics(t *testing.T) { + execution := x86FloatingHelperRuntimeTools(t) + var source strings.Builder + source.WriteString(`TEXT getmantsemantics(SB),$0-56 + MOVQ out+0(FP), AX + MOVQ source32+8(FP), BX + MOVQ source64+16(FP), CX + MOVQ old32+24(FP), DX + MOVQ old64+32(FP), R8 + MOVQ upper+40(FP), R9 + MOVQ mask+48(FP), R10 + KMOVQ R10, K1 + STC +`) + for immediate := 0; immediate < 16; immediate++ { + fmt.Fprintf(&source, "\tVGETMANTPS $%d, 0(BX), Z0\n", immediate) + fmt.Fprintf(&source, "\tVMOVDQU32 Z0, %d(AX)\n", immediate*64) + fmt.Fprintf(&source, "\tVGETMANTPD $%d, 0(CX), Z1\n", immediate) + fmt.Fprintf(&source, "\tVMOVDQU64 Z1, %d(AX)\n", 1024+immediate*64) + } + source.WriteString(` VMOVDQU32 0(DX), Z2 + VGETMANTPS $3, 0(BX), K1, Z2 + VMOVDQU32 Z2, 2048(AX) + VGETMANTPS.Z $3, 0(BX), K1, Z2 + VMOVDQU32 Z2, 2112(AX) + VGETMANTPS.BCST $5, 0(BX), Z2 + VMOVDQU32 Z2, 2176(AX) + VMOVDQU64 0(R8), Z3 + VGETMANTPD $3, 0(CX), K1, Z3 + VMOVDQU64 Z3, 2240(AX) + VGETMANTPD.Z $3, 0(CX), K1, Z3 + VMOVDQU64 Z3, 2304(AX) + VGETMANTPD.BCST $5, 0(CX), Z3 + VMOVDQU64 Z3, 2368(AX) + VMOVDQU64 0(R9), X4 + VGETMANTSS $11, 12(BX), X4, X5 + VMOVDQU64 X5, 2432(AX) + VGETMANTSD $11, 24(CX), X4, X6 + VMOVDQU64 X6, 2448(AX) + SETCS 2464(AX) + RET +`) + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := execution.triple + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"getmantsemantics": { + Name: "getmantsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: I64, Index: 6, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void getmantsemantics(uint8_t *, const uint32_t *, const uint64_t *, const uint32_t *, const uint64_t *, const uint64_t *, uint64_t); +static uint32_t getmant32(uint32_t u, unsigned imm, int daz) { + const uint32_t em=UINT32_C(0x7f800000), fm=UINT32_C(0x007fffff), qb=UINT32_C(0x00400000), sb=UINT32_C(0x80000000); + uint32_t exp=u&em, frac=u&fm, sign=u&sb; unsigned sc=(imm>>2)&3, interval=imm&3; + int nan=exp==em&&frac!=0, inf=exp==em&&frac==0, zero=exp==0&&(frac==0||daz), neg=sign!=0; + if (nan) return u|qb; + uint32_t one=UINT32_C(0x3f800000), signedOne=one|((sc&1)?0:sign); + if (neg && !zero && (sc&2)) return UINT32_C(0xffc00000); + if (zero||inf) return neg?signedOne:one; + int unbiased; + if (exp==0) { int shift=__builtin_clz(frac)-8; frac=(frac<>23)-127; + uint32_t outExp=127; + if (interval==1) outExp=(unbiased&1)?126:127; + else if (interval==2) outExp=126; + else if (interval==3) outExp=(frac&qb)?126:127; + return (outExp<<23)|frac|((sc&1)?0:sign); +} +static uint64_t getmant64(uint64_t u, unsigned imm, int daz) { + const uint64_t em=UINT64_C(0x7ff0000000000000), fm=UINT64_C(0x000fffffffffffff), qb=UINT64_C(0x0008000000000000), sb=UINT64_C(0x8000000000000000); + uint64_t exp=u&em, frac=u&fm, sign=u&sb; unsigned sc=(imm>>2)&3, interval=imm&3; + int nan=exp==em&&frac!=0, inf=exp==em&&frac==0, zero=exp==0&&(frac==0||daz), neg=sign!=0; + if (nan) return u|qb; + uint64_t one=UINT64_C(0x3ff0000000000000), signedOne=one|((sc&1)?0:sign); + if (neg && !zero && (sc&2)) return UINT64_C(0xfff8000000000000); + if (zero||inf) return neg?signedOne:one; + int unbiased; + if (exp==0) { int shift=__builtin_clzll(frac)-11; frac=(frac<>52)-1023; + uint64_t outExp=1023; + if (interval==1) outExp=(unbiased&1)?1022:1023; + else if (interval==2) outExp=1022; + else if (interval==3) outExp=(frac&qb)?1022:1023; + return (outExp<<52)|frac|((sc&1)?0:sign); +} +static uint32_t u32(const uint8_t *p) { uint32_t v; memcpy(&v,p,4); return v; } +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + const uint32_t source32[16]={ + 0x3f800000,0xbf800000,0x40400000,0xc0400000,0x3f400000,0xbf400000,0,0x80000000, + 0x7f800000,0xff800000,0x7fc12345,0xff812345,1,0x807fffff,0x00800000,0x7f7fffff}; + const uint64_t source64[8]={ + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x4008000000000000),UINT64_C(0xc008000000000000), + 0,UINT64_C(0x8000000000000000),UINT64_C(1),UINT64_C(0xfff0000000000000)}; + uint32_t old32[16]; uint64_t old64[8]; + const uint64_t upper[2]={UINT64_C(0x1122334455667788),UINT64_C(0x99aabbccddeeff00)}; + uint8_t out[2465]={0}, outDaz[2465]={0}; const uint64_t mask=UINT64_C(0xa55a); + for (int i=0;i<16;i++) old32[i]=UINT32_C(0x41000000)+(uint32_t)i; + for (int i=0;i<8;i++) old64[i]=UINT64_C(0x4020000000000000)+(uint64_t)i; + getmantsemantics(out,source32,source64,old32,old64,upper,mask); + unsigned csr=_mm_getcsr(); _mm_setcsr(csr|UINT32_C(0x40)); + getmantsemantics(outDaz,source32,source64,old32,old64,upper,mask); _mm_setcsr(csr); + for (unsigned imm=0;imm<16;imm++) { + for (int i=0;i<16;i++) { + if (u32(out+imm*64+i*4)!=getmant32(source32[i],imm,0)) return 10; + if (u32(outDaz+imm*64+i*4)!=getmant32(source32[i],imm,1)) return 11; + } + for (int i=0;i<8;i++) { + if (u64(out+1024+imm*64+i*8)!=getmant64(source64[i],imm,0)) return 12; + if (u64(outDaz+1024+imm*64+i*8)!=getmant64(source64[i],imm,1)) return 13; + } + } + for (int i=0;i<16;i++) { + uint32_t want=getmant32(source32[i],3,0); + if (u32(out+2048+i*4)!=(((mask>>i)&1)?want:old32[i])) return 20; + if (u32(out+2112+i*4)!=(((mask>>i)&1)?want:0)) return 21; + if (u32(out+2176+i*4)!=getmant32(source32[0],5,0)) return 22; + } + for (int i=0;i<8;i++) { + uint64_t want=getmant64(source64[i],3,0); + if (u64(out+2240+i*8)!=(((mask>>i)&1)?want:old64[i])) return 23; + if (u64(out+2304+i*8)!=(((mask>>i)&1)?want:0)) return 24; + if (u64(out+2368+i*8)!=getmant64(source64[0],5,0)) return 25; + } + if (u32(out+2432)!=getmant32(source32[3],11,0)) return 30; + if (memcmp(out+2436,(const uint8_t *)upper+4,12)) return 31; + if (u64(out+2448)!=getmant64(source64[3],11,0)) return 32; + if (memcmp(out+2456,(const uint8_t *)upper+8,8)) return 33; + if (out[2464]!=1) return 34; + return 0; +} +` + x86FloatingHelperRunNativePipelines(t, execution, "getmant_semantics", ir, mainC) +} diff --git a/amd64_lower_gfni.go b/amd64_lower_gfni.go new file mode 100644 index 000000000..6251a2090 --- /dev/null +++ b/amd64_lower_gfni.go @@ -0,0 +1,173 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64GFNIMode uint8 + +const ( + amd64GFNIMultiply amd64GFNIMode = iota + amd64GFNIAffine + amd64GFNIAffineInverse +) + +type amd64GFNISpec struct { + mode amd64GFNIMode + immediate bool + broadcast bool +} + +// amd64GFNISpecs models the whole Go 1.27 GFNI family. VGF2P8MULB uses the +// eight _yvandnpd rows; both affine instructions use the eight +// _yvgf2p8affineinvqb rows, including qword broadcast for their matrix source. +var amd64GFNISpecs = map[Op]amd64GFNISpec{ + "VGF2P8MULB": {mode: amd64GFNIMultiply}, + "VGF2P8AFFINEQB": {mode: amd64GFNIAffine, immediate: true, broadcast: true}, + "VGF2P8AFFINEINVQB": {mode: amd64GFNIAffineInverse, immediate: true, broadcast: true}, +} + +func (c *amd64Ctx) lowerGFNI(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86GFNIProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + spec, byteWidth := form.spec, form.byteWidth + first, second := ins.Args[form.first], ins.Args[form.second] + destination := ins.Args[form.destination] + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + + firstBytes, err := c.loadGFNIFirstSource(first, byteWidth, form.broadcast) + if err != nil { + return true, false, err + } + secondBytes, err := c.loadPackedCompareBytes(second, byteWidth) + if err != nil { + return true, false, err + } + computed := "" + switch spec.mode { + case amd64GFNIMultiply: + computed = c.emitGF2P8Multiply(byteWidth, firstBytes, secondBytes) + case amd64GFNIAffine, amd64GFNIAffineInverse: + if spec.mode == amd64GFNIAffineInverse { + secondBytes = c.emitGF2P8Inverse(byteWidth, secondBytes) + } + computed = c.emitGF2P8Affine(byteWidth, firstBytes, secondBytes, form.immediate) + } + if form.mask >= 0 { + old, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + computed = amd64ApplyIntegerLaneMask(c, byteWidth, 8, computed, old, mask, form.zeroing) + } + return true, false, c.storePackedMoveOperand(destination, byteWidth, computed) +} + +func (c *amd64Ctx) loadGFNIFirstSource(source Operand, byteWidth int, broadcast bool) (string, error) { + if !broadcast { + return c.loadX86PackedShiftBytes(source, byteWidth, true) + } + // GFNI is E4NF: masks never suppress this complete source/m64bcst read. + qwords, err := c.loadX86PackedShiftLanes(source, byteWidth, 64, true, "") + if err != nil { + return "", err + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", bytes, byteWidth/8, qwords, byteWidth) + return "%" + bytes, nil +} + +// emitGF2P8Multiply performs lane-wise carry-less multiplication reduced by +// x^8+x^4+x^3+x+1 (0x11b). Generic IR keeps translated programs runnable on +// hosts without GFNI while preserving exactly the instruction's byte lanes. +func (c *amd64Ctx) emitGF2P8Multiply(lanes int, first, second string) string { + vectorType := fmt.Sprintf("<%d x i8>", lanes) + result := "zeroinitializer" + multiplicand, multiplier := first, second + for bit := 0; bit < 8; bit++ { + lowBit := c.newTmp() + active := c.newTmp() + selected := c.newTmp() + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", lowBit, vectorType, multiplier, llvmSplatInteger(lanes, 8, 1)) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", active, vectorType, lowBit) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s zeroinitializer\n", selected, lanes, active, vectorType, multiplicand, vectorType) + fmt.Fprintf(c.b, " %%%s = xor %s %s, %%%s\n", combined, vectorType, result, selected) + result = "%" + combined + if bit == 7 { + break + } + highBit := c.newTmp() + reduce := c.newTmp() + shifted := c.newTmp() + reduced := c.newTmp() + nextMultiplicand := c.newTmp() + nextMultiplier := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", highBit, vectorType, multiplicand, llvmSplatInteger(lanes, 8, 0x80)) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", reduce, vectorType, highBit) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %s\n", shifted, vectorType, multiplicand, llvmSplatInteger(lanes, 8, 1)) + fmt.Fprintf(c.b, " %%%s = xor %s %%%s, %s\n", reduced, vectorType, shifted, llvmSplatInteger(lanes, 8, 0x1b)) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %%%s, %s %%%s\n", nextMultiplicand, lanes, reduce, vectorType, reduced, vectorType, shifted) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", nextMultiplier, vectorType, multiplier, llvmSplatInteger(lanes, 8, 1)) + multiplicand, multiplier = "%"+nextMultiplicand, "%"+nextMultiplier + } + return result +} + +func (c *amd64Ctx) emitGF2P8Inverse(lanes int, source string) string { + powers := make([]string, 7) + powers[0] = c.emitGF2P8Multiply(lanes, source, source) + for i := 1; i < len(powers); i++ { + powers[i] = c.emitGF2P8Multiply(lanes, powers[i-1], powers[i-1]) + } + result := powers[0] + for i := 1; i < len(powers); i++ { + result = c.emitGF2P8Multiply(lanes, result, powers[i]) + } + return result +} + +func (c *amd64Ctx) emitGF2P8Affine(lanes int, matrix, data string, immediate uint8) string { + vectorType := fmt.Sprintf("<%d x i8>", lanes) + result := "zeroinitializer" + for bit := 0; bit < 8; bit++ { + row := c.newTmp() + masked := c.newTmp() + parities := c.newTmp() + oneBit := c.newTmp() + positioned := c.newTmp() + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector %s %s, %s poison, <%d x i32> %s\n", row, vectorType, matrix, vectorType, lanes, llvmGFNIMatrixRowMask(lanes, bit)) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", masked, vectorType, data, row) + fmt.Fprintf(c.b, " %%%s = call %s @llvm.ctpop.v%di8(%s %%%s)\n", parities, vectorType, lanes, vectorType, masked) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", oneBit, vectorType, parities, llvmSplatInteger(lanes, 8, 1)) + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %s\n", positioned, vectorType, oneBit, llvmSplatInteger(lanes, 8, uint64(bit))) + fmt.Fprintf(c.b, " %%%s = or %s %s, %%%s\n", combined, vectorType, result, positioned) + result = "%" + combined + } + withImmediate := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", withImmediate, vectorType, result, llvmSplatInteger(lanes, 8, uint64(immediate))) + return "%" + withImmediate +} + +func llvmGFNIMatrixRowMask(lanes, bit int) string { + var b strings.Builder + b.WriteByte('<') + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "i32 %d", lane/8*8+7-bit) + } + b.WriteByte('>') + return b.String() +} diff --git a/amd64_lower_gfni_test.go b/amd64_lower_gfni_test.go new file mode 100644 index 000000000..f2e7d6ad0 --- /dev/null +++ b/amd64_lower_gfni_test.go @@ -0,0 +1,260 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64GFNIGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64GFNISpec{ + "VGF2P8MULB": {mode: amd64GFNIMultiply}, + "VGF2P8AFFINEQB": {mode: amd64GFNIAffine, immediate: true, broadcast: true}, + "VGF2P8AFFINEINVQB": {mode: amd64GFNIAffineInverse, immediate: true, broadcast: true}, + } + if len(amd64GFNISpecs) != len(expected) { + t.Fatalf("GFNI grammar has %d entries, want %d", len(amd64GFNISpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64GFNISpecs[op]; !ok { + t.Errorf("GFNI grammar omitted %s", op) + } else if got != want { + t.Errorf("GFNI grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86GFNICompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT gfniforms(SB),$0-0\n") + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\tVGF2P8MULB %s1, %s2, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8MULB 8(AX), %s2, %s%d\n", width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB $0x63, %s1, %s2, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB $0x63, 16(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB $0xa5, %s1, %s2, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB $0xa5, 24(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB.BCST $0x63, 32(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB.BCST $0xa5, 40(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8MULB %s1, %s2, K1, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8MULB.Z 48(AX), %s2, K2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB $0x63, %s1, %s2, K3, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB.Z $0x63, 56(AX), %s2, K4, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB $0xa5, %s1, %s2, K5, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB.Z $0xa5, 64(AX), %s2, K6, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB.BCST.Z $0x63, 72(AX), %s2, K7, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB.BCST.Z $0xa5, 80(AX), %s2, K1, %s%d\n", width, width, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"gfniforms": {Name: "gfniforms", Ret: Void}}, + }) + llc := findLLVM22Tool("llc") + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original GFNI format source invented initialized inputs", err) + } + ir = x86ZeroArgumentLowererObjectUnit(t, file, target.goarch, target.triple, "GFNIEmitterOnly", + func(consumer *amd64Ctx, instruction Instr) (bool, bool, error) { + return consumer.lowerGFNI(instruction.Op, instruction) + }) + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "gfni-"+target.name+".ll", "gfni-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86GFNIRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VGF2P8MULB X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VGF2P8MULB X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VGF2P8MULB.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VGF2P8MULB.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VGF2P8MULB X0, X1, 0(AX)"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEQB $256, X0, X1, X2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEINVQB $1, X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEQB.BCST $1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEINVQB.Z.BCST $1, 0(AX), X1, K1, X2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEQB $1, X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEQB $1, X0, X1"}, + {goarch: "386", instruction: "VGF2P8MULB X0, X1, K1, X2"}, + {goarch: "386", instruction: "VGF2P8AFFINEQB $1, X0, X1, X2"}, + {goarch: "386", instruction: "VGF2P8MULB Z0, Z1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's GFNI table", test.instruction) + } + }) + } +} + +func TestAMD64GFNIRuntimeSemantics(t *testing.T) { + // Portable LLVM execution on the actual host, not original-Go/native AVX512. + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT gfnisemantics(SB),$0-48 + MOVQ out+0(FP), AX + MOVQ matrix+8(FP), BX + MOVQ data+16(FP), CX + MOVQ other+24(FP), DX + MOVQ old+32(FP), R8 + MOVQ mask+40(FP), R9 + KMOVQ R9, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VMOVDQU64 0(DX), Z2 + VMOVDQU64 0(R8), Z3 + STC + VGF2P8MULB Z2, Z1, Z4 + VMOVDQU64 Z4, 0(AX) + VGF2P8AFFINEQB $0x63, Z0, Z1, Z4 + VMOVDQU64 Z4, 64(AX) + VGF2P8AFFINEINVQB $0xa5, Z0, Z1, Z4 + VMOVDQU64 Z4, 128(AX) + VGF2P8MULB Z2, Z1, K1, Z3 + VMOVDQU64 Z3, 192(AX) + VGF2P8AFFINEQB.Z $0x63, Z0, Z1, K1, Z4 + VMOVDQU64 Z4, 256(AX) + VGF2P8AFFINEQB.BCST $0x63, 0(BX), Z1, Z4 + VMOVDQU64 Z4, 320(AX) + VMOVDQU64 0(R8), Z3 + VGF2P8AFFINEINVQB.BCST.Z $0xa5, 0(BX), Z1, K1, Z3 + VMOVDQU64 Z3, 384(AX) + VGF2P8MULB Z2, Z1, Z2 + VMOVDQU64 Z2, 448(AX) + SETCS 512(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"gfnisemantics": { + Name: "gfnisemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void gfnisemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static uint8_t gf_mul(uint8_t a, uint8_t b) { + uint8_t r = 0; + for (int i = 0; i < 8; i++) { + if (b & 1) r ^= a; + uint8_t hi = a & 0x80; + a <<= 1; + if (hi) a ^= 0x1b; + b >>= 1; + } + return r; +} +static uint8_t gf_inv(uint8_t x) { + if (!x) return 0; + uint8_t r = 1, a = x; + for (unsigned e = 254; e; e >>= 1) { + if (e & 1) r = gf_mul(r, a); + a = gf_mul(a, a); + } + return r; +} +static unsigned parity8(uint8_t x) { + x ^= x >> 4; x &= 0xf; + return (0x6996u >> x) & 1u; +} +static uint8_t affine(const uint8_t *matrix, uint8_t x, uint8_t imm, int inverse) { + if (inverse) x = gf_inv(x); + uint8_t r = 0; + for (int bit = 0; bit < 8; bit++) r |= (uint8_t)(parity8(matrix[7-bit] & x) << bit); + return r ^ imm; +} +int main(void) { + uint8_t matrix[64], data[64], other[64], old[64], out[513] = {0}; + const uint64_t mask = UINT64_C(0xa55aa55af00f0ff0); + for (int i = 0; i < 64; i++) { + matrix[i] = (uint8_t)(0x81u + 29u * (unsigned)i); + data[i] = (uint8_t)(3u + 17u * (unsigned)i); + other[i] = (uint8_t)(7u + 11u * (unsigned)i); + old[i] = (uint8_t)(0xe0u - (unsigned)i); + } + gfnisemantics(out, matrix, data, other, old, mask); + for (int i = 0; i < 64; i++) { + const uint8_t mul = gf_mul(other[i], data[i]); + const uint8_t aq = affine(matrix + (i/8)*8, data[i], 0x63, 0); + const uint8_t ai = affine(matrix + (i/8)*8, data[i], 0xa5, 1); + if (out[i] != mul) return 10; + if (out[64+i] != aq) return 11; + if (out[128+i] != ai) return 12; + if (out[192+i] != (((mask>>i)&1) ? mul : old[i])) return 13; + if (out[256+i] != (((mask>>i)&1) ? aq : 0)) return 14; + if (out[320+i] != affine(matrix, data[i], 0x63, 0)) return 15; + if (out[384+i] != (((mask>>i)&1) ? affine(matrix, data[i], 0xa5, 1) : 0)) return 16; + if (out[448+i] != mul) return 17; + } + if (out[512] != 1) return 18; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "gfni_semantics", triple, ir, mainC, nil) +} diff --git a/amd64_lower_halt_test.go b/amd64_lower_halt_test.go new file mode 100644 index 000000000..baff7b503 --- /dev/null +++ b/amd64_lower_halt_test.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86HLTCompleteGoAssemblerForms(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT haltforms(SB),NOSPLIT,$0-0\n\tHLT\n") + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"haltforms": {Name: "haltforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, `asm sideeffect "hlt"`) || !strings.Contains(ll, "unreachable") { + t.Fatalf("HLT must retain the hardware instruction and terminate control flow:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "hlt-"+target.name+".ll", "hlt-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86HLTRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{"HLT AX", "HLT $1", "HLT.P"} { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n") + if err != nil { + continue + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's HLT ynone table", instruction) + } + } +} diff --git a/amd64_lower_horizontal_float.go b/amd64_lower_horizontal_float.go new file mode 100644 index 000000000..df335dcbd --- /dev/null +++ b/amd64_lower_horizontal_float.go @@ -0,0 +1,40 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64HorizontalFloatingSpec struct { + laneBits int + subtract bool + vector bool +} + +// amd64HorizontalFloatingSpecs is the complete Go 1.27 horizontal +// floating-point add/subtract family. Legacy forms use yxm; V-prefixed forms +// use _yvaddsubpd and are deliberately limited to VEX X/Y registers. +var amd64HorizontalFloatingSpecs = map[Op]amd64HorizontalFloatingSpec{ + "HADDPS": {laneBits: 32}, + "HADDPD": {laneBits: 64}, + "HSUBPS": {laneBits: 32, subtract: true}, + "HSUBPD": {laneBits: 64, subtract: true}, + "VHADDPS": {laneBits: 32, vector: true}, + "VHADDPD": {laneBits: 64, vector: true}, + "VHSUBPS": {laneBits: 32, subtract: true, vector: true}, + "VHSUBPD": {laneBits: 64, subtract: true, vector: true}, +} + +func (c *amd64Ctx) lowerHorizontalFloating(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerNativePackedReduction(op, ins) +} + +func (c *amd64Ctx) loadHorizontalFloatingOperand(operand Operand, byteWidth, laneBits int) (string, error) { + bytesValue, err := c.loadPackedCompareBytes(operand, byteWidth) + if err != nil { + return "", err + } + lanes := byteWidth * 8 / laneBits + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", result, byteWidth, bytesValue, amd64FMA3LLVMType(lanes, laneBits)) + return "%" + result, nil +} diff --git a/amd64_lower_horizontal_integer.go b/amd64_lower_horizontal_integer.go new file mode 100644 index 000000000..5798cc4fb --- /dev/null +++ b/amd64_lower_horizontal_integer.go @@ -0,0 +1,154 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerHorizontalInteger(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + _, recognized := amd64HorizontalIntegerSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("horizontal integer dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := x86HorizontalIntegerProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + spec, byteWidth := form.spec, form.byteWidth + if form.memory >= 0 && ins.Args[form.memory].Kind == OpFP { + if err := c.validateFPVectorSpan(ins.Args[form.memory].FPOffset, byteWidth); err != nil { + return true, false, err + } + } + destination := ins.Args[form.destination].Reg + if form.mmx { + memoryBytes := 0 + if form.memory >= 0 { + memoryBytes = byteWidth + } + firstBits, err := c.loadPackedUnpackMMXSource(ins.Args[form.first], memoryBytes) + if err != nil { + return true, false, err + } + secondBits, err := c.loadReg(destination) + if err != nil { + return true, false, err + } + first := c.bitcastI64ToIntegerLanes(2, 32, firstBits) + second := c.bitcastI64ToIntegerLanes(2, 32, secondBits) + result := c.emitHorizontalInteger(spec, 8, first, second) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i32> %s to i64\n", bits, result) + return true, false, c.storeReg(destination, "%"+bits) + } + firstOperand := ins.Args[form.first] + if !spec.vector && form.memory >= 0 && (firstOperand.Kind == OpFP || ins.x86AddressBits != 0 || + firstOperand.Kind == OpMem && !c.nativeMemoryOperand(ins, form.first)) { + return true, false, fmt.Errorf("%w: legacy horizontal m128 needs original native backing/alignment", ErrProbeNeedsContext) + } + second, err := c.loadHorizontalIntegerOperand(ins.Args[form.second], byteWidth, spec.laneBits) + if err != nil { + return true, false, err + } + var result string + if !spec.vector && form.memory >= 0 { + result, err = c.emitPackedBinaryNativeMemory(firstOperand, spec.laneBits, second, strings.ToLower(baseOp)) + } else { + var first string + first, err = c.loadHorizontalIntegerOperand(firstOperand, byteWidth, spec.laneBits) + if err == nil { + result = c.emitHorizontalInteger(spec, byteWidth, first, second) + } + } + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / spec.laneBits + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + if !spec.vector { + return true, false, c.storeLegacyPackedLogical(ins.Args[form.destination], "%"+out) + } + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], byteWidth, "%"+out) +} + +func (c *amd64Ctx) loadHorizontalIntegerOperand(operand Operand, byteWidth, laneBits int) (string, error) { + bytesValue, err := c.loadX86PackedShiftBytes(operand, byteWidth, true) + if err != nil { + return "", err + } + lanes := byteWidth * 8 / laneBits + return c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, bytesValue), nil +} + +func (c *amd64Ctx) emitHorizontalInteger(spec amd64HorizontalIntegerSpec, byteWidth int, first, second string) string { + lanes := byteWidth * 8 / spec.laneBits + lanesPer128 := 128 / spec.laneBits + // The legacy MMX PHADDD row is one 64-bit lane group rather than one + // 128-bit group, but the same half-from-each-source construction applies. + if byteWidth == 8 { + lanesPer128 = lanes + } + typeName := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + elementType := fmt.Sprintf("i%d", spec.laneBits) + result := "poison" + for lane := 0; lane < lanes; lane++ { + position := lane % lanesPer128 + block := lane - position + source := second + pair := position + if position >= lanesPer128/2 { + source = first + pair -= lanesPer128 / 2 + } + leftIndex := block + 2*pair + rightIndex := leftIndex + 1 + left := c.newTmp() + right := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", left, typeName, source, leftIndex) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", right, typeName, source, rightIndex) + value := c.emitHorizontalIntegerPair(spec, "%"+left, "%"+right) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %s, i32 %d\n", inserted, typeName, result, elementType, value, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitHorizontalIntegerPair(spec amd64HorizontalIntegerSpec, left, right string) string { + operation := "add" + if spec.subtract { + operation = "sub" + } + if !spec.saturate { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i%d %s, %s\n", value, operation, spec.laneBits, left, right) + return "%" + value + } + leftWide := c.newTmp() + rightWide := c.newTmp() + value := c.newTmp() + below := c.newTmp() + lowerClamped := c.newTmp() + above := c.newTmp() + clamped := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i16 %s to i32\n", leftWide, left) + fmt.Fprintf(c.b, " %%%s = sext i16 %s to i32\n", rightWide, right) + fmt.Fprintf(c.b, " %%%s = %s i32 %%%s, %%%s\n", value, operation, leftWide, rightWide) + fmt.Fprintf(c.b, " %%%s = icmp slt i32 %%%s, -32768\n", below, value) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 -32768, i32 %%%s\n", lowerClamped, below, value) + fmt.Fprintf(c.b, " %%%s = icmp sgt i32 %%%s, 32767\n", above, lowerClamped) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 32767, i32 %%%s\n", clamped, above, lowerClamped) + fmt.Fprintf(c.b, " %%%s = trunc i32 %%%s to i16\n", result, clamped) + return "%" + result +} diff --git a/amd64_lower_horizontal_integer_test.go b/amd64_lower_horizontal_integer_test.go new file mode 100644 index 000000000..68cd8129f --- /dev/null +++ b/amd64_lower_horizontal_integer_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86HorizontalIntegerCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + source.WriteString("TEXT horizontalintegerforms(SB),NOSPLIT,$0-0\n") + // PHADDD uniquely uses Go's ymmxmm0f38 table and therefore has + // both MMX/m64 and XMM/m128 legacy rows. + source.WriteString("\tPHADDD M0, M1\n\tPHADDD 8(AX), M2\n") + for _, op := range []string{"PHADDD", "PHADDSW", "PHADDW", "PHSUBD", "PHSUBSW", "PHSUBW"} { + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, last) + vector := "V" + op + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s%d\n", vector, width, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s1, %s%d\n", vector, width, width, last) + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "horizontalintegerforms": {Name: "horizontalintegerforms", Ret: Void}, + }, + }) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized original formats acquired numerical/MMX inputs", err) + } + ll := x86InstructionFormatObjectUnit(t, file, + map[string]FuncSig{"horizontalintegerforms": {Name: "horizontalintegerforms", Ret: Void}}, + target.goarch, target.triple, 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerHorizontalInteger(ins.Op, ins) + }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "horizontal-integer-"+target.name+".ll", "horizontal-integer-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86HorizontalIntegerRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PHADDD.Z X0, X1", + "PHADDD X0", + "PHADDSW M0, M1", + "PHADDW M0, M1", + "PHSUBD M0, M1", + "PHSUBSW M0, M1", + "PHSUBW M0, M1", + "PHADDW X0, M1", + "PHSUBD Y0, Y1", + "VPHADDD.Z X0, X1, X2", + "VPHADDSW X0, X1", + "VPHADDW X0, Y1, Y2", + "VPHSUBD Z0, Z1, Z2", + "VPHSUBSW X0, X1, K1, X2", + "VPHSUBW X0, X1, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86HorizontalIntegerRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PHADDD X8, X0", + "PHADDW X0, X8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86HorizontalIntegerRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86HorizontalIntegerRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's horizontal-integer forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_ifma.go b/amd64_lower_ifma.go new file mode 100644 index 000000000..57c43216c --- /dev/null +++ b/amd64_lower_ifma.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64IFMASpec struct { + x86PackedBinarySpec + high bool +} + +// amd64IFMASpecs is the complete Go 1.27 integer fused multiply-add family. +// Both instructions use _yvblendmpd and operate on the low unsigned 52 bits +// of each qword source. LUQ adds product bits 51:0; HUQ adds bits 103:52. +var amd64IFMASpecs = map[Op]amd64IFMASpec{ + "VPMADD52HUQ": { + x86PackedBinarySpec: x86PackedBinarySpec{ + encoding: x86PackedLogicalEVEX, broadcastBytes: 8, zeroing: true, + }, + high: true, + }, + "VPMADD52LUQ": { + x86PackedBinarySpec: x86PackedBinarySpec{ + encoding: x86PackedLogicalEVEX, broadcastBytes: 8, zeroing: true, + }, + }, +} + +func (c *amd64Ctx) lowerIFMA(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86IFMAProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + source, destination := ins.Args[form.first], ins.Args[form.destination] + if form.memory >= 0 { + if source.Kind == OpMem && !c.nativeMemoryOperand(ins, form.memory) && + !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + return true, false, fmt.Errorf("%w: IFMA source needs its original native storage binding", ErrProbeNeedsContext) + } + if source.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(source.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + } + mask := "" + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + first, err := c.loadX86PackedShiftLanes(source, form.byteWidth, 64, form.broadcast, mask) + if err != nil { + return true, false, err + } + second, err := c.loadPackedCompareLanes(ins.Args[form.second], form.byteWidth, 64, false) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, form.byteWidth) + if err != nil { + return true, false, err + } + lanes := form.byteWidth / 8 + old := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, 64, oldBytes) + computed := c.emitIFMA(form.spec, lanes, first, second, old) + if form.mask >= 0 { + computed = amd64ApplyIntegerLaneMask(c, lanes, 64, computed, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", out, lanes, computed, form.byteWidth) + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitIFMA(spec amd64IFMASpec, lanes int, first, second, accumulator string) string { + const low52Mask = "4503599627370495" + result := "poison" + for lane := 0; lane < lanes; lane++ { + firstLane := c.newTmp() + secondLane := c.newTmp() + oldLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", firstLane, lanes, first, lane) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", secondLane, lanes, second, lane) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", oldLane, lanes, accumulator, lane) + maskedFirst := c.newTmp() + maskedSecond := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %s\n", maskedFirst, firstLane, low52Mask) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %s\n", maskedSecond, secondLane, low52Mask) + wideFirst := c.newTmp() + wideSecond := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i64 %%%s to i128\n", wideFirst, maskedFirst) + fmt.Fprintf(c.b, " %%%s = zext i64 %%%s to i128\n", wideSecond, maskedSecond) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", product, wideFirst, wideSecond) + selected := "%" + product + if spec.high { + high := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i128 %%%s, 52\n", high, product) + selected = "%" + high + } else { + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i128 %%%s, %s\n", low, product, low52Mask) + selected = "%" + low + } + productPart := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i128 %s to i64\n", productPart, selected) + added := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", added, oldLane, productPart) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, lanes, result, added, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_ifma_test.go b/amd64_lower_ifma_test.go new file mode 100644 index 000000000..4b9494ad0 --- /dev/null +++ b/amd64_lower_ifma_test.go @@ -0,0 +1,244 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86IFMACompleteGo127Forms(t *testing.T) { + // VPMADD52HUQ/LUQ share Go 1.27's six-row _yvblendmpd table: X/Y/Z, + // each with an unmasked and K1-K7 masked row. Their EVEX attributes add + // scalar qword broadcast and zeroing. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT ifma_forms(SB),$0-0\n") + for _, op := range []string{"VPMADD52HUQ", "VPMADD52LUQ"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s 8(BX), %s3, %s4\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST 16(BX), %s5, %s6\n", op, width, width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 24(BX), %s23, K7, %s24\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(BX), %s25, K2, %s26\n", op, width, width) + } + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "ifma_forms": {Name: "ifma_forms", Ret: Void}, + }, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original IFMA formats invented native input addresses", err) + } + ir = x86InstructionFormatObjectUnit(t, file, + map[string]FuncSig{"ifma_forms": {Name: "ifma_forms", Ret: Void}}, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerIFMA(ins.Op, ins) }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "ifma-"+target.name+".ll", "ifma-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86IFMARejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VPMADD52HUQ X0, X1", + "VPMADD52LUQ X0, X1, X2, X3, X4", + "VPMADD52HUQ Y0, X1, X2", + "VPMADD52LUQ X0, Y1, X2", + "VPMADD52HUQ X0, X1, Y2", + "VPMADD52LUQ X0, 8(BX), X2", + "VPMADD52HUQ X0, X1, 8(BX)", + "VPMADD52LUQ X0, X1, K0, X2", + "VPMADD52HUQ X0, K1, X1, X2", + "VPMADD52LUQ.Z X0, X1, X2", + "VPMADD52HUQ.BCST X0, X1, X2", + "VPMADD52LUQ.BCST.Z 8(BX), X1, X2", + "VPMADD52HUQ.Z.BCST 8(BX), X1, K1, X2", + "VPMADD52LUQ.SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's IFMA table", instruction) + } + }) + } + for _, instruction := range []string{ + "VPMADD52HUQ X0, X1, K1, X2", + "VPMADD52LUQ.Z 8(BX), Y1, K7, Y2", + "VPMADD52HUQ.BCST.Z 16(BX), Z1, K2, Z2", + "VPMADD52LUQ Z8, Z0, Z1", + "VPMADD52HUQ Z0, Z8, Z1", + "VPMADD52LUQ Z0, Z1, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted 386 %q outside Go 1.27's IFMA frontend boundary", instruction) + } + }) + } +} + +func TestAMD64IFMARuntimeSemantics(t *testing.T) { + // Integer LLVM and the independent i128 C reference are portable on + // both supported 64-bit hosts; no AVX512 or Rosetta execution is implied. + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable runtime oracle requires its native 64-bit host counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT ifmaSemantics(SB),$0-48 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + MOVQ accumulator+24(FP), DX + MOVQ scalar+32(FP), SI + VMOVDQU64 (BX), X0 + VMOVDQU64 (CX), X1 + VMOVDQU64 (DX), X2 + VPMADD52LUQ X0, X1, X2 + VMOVDQU64 X2, 0(AX) + VMOVDQU64 (DX), X3 + VPMADD52HUQ X0, X1, X3 + VMOVDQU64 X3, 16(AX) + KMOVQ mask+40(FP), K1 + VMOVDQU64 (DX), X4 + VPMADD52LUQ X0, X1, K1, X4 + VMOVDQU64 X4, 32(AX) + VMOVDQU64 (DX), X5 + VPMADD52LUQ.Z X0, X1, K1, X5 + VMOVDQU64 X5, 48(AX) + VMOVDQU64 (DX), X6 + VPMADD52LUQ.BCST (SI), X1, X6 + VMOVDQU64 X6, 64(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "ifmaSemantics": { + Name: "ifmaSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, I64}, + Ret: Void, + Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void ifmaSemantics(uint64_t *, const uint64_t *, const uint64_t *, const uint64_t *, const uint64_t *, uint64_t); +static const uint64_t mask52 = (((uint64_t)1) << 52) - 1; +static uint64_t lo(uint64_t a, uint64_t b) { + return (uint64_t)((((unsigned __int128)(a & mask52)) * (b & mask52)) & mask52); +} +static uint64_t hi(uint64_t a, uint64_t b) { + return (uint64_t)((((unsigned __int128)(a & mask52)) * (b & mask52)) >> 52); +} +int main(void) { + const uint64_t first[2] = {(UINT64_C(1) << 60) | mask52, (UINT64_C(1) << 55) | (UINT64_C(1) << 51) | 3}; + const uint64_t second[2] = {(UINT64_C(1) << 59) | mask52, (UINT64_C(1) << 58) | (UINT64_C(1) << 51) | 5}; + const uint64_t accumulator[2] = {UINT64_MAX - 15, 100}; + const uint64_t scalar = (UINT64_C(1) << 63) | 7; + uint64_t out[10] = {0}; + const uint64_t want[10] = { + accumulator[0] + lo(first[0], second[0]), accumulator[1] + lo(first[1], second[1]), + accumulator[0] + hi(first[0], second[0]), accumulator[1] + hi(first[1], second[1]), + accumulator[0] + lo(first[0], second[0]), accumulator[1], + accumulator[0] + lo(first[0], second[0]), 0, + accumulator[0] + lo(scalar, second[0]), accumulator[1] + lo(scalar, second[1]), + }; + ifmaSemantics(out, first, second, accumulator, &scalar, 1); + for (int i = 0; i < 10; i++) if (out[i] != want[i]) return i + 1; + return 0; +} +` + compiler := []string{clang} + if triple != testTargetTriple(runtime.GOOS, runtime.GOARCH) { + compiler = append(compiler, "-target", triple) + } + for _, pipeline := range []struct { + name string + optimized bool + backend int + }{ + {name: "backend_O0"}, + {name: "backend_O2", backend: 2}, + {name: "IR_O2/backend_O2", optimized: true, backend: 2}, + } { + t.Run(pipeline.name, func(t *testing.T) { + text := ir + if pipeline.optimized { + text = x86DwordProductIROptimize(t, ir) + } + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, "ifma", triple, + text, mainC, nil, pipeline.backend) + }) + } +} diff --git a/amd64_lower_implicit_legacy_test.go b/amd64_lower_implicit_legacy_test.go new file mode 100644 index 000000000..7db18e967 --- /dev/null +++ b/amd64_lower_implicit_legacy_test.go @@ -0,0 +1,155 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64LeaveGrammarIsComplete(t *testing.T) { + want := map[Op]amd64LeaveSpec{ + "LEAVEW": {bits: 16}, + "LEAVEL": {bits: 32}, + "LEAVEQ": {bits: 64}, + } + if !reflect.DeepEqual(amd64LeaveSpecs, want) { + t.Fatalf("LEAVE grammar = %+v, want %+v", amd64LeaveSpecs, want) + } +} + +func TestTranslateX86LeaveAndXLATCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + secondLeave := "LEAVEQ" + if target.goarch == "386" { + secondLeave = "LEAVEL" + } + source := "TEXT implicitlegacyforms(SB),$0-0\n" + + "\tLEAVEW\n" + + "\t" + secondLeave + "\n" + + "\tXLAT\n" + + "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + x86PhysicalSPRequirePublicContext(t, source, "func implicitlegacyforms()", target.goarch, target.triple) + ir := x86ScalarFormatObjectUnit(t, file, map[string]FuncSig{ + "implicitlegacyforms": {Name: "implicitlegacyforms", Ret: Void}, + }, target.goarch, target.triple, 0) + for _, want := range []string{"load i16", "load volatile i8", "-256"} { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if target.goarch == "386" { + if !strings.Contains(ir, "load i32") { + t.Errorf("386 IR is missing LEAVEL load:\n%s", ir) + } + } else if !strings.Contains(ir, "load i64") { + t.Errorf("amd64 IR is missing LEAVEQ load:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "implicit-legacy-"+target.name+".ll", "implicit-legacy-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LeaveAndXLATRejectFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LEAVEW AX"}, + {goarch: "amd64", instruction: "LEAVEL"}, + {goarch: "amd64", instruction: "LEAVEQ.P"}, + {goarch: "amd64", instruction: "XLAT AX"}, + {goarch: "amd64", instruction: "XLAT.P"}, + {goarch: "386", instruction: "LEAVEQ"}, + {goarch: "386", instruction: "LEAVEL AX"}, + {goarch: "386", instruction: "XLAT AX"}, + } { + t.Run(test.goarch+"_"+strings.ReplaceAll(test.instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's LEAVE/XLAT tables", test.instruction) + } + }) + } +} + +func TestAMD64LeaveAndXLATRuntimeSemantics(t *testing.T) { + execution := x86ScalarCountHostExecution(t) + const source = `TEXT implicitlegacysemantics(SB),NOSPLIT,$0-16 + MOVQ out+0(FP), R8 + MOVQ data+8(FP), BP + LEAVEQ + MOVQ BP, 0(R8) + MOVQ SP, 8(R8) + MOVQ data+8(FP), BX + ADDQ $16, BX + MOVQ $0x1122334455667703, AX + XLAT + MOVQ AX, 16(R8) + RET +` + // LEAVE changes physical SP and invalidates the later Go FP/RET transport. + // Preserve this complete original body as Context and execute only its + // explicitly private virtual-register mathematics, never a native Go ABI. + x86PhysicalSPRequirePublicContext(t, source, "func implicitlegacysemantics(out,data *byte)", "amd64", execution.triple) + ir := x86PhysicalSPPrivateEmitter(t, source, + map[string]FuncSig{ + "implicitlegacysemantics": { + Name: "implicitlegacysemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + }, "amd64", execution.triple) + const mainC = ` +#include +#include +extern void implicitlegacysemantics(uint8_t *, uint8_t *); +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint8_t data[256] = {0}, out[24] = {0}; + const uint64_t saved = UINT64_C(0xa1b2c3d4e5f60718); memcpy(data,&saved,8); + data[19] = 0xaa; + implicitlegacysemantics(out,data); + if (load64(out) != saved) return 10; + if (load64(out+8) != (uintptr_t)(data+8)) return 11; + if (load64(out+16) != UINT64_C(0x11223344556677aa)) return 12; + return 0; +} +` + execution.compileAndRunLLVM(t, "implicit_legacy_private_math", ir, + "#define implicitlegacysemantics InstructionFormat0\n"+mainC, "-O2") + t.Log("private virtual-SP LEAVE/XLAT mathematics only; original public source remains Context, not native Go ABI") +} diff --git a/amd64_lower_implicit_mask_move.go b/amd64_lower_implicit_mask_move.go new file mode 100644 index 000000000..0898715fb --- /dev/null +++ b/amd64_lower_implicit_mask_move.go @@ -0,0 +1,54 @@ +package plan9asm + +import "fmt" + +type amd64ImplicitMaskMoveSpec struct { + byteWidth int + vector bool +} + +// Go's frontend exposes the legacy SSE spelling both as MASKMOVOU and via +// the Intel-name alias MASKMOVDQU. MASKMOVQ is the corresponding MMX form; +// VMASKMOVDQU is the VEX spelling. All use the first operand as a byte mask, +// the second as data, and DI as the implicit memory destination. +var amd64ImplicitMaskMoveSpecs = map[Op]amd64ImplicitMaskMoveSpec{ + "MASKMOVQ": {byteWidth: 8}, + "MASKMOVOU": {byteWidth: 16}, + "MASKMOVDQU": {byteWidth: 16}, + "VMASKMOVDQU": {byteWidth: 16, vector: true}, +} + +func (c *amd64Ctx) lowerImplicitMaskMove(op Op, ins Instr) (bool, bool, error) { + spec, known, err := x86ImplicitMaskMoveProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if spec.byteWidth == 8 || !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || + !x86ResultNamedPredicateInstruction(ins) { + return true, false, fmt.Errorf("%w: implicit mask store needs its original native source/target and MMX owner", ErrProbeNeedsContext) + } + mask, err := c.loadX(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + data, err := c.loadX(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + addressInteger, err := c.loadReg(DI) + if err != nil { + return true, false, err + } + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", address, addressInteger) + mnemonic := "maskmovdqu" + if spec.vector { + mnemonic = "vmaskmovdqu" + } + // Preserve WC/non-temporal behavior and implementation-dependent + // zero-mask address faults, not a removable masked store or a fabricated + // unconditional read of the destination. + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(<16 x i8> %s, <16 x i8> %s, ptr %%%s)\n", + mnemonic+" $1, $0", "x,x,{di},~{memory}", data, mask, address) + return true, false, nil +} diff --git a/amd64_lower_implicit_mask_move_test.go b/amd64_lower_implicit_mask_move_test.go new file mode 100644 index 000000000..672575d14 --- /dev/null +++ b/amd64_lower_implicit_mask_move_test.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86ImplicitMaskMoveCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + source := fmt.Sprintf("TEXT implicitmaskmoveforms(SB),NOSPLIT,$0-0\n"+ + "\tMASKMOVQ M0, M7\n"+ + "\tMASKMOVOU X0, X%d\n"+ + "\tMASKMOVDQU X0, X%d\n"+ + "\tVMASKMOVDQU X0, X%d\n"+ + "\tRET\n", last, last, last) + // Preserve all four original rows and their actual Go objects. + // These bare registers/DI cannot establish a successful public + // translation. The independently initialized full-family root + // compiles real translated objects at the same five targets. + file, _ := x86UnboundVoidFormatSources(t, source, target.goarch, target.triple) + count := 0 + for _, instruction := range file.Funcs[0].Instrs { + if instruction.Op == OpTEXT || instruction.Op == OpRET { + continue + } + if _, known, err := x86ImplicitMaskMoveProperties(instruction.Op, instruction, target.goarch); !known || err != nil { + t.Fatal("original implicit mask-store format disappeared", instruction.Raw, err) + } + count++ + } + if count != 4 { + t.Fatal("original complete implicit mask-store denominator changed", count) + } + }) + } +} + +func TestTranslateX86ImplicitMaskMoveRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "MASKMOVQ.Z M0, M1", + "MASKMOVQ M0", + "MASKMOVQ M0, X1", + "MASKMOVQ 8(AX), M1", + "MASKMOVOU.Z X0, X1", + "MASKMOVOU X0", + "MASKMOVOU M0, M1", + "MASKMOVOU X0, Y1", + "MASKMOVDQU.Z X0, X1", + "MASKMOVDQU X0, X1, X2", + "MASKMOVDQU 8(AX), X1", + "VMASKMOVDQU.Z X0, X1", + "VMASKMOVDQU X0", + "VMASKMOVDQU Y0, Y1", + "VMASKMOVDQU X0, K1, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ImplicitMaskMoveRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "MASKMOVOU X8, X0", + "MASKMOVDQU X0, X8", + "VMASKMOVDQU X8, X0", + "VMASKMOVDQU X0, X8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86ImplicitMaskMoveRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86ImplicitMaskMoveRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's implicit mask-move forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_implicit_system.go b/amd64_lower_implicit_system.go new file mode 100644 index 000000000..596f28f43 --- /dev/null +++ b/amd64_lower_implicit_system.go @@ -0,0 +1,255 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" +) + +type amd64ImplicitSystemOperand uint8 + +const ( + amd64ImplicitSystemNoOperand amd64ImplicitSystemOperand = iota + amd64ImplicitSystemGP + amd64ImplicitSystemAddressGP +) + +type amd64ImplicitRegisterSet uint8 + +const ( + amd64ImplicitAX amd64ImplicitRegisterSet = 1 << iota + amd64ImplicitCX + amd64ImplicitDX +) + +type amd64ImplicitSystemSpec struct { + operand amd64ImplicitSystemOperand + inputs amd64ImplicitRegisterSet + outputs amd64ImplicitRegisterSet + writesCarry bool + rawPrefix byte +} + +// amd64ImplicitSystemSpecs is the complete Go 1.27 grammar for the system +// instructions whose architectural state is carried by implicit AX/CX/DX +// registers, plus the WAITPKG instructions' one explicit Yrl register. The +// table separates operand syntax from data flow so parsing, register liveness, +// and emission all consume the same typed description. +var amd64ImplicitSystemSpecs = map[Op]amd64ImplicitSystemSpec{ + "MONITOR": {inputs: amd64ImplicitAX | amd64ImplicitCX | amd64ImplicitDX}, + "MWAIT": {inputs: amd64ImplicitAX | amd64ImplicitCX}, + "RDPMC": {inputs: amd64ImplicitCX, outputs: amd64ImplicitAX | amd64ImplicitDX}, + "RDPKRU": {inputs: amd64ImplicitCX, outputs: amd64ImplicitAX | amd64ImplicitDX}, + "WRPKRU": {inputs: amd64ImplicitAX | amd64ImplicitCX | amd64ImplicitDX}, + "XSETBV": {inputs: amd64ImplicitAX | amd64ImplicitCX | amd64ImplicitDX}, + "UMONITOR": {operand: amd64ImplicitSystemAddressGP, rawPrefix: 0xf3}, + "UMWAIT": {operand: amd64ImplicitSystemGP, inputs: amd64ImplicitAX | amd64ImplicitDX, writesCarry: true, rawPrefix: 0xf2}, + "TPAUSE": {operand: amd64ImplicitSystemGP, inputs: amd64ImplicitAX | amd64ImplicitDX, writesCarry: true, rawPrefix: 0x66}, +} + +type amd64ImplicitSystemForm struct { + register Reg + addressBits int + segmentPrefix byte +} + +func (c *amd64Ctx) lowerImplicitSystem(op Op, ins Instr) (ok bool, terminated bool, err error) { + spec, form, known, err := x86ImplicitSystemProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return c.emitImplicitSystem(strings.ToUpper(string(op)), spec, form) +} + +func x86ImplicitSystemProperties(op Op, ins Instr, arch string) (amd64ImplicitSystemSpec, amd64ImplicitSystemForm, bool, error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64ImplicitSystemSpecs[Op(baseOp)] + if !recognized { + return spec, amd64ImplicitSystemForm{}, false, nil + } + if rawOp != baseOp || arch != "386" && arch != "amd64" || + ins.Op != "" && !strings.EqualFold(string(ins.Op), rawOp) { + return spec, amd64ImplicitSystemForm{}, true, fmt.Errorf("%s %s has an invalid suffix, mode or dispatch identity: %q", arch, op, ins.Raw) + } + form, err := parseX86ImplicitSystemForm(baseOp, spec, ins, arch) + return spec, form, true, err +} + +func parseX86ImplicitSystemForm(op string, spec amd64ImplicitSystemSpec, ins Instr, arch string) (amd64ImplicitSystemForm, error) { + var form amd64ImplicitSystemForm + switch spec.operand { + case amd64ImplicitSystemNoOperand: + if len(ins.Args) != 0 { + return form, fmt.Errorf("%s %s takes no operands: %q", arch, op, ins.Raw) + } + case amd64ImplicitSystemGP, amd64ImplicitSystemAddressGP: + if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[0].Reg, arch) { + return form, fmt.Errorf("%s %s expects one Go 1.27 Yrl register: %q", arch, op, ins.Raw) + } + form.register = ins.Args[0].Reg + if spec.operand == amd64ImplicitSystemAddressGP { + form.addressBits = 64 + if arch == "386" { + form.addressBits = 32 + } + if ins.x86Encoded { + if ins.x86AddressBits != 0 { + if ins.x86AddressBits != form.addressBits && ins.x86AddressBits != form.addressBits/2 { + return form, fmt.Errorf("invalid %s address size %d", arch, ins.x86AddressBits) + } + form.addressBits = ins.x86AddressBits + } + form.segmentPrefix = ins.x86SegmentPrefix + } + } + default: + panic("unknown amd64 implicit-system operand kind") + } + return form, nil +} + +// UMWAIT/TPAUSE consume their complete dword control and EDX:EAX deadline. +// Intel SDM defines CF and clears AF/PF/SF/ZF/OF on continuation; GP, DF and +// storage are unchanged. This numerical descriptor grants no monitor address, +// raw/native entry or promise that the operation cannot fault. +func x86TypedWaitPackageByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + spec, form, known, err := x86ImplicitSystemProperties(op, ins, arch) + if !known || !spec.writesCarry { + return x86TypedByteEffect{}, false, nil + } + if err != nil { + return x86TypedByteEffect{}, true, err + } + if !x86WaitPackagePreservesStorage(ins) { + return x86TypedByteEffect{}, true, fmt.Errorf("%w: WAITPKG original/normalized numerical source differs", ErrProbeNeedsContext) + } + effect := x86TypedByteEffect{ + operands: []x86TypedByteOperand{ + {index: 0, operand: Operand{Kind: OpReg, Reg: form.register}, width: 4, read: true}, + }, + flagsDefined: x86ByteFlagsAll, flagsKilled: x86ByteFlagsAll, requiresResultProof: true, + } + for index, register := range []Reg{AX, CX, DX} { + if spec.inputs&(1<= 0 { + baseOp = rawOp[:dot] + } + bits := 0 + switch baseOp { + case "INCB", "DECB": + bits = 8 + case "INCW", "DECW": + bits = 16 + case "INCL", "DECL": + bits = 32 + case "INCQ", "DECQ": + bits = 64 + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 1 { + return true, false, fmt.Errorf("%s %s expects one destination: %q", c.goarch, baseOp, ins.Raw) + } + dst := ins.Args[0] + if dst.Kind == OpReg { + if bits == 8 { + if !isGoYmbRegisterForArch(dst.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Ymb class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isX86YrlRegisterForArch(dst.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yml class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(dst) { + return true, false, fmt.Errorf("%s %s expects a GP register or memory destination: %q", c.goarch, baseOp, ins.Raw) + } + + ty := amd64IntegerTypeForBits(bits) + old, store, err := c.loadScalarRMWDestination(op, ins, dst, ty) + if err != nil { + return true, false, err + } + result := c.newTmp() + llvmOp := "add" + if baseOp[:3] == "DEC" { + llvmOp = "sub" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s, 1\n", result, llvmOp, ty, old) + value := "%" + result + if err := store(value); err != nil { + return true, false, err + } + + zero := c.newTmp() + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", zero, ty, value) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, 0\n", sign, ty, value) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + c.setParityFlagSized(ty, value) + c.setAuxiliaryCarryFlag(ty, old, "1", value) + + overflowInput := int64(1) << (bits - 1) + if baseOp[:3] == "INC" { + overflowInput-- + } else { + overflowInput = -overflowInput + } + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, %d\n", overflow, ty, old, overflowInput) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) + return true, false, nil +} diff --git a/amd64_lower_insert.go b/amd64_lower_insert.go new file mode 100644 index 000000000..d54ecfc16 --- /dev/null +++ b/amd64_lower_insert.go @@ -0,0 +1,9 @@ +package plan9asm + +// lowerPackedVectorInsert implements every Go 1.27 form in _yvinsertf128, +// _yvinsertf32x4, and _yvinsertf32x8. The floating and integer spellings have +// identical bit-level insert behavior; their lane type only changes EVEX mask +// granularity. +func (c *amd64Ctx) lowerPackedVectorInsert(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerInsertExtractFamily(op, ins) +} diff --git a/amd64_lower_insert_extract.go b/amd64_lower_insert_extract.go new file mode 100644 index 000000000..006140818 --- /dev/null +++ b/amd64_lower_insert_extract.go @@ -0,0 +1,154 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerInsertExtractFamily(op Op, ins Instr) (bool, bool, error) { + ins, f, known, err := c.bindInsertExtractForm(op, ins) + if !known || err != nil { + return known, false, err + } + if f.undefinedOpcode() { + return c.lowerX86UndefinedTrap(op, ins) + } + if f.mask >= 0 && f.memoryWrite { + return true, false, c.lowerNativeMaskedExtract(ins, f) + } + if f.spec.scalar { + if f.spec.insert { + return c.lowerInsertPackedSingle(op, ins) + } + return c.lowerPackedScalarExtract(op, ins) + } + src, err := c.loadInsertExtractSource(ins.Args[f.source], f) + if err != nil { + return true, false, err + } + result := src + if f.spec.insert { + base, err := c.loadPackedCompareBytes(ins.Args[f.base], f.width) + if err != nil { + return true, false, err + } + // Widen the insertion source only with defined zero bytes; then one + // shuffle reads the two old inputs before an aliased destination write. + wide := c.newTmp() + indices := make([]int, f.width) + for i := range indices { + indices[i] = i + if i >= f.spec.chunk { + indices[i] = f.spec.chunk + } + } + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> %s\n", wide, f.spec.chunk, src, f.spec.chunk, f.width, llvmI32Mask(indices)) + for i := range indices { + indices[i] = i + if i >= f.offset && i < f.offset+f.spec.chunk { + indices[i] = f.width + i - f.offset + } + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> %%%s, <%d x i32> %s\n", out, f.width, base, f.width, wide, f.width, llvmI32Mask(indices)) + result = "%" + out + } else { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> poison, <%d x i32> %s\n", out, f.width, src, f.width, f.output, llvmI32RangeMask(f.offset, f.output)) + result = "%" + out + } + if f.mask >= 0 { + mask, err := c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + lanes := f.output * 8 / f.spec.laneBits + computed := c.bitcastVectorBytesToIntegerLanes(f.output, lanes, f.spec.laneBits, result) + old := "zeroinitializer" + if !f.zeroing { + bytes, err := c.loadPackedCompareBytes(ins.Args[f.destination], f.output) + if err != nil { + return true, false, err + } + old = c.bitcastVectorBytesToIntegerLanes(f.output, lanes, f.spec.laneBits, bytes) + } + masked := amd64ApplyIntegerLaneMask(c, lanes, f.spec.laneBits, computed, old, mask, f.zeroing) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, f.spec.laneBits, masked, f.output) + result = "%" + out + } + destination := ins.Args[f.destination] + if f.memoryWrite && destination.Kind != OpFP { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(destination) + if err != nil { + return true, false, err + } + // Preserve every original unmasked write, including overwritten + // chunks, without reading old destination bytes or adding alignment. + fmt.Fprintf(c.b, " store volatile <%d x i8> %s, %s %s, align 1\n", f.output, result, pointerType, pointer) + return true, false, nil + } + return true, false, c.storePackedMoveOperand(destination, f.output, result) +} + +func (c *amd64Ctx) lowerNativeMaskedExtract(ins Instr, f x86InsertExtractForm) error { + destination := ins.Args[f.destination] + if destination.Kind == OpFP && c.classicFrame == "" { + return fmt.Errorf("%w: %s E6NF masked extraction needs contiguous original FP storage, not separately allocated virtual slots", ErrProbeNeedsContext, ins.Op) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(destination) + if err != nil { + return err + } + mask, err := c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return err + } + index, _ := amd64VectorRegisterIndex(ins.Args[f.source].Reg, f.width) + buffer := c.yRegSlot[index] + if f.width == 64 { + buffer = c.zRegSlot[index] + } + if buffer == "" { + return fmt.Errorf("%w: missing private virtual source buffer for %s", ErrProbeNeedsContext, ins.Args[f.source].Reg) + } + bank := "ymm" + if f.width == 64 { + bank = "zmm" + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } + if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + // This write-only native instruction supplies E6NF's nonsuppressing + // address/fault behavior without inventing old destination read access + // or writing inactive lanes. Only private physical scratch is clobbered; + // virtual SIMD/K state and CPU flags are unchanged. No target feature is + // enabled globally. Native execution still requires the source's ISA. + assembly := fmt.Sprintf("vmovdqu64 ($0), %%%s0; kmovq $1, %%k1; %s $$%d, %%%s0, %s($2) {%%k1}", bank, strings.ToLower(string(ins.Op)), f.immediate, bank, segment) + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(ptr %s, i64 %s, %s %s)\n", assembly, "r,r,r,~{zmm0},~{k1},~{memory}", buffer, mask, pointerType, pointer) + return nil +} + +func (c *amd64Ctx) loadInsertExtractSource(arg Operand, f x86InsertExtractForm) (string, error) { + if f.spec.insert && f.memory >= 0 && arg.Kind != OpFP { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(arg) + if err != nil { + return "", err + } + value := c.newTmp() + // Intel E6NF retains the entire native access even if every result + // lane is inactive or subsequently overwritten. Volatile prevents + // LLVM O2 from deleting that original observable faulting load. + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, %s %s, align 1\n", value, f.spec.chunk, pointerType, pointer) + return "%" + value, nil + } + width := f.width + if f.spec.insert { + width = f.spec.chunk + } + return c.loadPackedCompareBytes(arg, width) +} diff --git a/amd64_lower_insert_integer.go b/amd64_lower_insert_integer.go new file mode 100644 index 000000000..79568f8aa --- /dev/null +++ b/amd64_lower_insert_integer.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedIntegerInsertSpec struct { + laneBits int + legacy bool +} + +var amd64PackedIntegerInsertSpecs = map[string]amd64PackedIntegerInsertSpec{ + "PINSRB": {laneBits: 8, legacy: true}, + "PINSRW": {laneBits: 16, legacy: true}, + "PINSRD": {laneBits: 32, legacy: true}, + "PINSRQ": {laneBits: 64, legacy: true}, + "VPINSRB": {laneBits: 8}, + "VPINSRW": {laneBits: 16}, + "VPINSRD": {laneBits: 32}, + "VPINSRQ": {laneBits: 64}, +} + +func (c *amd64Ctx) lowerPackedIntegerInsert(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64PackedIntegerInsertSpecs[base]; !known { + return false, false, nil + } + ins, f, _, err := c.bindScalarLaneForm(op, ins) + if err != nil { + return true, false, err + } + source := ins.Args[f.source] + if source.Kind == OpReg { + source.Reg = f.transferReg + } + // Exact native m8/m16/m32/m64 reads remain volatile; FP uses the + // independent complete-byte-span helper, not the first scalar slot. + var scalar string + if source.Kind == OpFP { + scalar, err = c.loadScalarLaneFPBits(source.FPOffset, f.bits) + } else { + scalar, err = c.loadGPSIMDTransferSource(source, f.bits) + } + if err != nil { + return true, false, err + } + baseBytes, err := c.loadX(ins.Args[f.base].Reg) + if err != nil { + return true, false, err + } + lanes := 128 / f.bits + baseLanes, inserted, out := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <%d x i%d>\n", baseLanes, baseBytes, lanes, f.bits) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %%%s, i%d %s, i32 %d\n", inserted, lanes, f.bits, baseLanes, f.bits, scalar, f.offset/(f.bits/8)) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <16 x i8>\n", out, lanes, f.bits, inserted) + if f.legacy { + return true, false, c.storeX(ins.Args[f.destination].Reg, "%"+out) + } + return true, false, c.storePackedMoveOperand(ins.Args[f.destination], 16, "%"+out) +} + +func amd64IsExtendedGPRegister(reg Reg) bool { + switch strings.ToUpper(string(reg)) { + case "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15": + return true + default: + return false + } +} diff --git a/amd64_lower_insert_integer_test.go b/amd64_lower_insert_integer_test.go new file mode 100644 index 000000000..736f2e40c --- /dev/null +++ b/amd64_lower_insert_integer_test.go @@ -0,0 +1,126 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedIntegerInsertCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + var source strings.Builder + source.WriteString("TEXT packedintegerinsertforms(SB),$0-0\n") + legacyOps := []string{"PINSRB", "PINSRW", "PINSRD", "PINSRQ"} + vectorOps := []string{"VPINSRB", "VPINSRW", "VPINSRD", "VPINSRQ"} + if target.goarch == "386" { + legacyOps = legacyOps[:3] + vectorOps = nil + } + for _, op := range legacyOps { + fmt.Fprintf(&source, "\t%s $0, AX, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s $255, 8(AX), X%d\n", op, legacyLast) + } + for _, op := range vectorOps { + fmt.Fprintf(&source, "\t%s $0, AX, X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s $255, 8(AX), X0, X%d\n", op, legacyLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $82, R9, X22, X31\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedintegerinsertforms": {Name: "packedintegerinsertforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-integer-insert.ll", "packed-integer-insert.o", ll) + }) + } +} + +func TestTranslateX86PackedIntegerInsertRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PINSRB $-1, AX, X0", + "PINSRW $256, AX, X0", + "PINSRD $1, AX, Y0", + "PINSRQ.Z $1, AX, X0", + "PINSRB $1, AL, X0", + "PINSRD $1, X0, X1", + "PINSRQ $1, AX, X16", + "VPINSRB $1, AX, X0, Y1", + "VPINSRW $1, AX, Y0, X1", + "VPINSRD $1, X0, X1, X2", + "VPINSRQ $1, AX, X0, K1, X1", + "VPINSRQ.Z $1, AX, X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's PINSR/VPINSR tables", instruction) + } + }) + } + + for _, instruction := range []string{ + "PINSRQ $1, AX, X0", + "VPINSRB $1, AX, X0, X1", + "VPINSRQ $1, AX, X0, X1", + "PINSRD $1, R8, X0", + "PINSRD $1, AX, X8", + } { + t.Run("386_"+strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's 386 PINSR/VPINSR tables", instruction) + } + }) + } +} diff --git a/amd64_lower_insert_test.go b/amd64_lower_insert_test.go new file mode 100644 index 000000000..deac8e303 --- /dev/null +++ b/amd64_lower_insert_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedVectorInsertCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 defines three complete insert tables: + // _yvinsertf128: imm8, X/m, Y, Y + // _yvinsertf32x4: imm8, X/m, Y|Z, [K1-K7,] Y|Z + // _yvinsertf32x8: imm8, Y/m, Z, [K1-K7,] Z + // Exercise register and memory members of every source class, every + // destination width, and both merge and zero masking for all ten opcodes. + for _, target := range []struct { + name string + goarch string + triple string + zreg int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", zreg: 21}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", zreg: 21}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", zreg: 21}, + } { + t.Run(target.name, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedvectorinsertforms(SB),NOSPLIT,$0-0\n") + if target.goarch == "amd64" { + for _, op := range []string{"VINSERTF128", "VINSERTI128"} { + fmt.Fprintf(&src, "\t%s $0, X0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), Y3, Y4\n", op) + } + } + for _, op := range []string{"VINSERTF32X4", "VINSERTF64X2", "VINSERTI32X4", "VINSERTI64X2"} { + fmt.Fprintf(&src, "\t%s $0, X0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), Y3, Y4\n", op) + fmt.Fprintf(&src, "\t%s $1, X20, Y20, K1, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z $2, (AX), Y20, K2, Y21\n", op) + fmt.Fprintf(&src, "\t%s $0, X0, Z0, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s $255, (AX), Z0, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s $3, X20, Z0, K3, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s.Z $1, (AX), Z0, K4, Z%d\n", op, target.zreg) + } + for _, op := range []string{"VINSERTF32X8", "VINSERTF64X4", "VINSERTI32X8", "VINSERTI64X4"} { + fmt.Fprintf(&src, "\t%s $0, Y0, Z0, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s $255, (AX), Z0, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s $1, Y20, Z0, K5, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s.Z $1, (AX), Z0, K6, Z%d\n", op, target.zreg) + } + src.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, strings.Replace(src.String(), ",NOSPLIT", "", 1), true) + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedvectorinsertforms": {Name: "packedvectorinsertforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-vector-insert-"+target.name+".ll", "packed-vector-insert-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedVectorInsertRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VINSERTI128 $0, Y0, Y1, Y2", + "VINSERTF128 $0, X0, Z1, Z2", + "VINSERTI128 $256, X0, Y1, Y2", + "VINSERTF128 $-1, X0, Y1, Y2", + "VINSERTI128.Z $0, X0, Y1, Y2", + "VINSERTF128 $0, X0, Y1, K1, Y2", + "VINSERTI32X4 $0, Y0, Y1, Y2", + "VINSERTF64X2 $0, X0, X1, X2", + "VINSERTI64X2 $0, X0, Y1, Z2", + "VINSERTF32X4 $0, X0, Y1, K0, Y2", + "VINSERTI32X4.Z $0, X0, Y1, Y2", + "VINSERTF64X2.BCST $0, (AX), Z1, Z2", + "VINSERTI32X8 $0, X0, Z1, Z2", + "VINSERTF64X4 $0, Y0, Y1, Y2", + "VINSERTI64X4 $0, Y0, Z1, K0, Z2", + "VINSERTF32X8.Z $0, Y0, Z1, Z2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT bad(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", strings.Replace(src, ",NOSPLIT", "", 1), false) + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed insert tables", instruction) + } + }) + } + + for _, instruction := range []string{ + "VINSERTI128 $0, X0, Y0, Y1", + "VINSERTF128 $0, X0, Y0, Y1", + "VINSERTI128 $0, X16, Y0, Y1", + "VINSERTF128 $0, X0, Y16, Y1", + "VINSERTF32X4 $0, X0, Y0, Y1", + "VINSERTF64X2 $0, X0, Y0, Y1", + "VINSERTI32X4 $0, X0, Y0, Y1", + "VINSERTI64X2 $0, X0, Y0, Y1", + "VINSERTF32X8 $0, Y0, Z0, Z1", + "VINSERTF64X4 $0, Y0, Z0, Z1", + "VINSERTI32X8 $0, Y0, Z0, Z1", + "VINSERTI64X4 $0, Y0, Z0, Z1", + "VINSERTI32X4 $0, X0, Z8, Z0", + "VINSERTF64X2 $0, X0, Z0, K1, Z8", + "VINSERTI32X8 $0, Y0, Z8, Z0", + "VINSERTI64X4.Z $0, Y0, Z0, K1, Z8", + } { + src := "TEXT bad386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", strings.Replace(src, ",NOSPLIT", "", 1), false) + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad386": {Name: "bad386", Ret: Void}}, + }); err == nil { + t.Fatalf("386 Translate accepted architecture-restricted %q", instruction) + } + } +} diff --git a/amd64_lower_insertps.go b/amd64_lower_insertps.go new file mode 100644 index 000000000..a1e0a2c29 --- /dev/null +++ b/amd64_lower_insertps.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerInsertPackedSingle implements all forms in Go 1.27's yxshuf and +// _yvinsertps tables. The imm8 selects the register-source lane in bits 7:6, +// the destination lane in bits 5:4, and lanes to clear in bits 3:0. +func (c *amd64Ctx) lowerInsertPackedSingle(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if baseOp != "INSERTPS" && baseOp != "VINSERTPS" { + return false, false, nil + } + var bindingErr error + ins, _, _, bindingErr = c.bindInsertExtractForm(op, ins) + if bindingErr != nil { + return true, false, bindingErr + } + if rawOp != baseOp { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", baseOp, ins.Raw) + } + wantArgs := 3 + if baseOp == "VINSERTPS" { + wantArgs = 4 + } + if len(ins.Args) != wantArgs || ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("amd64 %s expects unsigned imm8, X/m32 source, %s: %q", baseOp, map[bool]string{false: "X destination", true: "X base, X destination"}[baseOp == "VINSERTPS"], ins.Raw) + } + + source := ins.Args[1] + base := ins.Args[2] + destination := ins.Args[2] + if baseOp == "VINSERTPS" { + destination = ins.Args[3] + } + if base.Kind != OpReg || destination.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects X base/destination registers: %q", baseOp, ins.Raw) + } + if baseOp == "INSERTPS" { + if !c.isGoLegacyXReg(base.Reg) { + return true, false, fmt.Errorf("%s INSERTPS destination is outside Go 1.27's X register class: %q", c.goarch, ins.Raw) + } + } else if !amd64EVEXVectorRegister(base, 16) || !amd64EVEXVectorRegister(destination, 16) { + return true, false, fmt.Errorf("%s VINSERTPS base/destination is outside Go 1.27's VEX/EVEX X register class: %q", c.goarch, ins.Raw) + } + + var inserted string + if source.Kind == OpReg { + valid := c.isGoLegacyXReg(source.Reg) + if baseOp == "VINSERTPS" { + valid = amd64EVEXVectorRegister(source, 16) + } + if !valid { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's X register class: %q", c.goarch, baseOp, ins.Raw) + } + sourceBytes, err := c.loadX(source.Reg) + if err != nil { + return true, false, err + } + sourceLanes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", sourceLanes, sourceBytes) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <4 x i32> %%%s, i32 %d\n", selected, sourceLanes, (ins.Args[0].Imm>>6)&3) + inserted = "%" + selected + } else if source.Kind == OpFP { + // An original m32 FP span can comprise four independent ABI0 + // shadow fields. It is not a contiguous load from the first alloca. + value, err := c.loadFPVectorBytes(source.FPOffset, 4) + if err != nil { + return true, false, err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i8> %s to i32\n", bits, value) + inserted = "%" + bits + } else { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("amd64 %s expects an X register or memory source: %q", baseOp, ins.Raw) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return true, false, err + } + bits := c.newTmp() + // INSERTPS always reads m32, even when imm8 subsequently clears it. + fmt.Fprintf(c.b, " %%%s = load volatile i32, %s %s, align 1\n", bits, pointerType, pointer) + inserted = "%" + bits + } + + baseBytes, err := c.loadX(base.Reg) + if err != nil { + return true, false, err + } + baseLanes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", baseLanes, baseBytes) + placed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> %%%s, i32 %s, i32 %d\n", placed, baseLanes, inserted, (ins.Args[0].Imm>>4)&3) + result := "%" + placed + for lane := 0; lane < 4; lane++ { + if ins.Args[0].Imm&(1< %s, i32 0, i32 %d\n", cleared, result, lane) + result = "%" + cleared + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %s to <16 x i8>\n", out, result) + if baseOp == "VINSERTPS" { + return true, false, c.storePackedMoveOperand(destination, 16, "%"+out) + } + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) +} diff --git a/amd64_lower_insertps_test.go b/amd64_lower_insertps_test.go new file mode 100644 index 000000000..56c11cc8a --- /dev/null +++ b/amd64_lower_insertps_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86InsertPSCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's yxshuf and _yvinsertps tables contain exactly: + // INSERTPS imm8, X/m32, X + // VINSERTPS imm8, X/m32, X, X + // VINSERTPS can select either its VEX (<16) or EVEX (<32) entry. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + src := fmt.Sprintf(`TEXT insertpsforms(SB),NOSPLIT,$0-0 + INSERTPS $0, X0, X1 + INSERTPS $255, 4(AX), X%d +`, legacyLast) + vex := ` + VINSERTPS $0, X0, X1, X2 + VINSERTPS $255, 8(AX), X14, X15 + VINSERTPS $16, X20, X21, X22 + VINSERTPS $32, 12(AX), X30, X31 +` + if target.goarch == "386" { + // Preserve the previous mixed positive verbatim as an actual + // Go rejection: the 386 frontend cannot take four operands. + goAssemblerOracleOutput(t, "386", strings.Replace(src+vex+"RET\n", "NOSPLIT", "4", 1), "", false) + original, err := Parse(ArchAMD64, src+vex+"RET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(original, Options{Goarch: "386", TargetTriple: target.triple, Sigs: map[string]FuncSig{"insertpsforms": {Ret: Void}}}); err == nil { + t.Fatal("original Go-rejected mixed 386 fixture translated") + } + } else { + src += vex + } + src += "RET\n" + goAssemblerOracleOutput(t, target.goarch, strings.Replace(src, "NOSPLIT", "4", 1), "", true) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "insertpsforms": {Name: "insertpsforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "insertps-"+target.name+".ll", "insertps-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86InsertPSRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "INSERTPS X0, X1", + "INSERTPS $-1, X0, X1", + "INSERTPS $256, X0, X1", + "INSERTPS $0, Y0, X1", + "INSERTPS $0, X0, Y1", + "INSERTPS.Z $0, X0, X1", + "VINSERTPS $0, X0, X1", + "VINSERTPS $0, X0, X1, X2, X3", + "VINSERTPS $0, Y0, X1, X2", + "VINSERTPS $0, X0, Y1, X2", + "VINSERTPS $0, X0, X1, Y2", + "VINSERTPS $0, X0, K1, X1", + "VINSERTPS.Z $0, X0, X1, X2", + "VINSERTPS.BCST $0, (AX), X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86InsertPSRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "INSERTPS $0, X8, X0", + "INSERTPS $0, X0, X8", + } { + assertX86InsertPSRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86InsertPSRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's INSERTPS tables", instruction) + } +} diff --git a/amd64_lower_integer_dot.go b/amd64_lower_integer_dot.go new file mode 100644 index 000000000..832bfa4ae --- /dev/null +++ b/amd64_lower_integer_dot.go @@ -0,0 +1,48 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerIntegerPackedDotProduct(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86PackedDotProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := c.validateConflictFunnelMemory(ins, form.x86PackedBinaryForm); err != nil { + return true, false, err + } + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + // E4 reads one exact volatile dword per enabled output lane; broadcast + // reads m32 iff an output lane is enabled. VEX retains its full tuple. + first, err := c.loadX86PackedShiftLanes(ins.Args[form.first], form.byteWidth, 32, form.broadcast, mask) + if err != nil { + return true, false, err + } + lanes, inputBits := form.byteWidth/4, form.spec.inputBits + inputLanes := form.byteWidth * 8 / inputBits + firstBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x i%d>\n", firstBits, lanes, first, inputLanes, inputBits) + second, err := c.loadPackedCompareBytes(ins.Args[form.second], form.byteWidth) + if err != nil { + return true, false, err + } + destination := ins.Args[form.destination] + oldBytes, err := c.loadPackedCompareBytes(destination, form.byteWidth) + if err != nil { + return true, false, err + } + other := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, inputLanes, inputBits, second) + old := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, 32, oldBytes) + computed := c.emitPackedDotProduct(form.spec, lanes, "%"+firstBits, other, old) + if form.mask >= 0 { + computed = amd64ApplyI32LaneMask(c, lanes, computed, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x i8>\n", out, lanes, computed, form.byteWidth) + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} diff --git a/amd64_lower_lddqu.go b/amd64_lower_lddqu.go new file mode 100644 index 000000000..17995dce2 --- /dev/null +++ b/amd64_lower_lddqu.go @@ -0,0 +1,30 @@ +package plan9asm + +import "fmt" + +type amd64UnalignedLoadSpec struct { + maxBytes int + vector bool +} + +// amd64UnalignedLoadSpecs is the complete Go 1.27 ylddqu/_yvlddqu grammar. +// Both instructions are load-only; legacy LDDQU is X-width, while VLDDQU +// adds the VEX Y-width row. +var amd64UnalignedLoadSpecs = map[Op]amd64UnalignedLoadSpec{ + "LDDQU": {maxBytes: 16}, + "VLDDQU": {maxBytes: 32, vector: true}, +} + +func (c *amd64Ctx) lowerUnalignedVectorLoad(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86UnalignedLoadProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || ins.Args[0].Kind == OpFP { + return true, false, fmt.Errorf("%w: LDDQU needs its original x86 native access, not logical FP storage", ErrProbeNeedsContext) + } + // Share only the native vector transport. No alignment assumption is + // introduced: the actual opcode retains its wider/repeated reads and #AC. + move := amd64PackedVectorMoveForm{byteWidth: form.byteWidth, memory: 0, destination: 1, mask: -1} + return true, false, c.lowerNativeAlignedVectorMemory(ins, move, !form.spec.vector) +} diff --git a/amd64_lower_lddqu_test.go b/amd64_lower_lddqu_test.go new file mode 100644 index 000000000..87ff20e61 --- /dev/null +++ b/amd64_lower_lddqu_test.go @@ -0,0 +1,171 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64UnalignedLoadGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64UnalignedLoadSpec{ + "LDDQU": {maxBytes: 16}, + "VLDDQU": {maxBytes: 32, vector: true}, + } + if len(amd64UnalignedLoadSpecs) != len(expected) { + t.Fatalf("unaligned-load grammar has %d entries, want %d", len(amd64UnalignedLoadSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64UnalignedLoadSpecs[op]; !ok { + t.Errorf("unaligned-load grammar omitted %s", op) + } else if got != want { + t.Errorf("unaligned-load grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86LDDQUCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + source := fmt.Sprintf(`TEXT lddquforms(SB),$0-0 + LDDQU 1(AX), X%d + VLDDQU 2(AX), X%d + VLDDQU 3(AX), Y%d +`, last, last, last) + if target.goarch == "386" { + source += "\tVLDDQU 4(R9), Y8\n" + } + source += "\tRET\n" + file, signatures := x86UnboundVoidFormatSource(t, source, "lddquforms", target.goarch, target.triple) + if target.goarch == "386" { + // Keep the complete original Go object and all three public + // Context controls above. Go accepts the high names, but they + // do not describe a physical mode32 R9/Y8 instruction. The + // initialized low-bank five-target objects are independent. + instruction := file.Funcs[0].Instrs[len(file.Funcs[0].Instrs)-2] + if _, known, err := x86UnalignedLoadProperties(instruction.Op, instruction, "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go's named high registers invented a mode32 physical instruction", err) + } + return + } + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0, + func(c *amd64Ctx, instruction Instr) (bool, bool, error) { + return c.lowerUnalignedVectorLoad(instruction.Op, instruction) + }) + for _, want := range []string{"asm sideeffect \"lddqu ", "asm sideeffect \"vlddqu "} { + if !strings.Contains(ir, want) { + t.Fatalf("LDDQU/VLDDQU lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "lddqu-"+target.name+".ll", "lddqu-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LDDQURejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LDDQU X0, X1"}, + {goarch: "amd64", instruction: "LDDQU 0(AX), Y1"}, + {goarch: "amd64", instruction: "VLDDQU X0, X1"}, + {goarch: "amd64", instruction: "VLDDQU 0(AX), Z1"}, + {goarch: "amd64", instruction: "VLDDQU.Z 0(AX), X1"}, + {goarch: "386", instruction: "LDDQU 0(AX), X8"}, + {goarch: "386", instruction: "LDDQU 0(R9), X0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: test.goarch, TargetTriple: triple, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside the Go table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64LDDQURuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT lddqusemantics(SB),$0-16 + MOVQ out+0(FP), AX + MOVQ in+8(FP), CX + STC + LDDQU 1(CX), X0 + MOVUPS X0, 0(AX) + SETCS 16(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"lddqusemantics": { + Name: "lddqusemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void lddqusemantics(uint8_t *, const uint8_t *); +int main(void) { + uint8_t in[17], out[17] = {0}; + for (int i = 0; i < 17; i++) in[i] = (uint8_t)(i * 7 + 3); + lddqusemantics(out, in); + for (int i = 0; i < 16; i++) if (out[i] != in[i + 1]) return 10 + i; + if (out[16] != 1) return 30; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "lddqu_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_lea_complete_test.go b/amd64_lower_lea_complete_test.go new file mode 100644 index 000000000..25d2f95e6 --- /dev/null +++ b/amd64_lower_lea_complete_test.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func x86LEACompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA leadata+0(SB)/8, $1\n") + source.WriteString("GLOBL leadata(SB), $8\n") + source.WriteString("TEXT leaforms(SB),$0-0\n") + widths := []string{"W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + op := "LEA" + width + source.WriteString("\t" + op + " 8(BX)(CX*2), AX\n") + source.WriteString("\t" + op + " leadata(SB), DX\n") + source.WriteString("\t" + op + " 16(BX), SP\n") + if goarch == "amd64" { + source.WriteString("\t" + op + " 24(R11)(R12*4), R13\n") + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86LEACompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86LEACompleteFormsSource(target.goarch) + requireLEANativeGoObject(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"leaforms": {Name: "leaforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"trunc i64", "to i16", "to i32"} { + if !strings.Contains(ir, want) { + t.Fatalf("LEA lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "lea-complete-"+target.name+".ll", "lea-complete-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LEARejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LEAW AX, BX"}, + {goarch: "amd64", instruction: "LEAL 8(AX), X0"}, + {goarch: "amd64", instruction: "LEAQ 8(AX)"}, + {goarch: "amd64", instruction: "LEAW.Z 8(AX), BX"}, + {goarch: "amd64", instruction: "LEAL $ret+0(FP), AX"}, + {goarch: "386", instruction: "LEAQ 8(AX), BX"}, + {goarch: "386", instruction: "LEAL $ret+0(FP), AX"}, + {goarch: "386", instruction: "LEAW 8(R9), AX"}, + {goarch: "386", instruction: "LEAL 8(AX), R8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireLEANativeGoObject(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ym_rl table", test.instruction) + } + }) + } +} + +func TestAMD64LEARuntimeWidthsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT leasementics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVQ $0x1122334455667788, AX + LEAW 0x1234(DI), AX + MOVQ AX, 0(DI) + LEAL 0x1234(DI), BX + MOVQ BX, 8(DI) + LEAQ 0x1234(DI), CX + MOVQ CX, 16(DI) + SETCS 24(DI) + SETOS 25(DI) + SETEQ 26(DI) + SETMI 27(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "leasementics": { + Name: "leasementics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void leasementics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[32] = {0}; + uintptr_t address = (uintptr_t)(out + 0x1234); + leasementics(out); + uint64_t want16 = UINT64_C(0x1122334455660000) | (address & UINT64_C(0xffff)); + if (load64(out+0) != want16) return 10; + if (load64(out+8) != (uint32_t)address) return 11; + if (load64(out+16) != address) return 12; + if (out[24] != 1 || out[25] != 1 || out[26] != 0 || out[27] != 1) return 13; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "lea_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_leave.go b/amd64_lower_leave.go new file mode 100644 index 000000000..95696d254 --- /dev/null +++ b/amd64_lower_leave.go @@ -0,0 +1,73 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64LeaveSpec struct { + bits int +} + +// amd64LeaveSpecs is the complete Go 1.27 LEAVE grammar. W is shared, +// L is 386-only, and Q is amd64-only. The lowering models the architectural +// sequence SP = BP; POP BP against the translated register/address state. +var amd64LeaveSpecs = map[Op]amd64LeaveSpec{ + "LEAVEW": {bits: 16}, + "LEAVEL": {bits: 32}, + "LEAVEQ": {bits: 64}, +} + +func (c *amd64Ctx) lowerLeave(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64LeaveSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.bits == 64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + if c.goarch != "386" && spec.bits == 32 { + return true, false, fmt.Errorf("amd64 %s is absent from Go 1.27's 64-bit encoding table: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + + base, err := c.loadReg(BP) + if err != nil { + return true, false, err + } + address := base + if c.goarch == "386" { + low := c.truncI64(base, I32) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, low) + address = "%" + wide + } + pointer := c.ptrFromAddrI64(address) + typ := amd64StackLLVMType(spec.bits) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", value, typ, pointer) + if err := c.storeRegSized(BP, typ, "%"+value); err != nil { + return true, false, err + } + if c.goarch == "386" { + address32 := c.truncI64(address, I32) + next32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i32 %s, %d\n", next32, address32, spec.bits/8) + next64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", next64, next32) + return true, false, c.storeRegUnchecked(SP, "%"+next64) + } + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %d\n", next, address, spec.bits/8) + return true, false, c.storeRegUnchecked(SP, "%"+next) +} diff --git a/amd64_lower_legacy_float_memory.go b/amd64_lower_legacy_float_memory.go new file mode 100644 index 000000000..806a4c4ab --- /dev/null +++ b/amd64_lower_legacy_float_memory.go @@ -0,0 +1,64 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func x86LegacyPackedFloatingNativeMemoryBits(ins Instr, goarch string) int { + parsed, known, err := x86FloatArithmeticProperties(ins, goarch, "") + form := parsed.x86FloatBinaryForm + if !known || err != nil || !form.legacy || form.scalar || form.memory < 0 || ins.Args[form.memory].Kind == OpFP { + return 0 + } + return form.bits +} + +// Legacy packed SSE arithmetic consumes one complete native m128 operand. +// An align-1 LLVM load followed by floating arithmetic can select a different +// load/instruction sequence, losing the original alignment and fault contract. +// Execute the exact validated opcode, including its source-order NaNs, MXCSR +// exceptions/rounding/DAZ/FTZ, and observable read even when the result is dead. +func (c *amd64Ctx) lowerLegacyPackedFloatingMemory(ins Instr, form x86FloatBinaryForm) error { + source := ins.Args[form.first] + if source.Kind == OpFP { + // ABI0 virtual fields are independently allocated. Even a logical + // contiguous field span is not original native stack alignment. + return fmt.Errorf("%w: %s m128 needs contiguous original FP storage and native alignment context, not virtual ABI0 fields", ErrProbeNeedsContext, ins.Op) + } + if ins.x86AddressBits != 0 { + return fmt.Errorf("%w: %s m128 lacks its original native address-size binding", ErrProbeNeedsContext, ins.Op) + } + if source.Kind == OpMem && !x86CallerMemoryCertified(ins, form.first) { + if !c.nativeMemoryOperand(ins, form.first) { + return fmt.Errorf("%w: %s m128 needs original native stack backing, not virtual SP/BP", ErrProbeNeedsContext, ins.Op) + } + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return err + } + left, err := c.loadX(ins.Args[form.second].Reg) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + carrierType := amd64FMA3LLVMType(128/form.bits, form.bits) + carrier, result, bytes := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", carrier, left, carrierType) + assembly := strings.ToLower(string(ins.Op)) + " " + segment + "($2), $0" + // The tied private XMM value is read before its output is written. No + // original GP, flags, upper SIMD bytes, or additional pointer permission + // is created. Segment-aware pointers retain the original FS/GS prefix. + // SSE1 admits a <4 x float> XMM carrier on physical i386; a byte-vector + // constraint there requires SSE2 and fails LLVM legalization. These are + // only bitcasts, not extra arithmetic or an x87 transport ABI. + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %%%s, %s %s)\n", result, carrierType, assembly, "=x,0,r,~{memory},~{fpsr}", carrierType, carrier, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytes, carrierType, result) + return c.storeLegacyPackedLogical(ins.Args[form.destination], "%"+bytes) +} diff --git a/amd64_lower_legacy_mmx.go b/amd64_lower_legacy_mmx.go new file mode 100644 index 000000000..46627c8e6 --- /dev/null +++ b/amd64_lower_legacy_mmx.go @@ -0,0 +1,146 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64LegacyMMXOperandClass uint8 + +const ( + amd64LegacyMMXMMX amd64LegacyMMXOperandClass = 1 << iota + amd64LegacyMMXXMM + amd64LegacyMMXMemory +) + +type amd64LegacyMMXMoveSpec struct { + source amd64LegacyMMXOperandClass + destination amd64LegacyMMXOperandClass + transferBits int + zeroExtend bool + nonTemporal bool +} + +// amd64LegacyMMXMoveSpecs describes the complete Go 1.27 legacy cross-domain +// and non-temporal MMX move grammar. Operand classes and transfer semantics are +// data; one parser validates the owned spellings. SIMD non-temporal moves +// have their complete separate x86NonTemporalVectorSpecs grammar. +var amd64LegacyMMXMoveSpecs = map[Op]amd64LegacyMMXMoveSpec{ + "MOVQOZX": {source: amd64LegacyMMXMMX | amd64LegacyMMXXMM | amd64LegacyMMXMemory, destination: amd64LegacyMMXXMM, transferBits: 64, zeroExtend: true}, + "MOVDQ2Q": {source: amd64LegacyMMXXMM, destination: amd64LegacyMMXMMX, transferBits: 64}, + "MOVNTQ": {source: amd64LegacyMMXMMX, destination: amd64LegacyMMXMemory, transferBits: 64, nonTemporal: true}, +} + +type amd64LegacyMMXMoveForm struct { + source Operand + destination Operand +} + +func (c *amd64Ctx) lowerLegacyMMXMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64LegacyMMXMoveSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if baseOp == "MOVDQ2Q" && (ins.x86Encoded || ins.x86SourceRaw != nil) { + return true, false, fmt.Errorf("%w: raw Intel MOVDQ2Q requires its physical MMX/x87 entry and caller-state contract: %q", ErrProbeNeedsContext, ins.Raw) + } + if baseOp == "MOVDQ2Q" { + return false, false, nil // Named Go source has all AMOVQ rows, not this Intel-only shape. + } + form, err := c.parseLegacyMMXMoveForm(baseOp, spec, ins) + if err != nil { + return true, false, err + } + return c.emitLegacyMMXMove(spec, form) +} + +func (c *amd64Ctx) parseLegacyMMXMoveForm(op string, spec amd64LegacyMMXMoveSpec, ins Instr) (amd64LegacyMMXMoveForm, error) { + var form amd64LegacyMMXMoveForm + if len(ins.Args) != 2 { + return form, fmt.Errorf("%s %s expects exactly two operands: %q", c.goarch, op, ins.Raw) + } + sourceClass := c.legacyMMXOperandClass(ins.Args[0]) + destinationClass := c.legacyMMXOperandClass(ins.Args[1]) + if sourceClass&spec.source == 0 || destinationClass&spec.destination == 0 { + return form, fmt.Errorf("%s %s operands are outside its Go 1.27 legacy MMX table: %q", c.goarch, op, ins.Raw) + } + for _, operand := range ins.Args { + if operand.Kind == OpMem && !x86MemoryRegistersValidForArch(operand.Mem, c.goarch) { + return form, fmt.Errorf("%s %s memory address uses an out-of-range register: %q", c.goarch, op, ins.Raw) + } + } + return amd64LegacyMMXMoveForm{source: ins.Args[0], destination: ins.Args[1]}, nil +} + +func (c *amd64Ctx) legacyMMXOperandClass(operand Operand) amd64LegacyMMXOperandClass { + if operand.Kind == OpReg { + if _, ok := amd64ParseMReg(operand.Reg); ok { + return amd64LegacyMMXMMX + } + if c.isGoLegacyXReg(operand.Reg) { + return amd64LegacyMMXXMM + } + return 0 + } + if isAMD64MemoryOperand(operand) { + return amd64LegacyMMXMemory + } + return 0 +} + +func (c *amd64Ctx) emitLegacyMMXMove(spec amd64LegacyMMXMoveSpec, form amd64LegacyMMXMoveForm) (bool, bool, error) { + if spec.destination == amd64LegacyMMXXMM { + bits, err := c.loadLegacyMMXMoveLow64(form.source) + if err != nil { + return true, false, err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> zeroinitializer, i64 %s, i32 0\n", words, bits) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bytes, words) + return true, false, c.storeX(form.destination.Reg, "%"+bytes) + } + + if spec.destination == amd64LegacyMMXMMX { + bits, err := c.loadLegacyMMXMoveLow64(form.source) + if err != nil { + return true, false, err + } + return true, false, c.storeReg(form.destination.Reg, bits) + } + + // MOVNTQ is the only remaining memory destination in this MMX directory. + bits, err := c.evalIntSized(form.source, I64) + if err != nil { + return true, false, err + } + return true, false, c.storeMOVNTIMemory(form.destination, I64, bits) +} + +func (c *amd64Ctx) loadLegacyMMXMoveLow64(source Operand) (string, error) { + if source.Kind == OpReg { + if _, ok := amd64ParseMReg(source.Reg); ok { + return c.loadReg(source.Reg) + } + if c.isGoLegacyXReg(source.Reg) { + value, err := c.loadX(source.Reg) + if err != nil { + return "", err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", words, value) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", low, words) + return "%" + low, nil + } + } + return c.evalIntSized(source, I64) +} diff --git a/amd64_lower_legacy_mmx_test.go b/amd64_lower_legacy_mmx_test.go new file mode 100644 index 000000000..cd38b7394 --- /dev/null +++ b/amd64_lower_legacy_mmx_test.go @@ -0,0 +1,201 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "strings" + "testing" +) + +func TestAMD64LegacyMMXMoveGrammarIsComplete(t *testing.T) { + want := map[Op]amd64LegacyMMXMoveSpec{ + "MOVQOZX": {source: amd64LegacyMMXMMX | amd64LegacyMMXXMM | amd64LegacyMMXMemory, destination: amd64LegacyMMXXMM, transferBits: 64, zeroExtend: true}, + "MOVDQ2Q": {source: amd64LegacyMMXXMM, destination: amd64LegacyMMXMMX, transferBits: 64}, + "MOVNTQ": {source: amd64LegacyMMXMMX, destination: amd64LegacyMMXMemory, transferBits: 64, nonTemporal: true}, + } + if !reflect.DeepEqual(amd64LegacyMMXMoveSpecs, want) { + t.Fatalf("legacy MMX move grammar = %+v, want %+v", amd64LegacyMMXMoveSpecs, want) + } +} + +func TestTranslateX86LegacyMMXCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + base := "BX" + xreg := "X7" + if target.goarch == "amd64" { + base = "R11" + xreg = "X15" + } + source := `TEXT legacymmxforms(SB),$0-0 + PSHUFW $-128, M1, M2 + PSHUFW $127, 8(BX), M3 + MOVQOZX M2, X2 + MOVQOZX X1, X3 + MOVQOZX 16(BX), X4 + MOVNTQ M2, 24(BX) +` + "\tMOVQOZX M3, " + xreg + "\n" + + "\tMOVNTQ M3, 32(" + base + ")\n\tRET\n" + ir := x86UnboundVoidScalarFormatObject(t, source, "legacymmxforms", target.goarch, target.triple) + for _, want := range []string{"!nontemporal !0", "!0 = !{i32 1}"} { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "legacy-mmx-"+target.name+".ll", "legacy-mmx-"+target.name+".o", ir) + }) + } +} + +func TestAMD64LegacyMMXRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT legacymmxsemantics(SB),NOSPLIT,$0-16 + MOVQ value+0(FP), AX + MOVQ out+8(FP), DI + MOVQ AX, M1 + PSHUFW $0x1b, M1, M2 + MOVNTQ M2, 0(DI) + MOVQOZX M2, X1 + MOVDQ2Q X1, M3 + MOVNTQ M3, 8(DI) + EMMS + MOVNTDQ X1, 16(DI) + SFENCE + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"legacymmxsemantics": { + Name: "legacymmxsemantics", Args: []LLVMType{I64, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void legacymmxsemantics(uint64_t, uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + __attribute__((aligned(16))) uint8_t out[32] = {0xff}; + legacymmxsemantics(UINT64_C(0x1122334455667788), out); + const uint64_t shuffled = UINT64_C(0x7788556633441122); + if (load64(out) != shuffled) return 10; + if (load64(out + 8) != shuffled) return 11; + if (load64(out + 16) != shuffled) return 12; + if (load64(out + 24) != 0) return 13; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy_mmx_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateX86LegacyMMXEncoderAliases(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + const source = `TEXT encoderaliases(SB),$0-0 + MOVDQ2Q X1, M1 + MOVNTDQ X2, 8(BX) + RET +` + // Go accepts these Intel decoder names in addition to the MOVQ and + // MOVNTO spellings used by its primary optab entries. + ir := x86UnboundVoidScalarFormatObject(t, source, "encoderaliases", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "legacy-mmx-alias-"+target.name+".ll", "legacy-mmx-alias-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LegacyMMXRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PSHUFW $-129, M1, M2"}, + {goarch: "amd64", instruction: "PSHUFW $128, M1, M2"}, + {goarch: "amd64", instruction: "PSHUFW $1, X1, M2"}, + {goarch: "amd64", instruction: "PSHUFW $1, M1, X2"}, + {goarch: "amd64", instruction: "PSHUFW.Z $1, M1, M2"}, + {goarch: "amd64", instruction: "MOVQOZX M1, M2"}, + {goarch: "amd64", instruction: "MOVQOZX $1, X2"}, + {goarch: "amd64", instruction: "MOVNTQ X1, 8(BX)"}, + {goarch: "amd64", instruction: "MOVNTQ M1, M2"}, + {goarch: "amd64", instruction: "MOVDQ2Q M1, X1"}, + {goarch: "amd64", instruction: "MOVNTDQ 8(BX), X1"}, + {goarch: "386", instruction: "MOVQOZX M1, X8"}, + {goarch: "386", instruction: "MOVNTQ M1, 8(R11)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside the legacy MMX grammar", test.instruction) + } + }) + } +} diff --git a/amd64_lower_legacy_packed_logical_test.go b/amd64_lower_legacy_packed_logical_test.go new file mode 100644 index 000000000..40f5b84ce --- /dev/null +++ b/amd64_lower_legacy_packed_logical_test.go @@ -0,0 +1,154 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86LegacyPackedLogicalCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastX := 15 + if target.goarch == "386" { + lastX = 7 + } + var source strings.Builder + source.WriteString("TEXT legacypackedlogicalforms(SB),$0-0\n") + for _, op := range []string{"PAND", "PANDN", "POR", "PXOR"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M7\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M7\n", op) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, lastX) + fmt.Fprintf(&source, "\t%s 16(AX), X%d\n", op, lastX) + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "legacypackedlogicalforms", target.goarch, target.triple) + wantIR := []string{"and <16 x i8>", "or <16 x i8>", "xor <16 x i8>"} + if target.goarch == "amd64" { + wantIR = append(wantIR, "and i64", "or i64", "xor i64") + } + for _, want := range wantIR { + if !strings.Contains(ir, want) { + t.Fatalf("legacy packed logical lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "legacy-packed-logical-"+target.name+".ll", "legacy-packed-logical-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LegacyPackedLogicalRejectsFormsOutsideGo127Table(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PAND M0, X1"}, + {goarch: "amd64", instruction: "POR X0, M1"}, + {goarch: "amd64", instruction: "PXOR Y0, Y1"}, + {goarch: "amd64", instruction: "PAND.Z X0, X1"}, + {goarch: "amd64", instruction: "PAND M0"}, + {goarch: "amd64", instruction: "POR M0, M1, M2"}, + {goarch: "386", instruction: "PXOR X0, X8"}, + {goarch: "386", instruction: "PAND 8(R9), X0"}, + {goarch: "386", instruction: "PAND M0, M1"}, + {goarch: "386", instruction: "PANDN M0, M1"}, + {goarch: "386", instruction: "POR M0, M1"}, + {goarch: "386", instruction: "PXOR M0, M1"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed logical table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64LegacyPackedLogicalMMXRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86LegacyPackedLogicalMMXRuntimeSource + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "legacypackedlogicalsemantics": { + Name: "legacypackedlogicalsemantics", Args: []LLVMType{Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void legacypackedlogicalsemantics(uint8_t *, uint64_t, uint64_t); +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v; } +int main(void) { + uint8_t out[33] = {0}; + uint64_t a = UINT64_C(0x0ff00ff0a55aa55a); + uint64_t b = UINT64_C(0x33cc55aa5aa5f00f); + legacypackedlogicalsemantics(out, a, b); + if (load64(out+0) != (a & b)) return 10; + if (load64(out+8) != ((~b) & a)) return 11; + if (load64(out+16) != (a | b)) return 12; + if (load64(out+24) != (a ^ b)) return 13; + if (out[32] != 1) return 14; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy_packed_logical_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_legacy_packed_move.go b/amd64_lower_legacy_packed_move.go new file mode 100644 index 000000000..a8eb59f75 --- /dev/null +++ b/amd64_lower_legacy_packed_move.go @@ -0,0 +1,108 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// The paired native emitter preserves the exact whole128 opcode, including +// alignment faults. Continuation defines the vector bytes but grants no +// initialized pointee, native store or virtual-frame permission. Callers must +// separately close the original source/physical-mode identity. +func x86LoweredLegacyPackedMoveRead(ins Instr, state *x86ResultState, owned []bool, + resultOffset int64, arch string) (bool, bool) { + form, known, err := x86LegacyPackedMoveProperties(ins.Op, ins, arch) + if !known || err != nil || form.memory != 0 || form.source.Kind != OpMem || + form.source.Mem.Sym != "" || !x86OrdinaryNativeMemoryOperand(form.source, arch) { + return false, false + } + return x86ResultIncomingPointerReadAddressIdentity(form.source, state, arch) && + x86ResultStore(form.destination, 16, state, owned, resultOffset, arch), true +} + +// Both floating and integer dispatch use the same whole-128-bit yxmov grammar. +// Native operands keep the original opcode: align-1 loads lose aligned faults, +// and ordinary LLVM loads disappear when the source result is dead. +func (c *amd64Ctx) lowerLegacyPackedMove(op Op, ins Instr) (bool, bool, error) { + known, err := c.validateLegacyPackedMove(op, ins) + if !known || err != nil { + return known, false, err + } + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, _, err := x86LegacyPackedMoveProperties(op, ins, arch) + if err != nil { + return true, false, err + } + if form.memory >= 0 { + memory := ins.Args[form.memory] + if memory.Kind == OpFP && form.aligned { + return true, false, fmt.Errorf("%w: %s FP operand needs its original native alignment", ErrProbeNeedsContext, op) + } + // Existing checked virtual FP/AUTO views are not native addresses. + // They keep their independent layout handling; this rule gives them + // neither native alignment nor permission to read adjacent storage. + virtual := !c.nativeMemoryOperand(ins, form.memory) + if memory.Kind == OpMem && (x86CallerMemoryCertified(ins, form.memory) || c.fixedLocalMemoryCertified(ins, form.memory)) { + virtual = false + } + if !virtual { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return true, false, fmt.Errorf("%w: legacy packed move needs its original native address size", ErrProbeNeedsContext) + } + return true, false, c.lowerLegacyPackedMoveNative(op, form) + } + } + value, err := c.loadXVecOperand(form.source) + if err != nil { + return true, false, err + } + if form.destination.Kind == OpReg { + return true, false, c.storeX(form.destination.Reg, value) + } + return true, false, c.storeXVecOperand(form.destination, value) +} + +func (c *amd64Ctx) lowerLegacyPackedMoveNative(op Op, form x86LegacyPackedMoveForm) error { + memory := form.source + if form.memoryWrite { + memory = form.destination + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(memory) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + mnemonic := strings.ToLower(string(op)) + switch strings.ToUpper(string(op)) { + case "MOVOU": + mnemonic = "movdqu" + case "MOVO", "MOVOA": + mnemonic = "movdqa" + } + if form.memoryWrite { + value, err := c.loadX(form.source.Reg) + if err != nil { + return err + } + carrier := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x float>\n", carrier, value) + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(<4 x float> %%%s, %s %s)\n", + mnemonic+" $0, "+segment+"($1)", "x,r,~{memory}", carrier, pointerType, pointer) + return nil + } + // A float carrier preserves SSE1's XMM class even on 386. It is a bit + // transport only, with no floating arithmetic or change to MXCSR/FLAGS. + loaded, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(%s %s)\n", + loaded, mnemonic+" "+segment+"($1), $0", "=x,r,~{memory}", pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <16 x i8>\n", bytes, loaded) + return c.storeX(form.destination.Reg, "%"+bytes) +} diff --git a/amd64_lower_lods_test.go b/amd64_lower_lods_test.go new file mode 100644 index 000000000..b145dd293 --- /dev/null +++ b/amd64_lower_lods_test.go @@ -0,0 +1,213 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86LODSCompleteFormsSource(goarch string) string { + widths := []string{"B", "W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + var source strings.Builder + source.WriteString("TEXT lodsforms(SB),$0-0\n") + for _, width := range widths { + op := "LODS" + width + fmt.Fprintf(&source, "\t%s\n", op) + fmt.Fprintf(&source, "\tCLD; REP; %s\n", op) + fmt.Fprintf(&source, "\tSTD; REPN; %s\n", op) + source.WriteString("\tCLD\n") + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86LODSCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86LODSCompleteFormsSource(target.goarch) + ir := x86UnboundVoidScalarFormatObject(t, source, "lodsforms", target.goarch, target.triple) + for _, width := range []string{"b", "w", "l"} { + if want := "@__plan9asm_rep_lods" + width; !strings.Contains(ir, want) { + t.Fatalf("LODS lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "@__plan9asm_rep_lodsq") { + t.Fatalf("LODSQ lowering omitted qword helper:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "lods-"+target.name+".ll", "lods-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LODSRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LODSB AX"}, + {goarch: "amd64", instruction: "LODSW AX, BX"}, + {goarch: "amd64", instruction: "LODSL.Z"}, + {goarch: "386", instruction: "LODSQ"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's LODS ynone table", test.instruction) + } + }) + } +} + +func TestAMD64LODSRuntimeSemanticsWidthsPrefixesAndDirection(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT lodssemantics(SB),NOSPLIT,$0-16 + MOVQ out+0(FP), R8 + MOVQ data+8(FP), R9 + CLD + + MOVQ R9, SI + MOVQ $0x1122334455667788, AX + LODSB + MOVQ AX, 0(R8) + MOVQ SI, 8(R8) + + MOVQ R9, SI + ADDQ $8, SI + MOVQ $0x1122334455667788, AX + LODSW + MOVQ AX, 16(R8) + + MOVQ R9, SI + ADDQ $16, SI + MOVQ $-1, AX + LODSL + MOVQ AX, 24(R8) + + MOVQ R9, SI + ADDQ $24, SI + LODSQ + MOVQ AX, 32(R8) + + MOVQ R9, SI + ADDQ $32, SI + MOVQ $3, CX + REP; LODSB + MOVQ AX, 40(R8) + MOVQ SI, 48(R8) + MOVQ CX, 56(R8) + + MOVQ R9, SI + ADDQ $40, SI + MOVQ $3, CX + REPN; LODSW + MOVQ AX, 64(R8) + MOVQ SI, 72(R8) + MOVQ CX, 80(R8) + + MOVQ R9, SI + ADDQ $52, SI + MOVQ $2, CX + STD + REP; LODSL + CLD + MOVQ AX, 88(R8) + MOVQ SI, 96(R8) + MOVQ CX, 104(R8) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "lodssemantics": { + Name: "lodssemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void lodssemantics(uint8_t *, uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t data[64] = {0}; + data[0] = 0xaa; + const uint16_t w = UINT16_C(0xbbcc); memcpy(data+8, &w, sizeof(w)); + const uint32_t l = UINT32_C(0x89abcdef); memcpy(data+16, &l, sizeof(l)); + const uint64_t q = UINT64_C(0x0123456789abcdef); memcpy(data+24, &q, sizeof(q)); + data[32] = 1; data[33] = 2; data[34] = 3; + const uint16_t ws[3] = {0x1111,0x2222,0x3333}; memcpy(data+40, ws, sizeof(ws)); + const uint32_t ls[2] = {0x44444444,0x55555555}; memcpy(data+48, ls, sizeof(ls)); + uint8_t out[112] = {0}; + lodssemantics(out, data); + if (load64(out+0) != UINT64_C(0x11223344556677aa) || load64(out+8) != (uintptr_t)(data+1)) return 10; + if (load64(out+16) != UINT64_C(0x112233445566bbcc)) return 11; + if (load64(out+24) != UINT64_C(0x89abcdef)) return 12; + if (load64(out+32) != q) return 13; + if ((load64(out+40) & 0xff) != 3 || load64(out+48) != (uintptr_t)(data+35) || load64(out+56) != 0) return 14; + if ((load64(out+64) & 0xffff) != 0x3333 || load64(out+72) != (uintptr_t)(data+46) || load64(out+80) != 0) return 15; + if (load64(out+88) != UINT64_C(0x44444444) || load64(out+96) != (uintptr_t)(data+44) || load64(out+104) != 0) return 16; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "lods_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_logical_family.go b/amd64_lower_logical_family.go new file mode 100644 index 000000000..96cb8ed5e --- /dev/null +++ b/amd64_lower_logical_family.go @@ -0,0 +1,152 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPackedLogicalFamily(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := x86PackedLogicalSpecs[Op(base)]; !recognized { + return false, false, nil + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed logical dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := x86PackedLogicalProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return true, false, fmt.Errorf("%w: legacy packed logical m128 lacks original native address-size binding", ErrProbeNeedsContext) + } + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed logical m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.mmx { + a, err := c.evalIntSized(first, I64) + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + if form.spec.mode == amd64PackedLogicalAndNot { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, -1\n", inverted, b) + b = "%" + inverted + } + operation := "and" + if form.spec.mode == amd64PackedLogicalOr { + operation = "or" + } + if form.spec.mode == amd64PackedLogicalXor { + operation = "xor" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i64 %s, %s\n", result, operation, b, a) + return true, false, c.storeReg(destination.Reg, "%"+result) + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + lanes := form.byteWidth * 8 / form.spec.laneBits + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, form.spec.laneBits, false) + if err != nil { + return true, false, err + } + var result string + if form.legacy && form.memory >= 0 { + // Retain the single original memory opcode: an align-1 load and + // portable logical arithmetic lose its alignment/dead-read faults. + result, err = c.emitPackedBinaryNativeMemory(first, form.spec.laneBits, b, strings.ToLower(string(op))) + } else { + // Native reads remain observable with dead results. EVEX inactive + // lanes alone suppress their accesses, including scalar broadcasts. + var a string + if ins.x86NumericalRead != nil || c.goReadOnlyOperand.allows(first, int64(form.memoryBytes)) || + first.Kind == OpMem && c.nativeMemoryOperand(ins, form.first) { + a, err = c.loadX86PackedShiftLanes(first, form.byteWidth, form.spec.laneBits, form.broadcast, mask) + } else { + a, err = c.loadMaskedPackedCompareLanes(first, form.byteWidth, form.spec.laneBits, form.broadcast, mask) + } + if err == nil { + result = c.emitPackedLogical(lanes, amd64PackedLogicalSpec{form.spec.laneBits, form.spec.mode}, a, b) + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + old, err := c.loadPackedCompareLanes(destination, form.byteWidth, form.spec.laneBits, false) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.spec.laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.spec.laneBits, result, form.byteWidth) + if form.legacy { + // Legacy writes update low X bits in every wider view but preserve the + // upper bytes. storeX alone is a separate shadow slot, not an alias. + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) storeLegacyPackedLogical(dst Operand, value string) error { + index, _ := amd64ParseXReg(dst.Reg) + for _, view := range []struct { + width int + slots map[int]string + }{{32, c.yRegSlot}, {64, c.zRegSlot}} { + slot, live := view.slots[index] + if !live { + continue + } + old, merged := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = load <%d x i8>, ptr %s\n", old, view.width, slot) + extended := c.newTmp() + extendIndices := make([]string, view.width) + for i := range extendIndices { + lane := i + if i >= 16 { + lane = 16 + } + extendIndices[i] = fmt.Sprintf("i32 %d", lane) + } + fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, <%d x i32> <%s>\n", extended, value, view.width, strings.Join(extendIndices, ", ")) + indices := make([]string, view.width) + for i := range indices { + lane := i + if i < 16 { + lane += view.width + } + indices[i] = fmt.Sprintf("i32 %d", lane) + } + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %%%s, <%d x i8> %%%s, <%d x i32> <%s>\n", merged, view.width, old, view.width, extended, view.width, strings.Join(indices, ", ")) + fmt.Fprintf(c.b, " store <%d x i8> %%%s, ptr %s\n", view.width, merged, slot) + } + return c.storeX(dst.Reg, value) +} diff --git a/amd64_lower_logical_shift.go b/amd64_lower_logical_shift.go new file mode 100644 index 000000000..88bf80f1e --- /dev/null +++ b/amd64_lower_logical_shift.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64PackedLogicalShiftSpec struct { + laneBits int + left bool + vector bool +} + +// amd64PackedLogicalShiftSpecs covers all Go 1.27 uniform packed logical +// shifts. Legacy operations use yps (MMX/XMM, immediate or packed count), and +// V-prefixed operations use _yvpslld (X/Y/Z data and an immediate or X/m128 +// count, with the table's EVEX mask and D/Q broadcast forms). +var amd64PackedLogicalShiftSpecs = map[Op]amd64PackedLogicalShiftSpec{ + "PSLLW": {laneBits: 16, left: true}, + "PSLLL": {laneBits: 32, left: true}, + "PSLLQ": {laneBits: 64, left: true}, + "PSRLW": {laneBits: 16}, + "PSRLL": {laneBits: 32}, + "PSRLQ": {laneBits: 64}, + "VPSLLW": {laneBits: 16, left: true, vector: true}, + "VPSLLD": {laneBits: 32, left: true, vector: true}, + "VPSLLQ": {laneBits: 64, left: true, vector: true}, + "VPSRLW": {laneBits: 16, vector: true}, + "VPSRLD": {laneBits: 32, vector: true}, + "VPSRLQ": {laneBits: 64, vector: true}, +} + +func (c *amd64Ctx) lowerPackedLogicalShift(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerX86PackedShift(op, ins) +} + +func (c *amd64Ctx) loadPackedUniformLogicalShiftCount(count Operand, mmx bool) (string, error) { + if count.Kind == OpImm { + return fmt.Sprintf("%d", uint8(count.Imm)), nil + } + if mmx { + if count.Kind == OpMem || count.Kind == OpSym { + bytes, err := c.loadX86PackedShiftBytes(count, 8, true) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i8> %s to i64\n", value, bytes) + return "%" + value, nil + } + return c.loadLegacyMMXPackedSource(count) + } + // The ignored upper count qword still belongs to the complete native + // m128 read, even when a zero mask or saturated result is optimized. + countBytes, err := c.loadX86PackedShiftBytes(count, 16, true) + if err != nil { + return "", err + } + countWords := c.newTmp() + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", countWords, countBytes) + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", low, countWords) + return "%" + low, nil +} + +func (c *amd64Ctx) emitPackedLogicalShift(spec amd64PackedLogicalShiftSpec, lanes int, data, count64 string) string { + inRange := c.newTmp() + safe64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, %d\n", inRange, count64, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %s, i64 0\n", safe64, inRange, count64) + count := "%" + safe64 + if spec.laneBits < 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", narrow, count, spec.laneBits) + count = "%" + narrow + } + counts := amd64SplatInteger(c, lanes, spec.laneBits, count) + shifted := c.newTmp() + operation := "lshr" + if spec.left { + operation = "shl" + } + typeName := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", shifted, operation, typeName, data, counts) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s zeroinitializer\n", result, inRange, typeName, shifted, typeName) + return "%" + result +} diff --git a/amd64_lower_logical_shift_test.go b/amd64_lower_logical_shift_test.go new file mode 100644 index 000000000..88ae19137 --- /dev/null +++ b/amd64_lower_logical_shift_test.go @@ -0,0 +1,146 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedLogicalShiftCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, zLast, dataReg, dstReg := 15, 22, 20, 21 + if target.goarch == "386" { + legacyLast, zLast, dataReg, dstReg = 7, 7, 6, 7 + } + var source strings.Builder + source.WriteString("TEXT packedlogicalshiftforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSLLW", "PSLLL", "PSLLQ", "PSRLW", "PSRLL", "PSRLQ"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $-1, M1\n", op) + fmt.Fprintf(&source, "\t%s M0, M1\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M1\n", op) + } + fmt.Fprintf(&source, "\t%s $3, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + } + for _, op := range []string{"VPSLLW", "VPSLLD", "VPSLLQ", "VPSRLW", "VPSRLD", "VPSRLQ"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s $-1, %s0, %s1\n", op, width, width) + fmt.Fprintf(&source, "\t%s $255, 8(AX), %s%d\n", op, width, dstReg) + fmt.Fprintf(&source, "\t%s X0, %s%d, %s%d\n", op, width, dataReg, width, dstReg) + fmt.Fprintf(&source, "\t%s 8(AX), %s%d, %s%d\n", op, width, dataReg, width, dstReg) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, %s20, K1, %s22\n", op, width, width) + fmt.Fprintf(&source, "\t%s.Z X0, %s20, K7, %s22\n", op, width, width) + } + } + fmt.Fprintf(&source, "\t%s $3, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s $3, 8(AX), Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s X0, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, Z20, K2, Z22\n", op) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z20, K6, Z22\n", op) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), X%d\n", op, dstReg) + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), Y%d\n", op, dstReg) + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z $3, 8(AX), K5, Z22\n", op) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, strings.ReplaceAll(source.String(), "NOSPLIT", "4"), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func packedlogicalshiftforms()", target.goarch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original packed-shift formats invented native/vector/mask inputs", api, err) + } + } + ll := x86ScalarFormatObjectUnit(t, file, map[string]FuncSig{ + "packedlogicalshiftforms": {Name: "packedlogicalshiftforms", Ret: Void}, + }, target.goarch, target.triple, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, ir := range []string{ll, x86DwordProductIROptimize(t, ll)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + } + }) + } +} + +func TestTranslateX86PackedLogicalShiftRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSLLW.Z $3, X0", + "PSLLL $3", + "PSLLQ $-129, X0", + "PSRLW $128, X0", + "PSRLL M0, X1", + "PSRLQ X0, M1", + "VPSLLW $-129, X0, X1", + "VPSLLD $256, X0, X1", + "VPSLLQ Y0, X1, X2", + "VPSRLW X0, 8(AX), X2", + "VPSRLD Y0, X1, X2", + "VPSRLQ.BCST X0, X1, X2", + "VPSLLW.BCST $3, 8(AX), X2", + "VPSLLD.BCST $3, X1, X2", + "VPSRLQ.Z $3, X1, X2", + "VPSLLQ $3, X1, K0, X2", + "VPSRLD $3, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedLogicalShiftRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PSLLQ X8, X0", + "PSRLQ X0, X8", + "VPSLLQ $3, Z8, Z1", + "VPSRLQ X0, Z1, Z8", + "VPSLLD $3, X1, K1, X2", + "VPSRLD.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedLogicalShiftRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedLogicalShiftRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed logical-shift forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_low_multiply.go b/amd64_lower_low_multiply.go new file mode 100644 index 000000000..664c71fb3 --- /dev/null +++ b/amd64_lower_low_multiply.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPackedLowMultiply(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86PackedLowMultiplyProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + first := ins.Args[form.first] + if form.legacy && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed low multiply m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + bits := form.spec.laneBits + err = c.emitPackedBinaryE4(ins, form.x86PackedBinaryForm, bits, "pmulld", + func(lanes, bits int, first, second string) string { + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul <%d x i%d> %s, %s\n", product, lanes, bits, second, first) + return "%" + product + }) + return true, false, err +} + +func isAMD64MemoryOperand(op Operand) bool { + switch op.Kind { + case OpFP, OpMem: + return true + case OpSym: + // Go TYPE_ADDR is an address, not a TYPE_MEM read. + return !strings.HasPrefix(strings.TrimSpace(op.Sym), "$") + default: + return false + } +} diff --git a/amd64_lower_lzcnt_test.go b/amd64_lower_lzcnt_test.go new file mode 100644 index 000000000..7632cfda3 --- /dev/null +++ b/amd64_lower_lzcnt_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86LZCNTCompleteFormsSource(goarch string) string { + var source strings.Builder + symbol := "lzcntdata" + size := 8 + if goarch == "386" { + symbol = "lzcntdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $1\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT lzcntforms(SB),$0-0\n") + widths := []string{"W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + fmt.Fprintf(&source, "\tLZCNT%s AX, BX\n", width) + fmt.Fprintf(&source, "\tLZCNT%s 8(BX), CX\n", width) + fmt.Fprintf(&source, "\tLZCNT%s %s(SB), DX\n", width, symbol) + if goarch == "amd64" { + fmt.Fprintf(&source, "\tLZCNT%s R11, R12\n", width) + } else { + fmt.Fprintf(&source, "\tLZCNT%s AX, SP\n", width) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86LZCNTCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86LZCNTCompleteFormsSource(target.goarch) + ll := x86ScalarCountUnboundFormatObject(t, source, "lzcntforms", target.goarch, target.triple) + for _, want := range []string{"@llvm.ctlz.i16", "@llvm.ctlz.i32", `"target-features"="+lzcnt"`} { + if !strings.Contains(ll, want) { + t.Fatalf("LZCNT lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "@llvm.ctlz.i64") { + t.Fatalf("LZCNTQ lowering omitted llvm.ctlz.i64:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "lzcnt-"+target.name+".ll", "lzcnt-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86LZCNTRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LZCNTW AL, BX"}, + {goarch: "amd64", instruction: "LZCNTL AX, AH"}, + {goarch: "amd64", instruction: "LZCNTQ X0, AX"}, + {goarch: "amd64", instruction: "LZCNTW AX, 0(BX)"}, + {goarch: "amd64", instruction: "LZCNTL $1, AX"}, + {goarch: "amd64", instruction: "LZCNTQ AX"}, + {goarch: "amd64", instruction: "LZCNTW.Z AX, BX"}, + {goarch: "386", instruction: "LZCNTL R8, AX"}, + {goarch: "386", instruction: "LZCNTQ AX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's LZCNT yml_rl table", test.instruction) + } + }) + } +} + +// Keep the complete original failing runtime program byte-for-byte, including +// the read after a native store and all three widths' CF/ZF observations. +const x86LZCNTOriginalRuntimeSource = ` +TEXT lzcntsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0x1122334455660000, AX + MOVQ $0xaabbccddeeff1234, BX + LZCNTW AX, BX + MOVQ BX, 0(DI) + SETCS 8(DI) + SETEQ 9(DI) + MOVQ $0x80000000, AX + MOVQ $0x1122334455667788, CX + LZCNTL AX, CX + MOVQ CX, 16(DI) + SETCS 24(DI) + SETEQ 25(DI) + MOVQ $1, 32(DI) + LZCNTQ 32(DI), DX + MOVQ DX, 40(DI) + SETCS 48(DI) + SETEQ 49(DI) + RET +` + +func TestAMD64LZCNTRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86LZCNTOriginalRuntimeSource + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "lzcntsemantics": { + Name: "lzcntsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void lzcntsemantics(uint8_t *); +int main(void) { + uint8_t out[56] = {0}; + lzcntsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0xaabbccddeeff0010)) return 10; + if (out[8] != 1 || out[9] != 0) return 11; + if (*(uint64_t *)(out+16) != 0) return 12; + if (out[24] != 0 || out[25] != 1) return 13; + if (*(uint64_t *)(out+40) != 63) return 14; + if (out[48] != 0 || out[49] != 0) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "lzcnt_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_lzcnt_vector_test.go b/amd64_lower_lzcnt_vector_test.go new file mode 100644 index 000000000..b92f93cc1 --- /dev/null +++ b/amd64_lower_lzcnt_vector_test.go @@ -0,0 +1,222 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedBitCountGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64PackedBitCountSpec{ + "VPOPCNTB": {laneBits: 8, kind: amd64PackedPopulationCount}, + "VPOPCNTW": {laneBits: 16, kind: amd64PackedPopulationCount}, + "VPOPCNTD": {laneBits: 32, kind: amd64PackedPopulationCount, broadcast: true}, + "VPOPCNTQ": {laneBits: 64, kind: amd64PackedPopulationCount, broadcast: true}, + "VPLZCNTD": {laneBits: 32, kind: amd64PackedLeadingZeroCount, broadcast: true}, + "VPLZCNTQ": {laneBits: 64, kind: amd64PackedLeadingZeroCount, broadcast: true}, + } + if len(amd64PackedBitCountSpecs) != len(expected) { + t.Fatalf("packed-bit-count grammar has %d entries, want %d", len(amd64PackedBitCountSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64PackedBitCountSpecs[op]; !ok { + t.Errorf("packed-bit-count grammar omitted %s", op) + } else if got != want { + t.Errorf("packed-bit-count grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86PackedLeadingZeroCountCompleteGoFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + base := "R9" + if target.goarch == "386" { + base = "CX" + } + source.WriteString("TEXT packedlzcntforms(SB),$0-0\n") + for _, op := range []string{"VPLZCNTD", "VPLZCNTQ"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s0, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s 8(%s), %s%d\n", op, base, width, last) + fmt.Fprintf(&source, "\t%s %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 16(%s), K7, %s%d\n", op, base, width, last) + fmt.Fprintf(&source, "\t%s.BCST 24(%s), %s%d\n", op, base, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(%s), K2, %s%d\n", op, base, width, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"packedlzcntforms": {Name: "packedlzcntforms", Ret: Void}}, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original leading-zero format invented initialized inputs", err) + } + ir = x86ZeroArgumentLowererObjectUnit(t, file, target.goarch, target.triple, "LZCountEmitterOnly", + func(consumer *amd64Ctx, instruction Instr) (bool, bool, error) { + return consumer.lowerPackedBitCount(instruction.Op, instruction) + }) + for _, want := range []string{ + "@llvm.ctlz.v4i32", "@llvm.ctlz.v8i64", "select <16 x i1>", "select <8 x i1>", + } { + if !strings.Contains(ir, want) { + t.Fatalf("packed-LZCNT lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "packed-lzcnt-"+target.name+".ll", "packed-lzcnt-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedLeadingZeroCountRejectsFormsOutsideGoTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPLZCNTD X0, Y1"}, + {goarch: "amd64", instruction: "VPLZCNTQ Y0, Z1"}, + {goarch: "amd64", instruction: "VPLZCNTD X0, K0, X1"}, + {goarch: "amd64", instruction: "VPLZCNTQ.Z X0, X1"}, + {goarch: "amd64", instruction: "VPLZCNTD.BCST X0, X1"}, + {goarch: "amd64", instruction: "VPLZCNTQ.SAE X0, X1"}, + {goarch: "amd64", instruction: "VPLZCNTD X0, 0(AX)"}, + {goarch: "amd64", instruction: "VPLZCNTQ AX, X1"}, + {goarch: "amd64", instruction: "VPLZCNTD X0, K1, K2, X1"}, + {goarch: "386", instruction: "VPLZCNTD Z8, Z0"}, + {goarch: "386", instruction: "VPLZCNTQ Z0, K1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPLZCNT _yvexpandpd grammar", test.instruction) + } + }) + } +} + +func TestAMD64PackedLeadingZeroCountRuntimeSemantics(t *testing.T) { + // Portable LLVM execution on the actual host, not original-Go/native AVX512. + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT packedlzcntsemantics(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VPLZCNTD Z0, Z2 + VMOVDQU64 Z2, 0(AX) + VPLZCNTQ 0(BX), Z2 + VMOVDQU64 Z2, 64(AX) + VPLZCNTD Z0, K1, Z1 + VMOVDQU64 Z1, 128(AX) + VMOVDQU64 0(CX), Z1 + VPLZCNTQ.Z 0(BX), K1, Z1 + VMOVDQU64 Z1, 192(AX) + VPLZCNTD.BCST 0(BX), Z2 + VMOVDQU64 Z2, 256(AX) + VPLZCNTQ.BCST.Z 0(BX), K1, Z1 + VMOVDQU64 Z1, 320(AX) + STC + VPLZCNTD Z0, Z2 + SETCS 384(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"packedlzcntsemantics": { + Name: "packedlzcntsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void packedlzcntsemantics(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static unsigned lz32(uint32_t v) { unsigned n = 0; if (!v) return 32; while (!(v & UINT32_C(0x80000000))) { n++; v <<= 1; } return n; } +static unsigned lz64(uint64_t v) { unsigned n = 0; if (!v) return 64; while (!(v & UINT64_C(0x8000000000000000))) { n++; v <<= 1; } return n; } +int main(void) { + uint32_t source32[16], old32[16]; uint8_t out[385] = {0}; + const uint64_t mask = UINT64_C(0xa55a); + for (int i = 0; i < 16; i++) { source32[i] = i ? (UINT32_C(1) << ((i*7)%32)) : 0; old32[i] = UINT32_C(0x90000000) + i; } + packedlzcntsemantics(out, (const uint8_t *)source32, (const uint8_t *)old32, mask); + uint32_t *plain32=(uint32_t *)(out+0), *merge32=(uint32_t *)(out+128), *broadcast32=(uint32_t *)(out+256); + for (int i=0;i<16;i++) { + if (plain32[i] != lz32(source32[i])) return 10+i; + uint32_t want = (mask>>i)&1 ? lz32(source32[i]) : old32[i]; + if (merge32[i] != want) return 30+i; + if (broadcast32[i] != lz32(source32[0])) return 50+i; + } + uint64_t source64[8], old64[8], plain64[8], zero64[8], broadcast64[8]; + memcpy(source64, source32, 64); memcpy(old64, old32, 64); + memcpy(plain64,out+64,64); memcpy(zero64,out+192,64); memcpy(broadcast64,out+320,64); + for (int i=0;i<8;i++) { + if (plain64[i] != lz64(source64[i])) return 70+i; + uint64_t want = (mask>>i)&1 ? lz64(source64[i]) : 0; + if (zero64[i] != want) return 80+i; + want = (mask>>i)&1 ? lz64(source64[0]) : 0; + if (broadcast64[i] != want) return 90+i; + } + if (out[384] != 1) return 100; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_lzcnt_semantics", triple, ir, mainC, nil) +} diff --git a/amd64_lower_machine_register.go b/amd64_lower_machine_register.go new file mode 100644 index 000000000..8fef3a577 --- /dev/null +++ b/amd64_lower_machine_register.go @@ -0,0 +1,224 @@ +package plan9asm + +import ( + "fmt" + "strconv" + "strings" +) + +// lowerMachineRegisterMove covers the complete machine-state-register portion +// of Go 1.27's x86 ymovtab: CR, DR, and TR registers plus the descriptor-state +// aliases GDTR, IDTR, LDTR, MSW, and TASK. Segment and TLS moves are a separate +// family. +func (c *amd64Ctx) lowerMachineRegisterMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if baseOp != "MOVW" && baseOp != "MOVL" && baseOp != "MOVQ" { + return false, false, nil + } + if len(ins.Args) != 2 { + return false, false, nil + } + sourceClass, sourceIndex, sourceSpecial := x86MachineRegisterOperand(ins.Args[0]) + destinationClass, destinationIndex, destinationSpecial := x86MachineRegisterOperand(ins.Args[1]) + if !sourceSpecial && !destinationSpecial { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s machine-register %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if sourceSpecial == destinationSpecial { + return true, false, fmt.Errorf("%s %s machine-register form expects exactly one special register: %q", c.goarch, baseOp, ins.Raw) + } + + specialClass, specialIndex := sourceClass, sourceIndex + other := ins.Args[1] + readSpecial := sourceSpecial + if destinationSpecial { + specialClass, specialIndex = destinationClass, destinationIndex + other = ins.Args[0] + } + + switch specialClass { + case "cr": + return true, false, c.lowerControlDebugRegisterMove(baseOp, "cr", specialIndex, other, readSpecial, ins) + case "dr": + return true, false, c.lowerControlDebugRegisterMove(baseOp, "dr", specialIndex, other, readSpecial, ins) + case "tr": + return true, false, c.lowerTestRegisterMove(baseOp, specialIndex, other, readSpecial, ins) + case "gdtr", "idtr", "ldtr", "msw", "task": + return true, false, c.lowerDescriptorRegisterAlias(baseOp, specialClass, other, readSpecial, ins) + default: + panic("machine-register classifier drift") + } +} + +func x86MachineRegisterOperand(operand Operand) (class string, index int, ok bool) { + if operand.Kind != OpReg { + return "", 0, false + } + return x86MachineRegister(operand.Reg) +} + +func x86MachineRegister(reg Reg) (class string, index int, ok bool) { + name := strings.ToUpper(string(reg)) + switch name { + case "GDTR", "IDTR", "LDTR", "MSW", "TASK": + return strings.ToLower(name), 0, true + } + if len(name) < 3 { + return "", 0, false + } + prefix := strings.ToLower(name[:2]) + if prefix != "cr" && prefix != "dr" && prefix != "tr" { + return "", 0, false + } + index, err := strconv.Atoi(name[2:]) + if err != nil { + return "", 0, false + } + return prefix, index, true +} + +func (c *amd64Ctx) lowerControlDebugRegisterMove(op, class string, index int, other Operand, readSpecial bool, ins Instr) error { + if op != "MOVL" && op != "MOVQ" { + return fmt.Errorf("%s %s %s register form is absent from Go 1.27's ymovtab: %q", c.goarch, op, strings.ToUpper(class), ins.Raw) + } + allowed := false + switch class { + case "cr": + allowed = index == 0 || index == 2 || index == 3 || index == 4 || index == 8 + if index == 8 && c.goarch == "386" { + allowed = false + } + case "dr": + if op == "MOVL" { + allowed = index == 0 || index == 6 || index == 7 + } else { + allowed = index == 0 || index == 2 || index == 3 || index == 6 || index == 7 + } + } + if !allowed { + return fmt.Errorf("%s %s %s%d form is absent from Go 1.27's ymovtab: %q", c.goarch, op, strings.ToUpper(class), index, ins.Raw) + } + if other.Kind != OpReg || !isX86YrlRegisterForArch(other.Reg, c.goarch) { + return fmt.Errorf("%s %s %s%d expects a Go 1.27 Yrl GP register: %q", c.goarch, op, strings.ToUpper(class), index, ins.Raw) + } + + special := fmt.Sprintf("%%%s%d", class, index) + hardwareType := I64 + if c.goarch == "386" { + hardwareType = I32 + } + constraints := "~{memory},~{dirflag},~{fpsr},~{flags}" + if readSpecial { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", + call, hardwareType, "mov "+special+", $0", "=r,"+constraints) + value := "%" + call + resultType := hardwareType + if op == "MOVL" && hardwareType == I64 { + value = c.truncI64(value, I32) + resultType = I32 + } + return c.storeRegSized(other.Reg, resultType, value) + } + + value, err := c.loadReg(other.Reg) + if err != nil { + return err + } + if hardwareType == I32 { + value = c.truncI64(value, I32) + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s)\n", + "mov $0, "+special, "r,"+constraints, hardwareType, value) + return nil +} + +func (c *amd64Ctx) lowerTestRegisterMove(op string, index int, other Operand, readSpecial bool, ins Instr) error { + if op != "MOVL" || (index != 6 && index != 7) { + return fmt.Errorf("%s %s TR%d form is absent from Go 1.27's ymovtab: %q", c.goarch, op, index, ins.Raw) + } + if other.Kind == OpReg { + if !isX86YrlRegisterForArch(other.Reg, c.goarch) { + return fmt.Errorf("%s MOVL TR%d operand is outside Go 1.27's Yml class: %q", c.goarch, index, ins.Raw) + } + } else if !isAMD64MemoryOperand(other) { + return fmt.Errorf("%s MOVL TR%d expects a Go 1.27 Yml register or memory: %q", c.goarch, index, ins.Raw) + } + modRM := 0xc0 | byte(index<<3) + constraints := "~{memory},~{dirflag},~{fpsr},~{flags}" + if readSpecial { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 asm sideeffect %q, %q()\n", + call, fmt.Sprintf(".byte 0x0f, 0x24, 0x%02x", modRM), "={ax},"+constraints) + return c.storeScalarIntegerOperand(other, I32, "%"+call) + } + value, err := c.evalIntSized(other, I32) + if err != nil { + return err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(i32 %s)\n", + fmt.Sprintf(".byte 0x0f, 0x26, 0x%02x", modRM), "{ax},"+constraints, value) + return nil +} + +func (c *amd64Ctx) lowerDescriptorRegisterAlias(op, class string, other Operand, readSpecial bool, ins Instr) error { + directOp := "" + switch class { + case "gdtr": + if op != "MOVL" && op != "MOVQ" { + break + } + if readSpecial { + directOp = "SGDT" + } else { + directOp = "LGDT" + } + case "idtr": + if op != "MOVL" && op != "MOVQ" { + break + } + if readSpecial { + directOp = "SIDT" + } else { + directOp = "LIDT" + } + case "ldtr": + if op == "MOVW" { + if readSpecial { + directOp = "SLDTW" + } else { + directOp = "LLDT" + } + } + case "msw": + if op == "MOVW" { + if readSpecial { + directOp = "SMSWW" + } else { + directOp = "LMSW" + } + } + case "task": + if op == "MOVW" { + if readSpecial { + directOp = "STRW" + } else { + directOp = "LTR" + } + } + } + if directOp == "" { + return fmt.Errorf("%s %s %s form is absent from Go 1.27's ymovtab: %q", c.goarch, op, strings.ToUpper(class), ins.Raw) + } + ok, _, err := c.lowerDescriptorTable(Op(directOp), Instr{Op: Op(directOp), Args: []Operand{other}, Raw: ins.Raw}) + if !ok { + panic("descriptor MOV alias lowering drift") + } + return err +} diff --git a/amd64_lower_maddubsw_test.go b/amd64_lower_maddubsw_test.go new file mode 100644 index 000000000..5525325d3 --- /dev/null +++ b/amd64_lower_maddubsw_test.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86PackedUnsignedSignedByteMultiplyAddCompleteGoAssemblerForms(t *testing.T) { + amd64Source := `TEXT packedmaddubswforms(SB),NOSPLIT,$0-0 + PMADDUBSW X0, X1 + PMADDUBSW (AX), X15 + VPMADDUBSW X0, X1, X2 + VPMADDUBSW (AX), X1, X2 + VPMADDUBSW Y0, Y1, Y2 + VPMADDUBSW (AX), Y1, Y2 + VPMADDUBSW Z0, Z1, Z2 + VPMADDUBSW (AX), Z1, Z2 + VPMADDUBSW X20, X21, X22 + VPMADDUBSW Y20, Y21, Y22 + VPMADDUBSW Z20, Z21, Z22 + VPMADDUBSW X20, X21, K1, X22 + VPMADDUBSW.Z (AX), X21, K2, X22 + VPMADDUBSW Y20, Y21, K3, Y22 + VPMADDUBSW.Z (AX), Y21, K4, Y22 + VPMADDUBSW Z20, Z21, K5, Z22 + VPMADDUBSW.Z (AX), Z21, K6, Z22 + RET +` + ll := x86UnboundVoidScalarFormatObject(t, amd64Source, "packedmaddubswforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-maddubsw-amd64.ll", "packed-maddubsw-amd64.o", ll) + + source386 := `TEXT packedmaddubswforms386(SB),NOSPLIT,$0-0 + PMADDUBSW X0, X1 + PMADDUBSW (AX), X7 + VPMADDUBSW X0, X1, X2 + VPMADDUBSW (AX), X1, X2 + VPMADDUBSW Y0, Y1, Y2 + VPMADDUBSW (AX), Y1, Y2 + VPMADDUBSW Z0, Z1, Z7 + VPMADDUBSW (AX), Z6, Z7 + RET +` + ll386 := x86UnboundVoidScalarFormatObject(t, source386, "packedmaddubswforms386", "386", "i386-unknown-linux-gnu") + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-maddubsw-386.ll", "packed-maddubsw-386.o", ll386) +} + +func TestTranslateX86PackedUnsignedSignedByteMultiplyAddRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMADDUBSW.Z X0, X1", + "PMADDUBSW X0", + "PMADDUBSW M0, M1", + "PMADDUBSW Y0, Y1", + "PMADDUBSW X0, (AX)", + "VPMADDUBSW X0, X1", + "VPMADDUBSW X0, Y1, Y2", + "VPMADDUBSW X0, X1, (AX)", + "VPMADDUBSW.BCST (AX), X1, X2", + "VPMADDUBSW.Z X0, X1, X2", + "VPMADDUBSW X0, X1, K0, X2", + "VPMADDUBSW X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PMADDUBSW/VPMADDUBSW tables", instruction) + } + }) + } + + for _, instruction := range []string{ + "PMADDUBSW X8, X0", + "PMADDUBSW X0, X8", + "VPMADDUBSW Z8, Z1, Z2", + "VPMADDUBSW Z0, Z1, Z8", + "VPMADDUBSW X0, X1, K1, X2", + "VPMADDUBSW.Z X0, X1, K1, X2", + } { + file, err := Parse(ArchAMD64, "TEXT bad386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad386": {Name: "bad386", Ret: Void}}, + }); err == nil { + t.Fatalf("386 Translate accepted architecture-restricted %q", instruction) + } + } +} diff --git a/amd64_lower_maddword.go b/amd64_lower_maddword.go new file mode 100644 index 000000000..bd3175b9a --- /dev/null +++ b/amd64_lower_maddword.go @@ -0,0 +1,69 @@ +package plan9asm + +import "fmt" + +// Both Go packed multiply-add families share the same ordered operand, +// memory and write-generation contract. +func (c *amd64Ctx) lowerPackedWordMultiplyAdd(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedPhysical(op, ins, x86PackedMultiplyAdd) +} + +func (c *amd64Ctx) emitPackedWordMultiplyAdd(wordLanes int, first, second string) string { + wordType := fmt.Sprintf("<%d x i16>", wordLanes) + wideType := fmt.Sprintf("<%d x i32>", wordLanes) + firstWide := c.newTmp() + secondWide := c.newTmp() + products := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext %s %s to %s\n", firstWide, wordType, first, wideType) + fmt.Fprintf(c.b, " %%%s = sext %s %s to %s\n", secondWide, wordType, second, wideType) + fmt.Fprintf(c.b, " %%%s = mul %s %%%s, %%%s\n", products, wideType, firstWide, secondWide) + resultLanes := wordLanes / 2 + result := "zeroinitializer" + for lane := 0; lane < resultLanes; lane++ { + low := c.newTmp() + high := c.newTmp() + sum := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", low, wideType, products, lane*2) + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", high, wideType, products, lane*2+1) + fmt.Fprintf(c.b, " %%%s = add i32 %%%s, %%%s\n", sum, low, high) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i32> %s, i32 %%%s, i32 %d\n", inserted, resultLanes, result, sum, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitPackedUnsignedSignedByteMultiplyAdd(byteLanes int, signedBytes, unsignedBytes string) string { + byteType := fmt.Sprintf("<%d x i8>", byteLanes) + wideType := fmt.Sprintf("<%d x i32>", byteLanes) + signedWide := c.newTmp() + unsignedWide := c.newTmp() + products := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext %s %s to %s\n", signedWide, byteType, signedBytes, wideType) + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", unsignedWide, byteType, unsignedBytes, wideType) + fmt.Fprintf(c.b, " %%%s = mul %s %%%s, %%%s\n", products, wideType, signedWide, unsignedWide) + resultLanes := byteLanes / 2 + result := "zeroinitializer" + for lane := 0; lane < resultLanes; lane++ { + low := c.newTmp() + high := c.newTmp() + sum := c.newTmp() + below := c.newTmp() + lowerClamped := c.newTmp() + above := c.newTmp() + clamped := c.newTmp() + narrow := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", low, wideType, products, lane*2) + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", high, wideType, products, lane*2+1) + fmt.Fprintf(c.b, " %%%s = add i32 %%%s, %%%s\n", sum, low, high) + fmt.Fprintf(c.b, " %%%s = icmp slt i32 %%%s, -32768\n", below, sum) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 -32768, i32 %%%s\n", lowerClamped, below, sum) + fmt.Fprintf(c.b, " %%%s = icmp sgt i32 %%%s, 32767\n", above, lowerClamped) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 32767, i32 %%%s\n", clamped, above, lowerClamped) + fmt.Fprintf(c.b, " %%%s = trunc i32 %%%s to i16\n", narrow, clamped) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i16> %s, i16 %%%s, i32 %d\n", inserted, resultLanes, result, narrow, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_mask_arithmetic.go b/amd64_lower_mask_arithmetic.go new file mode 100644 index 000000000..f37af5e7c --- /dev/null +++ b/amd64_lower_mask_arithmetic.go @@ -0,0 +1,59 @@ +package plan9asm + +import "fmt" + +type amd64MaskArithmeticSpec struct { + bits int + shift string +} + +// amd64MaskArithmeticSpecs covers Go 1.27's complete KADD and KSHIFT +// families. KADD wraps at the named mask width; KSHIFT produces zero when the +// unsigned byte count is greater than or equal to that width. +var amd64MaskArithmeticSpecs = map[Op]amd64MaskArithmeticSpec{ + "KADDB": {bits: 8}, "KADDW": {bits: 16}, "KADDD": {bits: 32}, "KADDQ": {bits: 64}, + "KSHIFTLB": {bits: 8, shift: "shl"}, "KSHIFTLW": {bits: 16, shift: "shl"}, + "KSHIFTLD": {bits: 32, shift: "shl"}, "KSHIFTLQ": {bits: 64, shift: "shl"}, + "KSHIFTRB": {bits: 8, shift: "lshr"}, "KSHIFTRW": {bits: 16, shift: "lshr"}, + "KSHIFTRD": {bits: 32, shift: "lshr"}, "KSHIFTRQ": {bits: 64, shift: "lshr"}, +} + +func (c *amd64Ctx) lowerMaskArithmetic(op Op, ins Instr) error { + form, known, err := x86MaskValueProperties(op, ins, c.goarch) + if err != nil { + return err + } + if !known || form.unpack { + return fmt.Errorf("%s unknown mask-arithmetic opcode %q", c.goarch, op) + } + spec := form.spec + if spec.shift == "" { + first, err := c.loadK(ins.Args[0].Reg) + if err != nil { + return err + } + second, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return err + } + first = c.maskMoveI64(first, spec.bits) + second = c.maskMoveI64(second, spec.bits) + added := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %s\n", added, first, second) + return c.storeK(ins.Args[2].Reg, c.maskMoveI64("%"+added, spec.bits)) + } + + source, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return err + } + source = c.maskMoveI64(source, spec.bits) + result := "0" + count := int(uint8(ins.Args[0].Imm)) + if count < spec.bits { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i64 %s, %d\n", shifted, spec.shift, source, count) + result = c.maskMoveI64("%"+shifted, spec.bits) + } + return c.storeK(ins.Args[2].Reg, result) +} diff --git a/amd64_lower_mask_arithmetic_test.go b/amd64_lower_mask_arithmetic_test.go new file mode 100644 index 000000000..e973e4764 --- /dev/null +++ b/amd64_lower_mask_arithmetic_test.go @@ -0,0 +1,170 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86MaskArithmeticCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT maskArithmeticForms(SB),$0-0\n") + for _, width := range []string{"B", "W", "D", "Q"} { + fmt.Fprintf(&source, "\tKADD%s K0, K7, K3\n", width) + fmt.Fprintf(&source, "\tKSHIFTL%s $0, K3, K4\n", width) + fmt.Fprintf(&source, "\tKSHIFTR%s $255, K4, K7\n", width) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + original, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + options := Options{TargetTriple: target.triple, Goarch: target.goarch, + Sigs: map[string]FuncSig{"maskArithmeticForms": {Name: "maskArithmeticForms", Ret: Void}}} + if ir, err := Translate(original, options); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original format-only source invented mask inputs", err) + } + // Preserve every original row and separately supply actual source + // definitions. This is a complete initialized object counterpart. + initialized := strings.Replace(source.String(), "\n", "\nMOVL $7,AX\nKMOVD AX,K0\nKMOVD AX,K7\n", 1) + requireX86GoAssemblerResult(t, target.goarch, initialized, true) + file, err := Parse(ArchAMD64, initialized) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "maskArithmeticForms": {Name: "maskArithmeticForms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "mask-arithmetic-"+target.name+".ll", "mask-arithmetic-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86MaskArithmeticRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "KADDB K1, K2", + "KADDW K1, AX, K2", + "KADDD K1, K2, K8", + "KADDQ.Z K1, K2, K3", + "KSHIFTLB K1, K2", + "KSHIFTLW $-1, K1, K2", + "KSHIFTLD $1, AX, K2", + "KSHIFTRQ $1, K1, K8", + "KSHIFTRB.Z $1, K1, K2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + for _, goarch := range []string{"amd64", "386"} { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + assertX86MaskArithmeticRejected(t, goarch, instruction) + } + }) + } +} + +func assertX86MaskArithmeticRejected(t *testing.T, goarch, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %s %q outside Go 1.27's KADD/KSHIFT tables", goarch, instruction) + } +} + +func TestAMD64MaskArithmeticRuntimeSemantics(t *testing.T) { + // Portable LLVM integer execution, not original-Go/native opmask execution. + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT maskArithmetic(SB),$0-24 + MOVQ out+0(FP), AX + KMOVQ a+8(FP), K1 + KMOVQ b+16(FP), K2 + KADDB K1, K2, K3 + KADDW K1, K2, K4 + KSHIFTLW $3, K1, K5 + KSHIFTRW $4, K2, K6 + KSHIFTRB $9, K2, K7 + KMOVQ K3, 0(AX) + KMOVQ K4, 8(AX) + KMOVQ K5, 16(AX) + KMOVQ K6, 24(AX) + KMOVQ K7, 32(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "maskArithmetic": {Name: "maskArithmetic", Args: []LLVMType{Ptr, I64, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void maskArithmetic(uint64_t *, uint64_t, uint64_t); +int main(void) { + uint64_t out[5] = {0}; + uint64_t a = 0x123456789abcdef5ULL; + uint64_t b = 0xfedcba98765432f0ULL; + maskArithmetic(out, a, b); + if (out[0] != ((a + b) & 0xff)) return 1; + if (out[1] != ((a + b) & 0xffff)) return 2; + if (out[2] != ((a & 0xffff) << 3 & 0xffff)) return 3; + if (out[3] != ((b & 0xffff) >> 4)) return 4; + if (out[4] != 0) return 5; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask_arithmetic", triple, ll, mainC, nil) +} diff --git a/amd64_lower_mask_logical.go b/amd64_lower_mask_logical.go new file mode 100644 index 000000000..4ad903463 --- /dev/null +++ b/amd64_lower_mask_logical.go @@ -0,0 +1,147 @@ +package plan9asm + +import "fmt" + +type amd64MaskLogicalKind uint8 + +const ( + amd64MaskAnd amd64MaskLogicalKind = iota + amd64MaskAndNot + amd64MaskOr + amd64MaskXNOR + amd64MaskXOR + amd64MaskNot + amd64MaskTest + amd64MaskOrTest +) + +type amd64MaskLogicalSpec struct { + kind amd64MaskLogicalKind + bits int +} + +// amd64MaskLogicalSpecs is the complete Go 1.27 x86 mask-register logical +// family. Binary forms use _ykaddb (K,K,K); unary and flag-setting forms use +// _yknotb (K,K). Go accepts every listed form in both amd64 and 386 mode. +var amd64MaskLogicalSpecs = map[Op]amd64MaskLogicalSpec{ + "KANDB": {amd64MaskAnd, 8}, "KANDW": {amd64MaskAnd, 16}, "KANDD": {amd64MaskAnd, 32}, "KANDQ": {amd64MaskAnd, 64}, + "KANDNB": {amd64MaskAndNot, 8}, "KANDNW": {amd64MaskAndNot, 16}, "KANDND": {amd64MaskAndNot, 32}, "KANDNQ": {amd64MaskAndNot, 64}, + "KORB": {amd64MaskOr, 8}, "KORW": {amd64MaskOr, 16}, "KORD": {amd64MaskOr, 32}, "KORQ": {amd64MaskOr, 64}, + "KXNORB": {amd64MaskXNOR, 8}, "KXNORW": {amd64MaskXNOR, 16}, "KXNORD": {amd64MaskXNOR, 32}, "KXNORQ": {amd64MaskXNOR, 64}, + "KXORB": {amd64MaskXOR, 8}, "KXORW": {amd64MaskXOR, 16}, "KXORD": {amd64MaskXOR, 32}, "KXORQ": {amd64MaskXOR, 64}, + "KNOTB": {amd64MaskNot, 8}, "KNOTW": {amd64MaskNot, 16}, "KNOTD": {amd64MaskNot, 32}, "KNOTQ": {amd64MaskNot, 64}, + "KTESTB": {amd64MaskTest, 8}, "KTESTW": {amd64MaskTest, 16}, "KTESTD": {amd64MaskTest, 32}, "KTESTQ": {amd64MaskTest, 64}, + "KORTESTB": {amd64MaskOrTest, 8}, "KORTESTW": {amd64MaskOrTest, 16}, "KORTESTD": {amd64MaskOrTest, 32}, "KORTESTQ": {amd64MaskOrTest, 64}, +} + +func (c *amd64Ctx) lowerMaskLogical(op Op, ins Instr) error { + spec, known, err := x86MaskLogicalProperties(op, ins, c.goarch) + if err != nil { + return err + } + if !known { + return fmt.Errorf("%s unknown mask logical opcode %q", c.goarch, op) + } + + load := func(arg Operand) (string, error) { + value, err := c.loadK(arg.Reg) + if err != nil { + return "", err + } + if spec.bits == 64 { + return value, nil + } + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, value, amd64MaskWidthLiteral(spec.bits)) + return "%" + masked, nil + } + + first, err := load(ins.Args[0]) + if err != nil { + return err + } + var second string + if spec.kind != amd64MaskNot { + second, err = load(ins.Args[1]) + if err != nil { + return err + } + } + if spec.writesFlags() { + return c.lowerMaskLogicalFlags(spec, first, second) + } + + var result string + switch spec.kind { + case amd64MaskNot: + not := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", not, first, amd64MaskWidthLiteral(spec.bits)) + result = "%" + not + case amd64MaskAnd, amd64MaskOr, amd64MaskXOR, amd64MaskXNOR, amd64MaskAndNot: + operation := "and" + lhs, rhs := first, second + switch spec.kind { + case amd64MaskOr: + operation = "or" + case amd64MaskXOR, amd64MaskXNOR: + operation = "xor" + case amd64MaskAndNot: + // Go's first source is the r/m operand and its second source is + // VEX.vvvv, so KANDN computes first & ~second. + notSecond := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", notSecond, second, amd64MaskWidthLiteral(spec.bits)) + rhs = "%" + notSecond + } + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i64 %s, %s\n", combined, operation, lhs, rhs) + result = "%" + combined + if spec.kind == amd64MaskXNOR { + not := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", not, result, amd64MaskWidthLiteral(spec.bits)) + result = "%" + not + } + default: + return fmt.Errorf("amd64 %s has unknown mask logical operation", op) + } + if spec.bits != 64 { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, result, amd64MaskWidthLiteral(spec.bits)) + result = "%" + masked + } + return c.storeK(ins.Args[spec.argumentCount()-1].Reg, result) +} + +func (c *amd64Ctx) lowerMaskLogicalFlags(spec amd64MaskLogicalSpec, first, second string) error { + operation := "and" + if spec.kind == amd64MaskOrTest { + operation = "or" + } + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i64 %s, %s\n", combined, operation, first, second) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, 0\n", zero, combined) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + + carry := c.newTmp() + if spec.kind == amd64MaskTest { + notSecond := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", notSecond, second, amd64MaskWidthLiteral(spec.bits)) + andNot := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %%%s\n", andNot, first, notSecond) + fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, 0\n", carry, andNot) + } else { + // KORTEST observes the same OR result for both flags: zero for ZF, + // all width-selected bits set for CF. KTEST's AND-NOT stays separate. + fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, %s\n", carry, combined, amd64MaskWidthLiteral(spec.bits)) + } + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + c.clearX86StatusFlags(x86StatusAll &^ (x86StatusCF | x86StatusZF)) + return nil +} + +func amd64MaskWidthLiteral(bits int) string { + if bits == 64 { + return "-1" + } + return fmt.Sprintf("%d", uint64(1)<memory, K->GP, +// K-or-memory->K, and GP->K rows. Go accepts these rows in amd64 and 386 mode. +var amd64MaskMoveSpecs = map[Op]int{ + "KMOVB": 8, + "KMOVW": 16, + "KMOVD": 32, + "KMOVQ": 64, +} + +type x86MaskMoveOperandKind uint8 + +const ( + x86MaskMoveK x86MaskMoveOperandKind = iota + x86MaskMoveGP + x86MaskMoveMemory +) + +type x86MaskMoveForm struct { + bits int + source, destination x86MaskMoveOperandKind +} + +// Only a GP operand binds the mode-effective word width: VEX.W is ignored +// for GP transfers in mode32. K/K and K/memory keep the original native width. +func x86MaskMoveProperties(op Op, ins Instr, arch string) (x86MaskMoveForm, bool, error) { + bits, known := amd64MaskMoveSpecs[op] + form := x86MaskMoveForm{bits: bits} + if !known { + return form, false, nil + } + fail := func() (x86MaskMoveForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands are outside Go's _ykmovb table: %q", arch, op, ins.Raw) + } + if arch != "386" && arch != "amd64" || !strings.EqualFold(string(ins.Op), string(op)) || len(ins.Args) != 2 { + return fail() + } + kind := func(operand Operand) (x86MaskMoveOperandKind, bool) { + switch { + case amd64IsKOperand(operand): + return x86MaskMoveK, true + case amd64IsYrlOperand(operand, arch): + return x86MaskMoveGP, true + case isAMD64MemoryOperand(operand): + return x86MaskMoveMemory, true + } + return 0, false + } + var sourceOK, destinationOK bool + form.source, sourceOK = kind(ins.Args[0]) + form.destination, destinationOK = kind(ins.Args[1]) + if !sourceOK || !destinationOK || form.source != x86MaskMoveK && form.destination != x86MaskMoveK { + return fail() + } + if form.source == x86MaskMoveGP || form.destination == x86MaskMoveGP { + if word := int(x86ResultPointerWord(arch) * 8); form.bits > word { + form.bits = word + } + } + return form, true, nil +} + +func (c *amd64Ctx) lowerMaskMove(op Op, ins Instr) error { + if ins.Op != "" && Op(strings.ToUpper(string(ins.Op))) != op { + return fmt.Errorf("amd64 mask-move dispatch %q disagrees with instruction %q", op, ins.Op) + } + // The internal dispatcher owns the opcode argument, including for helper + // callers without Instr.Op. Bind a local copy before the shared validator; + // source/caller proofs still require their original complete instruction. + ins.Op = op + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86MaskMoveProperties(op, ins, arch) + if !known { + return fmt.Errorf("amd64 unknown mask-move opcode %q", op) + } + if err != nil { + return err + } + src, dst := ins.Args[0], ins.Args[1] + value, err := c.loadMaskMoveValue(src, form.bits, form.source == x86MaskMoveK) + if err != nil { + return err + } + switch form.destination { + case x86MaskMoveK: + return c.storeK(dst.Reg, value) + case x86MaskMoveGP: + // The GP-destination encodings zero-extend B/W/D results. Store the + // already-masked value as the complete modeled register value. + base, _ := amd64FullRegBase(dst.Reg) + return c.storeReg(base, value) + case x86MaskMoveMemory: + return c.storeMaskMoveMemory(dst, form.bits, value) + default: + return fmt.Errorf("amd64 %s has unsupported destination: %q", op, ins.Raw) + } +} + +func amd64MaskMoveOperandsValid(ins Instr, goarch string) bool { + _, known, err := x86MaskMoveProperties(ins.Op, ins, goarch) + return known && err == nil +} + +func x86TypedMaskMoveByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86MaskMoveProperties(op, ins, arch) + if !known { + return x86TypedByteEffect{}, false, nil + } + if err != nil { + return x86TypedByteEffect{}, true, err + } + width := int64(form.bits / 8) + first, last := ins.Args[0], ins.Args[1] + writeWidth := width + if form.destination == x86MaskMoveGP && writeWidth < x86ResultPointerWord(arch) { + writeWidth = x86ResultPointerWord(arch) // GP B/W/D writes zero-extend; memory stays exact-width. + } + return x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: 0, operand: first, width: width, read: true}, + {index: 1, operand: last, width: writeWidth, write: true}, + }}, true, nil +} + +func amd64IsKOperand(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + _, ok := amd64ParseKReg(operand.Reg) + return ok +} + +func amd64IsYrlOperand(operand Operand, goarch string) bool { + return operand.Kind == OpReg && isX86YrlRegisterForArch(operand.Reg, goarch) +} + +func (c *amd64Ctx) loadMaskMoveValue(src Operand, bits int, sourceIsK bool) (string, error) { + if sourceIsK { + value, err := c.loadK(src.Reg) + if err != nil { + return "", err + } + return c.maskMoveI64(value, bits), nil + } + + typ := amd64IntegerTypeForBits(bits) + var value string + var err error + if src.Kind == OpFP && c.fpScalarSource.x86SourceStack == nil && c.fpScalarSource.x86FixedLocal == nil { + // A mask memory width is not its first logical field's type. Gather + // precisely those bytes, including interior and cross-field reads. + value, err = c.loadScalarLaneFPBits(src.FPOffset, bits) + } else if src.Kind == OpMem || src.Kind == OpSym { + pointer, pointerType, _, pointerErr := c.packedVectorMemoryPointer(src) + if pointerErr != nil { + return "", pointerErr + } + temporary := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", temporary, typ, pointerType, pointer) + value = "%" + temporary + } else { + value, err = c.evalIntSized(src, typ) + } + if err != nil { + return "", err + } + if bits == 64 { + return value, nil + } + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", extended, typ, value) + return "%" + extended, nil +} + +func (c *amd64Ctx) maskMoveI64(value string, bits int) string { + if bits == 64 { + return value + } + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, value, amd64MaskWidthLiteral(bits)) + return "%" + masked +} + +func (c *amd64Ctx) storeMaskMoveMemory(dst Operand, bits int, value string) error { + typ := amd64IntegerTypeForBits(bits) + stored := value + if bits != 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", truncated, value, typ) + stored = "%" + truncated + } + switch dst.Kind { + case OpFP: + if c.fpScalarSource.x86SourceStack == nil && c.fpScalarSource.x86FixedLocal == nil { + // Scatter only the native write width; a B/W/D store must retain + // the initialized remainder of a larger logical result field. + return c.storeScalarLaneFPBits(dst.FPOffset, bits, stored) + } + return c.storeFPResult(dst.FPOffset, typ, stored) + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store volatile %s %s, %s %s, align 1\n", typ, stored, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store volatile %s %s, ptr %s, align 1\n", typ, stored, ptr) + return nil + default: + return fmt.Errorf("amd64 mask move expected a memory destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_mask_to_vector.go b/amd64_lower_mask_to_vector.go new file mode 100644 index 000000000..850c7fec8 --- /dev/null +++ b/amd64_lower_mask_to_vector.go @@ -0,0 +1,64 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerMaskVectorMove(op Op, ins Instr) (bool, bool, error) { + if ins.Op == "" { + ins.Op = op + } + form, recognized, err := x86MaskVectorProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + if !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("mask/vector dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + vectorOperand, maskOperand := ins.Args[form.vector], ins.Args[form.mask] + lanes := form.byteWidth * 8 / form.spec.laneBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, form.spec.laneBits) + if !form.spec.maskToVector { + bytes, err := c.loadPackedCompareBytes(vectorOperand, form.byteWidth) + if err != nil { + return true, false, err + } + vector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", vector, form.byteWidth, bytes, vectorType) + signBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, zeroinitializer\n", signBits, vectorType, vector) + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %%%s to %s\n", packed, lanes, signBits, packedType) + result := "%" + packed + if lanes < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + return true, false, c.storeK(maskOperand.Reg, result) + } + + mask, err := c.loadK(maskOperand.Reg) + if err != nil { + return true, false, err + } + vector := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", shifted, mask, lane) + bit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i1\n", bit, shifted) + element := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d -1, i%d 0\n", element, bit, form.spec.laneBits, form.spec.laneBits) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %%%s, i32 %d\n", inserted, vectorType, vector, form.spec.laneBits, element, lane) + vector = "%" + inserted + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytes, vectorType, vector, form.byteWidth) + // All widths are EVEX writes. Legacy storeX would preserve wider aliases + // and contradict the caller's complete defined-byte effect. + return true, false, c.storePackedMoveOperand(vectorOperand, form.byteWidth, "%"+bytes) +} diff --git a/amd64_lower_mask_to_vector_test.go b/amd64_lower_mask_to_vector_test.go new file mode 100644 index 000000000..5317487b9 --- /dev/null +++ b/amd64_lower_mask_to_vector_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86MaskToVectorMoveCompleteForms(t *testing.T) { + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT maskvectorforms(SB),$0-0\n") + index := 0 + for _, lane := range []string{"B", "W", "D", "Q"} { + for _, width := range []string{"X", "Y", "Z"} { + vector := index%7 + 1 + mask := index % 8 + fmt.Fprintf(&source, "\tVPMOVM2%s K%d, %s%d\n", lane, mask, width, vector) + fmt.Fprintf(&source, "\tVPMOV%s2M %s%d, K%d\n", lane, width, vector, mask) + index++ + } + } + source.WriteString("\tRET\n") + x86UnboundVoidScalarFormatObject(t, source.String(), "maskvectorforms", target.goarch, target.triple) + }) + } +} + +func TestTranslateX86MaskVectorMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPMOVM2B X0, X1", + "VPMOVM2W K0, 0(AX)", + "VPMOVM2D.Z K0, Z1", + "VPMOVM2Q K8, Z1", + "VPMOVB2M K0, K1", + "VPMOVW2M 0(AX), K1", + "VPMOVD2M.Z Z0, K1", + "VPMOVQ2M Z0, K8", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + for _, goarch := range []string{"amd64", "386"} { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + triple := "x86_64-unknown-linux-gnu" + if goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: goarch, TargetTriple: triple, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %s %q outside Go 1.27's mask/vector move table", goarch, instruction) + } + } + }) + } +} + +func TestAMD64MaskVectorMoveRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT maskVectorMoves(SB),NOSPLIT,$0-16 + MOVQ out+0(FP), AX + KMOVQ mask+8(FP), K1 + VPMOVM2B K1, Z0 + VPMOVB2M Z0, K2 + KMOVQ K2, 0(AX) + VPMOVM2W K1, Z1 + VPMOVW2M Z1, K3 + KMOVQ K3, 8(AX) + VPMOVM2D K1, Z2 + VPMOVD2M Z2, K4 + KMOVQ K4, 16(AX) + VPMOVM2Q K1, Z3 + VPMOVQ2M Z3, K5 + KMOVQ K5, 24(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "maskVectorMoves": {Name: "maskVectorMoves", Args: []LLVMType{Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void maskVectorMoves(uint64_t *, uint64_t); +int main(void) { + uint64_t out[4] = {0}; + uint64_t mask = 0xd6a539c7f04b821dULL; + maskVectorMoves(out, mask); + if (out[0] != mask) return 1; + if (out[1] != (mask & 0xffffffffULL)) return 2; + if (out[2] != (mask & 0xffffULL)) return 3; + if (out[3] != (mask & 0xffULL)) return 4; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask_vector_moves", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_mask_unpack.go b/amd64_lower_mask_unpack.go new file mode 100644 index 000000000..18e69ec44 --- /dev/null +++ b/amd64_lower_mask_unpack.go @@ -0,0 +1,43 @@ +package plan9asm + +import "fmt" + +// amd64MaskUnpackBits covers Go 1.27's complete KUNPCK family. The Plan 9 +// first source is the encoded r/m operand and becomes the low half; the second +// source is VEX.vvvv and becomes the high half of the destination mask. +var amd64MaskUnpackBits = map[Op]int{ + "KUNPCKBW": 8, + "KUNPCKWD": 16, + "KUNPCKDQ": 32, +} + +func (c *amd64Ctx) lowerMaskUnpack(op Op, ins Instr) error { + form, known, err := x86MaskValueProperties(op, ins, c.goarch) + if err != nil { + return err + } + if !known || !form.unpack { + return fmt.Errorf("%s unknown mask-unpack opcode %q", c.goarch, op) + } + bits := form.spec.bits + + low, err := c.loadK(ins.Args[0].Reg) + if err != nil { + return err + } + high, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return err + } + low = c.maskMoveI64(low, bits) + high = c.maskMoveI64(high, bits) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i64 %s, %d\n", shifted, high, bits) + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %s\n", combined, shifted, low) + result := "%" + combined + if bits < 32 { + result = c.maskMoveI64(result, bits*2) + } + return c.storeK(ins.Args[2].Reg, result) +} diff --git a/amd64_lower_mask_unpack_test.go b/amd64_lower_mask_unpack_test.go new file mode 100644 index 000000000..6012a8b24 --- /dev/null +++ b/amd64_lower_mask_unpack_test.go @@ -0,0 +1,168 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86MaskUnpackCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + const source = `TEXT maskunpackforms(SB),$0-0 + KUNPCKBW K0, K7, K3 + KUNPCKBW K2, K4, K4 + KUNPCKWD K7, K5, K3 + KUNPCKDQ K1, K6, K0 + KUNPCKDQ K5, K5, K5 + RET +` + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + original, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"maskunpackforms": {Name: "maskunpackforms", Ret: Void}}} + if ir, err := Translate(original, options); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original format-only source invented mask inputs", err) + } + var definitions strings.Builder + definitions.WriteString("\nMOVL $7,AX\n") + for index := 0; index < 8; index++ { + fmt.Fprintf(&definitions, "KMOVD AX,K%d\n", index) + } + initialized := strings.Replace(source, "\n", definitions.String(), 1) + requireX86GoAssemblerResult(t, target.goarch, initialized, true) + file, err := Parse(ArchAMD64, initialized) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"maskunpackforms": {Name: "maskunpackforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"shl i64", "or i64", "255", "65535", "4294967295"} { + if !strings.Contains(ir, want) { + t.Fatalf("mask unpack lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "mask-unpack-"+target.name+".ll", "mask-unpack-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86MaskUnpackRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "KUNPCKBW K1, K2", + "KUNPCKWD K1, K2, K3, K4", + "KUNPCKDQ AX, K2, K3", + "KUNPCKBW K1, K8, K3", + "KUNPCKWD K1, K2, K8", + "KUNPCKDQ.Z K1, K2, K3", + } { + for _, goarch := range []string{"amd64", "386"} { + t.Run(goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's KUNPCK table for %s", instruction, goarch) + } + }) + } + } +} + +func TestAMD64MaskUnpackRuntimeSemantics(t *testing.T) { + // Portable LLVM integer execution, not original-Go/native opmask execution. + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT maskunpacksemantics(SB),$0-24 + MOVQ out+0(FP), AX + KMOVQ a+8(FP), K1 + KMOVQ b+16(FP), K2 + STC + KUNPCKBW K1, K2, K3 + KMOVQ K3, 0(AX) + SETCS 24(AX) + KUNPCKWD K1, K2, K4 + KMOVQ K4, 8(AX) + KUNPCKDQ K1, K2, K2 + KMOVQ K2, 16(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "maskunpacksemantics": { + Name: "maskunpacksemantics", Args: []LLVMType{Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void maskunpacksemantics(uint8_t *, uint64_t, uint64_t); +int main(void) { + uint8_t storage[25] = {0}; + uint64_t *out = (uint64_t *)storage; + uint64_t a = UINT64_C(0x0123456789abcdef); + uint64_t b = UINT64_C(0xfedcba9876543210); + maskunpacksemantics(storage, a, b); + if (out[0] != ((b & 0xff) << 8 | (a & 0xff))) return 10; + if (out[1] != ((b & 0xffff) << 16 | (a & 0xffff))) return 11; + if (out[2] != ((b & UINT64_C(0xffffffff)) << 32 | (a & UINT64_C(0xffffffff)))) return 12; + if (storage[24] != 1) return 13; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask_unpack_semantics", triple, ir, mainC, nil) +} diff --git a/amd64_lower_minmax.go b/amd64_lower_minmax.go new file mode 100644 index 000000000..7756475c2 --- /dev/null +++ b/amd64_lower_minmax.go @@ -0,0 +1,74 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedIntegerMinMaxSpec struct { + laneBits int + signed bool + minimum bool + broadcast bool + mapNumber int + opcode int +} + +// The twelve legacy and sixteen VEX/EVEX members share typed operand and +// numerical rules. Legacy retains its RMW/upper aliases and original m128 +// opcode; only VEX/EVEX D/Q has scalar-memory broadcast. +func (c *amd64Ctx) lowerPackedIntegerMinMax(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86IntegerMinMaxProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := c.validateConflictFunnelMemory(ins, form.x86PackedBinaryForm); err != nil { + return true, false, err + } + first := ins.Args[form.first] + if form.legacy && form.memory >= 0 && + (first.Kind == OpFP || first.Kind == OpMem && !c.nativeMemoryOperand(ins, form.first) || + !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple)) { + return true, false, fmt.Errorf("%w: legacy integer min/max m128 needs its original native storage/target", ErrProbeNeedsContext) + } + spec := form.spec + predicate := "ult" + if spec.signed { + predicate = "slt" + } + if !spec.minimum { + predicate = strings.Replace(predicate, "lt", "gt", 1) + } + legacyMnemonic := "" + if form.legacy && form.memory >= 0 { + legacyMnemonic = strings.ToLower(string(op)) + } + err = c.emitPackedBinaryE4(ins, form.x86PackedBinaryForm, spec.laneBits, legacyMnemonic, + func(lanes, bits int, first, second string) string { + compared, computed := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s <%d x i%d> %s, %s\n", compared, predicate, lanes, bits, second, first) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %s, <%d x i%d> %s\n", computed, lanes, compared, lanes, bits, second, lanes, bits, first) + return "%" + computed + }) + return true, false, err +} + +var amd64PackedIntegerMinMaxSpecs = map[string]amd64PackedIntegerMinMaxSpec{ + "VPMINSB": {laneBits: 8, signed: true, minimum: true, mapNumber: 2, opcode: 0x38}, + "VPMINUB": {laneBits: 8, minimum: true, mapNumber: 1, opcode: 0xda}, + "VPMAXSB": {laneBits: 8, signed: true, mapNumber: 2, opcode: 0x3c}, + "VPMAXUB": {laneBits: 8, mapNumber: 1, opcode: 0xde}, + "VPMINSW": {laneBits: 16, signed: true, minimum: true, mapNumber: 1, opcode: 0xea}, + "VPMINUW": {laneBits: 16, minimum: true, mapNumber: 2, opcode: 0x3a}, + "VPMAXSW": {laneBits: 16, signed: true, mapNumber: 1, opcode: 0xee}, + "VPMAXUW": {laneBits: 16, mapNumber: 2, opcode: 0x3e}, + + "VPMINSD": {laneBits: 32, signed: true, minimum: true, broadcast: true, mapNumber: 2, opcode: 0x39}, + "VPMINUD": {laneBits: 32, minimum: true, broadcast: true, mapNumber: 2, opcode: 0x3b}, + "VPMAXSD": {laneBits: 32, signed: true, broadcast: true, mapNumber: 2, opcode: 0x3d}, + "VPMAXUD": {laneBits: 32, broadcast: true, mapNumber: 2, opcode: 0x3f}, + "VPMINSQ": {laneBits: 64, signed: true, minimum: true, broadcast: true, mapNumber: 2, opcode: 0x39}, + "VPMINUQ": {laneBits: 64, minimum: true, broadcast: true, mapNumber: 2, opcode: 0x3b}, + "VPMAXSQ": {laneBits: 64, signed: true, broadcast: true, mapNumber: 2, opcode: 0x3d}, + "VPMAXUQ": {laneBits: 64, broadcast: true, mapNumber: 2, opcode: 0x3f}, +} diff --git a/amd64_lower_minmax_test.go b/amd64_lower_minmax_test.go new file mode 100644 index 000000000..f8c97b210 --- /dev/null +++ b/amd64_lower_minmax_test.go @@ -0,0 +1,132 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedIntegerMinMaxCompleteGoAssemblerForms(t *testing.T) { + type operation struct { + name string + broadcast bool + } + operations := []operation{ + {name: "VPMINSB"}, {name: "VPMINUB"}, {name: "VPMAXSB"}, {name: "VPMAXUB"}, + {name: "VPMINSW"}, {name: "VPMINUW"}, {name: "VPMAXSW"}, {name: "VPMAXUW"}, + {name: "VPMINSD", broadcast: true}, {name: "VPMINUD", broadcast: true}, + {name: "VPMAXSD", broadcast: true}, {name: "VPMAXUD", broadcast: true}, + {name: "VPMINSQ", broadcast: true}, {name: "VPMINUQ", broadcast: true}, + {name: "VPMAXSQ", broadcast: true}, {name: "VPMAXUQ", broadcast: true}, + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + zLast := 22 + if target.goarch == "386" { + zLast = 7 + } + var source strings.Builder + source.WriteString("TEXT packedintegerminmaxforms(SB),NOSPLIT,$0-0\n") + for _, operation := range operations { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", operation.name, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s20, %s21\n", operation.name, width, width) + if operation.broadcast { + fmt.Fprintf(&source, "\t%s.BCST 16(AX), %s20, %s22\n", operation.name, width, width) + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", operation.name, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 8(AX), %s21, K7, %s22\n", operation.name, width, width) + if operation.broadcast { + fmt.Fprintf(&source, "\t%s.BCST.Z 16(AX), %s21, K2, %s22\n", operation.name, width, width) + } + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, Z%d\n", operation.name, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", operation.name, zLast) + if operation.broadcast { + fmt.Fprintf(&source, "\t%s.BCST 16(AX), Z6, Z%d\n", operation.name, zLast) + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K3, Z22\n", operation.name) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z21, K4, Z22\n", operation.name) + if operation.broadcast { + fmt.Fprintf(&source, "\t%s.BCST.Z 16(AX), Z21, K5, Z22\n", operation.name) + } + } + } + source.WriteString("\tRET\n") + assembleX87ControlBytes(t, target.goarch, strings.ReplaceAll(source.String(), "NOSPLIT", "4")) + file, signatures := x86UnboundVoidFormatSources(t, source.String(), target.goarch, target.triple) + if target.goarch == "386" { + // Go accepts the original high X/Y spellings without proving + // physical mode32 identities. Retain its whole Go object and + // Context result; initialized low-bank objects have five owners. + return + } + ll := x86ScalarFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-integer-minmax-"+target.name+".ll", "packed-integer-minmax-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedIntegerMinMaxRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VPMINUB X0, X1", + "VPMAXUB X0, X1, (AX)", + "VPMINSW X0, Y1, X2", + "VPMAXUW Z0, Z1, Y2", + "VPMINSD X0, X1, K0, X2", + "VPMAXUD X0, K1, X1, X2", + "VPMINSQ.Z X0, X1, X2", + "VPMAXUQ.SAE X0, X1, X2", + "VPMINUB.BCST 8(AX), X1, X2", + "VPMAXSW.BCST 8(AX), X1, X2", + "VPMINUD.BCST X0, X1, X2", + "VPMAXSQ.BCST 8(AX), Y1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedIntegerMinMaxRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VPMINUB Z8, Z0, Z1", + "VPMAXUW Z0, Z8, Z1", + "VPMINSD Z0, Z1, Z8", + "VPMAXUQ X0, X1, K1, X2", + "VPMINSQ.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedIntegerMinMaxRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedIntegerMinMaxRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed min/max forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_minpos.go b/amd64_lower_minpos.go new file mode 100644 index 000000000..3376192de --- /dev/null +++ b/amd64_lower_minpos.go @@ -0,0 +1,86 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64MinimumPositionSpec struct { + vector bool + mapNumber int + opcode int +} + +// Both encodings have one X/m128 input, a 128-bit output and eight unsigned +// word lanes. Named lowering and raw VEX decoding share the family table. +var amd64MinimumPositionSpecs = map[Op]amd64MinimumPositionSpec{ + "PHMINPOSUW": {mapNumber: 2, opcode: 0x41}, + "VPHMINPOSUW": {vector: true, mapNumber: 2, opcode: 0x41}, +} + +// lowerPackedUnsignedWordMinimumPosition implements every Go 1.27 form of +// PHMINPOSUW (yxm_q4) and VPHMINPOSUW (_yvaesimc). Both tables are restricted +// to a single X/m128 source and an X destination; the VEX spelling is also a +// two-operand, destructive-looking Plan 9 form. +func (c *amd64Ctx) lowerPackedUnsignedWordMinimumPosition(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, recognized, err := x86BlendMinposProperties(op, ins, c.goarch) + if !recognized || !form.minimum { + return false, false, nil + } + if err != nil { + return true, false, err + } + if form.legacy && form.memory { + return true, false, c.lowerLegacyBlendMinposMemory(ins, form) + } + bytesValue, err := c.loadX86PackedShiftBytes(ins.Args[0], 16, true) + if err != nil { + return true, false, err + } + words := c.bitcastVectorBytesToIntegerLanes(16, 8, 16, bytesValue) + result := c.emitPackedUnsignedWordMinimumPosition(words) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i16> %s to <16 x i8>\n", out, result) + if !form.legacy { + return true, false, c.storePackedMoveOperand(ins.Args[1], 16, "%"+out) + } + return true, false, c.storeLegacyXViews(ins.Args[1].Reg, "%"+out) +} + +// Legacy SSE replaces the low 128 bits in every live view without changing +// the upper bytes. A narrow store into each wider slot expresses that directly. +func (c *amd64Ctx) storeLegacyXViews(dst Reg, value string) error { + index, valid := amd64ParseXReg(dst) + if !valid { + return fmt.Errorf("invalid legacy X register %s", dst) + } + for _, slots := range []map[int]string{c.xRegSlot, c.yRegSlot, c.zRegSlot} { + if slot, live := slots[index]; live { + fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s\n", value, slot) + } + } + return nil +} + +func (c *amd64Ctx) emitPackedUnsignedWordMinimumPosition(words string) string { + minimum := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <8 x i16> %s, i32 0\n", minimum, words) + minimumValue := "%" + minimum + indexValue := "0" + for lane := 1; lane < 8; lane++ { + value := c.newTmp() + less := c.newTmp() + selectedMinimum := c.newTmp() + selectedIndex := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <8 x i16> %s, i32 %d\n", value, words, lane) + fmt.Fprintf(c.b, " %%%s = icmp ult i16 %%%s, %s\n", less, value, minimumValue) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %%%s, i16 %s\n", selectedMinimum, less, value, minimumValue) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %d, i16 %s\n", selectedIndex, less, lane, indexValue) + minimumValue = "%" + selectedMinimum + indexValue = "%" + selectedIndex + } + withMinimum := c.newTmp() + withIndex := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <8 x i16> zeroinitializer, i16 %s, i32 0\n", withMinimum, minimumValue) + fmt.Fprintf(c.b, " %%%s = insertelement <8 x i16> %%%s, i16 %s, i32 1\n", withIndex, withMinimum, indexValue) + return "%" + withIndex +} diff --git a/amd64_lower_minpos_test.go b/amd64_lower_minpos_test.go new file mode 100644 index 000000000..bfa5b94e5 --- /dev/null +++ b/amd64_lower_minpos_test.go @@ -0,0 +1,86 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedUnsignedWordMinimumPositionCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + source := fmt.Sprintf("TEXT packedminimumpositionforms(SB),NOSPLIT,$0-0\n"+ + "\tPHMINPOSUW X0, X%d\n"+ + "\tPHMINPOSUW 8(AX), X%d\n"+ + "\tVPHMINPOSUW X0, X%d\n"+ + "\tVPHMINPOSUW 8(AX), X%d\n"+ + "\tRET\n", last, last, last, last) + ll := x86UnboundBlendMinposFormatObject(t, source, target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-minimum-position-"+target.name+".ll", "packed-minimum-position-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedUnsignedWordMinimumPositionRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PHMINPOSUW.Z X0, X1", + "PHMINPOSUW X0", + "PHMINPOSUW M0, M1", + "PHMINPOSUW Y0, Y1", + "PHMINPOSUW X0, (AX)", + "VPHMINPOSUW.Z X0, X1", + "VPHMINPOSUW X0", + "VPHMINPOSUW X0, X1, X2", + "VPHMINPOSUW Y0, Y1", + "VPHMINPOSUW Z0, Z1", + "VPHMINPOSUW X0, K1, X1", + "VPHMINPOSUW X0, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedUnsignedWordMinimumPositionRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PHMINPOSUW X8, X0", + "PHMINPOSUW X0, X8", + "VPHMINPOSUW X16, X0", + "VPHMINPOSUW X0, X16", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedUnsignedWordMinimumPositionRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedUnsignedWordMinimumPositionRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's PHMINPOSUW forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_mov.go b/amd64_lower_mov.go index 82de66ebc..7813ff2bd 100644 --- a/amd64_lower_mov.go +++ b/amd64_lower_mov.go @@ -6,8 +6,25 @@ import ( ) func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err error) { + if ok, terminated, err := c.lowerMOVNTI(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerMOVBE(op, ins); ok { + return ok, terminated, err + } + if form, known, err := x86ScalarMoveAliasProperties(op, ins, c.goarch); known { + if err != nil { + return true, false, err + } + if !form.spec.load { + op = form.spec.canonical // Local lowering only; the original source remains intact. + } + } switch op { - case "MOVQ", "MOVD", "MOVL", "MOVLQZX", "MOVLQSX", "MOVBQZX", "MOVBLZX", "MOVB", "MOVW", "MOVWLZX", "MOVWQZX", "MOVWQSX", "CMOVQLT": + case "MOVQ", "MOVD", "MOVDQ2Q", "MOVL", "MOVB", "MOVW", "CMOVQLT", + "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", "MOVBQSX", "MOVBQZX", + "MOVWLSX", "MOVWLZX", "MOVWQSX", "MOVWQZX", "MOVLQSX", "MOVLQZX", + "MOVSWW", "MOVZWW": // ok default: return false, false, nil @@ -15,71 +32,55 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err if len(ins.Args) != 2 { return true, false, fmt.Errorf("amd64 %s expects 2 operands: %q", op, ins.Raw) } - src, dst := ins.Args[0], ins.Args[1] - zeroExtendSmallDst := false - if op == "MOVLQZX" { - op = "MOVL" + if view, known, err := x86NamedFrameMoveAddressInstruction(ins, c.goarch); known { + if err != nil { + return true, false, err + } + dispatch := ins + dispatch.Op = op + if x86SourceMoveInstruction(dispatch, c.goarch).Op != x86SourceMoveInstruction(ins, c.goarch).Op { + return true, false, fmt.Errorf("named frame MOV address dispatch disagrees with original instruction") + } + if view.Args[0].Kind == OpFP { + if err := c.scalarFPResultInitialized(); err != nil { + return true, false, err + } + } + // Reuse LEA's actual bound address and W/L/Q writeback. The source, + // frame proof and stored original instruction remain unchanged. + return c.lowerArith(view.Op, view) } - if op == "MOVD" { - op = "MOVQ" + if known, err := c.lowerGoReadOnlyLoad(ins); known { + return true, false, err } - if op == "MOVBQZX" || op == "MOVBLZX" { - op = "MOVB" - zeroExtendSmallDst = true + if _, ok := amd64ScalarExtensionMoveSpecs[op]; ok { + return true, false, c.lowerScalarExtensionMove(op, ins) } - if op == "MOVWQZX" || op == "MOVWLZX" { - op = "MOVW" - zeroExtendSmallDst = true + if known, err := c.lowerDataBPScalarMove(ins); known { + return true, false, err } - if op == "MOVLQSX" || op == "MOVWQSX" { - // Sign-extend i32/i16 source to i64 destination register. - if dst.Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects dst reg: %q", op, ins.Raw) + if form, known, err := x86ScalarStaticStoreProperties(op, ins, c.goarch); known { + if err != nil { + return true, false, err } - srcTy := I32 - if op == "MOVWQSX" { - srcTy = I16 + return true, false, c.lowerScalarNativeStore(form, ins) + } + if form, known, err := c.scalarNativeLoadProperties(op, ins); known { + if err != nil { + return true, false, err } - var sv string - switch src.Kind { - case OpImm: - switch srcTy { - case I16: - sv = fmt.Sprintf("%d", int16(src.Imm)) - default: - sv = fmt.Sprintf("%d", int32(src.Imm)) - } - case OpReg, OpFP: - v64, err := c.evalI64(src) - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", tr, v64, srcTy) - sv = "%" + tr - case OpMem: - p, ptrType, err := c.ptrFromMem(src.Mem) - if err != nil { - return true, false, err - } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load %s, %s %s, align 1\n", ld, srcTy, ptrType, p) - sv = "%" + ld - case OpSym: - p, err := c.ptrFromSB(src.Sym) - if err != nil { - return true, false, err - } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", ld, srcTy, p) - sv = "%" + ld - default: - return true, false, fmt.Errorf("amd64 %s unsupported src: %q", op, ins.Raw) + return true, false, c.lowerScalarNativeLoad(form, ins) + } + if form, known, err := c.scalarNativeStoreProperties(op, ins); known { + if err != nil { + return true, false, err } - se := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext %s %s to i64\n", se, srcTy, sv) - return true, false, c.storeReg(dst.Reg, "%"+se) + return true, false, c.lowerScalarNativeStore(form, ins) } + src, dst := ins.Args[0], ins.Args[1] + dispatch := ins + dispatch.Op = op + op = x86SourceMoveInstruction(dispatch, c.goarch).Op // Vector moves are handled in lowerVec. if dst.Kind == OpReg { @@ -123,6 +124,17 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err widthTy := I8 if op == "MOVW" { widthTy = I16 + } else { + // ymovb uses Yrb for register operands on both sides. The + // compatibility closure through Ymb is identical for registers, + // including Go's synthesized BP/SI/DI support on 386, but excludes + // the amd64-only explicit low-byte aliases when translating 386. + if src.Kind == OpReg && !isGoYmbRegisterForArch(src.Reg, c.goarch) { + return true, false, fmt.Errorf("amd64 MOVB source register is outside its Go 1.27 class: %q", ins.Raw) + } + if dst.Kind == OpReg && !isGoYmbRegisterForArch(dst.Reg, c.goarch) { + return true, false, fmt.Errorf("amd64 MOVB destination register is outside its Go 1.27 class: %q", ins.Raw) + } } var small string switch src.Kind { @@ -137,6 +149,13 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", tr, v64, widthTy) small = "%" + tr case OpFP: + if value, handled, err := c.loadIntegerFPBytes(src.FPOffset, widthTy); handled { + if err != nil { + return true, false, err + } + small = value + break + } v64, err := c.evalFPToI64(src.FPOffset) if err != nil { return true, false, err @@ -165,13 +184,11 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err } switch dst.Kind { case OpReg: - if zeroExtendSmallDst { - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", z, widthTy, small) - return true, false, c.storeReg(dst.Reg, "%"+z) - } return true, false, c.storeRegSized(dst.Reg, widthTy, small) case OpFP: + if handled, err := c.storeIntegerFPBytes(dst.FPOffset, widthTy, small); handled { + return true, false, err + } return true, false, c.storeFPResult(dst.FPOffset, widthTy, small) case OpMem: p, ptrType, err := c.ptrFromMem(dst.Mem) @@ -200,7 +217,13 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err case OpReg: var v string var err error - if src.Kind == OpMem && src.Mem.Segment != "" { + if src.Kind == OpFP { + var handled bool + v, handled, err = c.loadIntegerFPBytes(src.FPOffset, I64) + if !handled { + v, err = c.evalI64(src) + } + } else if src.Kind == OpMem && src.Mem.Segment != "" { p, ptrType, err2 := c.ptrFromMem(src.Mem) if err2 != nil { return true, false, err2 @@ -241,6 +264,9 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err if err != nil { return true, false, err } + if handled, err := c.storeIntegerFPBytes(dst.FPOffset, I64, v); handled { + return true, false, err + } return true, false, c.storeFPResult(dst.FPOffset, I64, v) case OpMem: // Store i64 to memory (common for writing return values via a pointer). @@ -274,19 +300,24 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err case "MOVL": // MOVL src, dst - if c.goarch != "386" && dst.Kind == OpFP { - // Preserve the established amd64 ABI result-slot lowering. - if src.Kind != OpReg { - return true, false, fmt.Errorf("amd64 MOVL expects reg, fp for stores: %q", ins.Raw) - } - v64, err := c.loadReg(src.Reg) - if err != nil { - return true, false, err - } - return true, false, c.storeFPResult(dst.FPOffset, I64, v64) + // FP result slots are memory operands in Go's ymovl table, so only a + // GP register or a 32-bit immediate can write them. Vector-register + // rows are handled by lowerVec before this scalar lowerer. + if dst.Kind == OpFP && src.Kind != OpReg && src.Kind != OpImm { + return true, false, fmt.Errorf("%s MOVL does not allow memory-to-result-memory: %q", c.goarch, ins.Raw) } - i32v, err := c.evalIntSized(src, I32) + var i32v string + var err error + if src.Kind == OpFP { + var handled bool + i32v, handled, err = c.loadIntegerFPBytes(src.FPOffset, I32) + if !handled { + i32v, err = c.evalIntSized(src, I32) + } + } else { + i32v, err = c.evalIntSized(src, I32) + } if err != nil { // Preserve MOVL's existing treatment of unresolved constants from // assembly include files. Explicit `$symbol(SB)` addresses are handled @@ -305,6 +336,9 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err case OpFP: // Store low 32 bits (common for ret slots). + if handled, err := c.storeIntegerFPBytes(dst.FPOffset, I32, i32v); handled { + return true, false, err + } return true, false, c.storeFPResult(dst.FPOffset, I32, i32v) case OpMem: p, ptrType, err := c.ptrFromMem(dst.Mem) diff --git a/amd64_lower_mov_extension.go b/amd64_lower_mov_extension.go new file mode 100644 index 000000000..5c838c601 --- /dev/null +++ b/amd64_lower_mov_extension.go @@ -0,0 +1,65 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerScalarExtensionMove(op Op, ins Instr) error { + form, recognized, err := x86ScalarExtensionProperties(op, ins, c.goarch) + if err != nil { + return err + } + if !recognized { + return fmt.Errorf("unrecognized scalar extension opcode %s: %q", op, ins.Raw) + } + spec := form.spec + sourceType := amd64IntegerTypeForBits(spec.sourceBits) + destinationType := amd64IntegerTypeForBits(spec.destinationBits) + var value string + native := c.nativeMemoryOperand(ins, 0) || + x86CallerMemoryCertified(ins, 0) || c.fixedLocalMemoryCertified(ins, 0) + if native { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return fmt.Errorf("%w: scalar extension needs its original native address size", ErrProbeNeedsContext) + } + pointer, pointerType, _, pointerErr := c.packedVectorMemoryPointer(form.source) + if pointerErr != nil { + return pointerErr + } + loaded := c.newTmp() + // Even a dead extension must read (and possibly fault on) its exact + // source width. No validity or alignment assumption follows. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", loaded, sourceType, pointerType, pointer) + value = "%" + loaded + } else { + value, err = c.evalIntSized(form.source, sourceType) + } + if err != nil { + return err + } + if spec.sourceBits < spec.destinationBits { + extended := c.newTmp() + extension := "zext" + if spec.signed { + extension = "sext" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", extended, extension, sourceType, value, destinationType) + value = "%" + extended + } + return c.storeRegSized(form.result, destinationType, value) +} + +// Only the paired volatile emitter can supply these conditional numerical +// result bytes. This creates no reusable pointee, bounds or caller proof. +func x86LoweredScalarExtensionRead(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) (bool, bool) { + form, known, err := x86ScalarExtensionProperties(ins.Op, ins, arch) + if !known || err != nil || form.source.Kind != OpMem || form.source.Mem.Sym != "" || + !x86OrdinaryNativeMemoryOperand(form.source, arch) || ins.x86NumericalRead != nil { + return false, false + } + if ins.x86Encoded || ins.x86SourceRaw != nil || ins.x86RawNearReturn != nil || + ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 || + !x86ResultReadWithNativeGPContinuation(ins, form.source, int64(form.spec.sourceBits/8), state, owned, arch) { + return false, true + } + return x86ResultStore(Operand{Kind: OpReg, Reg: form.result}, int64(form.spec.destinationBits/8), + state, owned, resultOffset, arch), true +} diff --git a/amd64_lower_movbe.go b/amd64_lower_movbe.go new file mode 100644 index 000000000..8720ba77b --- /dev/null +++ b/amd64_lower_movbe.go @@ -0,0 +1,46 @@ +package plan9asm + +import "fmt" + +// lowerMOVBE implements both rows of Go 1.27's ymovbe table: memory to +// general register and general register to memory, for W/L/Q widths. +func (c *amd64Ctx) lowerMOVBE(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86MOVBEProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + typ := amd64IntegerTypeForBits(form.bits) + value, err := c.evalSizedNativeGPSource(form.source, typ, ins.x86AddressBits) + if err != nil { + return true, false, err + } + swapped := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.bswap.%s(%s %s)\n", swapped, typ, typ, typ, value) + if !form.store { + return true, false, c.storeRegSized(form.destination.Reg, typ, "%"+swapped) + } + return true, false, c.storeMOVBEMemory(form.destination, typ, "%"+swapped) +} + +func (c *amd64Ctx) storeMOVBEMemory(destination Operand, typ LLVMType, value string) error { + switch destination.Kind { + case OpMem: + pointer, pointerType, _, err := c.packedVectorMemoryPointer(destination) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store volatile %s %s, %s %s, align 1\n", typ, value, pointerType, pointer) + return nil + case OpFP: + return c.storeFPResult(destination.FPOffset, typ, value) + case OpSym: + pointer, err := c.ptrFromSB(destination.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store volatile %s %s, ptr %s, align 1\n", typ, value, pointer) + return nil + default: + return fmt.Errorf("MOVBE destination is not memory: %s", destination.String()) + } +} diff --git a/amd64_lower_movbe_test.go b/amd64_lower_movbe_test.go new file mode 100644 index 000000000..63c59394b --- /dev/null +++ b/amd64_lower_movbe_test.go @@ -0,0 +1,141 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86MOVBECompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA movedata+0(SB)/8, $1\n") + source.WriteString("GLOBL movedata(SB), $8\n") + source.WriteString("TEXT moveforms(SB),$0-0\n") + widths := []string{"W", "WW", "L", "LL"} + if goarch == "amd64" { + widths = append(widths, "Q", "QQ") + } + for _, width := range widths { + op := "MOVBE" + width + fmt.Fprintf(&source, "\t%s 8(BX), AX\n", op) + fmt.Fprintf(&source, "\t%s AX, 16(BX)\n", op) + fmt.Fprintf(&source, "\t%s movedata(SB), CX\n", op) + fmt.Fprintf(&source, "\t%s DX, movedata(SB)\n", op) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s 24(R11), R12\n", op) + fmt.Fprintf(&source, "\t%s R13, 32(R14)\n", op) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86MOVBECompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86MOVBECompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + ir := x86UnboundVoidScalarFormatObject(t, source, "moveforms", target.goarch, target.triple) + for _, want := range []string{"@llvm.bswap.i16", "@llvm.bswap.i32", `"target-features"="+movbe"`} { + if !strings.Contains(ir, want) { + t.Fatalf("MOVBE lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "@llvm.bswap.i64") { + t.Fatalf("MOVBEQ lowering omitted i64 byte swap:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "movbe-"+target.name+".ll", "movbe-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86MOVBERejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "MOVBEW AX, BX"}, + {goarch: "amd64", instruction: "MOVBEL 0(AX), 0(BX)"}, + {goarch: "amd64", instruction: "MOVBEQ $1, 0(BX)"}, + {goarch: "amd64", instruction: "MOVBEW AL, 0(BX)"}, + {goarch: "amd64", instruction: "MOVBEL 0(AX), X0"}, + {goarch: "amd64", instruction: "MOVBEQ AX"}, + {goarch: "amd64", instruction: "MOVBEQQ AX"}, + {goarch: "amd64", instruction: "MOVBEL.Z 0(AX), BX"}, + {goarch: "386", instruction: "MOVBEQ 0(AX), BX"}, + {goarch: "386", instruction: "MOVBEQQ 0(AX), BX"}, + {goarch: "386", instruction: "MOVBEL R8, 0(AX)"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's MOVBE ymovbe table", test.instruction) + } + }) + } +} + +func TestAMD64MOVBERuntimeSemanticsWidthsDirectionsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86MOVBERepairedRuntimeSource(t) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "movesemantics": { + Name: "movesemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := x86MOVBEOriginalRuntimeDriver + compileAndRunRuntimeTestForTarget(t, llc, clang, "movbe_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_move_half_test.go b/amd64_lower_move_half_test.go new file mode 100644 index 000000000..b5aa9c312 --- /dev/null +++ b/amd64_lower_move_half_test.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86MoveHalfCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + last int + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", last: 15}, + {goarch: "386", triple: "i386-unknown-linux-gnu", last: 7}, + } { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT movehalfforms(SB),$0-0\n") + for _, op := range []string{"MOVLPS", "MOVHPS", "MOVLPD", "MOVHPD"} { + fmt.Fprintf(&source, "\t%s X1, X%d\n", op, target.last) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, target.last) + fmt.Fprintf(&source, "\t%s X%d, 16(BX)\n", op, target.last) + } + source.WriteString("\tRET\n") + + file, signatures := x86UnboundVoidFormatSource(t, source.String(), "movehalfforms", target.goarch, target.triple) + ll := x86MoveHalfFormatObjectUnit(t, file, signatures, target.goarch, target.triple) + if !strings.Contains(ll, "extractelement <2 x i64>") || !strings.Contains(ll, "insertelement <2 x i64>") { + t.Fatalf("translation omitted half-vector lane operations:\n%s", ll) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "move-half-"+target.goarch+".ll", "move-half-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86MoveHalfRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "MOVHPS Y0, X1", + "MOVLPS X0, Y1", + "MOVHPD (AX), (BX)", + "MOVLPD $1, X0", + "MOVHPS.Z X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yxmov table", instruction) + } + }) + } +} diff --git a/amd64_lower_move_packed_float.go b/amd64_lower_move_packed_float.go new file mode 100644 index 000000000..8c8c647d8 --- /dev/null +++ b/amd64_lower_move_packed_float.go @@ -0,0 +1,176 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerPackedVectorMove implements the complete Go 1.27 packed vector-move +// tables. VMOVAPD/APS/UPD/UPS share _yvmovapd, while VMOVDQA32/DQA64 and +// VMOVDQU8/DQU16/DQU32/DQU64 share _yvmovdqa32. All have bit-preserving move +// semantics; the element width only controls the granularity of an EVEX mask. +func (c *amd64Ctx) lowerPackedVectorMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := amd64PackedVectorMoveSpecs[Op(base)]; !recognized { + return false, false, nil + } + // Legacy helper calls may omit ins.Op. Never replace a conflicting source + // identity when binding the dispatch argument for the shared grammar. + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed move dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := amd64PackedVectorMoveProperties(ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + if form.undefinedOpcode() { + // Invalid encoding faults before any mask, vector or address access. + // An ordinary masked store would silently change the native semantics. + return c.lowerSyscall("UD2", Instr{Op: "UD2"}) + } + src, dst := ins.Args[0], ins.Args[form.destination] + byteWidth, elemBits := form.byteWidth, form.spec.elementBits + if form.spec.aligned && form.memory >= 0 { + if !c.nativeMemoryOperand(ins, form.memory) && + !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + return true, false, fmt.Errorf("%w: %s requires original native memory backing and alignment effects", ErrProbeNeedsContext, op) + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return true, false, fmt.Errorf("%w: aligned packed move needs its original native address size", ErrProbeNeedsContext) + } + return true, false, c.lowerAlignedPackedMoveMemory(ins, form) + } + for _, operand := range []Operand{src, dst} { + if operand.Kind != OpFP { + continue + } + if err := c.validateFPVectorSpan(operand.FPOffset, byteWidth); err != nil { + return true, false, fmt.Errorf("%s %s: %w: %q", c.goarch, op, err, ins.Raw) + } + } + if !form.spec.aligned && form.memory >= 0 && + (c.nativeMemoryOperand(ins, form.memory) || x86CallerMemoryCertified(ins, form.memory) || c.fixedLocalMemoryCertified(ins, form.memory)) { + return true, false, c.lowerUnalignedPackedMoveMemory(ins, form) + } + + if form.mask < 0 { + value, err := c.loadPackedCompareBytes(src, byteWidth) + if err != nil { + return true, false, err + } + return true, false, c.storePackedMoveOperand(dst, byteWidth, value) + } + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + // FP operands describe checked virtual slots, not necessarily contiguous + // native memory: each parameter/result can have its own alloca. Retain the + // per-slot loads, mask selection and stores below for those operands. + if (isAMD64MemoryOperand(src) && src.Kind != OpFP) || (isAMD64MemoryOperand(dst) && dst.Kind != OpFP) { + return true, false, c.lowerMaskedPackedMoveMemory(src, dst, byteWidth, elemBits, mask, form.zeroing) + } + value, err := c.loadPackedCompareBytes(src, byteWidth) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(dst, byteWidth) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / elemBits + computed := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, elemBits, value) + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, elemBits, oldBytes) + // The invalid .Z memory-destination encoding was handled before evaluating + // any operand. Valid zeroing forms always have a register destination. + result := amd64ApplyIntegerLaneMask(c, lanes, elemBits, computed, old, mask, form.zeroing && dst.Kind == OpReg) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, elemBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(dst, byteWidth, "%"+out) +} + +// VEX/EVEX register writes replace every overlapping view and zero all bits +// above the written width. Keep this separate from legacy SSE stores, which +// preserve those upper bits. +func (c *amd64Ctx) storePackedMoveOperand(dst Operand, byteWidth int, value string) error { + if dst.Kind != OpReg { + return c.storeVectorBytesOperand(dst, byteWidth, value) + } + index, valid := amd64VectorRegisterIndex(dst.Reg, byteWidth) + if !valid { + return fmt.Errorf("invalid packed move register %s", dst.Reg) + } + for _, view := range []struct { + bytes int + slots map[int]string + }{{16, c.xRegSlot}, {32, c.yRegSlot}, {64, c.zRegSlot}} { + slot, live := view.slots[index] + if !live { + continue + } + stored := value + if view.bytes != byteWidth { + indices := make([]string, view.bytes) + for i := range indices { + lane := i + if lane >= byteWidth { + lane = byteWidth // First byte of the zero second source. + } + indices[i] = fmt.Sprintf("i32 %d", lane) + } + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> <%s>\n", converted, byteWidth, value, byteWidth, view.bytes, strings.Join(indices, ", ")) + stored = "%" + converted + } + fmt.Fprintf(c.b, " store <%d x i8> %s, ptr %s\n", view.bytes, stored, slot) + } + return nil +} + +// Other EVEX families use this architecture/register axis without acquiring +// packed-move effects or caller-memory permissions. +func (c *amd64Ctx) isGoPackedVectorMoveRegister(arg Operand, byteWidth int) bool { + return amd64GoPackedVectorMoveRegister(arg, byteWidth, c.goarch) +} + +// Share the architectural lane mask across floating and integer moves. A +// select after an eager full-width load/store cannot suppress memory faults +// and must not be used for masked memory destinations (including write-only +// memory). LLVM 22's masked intrinsics access only enabled lanes. +func (c *amd64Ctx) lowerMaskedPackedMoveMemory(src, dst Operand, byteWidth, elemBits int, mask string, zeroing bool) error { + load := isAMD64MemoryOperand(src) + memory := dst + if load { + memory = src + } + pointer, pointerType, suffix, err := c.packedVectorMemoryPointer(memory) + if err != nil { + return err + } + if suffix == "" { + suffix = ".p0" + } + lanes := byteWidth * 8 / elemBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, elemBits) + predicate := amd64IntegerMaskVector(c, lanes, mask) + if load { + fallback := "zeroinitializer" + if !zeroing { + fallback, err = c.loadPackedCompareLanes(dst, byteWidth, elemBits, false) + if err != nil { + return err + } + } + loaded, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.masked.load.v%di%d%s(%s align 1 %s, <%d x i1> %s, %s %s)\n", loaded, vectorType, lanes, elemBits, suffix, pointerType, pointer, lanes, predicate, vectorType, fallback) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, vectorType, loaded, byteWidth) + return c.storePackedMoveOperand(dst, byteWidth, "%"+bytes) + } + value, err := c.loadPackedCompareLanes(src, byteWidth, elemBits, false) + if err != nil { + return err + } + fmt.Fprintf(c.b, " call void @llvm.masked.store.v%di%d%s(%s %s, %s align 1 %s, <%d x i1> %s)\n", lanes, elemBits, suffix, vectorType, value, pointerType, pointer, lanes, predicate) + return nil +} diff --git a/amd64_lower_move_sign_extend.go b/amd64_lower_move_sign_extend.go new file mode 100644 index 000000000..b11d1f79a --- /dev/null +++ b/amd64_lower_move_sign_extend.go @@ -0,0 +1,86 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerPackedExtendMove implements all six signed and all six unsigned legacy +// and vector widening instructions exposed by Go 1.27. Ratio-two V forms use +// _yvcvtdq2pd; ratio-four and ratio-eight forms use _yvbroadcastss. +func (c *amd64Ctx) lowerPackedExtendMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + _, zero := x86PackedZeroExtendSpecs[Op(base)] + _, signed := x86PackedSignExtendSpecs[Op(base)] + if !zero && !signed { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("packed widening dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op // Keep the original helper's empty-op dispatch contract. + form, known, err := x86PackedExtendProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return c.emitPackedExtendMove(ins, form) +} + +func (c *amd64Ctx) loadPackedExtendInputs(src Operand, registerBytes, lanes, inputBits int) (string, error) { + if src.Kind == OpReg { + bytesValue, err := c.loadPackedCompareBytes(src, registerBytes) + if err != nil { + return "", err + } + fullLanes := registerBytes * 8 / inputBits + full := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i%d>\n", full, registerBytes, bytesValue, fullLanes, inputBits) + if fullLanes == lanes { + return "%" + full, nil + } + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %%%s, i32 %d\n", value, fullLanes, inputBits, full, lane) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, inputBits, result, inputBits, value, lane) + result = "%" + inserted + } + return result, nil + } + vectorType := fmt.Sprintf("<%d x i%d>", lanes, inputBits) + switch src.Kind { + case OpMem: + ptr, ptrType, err := c.ptrFromMem(src.Mem) + if err != nil { + return "", err + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, %s %s, align 1\n", loaded, vectorType, ptrType, ptr) + return "%" + loaded, nil + case OpSym: + ptr, err := c.ptrFromSB(src.Sym) + if err != nil { + return "", err + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", loaded, vectorType, ptr) + return "%" + loaded, nil + case OpFP: + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + laneOperand := src + laneOperand.FPOffset += int64(lane * inputBits / 8) + value, err := c.evalIntSized(laneOperand, amd64IntegerTypeForBits(inputBits)) + if err != nil { + return "", err + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, result, inputBits, value, lane) + result = "%" + inserted + } + return result, nil + default: + return "", fmt.Errorf("expected packed extension register or memory source, got %s", src.String()) + } +} diff --git a/amd64_lower_move_zero_extend.go b/amd64_lower_move_zero_extend.go new file mode 100644 index 000000000..6c3f62aba --- /dev/null +++ b/amd64_lower_move_zero_extend.go @@ -0,0 +1,82 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPackedZeroExtendMove(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86PackedZeroExtendSpecs[Op(base)]; !known { + return false, false, nil + } + return c.lowerPackedExtendMove(op, ins) +} + +func (c *amd64Ctx) emitPackedExtendMove(ins Instr, form x86PackedExtendForm) (bool, bool, error) { + var err error + source, destination := ins.Args[0], ins.Args[form.destination] + lanes := form.byteWidth * 8 / form.spec.outputBits + if source.Kind == OpFP { + if err := c.validateFPVectorSpan(source.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + var mask, inputs string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + switch { + case source.Kind == OpFP: + // Independent ABI0 fields are not a contiguous native vector. Read + // each declared byte and only then assemble the integer source lanes. + var bytes string + bytes, err = c.loadFPVectorBytes(source.FPOffset, form.memoryBytes) + if err == nil { + inputs = c.bitcastVectorBytesToIntegerLanes(form.memoryBytes, lanes, form.spec.inputBits, bytes) + } + case form.memory >= 0: + // E5 inactive lanes suppress reads; active accesses remain observable + // even if the widened result is dead. Read the shrinking source tuple, + // not its larger physical vector container or widened destination. + inputs, err = c.loadX86PackedShiftLanes(source, form.memoryBytes, form.spec.inputBits, false, mask) + default: + inputs, err = c.loadPackedExtendInputs(source, form.sourceRegisterBytes, lanes, form.spec.inputBits) + } + if err != nil { + return true, false, err + } + result := "zeroinitializer" + extension := "zext" + if form.signed { + extension = "sext" + } + for lane := 0; lane < lanes; lane++ { + input, extended, inserted := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", input, lanes, form.spec.inputBits, inputs, lane) + fmt.Fprintf(c.b, " %%%s = %s i%d %%%s to i%d\n", extended, extension, form.spec.inputBits, input, form.spec.outputBits) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, form.spec.outputBits, result, form.spec.outputBits, extended, lane) + result = "%" + inserted + } + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + var bytes string + bytes, err = c.loadPackedCompareBytes(destination, form.byteWidth) + if err != nil { + return true, false, err + } + old = c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.spec.outputBits, bytes) + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.spec.outputBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.spec.outputBits, result, form.byteWidth) + if !form.spec.vector { + return true, false, c.storeLegacyXViews(destination.Reg, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} diff --git a/amd64_lower_move_zero_extend_test.go b/amd64_lower_move_zero_extend_test.go new file mode 100644 index 000000000..c9b64261d --- /dev/null +++ b/amd64_lower_move_zero_extend_test.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86PackedZeroExtendMoveCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedzeroextendmoveforms(SB),4,$0-0\n") + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, row := range x86ZeroExtendAcceptedForms(prefix+"MOVZX"+ratio.suffix, target.goarch) { + source.WriteString(row + "\n") + } + } + } + source.WriteString("\tRET\n") + // This unchanged format directory has no address, data or mask + // producers. Its Go object and public Context checks remain + // separate from the initialized three-API object matrices. + ll := x86UnboundVoidScalarFormatObject(t, source.String(), "packedzeroextendmoveforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-zero-extend-move-"+target.name+".ll", "packed-zero-extend-move-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedZeroExtendMoveRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMOVZXBW.Z X0, X1", + "PMOVZXBD X0", + "PMOVZXBQ M0, X1", + "PMOVZXWD Y0, X1", + "PMOVZXWQ X0, Y1", + "PMOVZXDQ X0, K1, X1", + "VPMOVZXBW Y0, Y1", + "VPMOVZXWD X0, Z1", + "VPMOVZXDQ Z0, Z1", + "VPMOVZXBD Y0, Z1", + "VPMOVZXBQ X0, K0, X1", + "VPMOVZXWQ.Z X0, X1", + "VPMOVZXBD.BCST 8(AX), X1", + "VPMOVZXWQ X0, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedZeroExtendMoveRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PMOVZXBW X8, X0", + "PMOVZXDQ X0, X8", + "VPMOVZXBW Y20, Z8", + "VPMOVZXBQ X20, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedZeroExtendMoveRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedZeroExtendMoveRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed zero-extension forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_movemask.go b/amd64_lower_movemask.go new file mode 100644 index 000000000..b8bb45eeb --- /dev/null +++ b/amd64_lower_movemask.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64MoveMaskSpec struct { + opcode int + prefix int + vector bool + floatingLaneBits int +} + +// The legacy and VEX spellings share the Go table's register-to-GP shape. +// The raw decoder uses the VEX entries; x/arch handles legacy raw bytes. +var amd64MoveMaskSpecs = map[Op]amd64MoveMaskSpec{ + "PMOVMSKB": {opcode: 0xd7, prefix: 1}, + "VPMOVMSKB": {opcode: 0xd7, prefix: 1, vector: true}, + "MOVMSKPS": {opcode: 0x50, floatingLaneBits: 32}, + "MOVMSKPD": {opcode: 0x50, prefix: 1, floatingLaneBits: 64}, + "VMOVMSKPS": {opcode: 0x50, vector: true, floatingLaneBits: 32}, + "VMOVMSKPD": {opcode: 0x50, prefix: 1, vector: true, floatingLaneBits: 64}, +} + +type x86MoveMaskForm struct { + amd64MoveMaskSpec + byteWidth int +} + +// Shape is shared by lowering and result definedness. Acceptance supplies no +// initialized vector, MMX state, physical mode32 bank or caller permission. +func x86MoveMaskProperties(op Op, ins Instr, arch string) (x86MoveMaskForm, bool, error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64MoveMaskSpecs[Op(baseOp)] + form := x86MoveMaskForm{amd64MoveMaskSpec: spec} + if !recognized { + return form, false, nil + } + if arch != "amd64" && arch != "386" || rawOp != baseOp || + ins.Op != "" && !strings.EqualFold(string(ins.Op), baseOp) { + return form, true, fmt.Errorf("%s %s has an unbound mode, dispatch or suffix: %q", arch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { + return form, true, fmt.Errorf("%s %s expects vector-or-MMX source and GP destination: %q", arch, baseOp, ins.Raw) + } + if !isX86YrlRegisterForArch(ins.Args[1].Reg, arch) { + return form, true, fmt.Errorf("%s %s destination is outside Go's Yrl class: %q", arch, baseOp, ins.Raw) + } + limit := 15 + if !spec.vector && arch == "386" { + limit = 7 + } + if index, valid := amd64ParseXReg(ins.Args[0].Reg); valid && index <= limit { + form.byteWidth = 16 + } else if index, valid := amd64ParseYReg(ins.Args[0].Reg); spec.vector && valid && index <= 15 { + form.byteWidth = 32 + } else if _, valid := amd64ParseMReg(ins.Args[0].Reg); !spec.vector && spec.floatingLaneBits == 0 && valid { + form.byteWidth = 8 + } else { + return form, true, fmt.Errorf("%s %s source is outside its Go table: %q", arch, baseOp, ins.Raw) + } + return form, true, nil +} + +// No Go move-mask row accepts EVEX, mask-register, suffix or memory forms. +func (c *amd64Ctx) lowerPackedMoveMask(op Op, ins Instr) (ok bool, terminated bool, err error) { + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86MoveMaskProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + if form.floatingLaneBits != 0 { + value, err := c.loadPackedCompareBytes(ins.Args[0], form.byteWidth) + if err != nil { + return true, false, err + } + mask32 := c.packedSignMask(value, form.byteWidth, form.floatingLaneBits) + return true, false, c.storeRegSized(ins.Args[1].Reg, I32, mask32) + } + if form.byteWidth != 8 && c.targetTriple != "" && !x86ScalarFloatIntegerNativeTarget(arch, c.targetTriple) { + value, err := c.loadPackedCompareBytes(ins.Args[0], form.byteWidth) + if err != nil { + return true, false, err + } + return true, false, c.storeRegSized(ins.Args[1].Reg, I32, c.packedSignMask(value, form.byteWidth, 8)) + } + + var mask32 string + if form.byteWidth == 16 { + value, err := c.loadX(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + mask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %s)\n", mask, value) + mask32 = "%" + mask + } else if form.byteWidth == 32 { + value, err := c.loadY(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", low, value, llvmI32RangeMask(0, 16)) + high := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", high, value, llvmI32RangeMask(16, 16)) + lowMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %%%s)\n", lowMask, low) + highMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %%%s)\n", highMask, high) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, 16\n", shifted, highMask) + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i32 %%%s, %%%s\n", combined, shifted, lowMask) + mask32 = "%" + combined + } else { + value, err := c.loadReg(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to <8 x i8>\n", bytes, value) + signBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <8 x i8> %%%s, zeroinitializer\n", signBits, bytes) + mask8 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i1> %%%s to i8\n", mask8, signBits) + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i32\n", widened, mask8) + mask32 = "%" + widened + } + + return true, false, c.storeRegSized(ins.Args[1].Reg, I32, mask32) +} + +// Lane i writes mask bit i explicitly. Byte lanes need no vector-to-integer +// bitcast, so this portable path does not depend on the target's byte order. +func (c *amd64Ctx) packedSignMask(value string, byteWidth, laneBits int) string { + lanes := byteWidth * 8 / laneBits + integerLanes := value + if laneBits != 8 { + integerLanes = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, value) + } + mask := "0" + for lane := 0; lane < lanes; lane++ { + laneValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", laneValue, lanes, laneBits, integerLanes, lane) + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i%d %%%s, 0\n", sign, laneBits, laneValue) + bit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %%%s to i32\n", bit, sign) + positioned := "%" + bit + if lane != 0 { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, %d\n", shifted, bit, lane) + positioned = "%" + shifted + } + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i32 %s, %s\n", combined, mask, positioned) + mask = "%" + combined + } + return mask +} + +func isX86YrlRegisterForArch(r Reg, goarch string) bool { + if !isAMD64YrlRegister(r) { + return false + } + if goarch != "386" { + return true + } + switch r { + case AX, BX, CX, DX, SP, BP, SI, DI: + return true + default: + return false + } +} diff --git a/amd64_lower_movhlps.go b/amd64_lower_movhlps.go new file mode 100644 index 000000000..ead29b338 --- /dev/null +++ b/amd64_lower_movhlps.go @@ -0,0 +1,41 @@ +package plan9asm + +import "fmt" + +// lowerMoveHighLowPackedSingle implements all forms in Go 1.27's legacy yxr +// and AVX _yvmovhlps tables for the HLPS/LHPS register-only move family. +func (c *amd64Ctx) lowerMoveHighLowPackedSingle(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86MoveHalfProperties(op, ins, c.goarch) + if !known || !form.spec.registerOnly { + return false, false, nil + } + if err != nil { + return true, false, err + } + first, err := c.loadXAsI64x2(ins.Args[form.source].Reg) + if err != nil { + return true, false, err + } + second, err := c.loadXAsI64x2(ins.Args[form.base].Reg) + if err != nil { + return true, false, err + } + mask := "" + if form.outputLane == 0 { + mask = "" + } + if !form.spec.vector { + first, second = second, first + mask = "" + if form.outputLane == 0 { + mask = "" + } + } + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <2 x i32> %s\n", result, first, second, mask) + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bytes, result) + if form.spec.vector { + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], 16, "%"+bytes) + } + return true, false, c.storeLegacyXViews(ins.Args[form.destination].Reg, "%"+bytes) +} diff --git a/amd64_lower_movnti.go b/amd64_lower_movnti.go new file mode 100644 index 000000000..288142fb5 --- /dev/null +++ b/amd64_lower_movnti.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +const x86NonTemporalMetadata = ", !nontemporal !0" + +func (c *amd64Ctx) lowerMOVNTI(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86MOVNTIProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if form.trap { + c.b.WriteString(" call void asm sideeffect \"ud2\", \"~{memory}\"()\n") + c.b.WriteString(" unreachable\n") + return true, true, nil + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || + !x86ResultNamedPredicateInstruction(ins) || ins.x86NumericalRead != nil { + return true, false, fmt.Errorf("%w: MOVNTI needs its original named x86 target and address size", ErrProbeNeedsContext) + } + if !c.nativeMemoryOperand(ins, 1) { + return true, false, fmt.Errorf("%w: MOVNTI needs native storage, not logical FP/AUTO bytes", ErrProbeNeedsContext) + } + return true, false, c.emitMOVNTI(form) +} + +// Intel SDM MOVNTI retains its actual WC/cache policy, dead write and faults. +// LLVM nontemporal metadata on an ordinary store makes none of those effects +// mandatory. Pass only the exact integer and opaque address to the native +// opcode, without asserting alignment, extent, aliasing or pointee validity. +func (c *amd64Ctx) emitMOVNTI(form x86MOVNTIForm) error { + typ := amd64IntegerTypeForBits(form.bits) + value, err := c.evalIntSized(form.source, typ) + if err != nil { + return err + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.destination) + if err != nil { + return err + } + segment, modifier := "", "k" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + if form.bits == 64 { + modifier = "q" + } + assembly := "movnti ${0:" + modifier + "}, " + segment + "($1)" + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s, %s %s)\n", assembly, + "r,r,~{memory}", typ, value, pointerType, pointer) + return nil +} + +func x86MemoryRegistersValidForArch(memory MemRef, goarch string) bool { + for _, register := range []Reg{memory.Base, memory.Index} { + if register != "" && !isX86YrlRegisterForArch(register, goarch) { + return false + } + } + return true +} + +func (c *amd64Ctx) storeMOVNTIMemory(destination Operand, typ LLVMType, value string) error { + switch destination.Kind { + case OpMem: + pointer, pointerType, err := c.ptrFromMem(destination.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, %s %s, align 1%s\n", typ, value, pointerType, pointer, x86NonTemporalMetadata) + return nil + case OpFP: + return c.storeFPResultWithMetadata(destination.FPOffset, typ, value, x86NonTemporalMetadata) + case OpSym: + pointer, err := c.ptrFromSB(destination.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1%s\n", typ, value, pointer, x86NonTemporalMetadata) + return nil + default: + return fmt.Errorf("MOVNTI destination is not memory: %s", destination.String()) + } +} + +func fileUsesX86NonTemporalMetadata(file *File) bool { + if file == nil || file.Arch != ArchAMD64 { + return false + } + for _, function := range file.Funcs { + for _, instruction := range function.Instrs { + op := strings.ToUpper(string(instruction.Op)) + switch op { + case "MOVNTQ", "MOVNTDQ", "MOVNTO", "MOVNTPD", "MOVNTPS", "VMOVNTDQ", "VMOVNTPD", "VMOVNTPS": + return true + } + } + } + return false +} diff --git a/amd64_lower_movnti_test.go b/amd64_lower_movnti_test.go new file mode 100644 index 000000000..55f707bb7 --- /dev/null +++ b/amd64_lower_movnti_test.go @@ -0,0 +1,242 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os/exec" + "runtime" + "strings" + "testing" +) + +func x86MOVNTICompleteFormsSource(goarch string) (string, map[string]FuncSig) { + var source strings.Builder + source.WriteString("DATA movntidata+0(SB)/8, $1\n") + source.WriteString("GLOBL movntidata(SB), $8\n") + source.WriteString("TEXT movntiforms(SB),$0-0\n") + source.WriteString("\tMOVNTIL AX, 8(BX)\n") + source.WriteString("\tMOVNTIL CX, movntidata(SB)\n") + if goarch == "amd64" { + source.WriteString("\tMOVNTIL R11, 16(R12)\n") + source.WriteString("\tMOVNTIQ AX, 24(BX)\n") + source.WriteString("\tMOVNTIQ CX, movntidata(SB)\n") + source.WriteString("\tMOVNTIQ R13, 32(R14)\n") + } + source.WriteString("\tRET\n") + + source.WriteString("TEXT movntiframe(SB),$0-4\n") + source.WriteString("\tMOVNTIL AX, ret+0(FP)\n") + source.WriteString("\tRET\n") + if goarch == "amd64" { + source.WriteString("TEXT movntiframeq(SB),$0-8\n") + source.WriteString("\tMOVNTIQ AX, ret+0(FP)\n") + source.WriteString("\tRET\n") + } + + // Yml's Go compatibility closure includes registers even though the ISA + // defines only memory destinations for MOVNTI. Keep this Go-accepted source + // form visible and require the translated function to trap explicitly. + source.WriteString("TEXT movntiregtrap(SB),$0-0\n") + source.WriteString("\tMOVNTIL AX, BX\n") + source.WriteString("\tRET\n") + + sigs := map[string]FuncSig{ + "movntiforms": {Name: "movntiforms", Ret: Void}, + "movntiframe": {Name: "movntiframe", Ret: I32, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}}}}, + "movntiregtrap": {Name: "movntiregtrap", Ret: Void}, + } + if goarch == "amd64" { + sigs["movntiframeq"] = FuncSig{Name: "movntiframeq", Ret: I64, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}}} + } + return source.String(), sigs +} + +func TestTranslateX86MOVNTICompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source, sigs := x86MOVNTICompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original unbound MOVNTI format source invented numerical/native inputs", err) + } + // Preserve every original source and Go object. This private + // emitter unit measures the accepted encoding rows, not a public + // native FP frame or initialized result. The register-only row is + // independently translated as its actual terminal #UD below. + formats := *file + formats.Funcs = append([]Func(nil), file.Funcs[:len(file.Funcs)-1]...) + ir := x86InstructionFormatObjectUnit(t, &formats, sigs, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + form, known, err := x86MOVNTIProperties(ins.Op, ins, target.goarch) + if !known || err != nil || form.trap { + return known, false, err + } + return true, false, c.emitMOVNTI(form) + }) + for _, want := range []string{`asm sideeffect "movnti ${0:k}`, `~{memory}`, `"target-features"="+sse2"`} { + if !strings.Contains(ir, want) { + t.Fatalf("MOVNTI lowering omitted %q:\n%s", want, ir) + } + } + if strings.Contains(ir, "!nontemporal") || target.goarch == "amd64" && !strings.Contains(ir, `asm sideeffect "movnti ${0:q}`) { + t.Fatalf("MOVNTI lost its mandatory original-width native store:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "movnti-"+target.name+".ll", "movnti-"+target.name+".o", ir) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, llc, "-mtriple="+target.triple, "-filetype=asm", "-o=-", "-") + command.Stdin = strings.NewReader(ir) + assembly, err := runtimeOracleCombinedOutput(ctx, command) + if err != nil { + t.Fatalf("llc MOVNTI assembly failed: %v\n%s", err, assembly) + } + if !strings.Contains(strings.ToLower(string(assembly)), "movnti") { + t.Fatalf("LLVM 22 did not preserve the non-temporal integer store:\n%s", assembly) + } + trap, err := Parse(ArchAMD64, "TEXT movntiregtrap(SB),$0-0\nMOVNTIL AX,BX\nRET\n") + if err != nil { + t.Fatal(err) + } + trapIR, err := Translate(trap, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil || !strings.Contains(trapIR, `asm sideeffect "ud2"`) || !strings.Contains(trapIR, "unreachable") { + t.Fatal("original Go register row lost its input-independent #UD", err) + } + compileLLVMToObject(t, llc, target.triple, "movnti-trap.ll", "movnti-trap.o", trapIR) + }) + } +} + +func TestTranslateX86MOVNTIRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "MOVNTIL 0(AX), BX"}, + {goarch: "amd64", instruction: "MOVNTIL 0(AX), 8(BX)"}, + {goarch: "amd64", instruction: "MOVNTIL $1, 8(BX)"}, + {goarch: "amd64", instruction: "MOVNTIL X0, 8(BX)"}, + {goarch: "amd64", instruction: "MOVNTIL AX"}, + {goarch: "amd64", instruction: "MOVNTIL AX, 0(BX), CX"}, + {goarch: "amd64", instruction: "MOVNTIL.Z AX, 8(BX)"}, + {goarch: "386", instruction: "MOVNTIQ AX, 8(BX)"}, + {goarch: "386", instruction: "MOVNTIL R8, 8(BX)"}, + {goarch: "386", instruction: "MOVNTIL AX, 8(R9)"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's MOVNTI yrl_ml table", test.instruction) + } + }) + } +} + +func TestAMD64MOVNTIRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT movntisemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVQ $0x1122334455667788, AX + MOVQ $0x8877665544332211, BX + MOVNTIL AX, 0(DI) + MOVNTIQ BX, 8(DI) + SFENCE + SETCS 16(DI) + SETOS 17(DI) + SETEQ 18(DI) + SETMI 19(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "movntisemantics": { + Name: "movntisemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void movntisemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[24] = {0}; + movntisemantics(out); + if (load64(out) != UINT64_C(0x0000000055667788)) return 10; + if (load64(out+8) != UINT64_C(0x8877665544332211)) return 11; + if (out[16] != 1 || out[17] != 1 || out[18] != 0 || out[19] != 1) return 12; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "movnti_semantics", triple, ir, mainC, runPrefix) +} + +func TestX86MOVNTIGoAssemblerCompatibilityClosure(t *testing.T) { + for _, goarch := range []string{"amd64", "386"} { + for _, instruction := range []string{"MOVNTIL AX, 8(BX)", "MOVNTIL AX, BX", "MOVNTIL AX, ret+0(FP)"} { + t.Run(fmt.Sprintf("%s/%s", goarch, strings.NewReplacer(" ", "_", ",", "").Replace(instruction)), func(t *testing.T) { + requireX86GoAssemblerResult(t, goarch, "TEXT compat(SB),$0-8\n\t"+instruction+"\n\tRET\n", true) + }) + } + } +} diff --git a/amd64_lower_mpsadbw_test.go b/amd64_lower_mpsadbw_test.go new file mode 100644 index 000000000..a68f806d7 --- /dev/null +++ b/amd64_lower_mpsadbw_test.go @@ -0,0 +1,356 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedSADGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64PackedSADSpec{ + "PSADBW": {kind: amd64PackedSADBlockSum, form: amd64PackedSADLegacyX}, + "VPSADBW": {kind: amd64PackedSADBlockSum, form: amd64PackedSADEVEXXYZ}, + "MPSADBW": {kind: amd64PackedSADMultiple, form: amd64PackedSADLegacyX}, + "VMPSADBW": {kind: amd64PackedSADMultiple, form: amd64PackedSADVEXXY}, + "VDBPSADBW": {kind: amd64PackedSADDoubleBlock, form: amd64PackedSADEVEXXYZ}, + } + if len(amd64PackedSADSpecs) != len(expected) { + t.Fatalf("packed SAD grammar has %d entries, want %d", len(amd64PackedSADSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64PackedSADSpecs[op]; !ok { + t.Errorf("packed SAD grammar omitted %s", op) + } else if got != want { + t.Errorf("packed SAD grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86MultipleSADCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + source.WriteString("TEXT mpsadforms(SB),$0-0\n") + fmt.Fprintf(&source, "\tMPSADBW $7, X0, X%d\n", last) + fmt.Fprintf(&source, "\tMPSADBW $7, 0(AX), X%d\n", last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\tVMPSADBW $7, X0, X1, X%d\n", last) + fmt.Fprintf(&source, "\tVMPSADBW $7, 16(AX), X1, X%d\n", last) + fmt.Fprintf(&source, "\tVMPSADBW $7, Y0, Y1, Y%d\n", last) + fmt.Fprintf(&source, "\tVMPSADBW $7, 32(AX), Y1, Y%d\n", last) + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "mpsadforms", target.goarch, target.triple) + if !strings.Contains(ir, "call <8 x i16> @llvm.x86.sse41.mpsadbw") { + t.Fatalf("MPSADBW lowering omitted the SSE4.1 intrinsic:\n%s", ir) + } + if target.goarch == "amd64" && !strings.Contains(ir, "call <16 x i16> @llvm.x86.avx2.mpsadbw") { + t.Fatalf("VMPSADBW Y lowering omitted the AVX2 intrinsic:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "mpsadbw-"+target.name+".ll", "mpsadbw-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86DoubleSADCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 23 + var source strings.Builder + source.WriteString("TEXT dbpsadforms(SB),$0-0\n") + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\tVDBPSADBW $0, %s0, %s1, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVDBPSADBW $85, 0(AX), %s1, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVDBPSADBW $170, %s2, %s1, K1, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVDBPSADBW.Z $255, 64(AX), %s1, K2, %s%d\n", width, width, last) + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "dbpsadforms", target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "dbpsadbw-"+target.name+".ll", "dbpsadbw-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86DoubleSADRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VDBPSADBW $-1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW $256, X0, X1, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW.Z $1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW.BCST $1, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW $1, X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VDBPSADBW $1, X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW $1, X0, X1, K1, X2, X3"}, + {goarch: "386", instruction: "VDBPSADBW $1, X0, X1, X2"}, + {goarch: "386", instruction: "VDBPSADBW.Z $1, X0, X1, K1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VDBPSADBW table", test.instruction) + } + }) + } +} + +func TestTranslateX86MultipleSADRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "MPSADBW $-1, X0, X1"}, + {goarch: "amd64", instruction: "MPSADBW $256, X0, X1"}, + {goarch: "amd64", instruction: "MPSADBW.Z $1, X0, X1"}, + {goarch: "amd64", instruction: "MPSADBW $1, Y0, Y1"}, + {goarch: "amd64", instruction: "VMPSADBW $1, Z0, Z1, Z2"}, + {goarch: "amd64", instruction: "VMPSADBW $1, X0, Y1, Y2"}, + {goarch: "386", instruction: "MPSADBW $1, X0, X8"}, + {goarch: "386", instruction: "VMPSADBW $1, X0, X1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: test.goarch, TargetTriple: triple, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside the Go table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestTranslateX86PackedSADRejects386OutOfRangeForms(t *testing.T) { + for _, instruction := range []string{ + "PSADBW X0, X8", + "PSADBW 8(R9), X0", + } { + t.Run(strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside the Go 386 SAD tables", instruction) + } + }) + } +} + +func TestAMD64MultipleSADRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT mpsadsemantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ a+8(FP), CX + MOVQ b+16(FP), DX + MOVUPS 0(CX), X1 + MOVUPS 0(DX), X0 + STC + MPSADBW $0, X0, X1 + MOVUPS X1, 0(AX) + SETCS 16(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"mpsadsemantics": { + Name: "mpsadsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void mpsadsemantics(uint8_t *, const uint8_t *, const uint8_t *); +int main(void) { + uint8_t out[17] = {0}; + const uint8_t a[16] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + const uint8_t b[16] = {0}; + const uint16_t want[8] = {6, 10, 14, 18, 22, 26, 30, 34}; + uint16_t got[8]; + mpsadsemantics(out, a, b); + memcpy(got, out, 16); + for (int i = 0; i < 8; i++) if (got[i] != want[i]) return 10 + i; + if (out[16] != 1) return 20; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "multiple_sad_semantics", triple, ir, mainC, runPrefix) +} + +func TestAMD64DoubleSADRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT dbpsadsemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ a+8(FP), CX + MOVQ b+16(FP), DX + MOVQ old+24(FP), R8 + MOVQ mask+32(FP), R9 + KMOVQ R9, K1 + VMOVDQU8 0(CX), Z1 + VMOVDQU8 0(DX), Z0 + VMOVDQU16 0(R8), Z2 + STC + VDBPSADBW $27, Z0, Z1, K1, Z2 + VMOVDQU16 Z2, 0(AX) + VDBPSADBW.Z $228, Z0, Z1, K1, Z3 + VMOVDQU16 Z3, 64(AX) + SETCS 128(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"dbpsadsemantics": { + Name: "dbpsadsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void dbpsadsemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint16_t *, uint64_t); +static void oracle(uint16_t *out, const uint8_t *src1, const uint8_t *src2, const uint16_t *old, unsigned imm, uint64_t mask, int zeroing) { + uint8_t shuffled[64]; + for (int lane = 0; lane < 64; lane += 16) + for (int dword = 0; dword < 4; dword++) + for (int byte = 0; byte < 4; byte++) + shuffled[lane + dword * 4 + byte] = src2[lane + ((imm >> (2 * dword)) & 3) * 4 + byte]; + for (int block = 0; block < 64; block += 8) { + for (int word = 0; word < 4; word++) { + unsigned sum = 0; + int src1_base = block + (word >= 2 ? 4 : 0); + int shuffled_base = block + word; + for (int byte = 0; byte < 4; byte++) { + int difference = (int)src1[src1_base + byte] - (int)shuffled[shuffled_base + byte]; + sum += difference < 0 ? -difference : difference; + } + int index = block / 2 + word; + out[index] = (mask >> index) & 1 ? (uint16_t)sum : (zeroing ? 0 : old[index]); + } + } +} +int main(void) { + uint8_t a[64], b[64], bytes[129] = {0}; uint16_t old[32], got[64], want[64]; + for (int i = 0; i < 64; i++) { a[i] = (uint8_t)(i * 7 + 3); b[i] = (uint8_t)(255 - i * 5); } + for (int i = 0; i < 32; i++) old[i] = (uint16_t)(1000 + i); + uint64_t mask = UINT64_C(0xa55aa55a); + dbpsadsemantics(bytes, a, b, old, mask); + memcpy(got, bytes, 128); + oracle(want, a, b, old, 27, mask, 0); + oracle(want + 32, a, b, old, 228, mask, 1); + for (int i = 0; i < 64; i++) if (got[i] != want[i]) return 10 + i; + if (bytes[128] != 1) return 80; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "double_sad_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_msr.go b/amd64_lower_msr.go new file mode 100644 index 000000000..d19eaa063 --- /dev/null +++ b/amd64_lower_msr.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerMSRAccess implements the complete RDMSR/WRMSR instruction family. +// Go 1.27 uses the operand-free ynone optab entry for both instructions on +// amd64 and 386; the architectural operands are ECX and EDX:EAX. +func (c *amd64Ctx) lowerMSRAccess(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + switch baseOp { + case "RDMSR", "WRMSR": + // handled below + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + + cx, err := c.loadReg(CX) + if err != nil { + return true, false, err + } + cx32 := c.truncI64(cx, I32) + if baseOp == "RDMSR" { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call { i32, i32 } asm sideeffect \"rdmsr\", \"={ax},={dx},{cx},~{dirflag},~{fpsr},~{flags}\"(i32 %s)\n", call, cx32) + for index, reg := range []Reg{AX, DX} { + part := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { i32, i32 } %%%s, %d\n", part, call, index) + if err := c.storeRegSized(reg, I32, "%"+part); err != nil { + return true, false, err + } + } + return true, false, nil + } + + ax, err := c.loadReg(AX) + if err != nil { + return true, false, err + } + dx, err := c.loadReg(DX) + if err != nil { + return true, false, err + } + ax32 := c.truncI64(ax, I32) + dx32 := c.truncI64(dx, I32) + fmt.Fprintf(c.b, " call void asm sideeffect \"wrmsr\", \"{ax},{dx},{cx},~{dirflag},~{fpsr},~{flags}\"(i32 %s, i32 %s, i32 %s)\n", ax32, dx32, cx32) + return true, false, nil +} diff --git a/amd64_lower_mul_dword_qword.go b/amd64_lower_mul_dword_qword.go new file mode 100644 index 000000000..576d5b291 --- /dev/null +++ b/amd64_lower_mul_dword_qword.go @@ -0,0 +1,147 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerDwordToQwordMultiply(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedPhysical(op, ins, x86PackedDwordProduct) +} + +func (c *amd64Ctx) lowerDwordToQwordMultiplyForm(op Op, ins Instr, form x86PackedPhysicalForm) (bool, bool, error) { + var err error + first, second, destination := form.source, ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy dword product m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + if form.mmx { + if first.Kind == OpReg && first.Reg != ins.Args[form.first].Reg { + return true, false, fmt.Errorf("%w: Go GP-spelled dword product source names the MMX bank", ErrProbeNeedsContext) + } + var a string + if first.Kind == OpReg { + a, err = c.loadReg(first.Reg) + } else { + var bytes string + bytes, err = c.loadX86PackedShiftBytes(first, 8, true) + if err == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i8> %s to i64\n", value, bytes) + a = "%" + value + } + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + a = c.extendDwordScalar(c.lowDwordFromI64(a), false) + b = c.extendDwordScalar(c.lowDwordFromI64(b), false) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i64 %s, %s\n", product, a, b) + return true, false, c.storeReg(destination.Reg, "%"+product) + } + lanes := form.byteWidth / 8 + var mask, result string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.legacy && form.memory >= 0 { + b, err := c.loadPackedCompareLanes(second, 16, 64, false) + if err != nil { + return true, false, err + } + // Keep the original SSE opcode's complete m128 access and alignment + // behavior, including when the numerical result is dead. + result, err = c.emitPackedBinaryNativeMemory(first, 64, b, form.spec.nativeOpcode) + } else { + // EVEX E4 uses qword fault-suppression lanes, including m64bcst; + // only each qword's low dword participates in the multiplication. + var a, b string + a, err = c.loadDwordMultiplyEvenLanes(first, form.byteWidth, form.spec.memoryBits, form.broadcast, mask) + if err == nil { + b, err = c.loadDwordMultiplyEvenLanes(second, form.byteWidth, form.spec.memoryBits, false, "") + } + if err == nil { + a = c.extendDwordMultiplyLanes(a, lanes, form.spec.signed) + b = c.extendDwordMultiplyLanes(b, lanes, form.spec.signed) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul <%d x i64> %s, %s\n", product, lanes, a, b) + result = "%" + product + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, 64, false) + if err != nil { + return true, false, err + } + } + result = amd64ApplyIntegerLaneMask(c, lanes, 64, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", out, lanes, result, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func isAMD64XRegOperand(op Operand) bool { + return op.Kind == OpReg && isAMD64XReg(op.Reg) +} + +func (c *amd64Ctx) loadDwordMultiplyEvenLanes(op Operand, byteWidth, memoryBits int, broadcast bool, mask string) (string, error) { + lanes := byteWidth / 8 + qwords, err := c.loadX86PackedShiftLanes(op, byteWidth, memoryBits, broadcast, mask) + if err != nil { + return "", err + } + even := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc <%d x i64> %s to <%d x i32>\n", even, lanes, qwords, lanes) + return "%" + even, nil +} + +func (c *amd64Ctx) extendDwordMultiplyLanes(value string, lanes int, signed bool) string { + extended := c.newTmp() + op := "zext" + if signed { + op = "sext" + } + fmt.Fprintf(c.b, " %%%s = %s <%d x i32> %s to <%d x i64>\n", extended, op, lanes, value, lanes) + return "%" + extended +} + +func (c *amd64Ctx) lowDwordFromI64(value string) string { + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", low, value) + return "%" + low +} + +func (c *amd64Ctx) extendDwordScalar(value string, signed bool) string { + extended := c.newTmp() + op := "zext" + if signed { + op = "sext" + } + fmt.Fprintf(c.b, " %%%s = %s i32 %s to i64\n", extended, op, value) + return "%" + extended +} diff --git a/amd64_lower_mul_qword_test.go b/amd64_lower_mul_qword_test.go new file mode 100644 index 000000000..6f0b7fc75 --- /dev/null +++ b/amd64_lower_mul_qword_test.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedQwordMultiplyCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + source := fmt.Sprintf(` +TEXT packedqwordmultiplyforms(SB),4,$0-0 + VPMULLQ X1, X20, X21 + VPMULLQ 8(AX), Y20, Y21 + VPMULLQ Z1, Z2, Z%d + VPMULLQ.BCST 40(AX), X20, X21 + VPMULLQ.BCST 48(AX), Y20, Y21 + VPMULLQ.BCST 56(AX), Z2, Z%d +`, lastZ, lastZ) + if target.goarch == "amd64" { + source += ` + VPMULLQ X1, X20, K1, X21 + VPMULLQ.Z Y1, Y20, K2, Y21 + VPMULLQ.BCST.Z 104(AX), Z20, K3, Z21 +` + } + source += "\tRET\n" + if target.goarch == "386" { + // Go accepts these high X/Y spellings, but syntax acceptance + // does not provide a physical high register in 32-bit mode. + // Their separate Context regression retains the original forms. + source = strings.NewReplacer("X20", "X6", "X21", "X7", "Y20", "Y6", "Y21", "Y7").Replace(source) + } + goAssemblerOracleOutput(t, target.goarch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"packedqwordmultiplyforms": {Name: "packedqwordmultiplyforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-qword-multiply-"+target.name+".ll", "packed-qword-multiply-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedQwordMultiplyRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPMULLQ X0, X1", + "VPMULLQ X0, Y1, Y2", + "VPMULLQ X0, (AX), X2", + "VPMULLQ X0, X1, K0, X2", + "VPMULLQ X0, X1, AX", + "VPMULLQ.Z X0, X1, X2", + "VPMULLQ.BCST X0, X1, X2", + "VPMULLQ.Z.BCST (AX), X1, K1, X2", + "VPMULLQ.RN_SAE X0, X1, X2", + } { + assertX86PackedQwordMultiplyRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + } + assertX86PackedQwordMultiplyRejected(t, "386", "i386-unknown-linux-gnu", "VPMULLQ X0, X1, K1, X2") + assertX86PackedQwordMultiplyRejected(t, "386", "i386-unknown-linux-gnu", "VPMULLQ Z0, Z1, Z8") +} + +func assertX86PackedQwordMultiplyRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPMULLQ forms", instruction) + } +} diff --git a/amd64_lower_muldiv_scalar.go b/amd64_lower_muldiv_scalar.go new file mode 100644 index 000000000..b3701b56c --- /dev/null +++ b/amd64_lower_muldiv_scalar.go @@ -0,0 +1,358 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerScalarMultiplyDivide implements the complete scalar x86 multiply and +// divide families described by Go 1.27's ydivb, ydivl, yimul, and yimul3 +// tables. The one-operand forms use the architectural accumulator pairs; +// IMULW/L/Q also have the table's explicit two- and three-operand forms. +func (c *amd64Ctx) lowerScalarMultiplyDivide(op Op, ins Instr) (ok bool, terminated bool, err error) { + if known, terminated, err := c.lowerImplicitDivide(op, ins); known { + return known, terminated, err + } + name := strings.ToUpper(string(op)) + stem, bits, threeOperandName, recognized := x86ScalarMultiplyDivideProperties(name) + if !recognized { + return false, false, nil + } + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", name, ins.Raw) + } + + if threeOperandName { + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects immediate, source, destination register: %q", c.goarch, name, ins.Raw) + } + form, _, err := x86ExplicitIMULProperties(Op(name), ins, c.goarch) + if err != nil { + return true, false, err + } + return true, false, c.emitX86ExplicitIMUL(bits, form.immediate, form.source, form.destination) + } + + if stem == "IMUL" && bits != 8 && len(ins.Args) != 1 { + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects one or two operands: %q", c.goarch, name, ins.Raw) + } + form, _, err := x86ExplicitIMULProperties(Op(name), ins, c.goarch) + if err != nil { + return true, false, err + } + return true, false, c.emitX86ExplicitIMUL(bits, Operand{}, form.source, form.destination) + } + + if len(ins.Args) != 1 { + return true, false, fmt.Errorf("%s %s expects one GP-or-memory source: %q", c.goarch, name, ins.Raw) + } + if !isX86ScalarMultiplyDivideSource(ins.Args[0], bits, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's %s class: %q", c.goarch, name, x86ScalarMultiplyDivideSourceClass(bits), ins.Raw) + } + if (stem == "MUL" || stem == "IMUL") && x86ResultNamedPredicateInstruction(ins) { + if _, _, err := x86ImplicitMultiplyProperties(op, ins, c.goarch); err != nil { + return true, false, err + } + } + + switch stem { + case "MUL": + return true, false, c.emitX86ImplicitMultiply(bits, false, ins.Args[0], x86ResultNamedPredicateInstruction(ins)) + case "IMUL": + return true, false, c.emitX86ImplicitMultiply(bits, true, ins.Args[0], x86ResultNamedPredicateInstruction(ins)) + default: + panic("unreachable scalar multiply/divide stem") + } +} + +func x86ScalarMultiplyDivideProperties(op string) (stem string, bits int, threeOperandName bool, ok bool) { + switch op { + case "MULB": + return "MUL", 8, false, true + case "MULW": + return "MUL", 16, false, true + case "MULL": + return "MUL", 32, false, true + case "MULQ": + return "MUL", 64, false, true + case "IMULB": + return "IMUL", 8, false, true + case "IMULW": + return "IMUL", 16, false, true + case "IMULL": + return "IMUL", 32, false, true + case "IMULQ": + return "IMUL", 64, false, true + case "IMUL3W": + return "IMUL", 16, true, true + case "IMUL3L": + return "IMUL", 32, true, true + case "IMUL3Q": + return "IMUL", 64, true, true + case "DIVB": + return "DIV", 8, false, true + case "DIVW": + return "DIV", 16, false, true + case "DIVL": + return "DIV", 32, false, true + case "DIVQ": + return "DIV", 64, false, true + case "IDIVB": + return "IDIV", 8, false, true + case "IDIVW": + return "IDIV", 16, false, true + case "IDIVL": + return "IDIV", 32, false, true + case "IDIVQ": + return "IDIV", 64, false, true + default: + return "", 0, false, false + } +} + +func x86ScalarMultiplyDivideSourceClass(bits int) string { + if bits == 8 { + return "Ymb" + } + return "Yml" +} + +func isX86ScalarMultiplyDivideSource(operand Operand, bits int, goarch string) bool { + if operand.Kind == OpReg { + if bits == 8 { + return isGoYmbRegisterForArch(operand.Reg, goarch) + } + return isX86YrlRegisterForArch(operand.Reg, goarch) + } + if operand.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(operand.Sym), "$") { + return false + } + return isAMD64MemoryOperand(operand) +} + +func validateX86ExplicitIMUL(goarch, op string, bits int, ins Instr, threeOperands bool) error { + immediate := Operand{} + source := ins.Args[0] + destination := ins.Args[1] + if threeOperands { + immediate = ins.Args[0] + source = ins.Args[1] + destination = ins.Args[2] + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, goarch) { + return fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", goarch, op, ins.Raw) + } + if threeOperands { + if !isX86IMULImmediate(immediate, goarch) { + return fmt.Errorf("%s %s immediate is outside Go 1.27's Yi8/Yi32 classes: %q", goarch, op, ins.Raw) + } + if !isX86ScalarMultiplyDivideSource(source, bits, goarch) { + return fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", goarch, op, ins.Raw) + } + return nil + } + if source.Kind == OpImm { + if !isX86IMULImmediate(source, goarch) { + return fmt.Errorf("%s %s immediate is outside Go 1.27's Yi8/Yi32 classes: %q", goarch, op, ins.Raw) + } + return nil + } + if !isX86ScalarMultiplyDivideSource(source, bits, goarch) { + return fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", goarch, op, ins.Raw) + } + return nil +} + +func isX86IMULImmediate(operand Operand, goarch string) bool { + if operand.Kind != OpImm { + return false + } + // The 386 assembler classifies constants after truncating them to int32. + // In amd64 mode Yi32 covers signed int32 through unsigned uint32. + return goarch == "386" || (operand.Imm >= -2147483648 && operand.Imm <= 4294967295) +} + +func (c *amd64Ctx) emitX86ImplicitMultiply(bits int, signed bool, source Operand, named bool) error { + narrow := amd64IntegerTypeForBits(bits) + wide := x86ScalarMultiplyDivideWideType(bits) + var src string + var err error + handled := false + if named && source.Kind == OpFP { + src, handled, err = c.loadIntegerFPBytesMode(source.FPOffset, narrow, true) + } else if named && x86RegisterAddressedNativeMemoryOperand(source, c.goarch) && source.Mem.Segment == "" { + // Keep the exact source access even for a zero accumulator or dead + // product. Address and AX inputs precede either implicit overwrite; + // this emitter creates no caller, frame or reusable memory fact. + src, err = c.evalSizedNativeGPSource(source, narrow, 0) + handled = true + } + if !handled && err == nil { + src, err = c.evalIntSized(source, narrow) + } + if err != nil { + return err + } + acc, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, narrow) + if err != nil { + return err + } + lhs := c.extendX86MulDivValue(acc, narrow, wide, signed) + rhs := c.extendX86MulDivValue(src, narrow, wide, signed) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul %s %s, %s\n", product, wide, lhs, rhs) + + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", low, wide, product, narrow) + if bits == 8 { + if err := c.storeRegSized(AX, I16, "%"+product); err != nil { + return err + } + } else { + shifted := c.newTmp() + shift := "lshr" + if signed { + shift = "ashr" + } + fmt.Fprintf(c.b, " %%%s = %s %s %%%s, %d\n", shifted, shift, wide, product, bits) + high := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", high, wide, shifted, narrow) + if err := c.storeRegSized(AX, narrow, "%"+low); err != nil { + return err + } + if err := c.storeRegSized(DX, narrow, "%"+high); err != nil { + return err + } + } + c.setX86MultiplyOverflowFlags(narrow, wide, "%"+low, "%"+product, signed) + return nil +} + +func (c *amd64Ctx) emitX86ExplicitIMUL(bits int, immediate, source, destination Operand) error { + typ := amd64IntegerTypeForBits(bits) + wide := x86ScalarMultiplyDivideWideType(bits) + right := "" + var err error + if source.Kind == OpImm { + right = x86IMULImmediateValue(source.Imm, bits) + } else { + right, err = c.evalSizedNativeGPSource(source, typ, 0) + if err != nil { + return err + } + } + left := "" + if immediate.Kind == OpImm { + left = x86IMULImmediateValue(immediate.Imm, bits) + } else { + left, err = c.evalIntSized(destination, typ) + if err != nil { + return err + } + } + lhs := c.extendX86MulDivValue(left, typ, wide, true) + rhs := c.extendX86MulDivValue(right, typ, wide, true) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul %s %s, %s\n", product, wide, lhs, rhs) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", low, wide, product, typ) + if err := c.storeRegSized(destination.Reg, typ, "%"+low); err != nil { + return err + } + c.setX86MultiplyOverflowFlags(typ, wide, "%"+low, "%"+product, true) + return nil +} + +func x86IMULImmediateValue(value int64, bits int) string { + return fmt.Sprintf("%d", x86IMULEncodedImmediate(value, bits)) +} + +// The explicit IMUL emitter uses evalSizedNativeGPSource's exact volatile read. +// Continuing past it supplies only this instruction's numerical source bytes; +// the existing effect still owns old-destination, immediate and FLAGS roles. +func x86LoweredExplicitIMULNativeRead(ins Instr, state *x86ResultState, owned []bool, + resultOffset int64, arch string) (bool, bool) { + form, known, err := x86ExplicitIMULProperties(ins.Op, ins, arch) + if !known || err != nil || form.source.Kind != OpMem || form.source.Mem.Sym != "" || + !x86OrdinaryNativeMemoryOperand(form.source, arch) || ins.x86NumericalRead != nil { + return false, false // Logical fields, static sources and actual source certificates keep their own consumers. + } + if !x86ResultNamedPredicateInstruction(ins) { + return false, true + } + effect := x86ExplicitIMULByteEffect(form) + if effect.dynamicMemory || state.flags&effect.flagsRead != effect.flagsRead { + return false, true + } + for _, operand := range effect.operands { + if !operand.read { + continue + } + if operand.operand.Kind == OpReg { + index, _, valid := x86ResultGP(operand.operand.Reg, operand.width, arch) + if !valid || state.pointerGP[index] { + return false, true + } + } + if !x86ResultReadWithNativeGPContinuation(ins, operand.operand, operand.width, state, owned, arch) { + return false, true + } + } + for _, operand := range effect.operands { + if operand.write && !x86ResultStore(operand.operand, operand.width, state, owned, resultOffset, arch) { + return false, true + } + } + state.flags = state.flags&^effect.flagsKilled | effect.flagsDefined + return true, true +} + +func x86IMULEncodedImmediate(value int64, bits int) int64 { + switch bits { + case 16: + return int64(int16(value)) + case 32: + return int64(int32(value)) + case 64: + // x86-64 IMUL encodes this table's immediate as imm32 and sign + // extends it to the qword operand width. + return int64(int32(value)) + default: + panic("unsupported explicit IMUL width") + } +} + +func (c *amd64Ctx) extendX86MulDivValue(value string, from, to LLVMType, signed bool) string { + tmp := c.newTmp() + op := "zext" + if signed { + op = "sext" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", tmp, op, from, value, to) + return "%" + tmp +} + +func x86ScalarMultiplyDivideWideType(bits int) LLVMType { + if bits == 64 { + return LLVMType("i128") + } + return amd64IntegerTypeForBits(bits * 2) +} + +func (c *amd64Ctx) setX86MultiplyOverflowFlags(narrow, wide LLVMType, low, product string, signed bool) { + var overflow string + if signed { + extended := c.extendX86MulDivValue(low, narrow, wide, true) + cmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne %s %s, %s\n", cmp, wide, product, extended) + overflow = "%" + cmp + } else { + extended := c.extendX86MulDivValue(low, narrow, wide, false) + cmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne %s %s, %s\n", cmp, wide, product, extended) + overflow = "%" + cmp + } + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", overflow, c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", overflow, c.flagsOFSlot) +} diff --git a/amd64_lower_mulword.go b/amd64_lower_mulword.go new file mode 100644 index 000000000..440b2a318 --- /dev/null +++ b/amd64_lower_mulword.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedWordMultiplyMode uint8 + +const ( + amd64PackedWordMultiplyLow amd64PackedWordMultiplyMode = iota + amd64PackedWordMultiplySignedHigh + amd64PackedWordMultiplyUnsignedHigh + amd64PackedWordMultiplyRoundedSignedHigh +) + +func (c *amd64Ctx) lowerPackedWordMultiply(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedPhysical(op, ins, x86PackedWordProduct) +} + +func (c *amd64Ctx) lowerPackedWordMultiplyForm(op Op, ins Instr, form x86PackedPhysicalForm) (bool, bool, error) { + first, second, destination := form.source, ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed word multiply m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + lanes := form.byteWidth / 2 + if form.mmx { + // A GP source accepted through ycover encodes an MMX register; it + // must never consume the unrelated GP shadow as its input value. + if first.Kind == OpReg && first.Reg != ins.Args[form.first].Reg { + return true, false, fmt.Errorf("%w: Go GP-spelled packed multiply source names the MMX bank", ErrProbeNeedsContext) + } + var a string + var err error + if first.Kind == OpReg { + a, err = c.loadReg(first.Reg) + } else { + a, err = c.loadPackedUnpackMMXSource(first, 8) + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + value := c.emitPackedWordMultiply(lanes, c.bitcastI64ToIntegerLanes(lanes, 16, a), c.bitcastI64ToIntegerLanes(lanes, 16, b), form.spec.mode) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to i64\n", out, lanes, value) + return true, false, c.storeReg(destination.Reg, "%"+out) + } + mnemonic := "" + if form.legacy && form.memory >= 0 { + mnemonic = strings.ToLower(string(op)) + } + return true, false, c.emitPackedBinaryE4(ins, form.x86PackedBinaryForm, form.spec.memoryBits, + mnemonic, func(lanes, bits int, first, second string) string { + return c.emitPackedWordMultiply(lanes, first, second, form.spec.mode) + }) +} + +func (c *amd64Ctx) emitPackedWordMultiply(lanes int, first, second string, mode amd64PackedWordMultiplyMode) string { + wordType := fmt.Sprintf("<%d x i16>", lanes) + if mode == amd64PackedWordMultiplyLow { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul %s %s, %s\n", result, wordType, second, first) + return "%" + result + } + + wideType := fmt.Sprintf("<%d x i32>", lanes) + extend := "sext" + shift := int64(16) + if mode == amd64PackedWordMultiplyUnsignedHigh { + extend = "zext" + } + firstWide := c.newTmp() + secondWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", firstWide, extend, wordType, first, wideType) + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", secondWide, extend, wordType, second, wideType) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul %s %%%s, %%%s\n", product, wideType, secondWide, firstWide) + value := "%" + product + if mode == amd64PackedWordMultiplyRoundedSignedHigh { + rounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", rounded, wideType, value, llvmSplatSignedInteger(lanes, 32, 16384)) + value = "%" + rounded + shift = 15 + } + shifted := c.newTmp() + shiftOp := "ashr" + if mode == amd64PackedWordMultiplyUnsignedHigh { + shiftOp = "lshr" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", shifted, shiftOp, wideType, value, llvmSplatSignedInteger(lanes, 32, shift)) + value = "%" + shifted + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", result, wideType, value, wordType) + return "%" + result +} diff --git a/amd64_lower_mxcsr.go b/amd64_lower_mxcsr.go new file mode 100644 index 000000000..c86f817ff --- /dev/null +++ b/amd64_lower_mxcsr.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerMXCSR(op Op, ins Instr) (ok bool, terminated bool, err error) { + if ins.Op == "" { + ins.Op = op + } else if !strings.EqualFold(string(ins.Op), string(op)) { + if _, known, _ := x86MXCSRProperties(Instr{Op: op}, c.goarch); known { + return true, false, fmt.Errorf("MXCSR helper/source identity differs: %s/%s", op, ins.Op) + } + return false, false, nil + } + write, known, err := x86MXCSRProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + baseOp := strings.ToUpper(string(op)) + if memory := ins.Args[0]; memory.Kind == OpMem && memory.Mem.Sym == "" && + c.nativeMemoryOperand(ins, 0) { + pointer, pointerType, err := c.ptrFromMem(memory.Mem) + if err != nil { + return true, false, err + } + constraint := "*m,~{memory}" + if write { + constraint = "=*m,~{memory}" + } + // One original native m32 operation retains exact access/fault and + // legacy/VEX identity, even when no numerical result is observed. + // MXCSR does not modify integer FLAGS, DF or any GP register. + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype(i32) %s)\n", + strings.ToLower(baseOp)+" $0", constraint, pointerType, pointer) + return true, false, nil + } + slot := c.newTmp() + fmt.Fprintf(c.b, " %%%s = alloca i32\n", slot) + switch baseOp { + case "LDMXCSR", "VLDMXCSR": + value, err := c.evalIntSized(ins.Args[0], I32) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store i32 %s, ptr %%%s\n", value, slot) + fmt.Fprintf(c.b, " call void @llvm.x86.sse.ldmxcsr(ptr %%%s)\n", slot) + return true, false, nil + case "STMXCSR", "VSTMXCSR": + value := c.newTmp() + fmt.Fprintf(c.b, " call void @llvm.x86.sse.stmxcsr(ptr %%%s)\n", slot) + fmt.Fprintf(c.b, " %%%s = load i32, ptr %%%s\n", value, slot) + if err := c.storeMXCSRMemory(ins.Args[0], "%"+value); err != nil { + return true, false, err + } + return true, false, nil + default: + panic("MXCSR opcode precheck drift") + } +} + +// The paired original opcode either faults or continues. This lowering-only +// consumer checks actual incoming pointer/index identity, not numerical address +// bits, pointee initializedness, bounds or a reusable caller-memory permission. +func x86LoweredNativeMXCSRStep(ins Instr, state *x86ResultState, arch string) (bool, bool) { + write, known, err := x86MXCSRProperties(ins, arch) + if !known || err != nil || len(ins.Args) != 1 || ins.Args[0].Kind != OpMem || + ins.Args[0].Mem.Sym != "" || !x86OrdinaryNativeMemoryOperand(ins.Args[0], arch) { + return false, false + } + if !x86ResultNamedPredicateInstruction(ins) || ins.x86NumericalRead != nil { + return false, true + } + if write { + return x86ResultIncomingPointerAddressIdentity(ins.Args[0], state, arch), true + } + return x86ResultIncomingPointerReadAddressIdentity(ins.Args[0], state, arch), true +} + +func (c *amd64Ctx) storeMXCSRMemory(dst Operand, value string) error { + switch dst.Kind { + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i32 %s, %s %s, align 1\n", value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i32 %s, ptr %s, align 1\n", value, ptr) + return nil + case OpFP: + return c.storeFPResult(dst.FPOffset, I32, value) + default: + return fmt.Errorf("amd64: unsupported MXCSR memory operand %s", dst.String()) + } +} diff --git a/amd64_lower_narrow_single_source.go b/amd64_lower_narrow_single_source.go new file mode 100644 index 000000000..2dbdf04bd --- /dev/null +++ b/amd64_lower_narrow_single_source.go @@ -0,0 +1,166 @@ +package plan9asm + +import ( + "fmt" +) + +type x86SingleSourceNarrowDomain uint8 + +const ( + x86NarrowTruncate x86SingleSourceNarrowDomain = iota + x86NarrowSigned + x86NarrowUnsigned +) + +type amd64SingleSourceNarrowSpec struct { + inputBits, outputBits int + domain x86SingleSourceNarrowDomain +} + +var amd64SingleSourceNarrowSpecs = map[string]amd64SingleSourceNarrowSpec{ + "VPMOVDB": {inputBits: 32, outputBits: 8}, + "VPMOVDW": {inputBits: 32, outputBits: 16}, + "VPMOVQB": {inputBits: 64, outputBits: 8}, + "VPMOVQD": {inputBits: 64, outputBits: 32}, + "VPMOVQW": {inputBits: 64, outputBits: 16}, + "VPMOVWB": {inputBits: 16, outputBits: 8}, + "VPMOVSDB": {inputBits: 32, outputBits: 8, domain: x86NarrowSigned}, + "VPMOVSDW": {inputBits: 32, outputBits: 16, domain: x86NarrowSigned}, + "VPMOVSQB": {inputBits: 64, outputBits: 8, domain: x86NarrowSigned}, + "VPMOVSQD": {inputBits: 64, outputBits: 32, domain: x86NarrowSigned}, + "VPMOVSQW": {inputBits: 64, outputBits: 16, domain: x86NarrowSigned}, + "VPMOVSWB": {inputBits: 16, outputBits: 8, domain: x86NarrowSigned}, + "VPMOVUSDB": {inputBits: 32, outputBits: 8, domain: x86NarrowUnsigned}, + "VPMOVUSDW": {inputBits: 32, outputBits: 16, domain: x86NarrowUnsigned}, + "VPMOVUSQB": {inputBits: 64, outputBits: 8, domain: x86NarrowUnsigned}, + "VPMOVUSQD": {inputBits: 64, outputBits: 32, domain: x86NarrowUnsigned}, + "VPMOVUSQW": {inputBits: 64, outputBits: 16, domain: x86NarrowUnsigned}, + "VPMOVUSWB": {inputBits: 16, outputBits: 8, domain: x86NarrowUnsigned}, +} + +// lowerSingleSourceIntegerNarrow implements the complete Go 1.27 +// _yvpmovdb/_yvpmovdw family: truncating, signed-saturating, and +// unsigned-saturating conversions from X/Y/Z into X/Y or memory. +func (c *amd64Ctx) lowerSingleSourceIntegerNarrow(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86SingleSourceNarrowProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if form.undefinedOpcode() { + return c.lowerX86UndefinedTrap(op, ins) + } + mask := "" + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + sourceBytes, err := c.loadPackedCompareBytes(ins.Args[0], form.inputBytes) + if err != nil { + return true, false, err + } + lanes := form.inputBytes * 8 / form.spec.inputBits + source := c.bitcastVectorBytesToIntegerLanes(form.inputBytes, lanes, form.spec.inputBits, sourceBytes) + narrowed := c.emitSingleSourceIntegerNarrow(form.spec, lanes, source) + destination := ins.Args[form.destination] + if form.memory < 0 { + return true, false, c.storeSingleSourceNarrowRegister(destination.Reg, form.carrierBytes, + lanes, form.spec.outputBits, narrowed, mask, form.zeroing) + } + return true, false, c.storeSingleSourceNarrowMemory(destination, form.outputBytes, + lanes, form.spec.outputBits, narrowed, mask) +} + +func (c *amd64Ctx) emitSingleSourceIntegerNarrow(spec amd64SingleSourceNarrowSpec, lanes int, source string) string { + inputType := fmt.Sprintf("<%d x i%d>", lanes, spec.inputBits) + outputType := fmt.Sprintf("<%d x i%d>", lanes, spec.outputBits) + if spec.domain != x86NarrowTruncate { + // Single-source US conversions consume unsigned inputs. PACKUS has + // signed inputs and deliberately keeps its separate saturation rule. + predicate := "ugt" + maximum := uint64(1)<>= 1 + } + limit := llvmSplatInteger(lanes, spec.inputBits, maximum) + above, capped := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s %s %s, %s\n", above, predicate, inputType, source, limit) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", capped, lanes, above, inputType, limit, inputType, source) + source = "%" + capped + if spec.domain == x86NarrowSigned { + minimum := llvmSplatIntegerText(lanes, spec.inputBits, fmt.Sprint(-int64(maximum)-1)) + below, floored := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, %s\n", below, inputType, source, minimum) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", floored, lanes, below, inputType, minimum, inputType, source) + source = "%" + floored + } + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", result, inputType, source, outputType) + return "%" + result +} + +func (c *amd64Ctx) storeSingleSourceNarrowRegister(destination Reg, destinationBytes, lanes, outputBits int, narrowed, mask string, zeroing bool) error { + physicalLanes := destinationBytes * 8 / outputBits + result := "zeroinitializer" + var old string + if mask != "" && !zeroing { + oldBytes, err := c.loadPackedCompareBytes(Operand{Kind: OpReg, Reg: destination}, destinationBytes) + if err != nil { + return err + } + old = c.bitcastVectorBytesToIntegerLanes(destinationBytes, physicalLanes, outputBits, oldBytes) + } + for lane := 0; lane < lanes; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", value, lanes, outputBits, narrowed, lane) + selected := "%" + value + if mask != "" { + fallback := "0" + if !zeroing { + oldValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", oldValue, physicalLanes, outputBits, old, lane) + fallback = "%" + oldValue + } + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %s, i%d %s\n", masked, amd64MaskBitI1(c, mask, lane), outputBits, selected, outputBits, fallback) + selected = "%" + masked + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %s, i32 %d\n", inserted, physicalLanes, outputBits, result, outputBits, selected, lane) + result = "%" + inserted + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, physicalLanes, outputBits, result, destinationBytes) + return c.storePackedMoveOperand(Operand{Kind: OpReg, Reg: destination}, destinationBytes, "%"+out) +} + +func (c *amd64Ctx) storeSingleSourceNarrowMemory(destination Operand, outputBytes, lanes, outputBits int, narrowed, mask string) error { + if mask != "" { + pointer, pointerType, suffix, err := c.packedVectorMemoryPointer(destination) + if err != nil { + return err + } + if suffix == "" { + suffix = ".p0" + } + predicate := amd64IntegerMaskVector(c, lanes, mask) + fmt.Fprintf(c.b, " call void @llvm.masked.store.v%di%d%s(<%d x i%d> %s, %s align 1 %s, <%d x i1> %s)\n", lanes, outputBits, suffix, lanes, outputBits, narrowed, pointerType, pointer, lanes, predicate) + return nil + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, outputBits, narrowed, outputBytes) + return c.storeNarrowMemoryBytes(destination, outputBytes, "%"+out) +} + +func (c *amd64Ctx) storeNarrowMemoryBytes(destination Operand, byteWidth int, value string) error { + if byteWidth >= 16 { + return c.storeVectorBytesOperand(destination, byteWidth, value) + } + typ := amd64IntegerTypeForBits(byteWidth * 8) + integer := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", integer, byteWidth, value, typ) + return c.storeVectorScalarMemory(destination, byteWidth*8, "%"+integer, "") +} diff --git a/amd64_lower_narrow_single_source_test.go b/amd64_lower_narrow_single_source_test.go new file mode 100644 index 000000000..361d3630e --- /dev/null +++ b/amd64_lower_narrow_single_source_test.go @@ -0,0 +1,169 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86SingleSourceIntegerNarrowCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 31 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + source.WriteString("TEXT singlesourcenarrowforms(SB),$0-0\n") + for _, op := range []string{ + "VPMOVDB", "VPMOVDW", "VPMOVQB", "VPMOVQD", "VPMOVQW", "VPMOVWB", + "VPMOVSDB", "VPMOVSDW", "VPMOVSQB", "VPMOVSQD", "VPMOVSQW", "VPMOVSWB", + "VPMOVUSDB", "VPMOVUSDW", "VPMOVUSQB", "VPMOVUSQD", "VPMOVUSQW", "VPMOVUSWB", + } { + inputBits, outputBits := x86SingleSourceNarrowTestBits(op) + for _, width := range []struct { + name string + bytes int + }{ + {name: "X", bytes: 16}, + {name: "Y", bytes: 32}, + {name: "Z", bytes: 64}, + } { + outputBytes := width.bytes * outputBits / inputBits + destinationWidth := "X" + if outputBytes > 16 { + destinationWidth = "Y" + } + fmt.Fprintf(&source, "\t%s %s%d, %s%d\n", op, width.name, last, destinationWidth, last) + fmt.Fprintf(&source, "\t%s %s%d, K1, %s%d\n", op, width.name, last, destinationWidth, last) + fmt.Fprintf(&source, "\t%s.Z %s%d, K2, %s%d\n", op, width.name, last, destinationWidth, last) + fmt.Fprintf(&source, "\t%s %s%d, 8(AX)\n", op, width.name, last) + fmt.Fprintf(&source, "\t%s %s%d, K3, 16(AX)\n", op, width.name, last) + fmt.Fprintf(&source, "\t%s.Z %s%d, K4, 24(AX)\n", op, width.name, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "singlesourcenarrowforms": {Name: "singlesourcenarrowforms", Ret: Void}, + }, + }) + if ll != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("bare Go formats invented initialized entry registers or memory", err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Preserve every original operand and all 324 Go-encoded shapes as + // isolated emitter units. Initialized public objects and executable + // numerical/fault owners are independent of these format objects. + units, signatures := x86SingleSourceNarrowingFormatUnits(file) + if len(units.Funcs) != 324 { + t.Fatal("an original Go format disappeared", len(units.Funcs)) + } + x86SingleSourceNarrowingFormatObjects(t, llc, units, signatures, target.goarch, target.triple) + }) + } +} + +func x86SingleSourceNarrowingFormatUnits(file *File) (*File, map[string]FuncSig) { + units := *file + units.Funcs = nil + signatures := make(map[string]FuncSig) + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + if ins.Op == OpTEXT || ins.Op == OpRET { + continue + } + name := fmt.Sprint("NarrowFormat", len(units.Funcs)) + units.Funcs = append(units.Funcs, Func{Sym: name, Instrs: []Instr{{Op: OpTEXT}, ins, {Op: OpRET}}}) + signatures[name] = FuncSig{Name: name, Ret: Void} + } + } + return &units, signatures +} + +func x86SingleSourceNarrowingFormatObjects(t *testing.T, llc string, units *File, signatures map[string]FuncSig, arch, triple string) { + t.Helper() + for start := 0; start < len(units.Funcs); start += 64 { + end := start + 64 + if end > len(units.Funcs) { + end = len(units.Funcs) + } + batch := *units + batch.Funcs = units.Funcs[start:end] + ir := x86SingleInstructionFormatObjectUnits(t, &batch, signatures, arch, triple, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerSingleSourceIntegerNarrow(ins.Op, ins) }) + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + } +} + +func x86SingleSourceNarrowTestBits(op string) (inputBits, outputBits int) { + pair := strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(op, "VPMOV"), "US"), "S") + bits := func(letter byte) int { + switch letter { + case 'B': + return 8 + case 'W': + return 16 + case 'D': + return 32 + case 'Q': + return 64 + default: + panic("bad narrow element letter") + } + } + return bits(pair[0]), bits(pair[1]) +} + +func TestTranslateX86SingleSourceIntegerNarrowRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPMOVSDB 8(AX), X0", + "VPMOVSDB X0, Y0", + "VPMOVSDW Z0, X0", + "VPMOVSDB X0, K0, X1", + "VPMOVSDB.Z X0, X1", + "VPMOVSDB.BCST X0, X1", + "VPMOVSDB X0, K1, K2, X1", + "VPMOVSDB X0, AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvpmovdb/_yvpmovdw tables", instruction) + } + }) + } +} diff --git a/amd64_lower_neg.go b/amd64_lower_neg.go new file mode 100644 index 000000000..dc3841d3e --- /dev/null +++ b/amd64_lower_neg.go @@ -0,0 +1,63 @@ +package plan9asm + +import "fmt" + +// Both complete unary families use Go's yscond(Ymb) grammar, including +// byte-register spellings of wide operations. Only NEG changes flags. +type x86UnaryYmbSpec struct { + bits int + negate bool +} + +var x86UnaryYmbSpecs = map[Op]x86UnaryYmbSpec{ + "NEGB": {bits: 8, negate: true}, + "NEGW": {bits: 16, negate: true}, + "NEGL": {bits: 32, negate: true}, + "NEGQ": {bits: 64, negate: true}, + "NOTB": {bits: 8}, + "NOTW": {bits: 16}, + "NOTL": {bits: 32}, + "NOTQ": {bits: 64}, +} + +func (c *amd64Ctx) lowerScalarUnaryYmb(op Op, ins Instr) (ok bool, terminated bool, err error) { + spec, recognized := x86UnaryYmbSpecs[op] + if !recognized { + return false, false, nil + } + bits := spec.bits + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", op, ins.Raw) + } + if len(ins.Args) != 1 { + return true, false, fmt.Errorf("%s %s expects one register/memory operand: %q", c.goarch, op, ins.Raw) + } + destination := ins.Args[0] + if destination.Kind == OpReg { + if !isGoYmbRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s register is outside Go 1.27's Ymb class: %q", c.goarch, op, ins.Raw) + } + destination.Reg = amd64YmbEffectiveRegister(destination.Reg, bits) + } else if !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("%s %s operand is outside Go 1.27's Ymb class: %q", c.goarch, op, ins.Raw) + } + + typ := amd64IntegerTypeForBits(bits) + value, store, err := c.loadScalarRMWDestination(op, ins, destination, typ) + if err != nil { + return true, false, err + } + result := c.newTmp() + if !spec.negate { + fmt.Fprintf(c.b, " %%%s = xor %s %s, -1\n", result, typ, value) + return true, false, store("%" + result) + } + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", result, typ, value) + if err := store("%" + result); err != nil { + return true, false, err + } + // NEG has the same arithmetic flags as subtraction from zero: CF is set + // exactly for nonzero inputs and OF is set only for the signed minimum. + c.setScalarAddSubFlags(typ, false, "0", value, "%"+result) + return true, false, nil +} diff --git a/amd64_lower_neg_test.go b/amd64_lower_neg_test.go new file mode 100644 index 000000000..4a4071609 --- /dev/null +++ b/amd64_lower_neg_test.go @@ -0,0 +1,285 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86UnaryYmbGrammarMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`\{A((?:NEG|NOT)[BWLQ]),\s*yscond,`).FindAllStringSubmatch(string(data), -1) + if len(rows) != len(x86UnaryYmbSpecs) || len(rows) != 8 { + t.Fatalf("Go encoder has %d NEG/NOT Ymb rows; grammar has %d", len(rows), len(x86UnaryYmbSpecs)) + } + for _, row := range rows { + name := row[1] + spec, ok := x86UnaryYmbSpecs[Op(name)] + width := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[name[len(name)-1]] + if !ok || spec.bits != width || spec.negate != strings.HasPrefix(name, "NEG") { + t.Errorf("Go %s encoder row disagrees with grammar: %+v, present=%v", name, spec, ok) + } + } +} + +func x86NEGCompleteFormsSource(goarch string) string { + var source strings.Builder + symbol := "negdata" + size := 8 + if goarch == "386" { + symbol = "negdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $1\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT negforms(SB),$0-0\n") + if goarch == "amd64" { + source.WriteString("\tNEGB AH\n\tNEGB R11\n") + } else { + source.WriteString("\tNEGB AH\n\tNEGB BP\n") + } + source.WriteString("\tNEGB 8(BX)\n\tNEGB " + symbol + "(SB)\n") + widths := []string{"W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + // yscond uses Ymb even for wide opcodes. Byte spellings therefore + // select the full register with the same raw ModRM number. + for _, reg := range []string{"AL", "CL", "DL", "BL", "AH", "CH", "DH", "BH"} { + fmt.Fprintf(&source, "\tNEG%s %s\n", width, reg) + } + fmt.Fprintf(&source, "\tNEG%s DX\n", width) + if goarch == "amd64" { + for _, reg := range []string{"SP", "BPB", "SIB", "DIB", "R8", "R15", "R8B", "R15B"} { + fmt.Fprintf(&source, "\tNEG%s %s\n", width, reg) + } + } else { + fmt.Fprintf(&source, "\tNEG%s SI\n", width) + } + fmt.Fprintf(&source, "\tNEG%s 8(BX)\n\tNEG%s %s(SB)\n", width, width, symbol) + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86NEGCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86NEGCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source, "func negforms()", target.goarch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original scalar format source invented mutable/native/frame inputs", api, err) + } + } + ll := x86ScalarFormatObjectUnit(t, file, map[string]FuncSig{"negforms": {Name: "negforms", Ret: Void}}, + target.goarch, target.triple, 0) + for _, want := range []string{"sub i8 0", "sub i16 0", "sub i32 0", "store i1"} { + if !strings.Contains(ll, want) { + t.Fatalf("NEG lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "sub i64 0") { + t.Fatalf("NEGQ lowering omitted sub i64:\n%s", ll) + } + for _, ir := range []string{ll, x86DwordProductIROptimize(t, ll)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + } + }) + } +} + +func TestTranslateX86NEGRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "NEGB X0"}, + {goarch: "amd64", instruction: "NEGW $1"}, + {goarch: "amd64", instruction: "NEGL AX, BX"}, + {goarch: "amd64", instruction: "NEGQ"}, + {goarch: "amd64", instruction: "NEGQ.Z AX"}, + {goarch: "386", instruction: "NEGB SP"}, + {goarch: "386", instruction: "NEGW SP"}, + {goarch: "386", instruction: "NEGL SP"}, + {goarch: "386", instruction: "NEGL R8"}, + {goarch: "386", instruction: "NEGQ AX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's NEG yscond table", test.instruction) + } + }) + } +} + +func TestAMD64NEGRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT negsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0x1122, AX + NEGB AH + MOVQ AX, 0(DI) + MOVQ $0x1122334455660001, AX + NEGW AX + MOVQ AX, 8(DI) + MOVQ $0x1122334480000000, AX + NEGL AX + MOVQ AX, 16(DI) + SETCS 32(DI) + SETOS 33(DI) + SETEQ 34(DI) + SETMI 35(DI) + SETPS 36(DI) + MOVQ $5, 24(DI) + NEGQ 24(DI) + MOVB $0, 40(DI) + NEGB 40(DI) + SETCS 41(DI) + SETOS 42(DI) + SETEQ 43(DI) + SETMI 44(DI) + SETPS 45(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "negsemantics": { + Name: "negsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void negsemantics(uint8_t *); +int main(void) { + uint8_t out[48] = {0}; + negsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0xef22)) return 10; + if (*(uint64_t *)(out+8) != UINT64_C(0x112233445566ffff)) return 11; + if (*(uint64_t *)(out+16) != UINT64_C(0x80000000)) return 12; + if (*(uint64_t *)(out+24) != UINT64_C(0xfffffffffffffffb)) return 13; + if (out[32] != 1 || out[33] != 1 || out[34] != 0 || out[35] != 1 || out[36] != 1) return 14; + if (out[40] != 0 || out[41] != 0 || out[42] != 0 || out[43] != 1 || out[44] != 0 || out[45] != 1) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "neg_semantics", triple, ll, mainC, runPrefix) +} + +func TestAMD64UnaryYmbAliasesRuntime(t *testing.T) { + execution := x86ScalarCountHostExecution(t) + var source, checks strings.Builder + source.WriteString("TEXT unaryaliases(SB),$0-8\n\tMOVQ out+0(FP), R14\n") + index := 0 + for _, family := range []string{"NEG", "NOT"} { + for _, width := range []struct { + suffix string + bits uint + }{{"W", 16}, {"L", 32}, {"Q", 64}} { + for _, reg := range []struct{ spelling, effective string }{ + {"AL", "AX"}, {"CL", "CX"}, {"DL", "DX"}, {"BL", "BX"}, + {"AH", "SP"}, {"CH", "BP"}, {"DH", "SI"}, {"BH", "DI"}, + {"BPB", "BP"}, {"SIB", "SI"}, {"DIB", "DI"}, {"R8B", "R8"}, {"R15B", "R15"}, + } { + const input uint64 = 0x1122334455667701 + want := ^input + if family == "NEG" { + want++ + } + if width.bits == 16 { + want = (input &^ 0xffff) | (want & 0xffff) + } + if width.bits == 32 { + want &= 0xffffffff + } + fmt.Fprintf(&source, "\tMOVQ $%#x, %s\n\t%s%s %s\n\tMOVQ %s, %d(R14)\n", input, reg.effective, family, width.suffix, reg.spelling, reg.effective, index*8) + fmt.Fprintf(&checks, " if (out[%d] != UINT64_C(%#x)) return %d;\n", index, want, index+1) + index++ + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + // Wide AH/CH aliases select physical SP/BP. The unchanged original body + // never restores its real return stack: this remains a private arithmetic + // oracle, separately from public Context and actual Go byte identities. + x86PhysicalSPRequirePublicContext(t, source.String(), "func unaryaliases(out *uint64)", "amd64", execution.triple) + ir := x86PhysicalSPPrivateEmitter(t, source.String(), map[string]FuncSig{ + "unaryaliases": {Name: "unaryaliases", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}}, + }, "amd64", execution.triple) + mainC := fmt.Sprintf("#include \nextern void unaryaliases(uint64_t *);\nint main(void) {\n uint64_t out[%d] = {0};\n unaryaliases(out);\n%s return 0;\n}\n", index, checks.String()) + execution.compileAndRunLLVM(t, "unary_aliases_private_math", ir, + "#define unaryaliases InstructionFormat0\n"+mainC, "-O2") + t.Log("all 78 original Ymb aliases execute private virtual-register mathematics only, not the invalid physical Go return sequence") +} diff --git a/amd64_lower_non_temporal_move.go b/amd64_lower_non_temporal_move.go new file mode 100644 index 000000000..829fdb81e --- /dev/null +++ b/amd64_lower_non_temporal_move.go @@ -0,0 +1,33 @@ +package plan9asm + +import "fmt" + +// A non-temporal hint does not make the instruction's native access optional. +// Reuse the aligned packed-move emitter, including original faults, exact +// width, segment address spaces and encoding-aware register writeback. +func (c *amd64Ctx) lowerNonTemporalVectorMove(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86NonTemporalVectorProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: non-temporal vector access needs its actual x86 target", ErrProbeNeedsContext) + } + if form.trap { + return c.lowerSyscall("UD2", Instr{Op: "UD2"}) + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + (ins.x86Encoded || ins.x86SourceRaw != nil) && ins.x86NumericalRead == nil && ins.x86SDKLiteral == nil { + return true, false, fmt.Errorf("%w: non-temporal vector access needs its actual x86 target/encoding", ErrProbeNeedsContext) + } + if !c.nativeMemoryOperand(ins, form.move.memory) && + !x86CallerMemoryCertified(ins, form.move.memory) && !c.fixedLocalMemoryCertified(ins, form.move.memory) { + return true, false, fmt.Errorf("%w: non-temporal vector access needs native backing, not logical FP/AUTO bytes", ErrProbeNeedsContext) + } + if err := x86Mode32VectorRegisterContext(ins, c.goarch); err != nil { + return true, false, err + } + native := ins + native.Op = Op(form.spec.native) // Local spelling adapter; never mutate original source identity. + return true, false, c.lowerNativeAlignedVectorMemory(native, form.move, !form.spec.vector) +} diff --git a/amd64_lower_non_temporal_move_test.go b/amd64_lower_non_temporal_move_test.go new file mode 100644 index 000000000..824b78c5a --- /dev/null +++ b/amd64_lower_non_temporal_move_test.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86NonTemporalVectorMoveCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses _yvmovntdq for VMOVNTDQ/PD/PS register-to-memory + // stores and _yvmovntdqa for the inverse aligned memory loads. Both + // tables cover VEX/EVEX X, Y, and Z widths. The legacy table covers the + // corresponding X-only forms (with MOVNTO as Go's MOVNTDQ spelling). + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 31 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT nontemporalmoveforms(SB),$0-0\n") + for index, op := range []string{"VMOVNTDQ", "VMOVNTPD", "VMOVNTPS"} { + fmt.Fprintf(&source, "\t%s X%d, %d(AX)\n", op, 20+index, index*256) + fmt.Fprintf(&source, "\t%s Y%d, %d(AX)\n", op, 20+index, index*256+64) + fmt.Fprintf(&source, "\t%s Z%d, %d(AX)\n", op, lastZ, index*256+128) + } + source.WriteString("\tVMOVNTDQA 1024(AX), X20\n") + source.WriteString("\tVMOVNTDQA 1088(AX), Y21\n") + fmt.Fprintf(&source, "\tVMOVNTDQA 1152(AX), Z%d\n", lastZ) + source.WriteString("\tMOVNTO X1, 1216(AX)\n") + source.WriteString("\tMOVNTPD X2, 1280(AX)\n") + source.WriteString("\tMOVNTPS X3, 1344(AX)\n") + source.WriteString("\tMOVNTDQA 1408(AX), X4\n") + source.WriteString("\tRET\n") + + file, signatures := x86UnboundVoidFormatSource(t, source.String(), "nontemporalmoveforms", target.goarch, target.triple) + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + form, known, err := x86NonTemporalVectorProperties(ins.Op, ins, target.goarch) + if !known || err != nil { + return known, false, err + } + // The original Go source and every public Context result are + // retained above. This unit tests LLVM carrier syntax only: + // high named mode32 registers have no physical entry binding. + ins.Op = Op(form.spec.native) + return true, false, c.lowerNativeAlignedVectorMemory(ins, form.move, !form.spec.vector) + }) + if strings.Count(ir, "asm sideeffect") != 16 { + t.Fatal("emitter-only object lost an original non-temporal native format") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "non-temporal-move-"+target.name+".ll", "non-temporal-move-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86NonTemporalVectorMoveRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VMOVNTDQ 8(AX), X0"}, + {goarch: "amd64", instruction: "VMOVNTPD X0, X1"}, + {goarch: "amd64", instruction: "VMOVNTPS X0, K1, 8(AX)"}, + {goarch: "amd64", instruction: "VMOVNTDQ.Z Z0, 8(AX)"}, + {goarch: "amd64", instruction: "VMOVNTDQA X0, X1"}, + {goarch: "amd64", instruction: "VMOVNTDQA 8(AX), K1, X0"}, + {goarch: "amd64", instruction: "MOVNTPD 8(AX), X0"}, + {goarch: "amd64", instruction: "MOVNTPS X0, X1"}, + {goarch: "amd64", instruction: "MOVNTDQA X0, 8(AX)"}, + {goarch: "386", instruction: "VMOVNTDQ Z8, 8(AX)"}, + {goarch: "386", instruction: "VMOVNTDQA 8(AX), Z8"}, + {goarch: "386", instruction: "MOVNTO X8, 8(AX)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's non-temporal vector-move tables for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestX86RawDecoderRecoversVEXNonTemporalVectorMoveWidths(t *testing.T) { + for _, test := range []struct { + name string + code []byte + op Op + from string + to string + }{ + {name: "vmovntdq x store", code: []byte{0xc5, 0xf9, 0xe7, 0x03}, op: "VMOVNTDQ", from: "X0", to: "0(BX)"}, + {name: "vmovntdq y store", code: []byte{0xc5, 0xfd, 0xe7, 0x03}, op: "VMOVNTDQ", from: "Y0", to: "0(BX)"}, + {name: "vmovntdq y sib store", code: []byte{0xc4, 0xa1, 0x7d, 0xe7, 0x04, 0x03}, op: "VMOVNTDQ", from: "Y0", to: "0(BX)(R8*1)"}, + {name: "vmovntpd x store", code: []byte{0xc5, 0xf9, 0x2b, 0x03}, op: "VMOVNTPD", from: "X0", to: "0(BX)"}, + {name: "vmovntpd y store", code: []byte{0xc5, 0xfd, 0x2b, 0x03}, op: "VMOVNTPD", from: "Y0", to: "0(BX)"}, + {name: "vmovntps x store", code: []byte{0xc5, 0xf8, 0x2b, 0x03}, op: "VMOVNTPS", from: "X0", to: "0(BX)"}, + {name: "vmovntps y store", code: []byte{0xc5, 0xfc, 0x2b, 0x03}, op: "VMOVNTPS", from: "Y0", to: "0(BX)"}, + {name: "vmovntdqa x load", code: []byte{0xc4, 0xe2, 0x79, 0x2a, 0x03}, op: "VMOVNTDQA", from: "0(BX)", to: "X0"}, + {name: "vmovntdqa y load", code: []byte{0xc4, 0xe2, 0x7d, 0x2a, 0x03}, op: "VMOVNTDQA", from: "0(BX)", to: "Y0"}, + } { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.op || len(got.Instrs[0].Args) != 2 || got.Instrs[0].Args[0].String() != test.from || got.Instrs[0].Args[1].String() != test.to { + t.Fatalf("decoded %#x as %#v, want %s %s, %s", test.code, got.Instrs, test.op, test.from, test.to) + } + }) + } +} diff --git a/amd64_lower_nop_width_test.go b/amd64_lower_nop_width_test.go new file mode 100644 index 000000000..1d0866518 --- /dev/null +++ b/amd64_lower_nop_width_test.go @@ -0,0 +1,160 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86SizedNOPCompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA nopdata+0(SB)/8, $1\n") + source.WriteString("GLOBL nopdata(SB), $8\n") + source.WriteString("TEXT sizednopforms(SB),$0-0\n") + for _, op := range []string{"NOPW", "NOPL"} { + fmt.Fprintf(&source, "\t%s AX\n", op) + fmt.Fprintf(&source, "\t%s SP\n", op) + fmt.Fprintf(&source, "\t%s 8(BX)\n", op) + fmt.Fprintf(&source, "\t%s nopdata(SB)\n", op) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11\n", op) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86SizedNOPCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86SizedNOPCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"sizednopforms": {Name: "sizednopforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "sized-nop-"+target.name+".ll", "sized-nop-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SizedNOPRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "NOPW"}, + {goarch: "amd64", instruction: "NOPL $1"}, + {goarch: "amd64", instruction: "NOPW X0"}, + {goarch: "amd64", instruction: "NOPL AX, BX"}, + {goarch: "amd64", instruction: "NOPL.Z AX"}, + {goarch: "386", instruction: "NOPW R8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's NOPW/NOPL ydivl table", test.instruction) + } + }) + } +} + +func TestAMD64SizedNOPRuntimePreservesState(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT sizednopsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0x123456789abcdef, AX + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + NOPW AX + NOPL 4096(DI) + MOVQ AX, 0(DI) + SETCS 8(DI) + SETOS 9(DI) + SETEQ 10(DI) + SETMI 11(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "sizednopsemantics": { + Name: "sizednopsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void sizednopsemantics(uint8_t *); +int main(void) { + uint8_t out[16] = {0}; + sizednopsemantics(out); + uint64_t value; + memcpy(&value, out, sizeof(value)); + if (value != UINT64_C(0x123456789abcdef)) return 10; + if (out[8] != 1 || out[9] != 1 || out[10] != 0 || out[11] != 1) return 11; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "sized_nop_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_numerical_read.go b/amd64_lower_numerical_read.go new file mode 100644 index 000000000..31937775d --- /dev/null +++ b/amd64_lower_numerical_read.go @@ -0,0 +1,97 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerNumericalRead(ins Instr) (bool, error) { + return c.lowerNumericalReadAt(ins, -1, -1) +} + +func (c *amd64Ctx) lowerNumericalReadAt(ins Instr, block, ordinal int) (bool, error) { + if ins.x86SDKLiteral != nil { + fact, valid := certifiedSDKLiteralAccess(ins, c.goarch) + if !valid || c.sig.Name != fact.Function || c.targetTriple != ins.x86SDKLiteral.binding.options.TargetTriple { + return true, numericalSBContext("emitted private literal source/function/profile differs") + } + if c.fixedLocal != nil || c.sourceStack != nil { + return true, numericalSBContext("literal address cannot borrow private/native frame authority") + } + if err := c.scalarFPResultInitialized(); err != nil { + return true, err + } + // Keep the original opcode/address emission. The byte fact supplies + // no align-1 replacement, fault suppression or constant-folded data. + return false, nil + } + if ins.x86NumericalRead == nil { + return false, nil + } + read, form, valid := numericalCertifiedRead(ins, c.goarch) + cert := ins.x86NumericalRead + if !valid || !c.numericalReadAtOriginalSource(ins, read, block, ordinal) || c.targetTriple != cert.binding.options.TargetTriple { + return true, numericalSBContext("emitted function/profile/read certificate changed") + } + var comparison x86ScalarCompareForm + if form.spec.compare { + var known bool + var err error + comparison, known, err = x86ScalarCompareProperties(ins.Op, ins, c.goarch) + if !known || err != nil { + return true, numericalSBContext("certified comparison lost its complete operand form: %v", err) + } + } + if c.fixedLocal == nil { + if err := c.scalarFPResultInitialized(); err != nil { + return true, err + } + } else if c.fixedLocal.binding == nil || c.fixedLocal.binding != c.sig.x86FixedLocal || + c.fixedLocal.binding.numericalReads == nil || c.fixedLocal.binding.numericalReads.source != cert.binding.source { + return true, numericalSBContext("numerical read lost its independent complete fixed-local proof") + } + if form.spec.vector == x86NumericalSBAES { + _, _, err := c.lowerAES(ins.Op, ins) + return true, err + } + if form.spec.vector == x86NumericalSBLogical { + _, _, err := c.lowerPackedLogicalFamily(ins.Op, ins) + return true, err + } + // No ptrFromSB call: only this exact typed read can consume the immutable + // native source offset. The global remains externally mutable, and every + // read is volatile even when its result/condition is later overwritten. + pointer := llvmGlobal(read.Variable.NativeName) + if read.Offset != 0 { + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, ptr %s, i64 %d\n", address, pointer, read.Offset) + pointer = "%" + address + } + sourceType := amd64IntegerTypeForBits(int(read.Width * 8)) + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile %s, ptr %s, align 1\n", loaded, sourceType, pointer) + value := "%" + loaded + if form.spec.compare { + // Certification already checked this shared complete operand grammar. + // Preserve Go's immediate truncation/sign extension and byte views. + partner := comparison.args[form.partner] + other, err := c.evalIntSized(partner, sourceType) + if err != nil { + return true, err + } + if form.operand == 0 { + c.setCmpFlagsSized(sourceType, value, other) + } else { + c.setCmpFlagsSized(sourceType, other, value) + } + return true, nil + } + destinationType := amd64IntegerTypeForBits(int(form.spec.gpBytes * 8)) + if read.Width < form.spec.gpBytes { + extension := "zext" + if amd64ScalarExtensionMoveSpecs[ins.Op].signed { + extension = "sext" + } + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", extended, extension, sourceType, value, destinationType) + value = "%" + extended + } + return true, c.storeRegSized(ins.Args[form.partner].Reg, destinationType, value) +} diff --git a/amd64_lower_pack.go b/amd64_lower_pack.go new file mode 100644 index 000000000..7f395343a --- /dev/null +++ b/amd64_lower_pack.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedNarrowMode uint8 + +const ( + amd64PackedNarrowSigned amd64PackedNarrowMode = iota + amd64PackedNarrowUnsigned +) + +// All legacy/VEX/EVEX forms share one typed grammar. Packing effects remain +// separate: clamp signed inputs, then concatenate each 128-bit source pair. +func (c *amd64Ctx) lowerPackedSaturatingNarrow(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86PackedNarrowSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("packed narrow dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, known, err := x86PackedNarrowProperties(ins, c.goarch) + if err != nil || !known { + return known, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.mmx && first.Kind == OpReg && first.Reg != form.source.Reg { + return true, false, fmt.Errorf("%w: packed narrow GP spelling needs its physical MMX transport", ErrProbeNeedsContext) + } + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy pack m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + inputBits, outputBits := form.spec.inputBits, form.spec.outputBits + inputLanes := form.byteWidth * 8 / inputBits + if form.mmx { + var firstBits string + if form.memory >= 0 { + firstBits, err = c.loadPackedUnpackMMXSource(first, form.memoryBytes) + } else { + firstBits, err = c.loadReg(first.Reg) + } + if err != nil { + return true, false, err + } + secondBits, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + value := c.emitPackedSaturatingNarrow(form.byteWidth, inputBits, outputBits, form.spec.mode, + c.bitcastI64ToIntegerLanes(inputLanes, inputBits, firstBits), + c.bitcastI64ToIntegerLanes(inputLanes, inputBits, secondBits)) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", out, 64/outputBits, outputBits, value) + return true, false, c.storeReg(destination.Reg, "%"+out) + } + secondValue, err := c.loadPackedCompareLanes(second, form.byteWidth, inputBits, false) + if err != nil { + return true, false, err + } + outputLanes := form.byteWidth * 8 / outputBits + var value string + if form.legacy && form.memory >= 0 { + mnemonic := strings.ToLower(base) + if base == "PACKSSLW" { + mnemonic = "packssdw" + } + // Preserve legacy native alignment/fault behavior. The native result + // is already packed: reinterpret it, do not narrow it a second time. + packed, loadErr := c.emitPackedBinaryNativeMemory(first, inputBits, secondValue, mnemonic) + err = loadErr + if err == nil { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i%d>\n", out, inputLanes, inputBits, packed, outputLanes, outputBits) + value = "%" + out + } + } else { + // Intel's E4NF/E4NF.nb pack reads the entire tuple even with a zero + // output mask or dead result. Never pass that mask to an E4 reader. + firstValue, loadErr := c.loadX86PackedShiftLanes(first, form.byteWidth, inputBits, form.broadcast, "") + err = loadErr + if err == nil { + value = c.emitPackedSaturatingNarrow(form.byteWidth, inputBits, outputBits, form.spec.mode, firstValue, secondValue) + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, outputBits, false) + if err != nil { + return true, false, err + } + } + value = amd64ApplyIntegerLaneMask(c, outputLanes, outputBits, value, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, outputLanes, outputBits, value, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedSaturatingNarrow(byteWidth, inputBits, outputBits int, mode amd64PackedNarrowMode, first, second string) string { + inputLanes := byteWidth * 8 / inputBits + outputLanes := byteWidth * 8 / outputBits + chunkBits := 128 + if byteWidth < 16 { + chunkBits = byteWidth * 8 + } + inputLanesPerChunk := chunkBits / inputBits + outputLanesPerChunk := chunkBits / outputBits + chunks := byteWidth * 8 / chunkBits + result := "zeroinitializer" + for chunk := 0; chunk < chunks; chunk++ { + for sourceHalf, source := range []string{second, first} { + for lane := 0; lane < inputLanesPerChunk; lane++ { + inputIndex := chunk*inputLanesPerChunk + lane + outputIndex := chunk*outputLanesPerChunk + sourceHalf*inputLanesPerChunk + lane + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", value, inputLanes, inputBits, source, inputIndex) + narrowed := c.saturatePackedInteger("%"+value, inputBits, outputBits, mode) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %s, i32 %d\n", inserted, outputLanes, outputBits, result, outputBits, narrowed, outputIndex) + result = "%" + inserted + } + } + } + return result +} + +func (c *amd64Ctx) saturatePackedInteger(value string, inputBits, outputBits int, mode amd64PackedNarrowMode) string { + minValue := -(int64(1) << (outputBits - 1)) + maxValue := (int64(1) << (outputBits - 1)) - 1 + clamped := value + if mode == amd64PackedNarrowUnsigned { + minValue = 0 + maxValue = (int64(1) << outputBits) - 1 + below := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i%d %s, 0\n", below, inputBits, clamped) + nonnegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 0, i%d %s\n", nonnegative, below, inputBits, inputBits, clamped) + clamped = "%" + nonnegative + } + above := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt i%d %s, %d\n", above, inputBits, clamped, maxValue) + capped := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %s\n", capped, above, inputBits, maxValue, inputBits, clamped) + clamped = "%" + capped + if mode == amd64PackedNarrowSigned { + below := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i%d %s, %d\n", below, inputBits, clamped, minValue) + floored := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %s\n", floored, below, inputBits, minValue, inputBits, clamped) + clamped = "%" + floored + } + narrowed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i%d %s to i%d\n", narrowed, inputBits, clamped, outputBits) + return "%" + narrowed +} diff --git a/amd64_lower_packed_align.go b/amd64_lower_packed_align.go new file mode 100644 index 000000000..fa98cd829 --- /dev/null +++ b/amd64_lower_packed_align.go @@ -0,0 +1,106 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerPackedAlignFamily(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86PackedAlignProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := c.validateConflictFunnelMemory(ins, form.x86PackedBinaryForm); err != nil { + return true, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && form.memory >= 0 && + (first.Kind == OpFP || !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple)) { + return true, false, fmt.Errorf("%w: PALIGNR m128 needs original native backing and its matching x86 target", ErrProbeNeedsContext) + } + high, err := c.loadPackedCompareBytes(second, form.byteWidth) + if err != nil { + return true, false, err + } + if form.legacy && form.memory >= 0 { + // Keep the original m128 opcode even for imm>=32. An unaligned + // volatile read cannot reproduce the legacy alignment exception. + value, err := c.emitPackedBinaryNativeMemory(first, 8, high, fmt.Sprintf("palignr $$%d,", form.immediate)) + if err != nil { + return true, false, err + } + return true, false, c.storeLegacyPackedLogical(destination, value) + } + // VEX/EVEX alignment is E4NF/E4NF.nb: the entire source tuple is + // observable, irrespective of selectors, zero masks or dead outputs. + low, err := c.loadX86PackedShiftBytes(first, form.memoryBytesOrWidth(), true) + if err != nil { + return true, false, err + } + if form.broadcast { + bytes, broadcast := form.spec.bits/8, c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> %s\n", + broadcast, bytes, low, bytes, form.byteWidth, llvmRepeatI8Mask(bytes, form.byteWidth)) + low = "%" + broadcast + } + result := c.emitPackedAlignSelectors(form, low, high) + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareBytes(destination, form.byteWidth) + if err != nil { + return true, false, err + } + } + lanes := form.byteWidth * 8 / form.spec.bits + computed := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.spec.bits, result) + prior := old + if !form.zeroing { + prior = c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.spec.bits, old) + } + masked := amd64ApplyIntegerLaneMask(c, lanes, form.spec.bits, computed, prior, mask, form.zeroing) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, form.spec.bits, masked, form.byteWidth) + result = "%" + bytes + } + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, result) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, result) +} + +func (f x86PackedAlignForm) memoryBytesOrWidth() int { + if f.memory >= 0 { + return f.memoryBytes + } + return f.byteWidth +} + +func (c *amd64Ctx) emitPackedAlignSelectors(form x86PackedAlignForm, first, second string) string { + indices := make([]int, form.byteWidth) + zeros := false + for output := range indices { + indices[output] = form.sourceByte(output) + zeros = zeros || indices[output] < 0 + } + width := form.byteWidth + if zeros { + // A second shuffle supplies constant padding without poison or one + // extract/insert chain per byte. Both numerical consumers use exactly + // the same selectors, including the known-zero large-count domain. + joined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> %s, <%d x i32> %s\n", + joined, width, first, width, second, width*2, llvmI32RangeMask(0, width*2)) + first, second, width = "%"+joined, "zeroinitializer", width*2 + for output := range indices { + if indices[output] < 0 { + indices[output] = width + } + } + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> %s, <%d x i32> %s\n", + result, width, first, width, second, form.byteWidth, llvmI32Mask(indices)) + return "%" + result +} diff --git a/amd64_lower_packed_binary_e4.go b/amd64_lower_packed_binary_e4.go new file mode 100644 index 000000000..075bac6e2 --- /dev/null +++ b/amd64_lower_packed_binary_e4.go @@ -0,0 +1,54 @@ +package plan9asm + +import "fmt" + +// emitPackedBinaryE4 shares the lane pipeline after the family's grammar and +// storage checks. Legacy memory keeps its original SSE opcode and alignment +// faults. VEX/EVEX uses E4 reads: even dead outputs retain active volatile +// accesses, while inactive lanes are suppressed. The operation callback owns +// only register arithmetic; this helper supplies no source or memory authority. +func (c *amd64Ctx) emitPackedBinaryE4(ins Instr, form x86PackedBinaryForm, bits int, + legacyMnemonic string, operation func(lanes, bits int, first, second string) string) error { + lanes := form.byteWidth * 8 / bits + second, err := c.loadPackedCompareLanes(ins.Args[form.second], form.byteWidth, bits, false) + if err != nil { + return err + } + var mask, value string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + } + first := ins.Args[form.first] + if form.legacy && form.memory >= 0 { + value, err = c.emitPackedBinaryNativeMemory(first, bits, second, legacyMnemonic) + } else { + var input string + input, err = c.loadX86PackedShiftLanes(first, form.byteWidth, bits, form.broadcast, mask) + if err == nil { + value = operation(lanes, bits, input, second) + } + } + if err != nil { + return err + } + destination := ins.Args[form.destination] + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, bits, false) + if err != nil { + return err + } + } + value = amd64ApplyIntegerLaneMask(c, lanes, bits, value, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, bits, value, form.byteWidth) + if form.legacy { + return c.storeLegacyPackedLogical(destination, "%"+out) + } + return c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} diff --git a/amd64_lower_packed_logical.go b/amd64_lower_packed_logical.go new file mode 100644 index 000000000..8605323b5 --- /dev/null +++ b/amd64_lower_packed_logical.go @@ -0,0 +1,90 @@ +package plan9asm + +import "fmt" + +type amd64PackedLogicalMode uint8 + +const ( + amd64PackedLogicalAnd amd64PackedLogicalMode = iota + amd64PackedLogicalAndNot + amd64PackedLogicalOr + amd64PackedLogicalXor +) + +type amd64PackedLogicalSpec struct { + laneBits int + mode amd64PackedLogicalMode +} + +// Compatibility helper entry points all use the same dispatch identity and +// complete typed grammar; none has a separate permissive operand fallback. +func (c *amd64Ctx) lowerPackedFloatingLogical(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedLogicalFamily(op, ins) +} + +func (c *amd64Ctx) lowerEVEXPackedLogical(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedLogicalFamily(op, ins) +} + +func (c *amd64Ctx) lowerPackedAndNot(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedLogicalFamily(op, ins) +} + +func (c *amd64Ctx) emitPackedLogical(lanes int, spec amd64PackedLogicalSpec, first, second string) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + if spec.mode == amd64PackedLogicalAndNot { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", inverted, vectorType, second, llvmSplatInteger(lanes, spec.laneBits, ^uint64(0))) + second = "%" + inverted + } + result := c.newTmp() + op := "and" + switch spec.mode { + case amd64PackedLogicalOr: + op = "or" + case amd64PackedLogicalXor: + op = "xor" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", result, op, vectorType, second, first) + return "%" + result +} + +func (c *amd64Ctx) isGoEVEXVectorRegister(op Operand, byteWidth int) bool { + return amd64GoPackedVectorMoveRegister(op, byteWidth, c.goarch) +} + +func amd64VectorByteWidth(r Reg) int { + switch { + case isAMD64XReg(r): + return 16 + case isAMD64YReg(r): + return 32 + case isAMD64ZReg(r): + return 64 + default: + return 0 + } +} + +func amd64VectorRegisterHasWidth(op Operand, byteWidth int) bool { + return op.Kind == OpReg && amd64VectorByteWidth(op.Reg) == byteWidth +} + +func (c *amd64Ctx) bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits int, value string) string { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i%d>\n", cast, byteWidth, value, lanes, laneBits) + return "%" + cast +} + +func (c *amd64Ctx) storeVectorBytes(dst Reg, byteWidth int, value string) error { + switch byteWidth { + case 16: + return c.storeX(dst, value) + case 32: + return c.storeY(dst, value) + case 64: + return c.storeZ(dst, value) + default: + return fmt.Errorf("unsupported packed vector width %d", byteWidth) + } +} diff --git a/amd64_lower_packed_move_frame_test.go b/amd64_lower_packed_move_frame_test.go new file mode 100644 index 000000000..eb0225950 --- /dev/null +++ b/amd64_lower_packed_move_frame_test.go @@ -0,0 +1,124 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86PackedMoveFrameRejectsUndeclaredSlots(t *testing.T) { + for _, op := range []string{"VMOVAPS", "VMOVAPD", "VMOVUPS", "VMOVUPD", "VMOVDQA32", "VMOVDQA64", "VMOVDQU8", "VMOVDQU16", "VMOVDQU32", "VMOVDQU64"} { + for _, reg := range []string{"X0", "Y0", "Z0"} { + for _, instruction := range []string{fmt.Sprintf("%s %s,K7,out+0(FP)", op, reg), fmt.Sprintf("%s out+0(FP),K7,%s", op, reg)} { + for _, arch := range []string{"386", "amd64"} { + file, err := Parse(ArchAMD64, "TEXT framebounds(SB),4,$0-8\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{ + "framebounds": {Name: "framebounds", Ret: I64, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}}}, + }}) + if err == nil { + t.Errorf("%s accepted %s beyond a single 8-byte frame slot", arch, instruction) + } + } + } + } + } +} + +func TestX86PackedMoveFrameSlotsRuntime(t *testing.T) { + x86PackedMoveFrameSlots(t, true) +} + +func TestX86PackedMoveFrameSlotsObjects(t *testing.T) { + x86PackedMoveFrameSlots(t, false) +} + +func x86PackedMoveFrameSlots(t *testing.T, execute bool) { + t.Helper() + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + for index, spec := range x86PackedMoveRuntimeRows { + name := fmt.Sprintf("packedframe%d", index) + body := fmt.Sprintf("TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),DI\nMOVQ mask+24(FP),BX\nKMOVQ BX,K7\nVMOVUPS (DI),X0\n%s a+8(FP),K7,X0\nVMOVUPS X0,(DI)\nVMOVUPS 16(DI),X1\n%s X1,K7,a+8(FP)\nMOVQ a+8(FP),AX\nMOVQ AX,16(DI)\nMOVQ b+16(FP),AX\nMOVQ AX,24(DI)\nRET\n", name, spec.op, spec.op) + sig := FuncSig{Name: name, Args: []LLVMType{Ptr, I64, I64, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}, {Offset: 16, Type: I64, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + if x86PackedMoveRuntimeNativeRow(spec.op, true) { + // Preserve each old fixture as an explicit original-Go accepted + // Context boundary, not an invented aligned LLVM aggregate. + goAssemblerOracleOutput(t, "amd64", body, "", true) + file, err := Parse(ArchAMD64, body) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{name: sig}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("virtual aligned FP fixture %s: %v", spec.op, err) + } + continue + } + source.WriteString(body) + sigs[name] = sig + fmt.Fprintf(&declarations, "extern void %s(void *,uint64_t,uint64_t,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d)) return %d;\n", name, spec.bits/8, index+1) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "x86_64-w64-windows-gnu"} { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "packed-frame.ll", "packed-frame.o", ir) + } + if !execute { + t.Log("complete original frame source compiled for all four targets; native execution has its separate runtime owner") + return + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; object compilation above is required on every host") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := `#include +#include +#include +` + declarations.String() + ` +static int check(void (*fn)(void *,uint64_t,uint64_t,uint64_t),int lane) { + uint64_t words[]={UINT64_C(0x123456789abcdef0),UINT64_C(0xfedcba9876543210)}; + uint64_t masks[]={0,1,UINT64_C(0xaaaa),UINT64_MAX}; + uint8_t frame[16]; memcpy(frame,words,16); + for(int m=0;m<4;m++) { + _Alignas(64) uint8_t out[32],old[32]; + for(int i=0;i<32;i++) out[i]=old[i]=(uint8_t)(19+i*7); + fn(out,words[0],words[1],masks[m]); + for(int i=0;i<16;i++) { + int active=(masks[m]>>(i/lane))&1; + if(out[i]!=(active ? frame[i] : old[i]) || out[16+i]!=(active ? old[16+i] : frame[i])) { + fprintf(stderr,"packed FP mismatch lane=%d mask=%d byte=%d\n",lane,m,i);return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_move_frame", triple, ir, mainC, prefix) +} diff --git a/amd64_lower_packed_move_memory_test.go b/amd64_lower_packed_move_memory_test.go new file mode 100644 index 000000000..3b503aae4 --- /dev/null +++ b/amd64_lower_packed_move_memory_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86PackedMoveMaskedMemoryRuntime(t *testing.T) { + x86PackedMoveMaskedMemoryRuntime(t, false) +} + +func x86PackedMoveMaskedMemoryRuntime(t *testing.T, native bool) { + t.Helper() + var execution x86FloatCompareExecution + if native { + execution = x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + } else { + execution = x86ScalarCountHostExecution(t) + } + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, spec := range x86PackedMoveRuntimeRows { + if x86PackedMoveRuntimeNativeRow(spec.op, true) != native { + continue + } + for width, reg := range []string{"X0", "Y0", "Z0"} { + bytes := 16 << width + name := fmt.Sprintf("packedmemory%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ in+0(FP),BX\nMOVQ mem+8(FP),AX\nMOVQ out+16(FP),DI\nMOVQ mask+24(FP),CX\nKMOVQ CX,K7\nVMOVUPS (BX),%s\n%s (AX),K7,%s\nVMOVUPS %s,(DI)\nVMOVUPS (BX),%s\n%s.Z (AX),K7,%s\nVMOVUPS %s,%d(DI)\nVMOVUPS (BX),%s\n%s %s,K7,(AX)\nRET\n", name, reg, spec.op, reg, reg, reg, spec.op, reg, reg, bytes, reg, spec.op, reg) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(const void *,void *,void *,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d,%d)) return %d;\n", name, bytes, spec.bits/8, index) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir := x86PackedMoveRuntimeIR(t, file, execution, sigs, native, "packed-move-masked-memory") + mainC := `#include +#include +#include +` + declarations.String() + ` +static int check(void (*fn)(const void *,void *,void *,uint64_t),int bytes,int lane) { + uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX}; + for (int m=0;m<4;m++) { + _Alignas(64) uint8_t in[64],mem[64],out[128],old[64]; + for(int i=0;i<64;i++) { in[i]=(uint8_t)(17+i*3); mem[i]=old[i]=(uint8_t)(255-i*7); } + fn(in,mem,out,masks[m]); + for(int i=0;i>(i/lane))&1; + if(out[i]!=(active ? old[i] : in[i]) || out[bytes+i]!=(active ? old[i] : 0) || mem[i]!=(active ? in[i] : old[i])) { + fprintf(stderr,"packed move bytes=%d lane=%d mask=%d byte=%d mismatch\n",bytes,lane,m,i);return 1; + } + } + if(!masks[m]) { fn(in,0,out,0); if(memcmp(in,out,bytes)) return 1; for(int i=0;i= 3 { + input = prefix + "1" + } + op, mask := spec.op, "" + if form%3 != 0 { + mask = "K7," + } + if form%3 == 2 { + op += ".Z" + } + fmt.Fprintf(&source, "%s %s,%s%s0\nVMOVUPS Z0,(DI)\nVMOVUPS Y0,64(DI)\nVMOVUPS X0,96(DI)\nRET\n", op, input, mask, prefix) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d,%d,%d)) return %d;\n", name, 16< +#include +` + declarations.String() + ` +static int check(void (*fn)(const void *,const void *,void *,uint64_t),int bytes,int lane,int form) { + uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX}; + for(int m=0;m<4;m++) { + _Alignas(64) uint8_t in[64],old[64],out[112]; + for(int i=0;i<64;i++) {in[i]=(uint8_t)(17+i*3);old[i]=(uint8_t)(255-i*7);} + fn(in,old,out,masks[m]); + for(int view=0;view<3;view++) for(int i=0;i<(64>>view);i++) { + int active=form==0 || ((masks[m]>>(i/lane))&1); + uint8_t want=i>=bytes ? 0 : active ? in[i] : form==2 ? 0 : old[i]; + int offset=view==0 ? 0 : view==1 ? 64 : 96; + if(out[offset+i]!=want) {fprintf(stderr,"packed views width=%d lane=%d form=%d mask=%d view=%d byte=%d got=%d want=%d\n",bytes,lane,form,m,view,i,out[offset+i],want);return 1;} + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + execution.compileAndRunLLVM(t, "packed-move-views", ir, mainC, "-O2") +} + +func x86PackedMoveRuntimeIR(t *testing.T, file *File, execution x86FloatCompareExecution, sigs map[string]FuncSig, native bool, name string) string { + t.Helper() + options := Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + if native { + return ir // Native memory faults and original ISA attributes stay intact. + } + profiles := make(map[string]string, len(file.Funcs)) + for _, function := range file.Funcs { + profile := "" + if x86ScalarFloatIntegerNativeTarget("amd64", execution.triple) || + x86ScalarFloatIntegerNativeTarget("386", execution.triple) { + profile = inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", function) + } + profiles[function.Sym] = profile + } + // On every host, including ARM64, reject native bodies. On x86 also remove + // only each complete function's exact checked ISA attributes for this + // numerical oracle; a Z-register spelling must not require an AVX512 host. + portable, err := x86PackedShiftPortableFeatureProjection(ir, profiles) + if err != nil { + t.Fatal(err) + } + t.Run("amd64-object", func(t *testing.T) { + const triple = "x86_64-unknown-linux-gnu" + original := ir + if execution.triple != triple { + options.TargetTriple = triple + original, err = Translate(file, options) + if err != nil { + t.Fatal(err) + } + } + // Compile the unprojected public module with its original features. + x86ResultInitializationObject(t, execution.llc, triple, original, name) + }) + return portable +} diff --git a/amd64_lower_packed_numeric_conversion.go b/amd64_lower_packed_numeric_conversion.go new file mode 100644 index 000000000..e069c1de9 --- /dev/null +++ b/amd64_lower_packed_numeric_conversion.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedNumericKind uint8 + +const ( + amd64PackedNumericFloat amd64PackedNumericKind = iota + amd64PackedNumericSigned + amd64PackedNumericUnsigned +) + +type amd64PackedNumericShape struct { + sourceBytes int + destinationBytes int + lanes int +} + +type amd64PackedNumericConversionSpec struct { + sourceKind amd64PackedNumericKind + destinationKind amd64PackedNumericKind + sourceBits int + destinationBits int + truncate bool + explicit string + shapes []amd64PackedNumericShape +} + +var ( + amd64PackedNumericSame32 = []amd64PackedNumericShape{ + {sourceBytes: 16, destinationBytes: 16, lanes: 4}, + {sourceBytes: 32, destinationBytes: 32, lanes: 8}, + {sourceBytes: 64, destinationBytes: 64, lanes: 16}, + } + amd64PackedNumericSame64 = []amd64PackedNumericShape{ + {sourceBytes: 16, destinationBytes: 16, lanes: 2}, + {sourceBytes: 32, destinationBytes: 32, lanes: 4}, + {sourceBytes: 64, destinationBytes: 64, lanes: 8}, + } + amd64PackedNumericWiden32To64 = []amd64PackedNumericShape{ + {sourceBytes: 16, destinationBytes: 16, lanes: 2}, + {sourceBytes: 16, destinationBytes: 32, lanes: 4}, + {sourceBytes: 32, destinationBytes: 64, lanes: 8}, + } +) + +// amd64PackedNumericConversionSpecs describes the complete Go 1.27 EVEX +// packed numeric-conversion family not covered by the legacy/VEX conversion +// handlers. Source/destination element types and physical width shapes remain +// independent axes, avoiding one lowerer per mnemonic and register width. +var amd64PackedNumericConversionSpecs = map[Op]amd64PackedNumericConversionSpec{ + "VCVTPD2QQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, sourceBits: 64, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericSame64}, + "VCVTPD2UQQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericSame64}, + "VCVTTPD2QQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, sourceBits: 64, destinationBits: 64, truncate: true, explicit: "sae", shapes: amd64PackedNumericSame64}, + "VCVTTPD2UQQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 64, truncate: true, explicit: "sae", shapes: amd64PackedNumericSame64}, + "VCVTPS2UDQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 32, destinationBits: 32, explicit: "round", shapes: amd64PackedNumericSame32}, + "VCVTTPS2UDQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 32, destinationBits: 32, truncate: true, explicit: "sae", shapes: amd64PackedNumericSame32}, + "VCVTUDQ2PS": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 32, destinationBits: 32, explicit: "round", shapes: amd64PackedNumericSame32}, + "VCVTQQ2PD": {sourceKind: amd64PackedNumericSigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericSame64}, + "VCVTUQQ2PD": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericSame64}, + "VCVTPS2QQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, sourceBits: 32, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericWiden32To64}, + "VCVTPS2UQQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 32, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericWiden32To64}, + "VCVTTPS2QQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, sourceBits: 32, destinationBits: 64, truncate: true, explicit: "sae", shapes: amd64PackedNumericWiden32To64}, + "VCVTTPS2UQQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 32, destinationBits: 64, truncate: true, explicit: "sae", shapes: amd64PackedNumericWiden32To64}, + "VCVTUDQ2PD": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 32, destinationBits: 64, shapes: amd64PackedNumericWiden32To64}, + "VCVTPD2UDQX": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, + "VCVTPD2UDQY": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}}}, + "VCVTPD2UDQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, explicit: "round", shapes: []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}}}, + "VCVTTPD2UDQX": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, truncate: true, shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, + "VCVTTPD2UDQY": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, truncate: true, shapes: []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}}}, + "VCVTTPD2UDQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, truncate: true, explicit: "sae", shapes: []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}}}, + "VCVTQQ2PSX": {sourceKind: amd64PackedNumericSigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, + "VCVTQQ2PSY": {sourceKind: amd64PackedNumericSigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}}}, + "VCVTQQ2PS": {sourceKind: amd64PackedNumericSigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, explicit: "round", shapes: []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}}}, + "VCVTUQQ2PSX": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, + "VCVTUQQ2PSY": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}}}, + "VCVTUQQ2PS": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, explicit: "round", shapes: []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}}}, +} + +func (c *amd64Ctx) lowerPackedNumericConversion(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86PackedConversionSpecification(Op(base)); !known { + return false, false, nil + } + if !strings.EqualFold(string(op), string(ins.Op)) { + return true, false, fmt.Errorf("packed conversion dispatch/source mismatch: %s/%s", op, ins.Op) + } + form, _, err := x86PackedConversionProperties(ins, c.goarch) + if err != nil { + return true, false, err + } + if form.mmx || !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: packed conversion lacks native target or physical MMX/x87 caller contract", ErrProbeNeedsContext) + } + source := ins.Args[0] + if source.Kind == OpFP { + if err := c.validateFPVectorSpan(source.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + return true, false, fmt.Errorf("%w: packed FP conversion source lacks original native backing", ErrProbeNeedsContext) + } + if ins.x86AddressBits != 0 || source.Kind == OpMem && + !c.nativeMemoryOperand(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + return true, false, fmt.Errorf("%w: packed conversion lacks original native address binding", ErrProbeNeedsContext) + } + base = strings.ToLower(base) + mnemonic := strings.TrimSuffix(strings.TrimSuffix(base, "x"), "y") + if form.spec.legacy { + mnemonic = strings.ReplaceAll(base, "pl", "dq") + } + // Narrowing memory leaves X/Y source length ambiguous to an assembler: + // encode the source vector length, not the destination carrier width. + if form.memory >= 0 && !form.spec.legacy && !form.properties.broadcast && + form.spec.sourceBits > form.spec.destinationBits && form.shape.sourceBytes < 64 { + mnemonic += map[int]string{16: "x", 32: "y"}[form.shape.sourceBytes] + } + return true, false, c.emitNativePackedConversion(ins, x86NativePackedConversion{ + byteWidth: form.shape.destinationBytes, sourceBytes: form.shape.sourceBytes, + memoryBytes: form.memoryBytes, outputBits: form.spec.destinationBits, + lanes: form.shape.lanes, + destination: form.destination, mask: form.mask, legacy: form.spec.legacy, evex: form.evex, + broadcast: form.properties.broadcast, zeroing: form.properties.zeroing, + rounding: form.properties.rounding, sae: form.properties.sae, mnemonic: mnemonic, + }) +} + +func (c *amd64Ctx) packedNumericRoundingMode(rounding string) string { + switch rounding { + case "RN_SAE": + return "0" + case "RD_SAE": + return "1" + case "RU_SAE": + return "2" + case "RZ_SAE": + return "3" + default: + mxcsr := c.loadMXCSR() + shifted, mode := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i32 %s, 13\n", shifted, mxcsr) + fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 3\n", mode, shifted) + return "%" + mode + } +} diff --git a/amd64_lower_packed_numeric_conversion_test.go b/amd64_lower_packed_numeric_conversion_test.go new file mode 100644 index 000000000..3fee9bbf7 --- /dev/null +++ b/amd64_lower_packed_numeric_conversion_test.go @@ -0,0 +1,289 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type packedNumericConversionTestShape struct { + source string + destination string +} + +type packedNumericConversionTestSpec struct { + op string + shapes []packedNumericConversionTestShape + explicit string +} + +func go127PackedNumericConversionSpecs() []packedNumericConversionTestSpec { + same := []packedNumericConversionTestShape{{"X", "X"}, {"Y", "Y"}, {"Z", "Z"}} + widen := []packedNumericConversionTestShape{{"X", "X"}, {"X", "Y"}, {"Y", "Z"}} + return []packedNumericConversionTestSpec{ + {op: "VCVTPD2QQ", shapes: same, explicit: "round"}, + {op: "VCVTPD2UQQ", shapes: same, explicit: "round"}, + {op: "VCVTTPD2QQ", shapes: same, explicit: "sae"}, + {op: "VCVTTPD2UQQ", shapes: same, explicit: "sae"}, + {op: "VCVTPS2UDQ", shapes: same, explicit: "round"}, + {op: "VCVTTPS2UDQ", shapes: same, explicit: "sae"}, + {op: "VCVTUDQ2PS", shapes: same, explicit: "round"}, + {op: "VCVTQQ2PD", shapes: same, explicit: "round"}, + {op: "VCVTUQQ2PD", shapes: same, explicit: "round"}, + {op: "VCVTPS2QQ", shapes: widen, explicit: "round"}, + {op: "VCVTPS2UQQ", shapes: widen, explicit: "round"}, + {op: "VCVTTPS2QQ", shapes: widen, explicit: "sae"}, + {op: "VCVTTPS2UQQ", shapes: widen, explicit: "sae"}, + {op: "VCVTUDQ2PD", shapes: widen}, + {op: "VCVTPD2UDQX", shapes: []packedNumericConversionTestShape{{"X", "X"}}}, + {op: "VCVTPD2UDQY", shapes: []packedNumericConversionTestShape{{"Y", "X"}}}, + {op: "VCVTPD2UDQ", shapes: []packedNumericConversionTestShape{{"Z", "Y"}}, explicit: "round"}, + {op: "VCVTTPD2UDQX", shapes: []packedNumericConversionTestShape{{"X", "X"}}}, + {op: "VCVTTPD2UDQY", shapes: []packedNumericConversionTestShape{{"Y", "X"}}}, + {op: "VCVTTPD2UDQ", shapes: []packedNumericConversionTestShape{{"Z", "Y"}}, explicit: "sae"}, + {op: "VCVTQQ2PSX", shapes: []packedNumericConversionTestShape{{"X", "X"}}}, + {op: "VCVTQQ2PSY", shapes: []packedNumericConversionTestShape{{"Y", "X"}}}, + {op: "VCVTQQ2PS", shapes: []packedNumericConversionTestShape{{"Z", "Y"}}, explicit: "round"}, + {op: "VCVTUQQ2PSX", shapes: []packedNumericConversionTestShape{{"X", "X"}}}, + {op: "VCVTUQQ2PSY", shapes: []packedNumericConversionTestShape{{"Y", "X"}}}, + {op: "VCVTUQQ2PS", shapes: []packedNumericConversionTestShape{{"Z", "Y"}}, explicit: "round"}, + } +} + +func TestAMD64PackedNumericConversionGrammarCoversCompleteGoFamily(t *testing.T) { + expected := go127PackedNumericConversionSpecs() + if len(amd64PackedNumericConversionSpecs) != len(expected) { + t.Fatalf("packed numeric conversion grammar has %d entries, want %d", len(amd64PackedNumericConversionSpecs), len(expected)) + } + for _, want := range expected { + if _, ok := amd64PackedNumericConversionSpecs[Op(want.op)]; !ok { + t.Errorf("packed numeric conversion grammar omitted %s", want.op) + } + } +} + +func TestTranslateX86PackedNumericConversionCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 23 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + source.WriteString("TEXT packednumericconversionforms(SB),$0-0\n") + for index, spec := range go127PackedNumericConversionSpecs() { + for _, shape := range spec.shapes { + fmt.Fprintf(&source, "\t%s %s1, %s%d\n", spec.op, shape.source, shape.destination, last) + fmt.Fprintf(&source, "\t%s %d(AX), %s%d\n", spec.op, 8+index*8, shape.destination, last) + fmt.Fprintf(&source, "\t%s.BCST %d(AX), %s%d\n", spec.op, 16+index*8, shape.destination, last) + fmt.Fprintf(&source, "\t%s %s2, K1, %s%d\n", spec.op, shape.source, shape.destination, last) + fmt.Fprintf(&source, "\t%s.Z %d(AX), K2, %s%d\n", spec.op, 24+index*8, shape.destination, last) + fmt.Fprintf(&source, "\t%s.BCST.Z %d(AX), K3, %s%d\n", spec.op, 32+index*8, shape.destination, last) + } + if spec.explicit == "round" { + shape := spec.shapes[len(spec.shapes)-1] + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s %s3, %s%d\n", spec.op, rounding, shape.source, shape.destination, last) + } + fmt.Fprintf(&source, "\t%s.RN_SAE.Z %s4, K4, %s%d\n", spec.op, shape.source, shape.destination, last) + } else if spec.explicit == "sae" { + shape := spec.shapes[len(spec.shapes)-1] + fmt.Fprintf(&source, "\t%s.SAE %s3, %s%d\n", spec.op, shape.source, shape.destination, last) + fmt.Fprintf(&source, "\t%s.SAE.Z %s4, K4, %s%d\n", spec.op, shape.source, shape.destination, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "packednumericconversionforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-numeric-conversion-"+target.name+".ll", "packed-numeric-conversion-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedNumericConversionRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VCVTPD2QQ X0"}, + {goarch: "amd64", instruction: "VCVTPD2QQ X0, Y1"}, + {goarch: "amd64", instruction: "VCVTPS2QQ Y0, Y1"}, + {goarch: "amd64", instruction: "VCVTUDQ2PD Y0, Y1"}, + {goarch: "amd64", instruction: "VCVTPD2UDQX Y0, X1"}, + {goarch: "amd64", instruction: "VCVTPD2UDQY X0, X1"}, + {goarch: "amd64", instruction: "VCVTPD2UDQ Z0, Z1"}, + {goarch: "amd64", instruction: "VCVTQQ2PSX X0, Y1"}, + {goarch: "amd64", instruction: "VCVTQQ2PSY X0, X1"}, + {goarch: "amd64", instruction: "VCVTQQ2PS Z0, Z1"}, + {goarch: "amd64", instruction: "VCVTUDQ2PS.Z X0, X1"}, + {goarch: "amd64", instruction: "VCVTUDQ2PS X0, K0, X1"}, + {goarch: "amd64", instruction: "VCVTUDQ2PS.BCST X0, X1"}, + {goarch: "amd64", instruction: "VCVTPD2QQ.RN_SAE Y0, Y1"}, + {goarch: "amd64", instruction: "VCVTPD2QQ.RN_SAE 0(AX), Z1"}, + {goarch: "amd64", instruction: "VCVTTPD2QQ.RN_SAE Z0, Z1"}, + {goarch: "amd64", instruction: "VCVTPD2UDQX.RN_SAE X0, X1"}, + {goarch: "386", instruction: "VCVTPD2QQ Z8, Z1"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed numeric conversion tables", test.instruction) + } + }) + } +} + +func amd64PackedNumericConversionNativeRuntimeSemantics(t *testing.T) { + execution := x86PackedConversionNativeTools(t) + const source = `TEXT packednumericconversionsemantics(SB),$0-64 + MOVQ out+0(FP), AX + MOVQ f64+8(FP), BX + MOVQ f32+16(FP), CX + MOVQ u32+24(FP), DX + MOVQ i64+32(FP), R8 + MOVQ u64+40(FP), R9 + MOVQ old64+48(FP), R10 + MOVQ mask+56(FP), R11 + KMOVQ R11, K1 + VMOVDQU64 0(BX), Z0 + VCVTPD2QQ Z0, Z1 + VMOVDQU64 Z1, 0(AX) + VCVTPD2UQQ Z0, Z2 + VMOVDQU64 Z2, 64(AX) + VCVTPD2QQ.RU_SAE Z0, Z1 + VMOVDQU64 Z1, 128(AX) + VCVTTPD2UQQ.SAE Z0, Z2 + VMOVDQU64 Z2, 192(AX) + VMOVDQU32 0(CX), Y3 + VCVTPS2QQ Y3, Z4 + VMOVDQU64 Z4, 256(AX) + VCVTTPS2UQQ Y3, Z5 + VMOVDQU64 Z5, 320(AX) + VMOVDQU32 0(CX), Z6 + VCVTPS2UDQ.BCST 0(CX), Z7 + VMOVDQU32 Z7, 384(AX) + VCVTTPS2UDQ Z6, Z7 + VMOVDQU32 Z7, 448(AX) + VMOVDQU32 0(DX), Z8 + VCVTUDQ2PS Z8, Z9 + VMOVDQU32 Z9, 512(AX) + VCVTUDQ2PS.RD_SAE Z8, Z9 + VMOVDQU32 Z9, 576(AX) + VCVTUDQ2PD 0(DX), Z10 + VMOVDQU64 Z10, 640(AX) + VMOVDQU64 0(R8), Z11 + VCVTQQ2PD Z11, Z12 + VMOVDQU64 Z12, 704(AX) + VMOVDQU64 0(R9), Z13 + VCVTUQQ2PD Z13, Z14 + VMOVDQU64 Z14, 768(AX) + VCVTQQ2PS Z11, Y15 + VMOVDQU32 Y15, 832(AX) + VCVTUQQ2PS.RU_SAE Z13, Y16 + VMOVDQU32 Y16, 864(AX) + VMOVDQU64 0(R10), Z17 + VCVTPD2QQ.Z Z0, K1, Z17 + VMOVDQU64 Z17, 896(AX) + VMOVDQU64 0(BX), X0 + VCVTPD2UDQX X0, X18 + VMOVDQU32 X18, 960(AX) + VMOVDQU64 0(BX), Y0 + VCVTTPD2UDQY Y0, X19 + VMOVDQU32 X19, 976(AX) + STC + SETCS 992(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := execution.triple + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"packednumericconversionsemantics": { + Name: "packednumericconversionsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: I64, Index: 7, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#pragma STDC FENV_ACCESS ON +#include +#include +#include +#include +#include +#include +extern void packednumericconversionsemantics(uint8_t *, const double *, const float *, const uint32_t *, const int64_t *, const uint64_t *, const uint64_t *, uint64_t); +static uint32_t u32(const uint8_t *p){uint32_t v;memcpy(&v,p,4);return v;} static uint64_t u64(const uint8_t *p){uint64_t v;memcpy(&v,p,8);return v;} +static double rd(double x,int truncating,int fixed){if(truncating)return trunc(x);if(fixed==1)return nearbyint(x);int rc=fegetround();return rc==FE_DOWNWARD?floor(x):(rc==FE_UPWARD?ceil(x):(rc==FE_TOWARDZERO?trunc(x):nearbyint(x)));} +static float rf(float x,int truncating){if(truncating)return truncf(x);int rc=fegetround();return rc==FE_DOWNWARD?floorf(x):(rc==FE_UPWARD?ceilf(x):(rc==FE_TOWARDZERO?truncf(x):nearbyintf(x)));} +static uint64_t f64i(double x,int uns,int truncating,int fixed){double r=rd(x,truncating,fixed);if(uns){if(!(r>=0.0&&r<18446744073709551616.0))return UINT64_MAX;return (uint64_t)r;}if(!(r>=-9223372036854775808.0&&r<9223372036854775808.0))return UINT64_C(0x8000000000000000);return (uint64_t)(int64_t)r;} +static uint64_t f32i64(float x,int uns,int truncating){float r=rf(x,truncating);if(uns){if(!(r>=0.0f&&r<18446744073709551616.0f))return UINT64_MAX;return (uint64_t)r;}if(!(r>=-9223372036854775808.0f&&r<9223372036854775808.0f))return UINT64_C(0x8000000000000000);return (uint64_t)(int64_t)r;} +static uint32_t f32u32(float x,int truncating){float r=rf(x,truncating);if(!(r>=0.0f&&r<4294967296.0f))return UINT32_MAX;return (uint32_t)r;} +static uint32_t fb(float x){uint32_t u;memcpy(&u,&x,4);return u;} static uint64_t db(double x){uint64_t u;memcpy(&u,&x,8);return u;} +static float cu32f(uint32_t x){volatile uint32_t v=x;return (float)v;} static double cu32d(uint32_t x){volatile uint32_t v=x;return (double)v;} +static float ci64f(int64_t x){volatile int64_t v=x;return (float)v;} static double ci64d(int64_t x){volatile int64_t v=x;return (double)v;} +static float cu64f(uint64_t x){volatile uint64_t v=x;return (float)v;} static double cu64d(uint64_t x){volatile uint64_t v=x;return (double)v;} +int main(void){ + const double d[8]={1.5,2.5,-1.5,-2.5,9223372036854775808.0,-9223372036854775808.0,18446744073709551616.0,NAN}; + const float f[16]={1.5f,2.5f,-1.5f,-2.5f,0.0f,-0.0f,4294967296.0f,NAN,16777217.0f,2147483648.0f,-2147483904.0f,3.75f,INFINITY,-INFINITY,0x1p-149f,0x1.fffffep127f}; + const uint32_t ui32[16]={0,1,16777215,16777217,UINT32_MAX,2147483648u,3,5,7,9,11,13,15,17,19,21}; + const int64_t si64[8]={0,1,-1,INT64_C(9007199254740993),-INT64_C(9007199254740993),INT64_MAX,INT64_MIN,123456789}; + const uint64_t ui64[8]={0,1,UINT64_C(9007199254740993),UINT64_C(16777217),UINT64_MAX,UINT64_C(1)<<63,3,123456789}; + const uint64_t old[8]={10,11,12,13,14,15,16,17},mask=UINT64_C(0xa5); unsigned original=_mm_getcsr(); const int modes[4]={FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO}; + for(int rc=0;rc<4;rc++){uint8_t out[993]={0};fesetround(modes[rc]);packednumericconversionsemantics(out,d,f,ui32,si64,ui64,old,mask); + for(int i=0;i<8;i++){if(u64(out+i*8)!=f64i(d[i],0,0,0))return 10+rc;if(u64(out+64+i*8)!=f64i(d[i],1,0,0))return 20+rc; + int save=fegetround();fesetround(FE_UPWARD);uint64_t ru=f64i(d[i],0,0,0);fesetround(save);if(u64(out+128+i*8)!=ru)return 30+rc;if(u64(out+192+i*8)!=f64i(d[i],1,1,0))return 40+rc; + if(u64(out+256+i*8)!=f32i64(f[i],0,0))return 50+rc;if(u64(out+320+i*8)!=f32i64(f[i],1,1))return 60+rc;} + for(int i=0;i<16;i++){if(u32(out+384+i*4)!=f32u32(f[0],0))return 70+rc;if(u32(out+448+i*4)!=f32u32(f[i],1))return 80+rc;if(u32(out+512+i*4)!=fb(cu32f(ui32[i])))return 90+rc; + int save=fegetround();fesetround(FE_DOWNWARD);uint32_t down=fb(cu32f(ui32[i]));fesetround(save);if(u32(out+576+i*4)!=down){fprintf(stderr,"rd rc=%d i=%d got=%08x want=%08x\n",rc,i,u32(out+576+i*4),down);return 100+i;}} + for(int i=0;i<8;i++){if(u64(out+640+i*8)!=db(cu32d(ui32[i])))return 110+rc;if(u64(out+704+i*8)!=db(ci64d(si64[i])))return 120+rc;if(u64(out+768+i*8)!=db(cu64d(ui64[i])))return 130+rc;if(u32(out+832+i*4)!=fb(ci64f(si64[i])))return 140+rc; + int save=fegetround();fesetround(FE_UPWARD);uint32_t up=fb(cu64f(ui64[i]));fesetround(save);if(u32(out+864+i*4)!=up)return 150+rc;if(u64(out+896+i*8)!=(((mask>>i)&1)?f64i(d[i],0,0,0):0))return 160+rc;} + if(u32(out+960)!=f32u32((float)d[0],0)||u32(out+964)!=f32u32((float)d[1],0)||u64(out+968)!=0){fprintf(stderr,"narrow rc=%d got=%08x,%08x,%016llx want=%08x,%08x,0\n",rc,u32(out+960),u32(out+964),(unsigned long long)u64(out+968),f32u32((float)d[0],0),f32u32((float)d[1],0));return 170+rc;} + for(int i=0;i<4;i++)if(u32(out+976+i*4)!=f32u32((float)d[i],1))return 180+rc;if(out[992]!=1)return 190+rc; + } + _mm_setcsr(original);fesetround(FE_TONEAREST);return 0; +} +` + x86PackedConversionNativePipelines(t, execution, ir, mainC, nil) +} diff --git a/amd64_lower_packed_precision.go b/amd64_lower_packed_precision.go new file mode 100644 index 000000000..908828b24 --- /dev/null +++ b/amd64_lower_packed_precision.go @@ -0,0 +1,78 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPackedPrecisionConvert(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86PackedPrecisionProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + ins.Op = op + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: %s needs its matching native x86 MXCSR target", ErrProbeNeedsContext, op) + } + if c.fpScalarSource.x86SourceStack != nil || c.fpScalarSource.x86FixedLocal != nil { + return true, false, fmt.Errorf("%w: %s retains its separate source/frame owner", ErrProbeNeedsContext, op) + } + source := ins.Args[0] + if source.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(source.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + for at := source.FPOffset; at < source.FPOffset+int64(form.memoryBytes); at++ { + if slot, _, _ := c.fpVectorSlot(at); slot.Type == Ptr { + return true, false, fmt.Errorf("%w: %s cannot use opaque pointer bits as floating input", ErrProbeNeedsContext, op) + } + } + } else if form.memory >= 0 && !c.nativeMemoryOperand(ins, form.memory) { + return true, false, fmt.Errorf("%w: %s needs its original native source address", ErrProbeNeedsContext, op) + } + base, _, _ := strings.Cut(strings.ToLower(string(op)), ".") + mnemonic := base + if base == "vcvtpd2psx" || base == "vcvtpd2psy" { + mnemonic = "vcvtpd2ps" + if form.memory >= 0 && source.Kind != OpFP && !form.properties.broadcast { + // AT&T memory syntax needs the source VL, not the narrower + // destination carrier, to distinguish X/m128 from Y/m256. + mnemonic = base + } + } + return true, false, c.emitNativePackedConversion(ins, x86NativePackedConversion{ + byteWidth: form.shape.destinationBytes, sourceBytes: form.shape.sourceBytes, + memoryBytes: form.memoryBytes, outputBits: form.spec.destinationBits, logicalFPBits: form.spec.sourceBits, + lanes: form.shape.lanes, destination: form.destination, mask: form.mask, + legacy: form.spec.legacy, evex: form.evex, broadcast: form.properties.broadcast, + zeroing: form.properties.zeroing, sae: form.properties.sae, rounding: form.properties.rounding, mnemonic: mnemonic, + }) +} + +// Independent scalar FP fields supply only their checked little-endian value +// bytes. Fill unused register-container bytes with private zeros, or replicate +// one declared scalar for a broadcast; never invent a physical FP address. +func (c *amd64Ctx) loadPackedConversionFPValue(source Operand, sourceBytes, memoryBytes, bits int, broadcast bool) (string, error) { + bytes, err := c.loadFPVectorBytes(source.FPOffset, memoryBytes) + if err != nil { + return "", err + } + if broadcast { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", value, memoryBytes, bytes, bits) + lanes := sourceBytes * 8 / bits + splat := amd64SplatInteger(c, lanes, bits, "%"+value) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x float>\n", out, lanes, bits, splat, sourceBytes/4) + return "%" + out, nil + } + if memoryBytes != sourceBytes { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> %s\n", + out, memoryBytes, bytes, memoryBytes, sourceBytes, llvmI32RangeMask(0, sourceBytes)) + bytes = "%" + out + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x float>\n", value, sourceBytes, bytes, sourceBytes/4) + return "%" + value, nil +} diff --git a/amd64_lower_packed_rotate.go b/amd64_lower_packed_rotate.go new file mode 100644 index 000000000..5db46f95e --- /dev/null +++ b/amd64_lower_packed_rotate.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "fmt" + "strconv" +) + +type amd64PackedRotateSpec struct { + laneBits int + left bool + variable bool +} + +// amd64PackedRotateSpecs covers Go 1.27's complete _yvprold immediate and +// _yvblendmpd per-lane variable rotate families. +var amd64PackedRotateSpecs = map[Op]amd64PackedRotateSpec{ + "VPROLD": {laneBits: 32, left: true}, + "VPROLQ": {laneBits: 64, left: true}, + "VPRORD": {laneBits: 32}, + "VPRORQ": {laneBits: 64}, + "VPROLVD": {laneBits: 32, left: true, variable: true}, + "VPROLVQ": {laneBits: 64, left: true, variable: true}, + "VPRORVD": {laneBits: 32, variable: true}, + "VPRORVQ": {laneBits: 64, variable: true}, +} + +func (c *amd64Ctx) lowerPackedRotate(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, recognized, err := x86PackedRotateProperties(op, ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + if err := c.validateConflictFunnelMemory(ins, form.x86PackedBinaryForm); err != nil { + return true, false, err + } + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + return c.emitPackedRotateInstruction(form, ins, mask) +} + +func (c *amd64Ctx) emitPackedRotateInstruction(form x86PackedRotateForm, ins Instr, mask string) (bool, bool, error) { + spec, byteWidth := form.spec, form.byteWidth + dstArg := ins.Args[form.destination] + lanes := byteWidth * 8 / spec.laneBits + // EVEX E4 suppresses inactive memory lanes, not merely their results. + // The shared reader retains exact volatile active reads and broadcasts. + first, err := c.loadX86PackedShiftLanes(ins.Args[form.first], byteWidth, spec.laneBits, form.broadcast, mask) + if err != nil { + return true, false, err + } + var data, counts string + if spec.variable { + counts = first + data, err = c.loadPackedCompareLanes(ins.Args[form.second], byteWidth, spec.laneBits, false) + } else { + data = first + counts = amd64SplatInteger(c, lanes, spec.laneBits, strconv.Itoa(int(form.immediate)&(spec.laneBits-1))) + } + if err != nil { + return true, false, err + } + result := c.emitPackedRotate(lanes, spec.laneBits, data, counts, spec.left) + if mask != "" { + old := "zeroinitializer" + if !form.zeroing { + oldBytes, err := c.loadPackedCompareBytes(dstArg, byteWidth) + if err != nil { + return true, false, err + } + old = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + } + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(dstArg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedRotate(lanes, laneBits int, data, counts string, left bool) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + laneMask := amd64SplatInteger(c, lanes, laneBits, strconv.Itoa(laneBits-1)) + normalized := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", normalized, vectorType, counts, laneMask) + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s zeroinitializer, %%%s\n", negated, vectorType, normalized) + inverse := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", inverse, vectorType, negated, laneMask) + + low, high := c.newTmp(), c.newTmp() + if left { + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", low, vectorType, data, normalized) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", high, vectorType, data, inverse) + } else { + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", low, vectorType, data, normalized) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", high, vectorType, data, inverse) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", result, vectorType, low, high) + return "%" + result +} diff --git a/amd64_lower_packed_shift.go b/amd64_lower_packed_shift.go new file mode 100644 index 000000000..bb9cdcd8b --- /dev/null +++ b/amd64_lower_packed_shift.go @@ -0,0 +1,212 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerX86PackedShift(op Op, ins Instr) (bool, bool, error) { + form, recognized, err := x86PackedShiftProperties(op, ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + local := ins + local.Op = op + if err := x86LegacyMMXReadEmitterContext(local, c.goarch); err != nil { + return true, false, err + } + if x86PackedShiftHasNativeCount(form) && ins.Args[form.count].Kind == OpFP { + return true, false, fmt.Errorf("legacy m128 count needs actual native FP backing: %w", ErrProbeNeedsContext) + } + if form.memory >= 0 && ins.Args[form.memory].Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(ins.Args[form.memory].FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + dataArg := ins.Args[form.data] + var data string + if form.mmx { + bits, err := c.loadReg(dataArg.Reg) + if err != nil { + return true, false, err + } + data = c.bitcastI64ToIntegerLanes(64/form.laneBits, form.laneBits, bits) + } else if form.countMode == x86PackedShiftBytes { + data, err = c.loadX86PackedShiftBytes(dataArg, form.byteWidth, true) + } else { + dataMask := "" + if form.faultSuppressed && form.memory == form.data { + dataMask = mask + } + if form.memory == form.data && !form.faultSuppressed { + bytes, loadErr := c.loadX86PackedShiftBytes(dataArg, form.byteWidth, true) + err = loadErr + if err == nil { + data = c.bitcastVectorBytesToIntegerLanes(form.byteWidth, form.byteWidth*8/form.laneBits, form.laneBits, bytes) + } + } else { + data, err = c.loadX86PackedShiftLanes(dataArg, form.byteWidth, form.laneBits, form.broadcast && form.memory == form.data, dataMask) + } + } + if err != nil { + return true, false, err + } + lanes := form.byteWidth * 8 / form.laneBits + var result string + if form.countMode == x86PackedShiftBytes { + result = c.emitPackedByteShift(form.byteWidth, data, uint8(form.countOperand.Imm), form.left) + } else { + if form.countMode == x86PackedShiftPerLane { + counts, err := c.loadX86PackedShiftLanes(ins.Args[form.count], form.byteWidth, form.laneBits, form.broadcast, mask) + if err != nil { + return true, false, err + } + operation := "lshr" + if form.left { + operation = "shl" + } else if form.arithmetic { + operation = "ashr" + } + result = c.emitPerLaneVariableShift(amd64PerLaneVariableShiftSpec{ + laneBits: form.laneBits, operation: operation, arithmetic: form.arithmetic, + }, lanes, counts, data) + } else if x86PackedShiftHasNativeCount(form) { + result, err = c.emitX86PackedShiftNativeCount(form, ins.Args[form.count], data) + if err != nil { + return true, false, err + } + } else { + count, err := c.loadPackedUniformLogicalShiftCount(form.countOperand, form.mmx) + if err != nil { + return true, false, err + } + if form.arithmetic { + result = c.emitPackedArithmeticRightShift(lanes, form.laneBits, data, count) + } else { + result = c.emitPackedLogicalShift(amd64PackedLogicalShiftSpec{laneBits: form.laneBits, left: form.left}, lanes, data, count) + } + } + if form.mask >= 0 { + old, err := c.loadPackedCompareLanes(ins.Args[form.destination], form.byteWidth, form.laneBits, false) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + if form.mmx { + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", out, lanes, form.laneBits, result) + return true, false, c.storeReg(ins.Args[form.destination].Reg, "%"+out) + } + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.laneBits, result, form.byteWidth) + result = "%" + out + } + if form.legacy { + return true, false, c.storeLegacyPackedLogical(ins.Args[form.destination], result) + } + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, result) +} + +// Native reads remain observable even when the shift result is a known zero. +// Only actual EVEX mask bits can suppress the corresponding memory accesses; +// LLVM dead-value elimination is not a native fault-suppression contract. +func (c *amd64Ctx) loadX86PackedShiftLanes(source Operand, width, bits int, broadcast bool, mask string) (string, error) { + if source.Kind != OpMem && source.Kind != OpSym && source.Kind != OpFP { + return c.loadMaskedPackedCompareLanes(source, width, bits, broadcast, mask) + } + lanes := width * 8 / bits + if mask == "" { + bytes := width + if broadcast { + bytes = bits / 8 + } + value, err := c.loadX86PackedShiftBytes(source, bytes, true) + if err != nil { + return "", err + } + if !broadcast { + return c.bitcastVectorBytesToIntegerLanes(width, lanes, bits, value), nil + } + scalar := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", scalar, bytes, value, bits) + return amd64SplatInteger(c, lanes, bits, "%"+scalar), nil + } + load := func(offset int) func() (string, error) { + return func() (string, error) { + operand := source + if operand.Kind == OpFP { + operand.FPOffset += int64(offset) + bytes, err := c.loadX86PackedShiftBytes(operand, bits/8, true) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", value, bits/8, bytes, bits) + return "%" + value, nil + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(operand) + if err != nil { + return "", err + } + at, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %d\n", at, pointerType, pointer, offset) + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %%%s, align 1\n", value, bits, pointerType, at) + return "%" + value, nil + } + } + if broadcast { + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)< %s, i%d %s, i32 %d\n", inserted, lanes, bits, result, bits, value, lane) + result = "%" + inserted + } + return result, nil +} + +func (c *amd64Ctx) loadX86PackedShiftBytes(source Operand, width int, observable bool) (string, error) { + if source.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(source.FPOffset, width); err != nil { + return "", err + } + if c.classicFrame == "" { + // Every native ABI0 byte is bound; preserve its little-endian + // value without pretending independent allocas are contiguous. + return c.loadFPVectorBytes(source.FPOffset, width) + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, ptr %s, align 1\n", + value, width, c.classicFramePtr(source.FPOffset)) + return "%" + value, nil + } + if !observable || source.Kind != OpMem && source.Kind != OpSym { + return c.loadPackedCompareBytes(source, width) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, %s %s, align 1\n", value, width, pointerType, pointer) + return "%" + value, nil +} diff --git a/amd64_lower_packed_testmask.go b/amd64_lower_packed_testmask.go new file mode 100644 index 000000000..fd99afe44 --- /dev/null +++ b/amd64_lower_packed_testmask.go @@ -0,0 +1,75 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64PackedTestMaskSpec struct { + laneBits int + testZero bool +} + +var amd64PackedTestMaskSpecs = map[string]amd64PackedTestMaskSpec{ + "VPTESTMB": {laneBits: 8}, + "VPTESTMW": {laneBits: 16}, + "VPTESTMD": {laneBits: 32}, + "VPTESTMQ": {laneBits: 64}, + "VPTESTNMB": {laneBits: 8, testZero: true}, + "VPTESTNMW": {laneBits: 16, testZero: true}, + "VPTESTNMD": {laneBits: 32, testZero: true}, + "VPTESTNMQ": {laneBits: 64, testZero: true}, +} + +// lowerPackedTestMask implements the complete Go 1.27 _yvpshufbitqmb +// operand table shared by VPTESTM{B,W,D,Q} and VPTESTNM{B,W,D,Q}. +// VPTESTM sets each K bit when the corresponding source-lane intersection is +// nonzero; VPTESTNM sets it when that intersection is zero. +func (c *amd64Ctx) lowerPackedTestMask(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86PackedTestMaskProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + spec, byteWidth := form.spec, form.byteWidth + first, second := form.source, ins.Args[form.second] + destination := ins.Args[form.destination] + writeMask := "" + if form.mask >= 0 { + writeMask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + firstValue, err := c.loadX86PackedShiftLanes(first, byteWidth, spec.laneBits, form.broadcast, writeMask) + if err != nil { + return true, false, err + } + secondValue, err := c.loadPackedCompareLanes(second, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / spec.laneBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + intersection := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", intersection, vectorType, firstValue, secondValue) + predicate := "ne" + if spec.testZero { + predicate = "eq" + } + tested := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s %s %%%s, zeroinitializer\n", tested, predicate, vectorType, intersection) + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %%%s to %s\n", packed, lanes, tested, packedType) + result := "%" + packed + if lanes < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + if writeMask != "" { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, result, writeMask) + result = "%" + masked + } + return true, false, c.storeK(destination.Reg, result) +} diff --git a/amd64_lower_packed_testmask_test.go b/amd64_lower_packed_testmask_test.go new file mode 100644 index 000000000..2de7a5b5a --- /dev/null +++ b/amd64_lower_packed_testmask_test.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedTestMaskCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 gives VPTESTM{B,W,D,Q} and VPTESTNM{B,W,D,Q} the same + // _yvpshufbitqmb operand table: X/Y/Z r/m first source, a matching + // vector second source, an optional K1-K7 write mask, and a K + // destination. D/Q additionally enable scalar-memory broadcast. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedtestmaskforms(SB),$0-0\n") + for _, op := range []string{ + "VPTESTMB", "VPTESTMW", "VPTESTMD", "VPTESTMQ", + "VPTESTNMB", "VPTESTNMW", "VPTESTNMD", "VPTESTNMQ", + } { + zLast := 31 + if target.goarch == "386" { + zLast = 7 + } + fmt.Fprintf(&source, "\t%s X1, X31, K0\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), X31, K2\n", op) + fmt.Fprintf(&source, "\t%s Y1, Y31, K3\n", op) + fmt.Fprintf(&source, "\t%s 40(AX), Y31, K5\n", op) + fmt.Fprintf(&source, "\t%s Z1, Z%d, K6\n", op, zLast) + fmt.Fprintf(&source, "\t%s 104(AX), Z%d, K0\n", op, zLast) + fmt.Fprintf(&source, "\t%s 0, Z%d, K0\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s 112(AX), X31, K1, K2\n", op) + fmt.Fprintf(&source, "\t%s 120(AX), Y31, K4, K5\n", op) + fmt.Fprintf(&source, "\t%s 128(AX), Z31, K7, K0\n", op) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + fmt.Fprintf(&source, "\t%s.BCST 168(AX), X31, K1\n", op) + fmt.Fprintf(&source, "\t%s.BCST 176(AX), Y31, K3\n", op) + fmt.Fprintf(&source, "\t%s.BCST 184(AX), Z%d, K4\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST 192(AX), Y31, K2, K3\n", op) + } + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedtestmaskforms": {Name: "packedtestmaskforms", Ret: Void}, + }, + }) + if ll != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("bare test-to-mask formats invented numerical entry inputs", err) + } + want, count := 96, 0 + if target.goarch == "386" { + want = 68 + } + for _, ins := range file.Funcs[0].Instrs { + if ins.Op != OpTEXT && ins.Op != OpRET { + count++ + } + } + if count != want { + t.Fatal("original Go forms disappeared", count, want) + } + // Keep the original Go-encoded operands as emitter-only objects. + // Independent initialized API/value/fault owners supply the proof + // that these formats execute correctly with actual input facts. + ll = x86InstructionFormatObjectUnit(t, file, map[string]FuncSig{ + "packedtestmaskforms": {Name: "packedtestmaskforms", Ret: Void}, + }, target.goarch, target.triple, 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerPackedTestMask(ins.Op, ins) + }) + for _, want := range []string{"icmp ne", "icmp eq", "store i64"} { + if !strings.Contains(ll, want) { + t.Fatalf("translation omitted %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-test-mask-"+target.name+".ll", "packed-test-mask-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedTestMaskRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPTESTMD X0, X1"}, + {goarch: "amd64", instruction: "VPTESTMD X0, Y1, K1"}, + {goarch: "amd64", instruction: "VPTESTMD X0, (AX), K1"}, + {goarch: "amd64", instruction: "VPTESTMD X0, X1, K0, K1"}, + {goarch: "amd64", instruction: "VPTESTMD X0, X1, AX"}, + {goarch: "amd64", instruction: "VPTESTMD.Z X0, X1, K1"}, + {goarch: "amd64", instruction: "VPTESTMB.BCST (AX), X1, K1"}, + {goarch: "amd64", instruction: "VPTESTMW.BCST (AX), X1, K1"}, + {goarch: "amd64", instruction: "VPTESTMD.BCST X0, X1, K1"}, + {goarch: "amd64", instruction: "VPTESTNMQ X32, X1, K1"}, + {goarch: "386", instruction: "VPTESTMD X1, X2, K1, K2"}, + {goarch: "386", instruction: "VPTESTMD Z8, Z0, K1"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed test-mask forms for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_paddd_alias_test.go b/amd64_lower_paddd_alias_test.go new file mode 100644 index 000000000..fb838e9d8 --- /dev/null +++ b/amd64_lower_paddd_alias_test.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedAddGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]struct { + encoding x86PackedLogicalEncoding + laneBits int + mode amd64PackedAddMode + broadcastBytes int + }{ + "PADDB": {x86PackedLogicalLegacyInteger, 8, amd64PackedAddWrap, 0}, + "PADDW": {x86PackedLogicalLegacyInteger, 16, amd64PackedAddWrap, 0}, + "PADDL": {x86PackedLogicalLegacyInteger, 32, amd64PackedAddWrap, 0}, + "PADDD": {x86PackedLogicalLegacyInteger, 32, amd64PackedAddWrap, 0}, + "PADDQ": {x86PackedLogicalLegacyFP, 64, amd64PackedAddWrap, 0}, + "PADDSB": {x86PackedLogicalLegacyInteger, 8, amd64PackedAddSignedSaturating, 0}, + "PADDSW": {x86PackedLogicalLegacyInteger, 16, amd64PackedAddSignedSaturating, 0}, + "PADDUSB": {x86PackedLogicalLegacyInteger, 8, amd64PackedAddUnsignedSaturating, 0}, + "PADDUSW": {x86PackedLogicalLegacyInteger, 16, amd64PackedAddUnsignedSaturating, 0}, + "VPADDB": {x86PackedLogicalVEXOrEVEX, 8, amd64PackedAddWrap, 0}, + "VPADDW": {x86PackedLogicalVEXOrEVEX, 16, amd64PackedAddWrap, 0}, + "VPADDD": {x86PackedLogicalVEXOrEVEX, 32, amd64PackedAddWrap, 4}, + "VPADDQ": {x86PackedLogicalVEXOrEVEX, 64, amd64PackedAddWrap, 8}, + "VPADDSB": {x86PackedLogicalVEXOrEVEX, 8, amd64PackedAddSignedSaturating, 0}, + "VPADDSW": {x86PackedLogicalVEXOrEVEX, 16, amd64PackedAddSignedSaturating, 0}, + "VPADDUSB": {x86PackedLogicalVEXOrEVEX, 8, amd64PackedAddUnsignedSaturating, 0}, + "VPADDUSW": {x86PackedLogicalVEXOrEVEX, 16, amd64PackedAddUnsignedSaturating, 0}, + } + addCount := 0 + for _, spec := range x86PackedArithmeticSpecs { + if spec.family == x86PackedArithmeticAdd { + addCount++ + } + } + if addCount != len(expected) { + t.Fatal("complete packed-add grammar inventory changed", addCount) + } + for op, want := range expected { + got, known := x86PackedArithmeticSpecForOp(op, x86PackedArithmeticAdd) + if !known || got.encoding != want.encoding || got.laneBits != want.laneBits || got.mode != want.mode || + got.broadcastBytes != want.broadcastBytes || got.zeroing != (want.encoding == x86PackedLogicalVEXOrEVEX) { + t.Errorf("packed-add grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86PADDDGoAliasCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastX := 15 + if target.goarch == "386" { + lastX = 7 + } + var source strings.Builder + source.WriteString("TEXT padddaliasforms(SB),$0-0\n") + for _, op := range []string{"PADDL", "PADDD"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M7\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M7\n", op) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, lastX) + fmt.Fprintf(&source, "\t%s 16(AX), X%d\n", op, lastX) + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "padddaliasforms", target.goarch, target.triple) + if !strings.Contains(ir, "add <4 x i32>") { + t.Fatalf("PADDL/PADDD lowering omitted dword addition:\n%s", ir) + } + if target.goarch == "amd64" && !strings.Contains(ir, "bitcast i64") { + t.Fatalf("PADDL/PADDD lowering omitted MMX forms:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "paddd-alias-"+target.name+".ll", "paddd-alias-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PADDDGoAliasRejectsFormsOutsidePADDLTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PADDD M0, X1"}, + {goarch: "amd64", instruction: "PADDD X0, M1"}, + {goarch: "amd64", instruction: "PADDD Y0, Y1"}, + {goarch: "amd64", instruction: "PADDD.Z X0, X1"}, + {goarch: "386", instruction: "PADDD M0, M1"}, + {goarch: "386", instruction: "PADDL M0, M1"}, + {goarch: "386", instruction: "PADDD X0, X8"}, + {goarch: "386", instruction: "PADDD 8(R9), X0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside PADDD/PADDL's Go table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64PADDDGoAliasRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86PADDDGoAliasRuntimeSource + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "padddaliassemantics": { + Name: "padddaliassemantics", Args: []LLVMType{Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void padddaliassemantics(uint8_t *, uint64_t, uint64_t); +int main(void) { + uint8_t out[9] = {0}; + uint64_t a = UINT64_C(0xffffffff00000002); + uint64_t b = UINT64_C(0x0000000200000003); + padddaliassemantics(out, a, b); + uint64_t got; + memcpy(&got, out, 8); + if (got != UINT64_C(0x0000000100000005)) return 10; + if (out[8] != 1) return 11; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "paddd_alias_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_parallel_bits.go b/amd64_lower_parallel_bits.go new file mode 100644 index 000000000..ed79e491c --- /dev/null +++ b/amd64_lower_parallel_bits.go @@ -0,0 +1,67 @@ +package plan9asm + +import "fmt" + +// Both BMI2 families share the complete three-word grammar with ANDN. +// Their operation and preserved FLAGS remain separate. +func (c *amd64Ctx) lowerParallelBitDepositExtract(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86TernaryGPProperties(op, ins, c.goarch) + if !known || form.spec.operation == x86TernaryANDN { + return false, false, nil + } + if err != nil { + return true, false, err + } + bits := form.spec.bits + typ := amd64IntegerTypeForBits(bits) + mask, err := c.evalSizedNativeGPSource(form.first, typ, ins.x86AddressBits) + if err != nil { + return true, false, err + } + source, err := c.evalIntSized(form.second, typ) + if err != nil { + return true, false, err + } + result := c.emitParallelBitOperation(form.spec.operation == x86TernaryPDEP, bits, mask, source) + return true, false, c.storeRegSized(form.destination.Reg, typ, result) +} + +// emitParallelBitOperation emits target-independent LLVM so translated code +// retains PDEP/PEXT semantics even when the LLVM execution host lacks BMI2. +// cursor is at most bit before each iteration, so every variable shift stays +// strictly below the integer width and cannot produce LLVM poison. +func (c *amd64Ctx) emitParallelBitOperation(deposit bool, bits int, mask, source string) string { + typ := fmt.Sprintf("i%d", bits) + result := "0" + cursor := "0" + for bit := 0; bit < bits; bit++ { + shiftedMask := c.newTmp() + maskBit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %d\n", shiftedMask, typ, mask, bit) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 1\n", maskBit, typ, shiftedMask) + + sourceShift := cursor + destinationShift := fmt.Sprintf("%d", bit) + if !deposit { + sourceShift = fmt.Sprintf("%d", bit) + destinationShift = cursor + } + shiftedSource := c.newTmp() + sourceBit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", shiftedSource, typ, source, sourceShift) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 1\n", sourceBit, typ, shiftedSource) + placed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %s\n", placed, typ, sourceBit, destinationShift) + enabledMask := c.newTmp() + enabled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s 0, %%%s\n", enabledMask, typ, maskBit) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", enabled, typ, placed, enabledMask) + nextResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %s, %%%s\n", nextResult, typ, result, enabled) + result = "%" + nextResult + nextCursor := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %%%s\n", nextCursor, typ, cursor, maskBit) + cursor = "%" + nextCursor + } + return result +} diff --git a/amd64_lower_permute.go b/amd64_lower_permute.go new file mode 100644 index 000000000..f1dc1b579 --- /dev/null +++ b/amd64_lower_permute.go @@ -0,0 +1,11 @@ +package plan9asm + +// Dword variable permutations share the complete full-vector data/index +// envelope and E4NF reads with qword variable permutations. +func (c *amd64Ctx) lowerVariableDwordPermute(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86VariableDwordPermuteProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return c.lowerElementPermute(form, ins) +} diff --git a/amd64_lower_permute128.go b/amd64_lower_permute128.go new file mode 100644 index 000000000..b2d361219 --- /dev/null +++ b/amd64_lower_permute128.go @@ -0,0 +1,53 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPermute128(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86Permute128Properties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := c.validateConflictFunnelMemory(ins, form.x86PackedBinaryForm); err != nil { + return true, false, err + } + // Type6 has no selector-based fault suppression: even two zero halves + // and dead results retain the complete unaligned m256 native access. + first, err := c.loadX86PackedShiftBytes(ins.Args[form.first], 32, true) + if err != nil { + return true, false, err + } + second, err := c.loadPackedCompareBytes(ins.Args[form.second], 32) + if err != nil { + return true, false, err + } + a, b, selected := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", a, first) + fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", b, second) + var indices []int + var zeros []string + hasZero := false + for lane := 0; lane < 4; lane++ { + at := form.sourceByte(lane * 8) + zero := at < 0 + if zero { + at = 0 // A constant select below does not observe this lane. + } + indices = append(indices, at/8) + zeros = append(zeros, fmt.Sprint("i1 ", zero)) + hasZero = hasZero || zero + } + fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i64> %%%s, <4 x i64> %%%s, <4 x i32> %s\n", + selected, a, b, llvmI32Mask(indices)) + if hasZero { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <4 x i1> <%s>, <4 x i64> zeroinitializer, <4 x i64> %%%s\n", + value, strings.Join(zeros, ", "), selected) + selected = value + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i64> %%%s to <32 x i8>\n", out, selected) + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], 32, "%"+out) +} diff --git a/amd64_lower_permute128_test.go b/amd64_lower_permute128_test.go new file mode 100644 index 000000000..15b029cd6 --- /dev/null +++ b/amd64_lower_permute128_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "errors" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86Permute128CompleteGoAssemblerForms(t *testing.T) { + const source = `TEXT permute128forms(SB),$0-0 + VPERM2F128 $0, Y0, Y1, Y2 + VPERM2F128 $255, 8(AX), Y15, Y15 + VPERM2I128 $0, Y0, Y1, Y2 + VPERM2I128 $255, 8(AX), Y15, Y15 + RET +` + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func permute128forms()", "amd64", target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original bare format acquired unknown Y bytes or a native pointer", api, err) + } + } + // Preserve all four original format rows and their target objects + // without claiming initialized public or executable semantics. + ll := x86InstructionFormatObjectUnit(t, file, + map[string]FuncSig{"permute128forms": {Name: "permute128forms", Ret: Void}}, + "amd64", target.triple, 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerPermute128(ins.Op, ins) + }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "permute-128-"+target.name+".ll", "permute-128-"+target.name+".o", ll) + }) + } + requireX86GoAssemblerResult(t, "386", source, false) +} + +func TestTranslateX86Permute128RejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERM2F128 $-1, Y0, Y1, Y2", + "VPERM2F128 $256, Y0, Y1, Y2", + "VPERM2I128 $-1, Y0, Y1, Y2", + "VPERM2I128 $256, Y0, Y1, Y2", + "VPERM2I128 $1, X0, Y1, Y2", + "VPERM2I128 $1, Y0, 8(AX), Y2", + "VPERM2I128 $1, Y0, X1, Y2", + "VPERM2I128 $1, Y0, Y1, X2", + "VPERM2I128.Z $1, Y0, Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86Permute128Rejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86Permute128Rejected(t, "386", "i386-unknown-linux-gnu", "VPERM2I128 $1, Y0, Y1, Y2") +} + +func assertX86Permute128Rejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPERM2F128/VPERM2I128 table for %s", instruction, goarch) + } +} + +func TestAMD64Permute128RuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT permute128integer(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + VMOVDQU (BX), Y0 + VMOVDQU (CX), Y1 + VPERM2I128 $0x21, Y0, Y1, Y2 + VMOVDQU Y2, (AX) + VZEROUPPER + RET +TEXT permute128float(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + VMOVDQU (BX), Y0 + VMOVDQU (CX), Y1 + VPERM2F128 $0x12, Y0, Y1, Y2 + VMOVDQU Y2, (AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "permute128integer": {Name: "permute128integer", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + "permute128float": {Name: "permute128float", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void permute128integer(uint8_t *, const uint8_t *, const uint8_t *); +extern void permute128float(uint8_t *, const uint8_t *, const uint8_t *); +int main(void) { + uint8_t first[32], second[32], out[32]; + for (int i = 0; i < 32; i++) { first[i] = (uint8_t)i; second[i] = (uint8_t)(100+i); out[i] = 0; } + permute128integer(out, first, second); + for (int i = 0; i < 16; i++) if (out[i] != second[i+16]) return 10+i; + for (int i = 16; i < 32; i++) if (out[i] != first[i-16]) return 50+i; + permute128float(out, first, second); + for (int i = 0; i < 16; i++) if (out[i] != first[i]) return 90+i; + for (int i = 16; i < 32; i++) if (out[i] != second[i]) return 130+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "permute_128", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_permute_index_family_test.go b/amd64_lower_permute_index_family_test.go new file mode 100644 index 000000000..2b7d8921c --- /dev/null +++ b/amd64_lower_permute_index_family_test.go @@ -0,0 +1,271 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64IndexedPermuteGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64IndexedPermuteSpec{ + "VPERMB": {laneBits: 8, mode: amd64IndexedPermuteSingle}, + "VPERMW": {laneBits: 16, mode: amd64IndexedPermuteSingle}, + "VPERMI2B": {laneBits: 8, mode: amd64IndexedPermuteI2}, + "VPERMI2W": {laneBits: 16, mode: amd64IndexedPermuteI2}, + "VPERMI2D": {laneBits: 32, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2PS": {laneBits: 32, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2Q": {laneBits: 64, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2PD": {laneBits: 64, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMT2B": {laneBits: 8, mode: amd64IndexedPermuteT2}, + "VPERMT2W": {laneBits: 16, mode: amd64IndexedPermuteT2}, + "VPERMT2D": {laneBits: 32, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2PS": {laneBits: 32, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2Q": {laneBits: 64, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2PD": {laneBits: 64, mode: amd64IndexedPermuteT2, broadcast: true}, + } + if len(amd64IndexedPermuteSpecs) != len(expected) { + t.Fatalf("indexed-permute grammar has %d entries, want %d", len(amd64IndexedPermuteSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64IndexedPermuteSpecs[op]; !ok { + t.Errorf("indexed-permute grammar omitted %s", op) + } else if got != want { + t.Errorf("indexed-permute grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86IndexedPermuteCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT indexedpermuteforms(SB),$0-0\n") + for _, family := range []struct { + op string + broadcast bool + }{ + {op: "VPERMB"}, {op: "VPERMW"}, + {op: "VPERMI2B"}, {op: "VPERMI2W"}, + {op: "VPERMI2D", broadcast: true}, {op: "VPERMI2Q", broadcast: true}, + {op: "VPERMI2PS", broadcast: true}, {op: "VPERMI2PD", broadcast: true}, + {op: "VPERMT2B"}, {op: "VPERMT2W"}, + {op: "VPERMT2D", broadcast: true}, {op: "VPERMT2Q", broadcast: true}, + {op: "VPERMT2PS", broadcast: true}, {op: "VPERMT2PD", broadcast: true}, + } { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s1, %s2, %s%d\n", family.op, width, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s2, %s%d\n", family.op, width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s1, %s2, K1, %s%d\n", family.op, width, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 16(AX), %s2, K2, %s%d\n", family.op, width, width, last) + } + if family.broadcast { + fmt.Fprintf(&source, "\t%s.BCST 24(AX), %s2, %s%d\n", family.op, width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z 32(AX), %s2, K3, %s%d\n", family.op, width, width, last) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"indexedpermuteforms": {Name: "indexedpermuteforms", Ret: Void}}, + }) + llc := findLLVM22Tool("llc") + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original indexed format source invented initialized inputs", err) + } + ir = x86ZeroArgumentLowererObjectUnit(t, file, target.goarch, target.triple, "IndexedEmitterOnly", + func(consumer *amd64Ctx, instruction Instr) (bool, bool, error) { + return consumer.lowerIndexedPermute(instruction.Op, instruction) + }) + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "indexed-permute-"+target.name+".ll", "indexed-permute-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86IndexedPermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPERMB X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VPERMW.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VPERMT2B.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VPERMT2D.BCST X0, X1, X2"}, + {goarch: "amd64", instruction: "VPERMT2D.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VPERMI2Q X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VPERMT2PS X0, X1, 0(AX)"}, + {goarch: "amd64", instruction: "VPERMI2D.Z.BCST 0(AX), X1, K1, X2"}, + {goarch: "amd64", instruction: "VPERMB X0, X1"}, + {goarch: "386", instruction: "VPERMT2D X0, X1, K1, X2"}, + {goarch: "386", instruction: "VPERMW Z0, Z1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's indexed-permute table", test.instruction) + } + }) + } +} + +func TestAMD64IndexedPermuteRuntimeSemantics(t *testing.T) { + // Portable LLVM execution on the actual host, not original-Go/native AVX512. + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT indexedpermutesemantics(SB),$0-48 + MOVQ out+0(FP), AX + MOVQ tableA+8(FP), BX + MOVQ tableB+16(FP), CX + MOVQ indices+24(FP), DX + MOVQ old+32(FP), R8 + MOVQ mask+40(FP), R9 + KMOVQ R9, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VMOVDQU64 0(DX), Z2 + VMOVDQU64 0(R8), Z3 + STC + VPERMB Z0, Z2, Z4 + VMOVDQU64 Z4, 0(AX) + VPERMW Z0, Z2, Z4 + VMOVDQU64 Z4, 64(AX) + VPERMI2B Z0, Z1, Z2 + VMOVDQU64 Z2, 128(AX) + VMOVDQU64 0(DX), Z2 + VPERMT2B Z0, Z2, Z3 + VMOVDQU64 Z3, 192(AX) + VMOVDQU64 0(R8), Z3 + VPERMB Z0, Z2, K1, Z3 + VMOVDQU64 Z3, 256(AX) + VMOVDQU64 0(R8), Z3 + VPERMT2W.Z Z0, Z2, K1, Z3 + VMOVDQU64 Z3, 320(AX) + VMOVDQU64 0(DX), Z2 + VPERMI2Q.BCST 0(BX), Z1, K1, Z2 + VMOVDQU64 Z2, 384(AX) + VMOVDQU64 0(DX), Z2 + VMOVDQU64 0(R8), Z3 + VPERMT2D.BCST 0(BX), Z2, K1, Z3 + VMOVDQU64 Z3, 448(AX) + SETCS 512(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"indexedpermutesemantics": { + Name: "indexedpermutesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void indexedpermutesemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static uint16_t u16(const uint8_t *p) { uint16_t v; memcpy(&v,p,2); return v; } +static uint32_t u32(const uint8_t *p) { uint32_t v; memcpy(&v,p,4); return v; } +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint64_t a64[8], b64[8], indices64[8], old64[8], output64[65] = {0}; + uint8_t *a=(uint8_t *)a64, *b=(uint8_t *)b64, *indices=(uint8_t *)indices64; + uint8_t *old=(uint8_t *)old64, *out=(uint8_t *)output64; + const uint64_t mask=UINT64_C(0xa55af00ff00fa55a); + for (int i=0;i<64;i++) { + a[i]=(uint8_t)(0x11u+3u*(unsigned)i); + b[i]=(uint8_t)(0x80u+5u*(unsigned)i); + indices[i]=(uint8_t)(7u+13u*(unsigned)i); + old[i]=(uint8_t)(0xe0u-(unsigned)i); + } + indexedpermutesemantics(out,a,b,indices,old,mask); + for (int i=0;i<64;i++) { + unsigned index=indices[i]&63u; + if (out[i]!=a[index]) return 10; + unsigned two=indices[i]&127u; + uint8_t want=two<64u ? b[two] : a[two-64u]; + if (out[128+i]!=want) return 11; + want=two<64u ? old[two] : a[two-64u]; + if (out[192+i]!=want) return 12; + want=(mask>>i)&1u ? a[index] : old[i]; + if (out[256+i]!=want) return 13; + } + for (int i=0;i<32;i++) { + unsigned index=u16(indices+2*i)&31u; + if (u16(out+64+2*i)!=u16(a+2*index)) return 20; + unsigned two=u16(indices+2*i)&63u; + uint16_t want=two<32u ? u16(old+2*two) : u16(a+2*(two-32u)); + if (u16(out+320+2*i)!=(((mask>>i)&1u)?want:0)) return 21; + } + for (int i=0;i<8;i++) { + unsigned index=(unsigned)(u64(indices+8*i)&15u); + uint64_t value=index<8u ? u64(b+8*index) : u64(a); + uint64_t want=(mask>>i)&1u ? value : u64(indices+8*i); + if (u64(out+384+8*i)!=want) return 30; + } + for (int i=0;i<16;i++) { + unsigned index=u32(indices+4*i)&31u; + uint32_t value=index<16u ? u32(old+4*index) : u32(a); + uint32_t want=(mask>>i)&1u ? value : u32(old+4*i); + if (u32(out+448+4*i)!=want) return 31; + } + if (out[512]!=1) return 40; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "indexed_permute_semantics", triple, ir, mainC, nil) +} diff --git a/amd64_lower_permute_inlane.go b/amd64_lower_permute_inlane.go new file mode 100644 index 000000000..3975de8f3 --- /dev/null +++ b/amd64_lower_permute_inlane.go @@ -0,0 +1,119 @@ +package plan9asm + +import ( + "fmt" +) + +// lowerInLaneFloatingPermute implements the complete Go 1.27 _yvpermilpd +// operand family shared by VPERMILPD and VPERMILPS. The permutation preserves +// element bits and never selects across a 128-bit lane. +func (c *amd64Ctx) lowerInLaneFloatingPermute(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86InLanePermuteProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return c.lowerElementPermute(form, ins) +} + +func (c *amd64Ctx) lowerElementPermute(form x86ElementPermuteForm, ins Instr) (bool, bool, error) { + lanes := form.byteWidth * 8 / form.bits + readWidth := form.byteWidth + if form.memory >= 0 { + readWidth = form.memoryBytes + } + // E4NF retains the complete source/control tuple even for a dead result, + // an immediate selecting one lane, or a zero opmask. Share exact volatile + // unaligned reads, not an LLVM load that optimization may discard. + bytes, err := c.loadX86PackedShiftBytes(ins.Args[form.source], readWidth, true) + if err != nil { + return true, false, err + } + var input string + if form.broadcast { + scalar := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", scalar, readWidth, bytes, form.bits) + input = amd64SplatInteger(c, lanes, form.bits, "%"+scalar) + } else { + input = c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.bits, bytes) + } + var result string + if form.immediate { + result = c.emitImmediateElementPermute(input, form) + } else { + dataBytes, err := c.loadPackedCompareBytes(ins.Args[form.data], form.byteWidth) + if err != nil { + return true, false, err + } + data := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.bits, dataBytes) + if form.wholeVector { + // Whole-vector r/m is data, whereas VPERMIL r/m is control. + data, input = input, data + } + result = c.emitVariableElementPermute(data, input, form) + } + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(ins.Args[form.destination], form.byteWidth, form.bits, false) + if err != nil { + return true, false, err + } + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.bits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.bits, result, form.byteWidth) + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitImmediateElementPermute(data string, form x86ElementPermuteForm) string { + lanes, laneBits := form.byteWidth*8/form.bits, form.bits + result := "poison" + for lane := 0; lane < lanes; lane++ { + sourceLane := form.sourceByte(lane*(laneBits/8)) / (laneBits / 8) + value := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", value, lanes, laneBits, data, sourceLane) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, value, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitVariableElementPermute(data, control string, form x86ElementPermuteForm) string { + lanes, laneBits := form.byteWidth*8/form.bits, form.bits + result := "poison" + groupLanes := 128 / laneBits + if form.wholeVector { + groupLanes = lanes + } + for lane := 0; lane < lanes; lane++ { + controlLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", controlLane, lanes, laneBits, control, lane) + selectorValue := "%" + controlLane + if laneBits == 64 && !form.wholeVector { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %%%s, 1\n", shifted, controlLane) + selectorValue = "%" + shifted + } + selector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %s, %d\n", selector, laneBits, selectorValue, groupLanes-1) + index := "%" + selector + groupBase := lane / groupLanes * groupLanes + if groupBase != 0 { + withBase := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i%d %%%s, %d\n", withBase, laneBits, selector, groupBase) + index = "%" + withBase + } + value := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i%d %s\n", value, lanes, laneBits, data, laneBits, index) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_permute_inlane_test.go b/amd64_lower_permute_inlane_test.go new file mode 100644 index 000000000..53c357d60 --- /dev/null +++ b/amd64_lower_permute_inlane_test.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86InLaneFloatingPermuteCompleteGoAssemblerForms(t *testing.T) { + // VPERMILPD and VPERMILPS share Go 1.27's 18-row _yvpermilpd table: + // immediate and variable-control VEX X/Y forms, EVEX X/Y/Z forms, + // scalar-memory broadcast, and K1-K7 merge/zero masking. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + zLast := 21 + if target.goarch == "386" { + zLast = 7 + } + var src strings.Builder + src.WriteString("TEXT inlanefloatingpermuteforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMILPD", "VPERMILPS"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\t%s $-128, %s0, %s1\n", op, width, width) + fmt.Fprintf(&src, "\t%s $255, (AX), %s2\n", op, width) + fmt.Fprintf(&src, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s (AX), %s20, %s21\n", op, width, width) + fmt.Fprintf(&src, "\t%s $7, %s20, %s21\n", op, width, width) + } + fmt.Fprintf(&src, "\t%s $255, Z0, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s Z0, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.BCST $7, (AX), Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.BCST (AX), Z6, Z%d\n", op, zLast) + if target.goarch == "amd64" { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s $7, %s20, K1, %s21\n", op, width, width) + fmt.Fprintf(&src, "\t%s.Z $7, (AX), K2, %s22\n", op, width) + fmt.Fprintf(&src, "\t%s %s20, %s21, K3, %s22\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), %s21, K4, %s22\n", op, width, width) + } + } + } + src.WriteString("\tRET\n") + + physical, contextual := 52, 0 + if target.goarch == "386" { + physical, contextual = 20, 8 + } + ll := x86InLanePermuteUnboundFormatObject(t, src.String(), target.goarch, target.triple, physical, contextual) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "in-lane-floating-permute-"+target.name+".ll", "in-lane-floating-permute-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86InLaneFloatingPermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERMILPS $-129, X0, X1", + "VPERMILPD $256, X0, X1", + "VPERMILPS $-1, X20, X21", + "VPERMILPD $-1, Z0, Z1", + "VPERMILPS X0, X1", + "VPERMILPD X0, Y1, Y2", + "VPERMILPS X0, X1, (AX)", + "VPERMILPD.BCST X0, X1, X2", + "VPERMILPS.BCST $7, X0, X1", + "VPERMILPD.Z $7, X0, X1", + "VPERMILPS X0, X1, K0, X2", + "VPERMILPD.SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86InLaneFloatingPermuteRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + + for _, instruction := range []string{ + "VPERMILPS $7, Z8, Z0", + "VPERMILPD Z0, Z1, Z8", + "VPERMILPS $-1, X20, X21", + "VPERMILPD X0, X1, K1, X2", + "VPERMILPS.Z $7, X0, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86InLaneFloatingPermuteRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86InLaneFloatingPermuteRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvpermilpd forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_permute_qword.go b/amd64_lower_permute_qword.go new file mode 100644 index 000000000..affe58d1f --- /dev/null +++ b/amd64_lower_permute_qword.go @@ -0,0 +1,11 @@ +package plan9asm + +// Qword permutations reuse the complete typed operand and E4NF read machinery +// with VPERMIL, not its 128-bit selector or variable control/data ordering. +func (c *amd64Ctx) lowerQwordPermute(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86QwordPermuteProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return c.lowerElementPermute(form, ins) +} diff --git a/amd64_lower_permute_two_source.go b/amd64_lower_permute_two_source.go new file mode 100644 index 000000000..e29b383a5 --- /dev/null +++ b/amd64_lower_permute_two_source.go @@ -0,0 +1,153 @@ +package plan9asm + +import "fmt" + +type amd64IndexedPermuteMode uint8 + +const ( + amd64IndexedPermuteSingle amd64IndexedPermuteMode = iota + amd64IndexedPermuteI2 + amd64IndexedPermuteT2 +) + +type amd64IndexedPermuteSpec struct { + laneBits int + mode amd64IndexedPermuteMode + broadcast bool +} + +// amd64IndexedPermuteSpecs models the complete Go 1.27 _yvblendmpd grammar +// for the single-table VPERMB/W and both two-table VPERMI2*/VPERMT2* +// families. Broadcast capability is an opcode property, not a consequence of +// lane width alone. +var amd64IndexedPermuteSpecs = map[Op]amd64IndexedPermuteSpec{ + "VPERMB": {laneBits: 8, mode: amd64IndexedPermuteSingle}, + "VPERMW": {laneBits: 16, mode: amd64IndexedPermuteSingle}, + "VPERMI2B": {laneBits: 8, mode: amd64IndexedPermuteI2}, + "VPERMI2W": {laneBits: 16, mode: amd64IndexedPermuteI2}, + "VPERMI2D": {laneBits: 32, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2PS": {laneBits: 32, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2Q": {laneBits: 64, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2PD": {laneBits: 64, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMT2B": {laneBits: 8, mode: amd64IndexedPermuteT2}, + "VPERMT2W": {laneBits: 16, mode: amd64IndexedPermuteT2}, + "VPERMT2D": {laneBits: 32, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2PS": {laneBits: 32, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2Q": {laneBits: 64, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2PD": {laneBits: 64, mode: amd64IndexedPermuteT2, broadcast: true}, +} + +// lowerIndexedPermute implements Go 1.27's complete single- and two-table +// indexed-permute family. +func (c *amd64Ctx) lowerIndexedPermute(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, spec, known, err := x86IndexedPermuteProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + destination := ins.Args[form.destination] + first, second, byteWidth := ins.Args[form.first], ins.Args[form.second], form.byteWidth + mask := "" + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + + lanes := byteWidth * 8 / spec.laneBits + // Indexed table reads are E4NF, not destination-lane masked loads. Keep + // the complete tuple/broadcast even for zero masks or unused tables. + firstValue, err := c.loadX86PackedShiftLanes(first, byteWidth, spec.laneBits, form.broadcast, "") + if err != nil { + return true, false, err + } + secondValue, err := c.loadPackedCompareLanes(second, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + computed := "" + switch spec.mode { + case amd64IndexedPermuteSingle: + // Plan 9 lists Intel's r/m table first and its index vector second. + computed = c.emitSingleTablePermute(lanes, spec.laneBits, firstValue, secondValue) + case amd64IndexedPermuteI2: + // VPERMI2 gets indices from the old destination. Intel calls the + // Plan 9 second operand table 0 and the first operand table 1. + computed = c.emitTwoTablePermute(lanes, spec.laneBits, secondValue, firstValue, old) + case amd64IndexedPermuteT2: + // VPERMT2 gets indices from the Plan 9 second operand. The old + // destination is table 0 and the Plan 9 first operand is table 1. + computed = c.emitTwoTablePermute(lanes, spec.laneBits, old, firstValue, secondValue) + } + if form.mask >= 0 { + computed = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computed, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, computed, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitSingleTablePermute(lanes, laneBits int, table, indices string) string { + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + indexValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", indexValue, lanes, laneBits, indices, lane) + bounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", bounded, laneBits, indexValue, lanes-1) + indexI32 := "%" + bounded + if laneBits < 32 { + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %%%s to i32\n", converted, laneBits, bounded) + indexI32 = "%" + converted + } else if laneBits > 32 { + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i%d %%%s to i32\n", converted, laneBits, bounded) + indexI32 = "%" + converted + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %s\n", selected, lanes, laneBits, table, indexI32) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, selected, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitTwoTablePermute(lanes, laneBits int, table0, table1, indices string) string { + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + indexValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", indexValue, lanes, laneBits, indices, lane) + bounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", bounded, laneBits, indexValue, 2*lanes-1) + useTable1 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp uge i%d %%%s, %d\n", useTable1, laneBits, bounded, lanes) + withinTable := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", withinTable, laneBits, bounded, lanes-1) + indexI32 := "%" + withinTable + if laneBits < 32 { + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %%%s to i32\n", converted, laneBits, withinTable) + indexI32 = "%" + converted + } else if laneBits > 32 { + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i%d %%%s to i32\n", converted, laneBits, withinTable) + indexI32 = "%" + converted + } + fromTable0 := c.newTmp() + fromTable1 := c.newTmp() + selected := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %s\n", fromTable0, lanes, laneBits, table0, indexI32) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %s\n", fromTable1, lanes, laneBits, table1, indexI32) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", selected, useTable1, laneBits, fromTable1, laneBits, fromTable0) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, selected, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_permute_two_source_test.go b/amd64_lower_permute_two_source_test.go new file mode 100644 index 000000000..eb829ef16 --- /dev/null +++ b/amd64_lower_permute_two_source_test.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86TwoSourcePermuteCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 shares _yvblendmpd across VPERMI2{B,W,D,Q,PS,PD}: matching + // X/Y/Z sources and destination, optional K1-K7 masking, and .Z. The + // D/Q/PS/PD forms also accept .BCST on a memory first source. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT twosourcepermuteforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMI2B", "VPERMI2W", "VPERMI2D", "VPERMI2Q", "VPERMI2PS", "VPERMI2PD"} { + fmt.Fprintf(&source, "\t%s X1, X20, X21\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s Z1, Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X20, K1, X21\n", op) + fmt.Fprintf(&source, "\t%s.Z Y1, Y20, K2, Y21\n", op) + fmt.Fprintf(&source, "\t%s.Z 40(AX), Z20, K3, Z21\n", op) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") || strings.HasSuffix(op, "PS") || strings.HasSuffix(op, "PD") { + fmt.Fprintf(&source, "\t%s.BCST 104(AX), X20, X21\n", op) + fmt.Fprintf(&source, "\t%s.BCST 112(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s.BCST 120(AX), Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z 128(AX), Z20, K4, Z21\n", op) + } + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "twosourcepermuteforms": {Name: "twosourcepermuteforms", Ret: Void}, + }, + }) + llc := findLLVM22Tool("llc") + if ll != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original two-source format invented initialized tables or indices", err) + } + requireX86GoAssemblerResult(t, target.goarch, "#include \"textflag.h\"\n"+source.String(), true) + ll = x86ZeroArgumentLowererObjectUnit(t, file, target.goarch, target.triple, "TwoSourceEmitterOnly", + func(consumer *amd64Ctx, instruction Instr) (bool, bool, error) { + return consumer.lowerIndexedPermute(instruction.Op, instruction) + }) + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "two-source-permute-"+target.name+".ll", "two-source-permute-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86TwoSourcePermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERMI2D X0, X1", + "VPERMI2D X0, Y1, Y2", + "VPERMI2D X0, (AX), X2", + "VPERMI2D X0, X1, K0, X2", + "VPERMI2D X0, X1, AX", + "VPERMI2D.Z X0, X1, X2", + "VPERMI2D.BCST X0, X1, X2", + "VPERMI2B.BCST (AX), X1, X2", + "VPERMI2W.BCST (AX), X1, X2", + "VPERMI2D.Z.BCST (AX), X1, K1, X2", + "VPERMI2D.RN_SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86TwoSourcePermuteRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86TwoSourcePermuteRejected(t, "386", "i386-unknown-linux-gnu", "VPERMI2D X0, X1, K1, X2") + assertX86TwoSourcePermuteRejected(t, "386", "i386-unknown-linux-gnu", "VPERMI2D Z0, Z1, Z8") +} + +func assertX86TwoSourcePermuteRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPERMI2 forms", instruction) + } +} diff --git a/amd64_lower_popcount_vector.go b/amd64_lower_popcount_vector.go new file mode 100644 index 000000000..9cd7a04b9 --- /dev/null +++ b/amd64_lower_popcount_vector.go @@ -0,0 +1,73 @@ +package plan9asm + +import "fmt" + +type amd64PackedBitCountKind uint8 + +const ( + amd64PackedPopulationCount amd64PackedBitCountKind = iota + amd64PackedLeadingZeroCount +) + +type amd64PackedBitCountSpec struct { + laneBits int + kind amd64PackedBitCountKind + broadcast bool +} + +// amd64PackedBitCountSpecs models the complete Go 1.27 _yvexpandpd grammar +// for packed population-count and leading-zero-count instructions. +var amd64PackedBitCountSpecs = map[Op]amd64PackedBitCountSpec{ + "VPOPCNTB": {laneBits: 8, kind: amd64PackedPopulationCount}, + "VPOPCNTW": {laneBits: 16, kind: amd64PackedPopulationCount}, + "VPOPCNTD": {laneBits: 32, kind: amd64PackedPopulationCount, broadcast: true}, + "VPOPCNTQ": {laneBits: 64, kind: amd64PackedPopulationCount, broadcast: true}, + "VPLZCNTD": {laneBits: 32, kind: amd64PackedLeadingZeroCount, broadcast: true}, + "VPLZCNTQ": {laneBits: 64, kind: amd64PackedLeadingZeroCount, broadcast: true}, +} + +// lowerPackedBitCount implements X/Y/Z register or memory sources, K1-K7 +// merge and .Z, and each opcode's scalar-memory broadcast forms. +func (c *amd64Ctx) lowerPackedBitCount(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86PackedBitCountProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + spec, byteWidth := form.spec, form.byteWidth + source, destination := ins.Args[form.first], ins.Args[form.destination] + mask := "" + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + + lanes := byteWidth * 8 / spec.laneBits + // E4 suppresses inactive native lanes; enabled reads remain observable + // even when the computed count is dead. FP fields keep their full span. + sourceValue, err := c.loadX86PackedShiftLanes(source, byteWidth, spec.laneBits, form.broadcast, mask) + if err != nil { + return true, false, err + } + computed := c.newTmp() + if spec.kind == amd64PackedLeadingZeroCount { + fmt.Fprintf(c.b, " %%%s = call <%d x i%d> @llvm.ctlz.v%di%d(<%d x i%d> %s, i1 false)\n", + computed, lanes, spec.laneBits, lanes, spec.laneBits, lanes, spec.laneBits, sourceValue) + } else { + fmt.Fprintf(c.b, " %%%s = call <%d x i%d> @llvm.ctpop.v%di%d(<%d x i%d> %s)\n", + computed, lanes, spec.laneBits, lanes, spec.laneBits, lanes, spec.laneBits, sourceValue) + } + result := "%" + computed + if form.mask >= 0 { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} diff --git a/amd64_lower_popcount_vector_test.go b/amd64_lower_popcount_vector_test.go new file mode 100644 index 000000000..fc468ce53 --- /dev/null +++ b/amd64_lower_popcount_vector_test.go @@ -0,0 +1,207 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedPopcountCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedpopcountforms(SB),$0-0\n") + for _, op := range []string{"VPOPCNTB", "VPOPCNTW", "VPOPCNTD", "VPOPCNTQ"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 22 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s0, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s %s0, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 8(AX), K7, %s%d\n", op, width, last) + if op == "VPOPCNTD" || op == "VPOPCNTQ" { + fmt.Fprintf(&source, "\t%s.BCST 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 8(AX), K2, %s%d\n", op, width, last) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedpopcountforms": {Name: "packedpopcountforms", Ret: Void}, + }, + }) + llc := findLLVM22Tool("llc") + if ll != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original population-count format invented initialized inputs", err) + } + ll = x86ZeroArgumentLowererObjectUnit(t, file, target.goarch, target.triple, "PopcountEmitterOnly", + func(consumer *amd64Ctx, instruction Instr) (bool, bool, error) { + return consumer.lowerPackedBitCount(instruction.Op, instruction) + }) + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-popcount-"+target.name+".ll", "packed-popcount-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedPopcountRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPOPCNTB.BCST 8(AX), X1", + "VPOPCNTW.BCST 8(AX), Z1", + "VPOPCNTD.BCST X0, X1", + "VPOPCNTQ X0, Y1", + "VPOPCNTB X0, K0, X1", + "VPOPCNTW.Z X0, X1", + "VPOPCNTD.SAE X0, X1", + "VPOPCNTQ X0, 8(AX)", + "VPOPCNTB AX, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + if currentGoMinorAtLeast(27) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } + assertX86PackedPopcountRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VPOPCNTD Z8, K1, Z0", + "VPOPCNTQ Z0, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + if currentGoMinorAtLeast(27) { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } + assertX86PackedPopcountRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedPopcountRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvexpandpd forms for %s", instruction, goarch) + } +} + +func TestAMD64PackedPopcountRuntimeSemantics(t *testing.T) { + // Portable LLVM execution on the actual host, not original-Go/native AVX512. + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT packedpopcount(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 (BX), Z0 + VMOVDQU64 (CX), Z1 + VPOPCNTB Z0, Z2 + VMOVDQU64 Z2, 0(AX) + VPOPCNTW Z0, K1, Z1 + VMOVDQU64 Z1, 64(AX) + VPOPCNTD.Z Z0, K1, Z1 + VMOVDQU64 Z1, 128(AX) + VPOPCNTQ (BX), Z2 + VMOVDQU64 Z2, 192(AX) + VPOPCNTD.BCST (BX), Z2 + VMOVDQU64 Z2, 256(AX) + VPOPCNTQ.BCST.Z (BX), K1, Z1 + VMOVDQU64 Z1, 320(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedpopcount": {Name: "packedpopcount", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +extern void packedpopcount(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static unsigned pop64(uint64_t v) { + unsigned n = 0; + while (v) { n += v & 1; v >>= 1; } + return n; +} +int main(void) { + uint8_t source[64], old[64], out[384]; + const uint64_t mask = 0xa55a5aa5ULL; + for (int i = 0; i < 64; i++) { source[i] = (uint8_t)(i * 37 + 11); old[i] = (uint8_t)(0xe0 - i); } + memset(out, 0xcc, sizeof(out)); + packedpopcount(out, source, old, mask); + for (int i = 0; i < 64; i++) if (out[i] != pop64(source[i])) return 10 + i; + for (int i = 0; i < 32; i++) { + uint16_t src, prior, got; memcpy(&src, source+2*i, 2); memcpy(&prior, old+2*i, 2); memcpy(&got, out+64+2*i, 2); + uint16_t want = (mask >> i) & 1 ? (uint16_t)pop64(src) : prior; + if (got != want) return 80 + i; + } + for (int i = 0; i < 16; i++) { + uint32_t src, got; memcpy(&src, source+4*i, 4); memcpy(&got, out+128+4*i, 4); + uint32_t want = (mask >> i) & 1 ? pop64(src) : 0; + if (got != want) return 120 + i; + } + for (int i = 0; i < 8; i++) { + uint64_t src, got; memcpy(&src, source+8*i, 8); memcpy(&got, out+192+8*i, 8); + if (got != pop64(src)) return 150 + i; + } + uint32_t first32; memcpy(&first32, source, 4); + for (int i = 0; i < 16; i++) { uint32_t got; memcpy(&got, out+256+4*i, 4); if (got != pop64(first32)) return 170 + i; } + uint64_t first64; memcpy(&first64, source, 8); + for (int i = 0; i < 8; i++) { uint64_t got; memcpy(&got, out+320+8*i, 8); uint64_t want = (mask >> i) & 1 ? pop64(first64) : 0; if (got != want) return 190 + i; } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_popcount", triple, ll, mainC, nil) +} diff --git a/amd64_lower_portio.go b/amd64_lower_portio.go new file mode 100644 index 000000000..c48177ec7 --- /dev/null +++ b/amd64_lower_portio.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func x86PortScalarProperties(op Op) (input bool, typ LLVMType, ok bool) { + switch strings.ToUpper(string(op)) { + case "INB": + return true, I8, true + case "INW": + return true, I16, true + case "INL": + return true, I32, true + case "OUTB": + return false, I8, true + case "OUTW": + return false, I16, true + case "OUTL": + return false, I32, true + default: + return false, "", false + } +} + +func x86PortStringProperties(op Op) (input bool, width int, ok bool) { + switch strings.ToUpper(string(op)) { + case "INSB": + return true, 1, true + case "INSW": + return true, 2, true + case "INSL": + return true, 4, true + case "OUTSB": + return false, 1, true + case "OUTSW": + return false, 2, true + case "OUTSL": + return false, 4, true + default: + return false, 0, false + } +} + +// lowerPortIO implements the complete x86 port-I/O rows in Go's yin and +// ynone operand tables. These instructions are inherently target-specific, so +// preserving them as constrained LLVM inline assembly is both more accurate +// and safer than pretending that they are ordinary memory accesses. +func (c *amd64Ctx) lowerPortIO(op Op, ins Instr) (ok bool, terminated bool, err error) { + if input, typ, scalar := x86PortScalarProperties(op); scalar { + if c.repeatPrefix != "" { + prefix := c.repeatPrefix + c.repeatPrefix = "" + return true, false, fmt.Errorf("%s %s prefix is unsupported for %s", c.goarch, prefix, op) + } + if len(ins.Args) > 1 || (len(ins.Args) == 1 && ins.Args[0].Kind != OpImm) { + return true, false, fmt.Errorf("%s %s expects no operand or one immediate: %q", c.goarch, op, ins.Raw) + } + if len(ins.Args) == 1 && (ins.Args[0].Imm < -1<<31 || ins.Args[0].Imm > 1<<32-1) { + return true, false, fmt.Errorf("%s %s immediate is outside the Go assembler's 32-bit form: %q", c.goarch, op, ins.Raw) + } + return true, false, c.lowerScalarPortIO(strings.ToLower(string(op)), input, typ, ins.Args) + } + + input, _, stringOp := x86PortStringProperties(op) + if !stringOp { + return false, false, nil + } + prefix := c.repeatPrefix + c.repeatPrefix = "" + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, op, ins.Raw) + } + return true, false, c.lowerStringPortIO(strings.ToLower(string(op)), input, prefix) +} + +func (c *amd64Ctx) lowerScalarPortIO(asm string, input bool, typ LLVMType, args []Operand) error { + constraints := "~{dirflag},~{fpsr},~{flags}" + if input { + call := c.newTmp() + if len(args) == 0 { + port, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I16) + if err != nil { + return err + } + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(i16 %s)\n", call, typ, asm+" $1, $0", "={ax},{dx},"+constraints, port) + } else { + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", call, typ, fmt.Sprintf("%s $$%d, $0", asm, int64(args[0].Imm)), "={ax},"+constraints) + } + return c.storeRegSized(AX, typ, "%"+call) + } + + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, typ) + if err != nil { + return err + } + if len(args) == 0 { + port, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I16) + if err != nil { + return err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s, i16 %s)\n", asm+" $0, $1", "{ax},{dx},"+constraints, typ, value, port) + return nil + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s)\n", fmt.Sprintf("%s $0, $$%d", asm, int64(args[0].Imm)), "{ax},"+constraints, typ, value) + return nil +} + +func (c *amd64Ctx) lowerStringPortIO(asm string, input bool, prefix string) error { + indexReg := SI + indexConstraint := "{si}" + if input { + indexReg = DI + indexConstraint = "{di}" + } + index, err := c.loadReg(indexReg) + if err != nil { + return err + } + port, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I16) + if err != nil { + return err + } + wordType := I64 + if c.goarch == "386" { + wordType = I32 + index = c.truncI64(index, I32) + } + clobbers := "~{memory},~{dirflag},~{fpsr},~{flags}" + if prefix == "" { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(i16 %s, %s %s)\n", call, wordType, asm, "="+indexConstraint+",{dx},0,"+clobbers, port, wordType, index) + return c.storeRegSized(indexReg, wordType, "%"+call) + } + + count, err := c.loadReg(CX) + if err != nil { + return err + } + if c.goarch == "386" { + count = c.truncI64(count, I32) + } + call := c.newTmp() + resultType := fmt.Sprintf("{ %s, %s }", wordType, wordType) + asmPrefix := "rep" + if prefix == "REPN" { + asmPrefix = "repne" + } + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(i16 %s, %s %s, %s %s)\n", call, resultType, asmPrefix+"; "+asm, "="+indexConstraint+",={cx},{dx},0,1,"+clobbers, port, wordType, index, wordType, count) + nextIndex := c.newTmp() + nextCount := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 0\n", nextIndex, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 1\n", nextCount, resultType, call) + if err := c.storeRegSized(indexReg, wordType, "%"+nextIndex); err != nil { + return err + } + return c.storeRegSized(CX, wordType, "%"+nextCount) +} diff --git a/amd64_lower_qword_bit_lookup.go b/amd64_lower_qword_bit_lookup.go new file mode 100644 index 000000000..ebd37c45e --- /dev/null +++ b/amd64_lower_qword_bit_lookup.go @@ -0,0 +1,132 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64QwordBitLookupOutput uint8 + +const ( + amd64QwordBitLookupBytes amd64QwordBitLookupOutput = iota + amd64QwordBitLookupMask +) + +type amd64QwordBitLookupSpec struct { + output amd64QwordBitLookupOutput + broadcast bool +} + +// amd64QwordBitLookupSpecs groups the two AVX-512 byte-control operations +// that select bits inside corresponding qword lanes. They share the same core +// rotate/lookup grammar but produce either bytes or a packed K mask. +var amd64QwordBitLookupSpecs = map[Op]amd64QwordBitLookupSpec{ + "VPMULTISHIFTQB": {output: amd64QwordBitLookupBytes, broadcast: true}, + "VPSHUFBITQMB": {output: amd64QwordBitLookupMask}, +} + +func (c *amd64Ctx) lowerQwordBitLookup(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86QwordLookupProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + spec, byteWidth := form.spec, form.byteWidth + first, second := ins.Args[form.first], ins.Args[form.second] + destination := ins.Args[form.destination] + var writeMask string + if form.mask >= 0 { + writeMask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.memory >= 0 && first.Kind == OpMem && !c.nativeMemoryOperand(ins, form.memory) && + !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + return true, false, fmt.Errorf("%w: qword lookup needs its original native storage binding", ErrProbeNeedsContext) + } + var dataQwords, controls string + switch spec.output { + case amd64QwordBitLookupBytes: + // VPMULTISHIFTQB is E4NF: even a zero write mask retains the full + // source tuple, or the independent m64 broadcast. + dataQwords, err = c.loadX86PackedShiftLanes(first, byteWidth, 64, form.broadcast, "") + if err == nil { + controls, err = c.loadPackedCompareBytes(second, byteWidth) + } + case amd64QwordBitLookupMask: + dataQwords, err = c.loadPackedCompareLanes(second, byteWidth, 64, false) + if err == nil { + // VPSHUFBITQMB is E4 with byte-granular fault suppression. Its + // mask is not a license to suppress the other family's E4NF read. + controls, err = c.loadX86PackedShiftLanes(first, byteWidth, 8, false, writeMask) + } + } + if err != nil { + return true, false, err + } + selectedBytes := c.emitQwordBitLookupBytes(byteWidth, dataQwords, controls) + + if spec.output == amd64QwordBitLookupBytes { + if form.mask >= 0 { + old, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + selectedBytes = amd64ApplyIntegerLaneMask(c, byteWidth, 8, selectedBytes, old, writeMask, form.zeroing) + } + return true, false, c.storePackedMoveOperand(destination, byteWidth, selectedBytes) + } + + ones := c.newTmp() + predicates := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and <%d x i8> %s, %s\n", ones, byteWidth, selectedBytes, llvmSplatInteger(byteWidth, 8, 1)) + fmt.Fprintf(c.b, " %%%s = icmp ne <%d x i8> %%%s, zeroinitializer\n", predicates, byteWidth, ones) + packedType := fmt.Sprintf("i%d", byteWidth) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %%%s to %s\n", packed, byteWidth, predicates, packedType) + result := "%" + packed + if byteWidth < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + if form.mask >= 0 { + maskedResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", maskedResult, result, writeMask) + result = "%" + maskedResult + } + return true, false, c.storeK(destination.Reg, result) +} + +// emitQwordBitLookupBytes rotates each corresponding data qword right by the +// low six bits of a control byte. Its low byte is VPMULTISHIFTQB's result; +// its low bit is VPSHUFBITQMB's gathered mask bit. +func (c *amd64Ctx) emitQwordBitLookupBytes(byteWidth int, dataQwords, controls string) string { + result := "zeroinitializer" + qwords := byteWidth / 8 + for lane := 0; lane < byteWidth; lane++ { + data := c.newTmp() + controlByte := c.newTmp() + controlWide := c.newTmp() + control := c.newTmp() + right := c.newTmp() + negated := c.newTmp() + wrappedLeftCount := c.newTmp() + left := c.newTmp() + rotated := c.newTmp() + selected := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", data, qwords, dataQwords, lane/8) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", controlByte, byteWidth, controls, lane) + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", controlWide, controlByte) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 63\n", control, controlWide) + fmt.Fprintf(c.b, " %%%s = lshr i64 %%%s, %%%s\n", right, data, control) + fmt.Fprintf(c.b, " %%%s = sub i64 0, %%%s\n", negated, control) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 63\n", wrappedLeftCount, negated) + fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, %%%s\n", left, data, wrappedLeftCount) + fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", rotated, right, left) + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i8\n", selected, rotated) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i8> %s, i8 %%%s, i32 %d\n", inserted, byteWidth, result, selected, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_qword_bit_lookup_test.go b/amd64_lower_qword_bit_lookup_test.go new file mode 100644 index 000000000..e782bc6e6 --- /dev/null +++ b/amd64_lower_qword_bit_lookup_test.go @@ -0,0 +1,251 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64QwordBitLookupGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64QwordBitLookupSpec{ + "VPMULTISHIFTQB": {output: amd64QwordBitLookupBytes, broadcast: true}, + "VPSHUFBITQMB": {output: amd64QwordBitLookupMask}, + } + if len(amd64QwordBitLookupSpecs) != len(expected) { + t.Fatalf("qword-bit-lookup grammar has %d entries, want %d", len(amd64QwordBitLookupSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64QwordBitLookupSpecs[op]; !ok { + t.Errorf("qword-bit-lookup grammar omitted %s", op) + } else if got != want { + t.Errorf("qword-bit-lookup grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86QwordBitLookupCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT qwordbitlookupforms(SB),$0-0\n") + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\tVPMULTISHIFTQB %s1, %s2, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVPMULTISHIFTQB 8(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVPMULTISHIFTQB.BCST 16(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVPSHUFBITQMB %s1, %s2, K0\n", width, width) + fmt.Fprintf(&source, "\tVPSHUFBITQMB 24(AX), %s2, K7\n", width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\tVPMULTISHIFTQB %s1, %s2, K1, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVPMULTISHIFTQB.Z 32(AX), %s2, K2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVPMULTISHIFTQB.BCST.Z 40(AX), %s2, K3, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVPSHUFBITQMB %s1, %s2, K4, K7\n", width, width) + fmt.Fprintf(&source, "\tVPSHUFBITQMB 48(AX), %s2, K5, K6\n", width) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"qwordbitlookupforms": {Name: "qwordbitlookupforms", Ret: Void}}, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized qword lookup formats invented numerical/native inputs", err) + } + if target.goarch == "386" { + // The retained original source includes X31/Y31 spellings: + // Go accepts their syntax without a physical mode32 binding. + // Check that precise boundary independently; the initialized + // low-bank matrix owns successful LLVM objects for both OSes. + for _, row := range []string{"VPMULTISHIFTQB X1,X2,X31", "VPMULTISHIFTQB Y1,Y2,Y31"} { + ins := x86ScalarLaneInstruction(t, row) + _, known, err := x86QwordLookupProperties(ins.Op, ins, "386") + if !known || !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "physical mode32 binding") { + t.Fatal("original high-bank syntax invented a physical mode32 vector", row, err) + } + } + t.Log("original Go high-bank formats retained as Context; initialized low-bank tests own LLVM objects") + return + } + ir = x86InstructionFormatObjectUnit(t, file, + map[string]FuncSig{"qwordbitlookupforms": {Name: "qwordbitlookupforms", Ret: Void}}, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerQwordBitLookup(ins.Op, ins) }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "qword-bit-lookup-"+target.name+".ll", "qword-bit-lookup-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86QwordBitLookupRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPMULTISHIFTQB X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VPMULTISHIFTQB.BCST X0, X1, X2"}, + {goarch: "amd64", instruction: "VPMULTISHIFTQB.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VPMULTISHIFTQB X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VPMULTISHIFTQB X0, X1, 0(AX)"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB.Z X0, X1, K2"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB.BCST 0(AX), X1, K2"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB X0, Y1, K2"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB X0, X1, K0, K2"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB X0, X1, AX"}, + {goarch: "386", instruction: "VPMULTISHIFTQB X0, X1, K1, X2"}, + {goarch: "386", instruction: "VPSHUFBITQMB X0, X1, K1, K2"}, + {goarch: "386", instruction: "VPMULTISHIFTQB Z0, Z1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's qword-bit-lookup table", test.instruction) + } + }) + } +} + +func TestAMD64QwordBitLookupRuntimeSemantics(t *testing.T) { + // This oracle executes portable integer LLVM, not native AVX512. Use + // the actual 64-bit host rather than an unrelated Rosetta requirement. + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable runtime oracle requires its native 64-bit host counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT qwordbitlookupsemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ data+8(FP), BX + MOVQ controls+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), R8 + KMOVQ R8, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VMOVDQU64 0(DX), Z2 + STC + VPMULTISHIFTQB Z0, Z1, Z3 + VMOVDQU64 Z3, 0(AX) + VPMULTISHIFTQB Z0, Z1, K1, Z2 + VMOVDQU64 Z2, 64(AX) + VPMULTISHIFTQB.Z Z0, Z1, K1, Z3 + VMOVDQU64 Z3, 128(AX) + VPMULTISHIFTQB.BCST 0(BX), Z1, Z3 + VMOVDQU64 Z3, 192(AX) + VPSHUFBITQMB Z1, Z0, K2 + KMOVQ K2, 256(AX) + VPSHUFBITQMB Z1, Z0, K1, K3 + KMOVQ K3, 264(AX) + SETCS 272(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"qwordbitlookupsemantics": { + Name: "qwordbitlookupsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void qwordbitlookupsemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static uint64_t ror64(uint64_t value, unsigned count) { + count &= 63u; + return (value >> count) | (value << ((0u-count)&63u)); +} +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint64_t data64[8], controls64[8], old64[8], output64[35] = {0}; + uint8_t *data=(uint8_t *)data64, *controls=(uint8_t *)controls64; + uint8_t *old=(uint8_t *)old64, *out=(uint8_t *)output64; + const uint64_t mask=UINT64_C(0xa55af00ff00fa55a); + for (int q=0;q<8;q++) data64[q]=UINT64_C(0x0123456789abcdef)^((uint64_t)q*UINT64_C(0x11100f0e0d0c0b0a)); + for (int i=0;i<64;i++) { controls[i]=(uint8_t)(13u*(unsigned)i+((unsigned)i>>2)); old[i]=(uint8_t)(0xe0u-(unsigned)i); } + qwordbitlookupsemantics(out,data,controls,old,mask); + uint64_t gathered=0; + for (int i=0;i<64;i++) { + uint64_t qword=data64[i/8]; + uint8_t selected=(uint8_t)ror64(qword,controls[i]); + if (out[i]!=selected) return 10; + if (out[64+i]!=(((mask>>i)&1u)?selected:old[i])) return 11; + if (out[128+i]!=(((mask>>i)&1u)?selected:0)) return 12; + uint8_t broadcast=(uint8_t)ror64(data64[0],controls[i]); + if (out[192+i]!=broadcast) return 13; + gathered|=((qword>>(controls[i]&63u))&1u)<= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64FloatingRangeSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return c.lowerNativeFloatingHelper(op, ins) + } + if c.goarch == "386" { + return true, false, fmt.Errorf("386 %s exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's range optab: %q", c.goarch, baseOp, ins.Raw) + } + if spec.scalar && properties.broadcast { + return true, false, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 4 && len(ins.Args) != 5 { + return true, false, fmt.Errorf("%s %s expects immediate, source2, source1, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("%s %s immediate must be an unsigned byte: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 5 + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[3]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + + first, second := ins.Args[1], ins.Args[2] + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if spec.scalar { + if byteWidth != 16 { + return true, false, fmt.Errorf("%s %s scalar destination must be an X register: %q", c.goarch, baseOp, ins.Raw) + } + } else if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s packed destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoEVEXVectorRegister(destination, byteWidth) || !c.isGoEVEXVectorRegister(second, byteWidth) { + return true, false, fmt.Errorf("%s %s source1 and destination must be same-width Go vector registers: %q", c.goarch, baseOp, ins.Raw) + } + if properties.sae && !spec.scalar && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s packed SAE is available only for Z registers: %q", c.goarch, baseOp, ins.Raw) + } + if properties.broadcast { + if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source2: %q", c.goarch, baseOp, ins.Raw) + } + } else if first.Kind == OpReg { + if !c.isGoEVEXVectorRegister(first, byteWidth) { + return true, false, fmt.Errorf("%s %s source2 register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s source2 must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if properties.sae && first.Kind != OpReg { + return true, false, fmt.Errorf("%s %s.SAE requires a register source2: %q", c.goarch, baseOp, ins.Raw) + } + + mask := "" + if masked { + mask, err = c.loadK(ins.Args[3].Reg) + if err != nil { + return true, false, err + } + } + if spec.scalar { + return c.lowerScalarFloatingRange(spec, properties, ins, first, second, destination, mask) + } + return c.lowerPackedFloatingRange(spec, properties, ins, first, second, destination, byteWidth, mask) +} + +func (c *amd64Ctx) lowerPackedFloatingRange(spec amd64FloatingRangeSpec, properties amd64BinaryFloatingSuffix, ins Instr, first, second, destination Operand, byteWidth int, mask string) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + source2, err := c.loadPackedCompareLanes(first, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + source1, err := c.loadPackedCompareLanes(second, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + result := c.emitFloatingRangeBits(lanes, spec.laneBits, source1, source2, uint8(ins.Args[0].Imm)) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarFloatingRange(spec amd64FloatingRangeSpec, properties amd64BinaryFloatingSuffix, ins Instr, first, second, destination Operand, mask string) (bool, bool, error) { + source2, err := c.loadFloatingScalarBits(first, spec.laneBits) + if err != nil { + return true, false, err + } + source1, err := c.loadFloatingScalarBits(second, spec.laneBits) + if err != nil { + return true, false, err + } + source2Vector := c.newTmp() + source1Vector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", source2Vector, spec.laneBits, spec.laneBits, source2) + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", source1Vector, spec.laneBits, spec.laneBits, source1) + result := c.emitFloatingRangeBits(1, spec.laneBits, "%"+source1Vector, "%"+source2Vector, uint8(ins.Args[0].Imm)) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + secondBytes, err := c.loadX(second.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, secondBytes) + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) loadFloatingScalarBits(operand Operand, laneBits int) (string, error) { + value, err := c.loadFloatingScalarOperand(operand, laneBits) + if err != nil { + return "", err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", bits, amd64FloatingScalarType(laneBits), value, laneBits) + return "%" + bits, nil +} + +// emitFloatingRangeBits follows Intel's range operation in integer bit space +// around one ordered floating comparison. This preserves NaN payloads, quiets +// signaling NaNs, and handles signed zero and equal-magnitude opposite signs. +func (c *amd64Ctx) emitFloatingRangeBits(lanes, laneBits int, source1, source2 string, immediate uint8) string { + integerType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + floatType := amd64FloatingVectorType(lanes, laneBits) + predicateType := fmt.Sprintf("<%d x i1>", lanes) + signBit := uint64(1) << (laneBits - 1) + absMask := signBit - 1 + quietBit := uint64(0x00400000) + infinity := uint64(0x7f800000) + if laneBits == 64 { + quietBit = uint64(0x0008000000000000) + infinity = uint64(0x7ff0000000000000) + } + signVector := llvmSplatInteger(lanes, laneBits, signBit) + absVector := llvmSplatInteger(lanes, laneBits, absMask) + quietVector := llvmSplatInteger(lanes, laneBits, quietBit) + infinityVector := llvmSplatInteger(lanes, laneBits, infinity) + + abs1, abs2 := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", abs1, integerType, source1, absVector) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", abs2, integerType, source2, absVector) + + compareLeft, compareRight := source1, source2 + if immediate&3 >= 2 { + compareLeft, compareRight = "%"+abs1, "%"+abs2 + } + leftFloat, rightFloat := c.newTmp(), c.newTmp() + ordered := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", leftFloat, integerType, compareLeft, floatType) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", rightFloat, integerType, compareRight, floatType) + fmt.Fprintf(c.b, " %%%s = fcmp ole %s %%%s, %%%s\n", ordered, floatType, leftFloat, rightFloat) + selected := c.newTmp() + if immediate&1 == 0 { + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", selected, predicateType, ordered, integerType, source1, integerType, source2) + } else { + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", selected, predicateType, ordered, integerType, source2, integerType, source1) + } + result := "%" + selected + + signXor, signDifference, oppositeSigns := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", signXor, integerType, source1, source2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", signDifference, integerType, signXor, signVector) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", oppositeSigns, integerType, signDifference) + if immediate&3 < 2 { + zero1, zero2, bothZero, oppositeZero := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", zero1, integerType, abs1) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", zero2, integerType, abs2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", bothZero, predicateType, zero1, zero2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", oppositeZero, predicateType, bothZero, oppositeSigns) + zeroResult := "zeroinitializer" + if immediate&1 == 0 { + zeroResult = signVector + } + overridden := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", overridden, predicateType, oppositeZero, integerType, zeroResult, integerType, result) + result = "%" + overridden + } else { + equalMagnitude, equalOpposite := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %%%s\n", equalMagnitude, integerType, abs1, abs2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", equalOpposite, predicateType, equalMagnitude, oppositeSigns) + tieResult := "%" + abs1 + if immediate&1 == 0 { + negative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", negative, integerType, abs1, signVector) + tieResult = "%" + negative + } + overridden := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", overridden, predicateType, equalOpposite, integerType, tieResult, integerType, result) + result = "%" + overridden + } + + isNaN1, isNaN2 := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %s\n", isNaN1, integerType, abs1, infinityVector) + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %s\n", isNaN2, integerType, abs2, infinityVector) + withoutNaN1 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", withoutNaN1, predicateType, isNaN1, integerType, source2, integerType, result) + withoutQuietNaNs := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %%%s\n", withoutQuietNaNs, predicateType, isNaN2, integerType, source1, integerType, withoutNaN1) + result = "%" + withoutQuietNaNs + + signMode := immediate >> 2 & 3 + if signMode != 1 { + magnitude := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", magnitude, integerType, result, absVector) + switch signMode { + case 0: + sourceSign := c.newTmp() + signed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", sourceSign, integerType, source1, signVector) + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", signed, integerType, magnitude, sourceSign) + result = "%" + signed + case 2: + result = "%" + magnitude + case 3: + signed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", signed, integerType, magnitude, signVector) + result = "%" + signed + } + } + + quiet1, quiet2 := c.newTmp(), c.newTmp() + quietField1, quietField2 := c.newTmp(), c.newTmp() + quietClear1, quietClear2 := c.newTmp(), c.newTmp() + signaling1, signaling2 := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %s, %s\n", quiet1, integerType, source1, quietVector) + fmt.Fprintf(c.b, " %%%s = or %s %s, %s\n", quiet2, integerType, source2, quietVector) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", quietField1, integerType, source1, quietVector) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", quietField2, integerType, source2, quietVector) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", quietClear1, integerType, quietField1) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", quietClear2, integerType, quietField2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", signaling1, predicateType, isNaN1, quietClear1) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", signaling2, predicateType, isNaN2, quietClear2) + withSecondSignaling := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %%%s, %s %s\n", withSecondSignaling, predicateType, signaling2, integerType, quiet2, integerType, result) + withFirstSignaling := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %%%s, %s %%%s\n", withFirstSignaling, predicateType, signaling1, integerType, quiet1, integerType, withSecondSignaling) + return "%" + withFirstSignaling +} diff --git a/amd64_lower_range_test.go b/amd64_lower_range_test.go new file mode 100644 index 000000000..0c608ca75 --- /dev/null +++ b/amd64_lower_range_test.go @@ -0,0 +1,338 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestAMD64FloatingRangeGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64FloatingRangeSpec{ + "VRANGEPS": {laneBits: 32}, + "VRANGEPD": {laneBits: 64}, + "VRANGESS": {laneBits: 32, scalar: true}, + "VRANGESD": {laneBits: 64, scalar: true}, + } + if len(amd64FloatingRangeSpecs) != len(expected) { + t.Fatalf("floating-range grammar has %d entries, want %d", len(amd64FloatingRangeSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64FloatingRangeSpecs[op]; !ok { + t.Errorf("floating-range grammar omitted %s", op) + } else if got != want { + t.Errorf("floating-range grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86FloatingRangeCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT floatingrangeforms(SB),$0-0\n") + for _, op := range []string{"VRANGEPS", "VRANGEPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $0, %s1, %s2, %s%d\n", op, width, width, width, last) + fmt.Fprintf(&source, "\t%s $255, 8(AX), %s2, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.BCST $2, 16(AX), %s2, %s%d\n", op, width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, %s1, %s2, K1, %s%d\n", op, width, width, width, last) + fmt.Fprintf(&source, "\t%s.Z $4, 24(AX), %s2, K2, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z $5, 32(AX), %s2, K3, %s%d\n", op, width, width, last) + if width == "Z" { + fmt.Fprintf(&source, "\t%s.SAE $6, Z1, Z2, Z%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE.Z $7, Z1, Z2, K4, Z%d\n", op, last) + } + } + } + } + for _, op := range []string{"VRANGESS", "VRANGESD"} { + last := 31 + if target.goarch == "386" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $8, X1, X2, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s $9, 40(AX), X2, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE $10, X1, X2, X%d\n", op, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $11, X1, X2, K5, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.Z $12, 48(AX), X2, K6, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE.Z $13, X1, X2, K7, X%d\n", op, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86FloatingHelperOriginalFormatObject(t, source.String(), "floatingrangeforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "floating-range-"+target.name+".ll", "floating-range-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FloatingRangeRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VRANGEPS $-1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS $256, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS $0, X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VRANGEPS.BCST $0, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS.SAE $0, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGESD.SAE $0, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS.RN_SAE $0, Z0, Z1, Z2"}, + {goarch: "amd64", instruction: "VRANGEPS.Z $0, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS $0, X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VRANGESS $0, Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGESS.BCST $0, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VRANGESD $0, X0, X1, Y2"}, + {goarch: "386", instruction: "VRANGEPS $0, X0, X1, X2"}, + {goarch: "386", instruction: "VRANGEPS $0, X0, X1, K1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's floating-range table", test.instruction) + } + }) + } +} + +func qualifyAMD64FloatingRangeRuntimeSemantics(t *testing.T) { + execution := x86FloatingHelperRuntimeTools(t) + var source strings.Builder + source.WriteString(`TEXT floatingrangesemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ source2+8(FP), BX + MOVQ source1+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), R8 + KMOVQ R8, K1 + VMOVDQU32 0(BX), Z0 + VMOVDQU32 0(CX), Z1 + STC +`) + for immediate := 0; immediate < 16; immediate++ { + fmt.Fprintf(&source, "\tVRANGEPS $%d, Z0, Z1, Z2\n", immediate) + fmt.Fprintf(&source, "\tVMOVDQU32 Z2, %d(AX)\n", immediate*64) + } + source.WriteString(` VMOVDQU32 0(DX), Z2 + VRANGEPS $2, Z0, Z1, K1, Z2 + VMOVDQU32 Z2, 1024(AX) + VRANGEPS.Z $3, Z0, Z1, K1, Z2 + VMOVDQU32 Z2, 1088(AX) + VRANGEPS.BCST $2, 0(BX), Z1, Z2 + VMOVDQU32 Z2, 1152(AX) + SETCS 1216(AX) + RET +`) + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := execution.triple + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"floatingrangesemantics": { + Name: "floatingrangesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void floatingrangesemantics(uint8_t *, const uint32_t *, const uint32_t *, const uint32_t *, uint64_t); +static float f32(uint32_t u) { float f; memcpy(&f,&u,4); return f; } +static uint32_t range32(uint32_t a, uint32_t b, unsigned imm) { + const uint32_t sign=UINT32_C(0x80000000), absMask=UINT32_C(0x7fffffff); + const uint32_t quiet=UINT32_C(0x00400000), inf=UINT32_C(0x7f800000); + uint32_t aa=a&absMask, ab=b&absMask, tmp; + int nanA=aa>inf, nanB=ab>inf, snanA=nanA && !(a&quiet), snanB=nanB && !(b&quiet); + unsigned op=imm&3, signs=(imm>>2)&3; + if (nanB) tmp=a; + else if (nanA) tmp=b; + else if (((a^b)&sign) && aa==0 && ab==0) tmp=(op&1)?0:sign; + else if (((a^b)&sign) && aa==ab && op>=2) tmp=(op&1)?aa:(aa|sign); + else { + int le=op>=2 ? f32(aa)<=f32(ab) : f32(a)<=f32(b); + tmp=(op&1) ? (le?b:a) : (le?a:b); + } + if (signs==0) tmp=(tmp&absMask)|(a&sign); + else if (signs==2) tmp&=absMask; + else if (signs==3) tmp|=sign; + if (snanB) tmp=b|quiet; + if (snanA) tmp=a|quiet; + return tmp; +} +int main(void) { + const uint32_t source1[16]={ + 0x00000000,0x80000000,0x3f800000,0xbf800000, + 0x40000000,0xc0000000,0x40400000,0xc0800000, + 0x7f800000,0xff800000,0x7fc12345,0xffc23456, + 0x7f812345,0xff823456,0x3f000000,0xbf400000}; + const uint32_t source2[16]={ + 0x80000000,0x00000000,0xbf800000,0x3f800000, + 0xc0400000,0x3f000000,0xc0400000,0x40a00000, + 0x7f000000,0xff000000,0x3f800000,0x7fc34567, + 0x40000000,0x7f834567,0xbf000000,0x3fc00000}; + uint32_t old[16], out[305]; + const uint64_t mask=UINT64_C(0xa55a); + for (int i=0;i<16;i++) old[i]=UINT32_C(0x41000000)+(uint32_t)i; + memset(out,0,sizeof(out)); + floatingrangesemantics((uint8_t*)out,source2,source1,old,mask); + for (unsigned imm=0;imm<16;imm++) for (int i=0;i<16;i++) + if (out[imm*16+i]!=range32(source1[i],source2[i],imm)) return 10+(int)imm; + for (int i=0;i<16;i++) { + uint32_t r2=range32(source1[i],source2[i],2); + uint32_t r3=range32(source1[i],source2[i],3); + if (out[256+i]!=(((mask>>i)&1)?r2:old[i])) return 40; + if (out[272+i]!=(((mask>>i)&1)?r3:0)) return 41; + if (out[288+i]!=range32(source1[i],source2[0],2)) return 42; + } + if (((uint8_t*)out)[1216]!=1) return 43; + return 0; +} +` + x86FloatingHelperRunNativePipelines(t, execution, "floating_range_semantics", ir, mainC) +} + +func qualifyAMD64FloatingRangeDoubleAndScalarRuntimeSemantics(t *testing.T) { + execution := x86FloatingHelperRuntimeTools(t) + var source strings.Builder + source.WriteString(`TEXT floatingrangedoublesemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ source2+8(FP), BX + MOVQ source1+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), R8 + KMOVQ R8, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + STC +`) + for immediate := 0; immediate < 16; immediate++ { + fmt.Fprintf(&source, "\tVRANGEPD $%d, Z0, Z1, Z2\n", immediate) + fmt.Fprintf(&source, "\tVMOVDQU64 Z2, %d(AX)\n", immediate*64) + } + source.WriteString(` VMOVDQU64 0(BX), X0 + VMOVDQU64 0(CX), X1 + VMOVDQU64 0(DX), X2 + VRANGESD $2, X0, X1, X2 + VMOVDQU64 X2, 1024(AX) + VMOVDQU64 0(DX), X2 + VRANGESD $2, X0, X1, K1, X2 + VMOVDQU64 X2, 1040(AX) + VRANGESD.Z $2, X0, X1, K1, X2 + VMOVDQU64 X2, 1056(AX) + SETCS 1072(AX) + RET +`) + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := execution.triple + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"floatingrangedoublesemantics": { + Name: "floatingrangedoublesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void floatingrangedoublesemantics(uint8_t *, const uint64_t *, const uint64_t *, const uint64_t *, uint64_t); +static double f64(uint64_t u) { double f; memcpy(&f,&u,8); return f; } +static uint64_t range64(uint64_t a, uint64_t b, unsigned imm) { + const uint64_t sign=UINT64_C(0x8000000000000000), absMask=UINT64_C(0x7fffffffffffffff); + const uint64_t quiet=UINT64_C(0x0008000000000000), inf=UINT64_C(0x7ff0000000000000); + uint64_t aa=a&absMask, ab=b&absMask, tmp; + int nanA=aa>inf, nanB=ab>inf, snanA=nanA && !(a&quiet), snanB=nanB && !(b&quiet); + unsigned op=imm&3, signs=(imm>>2)&3; + if (nanB) tmp=a; + else if (nanA) tmp=b; + else if (((a^b)&sign) && aa==0 && ab==0) tmp=(op&1)?0:sign; + else if (((a^b)&sign) && aa==ab && op>=2) tmp=(op&1)?aa:(aa|sign); + else { + int le=op>=2 ? f64(aa)<=f64(ab) : f64(a)<=f64(b); + tmp=(op&1) ? (le?b:a) : (le?a:b); + } + if (signs==0) tmp=(tmp&absMask)|(a&sign); + else if (signs==2) tmp&=absMask; + else if (signs==3) tmp|=sign; + if (snanB) tmp=b|quiet; + if (snanA) tmp=a|quiet; + return tmp; +} +int main(void) { + const uint64_t source1[8]={ + UINT64_C(0),UINT64_C(0x8000000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000), + UINT64_C(0x7ff8123456789abc),UINT64_C(0xfff823456789abcd),UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff023456789abcd)}; + const uint64_t source2[8]={ + UINT64_C(0x8000000000000000),UINT64_C(0),UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000000), + UINT64_C(0x4000000000000000),UINT64_C(0x7ff83456789abcde),UINT64_C(0x4008000000000000),UINT64_C(0x7ff0456789abcdef)}; + const uint64_t old[8]={UINT64_C(0x4020000000000000),UINT64_C(0x4030000000000000),0,0,0,0,0,0}; + uint64_t out[135]; + memset(out,0,sizeof(out)); + floatingrangedoublesemantics((uint8_t*)out,source2,source1,old,0); + for (unsigned imm=0;imm<16;imm++) for (int i=0;i<8;i++) + if (out[imm*8+i]!=range64(source1[i],source2[i],imm)) return 10+(int)imm; + if (out[128]!=range64(source1[0],source2[0],2)) return 40; + if (out[129]!=source1[1]) return 44; + if (out[130]!=old[0] || out[131]!=source1[1]) return 41; + if (out[132]!=0 || out[133]!=source1[1]) return 42; + if (((uint8_t*)out)[1072]!=1) return 43; + return 0; +} +` + x86FloatingHelperRunNativePipelines(t, execution, "floating_range_double_scalar_semantics", ir, mainC) +} diff --git a/amd64_lower_rdpid_test.go b/amd64_lower_rdpid_test.go new file mode 100644 index 000000000..e30ad61dd --- /dev/null +++ b/amd64_lower_rdpid_test.go @@ -0,0 +1,92 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86RDPIDCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + registers []string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"}}, + {goarch: "386", triple: "i686-pc-windows-msvc", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"}}, + {goarch: "amd64", triple: "x86_64-apple-darwin", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"}}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"}}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"}}, + } { + t.Run(target.goarch+"/"+target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rdpidforms(SB),$0-0\n") + for _, register := range target.registers { + fmt.Fprintf(&source, "\tRDPID %s\n", register) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "rdpidforms": {Name: "rdpidforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"@llvm.x86.rdpid()", `"target-features"="+rdpid"`} { + if !strings.Contains(ll, want) { + t.Fatalf("RDPID lowering omitted %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "x86-rdpid.ll", "x86-rdpid.o", ll) + }) + } +} + +func TestTranslateX86RDPIDRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "386", instruction: "RDPID AL"}, + {goarch: "386", instruction: "RDPID 0(AX)"}, + {goarch: "386", instruction: "RDPID R8"}, + {goarch: "amd64", instruction: "RDPID AL"}, + {goarch: "amd64", instruction: "RDPID 0(AX)"}, + {goarch: "amd64", instruction: "RDPID AX, DX"}, + {goarch: "amd64", instruction: "RDPID.Z AX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", ".", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT badrdpid(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{ + "badrdpid": {Name: "badrdpid", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go's RDPID optab", test.instruction) + } + }) + } +} diff --git a/amd64_lower_reciprocal.go b/amd64_lower_reciprocal.go new file mode 100644 index 000000000..652bef87c --- /dev/null +++ b/amd64_lower_reciprocal.go @@ -0,0 +1,172 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64Reciprocal14Spec struct { + laneBits int + scalar bool +} + +// amd64Reciprocal14Specs covers every opcode backed by Go 1.27's +// _yvexpandpd and _yvgetexpsd reciprocal-estimate tables. +var amd64Reciprocal14Specs = map[Op]amd64Reciprocal14Spec{ + "VRCP14PS": {laneBits: 32}, + "VRCP14PD": {laneBits: 64}, + "VRCP14SS": {laneBits: 32, scalar: true}, + "VRCP14SD": {laneBits: 64, scalar: true}, + "VRSQRT14PS": {laneBits: 32}, + "VRSQRT14PD": {laneBits: 64}, + "VRSQRT14SS": {laneBits: 32, scalar: true}, + "VRSQRT14SD": {laneBits: 64, scalar: true}, +} + +// The complete Go RCP14/RSQRT14 family uses the existing typed native helper. +// Its approximation observes dynamic DAZ/FTZ, and active native memory reads +// must remain observable even when the numerical result is dead. +func (c *amd64Ctx) lowerReciprocal14(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64Reciprocal14Specs[Op(base)]; !known { + return false, false, nil + } + return c.lowerNativeFloatingHelper(op, ins) +} + +// lowerReciprocalEstimate implements the complete Go 1.27 legacy and VEX +// RCP/RSQRT estimate family. Pure packed register estimates retain their +// intrinsics and legal CSE; memory estimates preserve their original opcode, +// full native access and legacy alignment fault even with a dead result. +func (c *amd64Ctx) lowerReciprocalEstimate(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + raw := strings.ToUpper(string(rawOp)) + base, _, _ := strings.Cut(raw, ".") + if _, scalar := x86ScalarReciprocalSpecs[Op(base)]; scalar { + if raw != base { + return true, false, fmt.Errorf("%s %s has no suffixed form in Go 1.27: %q", c.goarch, base, ins.Raw) + } + return c.lowerScalarReciprocalEstimate(base, ins) + } + form, known, err := x86PackedReciprocalProperties(rawOp, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return true, false, c.emitPackedReciprocalEstimate(ins, form) +} + +func (c *amd64Ctx) emitPackedReciprocalEstimate(ins Instr, form x86PackedReciprocalForm) error { + if ins.x86AddressBits != 0 { + return fmt.Errorf("%w: %s needs its original address-size binding", ErrProbeNeedsContext, ins.Op) + } + source, destination := ins.Args[0], ins.Args[1] + byteWidth := form.byteWidth + lanes := byteWidth / 4 + vectorType := fmt.Sprintf("<%d x float>", lanes) + result := c.newTmp() + if source.Kind != OpReg && source.Kind != OpFP { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return err + } + segment := "" + if source.Kind == OpMem && source.Mem.Segment != "" { + segment = "%" + strings.ToLower(string(source.Mem.Segment)) + ":" + } + assembly := strings.ToLower(string(form.op)) + " " + segment + "$1, $0" + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s elementtype([%d x i8]) %s)\n", + result, vectorType, assembly, "=x,*m,~{memory}", pointerType, byteWidth, pointer) + } else { + value, err := c.loadFloatingVectorOperand(source, byteWidth, lanes, 32) + if err != nil { + return err + } + intrinsic := "llvm.x86.sse.rcp.ps" + if strings.Contains(string(form.op), "RSQRT") { + intrinsic = "llvm.x86.sse.rsqrt.ps" + } + if byteWidth == 32 { + intrinsic = "llvm.x86.avx.rcp.ps.256" + if strings.Contains(string(form.op), "RSQRT") { + intrinsic = "llvm.x86.avx.rsqrt.ps.256" + } + } + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s)\n", result, vectorType, intrinsic, vectorType, value) + } + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytesValue, vectorType, result, byteWidth) + if form.legacy { + return c.storeLegacyXViews(destination.Reg, "%"+bytesValue) + } + return c.storePackedMoveOperand(destination, byteWidth, "%"+bytesValue) +} + +func (c *amd64Ctx) lowerScalarReciprocalEstimate(base string, ins Instr) (bool, bool, error) { + form, _, err := x86ScalarReciprocalProperties(Op(base), ins, c.goarch) + if err != nil { + return true, false, err + } + return true, false, c.emitScalarReciprocalEstimate(ins, form) +} + +func (c *amd64Ctx) emitScalarReciprocalEstimate(ins Instr, form x86ScalarReciprocalForm) error { + if ins.x86AddressBits != 0 { + return fmt.Errorf("%w: %s needs its original address-size binding", ErrProbeNeedsContext, ins.Op) + } + upper, err := c.loadFloatingVectorOperand(ins.Args[form.base], 16, 4, 32) + if err != nil { + return err + } + value, sourceType, constraint, template, err := c.scalarReciprocalNativeSource(ins.Args[0]) + if err != nil { + return err + } + // Keep the scalar hardware approximation, not a packed estimate or generic + // division/sqrt. The original opcode has no SIMD exceptions, ignores RC, + // treats denormals as signed zero and leaves all MXCSR status untouched. + assembly := strings.ToLower(string(ins.Op)) + " " + constraints, args := "=x,", "" + if form.spec.vector { + assembly += template + ", $2, $0" + constraints += constraint + ",x" + args = fmt.Sprintf("%s %s, <4 x float> %s", sourceType, value, upper) + } else { + assembly += strings.Replace(template, "$1", "$2", 1) + ", $0" + constraints += "0," + constraint + args = fmt.Sprintf("<4 x float> %s, %s %s", upper, sourceType, value) + } + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(%s)\n", result, assembly, constraints+",~{memory},~{fpsr}", args) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <16 x i8>\n", bytes, result) + if !form.spec.vector { + return c.storeLegacyXViews(ins.Args[form.destination].Reg, "%"+bytes) + } + return c.storePackedMoveOperand(ins.Args[form.destination], 16, "%"+bytes) +} + +func (c *amd64Ctx) scalarReciprocalNativeSource(source Operand) (value, typ, constraint, template string, err error) { + if source.Kind == OpReg { + value, err = c.loadFloatingVectorOperand(source, 16, 4, 32) + return value, "<4 x float>", "x", "$1", err + } + if source.Kind == OpFP { + bits, loadErr := c.loadScalarLaneFPBits(source.FPOffset, 32) + if loadErr != nil { + return "", "", "", "", loadErr + } + scalar, inserted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %s to float\n", scalar, bits) + fmt.Fprintf(c.b, " %%%s = insertelement <4 x float> zeroinitializer, float %%%s, i32 0\n", inserted, scalar) + return "%" + inserted, "<4 x float>", "x", "$1", nil + } + // A sideeffecting m32 operand keeps the original access even for Void. + // This pointer is not a widened virtual FP slot or invented backing. + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", "", "", "", err + } + segment := "" + if source.Kind == OpMem && source.Mem.Segment != "" { + segment = "%" + strings.ToLower(string(source.Mem.Segment)) + ":" + } + return pointer, pointerType + " elementtype([4 x i8])", "*m", segment + "$1", nil +} diff --git a/amd64_lower_reciprocal14_test.go b/amd64_lower_reciprocal14_test.go new file mode 100644 index 000000000..97130e550 --- /dev/null +++ b/amd64_lower_reciprocal14_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func x86Reciprocal14InitializeOriginalShapes(t *testing.T, original, arch string) (string, string, int) { + t.Helper() + var source, declarations strings.Builder + count := 0 + for _, line := range strings.Split(original, "\n") { + instruction := strings.TrimSpace(line) + if instruction == "" || instruction == "RET" || strings.HasPrefix(instruction, "TEXT ") { + continue + } + if arch == "386" { + // Keep the original high-bank source's Go assembler observation + // above. Its physical mode32 counterpart uses actual low registers. + instruction = strings.NewReplacer("X20", "X4", "Y20", "Y4", "X21", "X5").Replace(instruction) + } + ins := x86ScalarLaneInstruction(t, instruction) + registers := [3]int{0, 0, 0} + for index, operand := range ins.Args { + register, vector := amd64VectorRegisterIndex(operand.Reg, amd64VectorByteWidth(operand.Reg)) + if !vector { + continue + } + if index == len(ins.Args)-1 { + registers[2] = register + } else if index == 0 { + registers[0] = register + } else { + registers[1] = register + } + } + body, declaration := x86PackedAbsSignInitializedInstruction(count, instruction, arch, registers) + maskOffset := 32 + if arch == "386" { + maskOffset = 16 + } + var masks strings.Builder + for k := 2; k <= 5; k++ { + fmt.Fprintf(&masks, "KMOVQ mask+%d(FP),K%d\n", maskOffset, k) + } + body = strings.Replace(body, "STC\n", masks.String()+"STC\n", 1) + declaration = strings.Replace(declaration, "input,control,old *[64]byte", "input *[128]byte,control,old *[64]byte", 1) + source.WriteString(body) + declarations.WriteString(declaration) + count++ + } + return source.String(), declarations.String(), count +} + +func TestX86Reciprocal14Mode32HighSyntaxHasNoPhysicalBinding(t *testing.T) { + for _, member := range x86Reciprocal14TestMembers { + instruction := member.op + " X1,X20" + if member.scalar { + instruction = member.op + " X1,X20,X21" + } + source := "TEXT high(SB),$0-0\n" + instruction + "\nRET\n" + requireX86GoAssemblerResult(t, "386", source, true) + ins := x86ScalarLaneInstruction(t, instruction) + if _, known, err := x86TypedInstructionByteEffect(ins.Op, ins, "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("named high-bank syntax was mistaken for a physical mode32 register", instruction, known, err) + } + } +} + +func TestTranslateX86Reciprocal14CompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's _yvexpandpd table accepts EVEX X/Y/Z register or memory + // sources, optional scalar-memory broadcast, and optional K1-K7 merge/.Z + // masking. _yvgetexpsd accepts scalar memory/X sources, an X passthrough, + // and the same optional mask. The 386 frontend accepts packed three-operand + // masks, refuses scalar four-operand masks, and limits Z registers to Z0-Z7. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 31 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT reciprocal14forms(SB),$0-0\n") + for _, op := range []string{"VRCP14PS", "VRCP14PD", "VRSQRT14PS", "VRSQRT14PD"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s1, %s20\n", op, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s20\n", op, width) + fmt.Fprintf(&source, "\t%s.BCST 16(AX), %s20\n", op, width) + } + fmt.Fprintf(&source, "\t%s Z1, Z%d\n", op, lastZ) + fmt.Fprintf(&source, "\t%s 24(AX), Z%d\n", op, lastZ) + fmt.Fprintf(&source, "\t%s.BCST 32(AX), Z%d\n", op, lastZ) + fmt.Fprintf(&source, "\t%s X1, K1, X20\n", op) + fmt.Fprintf(&source, "\t%s.Z 40(AX), K2, Y20\n", op) + fmt.Fprintf(&source, "\t%s.BCST.Z 48(AX), K3, Z%d\n", op, lastZ) + } + for _, op := range []string{"VRCP14SS", "VRCP14SD", "VRSQRT14SS", "VRSQRT14SD"} { + fmt.Fprintf(&source, "\t%s X1, X20, X21\n", op) + fmt.Fprintf(&source, "\t%s 56(AX), X20, X21\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X20, K4, X21\n", op) + fmt.Fprintf(&source, "\t%s.Z 64(AX), X20, K5, X21\n", op) + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + initialized, declaration, count := x86Reciprocal14InitializeOriginalShapes(t, source.String(), target.goarch) + want := 64 + if target.goarch == "386" { + want = 56 + } + if count != want { + t.Fatal("original reciprocal14 shape count changed", count, want) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, initialized, declaration, target.goarch, target.triple, api) + if err != nil { + t.Fatal("complete initialized original reciprocal14 shapes rejected", err) + } + compileLLVMToObject(t, llc, target.triple, "reciprocal14-"+target.name+".ll", "reciprocal14-"+target.name+".o", ir) + }) + } + }) + } +} + +func TestTranslateX86Reciprocal14RejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VRCP14PS X0, Y1"}, + {goarch: "amd64", instruction: "VRSQRT14PD.Z Z0, Z1"}, + {goarch: "amd64", instruction: "VRCP14PS.BCST X0, X1"}, + {goarch: "amd64", instruction: "VRCP14PD.SAE Z0, Z1"}, + {goarch: "amd64", instruction: "VRSQRT14PS Z0, K0, Z1"}, + {goarch: "amd64", instruction: "VRCP14SS X0, X1"}, + {goarch: "amd64", instruction: "VRCP14SD Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRSQRT14SS.BCST 8(AX), X1, X2"}, + {goarch: "amd64", instruction: "VRSQRT14SD.SAE X0, X1, X2"}, + {goarch: "amd64", instruction: "VRCP14SS.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VRCP14SS X0, X1, K0, X2"}, + {goarch: "386", instruction: "VRCP14SS X0, X1, K1, X2"}, + {goarch: "386", instruction: "VRSQRT14PD Z0, Z8"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's reciprocal14 tables for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_reciprocal_scalar_test.go b/amd64_lower_reciprocal_scalar_test.go new file mode 100644 index 000000000..a6baa5e45 --- /dev/null +++ b/amd64_lower_reciprocal_scalar_test.go @@ -0,0 +1,249 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86ScalarReciprocalCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + const vexLast = 15 + var source strings.Builder + source.WriteString("DATA scalarreciprocaldata+0(SB)/4, $0\n") + source.WriteString("GLOBL scalarreciprocaldata(SB), $4\n") + source.WriteString("TEXT scalarreciprocalforms(SB),$0-4\n") + for _, op := range []string{"RCPSS", "RSQRTSS"} { + fmt.Fprintf(&source, "\t%s X1, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s scalarreciprocaldata(SB), X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s arg+0(FP), X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 0, X%d\n", op, legacyLast) + } + for _, op := range []string{"VRCPSS", "VRSQRTSS"} { + fmt.Fprintf(&source, "\t%s X1, X2, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 8(AX), X2, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s scalarreciprocaldata(SB), X2, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s arg+0(FP), X2, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 0, X2, X%d\n", op, vexLast) + if target.goarch == "386" { + // Go's VEX register classes retain their 16-register + // compatibility closure even in 32-bit mode. + fmt.Fprintf(&source, "\t%s 16(R9), X13, X14\n", op) + } + } + if target.goarch == "amd64" { + for _, op := range []string{"RCPSS", "RSQRTSS"} { + fmt.Fprintf(&source, "\t%s 16(R11)(R12*2), X13\n", op) + } + for _, op := range []string{"VRCPSS", "VRSQRTSS"} { + fmt.Fprintf(&source, "\t%s 16(R11)(R12*2), X13, X14\n", op) + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + signatures := map[string]FuncSig{ + "scalarreciprocalforms": { + Name: "scalarreciprocalforms", Args: []LLVMType{LLVMType("float")}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: LLVMType("float"), Index: 0, Field: -1}}}, + }, + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func scalarreciprocalforms(arg float32)\n", target.goarch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original uninitialized format rows acquired numerical or native input permission", api, err) + } + } + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + // Preserve Go's original mode32 VEX compatibility formats as + // private object rows, not initialized physical entry states. + grammarArch := target.goarch + if strings.HasPrefix(string(ins.Op), "V") { + grammarArch = "amd64" + } + form, known, err := x86ScalarReciprocalProperties(ins.Op, ins, grammarArch) + if !known || err != nil { + return known, false, err + } + return true, false, c.emitScalarReciprocalEstimate(ins, form) + }) + for _, want := range []string{ + `asm sideeffect "rcpss `, `asm sideeffect "rsqrtss `, + `asm sideeffect "vrcpss `, `asm sideeffect "vrsqrtss `, + "elementtype([4 x i8])", "insertelement <4 x float>", + `"target-features"="+avx,+sse"`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("scalar reciprocal lowering omitted %q:\n%s", want, ir) + } + } + if strings.Contains(ir, "call <4 x float> @llvm.x86.sse.rcp.ps") || + strings.Contains(ir, "call <4 x float> @llvm.x86.sse.rsqrt.ps") { + t.Fatal("scalar source opcode was replaced with a packed estimate") + } + compileLLVMToObject(t, llc, target.triple, "scalar-reciprocal-"+target.name+".ll", "scalar-reciprocal-"+target.name+".o", ir) + }) + } +} + +func TestAMD64ScalarReciprocalRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT scalarreciprocalsemantics(SB),$0-24 + MOVQ out+0(FP), DI + MOVQ source+8(FP), AX + MOVQ upper+16(FP), BX + + MOVUPS (BX), X0 + STC + RCPSS (AX), X0 + MOVUPS X0, 0(DI) + SETCS 64(DI) + + MOVUPS (BX), X1 + STC + RSQRTSS (AX), X1 + MOVUPS X1, 16(DI) + SETCS 65(DI) + + MOVUPS (BX), X2 + STC + VRCPSS (AX), X2, X3 + MOVUPS X3, 32(DI) + SETCS 66(DI) + + MOVUPS (BX), X4 + STC + VRSQRTSS (AX), X4, X5 + MOVUPS X5, 48(DI) + SETCS 67(DI) + VZEROUPPER + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "scalarreciprocalsemantics": { + Name: "scalarreciprocalsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void scalarreciprocalsemantics(uint8_t *, const float *, const float *); +static int close_enough(float got, float want) { + float delta = got - want; + if (delta < 0) delta = -delta; + return delta < 0.01f; +} +int main(void) { + float source[4] = {4.0f, 100.0f, 200.0f, 300.0f}; + float upper[4] = {99.0f, 10.0f, 20.0f, 30.0f}; + uint8_t out[68] = {0}; + scalarreciprocalsemantics(out, source, upper); + const float *values = (const float *)out; + for (int vector = 0; vector < 4; ++vector) { + float want = (vector & 1) ? 0.5f : 0.25f; + if (!close_enough(values[4*vector], want)) return 10+vector; + if (values[4*vector+1] != 10.0f || values[4*vector+2] != 20.0f || values[4*vector+3] != 30.0f) return 20+vector; + if (out[64+vector] != 1) return 30+vector; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_reciprocal_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateX86ScalarReciprocalRejectsFormsOutsideGo127Tables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "RCPSS X0"}, + {goarch: "amd64", instruction: "RSQRTSS X0, X1, X2"}, + {goarch: "amd64", instruction: "RCPSS Y0, X1"}, + {goarch: "amd64", instruction: "RCPSS X0, X16"}, + {goarch: "amd64", instruction: "VRCPSS X0, X1"}, + {goarch: "amd64", instruction: "VRSQRTSS X0, X1, Y2"}, + {goarch: "amd64", instruction: "VRCPSS Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRCPSS.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VRSQRTSS X0, X1, K1, X2"}, + {goarch: "386", instruction: "RCPSS X0, X8"}, + {goarch: "386", instruction: "RSQRTSS 8(R9), X0"}, + {goarch: "386", instruction: "VRCPSS X16, X1, X2"}, + {goarch: "386", instruction: "VRSQRTSS X0, X1, X16"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar reciprocal tables for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_reciprocal_test.go b/amd64_lower_reciprocal_test.go new file mode 100644 index 000000000..ee274955f --- /dev/null +++ b/amd64_lower_reciprocal_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedReciprocalCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yxm table accepts X/m128 -> X, while the VEX + // _yvptest table accepts both X/m128 -> X and Y/m256 -> Y. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + const vexLast = 15 + var source strings.Builder + source.WriteString("TEXT packedreciprocalforms(SB),$0-0\n") + for _, op := range []string{"RCPPS", "RSQRTPS"} { + fmt.Fprintf(&source, "\t%s X1, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 0, X%d\n", op, legacyLast) + } + for _, op := range []string{"VRCPPS", "VRSQRTPS"} { + fmt.Fprintf(&source, "\t%s X1, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s Y1, Y%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 16(AX), Y%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 0, Y%d\n", op, vexLast) + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func packedreciprocalforms()\n", target.goarch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("bare original formats acquired a native pointer or physical entry value", api, err) + } + } + // Keep every original Go shape and its actual LLVM object. The + // private format compiler supplies no public entry/storage proof; + // mode32 VEX high-register spellings remain frontend-only shapes. + ll := x86InstructionFormatObjectUnit(t, file, + map[string]FuncSig{"packedreciprocalforms": {Name: "packedreciprocalforms", Ret: Void}}, + target.goarch, target.triple, 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + grammarArch := target.goarch + if strings.HasPrefix(string(ins.Op), "V") { + grammarArch = "amd64" + } + form, known, err := x86PackedReciprocalProperties(ins.Op, ins, grammarArch) + if !known || err != nil { + return known, false, err + } + return true, false, c.emitPackedReciprocalEstimate(ins, form) + }) + for _, intrinsic := range []string{"@llvm.x86.sse.rcp.ps", "@llvm.x86.sse.rsqrt.ps", "@llvm.x86.avx.rcp.ps.256", "@llvm.x86.avx.rsqrt.ps.256"} { + if !strings.Contains(ll, intrinsic) { + t.Fatalf("translation omitted %s:\n%s", intrinsic, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-reciprocal-"+target.name+".ll", "packed-reciprocal-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedReciprocalRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "RCPPS Y0, X1"}, + {goarch: "amd64", instruction: "RSQRTPS X0, X16"}, + {goarch: "amd64", instruction: "VRCPPS X16, X1"}, + {goarch: "amd64", instruction: "VRCPPS Y0, X1"}, + {goarch: "amd64", instruction: "VRSQRTPS Z0, Z1"}, + {goarch: "amd64", instruction: "VRCPPS.Z X0, X1"}, + {goarch: "amd64", instruction: "VRSQRTPS X0, K1, X1"}, + {goarch: "386", instruction: "RCPPS X0, X8"}, + {goarch: "386", instruction: "VRCPPS X16, X0"}, + {goarch: "386", instruction: "VRCPPS Y16, Y0"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's reciprocal tables for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_rorx_test.go b/amd64_lower_rorx_test.go new file mode 100644 index 000000000..d2db19417 --- /dev/null +++ b/amd64_lower_rorx_test.go @@ -0,0 +1,222 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawRORXReportedEncoding(t *testing.T) { + const source = ` +TEXT rawRORX(SB),$0-0 + BYTE $0xc4 + BYTE $0xe3 + BYTE $0x7b + BYTE $0xf0 + BYTE $0xe8 + BYTE $0x19 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "386", triple: "i686-pc-windows-msvc"}, + {goarch: "amd64", triple: "x86_64-apple-darwin"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var ir string + if target.goarch == "amd64" { + // The original ModRM writes physical BP. Its encoding remains + // covered, but an ordinary logical entry has no native-frame role. + ir = x86UnboundVoidRawScalarFormatObject(t, source, target.goarch, target.triple) + } else { + ir, err = Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawRORX": {Name: "rawRORX", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + } + if !strings.Contains(ir, `"target-features"="+bmi2"`) { + t.Fatalf("raw RORX did not infer BMI2 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-rorx.ll", "raw-rorx.o", ir) + }) + } +} + +func TestX86RORXGo127CompleteNamedForms(t *testing.T) { + const source = ` +TEXT rorxforms(SB),$0-0 + RORXL $7, (BX), DX + RORXL $31, R11, R12 + RORXQ $7, 16(BX)(SI*2), DX + RORXQ $63, R11, R12 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + const triple = "x86_64-unknown-linux-gnu" + ir := x86UnboundVoidScalarFormatObject(t, source, "rorxforms", "amd64", triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } +} + +func TestX86RORXCompleteGo127FormsAcrossTargets(t *testing.T) { + registers386 := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} + registersAMD64 := append(append([]string{}, registers386...), "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + targets := []struct { + name string + goarch string + triple string + registers []string + }{ + {name: "386-linux", goarch: "386", triple: "i386-unknown-linux-gnu", registers: registers386}, + {name: "386-windows", goarch: "386", triple: "i686-pc-windows-msvc", registers: registers386}, + {name: "amd64-darwin", goarch: "amd64", triple: "x86_64-apple-darwin", registers: registersAMD64}, + {name: "amd64-linux", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registers: registersAMD64}, + {name: "amd64-windows", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registers: registersAMD64}, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rorxall(SB),$0-0\n") + for _, width := range []string{"L", "Q"} { + for index, sourceRegister := range target.registers { + destination := target.registers[(index+3)%len(target.registers)] + fmt.Fprintf(&source, "\tRORX%s $%d, %s, %s\n", width, index*17&255, sourceRegister, destination) + } + fmt.Fprintf(&source, "\tRORX%s $-1, 8(AX), DX\n", width) + fmt.Fprintf(&source, "\tRORX%s $63, 16(AX)(CX*4), BX\n", width) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "rorxall", target.goarch, target.triple) + if !strings.Contains(ir, `"target-features"="+bmi2"`) { + t.Fatalf("RORX family did not infer BMI2 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "rorx-family.ll", "rorx-family.o", ir) + }) + } +} + +func TestX86RawRORXCompleteRegisterFields(t *testing.T) { + for _, mode := range []int{32, 64} { + registerCount := 8 + if mode == 64 { + registerCount = 16 + } + for _, width64 := range []bool{false, true} { + for source := 0; source < registerCount; source++ { + for destination := 0; destination < registerCount; destination++ { + for _, immediate := range []byte{0, 1, 31, 32, 63, 255} { + p0 := byte((^destination>>3)&1)<<7 | 1<<6 | byte((^source>>3)&1)<<5 | 3 + p1 := byte(0x7b) + if width64 { + p1 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | source&7) + encoding := []byte{0xc4, p0, p1, 0xf0, modRM, immediate} + instruction, length, ok, err := decodedX86RORXInstruction(encoding, mode) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("mode=%d width64=%v source=%d destination=%d immediate=%d: ok=%v length=%d err=%v", mode, width64, source, destination, immediate, ok, length, err) + } + wantOp := Op("RORXL") + if width64 && mode == 64 { + wantOp = "RORXQ" + } + wantSource, _ := decodedX86GeneralRegister(source) + wantDestination, _ := decodedX86GeneralRegister(destination) + if instruction.Op != wantOp || len(instruction.Args) != 3 || instruction.Args[0].Imm != int64(immediate) || instruction.Args[1].Reg != wantSource || instruction.Args[2].Reg != wantDestination { + t.Fatalf("encoding %x decoded as %#v", encoding, instruction) + } + } + } + } + } + } +} + +func TestX86RawRORXMemoryForms(t *testing.T) { + for _, test := range []struct { + name string + encoding []byte + }{ + {name: "direct", encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0, 0x13, 0x07}}, + {name: "extended base", encoding: []byte{0xc4, 0xc3, 0x7b, 0xf0, 0x13, 0x07}}, + {name: "sib disp8", encoding: []byte{0xc4, 0x83, 0xfb, 0xf0, 0x54, 0x88, 0x10, 0xff}}, + {name: "base less", encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0, 0x14, 0x25, 0x78, 0x56, 0x34, 0x12, 0x1f}}, + {name: "segment", encoding: []byte{0x64, 0xc4, 0xe3, 0x7b, 0xf0, 0x54, 0x48, 0x08, 0x1f}}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86RORXInstruction(test.encoding, 64) + if !ok || err != nil || length != len(test.encoding) || len(instruction.Args) != 3 || instruction.Args[1].Kind != OpMem { + t.Fatalf("encoding %x: instruction=%#v ok=%v length=%d err=%v", test.encoding, instruction, ok, length, err) + } + }) + } +} + +func TestX86RawRORXRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + }{ + {name: "vex l", mode: 64, encoding: []byte{0xc4, 0xe3, 0x7f, 0xf0, 0xd2, 0x07}}, + {name: "vex pp", mode: 64, encoding: []byte{0xc4, 0xe3, 0x79, 0xf0, 0xd2, 0x07}}, + {name: "vex vvvv", mode: 64, encoding: []byte{0xc4, 0xe3, 0x73, 0xf0, 0xd2, 0x07}}, + {name: "missing modrm", mode: 64, encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0}}, + {name: "missing immediate", mode: 64, encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0, 0xd2}}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe3, 0x7b, 0xf0, 0xd2, 0x07}}, + {name: "rip relative", mode: 64, encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0, 0x15, 0, 0, 0, 0, 0x07}}, + {name: "extended source 386", mode: 32, encoding: []byte{0xc4, 0xc3, 0x7b, 0xf0, 0xc0, 0x07}}, + {name: "extended destination 386", mode: 32, encoding: []byte{0xc4, 0x63, 0x7b, 0xf0, 0xc0, 0x07}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86RORXInstruction(test.encoding, test.mode) + if !ok || err == nil { + t.Fatalf("encoding %x: ok=%v err=%v, want recognized rejection", test.encoding, ok, err) + } + }) + } +} + +func TestX86RORXGo127ModeOracle(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + want bool + }{ + {goarch: "386", instruction: "RORXL $7, (BX), DX", want: true}, + {goarch: "386", instruction: "RORXQ $7, (BX), DX", want: true}, + {goarch: "386", instruction: "RORXL $7, R8, DX", want: false}, + {goarch: "amd64", instruction: "RORXQ $7, R8, R15", want: true}, + } { + t.Run(fmt.Sprintf("%s/%s", test.goarch, test.instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, test.goarch, "TEXT oracle(SB),$0-0\n\t"+test.instruction+"\n\tRET\n", test.want) + }) + } +} diff --git a/amd64_lower_round.go b/amd64_lower_round.go new file mode 100644 index 000000000..dc62f9012 --- /dev/null +++ b/amd64_lower_round.go @@ -0,0 +1,118 @@ +package plan9asm + +import "fmt" + +type amd64RoundSpec struct { + laneBits int + scalar bool + vector bool +} + +// amd64RoundSpecs mirrors Go 1.27's yxshuf, _yvroundpd, and _yvdppd tables. +// The legacy family is X-only, the packed VEX family is X/Y, and the scalar +// VEX family has an explicit upper-lane source. +var amd64RoundSpecs = map[Op]amd64RoundSpec{ + "ROUNDPS": {laneBits: 32}, + "ROUNDPD": {laneBits: 64}, + "ROUNDSS": {laneBits: 32, scalar: true}, + "ROUNDSD": {laneBits: 64, scalar: true}, + "VROUNDPS": {laneBits: 32, vector: true}, + "VROUNDPD": {laneBits: 64, vector: true}, + "VROUNDSS": {laneBits: 32, scalar: true, vector: true}, + "VROUNDSD": {laneBits: 64, scalar: true, vector: true}, +} + +func (c *amd64Ctx) lowerRound(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + local := ins + local.Op = rawOp + form, known, err := x86RoundProperties(local, c.goarch) + if !known || err != nil { + return known, false, err + } + if !form.spec.vector && form.memory && form.source.Kind != OpFP { + memory := x86FloatBinaryForm{bits: form.spec.laneBits, scalar: form.spec.scalar, legacy: true, + first: 1, second: len(ins.Args) - 1, destination: len(ins.Args) - 1, mask: -1, + byteWidth: 16, memory: 1, memoryBytes: form.memoryBytes} + return true, false, c.emitLegacyAlternatingRoundMemory(local, memory, &form.immediate) + } + intrinsic := amd64RoundIntrinsic(form.immediate) + if form.spec.scalar { + return c.lowerScalarRound(form, intrinsic) + } + return c.lowerPackedRound(form, intrinsic) +} + +func amd64RoundIntrinsic(imm uint8) string { + if imm&4 != 0 { + return "rint" + } + switch imm & 3 { + case 0: + return "roundeven" + case 1: + return "floor" + case 2: + return "ceil" + default: + return "trunc" + } +} + +func (c *amd64Ctx) lowerScalarRound(form x86RoundForm, intrinsic string) (bool, bool, error) { + spec, destination := form.spec, form.destination + value, err := c.loadAlternatingRoundFloat(form.source, form.memoryBytes, spec.laneBits, spec.vector) + if err != nil { + return true, false, err + } + rounded := c.emitScalarFloatRound(value, spec.laneBits, intrinsic) + bits := c.newTmp() + floatingType := amd64FloatingTypeForBits(spec.laneBits) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", bits, floatingType, rounded, spec.laneBits) + baseBytes, err := c.loadX(form.upper.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, baseBytes) + if spec.vector { + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.laneBits, "%"+bits, base, "", false) + } + return true, false, c.storeVectorScalarRegister(destination.Reg, spec.laneBits, "%"+bits, base, "", false) +} + +func (c *amd64Ctx) lowerPackedRound(form x86RoundForm, intrinsic string) (bool, bool, error) { + spec, destination, byteWidth := form.spec, form.destination, form.byteWidth + bytesValue, err := c.loadRoundPackedBytes(form) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / spec.laneBits + floatingType := fmt.Sprintf("<%d x %s>", lanes, amd64FloatingTypeForBits(spec.laneBits)) + floating := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", floating, byteWidth, bytesValue, floatingType) + result := "poison" + for lane := 0; lane < lanes; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", value, floatingType, floating, lane) + rounded := c.emitScalarFloatRound("%"+value, spec.laneBits, intrinsic) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %s, i32 %d\n", inserted, floatingType, result, amd64FloatingTypeForBits(spec.laneBits), rounded, lane) + result = "%" + inserted + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, floatingType, result, byteWidth) + if spec.vector { + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) + } + return true, false, c.storeX(destination.Reg, "%"+out) +} + +func (c *amd64Ctx) emitScalarFloatRound(value string, bits int, intrinsic string) string { + typeName := amd64FloatingTypeForBits(bits) + intrinsicSuffix := "f64" + if bits == 32 { + intrinsicSuffix = "f32" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.%s.%s(%s %s)\n", result, typeName, intrinsic, intrinsicSuffix, typeName, value) + return "%" + result +} diff --git a/amd64_lower_round_test.go b/amd64_lower_round_test.go new file mode 100644 index 000000000..bfb9e9697 --- /dev/null +++ b/amd64_lower_round_test.go @@ -0,0 +1,248 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86RoundCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT roundforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"ROUNDPS", "ROUNDPD", "ROUNDSS", "ROUNDSD"} { + fmt.Fprintf(&source, "\t%s $0, X0, X1\n", op) + fmt.Fprintf(&source, "\t%s $255, 8(BX), X2\n", op) + } + for _, op := range []string{"VROUNDPS", "VROUNDPD"} { + fmt.Fprintf(&source, "\t%s $0, X0, X1\n", op) + fmt.Fprintf(&source, "\t%s $-1, 16(BX), X2\n", op) + fmt.Fprintf(&source, "\t%s $255, Y0, Y1\n", op) + fmt.Fprintf(&source, "\t%s $2, 32(BX), Y2\n", op) + } + for _, op := range []string{"VROUNDSS", "VROUNDSD"} { + fmt.Fprintf(&source, "\t%s $0, X0, X1, X2\n", op) + fmt.Fprintf(&source, "\t%s $-1, 8(BX), X1, X2\n", op) + fmt.Fprintf(&source, "\t%s $255, X0, X1, X2\n", op) + } + source.WriteString("\tRET\n") + ll := x86RoundOriginalSourceClassifiedObjects(t, source.String(), target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "round-"+target.name+".ll", "round-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86RoundRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + tests := []struct { + goarch string + triple string + instruction string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDSD $-1, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDSS $256, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDPD $0, Y0, Y1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDPS $0, X0, Y1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDSS.Z $0, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDPS $-129, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDPD $256, Y0, Y1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDPS $0, X0, Y1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDSS $0, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDSD $0, X0, Y1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDSS $0, X0, X1, Y2"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "ROUNDSS $0, X8, X0"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "VROUNDPS $0, Y8, Y0"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "VROUNDSD $0, X0, X8, X1"}, + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+test.instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ROUND tables for %s", test.instruction, test.goarch) + } + }) + } +} + +const x86LegacyRoundOriginalRuntimeSource = ` +TEXT roundsemantics(SB),NOSPLIT,$0-40 + MOVQ out+0(FP), AX + MOVQ a32+8(FP), BX + MOVQ a64+16(FP), CX + MOVQ upper32+24(FP), DX + MOVQ upper64+32(FP), SI + ROUNDPS $0, (BX), X0 + MOVUPS X0, 0(AX) + ROUNDPS $1, (BX), X0 + MOVUPS X0, 16(AX) + ROUNDPS $2, (BX), X0 + MOVUPS X0, 32(AX) + ROUNDPS $3, (BX), X0 + MOVUPS X0, 48(AX) + ROUNDPS $4, (BX), X0 + MOVUPS X0, 64(AX) + MOVUPS (DX), X1 + VROUNDSS $1, (BX), X1, X2 + MOVUPS X2, 80(AX) + ROUNDPD $0, (CX), X0 + MOVUPD X0, 96(AX) + ROUNDPD $1, (CX), X0 + MOVUPD X0, 112(AX) + ROUNDPD $2, (CX), X0 + MOVUPD X0, 128(AX) + ROUNDPD $3, (CX), X0 + MOVUPD X0, 144(AX) + ROUNDPD $4, (CX), X0 + MOVUPD X0, 160(AX) + MOVUPD (SI), X1 + VROUNDSD $2, (CX), X1, X2 + MOVUPD X2, 176(AX) + VZEROUPPER + RET +` + +func x86LegacyRoundOriginalRuntimeFrame() FrameLayout { + return FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }} +} + +// This is the exact existing five-opaque-Ptr runtime fixture, compiled on +// every host. It does not bypass the independent native runtime guard below. +func TestX86LegacyRoundOriginalRuntimeSourceObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + pkg := mustGoPackage(t, "bitview", "package bitview\nfunc roundsemantics(out,a32,a64,upper32,upper64 *byte)\n") + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + // Only the Go oracle adds its standard NOSPLIT macro definition. + // The original shared assembly body itself is byte-identical. + x86LEAActualTargetGoObject(t, "amd64", triple, "#include \"textflag.h\"\n"+x86LegacyRoundOriginalRuntimeSource) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + manualCalls := 0 + manual := func(name string) (FuncSig, bool) { + if name != "roundsemantics" { + return FuncSig{}, false + } + manualCalls++ + return FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, Ret: Void, + Frame: x86LegacyRoundOriginalRuntimeFrame()}, true + } + ir, err := x86ScalarFPBitViewPackageAPIWithManual(t, x86LegacyRoundOriginalRuntimeSource, + pkg, "amd64", triple, api, manual) + if err != nil || ir == "" || manualCalls == 0 { + t.Fatal("exact original runtime source/manual pointer frame rejected", api, manualCalls, err) + } + for _, level := range []string{"0", "2"} { + t.Run("IR"+level+"/backend"+level, func(t *testing.T) { + objectIR := ir + if level == "2" { + objectIR = x86DwordProductIROptimize(t, ir) + } + x86PackedStringCompileObject(t, llc, triple, objectIR, "-O"+level) + }) + } + }) + } + }) + } +} + +func TestAMD64RoundRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86LegacyRoundOriginalRuntimeSource + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := x86LegacyRoundOriginalRuntimeFrame() + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "roundsemantics": {Name: "roundsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void roundsemantics(uint8_t *, const float *, const double *, const float *, const double *); +static int check32(const float *got, const float *want, int base) { + for (int i = 0; i < 4; i++) if (got[i] != want[i]) return base+i; + return 0; +} +static int check64(const double *got, const double *want, int base) { + for (int i = 0; i < 2; i++) if (got[i] != want[i]) return base+i; + return 0; +} +int main(void) { + float a32[4] = {1.5f, -1.5f, 2.75f, -2.75f}; + double a64[2] = {2.5, -2.5}; + float upper32[4] = {99.0f, 10.0f, 20.0f, 30.0f}; + double upper64[2] = {99.0, 40.0}; + float want32[6][4] = { + {2, -2, 3, -3}, {1, -2, 2, -3}, {2, -1, 3, -2}, + {1, -1, 2, -2}, {2, -2, 3, -3}, {1, 10, 20, 30}, + }; + double want64[6][2] = { + {2, -2}, {2, -3}, {3, -2}, {2, -2}, {2, -2}, {3, 40}, + }; + uint8_t out[192] = {0}; + roundsemantics(out, a32, a64, upper32, upper64); + for (int i = 0; i < 6; i++) { + int failed = check32((const float *)(out + 16*i), want32[i], 10+4*i); + if (failed) return failed; + } + for (int i = 0; i < 6; i++) { + int failed = check64((const double *)(out + 96 + 16*i), want64[i], 40+2*i); + if (failed) return failed; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "round_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_rtm.go b/amd64_lower_rtm.go new file mode 100644 index 000000000..ade2d7701 --- /dev/null +++ b/amd64_lower_rtm.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64RTMKind uint8 + +const ( + amd64RTMBegin amd64RTMKind = iota + amd64RTMAbort + amd64RTMEnd + amd64RTMTest +) + +type amd64RTMOperand uint8 + +const ( + amd64RTMNoOperand amd64RTMOperand = iota + amd64RTMBranch + amd64RTMUnsignedByte +) + +type amd64RTMSpec struct { + kind amd64RTMKind + operand amd64RTMOperand + outputAX bool +} + +// amd64RTMSpecs is the complete Go 1.27 Restricted Transactional Memory +// grammar. XBEGIN is included even though Go's positive encoder corpus keeps +// its cases as TODOs: the optab accepts one Ybr target and writes abort status +// to EAX. XABORT accepts the complete Yu8 immediate domain. +var amd64RTMSpecs = map[Op]amd64RTMSpec{ + "XBEGIN": {kind: amd64RTMBegin, operand: amd64RTMBranch, outputAX: true}, + "XABORT": {kind: amd64RTMAbort, operand: amd64RTMUnsignedByte}, + "XEND": {kind: amd64RTMEnd}, + "XTEST": {kind: amd64RTMTest}, +} + +type amd64RTMForm struct { + branch Operand + immediate uint8 +} + +func (c *amd64Ctx) lowerRTM(bi int, ii int, op Op, ins Instr, emitCondBr amd64EmitCondBr) (ok bool, terminated bool, err error) { + spec, form, known, err := x86RTMProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return c.emitRTM(bi, ii, strings.ToUpper(string(op)), spec, form, emitCondBr) +} + +func x86RTMProperties(op Op, ins Instr, arch string) (amd64RTMSpec, amd64RTMForm, bool, error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64RTMSpecs[Op(baseOp)] + if !recognized { + return spec, amd64RTMForm{}, false, nil + } + if rawOp != baseOp || arch != "386" && arch != "amd64" || + ins.Op != "" && !strings.EqualFold(string(ins.Op), rawOp) { + return spec, amd64RTMForm{}, true, fmt.Errorf("%s %s has an invalid suffix, mode or dispatch identity: %q", arch, op, ins.Raw) + } + form, err := parseX86RTMForm(baseOp, spec, ins, arch) + return spec, form, true, err +} + +func parseX86RTMForm(op string, spec amd64RTMSpec, ins Instr, arch string) (amd64RTMForm, error) { + var form amd64RTMForm + switch spec.operand { + case amd64RTMNoOperand: + if len(ins.Args) != 0 { + return form, fmt.Errorf("%s %s takes no operands: %q", arch, op, ins.Raw) + } + case amd64RTMBranch: + if len(ins.Args) != 1 { + return form, fmt.Errorf("%s %s expects one Go 1.27 Ybr target: %q", arch, op, ins.Raw) + } + form.branch = ins.Args[0] + case amd64RTMUnsignedByte: + if len(ins.Args) != 1 || ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 || + ins.Args[0].ImmRaw != "" || ins.Args[0].ImmIsFloat { + return form, fmt.Errorf("%s %s expects one Go 1.27 Yu8 integer immediate: %q", arch, op, ins.Raw) + } + form.immediate = uint8(ins.Args[0].Imm) + default: + panic("unknown amd64 RTM operand kind") + } + return form, nil +} + +// Only the query kind is an ordinary numerical effect. Transaction begin, +// abort and end cannot inherit this straight-line storage/FLAGS continuation. +func x86TypedRTMStatusByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + spec, _, known, err := x86RTMProperties(op, ins, arch) + if !known || spec.kind != amd64RTMTest { + return x86TypedByteEffect{}, false, nil + } + if err != nil { + return x86TypedByteEffect{}, true, err + } + if !x86RegisterQueryPreservesSourceState(ins) { + return x86TypedByteEffect{}, true, fmt.Errorf("%w: RTM status query original/normalized identity differs", ErrProbeNeedsContext) + } + // XTEST reads no GP, numerical or incoming FLAGS input. Its hardware + // transaction observation defines ZF and clears the other five flags. + return x86TypedByteEffect{flagsDefined: x86ByteFlagsAll, flagsKilled: x86ByteFlagsAll}, true, nil +} + +func (c *amd64Ctx) rtmBranchTarget(bi int, ii int, operand Operand) (string, error) { + knownBlock := func(name string) bool { + for _, block := range c.blocks { + if block.name == name { + return true + } + } + return false + } + var target string + switch operand.Kind { + case OpIdent: + target = operand.Ident + case OpReg: + target = string(operand.Reg) + if !knownBlock(target) { + return "", fmt.Errorf("%s XBEGIN register is not a local branch target: %q", c.goarch, target) + } + case OpSym: + target = strings.TrimSpace(operand.Sym) + if strings.HasSuffix(target, "(SB)") { + return "", fmt.Errorf("%s XBEGIN target must be local: %q", c.goarch, target) + } + target = strings.TrimSuffix(target, "<>") + case OpMem: + if !strings.EqualFold(string(operand.Mem.Base), "PC") { + return "", fmt.Errorf("%s XBEGIN expects a label or PC-relative target", c.goarch) + } + current := c.blockBase[bi] + ii + index := current + int(operand.Mem.Off) + targetBlock, ok := c.blockByIdx[index] + if !ok || targetBlock < 0 || targetBlock >= len(c.blocks) { + return "", fmt.Errorf("%s XBEGIN invalid PC-relative target %d(PC)", c.goarch, operand.Mem.Off) + } + target = c.blocks[targetBlock].name + default: + return "", fmt.Errorf("%s XBEGIN expects one Go 1.27 Ybr target", c.goarch) + } + if target == "" || !knownBlock(target) { + return "", fmt.Errorf("%s XBEGIN unknown local target %q", c.goarch, target) + } + return target, nil +} + +func (c *amd64Ctx) emitRTM(bi int, ii int, op string, spec amd64RTMSpec, form amd64RTMForm, emitCondBr amd64EmitCondBr) (bool, bool, error) { + switch spec.kind { + case amd64RTMBegin: + if bi+1 >= len(c.blocks) { + return true, false, fmt.Errorf("%s XBEGIN has no success fallthrough", c.goarch) + } + target, err := c.rtmBranchTarget(bi, ii, form.branch) + if err != nil { + return true, false, err + } + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.xbegin()\n", status) + if err := c.storeRegSized(AX, I32, "%"+status); err != nil { + return true, false, err + } + success := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %%%s, -1\n", success, status) + if err := emitCondBr("%"+success, c.blocks[bi+1].name, target); err != nil { + return true, false, err + } + return true, true, nil + case amd64RTMAbort: + fmt.Fprintf(c.b, " call void @llvm.x86.xabort(i8 %d)\n", form.immediate) + case amd64RTMEnd: + c.b.WriteString(" call void @llvm.x86.xend()\n") + case amd64RTMTest: + active := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.xtest()\n", active) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %%%s, 0\n", zero, active) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + c.clearX86StatusFlags(x86StatusAll &^ x86StatusZF) + default: + panic("unknown amd64 RTM kind") + } + return true, false, nil +} diff --git a/amd64_lower_rtm_test.go b/amd64_lower_rtm_test.go new file mode 100644 index 000000000..82ff3d134 --- /dev/null +++ b/amd64_lower_rtm_test.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64RTMGrammarIsComplete(t *testing.T) { + want := map[Op]amd64RTMSpec{ + "XBEGIN": {kind: amd64RTMBegin, operand: amd64RTMBranch, outputAX: true}, + "XABORT": {kind: amd64RTMAbort, operand: amd64RTMUnsignedByte}, + "XEND": {kind: amd64RTMEnd}, + "XTEST": {kind: amd64RTMTest}, + } + if !reflect.DeepEqual(amd64RTMSpecs, want) { + t.Fatalf("RTM grammar = %+v, want %+v", amd64RTMSpecs, want) + } +} + +func TestTranslateX86RTMCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT rtmforms(SB),$0-0 + XBEGIN fallback + XTEST + XEND + RET +fallback: + XABORT $0 + XABORT $255 + RET +TEXT rtmpcrel(SB),$0-0 + XBEGIN 2(PC) + XEND + RET +` + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "rtmforms": {Name: "rtmforms", Ret: Void}, + "rtmpcrel": {Name: "rtmpcrel", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"target-features"="+rtm"`, + `call i32 @llvm.x86.xbegin()`, + `call i32 @llvm.x86.xtest()`, + `call void @llvm.x86.xend()`, + `call void @llvm.x86.xabort(i8 0)`, + `call void @llvm.x86.xabort(i8 -1)`, + `icmp eq i32`, + `store i1 %`, + `br i1 %`, + } { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "rtm-"+target.name+".ll", "rtm-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86RTMRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "XBEGIN AX"}, + {goarch: "amd64", instruction: "XBEGIN (BX)"}, + {goarch: "amd64", instruction: "XABORT"}, + {goarch: "amd64", instruction: "XABORT $-1"}, + {goarch: "amd64", instruction: "XABORT $256"}, + {goarch: "amd64", instruction: "XABORT AX"}, + {goarch: "amd64", instruction: "XEND AX"}, + {goarch: "amd64", instruction: "XTEST.P"}, + {goarch: "386", instruction: "XABORT $256"}, + {goarch: "386", instruction: "XTEST AX"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's RTM tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_sad.go b/amd64_lower_sad.go new file mode 100644 index 000000000..14403d6d6 --- /dev/null +++ b/amd64_lower_sad.go @@ -0,0 +1,260 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedSADKind uint8 + +const ( + amd64PackedSADBlockSum amd64PackedSADKind = iota + amd64PackedSADMultiple + amd64PackedSADDoubleBlock +) + +type amd64PackedSADForm uint8 + +const ( + amd64PackedSADLegacyX amd64PackedSADForm = iota + amd64PackedSADVEXXY + amd64PackedSADEVEXXYZ +) + +type amd64PackedSADSpec struct { + kind amd64PackedSADKind + form amd64PackedSADForm +} + +// amd64PackedSADSpecs is the complete Go 1.27 byte-SAD grammar. PSADBW uses +// fixed eight-byte block sums, MPSADBW uses one immediate-selected four-byte +// block and sliding windows, and VDBPSADBW independently selects four dwords +// before its sliding SAD. Their vector spellings intentionally differ: +// VPSADBW has EVEX X/Y/Z rows, VMPSADBW is VEX X/Y only, and VDBPSADBW adds +// EVEX writemasking to X/Y/Z. +var amd64PackedSADSpecs = map[Op]amd64PackedSADSpec{ + "PSADBW": {kind: amd64PackedSADBlockSum, form: amd64PackedSADLegacyX}, + "VPSADBW": {kind: amd64PackedSADBlockSum, form: amd64PackedSADEVEXXYZ}, + "MPSADBW": {kind: amd64PackedSADMultiple, form: amd64PackedSADLegacyX}, + "VMPSADBW": {kind: amd64PackedSADMultiple, form: amd64PackedSADVEXXY}, + "VDBPSADBW": {kind: amd64PackedSADDoubleBlock, form: amd64PackedSADEVEXXYZ}, +} + +// One ordered grammar governs both lowering and original-source effects. +func (c *amd64Ctx) lowerPackedSumAbsoluteDifferences(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64PackedSADSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("packed SAD dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, _, err := x86PackedSADProperties(ins, c.goarch) + if err != nil { + return true, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.constantZero(ins) { + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "zeroinitializer") + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "zeroinitializer") + } + if form.legacy && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: %s m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext, ins.Op) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + b, err := c.loadPackedCompareBytes(second, form.byteWidth) + if err != nil { + return true, false, err + } + var result string + if form.legacy && form.memory >= 0 { + // Keep the precise legacy opcode/read/fault behavior. A separately + // synthesized align-1 or MOVDQA load is not an equivalent contract. + result, err = c.emitPackedSADNativeMemory(first, b, form) + } else { + a, loadErr := c.loadPackedUnpackSource(first, x86PackedUnpackForm{ + x86PackedBinaryForm: form.x86PackedBinaryForm, spec: x86PackedUnpackSpec{laneBits: 8}, + }) + if loadErr != nil { + return true, false, loadErr + } + switch form.spec.kind { + case amd64PackedSADBlockSum: + result = c.emitPackedSumAbsoluteByteDifferences(form.byteWidth, a, b) + case amd64PackedSADMultiple: + result = c.emitMultipleSumAbsoluteDifferences(form.byteWidth, a, b, form.immediate) + case amd64PackedSADDoubleBlock: + words := c.emitDoubleBlockSumAbsoluteDifferences(form.byteWidth, b, a, form.immediate) + if form.mask >= 0 { + mask, loadErr := c.loadK(ins.Args[form.mask].Reg) + if loadErr != nil { + return true, false, loadErr + } + old := "zeroinitializer" + if !form.zeroing { + old, loadErr = c.loadPackedCompareLanes(destination, form.byteWidth, 16, false) + if loadErr != nil { + return true, false, loadErr + } + } + words = amd64ApplyIntegerLaneMask(c, form.byteWidth/2, 16, words, old, mask, form.zeroing) + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x i8>\n", bytes, form.byteWidth/2, words, form.byteWidth) + result = "%" + bytes + } + } + if err != nil { + return true, false, err + } + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, result) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, result) +} + +func (c *amd64Ctx) emitPackedSADNativeMemory(source Operand, data string, form x86PackedSADForm) (string, error) { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + switch pointerType { + case "ptr addrspace(256)": + segment = "%gs:" + case "ptr addrspace(257)": + segment = "%fs:" + } + opcode := "psadbw " + if form.spec.kind == amd64PackedSADMultiple { + opcode = fmt.Sprintf("mpsadbw $$%d, ", form.immediate) + } + carrier, value, bytes := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", carrier, data) + fmt.Fprintf(c.b, " %%%s = call <4 x i32> asm sideeffect %q, %q(<4 x i32> %%%s, %s %s)\n", + value, opcode+segment+"($2), $0", "=x,0,r,~{memory}", carrier, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bytes, value) + return "%" + bytes, nil +} + +func (c *amd64Ctx) emitDoubleBlockSumAbsoluteDifferences(byteWidth int, source1, source2 string, immediate uint8) string { + shuffled := "poison" + for laneBase := 0; laneBase < byteWidth; laneBase += 16 { + for destinationDWord := 0; destinationDWord < 4; destinationDWord++ { + sourceDWord := int(immediate>>(2*destinationDWord)) & 3 + for byteInDWord := 0; byteInDWord < 4; byteInDWord++ { + sourceIndex := laneBase + sourceDWord*4 + byteInDWord + destinationIndex := laneBase + destinationDWord*4 + byteInDWord + value, inserted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", value, byteWidth, source2, sourceIndex) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i8> %s, i8 %%%s, i32 %d\n", inserted, byteWidth, shuffled, value, destinationIndex) + shuffled = "%" + inserted + } + } + } + + lanes := byteWidth / 2 + result := "poison" + for blockBase := 0; blockBase < byteWidth; blockBase += 8 { + for outputInBlock := 0; outputInBlock < 4; outputInBlock++ { + source1Base := blockBase + if outputInBlock >= 2 { + source1Base += 4 + } + tmpBase := blockBase + outputInBlock + sum := "" + for byteInDWord := 0; byteInDWord < 4; byteInDWord++ { + firstByte, secondByte := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", firstByte, byteWidth, source1, source1Base+byteInDWord) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", secondByte, byteWidth, shuffled, tmpBase+byteInDWord) + firstWide, secondWide := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i16\n", firstWide, firstByte) + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i16\n", secondWide, secondByte) + firstMinusSecond, secondMinusFirst, firstGreater, difference := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i16 %%%s, %%%s\n", firstMinusSecond, firstWide, secondWide) + fmt.Fprintf(c.b, " %%%s = sub i16 %%%s, %%%s\n", secondMinusFirst, secondWide, firstWide) + fmt.Fprintf(c.b, " %%%s = icmp uge i16 %%%s, %%%s\n", firstGreater, firstWide, secondWide) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %%%s, i16 %%%s\n", difference, firstGreater, firstMinusSecond, secondMinusFirst) + if sum == "" { + sum = "%" + difference + } else { + added := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i16 %s, %%%s\n", added, sum, difference) + sum = "%" + added + } + } + outputLane := blockBase/2 + outputInBlock + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i16> %s, i16 %s, i32 %d\n", inserted, lanes, result, sum, outputLane) + result = "%" + inserted + } + } + return result +} + +func (c *amd64Ctx) emitMultipleSumAbsoluteDifferences(byteWidth int, first, second string, immediate uint8) string { + intrinsic := "llvm.x86.sse41.mpsadbw" + lanes := 8 + if byteWidth == 32 { + intrinsic = "llvm.x86.avx2.mpsadbw" + lanes = 16 + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <%d x i16> @%s(<%d x i8> %s, <%d x i8> %s, i8 %d)\n", result, lanes, intrinsic, byteWidth, second, byteWidth, first, immediate) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %%%s to <%d x i8>\n", bytesValue, lanes, result, byteWidth) + return "%" + bytesValue +} + +func (c *amd64Ctx) emitPackedSumAbsoluteByteDifferences(byteWidth int, first, second string) string { + firstWide := c.newTmp() + secondWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext <%d x i8> %s to <%d x i16>\n", firstWide, byteWidth, first, byteWidth) + fmt.Fprintf(c.b, " %%%s = zext <%d x i8> %s to <%d x i16>\n", secondWide, byteWidth, second, byteWidth) + firstMinusSecond := c.newTmp() + secondMinusFirst := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub <%d x i16> %%%s, %%%s\n", firstMinusSecond, byteWidth, firstWide, secondWide) + fmt.Fprintf(c.b, " %%%s = sub <%d x i16> %%%s, %%%s\n", secondMinusFirst, byteWidth, secondWide, firstWide) + firstGreater := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp uge <%d x i16> %%%s, %%%s\n", firstGreater, byteWidth, firstWide, secondWide) + differences := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i16> %%%s, <%d x i16> %%%s\n", differences, byteWidth, firstGreater, byteWidth, firstMinusSecond, byteWidth, secondMinusFirst) + + groups := byteWidth / 8 + result := "zeroinitializer" + for group := 0; group < groups; group++ { + sum := "" + for lane := group * 8; lane < (group+1)*8; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i16> %%%s, i32 %d\n", value, byteWidth, differences, lane) + if sum == "" { + sum = "%" + value + continue + } + added := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i16 %s, %%%s\n", added, sum, value) + sum = "%" + added + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i16 %s to i64\n", wide, sum) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, groups, result, wide, group) + result = "%" + inserted + } + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", bytesValue, groups, result, byteWidth) + return "%" + bytesValue +} diff --git a/amd64_lower_scalar_float_to_int.go b/amd64_lower_scalar_float_to_int.go new file mode 100644 index 000000000..a0e16e39a --- /dev/null +++ b/amd64_lower_scalar_float_to_int.go @@ -0,0 +1,81 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ScalarFloatToIntSpec struct { + inputBits int + outputBits int + truncating bool + unsigned bool + vector bool +} + +var amd64ScalarFloatToIntSpecs = map[Op]amd64ScalarFloatToIntSpec{ + "CVTSS2SL": {inputBits: 32, outputBits: 32}, + "CVTSS2SQ": {inputBits: 32, outputBits: 64}, + "CVTSD2SL": {inputBits: 64, outputBits: 32}, + "CVTSD2SQ": {inputBits: 64, outputBits: 64}, + "CVTTSS2SL": {inputBits: 32, outputBits: 32, truncating: true}, + "CVTTSS2SQ": {inputBits: 32, outputBits: 64, truncating: true}, + "CVTTSD2SL": {inputBits: 64, outputBits: 32, truncating: true}, + "CVTTSD2SQ": {inputBits: 64, outputBits: 64, truncating: true}, + + "VCVTSS2SI": {inputBits: 32, outputBits: 32, vector: true}, + "VCVTSS2SIQ": {inputBits: 32, outputBits: 64, vector: true}, + "VCVTSD2SI": {inputBits: 64, outputBits: 32, vector: true}, + "VCVTSD2SIQ": {inputBits: 64, outputBits: 64, vector: true}, + "VCVTTSS2SI": {inputBits: 32, outputBits: 32, truncating: true, vector: true}, + "VCVTTSS2SIQ": {inputBits: 32, outputBits: 64, truncating: true, vector: true}, + "VCVTTSD2SI": {inputBits: 64, outputBits: 32, truncating: true, vector: true}, + "VCVTTSD2SIQ": {inputBits: 64, outputBits: 64, truncating: true, vector: true}, + + "VCVTSS2USIL": {inputBits: 32, outputBits: 32, unsigned: true, vector: true}, + "VCVTSS2USIQ": {inputBits: 32, outputBits: 64, unsigned: true, vector: true}, + "VCVTSD2USIL": {inputBits: 64, outputBits: 32, unsigned: true, vector: true}, + "VCVTSD2USIQ": {inputBits: 64, outputBits: 64, unsigned: true, vector: true}, + "VCVTTSS2USIL": {inputBits: 32, outputBits: 32, truncating: true, unsigned: true, vector: true}, + "VCVTTSS2USIQ": {inputBits: 32, outputBits: 64, truncating: true, unsigned: true, vector: true}, + "VCVTTSD2USIL": {inputBits: 64, outputBits: 32, truncating: true, unsigned: true, vector: true}, + "VCVTTSD2USIQ": {inputBits: 64, outputBits: 64, truncating: true, unsigned: true, vector: true}, +} + +// lowerScalarFloatToInteger implements Go 1.27's complete signed legacy/VEX/ +// EVEX scalar float-to-integer family and its EVEX unsigned counterparts. +func (c *amd64Ctx) lowerScalarFloatToInteger(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(rawOp)), ".") + if _, known := amd64ScalarFloatToIntSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(rawOp), string(ins.Op)) { + return true, false, fmt.Errorf("scalar conversion dispatch/source mismatch: %s/%s", rawOp, ins.Op) + } + ins.Op = rawOp + f, known, err := x86ScalarFloatIntegerProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: scalar conversion lacks native MXCSR target", ErrProbeNeedsContext) + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return true, false, fmt.Errorf("%w: scalar conversion needs its original native address size", ErrProbeNeedsContext) + } + return true, false, c.emitNativeScalarFloatInteger(ins, f) +} + +func (c *amd64Ctx) evalScalarFloatForInteger(source Operand, bits int) (string, error) { + if bits == 32 { + return c.evalF32(source) + } + return c.evalF64(source) +} + +func amd64FloatingTypeForBits(bits int) LLVMType { + if bits == 32 { + return LLVMType("float") + } + return LLVMType("double") +} diff --git a/amd64_lower_scalar_float_to_int_test.go b/amd64_lower_scalar_float_to_int_test.go new file mode 100644 index 000000000..b1c73e752 --- /dev/null +++ b/amd64_lower_scalar_float_to_int_test.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86ScalarFloatToIntegerCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + xreg := "X16" + if target.goarch == "386" { + xreg = "X7" // A high named bank has no physical mode32 identity. + } + var source strings.Builder + source.WriteString("TEXT scalarfloattointforms(SB),NOSPLIT,$0-0\n") + for _, precision := range []string{"SS", "SD"} { + for _, truncating := range []bool{false, true} { + prefix := "CVT" + if truncating { + prefix = "CVTT" + } + for _, width := range []string{"L", "Q"} { + if target.goarch == "386" && width == "Q" { + continue + } + op := prefix + precision + "2S" + width + fmt.Fprintf(&source, "\t%s X0, AX\n", op) + fmt.Fprintf(&source, "\t%s 8(BX), DI\n", op) + vop := "V" + prefix + precision + "2SI" + if width == "Q" { + vop += "Q" + } + fmt.Fprintf(&source, "\t%s X0, AX\n", vop) + fmt.Fprintf(&source, "\t%s 16(BX), DI\n", vop) + if truncating { + fmt.Fprintf(&source, "\t%s.SAE %s, AX\n", vop, xreg) + } else { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s %s, AX\n", vop, rounding, xreg) + } + } + } + } + for _, truncating := range []bool{false, true} { + prefix := "VCVT" + if truncating { + prefix = "VCVTT" + } + for _, width := range []string{"L", "Q"} { + if target.goarch == "386" && width == "Q" { + continue + } + op := prefix + precision + "2USI" + width + fmt.Fprintf(&source, "\t%s %s, AX\n", op, xreg) + fmt.Fprintf(&source, "\t%s 24(BX), DI\n", op) + if truncating { + fmt.Fprintf(&source, "\t%s.SAE %s, AX\n", op, xreg) + } else { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s %s, AX\n", op, rounding, xreg) + } + } + } + } + } + source.WriteString("\tRET\n") + // This complete original format inventory has no input producers. + // Preserve Go objects and three-API Context separately from the + // same lowerer's private, compile-only format transport. + ll := x86UnboundVoidScalarFormatObject(t, source.String(), "scalarfloattointforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scalar-float-to-int-"+target.name+".ll", "scalar-float-to-int-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ScalarFloatToIntegerRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "CVTSS2SL Y0, AX", + "CVTTSD2SQ X16, AX", + "CVTSS2SL X0, X1", + "CVTSS2SL.Z X0, AX", + "VCVTSS2SI Y0, AX", + "VCVTSS2SI X0, X1", + "VCVTSS2SI.SAE X0, AX", + "VCVTTSS2SI.RN_SAE X0, AX", + "VCVTSD2SI.RD_SAE 8(BX), AX", + "VCVTTSD2SI.SAE 8(BX), AX", + "VCVTSS2USIL X0, X1", + "VCVTSS2USIL.Z X0, AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ScalarFloatToIntegerRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "CVTSS2SQ X0, AX", + "CVTTSD2SQ 8(BX), AX", + "VCVTTSD2SIQ.SAE X16, AX", + "VCVTSS2USIQ X16, AX", + "VCVTTSD2USIQ.SAE X16, AX", + "CVTSS2SL X8, AX", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ScalarFloatToIntegerRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86ScalarFloatToIntegerRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar float-to-integer tables for %s", instruction, goarch) + } +} diff --git a/amd64_lower_scalar_integer_float.go b/amd64_lower_scalar_integer_float.go new file mode 100644 index 000000000..609f65f24 --- /dev/null +++ b/amd64_lower_scalar_integer_float.go @@ -0,0 +1,166 @@ +package plan9asm + +import "fmt" + +type amd64ScalarIntegerFloatSpec struct { + sourceBits int + resultBits int + unsigned bool + rounding bool +} + +// Complete unsigned _yvcvtusi2sdl family. Signed legacy/VEX forms retain their +// independent table. Supported-op extraction reads both registries. +var amd64ScalarIntegerFloatOps = map[string]amd64ScalarIntegerFloatSpec{ + "VCVTUSI2SDL": {sourceBits: 32, resultBits: 64, unsigned: true}, + "VCVTUSI2SDQ": {sourceBits: 64, resultBits: 64, unsigned: true, rounding: true}, + "VCVTUSI2SSL": {sourceBits: 32, resultBits: 32, unsigned: true, rounding: true}, + "VCVTUSI2SSQ": {sourceBits: 64, resultBits: 32, unsigned: true, rounding: true}, +} + +func (c *amd64Ctx) lowerScalarIntegerToFloat(op Op, ins Instr) (bool, bool, error) { + if ok, term, err := c.lowerSignedIntegerScalarFloat(op, ins); ok { + return ok, term, err + } + return c.lowerUnsignedIntegerScalarFloat(op, ins) +} + +// emitRoundedScalarIntegerToFloatBits converts through an explicit integer +// significand so embedded rounding remains correct even when LLVM 22 lowers a +// constrained conversion to an instruction controlled by the ambient MXCSR. +func (c *amd64Ctx) emitRoundedScalarIntegerToFloatBits(integer string, spec amd64ScalarIntegerFloatSpec, rounding string) string { + integerType := amd64IntegerTypeForBits(spec.sourceBits) + precision := 24 + bias := 127 + if spec.resultBits == 64 { + precision = 53 + bias = 1023 + } + + negative := "false" + magnitude := integer + if !spec.unsigned { + isNegative := c.newTmp() + negated := c.newTmp() + absolute := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, 0\n", isNegative, integerType, integer) + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", negated, integerType, integer) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s %s\n", absolute, isNegative, integerType, negated, integerType, integer) + negative = "%" + isNegative + magnitude = "%" + absolute + } + + isZero := c.newTmp() + safeMagnitude := c.newTmp() + leadingZeros := c.newTmp() + msb := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", isZero, integerType, magnitude) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s 1, %s %s\n", safeMagnitude, isZero, integerType, integerType, magnitude) + fmt.Fprintf(c.b, " %%%s = call %s @llvm.ctlz.i%d(%s %%%s, i1 false)\n", leadingZeros, integerType, spec.sourceBits, integerType, safeMagnitude) + fmt.Fprintf(c.b, " %%%s = sub %s %d, %%%s\n", msb, integerType, spec.sourceBits-1, leadingZeros) + + needsRightShift := c.newTmp() + rawRightShift := c.newTmp() + rightShift := c.newTmp() + rawLeftShift := c.newTmp() + leftShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp uge %s %%%s, %d\n", needsRightShift, integerType, msb, precision-1) + fmt.Fprintf(c.b, " %%%s = sub %s %%%s, %d\n", rawRightShift, integerType, msb, precision-1) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s 0\n", rightShift, needsRightShift, integerType, rawRightShift, integerType) + fmt.Fprintf(c.b, " %%%s = sub %s %d, %%%s\n", rawLeftShift, integerType, precision-1, msb) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s 0, %s %%%s\n", leftShift, needsRightShift, integerType, integerType, rawLeftShift) + + shiftedRight := c.newTmp() + shiftedLeft := c.newTmp() + significand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", shiftedRight, integerType, magnitude, rightShift) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", shiftedLeft, integerType, magnitude, leftShift) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s %%%s\n", significand, needsRightShift, integerType, shiftedRight, integerType, shiftedLeft) + + oneAtShift := c.newTmp() + remainderMask := c.newTmp() + remainder := c.newTmp() + inexact := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s 1, %%%s\n", oneAtShift, integerType, rightShift) + fmt.Fprintf(c.b, " %%%s = sub %s %%%s, 1\n", remainderMask, integerType, oneAtShift) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", remainder, integerType, magnitude, remainderMask) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, 0\n", inexact, integerType, remainder) + + increment := "false" + switch rounding { + case "RN_SAE": + hasShift := c.newTmp() + rawHalfShift := c.newTmp() + halfShift := c.newTmp() + half := c.newTmp() + aboveHalf := c.newTmp() + equalHalf := c.newTmp() + lowBit := c.newTmp() + odd := c.newTmp() + tieAndOdd := c.newTmp() + roundNearest := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, 0\n", hasShift, integerType, rightShift) + fmt.Fprintf(c.b, " %%%s = sub %s %%%s, 1\n", rawHalfShift, integerType, rightShift) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s 0\n", halfShift, hasShift, integerType, rawHalfShift, integerType) + fmt.Fprintf(c.b, " %%%s = shl %s 1, %%%s\n", half, integerType, halfShift) + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %%%s\n", aboveHalf, integerType, remainder, half) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %%%s\n", equalHalf, integerType, remainder, half) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 1\n", lowBit, integerType, significand) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, 0\n", odd, integerType, lowBit) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", tieAndOdd, equalHalf, odd) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", roundNearest, aboveHalf, tieAndOdd) + increment = "%" + roundNearest + case "RD_SAE": + if !spec.unsigned { + roundDown := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %s, %%%s\n", roundDown, negative, inexact) + increment = "%" + roundDown + } + case "RU_SAE": + if spec.unsigned { + increment = "%" + inexact + } else { + positive := c.newTmp() + roundUp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", positive, negative) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", roundUp, positive, inexact) + increment = "%" + roundUp + } + } + + incrementInteger := c.newTmp() + roundedSignificand := c.newTmp() + overflow := c.newTmp() + normalizedOverflow := c.newTmp() + normalized := c.newTmp() + exponentIncrement := c.newTmp() + exponent := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", incrementInteger, increment, integerType) + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %%%s\n", roundedSignificand, integerType, significand, incrementInteger) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %d\n", overflow, integerType, roundedSignificand, uint64(1)< +#include +extern void scalarintegerfloat(uint8_t *, const int32_t *, const int64_t *, const uint32_t *, const uint64_t *, const float *, const double *); +int main(void) { + const int32_t signed32 = -16777217; + const int64_t signed64 = -16777217; + const uint32_t unsigned32 = UINT32_MAX; + const uint64_t unsigned64 = UINT64_MAX; + const float passthrough32[4] = {200.0f, 201.0f, 202.0f, 203.0f}; + const double passthrough64[2] = {100.0, 101.0}; + union { uint8_t bytes[128]; double d64[16]; float f32[32]; } out = {0}; + scalarintegerfloat(out.bytes, &signed32, &signed64, &unsigned32, &unsigned64, passthrough32, passthrough64); + if (out.d64[0] != -16777217.0 || out.d64[1] != 101.0) return 1; + if (out.d64[2] != -16777217.0 || out.d64[3] != 101.0) return 2; + if (out.f32[8] != -16777218.0f || out.f32[9] != 201.0f || out.f32[10] != 202.0f || out.f32[11] != 203.0f) return 3; + if (out.f32[12] != -16777216.0f || out.f32[13] != 201.0f || out.f32[14] != 202.0f || out.f32[15] != 203.0f) return 4; + if (out.d64[8] != 4294967295.0 || out.d64[9] != 101.0) return 5; + if (out.d64[10] != 18446744073709549568.0 || out.d64[11] != 101.0) return 6; + if (out.f32[24] != 4294967040.0f || out.f32[25] != 201.0f || out.f32[26] != 202.0f || out.f32[27] != 203.0f) { + fprintf(stderr, "unsigned32 RD result=%a upper={%a,%a,%a}\n", out.f32[24], out.f32[25], out.f32[26], out.f32[27]); + return 7; + } + if (out.f32[28] != 18446744073709551616.0f || out.f32[29] != 201.0f || out.f32[30] != 202.0f || out.f32[31] != 203.0f) return 8; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_integer_float", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_scalar_precision.go b/amd64_lower_scalar_precision.go new file mode 100644 index 000000000..152a1e3b3 --- /dev/null +++ b/amd64_lower_scalar_precision.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerScalarPrecisionConvert(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86ScalarPrecisionProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + ins.Op = op + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: %s needs its native x86 MXCSR target, not pure LLVM precision arithmetic", ErrProbeNeedsContext, op) + } + if form.memory >= 0 { + memory := x86PackedBinaryForm{memory: 0, memoryBytes: form.memoryBytes} + if err := c.validateConflictFunnelMemory(ins, memory); err != nil { + return true, false, err + } + } + return true, false, c.emitScalarPrecisionConvert(ins, form) +} + +// Keep the original conversion, rather than ambient pure fpext/fptrunc: +// DAZ/FTZ, RC, status, unmasked exceptions and explicit ER/SAE are hardware +// effects. The memory operand remains on that same instruction, so native +// E3 suppression and exact m32/m64 faults survive dead-value elimination. +func (c *amd64Ctx) emitScalarPrecisionConvert(ins Instr, form x86ScalarPrecisionForm) error { + const typ = "<4 x float>" // Bit-preserving 128-bit carrier, never arithmetic precision. + load := func(index int) (string, error) { + return c.loadFloatingVectorOperand(ins.Args[index], 16, 4, 32) + } + old := "zeroinitializer" + var err error + if form.legacy || form.mask >= 0 && !form.zeroing { + old, err = load(form.destination) + if err != nil { + return err + } + } + evex := x86ScalarPrecisionNativeEVEX(ins, form) + register := "x" + if evex { + register = "v" + } + value, inputType, inputConstraint, err := c.scalarPrecisionNativeSource(ins.Args[0], form.spec.inputBits, register) + if err != nil { + return err + } + constraints, args := "="+register+",0", typ+" "+old + inputIndex := 2 + operands := "$2, $0" + if !form.legacy { + upper, err := load(form.second) + if err != nil { + return err + } + constraints += "," + register + args += ", " + typ + " " + upper + inputIndex, operands = 3, "$3, $2, $0" + } + constraints += "," + inputConstraint + args += ", " + inputType + " " + value + mnemonic, _, _ := strings.Cut(strings.ToLower(string(ins.Op)), ".") + if evex { + mnemonic = "{evex} " + mnemonic + } + assembly := mnemonic + " " + if form.sae { + assembly += "{sae}, " + } else if form.rounding != "" { + assembly += "{" + strings.ToLower(strings.TrimSuffix(form.rounding, "_SAE")) + "-sae}, " + } + assembly += operands + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + assembly = fmt.Sprintf("kmovw ${%d:k}, %%k1; ", inputIndex+1) + assembly + " {%k1}" + if form.zeroing { + assembly += " {z}" + } + constraints += ",r,~{k1}" + args += ", i32 " + c.approximateMathMask32(mask) + } + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, typ, assembly, constraints+",~{memory},~{fpsr}", args) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytes, typ, result) + if form.legacy { + return c.storeLegacyXViews(ins.Args[form.destination].Reg, "%"+bytes) + } + return c.storePackedMoveOperand(ins.Args[form.destination], 16, "%"+bytes) +} + +func (c *amd64Ctx) scalarPrecisionNativeSource(source Operand, bits int, register string) (value, typ, constraint string, err error) { + const carrier = "<4 x float>" + if source.Kind == OpReg { + value, err = c.loadFloatingVectorOperand(source, 16, 4, 32) + return value, carrier, register, err + } + if source.Kind == OpFP { + value, err = c.loadScalarLaneFPBits(source.FPOffset, bits) + if err != nil { + return "", "", "", err + } + lanes := 128 / bits + inserted, converted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> zeroinitializer, i%d %s, i32 0\n", inserted, lanes, bits, bits, value) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to %s\n", converted, lanes, bits, inserted, carrier) + return "%" + converted, carrier, register, nil + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + return pointer, fmt.Sprintf("%s elementtype([%d x i8])", pointerType, bits/8), "*m", err +} diff --git a/amd64_lower_scaled_round.go b/amd64_lower_scaled_round.go new file mode 100644 index 000000000..34a08e824 --- /dev/null +++ b/amd64_lower_scaled_round.go @@ -0,0 +1,283 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ScaledRoundMode uint8 + +const ( + amd64RoundScale amd64ScaledRoundMode = iota + amd64Reduce +) + +type amd64ScaledRoundSpec struct { + laneBits int + scalar bool + mode amd64ScaledRoundMode +} + +var amd64ScaledRoundSpecs = map[Op]amd64ScaledRoundSpec{ + "VRNDSCALEPS": {laneBits: 32}, + "VRNDSCALEPD": {laneBits: 64}, + "VRNDSCALESS": {laneBits: 32, scalar: true}, + "VRNDSCALESD": {laneBits: 64, scalar: true}, + "VREDUCEPS": {laneBits: 32, mode: amd64Reduce}, + "VREDUCEPD": {laneBits: 64, mode: amd64Reduce}, + "VREDUCESS": {laneBits: 32, scalar: true, mode: amd64Reduce}, + "VREDUCESD": {laneBits: 64, scalar: true, mode: amd64Reduce}, +} + +func (c *amd64Ctx) lowerScaledRound(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64ScaledRoundSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return c.lowerNativeFloatingHelper(op, ins) + } + shape := amd64UnaryImmediateFloatingShape{laneBits: spec.laneBits, scalar: spec.scalar} + form, err := c.parseUnaryImmediateFloatingForm(baseOp, suffix, shape, ins) + if err != nil { + return true, false, err + } + mxcsr := c.loadMXCSR() + dazBits, daz := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i32 %s, 64\n", dazBits, mxcsr) + fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", daz, dazBits) + roundingMode := fmt.Sprintf("%d", form.immediate&3) + if form.immediate&4 != 0 { + shifted, extracted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i32 %s, 13\n", shifted, mxcsr) + fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 3\n", extracted, shifted) + roundingMode = "%" + extracted + } + if spec.scalar { + return c.lowerScalarScaledRound(spec, form.properties, form.source, form.passthrough, form.destination, form.mask, "%"+daz, roundingMode, form.immediate) + } + return c.lowerPackedScaledRound(spec, form.properties, form.source, form.destination, form.byteWidth, form.mask, "%"+daz, roundingMode, form.immediate) +} + +func (c *amd64Ctx) lowerPackedScaledRound(spec amd64ScaledRoundSpec, properties amd64BinaryFloatingSuffix, source, destination Operand, byteWidth int, mask, daz, roundingMode string, immediate uint8) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + input, err := c.loadPackedCompareLanes(source, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + result := c.emitScaledRoundBits(lanes, spec.laneBits, input, daz, roundingMode, immediate>>4, spec.mode) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarScaledRound(spec amd64ScaledRoundSpec, properties amd64BinaryFloatingSuffix, source, passthrough, destination Operand, mask, daz, roundingMode string, immediate uint8) (bool, bool, error) { + sourceBits, err := c.loadFloatingScalarBits(source, spec.laneBits) + if err != nil { + return true, false, err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", input, spec.laneBits, spec.laneBits, sourceBits) + result := c.emitScaledRoundBits(1, spec.laneBits, "%"+input, daz, roundingMode, immediate>>4, spec.mode) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + passthroughBytes, err := c.loadX(passthrough.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, passthroughBytes) + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitScaledRoundBits(lanes, laneBits int, input, daz, roundingMode string, scale uint8, mode amd64ScaledRoundMode) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + mantissaBits, bias := 23, 127 + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + if laneBits == 64 { + mantissaBits, bias = 52, 1023 + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + } + implicitBit := uint64(1) << mantissaBits + alreadyThreshold := bias + mantissaBits - int(scale) + regularThreshold := bias - int(scale) + stepBits := uint64(bias-int(scale)) << mantissaBits + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + bits, exponent, mantissa, sign := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", bits, vectorType, input, lane) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", exponent, laneBits, bits, exponentMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", mantissa, laneBits, bits, mantissaMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", sign, laneBits, bits, signBit) + exponentZero, exponentOnes, mantissaZero := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, exponent) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", exponentOnes, laneBits, exponent, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", mantissaZero, laneBits, mantissa) + mantissaNonzero, negative := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", mantissaNonzero, mantissaZero) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", negative, laneBits, sign) + isNaN, isInfinity := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, exponentOnes, mantissaNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, exponentOnes, mantissaZero) + effectiveZeroPart, effectiveZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %s\n", effectiveZeroPart, mantissaZero, daz) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", effectiveZero, exponentZero, effectiveZeroPart) + + exponentField := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %d\n", exponentField, laneBits, exponent, mantissaBits) + already, regularLower, regularUpper, regular := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp uge i%d %%%s, %d\n", already, laneBits, exponentField, alreadyThreshold) + fmt.Fprintf(c.b, " %%%s = icmp uge i%d %%%s, %d\n", regularLower, laneBits, exponentField, regularThreshold) + fmt.Fprintf(c.b, " %%%s = icmp ult i%d %%%s, %d\n", regularUpper, laneBits, exponentField, alreadyThreshold) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", regular, regularLower, regularUpper) + half := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", half, laneBits, exponentField, regularThreshold-1) + + safeExponent, drop := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %d\n", safeExponent, regular, laneBits, exponentField, laneBits, alreadyThreshold-1) + fmt.Fprintf(c.b, " %%%s = sub i%d %d, %%%s\n", drop, laneBits, mantissaBits+bias-int(scale), safeExponent) + significand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", significand, laneBits, mantissa, implicitBit) + quotient, oneAtDrop, remainderMask, remainder := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %%%s\n", quotient, laneBits, significand, drop) + fmt.Fprintf(c.b, " %%%s = shl i%d 1, %%%s\n", oneAtDrop, laneBits, drop) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, 1\n", remainderMask, laneBits, oneAtDrop) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %%%s\n", remainder, laneBits, significand, remainderMask) + halfRemainder, aboveHalf, equalHalf := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, 1\n", halfRemainder, laneBits, oneAtDrop) + fmt.Fprintf(c.b, " %%%s = icmp ugt i%d %%%s, %%%s\n", aboveHalf, laneBits, remainder, halfRemainder) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %%%s\n", equalHalf, laneBits, remainder, halfRemainder) + quotientOddValue, quotientOdd, tiedOdd := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, 1\n", quotientOddValue, laneBits, quotient) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", quotientOdd, laneBits, quotientOddValue) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", tiedOdd, equalHalf, quotientOdd) + nearestRegular := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", nearestRegular, aboveHalf, tiedOdd) + remainderNonzero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", remainderNonzero, laneBits, remainder) + incrementRegular := c.emitScaledRoundIncrement(roundingMode, "%"+negative, "%"+remainderNonzero, "%"+nearestRegular) + incrementValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to i%d\n", incrementValue, incrementRegular, laneBits) + roundedQuotient, roundedSignificand := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i%d %%%s, %%%s\n", roundedQuotient, laneBits, quotient, incrementValue) + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, %%%s\n", roundedSignificand, laneBits, roundedQuotient, drop) + carryValue, carry, carryInteger := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", carryValue, laneBits, roundedSignificand, implicitBit<<1) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", carry, laneBits, carryValue) + fmt.Fprintf(c.b, " %%%s = zext i1 %%%s to i%d\n", carryInteger, carry, laneBits) + roundedExponentField, roundedExponent := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i%d %%%s, %%%s\n", roundedExponentField, laneBits, exponentField, carryInteger) + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, %d\n", roundedExponent, laneBits, roundedExponentField, mantissaBits) + roundedFraction, roundedMagnitude, roundedRegular := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", roundedFraction, laneBits, roundedSignificand, mantissaMask) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", roundedMagnitude, laneBits, roundedExponent, roundedFraction) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", roundedRegular, laneBits, roundedMagnitude, sign) + + halfAbove := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ugt i%d %%%s, %d\n", halfAbove, laneBits, significand, implicitBit) + nonzero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", nonzero, effectiveZero) + incrementHalf := c.emitScaledRoundIncrement(roundingMode, "%"+negative, "%"+nonzero, "%"+halfAbove) + incrementSmall := c.emitScaledRoundIncrement(roundingMode, "%"+negative, "%"+nonzero, "false") + incrementNonRegular := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i1 %s, i1 %s\n", incrementNonRegular, half, incrementHalf, incrementSmall) + signedStep, stepOrZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", signedStep, laneBits, sign, stepBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", stepOrZero, incrementNonRegular, laneBits, signedStep, laneBits, sign) + roundedNeeded, roundedFinite, zeroAdjusted := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", roundedNeeded, regular, laneBits, roundedRegular, laneBits, stepOrZero) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", roundedFinite, already, laneBits, bits, laneBits, roundedNeeded) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", zeroAdjusted, effectiveZero, laneBits, sign, laneBits, roundedFinite) + withInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", withInfinity, isInfinity, laneBits, bits, laneBits, zeroAdjusted) + quietNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", quietNaN, laneBits, bits, quietBit) + roundedFinal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", roundedFinal, isNaN, laneBits, quietNaN, laneBits, withInfinity) + laneResult := "%" + roundedFinal + if mode == amd64Reduce { + effectiveSource := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", effectiveSource, effectiveZero, laneBits, sign, laneBits, bits) + laneResult = c.emitReduceResult(laneBits, "%"+effectiveSource, "%"+zeroAdjusted, "%"+isNaN, "%"+isInfinity, roundingMode, quietBit, signBit) + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, result, laneBits, laneResult, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitScaledRoundIncrement(roundingMode, negative, nonzero, nearest string) string { + isNearest, isDown, isUp := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 0\n", isNearest, roundingMode) + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 1\n", isDown, roundingMode) + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 2\n", isUp, roundingMode) + nearestIncrement, downIncrement := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %s\n", nearestIncrement, isNearest, nearest) + fmt.Fprintf(c.b, " %%%s = and i1 %s, %s\n", downIncrement, negative, nonzero) + downSelected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", downSelected, isDown, downIncrement) + nonnegative, upIncrement, upSelected := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", nonnegative, negative) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %s\n", upIncrement, nonnegative, nonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", upSelected, isUp, upIncrement) + combined, result := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", combined, nearestIncrement, downSelected) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", result, combined, upSelected) + return "%" + result +} + +func (c *amd64Ctx) emitReduceResult(laneBits int, sourceBits, finiteRoundedBits, isNaN, isInfinity, roundingMode string, quietBit, signBit uint64) string { + floatType := "float" + if laneBits == 64 { + floatType = "double" + } + special := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", special, isNaN, isInfinity) + safeSourceBits, safeRoundedBits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 0, i%d %s\n", safeSourceBits, special, laneBits, laneBits, sourceBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 0, i%d %s\n", safeRoundedBits, special, laneBits, laneBits, finiteRoundedBits) + sourceFloat, roundedFloat := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %%%s to %s\n", sourceFloat, laneBits, safeSourceBits, floatType) + fmt.Fprintf(c.b, " %%%s = bitcast i%d %%%s to %s\n", roundedFloat, laneBits, safeRoundedBits, floatType) + reducedFloat, reducedBits := c.newTmp(), c.newTmp() + // REDUCE's final subtraction obeys the selected immediate/MXCSR RC too. + // Its source has already undergone DAZ preprocessing; the exact helper + // must not apply the host's ambient rounding or flush its tiny result. + fmt.Fprintf(c.b, " %%%s = call %s @__plan9asm_add_sub_round_f%d(%s %%%s, %s %%%s, i1 true, i32 %s, i32 0)\n", reducedFloat, floatType, laneBits, floatType, sourceFloat, floatType, roundedFloat, roundingMode) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to i%d\n", reducedBits, floatType, reducedFloat, laneBits) + reducedMagnitude, reducedZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", reducedMagnitude, laneBits, reducedBits, signBit-1) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", reducedZero, laneBits, reducedMagnitude) + isDown := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 1\n", isDown, roundingMode) + zeroBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d 0\n", zeroBits, isDown, laneBits, signBit, laneBits) + zeroAdjusted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", zeroAdjusted, reducedZero, laneBits, zeroBits, laneBits, reducedBits) + withInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d 0, i%d %%%s\n", withInfinity, isInfinity, laneBits, laneBits, zeroAdjusted) + quietNaNBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", quietNaNBits, laneBits, sourceBits, quietBit) + final := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", final, isNaN, laneBits, quietNaNBits, laneBits, withInfinity) + return "%" + final +} diff --git a/amd64_lower_scaled_round_test.go b/amd64_lower_scaled_round_test.go new file mode 100644 index 000000000..ed6397cd4 --- /dev/null +++ b/amd64_lower_scaled_round_test.go @@ -0,0 +1,321 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateAMD64ScaledRoundRepresentativeForms(t *testing.T) { + const source = `TEXT scaledroundforms(SB),$0-0 + VREDUCEPS $3, Z1, K2, Z3 + VRNDSCALESD $20, X4, X5, K6, X7 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + ir := x86FloatingHelperOriginalFormatObject(t, source, "scaledroundforms", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "round-representative.ll", "round-representative.o", ir) +} + +func TestAMD64ScaledRoundGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64ScaledRoundSpec{ + "VRNDSCALEPS": {laneBits: 32}, + "VRNDSCALEPD": {laneBits: 64}, + "VRNDSCALESS": {laneBits: 32, scalar: true}, + "VRNDSCALESD": {laneBits: 64, scalar: true}, + "VREDUCEPS": {laneBits: 32, mode: amd64Reduce}, + "VREDUCEPD": {laneBits: 64, mode: amd64Reduce}, + "VREDUCESS": {laneBits: 32, scalar: true, mode: amd64Reduce}, + "VREDUCESD": {laneBits: 64, scalar: true, mode: amd64Reduce}, + } + if len(amd64ScaledRoundSpecs) != len(expected) { + t.Fatalf("scaled-round grammar has %d entries, want %d", len(amd64ScaledRoundSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64ScaledRoundSpecs[op]; !ok { + t.Errorf("scaled-round grammar omitted %s", op) + } else if got != want { + t.Errorf("scaled-round grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86ScaledRoundCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT scaledroundallforms(SB),$0-0\n") + for _, op := range []string{"VRNDSCALEPS", "VRNDSCALEPD", "VREDUCEPS", "VREDUCEPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $0, %s1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s $255, 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST $1, 16(AX), %s%d\n", op, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $2, %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z $3, 24(AX), K2, %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z $4, 32(AX), K3, %s%d\n", op, width, last) + } + if width == "Z" { + fmt.Fprintf(&source, "\t%s.SAE $5, Z1, Z%d\n", op, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.SAE.Z $6, Z1, K4, Z%d\n", op, last) + } + } + } + } + if target.goarch == "amd64" { + for _, op := range []string{"VRNDSCALESS", "VRNDSCALESD", "VREDUCESS", "VREDUCESD"} { + fmt.Fprintf(&source, "\t%s $0, X1, X2, X31\n", op) + fmt.Fprintf(&source, "\t%s $255, 40(AX), X2, X31\n", op) + fmt.Fprintf(&source, "\t%s.SAE $1, X1, X2, X31\n", op) + fmt.Fprintf(&source, "\t%s $2, X1, X2, K5, X31\n", op) + fmt.Fprintf(&source, "\t%s.Z $3, 48(AX), X2, K6, X31\n", op) + fmt.Fprintf(&source, "\t%s.SAE.Z $4, X1, X2, K7, X31\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86FloatingHelperOriginalFormatObject(t, source.String(), "scaledroundallforms", target.goarch, target.triple) + if ir == "" { + return // Go/public-Context coverage, not an LLVM object claim. + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scaled-round-"+target.name+".ll", "scaled-round-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ScaledRoundRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VREDUCEPS $-1, X0, X1"}, + {goarch: "amd64", instruction: "VRNDSCALEPD $256, X0, X1"}, + {goarch: "amd64", instruction: "VREDUCEPS X0, X1"}, + {goarch: "amd64", instruction: "VRNDSCALEPS $0, X0, Y1"}, + {goarch: "amd64", instruction: "VREDUCEPS.BCST $0, X0, X1"}, + {goarch: "amd64", instruction: "VRNDSCALEPS.SAE $0, Z0, X1"}, + {goarch: "amd64", instruction: "VREDUCEPD.SAE $0, 0(AX), Z1"}, + {goarch: "amd64", instruction: "VRNDSCALEPS.RN_SAE $0, Z0, Z1"}, + {goarch: "amd64", instruction: "VREDUCEPS.Z $0, X0, X1"}, + {goarch: "amd64", instruction: "VRNDSCALEPS $0, X0, K0, X1"}, + {goarch: "amd64", instruction: "VREDUCESS $0, Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRNDSCALESD.BCST $0, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VREDUCESD $0, X0, X1, Y2"}, + {goarch: "386", instruction: "VRNDSCALESS $0, X0, X1, X2"}, + {goarch: "386", instruction: "VREDUCEPS $0, X0, K1, X2"}, + {goarch: "386", instruction: "VRNDSCALEPS $0, Z8, Z0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scaled-round tables", test.instruction) + } + }) + } +} + +func qualifyAMD64ScaledRoundRuntimeSemantics(t *testing.T) { + execution := x86FloatingHelperRuntimeTools(t) + var source strings.Builder + source.WriteString(`TEXT scaledroundsemantics(SB),$0-56 + MOVQ out+0(FP), AX + MOVQ source32+8(FP), BX + MOVQ source64+16(FP), CX + MOVQ old32+24(FP), DX + MOVQ old64+32(FP), R8 + MOVQ upper+40(FP), R9 + MOVQ mask+48(FP), R10 + KMOVQ R10, K1 + VMOVDQU64 0(R9), X4 + STC +`) + for scale := 0; scale < 16; scale++ { + for rounding := 0; rounding < 4; rounding++ { + index := scale*4 + rounding + immediate := scale<<4 | rounding + fmt.Fprintf(&source, "\tVRNDSCALESS $%d, %d(BX), X4, X5\n", immediate, index%16*4) + fmt.Fprintf(&source, "\tVMOVSS X5, %d(AX)\n", index*4) + fmt.Fprintf(&source, "\tVREDUCESS $%d, %d(BX), X4, X5\n", immediate, index%16*4) + fmt.Fprintf(&source, "\tVMOVSS X5, %d(AX)\n", 256+index*4) + fmt.Fprintf(&source, "\tVRNDSCALESD $%d, %d(CX), X4, X5\n", immediate, index%16*8) + fmt.Fprintf(&source, "\tVMOVSD X5, %d(AX)\n", 512+index*8) + fmt.Fprintf(&source, "\tVREDUCESD $%d, %d(CX), X4, X5\n", immediate, index%16*8) + fmt.Fprintf(&source, "\tVMOVSD X5, %d(AX)\n", 1024+index*8) + } + } + source.WriteString(` VMOVDQU32 0(DX), Z2 + VRNDSCALEPS $35, 0(BX), K1, Z2 + VMOVDQU32 Z2, 1536(AX) + VREDUCEPS.Z $35, 0(BX), K1, Z2 + VMOVDQU32 Z2, 1600(AX) + VRNDSCALEPS.BCST $35, 0(BX), Z2 + VMOVDQU32 Z2, 1664(AX) + VMOVDQU64 0(R8), Z3 + VRNDSCALEPD $35, 0(CX), K1, Z3 + VMOVDQU64 Z3, 1728(AX) + VREDUCEPD.Z $35, 0(CX), K1, Z3 + VMOVDQU64 Z3, 1792(AX) + VRNDSCALEPD.BCST $35, 0(CX), Z3 + VMOVDQU64 Z3, 1856(AX) + VRNDSCALESS $0, 0(BX), X4, X5 + VMOVDQU64 X5, 1920(AX) + VREDUCESD $0, 0(CX), X4, X6 + VMOVDQU64 X6, 1936(AX) + VRNDSCALESS $36, 0(BX), X4, X5 + VMOVSS X5, 1952(AX) + VREDUCESS $36, 8(BX), X4, X5 + VMOVSS X5, 1956(AX) + VRNDSCALESD $36, 0(CX), X4, X5 + VMOVSD X5, 1960(AX) + VREDUCESD $36, 16(CX), X4, X5 + VMOVSD X5, 1968(AX) + SETCS 1976(AX) + RET +`) + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := execution.triple + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"scaledroundsemantics": { + Name: "scaledroundsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: I64, Index: 6, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void scaledroundsemantics(uint8_t *, const uint32_t *, const uint64_t *, const uint32_t *, const uint64_t *, const uint64_t *, uint64_t); +static int inc(unsigned rc,int neg,int nonzero,int nearest) { return rc==0?nearest:(rc==1?neg&&nonzero:(rc==2?!neg&&nonzero:0)); } +static uint32_t round32(uint32_t u,unsigned scale,unsigned rc,int daz) { + const uint32_t em=0x7f800000,fm=0x007fffff,qb=0x00400000,sb=0x80000000,ib=0x00800000; + uint32_t e=(u&em)>>23,f=u&fm,s=u&sb; if(e==255) return f?(u|qb):u; + if(e==0&&(f==0||daz)) return s; + unsigned already=127+23-scale, regular=127-scale; + if(e>=already) return u; + uint32_t sig=f|ib; + if(e>=regular) { unsigned drop=23+127-scale-e; uint32_t q=sig>>drop, rem=sig&((UINT32_C(1)<half||(rem==half&&(q&1)); q+=(uint32_t)inc(rc,s!=0,rem!=0,n); uint32_t rs=q<ib; return s|(inc(rc,s!=0,1,n)?((127-scale)<<23):0); +} +static uint64_t round64(uint64_t u,unsigned scale,unsigned rc,int daz) { + const uint64_t em=UINT64_C(0x7ff0000000000000),fm=UINT64_C(0x000fffffffffffff),qb=UINT64_C(0x0008000000000000),sb=UINT64_C(0x8000000000000000),ib=UINT64_C(0x0010000000000000); + uint64_t e=(u&em)>>52,f=u&fm,s=u&sb; if(e==2047) return f?(u|qb):u; + if(e==0&&(f==0||daz)) return s; + unsigned already=1023+52-scale, regular=1023-scale; + if(e>=already) return u; + uint64_t sig=f|ib; + if(e>=regular) { unsigned drop=52+1023-scale-(unsigned)e; uint64_t q=sig>>drop, rem=sig&((UINT64_C(1)<half||(rem==half&&(q&1)); q+=(uint64_t)inc(rc,s!=0,rem!=0,n); uint64_t rs=q<ib; return s|(inc(rc,s!=0,1,n)?((uint64_t)(1023-scale)<<52):0); +} +/* For |src|>fraction); + uint64_t significand=magnitude&((UINT64_C(1)<>drop)+(remainder!=0); + } + uint64_t step=(uint64_t)(bias-scale)<>i)&1)?r:old32[i]))return 20;if(u32(out+1600+i*4)!=(((mask>>i)&1)?d:0))return 21;if(u32(out+1664+i*4)!=round32(s32[0],2,3,0))return 22;} + for(int i=0;i<8;i++){uint64_t r=round64(s64[i],2,3,0),d=reduce64(s64[i],2,3,0);if(u64(out+1728+i*8)!=(((mask>>i)&1)?r:old64[i]))return 23;if(u64(out+1792+i*8)!=(((mask>>i)&1)?d:0))return 24;if(u64(out+1856+i*8)!=round64(s64[0],2,3,0))return 25;} + if(u32(out+1920)!=round32(s32[0],0,0,0)||memcmp(out+1924,(const uint8_t*)upper+4,12))return 30;if(u64(out+1936)!=reduce64(s64[0],0,0,0)||memcmp(out+1944,(const uint8_t*)upper+8,8))return 31;if(out[1976]!=1)return 32; + for(unsigned rc=0;rc<4;rc++){if(u32(dynamic[rc]+1952)!=round32(s32[0],2,rc,0)||u32(dynamic[rc]+1956)!=reduce32(s32[2],2,rc,0)||u64(dynamic[rc]+1960)!=round64(s64[0],2,rc,0)||u64(dynamic[rc]+1968)!=reduce64(s64[2],2,rc,0))return 40+rc;} + return 0; +} +` + x86FloatingHelperRunNativePipelines(t, execution, "scaled_round_semantics", ir, mainC) +} diff --git a/amd64_lower_scalef.go b/amd64_lower_scalef.go new file mode 100644 index 000000000..c09790d95 --- /dev/null +++ b/amd64_lower_scalef.go @@ -0,0 +1,383 @@ +package plan9asm + +import "fmt" + +type amd64ScaleFControls struct { + daz string + ftz string + rounding string +} + +func (c *amd64Ctx) scaleFControls(rounding string) amd64ScaleFControls { + mxcsr := c.loadMXCSR() + dazBits, daz := c.newTmp(), c.newTmp() + ftzBits, ftz := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i32 %s, 64\n", dazBits, mxcsr) + fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", daz, dazBits) + fmt.Fprintf(c.b, " %%%s = and i32 %s, 32768\n", ftzBits, mxcsr) + fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", ftz, ftzBits) + mode := "" + switch rounding { + case "RN_SAE": + mode = "0" + case "RD_SAE": + mode = "1" + case "RU_SAE": + mode = "2" + case "RZ_SAE": + mode = "3" + default: + shifted, extracted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i32 %s, 13\n", shifted, mxcsr) + fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 3\n", extracted, shifted) + mode = "%" + extracted + } + return amd64ScaleFControls{daz: "%" + daz, ftz: "%" + ftz, rounding: mode} +} + +func (c *amd64Ctx) lowerPackedScaleF(spec amd64BinaryFloatingSpec, properties amd64BinaryFloatingSuffix, source2, source1, destination Operand, byteWidth int, mask string) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + scale, err := c.loadPackedCompareLanes(source2, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + value, err := c.loadPackedCompareLanes(source1, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + result := c.emitScaleFBits(lanes, spec.laneBits, value, scale, c.scaleFControls(properties.rounding)) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarScaleF(spec amd64BinaryFloatingSpec, properties amd64BinaryFloatingSuffix, source2, source1, destination Operand, mask string) (bool, bool, error) { + scaleBits, err := c.loadFloatingScalarBits(source2, spec.laneBits) + if err != nil { + return true, false, err + } + valueBits, err := c.loadFloatingScalarBits(source1, spec.laneBits) + if err != nil { + return true, false, err + } + scale, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", scale, spec.laneBits, spec.laneBits, scaleBits) + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", value, spec.laneBits, spec.laneBits, valueBits) + result := c.emitScaleFBits(1, spec.laneBits, "%"+value, "%"+scale, c.scaleFControls(properties.rounding)) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + passthroughBytes, err := c.loadX(source1.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, passthroughBytes) + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitScaleFBits(lanes, laneBits int, source1, source2 string, controls amd64ScaleFControls) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + mantissaBits, bias, exponentMax, scaleClamp := 23, 127, 255, 512 + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + indefinite := uint64(0xffc00000) + if laneBits == 64 { + mantissaBits, bias, exponentMax, scaleClamp = 52, 1023, 2047, 4096 + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + indefinite = uint64(0xfff8000000000000) + } + implicitBit := uint64(1) << mantissaBits + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + x, y := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", x, vectorType, source1, lane) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", y, vectorType, source2, lane) + laneResult := c.emitScaleFLane(laneBits, mantissaBits, bias, exponentMax, scaleClamp, + exponentMask, mantissaMask, quietBit, signBit, indefinite, implicitBit, + "%"+x, "%"+y, controls) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, result, laneBits, laneResult, lane) + result = "%" + inserted + } + return result +} + +type amd64ScaleFClass struct { + bits, sign, fraction, exponent string + zero, nan, snan, infinity string + positiveInfinity, negativeInfinity string +} + +func (c *amd64Ctx) classifyScaleFValue(laneBits, mantissaBits, exponentMax int, exponentMask, mantissaMask, quietBit, signBit uint64, bits, daz string) amd64ScaleFClass { + sign, fraction, exponentField := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %s, %d\n", sign, laneBits, bits, signBit) + fmt.Fprintf(c.b, " %%%s = and i%d %s, %d\n", fraction, laneBits, bits, mantissaMask) + fmt.Fprintf(c.b, " %%%s = and i%d %s, %d\n", exponentField, laneBits, bits, exponentMask) + exponentZero, fractionNonzero, denormal := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, exponentField) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", fractionNonzero, laneBits, fraction) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", denormal, exponentZero, fractionNonzero) + flush := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %s\n", flush, denormal, daz) + effective := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %s\n", effective, flush, laneBits, sign, laneBits, bits) + + effectiveFraction, shiftedExponent, effectiveExponent := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", effectiveFraction, laneBits, effective, mantissaMask) + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %d\n", shiftedExponent, laneBits, effective, mantissaBits) + if laneBits == 32 { + fmt.Fprintf(c.b, " %%%s = and i32 %%%s, %d\n", effectiveExponent, shiftedExponent, exponentMax) + } else { + exponentWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %d\n", exponentWide, shiftedExponent, exponentMax) + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", effectiveExponent, exponentWide) + } + abs, isZero, isExponentMax, isFractionNonzero := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", abs, laneBits, effective, signBit-1) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", isZero, laneBits, abs) + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %%%s, %d\n", isExponentMax, effectiveExponent, exponentMax) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", isFractionNonzero, laneBits, effectiveFraction) + isNaN, isInfinity := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, isExponentMax, isFractionNonzero) + fractionQuiet, quietClear, isSNaN := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", fractionQuiet, laneBits, effectiveFraction, quietBit) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", quietClear, laneBits, fractionQuiet) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isSNaN, isNaN, quietClear) + fractionZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", fractionZero, laneBits, effectiveFraction) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, isExponentMax, fractionZero) + isNegative, positiveInfinity, negativeInfinity := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", isNegative, laneBits, sign) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", negativeInfinity, isInfinity, isNegative) + notNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notNegative, isNegative) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", positiveInfinity, isInfinity, notNegative) + return amd64ScaleFClass{ + bits: "%" + effective, sign: "%" + sign, fraction: "%" + effectiveFraction, exponent: "%" + effectiveExponent, + zero: "%" + isZero, nan: "%" + isNaN, snan: "%" + isSNaN, infinity: "%" + isInfinity, + positiveInfinity: "%" + positiveInfinity, negativeInfinity: "%" + negativeInfinity, + } +} + +func (c *amd64Ctx) emitScaleFLane(laneBits, mantissaBits, bias, exponentMax, scaleClamp int, exponentMask, mantissaMask, quietBit, signBit, indefinite, implicitBit uint64, source1, source2 string, controls amd64ScaleFControls) string { + x := c.classifyScaleFValue(laneBits, mantissaBits, exponentMax, exponentMask, mantissaMask, quietBit, signBit, source1, controls.daz) + y := c.classifyScaleFValue(laneBits, mantissaBits, exponentMax, exponentMask, mantissaMask, quietBit, signBit, source2, controls.daz) + + yNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %s, 0\n", yNegative, laneBits, y.sign) + yExponentNegative, yExponentLarge := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i32 %s, %d\n", yExponentNegative, y.exponent, bias) + largeExponent := 9 + if laneBits == 64 { + largeExponent = 12 + } + fmt.Fprintf(c.b, " %%%s = icmp sge i32 %s, %d\n", yExponentLarge, y.exponent, bias+largeExponent) + yUnbiased, yShift := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %s, %d\n", yUnbiased, y.exponent, bias) + fmt.Fprintf(c.b, " %%%s = sub i32 %d, %%%s\n", yShift, mantissaBits, yUnbiased) + yExponentOutOfRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", yExponentOutOfRange, yExponentNegative, yExponentLarge) + safeYShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %d, i32 %%%s\n", safeYShift, yExponentOutOfRange, mantissaBits, yShift) + yShiftLane := "%" + safeYShift + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, safeYShift) + yShiftLane = "%" + wide + } + ySignificand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", ySignificand, laneBits, y.fraction, implicitBit) + yIntegerWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %s\n", yIntegerWide, laneBits, ySignificand, yShiftLane) + yInteger := "%" + yIntegerWide + if laneBits == 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", narrow, yIntegerWide) + yInteger = "%" + narrow + } + oneShifted, remainderMask, remainder, remainderNonzero := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d 1, %s\n", oneShifted, laneBits, yShiftLane) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, 1\n", remainderMask, laneBits, oneShifted) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %%%s\n", remainder, laneBits, ySignificand, remainderMask) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", remainderNonzero, laneBits, remainder) + negativeInteger, negativeFloor := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 0, %s\n", negativeInteger, yInteger) + remainderAdjust := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %%%s to i32\n", remainderAdjust, remainderNonzero) + fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, %%%s\n", negativeFloor, negativeInteger, remainderAdjust) + signedInteger := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %s\n", signedInteger, yNegative, negativeFloor, yInteger) + smallScale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 -1, i32 0\n", smallScale, yNegative) + clampedScale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 -%d, i32 %d\n", clampedScale, yNegative, scaleClamp, scaleClamp) + nonzeroScale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %%%s\n", nonzeroScale, yExponentLarge, clampedScale, signedInteger) + finiteScale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %%%s\n", finiteScale, yExponentNegative, smallScale, nonzeroScale) + scale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i32 0, i32 %%%s\n", scale, y.zero, finiteScale) + + xExponentZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 0\n", xExponentZero, x.exponent) + xSignificandNormal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", xSignificandNormal, laneBits, x.fraction, implicitBit) + xSignificand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %s, i%d %%%s\n", xSignificand, xExponentZero, laneBits, x.fraction, laneBits, xSignificandNormal) + leadingZeros := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i%d @llvm.ctlz.i%d(i%d %%%s, i1 false)\n", leadingZeros, laneBits, laneBits, laneBits, xSignificand) + leadingZeros32 := "%" + leadingZeros + if laneBits == 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", narrow, leadingZeros) + leadingZeros32 = "%" + narrow + } + normalizeShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %s, %d\n", normalizeShift, leadingZeros32, laneBits-1-mantissaBits) + normalizeShiftLane := "%" + normalizeShift + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, normalizeShift) + normalizeShiftLane = "%" + wide + } + normalizedSignificand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, %s\n", normalizedSignificand, laneBits, xSignificand, normalizeShiftLane) + xBaseExponentNormal, xBaseExponent := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %s, %d\n", xBaseExponentNormal, x.exponent, bias) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %d, i32 %%%s\n", xBaseExponent, xExponentZero, 1-bias, xBaseExponentNormal) + xNormalizedExponent := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, %%%s\n", xNormalizedExponent, xBaseExponent, normalizeShift) + targetExponent := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i32 %%%s, %%%s\n", targetExponent, xNormalizedExponent, scale) + + overflow, normal := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt i32 %%%s, %d\n", overflow, targetExponent, bias) + fmt.Fprintf(c.b, " %%%s = icmp sge i32 %%%s, %d\n", normal, targetExponent, 1-bias) + normalExponent32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i32 %%%s, %d\n", normalExponent32, targetExponent, bias) + normalExponent := "%" + normalExponent32 + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, normalExponent32) + normalExponent = "%" + wide + } + normalExponentBits, normalFraction, normalMagnitude, normalBits := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d %s, %d\n", normalExponentBits, laneBits, normalExponent, mantissaBits) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", normalFraction, laneBits, normalizedSignificand, mantissaMask) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", normalMagnitude, laneBits, normalExponentBits, normalFraction) + fmt.Fprintf(c.b, " %%%s = or i%d %s, %%%s\n", normalBits, laneBits, x.sign, normalMagnitude) + + subnormalShiftRaw, subnormalShiftHigh := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %d, %%%s\n", subnormalShiftRaw, 1-bias, targetExponent) + fmt.Fprintf(c.b, " %%%s = icmp sgt i32 %%%s, %d\n", subnormalShiftHigh, subnormalShiftRaw, mantissaBits+2) + subnormalShiftMax := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %d, i32 %%%s\n", subnormalShiftMax, subnormalShiftHigh, mantissaBits+2, subnormalShiftRaw) + subnormalShiftLow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i32 %%%s, 1\n", subnormalShiftLow, subnormalShiftMax) + subnormalShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 1, i32 %%%s\n", subnormalShift, subnormalShiftLow, subnormalShiftMax) + subnormalShiftLane := "%" + subnormalShift + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, subnormalShift) + subnormalShiftLane = "%" + wide + } + quotient, subOneShifted, subRemainderMask, subRemainder := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %s\n", quotient, laneBits, normalizedSignificand, subnormalShiftLane) + fmt.Fprintf(c.b, " %%%s = shl i%d 1, %s\n", subOneShifted, laneBits, subnormalShiftLane) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, 1\n", subRemainderMask, laneBits, subOneShifted) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %%%s\n", subRemainder, laneBits, normalizedSignificand, subRemainderMask) + subRemainderNonzero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", subRemainderNonzero, laneBits, subRemainder) + halfShift32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, 1\n", halfShift32, subnormalShift) + halfShift := "%" + halfShift32 + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, halfShift32) + halfShift = "%" + wide + } + half, aboveHalf, equalHalf, quotientOdd, tieOdd, nearest := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d 1, %s\n", half, laneBits, halfShift) + fmt.Fprintf(c.b, " %%%s = icmp ugt i%d %%%s, %%%s\n", aboveHalf, laneBits, subRemainder, half) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %%%s\n", equalHalf, laneBits, subRemainder, half) + quotientLowBit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, 1\n", quotientLowBit, laneBits, quotient) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", quotientOdd, laneBits, quotientLowBit) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", tieOdd, equalHalf, quotientOdd) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", nearest, aboveHalf, tieOdd) + xNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %s, 0\n", xNegative, laneBits, x.sign) + increment := c.emitScaledRoundIncrement(controls.rounding, "%"+xNegative, "%"+subRemainderNonzero, "%"+nearest) + incrementBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to i%d\n", incrementBits, increment, laneBits) + roundedSubnormal, subnormalBits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i%d %%%s, %%%s\n", roundedSubnormal, laneBits, quotient, incrementBits) + fmt.Fprintf(c.b, " %%%s = or i%d %s, %%%s\n", subnormalBits, laneBits, x.sign, roundedSubnormal) + isSubnormal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i%d %%%s, %d\n", isSubnormal, laneBits, roundedSubnormal, implicitBit) + flushInexact, flushSubnormal := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %s, %%%s\n", flushInexact, controls.ftz, subRemainderNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", flushSubnormal, flushInexact, isSubnormal) + underflowBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %s, i%d %%%s\n", underflowBits, flushSubnormal, laneBits, x.sign, laneBits, subnormalBits) + + toInfinity := c.emitScaledRoundIncrement(controls.rounding, "%"+xNegative, "true", "true") + overflowInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d %d\n", overflowInfinity, toInfinity, laneBits, exponentMask, laneBits, exponentMask-1) + overflowBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %%%s\n", overflowBits, laneBits, x.sign, overflowInfinity) + finiteNonOverflow, finiteBits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", finiteNonOverflow, normal, laneBits, normalBits, laneBits, underflowBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", finiteBits, overflow, laneBits, overflowBits, laneBits, finiteNonOverflow) + + yInfinityResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d %s\n", yInfinityResult, y.positiveInfinity, laneBits, exponentMask, laneBits, x.sign) + yInfinitySigned := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %%%s\n", yInfinitySigned, laneBits, x.sign, yInfinityResult) + general := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", general, y.infinity, laneBits, yInfinitySigned, laneBits, finiteBits) + xZeroInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d %s\n", xZeroInfinity, y.positiveInfinity, laneBits, indefinite, laneBits, x.bits) + xZeroResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %s\n", xZeroResult, y.infinity, laneBits, xZeroInfinity, laneBits, x.bits) + withZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", withZero, x.zero, laneBits, xZeroResult, laneBits, general) + xInfinityResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d %s\n", xInfinityResult, y.negativeInfinity, laneBits, indefinite, laneBits, x.bits) + withInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", withInfinity, x.infinity, laneBits, xInfinityResult, laneBits, withZero) + quietY := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", quietY, laneBits, y.bits, quietBit) + withYNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", withYNaN, y.nan, laneBits, quietY, laneBits, withInfinity) + quietX := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", quietX, laneBits, x.bits, quietBit) + xQNaNInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d 0\n", xQNaNInfinity, y.positiveInfinity, laneBits, exponentMask, laneBits) + xQNaNResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", xQNaNResult, y.infinity, laneBits, xQNaNInfinity, laneBits, quietX) + xQNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %s, true\n", xQNaN, x.nan) + notSNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", notSNaN, x.snan) + isQNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isQNaN, xQNaN, notSNaN) + withXQNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", withXQNaN, isQNaN, laneBits, xQNaNResult, laneBits, withYNaN) + final := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", final, x.snan, laneBits, quietX, laneBits, withXQNaN) + return "%" + final +} diff --git a/amd64_lower_scalef_test.go b/amd64_lower_scalef_test.go new file mode 100644 index 000000000..ebace5bb7 --- /dev/null +++ b/amd64_lower_scalef_test.go @@ -0,0 +1,272 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestAMD64ScaleFGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64BinaryFloatingSpec{ + "VSCALEFPS": {laneBits: 32, mode: amd64BinaryFloatingScale}, + "VSCALEFPD": {laneBits: 64, mode: amd64BinaryFloatingScale}, + "VSCALEFSS": {laneBits: 32, mode: amd64BinaryFloatingScale, scalar: true}, + "VSCALEFSD": {laneBits: 64, mode: amd64BinaryFloatingScale, scalar: true}, + } + for op, want := range expected { + if got, ok := amd64BinaryFloatingSpecs[op]; !ok { + t.Errorf("SCALEF grammar omitted %s", op) + } else if got != want { + t.Errorf("SCALEF grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86ScaleFCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT scalefforms(SB),$0-0\n") + for _, op := range []string{"VSCALEFPS", "VSCALEFPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s1, %s2, %s%d\n", op, width, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s2, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.BCST 16(AX), %s2, %s%d\n", op, width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s1, %s2, K1, %s%d\n", op, width, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 24(AX), %s2, K2, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(AX), %s2, K3, %s%d\n", op, width, width, last) + } + if width == "Z" { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s Z1, Z2, Z%d\n", op, rounding, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.%s.Z Z1, Z2, K4, Z%d\n", op, rounding, last) + } + } + } + } + } + for _, op := range []string{"VSCALEFSS", "VSCALEFSD"} { + fmt.Fprintf(&source, "\t%s X1, X2, X7\n", op) + fmt.Fprintf(&source, "\t%s 40(AX), X2, X7\n", op) + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s X1, X2, X7\n", op, rounding) + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X2, K5, X31\n", op) + fmt.Fprintf(&source, "\t%s.Z 48(AX), X2, K6, X31\n", op) + fmt.Fprintf(&source, "\t%s.RN_SAE.Z X1, X2, K7, X31\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86FloatingHelperOriginalFormatObject(t, source.String(), "scalefforms", target.goarch, target.triple) + if ir == "" { + return // Go/public-Context coverage, not an LLVM object claim. + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scalef-"+target.name+".ll", "scalef-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ScaleFRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VSCALEFPS X0, X1"}, + {goarch: "amd64", instruction: "VSCALEFPS X0, X1, X2, X3, X4"}, + {goarch: "amd64", instruction: "VSCALEFPS X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VSCALEFPS X0, 8(AX), X2"}, + {goarch: "amd64", instruction: "VSCALEFPS X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VSCALEFPS.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFPS.BCST X0, X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFPS.RN_SAE 8(AX), Z1, Z2"}, + {goarch: "amd64", instruction: "VSCALEFPS.SAE Z0, Z1, Z2"}, + {goarch: "amd64", instruction: "VSCALEFPS.RN_SAE Y0, Y1, Y2"}, + {goarch: "amd64", instruction: "VSCALEFSS Y0, X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFSS.BCST 8(AX), X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFSS.RN_SAE 8(AX), X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFSS X0, Y1, X2"}, + {goarch: "386", instruction: "VSCALEFPS X0, X1, K1, X2"}, + {goarch: "386", instruction: "VSCALEFPS Z8, Z1, Z2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's SCALEF tables", test.instruction) + } + }) + } +} + +func qualifyAMD64ScaleFRuntimeSemantics(t *testing.T) { + execution := x86FloatingHelperRuntimeTools(t) + const source = `TEXT scalefsemantics(SB),$0-72 + MOVQ out+0(FP), AX + MOVQ value32+8(FP), BX + MOVQ scale32+16(FP), CX + MOVQ value64+24(FP), DX + MOVQ scale64+32(FP), R8 + MOVQ old32+40(FP), R9 + MOVQ old64+48(FP), R10 + MOVQ upper+56(FP), R11 + MOVQ mask+64(FP), R12 + KMOVQ R12, K1 + VMOVDQU32 0(BX), Z1 + VMOVDQU32 0(CX), Z2 + VSCALEFPS Z2, Z1, Z3 + VMOVDQU32 Z3, 0(AX) + VSCALEFPS.RN_SAE Z2, Z1, Z3 + VMOVDQU32 Z3, 64(AX) + VMOVDQU32 0(R9), Z3 + VSCALEFPS Z2, Z1, K1, Z3 + VMOVDQU32 Z3, 128(AX) + VSCALEFPS.BCST 0(CX), Z1, Z3 + VMOVDQU32 Z3, 192(AX) + VMOVDQU64 0(DX), Z4 + VMOVDQU64 0(R8), Z5 + VSCALEFPD Z5, Z4, Z6 + VMOVDQU64 Z6, 256(AX) + VSCALEFPD.RN_SAE Z5, Z4, Z6 + VMOVDQU64 Z6, 320(AX) + VMOVDQU64 0(R10), Z6 + VSCALEFPD.Z Z5, Z4, K1, Z6 + VMOVDQU64 Z6, 384(AX) + VSCALEFPD.BCST 0(R8), Z4, Z6 + VMOVDQU64 Z6, 448(AX) + VMOVDQU64 0(R11), X7 + VSCALEFSS 60(CX), X7, X8 + VMOVDQU64 X8, 512(AX) + VSCALEFSD 56(R8), X7, X9 + VMOVDQU64 X9, 528(AX) + STC + SETCS 544(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := execution.triple + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"scalefsemantics": { + Name: "scalefsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: Ptr, Index: 7, Field: -1}, + {Offset: 64, Type: I64, Index: 8, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +#include +#include +extern void scalefsemantics(uint8_t *, const uint32_t *, const uint32_t *, const uint64_t *, const uint64_t *, const uint32_t *, const uint64_t *, const uint64_t *, uint64_t); +static uint32_t u32(const uint8_t *p) { uint32_t v; memcpy(&v,p,4); return v; } +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +static uint32_t scale32(uint32_t x, uint32_t y, int daz) { + const uint32_t em=UINT32_C(0x7f800000), fm=UINT32_C(0x007fffff), qb=UINT32_C(0x00400000), sb=UINT32_C(0x80000000), ind=UINT32_C(0xffc00000); + if(daz && !(x&em) && (x&fm)) x&=sb; if(daz && !(y&em) && (y&fm)) y&=sb; + uint32_t xf=x&fm,yf=y&fm; int xn=(x&em)==em&&xf, xs=xn&&!(xf&qb), xi=(x&~sb)==em, xz=(x&~sb)==0; + int yn=(y&em)==em&&yf, yi=(y&~sb)==em, yp=yi&&!(y&sb), ym=yi&&(y&sb); + if(xs) return x|qb; + if(xn) return yi?(yp?em:0):(x|qb); + if(yn) return y|qb; + if(xi) return ym?ind:x; + if(xz) return yp?ind:x; + if(yp) return (x&sb)|em; if(ym) return x&sb; + float a,b,r; memcpy(&a,&x,4); memcpy(&b,&y,4); int n=b>=512?512:(b<=-512?-512:(int)floorf(b)); r=scalbnf(a,n); memcpy(&x,&r,4); return x; +} +static uint64_t scale64(uint64_t x, uint64_t y, int daz) { + const uint64_t em=UINT64_C(0x7ff0000000000000), fm=UINT64_C(0x000fffffffffffff), qb=UINT64_C(0x0008000000000000), sb=UINT64_C(0x8000000000000000), ind=UINT64_C(0xfff8000000000000); + if(daz && !(x&em) && (x&fm)) x&=sb; if(daz && !(y&em) && (y&fm)) y&=sb; + uint64_t xf=x&fm,yf=y&fm; int xn=(x&em)==em&&xf, xs=xn&&!(xf&qb), xi=(x&~sb)==em, xz=(x&~sb)==0; + int yn=(y&em)==em&&yf, yi=(y&~sb)==em, yp=yi&&!(y&sb), ym=yi&&(y&sb); + if(xs) return x|qb; + if(xn) return yi?(yp?em:0):(x|qb); + if(yn) return y|qb; + if(xi) return ym?ind:x; + if(xz) return yp?ind:x; + if(yp) return (x&sb)|em; if(ym) return x&sb; + double a,b,r; memcpy(&a,&x,8); memcpy(&b,&y,8); int n=b>=4096?4096:(b<=-4096?-4096:(int)floor(b)); r=scalbn(a,n); memcpy(&x,&r,8); return x; +} +int main(void) { + const uint32_t v32[16]={0x3f800000,0xbf800000,0x7f7fffff,0xff7fffff,0x00800000,0x80800000,1,0x80000001,0,0x80000000,0x7f800000,0xff800000,0x7fc12345,0x7f812345,0x3fc00000,0xbfc00000}; + const uint32_t s32[16]={0x3ff33333,0xbf8ccccd,0x3f800000,0x3f800000,0xbf800000,0xbf800000,0x3f800000,0x3f800000,0x7f800000,0xff800000,0xff800000,0x7f800000,0x7f800000,0,0xc3160000,0xc3160000}; + const uint64_t v64[8]={UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff),UINT64_C(0x0010000000000000),1,UINT64_C(0x7ff8123456789abc),UINT64_C(0x7ff0123456789abc)}; + const uint64_t s64[8]={UINT64_C(0x3ffe666666666666),UINT64_C(0xbff199999999999a),UINT64_C(0x3ff0000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0x7ff0000000000000),0}; + uint32_t old32[16]; uint64_t old64[8]; + uint64_t upper[2]={UINT64_C(0x112233443fc00000),UINT64_C(0x99aabbccddeeff00)}; + for(int i=0;i<16;i++)old32[i]=UINT32_C(0x41000000)+(uint32_t)i; for(int i=0;i<8;i++)old64[i]=UINT64_C(0x4020000000000000)+(uint64_t)i; + const uint64_t mask=UINT64_C(0xa55a); unsigned original=_mm_getcsr(); + const int fes[4]={FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO}; + for(unsigned rc=0;rc<4;rc++) for(unsigned daz=0;daz<2;daz++) { + uint8_t out[545]={0}; unsigned csr=(original&~UINT32_C(0xe040))|(rc<<13)|(daz?UINT32_C(0x40):0); _mm_setcsr(csr); fesetround(fes[rc]); + scalefsemantics(out,v32,s32,v64,s64,old32,old64,upper,mask); + for(int i=0;i<16;i++) { + uint32_t want=scale32(v32[i],s32[i],daz); if(u32(out+i*4)!=want)return 10+rc; + int saved=fegetround(); fesetround(FE_TONEAREST); uint32_t rn=scale32(v32[i],s32[i],daz); fesetround(saved); if(u32(out+64+i*4)!=rn)return 20+rc; + if(u32(out+128+i*4)!=(((mask>>i)&1)?want:old32[i]))return 30+rc; + if(u32(out+192+i*4)!=scale32(v32[i],s32[0],daz))return 40+rc; + } + for(int i=0;i<8;i++) { + uint64_t want=scale64(v64[i],s64[i],daz); if(u64(out+256+i*8)!=want)return 50+rc; + int saved=fegetround(); fesetround(FE_TONEAREST); uint64_t rn=scale64(v64[i],s64[i],daz); fesetround(saved); if(u64(out+320+i*8)!=rn)return 60+rc; + if(u64(out+384+i*8)!=(((mask>>i)&1)?want:0))return 70+rc; + if(u64(out+448+i*8)!=scale64(v64[i],s64[0],daz))return 80+rc; + } + if(u32(out+512)!=scale32((uint32_t)upper[0],s32[15],daz)||memcmp(out+516,(uint8_t*)upper+4,12))return 90+rc; + if(u64(out+528)!=scale64(upper[0],s64[7],daz)||u64(out+536)!=upper[1])return 100+rc; + if(out[544]!=1)return 110+rc; + } + _mm_setcsr(original); fesetround(FE_TONEAREST); return 0; +} +` + x86FloatingHelperRunNativePipelines(t, execution, "scalef_semantics", ir, mainC) +} diff --git a/amd64_lower_scatter.go b/amd64_lower_scatter.go new file mode 100644 index 000000000..6a5727631 --- /dev/null +++ b/amd64_lower_scatter.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64ScatterSpecs = map[string]amd64GatherSpec{ + "VPSCATTERDD": {table: amd64GatherDPS, elemBits: 32, indexBits: 32}, + "VPSCATTERQQ": {table: amd64GatherDPS, elemBits: 64, indexBits: 64}, + "VSCATTERDPS": {table: amd64GatherDPS, elemBits: 32, indexBits: 32}, + "VSCATTERQPD": {table: amd64GatherDPS, elemBits: 64, indexBits: 64}, + "VPSCATTERDQ": {table: amd64GatherDPD, elemBits: 64, indexBits: 32}, + "VSCATTERDPD": {table: amd64GatherDPD, elemBits: 64, indexBits: 32}, + "VPSCATTERQD": {table: amd64GatherQPS, elemBits: 32, indexBits: 64}, + "VSCATTERQPS": {table: amd64GatherQPS, elemBits: 32, indexBits: 64}, +} + +// lowerScatter implements all Go 1.27 packed integer and floating scatter +// spellings. Values are kept as integer bit patterns, which is exact for the +// floating variants as well. +func (c *amd64Ctx) lowerScatter(op Op, ins Instr) (ok bool, terminated bool, err error) { + name, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64ScatterSpecs[name]; !known { + return false, false, nil + } + local := ins + local.Op = op + form, _, err := x86VSIBProperties(local, c.goarch) + if err != nil { + return true, false, err + } + return true, false, c.emitScatter(form.spec, form.memory.Mem, form.mask.Reg, form.data.Reg, form.dataBytes, form.indexBytes) +} + +func (c *amd64Ctx) emitScatter(spec amd64GatherSpec, memory MemRef, mask, data Reg, dataBytes, indexBytes int) error { + dataValue, err := c.loadGatherVector(data, dataBytes) + if err != nil { + return err + } + dataLanes := dataBytes * 8 / spec.elemBits + dataType := fmt.Sprintf("<%d x i%d>", dataLanes, spec.elemBits) + dataValues := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", dataValues, dataBytes, dataValue, dataType) + + indexValue, err := c.loadGatherVector(memory.Index, indexBytes) + if err != nil { + return err + } + indexLanes := indexBytes * 8 / spec.indexBits + indexType := fmt.Sprintf("<%d x i%d>", indexLanes, spec.indexBits) + indexValues := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", indexValues, indexBytes, indexValue, indexType) + activeLanes := dataLanes + if indexLanes < activeLanes { + activeLanes = indexLanes + } + + maskValue, err := c.loadK(mask) + if err != nil { + return err + } + base, err := c.gatherBaseAddress(memory) + if err != nil { + return err + } + for lane := 0; lane < activeLanes; lane++ { + indexLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", indexLane, indexType, indexValues, lane) + wideIndex := "%" + indexLane + if spec.indexBits == 32 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i32 %%%s to i64\n", extended, indexLane) + wideIndex = "%" + extended + } + if memory.Scale != 0 && memory.Scale != 1 { + scaled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i64 %s, %d\n", scaled, wideIndex, memory.Scale) + wideIndex = "%" + scaled + } + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %s\n", address, base, wideIndex) + pointer, pointerType := c.gatherPointer(memory, "%"+address) + + condition := amd64MaskBitI1(c, maskValue, lane) + dataLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", dataLane, dataType, dataValues, lane) + storeLabel := c.newTmp() + "_scatter_store" + skipLabel := c.newTmp() + "_scatter_skip" + joinLabel := c.newTmp() + "_scatter_join" + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n", condition, storeLabel, skipLabel) + fmt.Fprintf(c.b, "%s:\n", storeLabel) + fmt.Fprintf(c.b, " store i%d %%%s, %s %s, align 1\n", spec.elemBits, dataLane, pointerType, pointer) + fmt.Fprintf(c.b, " br label %%%s\n", joinLabel) + fmt.Fprintf(c.b, "%s:\n", skipLabel) + fmt.Fprintf(c.b, " br label %%%s\n", joinLabel) + fmt.Fprintf(c.b, "%s:\n", joinLabel) + } + // Like EVEX gather, scatter clears completed mask bits to support precise + // restart after a fault. A normally completed instruction leaves zero. + return c.storeK(mask, "0") +} diff --git a/amd64_lower_scatter_prefetch_test.go b/amd64_lower_scatter_prefetch_test.go new file mode 100644 index 000000000..252126a46 --- /dev/null +++ b/amd64_lower_scatter_prefetch_test.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "fmt" + "sort" + "strings" + "testing" +) + +func TestAMD64VSIBPrefetchGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64VSIBPrefetchSpecs)) + for op := range amd64VSIBPrefetchSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{ + "VGATHERPF0DPD", "VGATHERPF0DPS", "VGATHERPF0QPD", "VGATHERPF0QPS", + "VGATHERPF1DPD", "VGATHERPF1DPS", "VGATHERPF1QPD", "VGATHERPF1QPS", + "VSCATTERPF0DPD", "VSCATTERPF0DPS", "VSCATTERPF0QPD", "VSCATTERPF0QPS", + "VSCATTERPF1DPD", "VSCATTERPF1DPS", "VSCATTERPF1QPD", "VSCATTERPF1QPS", + } + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("VSIB-prefetch grammar opcodes = %v, want %v", got, want) + } + for op, spec := range amd64VSIBPrefetchSpecs { + name := string(op) + wantDirection := amd64VSIBPrefetchGather + if strings.HasPrefix(name, "VSCATTER") { + wantDirection = amd64VSIBPrefetchScatter + } + wantHint := 0 + if strings.Contains(name, "PF1") { + wantHint = 1 + } + shape := name[strings.Index(name, "PF")+3:] + wantIndexBits := 32 + if shape[0] == 'Q' { + wantIndexBits = 64 + } + wantElemBits := 32 + if shape[2] == 'D' { + wantElemBits = 64 + } + wantIndexBytes := 64 + if shape == "DPD" { + wantIndexBytes = 32 + } + if spec.direction != wantDirection || spec.hint != wantHint || spec.indexBits != wantIndexBits || + spec.elemBits != wantElemBits || spec.indexBytes != wantIndexBytes { + t.Errorf("VSIB-prefetch grammar %s = %+v, want direction=%d hint=%d elem=%d index=%d vector=%d", + op, spec, wantDirection, wantHint, wantElemBits, wantIndexBits, wantIndexBytes) + } + } +} + +func TestTranslateX86ScatterPrefetchCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT scatterprefetchforms(SB),$0-0\n") + for _, op := range []string{"VSCATTERPF0DPD", "VSCATTERPF1DPD"} { + fmt.Fprintf(&source, "\t%s K1, 8(SP)(Y2*8)\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s K7, 16(R11)(Y20*4)\n", op) + } + } + for _, op := range []string{ + "VSCATTERPF0DPS", "VSCATTERPF0QPD", "VSCATTERPF0QPS", + "VSCATTERPF1DPS", "VSCATTERPF1QPD", "VSCATTERPF1QPS", + } { + fmt.Fprintf(&source, "\t%s K2, 24(BX)(Z3*4)\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s K6, 32(SP)(Z20*8)\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"scatterprefetchforms": {Name: "scatterprefetchforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "scatter-prefetch-"+target.name+".ll", "scatter-prefetch-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ScatterPrefetchRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VSCATTERPF0DPD K0, (AX)(Y2*8)"}, + {goarch: "amd64", instruction: "VSCATTERPF1DPD K1, (AX)(Z2*8)"}, + {goarch: "amd64", instruction: "VSCATTERPF0DPS K1, (AX)(Y2*4)"}, + {goarch: "amd64", instruction: "VSCATTERPF0QPD K1, (AX)"}, + {goarch: "amd64", instruction: "VSCATTERPF0QPS (AX)(Z2*4), K1"}, + {goarch: "amd64", instruction: "VSCATTERPF1DPS.Z K1, (AX)(Z2*4)"}, + {goarch: "amd64", instruction: "VSCATTERPF1QPD K1, (AX)(Z2*4), Z3"}, + {goarch: "amd64", instruction: "VSCATTERPF1QPS X1, (AX)(Z2*4)"}, + {goarch: "386", instruction: "VSCATTERPF0DPD K1, (R8)(Y2*8)"}, + {goarch: "386", instruction: "VSCATTERPF1DPS K1, (AX)(Z8*4)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scatter-prefetch tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_scatter_test.go b/amd64_lower_scatter_test.go new file mode 100644 index 000000000..da358526c --- /dev/null +++ b/amd64_lower_scatter_test.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86ScatterCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 exposes eight integer/floating spellings over three exact + // data/index width tables. Every form is data, K1-K7, VSIB memory. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT scatterforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPSCATTERDD", "VPSCATTERQQ", "VSCATTERDPS", "VSCATTERQPD"} { + fmt.Fprintf(&source, "\t%s X20, K1, (AX)(X2*1)\n", op) + fmt.Fprintf(&source, "\t%s Y20, K2, 8(BX)(Y3*2)\n", op) + fmt.Fprintf(&source, "\t%s Z%d, K3, 16(CX)(Z4*4)\n", op, lastZ) + } + for _, op := range []string{"VPSCATTERDQ", "VSCATTERDPD"} { + fmt.Fprintf(&source, "\t%s X20, K4, 24(DX)(X5*8)\n", op) + fmt.Fprintf(&source, "\t%s Y20, K5, 32(SI)(X6*1)\n", op) + fmt.Fprintf(&source, "\t%s Z%d, K6, 40(DI)(Y7*2)\n", op, lastZ) + } + for _, op := range []string{"VPSCATTERQD", "VSCATTERQPS"} { + fmt.Fprintf(&source, "\t%s X20, K7, 48(AX)(X2*4)\n", op) + fmt.Fprintf(&source, "\t%s X20, K1, 56(BX)(Y3*8)\n", op) + fmt.Fprintf(&source, "\t%s Y20, K2, 64(CX)(Z4*1)\n", op) + } + source.WriteString("\tRET\n") + + // Retain every original format, including mode32 high data banks, + // without treating unknown data, masks and addresses as an entry. + // Initialized public objects belong to the independent official-row owner. + ll := x86UnboundVoidScalarFormatObject(t, source.String(), "scatterforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scatter-"+target.name+".ll", "scatter-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ScatterRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VPSCATTERDD X1, (AX)(X2*1)", + "VPSCATTERDD X1, K0, (AX)(X2*1)", + "VPSCATTERDD X1, K1, (AX)", + "VPSCATTERDD Y1, K1, (AX)(X2*1)", + "VPSCATTERDQ X1, K1, (AX)(Y2*1)", + "VPSCATTERQD Y1, K1, (AX)(Y2*1)", + "VPSCATTERDD X1, K1, (AX)(X2*3)", + "VPSCATTERDD.Z X1, K1, (AX)(X2*1)", + "VPSCATTERDD X1, K1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ScatterRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86ScatterRejected(t, "386", "i386-unknown-linux-gnu", "VPSCATTERDD X1, K1, (R8)(X2*1)") + assertX86ScatterRejected(t, "386", "i386-unknown-linux-gnu", "VPSCATTERDD X1, K1, (AX)(X8*1)") + assertX86ScatterRejected(t, "386", "i386-unknown-linux-gnu", "VPSCATTERDD Z8, K1, (AX)(Z7*1)") +} + +func assertX86ScatterRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scatter forms", instruction) + } +} diff --git a/amd64_lower_segment_base.go b/amd64_lower_segment_base.go new file mode 100644 index 000000000..a56e91f8a --- /dev/null +++ b/amd64_lower_segment_base.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SegmentBaseDirection uint8 + +const ( + amd64SegmentBaseRead amd64SegmentBaseDirection = iota + amd64SegmentBaseWrite +) + +type amd64SegmentBaseSpec struct { + segment Reg + direction amd64SegmentBaseDirection + bits int +} + +// amd64SegmentBaseSpecs is the complete Go 1.27 FSGSBASE grammar. Segment, +// transfer direction, and width are data; all eight opcodes share one Yrl +// register parser and one exact inline-assembly emitter. +var amd64SegmentBaseSpecs = map[Op]amd64SegmentBaseSpec{ + "RDFSBASEL": {segment: FS, direction: amd64SegmentBaseRead, bits: 32}, + "RDFSBASEQ": {segment: FS, direction: amd64SegmentBaseRead, bits: 64}, + "RDGSBASEL": {segment: GS, direction: amd64SegmentBaseRead, bits: 32}, + "RDGSBASEQ": {segment: GS, direction: amd64SegmentBaseRead, bits: 64}, + "WRFSBASEL": {segment: FS, direction: amd64SegmentBaseWrite, bits: 32}, + "WRFSBASEQ": {segment: FS, direction: amd64SegmentBaseWrite, bits: 64}, + "WRGSBASEL": {segment: GS, direction: amd64SegmentBaseWrite, bits: 32}, + "WRGSBASEQ": {segment: GS, direction: amd64SegmentBaseWrite, bits: 64}, +} + +type amd64SegmentBaseForm struct { + register Reg +} + +func (c *amd64Ctx) lowerSegmentBase(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64SegmentBaseSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.bits == 64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + form, err := c.parseSegmentBaseForm(baseOp, ins) + if err != nil { + return true, false, err + } + return c.emitSegmentBase(spec, form) +} + +func (c *amd64Ctx) parseSegmentBaseForm(op string, ins Instr) (amd64SegmentBaseForm, error) { + var form amd64SegmentBaseForm + if len(ins.Args) != 1 { + return form, fmt.Errorf("%s %s expects one Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + operand := ins.Args[0] + if operand.Kind != OpReg || !isX86YrlRegisterForArch(operand.Reg, c.goarch) { + return form, fmt.Errorf("%s %s expects a Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + return amd64SegmentBaseForm{register: operand.Reg}, nil +} + +func (spec amd64SegmentBaseSpec) mnemonic() string { + prefix := "rd" + if spec.direction == amd64SegmentBaseWrite { + prefix = "wr" + } + width := "l" + if spec.bits == 64 { + width = "q" + } + return prefix + strings.ToLower(string(spec.segment)) + "base" + width +} + +func (c *amd64Ctx) emitSegmentBase(spec amd64SegmentBaseSpec, form amd64SegmentBaseForm) (bool, bool, error) { + typ := amd64IntegerTypeForBits(spec.bits) + constraints := "~{dirflag},~{fpsr},~{flags}" + assembly := spec.mnemonic() + " $0" + registerConstraint := "r" + if c.goarch == "386" { + // Intel defines FSGSBASE only in 64-bit mode, but Go 1.27's 386 + // assembler accepts the L spellings and emits their F3 0F AE /r + // encodings. LLVM intentionally rejects the mnemonic for i386. Emit + // Go's exact EAX encoding and use a fixed-register constraint so the + // object remains byte-compatible without claiming runtime viability. + modRM := byte(0xc0) + if spec.segment == GS { + modRM += 8 + } + if spec.direction == amd64SegmentBaseWrite { + modRM += 16 + } + assembly = fmt.Sprintf(".byte 0xf3, 0x0f, 0xae, 0x%02x", modRM) + registerConstraint = "{ax}" + } + if spec.direction == amd64SegmentBaseRead { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", + result, typ, assembly, "="+registerConstraint+","+constraints) + if err := c.storeRegSized(form.register, typ, "%"+result); err != nil { + return true, false, err + } + return true, false, nil + } + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: form.register}, typ) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s)\n", + assembly, registerConstraint+","+constraints, typ, value) + return true, false, nil +} diff --git a/amd64_lower_segment_base_test.go b/amd64_lower_segment_base_test.go new file mode 100644 index 000000000..f17f24329 --- /dev/null +++ b/amd64_lower_segment_base_test.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "reflect" + "sort" + "strings" + "testing" +) + +func TestAMD64SegmentBaseGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64SegmentBaseSpecs)) + for op := range amd64SegmentBaseSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{ + "RDFSBASEL", "RDFSBASEQ", "RDGSBASEL", "RDGSBASEQ", + "WRFSBASEL", "WRFSBASEQ", "WRGSBASEL", "WRGSBASEQ", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("segment-base grammar opcodes = %v, want %v", got, want) + } + for op, want := range map[Op]amd64SegmentBaseSpec{ + "RDFSBASEL": {segment: FS, direction: amd64SegmentBaseRead, bits: 32}, + "RDFSBASEQ": {segment: FS, direction: amd64SegmentBaseRead, bits: 64}, + "RDGSBASEL": {segment: GS, direction: amd64SegmentBaseRead, bits: 32}, + "RDGSBASEQ": {segment: GS, direction: amd64SegmentBaseRead, bits: 64}, + "WRFSBASEL": {segment: FS, direction: amd64SegmentBaseWrite, bits: 32}, + "WRFSBASEQ": {segment: FS, direction: amd64SegmentBaseWrite, bits: 64}, + "WRGSBASEL": {segment: GS, direction: amd64SegmentBaseWrite, bits: 32}, + "WRGSBASEQ": {segment: GS, direction: amd64SegmentBaseWrite, bits: 64}, + } { + if got := amd64SegmentBaseSpecs[op]; got != want { + t.Errorf("segment-base grammar %s = %+v, want %+v", op, got, want) + } + if got := want.mnemonic(); strings.ToUpper(got) != string(op) { + t.Errorf("segment-base grammar %s emits %q", op, got) + } + } +} + +func TestTranslateX86SegmentBaseCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT segmentbaseforms(SB),$0-0 + RDFSBASEL AX + RDGSBASEL SP + WRFSBASEL CX + WRGSBASEL SP +` + if target.goarch == "amd64" { + source += ` RDFSBASEQ R11 + RDGSBASEQ R12 + WRFSBASEQ R13 + WRGSBASEQ R14 +` + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"segmentbaseforms": {Name: "segmentbaseforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if want := `"target-features"="+fsgsbase"`; !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + if target.goarch == "386" { + for _, encoding := range []string{ + ".byte 0xf3, 0x0f, 0xae, 0xc0", + ".byte 0xf3, 0x0f, 0xae, 0xc8", + ".byte 0xf3, 0x0f, 0xae, 0xd0", + ".byte 0xf3, 0x0f, 0xae, 0xd8", + } { + if !strings.Contains(ir, encoding) { + t.Errorf("386 IR is missing Go-compatible encoding %q:\n%s", encoding, ir) + } + } + } + compileLLVMToObject(t, llc, target.triple, "segment-base-"+target.name+".ll", "segment-base-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SegmentBaseRejectsFormsOutsideGoTables(t *testing.T) { + tests := []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "RDFSBASEL $1"}, + {goarch: "amd64", instruction: "RDFSBASEQ (BX)"}, + {goarch: "amd64", instruction: "RDGSBASEL X0"}, + {goarch: "amd64", instruction: "RDGSBASEQ"}, + {goarch: "amd64", instruction: "WRFSBASEL $1"}, + {goarch: "amd64", instruction: "WRFSBASEQ (BX)"}, + {goarch: "amd64", instruction: "WRGSBASEL X0"}, + {goarch: "amd64", instruction: "WRGSBASEQ.Z AX"}, + } + for _, op := range []string{ + "RDFSBASEQ", "RDGSBASEQ", "WRFSBASEQ", "WRGSBASEQ", + } { + tests = append(tests, struct { + goarch string + instruction string + }{goarch: "386", instruction: op + " AX"}) + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's segment-base tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_segment_query.go b/amd64_lower_segment_query.go new file mode 100644 index 000000000..1c9d87cd8 --- /dev/null +++ b/amd64_lower_segment_query.go @@ -0,0 +1,123 @@ +package plan9asm + +import "fmt" + +type amd64SegmentQueryKind uint8 + +const ( + amd64SegmentQueryAccessRights amd64SegmentQueryKind = iota + amd64SegmentQueryLimit + amd64SegmentQueryReadable + amd64SegmentQueryWritable +) + +type amd64SegmentQuerySpec struct { + kind amd64SegmentQueryKind + bits int +} + +// amd64SegmentQuerySpecs is the complete Go 1.27 LAR/LSL/VERR/VERW grammar. +// The table records only query kind and optional destination width; all four +// families share one selector-source parser and one exact lowering mechanism. +var amd64SegmentQuerySpecs = map[Op]amd64SegmentQuerySpec{ + "LARW": {kind: amd64SegmentQueryAccessRights, bits: 16}, + "LARL": {kind: amd64SegmentQueryAccessRights, bits: 32}, + "LARQ": {kind: amd64SegmentQueryAccessRights, bits: 64}, + "LSLW": {kind: amd64SegmentQueryLimit, bits: 16}, + "LSLL": {kind: amd64SegmentQueryLimit, bits: 32}, + "LSLQ": {kind: amd64SegmentQueryLimit, bits: 64}, + "VERR": {kind: amd64SegmentQueryReadable}, + "VERW": {kind: amd64SegmentQueryWritable}, +} + +type amd64SegmentQueryForm struct { + spec amd64SegmentQuerySpec + selector Operand + destination Reg +} + +func (c *amd64Ctx) lowerSegmentQuery(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86SegmentQueryProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return c.emitSegmentQuery(ins, form) +} + +func (c *amd64Ctx) emitSegmentQuery(ins Instr, form amd64SegmentQueryForm) (bool, bool, error) { + spec := form.spec + selector, err := c.evalSegmentQuerySelector(ins, form.selector) + if err != nil { + return true, false, err + } + if spec.bits == 0 { + mnemonic := "verr" + if spec.kind == amd64SegmentQueryWritable { + mnemonic = "verw" + } + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i8 asm sideeffect %q, %q(i16 %s)\n", + status, mnemonic+" $1; sete $0", "=q,r,~{dirflag},~{fpsr},~{flags}", selector) + c.storeSegmentQueryZF("%" + status) + return true, false, nil + } + + // A query can fail without writing its destination. Retain the whole + // native GP carrier in the tied operand, rather than unconditionally + // zero-extending an i32 result after a failed LARL/LSLL. The actual opcode + // determines the physical upper-half behavior on this processor. + typ, destination := I64, "$0" + if c.goarch == "386" { + typ = I32 + } + if spec.bits == 16 { + destination = "${0:w}" + } else if spec.bits == 32 { + destination = "${0:k}" + } + previousDestination, err := c.evalIntSized(Operand{Kind: OpReg, Reg: form.destination}, typ) + if err != nil { + return true, false, err + } + + // Intel specifies that a failed descriptor check clears ZF but leaves the + // destination unchanged. Constraint 0 therefore ties the old destination + // value to the result instead of declaring a write-only output. SETE does + // not alter flags, so the second result captures the instruction's ZF while + // the translator's other virtual flag slots remain unchanged. + result := c.newTmp() + mnemonic := "lar" + if spec.kind == amd64SegmentQueryLimit { + mnemonic = "lsl" + } + width := map[int]string{16: "w", 32: "l", 64: "q"}[spec.bits] + assembly := mnemonic + width + " $3, " + destination + "; sete $1" + fmt.Fprintf(c.b, " %%%s = call { %s, i8 } asm sideeffect %q, %q(%s %s, i16 %s)\n", + result, typ, assembly, "=r,=q,0,r,~{dirflag},~{fpsr},~{flags}", typ, previousDestination, selector) + value := c.newTmp() + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i8 } %%%s, 0\n", value, typ, result) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i8 } %%%s, 1\n", status, typ, result) + c.storeSegmentQueryZF("%" + status) + if err := c.storeRegSized(form.destination, typ, "%"+value); err != nil { + return true, false, err + } + return true, false, nil +} + +func (c *amd64Ctx) evalSegmentQuerySelector(ins Instr, selector Operand) (string, error) { + if selector.Kind == OpFP && x86ResultNamedPredicateInstruction(ins) { + if value, handled, err := c.loadIntegerFPBytesMode(selector.FPOffset, I16, true); handled || err != nil { + return value, err + } + } + // Share the exact unaligned volatile scalar emitter. This is a two-byte + // read even for Q spellings and dead results; it creates no backing fact. + return c.evalSizedNativeGPSource(selector, I16, ins.x86AddressBits) +} + +func (c *amd64Ctx) storeSegmentQueryZF(status string) { + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %s, 0\n", zero, status) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) +} diff --git a/amd64_lower_segment_query_test.go b/amd64_lower_segment_query_test.go new file mode 100644 index 000000000..7d23e8780 --- /dev/null +++ b/amd64_lower_segment_query_test.go @@ -0,0 +1,289 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +func TestAMD64SegmentQueryGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64SegmentQuerySpecs)) + for op := range amd64SegmentQuerySpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{"LARL", "LARQ", "LARW", "LSLL", "LSLQ", "LSLW", "VERR", "VERW"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("segment-query grammar opcodes = %v, want %v", got, want) + } + for op, want := range map[Op]amd64SegmentQuerySpec{ + "LARW": {kind: amd64SegmentQueryAccessRights, bits: 16}, + "LARL": {kind: amd64SegmentQueryAccessRights, bits: 32}, + "LARQ": {kind: amd64SegmentQueryAccessRights, bits: 64}, + "LSLW": {kind: amd64SegmentQueryLimit, bits: 16}, + "LSLL": {kind: amd64SegmentQueryLimit, bits: 32}, + "LSLQ": {kind: amd64SegmentQueryLimit, bits: 64}, + "VERR": {kind: amd64SegmentQueryReadable}, + "VERW": {kind: amd64SegmentQueryWritable}, + } { + if got := amd64SegmentQuerySpecs[op]; got != want { + t.Errorf("segment-query grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86SegmentQueryCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT segmentqueryforms(SB),$0-0 + LARW AX, DX + LARW 2(BX), CX + LARL CX, AX + LARL 4(BX), DX + LSLW DX, CX + LSLW 6(BX), AX + LSLL AX, DX + LSLL 8(BX), CX +` + if target.goarch == "386" { + source += "\tLARW AX, SP\n\tLARL AX, SP\n\tLSLW AX, SP\n\tLSLL AX, SP\n" + } + if target.goarch == "amd64" { + source += ` LARW R11, R12 + LARW 10(R13), R14 + LARQ R11, R12 + LARQ 12(R13), R14 + LSLW R11, R12 + LSLW 14(R13), R14 + LSLQ R11, R12 + LSLQ 16(R13), R14 +` + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, signatures := x86UnboundVoidFormatSource(t, source, "segmentqueryforms", target.goarch, target.triple) + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0, + x86SegmentQueryOriginalFormatRow) + compileLLVMToObject(t, llc, target.triple, "segment-query-"+target.name+".ll", "segment-query-"+target.name+".o", ir) + }) + } +} + +// The original Go Yrl format includes SP in mode32. This private object-only +// carrier observes that physical destination through a virtual slot; the +// unchanged original source still has Go objects and public Context checks. +// It supplies no public stack, initialized input or native-entry contract. +func x86SegmentQueryOriginalFormatRow(c *amd64Ctx, ins Instr) (bool, bool, error) { + form, known, err := x86SegmentQueryProperties(ins.Op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + previous := c.allowSPWrite + c.allowSPWrite = c.goarch == "386" && form.destination == SP + defer func() { c.allowSPWrite = previous }() + return c.lowerSegmentQuery(ins.Op, ins) +} + +func TestTranslateX86SegmentQueryRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LARW $1, AX"}, + {goarch: "amd64", instruction: "LARL AX, (BX)"}, + {goarch: "amd64", instruction: "LSLQ X0, AX"}, + {goarch: "amd64", instruction: "LSLW AX, X0"}, + {goarch: "amd64", instruction: "LARW.Z AX, DX"}, + {goarch: "386", instruction: "LARQ AX, DX"}, + {goarch: "386", instruction: "LSLQ AX, DX"}, + {goarch: "386", instruction: "LARW R11, AX"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's segment-query tables", test.instruction) + } + }) + } +} + +func TestTranslateX86SegmentValidationCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + high := "DI" + if target.goarch == "amd64" { + high = "R11" + } + source := "TEXT segmentvalidationforms(SB),$0-0\n" + + "\tVERR AX\n\tVERR (BX)\n\tVERR " + high + "\n" + + "\tVERW DX\n\tVERW 2(BX)\n\tVERW " + high + "\n\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, signatures := x86UnboundVoidFormatSource(t, source, "segmentvalidationforms", target.goarch, target.triple) + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerSegmentQuery(ins.Op, ins) }) + compileLLVMToObject(t, llc, target.triple, "segment-validation-"+target.name+".ll", "segment-validation-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SegmentValidationRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VERR $1"}, + {goarch: "amd64", instruction: "VERW X0"}, + {goarch: "amd64", instruction: "VERR AX, DX"}, + {goarch: "amd64", instruction: "VERW.Z AX"}, + {goarch: "386", instruction: "VERR R11"}, + {goarch: "386", instruction: "VERW (R11)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's segment-validation table", test.instruction) + } + }) + } +} + +func TestAMD64SegmentQueryRuntimeSemantics(t *testing.T) { + if runtime.GOARCH != "amd64" { + // Rosetta advertises amd64 execution but raises SIGILL for LAR/LSL. + // Object compilation remains covered above; execute the architectural + // descriptor checks only on native amd64 hardware. + t.Skip("LAR/LSL runtime semantics require native amd64 hardware") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT segmentquerysemantics(SB),NOSPLIT,$0-16 + MOVQ selector+0(FP), AX + MOVQ out+8(FP), DI + MOVQ $0x1122334455667788, CX + LARQ AX, CX + MOVQ CX, 0(DI) + SETEQ 8(DI) + MOVQ $0x2233445566778899, CX + XORQ AX, AX + LARQ AX, CX + MOVQ CX, 16(DI) + SETEQ 24(DI) + MOVQ selector+0(FP), AX + MOVQ $0x33445566778899aa, DX + LSLQ AX, DX + MOVQ DX, 32(DI) + SETEQ 40(DI) + MOVQ $0x445566778899aabb, DX + XORQ AX, AX + LSLQ AX, DX + MOVQ DX, 48(DI) + SETEQ 56(DI) + MOVQ selector+0(FP), AX + STC + VERR AX + SETEQ 57(DI) + SETCS 58(DI) + VERW AX + SETEQ 59(DI) + XORQ AX, AX + VERR AX + SETEQ 60(DI) + VERW AX + SETEQ 61(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"segmentquerysemantics": { + Name: "segmentquerysemantics", Args: []LLVMType{I64, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void segmentquerysemantics(uint64_t, uint8_t *); +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v; } +int main(void) { + uint16_t cs; + uint8_t out[64] = {0}; + __asm__ volatile ("mov %%cs,%0" : "=r"(cs)); + segmentquerysemantics(cs, out); + if (out[8] != 1 || out[40] != 1) return 10; + if (out[24] != 0 || out[56] != 0) return 11; + if (load64(out + 16) != UINT64_C(0x2233445566778899)) return 12; + if (load64(out + 48) != UINT64_C(0x445566778899aabb)) return 13; + if (out[57] != 1 || out[58] != 1) return 14; + if (out[59] != 0 || out[60] != 0 || out[61] != 0) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "segment_query_semantics", triple, ir, mainC, nil) +} diff --git a/amd64_lower_segment_register.go b/amd64_lower_segment_register.go new file mode 100644 index 000000000..99c3db24a --- /dev/null +++ b/amd64_lower_segment_register.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerSegmentRegisterMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if (baseOp != "MOVW" && baseOp != "MOVL" && baseOp != "MOVQ") || len(ins.Args) != 2 { + return false, false, nil + } + sourceSegment := x86SegmentRegisterOperand(ins.Args[0]) + destinationSegment := x86SegmentRegisterOperand(ins.Args[1]) + if !sourceSegment && !destinationSegment { + return false, false, nil + } + if baseOp != "MOVW" { + return true, false, fmt.Errorf("%s segment-register moves use MOVW in Go 1.27's ymovtab: %q", c.goarch, ins.Raw) + } + if !x86SegmentMoveSuffixAccepted(rawOp) { + return true, false, fmt.Errorf("%s segment-register MOVW has an invalid x86 suffix: %q", c.goarch, ins.Raw) + } + if sourceSegment == destinationSegment { + return true, false, fmt.Errorf("%s MOVW segment-register form expects exactly one segment register: %q", c.goarch, ins.Raw) + } + if destinationSegment { + source := ins.Args[0] + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, c.goarch) { + return true, false, fmt.Errorf("%s MOVW to a segment register expects a Yml source: %q", c.goarch, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s MOVW to a segment register expects a Yml source: %q", c.goarch, ins.Raw) + } + value, err := c.evalIntSized(source, I16) + if err != nil { + return true, false, err + } + return true, false, c.storeX86SegmentSelector(ins.Args[1].Reg, value) + } + + destination := ins.Args[1] + if destination.Kind == OpReg { + if !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s MOVW from a segment register expects a Yml destination: %q", c.goarch, ins.Raw) + } + } else if !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("%s MOVW from a segment register expects a Yml destination: %q", c.goarch, ins.Raw) + } + call := c.loadX86SegmentSelector(ins.Args[0].Reg) + if destination.Kind == OpReg { + return true, false, c.storeRegSized(destination.Reg, I32, call) + } + selector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i32 %s to i16\n", selector, call) + return true, false, c.storeScalarIntegerOperand(destination, I16, "%"+selector) +} + +func (c *amd64Ctx) loadX86SegmentSelector(segment Reg) string { + call := c.newTmp() + assembly := fmt.Sprintf("movl %%%s, ${0:k}", strings.ToLower(string(segment))) + fmt.Fprintf(c.b, " %%%s = call i32 asm sideeffect %q, %q()\n", + call, assembly, "=r,~{memory},~{dirflag},~{fpsr},~{flags}") + return "%" + call +} + +func (c *amd64Ctx) storeX86SegmentSelector(segment Reg, selector string) error { + assembly := fmt.Sprintf("movw $0, %%%s", strings.ToLower(string(segment))) + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(i16 %s)\n", + assembly, "r,~{memory},~{dirflag},~{fpsr},~{flags}", selector) + return nil +} + +func x86SegmentMoveSuffixAccepted(rawOp string) bool { + if rawOp == "MOVW" { + return true + } + suffix := strings.TrimPrefix(rawOp, "MOVW.") + switch suffix { + case "Z", "SAE", "SAE.Z", + "RN_SAE", "RZ_SAE", "RD_SAE", "RU_SAE", + "RN_SAE.Z", "RZ_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", + "BCST", "BCST.Z": + // Go 1.27's ymovtab path accepts every syntactically valid x86 suffix + // here and emits the same 8C/8E bytes. Preserve that compatibility. + return true + default: + return false + } +} + +func x86SegmentRegisterOperand(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + switch operand.Reg { + case ES, CS, SS, DS, FS, GS: + return true + default: + return false + } +} diff --git a/amd64_lower_segment_register_test.go b/amd64_lower_segment_register_test.go new file mode 100644 index 000000000..bc933755b --- /dev/null +++ b/amd64_lower_segment_register_test.go @@ -0,0 +1,112 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86SegmentRegisterCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT segmentregisterforms(SB),$0-0\n") + for _, segment := range []string{"ES", "CS", "SS", "DS", "FS", "GS"} { + fmt.Fprintf(&source, "\tMOVW AX, %s\n", segment) + fmt.Fprintf(&source, "\tMOVW %s, AX\n", segment) + fmt.Fprintf(&source, "\tMOVW 8(BX), %s\n", segment) + fmt.Fprintf(&source, "\tMOVW %s, 16(BX)\n", segment) + } + for _, suffix := range []string{ + "Z", "SAE", "SAE.Z", + "RN_SAE", "RZ_SAE", "RD_SAE", "RU_SAE", + "RN_SAE.Z", "RZ_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", + "BCST", "BCST.Z", + } { + fmt.Fprintf(&source, "\tMOVW.%s AX, DS\n", suffix) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "segmentregisterforms": {Name: "segmentregisterforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "segment-register-"+target.name+".ll", "segment-register-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86SegmentRegisterRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "MOVL AX, DS", + "MOVQ AX, DS", + "MOVW AL, DS", + "MOVW X0, DS", + "MOVW DS, X0", + "MOVW DS, FS", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + requireX86GoAssemblerResult(t, goarch, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + triple := "i386-unknown-linux-gnu" + if goarch == "amd64" { + triple = "x86_64-unknown-linux-gnu" + } + assertX86SegmentRegisterRejected(t, goarch, triple, instruction) + } + }) + } +} + +func assertX86SegmentRegisterRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's segment-register MOVW table for %s", instruction, goarch) + } +} + +func TestX86SegmentRegistersDoNotShadowARM64Conditions(t *testing.T) { + file, err := Parse(ArchARM64, "TEXT select(SB),$0-0\n\tCSEL CS, R11, R3, R3\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if len(file.Funcs) != 1 || len(file.Funcs[0].Instrs) < 2 || len(file.Funcs[0].Instrs[1].Args) != 4 { + t.Fatalf("unexpected ARM64 parse: %+v", file) + } + condition := file.Funcs[0].Instrs[1].Args[0] + if condition.Kind == OpReg { + t.Fatalf("ARM64 CS condition was parsed as an x86 segment register: %+v", condition) + } +} diff --git a/amd64_lower_set.go b/amd64_lower_set.go new file mode 100644 index 000000000..5de63df9d --- /dev/null +++ b/amd64_lower_set.go @@ -0,0 +1,53 @@ +package plan9asm + +import ( + "fmt" +) + +// lowerConditionalSet implements the complete Go 1.27 SETcc family. All 16 +// names use yscond, whose single operand is the byte-register-or-memory Ymb +// class. +func (c *amd64Ctx) lowerConditionalSet(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, recognized, err := x86ConditionalSetProperties(op, ins, c.goarch) + if !recognized { + return false, false, nil + } + if err != nil { + return true, false, err + } + condition, err := c.x86Condition(form.condition) + if err != nil { + return true, false, err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i8 1, i8 0\n", value, condition) + if err := c.storeConditionalSetByte(form.destination, "%"+value); err != nil { + return true, false, err + } + return true, false, nil +} + +func (c *amd64Ctx) storeConditionalSetByte(dst Operand, value string) error { + switch dst.Kind { + case OpReg: + return c.storeRegSized(dst.Reg, I8, value) + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store volatile i8 %s, %s %s, align 1\n", value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store volatile i8 %s, ptr %s, align 1\n", value, ptr) + return nil + case OpFP: + return c.storeFPResult(dst.FPOffset, I8, value) + default: + return fmt.Errorf("expected Ymb destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_sha.go b/amd64_lower_sha.go new file mode 100644 index 000000000..f723bbd05 --- /dev/null +++ b/amd64_lower_sha.go @@ -0,0 +1,97 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerSHA(op Op, ins Instr) (bool, bool, error) { + f, known, err := x86SHAProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if c.fixedLocal == nil && len(c.fpResults) != 0 { + if err := c.scalarFPResultInitialized(); err != nil { + return true, false, err + } + } + state, err := c.loadPackedCompareLanes(ins.Args[f.destination], 16, 32, false) + if err != nil { + return true, false, err + } + args := "<4 x i32> " + state + implicit := "" + if f.encoding == 0x38cb { + implicit, err = c.loadPackedCompareLanes(ins.Args[0], 16, 32, false) + if err != nil { + return true, false, err + } + } + call, result := c.newTmp(), c.newTmp() + name := strings.ToLower(string(f.op)) + if f.memory >= 0 { + source := f.sourceOperand + if source.Kind == OpFP || !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) || + source.Kind == OpMem && !c.nativeMemoryOperand(ins, f.source) && + !x86CallerMemoryCertified(ins, f.source) && !c.fixedLocalMemoryCertified(ins, f.source) { + return true, false, fmt.Errorf("%w: SHA m128 needs original native storage and x86 target", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return true, false, err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + if f.encoding == 0x3acc { + name += fmt.Sprintf(" $$%d,", f.immediate) + } + assembly, constraints := name+" "+segment+"($2), $0", "=x,0,r" + args += fmt.Sprintf(", %s %s", pointerType, pointer) + if implicit != "" { + constraints += ",{xmm0}" + args += ", <4 x i32> " + implicit + } + // The actual Type 4 opcode retains m128 extent, alignment and faults, + // even when numerical dependencies or later dead code use fewer bytes. + fmt.Fprintf(c.b, " %%%s = call <4 x i32> asm sideeffect %q, %q(%s)\n", call, assembly, constraints+",~{memory}", args) + } else { + source, err := c.loadPackedCompareLanes(f.sourceOperand, 16, 32, false) + if err != nil { + return true, false, err + } + args += ", <4 x i32> " + source + if implicit != "" { + args += ", <4 x i32> " + implicit + } else if f.encoding == 0x3acc { + args += fmt.Sprintf(", i8 %d", f.immediate) + } + fmt.Fprintf(c.b, " %%%s = call <4 x i32> @llvm.x86.%s(%s)\n", call, name, args) + } + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", result, call) + return true, false, c.storeLegacyPackedLogical(ins.Args[f.destination], "%"+result) +} + +func emitX86SHADeclarations(b *strings.Builder, file *File) { + seen := make(map[Op]bool) + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + op := Op(strings.ToUpper(string(ins.Op))) + encoding, known := x86SHAEncodings[op] + if !known || seen[op] { + continue + } + seen[op] = true + args := "<4 x i32>, <4 x i32>" + if encoding == 0x38cb { + args += ", <4 x i32>" + } else if encoding == 0x3acc { + args += ", i8 immarg" + } + fmt.Fprintf(b, "declare <4 x i32> @llvm.x86.%s(%s)\n", strings.ToLower(string(op)), args) + } + } +} diff --git a/amd64_lower_shiftrotate.go b/amd64_lower_shiftrotate.go new file mode 100644 index 000000000..a997d325c --- /dev/null +++ b/amd64_lower_shiftrotate.go @@ -0,0 +1,315 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerScalarShiftRotate implements the complete Go 1.27 yshb/yshl family: +// RCL/RCR/ROL/ROR/SAR/SAL/SHL/SHR at B/W/L/Q widths, plus the special +// three-operand SHL/SHR W/L/Q double-shift table. +func (c *amd64Ctx) lowerScalarShiftRotate(op Op, ins Instr) (ok bool, terminated bool, err error) { + baseOp, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + _, _, recognized := amd64ScalarShiftRotateProperties(baseOp) + if !recognized { + return false, false, nil + } + form, _, err := x86ScalarShiftProperties(op, ins, c.goarch) + if err != nil { + return true, false, err + } + if form.double { + return c.lowerScalarDoubleShift(ins, form) + } + count, err := c.loadScalarShiftCount(form) + if err != nil { + return true, false, fmt.Errorf("%s %s count: %w", c.goarch, form.name, err) + } + ty := amd64IntegerTypeForBits(form.bits) + value, store, err := c.loadScalarShiftOperand(ins, form.destinationIndex, ty) + if err != nil { + return true, false, err + } + var result string + if form.stem == "RCL" || form.stem == "RCR" { + result = c.emitScalarCarryRotate(form.stem, form.bits, value, count) + } else { + result = c.emitScalarShiftRotate(form.stem, form.bits, value, count) + c.emitScalarShiftFlags(form.stem, form.bits, value, result, count, false) + } + if err := store(result); err != nil { + return true, false, err + } + return true, false, nil +} + +func amd64ScalarShiftRotateProperties(op string) (stem string, bits int, ok bool) { + switch op { + case "RCLB", "RCRB", "ROLB", "RORB", "SARB", "SALB", "SHLB", "SHRB": + bits = 8 + case "RCLW", "RCRW", "ROLW", "RORW", "SARW", "SALW", "SHLW", "SHRW": + bits = 16 + case "RCLL", "RCRL", "ROLL", "RORL", "SARL", "SALL", "SHLL", "SHRL": + bits = 32 + case "RCLQ", "RCRQ", "ROLQ", "RORQ", "SARQ", "SALQ", "SHLQ", "SHRQ": + bits = 64 + default: + return "", 0, false + } + return op[:len(op)-1], bits, true +} + +func (c *amd64Ctx) emitScalarCarryRotate(stem string, bits int, value, count64 string) string { + // Model RCL/RCR as a rotate of the destination plus CF. Intel masks the + // count to five (six for Q) bits first; loadScalarShiftCount already did that. + // B/W then reduce it modulo 9/17, while L/Q values are already below 33/65. + totalBits := bits + 1 + effective64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = urem i64 %s, %d\n", effective64, count64, totalBits) + + wideType := LLVMType(fmt.Sprintf("i%d", totalBits)) + valueWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", valueWide, amd64IntegerTypeForBits(bits), value, wideType) + oldCF := c.loadFlag(c.flagsCFSlot) + cfWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", cfWide, oldCF, wideType) + cfHigh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %d\n", cfHigh, wideType, cfWide, bits) + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", combined, wideType, valueWide, cfHigh) + + toWideCount := func(value string) string { + converted := c.newTmp() + if totalBits <= 64 { + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", converted, value, wideType) + } else { + fmt.Fprintf(c.b, " %%%s = zext i64 %s to %s\n", converted, value, wideType) + } + return "%" + converted + } + effective := toWideCount("%" + effective64) + inverseRaw := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i64 %d, %%%s\n", inverseRaw, totalBits, effective64) + inverse64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = urem i64 %%%s, %d\n", inverse64, inverseRaw, totalBits) + inverse := toWideCount("%" + inverse64) + + left := c.newTmp() + right := c.newTmp() + if stem == "RCL" { + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %s\n", left, wideType, combined, effective) + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %s\n", right, wideType, combined, inverse) + } else { + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %s\n", right, wideType, combined, effective) + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %s\n", left, wideType, combined, inverse) + } + rotated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", rotated, wideType, left, right) + + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", result, wideType, rotated, amd64IntegerTypeForBits(bits)) + cfShifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %d\n", cfShifted, wideType, rotated, bits) + newCF := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to i1\n", newCF, wideType, cfShifted) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", newCF, c.flagsCFSlot) + + // OF uses the masked count, not the modulo-9/17 count: e.g. RCLB $10 + // rotates once but has undefined OF. A zero-bit carry rotation preserves + // old FLAGS; only the other count>1 arm chooses a new finite OF value. + msbShifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %d\n", msbShifted, amd64IntegerTypeForBits(bits), result, bits-1) + msb := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to i1\n", msb, amd64IntegerTypeForBits(bits), msbShifted) + definedOF := c.newTmp() + if stem == "RCL" { + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, %%%s\n", definedOF, msb, newCF) + } else { + nextShifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %d\n", nextShifted, amd64IntegerTypeForBits(bits), result, bits-2) + nextMSB := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to i1\n", nextMSB, amd64IntegerTypeForBits(bits), nextShifted) + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, %%%s\n", definedOF, msb, nextMSB) + } + multiple := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ugt i64 %s, 1\n", multiple, count64) + chosenOF := c.emitScalarShiftArchitecturalChoice(I1, "%"+multiple, "%"+definedOF) + nonzero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", nonzero, effective64) + oldOF := c.loadFlag(c.flagsOFSlot) + newOF := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i1 %s, i1 %s\n", newOF, nonzero, chosenOF, oldOF) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", newOF, c.flagsOFSlot) + return "%" + result +} + +// The complete typed grammar has already validated the original count. Loading +// it does not change its numerical dependency or the hardware's five/six-bit mask. +func (c *amd64Ctx) loadScalarShiftCount(form x86ScalarShiftForm) (string, error) { + if form.count.Kind == OpImm { + return fmt.Sprintf("%d", form.maskedCount()), nil + } + mask := int64(31) + if form.bits == 64 { + mask = 63 + } + value, err := c.loadReg(form.count.Reg) + if err != nil { + return "", err + } + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", masked, value, mask) + return "%" + masked, nil +} + +func (c *amd64Ctx) emitScalarShiftRotate(stem string, bits int, value, count64 string) string { + ty := amd64IntegerTypeForBits(bits) + amount64 := count64 + if stem == "ROL" || stem == "ROR" { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", masked, count64, bits-1) + amount64 = "%" + masked + } + amount := amount64 + if bits != 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", truncated, amount64, ty) + amount = "%" + truncated + } + + if stem == "ROL" || stem == "ROR" { + inverseNeg := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", inverseNeg, ty, amount) + inverse := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", inverse, ty, inverseNeg, bits-1) + left := c.newTmp() + right := c.newTmp() + if stem == "ROL" { + fmt.Fprintf(c.b, " %%%s = shl %s %s, %s\n", left, ty, value, amount) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", right, ty, value, inverse) + } else { + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", right, ty, value, amount) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", left, ty, value, inverse) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", result, ty, left, right) + return "%" + result + } + + inRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, %d\n", inRange, count64, bits) + safe64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %s, i64 %d\n", safe64, inRange, count64, bits-1) + safe := "%" + safe64 + if bits != 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to %s\n", truncated, safe64, ty) + safe = "%" + truncated + } + shifted := c.newTmp() + switch stem { + case "SHL", "SAL": + fmt.Fprintf(c.b, " %%%s = shl %s %s, %s\n", shifted, ty, value, safe) + case "SHR": + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", shifted, ty, value, safe) + case "SAR": + fmt.Fprintf(c.b, " %%%s = ashr %s %s, %s\n", shifted, ty, value, safe) + return "%" + shifted + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s 0\n", result, inRange, ty, shifted, ty) + return "%" + result +} + +func (c *amd64Ctx) lowerScalarDoubleShift(ins Instr, form x86ScalarShiftForm) (bool, bool, error) { + count, err := c.loadScalarShiftCount(form) + if err != nil { + return true, false, fmt.Errorf("%s %s count: %w", c.goarch, form.name, err) + } + bits := form.bits + ty := amd64IntegerTypeForBits(bits) + source, err := c.evalIntSized(form.source, ty) + if err != nil { + return true, false, err + } + destination, store, err := c.loadScalarShiftOperand(ins, form.destinationIndex, ty) + if err != nil { + return true, false, err + } + + // W double shifts mask the hardware count to five bits. Keep the temporary + // arithmetic bounded, then replace only the ISA-undefined 17..31 output + // with one finite choice. No shift-by-width LLVM poison reaches a select. + effective64 := count + if bits == 16 { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, 15\n", masked, count) + effective64 = "%" + masked + } + effective := effective64 + if bits != 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", truncated, effective64, ty) + effective = "%" + truncated + } + inverseNeg := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", inverseNeg, ty, effective) + inverse := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", inverse, ty, inverseNeg, bits-1) + first := c.newTmp() + second := c.newTmp() + if form.stem == "SHL" { + fmt.Fprintf(c.b, " %%%s = shl %s %s, %s\n", first, ty, destination, effective) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", second, ty, source, inverse) + } else { + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", first, ty, destination, effective) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", second, ty, source, inverse) + } + merged := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", merged, ty, first, second) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", zero, ty, effective) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %s, %s %%%s\n", result, zero, ty, destination, ty, merged) + value := "%" + result + if bits == 16 { + full := c.newTmp() + corrected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i64 %s, 16\n", full, count) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %s, i16 %s\n", corrected, full, source, value) + value = "%" + corrected + outside := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ugt i64 %s, 16\n", outside, count) + value = c.emitScalarShiftArchitecturalChoice(I16, "%"+outside, value) + } + c.emitScalarShiftFlags(form.stem, bits, destination, value, count, true) + if err := store(value); err != nil { + return true, false, err + } + return true, false, nil +} + +func (c *amd64Ctx) storeScalarIntegerOperand(dst Operand, ty LLVMType, value string) error { + switch dst.Kind { + case OpReg: + return c.storeRegSized(dst.Reg, ty, value) + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, %s %s, align 1\n", ty, value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", ty, value, ptr) + return nil + case OpFP: + return c.storeFPResult(dst.FPOffset, ty, value) + default: + return fmt.Errorf("expected GP register or memory destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_shuffle_blocks.go b/amd64_lower_shuffle_blocks.go new file mode 100644 index 000000000..6c67632be --- /dev/null +++ b/amd64_lower_shuffle_blocks.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Complete E4NF data reads survive zero masks and dead results. The same +// validated form supplies byte effects, result dependencies and lowering. +func (c *amd64Ctx) lowerShuffle128BitBlocks(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86Shuffle128BitBlocksProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := c.validateConflictFunnelMemory(ins, form.x86PackedBinaryForm); err != nil { + return true, false, err + } + readWidth := form.byteWidth + if form.memory >= 0 { + readWidth = form.memoryBytes + } + firstBytes, err := c.loadX86PackedShiftBytes(ins.Args[form.first], readWidth, true) + if err != nil { + return true, false, err + } + lanes := form.byteWidth * 8 / form.laneBits + var first string + if form.broadcast { + scalar := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", scalar, readWidth, firstBytes, form.laneBits) + first = amd64SplatInteger(c, lanes, form.laneBits, "%"+scalar) + } else { + first = c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.laneBits, firstBytes) + } + secondBytes, err := c.loadX86PackedShiftBytes(ins.Args[form.second], form.byteWidth, true) + if err != nil { + return true, false, err + } + second := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.laneBits, secondBytes) + result := c.emitShuffle128BitBlocks(lanes, form.laneBits, first, second, form.immediate) + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(ins.Args[form.destination], form.byteWidth, form.laneBits, false) + if err != nil { + return true, false, err + } + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.laneBits, result, form.byteWidth) + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitShuffle128BitBlocks(lanes, laneBits int, firstPlan9, secondPlan9 string, immediate uint8) string { + blockLanes := 128 / laneBits + blocks := lanes / blockLanes + indices := make([]string, 0, lanes) + for outputBlock := 0; outputBlock < blocks; outputBlock++ { + var sourceBlock int + secondOperandOffset := 0 + if blocks == 2 { + sourceBlock = int(immediate>>outputBlock) & 1 + if outputBlock == 1 { + secondOperandOffset = lanes + } + } else { + sourceBlock = int(immediate>>(2*outputBlock)) & 3 + if outputBlock >= 2 { + secondOperandOffset = lanes + } + } + for lane := 0; lane < blockLanes; lane++ { + indices = append(indices, fmt.Sprintf("i32 %d", secondOperandOffset+sourceBlock*blockLanes+lane)) + } + } + shuffled := c.newTmp() + // The second Plan 9 source is Intel SRC1 and therefore LLVM operand 0; + // the first Plan 9 (r/m) source is Intel SRC2 / LLVM operand 1. + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %s, <%d x i%d> %s, <%d x i32> <%s>\n", + shuffled, lanes, laneBits, secondPlan9, lanes, laneBits, firstPlan9, lanes, strings.Join(indices, ", ")) + return "%" + shuffled +} diff --git a/amd64_lower_shuffle_blocks_test.go b/amd64_lower_shuffle_blocks_test.go new file mode 100644 index 000000000..9bd161557 --- /dev/null +++ b/amd64_lower_shuffle_blocks_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86Shuffle128BitBlocksCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 shares _yvshuff32x4 across the floating and integer D/Q + // spellings. It has Y/Z rows, an unsigned-byte immediate, optional K + // masking/.Z, and scalar broadcast from the first Plan 9 memory source. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT shuffle128bitblockforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VSHUFF32X4", "VSHUFF64X2", "VSHUFI32X4", "VSHUFI64X2"} { + fmt.Fprintf(&source, "\t%s $0, Y1, Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s $2, 8(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s $255, Z1, Z2, Z%d\n", op, lastZ) + fmt.Fprintf(&source, "\t%s.BCST $3, 40(AX), Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $4, Y1, Y20, K1, Y21\n", op) + fmt.Fprintf(&source, "\t%s.Z $5, Z1, Z20, K2, Z21\n", op) + fmt.Fprintf(&source, "\t%s.BCST.Z $6, 104(AX), Z20, K3, Z21\n", op) + } + } + source.WriteString("\tRET\n") + + if target.goarch == "386" { + // Go's mode32 frontend rejects every original four-operand + // spelling here. Keep those source observations; they are + // neither numerical Context nor physical LLVM object passes. + goAssemblerOracleOutput(t, "386", "#include \"textflag.h\"\n"+source.String(), "", false) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, instruction := range file.Funcs[0].Instrs { + if instruction.Op == OpTEXT || instruction.Op == OpRET { + continue + } + t.Run(fmt.Sprint("GoFrontendRejects/", count), func(t *testing.T) { + goAssemblerOracleOutput(t, "386", "TEXT ·rejected(SB),4,$0-0\n"+instruction.Raw+"\nRET\n", "", false) + _, known, err := x86Shuffle128BitBlocksProperties(instruction.Op, instruction, "386") + if !known || err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go-refused frontend arity became an input Context or accepted form", instruction.Raw, err) + } + }) + count++ + } + if count != 16 { + t.Fatal("original mode32 source directory changed", count) + } + return + } + ll := x86ElementPermuteUnboundFormatObject(t, source.String(), target.goarch, target.triple, 28, 0) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "shuffle-128-bit-blocks-"+target.name+".ll", "shuffle-128-bit-blocks-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86Shuffle128BitBlocksRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VSHUFF64X2 $1, X0, X1, X2", + "VSHUFF64X2 $1, Y0, Z1, Z2", + "VSHUFF64X2 $1, Y0, (AX), Y2", + "VSHUFF64X2 $1, Y0, Y1, K0, Y2", + "VSHUFF64X2 $1, Y0, Y1, AX", + "VSHUFF64X2.Z $1, Y0, Y1, Y2", + "VSHUFF64X2.BCST $1, Y0, Y1, Y2", + "VSHUFF64X2.Z.BCST $1, (AX), Y1, K1, Y2", + "VSHUFF64X2 $256, Y0, Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86Shuffle128BitBlocksRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86Shuffle128BitBlocksRejected(t, "386", "i386-unknown-linux-gnu", "VSHUFF64X2 $1, Y0, Y1, K1, Y2") + assertX86Shuffle128BitBlocksRejected(t, "386", "i386-unknown-linux-gnu", "VSHUFF64X2 $1, Z0, Z1, Z8") +} + +func assertX86Shuffle128BitBlocksRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VSHUFF/VSHUFI forms", instruction) + } +} diff --git a/amd64_lower_shuffle_bytes.go b/amd64_lower_shuffle_bytes.go new file mode 100644 index 000000000..bb95781e5 --- /dev/null +++ b/amd64_lower_shuffle_bytes.go @@ -0,0 +1,115 @@ +package plan9asm + +import "fmt" + +// lowerPackedByteShuffle implements all Go 1.27 PSHUFB/VPSHUFB forms. +// VPSHUFB applies the byte shuffle independently to each 128-bit lane; EVEX +// writemasks operate at byte granularity. +func (c *amd64Ctx) lowerPackedByteShuffle(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86ByteShuffleProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy byte shuffle m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + byteWidth := form.byteWidth + data, err := c.loadPackedCompareBytes(second, byteWidth) + if err != nil { + return true, false, err + } + var result string + if form.legacy && form.memory >= 0 { + // Keep the original opcode, including m128 alignment and dead faults. + // The portable E4NF reader alone cannot reproduce this legacy effect. + result, err = c.emitPackedBinaryNativeMemory(first, 8, data, "pshufb") + } else if ins.x86RIPLiteral && len(ins.x86RIPLiteralData) == byteWidth { + result = c.emitPackedByteShuffleLiteral(byteWidth, data, ins.x86RIPLiteralData) + } else { + // E4NF: all control bytes are read, even with a dead or zero-masked + // result. Reuse the exact full-tuple reader, not the E4 lane reader. + control, err := c.loadPackedUnpackSource(first, x86PackedUnpackForm{ + x86PackedBinaryForm: form, spec: x86PackedUnpackSpec{laneBits: 8}, + }) + if err != nil { + return true, false, err + } + result = c.emitPackedByteShuffle(byteWidth, data, control) + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, byteWidth, 8, result, old, mask, form.zeroing) + } + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, result) + } + return true, false, c.storePackedMoveOperand(destination, byteWidth, result) +} + +func (c *amd64Ctx) emitPackedByteShuffleLiteral(byteWidth int, data string, control []byte) string { + indices := make([]int, byteWidth) + for index, selected := range control { + if selected&0x80 != 0 { + indices[index] = byteWidth // Select a zero from the second vector. + } else { + indices[index] = index/16*16 + int(selected&0x0f) + } + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> %s\n", + result, byteWidth, data, byteWidth, byteWidth, llvmI32Mask(indices)) + return "%" + result +} + +func (c *amd64Ctx) emitPackedByteShuffle(byteWidth int, data, control string) string { + if byteWidth == 16 { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %s, <16 x i8> %s)\n", call, data, control) + return "%" + call + } + + parts := make([]string, 0, byteWidth/16) + for lane := 0; lane < byteWidth/16; lane++ { + dataLane := c.newTmp() + controlLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> poison, <16 x i32> %s\n", dataLane, byteWidth, data, byteWidth, llvmI32RangeMask(lane*16, 16)) + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> poison, <16 x i32> %s\n", controlLane, byteWidth, control, byteWidth, llvmI32RangeMask(lane*16, 16)) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %%%s, <16 x i8> %%%s)\n", call, dataLane, controlLane) + parts = append(parts, "%"+call) + } + partWidth := 16 + for len(parts) > 1 { + combined := make([]string, 0, len(parts)/2) + for i := 0; i < len(parts); i += 2 { + joined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> %s, <%d x i32> %s\n", joined, partWidth, parts[i], partWidth, parts[i+1], partWidth*2, llvmI32RangeMask(0, partWidth*2)) + combined = append(combined, "%"+joined) + } + parts = combined + partWidth *= 2 + } + return parts[0] +} diff --git a/amd64_lower_shuffle_bytes_test.go b/amd64_lower_shuffle_bytes_test.go new file mode 100644 index 000000000..5f2763e78 --- /dev/null +++ b/amd64_lower_shuffle_bytes_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedByteShuffleCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses ymshufb for PSHUFB and _yvandnpd for VPSHUFB. + // The legacy instruction accepts X/m128, X. The vector instruction has + // VEX X/Y forms and EVEX X/Y/Z forms, including K1-K7 merge/zero masks. + // In 386 mode the Go frontend still exposes EVEX X/Y0-31 and Z0-7, but + // cannot express the four-operand mask rows. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + zLast := 21 + vectorData, vectorDestination := 20, 21 + if target.goarch == "386" { + legacyLast = 7 + zLast = 7 + vectorData, vectorDestination = 6, 7 + } + var src strings.Builder + src.WriteString("TEXT packedbyteshuffleforms(SB),NOSPLIT,$0-0\n") + fmt.Fprintf(&src, "\tPSHUFB X0, X%d\n", legacyLast) + fmt.Fprintf(&src, "\tPSHUFB (AX), X%d\n", legacyLast) + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\tVPSHUFB %s0, %s1, %s2\n", width, width, width) + fmt.Fprintf(&src, "\tVPSHUFB (AX), %s%d, %s%d\n", width, vectorData, width, vectorDestination) + } + fmt.Fprintf(&src, "\tVPSHUFB Z0, Z1, Z%d\n", zLast) + fmt.Fprintf(&src, "\tVPSHUFB (AX), Z6, Z%d\n", zLast) + if target.goarch == "amd64" { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\tVPSHUFB %s20, %s21, K1, %s22\n", width, width, width) + fmt.Fprintf(&src, "\tVPSHUFB.Z (AX), %s21, K2, %s22\n", width, width) + } + } + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedbyteshuffleforms": {Name: "packedbyteshuffleforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-byte-shuffle-"+target.name+".ll", "packed-byte-shuffle-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedByteShuffleRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSHUFB.Z X0, X1", + "PSHUFB X0", + "PSHUFB Y0, Y1", + "PSHUFB X0, (AX)", + "VPSHUFB X0, X1", + "VPSHUFB X0, Y1, Y2", + "VPSHUFB X0, X1, (AX)", + "VPSHUFB.BCST (AX), X1, X2", + "VPSHUFB.SAE X0, X1, X2", + "VPSHUFB.Z X0, X1, X2", + "VPSHUFB X0, X1, K0, X2", + "VPSHUFB X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedByteShuffleRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + + for _, instruction := range []string{ + "PSHUFB X8, X0", + "PSHUFB X0, X8", + "VPSHUFB Z8, Z0, Z1", + "VPSHUFB Z0, Z1, Z8", + "VPSHUFB X0, X1, K1, X2", + "VPSHUFB.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedByteShuffleRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedByteShuffleRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ymshufb/_yvandnpd forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_shuffle_float.go b/amd64_lower_shuffle_float.go new file mode 100644 index 000000000..c3d1aef69 --- /dev/null +++ b/amd64_lower_shuffle_float.go @@ -0,0 +1,112 @@ +package plan9asm + +import "fmt" + +// Every consumer shares the complete Go grammar and exact two-source selector. +// Integer byte lanes preserve NaN payloads without creating FP/MXCSR effects. +func (c *amd64Ctx) lowerPackedFloatShuffle(op Op, ins Instr) (bool, bool, error) { + f, known, err := x86PackedFloatShuffleProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if err := c.validateConflictFunnelMemory(ins, f.x86PackedBinaryForm); err != nil { + return true, false, err + } + first, second, destination := ins.Args[f.first], ins.Args[f.second], ins.Args[f.destination] + secondBytes, err := c.loadPackedCompareBytes(second, f.byteWidth) + if err != nil { + return true, false, err + } + if f.legacy && f.memory >= 0 { + if first.Kind == OpFP || !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: legacy floating shuffle m128 needs its original native storage/target", ErrProbeNeedsContext) + } + lanes := c.bitcastVectorBytesToIntegerLanes(16, 128/f.bits, f.bits, secondBytes) + mnemonic := "shufps" + if f.bits == 64 { + mnemonic = "shufpd" + } + // Keep the complete original Type4 opcode and its RMW destination: + // align-1 loads do not reproduce legacy m128 alignment exceptions. + value, err := c.emitPackedBinaryNativeMemory(first, f.bits, lanes, + fmt.Sprintf("%s $$%d,", mnemonic, f.selector)) + if err != nil { + return true, false, err + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <16 x i8>\n", bytes, 128/f.bits, f.bits, value) + return true, false, c.storeLegacyPackedLogical(destination, "%"+bytes) + } + readWidth := f.byteWidth + if f.memory >= 0 { + readWidth = f.memoryBytes + } + // EVEX E4NF does not suppress faults. Even a zero mask, selected scalar + // or dead result retains this single full/broadcast unaligned read. + firstBytes, err := c.loadX86PackedShiftBytes(first, readWidth, true) + if err != nil { + return true, false, err + } + indices := make([]int, f.byteWidth) + width := f.byteWidth + if f.broadcast { + // Extend only the representation to a common shufflevector width; + // no additional memory byte or initialized input is introduced. + expanded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> %s\n", + expanded, readWidth, firstBytes, readWidth, width, llvmRepeatI8Mask(readWidth, width)) + firstBytes = "%" + expanded + } + for output := range indices { + indices[output] = f.sourceByte(output) + } + shuffled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> %s, <%d x i32> %s\n", + shuffled, width, firstBytes, width, secondBytes, width, llvmI32Mask(indices)) + result := "%" + shuffled + if f.mask >= 0 { + mask, err := c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + lanes := width * 8 / f.bits + computed := c.bitcastVectorBytesToIntegerLanes(width, lanes, f.bits, result) + old := "zeroinitializer" + if !f.zeroing { + old, err = c.loadPackedCompareLanes(destination, width, f.bits, false) + if err != nil { + return true, false, err + } + } + masked := amd64ApplyIntegerLaneMask(c, lanes, f.bits, computed, old, mask, f.zeroing) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, f.bits, masked, width) + result = "%" + bytes + } + if f.legacy { + return true, false, c.storeLegacyPackedLogical(destination, result) + } + return true, false, c.storePackedMoveOperand(destination, width, result) +} + +func amd64PackedFloatShuffleProperties(op string) (laneBits int, vector, ok bool) { + switch op { + case "SHUFPS": + return 32, false, true + case "SHUFPD": + return 64, false, true + case "VSHUFPS": + return 32, true, true + case "VSHUFPD": + return 64, true, true + default: + return 0, false, false + } +} + +func amd64UnsignedImm8(arg Operand) (uint8, bool) { + if arg.Kind != OpImm || arg.Imm < 0 || arg.Imm > 255 { + return 0, false + } + return uint8(arg.Imm), true +} diff --git a/amd64_lower_shuffle_words.go b/amd64_lower_shuffle_words.go new file mode 100644 index 000000000..8e1076d2e --- /dev/null +++ b/amd64_lower_shuffle_words.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Preserve the helper entry point while all dword/word/MMX siblings use one +// complete grammar. Dispatch identity cannot grant a contradictory spelling. +func (c *amd64Ctx) lowerPackedWordShuffle(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86ImmediateShuffleSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("immediate-shuffle dispatch/source mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, known, err := x86ImmediateShuffleProperties(ins, c.goarch) + if err != nil || !known { + return known, false, err + } + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + destination := ins.Args[form.destination] + if form.spec.legacy && !form.spec.mmx && form.memory >= 0 { + return true, false, c.lowerLegacyImmediateShuffleMemory(ins, form) + } + var source string + if form.spec.mmx { + source, err = c.loadPackedUnpackMMXSource(form.source, form.memoryBytes) + if err == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to <8 x i8>\n", value, source) + source = "%" + value + } + } else if form.broadcast { + bytes, loadErr := c.loadX86PackedShiftBytes(form.source, 4, true) + err = loadErr + if err == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i8> %s, <4 x i8> zeroinitializer, <%d x i32> %s\n", + value, bytes, form.byteWidth, llvmRepeatI8Mask(4, form.byteWidth)) + source = "%" + value + } + } else { + // E4NF/E4NF.nb does not suppress memory faults, even at K=0 or when + // the immediate selects only one lane. Keep the full read observable. + source, err = c.loadX86PackedShiftBytes(form.source, form.byteWidth, true) + } + if err != nil { + return true, false, err + } + indices := make([]string, form.byteWidth) + for index := range indices { + // A broadcast has already expanded the m32 to full logical lanes. + if form.broadcast { + indices[index] = fmt.Sprintf("i32 %d", index) + } else { + indices[index] = fmt.Sprintf("i32 %d", form.sourceByte(index)) + } + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> <%s>\n", + value, form.byteWidth, source, form.byteWidth, form.byteWidth, strings.Join(indices, ", ")) + result := "%" + value + if form.spec.mmx { + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i8> %s to i64\n", bits, result) + return true, false, c.storeReg(destination.Reg, "%"+bits) + } + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + laneBits := form.spec.laneBytes * 8 + lanes := form.byteWidth / form.spec.laneBytes + computed := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, laneBits, result) + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, laneBits, false) + if err != nil { + return true, false, err + } + } + masked := amd64ApplyIntegerLaneMask(c, lanes, laneBits, computed, old, mask, form.zeroing) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, laneBits, masked, form.byteWidth) + result = "%" + bytes + } + if form.spec.legacy { + return true, false, c.storeLegacyPackedLogical(destination, result) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, result) +} + +func amd64PackedWordShuffleRequiresEVEX(source, destination Operand, byteWidth int, masked bool) bool { + if byteWidth == 64 || masked { + return true + } + for _, operand := range []Operand{source, destination} { + if operand.Kind == OpReg { + index, ok := amd64VectorRegisterIndex(operand.Reg, byteWidth) + if !ok || index >= 16 { + return true + } + } + } + return false +} + +// Legacy m128 must retain the actual single SSE instruction's alignment and +// fault behavior, not an align-1 LLVM read or a fabricated virtual FP buffer. +func (c *amd64Ctx) lowerLegacyImmediateShuffleMemory(ins Instr, form x86ImmediateShuffleForm) error { + source := form.source + if source.Kind == OpFP || ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + source.Kind == OpMem && !c.fixedLocalMemoryCertified(ins, form.memory) && !c.nativeMemoryOperand(ins, form.memory) { + return fmt.Errorf("%w: legacy immediate shuffle m128 needs original native backing/alignment", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + mnemonic := "pshufd" + if form.spec.laneBytes == 2 { + mnemonic = "pshuflw" + if form.spec.high { + mnemonic = "pshufhw" + } + } + value, bytes := c.newTmp(), c.newTmp() + assembly := fmt.Sprintf("%s $$%d, %s($1), $0", mnemonic, form.immediate, segment) + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(%s %s)\n", + value, assembly, "=x,r,~{memory}", pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <16 x i8>\n", bytes, value) + return c.storeLegacyPackedLogical(ins.Args[form.destination], "%"+bytes) +} diff --git a/amd64_lower_shuffle_words_test.go b/amd64_lower_shuffle_words_test.go new file mode 100644 index 000000000..2bbf7254a --- /dev/null +++ b/amd64_lower_shuffle_words_test.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedWordShuffleCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + zLast := 31 + if target.goarch == "386" { + legacyLast = 7 + zLast = 7 + } + var source strings.Builder + source.WriteString("TEXT packedwordshuffleforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSHUFHW", "PSHUFLW"} { + fmt.Fprintf(&source, "\t%s $0, X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s $255, 8(AX), X%d\n", op, legacyLast) + } + for _, op := range []string{"VPSHUFHW", "VPSHUFLW"} { + source.WriteString("\t" + op + " $-128, X0, X1\n") + source.WriteString("\t" + op + " $127, 8(AX), X1\n") + source.WriteString("\t" + op + " $255, Y0, Y1\n") + source.WriteString("\t" + op + " $1, 16(AX), Y2\n") + if target.goarch == "amd64" { + source.WriteString("\t" + op + " $2, X16, X31\n") + } + fmt.Fprintf(&source, "\t%s $3, Z0, Z%d\n", op, zLast) + if target.goarch == "amd64" { + source.WriteString("\t" + op + " $4, X20, K1, X21\n") + source.WriteString("\t" + op + ".Z $5, 24(AX), K2, Y22\n") + source.WriteString("\t" + op + " $6, Z16, K3, Z31\n") + } + } + source.WriteString("\tRET\n") + goAssemblerOracleOutput(t, target.goarch, strings.ReplaceAll(source.String(), "NOSPLIT", "4"), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedwordshuffleforms": {Name: "packedwordshuffleforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-word-shuffle-"+target.name+".ll", "packed-word-shuffle-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedWordShuffleRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSHUFHW $-1, X0, X1", + "PSHUFLW $256, X0, X1", + "PSHUFHW $1, X16, X1", + "PSHUFLW $1, Y0, Y1", + "PSHUFHW.Z $1, X0, X1", + "PSHUFLW $1, X0, (AX)", + "VPSHUFHW $-129, X0, X1", + "VPSHUFLW $256, X0, X1", + "VPSHUFHW $-1, X16, X1", + "VPSHUFLW $-1, Z0, Z1", + "VPSHUFHW $1, X0, Y1", + "VPSHUFLW $1, X0, K0, X1", + "VPSHUFHW.Z $1, X0, X1", + "VPSHUFLW.BCST $1, (AX), X1", + "VPSHUFHW $1, X0, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedWordShuffleRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PSHUFHW $1, X8, X0", + "PSHUFLW $1, X0, X8", + "VPSHUFHW $1, X0, K1, X1", + "VPSHUFLW.Z $1, X0, K1, X1", + "VPSHUFHW $1, Z8, Z0", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedWordShuffleRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedWordShuffleRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's PSHUFHW/PSHUFLW tables for %s", instruction, goarch) + } +} diff --git a/amd64_lower_sqrt.go b/amd64_lower_sqrt.go new file mode 100644 index 000000000..6bb745293 --- /dev/null +++ b/amd64_lower_sqrt.go @@ -0,0 +1,124 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerSqrt(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86SqrtSpecFor(Op(base)); !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(op), string(ins.Op)) { + return true, false, fmt.Errorf("sqrt dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + f, _, err := x86SqrtProperties(ins, c.goarch) + if err != nil { + return true, false, err + } + if f.bits == 16 && !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return false, false, nil // Retain the existing cross-target numerical model. + } + if f.bits == 16 && f.scalar { + return true, false, c.emitNativeFP16ThreeOperand(ins, x86NativeFP16ThreeOperand{ + byteWidth: 16, lanes: 1, memoryBytes: f.memoryBytes, second: f.second, + destination: f.destination, mask: f.mask, + controls: amd64FMA3Suffix{zeroing: f.zeroing, rounding: f.rounding}, + }) + } + if f.scalar { + return true, false, c.lowerScalarSqrt(ins, f) + } + if ins.Args[0].Kind == OpFP || ins.x86AddressBits != 0 || f.memory >= 0 && + ins.Args[0].Kind == OpMem && !c.nativeMemoryOperand(ins, f.memory) { + return true, false, fmt.Errorf("%w: %s packed source lacks its original native memory binding", ErrProbeNeedsContext, op) + } + carrierBits := f.bits + if carrierBits == 16 { + carrierBits = 32 // Register bit carrier, not an arithmetic precision change. + } + return true, false, c.emitNativePackedConversion(ins, x86NativePackedConversion{ + byteWidth: f.byteWidth, sourceBytes: f.byteWidth, memoryBytes: f.memoryBytes, + outputBits: carrierBits, destination: f.destination, mask: f.mask, lanes: f.byteWidth * 8 / f.bits, + legacy: f.legacy, evex: x86SqrtNativeEVEX(ins, f), broadcast: f.broadcast, zeroing: f.zeroing, + rounding: f.rounding, mnemonic: strings.ToLower(base), + }) +} + +// Retain the actual scalar opcode even for dead results: default rounding, +// DAZ/FTZ, exceptions and explicit ER+SAE are not pure llvm.sqrt semantics. +// Masked source reads use the existing exact-width conditional load; the +// native mask separately suppresses inactive arithmetic and its exceptions. +func (c *amd64Ctx) lowerScalarSqrt(ins Instr, f x86FloatBinaryForm) error { + mask := "" + var err error + if f.mask >= 0 { + mask, err = c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return err + } + } + source, err := c.loadFloatBinaryScalar(ins.Args[0], f.bits, mask) + if err != nil { + return err + } + typ := amd64FloatingVectorType(128/f.bits, f.bits) + load := func(index int) (string, error) { + bytes, err := c.loadX(ins.Args[index].Reg) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", value, bytes, typ) + return "%" + value, nil + } + old := "zeroinitializer" + if f.legacy || mask != "" && !f.zeroing { + old, err = load(f.destination) + if err != nil { + return err + } + } + registerClass := "x" + if x86SqrtNativeEVEX(ins, f) { + registerClass = "v" + } + constraints := "=" + registerClass + ",0," + registerClass + args := fmt.Sprintf("%s %s, %s %s", typ, old, amd64FMA3LLVMType(1, f.bits), source) + operands, maskIndex := "$2, $0", 3 + if !f.legacy { + upper, err := load(f.second) + if err != nil { + return err + } + constraints += "," + registerClass + args += ", " + typ + " " + upper + operands, maskIndex = "$2, $3, $0", 4 + } + mnemonic, _, _ := strings.Cut(strings.ToLower(string(ins.Op)), ".") + if x86SqrtNativeEVEX(ins, f) { + mnemonic = "{evex} " + mnemonic + } + assembly := mnemonic + " " + if f.rounding != "" { + assembly += "{" + strings.ToLower(strings.TrimSuffix(f.rounding, "_SAE")) + "-sae}, " + } + assembly += operands + if mask != "" { + assembly = fmt.Sprintf("kmovw ${%d:k}, %%k1; ", maskIndex) + assembly + " {%k1}" + if f.zeroing { + assembly += " {z}" + } + constraints += ",r,~{k1}" + args += ", i32 " + c.approximateMathMask32(mask) + } + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, typ, assembly, constraints+",~{memory},~{fpsr}", args) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytes, typ, result) + if f.legacy { + return c.storeLegacyXViews(ins.Args[f.destination].Reg, "%"+bytes) + } + return c.storePackedMoveOperand(ins.Args[f.destination], 16, "%"+bytes) +} diff --git a/amd64_lower_sqrt_scalar_test.go b/amd64_lower_sqrt_scalar_test.go new file mode 100644 index 000000000..566e6934f --- /dev/null +++ b/amd64_lower_sqrt_scalar_test.go @@ -0,0 +1,231 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86ScalarSqrtCompleteGo127Forms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT scalarsqrtforms(SB),$0-0\n") + for _, op := range []string{"VSQRTSS", "VSQRTSD"} { + fmt.Fprintf(&source, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&source, "\t%s 8(BX), X3, X4\n", op) + fmt.Fprintf(&source, "\t%s X29, X30, X31\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X5, X6, K1, X7\n", op) + fmt.Fprintf(&source, "\t%s.Z 16(BX), X8, K2, X9\n", op) + } + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s X10, X11, X12\n", op, rounding) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.%s.Z X13, X14, K3, X15\n", op, rounding) + } + } + } + source.WriteString("\tRET\n") + ll := x86SqrtOriginalFormatObjects(t, source.String(), "scalarsqrtforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scalar-sqrt-"+target.name+".ll", "scalar-sqrt-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ScalarSqrtRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VSQRTSS X0, X1", + "VSQRTSD X0, X1, X2, X3, X4", + "VSQRTSS Y0, X1, X2", + "VSQRTSD X0, Y1, X2", + "VSQRTSS X0, X1, K0, X2", + "VSQRTSD.Z X0, X1, X2", + "VSQRTSS.BCST 8(BX), X1, X2", + "VSQRTSD.SAE X0, X1, X2", + "VSQRTSS.RN_SAE 8(BX), X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86ScalarSqrtRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86ScalarSqrtRejected(t, "386", "i386-unknown-linux-gnu", "VSQRTSS X0, X1, K1, X2") +} + +func assertX86ScalarSqrtRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VSQRTSS/VSQRTSD _yvaddsd forms for %s", instruction, goarch) + } +} + +func qualifyAMD64ScalarSqrtRuntimeSemantics(t *testing.T) { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Skip("native AVX512 execution belongs to TestCrossLinuxRuntimeMatrixX86ScalarSqrtNativeSemantics") + } + x86ScalarSqrtNativeRuntime(t) +} + +func x86ScalarSqrtNativeRuntime(t *testing.T) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + ir, main := x86ScalarSqrtRuntimeFixture(t, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "scalar-sqrt", ir, main, level) + }) + } +} + +func TestX86ScalarSqrtNativeFixtureObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue // The original runtime fixture uses native qword parameters. + } + t.Run(target.triple, func(t *testing.T) { + ir, _ := x86ScalarSqrtRuntimeFixture(t, target.triple) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + }) + } +} + +func x86ScalarSqrtRuntimeFixture(t *testing.T, triple string) (string, string) { + t.Helper() + source := ` +TEXT scalarsqrt(SB),NOSPLIT,$0-64 + MOVQ out+0(FP), AX + MOVQ source64+8(FP), BX + MOVQ passthrough64+16(FP), CX + MOVQ source32+24(FP), DX + MOVQ passthrough32+32(FP), SI + MOVQ old64+40(FP), DI + MOVQ old32+48(FP), R8 + MOVQ mask+56(FP), R9 + KMOVQ R9, K1 + + VMOVUPD (BX), X0 + VMOVUPD (CX), X1 + VSQRTSD.RD_SAE X0, X1, X2 + VMOVUPD X2, 0(AX) + + VMOVUPS (DX), X0 + VMOVUPS (SI), X1 + VSQRTSS.RU_SAE X0, X1, X2 + VMOVUPS X2, 16(AX) + + VMOVUPD (BX), X0 + VMOVUPD (CX), X1 + VMOVUPD (DI), X4 + VSQRTSD X0, X1, K1, X4 + VMOVUPD X4, 32(AX) + + VMOVUPS (DX), X0 + VMOVUPS (SI), X1 + VMOVUPS (R8), X4 + VSQRTSS.Z X0, X1, K1, X4 + VMOVUPS X4, 48(AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: I64, Index: 7, Field: -1}, + }} + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "scalarsqrt": {Name: "scalarsqrt", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +extern void scalarsqrt(uint8_t *, const double *, const double *, const float *, const float *, const double *, const float *, uint64_t); +int main(void) { + double source64[2] = {9.0, 77.0}; + const double passthrough64[2] = {100.0, 101.0}; + float source32[4] = {16.0f, 17.0f, 18.0f, 19.0f}; + const float passthrough32[4] = {200.0f, 201.0f, 202.0f, 203.0f}; + const double old64[2] = {300.0, 301.0}; + const float old32[4] = {400.0f, 401.0f, 402.0f, 403.0f}; + union { uint8_t bytes[64]; double d64[8]; float f32[16]; } out = {0}; + scalarsqrt(out.bytes, source64, passthrough64, source32, passthrough32, old64, old32, 0); + if (out.d64[0] != 3.0 || out.d64[1] != 101.0) return 1; + if (out.f32[4] != 4.0f || out.f32[5] != 201.0f || out.f32[6] != 202.0f || out.f32[7] != 203.0f) return 2; + if (out.d64[4] != 300.0 || out.d64[5] != 101.0) return 3; + if (out.f32[12] != 0.0f || out.f32[13] != 201.0f || out.f32[14] != 202.0f || out.f32[15] != 203.0f) return 4; + scalarsqrt(out.bytes, source64, passthrough64, source32, passthrough32, old64, old32, 1); + if (out.d64[4] != 3.0 || out.d64[5] != 101.0 || out.f32[12] != 4.0f || out.f32[13] != 201.0f) return 5; + + source64[0] = 2.0; + source32[0] = 2.0f; + scalarsqrt(out.bytes, source64, passthrough64, source32, passthrough32, old64, old32, 1); + uint64_t down; + uint32_t up; + memcpy(&down, out.bytes, sizeof(down)); + memcpy(&up, out.bytes + 16, sizeof(up)); + if (down != UINT64_C(0x3ff6a09e667f3bcc) || up != UINT32_C(0x3fb504f4)) return 6; + + unsigned saved = _mm_getcsr(); + source64[0] = -1.0; + source32[0] = -1.0f; + _mm_setcsr(0x1f80); + scalarsqrt(out.bytes, source64, passthrough64, source32, passthrough32, old64, old32, 0); + unsigned inactive = _mm_getcsr(); + _mm_setcsr(0x1f80); + scalarsqrt(out.bytes, source64, passthrough64, source32, passthrough32, old64, old32, 1); + unsigned active = _mm_getcsr(); + _mm_setcsr(saved); + if ((inactive & 0x3f) != 0 || (active & 0x3f) != 1) return 7; + return 0; +} +` + return ll, mainC +} diff --git a/amd64_lower_stack_width.go b/amd64_lower_stack_width.go new file mode 100644 index 000000000..25e4b1039 --- /dev/null +++ b/amd64_lower_stack_width.go @@ -0,0 +1,381 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64StackWidthMode uint8 + +const ( + amd64StackPushValue amd64StackWidthMode = iota + amd64StackPopValue + amd64StackPushFlags + amd64StackPopFlags +) + +type amd64StackWidthSpec struct { + bits int + mode amd64StackWidthMode + segments amd64StackSegmentSet + segmentNativeWidth bool +} + +type amd64StackSegmentSet uint8 + +const ( + amd64StackSegmentES amd64StackSegmentSet = 1 << iota + amd64StackSegmentCS + amd64StackSegmentSS + amd64StackSegmentDS + amd64StackSegmentFS + amd64StackSegmentGS + + amd64StackPushSegments = amd64StackSegmentES | amd64StackSegmentCS | + amd64StackSegmentSS | amd64StackSegmentDS | amd64StackSegmentFS | + amd64StackSegmentGS + amd64StackPopSegments = amd64StackPushSegments &^ amd64StackSegmentCS + amd64StackLongSegments = amd64StackSegmentFS | amd64StackSegmentGS +) + +// amd64StackWidthSpecs models the value/flags, push/pop, width, and segment-set +// axes once for all Go x86 stack forms. Ordinary L spellings are 386-only and +// ordinary Q spellings are amd64-only. Go's separate ymovtab accepts its listed +// segment-register forms in both modes; L/Q use the target's native stack width. +var amd64StackWidthSpecs = map[Op]amd64StackWidthSpec{ + "PUSHW": {bits: 16, mode: amd64StackPushValue, segments: amd64StackPushSegments}, + "POPW": {bits: 16, mode: amd64StackPopValue, segments: amd64StackPopSegments}, + "PUSHL": {bits: 32, mode: amd64StackPushValue, segments: amd64StackPushSegments, segmentNativeWidth: true}, + "POPL": {bits: 32, mode: amd64StackPopValue, segments: amd64StackPopSegments, segmentNativeWidth: true}, + "PUSHQ": {bits: 64, mode: amd64StackPushValue, segments: amd64StackLongSegments, segmentNativeWidth: true}, + "POPQ": {bits: 64, mode: amd64StackPopValue, segments: amd64StackLongSegments, segmentNativeWidth: true}, + "PUSHFW": {bits: 16, mode: amd64StackPushFlags}, + "POPFW": {bits: 16, mode: amd64StackPopFlags}, + "PUSHFL": {bits: 32, mode: amd64StackPushFlags}, + "POPFL": {bits: 32, mode: amd64StackPopFlags}, + "PUSHFQ": {bits: 64, mode: amd64StackPushFlags}, + "POPFQ": {bits: 64, mode: amd64StackPopFlags}, +} + +// This describes Go's complete value/FLAGS operand grammar, independently of +// a consumer's private-stack, segment-state or native-memory permissions. +func x86StackWidthProperties(ins Instr, arch string) (amd64StackWidthSpec, bool, error) { + name := strings.ToUpper(string(ins.Op)) + base := strings.SplitN(name, ".", 2)[0] + spec, known := amd64StackWidthSpecs[Op(base)] + if !known { + return spec, false, nil + } + fail := func() (amd64StackWidthSpec, bool, error) { + return spec, true, fmt.Errorf("%s invalid %s stack form: %q", arch, base, ins.Raw) + } + if name != base { + return fail() + } + flags := spec.mode == amd64StackPushFlags || spec.mode == amd64StackPopFlags + if flags { + if err := validateX86OrdinaryStackWidthMode(arch, base, spec.bits, ins); err != nil { + return spec, true, err + } + if len(ins.Args) != 0 { + return spec, true, fmt.Errorf("%s %s takes no operands: %q", arch, base, ins.Raw) + } + return spec, true, nil + } + if len(ins.Args) != 1 { + role := "src" + if spec.mode == amd64StackPopValue { + role = "dst" + } + return spec, true, fmt.Errorf("%s %s expects %s: %q", arch, base, role, ins.Raw) + } + arg := ins.Args[0] + if arg.Kind == OpImm && (arg.ImmRaw != "" || arg.ImmIsFloat || arg.Imm < -2147483648 || arg.Imm > 4294967295) { + return fail() + } + if x86SegmentRegisterOperand(arg) { + if !spec.segments.contains(arg.Reg) { + return fail() + } + return spec, true, nil + } + if err := validateX86OrdinaryStackWidthMode(arch, base, spec.bits, ins); err != nil { + return spec, true, err + } + if arg.Kind == OpReg { + if !isX86YrlRegisterForArch(arg.Reg, arch) { + return fail() + } + } else if !isAMD64MemoryOperand(arg) && !(spec.mode == amd64StackPushValue && + (arg.Kind == OpImm || x86AbsoluteRelocationSource(arg, arch, spec.bits))) { + if spec.mode == amd64StackPopValue { + return spec, true, fmt.Errorf("%s %s expects reg/mem dst in the Yrl class: %q", arch, base, ins.Raw) + } + return fail() + } + if arg.Kind == OpMem && !x86MemoryRegistersValidForArch(arg.Mem, arch) { + return fail() + } + return spec, true, nil +} + +func (c *amd64Ctx) lowerStackWidth(op Op, ins Instr) (ok bool, terminated bool, err error) { + if _, known, err := x86StackWidthProperties(ins, c.goarch); known && err != nil { + return true, false, err + } + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64StackWidthSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed form in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + + switch spec.mode { + case amd64StackPushFlags: + if err := c.validateOrdinaryStackWidthMode(baseOp, spec.bits, ins); err != nil { + return true, false, err + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + value := c.packX86Flags() + if spec.bits != 64 { + value = c.truncI64(value, amd64StackLLVMType(spec.bits)) + } + return true, false, c.pushStackWidth(spec.bits, value) + case amd64StackPopFlags: + if err := c.validateOrdinaryStackWidthMode(baseOp, spec.bits, ins); err != nil { + return true, false, err + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.popStackWidth(spec.bits) + if err != nil { + return true, false, err + } + if spec.bits != 64 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", extended, amd64StackLLVMType(spec.bits), value) + value = "%" + extended + } + c.unpackX86Flags(value, spec.bits) + return true, false, nil + case amd64StackPushValue, amd64StackPopValue: + if len(ins.Args) != 1 { + role := "src" + if spec.mode == amd64StackPopValue { + role = "dst" + } + return true, false, fmt.Errorf("%s %s expects %s: %q", c.goarch, baseOp, role, ins.Raw) + } + default: + panic("unknown x86 stack-width mode") + } + + operand := ins.Args[0] + segmentStack := x86SegmentRegisterOperand(operand) + if segmentStack { + if !spec.segments.contains(operand.Reg) { + return true, false, fmt.Errorf("%s %s does not accept segment register %s in Go 1.27's ymovtab: %q", c.goarch, baseOp, operand.Reg, ins.Raw) + } + bits := spec.bits + if spec.segmentNativeWidth { + bits = c.x86NativeStackBits() + } + if spec.mode == amd64StackPushValue { + selector := c.loadX86SegmentSelector(operand.Reg) + value := selector + switch bits { + case 16: + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i32 %s to i16\n", narrow, selector) + value = "%" + narrow + case 32: + case 64: + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, selector) + value = "%" + wide + default: + panic("invalid x86 segment stack width") + } + return true, false, c.pushStackWidth(bits, value) + } + value, err := c.popStackWidth(bits) + if err != nil { + return true, false, err + } + if bits != 16 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to i16\n", narrow, amd64StackLLVMType(bits), value) + value = "%" + narrow + } + return true, false, c.storeX86SegmentSelector(operand.Reg, value) + } + if err := c.validateOrdinaryStackWidthMode(baseOp, spec.bits, ins); err != nil { + return true, false, err + } + if spec.mode == amd64StackPushValue { + if operand.Kind == OpReg { + if !isX86YrlRegisterForArch(operand.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yrl class: %q", c.goarch, baseOp, ins.Raw) + } + } else if operand.Kind != OpImm && !isAMD64MemoryOperand(operand) && + !x86AbsoluteRelocationSource(operand, c.goarch, spec.bits) { + return true, false, fmt.Errorf("%s %s expects Yrl/memory/immediate src: %q", c.goarch, baseOp, ins.Raw) + } + if operand.Kind == OpMem && !x86MemoryRegistersValidForArch(operand.Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s memory source uses an out-of-range address register: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.loadStackValue(operand, spec.bits) + if err != nil { + return true, false, err + } + return true, false, c.pushStackWidth(spec.bits, value) + } + + if operand.Kind == OpReg { + if !isX86YrlRegisterForArch(operand.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(operand) { + return true, false, fmt.Errorf("%s %s expects reg/mem dst in the Yrl class: %q", c.goarch, baseOp, ins.Raw) + } + if operand.Kind == OpMem && !x86MemoryRegistersValidForArch(operand.Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s memory destination uses an out-of-range address register: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.popStackWidth(spec.bits) + if err != nil { + return true, false, err + } + return true, false, c.storeStackWidth(operand, spec.bits, value) +} + +func (c *amd64Ctx) validateOrdinaryStackWidthMode(op string, bits int, ins Instr) error { + return validateX86OrdinaryStackWidthMode(c.goarch, op, bits, ins) +} + +func validateX86OrdinaryStackWidthMode(arch, op string, bits int, ins Instr) error { + if arch == "386" && bits == 64 { + return fmt.Errorf("386 %s requires GOARCH=amd64: %q", op, ins.Raw) + } + if arch != "386" && bits == 32 { + return fmt.Errorf("%s requires GOARCH=386: %q", op, ins.Raw) + } + return nil +} + +func (c *amd64Ctx) x86NativeStackBits() int { + if c.goarch == "386" { + return 32 + } + return 64 +} + +func (set amd64StackSegmentSet) contains(reg Reg) bool { + var bit amd64StackSegmentSet + switch reg { + case ES: + bit = amd64StackSegmentES + case CS: + bit = amd64StackSegmentCS + case SS: + bit = amd64StackSegmentSS + case DS: + bit = amd64StackSegmentDS + case FS: + bit = amd64StackSegmentFS + case GS: + bit = amd64StackSegmentGS + default: + return false + } + return set&bit != 0 +} + +func amd64StackLLVMType(bits int) LLVMType { + switch bits { + case 16: + return I16 + case 32: + return I32 + case 64: + return I64 + default: + panic("invalid x86 stack width") + } +} + +func (c *amd64Ctx) pushStackWidth(bits int, value string) error { + if c.sourceStack != nil { + return c.pushSourceStackValue(amd64StackLLVMType(bits), int64(bits/8), value) + } + if c.goarch == "386" { + switch bits { + case 16: + return c.pushI16(value) + case 32: + return c.pushI32(value) + } + return fmt.Errorf("386 cannot push %d-bit stack values", bits) + } + if bits != 64 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", extended, amd64StackLLVMType(bits), value) + value = "%" + extended + } + c.pushI64(value) + return nil +} + +func (c *amd64Ctx) popStackWidth(bits int) (string, error) { + if c.sourceStack != nil { + return c.popSourceStackValue(amd64StackLLVMType(bits), int64(bits/8)) + } + if c.goarch == "386" { + switch bits { + case 16: + return c.popI16() + case 32: + return c.popI32() + } + return "", fmt.Errorf("386 cannot pop %d-bit stack values", bits) + } + value := c.popI64() + if bits == 64 { + return value, nil + } + return c.truncI64(value, amd64StackLLVMType(bits)), nil +} + +func (c *amd64Ctx) storeStackWidth(destination Operand, bits int, value string) error { + typ := amd64StackLLVMType(bits) + switch destination.Kind { + case OpReg: + return c.storeRegSized(destination.Reg, typ, value) + case OpMem: + ptr, ptrType, err := c.ptrFromMem(destination.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store volatile %s %s, %s %s, align 1\n", typ, value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(destination.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store volatile %s %s, ptr %s, align 1\n", typ, value, ptr) + return nil + case OpFP: + return c.storeFPResult(destination.FPOffset, typ, value) + default: + return fmt.Errorf("%s stack destination is not writable", destination.String()) + } +} diff --git a/amd64_lower_stack_width_test.go b/amd64_lower_stack_width_test.go new file mode 100644 index 000000000..4aff8002c --- /dev/null +++ b/amd64_lower_stack_width_test.go @@ -0,0 +1,267 @@ +package plan9asm + +import ( + "errors" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestAMD64StackWidthGrammarIsComplete(t *testing.T) { + want := map[Op]amd64StackWidthSpec{ + "PUSHW": {bits: 16, mode: amd64StackPushValue, segments: amd64StackPushSegments}, + "POPW": {bits: 16, mode: amd64StackPopValue, segments: amd64StackPopSegments}, + "PUSHL": {bits: 32, mode: amd64StackPushValue, segments: amd64StackPushSegments, segmentNativeWidth: true}, + "POPL": {bits: 32, mode: amd64StackPopValue, segments: amd64StackPopSegments, segmentNativeWidth: true}, + "PUSHQ": {bits: 64, mode: amd64StackPushValue, segments: amd64StackLongSegments, segmentNativeWidth: true}, + "POPQ": {bits: 64, mode: amd64StackPopValue, segments: amd64StackLongSegments, segmentNativeWidth: true}, + "PUSHFW": {bits: 16, mode: amd64StackPushFlags}, + "POPFW": {bits: 16, mode: amd64StackPopFlags}, + "PUSHFL": {bits: 32, mode: amd64StackPushFlags}, + "POPFL": {bits: 32, mode: amd64StackPopFlags}, + "PUSHFQ": {bits: 64, mode: amd64StackPushFlags}, + "POPFQ": {bits: 64, mode: amd64StackPopFlags}, + } + if !reflect.DeepEqual(amd64StackWidthSpecs, want) { + t.Fatalf("stack-width grammar = %+v, want %+v", amd64StackWidthSpecs, want) + } +} + +func TestTranslateX86StackWidthCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + high := "R11" + if target.goarch == "386" { + high = "DX" + } + source := "TEXT stackwidthforms(SB),$0-0\n" + + "\tPUSHW AX\n\tPOPW " + high + "\n" + + "\tPUSHW $61731\n\tPOPW (BX)\n" + + "\tPUSHW (BX)\n\tPOPW AX\n" + + "\tPUSHFW\n\tPOPFW\n" + if target.goarch == "amd64" { + source += "\tPUSHQ R11\n\tPOPQ (BX)\n\tPUSHQ FS\n\tPUSHQ GS\n\tPOPQ FS\n\tPOPQ GS\n\tPUSHFQ\n\tPOPFQ\n" + } else { + source += "\tPUSHL DX\n\tPOPL (BX)\n\tPUSHFL\n\tPOPFL\n" + } + source += "\tRET\n" + + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source, "func stackwidthforms()", target.goarch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original stack format source invented native/segment inputs", api, err) + } + } + ir := x86ScalarFormatObjectUnit(t, file, map[string]FuncSig{"stackwidthforms": {Name: "stackwidthforms", Ret: Void}}, + target.goarch, target.triple, 0) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + }) + } +} + +func TestTranslateX86StackSegmentCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + const source = `TEXT stacksegmentforms(SB),$0-0 + PUSHW CS + PUSHW SS + PUSHW DS + PUSHW ES + PUSHW FS + PUSHW GS + POPW SS + POPW DS + POPW ES + POPW FS + POPW GS + PUSHL CS + PUSHL SS + PUSHL DS + PUSHL ES + PUSHL FS + PUSHL GS + POPL SS + POPL DS + POPL ES + POPL FS + POPL GS + PUSHQ FS + PUSHQ GS + POPQ FS + POPQ GS + POPW DS + POPL DS + RET +` + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source, "func stacksegmentforms()", target.goarch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original segment formats invented physical selector/stack state", api, err) + } + } + ir := x86ScalarFormatObjectUnit(t, file, map[string]FuncSig{"stacksegmentforms": {Name: "stacksegmentforms", Ret: Void}}, + target.goarch, target.triple, 0) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + }) + } +} + +func TestTranslateX86StackWidthRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PUSHW X0"}, + {goarch: "amd64", instruction: "POPW $1"}, + {goarch: "amd64", instruction: "PUSHFW AX"}, + {goarch: "amd64", instruction: "POPQ X0"}, + {goarch: "amd64", instruction: "PUSHL AX"}, + {goarch: "amd64", instruction: "PUSHQ DS"}, + {goarch: "amd64", instruction: "POPQ ES"}, + {goarch: "amd64", instruction: "POPW CS"}, + {goarch: "amd64", instruction: "POPL CS"}, + {goarch: "386", instruction: "PUSHQ AX"}, + {goarch: "386", instruction: "POPW R11"}, + {goarch: "386", instruction: "POPFQ"}, + {goarch: "386", instruction: "PUSHQ DS"}, + {goarch: "386", instruction: "POPQ ES"}, + {goarch: "386", instruction: "POPW CS"}, + {goarch: "386", instruction: "POPL CS"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's stack tables", test.instruction) + } + }) + } +} + +func TestAMD64StackWidthRuntimeValuesAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT stackwidthsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVW $0x1234, AX + PUSHW AX + POPW CX + MOVW CX, 0(DI) + MOVW $0x5678, 2(DI) + PUSHW 2(DI) + POPW DX + MOVW DX, 4(DI) + PUSHW $61731 + POPW 6(DI) + MOVQ $0x123456789abcdef, AX + PUSHQ AX + POPQ 8(DI) + STC + PUSHFW + CLC + POPFW + SETCS 16(DI) + PUSHFQ + CLC + POPFQ + SETCS 17(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"stackwidthsemantics": { + Name: "stackwidthsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void stackwidthsemantics(uint8_t *); +static uint16_t load16(const uint8_t *p) { uint16_t v; memcpy(&v,p,2); return v; } +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint8_t out[24] = {0}; stackwidthsemantics(out); + if (load16(out) != 0x1234 || load16(out+4) != 0x5678 || load16(out+6) != 61731) return 10; + if (load64(out+8) != UINT64_C(0x123456789abcdef)) return 11; + if (out[16] != 1 || out[17] != 1) return 12; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "stack_width_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_string.go b/amd64_lower_string.go index 5ae46f59d..baadf7072 100644 --- a/amd64_lower_string.go +++ b/amd64_lower_string.go @@ -1,50 +1,45 @@ package plan9asm -import "fmt" +import ( + "fmt" +) -func (c *amd64Ctx) lowerString(op Op, ins Instr) (ok bool, terminated bool, err error) { - if c.goarch != "386" { +func (c *amd64Ctx) lowerString(ins Instr) (ok bool, terminated bool, err error) { + form, ok, err := x86StringInstructionProperties(ins, c.goarch) + if !ok { return false, false, nil } - switch op { - case "MOVSB", "MOVSL", "STOSL", "SCASB": - default: - return false, false, nil + if form.kind == "" { + return true, false, err } - + // Preserve the existing consumption order: a malformed opcode suffix has + // not consumed a prefix; mode/operand rejection after a valid head has. prefix := c.repeatPrefix c.repeatPrefix = "" - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) - } - repeated := prefix != "" - if prefix == "REPN" && op != "SCASB" { - return true, false, fmt.Errorf("386 REPN is unsupported for %s", op) - } - if prefix == "REP" && op == "SCASB" { - return true, false, fmt.Errorf("386 REP SCASB is not yet supported") + if err != nil { + return true, false, err } + kind, typ, width := form.kind, form.typ, form.width count := "1" - if repeated { - cx32, err := c.evalIntSized(Operand{Kind: OpReg, Reg: CX}, I32) + if prefix != "" { + cx, err := c.loadReg(CX) if err != nil { return true, false, err } - wide := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, cx32) - count = "%" + wide + if c.goarch == "386" { + cx32 := c.truncI64(cx, I32) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, cx32) + count = "%" + wide + } else { + count = cx + } } direction := c.loadDirectionFlag() - switch op { - case "MOVSB", "MOVSL": - width := 1 - helper := "__plan9asm_movsb" - if op == "MOVSL" { - width = 4 - helper = "__plan9asm_movsl" - } + switch kind { + case "movs": si, err := c.loadReg(SI) if err != nil { return true, false, err @@ -53,7 +48,7 @@ func (c *amd64Ctx) lowerString(op Op, ins Instr) (ok bool, terminated bool, err if err != nil { return true, false, err } - fmt.Fprintf(c.b, " call void @%s(i64 %s, i64 %s, i64 %s, i1 %s)\n", helper, di, si, count, direction) + fmt.Fprintf(c.b, " call void @%s(i64 %s, i64 %s, i64 %s, i1 %s)\n", x86StringHelperName(kind, width), di, si, count, direction) bytes := c.newTmp() fmt.Fprintf(c.b, " %%%s = mul i64 %s, %d\n", bytes, count, width) if err := c.storeStringIndex(SI, si, "%"+bytes, direction); err != nil { @@ -63,67 +58,186 @@ func (c *amd64Ctx) lowerString(op Op, ins Instr) (ok bool, terminated bool, err return true, false, err } - case "STOSL": + case "stos": di, err := c.loadReg(DI) if err != nil { return true, false, err } - ax, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, typ) if err != nil { return true, false, err } - fmt.Fprintf(c.b, " call void @__plan9asm_rep_stosl(i64 %s, i32 %s, i64 %s, i1 %s)\n", di, ax, count, direction) + fmt.Fprintf(c.b, " call void @%s(i64 %s, %s %s, i64 %s, i1 %s)\n", x86StringHelperName(kind, width), di, typ, value, count, direction) bytes := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i64 %s, 4\n", bytes, count) + fmt.Fprintf(c.b, " %%%s = mul i64 %s, %d\n", bytes, count, width) if err := c.storeStringIndex(DI, di, "%"+bytes, direction); err != nil { return true, false, err } - case "SCASB": + case "scas": di, err := c.loadReg(DI) if err != nil { return true, false, err } - needle, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AL}, I8) + needle, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, typ) if err != nil { return true, false, err } + whileEqual := prefix == "REP" + resultType := fmt.Sprintf("{ i64, i64, %s, i1 }", typ) call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call { i64, i64, i1 } @__plan9asm_repne_scasb(i64 %s, i8 %s, i64 %s, i1 %s)\n", call, di, needle, count, direction) + fmt.Fprintf(c.b, " %%%s = call %s @%s(i64 %s, %s %s, i64 %s, i1 %s, i1 %t)\n", call, resultType, x86StringHelperName(kind, width), di, typ, needle, count, direction, whileEqual) next := c.newTmp() remaining := c.newTmp() - equal := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i64, i1 } %%%s, 0\n", next, call) - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i64, i1 } %%%s, 1\n", remaining, call) - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i64, i1 } %%%s, 2\n", equal, call) + lastValue := c.newTmp() + executed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 0\n", next, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 1\n", remaining, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 2\n", lastValue, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 3\n", executed, resultType, call) if err := c.storeReg(DI, "%"+next); err != nil { return true, false, err } - zf := "%" + equal - if repeated { - oldZF := c.loadFlag(c.flagsZSlot) - executed := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %s, 0\n", executed, count) - preserved := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i1 %%%s, i1 %s\n", preserved, executed, equal, oldZF) - zf = "%" + preserved + // SCAS sets all six status flags from accumulator minus the last + // element read, not just the equality flag used to control repetition. + c.setStringCompareFlags(typ, needle, "%"+lastValue, "%"+executed, prefix != "") + if prefix != "" { + if err := c.storeStringCount("%" + remaining); err != nil { + return true, false, err + } } - fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", zf, c.flagsZSlot) - if repeated { - if err := c.storeRegSized(CX, I32, c.truncI64("%"+remaining, I32)); err != nil { + + case "cmps": + si, err := c.loadReg(SI) + if err != nil { + return true, false, err + } + di, err := c.loadReg(DI) + if err != nil { + return true, false, err + } + whileEqual := prefix == "REP" + resultType := fmt.Sprintf("{ i64, i64, i64, %s, %s, i1 }", typ, typ) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(i64 %s, i64 %s, i64 %s, i1 %s, i1 %t)\n", call, resultType, x86StringHelperName(kind, width), si, di, count, direction, whileEqual) + nextSI := c.newTmp() + nextDI := c.newTmp() + remaining := c.newTmp() + left := c.newTmp() + right := c.newTmp() + executed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 0\n", nextSI, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 1\n", nextDI, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 2\n", remaining, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 3\n", left, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 4\n", right, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 5\n", executed, resultType, call) + if err := c.storeReg(SI, "%"+nextSI); err != nil { + return true, false, err + } + if err := c.storeReg(DI, "%"+nextDI); err != nil { + return true, false, err + } + if prefix != "" { + if err := c.storeStringCount("%" + remaining); err != nil { + return true, false, err + } + } + c.setStringCompareFlags(typ, "%"+left, "%"+right, "%"+executed, prefix != "") + + case "lods": + si, err := c.loadReg(SI) + if err != nil { + return true, false, err + } + resultType := fmt.Sprintf("{ i64, i64, %s, i1 }", typ) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(i64 %s, i64 %s, i1 %s)\n", call, resultType, x86StringHelperName(kind, width), si, count, direction) + nextSI := c.newTmp() + remaining := c.newTmp() + loaded := c.newTmp() + executed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 0\n", nextSI, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 1\n", remaining, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 2\n", loaded, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 3\n", executed, resultType, call) + if err := c.storeReg(SI, "%"+nextSI); err != nil { + return true, false, err + } + var oldAccumulator string + if prefix != "" { + old, err := c.loadReg(AX) + if err != nil { + return true, false, err + } + oldAccumulator = old + } + if err := c.storeRegSized(AX, typ, "%"+loaded); err != nil { + return true, false, err + } + if prefix != "" { + written, err := c.loadReg(AX) + if err != nil { + return true, false, err + } + // An empty repeat executes no accumulator write at all. Selecting + // only the low operand width before storeRegSized would still clear + // AMD64's high 32 bits for LODSL even on that empty path. + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %s, i64 %s\n", selected, executed, written, oldAccumulator) + if err := c.storeReg(AX, "%"+selected); err != nil { + return true, false, err + } + if err := c.storeStringCount("%" + remaining); err != nil { return true, false, err } } } - if repeated && op != "SCASB" { - if err := c.storeRegSized(CX, I32, "0"); err != nil { + if prefix != "" && kind != "scas" && kind != "cmps" { + if err := c.storeStringCount("0"); err != nil { return true, false, err } } return true, false, nil } +func x86StringHelperName(kind string, width int) string { + suffix := map[int]string{1: "b", 2: "w", 4: "l", 8: "q"}[width] + if kind == "movs" { + return "__plan9asm_movs" + suffix + } + return "__plan9asm_rep_" + kind + suffix +} + +func (c *amd64Ctx) setStringCompareFlags(typ LLVMType, left, right, executed string, preserveEmpty bool) { + flags := c.x86StatusFlagSlots() + old := make([]string, len(flags)) + if preserveEmpty { + for index, flag := range flags { + old[index] = c.loadFlag(flag.slot) + } + } + c.setCmpFlagsSized(typ, left, right) + if preserveEmpty { + // REP/REPN with a zero initial count performs no comparison and + // preserves all six status flags. The helper also performs no load. + for index, flag := range flags { + current := c.loadFlag(flag.slot) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i1 %s, i1 %s\n", selected, executed, current, old[index]) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", selected, flag.slot) + } + } +} + +func (c *amd64Ctx) storeStringCount(value string) error { + if c.goarch == "386" { + return c.storeRegSized(CX, I32, c.truncI64(value, I32)) + } + return c.storeRegSized(CX, I64, value) +} + func (c *amd64Ctx) loadDirectionFlag() string { t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load i1, ptr %s\n", t, c.directionSlot) diff --git a/amd64_lower_string_compare.go b/amd64_lower_string_compare.go new file mode 100644 index 000000000..754321be4 --- /dev/null +++ b/amd64_lower_string_compare.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedStringCompareSpec struct { + explicitLength bool + maskResult bool + vector bool +} + +// amd64PackedStringCompareSpecs models Go 1.27's complete SSE4.2 packed-string +// grammar as three independent axes: explicit/implicit lengths, index/mask +// result, and legacy/VEX encoding. Every spelling shares the same imm8 and +// X/m128 operand grammar and the same six architectural flag results. +var amd64PackedStringCompareSpecs = map[Op]amd64PackedStringCompareSpec{ + "PCMPESTRI": {explicitLength: true}, + "PCMPESTRM": {explicitLength: true, maskResult: true}, + "PCMPISTRI": {}, + "PCMPISTRM": {maskResult: true}, + "VPCMPESTRI": {explicitLength: true, vector: true}, + "VPCMPESTRM": {explicitLength: true, maskResult: true, vector: true}, + "VPCMPISTRI": {vector: true}, + "VPCMPISTRM": {maskResult: true, vector: true}, +} + +func (c *amd64Ctx) lowerPackedStringCompare(op Op, ins Instr) (ok bool, terminated bool, err error) { + spec, recognized, err := amd64PackedStringCompareProperties(op, ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + source, destination := ins.Args[1], ins.Args[2] + left, err := c.loadX(destination.Reg) + if err != nil { + return true, false, err + } + right, err := c.loadPackedStringCompareBytes(source) + if err != nil { + return true, false, err + } + lengthLeft, lengthRight := "", "" + if spec.explicitLength { + lengthLeft, err = c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) + if err != nil { + return true, false, err + } + lengthRight, err = c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I32) + if err != nil { + return true, false, err + } + } + imm := uint8(ins.Args[0].Imm) + stem := "pcmpi" + if spec.explicitLength { + stem = "pcmpe" + } + + output, width, zeroUpper := spec.resultEffect() + if spec.maskResult { + result := c.emitPackedStringCompareIntrinsic(stem+"strm128", "<16 x i8>", left, right, lengthLeft, lengthRight, imm) + if zeroUpper { + err = c.storePackedMoveOperand(Operand{Kind: OpReg, Reg: output}, width, result) + } else { + // The widest shared bank keeps legacy writes limited to low X0. + err = c.storeX(output, result) + } + if err != nil { + return true, false, err + } + } else { + result := c.emitPackedStringCompareIntrinsic(stem+"stri128", "i32", left, right, lengthLeft, lengthRight, imm) + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", extended, result) + if err := c.storeReg(output, "%"+extended); err != nil { + return true, false, err + } + } + + carry := c.emitPackedStringCompareFlag(stem+"stric128", left, right, lengthLeft, lengthRight, imm) + overflow := c.emitPackedStringCompareFlag(stem+"strio128", left, right, lengthLeft, lengthRight, imm) + sign := c.emitPackedStringCompareFlag(stem+"stris128", left, right, lengthLeft, lengthRight, imm) + zero := c.emitPackedStringCompareFlag(stem+"striz128", left, right, lengthLeft, lengthRight, imm) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", carry, c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", zero, c.flagsZSlot) + // This slot stores architectural SF. Signed conditions apply SF XOR OF + // when observing it; precomputing that condition would corrupt PUSHF/LAHF. + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", sign, c.flagsSltSlot) + c.clearX86StatusFlags(x86StatusPF | x86StatusAF) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", overflow, c.flagsOFSlot) + return true, false, nil +} + +// Intel's packed-string exception row permits unaligned m128, but neither +// explicit lengths nor dead results suppress its full observable memory read. +// Logical FP is its existing separate field role; all native addresses use an +// unaligned volatile tuple, without dereferenceability or extent assumptions. +func (c *amd64Ctx) loadPackedStringCompareBytes(source Operand) (string, error) { + if source.Kind == OpReg || source.Kind == OpFP { + return c.loadPackedCompareBytes(source, 16) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile <16 x i8>, %s %s, align 1\n", value, pointerType, pointer) + return "%" + value, nil +} + +// This descriptor is syntax and architectural effects, not caller definedness +// or address permission. Unregistered source-stack consumers remain Context. +func (spec amd64PackedStringCompareSpec) resultEffect() (Reg, int, bool) { + if spec.maskResult { + return "X0", 16, spec.vector + } + return CX, 4, false +} + +func amd64PackedStringCompareProperties(op Op, ins Instr, goarch string) (amd64PackedStringCompareSpec, bool, error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64PackedStringCompareSpecs[Op(baseOp)] + if !recognized { + return spec, false, nil + } + if rawOp != baseOp || goarch != "amd64" && goarch != "386" { + return spec, true, fmt.Errorf("%s %s has no suffixed or non-x86 form in Go: %q", goarch, baseOp, ins.Raw) + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != rawOp { + return spec, true, fmt.Errorf("packed-string dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + minimumImmediate := int64(0) + if spec.vector { + // _yvaeskeygenassist has both Yu8 and Yi8 rows, while the legacy + // yxshuf table has only Yu8. + minimumImmediate = -128 + } + if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[0].ImmIsFloat || ins.Args[0].ImmRaw != "" || + ins.Args[0].Imm < minimumImmediate || ins.Args[0].Imm > 255 { + return spec, true, fmt.Errorf("%s %s immediate is outside its Go byte domain, or operands are not imm8, X/m128, X: %q", goarch, baseOp, ins.Raw) + } + + source, destination := ins.Args[1], ins.Args[2] + if destination.Kind != OpReg || !amd64PackedStringCompareRegister(spec, destination, goarch) { + return spec, true, fmt.Errorf("%s %s destination is outside its Go X register class: %q", goarch, baseOp, ins.Raw) + } + if source.Kind == OpReg { + if !amd64PackedStringCompareRegister(spec, source, goarch) { + return spec, true, fmt.Errorf("%s %s source is outside its Go X register class: %q", goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return spec, true, fmt.Errorf("%s %s source must be X or m128: %q", goarch, baseOp, ins.Raw) + } + if !spec.vector && source.Kind == OpMem && !x86MemoryRegistersValidForArch(source.Mem, goarch) { + return spec, true, fmt.Errorf("%s %s memory source uses a register outside the legacy address class: %q", goarch, baseOp, ins.Raw) + } + ins.Op = op + return spec, true, x86Mode32VectorRegisterContext(ins, goarch) +} + +func amd64PackedStringCompareRegister(spec amd64PackedStringCompareSpec, operand Operand, goarch string) bool { + if spec.vector { + return amd64VEXVectorRegister(operand, 16) + } + index, valid := amd64ParseXReg(operand.Reg) + limit := 16 + if goarch == "386" { + limit = 8 + } + return operand.Kind == OpReg && valid && index < limit +} + +func (c *amd64Ctx) emitPackedStringCompareIntrinsic(name, resultType, left, right, lengthLeft, lengthRight string, imm uint8) string { + result := c.newTmp() + if lengthLeft == "" { + fmt.Fprintf(c.b, " %%%s = call %s @llvm.x86.sse42.%s(<16 x i8> %s, <16 x i8> %s, i8 %d)\n", + result, resultType, name, left, right, imm) + } else { + fmt.Fprintf(c.b, " %%%s = call %s @llvm.x86.sse42.%s(<16 x i8> %s, i32 %s, <16 x i8> %s, i32 %s, i8 %d)\n", + result, resultType, name, left, lengthLeft, right, lengthRight, imm) + } + return "%" + result +} + +func (c *amd64Ctx) emitPackedStringCompareFlag(name, left, right, lengthLeft, lengthRight string, imm uint8) string { + result := c.emitPackedStringCompareIntrinsic(name, "i32", left, right, lengthLeft, lengthRight, imm) + flag := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i32 %s to i1\n", flag, result) + return "%" + flag +} diff --git a/amd64_lower_string_compare_test.go b/amd64_lower_string_compare_test.go new file mode 100644 index 000000000..583c69481 --- /dev/null +++ b/amd64_lower_string_compare_test.go @@ -0,0 +1,253 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +type packedStringCompareTestSpec struct { + op string + explicit bool + mask bool + vector bool +} + +func go127PackedStringCompareSpecs() []packedStringCompareTestSpec { + return []packedStringCompareTestSpec{ + {op: "PCMPESTRI", explicit: true}, + {op: "PCMPESTRM", explicit: true, mask: true}, + {op: "PCMPISTRI"}, + {op: "PCMPISTRM", mask: true}, + {op: "VPCMPESTRI", explicit: true, vector: true}, + {op: "VPCMPESTRM", explicit: true, mask: true, vector: true}, + {op: "VPCMPISTRI", vector: true}, + {op: "VPCMPISTRM", mask: true, vector: true}, + } +} + +func TestAMD64PackedStringCompareGrammarCoversCompleteGoFamily(t *testing.T) { + expected := go127PackedStringCompareSpecs() + if len(amd64PackedStringCompareSpecs) != len(expected) { + t.Fatalf("packed-string grammar has %d entries, want %d", len(amd64PackedStringCompareSpecs), len(expected)) + } + for _, want := range expected { + got, ok := amd64PackedStringCompareSpecs[Op(want.op)] + if !ok { + t.Errorf("packed-string grammar omitted %s", want.op) + continue + } + if got.explicitLength != want.explicit || got.maskResult != want.mask || got.vector != want.vector { + t.Errorf("packed-string grammar %s = %+v, want explicit=%v mask=%v vector=%v", want.op, got, want.explicit, want.mask, want.vector) + } + } +} + +func TestTranslateX86PackedStringCompareCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedstringcompareforms(SB),$0-0\n") + for index, spec := range go127PackedStringCompareSpecs() { + last := 15 + if target.goarch == "386" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $0, X1, X%d\n", spec.op, last) + fmt.Fprintf(&source, "\t%s $255, %d(AX), X%d\n", spec.op, index*16, last) + if spec.vector { + fmt.Fprintf(&source, "\t%s $-128, X1, X%d\n", spec.op, last) + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"packedstringcompareforms": {Name: "packedstringcompareforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, intrinsic := range []string{ + "@llvm.x86.sse42.pcmpestri128", "@llvm.x86.sse42.pcmpestrm128", + "@llvm.x86.sse42.pcmpistri128", "@llvm.x86.sse42.pcmpistrm128", + "@llvm.x86.sse42.pcmpestric128", "@llvm.x86.sse42.pcmpistriz128", + } { + if !strings.Contains(ir, intrinsic) { + t.Fatalf("translation omitted %s:\n%s", intrinsic, ir) + } + } + wantFeatures := `"target-features"="+avx,+sse4.2"` + if !strings.Contains(ir, wantFeatures) { + t.Fatalf("translation omitted %s:\n%s", wantFeatures, ir) + } + compileLLVMToObject(t, llc, target.triple, "packed-string-compare-"+target.name+".ll", "packed-string-compare-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedStringCompareRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PCMPESTRI $-1, X0, X1"}, + {goarch: "amd64", instruction: "PCMPISTRM $256, X0, X1"}, + {goarch: "amd64", instruction: "VPCMPESTRI.Z $7, X0, X1"}, + {goarch: "amd64", instruction: "VPCMPESTRI $-129, X0, X1"}, + {goarch: "amd64", instruction: "PCMPISTRI $7, Y0, X1"}, + {goarch: "amd64", instruction: "VPCMPISTRM $7, X0, X16"}, + {goarch: "386", instruction: "PCMPESTRM $7, X0, X8"}, + {goarch: "386", instruction: "VPCMPESTRM $7, X0, X16"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", ",", "", "$", "", ".", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed-string tables", test.instruction) + } + }) + } +} + +func TestAMD64PackedStringCompareRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT packedstringcomparesemantics(SB),NOSPLIT,$0-40 + MOVQ left+0(FP), SI + MOVOU (SI), X1 + MOVQ right+8(FP), DI + MOVOU (DI), X2 + MOVQ out+16(FP), BX + MOVQ lenLeft+24(FP), AX + MOVQ lenRight+32(FP), DX + + PCMPESTRI $12, X2, X1 + MOVQ CX, 0(BX) + SETCS 8(BX) + SETEQ 9(BX) + SETLT 10(BX) + SETOS 11(BX) + SETPS 12(BX) + + PCMPESTRM $64, X2, X1 + MOVOU X0, 16(BX) + SETCS 32(BX) + SETEQ 33(BX) + SETLT 34(BX) + SETOS 35(BX) + SETPS 36(BX) + + VPCMPISTRI $0, X2, X1 + MOVQ CX, 40(BX) + SETCS 48(BX) + SETEQ 49(BX) + SETLT 50(BX) + SETOS 51(BX) + SETPS 52(BX) + + VPCMPISTRM $64, X2, X1 + MOVOU X0, 56(BX) + SETCS 72(BX) + SETEQ 73(BX) + SETLT 74(BX) + SETOS 75(BX) + SETPS 76(BX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"packedstringcomparesemantics": { + Name: "packedstringcomparesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void packedstringcomparesemantics(const uint8_t *, const uint8_t *, uint8_t *, uint64_t, uint64_t); +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v; } +static int flags_equal(const uint8_t *p, int c, int z, int s, int o) { + return p[0] == c && p[1] == z && p[2] == (s ^ o) && p[3] == o && p[4] == 0; +} +__attribute__((target("sse4.2"))) static int check(void) { + const uint8_t left[16] = {'a','b','c','d','e',0,'x','y','z',0,1,2,3,4,5,6}; + const uint8_t right[16] = {'q','q','a','b','c','d','e',0,'x',0,9,8,7,6,5,4}; + const int la = 5, lb = 9; + uint8_t out[80] = {0}, want[16]; + __m128i a = _mm_loadu_si128((const __m128i *)left); + __m128i b = _mm_loadu_si128((const __m128i *)right); + packedstringcomparesemantics(left, right, out, la, lb); + if (load64(out) != (uint64_t)_mm_cmpestri(a, la, b, lb, 12)) return 10; + if (!flags_equal(out + 8, _mm_cmpestrc(a,la,b,lb,12), _mm_cmpestrz(a,la,b,lb,12), _mm_cmpestrs(a,la,b,lb,12), _mm_cmpestro(a,la,b,lb,12))) return 11; + _mm_storeu_si128((__m128i *)want, _mm_cmpestrm(a,la,b,lb,64)); + if (memcmp(out + 16, want, 16)) return 12; + if (!flags_equal(out + 32, _mm_cmpestrc(a,la,b,lb,64), _mm_cmpestrz(a,la,b,lb,64), _mm_cmpestrs(a,la,b,lb,64), _mm_cmpestro(a,la,b,lb,64))) return 13; + if (load64(out + 40) != (uint64_t)_mm_cmpistri(a,b,0)) return 14; + if (!flags_equal(out + 48, _mm_cmpistrc(a,b,0), _mm_cmpistrz(a,b,0), _mm_cmpistrs(a,b,0), _mm_cmpistro(a,b,0))) return 15; + _mm_storeu_si128((__m128i *)want, _mm_cmpistrm(a,b,64)); + if (memcmp(out + 56, want, 16)) return 16; + if (!flags_equal(out + 72, _mm_cmpistrc(a,b,64), _mm_cmpistrz(a,b,64), _mm_cmpistrs(a,b,64), _mm_cmpistro(a,b,64))) return 17; + return 0; +} +int main(void) { return check(); } +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_string_compare_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_subvec.go b/amd64_lower_subvec.go new file mode 100644 index 000000000..4d92f7c80 --- /dev/null +++ b/amd64_lower_subvec.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedSubtractMode = x86PackedArithmeticMode + +const ( + amd64PackedSubtractWrap = x86PackedWrap + amd64PackedSubtractSignedSaturating = x86PackedSignedSaturating + amd64PackedSubtractUnsignedSaturating = x86PackedUnsignedSaturating +) + +// lowerPackedIntegerSubtract implements the complete Go 1.27 packed-integer +// subtract family: legacy yxm XMM forms and the shared _yvandnpd VEX/EVEX +// forms. Go's 386 frontend accepts low-bank X/Y/Z three-operand forms but +// rejects explicit source masks; high banks retain their physical Context. +func (c *amd64Ctx) lowerPackedIntegerSubtract(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, _, _, known := amd64PackedSubtractProperties(base); !known { + return false, false, nil + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed subtract dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, known, err := x86PackedSubtractProperties(ins, c.goarch) + if err != nil || !known { + return known, false, err + } + first := ins.Args[form.first] + if form.legacy && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed subtract needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + bits := form.laneBits + mnemonic := "" + if form.legacy && form.memory >= 0 { + mnemonic = strings.ToLower(base) + if base == "PSUBL" { + mnemonic = "psubd" + } + } + err = c.emitPackedBinaryE4(ins, form.x86PackedBinaryForm, bits, mnemonic, + func(lanes, bits int, first, second string) string { + return c.emitPackedIntegerSubtract(lanes, bits, first, second, form.mode) + }) + return true, false, err +} + +func amd64PackedSubtractProperties(op string) (int, amd64PackedSubtractMode, bool, bool) { + spec, known := x86PackedArithmeticSpecs[Op(op)] + if !known || spec.family != x86PackedArithmeticSubtract { + return 0, 0, false, false + } + return spec.laneBits, spec.mode, spec.encoding == x86PackedLogicalVEXOrEVEX, true +} + +// emitPackedIntegerSubtract computes second-first, matching Go assembler's +// rm,v,destination operand order. +func (c *amd64Ctx) emitPackedIntegerSubtract(lanes, laneBits int, first, second string, mode amd64PackedSubtractMode) string { + vecType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + if mode == amd64PackedSubtractWrap { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %s, %s\n", result, vecType, second, first) + return "%" + result + } + + wideBits := laneBits * 2 + wideType := fmt.Sprintf("<%d x i%d>", lanes, wideBits) + extend := "sext" + if mode == amd64PackedSubtractUnsignedSaturating { + extend = "zext" + } + firstWide := c.newTmp() + secondWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", firstWide, extend, vecType, first, wideType) + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", secondWide, extend, vecType, second, wideType) + difference := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %%%s, %%%s\n", difference, wideType, secondWide, firstWide) + + clamped := "%" + difference + minValue := int64(0) + if mode == amd64PackedSubtractSignedSaturating { + minValue = -int64(uint64(1) << (laneBits - 1)) + } + below := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, %s\n", below, wideType, clamped, llvmSplatSignedInteger(lanes, wideBits, minValue)) + boundedBelow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", boundedBelow, lanes, below, wideType, llvmSplatSignedInteger(lanes, wideBits, minValue), wideType, clamped) + clamped = "%" + boundedBelow + if mode == amd64PackedSubtractSignedSaturating { + maxValue := int64((uint64(1) << (laneBits - 1)) - 1) + above := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt %s %s, %s\n", above, wideType, clamped, llvmSplatSignedInteger(lanes, wideBits, maxValue)) + boundedAbove := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", boundedAbove, lanes, above, wideType, llvmSplatSignedInteger(lanes, wideBits, maxValue), wideType, clamped) + clamped = "%" + boundedAbove + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", result, wideType, clamped, vecType) + return "%" + result +} diff --git a/amd64_lower_svm.go b/amd64_lower_svm.go new file mode 100644 index 000000000..365790d55 --- /dev/null +++ b/amd64_lower_svm.go @@ -0,0 +1,86 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerX86SystemManagement implements AMD's fixed no-operand 0F 01 D8-DF +// family and Intel's VMCALL. Their architectural operands are implicit: +// VMRUN/VMLOAD/VMSAVE use rAX, SKINIT uses EAX, INVLPGA uses rAX and ECX, +// and VMCALL/VMMCALL follow the common +// KVM hypercall convention with rAX as the result and rBX/rCX/rDX/rSI as input +// arguments. The remaining instructions have no general-register operands. +func (c *amd64Ctx) lowerX86SystemManagement(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + switch baseOp { + case "VMCALL", "VMRUN", "VMMCALL", "VMLOAD", "VMSAVE", "STGI", "CLGI", "SKINIT", "INVLPGA": + // handled below + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + + wordType := I64 + if c.goarch == "386" { + wordType = I32 + } + loadWordReg := func(reg Reg) (string, error) { + return c.evalIntSized(Operand{Kind: OpReg, Reg: reg}, wordType) + } + clobbers := "~{dirflag},~{fpsr},~{flags},~{memory}" + switch baseOp { + case "STGI", "CLGI": + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", strings.ToLower(baseOp), clobbers) + case "VMRUN", "VMLOAD", "VMSAVE": + ax, err := loadWordReg(AX) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s)\n", strings.ToLower(baseOp), "{ax},"+clobbers, wordType, ax) + case "SKINIT": + ax, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect \"skinit\", %q(i32 %s)\n", "{ax},"+clobbers, ax) + case "INVLPGA": + ax, err := loadWordReg(AX) + if err != nil { + return true, false, err + } + cx, err := c.evalIntSized(Operand{Kind: OpReg, Reg: CX}, I32) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect \"invlpga\", %q(%s %s, i32 %s)\n", "{ax},{cx},"+clobbers, wordType, ax, cx) + case "VMCALL", "VMMCALL": + regs := []Reg{AX, BX, CX, DX, SI} + args := make([]string, len(regs)) + for i, reg := range regs { + value, err := loadWordReg(reg) + if err != nil { + return true, false, err + } + args[i] = value + } + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %s, %s %s, %s %s, %s %s, %s %s)\n", + call, wordType, strings.ToLower(baseOp), "={ax},0,{bx},{cx},{dx},{si},"+clobbers, + wordType, args[0], wordType, args[1], wordType, args[2], wordType, args[3], wordType, args[4], + ) + if err := c.storeRegSized(AX, wordType, "%"+call); err != nil { + return true, false, err + } + } + return true, false, nil +} diff --git a/amd64_lower_syscall.go b/amd64_lower_syscall.go index 8fa1ce4ad..56981d916 100644 --- a/amd64_lower_syscall.go +++ b/amd64_lower_syscall.go @@ -7,6 +7,21 @@ import ( func (c *amd64Ctx) lowerSyscall(op Op, ins Instr) (ok bool, terminated bool, err error) { switch op { + case "WAIT": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("x86 WAIT expects no operands: %q", ins.Raw) + } + c.b.WriteString(" call void asm sideeffect \"wait\", \"~{memory}\"()\n") + return true, false, nil + case "HLT": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("x86 HLT expects no operands: %q", ins.Raw) + } + c.b.WriteString(" call void asm sideeffect \"hlt\", \"~{memory}\"()\n") + c.b.WriteString(" unreachable\n") + return true, true, nil + case "UD2": + return c.lowerX86UndefinedTrap(op, ins) case "INT": if c.goarch == "386" && (len(ins.Args) != 1 || ins.Args[0].Kind != OpImm) { return true, false, fmt.Errorf("386 INT expects an immediate vector: %q", ins.Raw) @@ -27,56 +42,31 @@ func (c *amd64Ctx) lowerSyscall(op Op, ins Instr) (ok bool, terminated bool, err } return true, true, nil case "SYSCALL": - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("amd64 SYSCALL expects no operands: %q", ins.Raw) - } - num, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - a1, err := c.loadReg(DI) - if err != nil { - return true, false, err + source := ins + if source.Op == "" { + source.Op = op } - a2, err := c.loadReg(SI) - if err != nil { - return true, false, err + effect, known, err := x86LinuxSyscallByteEffect(source, c.goarch, c.targetTriple) + if !known || err != nil { + return true, false, fmt.Errorf("%w: raw SYSCALL lacks its closed source mode/form/target: %v", ErrProbeNeedsContext, err) } - a3, err := c.loadReg(DX) - if err != nil { - return true, false, err - } - a4, err := c.loadReg(Reg("R10")) - if err != nil { - return true, false, err - } - a5, err := c.loadReg(Reg("R8")) - if err != nil { - return true, false, err - } - a6, err := c.loadReg(Reg("R9")) - if err != nil { - return true, false, err - } - - r := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @syscall(i64 %s, i64 %s, i64 %s, i64 %s, i64 %s, i64 %s, i64 %s)\n", r, num, a1, a2, a3, a4, a5, a6) - isErr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, -1\n", isErr, r) - errno32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @cliteErrno()\n", errno32) - errno64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", errno64, errno32) - negErr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 0, %%%s\n", negErr, errno64) - rax := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 %%%s\n", rax, isErr, negErr, r) - if err := c.storeReg(AX, "%"+rax); err != nil { - return true, false, err + arguments := make([]string, 0, 7) + for _, operand := range effect.operands { + if !operand.read { + continue + } + value, err := c.loadReg(operand.operand.Reg) + if err != nil { + return true, false, err + } + arguments = append(arguments, "i64 "+value) } - // Linux raw syscall ABI writes r2 in DX. libc syscall provides only one - // return value, so keep DX at zero. - if err := c.storeReg(DX, "0"); err != nil { + // The shared ordered input roles match Linux's AX/DI/SI/DX/R10/R8/R9. + // Preserve the raw -errno result and DX; libc must not change TLS errno. + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i64 asm sideeffect %q, %q(%s)\n", result, "syscall", + "={ax},0,{di},{si},{dx},{r10},{r8},{r9},~{rcx},~{r11},~{dirflag},~{fpsr},~{flags},~{memory}", strings.Join(arguments, ", ")) + if err := c.storeReg(AX, "%"+result); err != nil { return true, false, err } return true, false, nil diff --git a/amd64_lower_system_address.go b/amd64_lower_system_address.go new file mode 100644 index 000000000..c5ed43fab --- /dev/null +++ b/amd64_lower_system_address.go @@ -0,0 +1,134 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SystemAddressKind uint8 + +const ( + amd64SystemAddressInvalidatePage amd64SystemAddressKind = iota + amd64SystemAddressInvalidatePCID + amd64SystemAddressDemoteCacheLine +) + +type amd64SystemAddressSpec struct { + kind amd64SystemAddressKind + memoryBytes int + register bool +} + +// amd64SystemAddressSpecs is the complete Go 1.27 address-shaped system +// instruction grammar. The single-memory and memory-plus-register rows share +// memory/address validation; register compatibility closures are represented +// explicitly and trap just as their architecturally invalid ModRM encodings do. +var amd64SystemAddressSpecs = map[Op]amd64SystemAddressSpec{ + "INVLPG": {kind: amd64SystemAddressInvalidatePage, memoryBytes: 1}, + "INVPCID": {kind: amd64SystemAddressInvalidatePCID, memoryBytes: 16, register: true}, + "CLDEMOTE": {kind: amd64SystemAddressDemoteCacheLine, memoryBytes: 1}, +} + +type amd64SystemAddressForm struct { + memory Operand + register Reg + invalidModRM bool +} + +func (c *amd64Ctx) lowerSystemAddress(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64SystemAddressSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + form, err := c.parseSystemAddressForm(baseOp, spec, ins) + if err != nil { + return true, false, err + } + return c.emitSystemAddress(baseOp, spec, form) +} + +func (c *amd64Ctx) parseSystemAddressForm(op string, spec amd64SystemAddressSpec, ins Instr) (amd64SystemAddressForm, error) { + var form amd64SystemAddressForm + wantArgs := 1 + if spec.register { + wantArgs = 2 + } + if len(ins.Args) != wantArgs { + return form, fmt.Errorf("%s %s expects %d operand(s): %q", c.goarch, op, wantArgs, ins.Raw) + } + address := ins.Args[0] + if isAMD64MemoryOperand(address) { + if address.Kind == OpMem && !x86MemoryRegistersValidForArch(address.Mem, c.goarch) { + return form, fmt.Errorf("%s %s address uses an out-of-range register: %q", c.goarch, op, ins.Raw) + } + form.memory = address + } else if address.Kind == OpReg && spec.kind != amd64SystemAddressDemoteCacheLine { + valid := isX86YrlRegisterForArch(address.Reg, c.goarch) + if spec.kind == amd64SystemAddressInvalidatePage { + valid = isGoYmbRegisterForArch(address.Reg, c.goarch) + } + if !valid { + return form, fmt.Errorf("%s %s first operand is outside Go 1.27's compatibility class: %q", c.goarch, op, ins.Raw) + } + form.invalidModRM = true + } else { + return form, fmt.Errorf("%s %s expects Go 1.27 memory/address form: %q", c.goarch, op, ins.Raw) + } + if spec.register { + typeOperand := ins.Args[1] + if typeOperand.Kind != OpReg || !isX86YrlRegisterForArch(typeOperand.Reg, c.goarch) { + return form, fmt.Errorf("%s %s type operand must be a Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + form.register = typeOperand.Reg + } + return form, nil +} + +func (c *amd64Ctx) emitSystemAddress(op string, spec amd64SystemAddressSpec, form amd64SystemAddressForm) (bool, bool, error) { + if form.invalidModRM { + c.b.WriteString(" call void asm sideeffect \"ud2\", \"~{memory}\"()\n") + c.b.WriteString(" unreachable\n") + return true, true, nil + } + pointer, pointerType, err := c.x86DescriptorMemoryPointer(form.memory) + if err != nil { + return true, false, err + } + clobbers := "~{memory},~{dirflag},~{fpsr},~{flags}" + if spec.kind != amd64SystemAddressInvalidatePCID { + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype([%d x i8]) %s)\n", + strings.ToLower(op)+" $0", "*m,"+clobbers, pointerType, spec.memoryBytes, pointer) + return true, false, nil + } + + if c.goarch == "386" { + // INVPCID is a 64-bit-mode instruction, but Go 1.27 accepts it for + // 386. Reproduce Go's 66 0F 38 82 /r encoding with the calculated + // address fixed in EBX and the type fixed in EAX. In 32-bit mode the + // bytes fault architecturally, but remain object-level compatible. + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint %s %s to i32\n", address, pointerType, pointer) + typeValue, err := c.evalIntSized(Operand{Kind: OpReg, Reg: form.register}, I32) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(i32 %%%s, i32 %s)\n", + ".byte 0x66, 0x0f, 0x38, 0x82, 0x03", "{bx},{ax},"+clobbers, address, typeValue) + return true, false, nil + } + typeValue, err := c.evalIntSized(Operand{Kind: OpReg, Reg: form.register}, I64) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype([16 x i8]) %s, i64 %s)\n", + "invpcid $0, $1", "*m,r,"+clobbers, pointerType, pointer, typeValue) + return true, false, nil +} diff --git a/amd64_lower_system_address_test.go b/amd64_lower_system_address_test.go new file mode 100644 index 000000000..c18ffd917 --- /dev/null +++ b/amd64_lower_system_address_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64SystemAddressGrammarIsComplete(t *testing.T) { + want := map[Op]amd64SystemAddressSpec{ + "INVLPG": {kind: amd64SystemAddressInvalidatePage, memoryBytes: 1}, + "INVPCID": {kind: amd64SystemAddressInvalidatePCID, memoryBytes: 16, register: true}, + "CLDEMOTE": {kind: amd64SystemAddressDemoteCacheLine, memoryBytes: 1}, + } + if !reflect.DeepEqual(amd64SystemAddressSpecs, want) { + t.Fatalf("system-address grammar = %+v, want %+v", amd64SystemAddressSpecs, want) + } +} + +func TestTranslateX86SystemAddressCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + base := "BX" + if target.goarch == "amd64" { + base = "R11" + } + source := "TEXT systemaddressforms(SB),$0-0\n" + + "\tINVLPG 8(" + base + ")\n" + + "\tCLDEMOTE 16(" + base + ")\n" + + "\tINVPCID 32(" + base + "), AX\n" + + "\tRET\n" + + "TEXT invlpgcompat(SB),$0-0\n\tINVLPG AX\n" + + "TEXT invpcidcompat(SB),$0-0\n\tINVPCID AX, BX\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "systemaddressforms": {Name: "systemaddressforms", Ret: Void}, + "invlpgcompat": {Name: "invlpgcompat", Ret: Void}, + "invpcidcompat": {Name: "invpcidcompat", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"target-features"="+cldemote,+invpcid"`, `asm sideeffect "ud2"`, "unreachable"} { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if target.goarch == "386" && !strings.Contains(ir, ".byte 0x66, 0x0f, 0x38, 0x82, 0x03") { + t.Errorf("386 IR is missing Go-compatible INVPCID encoding:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "system-address-"+target.name+".ll", "system-address-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SystemAddressRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "INVLPG X1"}, + {goarch: "amd64", instruction: "INVLPG.Z (BX)"}, + {goarch: "amd64", instruction: "CLDEMOTE AX"}, + {goarch: "amd64", instruction: "CLDEMOTE (BX), (CX)"}, + {goarch: "amd64", instruction: "INVPCID (BX), X1"}, + {goarch: "amd64", instruction: "INVPCID X1, BX"}, + {goarch: "amd64", instruction: "INVPCID (BX)"}, + {goarch: "386", instruction: "INVLPG (R11)"}, + {goarch: "386", instruction: "CLDEMOTE (R11)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's system-address tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_system_control_test.go b/amd64_lower_system_control_test.go new file mode 100644 index 000000000..e6d4c560e --- /dev/null +++ b/amd64_lower_system_control_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86SystemControlLRETRawAndWAIT(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, test := range []struct { + name string + source string + want string + }{ + { + name: "lret", + source: "TEXT lret(SB),NOSPLIT,$0-0\n\tBYTE $0x48; BYTE $0xcb\n", + want: `asm sideeffect ".byte 0x48, 0xcb"`, + }, + { + name: "wait", + source: "TEXT wait(SB),NOSPLIT,$0-0\n\tWAIT\n\tRET\n", + want: `asm sideeffect "wait"`, + }, + } { + t.Run(test.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, test.source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{test.name: {Name: test.name, Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, test.want) { + t.Fatalf("translation omitted %q:\n%s", test.want, ir) + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "x86-system-control-"+test.name+".ll", "x86-system-control-"+test.name+".o", ir) + }) + } +} diff --git a/amd64_lower_system_transfer.go b/amd64_lower_system_transfer.go new file mode 100644 index 000000000..149b571ef --- /dev/null +++ b/amd64_lower_system_transfer.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SystemTransferKind uint8 + +const ( + amd64SystemEnter amd64SystemTransferKind = iota + amd64SystemExit + amd64SystemReturn +) + +type amd64SystemTransferInputs uint8 + +const ( + amd64SystemTransferCX amd64SystemTransferInputs = 1 << iota + amd64SystemTransferDX + amd64SystemTransferR11 +) + +type amd64SystemTransferSpec struct { + kind amd64SystemTransferKind + mode64 bool + inputs amd64SystemTransferInputs +} + +// amd64SystemTransferSpecs is the complete Go 1.27 SYSENTER/SYSEXIT/SYSRET +// grammar. SYSCALL is intentionally separate because plan9asm provides a +// hosted libc-backed syscall implementation for it. These raw architectural +// transfers retain exact Go bytes and explicitly materialize the GP registers +// consumed by their exit/return conventions. +var amd64SystemTransferSpecs = map[Op]amd64SystemTransferSpec{ + "SYSENTER": {kind: amd64SystemEnter}, + "SYSENTER64": {kind: amd64SystemEnter, mode64: true}, + "SYSEXIT": {kind: amd64SystemExit, inputs: amd64SystemTransferCX | amd64SystemTransferDX}, + "SYSEXIT64": {kind: amd64SystemExit, mode64: true, inputs: amd64SystemTransferCX | amd64SystemTransferDX}, + "SYSRET": {kind: amd64SystemReturn, inputs: amd64SystemTransferCX | amd64SystemTransferR11}, +} + +func (spec amd64SystemTransferSpec) terminates() bool { + return spec.kind != amd64SystemEnter +} + +func (spec amd64SystemTransferSpec) encoding() string { + var opcode byte + switch spec.kind { + case amd64SystemEnter: + opcode = 0x34 + case amd64SystemExit: + opcode = 0x35 + case amd64SystemReturn: + opcode = 0x07 + default: + panic("unknown amd64 system-transfer kind") + } + if spec.mode64 { + return fmt.Sprintf(".byte 0x48, 0x0f, 0x%02x", opcode) + } + return fmt.Sprintf(".byte 0x0f, 0x%02x", opcode) +} + +func (c *amd64Ctx) lowerSystemTransfer(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64SystemTransferSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.mode64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + constraints := make([]string, 0, 7) + arguments := make([]string, 0, 3) + for _, input := range []struct { + bit amd64SystemTransferInputs + reg Reg + constraint string + }{ + {bit: amd64SystemTransferCX, reg: CX, constraint: "{cx}"}, + {bit: amd64SystemTransferDX, reg: DX, constraint: "{dx}"}, + {bit: amd64SystemTransferR11, reg: Reg("R11"), constraint: "{r11}"}, + } { + if spec.inputs&input.bit == 0 || (c.goarch == "386" && input.bit == amd64SystemTransferR11) { + continue + } + typ := I32 + if spec.mode64 || spec.kind == amd64SystemReturn { + typ = I64 + } + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: input.reg}, typ) + if err != nil { + return true, false, err + } + constraints = append(constraints, input.constraint) + arguments = append(arguments, fmt.Sprintf("%s %s", typ, value)) + } + constraints = append(constraints, "~{memory}", "~{dirflag}", "~{fpsr}", "~{flags}") + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s)\n", + spec.encoding(), strings.Join(constraints, ","), strings.Join(arguments, ", ")) + if !spec.terminates() { + return true, false, nil + } + c.b.WriteString(" unreachable\n") + return true, true, nil +} diff --git a/amd64_lower_system_transfer_test.go b/amd64_lower_system_transfer_test.go new file mode 100644 index 000000000..58823da96 --- /dev/null +++ b/amd64_lower_system_transfer_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64SystemTransferGrammarIsComplete(t *testing.T) { + want := map[Op]amd64SystemTransferSpec{ + "SYSENTER": {kind: amd64SystemEnter}, + "SYSENTER64": {kind: amd64SystemEnter, mode64: true}, + "SYSEXIT": {kind: amd64SystemExit, inputs: amd64SystemTransferCX | amd64SystemTransferDX}, + "SYSEXIT64": {kind: amd64SystemExit, mode64: true, inputs: amd64SystemTransferCX | amd64SystemTransferDX}, + "SYSRET": {kind: amd64SystemReturn, inputs: amd64SystemTransferCX | amd64SystemTransferR11}, + } + if !reflect.DeepEqual(amd64SystemTransferSpecs, want) { + t.Fatalf("system-transfer grammar = %+v, want %+v", amd64SystemTransferSpecs, want) + } +} + +func TestTranslateX86SystemTransferCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT sysenterform(SB),$0-0 + SYSENTER + RET +TEXT sysexitform(SB),$0-0 + SYSEXIT +TEXT sysretform(SB),$0-0 + SYSRET +` + sigs := map[string]FuncSig{ + "sysenterform": {Name: "sysenterform", Ret: Void}, + "sysexitform": {Name: "sysexitform", Ret: Void}, + "sysretform": {Name: "sysretform", Ret: Void}, + } + if target.goarch == "amd64" { + source += `TEXT sysenter64form(SB),$0-0 + SYSENTER64 + RET +TEXT sysexit64form(SB),$0-0 + SYSEXIT64 +` + sigs["sysenter64form"] = FuncSig{Name: "sysenter64form", Ret: Void} + sigs["sysexit64form"] = FuncSig{Name: "sysexit64form", Ret: Void} + } + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `.byte 0x0f, 0x34`, + `.byte 0x0f, 0x35`, + `.byte 0x0f, 0x07`, + `{cx},{dx},~{memory}`, + } { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + wantUnreachable := 2 + if target.goarch == "amd64" { + wantUnreachable = 3 + for _, want := range []string{ + `.byte 0x48, 0x0f, 0x34`, + `.byte 0x48, 0x0f, 0x35`, + `{cx},{r11},~{memory}`, + } { + if !strings.Contains(ir, want) { + t.Errorf("amd64 IR is missing %s:\n%s", want, ir) + } + } + } + if got := strings.Count(ir, "unreachable"); got != wantUnreachable { + t.Errorf("unreachable count = %d, want %d", got, wantUnreachable) + } + compileLLVMToObject(t, llc, target.triple, "system-transfer-"+target.name+".ll", "system-transfer-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SystemTransferRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "SYSENTER AX"}, + {goarch: "amd64", instruction: "SYSENTER64.P"}, + {goarch: "amd64", instruction: "SYSEXIT AX"}, + {goarch: "amd64", instruction: "SYSEXIT64 AX"}, + {goarch: "amd64", instruction: "SYSRET AX"}, + {goarch: "386", instruction: "SYSENTER64"}, + {goarch: "386", instruction: "SYSEXIT64"}, + {goarch: "386", instruction: "SYSRET.P"}, + } { + t.Run(test.goarch+"_"+strings.ReplaceAll(test.instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's system-transfer tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_ternary_logic.go b/amd64_lower_ternary_logic.go new file mode 100644 index 000000000..53df324d9 --- /dev/null +++ b/amd64_lower_ternary_logic.go @@ -0,0 +1,114 @@ +package plan9asm + +import "fmt" + +// Both Go _yvalignd opcodes have the same forms; D/Q changes mask lanes and +// the scalar broadcast tuple, not the bitwise Boolean truth table. +var amd64TernaryLogicSpecs = map[Op]int{"VPTERNLOGD": 32, "VPTERNLOGQ": 64} + +func (c *amd64Ctx) lowerTernaryLogic(op Op, ins Instr) (bool, bool, error) { + form, recognized, err := x86TernaryLogicProperties(op, ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + if form.memory >= 0 && ins.Args[form.memory].Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(ins.Args[form.memory].FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + dependencies := form.dependencies() + if form.mask >= 0 && !form.zeroing { + dependencies |= 1 + } + inputs := [3]string{"zeroinitializer", "zeroinitializer", "zeroinitializer"} + for role, argument := range form.inputs { + if argument == form.memory { + // E4 reads are observable even if the truth table ignores C. + // Only active mask lanes suppress faults; never LLVM DCE. + // Intel SDM Vol.2A 2.8.4, Tables 2-43/2-49 (E4, not E4NF): + // https://cdrdv2-public.intel.com/789573/252046-sdm-change-document.pdf + lanes, err := c.loadX86PackedShiftLanes(ins.Args[argument], form.byteWidth, form.laneBits, form.broadcast, mask) + if err != nil { + return true, false, err + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, form.byteWidth*8/form.laneBits, form.laneBits, lanes, form.byteWidth) + inputs[role] = "%" + bytes + } else if dependencies>>uint(role)&1 != 0 { + inputs[role], err = c.loadPackedCompareBytes(ins.Args[argument], form.byteWidth) + if err != nil { + return true, false, err + } + } + } + result := c.emitTernaryLogicTerms(form.byteWidth, form.terms, inputs) + if form.mask >= 0 { + lanes := form.byteWidth * 8 / form.laneBits + computed := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.laneBits, result) + old := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.laneBits, inputs[0]) + masked := amd64ApplyIntegerLaneMask(c, lanes, form.laneBits, computed, old, mask, form.zeroing) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, form.laneBits, masked, form.byteWidth) + result = "%" + bytes + } + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, result) +} + +// Normalization removes irrelevant inputs and aliases before LLVM sees undef. +func (c *amd64Ctx) emitTernaryLogicBytes(width int, immediate uint8, a, b, third string) string { + return c.emitTernaryLogicTerms(width, x86TernaryLogicTerms(immediate, [3]int{0, 1, 2}), [3]string{a, b, third}) +} + +func (c *amd64Ctx) emitTernaryLogicTerms(width int, terms uint8, inputs [3]string) string { + result := "" + for subset := 0; subset < 8; subset++ { + if terms>>uint(subset)&1 == 0 { + continue + } + term := "" + for role, input := range inputs { + if subset>>uint(role)&1 == 0 { + continue + } + if term == "" { + term = input + continue + } + name := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and <%d x i8> %s, %s\n", name, width, term, input) + term = "%" + name + } + if term == "" { + term = llvmAllOnesI8Vec(width) + } + if result == "" { + result = term + continue + } + name := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor <%d x i8> %s, %s\n", name, width, result, term) + result = "%" + name + } + if result == "" { + return "zeroinitializer" + } + return result +} + +func amd64TernaryLogicBit(immediate uint8, a, b, third bool) bool { + index := 0 + for _, value := range []bool{a, b, third} { + index <<= 1 + if value { + index++ + } + } + return immediate>>uint(index)&1 != 0 +} diff --git a/amd64_lower_unary_imm_float.go b/amd64_lower_unary_imm_float.go new file mode 100644 index 000000000..6edc76ce1 --- /dev/null +++ b/amd64_lower_unary_imm_float.go @@ -0,0 +1,109 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64UnaryImmediateFloatingShape struct { + laneBits int + scalar bool + twoSource bool +} + +type amd64UnaryImmediateFloatingForm struct { + properties amd64BinaryFloatingSuffix + source Operand + passthrough Operand + destination Operand + byteWidth int + mask string + immediate uint8 +} + +// parseUnaryImmediateFloatingForm models Go 1.27's related unary-immediate +// and FIXUPIMM operand grammars. Semantic families supply only their lane +// width, packed/scalar shape, and whether packed forms have two sources; +// width, memory, BCST, SAE, mask, zeroing, and 386 frontend constraints are +// validated once here. +func (c *amd64Ctx) parseUnaryImmediateFloatingForm(baseOp, suffix string, shape amd64UnaryImmediateFloatingShape, ins Instr) (amd64UnaryImmediateFloatingForm, error) { + var form amd64UnaryImmediateFloatingForm + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.rounding != "" { + return form, fmt.Errorf("%s %s has a suffix absent from Go 1.27's unary-immediate floating optabs: %q", c.goarch, baseOp, ins.Raw) + } + if shape.scalar && properties.broadcast { + return form, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + hasPassthrough := shape.scalar || shape.twoSource + wantArgs := 3 + if hasPassthrough { + wantArgs = 4 + } + if len(ins.Args) != wantArgs && len(ins.Args) != wantArgs+1 { + return form, fmt.Errorf("%s %s has the wrong operand count for its packed/scalar grammar: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return form, fmt.Errorf("%s %s immediate must be an unsigned byte: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == wantArgs+1 + if c.goarch == "386" && (hasPassthrough || masked) { + return form, fmt.Errorf("386 %s form exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return form, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + maskIndex := len(ins.Args) - 2 + if masked && !amd64NonzeroKOperand(ins.Args[maskIndex]) { + return form, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return form, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + passthrough := Operand{} + if shape.scalar { + passthrough = ins.Args[2] + if byteWidth != 16 || !c.isGoEVEXVectorRegister(destination, 16) || !c.isGoEVEXVectorRegister(passthrough, 16) { + return form, fmt.Errorf("%s %s scalar pass-through and destination must be X registers: %q", c.goarch, baseOp, ins.Raw) + } + } else { + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return form, fmt.Errorf("%s %s packed destination must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + if shape.twoSource { + passthrough = ins.Args[2] + if !c.isGoEVEXVectorRegister(passthrough, byteWidth) { + return form, fmt.Errorf("%s %s second packed source must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } + } + source := ins.Args[1] + if properties.broadcast { + if !isAMD64MemoryOperand(source) { + return form, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, byteWidth) { + return form, fmt.Errorf("%s %s source register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return form, fmt.Errorf("%s %s source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if properties.sae && (source.Kind != OpReg || !shape.scalar && byteWidth != 64) { + return form, fmt.Errorf("%s %s.SAE requires a register source and scalar or Z form: %q", c.goarch, baseOp, ins.Raw) + } + mask := "" + if masked { + var err error + mask, err = c.loadK(ins.Args[maskIndex].Reg) + if err != nil { + return form, err + } + } + return amd64UnaryImmediateFloatingForm{ + properties: properties, source: source, passthrough: passthrough, + destination: destination, byteWidth: byteWidth, mask: mask, + immediate: uint8(ins.Args[0].Imm), + }, nil +} diff --git a/amd64_lower_unpack.go b/amd64_lower_unpack.go new file mode 100644 index 000000000..8a5229d0b --- /dev/null +++ b/amd64_lower_unpack.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// One complete grammar is consumed by lowering and caller byte proofs. +// Interleaving is bitwise, including floating spellings: it never changes MXCSR. +func (c *amd64Ctx) lowerPackedUnpack(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := x86PackedUnpackSpecs[Op(base)]; !recognized { + return false, false, nil + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed unpack dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := x86PackedUnpackProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy unpack m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + laneBits := form.spec.laneBits + lanes := form.byteWidth * 8 / laneBits + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + if form.mmx { + firstBits, err := c.loadPackedUnpackMMXSource(first, form.memoryBytes) + if err != nil { + return true, false, err + } + secondBits, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + a := c.bitcastI64ToIntegerLanes(lanes, laneBits, firstBits) + b := c.bitcastI64ToIntegerLanes(lanes, laneBits, secondBits) + result := c.emitPackedUnpackLanes(form.byteWidth, laneBits, form.spec.high, a, b) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", bits, lanes, laneBits, result) + return true, false, c.storeReg(destination.Reg, "%"+bits) + } + // Intel SDM Vol.2A Tables 2-44/45 classify these EVEX permutations as + // E4NF, not E4. A mask (even all zero) is not memory-fault suppression. + // Do not reuse the logical family's masked-memory reader here. + b, err := c.loadPackedCompareLanes(second, form.byteWidth, laneBits, false) + if err != nil { + return true, false, err + } + var result string + if form.legacy && form.memory >= 0 { + mnemonic := strings.ToLower(base) + switch base { + case "PUNPCKLWL": + mnemonic = "punpcklwd" + case "PUNPCKHWL": + mnemonic = "punpckhwd" + case "PUNPCKLLQ": + mnemonic = "punpckldq" + case "PUNPCKHLQ": + mnemonic = "punpckhdq" + } + result, err = c.emitPackedBinaryNativeMemory(first, laneBits, b, mnemonic) + } else { + var a string + a, err = c.loadPackedUnpackSource(first, form) + if err == nil { + result = c.emitPackedUnpackLanes(form.byteWidth, laneBits, form.spec.high, a, b) + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old, err := c.loadPackedCompareLanes(destination, form.byteWidth, laneBits, false) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, laneBits, result, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) loadPackedUnpackSource(source Operand, form x86PackedUnpackForm) (string, error) { + if source.Kind == OpReg || source.Kind == OpFP { + return c.loadPackedCompareLanes(source, form.byteWidth, form.spec.laneBits, form.broadcast) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + value := c.newTmp() + bits := form.spec.laneBits + lanes := form.byteWidth * 8 / bits + // An E4NF access is mandatory even when every output lane is masked off + // or observes only part of the source. A plain LLVM load can disappear. + if form.broadcast { + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %s, align 1\n", value, bits, pointerType, pointer) + return amd64SplatInteger(c, lanes, bits, "%"+value), nil + } + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i%d>, %s %s, align 1\n", value, lanes, bits, pointerType, pointer) + return "%" + value, nil +} + +func (c *amd64Ctx) loadPackedUnpackMMXSource(source Operand, memoryBytes int) (string, error) { + if source.Kind == OpFP { + bytes, err := c.loadFPVectorBytes(source.FPOffset, memoryBytes) + if err != nil { + return "", err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", bits, memoryBytes, bytes, 8*memoryBytes) + if memoryBytes == 8 { + return "%" + bits, nil + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, bits) + return "%" + wide, nil + } + if memoryBytes == 0 { + return c.loadLegacyMMXPackedSource(source) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %s, align 1\n", value, 8*memoryBytes, pointerType, pointer) + if memoryBytes == 8 { + return "%" + value, nil + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, value) + return "%" + wide, nil +} + +// emitPackedUnpackLanes interleaves the second (vvvv/old destination) source +// with the first (r/m) source independently within each 128-bit vector lane. +func (c *amd64Ctx) emitPackedUnpackLanes(byteWidth, laneBits int, high bool, first, second string) string { + lanes := byteWidth * 8 / laneBits + groupBits := 128 + if byteWidth == 8 { + groupBits = 64 + } + groupLanes := groupBits / laneBits + half := groupLanes / 2 + start := 0 + if high { + start = half + } + mask := make([]string, 0, lanes) + for group := 0; group < lanes/groupLanes; group++ { + base := group * groupLanes + for lane := start; lane < start+half; lane++ { + index := base + lane + mask = append(mask, fmt.Sprintf("i32 %d", index), fmt.Sprintf("i32 %d", lanes+index)) + } + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %s, <%d x i%d> %s, <%d x i32> <%s>\n", + result, lanes, laneBits, second, lanes, laneBits, first, lanes, strings.Join(mask, ", ")) + return "%" + result +} diff --git a/amd64_lower_unpack_float_test.go b/amd64_lower_unpack_float_test.go new file mode 100644 index 000000000..a11c2cd65 --- /dev/null +++ b/amd64_lower_unpack_float_test.go @@ -0,0 +1,108 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86FloatingUnpackCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, zLast, second, destination := 15, 22, 20, 21 + if target.goarch == "386" { + legacyLast, zLast = 7, 7 + second, destination = 6, 7 + } + var source strings.Builder + source.WriteString("TEXT floatingunpackforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"UNPCKLPS", "UNPCKHPS", "UNPCKLPD", "UNPCKHPD"} { + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + vector := "V" + op + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", vector, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s%d, %s%d\n", vector, width, second, width, destination) + fmt.Fprintf(&source, "\t%s.BCST 8(AX), %s%d, %s%d\n", vector, width, second, width, destination) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", vector, width, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z 8(AX), %s21, K7, %s22\n", vector, width, width) + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, Z%d\n", vector, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", vector, zLast) + fmt.Fprintf(&source, "\t%s.BCST 8(AX), Z6, Z%d\n", vector, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K2, Z22\n", vector) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z21, K6, Z22\n", vector) + } + } + source.WriteString("\tRET\n") + + ll := x86UnboundVoidScalarFormatObject(t, source.String(), "floatingunpackforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "floating-unpack-"+target.name+".ll", "floating-unpack-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86FloatingUnpackRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "UNPCKLPS.Z X0, X1", + "UNPCKHPD X0", + "UNPCKLPD M0, M1", + "UNPCKHPS Y0, X1", + "UNPCKLPS X0, (AX)", + "VUNPCKLPD X0, X1", + "VUNPCKHPS X0, Y1, Y2", + "VUNPCKLPD X0, X1, (AX)", + "VUNPCKHPD X0, X1, K0, X2", + "VUNPCKLPS.Z X0, X1, X2", + "VUNPCKHPS.BCST X0, X1, X2", + "VUNPCKLPD.BCST 8(AX), X1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86FloatingUnpackRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "UNPCKLPS X8, X0", + "UNPCKHPD X0, X8", + "VUNPCKLPD Z8, Z1, Z2", + "VUNPCKHPS Z0, Z1, Z8", + "VUNPCKLPS X0, X1, K1, X2", + "VUNPCKHPD.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86FloatingUnpackRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86FloatingUnpackRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's floating unpack forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_variable_shift.go b/amd64_lower_variable_shift.go new file mode 100644 index 000000000..71b38a901 --- /dev/null +++ b/amd64_lower_variable_shift.go @@ -0,0 +1,52 @@ +package plan9asm + +import "fmt" + +type amd64PerLaneVariableShiftSpec struct { + laneBits int + operation string + arithmetic bool + // Shared raw grammar: 66.0F38 opcode and optional VEX generation. + // W is fixed: W0 for dwords, W1 for words/qwords. + opcode byte + vex bool +} + +var amd64PerLaneVariableShiftSpecs = map[string]amd64PerLaneVariableShiftSpec{ + "VPSLLVW": {laneBits: 16, operation: "shl", opcode: 0x12}, + "VPSLLVD": {laneBits: 32, operation: "shl", opcode: 0x47, vex: true}, + "VPSLLVQ": {laneBits: 64, operation: "shl", opcode: 0x47, vex: true}, + "VPSRLVW": {laneBits: 16, operation: "lshr", opcode: 0x10}, + "VPSRLVD": {laneBits: 32, operation: "lshr", opcode: 0x45, vex: true}, + "VPSRLVQ": {laneBits: 64, operation: "lshr", opcode: 0x45, vex: true}, + "VPSRAVW": {laneBits: 16, operation: "ashr", arithmetic: true, opcode: 0x11}, + "VPSRAVD": {laneBits: 32, operation: "ashr", arithmetic: true, opcode: 0x46, vex: true}, + "VPSRAVQ": {laneBits: 64, operation: "ashr", arithmetic: true, opcode: 0x46}, +} + +// lowerPerLaneVariableShift implements the full Go 1.27 per-lane packed +// variable-shift family. The first Plan 9 source supplies one count per lane. +func (c *amd64Ctx) lowerPerLaneVariableShift(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerX86PackedShift(op, ins) +} + +func (c *amd64Ctx) emitPerLaneVariableShift(spec amd64PerLaneVariableShiftSpec, lanes int, counts, data string) string { + typeName := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + limit := amd64SplatInteger(c, lanes, spec.laneBits, fmt.Sprintf("%d", spec.laneBits)) + inRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %s\n", inRange, typeName, counts, limit) + fallbackCount := "zeroinitializer" + if spec.arithmetic { + fallbackCount = amd64SplatInteger(c, lanes, spec.laneBits, fmt.Sprintf("%d", spec.laneBits-1)) + } + safeCounts := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", safeCounts, lanes, inRange, typeName, counts, typeName, fallbackCount) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s, %%%s\n", shifted, spec.operation, typeName, data, safeCounts) + if spec.arithmetic { + return "%" + shifted + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %%%s, %s zeroinitializer\n", result, lanes, inRange, typeName, shifted, typeName) + return "%" + result +} diff --git a/amd64_lower_variable_shift_test.go b/amd64_lower_variable_shift_test.go new file mode 100644 index 000000000..1a902b768 --- /dev/null +++ b/amd64_lower_variable_shift_test.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PerLaneVariableShiftCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 exposes the W/D/Q per-lane left, logical-right, and + // arithmetic-right shifts. Every opcode has X/Y/Z rows and optional K + // masking/.Z; D/Q additionally enable scalar count broadcast. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ, dataReg, dstReg := 20, 20, 21 + if target.goarch == "386" { + lastZ, dataReg, dstReg = 7, 6, 7 + } + var source strings.Builder + source.WriteString("TEXT perlanevariableshiftforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{ + "VPSLLVW", "VPSLLVD", "VPSLLVQ", + "VPSRLVW", "VPSRLVD", "VPSRLVQ", + "VPSRAVW", "VPSRAVD", "VPSRAVQ", + } { + fmt.Fprintf(&source, "\t%s X1, X%d, X%d\n", op, dataReg, dstReg) + fmt.Fprintf(&source, "\t%s 8(AX), Y%d, Y%d\n", op, dataReg, dstReg) + fmt.Fprintf(&source, "\t%s Z1, Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X20, K1, X21\n", op) + fmt.Fprintf(&source, "\t%s.Z Y1, Y20, K2, Y21\n", op) + fmt.Fprintf(&source, "\t%s.Z 40(AX), Z20, K3, Z21\n", op) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + fmt.Fprintf(&source, "\t%s.BCST 104(AX), X%d, X%d\n", op, dataReg, dstReg) + fmt.Fprintf(&source, "\t%s.BCST 112(AX), Y%d, Y%d\n", op, dataReg, dstReg) + fmt.Fprintf(&source, "\t%s.BCST 120(AX), Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z 128(AX), Z20, K4, Z21\n", op) + } + } + } + source.WriteString("\tRET\n") + assembleX87ControlBytes(t, target.goarch, strings.ReplaceAll(source.String(), "NOSPLIT", "4")) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + options := Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "perlanevariableshiftforms": {Name: "perlanevariableshiftforms", Ret: Void}, + }, + } + for _, api := range []string{"text", "module", "Go"} { + original := strings.Replace(source.String(), "perlanevariableshiftforms(SB)", "·view(SB)", 1) + if ir, err := x86ScalarFPBitViewAPI(t, original, "func view()", target.goarch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized original formats acquired numerical/native inputs", api, err) + } + } + ll := x86InstructionFormatObjectUnit(t, file, options.Sigs, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerPerLaneVariableShift(ins.Op, ins) + }) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "per-lane-variable-shift-"+target.name+".ll", "per-lane-variable-shift-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PerLaneVariableShiftRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VPSLLVQ X0, X1", + "VPSLLVQ $1, X1, X2", + "VPSLLVQ X0, Y1, Y2", + "VPSLLVQ X0, (AX), X2", + "VPSLLVQ X0, X1, K0, X2", + "VPSLLVQ X0, X1, AX", + "VPSLLVQ.Z X0, X1, X2", + "VPSLLVQ.BCST X0, X1, X2", + "VPSLLVW.BCST (AX), X1, X2", + "VPSLLVQ.Z.BCST (AX), X1, K1, X2", + "VPSLLVQ.RN_SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PerLaneVariableShiftRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86PerLaneVariableShiftRejected(t, "386", "i386-unknown-linux-gnu", "VPSLLVQ X0, X1, K1, X2") + assertX86PerLaneVariableShiftRejected(t, "386", "i386-unknown-linux-gnu", "VPSLLVQ Z0, Z1, Z8") +} + +func assertX86PerLaneVariableShiftRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's per-lane variable shift forms", instruction) + } +} diff --git a/amd64_lower_vcomi.go b/amd64_lower_vcomi.go new file mode 100644 index 000000000..c2a59488f --- /dev/null +++ b/amd64_lower_vcomi.go @@ -0,0 +1,83 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerVectorScalarFlagCompare(op Op, ins Instr) (ok bool, terminated bool, err error) { + name, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86COMISpecs[Op(name)]; !known { + return false, false, nil + } + if !strings.EqualFold(string(op), string(ins.Op)) { + return true, false, fmt.Errorf("scalar flag comparison dispatch/source mismatch") + } + form, known, err := x86COMIProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: scalar flag comparison requires its native x86 MXCSR target", ErrProbeNeedsContext) + } + if ins.x86AddressBits != 0 { + return true, false, fmt.Errorf("%w: scalar flag comparison needs original address-size binding", ErrProbeNeedsContext) + } + return true, false, c.emitNativeCOMI(ins, form) +} + +// A relation-only LLVM fcmp neither distinguishes COMI/UCOMI signaling nor +// preserves exceptions after FLAGS die. Keep one whole native comparison and +// capture CF/ZF/PF before compiler-generated instructions can overwrite them. +func (c *amd64Ctx) emitNativeCOMI(ins Instr, form x86COMIForm) error { + destination, err := c.evalScalarFloatForInteger(ins.Args[1], form.spec.bits) + if err != nil { + return err + } + typ := string(amd64FloatingTypeForBits(form.spec.bits)) + registerConstraint := "x" + if form.evex { + registerConstraint = "v" + } + source := ins.Args[0] + var value, sourceType, constraint, operand string + if source.Kind == OpReg || source.Kind == OpFP { + value, err = c.evalScalarFloatForInteger(source, form.spec.bits) + sourceType, constraint, operand = typ, registerConstraint, "$3" + } else { + var pointerType string + value, pointerType, _, err = c.packedVectorMemoryPointer(source) + sourceType = fmt.Sprintf("%s elementtype([%d x i8])", pointerType, form.spec.bits/8) + constraint, operand = "*m", "$3" + if pointerType == "ptr addrspace(256)" { + operand = "%gs:" + operand + } else if pointerType == "ptr addrspace(257)" { + operand = "%fs:" + operand + } + } + if err != nil { + return err + } + assembly := strings.ToLower(form.name) + if form.evex { + assembly = "{evex} " + assembly + } + if form.sae { + assembly += " {sae}," + } + assembly += " " + operand + ", $4; setb $0; sete $1; setp $2" + result := c.newTmp() + constraints := "=q,=q,=q," + constraint + "," + registerConstraint + ",~{flags},~{fpsr},~{memory}" + fmt.Fprintf(c.b, " %%%s = call { i8, i8, i8 } asm sideeffect %q, %q(%s %s, %s %s)\n", + result, assembly, constraints, sourceType, value, typ, destination) + for index, slot := range []string{c.flagsCFSlot, c.flagsZSlot, c.flagsPFSlot} { + byteValue, boolean := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { i8, i8, i8 } %%%s, %d\n", byteValue, result, index) + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %%%s, 0\n", boolean, byteValue) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", boolean, slot) + } + for _, slot := range []string{c.flagsSltSlot, c.flagsOFSlot, c.flagsAFSlot} { + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", slot) + } + return nil +} diff --git a/amd64_lower_vcvtss2sd_test.go b/amd64_lower_vcvtss2sd_test.go new file mode 100644 index 000000000..3cd8859be --- /dev/null +++ b/amd64_lower_vcvtss2sd_test.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateAMD64VCVTSS2SDCompleteGo127Forms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT vcvtss2sdforms(SB),$0-0\n") + for _, op := range []string{"VCVTSS2SD", "VCVTSD2SS"} { + fmt.Fprintf(&source, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&source, "\t%s (AX), X3, X4\n", op) + fmt.Fprintf(&source, "\t%s X31, X30, X29\n", op) + if op == "VCVTSS2SD" { + fmt.Fprintf(&source, "\t%s.SAE X31, X30, X29\n", op) + } else { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s X31, X30, X29\n", op, rounding) + } + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X31, X30, K1, X29\n", op) + fmt.Fprintf(&source, "\t%s.Z (AX), X30, K2, X29\n", op) + if op == "VCVTSS2SD" { + fmt.Fprintf(&source, "\t%s.SAE.Z X31, X30, K3, X29\n", op) + } else { + fmt.Fprintf(&source, "\t%s.RD_SAE.Z X31, X30, K3, X29\n", op) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86ScalarPrecisionOriginalFormatObject(t, source.String(), "vcvtss2sdforms", target.goarch, target.triple, false) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "vcvtss2sd-"+target.name+".ll", "vcvtss2sd-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ScalarPrecisionConvertRejectsFormsOutsideGo127(t *testing.T) { + tests := []struct { + goarch string + triple string + instruction string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSS2SD.RD_SAE X0, X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSD2SS.SAE X0, X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSS2SD.SAE (AX), X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSD2SS.RN_SAE (AX), X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSS2SD.Z X0, X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSD2SS X0, X1, K0, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSS2SD Y0, X1, X2"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "VCVTSD2SS X0, X1, K1, X2"}, + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar precision-conversion forms for %s", test.instruction, test.goarch) + } + }) + } +} + +func qualifyAMD64ScalarPrecisionConvertRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT scalarPrecisionConvertSemantics(SB),NOSPLIT,$0-48 + MOVQ out+0(FP), AX + MOVQ doubles+8(FP), BX + MOVQ floats+16(FP), CX + MOVQ upper+24(FP), DX + MOVQ old+32(FP), SI + KMOVQ mask+40(FP), K1 + MOVSD (BX), X0 + MOVUPS (DX), X1 + MOVUPS (SI), X2 + VCVTSD2SS.RN_SAE X0, X1, X2 + MOVUPS X2, 0(AX) + VCVTSD2SS.RD_SAE X0, X1, X2 + MOVUPS X2, 16(AX) + VCVTSD2SS.RU_SAE X0, X1, X2 + MOVUPS X2, 32(AX) + VCVTSD2SS.RZ_SAE X0, X1, X2 + MOVUPS X2, 48(AX) + MOVUPS (SI), X2 + VCVTSD2SS.RU_SAE X0, X1, K1, X2 + MOVUPS X2, 64(AX) + MOVUPS (SI), X2 + VCVTSD2SS.RU_SAE.Z X0, X1, K1, X2 + MOVUPS X2, 80(AX) + MOVSS (CX), X0 + VCVTSS2SD.SAE X0, X1, X2 + MOVUPS X2, 96(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "scalarPrecisionConvertSemantics": {Name: "scalarPrecisionConvertSemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void scalarPrecisionConvertSemantics(uint32_t *, const double *, const float *, const uint32_t *, const uint32_t *, uint64_t); +int main(void) { + double input = 1.0 + 0x1p-24; + float input32 = 1.5f; + uint32_t upper[4] = {0xaaaaaaaaU, 0x11111111U, 0x22222222U, 0x33333333U}; + uint32_t old[4] = {0xdeadbeefU, 0xbbbbbbbbU, 0xccccccccU, 0xddddddddU}; + uint32_t out[28] = {0}; + scalarPrecisionConvertSemantics(out, &input, &input32, upper, old, 0); + uint32_t low[4] = {0x3f800000U, 0x3f800000U, 0x3f800001U, 0x3f800000U}; + for (int group = 0; group < 4; group++) { + if (out[group*4] != low[group]) return 1 + group; + for (int lane = 1; lane < 4; lane++) if (out[group*4+lane] != upper[lane]) return 10 + group*4 + lane; + } + if (out[16] != old[0]) return 40; + if (out[20] != 0) return 41; + for (int lane = 1; lane < 4; lane++) { + if (out[16+lane] != upper[lane]) return 42 + lane; + if (out[20+lane] != upper[lane]) return 46 + lane; + } + uint64_t converted = 0, want = 0x3ff8000000000000ULL; + memcpy(&converted, &out[24], sizeof(converted)); + if (converted != want) return 60; + if (out[26] != upper[2] || out[27] != upper[3]) return 61; + return 0; +} +` + x86OriginalFloatingRuntimeRequireCPU(t, "scalar-precision", clang, triple, runPrefix) + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_precision_convert", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_vec.go b/amd64_lower_vec.go index ebfb7a52f..d469d3299 100644 --- a/amd64_lower_vec.go +++ b/amd64_lower_vec.go @@ -2,126 +2,192 @@ package plan9asm import ( "fmt" + "strconv" "strings" ) func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err error) { + if known, term, err := c.lowerSHA(op, ins); known { + return known, term, err + } + if known, term, err := c.lowerSqrt(op, ins); known { + return known, term, err + } + if known, term, err := c.lowerLegacyPackedMove(op, ins); known { + return known, term, err + } + if ok, term, err := c.lowerFloatArithmetic(op, ins, ""); ok { + return ok, term, err + } + if ok, terminated, err := c.lowerVectorZero(op, ins); ok { + return ok, terminated, err + } + if ok, term, err := c.lowerInsertExtractFamily(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedLogicalFamily(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFourSource(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedStringCompare(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerExplicitMaskMove(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPermute128(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerIndexedPermute(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerQwordBitLookup(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerMaskVectorMove(op, ins); ok { + return ok, term, err + } + rawOp := op + if ok, term, err := c.lowerSingleSourceIntegerNarrow(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedIntegerInsert(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerScalarFloatBroadcast(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerAES(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedWordShuffle(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedByteShuffle(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedBitCount(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedConflict(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedExpand(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerGFNI(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedCompress(rawOp, ins); ok { + return ok, term, err + } + zeroMasking := false if raw := string(op); strings.Contains(raw, ".") { + for _, suffix := range strings.Split(raw, ".")[1:] { + if suffix == "Z" { + zeroMasking = true + } + } if i := strings.IndexByte(raw, '.'); i >= 0 { op = Op(raw[:i]) } } - switch op { - case "MOVOU", "MOVOA", "MOVUPS", "MOVAPS", "MOVO", "MOVQ", "MOVL", "MOVD", - "VMOVDQU", "VMOVDQA", "VMOVNTDQ", "VMOVDQU64", "VMOVDQA64", "VMOVAPS", "VMOVAPD", - "VPCMPEQB", "VPCMPGTB", "VPMOVMSKB", "VZEROUPPER", "VZEROALL", "VPBROADCASTB", "VPBROADCASTQ", "VPAND", "VPXOR", "VPOR", "VPADDD", "VPADDQ", "VPTEST", - "VPANDQ", "VPXORQ", "VPORQ", "VPCLMULQDQ", "VPTERNLOGD", "VEXTRACTF32X4", - "VPERMB", "VGF2P8AFFINEQB", "VPERMI2B", "VPCOMPRESSQ", "VPOPCNTB", "VPCMPUQ", - "VBROADCASTI128", "VBROADCASTF32X2", "VBROADCASTSD", "VXORPD", - "VPSHUFB", "VPSHUFD", "VPSLLD", "VPSRLD", "VPSRLQ", "VPSLLQ", "VPSRLDQ", "VPSLLDQ", "VPUNPCKLBW", "VPUNPCKHBW", - "VPALIGNR", "VPERM2I128", "VPERM2F128", "VINSERTI128", "VPBLENDD", - "PXOR", "POR", "PAND", "PANDN", "PADDD", "PADDL", "PSUBB", "PSUBL", "PMULULQ", "PCLMULQDQ", "PCMPEQB", "PCMPEQL", "PCMPGTB", "PMOVMSKB", - "PSHUFB", "PSRLDQ", "PSLLDQ", "PSRLQ", "PSLLW", "PSRLW", "PSRLL", "PSLLL", "PSRAL", "PEXTRD", "PEXTRB", "PEXTRQ", - "PINSRQ", "PINSRD", "PINSRB", "PINSRW", "PALIGNR", "PUNPCKLBW", "PUNPCKHBW", "PUNPCKLLQ", "PUNPCKHLQ", "PUNPCKLQDQ", "PSHUFL", "PSHUFD", "PSHUFHW", "SHUFPS", - "PBLENDW", "PADDQ", "KMOVB", "KMOVW", "KMOVQ", "KXORQ", - "SHA1NEXTE", "SHA1MSG1", "SHA1MSG2", "SHA1RNDS4", "SHA256MSG1", "SHA256MSG2", "SHA256RNDS2", - "AESENC", "AESENCLAST", "AESDEC", "AESDECLAST", "AESIMC", "AESKEYGENASSIST", "PCMPESTRI": - // ok - default: - return false, false, nil + if _, alias := x86GoMOVQSourceAliases[op]; alias { + if rawOp != op { + return true, false, fmt.Errorf("%s MOVD does not accept instruction suffixes: %q", c.goarch, ins.Raw) + } + if known, terminated, err := c.lowerMachineRegisterMove("MOVQ", ins); known { + return known, terminated, err + } + if err := c.validateMOVDAliasForm(ins); err != nil { + return true, false, err + } + // Go source aliases share AMOVQ's rows, not Intel's mnemonic scope. + // Change local dispatch only; keep source/caller certificate identity. + dispatch := ins + dispatch.Op = op + op = x86SourceMoveInstruction(dispatch, c.goarch).Op } - - // MOVL src, Xn (seed vector low 32 bits). - if op == "MOVL" && len(ins.Args) == 2 && ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[1].Reg); ok { - var v64 string - var err error - switch ins.Args[0].Kind { - case OpImm, OpReg, OpFP, OpMem, OpSym: - v64, err = c.evalI64(ins.Args[0]) - default: - return true, false, fmt.Errorf("amd64 MOVL to X reg unsupported src: %q", ins.Raw) - } - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) - // Build <4 x i32> { crc, 0, 0, 0 } then bitcast to <16 x i8>. - v0 := "%" + tr - tvec := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> zeroinitializer, i32 %s, i32 0\n", tvec, v0) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bc, tvec) - return true, false, c.storeX(ins.Args[1].Reg, "%"+bc) + if _, ok := amd64MaskMoveSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", op, ins.Raw) } + return true, false, c.lowerMaskMove(op, ins) } - // MOVL Xn, dst stores the low 32 bits of the vector register. - if op == "MOVL" && len(ins.Args) == 2 && ins.Args[0].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[0].Reg); ok { - vec, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - words := c.newTmp() - low := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", words, vec) - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i32> %%%s, i32 0\n", low, words) - switch ins.Args[1].Kind { - case OpReg: - return true, false, c.storeRegSized(ins.Args[1].Reg, I32, "%"+low) - case OpFP: - return true, false, c.storeFPResult(ins.Args[1].FPOffset, I32, "%"+low) - case OpMem: - p, ptrType, err := c.ptrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i32 %%%s, %s %s, align 1\n", low, ptrType, p) - return true, false, nil - case OpSym: - p, err := c.ptrFromSB(ins.Args[1].Sym) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i32 %%%s, ptr %s, align 1\n", low, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 MOVL from X reg unsupported dst: %q", ins.Raw) - } + if _, ok := amd64GatherSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", op, ins.Raw) } + return true, false, c.lowerGather(op, ins) } - - // MOVD reg, Xn (seed vector with low 32-bit value) - if op == "MOVD" && len(ins.Args) == 2 && ins.Args[0].Kind == OpReg && ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[1].Reg); ok { - v64, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) - // Build <4 x i32> { v, 0, 0, 0 } then bitcast to <16 x i8>. - v0 := "%" + tr - tvec := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> zeroinitializer, i32 %s, i32 0\n", tvec, v0) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bc, tvec) - return true, false, c.storeX(ins.Args[1].Reg, "%"+bc) - } - } - - if op == "VZEROUPPER" || op == "VZEROALL" { - // These instructions affect registers 0-15, including their ZMM view. - for i := 0; i < 16; i++ { - c.clearVectorUpper(i, 16) - if op == "VZEROALL" { - if slot := c.xRegSlot[i]; slot != "" { - fmt.Fprintf(c.b, " store <16 x i8> zeroinitializer, ptr %s\n", slot) - } - } + if _, ok := amd64VSIBPrefetchSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", op, ins.Raw) + } + return true, false, c.lowerGather(op, ins) + } + if _, ok := amd64MaskLogicalSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", op, ins.Raw) + } + return true, false, c.lowerMaskLogical(op, ins) + } + if _, ok := amd64MaskArithmeticSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, op, ins.Raw) + } + return true, false, c.lowerMaskArithmetic(op, ins) + } + if _, ok := amd64MaskUnpackBits[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, op, ins.Raw) } - return true, false, nil + return true, false, c.lowerMaskUnpack(op, ins) + } + if _, ok := amd64TernaryLogicSpecs[op]; ok { + return c.lowerTernaryLogic(rawOp, ins) + } + if _, ok := amd64VectorAlignSpecs[op]; ok { + return c.lowerVectorAlign(rawOp, ins) + } + if _, ok := amd64IFMASpecs[op]; ok { + return c.lowerIFMA(rawOp, ins) + } + if _, ok := amd64FunnelShiftSpecs[op]; ok { + return c.lowerFunnelShift(rawOp, ins) + } + if _, ok := amd64PackedRotateSpecs[op]; ok { + return c.lowerPackedRotate(rawOp, ins) + } + if _, ok := amd64FMA3Specs[op]; ok { + return c.lowerFMA3(rawOp, ins) + } + if _, ok := amd64BinaryFloatingSpecs[op]; ok { + return c.lowerBinaryFloating(rawOp, ins) + } + if _, ok := amd64HorizontalFloatingSpecs[op]; ok { + return c.lowerHorizontalFloating(rawOp, ins) + } + switch op { + case "MOVOU", "MOVOA", "MOVUPS", "MOVAPS", "MOVO", "MOVQ", "VMOVQ", "MOVL", "MOVD", + "VMOVDQU", "VMOVDQA", "VMOVNTDQ", "VMOVDQU64", "VMOVDQA64", "VMOVAPS", "VMOVAPD", + "VPCMPEQB", "VPCMPGTB", "VZEROUPPER", "VZEROALL", "VPBROADCASTB", "VPBROADCASTD", "VPBROADCASTQ", "VPAND", "VPXOR", "VPOR", "VPADDD", "VPADDQ", "VPSUBB", "VPTEST", "VTESTPD", "VTESTPS", + "VPCLMULQDQ", "VEXTRACTF32X4", + "VPCMPUQ", + "VBROADCASTF32X2", "VBROADCASTSD", + "VPSHUFB", "VPSHUFD", "VPSLLD", "VPSLLQ", "VPSLLW", "VPSRAD", "VPSRAW", "VPSRLD", "VPSRLQ", "VPSRLW", "VPSRLDQ", "VPSLLDQ", "VPUNPCKLBW", "VPUNPCKHBW", "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ", + "VPALIGNR", "VPERM2I128", "VPERM2F128", "VINSERTI128", "VPBLENDD", + "PXOR", "POR", "PAND", "PADDB", "PADDD", "PADDL", "PSUBB", "PSUBL", "PSUBUSB", "PMULLW", "PCLMULQDQ", "PCMPEQB", "PCMPEQW", "PCMPEQL", "PCMPGTB", "PABSD", "PTEST", + "PMINUB", "PMINSB", "PMINUW", "PMINSW", "PMINUD", "PMINSD", "PMAXUB", "PMAXSB", "PMAXUW", "PMAXSW", "PMAXUD", "PMAXSD", + "PSHUFB", "PSRLDQ", "PSLLDQ", "PSRLO", "PSLLO", "PSRLQ", "PSLLQ", "PSLLW", "PSRLW", "PSRLL", "PSLLL", "PSRAL", "PEXTRD", "PEXTRB", "PEXTRQ", + "PALIGNR", "PUNPCKLBW", "PUNPCKHBW", "PUNPCKLLQ", "PUNPCKHLQ", "PUNPCKLQDQ", "PUNPCKHQDQ", "PSHUFL", "PSHUFD", "SHUFPS", + "PBLENDW", "PADDQ", + "AESENC", "AESENCLAST", "AESDEC", "AESDECLAST", "AESIMC", "AESKEYGENASSIST": + // ok + default: + return false, false, nil } if op == "VMOVDQA" { @@ -130,8 +196,11 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err if op == "VMOVDQA64" { op = "VMOVDQU64" } - if op == "VPERM2F128" { - op = "VPERM2I128" + if op == "PSLLO" { + op = "PSLLDQ" + } + if op == "PSRLO" { + op = "PSRLDQ" } if op == "VMOVAPS" || op == "VMOVAPD" { if len(ins.Args) == 2 { @@ -143,324 +212,8 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err } } } - if op == "VPXORQ" || op == "VPANDQ" || op == "VPORQ" { - if len(ins.Args) == 3 && ins.Args[2].Kind == OpReg && !isAMD64ZReg(ins.Args[2].Reg) { - switch op { - case "VPXORQ": - op = "VPXOR" - case "VPANDQ": - op = "VPAND" - case "VPORQ": - op = "VPOR" - } - } - } - - // MOVUPS/MOVAPS/MOVO are aliases for unaligned/aligned 128-bit xmm moves. - if op == "MOVUPS" || op == "MOVAPS" || op == "MOVO" { - op = "MOVOU" - } - - // MOVQ src, Xn (load low 64 bits into Xn) - if op == "MOVQ" && len(ins.Args) == 2 && ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[1].Reg); ok { - var low string - switch ins.Args[0].Kind { - case OpImm: - low = fmt.Sprintf("%d", ins.Args[0].Imm) - case OpReg: - v, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - low = v - case OpFP: - v, err := c.evalFPToI64(ins.Args[0].FPOffset) - if err != nil { - return true, false, err - } - low = v - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - low = "%" + ld - case OpSym: - p, err := c.ptrFromSB(ins.Args[0].Sym) - if err != nil { - return true, false, err - } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - low = "%" + ld - default: - return true, false, fmt.Errorf("amd64 MOVQ to X reg unsupported src: %q", ins.Raw) - } - // Build <2 x i64> { low, 0 }. - ins0 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> zeroinitializer, i64 %s, i32 0\n", ins0, low) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, ins0) - return true, false, c.storeX(ins.Args[1].Reg, "%"+bc) - } - } - - // MOVQ Xn, dst (extract low 64 bits from Xn). - if op == "MOVQ" && len(ins.Args) == 2 && ins.Args[0].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[0].Reg); ok { - xv, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bc, xv) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", lo, bc) - switch ins.Args[1].Kind { - case OpReg: - return true, false, c.storeReg(ins.Args[1].Reg, "%"+lo) - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s, align 1\n", lo, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 MOVQ from X reg unsupported dst: %q", ins.Raw) - } - } - } switch op { - case "KXORQ": - if len(ins.Args) != 3 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 KXORQ expects Ksrc1, Ksrc2, Kdst: %q", ins.Raw) - } - if _, ok := amd64ParseKReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseKReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseKReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - a, err := c.loadK(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - b, err := c.loadK(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - x := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", x, a, b) - return true, false, c.storeK(ins.Args[2].Reg, "%"+x) - - case "KMOVB", "KMOVW", "KMOVQ": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects 2 operands: %q", op, ins.Raw) - } - bits := 64 - loadTy := "i64" - switch op { - case "KMOVB": - bits, loadTy = 8, "i8" - case "KMOVW": - bits, loadTy = 16, "i16" - } - maskValue := func(v string) string { - if bits == 64 { - return v - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", t, v, (uint64(1)< %s to <8 x i64>\n", sv, src) - outVec := "zeroinitializer" - writePos := "0" - for i := 0; i < 8; i++ { - bit := amd64MaskBitI1(c, maskv, i) - lane := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractelement <8 x i64> %%%s, i32 %d\n", lane, sv, i) - inserted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <8 x i64> %s, i64 %%%s, i32 %s\n", inserted, outVec, lane, writePos) - selected := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %s, <8 x i64> %%%s, <8 x i64> %s\n", selected, bit, inserted, outVec) - step := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i1 %s to i32\n", step, bit) - nextPos := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i32 %s, %%%s\n", nextPos, writePos, step) - outVec = "%" + selected - writePos = "%" + nextPos - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i64> %s to <64 x i8>\n", out, outVec) - return true, false, c.storeZ(ins.Args[2].Reg, "%"+out) - - case "VPXORQ", "VPANDQ", "VPORQ": - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src1, src2, dstReg: %q", op, ins.Raw) - } - if _, ok := amd64ParseZReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - a, err := c.loadZVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadZVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - ab := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <64 x i8> %s to <8 x i64>\n", ab, a) - bb := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <64 x i8> %s to <8 x i64>\n", bb, b) - t := c.newTmp() - switch op { - case "VPXORQ": - fmt.Fprintf(c.b, " %%%s = xor <8 x i64> %%%s, %%%s\n", t, ab, bb) - case "VPANDQ": - fmt.Fprintf(c.b, " %%%s = and <8 x i64> %%%s, %%%s\n", t, ab, bb) - case "VPORQ": - fmt.Fprintf(c.b, " %%%s = or <8 x i64> %%%s, %%%s\n", t, ab, bb) + var low32 string + if ins.Args[0].Kind == OpReg { + if _, ok := amd64ParseXReg(ins.Args[0].Reg); ok { + value, err := c.loadX(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", words, value) + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <4 x i32> %%%s, i32 0\n", extracted, words) + low32 = "%" + extracted + } } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i64> %%%s to <64 x i8>\n", out, t) - return true, false, c.storeZ(ins.Args[2].Reg, "%"+out) - - case "VBROADCASTI128": - // VBROADCASTI128 Xsrc|mem, Ydst - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VBROADCASTI128 expects Xsrc|mem, Ydst: %q", ins.Raw) + if low32 == "" { + var value64 string + var err error + switch ins.Args[0].Kind { + case OpMem: + addr, addrErr := c.addrFromMem(ins.Args[0].Mem) + if addrErr != nil { + return true, false, addrErr + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", loaded, c.ptrFromAddrI64(addr)) + low32 = "%" + loaded + case OpSym: + p, ptrErr := c.ptrFromSB(ins.Args[0].Sym) + if ptrErr != nil { + return true, false, ptrErr + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", loaded, p) + low32 = "%" + loaded + default: + value64, err = c.evalI64(ins.Args[0]) + if err != nil { + return true, false, err + } + } + if low32 == "" { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", truncated, value64) + low32 = "%" + truncated + } } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { + lanes := 0 + store := func(string) error { return nil } + switch { + case isAMD64XReg(ins.Args[1].Reg): + lanes = 4 + store = func(value string) error { + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %s to <16 x i8>\n", bytesValue, value) + return c.storeX(ins.Args[1].Reg, "%"+bytesValue) + } + case isAMD64YReg(ins.Args[1].Reg): + lanes = 8 + store = func(value string) error { + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i32> %s to <32 x i8>\n", bytesValue, value) + return c.storeY(ins.Args[1].Reg, "%"+bytesValue) + } + case isAMD64ZReg(ins.Args[1].Reg): + lanes = 16 + store = func(value string) error { + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i32> %s to <64 x i8>\n", bytesValue, value) + return c.storeZ(ins.Args[1].Reg, "%"+bytesValue) + } + default: return false, false, nil } - src, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err + seed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i32> zeroinitializer, i32 %s, i32 0\n", seed, lanes, low32) + mask := make([]string, lanes) + for i := range mask { + mask[i] = "i32 0" } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> %s, <32 x i32> %s\n", out, src, src, llvmRepeatI8Mask(16, 32)) - return true, false, c.storeY(ins.Args[1].Reg, "%"+out) + broadcast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i32> %%%s, <%d x i32> zeroinitializer, <%d x i32> <%s>\n", broadcast, lanes, seed, lanes, lanes, strings.Join(mask, ", ")) + return true, false, store("%" + broadcast) case "VBROADCASTF32X2", "VBROADCASTSD", "VPBROADCASTQ": // These instructions broadcast the low 64 source bits. Their lane @@ -612,769 +363,182 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err if ins.Args[0].Kind == OpSym && strings.HasPrefix(strings.TrimSpace(ins.Args[0].Sym), "$") { return false, false, nil } - var v64 string - if ins.Args[0].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[0].Reg); ok { - xv, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - words := c.newTmp() - low := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", words, xv) - fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", low, words) - v64 = "%" + low - } else if op != "VPBROADCASTQ" { - return false, false, nil - } - } - if v64 == "" { - var err error - v64, err = c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - } - chunk := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to <8 x i8>\n", chunk, v64) - if dstIsX { - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i8> %%%s, <8 x i8> %%%s, <16 x i32> %s\n", out, chunk, chunk, llvmRepeatI8Mask(8, 16)) - return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); ok { - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i8> %%%s, <8 x i8> %%%s, <32 x i32> %s\n", out, chunk, chunk, llvmRepeatI8Mask(8, 32)) - return true, false, c.storeY(ins.Args[1].Reg, "%"+out) - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i8> %%%s, <8 x i8> %%%s, <64 x i32> %s\n", out, chunk, chunk, llvmRepeatI8Mask(8, 64)) - return true, false, c.storeZ(ins.Args[1].Reg, "%"+out) - - case "VXORPD": - // VXORPD src1, src2, dst; this is a bitwise operation despite the - // floating-point mnemonic, so byte-vector xor preserves all bits. - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VXORPD expects src1, src2, dst: %q", ins.Raw) - } - if _, ok := amd64ParseZReg(ins.Args[2].Reg); ok { - a, err := c.loadZVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadZVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <64 x i8> %s, %s\n", t, a, b) - return true, false, c.storeZ(ins.Args[2].Reg, "%"+t) - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); ok { - a, err := c.loadYVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadYVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <32 x i8> %s, %s\n", t, a, b) - return true, false, c.storeY(ins.Args[2].Reg, "%"+t) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); ok { - a, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadXVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", t, a, b) - return true, false, c.storeX(ins.Args[2].Reg, "%"+t) - } - return false, false, nil - - case "VPXOR", "VPOR", "VPADDD", "VPADDQ": - // V* three-operand op; support both X and Y destinations. - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src1, src2, dstReg: %q", op, ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); ok { - a, err := c.loadYVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadYVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - switch op { - case "VPXOR": - fmt.Fprintf(c.b, " %%%s = xor <32 x i8> %s, %s\n", t, a, b) - case "VPOR": - fmt.Fprintf(c.b, " %%%s = or <32 x i8> %s, %s\n", t, a, b) - case "VPADDD": - ab := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <8 x i32>\n", ab, a) - bb := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <8 x i32>\n", bb, b) - add := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add <8 x i32> %%%s, %%%s\n", add, ab, bb) - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i32> %%%s to <32 x i8>\n", t, add) - case "VPADDQ": - ab := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", ab, a) - bb := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", bb, b) - add := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add <4 x i64> %%%s, %%%s\n", add, ab, bb) - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i64> %%%s to <32 x i8>\n", t, add) - } - return true, false, c.storeY(ins.Args[2].Reg, "%"+t) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); ok { - a, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadXVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - switch op { - case "VPXOR": - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", t, a, b) - case "VPOR": - fmt.Fprintf(c.b, " %%%s = or <16 x i8> %s, %s\n", t, a, b) - case "VPADDD": - ab := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", ab, a) - bb := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", bb, b) - add := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add <4 x i32> %%%s, %%%s\n", add, ab, bb) - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", t, add) - case "VPADDQ": - ab := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", ab, a) - bb := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bb, b) - add := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add <2 x i64> %%%s, %%%s\n", add, ab, bb) - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", t, add) - } - return true, false, c.storeX(ins.Args[2].Reg, "%"+t) - } - return false, false, nil - - case "VPSHUFB": - // VPSHUFB Ymask, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPSHUFB expects Ymask, Ysrc, Ydst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); ok { - mask, err := c.loadYVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - src, err := c.loadYVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - // Emulate 256-bit PSHUFB by lane-splitting into two 128-bit shuffles. - srcLo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", srcLo, src, llvmI32RangeMask(0, 16)) - srcHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", srcHi, src, llvmI32RangeMask(16, 16)) - maskLo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", maskLo, mask, llvmI32RangeMask(0, 16)) - maskHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", maskHi, mask, llvmI32RangeMask(16, 16)) - outLo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %%%s, <16 x i8> %%%s)\n", outLo, srcLo, maskLo) - outHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %%%s, <16 x i8> %%%s)\n", outHi, srcHi, maskHi) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <32 x i32> %s\n", out, outLo, outHi, llvmI32RangeMask(0, 32)) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); ok { - mask, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - src, err := c.loadXVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %s, <16 x i8> %s)\n", call, src, mask) - return true, false, c.storeX(ins.Args[2].Reg, "%"+call) - } - return false, false, nil - - case "VPSHUFD": - // VPSHUFD $imm, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPSHUFD expects $imm, Ysrc, Ydst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - imm := uint64(ins.Args[0].Imm) & 0xff - idx := func(k uint) uint64 { return (imm >> (2 * k)) & 3 } - src, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <8 x i32>\n", bc, src) - mask := fmt.Sprintf("<8 x i32> ", - idx(0), idx(1), idx(2), idx(3), - 4+idx(0), 4+idx(1), 4+idx(2), 4+idx(3)) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i32> %%%s, <8 x i32> zeroinitializer, %s\n", sh, bc, mask) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i32> %%%s to <32 x i8>\n", out, sh) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) - - case "VPSLLD", "VPSRLD": - // VPS*D $imm, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, Ysrc, Ydst: %q", op, ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm & 31 - src, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <8 x i32>\n", bc, src) - sh := c.newTmp() - if op == "VPSLLD" { - fmt.Fprintf(c.b, " %%%s = shl <8 x i32> %%%s, \n", sh, bc, n, n, n, n, n, n, n, n) - } else { - fmt.Fprintf(c.b, " %%%s = lshr <8 x i32> %%%s, \n", sh, bc, n, n, n, n, n, n, n, n) - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i32> %%%s to <32 x i8>\n", out, sh) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) - - case "VPSRLQ", "VPSLLQ": - // VPSR/LQ $imm, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, Ysrc, Ydst: %q", op, ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm & 63 - src, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", bc, src) - sh := c.newTmp() - if op == "VPSLLQ" { - fmt.Fprintf(c.b, " %%%s = shl <4 x i64> %%%s, \n", sh, bc, n, n, n, n) - } else { - fmt.Fprintf(c.b, " %%%s = lshr <4 x i64> %%%s, \n", sh, bc, n, n, n, n) - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i64> %%%s to <32 x i8>\n", out, sh) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) - - case "VPALIGNR": - // VPALIGNR $imm, Ysrc1, Ysrc2, Ydst ; dst = alignr(src2, src1, imm) - if len(ins.Args) != 4 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg || ins.Args[3].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPALIGNR expects $imm, Ysrc1, Ysrc2, Ydst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[3].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm - if n < 0 { - n = 0 - } - if n > 255 { - n = 255 - } - src1, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - src2, err := c.loadY(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - lo1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", lo1, src1, llvmI32RangeMask(0, 16)) - hi1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", hi1, src1, llvmI32RangeMask(16, 16)) - lo2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", lo2, src2, llvmI32RangeMask(0, 16)) - hi2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", hi2, src2, llvmI32RangeMask(16, 16)) - outLo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <16 x i32> %s\n", outLo, lo2, lo1, llvmAlignRightBytesMask(n)) - outHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <16 x i32> %s\n", outHi, hi2, hi1, llvmAlignRightBytesMask(n)) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <32 x i32> %s\n", out, outLo, outHi, llvmI32RangeMask(0, 32)) - return true, false, c.storeY(ins.Args[3].Reg, "%"+out) - - case "VPERM2I128": - // VPERM2I128 $imm, Ysrc1, Ysrc2, Ydst - if len(ins.Args) != 4 || ins.Args[0].Kind != OpImm || ins.Args[3].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPERM2I128 expects $imm, Ysrc1, Ysrc2, Ydst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[3].Reg); !ok { - return false, false, nil - } - imm := uint64(ins.Args[0].Imm) & 0xff - // AT&T order: imm, src1, src2, dst ; choose lanes from src2 (arg2) + src1 (arg1). - src1, err := c.loadYVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - src2, err := c.loadYVecOperand(ins.Args[2]) - if err != nil { - return true, false, err - } - sel := func(bits uint64) int { - switch bits & 0x3 { - case 0: - return 0 // src2 low - case 1: - return 1 // src2 high - case 2: - return 2 // src1 low - default: - return 3 // src1 high - } - } - lowSel := sel(imm) - hiSel := sel(imm >> 4) - mask := fmt.Sprintf("<4 x i32> ", lowSel, lowSel+4, hiSel, hiSel+4) - b2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", b2, src2) - b1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", b1, src1) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i64> %%%s, <4 x i64> %%%s, %s\n", sh, b2, b1, mask) - // Zeroing controls. - zeroLo := (imm>>3)&1 == 1 - zeroHi := (imm>>7)&1 == 1 - if zeroLo || zeroHi { - cur := "%" + sh - if zeroLo { - i0 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i64> %s, i64 0, i32 0\n", i0, cur) - i1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i64> %%%s, i64 0, i32 1\n", i1, i0) - cur = "%" + i1 - } - if zeroHi { - i2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i64> %s, i64 0, i32 2\n", i2, cur) - i3 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i64> %%%s, i64 0, i32 3\n", i3, i2) - cur = "%" + i3 - } - sh = strings.TrimPrefix(cur, "%") - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i64> %%%s to <32 x i8>\n", out, sh) - return true, false, c.storeY(ins.Args[3].Reg, "%"+out) - - case "VINSERTI128": - // VINSERTI128 $imm, Xsrc, Ysrc, Ydst - if len(ins.Args) != 4 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg || ins.Args[3].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VINSERTI128 expects $imm, Xsrc, Ysrc, Ydst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[3].Reg); !ok { - return false, false, nil - } - imm := ins.Args[0].Imm & 1 - xsrc, err := c.loadXVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - ysrc, err := c.loadY(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - y64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", y64, ysrc) - x64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", x64, xsrc) - i0 := c.newTmp() - i1 := c.newTmp() - if imm == 0 { - fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", i0, x64) - fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 1\n", i1, x64) - s0 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i64> %%%s, i64 %%%s, i32 0\n", s0, y64, i0) - s1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i64> %%%s, i64 %%%s, i32 1\n", s1, s0, i1) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i64> %%%s to <32 x i8>\n", out, s1) - return true, false, c.storeY(ins.Args[3].Reg, "%"+out) - } - fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", i0, x64) - fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 1\n", i1, x64) - s0 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i64> %%%s, i64 %%%s, i32 2\n", s0, y64, i0) - s1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i64> %%%s, i64 %%%s, i32 3\n", s1, s0, i1) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i64> %%%s to <32 x i8>\n", out, s1) - return true, false, c.storeY(ins.Args[3].Reg, "%"+out) - - case "VMOVNTDQ": - // VMOVNTDQ Ysrc, mem - if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VMOVNTDQ expects Ysrc, dst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - src, err := c.loadY(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - switch ins.Args[1].Kind { - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - fmt.Fprintf(c.b, " store <32 x i8> %s, ptr %s, align 1\n", src, p) - return true, false, nil - case OpSym: - p, err := c.ptrFromSB(ins.Args[1].Sym) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store <32 x i8> %s, ptr %s, align 1\n", src, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 VMOVNTDQ unsupported dst: %q", ins.Raw) - } - - case "AESENC", "AESENCLAST", "AESDEC", "AESDECLAST": - // Two-operand form: OP Xsrc, Xdst - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - src, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - dstv, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - src2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", src2, src) - dst2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", dst2, dstv) - intr := map[Op]string{ - "AESENC": "@llvm.x86.aesni.aesenc", - "AESENCLAST": "@llvm.x86.aesni.aesenclast", - "AESDEC": "@llvm.x86.aesni.aesdec", - "AESDECLAST": "@llvm.x86.aesni.aesdeclast", - }[op] - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <2 x i64> %s(<2 x i64> %%%s, <2 x i64> %%%s)\n", call, intr, dst2, src2) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, call) - return true, false, c.storeX(ins.Args[1].Reg, "%"+bc) - - case "AESIMC": - // Two-operand form: AESIMC Xsrc, Xdst - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 AESIMC expects Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - src, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - src2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", src2, src) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64> %%%s)\n", call, src2) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, call) - return true, false, c.storeX(ins.Args[1].Reg, "%"+bc) - - case "AESKEYGENASSIST": - // Three-operand form: AESKEYGENASSIST $imm, Xsrc, Xdst - if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 AESKEYGENASSIST expects $imm, Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - if ins.Args[0].Kind != OpImm { - return true, false, fmt.Errorf("amd64 AESKEYGENASSIST expects immediate first operand: %q", ins.Raw) - } - imm := ins.Args[0].Imm & 0xff - src, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - src2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", src2, src) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %%%s, i8 %d)\n", call, src2, imm) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, call) - return true, false, c.storeX(ins.Args[2].Reg, "%"+bc) - - case "VPTEST": - // VPTEST Ysrc1, Ysrc2 (sets flags based on bitwise tests; used with JZ/JNZ). - // We implement the ZF behavior (ZF=1 iff (a&b)==0) since stdlib uses JNZ. - if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPTEST expects Ysrc1, Ysrc2: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - a, err := c.loadY(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - b, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - and := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and <32 x i8> %s, %s\n", and, a, b) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %%%s to <4 x i64>\n", bc, and) - e0, e1, e2, e3 := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i64> %%%s, i32 0\n", e0, bc) - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i64> %%%s, i32 1\n", e1, bc) - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i64> %%%s, i32 2\n", e2, bc) - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i64> %%%s, i32 3\n", e3, bc) - o01 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", o01, e0, e1) - o23 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", o23, e2, e3) - o := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", o, o01, o23) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, 0\n", z, o) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", z, c.flagsZSlot) - // CF/other flags are not used by stdlib here; clear CF to avoid stale reads. - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) - return true, false, nil - - case "PCMPESTRI": - // PCMPESTRI $imm, mem, Xsrc (implicit lengths; result index in ECX). - // - // LLVM 19 has backend issues selecting the native X86ISD::PCMPESTR node - // for the corresponding intrinsic in some environments. For now we - // emulate the subset used by the Go stdlib: - // imm=0x0c: unsigned byte compare, equal ordered, first match. - // - // We compute the first offset i in [0..15] where: - // - bytes(mem[i:]) has a prefix that matches Xsrc[0:lenA] - // - or a partial match at the end of the 16-byte block (to drive the - // stdlib's "partial match" control flow). - // If no match/partial match exists, return 16. - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PCMPESTRI expects $imm, mem, Xsrc: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - imm := int64(ins.Args[0].Imm) & 0xff - if imm != 0x0c { - return true, false, fmt.Errorf("amd64 PCMPESTRI: unsupported imm 0x%x (only 0x0c supported): %q", imm, ins.Raw) - } - - // A: needle bytes. - a, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - // B: 16-byte haystack chunk. - var bvec string - switch ins.Args[1].Kind { - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err + var v64 string + if ins.Args[0].Kind == OpReg { + if _, ok := amd64ParseXReg(ins.Args[0].Reg); ok { + xv, err := c.loadX(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + words := c.newTmp() + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", words, xv) + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", low, words) + v64 = "%" + low + } else if op != "VPBROADCASTQ" { + return false, false, nil } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", ld, p) - bvec = "%" + ld - case OpSym: - p, err := c.ptrFromSB(ins.Args[1].Sym) + } + if v64 == "" { + var err error + v64, err = c.evalI64(ins.Args[0]) if err != nil { return true, false, err } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", ld, p) - bvec = "%" + ld - default: - return true, false, fmt.Errorf("amd64 PCMPESTRI unsupported mem operand: %q", ins.Raw) } + chunk := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to <8 x i8>\n", chunk, v64) + if dstIsX { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i8> %%%s, <8 x i8> %%%s, <16 x i32> %s\n", out, chunk, chunk, llvmRepeatI8Mask(8, 16)) + return true, false, c.storeX(ins.Args[1].Reg, "%"+out) + } + if _, ok := amd64ParseYReg(ins.Args[1].Reg); ok { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i8> %%%s, <8 x i8> %%%s, <32 x i32> %s\n", out, chunk, chunk, llvmRepeatI8Mask(8, 32)) + return true, false, c.storeY(ins.Args[1].Reg, "%"+out) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i8> %%%s, <8 x i8> %%%s, <64 x i32> %s\n", out, chunk, chunk, llvmRepeatI8Mask(8, 64)) + return true, false, c.storeZ(ins.Args[1].Reg, "%"+out) - ax, err := c.loadReg(AX) + case "VPSLLD", "VPSLLQ", "VPSLLW", "VPSRAD", "VPSRAW", "VPSRLD", "VPSRLQ", "VPSRLW": + // Go's x86 assembler uses _yvpslld for this whole family: + // $imm8, X|Y|Z src/mem, [Kmask,] X|Y|Z dst + // X|m128 count, X|Y|Z src, [Kmask,] X|Y|Z dst + // The count is uniform: variable forms use the low 64 bits of X|m128. + if (len(ins.Args) != 3 && len(ins.Args) != 4) || ins.Args[len(ins.Args)-1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects count, vector src, [K mask,] vector dst: %q", op, ins.Raw) + } + dst := ins.Args[len(ins.Args)-1].Reg + byteWidth := 0 + var src string + var err error + loadDst := func() (string, error) { return "", nil } + store := func(string) error { return nil } + switch { + case isAMD64XReg(dst): + byteWidth = 16 + src, err = c.loadXVecOperand(ins.Args[1]) + loadDst = func() (string, error) { return c.loadX(dst) } + store = func(v string) error { return c.storeX(dst, v) } + case isAMD64YReg(dst): + byteWidth = 32 + src, err = c.loadYVecOperand(ins.Args[1]) + loadDst = func() (string, error) { return c.loadY(dst) } + store = func(v string) error { return c.storeY(dst, v) } + case isAMD64ZReg(dst): + byteWidth = 64 + src, err = c.loadZVecOperand(ins.Args[1]) + loadDst = func() (string, error) { return c.loadZ(dst) } + store = func(v string) error { return c.storeZ(dst, v) } + default: + return false, false, nil + } if err != nil { return true, false, err } - la := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", la, ax) - allOnes := llvmAllOnesI8Vec(16) - idx := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i32 0, 16\n", idx) - prev := "%" + idx - - // Build first-match index using a select chain. - for i := 0; i < 16; i++ { - // Shuffle B so that element 0 is B[i], padding beyond 16 with 0. - maskElts := make([]string, 0, 16) - for k := 0; k < 16; k++ { - j := i + k - if j < 16 { - maskElts = append(maskElts, fmt.Sprintf("i32 %d", j)) - } else { - // 16 selects element 0 of the second (zero) vector. - maskElts = append(maskElts, "i32 16") - } - } - mask := "<16 x i32> <" + strings.Join(maskElts, ", ") + ">" + elemBits := 32 + switch op[len(op)-1] { + case 'Q': + elemBits = 64 + case 'W': + elemBits = 16 + } + lanes := byteWidth * 8 / elemBits + vecType := fmt.Sprintf("<%d x i%d>", lanes, elemBits) + bc := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", bc, byteWidth, src, vecType) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, %s\n", sh, bvec, mask) - cmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq <16 x i8> %%%s, %s\n", cmp, sh, a) - sel := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select <16 x i1> %%%s, <16 x i8> %s, <16 x i8> zeroinitializer\n", sel, cmp, allOnes) - pm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %%%s)\n", pm, sel) - - // minLen = min(lenA, 16-i) - cap := 16 - i - lt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i32 %%%s, %d\n", lt, la, cap) - min := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %d\n", min, lt, la, cap) - sh1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i32 1, %%%s\n", sh1, min) - req := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, 1\n", req, sh1) - have := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i32 %%%s, %%%s\n", have, pm, req) - okt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i32 %%%s, %%%s\n", okt, have, req) - - unset := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 16\n", unset, prev) - take := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", take, okt, unset) - next := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %d, i32 %s\n", next, take, i, prev) - prev = "%" + next - } - - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", z, prev) - return true, false, c.storeReg(CX, "%"+z) - - case "VPAND": - // VPAND src1, src2, dst supports both XMM and YMM widths. - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPAND expects src1, src2, dst: %q", ins.Raw) + llvmOp := "shl" + arithmetic := op == "VPSRAD" || op == "VPSRAW" + if arithmetic { + llvmOp = "ashr" + } else if strings.HasPrefix(string(op), "VPSRL") { + llvmOp = "lshr" } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); ok { - for _, arg := range ins.Args[:2] { - if arg.Kind == OpReg { - if _, ok := amd64ParseYReg(arg.Reg); !ok { - return false, false, nil - } - } + + shifted := "zeroinitializer" + if ins.Args[0].Kind == OpImm { + count := uint64(ins.Args[0].Imm) & 0xff + if arithmetic && count >= uint64(elemBits) { + count = uint64(elemBits - 1) } - a, err := c.loadYVecOperand(ins.Args[0]) - if err != nil { - return true, false, err + if arithmetic || count < uint64(elemBits) { + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %%%s, %s\n", sh, llvmOp, vecType, bc, llvmSplatInteger(lanes, elemBits, count)) + shifted = "%" + sh } - b, err := c.loadYVecOperand(ins.Args[1]) + } else { + countVec, err := c.loadXVecOperand(ins.Args[0]) if err != nil { - return true, false, err + return true, false, fmt.Errorf("amd64 %s count: %w", op, err) + } + countWords := c.newTmp() + count := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", countWords, countVec) + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", count, countWords) + inRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i64 %%%s, %d\n", inRange, count, elemBits) + safeCount := c.newTmp() + fallback := 0 + if arithmetic { + fallback = elemBits - 1 + } + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 %d\n", safeCount, inRange, count, fallback) + shiftCount := "%" + safeCount + if elemBits < 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i%d\n", truncated, safeCount, elemBits) + shiftCount = "%" + truncated + } + countSplat := amd64SplatInteger(c, lanes, elemBits, shiftCount) + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %%%s, %s\n", sh, llvmOp, vecType, bc, countSplat) + shifted = "%" + sh + if !arithmetic { + condition := amd64SplatI1(c, lanes, "%"+inRange) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %s, %s %s, %s zeroinitializer\n", selected, lanes, condition, vecType, shifted, vecType) + shifted = "%" + selected } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and <32 x i8> %s, %s\n", t, a, b) - return true, false, c.storeY(ins.Args[2].Reg, "%"+t) } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); ok { - for _, arg := range ins.Args[:2] { - if arg.Kind == OpReg { - if _, ok := amd64ParseXReg(arg.Reg); !ok { - return false, false, nil - } - } + + if len(ins.Args) == 4 { + if ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s masked form expects K mask: %q", op, ins.Raw) } - a, err := c.loadXVecOperand(ins.Args[0]) + if _, ok := amd64ParseKReg(ins.Args[2].Reg); !ok { + return false, false, nil + } + maskValue, err := c.loadK(ins.Args[2].Reg) if err != nil { return true, false, err } - b, err := c.loadXVecOperand(ins.Args[1]) + oldBytes, err := loadDst() if err != nil { return true, false, err } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %s, %s\n", t, a, b) - return true, false, c.storeX(ins.Args[2].Reg, "%"+t) + old := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", old, byteWidth, oldBytes, vecType) + shifted = amd64ApplyIntegerLaneMask(c, lanes, elemBits, shifted, "%"+old, maskValue, zeroMasking) } - return false, false, nil + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, vecType, shifted, byteWidth) + return true, false, store("%" + out) + + case "VPALIGNR": + return c.lowerPackedAlignRight(rawOp, ins) + + case "VMOVNTDQ": + // Compatibility entry point for direct lowerVec unit tests. Ordinary + // translation dispatches the whole family through the dedicated + // lowerer before reaching this legacy switch. + return c.lowerNonTemporalVectorMove(op, ins) + + case "PTEST", "VPTEST", "VTESTPD", "VTESTPS": + return c.lowerVectorTest(rawOp, ins) case "VPBLENDD": // VPBLENDD $imm, Ysrc1, Ysrc2, Ydst @@ -1435,43 +599,7 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, c.storeY(ins.Args[1].Reg, "%"+spl) case "VPSRLDQ", "VPSLLDQ": - // VPSR/LLDQ $imm, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, Ysrc, Ydst: %q", op, ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm - if n < 0 { - n = 0 - } - if n > 16 { - n = 16 - } - src, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", lo, src, llvmI32RangeMask(0, 16)) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", hi, src, llvmI32RangeMask(16, 16)) - lo2 := c.newTmp() - hi2 := c.newTmp() - if op == "VPSLLDQ" { - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> zeroinitializer, <16 x i32> %s\n", lo2, lo, llvmShiftLeftBytesMask(n)) - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> zeroinitializer, <16 x i32> %s\n", hi2, hi, llvmShiftLeftBytesMask(n)) - } else { - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> zeroinitializer, <16 x i32> %s\n", lo2, lo, llvmShiftRightBytesMask(n)) - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> zeroinitializer, <16 x i32> %s\n", hi2, hi, llvmShiftRightBytesMask(n)) - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <32 x i32> %s\n", out, lo2, hi2, llvmI32RangeMask(0, 32)) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) + return c.lowerPackedByteShift(rawOp, ins) case "VPUNPCKLBW", "VPUNPCKHBW": if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { @@ -1571,46 +699,6 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> %s, %s\n", sh, dstv, src, mask) return true, false, c.storeX(ins.Args[1].Reg, "%"+sh) - case "PMULULQ": - // PMULULQ Xsrc/m128, Xdst multiplies the unsigned even-numbered - // 32-bit lanes and writes the two 64-bit products. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PMULULQ expects Xsrc/m128, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if ins.Args[0].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - } - src, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - dstv, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - dst32 := c.newTmp() - src32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", dst32, dstv) - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", src32, src) - dstEven := c.newTmp() - srcEven := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i32> %%%s, <4 x i32> zeroinitializer, <2 x i32> \n", dstEven, dst32) - fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i32> %%%s, <4 x i32> zeroinitializer, <2 x i32> \n", srcEven, src32) - dst64 := c.newTmp() - src64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext <2 x i32> %%%s to <2 x i64>\n", dst64, dstEven) - fmt.Fprintf(c.b, " %%%s = zext <2 x i32> %%%s to <2 x i64>\n", src64, srcEven) - product := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul <2 x i64> %%%s, %%%s\n", product, dst64, src64) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", out, product) - return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - case "PUNPCKLLQ", "PUNPCKHLQ": // PUNPCKL/H LQ interleaves the low/high pair of 32-bit lanes. if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { @@ -1673,127 +761,11 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", out, sh) return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - case "PSHUFL", "PSHUFD": - // PSHUFL $imm, Xsrc, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, Xsrc, Xdst: %q", op, ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - imm := uint64(ins.Args[0].Imm) & 0xff - idx := func(k uint) uint64 { return (imm >> (2 * k)) & 3 } - src, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - bc1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", bc1, src) - mask := fmt.Sprintf("<4 x i32> ", idx(0), idx(1), idx(2), idx(3)) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i32> %%%s, <4 x i32> zeroinitializer, %s\n", sh, bc1, mask) - bc2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bc2, sh) - return true, false, c.storeX(ins.Args[2].Reg, "%"+bc2) - - case "PSHUFHW": - // PSHUFHW $imm, Xsrc, Xdst (shuffle high 4 words; low 4 words unchanged). - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSHUFHW expects $imm, Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - imm := uint64(ins.Args[0].Imm) & 0xff - idx := func(k uint) uint64 { return (imm >> (2 * k)) & 3 } - src, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - bc1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <8 x i16>\n", bc1, src) - mask := fmt.Sprintf("<8 x i32> ", - 4+idx(0), 4+idx(1), 4+idx(2), 4+idx(3)) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i16> %%%s, <8 x i16> zeroinitializer, %s\n", sh, bc1, mask) - bc2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i16> %%%s to <16 x i8>\n", bc2, sh) - return true, false, c.storeX(ins.Args[2].Reg, "%"+bc2) - - case "SHUFPS": - // SHUFPS $imm, Xsrc, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 SHUFPS expects $imm, Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - imm := uint64(ins.Args[0].Imm) & 0xff - idx := func(k uint) uint64 { return (imm >> (2 * k)) & 3 } - src, err := c.loadXVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - dstv, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - ds := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", ds, dstv) - ss := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", ss, src) - mask := fmt.Sprintf("<4 x i32> ", idx(0), idx(1), 4+idx(2), 4+idx(3)) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i32> %%%s, <4 x i32> %%%s, %s\n", sh, ds, ss, mask) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bc, sh) - return true, false, c.storeX(ins.Args[2].Reg, "%"+bc) - - case "PBLENDW": - // PBLENDW $imm, Xsrc, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PBLENDW expects $imm, Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - src, err := c.loadXVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - dstv, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - imm := uint64(ins.Args[0].Imm) & 0xff - sv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <8 x i16>\n", sv, src) - dv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <8 x i16>\n", dv, dstv) - mask := make([]string, 0, 8) - for i := 0; i < 8; i++ { - if ((imm >> i) & 1) != 0 { - mask = append(mask, fmt.Sprintf("i32 %d", 8+i)) - } else { - mask = append(mask, fmt.Sprintf("i32 %d", i)) - } - } - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i16> %%%s, <8 x i16> %%%s, <8 x i32> <%s>\n", sh, dv, sv, strings.Join(mask, ", ")) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i16> %%%s to <16 x i8>\n", out, sh) - return true, false, c.storeX(ins.Args[2].Reg, "%"+out) - - case "SHA256MSG1", "SHA256MSG2": - // Approximate scheduling helpers as per-lane adds to keep SSA flow. + case "PUNPCKHQDQ": + // PUNPCKHQDQ Xsrc/m128, Xdst interleaves the high quadwords: + // dst = {dst[127:64], src[127:64]}. if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) + return true, false, fmt.Errorf("amd64 PUNPCKHQDQ expects Xsrc/m128, Xdst: %q", ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil @@ -1806,48 +778,94 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err if err != nil { return true, false, err } - s32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", s32, src) - d32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", d32, dstv) - add := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add <4 x i32> %%%s, %%%s\n", add, d32, s32) + dstQ := c.newTmp() + srcQ := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", dstQ, dstv) + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", srcQ, src) + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %%%s, <2 x i64> %%%s, <2 x i32> \n", sh, dstQ, srcQ) out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, add) + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", out, sh) return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - case "SHA1NEXTE", "SHA1MSG1", "SHA1MSG2": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil + case "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ": + if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects src1, src2, dst: %q", op, ins.Raw) } - src, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err + if _, ok := amd64ParseXReg(ins.Args[2].Reg); ok { + a, err := c.loadXVecOperand(ins.Args[0]) + if err != nil { + return true, false, err + } + b, err := c.loadXVecOperand(ins.Args[1]) + if err != nil { + return true, false, err + } + laneType := "i64" + lanes := 2 + mask := "" + if op == "VPUNPCKLQDQ" { + mask = "" + } else if op == "VPUNPCKLDQ" { + laneType, lanes = "i32", 4 + mask = "" + } else if op == "VPUNPCKHDQ" { + laneType, lanes = "i32", 4 + mask = "" + } + aq := c.newTmp() + bq := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <%d x %s>\n", aq, a, lanes, laneType) + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <%d x %s>\n", bq, b, lanes, laneType) + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x %s> %%%s, <%d x %s> %%%s, <%d x i32> %s\n", sh, lanes, laneType, bq, lanes, laneType, aq, lanes, mask) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x %s> %%%s to <16 x i8>\n", out, lanes, laneType, sh) + return true, false, c.storeX(ins.Args[2].Reg, "%"+out) } - dstv, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err + if _, ok := amd64ParseYReg(ins.Args[2].Reg); ok { + a, err := c.loadYVecOperand(ins.Args[0]) + if err != nil { + return true, false, err + } + b, err := c.loadYVecOperand(ins.Args[1]) + if err != nil { + return true, false, err + } + laneType := "i64" + lanes := 4 + mask := "" + if op == "VPUNPCKLQDQ" { + mask = "" + } else if op == "VPUNPCKLDQ" { + laneType, lanes = "i32", 8 + mask = "" + } else if op == "VPUNPCKHDQ" { + laneType, lanes = "i32", 8 + mask = "" + } + aq := c.newTmp() + bq := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <%d x %s>\n", aq, a, lanes, laneType) + fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <%d x %s>\n", bq, b, lanes, laneType) + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x %s> %%%s, <%d x %s> %%%s, <%d x i32> %s\n", sh, lanes, laneType, bq, lanes, laneType, aq, lanes, mask) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x %s> %%%s to <32 x i8>\n", out, lanes, laneType, sh) + return true, false, c.storeY(ins.Args[2].Reg, "%"+out) } - s32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", s32, src) - d32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", d32, dstv) - add := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add <4 x i32> %%%s, %%%s\n", add, d32, s32) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, add) - return true, false, c.storeX(ins.Args[1].Reg, "%"+out) + return false, false, nil - case "SHA1RNDS4": + case "SHUFPS": + // SHUFPS $imm, Xsrc, Xdst if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 SHA1RNDS4 expects $imm, Xsrc, Xdst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 SHUFPS expects $imm, Xsrc, Xdst: %q", ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { return false, false, nil } + imm := uint64(ins.Args[0].Imm) & 0xff + idx := func(k uint) uint64 { return (imm >> (2 * k)) & 3 } src, err := c.loadXVecOperand(ins.Args[1]) if err != nil { return true, false, err @@ -1856,42 +874,50 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err if err != nil { return true, false, err } - s32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", s32, src) - d32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", d32, dstv) - add := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add <4 x i32> %%%s, %%%s\n", add, d32, s32) - immv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add <4 x i32> %%%s, \n", immv, add, ins.Args[0].Imm, ins.Args[0].Imm, ins.Args[0].Imm, ins.Args[0].Imm) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, immv) - return true, false, c.storeX(ins.Args[2].Reg, "%"+out) + ds := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", ds, dstv) + ss := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", ss, src) + mask := fmt.Sprintf("<4 x i32> ", idx(0), idx(1), 4+idx(2), 4+idx(3)) + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i32> %%%s, <4 x i32> %%%s, %s\n", sh, ds, ss, mask) + bc := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bc, sh) + return true, false, c.storeX(ins.Args[2].Reg, "%"+bc) - case "SHA256RNDS2": - // SHA256RNDS2 Xsrc, Xstate, Xdst - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 SHA256RNDS2 expects Xsrc, Xstate, Xdst: %q", ins.Raw) + case "PBLENDW": + // PBLENDW $imm, Xsrc, Xdst + if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("amd64 PBLENDW expects $imm, Xsrc, Xdst: %q", ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { return false, false, nil } - a, err := c.loadXVecOperand(ins.Args[0]) + src, err := c.loadXVecOperand(ins.Args[1]) if err != nil { return true, false, err } - b, err := c.loadXVecOperand(ins.Args[1]) + dstv, err := c.loadX(ins.Args[2].Reg) if err != nil { return true, false, err } - a32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", a32, a) - b32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", b32, b) - add := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add <4 x i32> %%%s, %%%s\n", add, a32, b32) + imm := uint64(ins.Args[0].Imm) & 0xff + sv := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <8 x i16>\n", sv, src) + dv := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <8 x i16>\n", dv, dstv) + mask := make([]string, 0, 8) + for i := 0; i < 8; i++ { + if ((imm >> i) & 1) != 0 { + mask = append(mask, fmt.Sprintf("i32 %d", 8+i)) + } else { + mask = append(mask, fmt.Sprintf("i32 %d", i)) + } + } + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i16> %%%s, <8 x i16> %%%s, <8 x i32> <%s>\n", sh, dv, sv, strings.Join(mask, ", ")) out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, add) + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i16> %%%s to <16 x i8>\n", out, sh) return true, false, c.storeX(ins.Args[2].Reg, "%"+out) case "VMOVDQU64": @@ -2152,138 +1178,9 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err } return false, false, nil - case "VPMOVMSKB": - // VPMOVMSKB Ysrc, dstReg - if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPMOVMSKB expects Ysrc, dstReg: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - v, err := c.loadY(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - // Work around LLVM backend issues with the AVX2 pmovmskb intrinsic by - // splitting the 256-bit vector into two 128-bit halves and using the SSE2 - // pmovmskb.128 intrinsic. - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", lo, v, llvmI32RangeMask(0, 16)) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", hi, v, llvmI32RangeMask(16, 16)) - ml := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %%%s)\n", ml, lo) - mh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %%%s)\n", mh, hi) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, 16\n", sh, mh) - or := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i32 %%%s, %%%s\n", or, sh, ml) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, or) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+z) - - case "MOVOU", "MOVOA": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) - } - if ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - dst := ins.Args[1].Reg - switch ins.Args[0].Kind { - case OpReg: - v, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - return true, false, c.storeX(dst, v) - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", ld, p) - return true, false, c.storeX(dst, "%"+ld) - case OpSym: - p, err := c.ptrFromSB(ins.Args[0].Sym) - if err != nil { - return true, false, err - } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", ld, p) - return true, false, c.storeX(dst, "%"+ld) - default: - return true, false, fmt.Errorf("amd64 %s unsupported src: %q", op, ins.Raw) - } - } - - if ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects Xsrc for store form: %q", op, ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - srcv, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - switch ins.Args[1].Kind { - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s, align 1\n", srcv, p) - return true, false, nil - case OpSym: - p, err := c.ptrFromSB(ins.Args[1].Sym) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s, align 1\n", srcv, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 %s unsupported dst: %q", op, ins.Raw) - } - - case "PXOR", "POR", "PAND", "PANDN": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - src, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - dstv, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - t := c.newTmp() - if op == "PXOR" { - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", t, dstv, src) - } else if op == "POR" { - fmt.Fprintf(c.b, " %%%s = or <16 x i8> %s, %s\n", t, dstv, src) - } else if op == "PANDN" { - notv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", notv, dstv, llvmAllOnesI8Vec(16)) - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %%%s, %s\n", t, notv, src) - } else { - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %s, %s\n", t, dstv, src) - } - return true, false, c.storeX(ins.Args[1].Reg, "%"+t) - - case "PADDL", "PADDD", "PADDQ", "PSUBB", "PSUBL": - // Two-operand packed arithmetic: dst = dst (+|-) src. + case "PABSD": if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) + return true, false, fmt.Errorf("amd64 PABSD expects Xsrc, Xdst: %q", ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil @@ -2292,272 +1189,113 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err if err != nil { return true, false, err } - dstv, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - vecTy := "<4 x i32>" - if op == "PSUBB" { - vecTy = "<16 x i8>" - } else if op == "PADDQ" { - vecTy = "<2 x i64>" - } - as := src - bs := dstv - if vecTy != "<16 x i8>" { - at := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", at, src, vecTy) - as = "%" + at - bt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", bt, dstv, vecTy) - bs = "%" + bt - } - x := c.newTmp() - if op == "PADDL" || op == "PADDD" || op == "PADDQ" { - fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", x, vecTy, bs, as) - } else { - fmt.Fprintf(c.b, " %%%s = sub %s %s, %s\n", x, vecTy, bs, as) - } - if vecTy == "<16 x i8>" { - return true, false, c.storeX(ins.Args[1].Reg, "%"+x) - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", out, vecTy, x) - return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - - case "PSLLW", "PSRLW", "PSLLL", "PSRLL", "PSRAL": - // Two-operand packed integer shift immediate. - if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, Xdst: %q", op, ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - laneBits := int64(32) - vecTy := "<4 x i32>" - if op == "PSLLW" || op == "PSRLW" { - laneBits = 16 - vecTy = "<8 x i16>" - } - n := ins.Args[0].Imm - v, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", bc, v, vecTy) - if n < 0 { - return true, false, fmt.Errorf("amd64 %s shift count must be non-negative: %q", op, ins.Raw) - } - if n >= laneBits { - if op != "PSRAL" { - return true, false, c.storeX(ins.Args[1].Reg, "zeroinitializer") - } - // Packed arithmetic right shifts saturate an oversized count to - // laneBits-1, producing an all-sign-bit lane instead of zero. - n = laneBits - 1 - } - sh := c.newTmp() - switch op { - case "PSLLW": - fmt.Fprintf(c.b, " %%%s = shl <8 x i16> %%%s, \n", sh, bc, n, n, n, n, n, n, n, n) - case "PSRLW": - fmt.Fprintf(c.b, " %%%s = lshr <8 x i16> %%%s, \n", sh, bc, n, n, n, n, n, n, n, n) - case "PSLLL": - fmt.Fprintf(c.b, " %%%s = shl <4 x i32> %%%s, \n", sh, bc, n, n, n, n) - case "PSRLL": - fmt.Fprintf(c.b, " %%%s = lshr <4 x i32> %%%s, \n", sh, bc, n, n, n, n) - case "PSRAL": - fmt.Fprintf(c.b, " %%%s = ashr <4 x i32> %%%s, \n", sh, bc, n, n, n, n) - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", out, vecTy, sh) - return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - - case "PCMPEQL": - if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PCMPEQL expects Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - src, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - dstv, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - as := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", as, src) - bs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", bs, dstv) - cmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq <4 x i32> %%%s, %%%s\n", cmp, bs, as) - sext := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext <4 x i1> %%%s to <4 x i32>\n", sext, cmp) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, sext) - return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - - case "VPCLMULQDQ": - if len(ins.Args) != 4 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg || ins.Args[3].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPCLMULQDQ expects $imm, Zsrc, Zdst, Zout: %q", ins.Raw) - } - if _, ok := amd64ParseZReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseZReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseZReg(ins.Args[3].Reg); !ok { - return false, false, nil - } - imm := ins.Args[0].Imm & 0xff - src, err := c.loadZ(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - dstv, err := c.loadZ(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - bd := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <64 x i8> %s to <8 x i64>\n", bd, dstv) - bs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <64 x i8> %s to <8 x i64>\n", bs, src) - lanes := make([]string, 0, 4) - for i := 0; i < 4; i++ { - dlane := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i64> %%%s, <8 x i64> zeroinitializer, <2 x i32> \n", dlane, bd, i*2, i*2+1) - slane := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i64> %%%s, <8 x i64> zeroinitializer, <2 x i32> \n", slane, bs, i*2, i*2+1) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %%%s, <2 x i64> %%%s, i8 %d)\n", call, dlane, slane, imm) - lanes = append(lanes, "%"+call) - } - merged0 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <4 x i32> \n", merged0, lanes[0], lanes[1]) - merged1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <4 x i32> \n", merged1, lanes[2], lanes[3]) - merged := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i64> %%%s, <4 x i64> %%%s, <8 x i32> \n", merged, merged0, merged1) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i64> %%%s to <64 x i8>\n", out, merged) - return true, false, c.storeZ(ins.Args[3].Reg, "%"+out) - - case "VPTERNLOGD": - if len(ins.Args) != 4 || ins.Args[0].Kind != OpImm || ins.Args[3].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPTERNLOGD expects $imm, src1, src2, dst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[3].Reg); ok { - if ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - } - if ins.Args[2].Kind == OpReg { - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - } - if ins.Args[0].Imm != 0x96 { - return true, false, fmt.Errorf("amd64 VPTERNLOGD only supports imm 0x96 for now: %q", ins.Raw) - } - a, err := c.loadYVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - b, err := c.loadYVecOperand(ins.Args[2]) - if err != nil { - return true, false, err - } - dstv, err := c.loadY(ins.Args[3].Reg) - if err != nil { - return true, false, err - } - x1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <32 x i8> %s, %s\n", x1, dstv, a) - x2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <32 x i8> %%%s, %s\n", x2, x1, b) - return true, false, c.storeY(ins.Args[3].Reg, "%"+x2) - } - if _, ok := amd64ParseZReg(ins.Args[3].Reg); !ok { - return false, false, nil - } - if ins.Args[0].Imm != 0x96 { - return true, false, fmt.Errorf("amd64 VPTERNLOGD only supports imm 0x96 for now: %q", ins.Raw) - } - a, err := c.loadZVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - b, err := c.loadZVecOperand(ins.Args[2]) - if err != nil { - return true, false, err - } - dstv, err := c.loadZ(ins.Args[3].Reg) - if err != nil { - return true, false, err - } - x1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <64 x i8> %s, %s\n", x1, dstv, a) - x2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <64 x i8> %%%s, %s\n", x2, x1, b) - return true, false, c.storeZ(ins.Args[3].Reg, "%"+x2) + values := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", values, src) + negative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <4 x i32> %%%s, zeroinitializer\n", negative, values) + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub <4 x i32> zeroinitializer, %%%s\n", negated, values) + absolute := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <4 x i1> %%%s, <4 x i32> %%%s, <4 x i32> %%%s\n", absolute, negative, negated, values) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, absolute) + return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - case "VEXTRACTF32X4": - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VEXTRACTF32X4 expects $imm, Zsrc, Xdst: %q", ins.Raw) + case "PMINUB", "PMINSB", "PMINUW", "PMINSW", "PMINUD", "PMINSD", + "PMAXUB", "PMAXSB", "PMAXUW", "PMAXSW", "PMAXUD", "PMAXSD": + return c.lowerPackedIntegerMinMax(rawOp, ins) + + case "PSLLW", "PSRLW", "PSLLL", "PSRLL", "PSRAL": + // Two-operand packed integer shift immediate. + if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects $imm, Xdst: %q", op, ins.Raw) } - if _, ok := amd64ParseZReg(ins.Args[1].Reg); !ok { + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil + laneBits := int64(32) + vecTy := "<4 x i32>" + if op == "PSLLW" || op == "PSRLW" { + laneBits = 16 + vecTy = "<8 x i16>" } - idx := int(ins.Args[0].Imm & 0x3) - src, err := c.loadZ(ins.Args[1].Reg) + n := ins.Args[0].Imm + v, err := c.loadX(ins.Args[1].Reg) if err != nil { return true, false, err } + bc := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", bc, v, vecTy) + if n < 0 { + return true, false, fmt.Errorf("amd64 %s shift count must be non-negative: %q", op, ins.Raw) + } + if n >= laneBits { + if op != "PSRAL" { + return true, false, c.storeX(ins.Args[1].Reg, "zeroinitializer") + } + // Packed arithmetic right shifts saturate an oversized count to + // laneBits-1, producing an all-sign-bit lane instead of zero. + n = laneBits - 1 + } + sh := c.newTmp() + switch op { + case "PSLLW": + fmt.Fprintf(c.b, " %%%s = shl <8 x i16> %%%s, \n", sh, bc, n, n, n, n, n, n, n, n) + case "PSRLW": + fmt.Fprintf(c.b, " %%%s = lshr <8 x i16> %%%s, \n", sh, bc, n, n, n, n, n, n, n, n) + case "PSLLL": + fmt.Fprintf(c.b, " %%%s = shl <4 x i32> %%%s, \n", sh, bc, n, n, n, n) + case "PSRLL": + fmt.Fprintf(c.b, " %%%s = lshr <4 x i32> %%%s, \n", sh, bc, n, n, n, n) + case "PSRAL": + fmt.Fprintf(c.b, " %%%s = ashr <4 x i32> %%%s, \n", sh, bc, n, n, n, n) + } out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <64 x i8> %s, <64 x i8> zeroinitializer, <16 x i32> %s\n", out, src, llvmI32RangeMask(idx*16, 16)) - return true, false, c.storeX(ins.Args[2].Reg, "%"+out) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", out, vecTy, sh) + return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - case "PCLMULQDQ": - // PCLMULQDQ $imm, Xsrc, Xdst => Xdst = pclmul(Xdst, Xsrc, imm) - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PCLMULQDQ expects $imm, Xsrc, Xdst: %q", ins.Raw) + case "PCMPEQW", "PCMPEQL": + if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + if _, ok := amd64ParseXReg(ins.Args[0].Reg); !ok { return false, false, nil } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil } - imm := ins.Args[0].Imm & 0xff - src, err := c.loadX(ins.Args[1].Reg) + src, err := c.loadX(ins.Args[0].Reg) if err != nil { return true, false, err } - dstv, err := c.loadX(ins.Args[2].Reg) + dstv, err := c.loadX(ins.Args[1].Reg) if err != nil { return true, false, err } - bd := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bd, dstv) + laneType := "i32" + lanes := 4 + if op == "PCMPEQW" { + laneType = "i16" + lanes = 8 + } + vecType := fmt.Sprintf("<%d x %s>", lanes, laneType) + as := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", as, src, vecType) bs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bs, src) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %%%s, <2 x i64> %%%s, i8 %d)\n", call, bd, bs, imm) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, call) - return true, false, c.storeX(ins.Args[2].Reg, "%"+bc) + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", bs, dstv, vecType) + cmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %%%s\n", cmp, vecType, bs, as) + sext := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext <%d x i1> %%%s to %s\n", sext, lanes, cmp, vecType) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", out, vecType, sext) + return true, false, c.storeX(ins.Args[1].Reg, "%"+out) + + case "VPCLMULQDQ": + return c.lowerPackedCarrylessMultiply(rawOp, ins) + + case "PCLMULQDQ": + return c.lowerPackedCarrylessMultiply(rawOp, ins) case "PCMPEQB", "PCMPGTB": if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { @@ -2594,146 +1332,6 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = select <16 x i1> %%%s, <16 x i8> %s, <16 x i8> zeroinitializer\n", sel, cmp, allOnes) return true, false, c.storeX(ins.Args[1].Reg, "%"+sel) - case "PMOVMSKB": - // PMOVMSKB Xsrc, dstReg - if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PMOVMSKB expects Xsrc, dstReg: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - v, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %s)\n", call, v) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+z) - - case "PSHUFB": - // PSHUFB Xmask, Xdst - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSHUFB expects Xmask, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - mask, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - dst, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %s, <16 x i8> %s)\n", call, dst, mask) - return true, false, c.storeX(ins.Args[1].Reg, "%"+call) - - case "PINSRQ": - // PINSRQ $imm, src64, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PINSRQ expects $imm, src, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - idx := ins.Args[0].Imm & 1 - src, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bc, dst) - insv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> %%%s, i64 %s, i32 %d\n", insv, bc, src, idx) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", out, insv) - return true, false, c.storeX(ins.Args[2].Reg, "%"+out) - - case "PINSRD": - // PINSRD $imm, src32, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PINSRD expects $imm, src, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - idx := ins.Args[0].Imm & 3 - src64, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - src32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", src32, src64) - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", bc, dst) - insv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> %%%s, i32 %%%s, i32 %d\n", insv, bc, src32, idx) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, insv) - return true, false, c.storeX(ins.Args[2].Reg, "%"+out) - - case "PINSRW": - // PINSRW $imm, src16, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PINSRW expects $imm, src, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - idx := ins.Args[0].Imm & 7 - src64, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - src16 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i16\n", src16, src64) - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <8 x i16>\n", bc, dst) - insv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <8 x i16> %%%s, i16 %%%s, i32 %d\n", insv, bc, src16, idx) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i16> %%%s to <16 x i8>\n", out, insv) - return true, false, c.storeX(ins.Args[2].Reg, "%"+out) - - case "PINSRB": - // PINSRB $imm, src8, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PINSRB expects $imm, src, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - idx := ins.Args[0].Imm & 15 - src64, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - src8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", src8, src64) - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - insv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <16 x i8> %s, i8 %%%s, i32 %d\n", insv, dst, src8, idx) - return true, false, c.storeX(ins.Args[2].Reg, "%"+insv) - case "PEXTRB": // PEXTRB $imm, Xsrc, dst if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { @@ -2806,84 +1404,27 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err } case "PALIGNR": - // PALIGNR $imm, Xsrc, Xdst ; dst = alignr(dst, src, imm) - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PALIGNR expects $imm, Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm - if n < 0 { - n = 0 - } - if n > 255 { - n = 255 - } - src, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> %s, <16 x i32> %s\n", sh, dst, src, llvmAlignRightBytesMask(n)) - return true, false, c.storeX(ins.Args[2].Reg, "%"+sh) + return c.lowerPackedAlignRight(rawOp, ins) - case "PSRLDQ": - // PSRLDQ $imm, Xdst - if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSRLDQ expects $imm, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm - if n < 0 || n > 16 { - return true, false, fmt.Errorf("amd64 PSRLDQ invalid imm %d: %q", n, ins.Raw) - } - v, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - shuf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, <16 x i32> %s\n", shuf, v, llvmShiftRightBytesMask(n)) - return true, false, c.storeX(ins.Args[1].Reg, "%"+shuf) + case "PSRLDQ", "PSLLDQ": + return c.lowerPackedByteShift(rawOp, ins) - case "PSLLDQ": - // PSLLDQ $imm, Xdst + case "PSRLQ", "PSLLQ": + // PSR/LLQ $imm, Xdst (shift each 64-bit lane). Counts at or above + // the lane width clear the destination rather than wrapping modulo 64. if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSLLDQ expects $imm, Xdst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 %s expects $imm, Xdst: %q", op, ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil } n := ins.Args[0].Imm - if n < 0 || n > 16 { - return true, false, fmt.Errorf("amd64 PSLLDQ invalid imm %d: %q", n, ins.Raw) - } - v, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - shuf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, <16 x i32> %s\n", shuf, v, llvmShiftLeftBytesMask(n)) - return true, false, c.storeX(ins.Args[1].Reg, "%"+shuf) - - case "PSRLQ": - // PSRLQ $imm, Xdst (shift each 64-bit lane right) - if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSRLQ expects $imm, Xdst: %q", ins.Raw) + if n < 0 { + return true, false, fmt.Errorf("amd64 %s shift count must be non-negative: %q", op, ins.Raw) } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil + if n >= 64 { + return true, false, c.storeX(ins.Args[1].Reg, "zeroinitializer") } - n := ins.Args[0].Imm & 63 v, err := c.loadX(ins.Args[1].Reg) if err != nil { return true, false, err @@ -2891,7 +1432,11 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err bc := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bc, v) sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr <2 x i64> %%%s, \n", sh, bc, n, n) + shiftOp := "lshr" + if op == "PSLLQ" { + shiftOp = "shl" + } + fmt.Fprintf(c.b, " %%%s = %s <2 x i64> %%%s, \n", sh, shiftOp, bc, n, n) back := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", back, sh) return true, false, c.storeX(ins.Args[1].Reg, "%"+back) @@ -2935,6 +1480,67 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err return false, false, nil } +func (c *amd64Ctx) validateMOVDAliasForm(ins Instr) error { + if len(ins.Args) != 2 { + return fmt.Errorf("%s MOVD expects 2 operands: %q", c.goarch, ins.Raw) + } + src, dst := ins.Args[0], ins.Args[1] + srcMem, dstMem := isAMD64MemoryOperand(src), isAMD64MemoryOperand(dst) + srcX, dstX := isAMD64MOVDXRegister(src), isAMD64MOVDXRegister(dst) + srcM, dstM := isAMD64MOVDMRegister(src), isAMD64MOVDMRegister(dst) + srcGP, dstGP := c.isGoYrlRegister(src), c.isGoYrlRegister(dst) + srcAddress := x86GoMOVQAddressSource(src, c.goarch) + + valid := false + if c.goarch == "386" { + // In 32-bit mode AMOVQ has only its memory/MMX/XMM rows. Go rejects + // every GP and immediate form as an illegal 64-bit instruction. + valid = (dstMem && (srcX || srcM)) || + (dstX && (srcMem || srcX)) || + (dstM && (srcMem || srcX)) + } else { + valid = (dstGP && (src.Kind == OpImm || srcAddress || srcGP || srcMem || srcX || srcM)) || + (dstMem && (src.Kind == OpImm || srcGP || srcX || srcM)) || + (dstX && (srcGP || srcMem || srcX)) || + (dstM && (srcGP || srcMem || srcX || srcM)) + } + if !valid { + return fmt.Errorf("%s MOVD operands are outside Go 1.27's MOVD/ymovq forms: %q", c.goarch, ins.Raw) + } + return nil +} + +func isAMD64MOVDXRegister(op Operand) bool { + if op.Kind != OpReg { + return false + } + _, ok := amd64ParseXReg(op.Reg) + return ok +} + +func isAMD64MOVDMRegister(op Operand) bool { + if op.Kind != OpReg { + return false + } + _, ok := amd64ParseMReg(op.Reg) + return ok +} + +func (c *amd64Ctx) isGoYrlRegister(op Operand) bool { + if op.Kind != OpReg || !isAMD64YrlRegister(op.Reg) { + return false + } + if c.goarch != "386" { + return true + } + switch op.Reg { + case AX, BX, CX, DX, SP, BP, SI, DI: + return true + default: + return false + } +} + func llvmShiftRightBytesMask(n int64) string { // shufflevector mask for right shift by n bytes. // Use second vector's element 0 (index 16) as the "zero" source. @@ -2975,29 +1581,6 @@ func llvmShiftLeftBytesMask(n int64) string { return sb.String() } -func llvmAlignRightBytesMask(n int64) string { - // shufflevector mask for alignr(dst, src, n): - // result bytes are selected from concatenation [dst, src], right shifted by n. - var sb strings.Builder - sb.WriteString("<") - for i := 0; i < 16; i++ { - if i != 0 { - sb.WriteString(", ") - } - idx := int64(i) + n - switch { - case idx < 16: - fmt.Fprintf(&sb, "i32 %d", idx) - case idx < 32: - fmt.Fprintf(&sb, "i32 %d", idx) - default: - sb.WriteString("i32 16") - } - } - sb.WriteString(">") - return sb.String() -} - func llvmRepeatI8Mask(chunk, width int) string { if chunk <= 0 { chunk = 1 @@ -3014,6 +1597,45 @@ func llvmRepeatI8Mask(chunk, width int) string { return sb.String() } +func llvmSplatInteger(lanes, bits int, value uint64) string { + return llvmSplatIntegerText(lanes, bits, strconv.FormatUint(value, 10)) +} + +func llvmSplatIntegerText(lanes, bits int, value string) string { + if lanes <= 0 { + return "<>" + } + element := "i" + strconv.Itoa(bits) + " " + value + var b strings.Builder + // The two brackets replace the separator omitted before the first lane. + b.Grow(lanes * (len(element) + 2)) + b.WriteByte('<') + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + b.WriteString(", ") + } + b.WriteString(element) + } + b.WriteByte('>') + return b.String() +} + +func amd64SplatInteger(c *amd64Ctx, lanes, bits int, value string) string { + seed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> poison, i%d %s, i32 0\n", seed, lanes, bits, bits, value) + splat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %%%s, <%d x i%d> poison, <%d x i32> zeroinitializer\n", splat, lanes, bits, seed, lanes, bits, lanes) + return "%" + splat +} + +func amd64SplatI1(c *amd64Ctx, lanes int, value string) string { + seed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i1> poison, i1 %s, i32 0\n", seed, lanes, value) + splat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i1> %%%s, <%d x i1> poison, <%d x i32> zeroinitializer\n", splat, lanes, seed, lanes, lanes) + return "%" + splat +} + func llvmAllOnesI8Vec(n int) string { if n <= 0 { return "<>" @@ -3035,6 +1657,16 @@ func isAMD64ZReg(r Reg) bool { return ok } +func isAMD64XReg(r Reg) bool { + _, ok := amd64ParseXReg(r) + return ok +} + +func isAMD64YReg(r Reg) bool { + _, ok := amd64ParseYReg(r) + return ok +} + func llvmI32RangeMask(start int, n int) string { // Build . if n <= 0 { @@ -3074,6 +1706,40 @@ func amd64MaskBitI1(c *amd64Ctx, mask string, idx int) string { return "%" + bit } +func amd64ApplyI32LaneMask(c *amd64Ctx, lanes int, computed, old, mask string, zeroing bool) string { + return amd64ApplyIntegerLaneMask(c, lanes, 32, computed, old, mask, zeroing) +} + +func amd64ApplyIntegerLaneMask(c *amd64Ctx, lanes, bits int, computed, old, mask string, zeroing bool) string { + fallback := old + if zeroing { + fallback = "zeroinitializer" + } + if lanes <= 0 { + return fallback + } + + // Each predicate lane selects its own numerical mask bit. A scalar-to- + // vector bitcast would instead depend on the target's byte order. Typed + // callers supply at most 64 lanes, so every logical shift is below 64. + splat := amd64SplatInteger(c, lanes, 64, mask) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr <%d x i64> %s, <", shifted, lanes, splat) + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + c.b.WriteString(", ") + } + fmt.Fprintf(c.b, "i64 %d", lane) + } + c.b.WriteString(">\n") + predicate := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc <%d x i64> %%%s to <%d x i1>\n", predicate, lanes, shifted, lanes) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %s, <%d x i%d> %s\n", + selected, lanes, predicate, lanes, bits, computed, lanes, bits, fallback) + return "%" + selected +} + func amd64PackI1x8ToI64(c *amd64Ctx, pred string) string { acc := "0" for i := 0; i < 8; i++ { diff --git a/amd64_lower_vecflags.go b/amd64_lower_vecflags.go new file mode 100644 index 000000000..0147cc6c2 --- /dev/null +++ b/amd64_lower_vecflags.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64VectorTestSpec struct { + maxBytes int + vector bool + signLaneBits int +} + +// amd64VectorTestSpecs models the complete Go 1.27 PTEST/VPTEST/VTEST grammar. +// PTEST uses the legacy yxm_q4 X/m128 row. The VEX instructions share +// _yvptest's X/m128,X and Y/m256,Y rows. VTESTPD and VTESTPS consider only the +// sign bit in each floating-point lane; PTEST and VPTEST consider every bit. +var amd64VectorTestSpecs = map[Op]amd64VectorTestSpec{ + "PTEST": {maxBytes: 16}, + "VPTEST": {maxBytes: 32, vector: true}, + "VTESTPD": {maxBytes: 32, vector: true, signLaneBits: 64}, + "VTESTPS": {maxBytes: 32, vector: true, signLaneBits: 32}, +} + +type x86VectorTestForm struct { + amd64VectorTestSpec + byteWidth int +} + +// The lowerer and numerical observer share operand shape only. Original +// memory backing, initialized bytes and physical mode remain separate roles. +func x86VectorTestProperties(op Op, ins Instr, arch string) (x86VectorTestForm, bool, error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64VectorTestSpecs[Op(baseOp)] + form := x86VectorTestForm{amd64VectorTestSpec: spec} + if !recognized { + return form, false, nil + } + if rawOp != baseOp || arch != "amd64" && arch != "386" || + ins.Op != "" && !strings.EqualFold(string(ins.Op), baseOp) { + return form, true, fmt.Errorf("%s %s has an unbound mode, dispatch or suffix: %q", arch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return form, true, fmt.Errorf("%s %s expects vector/memory source and vector destination: %q", arch, baseOp, ins.Raw) + } + + source, destination := ins.Args[0], ins.Args[1] + byteWidth := amd64VectorByteWidth(destination.Reg) + form.byteWidth = byteWidth + legacy := func(reg Reg) bool { + index, valid := amd64ParseXReg(reg) + return valid && index < 16 && (arch != "386" || index < 8) + } + if (byteWidth != 16 && byteWidth != 32) || byteWidth > spec.maxBytes { + return form, true, fmt.Errorf("%s %s destination width is outside its Go 1.27 table: %q", arch, baseOp, ins.Raw) + } + if spec.vector { + // Go's VEX classes admit X/Y0-15 even in 386 mode. They also admit + // extended address registers there, so do not apply the legacy 386 + // register filter to VEX memory operands. + if !amd64VEXVectorRegister(destination, byteWidth) { + return form, true, fmt.Errorf("%s %s destination is outside Go's VEX X/Y class: %q", arch, baseOp, ins.Raw) + } + } else if byteWidth != 16 || !legacy(destination.Reg) { + return form, true, fmt.Errorf("%s %s destination is outside Go's legacy X class: %q", arch, baseOp, ins.Raw) + } + + if source.Kind == OpReg { + if spec.vector { + if !amd64VEXVectorRegister(source, byteWidth) { + return form, true, fmt.Errorf("%s %s source must match the destination's VEX width: %q", arch, baseOp, ins.Raw) + } + } else if byteWidth != 16 || !legacy(source.Reg) { + return form, true, fmt.Errorf("%s %s source is outside Go's legacy X class: %q", arch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return form, true, fmt.Errorf("%s %s source must be a same-width vector register or memory: %q", arch, baseOp, ins.Raw) + } + if !spec.vector && source.Kind == OpMem && !x86MemoryRegistersValidForArch(source.Mem, arch) { + return form, true, fmt.Errorf("%s %s source uses an out-of-range legacy address register: %q", arch, baseOp, ins.Raw) + } + return form, true, nil +} + +func x86TypedVectorTestByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86VectorTestProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.byteWidth) + return x86TypedByteEffect{requiresResultProof: true, flagsDefined: x86ByteFlagsAll, + operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: width, read: true}, + {index: 1, operand: ins.Args[1], width: width, read: true}, + }}, true, nil +} + +func (c *amd64Ctx) lowerVectorTest(op Op, ins Instr) (ok bool, terminated bool, err error) { + arch := c.goarch + if arch == "" { + arch = "amd64" // Preserve the existing direct-emitter default. + } + form, known, err := x86VectorTestProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + source, destination := ins.Args[0], ins.Args[1] + spec, byteWidth := form.amd64VectorTestSpec, form.byteWidth + if source.Kind != OpReg && source.Kind != OpFP { + if source.Kind == OpMem && !c.nativeMemoryOperand(ins, 0) && + !x86CallerMemoryCertified(ins, 0) && !c.fixedLocalMemoryCertified(ins, 0) { + return true, false, fmt.Errorf("%w: vector test needs its original native memory binding", ErrProbeNeedsContext) + } + return true, false, c.emitVectorTestNativeMemory(ins, form, arch) + } + if source.Kind == OpFP { + if !spec.vector { + return true, false, fmt.Errorf("%w: legacy PTEST needs its original memory envelope", ErrProbeNeedsContext) + } + if err := c.validateX86PackedShiftFPSpan(source.FPOffset, byteWidth); err != nil { + return true, false, err + } + } + sourceBytes, err := c.loadX86PackedShiftBytes(source, byteWidth, true) + if err != nil { + return true, false, err + } + destinationBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + + intersection := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and <%d x i8> %s, %s\n", intersection, byteWidth, destinationBytes, sourceBytes) + invertedDestination := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor <%d x i8> %s, %s\n", invertedDestination, byteWidth, destinationBytes, llvmAllOnesI8Vec(byteWidth)) + outsideDestination := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and <%d x i8> %%%s, %s\n", outsideDestination, byteWidth, invertedDestination, sourceBytes) + + var zero, carry string + if spec.signLaneBits == 0 { + // Avoid an i256 scalar: LLVM 22 aborts while legalizing that type for + // i386. Reducing individual bytes compiles on every supported target. + zero = c.emitVectorBytesAreZero(byteWidth, "%"+intersection) + carry = c.emitVectorBytesAreZero(byteWidth, "%"+outsideDestination) + } else { + zero = c.emitNoSignBits(byteWidth, spec.signLaneBits, "%"+intersection) + carry = c.emitNoSignBits(byteWidth, spec.signLaneBits, "%"+outsideDestination) + } + + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", carry, c.flagsCFSlot) + // All four instructions compute CF/ZF and clear AF/OF/SF/PF. + c.clearX86StatusFlags(x86StatusAll &^ (x86StatusCF | x86StatusZF)) + return true, false, nil +} + +func (c *amd64Ctx) emitVectorBytesAreZero(byteWidth int, bytes string) string { + // Extract bytes directly. LLVM 22's i386 backend also aborts on some + // otherwise-valid bitcasts between illegal 256-bit vector types. + combined := "0" + for index := 0; index < byteWidth; index++ { + element := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", element, byteWidth, bytes, index) + merged := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i8 %s, %%%s\n", merged, combined, element) + combined = "%" + merged + } + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i8 %s, 0\n", zero, combined) + return "%" + zero +} + +func (c *amd64Ctx) emitNoSignBits(byteWidth, laneBits int, bytes string) string { + lanes := byteWidth * 8 / laneBits + any := "false" + for lane := 0; lane < lanes; lane++ { + // X86 is little-endian, so each lane's sign bit is the high bit of + // its final byte. Testing that byte avoids illegal 256-bit vector + // bitcasts in LLVM 22's i386 legalizer. + index := (lane+1)*(laneBits/8) - 1 + element := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", element, byteWidth, bytes, index) + negative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i8 %%%s, 0\n", negative, element) + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %s, %%%s\n", combined, any, negative) + any = "%" + combined + } + none := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", none, any) + return "%" + none +} diff --git a/amd64_lower_vecflags_test.go b/amd64_lower_vecflags_test.go new file mode 100644 index 000000000..393d482eb --- /dev/null +++ b/amd64_lower_vecflags_test.go @@ -0,0 +1,296 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64VectorTestGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64VectorTestSpec{ + "PTEST": {maxBytes: 16}, + "VPTEST": {maxBytes: 32, vector: true}, + "VTESTPD": {maxBytes: 32, vector: true, signLaneBits: 64}, + "VTESTPS": {maxBytes: 32, vector: true, signLaneBits: 32}, + } + if len(amd64VectorTestSpecs) != len(expected) { + t.Fatalf("vector-test grammar has %d entries, want %d", len(amd64VectorTestSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64VectorTestSpecs[op]; !ok { + t.Errorf("vector-test grammar omitted %s", op) + } else if got != want { + t.Errorf("vector-test grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86VectorTestCompleteGoFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + source := fmt.Sprintf(`TEXT vectortestforms(SB),$0-0 + PTEST X0, X%d + PTEST 0(AX), X%d + VPTEST X0, X15 + VPTEST 16(R9), X15 + VPTEST Y0, Y15 + VPTEST 32(R9), Y15 + VTESTPD X0, X15 + VTESTPD 48(R9), X15 + VTESTPD Y0, Y15 + VTESTPD 64(R9), Y15 + VTESTPS X0, X15 + VTESTPS 80(R9), X15 + VTESTPS Y0, Y15 + VTESTPS 96(R9), Y15 + RET +`, legacyLast, legacyLast) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "vectortestforms": {Name: "vectortestforms", Ret: Void}, + }, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original vector-test formats invented register or native-pointer inputs", err) + } + ir = x86InstructionFormatObjectUnit(t, file, + map[string]FuncSig{"vectortestforms": {Name: "vectortestforms", Ret: Void}}, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerVectorTest(ins.Op, ins) }) + for _, want := range []string{"<16 x i8>", "<32 x i8>", "store i1"} { + if !strings.Contains(ir, want) { + t.Fatalf("vector-test lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "vector-test-"+target.name+".ll", "vector-test-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86VectorTestRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PTEST Y0, Y1"}, + {goarch: "amd64", instruction: "PTEST X0, 0(AX)"}, + {goarch: "amd64", instruction: "PTEST X16, X0"}, + {goarch: "amd64", instruction: "PTEST.Z X0, X1"}, + {goarch: "386", instruction: "PTEST X8, X0"}, + {goarch: "386", instruction: "PTEST 0(R9), X0"}, + {goarch: "amd64", instruction: "VPTEST X0, Y1"}, + {goarch: "amd64", instruction: "VPTEST Z0, Z1"}, + {goarch: "amd64", instruction: "VPTEST X16, X0"}, + {goarch: "amd64", instruction: "VPTEST.Z X0, X1"}, + {goarch: "amd64", instruction: "VTESTPD X0, Y1"}, + {goarch: "amd64", instruction: "VTESTPD Z0, Z1"}, + {goarch: "amd64", instruction: "VTESTPS X16, X0"}, + {goarch: "amd64", instruction: "VTESTPS X0, X1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside the Go 1.27 vector-test tables", test.instruction) + } + }) + } +} + +func TestAMD64VectorTestRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT vectortestsemantics(SB),$0-24 + MOVQ out+0(FP), CX + MOVQ source+8(FP), AX + MOVQ destination+16(FP), BX + MOVUPS 0(AX), X0 + MOVUPS 0(BX), X1 + MOVB $0x7f, DL + ADDB $1, DL + PTEST X0, X1 + SETEQ 0(CX) + SETCS 1(CX) + SETMI 2(CX) + SETOS 3(CX) + SETPS 4(CX) + + MOVQ source+8(FP), AX + MOVQ destination+16(FP), BX + VMOVDQU 32(BX), Y1 + XORB DL, DL + VPTEST 32(AX), Y1 + SETEQ 5(CX) + SETCS 6(CX) + SETMI 7(CX) + SETOS 8(CX) + SETPS 9(CX) + + MOVQ source+8(FP), AX + MOVQ destination+16(FP), BX + MOVUPS 64(BX), X1 + VTESTPS 64(AX), X1 + SETEQ 10(CX) + SETCS 11(CX) + SETMI 12(CX) + SETOS 13(CX) + SETPS 14(CX) + + MOVQ source+8(FP), AX + MOVQ destination+16(FP), BX + VMOVDQU 96(BX), Y1 + VTESTPD 96(AX), Y1 + SETEQ 15(CX) + SETCS 16(CX) + SETMI 17(CX) + SETOS 18(CX) + SETPS 19(CX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"vectortestsemantics": { + Name: "vectortestsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void vectortestsemantics(uint8_t *, const uint8_t *, const uint8_t *); +int main(void) { + uint8_t source[128] = {0}, destination[128] = {0}, out[20] = {0}; + source[0] = 1; + source[32] = 1; destination[32] = 1; + source[64] = 1; destination[64] = 1; + source[103] = 0x80; + vectortestsemantics(out, source, destination); + const uint8_t want[20] = { + 1, 0, 0, 0, 0, + 0, 1, 0, 0, 0, + 1, 1, 0, 0, 0, + 1, 0, 0, 0, 0, + }; + for (int i = 0; i < 20; i++) if (out[i] != want[i]) return 10 + i; + return 0; +} +` + compiler := []string{clang} + if triple != testTargetTriple(runtime.GOOS, runtime.GOARCH) { + compiler = append(compiler, "-target", triple) + } + runKernel := func(encoding, input string) { + for _, pipeline := range []struct { + name string + optimized bool + backend int + }{ + {name: "backend_O0"}, + {name: "backend_O2", backend: 2}, + {name: "IR_O2/backend_O2", optimized: true, backend: 2}, + } { + t.Run(encoding+"/"+pipeline.name, func(t *testing.T) { + text := input + if pipeline.optimized { + text = x86DwordProductIROptimize(t, input) + } + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, "vector_test_"+encoding, + triple, text, mainC, runPrefix, pipeline.backend) + }) + } + } + runKernel("named", ir) + + // Exercise the raw-byte path with the same independently checked flag + // oracle. Keep the displacements and operand order identical to the named + // Go source above, including both 128-bit and 256-bit memory forms. + rawSource := strings.NewReplacer( + "PTEST X0, X1", "BYTE $0x66\n\tBYTE $0x0f\n\tBYTE $0x38\n\tBYTE $0x17\n\tBYTE $0xc8", + "VPTEST 32(AX), Y1", "BYTE $0xc4\n\tBYTE $0xe2\n\tBYTE $0x7d\n\tBYTE $0x17\n\tBYTE $0x48\n\tBYTE $0x20", + "VTESTPS 64(AX), X1", "BYTE $0xc4\n\tBYTE $0xe2\n\tBYTE $0x79\n\tBYTE $0x0e\n\tBYTE $0x48\n\tBYTE $0x40", + "VTESTPD 96(AX), Y1", "BYTE $0xc4\n\tBYTE $0xe2\n\tBYTE $0x7d\n\tBYTE $0x0f\n\tBYTE $0x48\n\tBYTE $0x60", + ).Replace(source) + requireX86GoAssemblerResult(t, "amd64", rawSource, true) + rawFile, err := Parse(ArchAMD64, rawSource) + if err != nil { + t.Fatal(err) + } + rawIR, err := Translate(rawFile, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"vectortestsemantics": { + Name: "vectortestsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + runKernel("raw", rawIR) +} diff --git a/amd64_lower_vextractps_test.go b/amd64_lower_vextractps_test.go new file mode 100644 index 000000000..922eced44 --- /dev/null +++ b/amd64_lower_vextractps_test.go @@ -0,0 +1,199 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +func TestAMD64PackedScalarExtractGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64PackedScalarExtractSpecs)) + for op := range amd64PackedScalarExtractSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{ + "EXTRACTPS", "PEXTRB", "PEXTRD", "PEXTRQ", "PEXTRW", + "VEXTRACTPS", "VPEXTRB", "VPEXTRD", "VPEXTRQ", "VPEXTRW", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("packed scalar-extract grammar opcodes = %v, want %v", got, want) + } +} + +func TestTranslateX86ExtractPSCompleteGo127Forms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + highX, highGP := "X15", "R15" + if target.goarch == "386" { + highX, highGP = "X7", "DI" + } + source := "TEXT extractpsforms(SB),$0-0\n" + + "\tEXTRACTPS $0, X0, AX\n" + + "\tEXTRACTPS $3, " + highX + ", (BX)\n" + + "\tEXTRACTPS $2, " + highX + ", " + highGP + "\n" + + "\tVEXTRACTPS $-128, X0, AX\n" + + "\tVPEXTRQ $0, X0, (BX)\n" + + "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + ir := x86UnboundVoidScalarFormatObject(t, source, "extractpsforms", target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "extractps-"+target.name+".ll", "extractps-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ExtractPSRejectsFormsOutsideGo127Table(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "EXTRACTPS $-1, X0, AX"}, + {goarch: "amd64", instruction: "EXTRACTPS $4, X0, AX"}, + {goarch: "amd64", instruction: "EXTRACTPS $0, X16, AX"}, + {goarch: "amd64", instruction: "EXTRACTPS $0, X0, M0"}, + {goarch: "386", instruction: "EXTRACTPS $0, X8, AX"}, + {goarch: "386", instruction: "EXTRACTPS $0, X0, R8"}, + {goarch: "386", instruction: "PEXTRQ $0, X0, AX"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar-extract tables", test.instruction) + } + }) + } +} + +func TestAMD64ExtractPSRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT extractpssemantics(SB),NOSPLIT,$0-16 + MOVQ in+0(FP), AX + MOVQ out+8(FP), BX + MOVUPS (AX), X0 + EXTRACTPS $0, X0, CX + MOVL CX, 0(BX) + EXTRACTPS $3, X0, 4(BX) + VEXTRACTPS $2, X0, DX + MOVL DX, 8(BX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"extractpssemantics": { + Name: "extractpssemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void extractpssemantics(void *, void *); +int main(void) { + uint32_t in[4] = {0x3f800001,0x40000002,0x40400003,0x40800004}, out[3] = {0}; + extractpssemantics(in, out); + return out[0] == in[0] && out[1] == in[3] && out[2] == in[2] ? 0 : 10; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "extractps_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateAMD64VExtractPSCompleteGo127Forms(t *testing.T) { + const source = `TEXT vextractpsforms(SB),$0-0 + VEXTRACTPS $-128, X0, AX + VEXTRACTPS $255, X1, (AX) + VEXTRACTPS $255, X31, R15 + RET +` + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", source, true) + ir := x86UnboundVoidScalarFormatObject(t, source, "vextractpsforms", "amd64", target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "vextractps-"+target.name+".ll", "vextractps-"+target.name+".o", ir) + }) + } +} + +func TestTranslateAMD64VExtractPSRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VEXTRACTPS $-129, X1, AX", + "VEXTRACTPS $-1, X31, AX", + "VEXTRACTPS $256, X1, AX", + "VEXTRACTPS $1, Y1, AX", + "VEXTRACTPS $1, X1, X2", + } { + t.Run(instruction, func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvextractps table", instruction) + } + }) + } +} diff --git a/amd64_lower_vinserti128_test.go b/amd64_lower_vinserti128_test.go new file mode 100644 index 000000000..86a065293 --- /dev/null +++ b/amd64_lower_vinserti128_test.go @@ -0,0 +1,161 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func TestTranslateX86VINSERTI128CompleteGo127Forms(t *testing.T) { + const source = `TEXT vinserti128_forms(SB),$0-0 + VINSERTI128 $0, X0, Y1, Y2 + VINSERTI128 $255, 8(BX), Y15, Y15 + VINSERTI128 $1, X15, Y0, Y1 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "vinserti128_forms": {Name: "vinserti128_forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "vinserti128-"+target.name+".ll", "vinserti128-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86VINSERTI128RejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VINSERTI128 X0, Y1, Y2", + "VINSERTI128 $0, X0, Y1", + "VINSERTI128 $-1, X0, Y1, Y2", + "VINSERTI128 $256, X0, Y1, Y2", + "VINSERTI128 AX, X0, Y1, Y2", + "VINSERTI128 $0, Y0, Y1, Y2", + "VINSERTI128 $0, X16, Y1, Y2", + "VINSERTI128 $0, X0, X1, Y2", + "VINSERTI128 $0, X0, Y16, Y2", + "VINSERTI128 $0, X0, Y1, X2", + "VINSERTI128 $0, X0, Y1, Y16", + "VINSERTI128 $0, X0, Y1, 8(BX)", + "VINSERTI128.Z $0, X0, Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VINSERTI128 table", instruction) + } + }) + } + const source386 = "TEXT bad(SB),$0-0\n\tVINSERTI128 $0, X0, Y1, Y2\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source386, false) + file, err := Parse(ArchAMD64, source386) + if err == nil { + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatal("Translate accepted 386 VINSERTI128 despite Go 1.27's frontend operand limit") + } + } +} + +func TestAMD64VINSERTI128RuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT vinserti128Semantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ narrow+8(FP), BX + MOVQ wide+16(FP), CX + VMOVDQU64 (BX), X0 + VMOVDQU64 (CX), Y1 + VINSERTI128 $0, X0, Y1, Y2 + VMOVDQU64 Y2, 0(AX) + VINSERTI128 $255, X0, Y1, Y3 + VMOVDQU64 Y3, 32(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "vinserti128Semantics": { + Name: "vinserti128Semantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void vinserti128Semantics(uint64_t *, const uint64_t *, const uint64_t *); +int main(void) { + const uint64_t narrow[2] = {10, 11}; + const uint64_t wide[4] = {20, 21, 22, 23}; + uint64_t out[8] = {0}; + const uint64_t want[8] = {10, 11, 22, 23, 20, 21, 10, 11}; + vinserti128Semantics(out, narrow, wide); + for (int i = 0; i < 8; i++) if (out[i] != want[i]) return i + 1; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vinserti128", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_vmov_fp_test.go b/amd64_lower_vmov_fp_test.go new file mode 100644 index 000000000..52510b804 --- /dev/null +++ b/amd64_lower_vmov_fp_test.go @@ -0,0 +1,192 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "testing" +) + +const x86VMOVDQUAggregateFPOriginalSource = `TEXT vmovAggregateFP(SB),4,$0-24 +MOVQ out+0(FP),AX +VMOVDQU key+8(FP),X0 +VMOVDQU X0,(AX) +PXOR X0,X0 +VMOVDQU X0,key+8(FP) +VMOVDQU key+8(FP),X1 +VMOVDQU X1,16(AX) +RET +` + +func TestX86VEXPackedMoveAggregateFPForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + pointerBytes := 8 + if target.arch == "386" { + pointerBytes = 4 + } + for _, op := range []string{"VMOVDQA", "VMOVDQU"} { + for _, width := range []struct { + name, reg string + bytes int + }{{"x", "X0", 16}, {"y", "Y0", 32}} { + t.Run(op+"/"+width.name, func(t *testing.T) { + name := "packedFrame" + var slots []FrameSlot + args := []LLVMType{Ptr} + slots = append(slots, FrameSlot{Offset: 0, Type: Ptr, Index: 0, Field: -1}) + for i := 0; i < width.bytes/4; i++ { + args = append(args, I32) + slots = append(slots, FrameSlot{ + Offset: int64(pointerBytes + i*4), + Type: I32, + Index: i + 1, + Field: -1, + }) + } + mov := "MOVQ" + if target.arch == "386" { + mov = "MOVL" + } + source := fmt.Sprintf("TEXT %s(SB),4,$0-%d\n%s out+0(FP),AX\n%s key+%d(FP),%s\n%s %s,(AX)\n%s (AX),%s\n%s %s,key+%d(FP)\nRET\n", + name, pointerBytes+width.bytes, mov, op, pointerBytes, width.reg, + op, width.reg, op, width.reg, op, width.reg, pointerBytes) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.arch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{name: { + Name: name, Args: args, Ret: Void, + Frame: FrameLayout{Params: slots}, + }}, + }) + if op == "VMOVDQA" { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("virtual aggregate acquired native alignment: %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "vmov-fp.ll", "vmov-fp.o", ir) + }) + } + } + }) + } +} + +func TestAMD64VMOVDQUAggregateFPBytesRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + file, err := Parse(ArchAMD64, x86VMOVDQUAggregateFPOriginalSource) + if err != nil { + t.Fatal(err) + } + sig := FuncSig{ + Name: "vmovAggregateFP", + Args: []LLVMType{Ptr, I32, I32, I32, I32}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I32, Index: 1, Field: -1}, + {Offset: 12, Type: I32, Index: 2, Field: -1}, + {Offset: 16, Type: I32, Index: 3, Field: -1}, + {Offset: 20, Type: I32, Index: 4, Field: -1}, + }}, + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if runtime.GOARCH != "amd64" { + if runtime.GOOS != "darwin" || !rosettaAvailable() { + t.Skip("execution requires amd64 or Rosetta; five-target object tests are separate") + } + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{sig.Name: sig}, + }) + if err != nil { + t.Fatal(err) + } + mainC := `#include +#include +#include +extern void vmovAggregateFP(uint8_t *, uint32_t, uint32_t, uint32_t, uint32_t); +int main(void) { + uint32_t key[4] = {0x12345678, 0x90abcdef, 0x13579bdf, 0x2468ace0}; + uint8_t out[32]; + memset(out, 0xa5, sizeof out); + vmovAggregateFP(out, key[0], key[1], key[2], key[3]); + for (int i = 0; i < 16; i++) { + if (out[i] != ((uint8_t *)key)[i] || out[16+i] != 0) { + fprintf(stderr, "VMOVDQU FP byte %d: got %u and %u\n", i, out[i], out[16+i]); + return 1; + } + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vmov_aggregate_fp", triple, ir, mainC, prefix) +} + +func TestX86VEXPackedMoveAggregateFPRejectsInvalidForms(t *testing.T) { + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"386", "i386-unknown-linux-gnu"}, + } { + for _, form := range []string{ + "VMOVDQU key+8(FP),X0", // Four-byte slot cannot supply 16 bytes. + "VMOVDQA key+8(FP),Y0", // Four-byte slot cannot supply 32 bytes. + "VMOVDQU key+8(FP),Z0", // No Z form in Go's VEX table. + "VMOVDQU.Z key+8(FP),X0", // No VEX zeroing form. + "VMOVDQU key+8(FP),K1,X0", // No VEX mask form. + } { + t.Run(target.arch+"/"+form, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalid(SB),4,$0-12\n"+form+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: target.arch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"invalid": { + Name: "invalid", Args: []LLVMType{Ptr, I32}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I32, Index: 1, Field: -1}, + }}, + }}, + }) + if err == nil { + t.Fatalf("accepted unsupported form %q", form) + } + }) + } + } +} diff --git a/amd64_lower_vmov_half_test.go b/amd64_lower_vmov_half_test.go new file mode 100644 index 000000000..b3896eefb --- /dev/null +++ b/amd64_lower_vmov_half_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64MoveHalfGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64MoveHalfSpec{ + "MOVHLPS": {registerOnly: true}, + "MOVLHPS": {high: true, registerOnly: true}, + "VMOVHLPS": {vector: true, registerOnly: true}, + "VMOVLHPS": {high: true, vector: true, registerOnly: true}, + "MOVLPS": {}, + "MOVHPS": {high: true}, + "MOVLPD": {}, + "MOVHPD": {high: true}, + "VMOVLPS": {vector: true}, + "VMOVHPS": {high: true, vector: true}, + "VMOVLPD": {vector: true}, + "VMOVHPD": {high: true, vector: true}, + } + if len(amd64MoveHalfSpecs) != len(expected) { + t.Fatalf("move-half grammar has %d entries, want %d", len(amd64MoveHalfSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64MoveHalfSpecs[op]; !ok { + t.Errorf("move-half grammar omitted %s", op) + } else if got != want { + t.Errorf("move-half grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86VEXMoveHalfCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + base, last := 30, 31 + if target.goarch == "386" { + base, last = 6, 7 + } + var source strings.Builder + source.WriteString("TEXT vmovhalfforms(SB),$0-0\n") + for _, op := range []string{"VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD"} { + fmt.Fprintf(&source, "\t%s X%d, 0(AX)\n", op, last) + fmt.Fprintf(&source, "\t%s 8(AX), X%d, X%d\n", op, base, last) + } + source.WriteString("\tRET\n") + file, signatures := x86UnboundVoidFormatSource(t, source.String(), "vmovhalfforms", target.goarch, target.triple) + ir := x86MoveHalfFormatObjectUnit(t, file, signatures, target.goarch, target.triple) + if !strings.Contains(ir, "extractelement <2 x i64>") || !strings.Contains(ir, "insertelement <2 x i64>") { + t.Fatalf("VEX half-move lowering omitted lane operations:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "vmov-half-"+target.name+".ll", "vmov-half-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86VEXMoveHalfRejectsFormsOutsideGoTable(t *testing.T) { + for _, instruction := range []string{ + "VMOVHPS.Z 0(AX), X1, X2", + "VMOVLPS Y0, 0(AX)", + "VMOVHPD 0(AX), Y1, Y2", + "VMOVLPD X0, X1, X2", + "VMOVHPS 0(AX), X1", + "VMOVLPS X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvmovhpd table", instruction) + } + }) + } +} + +func TestAMD64VEXMoveHalfRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT vmovhalfsemantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ base+8(FP), CX + MOVQ half+16(FP), DX + MOVUPS 0(CX), X1 + STC + VMOVHPS 0(DX), X1, X2 + MOVUPS X2, 0(AX) + VMOVLPS 0(DX), X1, X3 + MOVUPS X3, 16(AX) + SETCS 32(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"vmovhalfsemantics": { + Name: "vmovhalfsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void vmovhalfsemantics(uint8_t *, const uint64_t *, const uint64_t *); +int main(void) { + uint8_t out[33] = {0}; + const uint64_t base[2] = {UINT64_C(0x1111222233334444), UINT64_C(0x5555666677778888)}; + const uint64_t half[1] = {UINT64_C(0xaaaabbbbccccdddd)}; + uint64_t got[4]; + vmovhalfsemantics(out, base, half); + memcpy(got, out, 32); + if (got[0] != base[0] || got[1] != half[0]) return 10; + if (got[2] != half[0] || got[3] != base[1]) return 11; + if (out[32] != 1) return 12; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vex_move_half_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_vmov_integer.go b/amd64_lower_vmov_integer.go new file mode 100644 index 000000000..e9c840f40 --- /dev/null +++ b/amd64_lower_vmov_integer.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Every named GP/X consumer shares exact width and encoding writeback rules. +// Raw VMOVW retains its separate, decoder-bound 16-bit role. +func (c *amd64Ctx) lowerVectorScalarIntegerMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + if !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("contradictory GP/X transfer dispatch %s/%s", op, ins.Op) + } + f, known, err := x86GPSIMDTransferProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if f.mmx && (ins.x86Encoded || ins.x86SourceRaw != nil) { + // A physical X-to-MMX destination remains native x87/MMX state; + // no source-compatible spelling grants a virtual GP register slot. + return true, false, fmt.Errorf("%w: raw Intel MOVDQ2Q requires its physical MMX/x87 entry and caller-state contract: %q", ErrProbeNeedsContext, ins.Raw) + } + src, dst := ins.Args[0], ins.Args[1] + if f.memory >= 0 && ins.Args[f.memory].Kind == OpFP { + if _, err := c.scalarFPSpan(ins.Args[f.memory].FPOffset, f.bits); err != nil { + return true, false, err + } + } + var low string + if f.sourceX { + low, err = c.loadXLowInteger(src.Reg, f.bits) + } else { + low, err = c.loadGPSIMDTransferSource(src, f.bits) + } + if err != nil { + return true, false, err + } + if f.destinationX { + lanes := 128 / f.bits + inserted, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> zeroinitializer, i%d %s, i32 0\n", inserted, lanes, f.bits, f.bits, low) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <16 x i8>\n", bytes, lanes, f.bits, inserted) + if f.spec.legacy { + return true, false, c.storeX(dst.Reg, "%"+bytes) + } + return true, false, c.storePackedMoveOperand(dst, 16, "%"+bytes) + } + if f.mmx { + return true, false, c.storeReg(dst.Reg, low) + } + if dst.Kind == OpFP { + return true, false, c.storeScalarFPBits(dst.FPOffset, f.bits, low) + } + return true, false, c.storeVMOVIntegerScalar(dst, f.bits, low) +} + +func (c *amd64Ctx) loadGPSIMDTransferSource(source Operand, bits int) (string, error) { + if source.Kind == OpFP { + return c.loadScalarFPBits(source.FPOffset, bits) + } + var pointer, pointerType string + var err error + switch source.Kind { + case OpMem: + pointer, pointerType, err = c.ptrFromMem(source.Mem) + case OpSym: + pointer, err = c.ptrFromSB(source.Sym) + pointerType = "ptr" + default: + return c.evalIntSized(source, amd64IntegerTypeForBits(bits)) + } + if err != nil { + return "", err + } + value := c.newTmp() + // Unmasked MOVD/Q reads are observable at their original width even + // when the destination is dead. They have no alignment or MXCSR effect. + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %s, align 1\n", value, bits, pointerType, pointer) + return "%" + value, nil +} + +func isAMD64GoXRegister(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + _, ok := amd64ParseXReg(operand.Reg) + return ok +} + +func (c *amd64Ctx) storeVMOVIntegerScalar(dst Operand, bits int, value string) error { + typ := amd64IntegerTypeForBits(bits) + if dst.Kind == OpMem || dst.Kind == OpSym { + var pointer, pointerType string + var err error + if dst.Kind == OpMem { + pointer, pointerType, err = c.ptrFromMem(dst.Mem) + } else { + pointer, err = c.ptrFromSB(dst.Sym) + pointerType = "ptr" + } + if err != nil { + return err + } + // Keep each original unmasked native store, including overwritten + // stores or private symbols whose value becomes otherwise dead. + fmt.Fprintf(c.b, " store volatile %s %s, %s %s, align 1\n", typ, value, pointerType, pointer) + return nil + } + if dst.Kind != OpReg { + return c.storeScalarIntegerOperand(dst, typ, value) + } + if bits == 64 { + return c.storeRegUnchecked(dst.Reg, value) + } + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %s to i64\n", widened, bits, value) + return c.storeRegUnchecked(dst.Reg, "%"+widened) +} diff --git a/amd64_lower_vmov_integer_test.go b/amd64_lower_vmov_integer_test.go new file mode 100644 index 000000000..652b2ff29 --- /dev/null +++ b/amd64_lower_vmov_integer_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86VMOVDVMOVQCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's _yvmovd and _yvmovq tables allow scalar GP/memory transfers + // in both directions. VMOVQ additionally allows X-to-X low-qword moves. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT vmovintegerscalarforms(SB),4,$0-0 + VMOVD AX, X0 + VMOVD X0, AX + VMOVD SP, X1 + VMOVD X1, SP + VMOVD 8(AX), X20 + VMOVD X20, 12(AX) + VMOVQ AX, X2 + VMOVQ X2, AX + VMOVQ SP, X3 + VMOVQ X3, SP + VMOVQ 16(AX), X30 + VMOVQ X30, 24(AX) + VMOVQ X0, X1 + VMOVQ X20, X31 +` + if target.goarch == "amd64" { + source += "\tVMOVD R8, X31\n\tVMOVD X31, R9\n\tVMOVQ R10, X30\n\tVMOVQ X30, R11\n" + } else { + // Go accepts these high spellings but they lack a bound physical + // mode32 identity. Their independent Context cases live beside + // the shared grammar; ordinary object coverage uses X0-X7. + source = strings.NewReplacer("X20", "X4", "X30", "X5", "X31", "X6").Replace(source) + } + source += "\tRET\n" + goAssemblerOracleOutput(t, target.goarch, source, "", true) + // Retain every original GP/SP/X/native-memory spelling. This + // format program has no actual data, pointer or physical SP entry + // contract. Complete initialized GP/X and native transfer matrices + // independently cover public success on all five targets. + ll := x86UnboundVoidScalarFormatObject(t, source, "vmovintegerscalarforms", target.goarch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "vmov-integer-scalar-"+target.name+".ll", "vmov-integer-scalar-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86VMOVDVMOVQRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VMOVD X0, X1", + "VMOVD Y0, AX", + "VMOVD AX, Y0", + "VMOVD Z0, AX", + "VMOVD AX, Z0", + "VMOVD AX, BX", + "VMOVD (AX), 8(BX)", + "VMOVD.Z AX, X0", + "VMOVD AX, K1, X0", + "VMOVQ Y0, AX", + "VMOVQ AX, Y0", + "VMOVQ Z0, X0", + "VMOVQ AX, BX", + "VMOVQ (AX), 8(BX)", + "VMOVQ.Z X0, X1", + "VMOVQ X0, K1, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86VMOVIntegerRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VMOVD R8, X0", + "VMOVD X0, R8", + "VMOVQ R8, X0", + "VMOVQ X0, R8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86VMOVIntegerRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86VMOVIntegerRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VMOVD/VMOVQ forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_vmov_scalar.go b/amd64_lower_vmov_scalar.go new file mode 100644 index 000000000..4ca03c771 --- /dev/null +++ b/amd64_lower_vmov_scalar.go @@ -0,0 +1,330 @@ +package plan9asm + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// lowerVectorScalarMove implements all Go 1.27 _yvmovsd forms shared by +// VMOVSD and VMOVSS: scalar loads/stores, three-register upper merges, and +// their EVEX masked variants. +func (c *amd64Ctx) lowerVectorScalarMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64ScalarVectorMoveSpecs[Op(baseOp)] + if !recognized || spec.legacy { + return false, false, nil + } + form, _, err := amd64ScalarVectorMoveProperties(ins, c.goarch) + if err != nil { + return true, false, err + } + if form.literal { + if form.legacyZero { + // The actual Go encoder emits legacy XORPS, even for a VMOV + // spelling. Do not zero the wider Y/Z views as VEX would. + var value string + if spec.bits == 32 { + value, err = c.evalF32(ins.Args[0]) + } else { + value, err = c.evalF64(ins.Args[0]) + } + if err != nil { + return true, false, err + } + return true, false, c.storeLegacyScalarMoveRegister(ins.Args[form.destination].Reg, value, spec.bits, false) + } + bits := uint64(ins.Args[0].Imm) + if spec.bits == 32 { + bits = uint64(math.Float32bits(float32(math.Float64frombits(bits)))) + } + mask := "" + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + return true, false, c.storeScalarMoveRegister(ins.Args[form.destination].Reg, spec.bits, + strconv.FormatUint(bits, 10), "zeroinitializer", mask, form.zeroing) + } + if form.upper >= 0 { + mask := Operand{} + if form.mask >= 0 { + mask = ins.Args[form.mask] + } + return c.lowerVectorScalarMoveRegisterMerge(spec.bits, ins.Args[form.low], ins.Args[form.upper], mask, + ins.Args[form.destination], form.zeroing) + } + if form.mask >= 0 { + return c.lowerVectorScalarMoveMaskedMemory(baseOp, spec.bits, form.zeroing, ins) + } + return c.lowerVectorScalarMoveTwoOperand(baseOp, spec.bits, ins) +} + +func (c *amd64Ctx) lowerVectorScalarMoveTwoOperand(baseOp string, elemBits int, ins Instr) (bool, bool, error) { + src, dst := ins.Args[0], ins.Args[1] + if amd64ScalarXOperand(src) && isAMD64MemoryOperand(dst) { + value, err := c.loadXLowInteger(src.Reg, elemBits) + if err != nil { + return true, false, err + } + return true, false, c.storeVectorScalarMoveMemory(dst, elemBits, value, "") + } + if isAMD64MemoryOperand(src) && amd64ScalarXOperand(dst) { + value, err := c.loadScalarMoveMemory(src, elemBits) + if err != nil { + return true, false, err + } + return true, false, c.storeScalarMoveRegister(dst.Reg, elemBits, value, "zeroinitializer", "", false) + } + return true, false, fmt.Errorf("amd64 %s two-operand form requires X-to-memory or memory-to-X: %q", baseOp, ins.Raw) +} + +func (c *amd64Ctx) lowerVectorScalarMoveMaskedMemory(baseOp string, elemBits int, zeroing bool, ins Instr) (bool, bool, error) { + if !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 %s three-operand masked memory form expects K1-K7 in the middle: %q", baseOp, ins.Raw) + } + mask, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + if amd64ScalarXOperand(ins.Args[0]) && isAMD64MemoryOperand(ins.Args[2]) { + if zeroing { + return true, false, fmt.Errorf("amd64 %s cannot use .Z with a memory destination: %q", baseOp, ins.Raw) + } + value, err := c.loadXLowInteger(ins.Args[0].Reg, elemBits) + if err != nil { + return true, false, err + } + return true, false, c.storeVectorScalarMoveMemory(ins.Args[2], elemBits, value, mask) + } + if isAMD64MemoryOperand(ins.Args[0]) && amd64ScalarXOperand(ins.Args[2]) { + value, err := c.loadVectorScalarMemory(ins.Args[0], elemBits, mask) + if err != nil { + return true, false, err + } + return true, false, c.storeScalarMoveRegister(ins.Args[2].Reg, elemBits, value, "zeroinitializer", mask, zeroing) + } + return true, false, fmt.Errorf("amd64 %s masked memory form requires X, K, memory or memory, K, X: %q", baseOp, ins.Raw) +} + +// A masked-off scalar move must not access memory at all. Selecting a loaded +// value afterwards does not suppress faults, and a read/modify/write store +// spuriously reads write-only memory even when the mask is active. +func (c *amd64Ctx) loadVectorScalarMemory(src Operand, elemBits int, mask string) (string, error) { + condition := amd64MaskBitI1(c, mask, 0) + return c.loadX86ScalarMemoryIf(src, elemBits, condition) +} + +func (c *amd64Ctx) loadX86ScalarMemoryIf(src Operand, elemBits int, condition string) (string, error) { + if src.Kind == OpFP && elemBits != 16 { + if _, err := c.scalarFPSpan(src.FPOffset, elemBits); err != nil { + return "", err + } + // Parameter-only loads also need a source-defined mask and merge + // lane. Validate before creating either masked control-flow branch. + if err := c.scalarFPResultInitialized(); err != nil { + return "", err + } + } + return c.loadX86ScalarIf(elemBits, condition, func() (string, error) { + return c.loadScalarMoveMemory(src, elemBits) + }) +} + +func (c *amd64Ctx) loadScalarMoveMemory(src Operand, bits int) (string, error) { + if x86ScalarVectorMoveOrdinaryMemory(src, bits, c.goarch) { + return c.loadScalarVectorMoveNativeMemory(src, bits) + } + if src.Kind == OpFP && bits != 16 { + return c.loadScalarFPBits(src.FPOffset, bits) + } + return c.evalIntSized(src, amd64IntegerTypeForBits(bits)) +} + +func (c *amd64Ctx) loadX86ScalarIf(elemBits int, condition string, load func() (string, error)) (string, error) { + loadLabel := c.newTmp() + "_scalar_load" + skipLabel := c.newTmp() + "_scalar_skip" + joinLabel := c.newTmp() + "_scalar_join" + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n%s:\n", condition, loadLabel, skipLabel, loadLabel) + value, err := load() + if err != nil { + return "", err + } + fmt.Fprintf(c.b, " br label %%%s\n%s:\n br label %%%s\n%s:\n", joinLabel, skipLabel, joinLabel, joinLabel) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = phi i%d [ %s, %%%s ], [ 0, %%%s ]\n", selected, elemBits, value, loadLabel, skipLabel) + return "%" + selected, nil +} + +func (c *amd64Ctx) lowerVectorScalarMoveRegisterMerge(elemBits int, lowSource, upperSource, maskArg, dst Operand, zeroing bool) (bool, bool, error) { + low, err := c.loadXLowInteger(lowSource.Reg, elemBits) + if err != nil { + return true, false, err + } + upperBytes, err := c.loadX(upperSource.Reg) + if err != nil { + return true, false, err + } + lanes := 128 / elemBits + upper := c.bitcastVectorBytesToIntegerLanes(16, lanes, elemBits, upperBytes) + mask := "" + if maskArg.Kind == OpReg { + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + return true, false, c.storeScalarMoveRegister(dst.Reg, elemBits, low, upper, mask, zeroing) +} + +func amd64ScalarXOperand(op Operand) bool { + if op.Kind != OpReg { + return false + } + index, ok := amd64ParseXReg(op.Reg) + return ok && index < 32 +} + +func amd64NonzeroKOperand(op Operand) bool { + if op.Kind != OpReg { + return false + } + index, ok := amd64ParseKReg(op.Reg) + return ok && index != 0 +} + +func (c *amd64Ctx) loadXLowInteger(reg Reg, elemBits int) (string, error) { + bytesValue, err := c.loadX(reg) + if err != nil { + return "", err + } + lanes := 128 / elemBits + values := c.bitcastVectorBytesToIntegerLanes(16, lanes, elemBits, bytesValue) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 0\n", low, lanes, elemBits, values) + return "%" + low, nil +} + +func (c *amd64Ctx) storeVectorScalarRegister(dst Reg, elemBits int, low, base, mask string, zeroing bool) error { + lanes := 128 / elemBits + if mask != "" { + fallback := "0" + if !zeroing { + old, err := c.loadXLowInteger(dst, elemBits) + if err != nil { + return err + } + fallback = old + } + maskBit := amd64MaskBitI1(c, mask, 0) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %s, i%d %s\n", selected, maskBit, elemBits, low, elemBits, fallback) + low = "%" + selected + } + updated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %s, i32 0\n", updated, lanes, elemBits, base, elemBits, low) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <16 x i8>\n", bytesValue, lanes, elemBits, updated) + return c.storeX(dst, "%"+bytesValue) +} + +func (c *amd64Ctx) storeScalarMoveRegister(dst Reg, elemBits int, low, base, mask string, zeroing bool) error { + if err := c.storeVectorScalarRegister(dst, elemBits, low, base, mask, zeroing); err != nil { + return err + } + // VEX/EVEX scalar writes clear bits above 127, unlike legacy MOVSS/SD. + // Keep every live wider view coherent with the newly written low X value. + index, _ := amd64ParseXReg(dst) + for _, view := range []struct { + bytes int + slots map[int]string + }{{32, c.yRegSlot}, {64, c.zRegSlot}} { + if slot, live := view.slots[index]; live { + bytesValue, err := c.loadX(dst) + if err != nil { + return err + } + indices := make([]string, view.bytes) + for i := range indices { + lane := i + if lane >= 16 { + lane = 16 // First byte of the zero second source. + } + indices[i] = fmt.Sprintf("i32 %d", lane) + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, <%d x i32> <%s>\n", wide, bytesValue, view.bytes, strings.Join(indices, ", ")) + fmt.Fprintf(c.b, " store <%d x i8> %%%s, ptr %s\n", view.bytes, wide, slot) + } + } + return nil +} + +func (c *amd64Ctx) storeVectorScalarMemory(dst Operand, elemBits int, value, mask string) error { + return c.storeVectorScalarMemoryWithNativeAccess(dst, elemBits, value, mask, false) +} + +func (c *amd64Ctx) storeVectorScalarMoveMemory(dst Operand, elemBits int, value, mask string) error { + return c.storeVectorScalarMemoryWithNativeAccess(dst, elemBits, value, mask, true) +} + +// Other users of the scalar byte-store helper retain their original emitter. +// Only the typed VMOVSS/SD native path requests an observable access here. +func (c *amd64Ctx) storeVectorScalarMemoryWithNativeAccess(dst Operand, elemBits int, value, mask string, nativeMove bool) error { + if dst.Kind == OpFP && elemBits != 16 { + result, err := c.scalarFPSpan(dst.FPOffset, elemBits) + if err != nil { + return err + } + if result && mask != "" { + if err := c.scalarFPResultInitialized(); err != nil { + return err + } + } + } + if mask != "" { + condition := amd64MaskBitI1(c, mask, 0) + storeLabel := c.newTmp() + "_scalar_store" + joinLabel := c.newTmp() + "_scalar_join" + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n%s:\n", condition, storeLabel, joinLabel, storeLabel) + if err := c.storeVectorScalarMemoryWithNativeAccess(dst, elemBits, value, "", nativeMove); err != nil { + return err + } + fmt.Fprintf(c.b, " br label %%%s\n%s:\n", joinLabel, joinLabel) + return nil + } + switch dst.Kind { + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + volatile := "" + if nativeMove && x86ScalarVectorMoveOrdinaryMemory(dst, elemBits, c.goarch) { + volatile = "volatile " + } + fmt.Fprintf(c.b, " store %si%d %s, %s %s, align 1\n", volatile, elemBits, value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i%d %s, ptr %s, align 1\n", elemBits, value, ptr) + return nil + case OpFP: + if elemBits != 16 { + return c.storeScalarFPBits(dst.FPOffset, elemBits, value) + } + return c.storeFPResult(dst.FPOffset, amd64IntegerTypeForBits(elemBits), value) + default: + return fmt.Errorf("expected scalar memory destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_vmov_scalar_aggregate_test.go b/amd64_lower_vmov_scalar_aggregate_test.go new file mode 100644 index 000000000..79f3e0fe0 --- /dev/null +++ b/amd64_lower_vmov_scalar_aggregate_test.go @@ -0,0 +1,162 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "testing" +) + +func TestTranslateX86WideStoresSpanComplex64Result(t *testing.T) { + for _, tc := range []struct { + name string + instruction string + amd64Only bool + needsZeroXMM bool + }{ + {name: "movsd", instruction: "MOVSD X0, ret+0(FP)", needsZeroXMM: true}, + {name: "movq-xmm", instruction: "MOVQ X0, ret+0(FP)", needsZeroXMM: true}, + {name: "vmovq-xmm", instruction: "VMOVQ X0, ret+0(FP)", needsZeroXMM: true}, + {name: "movq-gp", instruction: "MOVQ AX, ret+0(FP)", amd64Only: true}, + {name: "vmovsd", instruction: "VMOVSD X0, ret+0(FP)", amd64Only: true, needsZeroXMM: true}, + } { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + if tc.amd64Only && target.goarch != "amd64" { + continue + } + t.Run(tc.name+"/"+target.name, func(t *testing.T) { + functionName := "widestorecomplex64result" + source := fmt.Sprintf("TEXT %s(SB),$0-8\n", functionName) + if tc.needsZeroXMM { + source += "\tPXOR X0, X0\n" + } else if tc.name == "movq-gp" { + // Like every vector variant, the GP format witness must + // define its source rather than publish unknown entry bits. + source += "\tXORQ AX, AX\n" + } + source += "\t" + tc.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + functionName: { + Name: functionName, + Ret: LLVMType("{ float, float }"), + Frame: FrameLayout{Results: []FrameSlot{ + {Offset: 0, Type: LLVMType("float"), Index: 0, Field: -1, Name: "ret"}, + {Offset: 4, Type: LLVMType("float"), Index: 1, Field: -1, Name: "ret"}, + }}, + }, + }, + } + if tc.name == "movq-gp" { + mod, directErr := translateModuleDirect(file, options) + if directErr == nil { + mod.Dispose() + t.Fatal("direct linear prototype unexpectedly accepted a wide aggregate-result store") + } + if !errors.Is(directErr, errDirectModuleUnsupported) { + t.Fatalf("direct linear prototype error = %v, want fallback request", directErr) + } + } + ll, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "wide-store-complex64-result.ll", "wide-store-complex64-result.o", ll) + }) + } + } +} + +func TestAMD64VMOVSDStoresComplex64ResultRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT vmovsdcomplex64result(SB),$0-16 + MOVQ input+0(FP), AX + VMOVSD (AX), X0 + VMOVSD X0, ret+8(FP) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "vmovsdcomplex64result": { + Name: "vmovsdcomplex64result", + Args: []LLVMType{Ptr}, + Ret: LLVMType("{ float, float }"), + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1, Name: "input"}}, + Results: []FrameSlot{ + {Offset: 8, Type: LLVMType("float"), Index: 0, Field: -1, Name: "ret"}, + {Offset: 12, Type: LLVMType("float"), Index: 1, Field: -1, Name: "ret"}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + ll += ` +define void @call_vmovsdcomplex64result(ptr %input, ptr %output) { +entry: + %result = call { float, float } @vmovsdcomplex64result(ptr %input) + %real = extractvalue { float, float } %result, 0 + %imag = extractvalue { float, float } %result, 1 + store float %real, ptr %output, align 4 + %imagptr = getelementptr float, ptr %output, i64 1 + store float %imag, ptr %imagptr, align 4 + ret void +} +` + mainC := ` +extern void call_vmovsdcomplex64result(const float *input, float *output); +int main(void) { + const float input[2] = { 1.25f, -2.5f }; + float got[2] = { 0.0f, 0.0f }; + call_vmovsdcomplex64result(input, got); + if (got[0] != input[0]) return 1; + if (got[1] != input[1]) return 2; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vmovsd_complex64_result", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_word_madd_family.go b/amd64_lower_word_madd_family.go new file mode 100644 index 000000000..dfc9d4533 --- /dev/null +++ b/amd64_lower_word_madd_family.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "fmt" +) + +func (c *amd64Ctx) lowerWordMultiplyAddForm(op Op, ins Instr, form x86PackedPhysicalForm) (bool, bool, error) { + first, second, destination := form.source, ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: %s m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext, op) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + if form.mmx { + if first.Reg != ins.Args[form.first].Reg { + return true, false, fmt.Errorf("%w: %s GP spelling encodes a separate MMX source bank", ErrProbeNeedsContext, op) + } + var a string + var err error + if form.memory < 0 { + a, err = c.loadReg(first.Reg) + } else { + a, err = c.loadPackedUnpackMMXSource(first, 8) + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + result := c.emitPackedWordMultiplyAdd(4, c.bitcastI64ToIntegerLanes(4, 16, a), c.bitcastI64ToIntegerLanes(4, 16, b)) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i32> %s to i64\n", bits, result) + return true, false, c.storeReg(destination.Reg, "%"+bits) + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, form.spec.inputBits, false) + if err != nil { + return true, false, err + } + var result string + if form.legacy && form.memory >= 0 && first.Kind != OpFP { + result, err = c.emitMultiplyAddNativeMemory(first, b, form.spec) + } else { + // Intel SDM's E4NF.nb is non-fault-suppressing. A plain eager LLVM + // load can disappear under a constant zero mask, so use the reviewed + // complete volatile native reader, never logical masked loads. + a, loadErr := c.loadPackedUnpackSource(first, x86PackedUnpackForm{ + x86PackedBinaryForm: form.x86PackedBinaryForm, spec: x86PackedUnpackSpec{laneBits: form.spec.inputBits}, + }) + if loadErr != nil { + return true, false, loadErr + } + if form.spec.inputBits == 8 { + result = c.emitPackedUnsignedSignedByteMultiplyAdd(form.byteWidth, a, b) + } else { + result = c.emitPackedWordMultiplyAdd(form.byteWidth/2, a, b) + } + } + if err != nil { + return true, false, err + } + lanes := form.byteWidth * 8 / form.spec.outputBits + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, form.spec.outputBits, false) + if err != nil { + return true, false, err + } + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.spec.outputBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.spec.outputBits, result, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitMultiplyAddNativeMemory(source Operand, data string, spec x86PackedPhysicalSpec) (string, error) { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + switch pointerType { + case "ptr addrspace(256)": + segment = "%gs:" + case "ptr addrspace(257)": + segment = "%fs:" + } + carrier, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <4 x i32>\n", carrier, 128/spec.inputBits, spec.inputBits, data) + fmt.Fprintf(c.b, " %%%s = call <4 x i32> asm sideeffect %q, %q(<4 x i32> %%%s, %s %s)\n", + value, spec.nativeOpcode+" "+segment+"($2), $0", "=x,0,r,~{memory}", carrier, pointerType, pointer) + if spec.outputBits != 32 { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <%d x i%d>\n", result, value, 128/spec.outputBits, spec.outputBits) + return "%" + result, nil + } + return "%" + value, nil +} diff --git a/amd64_lower_wrapping_add.go b/amd64_lower_wrapping_add.go new file mode 100644 index 000000000..46cd50a1e --- /dev/null +++ b/amd64_lower_wrapping_add.go @@ -0,0 +1,98 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPackedIntegerAddForm(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86PackedArithmeticSpecForOp(Op(base), x86PackedArithmeticAdd); !known { + return false, false, nil + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed add dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86PackedIntegerAddProperties(ins, arch) + if err != nil || !known { + return known, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !c.nativeMemoryOperand(ins, form.first) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed add m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + bits := form.spec.laneBits + lanes := form.byteWidth * 8 / bits + if form.mmx { + var a string + if first.Kind == OpReg { + a, err = c.loadReg(first.Reg) + } else { + a, err = c.loadPackedUnpackMMXSource(first, 8) + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + value := c.emitPackedIntegerAdd(lanes, bits, c.bitcastI64ToIntegerLanes(lanes, bits, a), c.bitcastI64ToIntegerLanes(lanes, bits, b), form.spec.mode) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", out, lanes, bits, value) + return true, false, c.storeReg(destination.Reg, "%"+out) + } + mnemonic := "" + if form.legacy && form.memory >= 0 { + mnemonic = strings.ToLower(base) + if mnemonic == "paddl" { + mnemonic = "paddd" + } + } + err = c.emitPackedBinaryE4(ins, form.x86PackedBinaryForm, bits, mnemonic, + func(lanes, bits int, first, second string) string { + return c.emitPackedIntegerAdd(lanes, bits, first, second, form.spec.mode) + }) + return true, false, err +} + +// The caller supplies a registered legacy mnemonic. Keeping its memory form +// native preserves alignment/fault effects which an unaligned vector load +// followed by portable arithmetic would erase. +func (c *amd64Ctx) emitPackedBinaryNativeMemory(source Operand, bits int, data, mnemonic string) (string, error) { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + carrier, result := c.newTmp(), c.newTmp() + // LLVM's i386 v4i32 inline-asm carrier requires SSE2. A bitcast-only + // v4f32 carrier also represents original SSE1 PS operations without + // strengthening their CPU requirements. No floating arithmetic occurs. + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <4 x float>\n", carrier, 128/bits, bits, data) + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(<4 x float> %%%s, %s %s)\n", result, mnemonic+" "+segment+"($2), $0", "=x,0,r,~{memory}", carrier, pointerType, pointer) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <%d x i%d>\n", value, result, 128/bits, bits) + return "%" + value, nil +} diff --git a/amd64_lower_x87.go b/amd64_lower_x87.go index 0ef86ec04..072f0ebca 100644 --- a/amd64_lower_x87.go +++ b/amd64_lower_x87.go @@ -1,12 +1,31 @@ package plan9asm -import "fmt" +import ( + "fmt" + "strings" +) func isX87Op(op Op) bool { switch op { - case "FMOVD", "FMOVDP", "FMOVV", "FMOVVP", "FXCHD", "FADDDP", - "FDIVD", "FMULD", "FMULDP", "FSTCW", "FLDCW", "FRNDINT", - "FABS", "FUCOMI", "FTST", "FSTSW", "FLD1", "FSQRT": + case "FMOVB", "FMOVBP", "FMOVD", "FMOVDP", "FMOVF", "FMOVFP", + "FMOVL", "FMOVLP", "FMOVV", "FMOVVP", "FMOVW", "FMOVWP", + "FMOVX", "FMOVXP", "FBLD", "FBSTP", "FXCHD", + "FADDW", "FADDL", "FADDF", "FADDD", "FADDDP", + "FMULW", "FMULL", "FMULF", "FMULD", "FMULDP", + "FSUBW", "FSUBL", "FSUBF", "FSUBD", "FSUBDP", + "FSUBRW", "FSUBRL", "FSUBRF", "FSUBRD", "FSUBRDP", + "FDIVW", "FDIVL", "FDIVF", "FDIVD", "FDIVDP", + "FDIVRW", "FDIVRL", "FDIVRF", "FDIVRD", "FDIVRDP", + "FCOMD", "FCOMDP", "FCOMDPP", "FCOMF", "FCOMFP", "FCOMI", "FCOMIP", + "FCOML", "FCOMLP", "FCOMW", "FCOMWP", + "FUCOM", "FUCOMI", "FUCOMIP", "FUCOMP", "FUCOMPP", + "FCMOVCC", "FCMOVCS", "FCMOVEQ", "FCMOVHI", "FCMOVLS", + "FCMOVB", "FCMOVBE", "FCMOVNB", "FCMOVNBE", "FCMOVE", "FCMOVNE", "FCMOVNU", "FCMOVU", "FCMOVUN", + "FSTCW", "FLDCW", "FLDENV", "FRSTOR", "FSAVE", "FSTENV", "FRNDINT", + "F2XM1", "FABS", "FCHS", "FCLEX", "FCOS", "FDECSTP", "FINCSTP", "FINIT", + "FTST", "FSTSW", "FLD1", "FLDL2E", "FLDL2T", "FLDLG2", "FLDLN2", + "FLDPI", "FLDZ", "FNOP", "FPATAN", "FPREM", "FPREM1", "FPTAN", "FSCALE", + "FSIN", "FSINCOS", "FSQRT", "FXAM", "FXTRACT", "FYL2X", "FYL2XP1": return true default: return false @@ -17,137 +36,195 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err if !isX87Op(op) { return false, false, nil } - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64: x87 instruction %s requires GOARCH=386", op) + dispatch := ins + if dispatch.Op == "" { + dispatch.Op = op // Legacy direct-helper callers have no second source head. + } else if !strings.EqualFold(string(dispatch.Op), string(op)) { + return true, false, fmt.Errorf("%s x87 dispatch/source opcode differs: %q", c.goarch, ins.Raw) + } + arch := c.goarch + if arch == "" { + arch = "amd64" // Legacy private helpers have always used the default x86 mode. + } + if _, known, err := x86X87WordProperties(dispatch, arch); known && err != nil { + return true, false, err + } + if _, known, err := x86X87EnvironmentReadProperties(dispatch, arch); known && err != nil { + return true, false, err + } + valueForm, typedValue, formErr := x86X87ValueProperties(dispatch, arch) + if typedValue && formErr != nil { + return true, false, formErr + } + if typedValue && valueForm.spec.kind == x86X87IntegerMove { + return true, false, c.lowerX87IntegerMove(valueForm) } - switch op { - case "FMOVD": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FMOVD expects src, dst: %q", ins.Raw) + case "FBLD": + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 FBLD expects memory source: %q", ins.Raw) } - v, err := c.evalX87F64(ins.Args[0]) + value, err := c.loadX87PackedBCD(ins.Args[0]) if err != nil { return true, false, err } - if idx, isF := x87OperandReg(ins.Args[1]); isF && idx == 0 && ins.Args[0].Kind != OpReg { - return true, false, c.pushX87(v) - } - return true, false, c.storeX87F64(ins.Args[1], v) + return true, false, c.pushX87(value) - case "FMOVDP": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FMOVDP expects src, dst: %q", ins.Raw) - } - v, err := c.evalX87F64(ins.Args[0]) - if err != nil { - return true, false, err + case "FBSTP": + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 FBSTP expects memory destination: %q", ins.Raw) } - if err := c.storeX87F64(ins.Args[1], v); err != nil { + if err := c.storeX87PackedBCD(ins.Args[0], c.loadX87(0)); err != nil { return true, false, err } c.popX87() return true, false, nil - case "FMOVV": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FMOVV expects src, dst: %q", ins.Raw) + case "FMOVB", "FMOVX": + if len(ins.Args) != 2 || !isX87MemoryOperand(ins.Args[0]) || !isX87F0(ins.Args[1]) { + return true, false, fmt.Errorf("386 %s expects memory source and F0 destination: %q", op, ins.Raw) } - if idx, ok := x87OperandReg(ins.Args[1]); !ok || idx != 0 { - return true, false, fmt.Errorf("386 FMOVV expects F0 destination: %q", ins.Raw) + var value string + var err error + if op == "FMOVB" { + value, err = c.loadX87PackedBCD(ins.Args[0]) + } else { + value, err = c.loadX87Extended(ins.Args[0]) } - v, err := c.evalIntSized(ins.Args[0], I64) if err != nil { return true, false, err } - f := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sitofp i64 %s to double\n", f, v) - return true, false, c.pushX87("%" + f) + return true, false, c.pushX87(value) - case "FMOVVP": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FMOVVP expects src, dst: %q", ins.Raw) + case "FMOVBP", "FMOVXP": + if len(ins.Args) != 2 || !isX87F0(ins.Args[0]) || !isX87MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("386 %s expects F0 source and memory destination: %q", op, ins.Raw) } - if index, ok := x87OperandReg(ins.Args[0]); !ok || index != 0 { - return true, false, fmt.Errorf("386 FMOVVP expects F0 source: %q", ins.Raw) - } - var iv string - if c.useHardwareX87() { - iv = c.convertX87ToI64Hardware() + var err error + if op == "FMOVBP" { + err = c.storeX87PackedBCD(ins.Args[1], c.loadX87(0)) } else { - rounded := c.roundX87Software(c.loadX87(0)) - converted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fptosi double %s to i64\n", converted, rounded) - iv = "%" + converted + err = c.storeX87Extended(ins.Args[1], c.loadX87(0)) } - if err := c.storeX87I64(ins.Args[1], iv); err != nil { + if err != nil { return true, false, err } c.popX87() return true, false, nil - case "FXCHD": + case "FMOVD": if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FXCHD expects Fsrc, Fdst: %q", ins.Raw) + return true, false, fmt.Errorf("386 FMOVD expects src, dst: %q", ins.Raw) } - a, okA := x87OperandReg(ins.Args[0]) - bv, okB := x87OperandReg(ins.Args[1]) - if !okA || !okB { - return true, false, fmt.Errorf("386 FXCHD expects x87 registers: %q", ins.Raw) + srcReg, srcIsReg := x87OperandReg(ins.Args[0]) + dstReg, dstIsReg := x87OperandReg(ins.Args[1]) + valid := (isX87MemoryOperand(ins.Args[0]) || ins.Args[0].Kind == OpImm) && dstIsReg && dstReg == 0 || + srcIsReg && srcReg == 0 && (isX87MemoryOperand(ins.Args[1]) || dstIsReg) || + srcIsReg && dstIsReg && dstReg == 0 + if !valid { + return true, false, fmt.Errorf("386 FMOVD expects memory,F0, Freg,F0, F0,memory, or F0,Freg: %q", ins.Raw) } - av := c.loadX87(a) - b := c.loadX87(bv) - c.storeX87(a, b) - c.storeX87(bv, av) - return true, false, nil - - case "FDIVD", "FMULD": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 %s expects src, dst: %q", op, ins.Raw) + if srcIsReg { + value := c.loadX87IntegerValue(srcReg) + if valueForm.push { + return true, false, c.pushX87IntegerValue(value) + } + if dstIsReg { + c.storeX87IntegerValue(dstReg, value) + return true, false, nil + } } - src, err := c.evalX87F64(ins.Args[0]) + v, err := c.loadX87FloatingMove(ins.Args[0], 64) if err != nil { return true, false, err } - dst, err := c.evalX87F64(ins.Args[1]) + if valueForm.push { + return true, false, c.pushX87(v) + } + return true, false, c.storeX87F64(ins.Args[1], v) + + case "FMOVDP": + if len(ins.Args) != 2 || !isX87F0(ins.Args[0]) || !(isX87MemoryOperand(ins.Args[1]) || isX87RegisterOperand(ins.Args[1])) { + return true, false, fmt.Errorf("386 FMOVDP expects F0 source and memory or x87-register destination: %q", ins.Raw) + } + if index, register := x87OperandReg(ins.Args[1]); register { + value := c.loadX87IntegerValue(0) + c.storeX87IntegerValue(index, value) + c.popX87() + return true, false, nil + } + v, err := c.evalX87F64(ins.Args[0]) if err != nil { return true, false, err } - out := c.newTmp() - if op == "FDIVD" { - fmt.Fprintf(c.b, " %%%s = fdiv double %s, %s\n", out, dst, src) - } else { - fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", out, dst, src) + if err := c.storeX87F64(ins.Args[1], v); err != nil { + return true, false, err } - return true, false, c.storeX87F64(ins.Args[1], "%"+out) + c.popX87() + return true, false, nil - case "FADDDP", "FMULDP": + case "FMOVF": if len(ins.Args) != 2 { return true, false, fmt.Errorf("386 %s expects src, dst: %q", op, ins.Raw) } - src, err := c.evalX87F64(ins.Args[0]) - if err != nil { - return true, false, err + if (isX87MemoryOperand(ins.Args[0]) || op == "FMOVF" && valueForm.push) && isX87F0(ins.Args[1]) { + value, err := c.loadX87MoveValue(op, ins.Args[0]) + if err != nil { + return true, false, err + } + return true, false, c.pushX87(value) } - dst, err := c.evalX87F64(ins.Args[1]) - if err != nil { - return true, false, err + if isX87F0(ins.Args[0]) && isX87MemoryOperand(ins.Args[1]) { + return true, false, c.storeX87MoveValue(op, ins.Args[1], c.loadX87(0)) } - out := c.newTmp() - if op == "FADDDP" { - fmt.Fprintf(c.b, " %%%s = fadd double %s, %s\n", out, dst, src) - } else { - fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", out, dst, src) + return true, false, fmt.Errorf("386 %s expects memory source and F0 destination, or F0 source and memory destination: %q", op, ins.Raw) + + case "FMOVFP": + if len(ins.Args) != 2 || !isX87F0(ins.Args[0]) || !isX87MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("386 %s expects F0 source and memory destination: %q", op, ins.Raw) } - if err := c.storeX87F64(ins.Args[1], "%"+out); err != nil { + baseOp := Op(strings.TrimSuffix(string(op), "P")) + if err := c.storeX87MoveValue(baseOp, ins.Args[1], c.loadX87(0)); err != nil { return true, false, err } c.popX87() return true, false, nil + case "FXCHD": + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("386 FXCHD expects Fsrc, Fdst: %q", ins.Raw) + } + a, okA := x87OperandReg(ins.Args[0]) + bv, okB := x87OperandReg(ins.Args[1]) + if !okA || !okB { + return true, false, fmt.Errorf("386 FXCHD expects x87 registers: %q", ins.Raw) + } + av := c.loadX87IntegerValue(a) + b := c.loadX87IntegerValue(bv) + c.storeX87IntegerValue(a, b) + c.storeX87IntegerValue(bv, av) + return true, false, nil + + case "FADDW", "FADDL", "FADDF", "FADDD", "FADDDP", + "FMULW", "FMULL", "FMULF", "FMULD", "FMULDP", + "FSUBW", "FSUBL", "FSUBF", "FSUBD", "FSUBDP", + "FSUBRW", "FSUBRL", "FSUBRF", "FSUBRD", "FSUBRDP", + "FDIVW", "FDIVL", "FDIVF", "FDIVD", "FDIVDP", + "FDIVRW", "FDIVRL", "FDIVRF", "FDIVRD", "FDIVRDP": + return true, false, c.lowerX87Arithmetic(op, ins, valueForm) + + case "FCOMD", "FCOMDP", "FCOMDPP", "FCOMF", "FCOMFP", "FCOMI", "FCOMIP", + "FCOML", "FCOMLP", "FCOMW", "FCOMWP", + "FUCOM", "FUCOMI", "FUCOMIP", "FUCOMP", "FUCOMPP": + return true, false, c.lowerX87Compare(op, ins, valueForm) + + case "FCMOVCC", "FCMOVCS", "FCMOVEQ", "FCMOVHI", "FCMOVLS", + "FCMOVB", "FCMOVBE", "FCMOVNB", "FCMOVNBE", "FCMOVE", "FCMOVNE", "FCMOVNU", "FCMOVU", "FCMOVUN": + return true, false, c.lowerX87ConditionalMove(op, ins) + case "FSTCW": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("386 FSTCW expects dst: %q", ins.Raw) + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 FSTCW expects memory destination: %q", ins.Raw) } if c.useHardwareX87() { c.storeHardwareX87ControlWord() @@ -157,8 +234,8 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, c.storeX87I16(ins.Args[0], "%"+cw) case "FLDCW": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("386 FLDCW expects src: %q", ins.Raw) + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 FLDCW expects memory source: %q", ins.Raw) } cw, err := c.evalIntSized(ins.Args[0], I16) if err != nil { @@ -170,6 +247,28 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err } return true, false, nil + case "FLDENV", "FRSTOR": + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 %s expects memory source: %q", op, ins.Raw) + } + return true, false, c.loadX87Environment(ins.Args[0]) + + case "FSAVE", "FSTENV": + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 %s expects memory destination: %q", op, ins.Raw) + } + if err := c.storeX87Environment(ins.Args[0]); err != nil { + return true, false, err + } + if op == "FSAVE" { + for i := range c.x87Slot { + c.storeX87(i, "0.000000e+00") + } + fmt.Fprintf(c.b, " store i16 895, ptr %s\n", c.x87ControlSlot) + fmt.Fprintf(c.b, " store i16 0, ptr %s\n", c.x87StatusSlot) + } + return true, false, nil + case "FRNDINT": if len(ins.Args) != 0 { return true, false, fmt.Errorf("386 FRNDINT takes no operands: %q", ins.Raw) @@ -178,7 +277,22 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err c.roundX87Hardware(c.x87Slot[0]) return true, false, nil } - return true, false, c.storeX87F64(Operand{Kind: OpReg, Reg: Reg("F0")}, c.roundX87Software(c.loadX87(0))) + value := c.loadX87IntegerValue(0) + value.floating = c.roundX87Software(value.floating) + c.storeX87IntegerValue(0, value) + return true, false, nil + + case "F2XM1": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 F2XM1 takes no operands: %q", ins.Raw) + } + value := c.loadX87(0) + power := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.exp2.f64(double %s)\n", power, value) + fmt.Fprintf(c.b, " %%%s = fsub double %%%s, 1.000000e+00\n", result, power) + c.storeX87(0, "%"+result) + return true, false, nil case "FABS", "FSQRT": if len(ins.Args) != 0 { @@ -194,27 +308,177 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err c.storeX87(0, "%"+out) return true, false, nil - case "FUCOMI": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FUCOMI expects lhs, rhs: %q", ins.Raw) + case "FCHS": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FCHS takes no operands: %q", ins.Raw) } - lhs, err := c.evalX87F64(ins.Args[0]) - if err != nil { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg double %s\n", result, c.loadX87(0)) + c.storeX87(0, "%"+result) + return true, false, nil + + case "FCLEX": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FCLEX takes no operands: %q", ins.Raw) + } + if c.useHardwareX87() { + fmt.Fprintln(c.b, ` call void asm sideeffect "fnclex", "~{fpsr}"()`) + } + status := c.newTmp() + conditionCodes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", status, c.x87StatusSlot) + fmt.Fprintf(c.b, " %%%s = and i16 %%%s, 32512\n", conditionCodes, status) // Preserve TOP and C3,C2,C1,C0. + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", conditionCodes, c.x87StatusSlot) + return true, false, nil + + case "FCOS", "FSIN": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + intrinsic := "llvm.cos.f64" + if op == "FSIN" { + intrinsic = "llvm.sin.f64" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @%s(double %s)\n", result, intrinsic, c.loadX87(0)) + c.storeX87(0, "%"+result) + c.clearX87C2() + return true, false, nil + + case "FDECSTP", "FINCSTP": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + c.rotateX87Stack(op == "FINCSTP") + return true, false, nil + + case "FINIT": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FINIT takes no operands: %q", ins.Raw) + } + if c.useHardwareX87() { + fmt.Fprintln(c.b, ` call void asm sideeffect "fninit", "~{st},~{st(1)},~{st(2)},~{st(3)},~{st(4)},~{st(5)},~{st(6)},~{st(7)},~{fpsr}"()`) + } + for i := range c.x87Slot { + c.storeX87(i, "0.000000e+00") + } + fmt.Fprintf(c.b, " store i16 895, ptr %s\n", c.x87ControlSlot) + fmt.Fprintf(c.b, " store i16 0, ptr %s\n", c.x87StatusSlot) + return true, false, nil + + case "FNOP": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FNOP takes no operands: %q", ins.Raw) + } + return true, false, nil + + case "FPATAN": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FPATAN takes no operands: %q", ins.Raw) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @atan2(double %s, double %s)\n", result, c.loadX87(1), c.loadX87(0)) + c.storeX87(1, "%"+result) + c.popX87() + return true, false, nil + + case "FPREM", "FPREM1": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + function := "fmod" + if op == "FPREM1" { + function = "remainder" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @%s(double %s, double %s)\n", result, function, c.loadX87(0), c.loadX87(1)) + c.storeX87(0, "%"+result) + c.clearX87C2() + return true, false, nil + + case "FPTAN": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FPTAN takes no operands: %q", ins.Raw) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @tan(double %s)\n", result, c.loadX87(0)) + c.storeX87(0, "%"+result) + if err := c.pushX87("1.000000e+00"); err != nil { return true, false, err } - rhs, err := c.evalX87F64(ins.Args[1]) - if err != nil { + c.clearX87C2() + return true, false, nil + + case "FSCALE": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FSCALE takes no operands: %q", ins.Raw) + } + exponent := c.newTmp() + scale := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.trunc.f64(double %s)\n", exponent, c.loadX87(1)) + fmt.Fprintf(c.b, " %%%s = call double @llvm.exp2.f64(double %%%s)\n", scale, exponent) + fmt.Fprintf(c.b, " %%%s = fmul double %s, %%%s\n", result, c.loadX87(0), scale) + c.storeX87(0, "%"+result) + return true, false, nil + + case "FSINCOS": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FSINCOS takes no operands: %q", ins.Raw) + } + value := c.loadX87(0) + sine := c.newTmp() + cosine := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.sin.f64(double %s)\n", sine, value) + fmt.Fprintf(c.b, " %%%s = call double @llvm.cos.f64(double %s)\n", cosine, value) + c.storeX87(0, "%"+sine) + if err := c.pushX87("%" + cosine); err != nil { return true, false, err } - z := c.newTmp() - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp ueq double %s, %s\n", z, lhs, rhs) - fmt.Fprintf(c.b, " %%%s = fcmp ult double %s, %s\n", cf, lhs, rhs) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", z, c.flagsZSlot) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - // FUCOMI clears both SF and OF. flagsSltSlot models SF xor OF for - // signed conditions, so those conditions must observe false here. - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsSltSlot) + c.clearX87C2() + return true, false, nil + + case "FXAM": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FXAM takes no operands: %q", ins.Raw) + } + c.classifyX87F0() + return true, false, nil + + case "FXTRACT": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FXTRACT takes no operands: %q", ins.Raw) + } + value := c.loadX87(0) + abs := c.newTmp() + log := c.newTmp() + exponent := c.newTmp() + scale := c.newTmp() + significand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.fabs.f64(double %s)\n", abs, value) + fmt.Fprintf(c.b, " %%%s = call double @llvm.log2.f64(double %%%s)\n", log, abs) + fmt.Fprintf(c.b, " %%%s = call double @llvm.floor.f64(double %%%s)\n", exponent, log) + fmt.Fprintf(c.b, " %%%s = call double @llvm.exp2.f64(double %%%s)\n", scale, exponent) + fmt.Fprintf(c.b, " %%%s = fdiv double %s, %%%s\n", significand, value, scale) + c.storeX87(0, "%"+significand) + return true, false, c.pushX87("%" + exponent) + + case "FYL2X", "FYL2XP1": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + x := c.loadX87(0) + if op == "FYL2XP1" { + incremented := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fadd double %s, 1.000000e+00\n", incremented, x) + x = "%" + incremented + } + log := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.log2.f64(double %s)\n", log, x) + fmt.Fprintf(c.b, " %%%s = fmul double %s, %%%s\n", result, c.loadX87(1), log) + c.storeX87(1, "%"+result) + c.popX87() return true, false, nil case "FTST": @@ -246,22 +510,176 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, nil case "FSTSW": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("386 FSTSW expects dst: %q", ins.Raw) + if len(ins.Args) != 1 || !(isX87MemoryOperand(ins.Args[0]) || ins.Args[0].Kind == OpReg && ins.Args[0].Reg == AX) { + return true, false, fmt.Errorf("386 FSTSW expects memory destination or AX: %q", ins.Raw) } status := c.newTmp() fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", status, c.x87StatusSlot) return true, false, c.storeX87I16(ins.Args[0], "%"+status) - case "FLD1": + case "FLD1", "FLDL2E", "FLDL2T", "FLDLG2", "FLDLN2", "FLDPI", "FLDZ": if len(ins.Args) != 0 { - return true, false, fmt.Errorf("386 FLD1 takes no operands: %q", ins.Raw) + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + constant := map[Op]string{ + "FLD1": "1.0000000000000000e+00", + "FLDL2E": "1.4426950408889634e+00", + "FLDL2T": "3.3219280948873623e+00", + "FLDLG2": "3.0102999566398120e-01", + "FLDLN2": "6.9314718055994531e-01", + "FLDPI": "3.1415926535897931e+00", + "FLDZ": "0.0000000000000000e+00", } - return true, false, c.pushX87("1.000000e+00") + return true, false, c.pushX87(constant[op]) } return false, false, nil } +func (c *amd64Ctx) lowerX87Arithmetic(op Op, ins Instr, form x86X87ValueForm) error { + width := form.spec.width + var src string + var err error + switch width { + case 'W', 'L': + typ := I16 + if width == 'L' { + typ = I32 + } + integer, evalErr := c.evalIntSized(ins.Args[0], typ) + if evalErr != nil { + return evalErr + } + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sitofp %s %s to double\n", converted, typ, integer) + src = "%" + converted + case 'F': + value, evalErr := c.evalF32(ins.Args[0]) + if evalErr != nil { + return evalErr + } + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext float %s to double\n", converted, value) + src = "%" + converted + case 'D': + src, err = c.evalX87F64(ins.Args[0]) + if err != nil { + return err + } + default: + return fmt.Errorf("386 %s has unknown x87 arithmetic width", op) + } + dst, err := c.evalX87F64(ins.Args[1]) + if err != nil { + return err + } + lhs, rhs := dst, src + if form.spec.reverse { + lhs, rhs = src, dst + } + operation := "" + switch { + case form.spec.stem == "FADD": + operation = "fadd" + case form.spec.stem == "FMUL": + operation = "fmul" + case form.spec.stem == "FSUB" || form.spec.stem == "FSUBR": + operation = "fsub" + case form.spec.stem == "FDIV" || form.spec.stem == "FDIVR": + operation = "fdiv" + } + if operation == "" { + return fmt.Errorf("386 %s has unknown x87 arithmetic operation", op) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s double %s, %s\n", result, operation, lhs, rhs) + if err := c.storeX87F64(ins.Args[1], "%"+result); err != nil { + return err + } + if form.spec.pop != 0 { + c.popX87() + } + return nil +} + +func (c *amd64Ctx) lowerX87Compare(op Op, ins Instr, form x86X87ValueForm) error { + lhs, err := c.evalX87F64(form.left) + if err != nil { + return err + } + var rhs string + switch form.spec.width { + case 'D': + rhs, err = c.evalX87F64(form.right) + case 'F': + var narrow string + narrow, err = c.evalF32(form.right) + if err == nil { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext float %s to double\n", wide, narrow) + rhs = "%" + wide + } + case 'L', 'W': + typ := I16 + if form.spec.width == 'L' { + typ = I32 + } + var integer string + integer, err = c.evalIntSized(form.right, typ) + if err == nil { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sitofp %s %s to double\n", wide, typ, integer) + rhs = "%" + wide + } + default: + return fmt.Errorf("386 %s has unknown x87 compare width", op) + } + if err != nil { + return err + } + if form.spec.directFlags { + c.setX87CompareFlags(lhs, rhs) + } else { + c.setX87CompareStatus(lhs, rhs) + } + for i := uint8(0); i < form.spec.pop; i++ { + c.popX87() + } + return nil +} + +func (c *amd64Ctx) x87CompareBits(lhs, rhs string) (equal, less, unordered string) { + equalTmp := c.newTmp() + lessTmp := c.newTmp() + unorderedTmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fcmp ueq double %s, %s\n", equalTmp, lhs, rhs) + fmt.Fprintf(c.b, " %%%s = fcmp ult double %s, %s\n", lessTmp, lhs, rhs) + fmt.Fprintf(c.b, " %%%s = fcmp uno double %s, %s\n", unorderedTmp, lhs, rhs) + return equalTmp, lessTmp, unorderedTmp +} + +func (c *amd64Ctx) setX87CompareFlags(lhs, rhs string) { + equal, less, unordered := c.x87CompareBits(lhs, rhs) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", equal, c.flagsZSlot) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", less, c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", unordered, c.flagsPFSlot) + // FCOMI/FUCOMI clear OF, SF, and AF. flagsSltSlot is architectural SF. + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsSltSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsAFSlot) +} + +func (c *amd64Ctx) setX87CompareStatus(lhs, rhs string) { + equal, less, unordered := c.x87CompareBits(lhs, rhs) + c3Word := c.x87StatusBit(equal, 16384) + c2Word := c.x87StatusBit(unordered, 1024) + c0Word := c.x87StatusBit(less, 256) + partial := c.newTmp() + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i16 %s, %s\n", partial, c3Word, c2Word) + fmt.Fprintf(c.b, " %%%s = or i16 %%%s, %s\n", status, partial, c0Word) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", status, c.x87StatusSlot) +} + func x87OperandReg(op Operand) (int, bool) { if op.Kind != OpReg { return 0, false @@ -269,6 +687,256 @@ func x87OperandReg(op Operand) (int, bool) { return amd64ParseX87Reg(op.Reg) } +func isX87RegisterOperand(op Operand) bool { + _, ok := x87OperandReg(op) + return ok +} + +func isX87F0(op Operand) bool { + index, ok := x87OperandReg(op) + return ok && index == 0 +} + +func isX87MemoryOperand(op Operand) bool { + return isAMD64MemoryOperand(op) +} + +func (c *amd64Ctx) loadX87MoveValue(op Op, src Operand) (string, error) { + switch op { + case "FMOVF": + value, err := c.loadX87FloatingMove(src, 32) + if err != nil { + return "", err + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext float %s to double\n", wide, value) + return "%" + wide, nil + case "FMOVL", "FMOVW": + typ := I32 + if op == "FMOVW" { + typ = I16 + } + value, err := c.evalIntSized(src, typ) + if err != nil { + return "", err + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sitofp %s %s to double\n", wide, typ, value) + return "%" + wide, nil + default: + return "", fmt.Errorf("386: unsupported x87 move load %s", op) + } +} + +func (c *amd64Ctx) storeX87MoveValue(op Op, dst Operand, value string) error { + switch op { + case "FMOVF": + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fptrunc double %s to float\n", narrow, value) + return c.storeX87TypedMemory(dst, LLVMType("float"), "%"+narrow) + case "FMOVL", "FMOVW": + bits := 32 + if op == "FMOVW" { + bits = 16 + } + rounded := c.roundX87Software(value) + return c.storeX87TypedMemory(dst, amd64IntegerTypeForBits(bits), c.convertRoundedX87Integer(rounded, bits)) + default: + return fmt.Errorf("386: unsupported x87 move store %s", op) + } +} + +func (c *amd64Ctx) storeX87TypedMemory(dst Operand, typ LLVMType, value string) error { + if typ == LLVMType("float") || typ == LLVMType("double") { + if handled, err := c.storeX87NativeFloatingMove(dst, typ, value); handled { + return err + } + } + if dst.Kind == OpFP { + if typ == LLVMType("float") { + return c.storeScalarFPFloat(dst.FPOffset, 32, value) + } + return c.storeFPResult(dst.FPOffset, typ, value) + } + ptr, ptrType, err := c.x87MemoryPointer(dst) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, %s %s, align 1\n", typ, value, ptrType, ptr) + return nil +} + +func (c *amd64Ctx) x87MemoryPointer(op Operand) (string, string, error) { + if !isX87MemoryOperand(op) { + return "", "", fmt.Errorf("386 x87 instruction expected memory, got %s", op.String()) + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(op) + if err != nil { + return "", "", fmt.Errorf("386 x87 memory operand %s: %w", op.String(), err) + } + return ptr, ptrType, nil +} + +func (c *amd64Ctx) loadX87Environment(src Operand) error { + ptr, ptrType, err := c.x87MemoryPointer(src) + if err != nil { + return err + } + // The 32-bit environment is one observable 28-byte read, including the + // fields that this software model does not interpret. Keep it exact and + // volatile even when both captured words are dead. An aggregate-width + // integer load avoids 28 separate IR accesses and preserves address space. + environment := c.newTmp() + control := c.newTmp() + statusBits := c.newTmp() + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile i224, %s %s, align 1\n", environment, ptrType, ptr) + fmt.Fprintf(c.b, " %%%s = trunc i224 %%%s to i16\n", control, environment) + fmt.Fprintf(c.b, " %%%s = lshr i224 %%%s, 32\n", statusBits, environment) + fmt.Fprintf(c.b, " %%%s = trunc i224 %%%s to i16\n", status, statusBits) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", control, c.x87ControlSlot) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", status, c.x87StatusSlot) + if c.useHardwareX87() { + c.loadHardwareX87ControlWord() + } + return nil +} + +func (c *amd64Ctx) storeX87Environment(dst Operand) error { + ptr, ptrType, err := c.x87MemoryPointer(dst) + if err != nil { + return err + } + if c.useHardwareX87() { + c.storeHardwareX87ControlWord() + } + control := c.newTmp() + status := c.newTmp() + statusPtr := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", control, c.x87ControlSlot) + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", status, c.x87StatusSlot) + fmt.Fprintf(c.b, " store i16 %%%s, %s %s, align 1\n", control, ptrType, ptr) + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 4\n", statusPtr, ptrType, ptr) + fmt.Fprintf(c.b, " store i16 %%%s, %s %%%s, align 1\n", status, ptrType, statusPtr) + return nil +} + +func (c *amd64Ctx) loadX87Extended(src Operand) (string, error) { + ptr, ptrType, err := c.x87MemoryPointer(src) + if err != nil { + return "", err + } + extended := c.newTmp() + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load x86_fp80, %s %s, align 1\n", extended, ptrType, ptr) + fmt.Fprintf(c.b, " %%%s = fptrunc x86_fp80 %%%s to double\n", value, extended) + return "%" + value, nil +} + +func (c *amd64Ctx) storeX87Extended(dst Operand, value string) error { + ptr, ptrType, err := c.x87MemoryPointer(dst) + if err != nil { + return err + } + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext double %s to x86_fp80\n", extended, value) + fmt.Fprintf(c.b, " store x86_fp80 %%%s, %s %s, align 1\n", extended, ptrType, ptr) + return nil +} + +func (c *amd64Ctx) loadX87PackedBCD(src Operand) (string, error) { + ptr, ptrType, err := c.x87MemoryPointer(src) + if err != nil { + return "", err + } + accumulator := "0" + for i := 8; i >= 0; i-- { + bytePtr := c.newTmp() + packed := c.newTmp() + low := c.newTmp() + highShifted := c.newTmp() + high := c.newTmp() + low64 := c.newTmp() + high64 := c.newTmp() + digit10 := c.newTmp() + digits := c.newTmp() + scaled := c.newTmp() + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %d\n", bytePtr, ptrType, ptr, i) + fmt.Fprintf(c.b, " %%%s = load i8, %s %%%s, align 1\n", packed, ptrType, bytePtr) + fmt.Fprintf(c.b, " %%%s = and i8 %%%s, 15\n", low, packed) + fmt.Fprintf(c.b, " %%%s = lshr i8 %%%s, 4\n", highShifted, packed) + fmt.Fprintf(c.b, " %%%s = and i8 %%%s, 15\n", high, highShifted) + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", low64, low) + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", high64, high) + fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, 10\n", digit10, high64) + fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", digits, digit10, low64) + fmt.Fprintf(c.b, " %%%s = mul i64 %s, 100\n", scaled, accumulator) + fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", next, scaled, digits) + accumulator = "%" + next + } + signPtr := c.newTmp() + signByte := c.newTmp() + signBits := c.newTmp() + isNegative := c.newTmp() + negative := c.newTmp() + signed := c.newTmp() + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 9\n", signPtr, ptrType, ptr) + fmt.Fprintf(c.b, " %%%s = load i8, %s %%%s, align 1\n", signByte, ptrType, signPtr) + fmt.Fprintf(c.b, " %%%s = and i8 %%%s, -128\n", signBits, signByte) + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %%%s, 0\n", isNegative, signBits) + fmt.Fprintf(c.b, " %%%s = sub i64 0, %s\n", negative, accumulator) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 %s\n", signed, isNegative, negative, accumulator) + fmt.Fprintf(c.b, " %%%s = sitofp i64 %%%s to double\n", value, signed) + return "%" + value, nil +} + +func (c *amd64Ctx) storeX87PackedBCD(dst Operand, value string) error { + ptr, ptrType, err := c.x87MemoryPointer(dst) + if err != nil { + return err + } + rounded := c.roundX87Software(value) + integer := c.newTmp() + isNegative := c.newTmp() + negated := c.newTmp() + absolute := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fptosi double %s to i64\n", integer, rounded) + fmt.Fprintf(c.b, " %%%s = icmp slt i64 %%%s, 0\n", isNegative, integer) + fmt.Fprintf(c.b, " %%%s = sub i64 0, %%%s\n", negated, integer) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 %%%s\n", absolute, isNegative, negated, integer) + remaining := "%" + absolute + for i := 0; i < 9; i++ { + digits := c.newTmp() + tens := c.newTmp() + ones := c.newTmp() + tensByte := c.newTmp() + onesByte := c.newTmp() + shifted := c.newTmp() + packed := c.newTmp() + bytePtr := c.newTmp() + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = urem i64 %s, 100\n", digits, remaining) + fmt.Fprintf(c.b, " %%%s = udiv i64 %%%s, 10\n", tens, digits) + fmt.Fprintf(c.b, " %%%s = urem i64 %%%s, 10\n", ones, digits) + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i8\n", tensByte, tens) + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i8\n", onesByte, ones) + fmt.Fprintf(c.b, " %%%s = shl i8 %%%s, 4\n", shifted, tensByte) + fmt.Fprintf(c.b, " %%%s = or i8 %%%s, %%%s\n", packed, shifted, onesByte) + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %d\n", bytePtr, ptrType, ptr, i) + fmt.Fprintf(c.b, " store i8 %%%s, %s %%%s, align 1\n", packed, ptrType, bytePtr) + fmt.Fprintf(c.b, " %%%s = udiv i64 %s, 100\n", next, remaining) + remaining = "%" + next + } + sign := c.newTmp() + signPtr := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i8 -128, i8 0\n", sign, isNegative) + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 9\n", signPtr, ptrType, ptr) + fmt.Fprintf(c.b, " store i8 %%%s, %s %%%s, align 1\n", sign, ptrType, signPtr) + return nil +} + func (c *amd64Ctx) loadX87(index int) string { t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load double, ptr %s\n", t, c.x87Slot[index]) @@ -277,23 +945,89 @@ func (c *amd64Ctx) loadX87(index int) string { func (c *amd64Ctx) storeX87(index int, value string) { fmt.Fprintf(c.b, " store double %s, ptr %s\n", value, c.x87Slot[index]) + if c.x87ExactSlot[index] != "" { + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.x87ExactSlot[index]) + } } func (c *amd64Ctx) pushX87(value string) error { - for i := len(c.x87Slot) - 1; i > 0; i-- { - c.storeX87(i, c.loadX87(i-1)) - } - c.storeX87(0, value) - return nil + return c.pushX87IntegerValue(x87IntegerValue{value, "0", "false"}) } func (c *amd64Ctx) popX87() { for i := 0; i < len(c.x87Slot)-1; i++ { - c.storeX87(i, c.loadX87(i+1)) + c.storeX87IntegerValue(i, c.loadX87IntegerValue(i+1)) } c.storeX87(len(c.x87Slot)-1, "0.000000e+00") } +func (c *amd64Ctx) rotateX87Stack(increment bool) { + values := make([]x87IntegerValue, len(c.x87Slot)) + for i := range values { + values[i] = c.loadX87IntegerValue(i) + } + if increment { + for i := range values { + c.storeX87IntegerValue(i, values[(i+1)%len(values)]) + } + return + } + for i := range values { + c.storeX87IntegerValue(i, values[(i+len(values)-1)%len(values)]) + } +} + +func (c *amd64Ctx) clearX87C2() { + status := c.newTmp() + cleared := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", status, c.x87StatusSlot) + fmt.Fprintf(c.b, " %%%s = and i16 %%%s, -1025\n", cleared, status) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", cleared, c.x87StatusSlot) +} + +func (c *amd64Ctx) classifyX87F0() { + value := c.loadX87(0) + isNaN := c.newTmp() + isZero := c.newTmp() + abs := c.newTmp() + isInfinity := c.newTmp() + notNaN := c.newTmp() + notZero := c.newTmp() + c2 := c.newTmp() + c0 := c.newTmp() + bits := c.newTmp() + signBits := c.newTmp() + isNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fcmp uno double %s, %s\n", isNaN, value, value) + fmt.Fprintf(c.b, " %%%s = fcmp oeq double %s, 0.000000e+00\n", isZero, value) + fmt.Fprintf(c.b, " %%%s = call double @llvm.fabs.f64(double %s)\n", abs, value) + fmt.Fprintf(c.b, " %%%s = fcmp oeq double %%%s, 0x7FF0000000000000\n", isInfinity, abs) + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notNaN, isNaN) + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notZero, isZero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", c2, notNaN, notZero) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", c0, isNaN, isInfinity) + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", bits, value) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, -9223372036854775808\n", signBits, bits) + fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", isNegative, signBits) + c3Word := c.x87StatusBit(isZero, 16384) + c2Word := c.x87StatusBit(c2, 1024) + c1Word := c.x87StatusBit(isNegative, 512) + c0Word := c.x87StatusBit(c0, 256) + first := c.newTmp() + second := c.newTmp() + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i16 %s, %s\n", first, c3Word, c2Word) + fmt.Fprintf(c.b, " %%%s = or i16 %%%s, %s\n", second, first, c1Word) + fmt.Fprintf(c.b, " %%%s = or i16 %%%s, %s\n", status, second, c0Word) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", status, c.x87StatusSlot) +} + +func (c *amd64Ctx) x87StatusBit(condition string, mask int) string { + word := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %d, i16 0\n", word, condition, mask) + return "%" + word +} + func (c *amd64Ctx) evalX87F64(op Operand) (string, error) { if index, ok := x87OperandReg(op); ok { return c.loadX87(index), nil @@ -302,13 +1036,16 @@ func (c *amd64Ctx) evalX87F64(op Operand) (string, error) { } func (c *amd64Ctx) storeX87F64(dst Operand, value string) error { + if handled, err := c.storeX87NativeFloatingMove(dst, LLVMType("double"), value); handled { + return err + } if index, ok := x87OperandReg(dst); ok { c.storeX87(index, value) return nil } switch dst.Kind { case OpFP: - return c.storeFPResult(dst.FPOffset, LLVMType("double"), value) + return c.storeScalarFPFloat(dst.FPOffset, 64, value) case OpMem: p, ptrType, err := c.ptrFromMem(dst.Mem) if err != nil { @@ -332,6 +1069,8 @@ func (c *amd64Ctx) storeX87I16(dst Operand, value string) error { switch dst.Kind { case OpReg: return c.storeRegSized(dst.Reg, I16, value) + case OpFP: + return c.storeFPResult(dst.FPOffset, I16, value) case OpMem: p, ptrType, err := c.ptrFromMem(dst.Mem) if err != nil { @@ -387,7 +1126,12 @@ func (c *amd64Ctx) roundX87Hardware(slot string) { } func (c *amd64Ctx) convertX87ToI64Hardware() string { - fmt.Fprintf(c.b, " call void asm sideeffect \"fldl $1; fistpll $0\", \"=*m,*m,~{dirflag},~{fpsr},~{flags}\"(ptr elementtype(i64) %s, ptr elementtype(double) %s)\n", c.x87IntegerSlot, c.x87Slot[0]) + if c.x87ExactSlot[0] == "" { + fmt.Fprintf(c.b, " call void asm sideeffect \"fldl $1; fistpll $0\", \"=*m,*m,~{dirflag},~{fpsr},~{flags}\"(ptr elementtype(i64) %s, ptr elementtype(double) %s)\n", c.x87IntegerSlot, c.x87Slot[0]) + } else { + fmt.Fprintf(c.b, " call void asm sideeffect \"cmpb $$0, $3; je 1f; fildll $2; jmp 2f; 1: fldl $1; 2: fistpll $0\", \"=*m,*m,*m,*m,~{dirflag},~{fpsr},~{flags}\"(ptr elementtype(i64) %s, ptr elementtype(double) %s, ptr elementtype(i64) %s, ptr elementtype(i1) %s)\n", + c.x87IntegerSlot, c.x87Slot[0], c.x87IntegerValueSlot[0], c.x87ExactSlot[0]) + } value := c.newTmp() fmt.Fprintf(c.b, " %%%s = load i64, ptr %s\n", value, c.x87IntegerSlot) return "%" + value @@ -436,7 +1180,11 @@ func (c *amd64Ctx) roundNearestEvenX87(value, floor string) string { floorInt := c.newTmp() oddBits := c.newTmp() odd := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fptosi double %s to i64\n", floorInt, floor) + // Only values below 2^52 need a tie-parity calculation. Keep even the + // discarded large/NaN path free of an out-of-range fptosi operand. + parityInput := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, double 0.000000e+00, double %s\n", parityInput, large, floor) + fmt.Fprintf(c.b, " %%%s = fptosi double %%%s to i64\n", floorInt, parityInput) fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 1\n", oddBits, floorInt) fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", odd, oddBits) tieUp := c.newTmp() diff --git a/amd64_lower_x87_conditional_move.go b/amd64_lower_x87_conditional_move.go new file mode 100644 index 000000000..d0b76ee15 --- /dev/null +++ b/amd64_lower_x87_conditional_move.go @@ -0,0 +1,59 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerX87ConditionalMove(op Op, ins Instr) error { + if len(ins.Args) != 2 { + return fmt.Errorf("%s %s expects one x87 source and F0 destination: %q", c.goarch, op, ins.Raw) + } + source, sourceOK := x87OperandReg(ins.Args[0]) + destination, destinationOK := x87OperandReg(ins.Args[1]) + if !sourceOK || !destinationOK || destination != 0 { + return fmt.Errorf("%s %s operands are outside Go 1.27's Yrf,F0 row: %q", c.goarch, op, ins.Raw) + } + + cf := func() string { return c.loadFlag(c.flagsCFSlot) } + zf := func() string { return c.loadFlag(c.flagsZSlot) } + pf := func() string { return c.loadFlag(c.flagsPFSlot) } + not := func(value string) string { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", inverted, value) + return "%" + inverted + } + combine := func(instruction, left, right string) string { + condition := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i1 %s, %s\n", condition, instruction, left, right) + return "%" + condition + } + + condition := "" + switch op { + case "FCMOVB", "FCMOVCS": + condition = cf() + case "FCMOVBE", "FCMOVLS": + condition = combine("or", cf(), zf()) + case "FCMOVE", "FCMOVEQ": + condition = zf() + case "FCMOVNB", "FCMOVCC": + condition = not(cf()) + case "FCMOVNBE", "FCMOVHI": + condition = combine("and", not(cf()), not(zf())) + case "FCMOVNE": + condition = not(zf()) + case "FCMOVNU": + condition = not(pf()) + case "FCMOVU", "FCMOVUN": + condition = pf() + default: + return fmt.Errorf("unsupported x87 conditional move %s", op) + } + sourceValue := c.loadX87IntegerValue(source) + destinationValue := c.loadX87IntegerValue(0) + selected := c.newTmp() + integer, exact := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, double %s, double %s\n", selected, condition, sourceValue.floating, destinationValue.floating) + fmt.Fprintf(c.b, " %%%s = select i1 %s, i64 %s, i64 %s\n", integer, condition, sourceValue.integer, destinationValue.integer) + fmt.Fprintf(c.b, " %%%s = select i1 %s, i1 %s, i1 %s\n", exact, condition, sourceValue.exact, destinationValue.exact) + c.storeX87IntegerValue(0, x87IntegerValue{"%" + selected, "%" + integer, "%" + exact}) + return nil +} diff --git a/amd64_lower_x87_conditional_move_test.go b/amd64_lower_x87_conditional_move_test.go new file mode 100644 index 000000000..1aca8ba98 --- /dev/null +++ b/amd64_lower_x87_conditional_move_test.go @@ -0,0 +1,217 @@ +package plan9asm + +import ( + "errors" + "runtime" + "strings" + "testing" +) + +var x87ConditionalMoveGo127Ops = []string{ + "FCMOVCC", "FCMOVCS", "FCMOVEQ", "FCMOVHI", "FCMOVLS", + "FCMOVB", "FCMOVBE", "FCMOVNB", "FCMOVNBE", "FCMOVE", + "FCMOVNE", "FCMOVNU", "FCMOVU", "FCMOVUN", +} + +func x86X87ConditionalMoveCompleteFormsSource() string { + var source strings.Builder + source.WriteString("TEXT x87conditionalmoveforms(SB),$0-0\n") + for _, op := range x87ConditionalMoveGo127Ops { + source.WriteString("\t" + op + " F0, F0\n") + source.WriteString("\t" + op + " F7, F0\n") + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86X87ConditionalMoveCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86X87ConditionalMoveCompleteFormsSource() + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"x87conditionalmoveforms": {Name: "x87conditionalmoveforms", Ret: Void}}, + } + if ir, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("original complete Go format fixture borrowed undefined F0/F7 or FLAGS", err) + } + // Retain every original conditional form, but supply actual source + // values and predicate bits in a separately labeled counterpart. + const header = "TEXT x87conditionalmoveforms(SB),$0-0\n" + initialized := strings.Replace(source, header, header+"FINIT\n"+strings.Repeat("FLD1\n", 8)+ + "MOVL $0,AX\nCMPL AX,AX\n", 1) + initialized = strings.Replace(initialized, "\tRET\n", "FINIT\n\tRET\n", 1) + if strings.Count(source, header) != 1 || strings.Count(initialized, "FLD1\n") != 8 { + t.Fatal("initialized counterpart lost its complete original source anchor") + } + file, err = Parse(ArchAMD64, initialized) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"select i1", "or i1", "and i1", "xor i1"} { + if !strings.Contains(ir, want) { + t.Fatalf("x87 conditional-move lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "x87-conditional-move-"+target.name+".ll", "x87-conditional-move-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86X87ConditionalMoveRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, instruction := range []string{ + "FCMOVB 0(AX), F0", + "FCMOVBE F8, F0", + "FCMOVE F1, F2", + "FCMOVNB F1", + "FCMOVNBE F1, F0, F2", + "FCMOVU.Z F1, F0", + } { + for _, goarch := range []string{"amd64", "386"} { + t.Run(goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yfcmv table", instruction) + } + }) + } + } +} + +func TestAMD64X87ConditionalMoveRuntimeConditions(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT x87conditionalmovesemantics(SB),NOSPLIT,$0-8 + MOVQ data+0(FP), DI + + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + STC + FCMOVB F1, F0 + FMOVDP F0, 0(DI) + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + STC + FCMOVNB F1, F0 + FMOVDP F0, 8(DI) + + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVE F1, F0 + FMOVDP F0, 16(DI) + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVNE F1, F0 + FMOVDP F0, 24(DI) + + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVBE F1, F0 + FMOVDP F0, 32(DI) + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVNBE F1, F0 + FMOVDP F0, 40(DI) + + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVU F1, F0 + FMOVDP F0, 48(DI) + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVNU F1, F0 + FMOVDP F0, 56(DI) + RET +` + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + if ir, err := x86X87InitializedWordAPI(t, source, "func x87conditionalmovesemantics(data *byte)", + "amd64", triple, "text", X87Auto); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("original pointer fixture acquired an unproved native region", err) + } + const mainC = ` +#include +#include +extern void x87conditionalmovesemantics(uint8_t *); +int main(void) { + double data[10] = {0}; + data[8] = 1.25; + data[9] = 9.5; + x87conditionalmovesemantics((uint8_t *)data); + const double expected[8] = {1.25, 9.5, 1.25, 9.5, 1.25, 9.5, 1.25, 9.5}; + for (int i = 0; i < 8; ++i) if (data[i] != expected[i]) return 10+i; + return 0; +} +` + // The original C checks remain unchanged. A separate typed-scalar source + // returns values; C owns the output array instead of granting native stores + // from a pointer declaration or a translator-private scratch allocation. + assembly, declaration := x86X87ConditionalScalarCounterpart() + ir, err := x86X87InitializedWordAPI(t, assembly, declaration, "amd64", triple, "text", X87Auto) + if err != nil || ir == "" { + t.Fatal("typed conditional counterpart did not reach native execution", err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "x87_conditional_move_semantics", triple, ir, + x86X87ConditionalScalarC(mainC), runPrefix) +} diff --git a/amd64_lower_xadd_test.go b/amd64_lower_xadd_test.go new file mode 100644 index 000000000..33fbba680 --- /dev/null +++ b/amd64_lower_xadd_test.go @@ -0,0 +1,301 @@ +package plan9asm + +import ( + "errors" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86XADDCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + source string + }{ + { + name: "darwin-amd64", + goarch: "amd64", + triple: "x86_64-apple-darwin", + source: ` +DATA xadddata+0(SB)/8, $0 +GLOBL xadddata(SB), $8 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB R11, R12 + XADDB DL, 8(BX) + XADDB R11, xadddata(SB) + XADDW DX, BX + XADDW R11, R12 + XADDW DX, 8(BX) + XADDW R11, xadddata(SB) + XADDL DX, BX + XADDL R11, R12 + XADDL DX, 8(BX) + XADDL R11, xadddata(SB) + XADDQ DX, BX + XADDQ R11, R12 + XADDQ DX, 8(BX) + XADDQ R11, xadddata(SB) + RET +`, + }, + { + name: "linux-amd64", + goarch: "amd64", + triple: "x86_64-unknown-linux-gnu", + source: ` +DATA xadddata+0(SB)/8, $0 +GLOBL xadddata(SB), $8 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB R11, R12 + XADDB DL, 8(BX) + XADDB R11, xadddata(SB) + XADDW DX, BX + XADDW R11, R12 + XADDW DX, 8(BX) + XADDW R11, xadddata(SB) + XADDL DX, BX + XADDL R11, R12 + XADDL DX, 8(BX) + XADDL R11, xadddata(SB) + XADDQ DX, BX + XADDQ R11, R12 + XADDQ DX, 8(BX) + XADDQ R11, xadddata(SB) + RET +`, + }, + { + name: "windows-amd64", + goarch: "amd64", + triple: "x86_64-pc-windows-msvc", + source: ` +DATA xadddata+0(SB)/8, $0 +GLOBL xadddata(SB), $8 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB R11, R12 + XADDB DL, 8(BX) + XADDB R11, xadddata(SB) + XADDW DX, BX + XADDW R11, R12 + XADDW DX, 8(BX) + XADDW R11, xadddata(SB) + XADDL DX, BX + XADDL R11, R12 + XADDL DX, 8(BX) + XADDL R11, xadddata(SB) + XADDQ DX, BX + XADDQ R11, R12 + XADDQ DX, 8(BX) + XADDQ R11, xadddata(SB) + RET +`, + }, + { + name: "linux-386", + goarch: "386", + triple: "i386-unknown-linux-gnu", + source: ` +DATA xadddata386+0(SB)/4, $0 +GLOBL xadddata386(SB), $4 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB BP, DI + XADDB DL, 8(BX) + XADDB SI, xadddata386(SB) + XADDW DX, BX + XADDW SI, DI + XADDW DX, 8(BX) + XADDW SI, xadddata386(SB) + XADDL DX, BX + XADDL SI, DI + XADDL DX, 8(BX) + XADDL SI, xadddata386(SB) + RET +`, + }, + { + name: "windows-386", + goarch: "386", + triple: "i686-pc-windows-msvc", + source: ` +DATA xadddata386+0(SB)/4, $0 +GLOBL xadddata386(SB), $4 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB BP, DI + XADDB DL, 8(BX) + XADDB SI, xadddata386(SB) + XADDW DX, BX + XADDW SI, DI + XADDW DX, 8(BX) + XADDW SI, xadddata386(SB) + XADDL DX, BX + XADDL SI, DI + XADDL DX, 8(BX) + XADDL SI, xadddata386(SB) + RET +`, + }, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, target.source, true) + file, err := Parse(ArchAMD64, target.source) + if err != nil { + t.Fatal(err) + } + signatures := map[string]FuncSig{"xaddforms": {Name: "xaddforms", Ret: Void}} + ll, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: signatures, + }) + if ll != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original uninitialized exchange formats acquired source inputs", err) + } + ll = x86ScalarFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0) + for _, want := range []string{"add i8", "add i16", "add i32", "store i1"} { + if !strings.Contains(ll, want) { + t.Fatalf("XADD lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "add i64") { + t.Fatalf("XADDQ lowering omitted add i64:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "xadd-"+target.name+".ll", "xadd-"+target.name+".o", ll) + fixture := x86InitializedExchangeFixture(t, target.source, file, target.goarch, "xaddforms") + x86OriginalGPInitializedFormatObjects(t, []x86OriginalGPInitializedFormat{fixture}, target.goarch, target.triple) + }) + } +} + +func TestTranslateX86XADDRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "XADDB (BX), DL"}, + {goarch: "amd64", instruction: "XADDW $1, DX"}, + {goarch: "amd64", instruction: "XADDL X0, DX"}, + {goarch: "amd64", instruction: "XADDQ DX, $1"}, + {goarch: "amd64", instruction: "XADDQ DX"}, + {goarch: "amd64", instruction: "XADDL DX, BX, AX"}, + {goarch: "amd64", instruction: "XADDL.Z DX, BX"}, + {goarch: "386", instruction: "XADDB SP, (BX)"}, + {goarch: "386", instruction: "XADDL R8, (BX)"}, + {goarch: "386", instruction: "XADDL DX, 1(R8)"}, + {goarch: "386", instruction: "XADDQ DX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's XADD optab", test.instruction) + } + }) + } +} + +func TestAMD64XADDRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT xaddsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $5, AX + MOVQ $7, BX + XADDQ AX, BX + MOVQ AX, 0(DI) + MOVQ BX, 8(DI) + MOVB $0xff, 16(DI) + MOVB $1, CL + XADDB CL, 16(DI) + MOVB CL, 17(DI) + SETCS 18(DI) + SETEQ 19(DI) + SETPS 20(DI) + SETMI 21(DI) + SETOS 22(DI) + MOVQ $0x1122, AX + XADDB AH, AL + MOVQ AX, 24(DI) + MOVQ $0x1122334455660001, AX + MOVQ $0x887766554433ffff, BX + XADDW AX, BX + MOVQ AX, 32(DI) + MOVQ BX, 40(DI) + MOVQ $3, AX + XADDQ AX, AX + MOVQ AX, 48(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "xaddsemantics": { + Name: "xaddsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void xaddsemantics(uint8_t *); +int main(void) { + uint8_t out[56] = {0}; + xaddsemantics(out); + if (*(uint64_t *)(out+0) != 7 || *(uint64_t *)(out+8) != 12) return 10; + if (out[16] != 0 || out[17] != 0xff) return 11; + if (out[18] != 1 || out[19] != 1 || out[20] != 1) return 12; + if (out[21] != 0 || out[22] != 0) return 13; + if (*(uint64_t *)(out+24) != 0x2233) return 14; + if (*(uint64_t *)(out+32) != UINT64_C(0x112233445566ffff)) return 15; + if (*(uint64_t *)(out+40) != UINT64_C(0x8877665544330000)) return 16; + if (*(uint64_t *)(out+48) != 6) return 17; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "xadd_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_xchg_test.go b/amd64_lower_xchg_test.go new file mode 100644 index 000000000..b745b36b0 --- /dev/null +++ b/amd64_lower_xchg_test.go @@ -0,0 +1,227 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func x86XCHGCompleteFormsSource(goarch string) string { + var source strings.Builder + symbol := "xchgdata" + size := 8 + if goarch == "386" { + symbol = "xchgdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $0\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT xchgforms(SB),$0-0\n") + if goarch == "amd64" { + source.WriteString("\tXCHGB AH, BL\n\tXCHGB R11, R12\n") + } else { + source.WriteString("\tXCHGB AH, BL\n\tXCHGB BP, DI\n") + } + source.WriteString("\tXCHGB DL, 8(BX)\n\tXCHGB 8(BX), DL\n") + if goarch == "amd64" { + source.WriteString("\tXCHGB R11, " + symbol + "(SB)\n\tXCHGB " + symbol + "(SB), R11\n") + } else { + source.WriteString("\tXCHGB SI, " + symbol + "(SB)\n\tXCHGB " + symbol + "(SB), SI\n") + } + widths := []string{"W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + if goarch == "amd64" { + fmt.Fprintf(&source, "\tXCHG%s DX, BX\n\tXCHG%s R11, R12\n", width, width) + } else { + fmt.Fprintf(&source, "\tXCHG%s DX, BX\n\tXCHG%s SI, DI\n", width, width) + } + fmt.Fprintf(&source, "\tXCHG%s DX, 8(BX)\n\tXCHG%s 8(BX), DX\n", width, width) + if goarch == "amd64" { + fmt.Fprintf(&source, "\tXCHG%s R11, %s(SB)\n\tXCHG%s %s(SB), R11\n", width, symbol, width, symbol) + } else { + fmt.Fprintf(&source, "\tXCHG%s SI, %s(SB)\n\tXCHG%s %s(SB), SI\n", width, symbol, width, symbol) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86XCHGCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86XCHGCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + signatures := map[string]FuncSig{"xchgforms": {Name: "xchgforms", Ret: Void}} + ll, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: signatures, + }) + if ll != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original uninitialized exchange formats acquired source inputs", err) + } + ll = x86ScalarFormatObjectUnit(t, file, signatures, target.goarch, target.triple, 0) + for _, want := range []string{"asm sideeffect \"xchg", "i8", "i16", "i32"} { + if !strings.Contains(ll, want) { + t.Fatalf("XCHG lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "i64") { + t.Fatalf("XCHGQ lowering omitted i64 operations:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "xchg-"+target.name+".ll", "xchg-"+target.name+".o", ll) + fixture := x86InitializedExchangeFixture(t, source, file, target.goarch, "xchgforms") + x86OriginalGPInitializedFormatObjects(t, []x86OriginalGPInitializedFormat{fixture}, target.goarch, target.triple) + }) + } +} + +func TestTranslateX86XCHGRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "XCHGB X0, DL"}, + {goarch: "amd64", instruction: "XCHGW $1, DX"}, + {goarch: "amd64", instruction: "XCHGL 0(BX), 8(CX)"}, + {goarch: "amd64", instruction: "XCHGQ DX, X0"}, + {goarch: "amd64", instruction: "XCHGW DX"}, + {goarch: "amd64", instruction: "XCHGL DX, BX, AX"}, + {goarch: "amd64", instruction: "XCHGL.Z DX, BX"}, + {goarch: "386", instruction: "XCHGB SP, (BX)"}, + {goarch: "386", instruction: "XCHGL R8, (BX)"}, + {goarch: "386", instruction: "XCHGL 1(R8), DX"}, + {goarch: "386", instruction: "XCHGQ DX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's XCHG optab", test.instruction) + } + }) + } +} + +func TestAMD64XCHGRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT xchgsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0x1122, AX + MOVQ $0x3344, BX + XCHGB AH, BL + MOVQ AX, 0(DI) + MOVQ BX, 8(DI) + MOVQ $0x1122334455667788, AX + MOVQ $0x887766554433aabb, BX + XCHGW AX, BX + MOVQ AX, 16(DI) + MOVQ BX, 24(DI) + MOVQ $0x1122334455667788, AX + MOVQ $0x88776655aabbccdd, BX + XCHGL AX, BX + MOVQ AX, 32(DI) + MOVQ BX, 40(DI) + MOVQ $5, AX + MOVQ $7, BX + XCHGQ AX, BX + MOVQ AX, 48(DI) + MOVQ BX, 56(DI) + MOVB $0x5a, 64(DI) + MOVQ $0x11223344556677c3, AX + CMPQ AX, AX + STC + XCHGB 64(DI), AL + MOVQ AX, 72(DI) + SETCS 80(DI) + SETEQ 81(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "xchgsemantics": { + Name: "xchgsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void xchgsemantics(uint8_t *); +int main(void) { + uint8_t out[88] = {0}; + xchgsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0x4422)) return 10; + if (*(uint64_t *)(out+8) != UINT64_C(0x3311)) return 11; + if (*(uint64_t *)(out+16) != UINT64_C(0x112233445566aabb)) return 12; + if (*(uint64_t *)(out+24) != UINT64_C(0x8877665544337788)) return 13; + if (*(uint64_t *)(out+32) != UINT64_C(0xaabbccdd)) return 14; + if (*(uint64_t *)(out+40) != UINT64_C(0x55667788)) return 15; + if (*(uint64_t *)(out+48) != 7 || *(uint64_t *)(out+56) != 5) return 16; + if (out[64] != 0xc3 || *(uint64_t *)(out+72) != UINT64_C(0x112233445566775a)) return 17; + if (out[80] != 1 || out[81] != 1) return 18; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "xchg_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_xlat.go b/amd64_lower_xlat.go new file mode 100644 index 000000000..c93291352 --- /dev/null +++ b/amd64_lower_xlat.go @@ -0,0 +1,35 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerXLAT(op Op, ins Instr) (ok bool, terminated bool, err error) { + known, err := x86XLATProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + addressType := I64 + if c.goarch == "386" { + addressType = I32 + } + base, err := c.evalIntSized(Operand{Kind: OpReg, Reg: BX}, addressType) + if err != nil { + return true, false, err + } + index, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AL}, I8) + if err != nil { + return true, false, err + } + wideIndex, address := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i8 %s to %s\n", wideIndex, index, addressType) + fmt.Fprintf(c.b, " %%%s = add %s %s, %%%s\n", address, addressType, base, wideIndex) + addressValue := "%" + address + if addressType == I32 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, addressValue) + addressValue = "%" + wide + } + pointer := c.ptrFromAddrI64(addressValue) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile i8, ptr %s, align 1\n", value, pointer) + return true, false, c.storeRegSized(AL, I8, "%"+value) +} diff --git a/amd64_lower_xstate.go b/amd64_lower_xstate.go new file mode 100644 index 000000000..da007ebbf --- /dev/null +++ b/amd64_lower_xstate.go @@ -0,0 +1,155 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64XStateFamily uint8 + +const ( + amd64XStateFX amd64XStateFamily = iota + amd64XStateXSAVE + amd64XStateXSAVEOPT + amd64XStateXSAVEC + amd64XStateXSAVES + amd64XStateXRSTOR + amd64XStateXRSTORS +) + +type amd64XStateDirection uint8 + +const ( + amd64XStateSave amd64XStateDirection = iota + amd64XStateRestore +) + +type amd64XStateSpec struct { + family amd64XStateFamily + direction amd64XStateDirection + mode64 bool + implicitMask bool + memoryBytes int +} + +// amd64XStateSpecs is the complete Go 1.27 extended-state memory grammar. +// Opcode spelling selects only the state family, transfer direction, and +// encoding mode; one parser owns the shared Ym memory operand rule. +var amd64XStateSpecs = map[Op]amd64XStateSpec{ + "FXSAVE": {family: amd64XStateFX, direction: amd64XStateSave, memoryBytes: 512}, + "FXSAVE64": {family: amd64XStateFX, direction: amd64XStateSave, mode64: true, memoryBytes: 512}, + "FXRSTOR": {family: amd64XStateFX, direction: amd64XStateRestore, memoryBytes: 512}, + "FXRSTOR64": {family: amd64XStateFX, direction: amd64XStateRestore, mode64: true, memoryBytes: 512}, + "XSAVE": {family: amd64XStateXSAVE, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVE64": {family: amd64XStateXSAVE, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVEOPT": {family: amd64XStateXSAVEOPT, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVEOPT64": {family: amd64XStateXSAVEOPT, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVEC": {family: amd64XStateXSAVEC, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVEC64": {family: amd64XStateXSAVEC, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVES": {family: amd64XStateXSAVES, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVES64": {family: amd64XStateXSAVES, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XRSTOR": {family: amd64XStateXRSTOR, direction: amd64XStateRestore, implicitMask: true, memoryBytes: 1}, + "XRSTOR64": {family: amd64XStateXRSTOR, direction: amd64XStateRestore, mode64: true, implicitMask: true, memoryBytes: 1}, + "XRSTORS": {family: amd64XStateXRSTORS, direction: amd64XStateRestore, implicitMask: true, memoryBytes: 1}, + "XRSTORS64": {family: amd64XStateXRSTORS, direction: amd64XStateRestore, mode64: true, implicitMask: true, memoryBytes: 1}, +} + +type amd64XStateForm struct { + memory Operand +} + +func (c *amd64Ctx) lowerXState(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64XStateSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.mode64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + form, err := c.parseXStateForm(baseOp, ins) + if err != nil { + return true, false, err + } + return c.emitXState(spec, form) +} + +func (c *amd64Ctx) parseXStateForm(op string, ins Instr) (amd64XStateForm, error) { + var form amd64XStateForm + if len(ins.Args) != 1 { + return form, fmt.Errorf("%s %s expects one Go 1.27 Ym memory operand: %q", c.goarch, op, ins.Raw) + } + memory := ins.Args[0] + if !isAMD64MemoryOperand(memory) { + return form, fmt.Errorf("%s %s expects Go 1.27 Ym memory: %q", c.goarch, op, ins.Raw) + } + if memory.Kind == OpMem && !x86MemoryRegistersValidForArch(memory.Mem, c.goarch) { + return form, fmt.Errorf("%s %s address uses an out-of-range register: %q", c.goarch, op, ins.Raw) + } + return amd64XStateForm{memory: memory}, nil +} + +func (spec amd64XStateSpec) mnemonic() string { + var mnemonic string + switch spec.family { + case amd64XStateFX: + mnemonic = "fxsave" + if spec.direction == amd64XStateRestore { + mnemonic = "fxrstor" + } + case amd64XStateXSAVE: + mnemonic = "xsave" + case amd64XStateXSAVEOPT: + mnemonic = "xsaveopt" + case amd64XStateXSAVEC: + mnemonic = "xsavec" + case amd64XStateXSAVES: + mnemonic = "xsaves" + case amd64XStateXRSTOR: + mnemonic = "xrstor" + case amd64XStateXRSTORS: + mnemonic = "xrstors" + default: + panic("unknown amd64 xstate family") + } + if spec.mode64 { + mnemonic += "64" + } + return mnemonic +} + +func (c *amd64Ctx) emitXState(spec amd64XStateSpec, form amd64XStateForm) (bool, bool, error) { + pointer, pointerType, err := c.x86DescriptorMemoryPointer(form.memory) + if err != nil { + return true, false, err + } + memoryConstraint := "=*m" + if spec.direction == amd64XStateRestore { + memoryConstraint = "*m" + } + constraints := memoryConstraint + arguments := fmt.Sprintf("%s elementtype([%d x i8]) %s", pointerType, spec.memoryBytes, pointer) + if spec.implicitMask { + ax, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) + if err != nil { + return true, false, err + } + dx, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I32) + if err != nil { + return true, false, err + } + constraints += ",{ax},{dx}" + arguments += fmt.Sprintf(", i32 %s, i32 %s", ax, dx) + } + constraints += ",~{memory},~{dirflag},~{fpsr},~{flags}" + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s)\n", + spec.mnemonic()+" $0", constraints, arguments) + return true, false, nil +} diff --git a/amd64_lower_xstate_test.go b/amd64_lower_xstate_test.go new file mode 100644 index 000000000..d8d37f09a --- /dev/null +++ b/amd64_lower_xstate_test.go @@ -0,0 +1,211 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +func TestAMD64XStateGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64XStateSpecs)) + for op := range amd64XStateSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{ + "FXRSTOR", "FXRSTOR64", "FXSAVE", "FXSAVE64", + "XRSTOR", "XRSTOR64", "XRSTORS", "XRSTORS64", + "XSAVE", "XSAVE64", "XSAVEC", "XSAVEC64", "XSAVEOPT", "XSAVEOPT64", "XSAVES", "XSAVES64", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("xstate grammar opcodes = %v, want %v", got, want) + } + for op, want := range map[Op]amd64XStateSpec{ + "FXSAVE": {family: amd64XStateFX, direction: amd64XStateSave, memoryBytes: 512}, + "FXSAVE64": {family: amd64XStateFX, direction: amd64XStateSave, mode64: true, memoryBytes: 512}, + "FXRSTOR": {family: amd64XStateFX, direction: amd64XStateRestore, memoryBytes: 512}, + "FXRSTOR64": {family: amd64XStateFX, direction: amd64XStateRestore, mode64: true, memoryBytes: 512}, + "XSAVE": {family: amd64XStateXSAVE, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVE64": {family: amd64XStateXSAVE, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVEOPT": {family: amd64XStateXSAVEOPT, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVEOPT64": {family: amd64XStateXSAVEOPT, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVEC": {family: amd64XStateXSAVEC, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVEC64": {family: amd64XStateXSAVEC, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVES": {family: amd64XStateXSAVES, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVES64": {family: amd64XStateXSAVES, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XRSTOR": {family: amd64XStateXRSTOR, direction: amd64XStateRestore, implicitMask: true, memoryBytes: 1}, + "XRSTOR64": {family: amd64XStateXRSTOR, direction: amd64XStateRestore, mode64: true, implicitMask: true, memoryBytes: 1}, + "XRSTORS": {family: amd64XStateXRSTORS, direction: amd64XStateRestore, implicitMask: true, memoryBytes: 1}, + "XRSTORS64": {family: amd64XStateXRSTORS, direction: amd64XStateRestore, mode64: true, implicitMask: true, memoryBytes: 1}, + } { + if got := amd64XStateSpecs[op]; got != want { + t.Errorf("xstate grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86XStateCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT xstateforms(SB),$0-0 + FXSAVE (BX) + FXRSTOR 8(BP) + XSAVE 16(SI) + XRSTOR 24(DI) + XSAVEOPT 32(BX) + XSAVEC 40(BP) + XSAVES 48(SI) + XRSTORS 56(DI) +` + if target.goarch == "amd64" { + source += ` FXSAVE64 64(R11) + FXRSTOR64 72(R12) + XSAVE64 80(R13) + XRSTOR64 88(R14) + XSAVEOPT64 96(R15) + XSAVEC64 104(R11) + XSAVES64 112(R12) + XRSTORS64 120(R13) +` + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"xstateforms": {Name: "xstateforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"fxsave", "fxrstor", "xsave", "xrstor", "xsaveopt", "xsavec", "xsaves", "xrstors"} { + if !strings.Contains(ir, want+" $0") { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if want := `"target-features"="+fxsr,+xsave,+xsavec,+xsaveopt,+xsaves"`; !strings.Contains(ir, want) { + t.Errorf("IR is missing complete xstate feature set %s:\n%s", want, ir) + } + compileLLVMToObject(t, llc, target.triple, "xstate-"+target.name+".ll", "xstate-"+target.name+".o", ir) + }) + } +} + +func TestAMD64FXStateRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT fxstatesemantics(SB),NOSPLIT,$0-8 + MOVQ area+0(FP), DI + FXSAVE (DI) + FXRSTOR (DI) + FXSAVE64 512(DI) + FXRSTOR64 512(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"fxstatesemantics": { + Name: "fxstatesemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void fxstatesemantics(uint8_t *); +int main(void) { + __attribute__((aligned(16))) uint8_t area[1024]; + memset(area, 0xa5, sizeof(area)); + fxstatesemantics(area); + uint16_t control0 = 0, control1 = 0; + memcpy(&control0, area, sizeof(control0)); + memcpy(&control1, area + 512, sizeof(control1)); + if (control0 == UINT16_C(0xa5a5)) return 10; + if (control1 == UINT16_C(0xa5a5)) return 11; + if (control0 != control1) return 12; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "fxstate_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateX86XStateRejectsFormsOutsideGoTables(t *testing.T) { + tests := []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "FXSAVE AX"}, + {goarch: "amd64", instruction: "FXRSTOR $1"}, + {goarch: "amd64", instruction: "XSAVE"}, + {goarch: "amd64", instruction: "XRSTOR (BX), (CX)"}, + {goarch: "amd64", instruction: "XSAVEOPT.Z (BX)"}, + {goarch: "amd64", instruction: "XSAVEC X0"}, + {goarch: "amd64", instruction: "XSAVES AX"}, + {goarch: "amd64", instruction: "XRSTORS $1"}, + {goarch: "386", instruction: "XSAVE (R11)"}, + } + for _, op := range []string{"FXSAVE64", "FXRSTOR64", "XSAVE64", "XRSTOR64", "XSAVEOPT64", "XSAVEC64", "XSAVES64", "XRSTORS64"} { + tests = append(tests, struct { + goarch string + instruction string + }{goarch: "386", instruction: op + " (BX)"}) + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's xstate tables", test.instruction) + } + }) + } +} diff --git a/amd64_madd_typed_forms_test.go b/amd64_madd_typed_forms_test.go new file mode 100644 index 000000000..dd9f875f9 --- /dev/null +++ b/amd64_madd_typed_forms_test.go @@ -0,0 +1,420 @@ +package plan9asm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86MADDOfficialRows(t *testing.T, arch string) []string { + t.Helper() + byOpcode := x86OfficialPackedBinaryWitnesses(t) + var rows []string + for _, name := range []string{"VPMADDWD", "VPMADDUBSW"} { + if len(byOpcode[name]) == 0 { + t.Fatal("official MADD table missing", name) + } + for _, row := range byOpcode[name] { + if arch != "386" || strings.Count(row, ",") != 3 { + rows = append(rows, row) + } + } + } + return rows +} + +// Full Go source classes, including valid forms lacking native backing or +// physical mode32 identity. These are not all object/runtime positives. +func x86MADDWholeFormats(t *testing.T, arch string) []string { + rows := x86MADDOfficialRows(t, arch) + for _, name := range []string{"PMADDWL", "PMADDUBSW"} { + for _, source := range []string{"X0", "X7", "(AX)", "16(AX)(CX*2)", "value+0(FP)", "local-16(SP)", "data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+",X1") + } + if arch == "amd64" { + rows = append(rows, name+" X15,X0") + } + } + if arch == "amd64" { + for _, source := range []string{"M0", "M7", "(AX)", "value+0(FP)", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + rows = append(rows, "PMADDWL "+source+",M1") + } + } + for _, name := range []string{"VPMADDWD", "VPMADDUBSW"} { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{"value+0(FP)", "local-64(SP)", "data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+","+bank+"1,"+bank+"2") + } + if arch != "386" || bank != "Z" { + rows = append(rows, name+" "+bank+"20,"+bank+"21,"+bank+"22") + } + } + } + return rows +} + +func TestX86MADDTypedCompleteGoAttributes(t *testing.T) { + if len(x86PackedMultiplyAddSpecs) != 4 { + t.Fatal("MADD source family must have four canonical Go names") + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APMADDWL, ymm, Py1, opBytes{0xf5, Pe, 0xf5}}", + "{APMADDUBSW, yxm_q4, Pq4, opBytes{0x04}}", + "{Zm_r_xm, 1, argList{Ymm, Ymr}}", "{Zm_r_xm, 2, argList{Yxm, Yxr}}", + "{Zm_r, 1, argList{Yxm, Yxr}}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatal("actual legacy opcode/class row changed", row) + } + } + for _, class := range []string{"Yax", "Ycx", "Yrx", "Yrl", "Yrl32", "Ym", "Ymr"} { + if !strings.Contains(string(legacy), "ycover["+class+"*Ymax+Ymm] = 1") { + t.Fatal("actual MMX source class changed", class) + } + } + for _, item := range []struct{ name, mapName, opcode string }{ + {"VPMADDWD", "0F", "F5"}, {"VPMADDUBSW", "0F38", "04"}, + } { + start := strings.Index(string(avx), "{as: A"+item.name+",") + if start < 0 { + t.Fatal("actual AVX opcode missing", item.name) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(row, "ytab: _yvandnpd, prefix: Pavx") || strings.Contains(row, "Bcst") || strings.Contains(row, "Sae") { + t.Fatal("actual opcode-specific attributes changed", row) + } + for _, width := range []string{"128", "256"} { + want := fmt.Sprintf("avxEscape | vex%s | vex66 | vex%s | vexW0, 0x%s", width, item.mapName, item.opcode) + if !strings.Contains(row, want) { + t.Fatal("actual VEX opcode attributes changed", want) + } + } + for i, width := range []string{"128", "256", "512"} { + want := fmt.Sprintf("avxEscape | evex%s | evex66 | evex%s | evexW0, evexN%d | evexZeroingEnabled, 0x%s", width, item.mapName, 16<= 3 { + physical = (index - 3) & 7 + } + if len(code) == 5 && code[0] == 0x41 { + code = code[1:] // Actual Go's ignored MMX REX.B annotation. + } + if want := []byte{0x0f, 0xf5, byte(0xc8 | physical), 0xc3}; !bytes.Equal(code, want) { + t.Fatalf("actual MMX physical bank changed: %s %x want%x", register, code, want) + } + if index >= 3 { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + context, ir := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := ir.String() + if _, _, err := context.lowerPackedWordMultiplyAdd("PMADDWL", file.Funcs[0].Instrs[1]); !errors.Is(err, ErrProbeNeedsContext) || ir.String() != before { + t.Fatal("GP spelling fabricated an initialized MMX source", register, err) + } + } + } +} + +func TestX86MADDTypedRetainsRejectedFormsAndVirtualMemory(t *testing.T) { + for _, row := range []string{ + "PMADDWD X0,X1", "PMADDWL X0,Y1", "PMADDWL M0,AX", "PMADDUBSW M0,M1", + "PMADDUBSW X0,(AX)", "PMADDUBSW.Z X0,X1", "VPMADDWD X0,Y1,Y2", + "VPMADDUBSW.BCST (AX),X0,K1,X1", "VPMADDWD.SAE Z0,Z1,Z2", + "VPMADDUBSW X0,X1,K0,X2", "VPMADDWD $1,X1,X2", "VPMADDUBSW X0,X1,(AX)", + } { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if err := ProbeInstruction(ArchAMD64, "amd64", file.Funcs[0].Instrs[1]); err == nil { + t.Fatal("Go-rejected MADD form obtained lowering", row) + } + } + for _, row := range []string{"PMADDUBSW source+0(FP),X1", "PMADDUBSW (SP),X1", "PMADDUBSW (BP),X1", "PMADDUBSW (AX)(BP*2),X1"} { + source := "TEXT f(SB),4,$0-64\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + context, ir := newAMD64CtxWithFuncForTest(t, file.Funcs[0], x86WordMaddFPType(t), nil) + before := ir.String() + if _, _, err := context.lowerPackedWordMultiplyAdd("PMADDUBSW", file.Funcs[0].Instrs[1]); !errors.Is(err, ErrProbeNeedsContext) || ir.String() != before { + t.Fatal("virtual storage became an original native m128 pointer", row, err) + } + } +} + +func TestX86MADDTypedByteCallerRetainsUnknownAndPartialInputs(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "VMOVUPS (DI),Z1\nVPMADDUBSW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Z0\nVPMADDUBSW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "partial pair": "MOVSS (DI),X0\nMOVSS (SI),X1\nVPMADDUBSW Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVPMADDUBSW.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nMOVL $1,AX\nKMOVQ AX,K1\nVPMADDUBSW Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPMADDUBSW (BX),Z0,Z2\nVMOVSS X2,(DX)", + "zero still full read": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPMADDUBSW.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "private SP": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPMADDUBSW (BX),X0\nVMOVSS X0,(DX)", + "join missing source": "VMOVUPS (DI),Z1\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z0\njoined:\nVPMADDUBSW Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("typed byte-MADD grammar became input/address permission", err) + } + }) + } +} + +func TestX86MADDTypedMalformedIdentityAndPhysicalBanks(t *testing.T) { + for _, name := range []string{"PMADDWL", "PMADDUBSW", "VPMADDWD", "VPMADDUBSW"} { + operands := "X0,X1" + if strings.HasPrefix(name, "V") { + operands = "X0,X1,X2" + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+name+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", original.Op, Op(strings.ToLower(name)), "MOVQ", Op(name + ".BAD")} { + ins := original + ins.Op = identity + before := fmt.Sprintf("%#v", ins) + context, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ir, temporary := out.String(), context.tmp + known, terminated, err := context.lowerPackedWordMultiplyAdd(original.Op, ins) + bad := identity != "" && !strings.EqualFold(string(identity), string(original.Op)) + if !known || terminated || (err != nil) != bad || bad && (out.String() != ir || context.tmp != temporary) || before != fmt.Sprintf("%#v", ins) { + t.Fatal("dispatch identity changed input or emitted before rejection", name, identity, err) + } + } + } + for _, opcode := range []string{"VPMADDWD", "VPMADDUBSW"} { + for _, bank := range []string{"X", "Y", "Z"} { + row := opcode + " " + bank + "20," + bank + "21," + bank + "22" + accepted := bank != "Z" + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", accepted) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, _, err = x86PackedMultiplyAddProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], "386") + if accepted && !errors.Is(err, ErrProbeNeedsContext) || !accepted && err == nil { + t.Fatal("Go syntax is not physical mode32 permission", row, err) + } + } + code := encodeX86RawEVEXPackedMADD(Op(opcode), 2, 1, 2, 0, 1, true, true) + decoded, _, _, err := decodedX86EVEXPackedMADDInstruction(code, 32) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(decoded) + if err != nil { + t.Fatal(err) + } + var detached Instr + if err := json.Unmarshal(data, &detached); err != nil { + t.Fatal(err) + } + if _, _, err := x86PackedMultiplyAddProperties(detached.Op, detached, "386"); err == nil { + t.Fatal("JSON/raw text fabricated physical mode32 mask provenance") + } + for _, address := range []int{16, 32, 64} { + changed := decoded + changed.x86AddressBits = address + if _, _, err := x86PackedMultiplyAddProperties(changed.Op, changed, "386"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound address-size marker became memory permission", err) + } + } + } +} + +func TestX86MADDTypedNamedAndPhysicalThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, raw := range []bool{false, true} { + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(fmt.Sprintf("raw=%t/%s", raw, target.triple), func(t *testing.T) { + rows := x86MADDOfficialRows(t, target.arch) + if raw { + rows = nil + for _, row := range x86MADDOfficialRows(t, "amd64") { + code := assembleX87ControlBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + code = code[:len(code)-1] + for w := 0; w < 2; w++ { + physical := append([]byte(nil), code...) + if physical[0] == 0xc5 { + physical = append([]byte{0xc4, physical[1]&0x80 | 0x61, physical[1] & 0x7f}, physical[2:]...) + } + physical[2] |= byte(w << 7) + var bytes strings.Builder + for _, value := range physical { + fmt.Fprintf(&bytes, "BYTE $%d\n", value) + } + rows = append(rows, bytes.String()) + } + } + } else { + rows = append(rows, "PMADDWL X0,X1", "PMADDWL (AX),X1", "PMADDUBSW X0,X1", "PMADDUBSW (AX),X1") + } + var source, declarations strings.Builder + declarations.WriteString("package madd\n") + for index, row := range rows { + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", index, row) + fmt.Fprintf(&declarations, "func F%d()\n", index) + } + if !raw { + file, sigs := x86UnboundVoidFormatSources(t, source.String(), target.arch, target.triple) + ir := x86ScalarFormatObjectUnit(t, file, sigs, target.arch, target.triple, 0) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + t.Logf("%d original named formats: Go objects, three public Context controls and emitter-only objects", len(rows)) + return + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + pkg := mustGoPackage(t, "example.invalid/madd", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for index := range rows { + name := fmt.Sprintf("F%d", index) + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), target.arch, types.SizesFor("gc", target.arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "madd-text.ll", "madd-text.o", ir) + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "madd-module.ll", "madd-module.o", module.String()) + result, err := translateGoModuleInContext(context, pkg, []byte(source.String()), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve, + }) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "madd-go.ll", "madd-go.o", result.Module.String()) + t.Logf("%d named/actual-Go-derived physical forms, three APIs/object axes; no caller or native runtime grant", len(rows)) + }) + } + } +} diff --git a/amd64_madd_typed_mode32_test.go b/amd64_madd_typed_mode32_test.go new file mode 100644 index 000000000..573334385 --- /dev/null +++ b/amd64_madd_typed_mode32_test.go @@ -0,0 +1,45 @@ +package plan9asm + +import "testing" + +func TestX86MADDTypedMode32LegacySourceRules(t *testing.T) { + for _, opcode := range []string{"PMADDWL", "PMADDUBSW"} { + for _, operands := range []string{"X0,X1", "X0,X7", "(AX),X7"} { + row := opcode + " " + operands + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", true) + code := assembleX87ControlBytes(t, "386", source) + if len(code) == 0 { + t.Fatal("accepted legacy source has no original Go machine bytes", row) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, known, err := x86PackedMultiplyAddProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], "386") + if !known || err != nil { + t.Errorf("accepted low mode32 source lost its typed shape: %s bytes%x: %v", row, code, err) + } + } + // Unlike the AVX name rows, the legacy encoder rejects these high + // source spellings; do not label them accepted physical Context rows. + for _, operands := range []string{"X8,X1", "X0,X15", "(AX),X8", "X16,X0", "X0,X16"} { + source := "TEXT f(SB),4,$0-0\n" + opcode + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, known, err := x86PackedMultiplyAddProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], "386") + if !known || err == nil { + t.Fatal("Go-rejected high legacy source obtained typed admission", source) + } + } + } + // The shared ymm is not a blanket 386 MMX row: Py1's original row + // selection rejects its z<1 MMX forms in 32-bit mode. + for _, source := range []string{"M0", "(AX)", "AX", "CX", "SI"} { + row := "PMADDWL " + source + ",M1" + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + } +} diff --git a/amd64_madd_typed_native_linux_amd64_test.go b/amd64_madd_typed_native_linux_amd64_test.go new file mode 100644 index 000000000..e9cf1301f --- /dev/null +++ b/amd64_madd_typed_native_linux_amd64_test.go @@ -0,0 +1,132 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The filename is the Linux/amd64 build role and the cross-runtime prefix is +// selected by required CI. Native AVX must be available: no Skip, Rosetta +// fallback or object-only substitution satisfies this original-Go axis. +func TestCrossLinuxRuntimeMatrixX86MADDTypedOriginalVEX(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + execution.requireAVX(t) + var declarations, assembly, table strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, row := range x86MADDOracleRows() { + if row.legacy || row.masking != 0 || row.width > 32 { + continue + } + name := fmt.Sprintf("V%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[48]byte,a,b *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-36\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nSTMXCSR saved-4(SP)\nLDMXCSR csr+32(FP)\nVMOVDQU (CX),Y0\nVMOVDQU (DX),Y1\nVMOVDQU (DX),Y2\nPUSHQ flags+24(FP)\nPOPFQ\n", name) + instruction, destination := row.instruction() + fmt.Fprintf(&assembly, "%s\nVMOVDQU Y%s,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,32(AX)\nSTMXCSR 40(AX)\nVZEROUPPER\nLDMXCSR saved-4(SP)\nRET\n", instruction, destination[1:]) + fmt.Fprintf(&table, "{%s,%d,%d},\n", name, row.width, row.inputBytes) + } + tests := `package oracle +import ( + "testing" + "encoding/binary" + "runtime" + "unsafe" +) + +var cases = []struct { + run func(*[48]byte, *[64]byte, *[64]byte, uint64, uint32) + width, bytes int +}{ +` + table.String() + `} + +func TestOriginal(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + edges := [...]int32{0, 1, -1, 127, -128, 255, 256, -257, 32767, -32768, 16384, -16384} + profiles := []uint32{0x1f80, 0x3fc1, 0x5f82, 0x9fe5} + observations := 0 + for row, entry := range cases { + for sample := 0; sample < 32; sample++ { + for _, csr := range profiles { + for seed := 0; seed < 2; seed++ { + var a, b [64]byte + var out, want [50]byte + for i := 0; i < 64; i += entry.bytes { + x, y := edges[(sample+i*3)%len(edges)], edges[(sample+7+i*3)%len(edges)] + a[i], b[i] = byte(x), byte(y) + if entry.bytes == 2 { + a[i+1], b[i+1] = byte(uint32(x)>>8), byte(uint32(y)>>8) + } + } + if entry.bytes == 2 && sample == 0 { + for i := 0; i < 64; i += 2 { + a[i] = 0 + b[i] = 0 + a[i+1] = 0x80 + b[i+1] = 0x80 + } + } + if entry.bytes == 1 && sample < 2 { + for i := range a { + a[i] = 0x7f + if sample == 1 { + a[i] = 0x80 + } + b[i] = 0xff + } + } + beforeA, beforeB := a, b + for i := range out { + out[i] = 0xc3 + want[i] = 0xc3 + } + + flags := uint64(2) + if seed != 0 { + flags |= 0x8d5 + } + entry.run((*[48]byte)(unsafe.Pointer(&out[1])), &a, &b, flags, csr) + for i := 0; i < 32; i++ { + want[i+1] = 0 + } + for i := 0; i < entry.width; i += entry.bytes * 2 { + var sum int64 + if entry.bytes == 2 { + sum = int64(int16(binary.LittleEndian.Uint16(a[i:]))) * int64(int16(binary.LittleEndian.Uint16(b[i:]))) + + int64(int16(binary.LittleEndian.Uint16(a[i+2:]))) * int64(int16(binary.LittleEndian.Uint16(b[i+2:]))) + } else { + sum = int64(int8(a[i]))*int64(b[i]) + int64(int8(a[i+1]))*int64(b[i+1]) + if sum < -32768 { + sum = -32768 + } + if sum > 32767 { + sum = 32767 + } + } + for j := 0; j < entry.bytes * 2; j++ { + want[i+j+1] = byte(uint32(sum) >> uint(j*8)) + } + } + + actualFlags := binary.LittleEndian.Uint64(out[33:41]) + actualCSR := binary.LittleEndian.Uint32(out[41:45]) + copy(want[33:45], out[33:45]) + observations++ + if out != want || a != beforeA || b != beforeB || actualFlags&0x8d5 != flags&0x8d5 || actualCSR != csr { + t.Fatalf("original VEX MADD row%d sample%d flags%x CSR%x got%x want%x", + row, sample, actualFlags, actualCSR, out, want) + } + } + } + } + } + if observations != len(cases)*32*4*2 { + t.Fatal("incomplete original VEX matrix", observations) + } +} +` + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), tests) + t.Logf("%d original VEX X/Y kernels, register/memory/all destination aliases ×32 samples ×4 CSR profiles ×2 six-flag seeds; full Y upper-zero/input/canaries. No original EVEX execution claim.", count) +} diff --git a/amd64_madd_typed_rules_test.go b/amd64_madd_typed_rules_test.go new file mode 100644 index 000000000..c0ea8286b --- /dev/null +++ b/amd64_madd_typed_rules_test.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +// Intel's PMADDWD and PMADDUBSW tables specify WIG, unlike EVEX PACK*DW. +// Bind W0 to stock Go's actual instruction, then independently compare the +// complete LLVM MC opcode/operands of the W1 physical counterpart. +func TestX86MADDTypedPhysicalWIGCompleteOperands(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM22 MC required") + } + for _, opcode := range []string{"VPMADDWD", "VPMADDUBSW"} { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "1", "16(AX)(CX*2)"} { + row := opcode + ".Z " + source + "," + bank + "2,K3," + bank + "4" + code := assembleX87ControlBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + code = code[:len(code)-1] + if code[0] != 0x62 || code[2]&0x80 != 0 { + t.Fatal("Go original is not the expected EVEX W0 row", row, code) + } + for _, mode := range []int{32, 64} { + t.Run(fmt.Sprintf("%d/%s", mode, row), func(t *testing.T) { + var texts [2]string + for w := 0; w < 2; w++ { + physical := append([]byte(nil), code...) + physical[2] |= byte(w << 7) + var input strings.Builder + for _, value := range physical { + fmt.Fprintf(&input, "0x%02x ", value) + } + triple := "x86_64" + if mode == 32 { + triple = "i386" + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple="+triple, + "--mattr=+avx2,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(input.String() + "\n") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || strings.Contains(string(output), "warning:") || + !strings.Contains(string(output), strings.ToLower(opcode)) { + t.Fatalf("independent MC rejected %x: %v\n%s", physical, err, output) + } + texts[w] = strings.TrimSpace(string(output)) + instruction, size, known, err := decodedX86EVEXPackedMADDInstruction(physical, mode) + if err != nil || !known || size != len(physical) { + t.Errorf("Go-bound WIG physical form rejected: %x %d/%t/%v", physical, size, known, err) + continue + } + arch := map[int]string{32: "386", 64: "amd64"}[mode] + if form, known, err := x86PackedMultiplyAddProperties(instruction.Op, instruction, arch); err != nil || !known || form.mask != 2 { + t.Errorf("physical MADD missing complete typed grammar: %+v %t/%v", form, known, err) + } + if width, write, valid := x86InstructionMemoryShape(instruction, 0, arch); source[0] == '1' && + (!valid || write || width != int64(amd64VectorByteWidth(Reg(bank+"0")))) { + t.Errorf("complete physical memory envelope missing: %d/%t/%t", width, write, valid) + } + } + if texts[0] != texts[1] { + t.Fatalf("WIG changed complete disassembled operands: W0=%q W1=%q", texts[0], texts[1]) + } + }) + } + } + } + } +} + +func TestX86MADDTypedCallerByteFamily(t *testing.T) { + rows := []string{"PMADDUBSW X0,X1", "PMADDUBSW X1,X1", "PMADDUBSW (DI),X1"} + for _, bank := range []string{"X", "Y", "Z"} { + for _, first := range []string{bank + "0", "(DI)"} { + rows = append(rows, + "VPMADDUBSW "+first+","+bank+"1,"+bank+"2", + "VPMADDUBSW "+first+","+bank+"1,K1,"+bank+"2", + "VPMADDUBSW.Z "+first+","+bank+"1,K1,"+bank+"2") + } + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + result := "X2" + if strings.HasPrefix(row, "PMADD") { + result = "X1" + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("Go-accepted byte-MADD is missing caller effects", triple, err) + } + } + }) + } +} + +func TestCrossLinuxRuntimeMatrixX86MADDTypedXUpperZero(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + assembly := "TEXT ·upper(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (DX),Z2\nVPMADDUBSW X0,X1,X2\nVMOVUPS Z2,(AX)\nRET\n" + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + ir := x86NarrowOracleIR(t, assembly, "package narroworacle\nfunc upper(out,a,b *[64]byte)\n", execution.triple) + main := `#include +#include +#include +extern void upper(void*,const void*,const void*); +int main(void) { + uint8_t a[64],b[64],out[66];memset(a,2,64);memset(b,3,64);memset(out,0xc3,66); + upper(out+1,a,b); + for(int i=0;i<64;++i) { + int want=i<16?(i&1?0:12):0; + if(out[i+1]!=want) {fprintf(stderr,"MADD alias byte%d got=%u want=%u\n",i,out[i+1],want);return 1;} + } + if(out[0]!=0xc3||out[65]!=0xc3)return 2; + return 0; +} +` + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "madd-x-upper", ir, main, level) + }) + } +} diff --git a/amd64_madd_typed_runtime_test.go b/amd64_madd_typed_runtime_test.go new file mode 100644 index 000000000..272e4766f --- /dev/null +++ b/amd64_madd_typed_runtime_test.go @@ -0,0 +1,434 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +type x86MADDOracleRow struct { + width, inputBytes, source, alias, masking int + legacy, mmx bool +} + +func x86MADDOracleRows() []x86MADDOracleRow { + var rows []x86MADDOracleRow + for _, inputBytes := range []int{1, 2} { + for _, width := range []int{8, 16, 32, 64} { + for _, legacy := range []bool{true, false} { + if legacy && width > 16 || !legacy && width == 8 || width == 8 && inputBytes == 1 { + continue + } + for source := 0; source < 2; source++ { + for alias := 0; alias < 3; alias++ { + if legacy && alias == 2 { + continue + } + for mask := 0; mask < 3; mask++ { + if legacy && mask != 0 { + continue + } + rows = append(rows, x86MADDOracleRow{width, inputBytes, source, alias, mask, legacy, width == 8}) + } + } + } + } + } + } + return rows +} + +func (row x86MADDOracleRow) instruction() (string, string) { + bank := map[int]string{8: "M", 16: "X", 32: "Y", 64: "Z"}[row.width] + first, destination := bank+"0", fmt.Sprintf("%s%d", bank, row.alias) + if row.legacy { + destination = bank + "1" + if row.alias == 1 { + first = destination + } + } + if row.source == 1 { + first = "(CX)" + if row.legacy && row.alias == 1 { + first = "(DX)" + } + } + name := "PMADDWL" + if row.inputBytes == 1 { + name = "PMADDUBSW" + } + if row.legacy { + return name + " " + first + "," + destination, destination + } + name = "VPMADDWD" + if row.inputBytes == 1 { + name = "VPMADDUBSW" + } + if row.masking == 2 { + name += ".Z" + } + mask := "" + if row.masking != 0 { + mask = "K1," + } + return name + " " + first + "," + bank + "1," + mask + destination, destination +} + +// Independent widened integer arithmetic, not a production helper/intrinsic. +// Signed word pairs wrap modulo 2^32; signed-byte * unsigned-byte pairs +// saturate to signed16. Every pair stays in its source byte position. +const x86MADDIndependentCReference = ` +static const int32_t edges[] = { + 0, 1, -1, 2, -2, 127, -128, 255, 256, -257, 32767, -32768, 16384, -16384 +}; + +static void fill(uint8_t *out, unsigned sample, unsigned salt, int bytes) { + for (int i = 0; i < 64; i += bytes) { + int32_t v = edges[(sample + salt + i * 3) % 14]; + if (sample >= 14) v = (int32_t)(sample * 7919 + i * 104729 + salt * 65537); + out[i] = (uint8_t)v; + if (bytes == 2) out[i + 1] = (uint8_t)(v >> 8); + } +} + +static int64_t signed_word(const uint8_t *p) { + uint32_t bits = (uint32_t)p[0] | (uint32_t)p[1] << 8; + return bits < 32768 ? (int64_t)bits : (int64_t)bits - 65536; +} + +static void independent_madd(uint8_t *out, const uint8_t *first, + const uint8_t *second, int width, int bytes) { + int group = bytes * 2; + for (int i = 0; i < width; i += group) { + int64_t sum; + if (bytes == 2) sum = signed_word(first + i) * signed_word(second + i) + + signed_word(first + i + 2) * signed_word(second + i + 2); + else { + int32_t a = first[i] < 128 ? first[i] : (int32_t)first[i] - 256; + int32_t b = first[i + 1] < 128 ? first[i + 1] : (int32_t)first[i + 1] - 256; + sum = (int64_t)a * second[i] + (int64_t)b * second[i + 1]; + if (sum < -32768) sum = -32768; + if (sum > 32767) sum = 32767; + } + uint32_t bits = (uint32_t)sum; + for (int j = 0; j < group; ++j) out[i + j] = (uint8_t)(bits >> (j * 8)); + } +} +` + +func TestCrossLinuxRuntimeMatrixX86MADDTypedIndependentWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + rows := x86MADDOracleRows() + var assembly, declarations, table, main strings.Builder + declarations.WriteString("package narroworacle\n") + main.WriteString("#include \n#include \n#include \n") + for index, row := range rows { + name := fmt.Sprintf("madd%d", index) + fmt.Fprintf(&declarations, "func %s(out *[80]byte,a,b,old *[64]byte,mask,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-52\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVQ mask+32(FP),R9\n", name) + // Set the independent state before entering MMX. Its physical x87 + // region must remain uninterrupted through the real EMMS instruction. + assembly.WriteString("LDMXCSR csr+48(FP)\nPUSHQ flags+40(FP)\nPOPFQ\n") + if row.mmx { + assembly.WriteString("MOVQ (CX),M0\nMOVQ (DX),M1\n") + } else { + assembly.WriteString("VMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nKMOVQ R9,K1\n") + } + instruction, destination := row.instruction() + fmt.Fprintf(&assembly, "%s\n", instruction) + if row.mmx { + fmt.Fprintf(&assembly, "MOVQ %s,(AX)\nEMMS\n", destination) + } else { + fmt.Fprintf(&assembly, "VMOVUPS Z%s,(AX)\n", destination[1:]) + } + assembly.WriteString("PUSHFQ\nPOPQ R10\nMOVQ R10,64(AX)\nSTMXCSR 72(AX)\nRET\n") + fmt.Fprintf(&main, "extern void %s(void*,const void*,const void*,const void*,uint64_t,uint64_t,uint32_t);\n", name) + fmt.Fprintf(&table, "{%s,%d,%d,%d,%d,%d,%d},\n", name, row.width, row.inputBytes, row.source, row.alias, row.masking, x86FloatAddSubBoolInt(row.legacy)) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + main.WriteString("static const struct {void (*run)(void*,const void*,const void*,const void*,uint64_t,uint64_t,uint32_t);int width,bytes,source,alias,masking,legacy;} cases[]={\n" + table.String() + "};\n") + main.WriteString(x86MADDIndependentCReference) + main.WriteString(` +int main(void) { + unsigned saved; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + const uint32_t profiles[] = {0x1f80, 0x3fc1, 0x5f82, 0x9fe5}; + unsigned long long observations = 0; + for (unsigned row = 0; row < sizeof(cases) / sizeof(cases[0]); ++row) { + int lanes = cases[row].width / (cases[row].bytes * 2); + for (unsigned sample = 0; sample < 32; ++sample) + for (int select = 0; select < lanes + 4; ++select) + for (unsigned profile = 0; profile < 4; ++profile) + for (unsigned seed = 0; seed < 2; ++seed) { + _Alignas(64) uint8_t a[64], b[64], old[64], beforeA[64], beforeB[64], beforeOld[64]; + uint8_t out[82], want[82], packed[64]; + fill(a, sample, 0, cases[row].bytes); + fill(b, sample, 7, cases[row].bytes); + for (int i = 0; i < 64; ++i) old[i] = (uint8_t)(0x91 + i * 11); + if (cases[row].bytes == 2 && sample == 0) + for (int i = 0; i < 64; i += 2) { + a[i] = b[i] = 0; + a[i + 1] = b[i + 1] = 0x80; + } + if (cases[row].bytes == 1 && sample < 2) { + memset(a, sample ? 0x80 : 0x7f, 64); + memset(b, 0xff, 64); + } + memcpy(beforeA, a, 64); + memcpy(beforeB, b, 64); + memcpy(beforeOld, old, 64); + memset(out, 0xc3, sizeof(out)); + memcpy(want, out, sizeof(want)); + + uint64_t mask = select == 0 ? 0 : select == 1 ? UINT64_MAX : + select == 2 ? UINT64_C(0xaaaaaaaaaaaaaaaa) : + select == 3 ? UINT64_MAX << lanes : UINT64_C(1) << (select - 4); + uint64_t flags = 2 | (seed ? 0x8d5 : 0); + cases[row].run(out + 1, a, b, old, mask, flags, profiles[profile]); + const uint8_t *first = cases[row].legacy && cases[row].alias ? b : a; + independent_madd(packed, first, b, cases[row].width, cases[row].bytes); + const uint8_t *previous = cases[row].legacy ? b : + cases[row].alias == 0 ? a : cases[row].alias == 1 ? b : old; + for (int i = 0; i < 64; ++i) { + uint8_t value = cases[row].width == 8 ? 0xc3 : cases[row].legacy ? previous[i] : 0; + if (i < cases[row].width) { + value = packed[i]; + if (cases[row].masking && !(mask >> (i / (cases[row].bytes * 2)) & 1)) + value = cases[row].masking == 2 ? 0 : previous[i]; + } + want[i + 1] = value; + } + + uint64_t actualFlags; + uint32_t actualCSR; + memcpy(&actualFlags, out + 65, 8); + memcpy(&actualCSR, out + 73, 4); + memcpy(want + 65, out + 65, 12); + ++observations; + if ((actualFlags & 0x8d5) != (flags & 0x8d5) || actualCSR != profiles[profile] || + memcmp(out, want, sizeof(out)) || memcmp(a, beforeA, 64) || + memcmp(b, beforeB, 64) || memcmp(old, beforeOld, 64)) { + fprintf(stderr, "MADD row%u sample%u width%d bytes%d source%d alias%d mask%llx flags%llx csr%x\n", + row, sample, cases[row].width, cases[row].bytes, cases[row].source, + cases[row].alias, (unsigned long long)mask, (unsigned long long)actualFlags, actualCSR); + for (unsigned i = 0; i < sizeof(out); ++i) + if (out[i] != want[i]) fprintf(stderr, "byte%u got%u want%u\n", i, out[i], want[i]); + return 1; + } + } + } + __asm__ volatile("ldmxcsr %0" :: "m"(saved)); + printf("MADD kernels=%zu observations=%llu aliases64 sixFlags fullCSR canaries\n", + sizeof(cases) / sizeof(cases[0]), observations); + return 0; +} +`) + observations := 0 + for _, row := range rows { + observations += (row.width/(row.inputBytes*2) + 4) * 32 * 4 * 2 + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + ir := ir + if level == "-O2" { + ir = x86MulDivRoundOptimize(t, ir) + } + output := execution.compileAndRunLLVM(t, "madd-independent", ir, main.String(), level) + want := fmt.Sprintf("MADD kernels=%d observations=%d aliases64 sixFlags fullCSR canaries\n", len(rows), observations) + if string(output) != want { + t.Fatalf("incomplete MADD denominator: %q want %q", output, want) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + t.Log("software VEX/EVEX is not original native AVX/AVX512 execution") +} + +func TestCrossLinuxRuntimeMatrixX86MADDTypedOriginalLegacy(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, table strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, row := range x86MADDOracleRows() { + if !row.legacy { + continue + } + name := fmt.Sprintf("original%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[32]byte,a,b *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-36\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nSTMXCSR saved-4(SP)\nLDMXCSR csr+32(FP)\n", name) + move, bank := "MOVOU", "X" + if row.mmx { + move, bank = "MOVQ", "M" + } + fmt.Fprintf(&assembly, "%s (CX),%s0\n%s (DX),%s1\nPUSHQ flags+24(FP)\nPOPFQ\n", move, bank, move, bank) + instruction, destination := row.instruction() + fmt.Fprintf(&assembly, "%s\n%s %s,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,16(AX)\nSTMXCSR 24(AX)\n", instruction, move, destination) + if row.mmx { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("LDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&table, "{%s,%d,%d,%d,-1},\n", name, row.width, row.inputBytes, row.alias) + } + // These accepted GP spellings encode MMX ModRM source banks. Initialize + // each physical M register independently; do not interpret GP contents as + // numeric MMX inputs or grant this source spelling LLVM admission. + for index, register := range []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + name := fmt.Sprintf("original%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[32]byte,a,b *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-36\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nSTMXCSR saved-4(SP)\nLDMXCSR csr+32(FP)\n", name) + for bank := 0; bank < 8; bank++ { + fmt.Fprintf(&assembly, "MOVQ %d(CX),M%d\n", bank*8, bank) + } + assembly.WriteString("MOVQ (DX),M1\nPUSHQ flags+24(FP)\nPOPFQ\n") + fmt.Fprintf(&assembly, "PMADDWL %s,M1\nMOVQ M1,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,16(AX)\nSTMXCSR 24(AX)\nEMMS\nLDMXCSR saved-4(SP)\nRET\n", register) + fmt.Fprintf(&table, "{%s,8,2,0,%d},\n", name, index&7) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + tests := `package oracle +import ( + "testing" + "encoding/binary" + "runtime" + "unsafe" +) + +var cases = []struct { + run func(*[32]byte, *[64]byte, *[64]byte, uint64, uint32) + width, bytes, alias, bank int +}{ +` + table.String() + `} + +func aligned(storage *[128]byte) *[64]byte { + offset := int((-uintptr(unsafe.Pointer(&storage[0]))) & 63) + return (*[64]byte)(unsafe.Pointer(&storage[offset])) +} + +func TestOriginal(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + edges := [...]int32{0, 1, -1, 2, -2, 127, -128, 255, 256, -257, 32767, -32768, 16384, -16384} + profiles := []uint32{0x1f80, 0x3fc1, 0x5f82, 0x9fe5} + var aa, bb [128]byte + a, b := aligned(&aa), aligned(&bb) + observations := 0 + + for row, entry := range cases { + for sample := 0; sample < 32; sample++ { + for _, csr := range profiles { + for seed := 0; seed < 2; seed++ { + for i := range aa { + aa[i] = 0xa7 + bb[i] = 0xb9 + } + for i := 0; i < 64; i += entry.bytes { + x, y := edges[(sample+i*3)%14], edges[(sample+7+i*3)%14] + if sample >= 14 { + x = int32(sample*7919+i*104729) + y = int32(sample*7919+i*104729+7*65537) + } + a[i], b[i] = byte(x), byte(y) + if entry.bytes == 2 { + a[i+1], b[i+1] = byte(uint32(x)>>8), byte(uint32(y)>>8) + } + } + if entry.bytes == 2 && sample == 0 { + for i := 0; i < 64; i += 2 { + a[i] = 0 + b[i] = 0 + a[i+1] = 0x80 + b[i+1] = 0x80 + } + } + if entry.bytes == 1 && sample < 2 { + for i := range a { + a[i] = 0x7f + if sample == 1 { + a[i] = 0x80 + } + b[i] = 0xff + } + } + beforeA, beforeB := aa, bb + var out, want [34]byte + for i := range out { + out[i] = 0xc3 + want[i] = 0xc3 + } + + flags := uint64(2) + if seed != 0 { + flags |= 0x8d5 + } + entry.run((*[32]byte)(unsafe.Pointer(&out[1])), a, b, flags, csr) + first := a[:] + if entry.alias == 1 { + first = b[:] + } + if entry.bank >= 0 { + first = a[entry.bank*8:] + if entry.bank == 1 { + first = b[:] + } + } + for i := 0; i < entry.width; i += entry.bytes * 2 { + var sum int64 + if entry.bytes == 2 { + sum = int64(int16(binary.LittleEndian.Uint16(first[i:]))) * int64(int16(binary.LittleEndian.Uint16(b[i:]))) + + int64(int16(binary.LittleEndian.Uint16(first[i+2:]))) * int64(int16(binary.LittleEndian.Uint16(b[i+2:]))) + } else { + sum = int64(int8(first[i]))*int64(b[i]) + int64(int8(first[i+1]))*int64(b[i+1]) + if sum < -32768 { + sum = -32768 + } + if sum > 32767 { + sum = 32767 + } + } + for j := 0; j < entry.bytes * 2; j++ { + want[i+j+1] = byte(uint32(sum) >> uint(j*8)) + } + } + + actualFlags := binary.LittleEndian.Uint64(out[17:25]) + actualCSR := binary.LittleEndian.Uint32(out[25:29]) + copy(want[17:29], out[17:29]) + observations++ + if out != want || aa != beforeA || bb != beforeB || actualFlags&0x8d5 != flags&0x8d5 || actualCSR != csr { + t.Fatalf("original MADD row%d sample%d width%d bytes%d flags%x CSR%x got%x want%x", + row, sample, entry.width, entry.bytes, actualFlags, actualCSR, out, want) + } + } + } + } + } + t.Logf("original MADD kernels=%d observations=%d sixFlags fullCSR canaries", len(cases), observations) +} +` + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, gotoolchain.HostExecutable("original")) + command := exec.Command("go", "test", "-c", "-p=2", "-vet=off", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH=amd64", "CGO_ENABLED=0", "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOMAXPROCS=2") + x86FloatCompareCaptureCommand(t, command) + output := execution.run(t, executable, "-test.v", "-test.run=^TestOriginal$", "-test.count=1") + want := fmt.Sprintf("original MADD kernels=%d observations=%d sixFlags fullCSR canaries", count, count*32*4*2) + if !strings.Contains(string(output), want) || !strings.Contains(string(output), "--- PASS: TestOriginal") || strings.Contains(string(output), "SKIP") { + t.Fatalf("incomplete original-Go denominator: %q", output) + } + t.Log(strings.TrimSpace(string(output))) + t.Log("original SSE/MMX including all16 GP-to-MMX encoded bank spellings only; this gate does not execute VEX/EVEX") +} diff --git a/amd64_maddword_context_test.go b/amd64_maddword_context_test.go new file mode 100644 index 000000000..9c639f082 --- /dev/null +++ b/amd64_maddword_context_test.go @@ -0,0 +1,23 @@ +package plan9asm + +import "testing" + +func TestX86WordMultiplyAddLegacyVirtualMemoryCannotProveAlignment(t *testing.T) { + for _, operand := range []string{"source+0(FP)", "(SP)", "(BP)"} { + for _, target := range x86RawReturnTargets { + source := "TEXT ·f(SB),4,$0-64\nPXOR X1,X1\nPMADDWL " + operand + ",X1\nRET\n" + declaration := "func f(source [64]byte)" + if operand != "source+0(FP)" { + source = x86OrdinaryBPNativeSource(target.arch, "f", "PMADDWL "+operand+",X1", operand, 0) + declaration = "func f(p *byte)" + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+operand+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + x86RequireOrdinaryBPNativeOutcome(t, target.arch, operand, ir, err) + }) + } + } + } +} diff --git a/amd64_maddword_fault_runtime_test.go b/amd64_maddword_fault_runtime_test.go new file mode 100644 index 000000000..2b4e0d3da --- /dev/null +++ b/amd64_maddword_fault_runtime_test.go @@ -0,0 +1,118 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddNativeMemoryStatus(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for i, row := range []struct { + instruction string + width int + }{ + {"PMADDWL (BX),X0", 16}, + {"VPMADDWD.Z (BX),X0,K1,X1", 16}, + {"VPMADDWD.Z (BX),Y0,K1,Y1", 32}, + {"VPMADDWD.Z (BX),Z0,K1,Z1", 64}, + {"PMADDUBSW (BX),X0", 16}, + {"VPMADDUBSW.Z (BX),X0,K1,X1", 16}, + {"VPMADDUBSW.Z (BX),Y0,K1,Y1", 32}, + {"VPMADDUBSW.Z (BX),Z0,K1,Z1", 64}, + } { + name := fmt.Sprintf("wordmaddfault%d", i) + // Deliberately discard the computed result. Sideeffect/volatile reads + // must remain after O2, even with the constant zero K mask. + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-8\nMOVQ input+0(FP),BX\nVPXORD Z0,Z0,Z0\nMOVL $0,CX\nKMOVQ CX,K1\n%s\nRET\n", name, row.instruction) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(const uint8_t*);\n", name) + reference := fmt.Sprintf("read%d", row.width) + if i == 0 { + reference = "originalPMADDWD" + } else if i == 4 { + reference = "originalPMADDUBSW" + } + fmt.Fprintf(&checks, "for(int offset=0;offset<16;++offset) if(compareStatus(%s,%s,memory+offset)) return 1;\n", reference, name) + fmt.Fprintf(&checks, "if(compareStatus(%s,%s,0) || compareStatus(%s,%s,boundary-%d+1) || compareStatus(%s,%s,boundary-%d)) return 1;\n", reference, name, reference, name, row.width, reference, name, row.width) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + ` +__attribute__((noinline)) static void originalPMADDWD(const uint8_t *p) { + typedef int32_t V __attribute__((vector_size(16))); + V carrier={0,0,0,0}; + // This is the original PMADDWD memory opcode, not MOVDQA plus PMADDWD. + __asm__ volatile("pmaddwd (%1),%0" : "+x"(carrier) : "r"(p) : "memory"); +} +__attribute__((noinline)) static void originalPMADDUBSW(const uint8_t *p) { + typedef int32_t V __attribute__((vector_size(16))); + V carrier={0,0,0,0}; + __asm__ volatile("pmaddubsw (%1),%0" : "+x"(carrier) : "r"(p) : "memory"); +} +__attribute__((noinline)) static void read16(const uint8_t *p) {for(int i=0;i<16;++i) (void)((const volatile uint8_t*)p)[i];} +__attribute__((noinline)) static void read32(const uint8_t *p) {for(int i=0;i<32;++i) (void)((const volatile uint8_t*)p)[i];} +__attribute__((noinline)) static void read64(const uint8_t *p) {for(int i=0;i<64;++i) (void)((const volatile uint8_t*)p)[i];} +typedef struct {int signal;int code;} Status; +static int observe(void (*program)(const uint8_t*),const uint8_t *input,Status *out) { + pid_t child=fork(); if(child<0) return 1; + if(child==0) { + struct rlimit limit={0,0}; if(setrlimit(RLIMIT_CORE,&limit)) _exit(3); + program(input); _exit(0); + } + int status=0; if(waitpid(child,&status,0)!=child) return 1; + if(WIFSIGNALED(status)) {*out=(Status){WTERMSIG(status),0};return 0;} + if(WIFEXITED(status)) {*out=(Status){0,WEXITSTATUS(status)};return 0;} + return 1; +} +static int compareStatus(void (*original)(const uint8_t*),void (*translated)(const uint8_t*),const uint8_t *input) { + Status a,b; + if(observe(original,input,&a) || observe(translated,input,&b)) return 1; + if(a.signal!=b.signal || a.code!=b.code) { + fprintf(stderr,"input=%p mod16=%lu original=(%d,%d) translated=(%d,%d)\n",input,(unsigned long)((uintptr_t)input%16),a.signal,a.code,b.signal,b.code); return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); if(page<=0 || page%64) return 1; + uint8_t *memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED) return 1; + memset(memory,0x83,(size_t)page); + if(mprotect(memory,(size_t)page,PROT_READ) || mprotect(memory+page,(size_t)page,PROT_NONE)) return 1; + uint8_t *boundary=memory+page; +` + checks.String() + ` + if(munmap(memory,(size_t)page*2)) return 1; + return 0; +} +` + for _, opt := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", opt), func(t *testing.T) { + text := ir + if opt { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "word-madd-read-status", triple, text, main, prefix) + }) + } + t.Log("legacy exact opcode original/translated typed status at each mod16 and real guard-page boundaries; EVEX zero-mask full-read contract uses independent volatile reader, not unexecuted AVX512 hardware as oracle") +} diff --git a/amd64_maddword_forms.go b/amd64_maddword_forms.go new file mode 100644 index 000000000..1e8b9ba00 --- /dev/null +++ b/amd64_maddword_forms.go @@ -0,0 +1,45 @@ +package plan9asm + +import "strings" + +// Go calls Intel's legacy PMADDWD instruction PMADDWL. The raw decoder keeps +// that canonical spelling. ymm supplies MMX/m64 and X/m128; _yvandnpd supplies +// all VEX/EVEX widths. Each VPMADDWD row permits zeroing, never broadcasting. +var x86WordMultiplyAddSpecs = map[Op]x86PackedBinarySpec{ + "PMADDWL": {encoding: x86PackedLogicalLegacyInteger}, + "VPMADDWD": {encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, +} + +type x86PackedMultiplyAddSpec struct { + x86PackedBinarySpec + inputBits, outputBits int + nativeOpcode string +} + +var x86PackedMultiplyAddSpecs = map[Op]x86PackedMultiplyAddSpec{ + "PMADDWL": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16, 32, "pmaddwd"}, + "PMADDUBSW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 8, 16, "pmaddubsw"}, + "VPMADDWD": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, 32, ""}, + "VPMADDUBSW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, + 8, 16, ""}, +} + +type x86PackedMultiplyAddForm = x86PackedPhysicalForm + +// This describes Go's actual ymm/yxm_q4/_yvandnpd operand rows only. WIG +// and the decoder's private physical marker do not grant a native address, +// initialized input, caller frame, or a new MMX transport. +func x86PackedMultiplyAddProperties(op Op, ins Instr, arch string) (x86PackedMultiplyAddForm, bool, error) { + return x86PackedPhysicalProperties(op, ins, arch, x86PackedMultiplyAdd) +} + +func x86WordMultiplyAddProperties(ins Instr, goarch string) (x86PackedBinaryForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + _, recognized := x86WordMultiplyAddSpecs[Op(base)] + if !recognized { + return x86PackedBinaryForm{}, false, nil + } + // Keep this word-only consumer boundary for the existing frame proof. + form, known, err := x86PackedMultiplyAddProperties(ins.Op, ins, goarch) + return form.x86PackedBinaryForm, known, err +} diff --git a/amd64_maddword_forms_test.go b/amd64_maddword_forms_test.go new file mode 100644 index 000000000..281dda3cf --- /dev/null +++ b/amd64_maddword_forms_test.go @@ -0,0 +1,283 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "testing" +) + +func x86WordMaddFormats(arch string) []string { + rows := []string{"PMADDWL X0,X7", "PMADDWL (AX),X7"} + if arch == "amd64" { + rows = append(rows, "PMADDWL M0,M7", "PMADDWL (AX),M7", "PMADDWL source+0(FP),M7", "PMADDWL X15,X0", "PMADDWL X7,X7", "PMADDWL (FS),X1", "PMADDWL (GS),X1", "PMADDWL data(SB),X1") + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "16(AX)(BX*2)", "source+0(FP)"} { + rows = append(rows, fmt.Sprintf("VPMADDWD %s,%s1,%s7", source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("VPMADDWD %s,%s31,K1,%s30", source, bank, bank), fmt.Sprintf("VPMADDWD.Z %s,%s31,K7,%s30", source, bank, bank)) + } + } + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("VPMADDWD %s31,%s30,%s29", bank, bank, bank)) + } + } + return rows +} + +func x86WordMaddFPType(t *testing.T) FuncSig { + t.Helper() + pkg := mustGoPackage(t, "oracle", "package oracle\nfunc f(source [64]byte)\n") + sig, err := DeriveGoFuncABI0Signature("f", pkg.Types.Scope().Lookup("f").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + if len(sig.Frame.Params) != 64 { + t.Fatalf("actual Go 64-byte field layout changed: %+v", sig.Frame) + } + return sig +} + +func TestX86WordMultiplyAddActualGoCompleteAttributes(t *testing.T) { + if len(x86WordMultiplyAddSpecs) != 2 { + t.Fatal("word multiply-add family must contain exactly the two Go spellings") + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(legacy), "{APMADDWL, ymm, Py1, opBytes{0xf5, Pe, 0xf5}}") { + t.Fatal("actual Go legacy ymm registration changed") + } + start := strings.Index(string(avx), "{as: AVPMADDWD,") + if start < 0 { + t.Fatal("actual Go omits VPMADDWD") + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(row, "ytab: _yvandnpd") || strings.Contains(row, "Bcst") || strings.Contains(row, "Sae") { + t.Fatalf("actual Go operand attributes changed: %s", row) + } + for _, width := range []string{"vex128", "vex256"} { + if !strings.Contains(row, "avxEscape | "+width+" | vex66 | vex0F | vexW0, 0xF5") { + t.Fatalf("actual Go VEX attribute missing: %s", width) + } + } + for i, width := range []string{"evex128", "evex256", "evex512"} { + if !strings.Contains(row, fmt.Sprintf("avxEscape | %s | evex66 | evex0F | evexW0, evexN%d | evexZeroingEnabled, 0xF5", width, 16<= 0 { + sig.Frame.Params = append(sig.Frame.Params[:missing], sig.Frame.Params[missing+1:]...) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := out.String() + _, _, err := ctx.lowerPackedWordMultiplyAdd(ins.Op, ins) + if missing < 0 { + if err != nil { + t.Fatal(err) + } + } else if err == nil || out.String() != before { + t.Fatalf("%s missing FP byte%d emitted or invented backing: %v", row, missing, err) + } + } + } +} diff --git a/amd64_maddword_go_oracle_test.go b/amd64_maddword_go_oracle_test.go new file mode 100644 index 000000000..0704e0bc3 --- /dev/null +++ b/amd64_maddword_go_oracle_test.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddOriginalGoBits(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle +import "testing" +func signed(p []byte) int64 { v:=int64(uint16(p[0])|uint16(p[1])<<8); if v>=32768 {v-=65536};return v } +func TestOriginalPairWrap(t *testing.T) { + var patterns [8][16]byte + state:=uint32(0x12345678) + for p:=range patterns {for i:=range patterns[p] {state=state*1664525+1013904223;patterns[p][i]=byte(state>>24)}} + for i:=0;i<16;i+=2 {patterns[0][i]=0;patterns[0][i+1]=0x80;patterns[1][i]=0;patterns[1][i+1]=0x80} + for i:=range patterns[2] {patterns[2][i]=0xff} +`) + count := 0 + for _, bank := range []string{"X", "M"} { + forms := []string{"register"} + if bank == "M" { + forms = append(forms, "memory") + } + for _, form := range forms { + name := fmt.Sprintf("wordmadd%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(a,b,out *byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\nSTMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\n", name) + move, width := "MOVOU", 16 + if bank == "M" { + move, width = "MOVQ", 8 + } + fmt.Fprintf(&assembly, "%s (BX),%s0\n", move, bank) + first := "(AX)" + if form == "register" { + first = bank + "1" + fmt.Fprintf(&assembly, "%s (AX),%s1\n", move, bank) + } + fmt.Fprintf(&assembly, "CMPQ AX,AX\nSTC\nPMADDWL %s,%s0\n%s %s0,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\nSTMXCSR 18(CX)\n", first, bank, move, bank) + if bank == "M" { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("LDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&tests, ` + for _,a:=range patterns {for _,b:=range patterns {for mode:=0;mode<4;mode++ { + var out [24]byte;for i:=range out {out[i]=0xc3} + csr:=uint32(0x1f80|mode<<13|mode) + %[1]s(&a[0],&b[0],&out[1],csr) + if out[0]!=0xc3 || out[23]!=0xc3 || out[17]!=1 || out[18]!=0 {t.Fatalf("%[1]s flags/canary: %%x",out)} + gotCSR:=uint32(out[19])|uint32(out[20])<<8|uint32(out[21])<<16|uint32(out[22])<<24 + if gotCSR!=csr {t.Fatalf("%[1]s CSR %%x != %%x",gotCSR,csr)} + for i:=0;i<%[2]d;i+=4 { + want:=uint32(signed(a[i:])*signed(b[i:])+signed(a[i+2:])*signed(b[i+2:])) + for j:=0;j<4;j++ {if out[i+j+1]!=byte(want>>uint(8*j)) {t.Fatalf("%[1]s pair %%d got %%x want %%x",i,out,want)}} + } + for i:=%[2]d;i<16;i++ {if out[i+1]!=0xc3 {t.Fatalf("%[1]s touched upper MMX bytes")}} + }}} +`, name, width) + } + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go signed-pair oracle: %v\n%s", err, output) + } + t.Logf("%d actual original SSE/MMX Go functions × 256 independent signed-pair/CSR/flags/canary cases", count) +} diff --git a/amd64_maddword_runtime_test.go b/amd64_maddword_runtime_test.go new file mode 100644 index 000000000..155050bf2 --- /dev/null +++ b/amd64_maddword_runtime_test.go @@ -0,0 +1,230 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" + "testing" +) + +const x86WordMaddIndependentRuntimeOracle = ` +static uint8_t patterns[8][64]; +static const uint64_t masks[]={0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; +static int32_t word(const uint8_t *p) { + uint32_t bits=(uint32_t)p[0]|(uint32_t)p[1]<<8; + return bits<32768 ? (int32_t)bits : (int32_t)bits-65536; +} +__attribute__((noinline)) static int checkMadd( + void (*program)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t), + const char *name,int width,int mode,int legacy,int destination,int shift) { + for(int round=0;round<8;++round) for(int m=0;m<4;++m) { + _Alignas(64) uint8_t aStorage[96],bStorage[96],output[68]; + uint8_t *a=aStorage+shift,*b=bStorage+3; + memset(aStorage,0xa7,sizeof(aStorage)); memset(bStorage,0xb9,sizeof(bStorage)); + memset(output,0xc3,sizeof(output)); + memcpy(a,patterns[round],64); memcpy(b,patterns[(round+1)%8],64); + uint64_t mask=masks[m]; + uint32_t saved,before=0x1f80|((round%4)<<13)|round,after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + program(output+1,a,b,mask); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + if(after!=before || output[0]!=0xc3 || output[67]!=0xc3 || output[65]!=1 || output[66]!=0) { + fprintf(stderr,"%s round=%d CSR/flags/canary failure\n",name,round); return 1; + } + for(int i=0;i<64;i+=4) { + // Widen independently to signed int64 before summing. Conversion to + // uint32 implements the specified pair wrap, including 0x80000000. + uint32_t result=(uint32_t)((int64_t)word(a+i)*word(b+i)+(int64_t)word(a+i+2)*word(b+i+2)); + const uint8_t *old=destination==1 ? a : b; + int active=mode==0 || ((mask>>(i/4))&1); + for(int j=0;j<4;++j) { + uint8_t want=i>(j*8)) : (mode==2 ? 0 : old[i+j])) : (legacy ? b[i+j] : 0); + if(output[i+j+1]!=want) { + fprintf(stderr,"%s round=%d mask=%llx byte=%d got=%x want=%x\n",name,round,(unsigned long long)mask,i+j,output[i+j+1],want); return 1; + } + } + } + for(int i=0;i<96;++i) { + if((i=shift+64) && aStorage[i]!=0xa7) return 1; + if((i<3 || i>=67) && bStorage[i]!=0xb9) return 1; + } + if(memcmp(a,patterns[round],64) || memcmp(b,patterns[(round+1)%8],64)) return 1; + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddCompleteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, legacy := range []bool{true, false} { + widths := []int{16} + if !legacy { + widths = []int{16, 32, 64} + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, memory := range []bool{false, true} { + for mode := 0; mode < 3; mode++ { + if legacy && mode != 0 { + continue + } + for destination := 0; destination < 3; destination++ { + if legacy && destination != 0 { + continue + } + name := fmt.Sprintf("wordmadd%d", count) + count++ + first := bank + "1" + if memory { + first = "(BX)" + } + row := "PMADDWL " + first + ",X0" + if !legacy { + opcode := "VPMADDWD" + if mode == 2 { + opcode += ".Z" + } + row = opcode + " " + first + "," + bank + "0" + if mode != 0 { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + } + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", name, row, destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + shift := 1 + if legacy && memory { + shift = 16 + } + fmt.Fprintf(&checks, "if(checkMadd(%s,\"%s\",%d,%d,%d,%d,%d)) return 1;\n", name, name, width, mode, x86FloatAddSubBoolInt(legacy), destination, shift) + } + } + } + } + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := "#include \n#include \n#include \n" + declarations.String() + x86WordMaddIndependentRuntimeOracle + ` +int main(void) { + uint32_t state=0x12345678; + for(int p=0;p<8;++p) for(int i=0;i<64;++i) {state=state*1664525+1013904223; patterns[p][i]=(uint8_t)(state>>24);} + // Explicit signed extrema and the unique overflowing pair, not merely random data. + memset(patterns[0],0,64); memset(patterns[1],0,64); + for(int i=1;i<64;i+=2) {patterns[0][i]=0x80;patterns[1][i]=0x80;} + memset(patterns[2],0xff,64); + for(int i=0;i<64;i+=2) {patterns[3][i]=0xff;patterns[3][i+1]=0x7f;} +` + checks.String() + "return 0;\n}\n" + for _, opt := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", opt), func(t *testing.T) { + text := ir + if opt { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "word-madd", triple, text, main, prefix) + }) + } + t.Logf("%d opcode/width/memory/mask/alias programs × 32 signed-pair/MXCSR/flags/canary cases at O0 and O2; translated wide-vector execution is not native AVX512 evidence", count) +} + +func x86WordMaddLogicalFPAssembly(name string, width, mode int) string { + bank := map[int]string{8: "M", 16: "X", 32: "Y", 64: "Z"}[width] + row := "PMADDWL source+0(FP),M1" + load, store := "MOVQ (AX),M1", "MOVQ M1,(AX)\nEMMS" + if width != 8 { + row = "VPMADDWD source+0(FP)," + bank + "1" + if mode == 2 { + row = strings.Replace(row, "VPMADDWD ", "VPMADDWD.Z ", 1) + } + if mode != 0 { + row += ",K1" + } + row += "," + bank + "1" + load = "VMOVUPS (AX),Z1" + store = "VMOVUPS " + bank + "1,(AX)" + } + return fmt.Sprintf("TEXT %s(SB),4,$0-80\nMOVQ out+64(FP),AX\nMOVQ mask+72(FP),DX\nKMOVQ DX,K1\n%s\n%s\n%s\nRET\n", name, load, row, store) +} + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddLogicalFPFields(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var definitions, wrappers, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for _, width := range []int{8, 16, 32, 64} { + for mode := 0; mode < 3; mode++ { + if width == 8 && mode != 0 { + continue + } + name := fmt.Sprintf("wordmaddfp%d", index) + index++ + declaration := fmt.Sprintf("func %s(source [64]byte,out *byte,mask uint64)\n", name) + pkg := mustGoPackage(t, "oracle", "package oracle\n"+declaration) + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + definitions.WriteString(x86WordMaddLogicalFPAssembly(name, width, mode)) + fmt.Fprintf(&wrappers, "define void @check_%s(ptr %%input,ptr %%output,i64 %%mask){\n", name) + packed := "poison" + for i := 0; i < 64; i++ { + fmt.Fprintf(&wrappers, " %%address%d=getelementptr i8,ptr %%input,i64 %d\n %%byte%d=load i8,ptr %%address%d,align 1\n %%pack%d=insertvalue [64 x i8] %s,i8 %%byte%d,%d\n", i, i, i, i, i, packed, i, i) + packed = fmt.Sprintf("%%pack%d", i) + } + fmt.Fprintf(&wrappers, " call void @%s([64 x i8] %s,ptr %%output,i64 %%mask)\n ret void\n}\n", name, packed) + fmt.Fprintf(&declarations, "extern void check_%s(const uint8_t*,uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, ` + for(int round=0;round<16;++round) for(unsigned m=0;m<4;++m) { + uint8_t input[66],output[66];memset(input,0xa7,66);memset(output,0xc3,66); + for(int i=0;i<64;++i) input[i+1]=(uint8_t)(i*17+round*31); + for(int i=0;i<64;i+=2) {output[i+1]=1;output[i+2]=0;} + check_%[1]s(input+1,output+1,masks[m]); + for(int i=0;i<64;i+=4) { + int32_t pair=word(input+i+1)+word(input+i+3); + int active=%[2]d==0 || ((masks[m]>>(i/4))&1); + uint32_t want=i<%[3]d ? (active ? (uint32_t)pair : (%[2]d==2 ? 0 : UINT32_C(0x00010001))) : UINT32_C(0x00010001); + for(int j=0;j<4;++j) if(output[i+j+1]!=(uint8_t)(want>>(8*j))) return 1; + } + if(input[0]!=0xa7 || input[65]!=0xa7 || output[0]!=0xc3 || output[65]!=0xc3) return 1; + for(int i=0;i<64;++i) if(input[i+1]!=(uint8_t)(i*17+round*31)) return 1; + } +`, name, mode, width) + } + } + requireX86GoAssemblerResult(t, "amd64", definitions.String(), true) + file, err := Parse(ArchAMD64, definitions.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := "#include \n#include \n#include \n" + declarations.String() + x86WordMaddIndependentRuntimeOracle + "int main(void){\n" + checks.String() + "return 0;}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + wrappers.String() + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "word-madd-logical-fp", triple, text, main, prefix) + }) + } + t.Logf("%d logical FP programs × 64 independent signed-pair cases; signatures derived from actual Go [64]byte declarations; explicit LLVM array carrier is not native stack/alignment or MMX tag proof", index) +} diff --git a/amd64_mask_blend_forms.go b/amd64_mask_blend_forms.go new file mode 100644 index 000000000..151f99c22 --- /dev/null +++ b/amd64_mask_blend_forms.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86MaskBlendForm struct { + x86PackedBinaryForm + spec amd64MaskBlendSpec +} + +func x86MaskBlendProperties(op Op, ins Instr, arch string) (x86MaskBlendForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64MaskBlendSpecs[Op(base)] + result := x86MaskBlendForm{spec: spec} + if !known { + return result, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return result, true, fmt.Errorf("mask blend dispatch/source opcode mismatch") + } + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, zeroing: true} + if spec.broadcast { + grammar.broadcastBytes = spec.laneBits / 8 + } + local := ins + local.Op = op + var err error + // This shared validator describes only Go's operand and address grammar; + // numerical bytes and native storage remain separate consumer obligations. + result.x86PackedBinaryForm, err = x86ConflictFunnelOperandGrammar(local, arch, grammar, false) + return result, true, err +} + +// The K operand selects between two sources; it is not an old-destination +// merge. No mask selects only the first Plan 9 source, and .Z drops the second. +func x86TypedMaskBlendByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86MaskBlendProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width, firstWidth := int64(form.byteWidth), int64(form.byteWidth) + if form.memory >= 0 { + firstWidth = int64(form.memoryBytes) + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: firstWidth, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: width, write: true}, + }} + if form.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + if !form.zeroing { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.second, operand: ins.Args[form.second], width: width, read: true}) + } + } + return effect, true, nil +} diff --git a/amd64_mask_logical_grammar_test.go b/amd64_mask_logical_grammar_test.go new file mode 100644 index 000000000..826abd279 --- /dev/null +++ b/amd64_mask_logical_grammar_test.go @@ -0,0 +1,77 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86MaskLogicalIndependentEncoderRows(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, oracle := range x86MaskLogicalOracle() { + stem := oracle.op[:len(oracle.op)-1] + opcode := map[string]string{ + "KAND": "0x41", "KANDN": "0x42", "KOR": "0x45", "KXNOR": "0x46", + "KXOR": "0x47", "KNOT": "0x44", "KTEST": "0x99", "KORTEST": "0x98", + }[stem] + table, length, classes := "_ykaddb", "vex256", []string{"Yk", "Yk", "Yk"} + if len(strings.Split(oracle.operands, ",")) == 2 { + table, length, classes = "_yknotb", "vex128", []string{"Yk", "Yk"} + } + prefix, w := "", "vexW0" + switch oracle.op[len(oracle.op)-1] { + case 'B': + prefix = " | vex66" + case 'D': + prefix, w = " | vex66", "vexW1" + case 'Q': + w = "vexW1" + } + bytes := []string{fmt.Sprintf("avxEscape | %s%s | vex0F | %s", length, prefix, w), opcode} + found := 0 + for _, row := range rules { + if row.Opcode != oracle.op { + continue + } + if row.Ytab != table || row.Prefix != "Pavx" || row.RowIndex != 0 || row.EncodingOffset != 0 || + row.ZCase != "Zvex_rm_v_r" || row.ZOffset != 2 || + !reflect.DeepEqual(row.Classes, classes) || !reflect.DeepEqual(row.OpBytes, bytes) { + t.Fatalf("complete mask encoding row changed: %+v", row) + } + found++ + } + if found != 1 { + t.Fatal("missing or additional mask encoding row", oracle.op, found) + } + count += found + } + if count != 32 || len(amd64MaskLogicalSpecs) != 32 { + t.Fatal("complete mask family inventory changed", count, len(amd64MaskLogicalSpecs)) + } +} + +func TestX86MaskLogicalLowererDispatchIdentity(t *testing.T) { + for _, row := range x86MaskLogicalOracle() { + t.Run(row.op, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, row.op+" "+row.operands) + ctx, ir := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + ctx.goarch = "amd64" + if err := ctx.lowerMaskLogical(ins.Op, ins); err != nil { + t.Fatal("matching source did not reach the actual emitter", err) + } + ir.Reset() + ins.Op = "MOVQ" + if err := ctx.lowerMaskLogical(Op(row.op), ins); err == nil || ir.Len() != 0 { + t.Fatal("mismatched source reached mask lowering", err) + } + }) + } +} diff --git a/amd64_mask_move_dispatch_test.go b/amd64_mask_move_dispatch_test.go new file mode 100644 index 000000000..a21fff65a --- /dev/null +++ b/amd64_mask_move_dispatch_test.go @@ -0,0 +1,73 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestAMD64MaskMoveDispatchBindsCompleteFamily(t *testing.T) { + // The internal lowerer takes the dispatched opcode separately. Older + // helper callers intentionally leave Instr.Op unset; parsed instructions + // retain their own opcode. Both must use the same complete _ykmovb grammar. + forms := []struct { + name string + src Operand + dst Operand + }{ + {"mask_to_mask", Operand{Kind: OpReg, Reg: "K1"}, Operand{Kind: OpReg, Reg: "K2"}}, + {"mask_to_gp", Operand{Kind: OpReg, Reg: "K1"}, Operand{Kind: OpReg, Reg: AX}}, + {"gp_to_mask", Operand{Kind: OpReg, Reg: AX}, Operand{Kind: OpReg, Reg: "K2"}}, + {"mask_to_memory", Operand{Kind: OpReg, Reg: "K1"}, Operand{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}, + {"memory_to_mask", Operand{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, Operand{Kind: OpReg, Reg: "K2"}}, + } + for _, arch := range []string{"amd64", "386"} { + for _, op := range []Op{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + for _, form := range forms { + for _, instructionOp := range []Op{"", op, Op(strings.ToLower(string(op)))} { + name := fmt.Sprintf("%s/%s/%s/op=%q", arch, op, form.name, instructionOp) + t.Run(name, func(t *testing.T) { + ins := Instr{Op: instructionOp, Args: []Operand{form.src, form.dst}} + registered := ins + registered.Op = op + c, _ := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{registered}}, FuncSig{Ret: Void}, nil) + c.goarch = arch + if ok, terminated, err := c.lowerVec(op, ins); !ok || terminated || err != nil { + t.Fatalf("lowerVec = (%v, %v, %v)", ok, terminated, err) + } + if ins.Op != instructionOp { + t.Fatal("dispatch mutated its caller's instruction") + } + }) + } + } + } + } +} + +func TestAMD64MaskMoveDispatchRejectsContradictoryIdentity(t *testing.T) { + for _, tc := range []struct { + name string + dispatch Op + source Op + }{ + {"different_width", "KMOVB", "KMOVQ"}, + {"different_family", "KMOVB", "MOVQ"}, + {"unregistered_suffix", "KMOVB", "KMOVB.Z"}, + {"unknown_source", "KMOVB", "UNKNOWN"}, + {"unknown_dispatch", "UNKNOWN", "KMOVB"}, + {"both_unknown", "UNKNOWN", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + ins := Instr{Op: tc.source, Args: []Operand{{Kind: OpReg, Reg: "K1"}, {Kind: OpReg, Reg: "K2"}}} + c, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before := output.String() + if err := c.lowerMaskMove(tc.dispatch, ins); err == nil { + t.Fatal("accepted an unknown or contradictory mask-move opcode") + } + if output.String() != before { + t.Fatal("invalid opcode emitted IR before rejection") + } + }) + } +} diff --git a/amd64_mask_vector_forms.go b/amd64_mask_vector_forms.go new file mode 100644 index 000000000..6168ba7cb --- /dev/null +++ b/amd64_mask_vector_forms.go @@ -0,0 +1,108 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86MaskVectorSpec struct { + laneBits int + maskToVector bool +} + +// Go's complete _yvpbroadcastmb2q/_yvpmovb2m rows are EVEX register-only +// moves. K0 is data, not the absence of writemasking; no suffix is permitted. +var x86MaskVectorSpecs = map[Op]x86MaskVectorSpec{ + "VPMOVM2B": {8, true}, "VPMOVM2W": {16, true}, + "VPMOVM2D": {32, true}, "VPMOVM2Q": {64, true}, + "VPMOVB2M": {8, false}, "VPMOVW2M": {16, false}, + "VPMOVD2M": {32, false}, "VPMOVQ2M": {64, false}, +} + +type x86MaskVectorForm struct { + spec x86MaskVectorSpec + vector, mask int + byteWidth int +} + +// Syntax and physical register roles are shared by lowering and caller effects. +// The form supplies no mask value, initialized vector bytes or entry contract. +func x86MaskVectorProperties(ins Instr, arch string) (x86MaskVectorForm, bool, error) { + base, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86MaskVectorSpecs[Op(base)] + if !recognized { + return x86MaskVectorForm{}, false, nil + } + form := x86MaskVectorForm{spec: spec, vector: 0, mask: 1} + if spec.maskToVector { + form.vector, form.mask = 1, 0 + } + fail := func() (x86MaskVectorForm, bool, error) { + return form, true, fmt.Errorf("%s %s requires register-only K0-K7 and X/Y/Z operands without suffix: %q", arch, base, ins.Raw) + } + if arch == "" { + arch = "amd64" + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != 2 || !amd64IsKOperand(ins.Args[form.mask]) { + return fail() + } + vector := ins.Args[form.vector] + form.byteWidth = amd64VectorByteWidth(vector.Reg) + if !amd64GoPackedVectorMoveRegister(vector, form.byteWidth, arch) { + return fail() + } + return form, true, x86Mode32VectorRegisterContext(ins, arch) +} + +func x86CallerMaskVectorStep(ins *Instr, form x86MaskVectorForm, state *x86SourceStackState) bool { + mask := ins.Args[form.mask].Reg + vector := x86CallerVectorRoot(ins.Args[form.vector].Reg) + if form.spec.maskToVector { + if !state.masks[mask] { + return false + } + // EVEX writes the selected vector and clears every wider alias byte. + state.vectors[vector] = ^uint64(0) + return true + } + required := form.signByteInputs() + if state.vectors[vector]&required != required { + return false // An entry/partially initialized vector is not numeric data. + } + // Sign bits fill the low mask lanes and all remaining K bits become zero. + state.masks[mask] = true + return true +} + +// Only each lane's sign bit contributes to a vector-to-mask result. At byte +// granularity its containing byte is sufficient; unrelated bytes stay unknown. +func (form x86MaskVectorForm) signByteInputs() uint64 { + lane := form.spec.laneBits / 8 + var required uint64 + for offset := lane - 1; offset < form.byteWidth; offset += lane { + required |= uint64(1) << uint(offset) + } + return required +} + +func x86MaskVectorResultStep(ins Instr, state *x86ResultState, arch string) (bool, bool) { + form, known, err := x86MaskVectorProperties(ins, arch) + if !known || err != nil || !x86ResultNamedPredicateInstruction(ins) { + return false, known + } + mask, _ := amd64ParseKReg(ins.Args[form.mask].Reg) + vector, _ := amd64VectorRegisterIndex(ins.Args[form.vector].Reg, form.byteWidth) + if form.spec.maskToVector { + if !state.masks[mask] { + return false, true + } + state.writeVector(vector, form.byteWidth, x86CallerVectorByteMask(int64(form.byteWidth)), false) + return true, true // EVEX clears wider aliases; this is not writemasking. + } + required := form.signByteInputs() + if state.vectorBytes(vector)&required != required { + return false, true + } + state.masks[mask] = true // Every inactive high K bit is architecturally zero. + return true, true +} diff --git a/amd64_mask_vector_runtime_test.go b/amd64_mask_vector_runtime_test.go new file mode 100644 index 000000000..d1428f16a --- /dev/null +++ b/amd64_mask_vector_runtime_test.go @@ -0,0 +1,132 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Observe both directions independently: a round trip alone could let two +// inverse mistakes agree. The reference does not consult the production spec. +func x86MaskVectorRuntimeFixture(t *testing.T) (string, map[string]FuncSig, string) { + t.Helper() + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, row := range x86OfficialMaskVectorRows(t) { + for _, mask := range []string{"K0", "K7"} { + name := fmt.Sprintf("maskvector%d", count) + count++ + instruction := strings.ReplaceAll(row, "K1", mask) + toVector := strings.HasPrefix(instruction, "VPMOVM2") + fields := strings.Fields(instruction) + vector := strings.TrimPrefix(strings.TrimSuffix(fields[1], ","+mask), mask+",") + width := map[byte]int{'X': 16, 'Y': 32, 'Z': 64}[vector[0]] + lane := strings.TrimPrefix(fields[0], "VPMOV") + lane = strings.TrimPrefix(strings.TrimSuffix(lane, "2M"), "M2") + laneBytes := map[string]int{"B": 1, "W": 2, "D": 4, "Q": 8}[lane] + if width == 0 || laneBytes == 0 { + t.Fatal("unmodeled independent mask/vector runtime spelling", row) + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ in+8(FP),BX\nMOVQ mask+16(FP),DX\n", name) + fmt.Fprintf(&source, "KMOVQ DX,%s\nMOVQ $0x8123456789abcdef,DI\nKMOVQ DI,K2\nVMOVUPS (BX),Z1\nCMPQ AX,AX\nSTC\n", mask) + fmt.Fprintln(&source, instruction) + fmt.Fprintf(&source, "VMOVUPS Z1,(AX)\nKMOVQ %s,DX\nMOVQ DX,64(AX)\nKMOVQ K2,DX\nMOVQ DX,72(AX)\nSETCS 80(AX)\nSETNE 81(AX)\nRET\n", mask) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(checkMaskVector(%s,\"%s\",%d,%d,%d)) return 1;\n", name, name, width, laneBytes, x86FloatAddSubBoolInt(toVector)) + } + } + if count != 48 { + t.Fatalf("independent complete runtime program denominator: %d", count) + } + oracle := `#include +#include +#include +` + declarations.String() + ` +static int checkMaskVector(void (*program)(uint8_t*,const uint8_t*,uint64_t), + const char *name,unsigned width,unsigned lane,int toVector) { + const uint64_t masks[]={0,1,UINT64_MAX,UINT64_C(0x55aa55aa55aa55aa),UINT64_C(1)<<63}; + for(unsigned round=0;round<4;round++) for(unsigned m=0;m<5;m++) { + uint8_t input[66],original[66],output[84]; + for(unsigned i=0;i<66;i++) input[i]=(uint8_t)(i*19+round*83+129); + memcpy(original,input,sizeof(input)); + memset(output,0xc3,sizeof(output)); + uint32_t saved,before=0x1f80|(round<<13)|round,after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + program(output+1,input+1,masks[m]); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + uint64_t gotMask,otherMask,wantMask=masks[m]; + memcpy(&gotMask,output+65,8); + memcpy(&otherMask,output+73,8); + if(!toVector) { + wantMask=0; + for(unsigned element=0;element>7)<=width ? 0 : (masks[m]>>(i/lane)&1) ? 255 : 0; + if(output[1+i]!=want) { + fprintf(stderr,"%s byte=%u got=%x want=%x\n",name,i,output[1+i],want); + return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + return source.String(), sigs, oracle +} + +func runX86MaskVectorRuntimeFixture(t *testing.T, llc, clang, triple string, prefix []string) { + t.Helper() + source, sigs, oracle := x86MaskVectorRuntimeFixture(t) + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask-vector-complete-bytes", triple, ir, oracle, prefix) + t.Log("48 independent direction/lane/width/K0-K7 programs; 20 complete byte/alias/mask/flags/MXCSR/canary observations each") +} + +func TestX86MaskVectorCompleteByteRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the mandatory cross-runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang required") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + runX86MaskVectorRuntimeFixture(t, llc, clang, triple, prefix) +} + +func TestCrossLinuxRuntimeMatrixX86MaskVectorCompleteByteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + runX86MaskVectorRuntimeFixture(t, llc, clang, triple, prefix) +} diff --git a/amd64_md5_data_bp_source_go127_test.go b/amd64_md5_data_bp_source_go127_test.go new file mode 100644 index 000000000..6e1e0e363 --- /dev/null +++ b/amd64_md5_data_bp_source_go127_test.go @@ -0,0 +1,106 @@ +//go:build go1.27 + +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// Retain the exact SDK source that exposed the benchmark regression. Extract +// only its original language declarations for this test package: that is not +// an SDK selection, linked frame, caller or pointee-extent capability. +func amd64MD5OriginalDataBPSource(t *testing.T) (string, GoPackage) { + t.Helper() + read := func(name string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src", "crypto", "md5", name)) + if err != nil { + t.Fatal(err) + } + t.Logf("original MD5 SDK source %s bytes=%d SHA256=%x", name, len(data), sha256.Sum256(data)) + return data + } + + assembly := read("md5block_amd64.s") + if got := fmt.Sprintf("%x", sha256.Sum256(assembly)); got != "36c7e14eaf58fd6bfc63a70c651dd36c6ab99421e1c455a3d203b15fd4ca4d81" { + t.Fatal("the complete original MD5 source changed; re-audit rather than alter its BP/frame usage", got) + } + declarations := map[string]string{} + for _, name := range []string{"md5.go", "md5block_decl.go"} { + data := read(name) + positions := token.NewFileSet() + parsed, err := parser.ParseFile(positions, name, data, 0) + if err != nil { + t.Fatal(err) + } + for _, declaration := range parsed.Decls { + selected := "" + switch declaration := declaration.(type) { + case *ast.FuncDecl: + if declaration.Name.Name == "block" && declaration.Body == nil && declaration.Recv == nil { + selected = "block" + } + case *ast.GenDecl: + if len(declaration.Specs) != 1 { + continue + } + switch spec := declaration.Specs[0].(type) { + case *ast.TypeSpec: + if spec.Name.Name == "digest" { + selected = "digest" + } + case *ast.ValueSpec: + if len(spec.Names) == 1 && spec.Names[0].Name == "BlockSize" { + selected = "BlockSize" + } + } + } + if selected != "" { + if declarations[selected] != "" { + t.Fatal("duplicate original MD5 declaration", selected) + } + declarations[selected] = string(data[positions.Position(declaration.Pos()).Offset:positions.Position(declaration.End()).Offset]) + } + } + } + var goSource strings.Builder + goSource.WriteString("package md5\n") + for _, name := range []string{"BlockSize", "digest", "block"} { + if declarations[name] == "" { + t.Fatal("missing original MD5 declaration", name) + } + goSource.WriteString(declarations[name] + "\n") + } + if declarations["block"] != "func block(dig *digest, p []byte)" || declarations["BlockSize"] != "const BlockSize = 64" { + t.Fatal("actual original MD5 ABI or block-size declaration changed") + } + return string(assembly), mustGoPackage(t, "crypto/md5", goSource.String()) +} + +func TestAMD64MD5OriginalDataBPSourceObjectsThreeAPIs(t *testing.T) { + source, pkg := amd64MD5OriginalDataBPSource(t) + for _, triple := range []string{ + "x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, "amd64", triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewPackageAPI(t, source, pkg, "amd64", triple, api) + if err != nil || ir == "" { + t.Fatalf("complete original MD5 initialized data BP is not closed: %v", err) + } + }) + } + }) + } +} diff --git a/amd64_memory_address_role.go b/amd64_memory_address_role.go new file mode 100644 index 000000000..1b47d3939 --- /dev/null +++ b/amd64_memory_address_role.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "math" + "strings" +) + +// NAME_EXTERN/NAME_STATIC in the non-shared Go 386 encoder discards Reg, +// including a nonscaled SP/BP annotation. Index remains an actual address +// input. AMD64 uses RIP and rejects either annotation. This describes address +// roles, not backing, initialized bytes, alignment or caller permissions. +func x86SBMemoryAddressRole(operand Operand, arch string) (MemRef, bool) { + var memory MemRef + switch operand.Kind { + case OpSym: + memory.Sym = operand.Sym + case OpMem: + memory = operand.Mem + if x86SymbolIndexedMemoryError(memory, arch) != nil { + return MemRef{}, false + } + default: + return MemRef{}, false + } + if arch != "386" && arch != "amd64" || + !strings.HasSuffix(strings.TrimSpace(memory.Sym), "(SB)") { + return MemRef{}, false + } + if arch == "386" { + memory.Base = "" + } + return memory, true +} + +// Native emitters may additionally possess a checked physical frame token. +// This ordinary branch does not create one, including for an effective Index. +// BP is an ordinary GP in mode32, not an implicit physical frame binding. +// It classifies already validated operands; the typed instruction grammar must +// first reject a bare name or malformed address instead of inventing storage. +func x86OrdinaryNativeMemoryOperand(operand Operand, arch string) bool { + if operand.Kind == OpSym { + return true + } + if operand.Kind != OpMem { + return false + } + memory := operand.Mem + if named, valid := x86SBMemoryAddressRole(operand, arch); valid { + memory = named + } + return !x86LogicalSourceFrameRegister(memory.Base, arch) && !x86LogicalSourceFrameRegister(memory.Index, arch) +} + +func x86RegisterAddressedNativeMemoryOperand(operand Operand, arch string) bool { + return operand.Kind == OpMem && operand.Mem.Sym == "" && x86OrdinaryNativeMemoryOperand(operand, arch) +} + +// Select an emitter's address role, never pointee, extent or alignment facts. +// Logical BP stays non-native unless this exact bound access records otherwise. +func (c *amd64Ctx) nativeMemoryOperand(ins Instr, index int) bool { + if index < 0 || index >= len(ins.Args) { + return false + } + if x86OrdinaryNativeMemoryOperand(ins.Args[index], c.goarch) { + return true + } + if access, found := ins.x86FixedLocalMemory[index]; !found || !access.nativeInput { + return false + } + access, valid := c.fixedLocalMemoryAccess(ins, index) + return valid && access.nativeInput +} + +// The key includes the complete effective displacement, not Operand.String: +// raw AST memory can hold a displacement in addition to its symbolic offset. +type x86StaticSBAddress struct { + symbol string + offset int64 +} + +func x86StaticSBReadAddress(operand Operand, arch string) (x86StaticSBAddress, bool) { + address, valid := x86StaticSBSourceAddress(operand, arch) + return address, valid && address.offset >= 0 +} + +// Address formation permits signed addends. Only a read additionally needs +// a nonnegative offset and an independently checked complete storage extent. +func x86StaticSBSourceAddress(operand Operand, arch string) (x86StaticSBAddress, bool) { + memory, valid := x86SBMemoryAddressRole(operand, arch) + if !valid || memory.Index != "" || memory.Scale != 0 || memory.IndexExt != "" || + memory.OffRaw != "" || memory.Segment != "" { + return x86StaticSBAddress{}, false + } + base, offset, valid := parseSBRef(memory.Sym) + if !valid || base == "" || strings.IndexAny(base, "$+(),[] \t") >= 0 || + memory.Off > 0 && offset > math.MaxInt64-memory.Off || + memory.Off < 0 && offset < math.MinInt64-memory.Off { + return x86StaticSBAddress{}, false + } + offset += memory.Off + return x86StaticSBAddress{symbol: base, offset: offset}, true +} diff --git a/amd64_memory_fault_observer_test.go b/amd64_memory_fault_observer_test.go new file mode 100644 index 000000000..9317e2c68 --- /dev/null +++ b/amd64_memory_fault_observer_test.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "context" + "fmt" + "os/exec" + "runtime" + "strconv" + "testing" + "time" +) + +// Each probe is an owned, directly launched process. Do not catch native +// exceptions in C or accept setup exits as evidence of a memory fault. +func x86ObserveMemoryFault(t *testing.T, execution x86FloatCompareExecution, executable string, row, probe, original int) string { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + child, stop := context.WithTimeout(ctx, 30*time.Second) + defer stop() + args := execution.command(executable, strconv.Itoa(row), strconv.Itoa(probe), strconv.Itoa(original)) + stdout, stderr, err := runtimeOracleMachineOutput(child, exec.CommandContext(child, args[0], args[1:]...)) + if child.Err() != nil { + t.Fatal(child.Err()) + } + witness := fmt.Sprintf("memory enter %d %d %d\n", row, probe, original) + result, cause := x86NativeFaultResult(runtime.GOOS, string(stdout), witness, "memory complete\n", err) + if cause != nil { + t.Fatalf("row=%d probe=%d original=%d: %v\nstdout: %q\nstderr: %q", row, probe, original, cause, stdout, stderr) + } + return result +} + +func x86CheckMemoryFaultMatrix(t *testing.T, execution x86FloatCompareExecution, executable string, rows int) { + t.Helper() + required := make([]bool, rows) + for row := range required { + required[row] = true + } + x86CheckMemoryAccessMatrix(t, execution, executable, required) +} + +func x86CheckMemoryAccessMatrix(t *testing.T, execution x86FloatCompareExecution, executable string, required []bool) { + t.Helper() + for row, reads := range required { + for probe := 0; probe < 3; probe++ { + // A volatile byte read supplies an independent access-extent + // reference, not evidence of original AVX/EVEX execution. + want := x86ObserveMemoryFault(t, execution, executable, row, probe, 1) + if probe == 2 || !reads { + if want != "return" { + t.Fatalf("row %d probe %d: accessible or suppressed source faulted: %s", row, probe, want) + } + } else if want != "segv" && want != "bus" { + t.Fatalf("row %d probe %d: native access did not fault: %s", row, probe, want) + } + if got := x86ObserveMemoryFault(t, execution, executable, row, probe, 0); got != want { + t.Fatalf("row %d probe %d: translated %s, native access %s", row, probe, got, want) + } + } + } + t.Logf("%d rows: NULL, partial-page and complete-page inputs; independent native access and translated outcomes agree", len(required)) +} + +func x86MemoryFaultDriver(declarations, rows string) string { + return x86MemoryFaultSupportC + declarations + + "static const struct memory_case cases[] = {\n" + rows + "};\n" + x86MemoryFaultMainC +} + +func TestX86MemoryFaultObserverRejectsSetupAndCanaryFailures(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + const callbacks = ` +static int canary_failure(const uint8_t *p) { (void)p; return 91; } +static int fake_signal_exit(const uint8_t *p) { (void)p; return 139; } +` + driver := x86MemoryFaultDriver(callbacks, "{canary_failure, 4, 3}, {fake_signal_exit, 4, 3},") + executable := execution.compileLLVM(t, "memory-fault-controls", "", driver, "-O2") + for _, row := range []int{0, 1, 99} { + ctx, cancel := runtimeOracleStageContext(t) + args := execution.command(executable, strconv.Itoa(row), "2", "0") + stdout, _, err := runtimeOracleMachineOutput(ctx, exec.CommandContext(ctx, args[0], args[1:]...)) + cancel() + witness := fmt.Sprintf("memory enter %d 2 0\n", row) + if result, cause := x86NativeFaultResult(runtime.GOOS, string(stdout), witness, "memory complete\n", err); cause == nil { + t.Fatalf("setup/canary error became memory evidence: row=%d result=%s", row, result) + } + } +} + +const x86MemoryFaultSupportC = ` +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif +` + x86TestGuardPagesC + ` +struct memory_case { + int (*run)(const uint8_t *); + unsigned bytes; + unsigned accessible; +}; + +static int memory_fault_setup(unsigned leading) { +#ifdef _WIN32 + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); +#else + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) return 120; +#endif + if (setup_guard_before(leading) || page < 64) return 81; + for (size_t i = 0; i < page; ++i) guard[i] = (uint8_t)(i * 13 + 17); + return 0; +} +` + +const x86MemoryFaultMainC = ` +int main(int argc, char **argv) { + if (argc != 4) return 80; + unsigned row, probe, original; + char tail; + if (sscanf(argv[1], "%u%c", &row, &tail) != 1 || + sscanf(argv[2], "%u%c", &probe, &tail) != 1 || + sscanf(argv[3], "%u%c", &original, &tail) != 1 || + row >= sizeof(cases) / sizeof(cases[0]) || probe > 2 || original > 1) return 80; + int setup = memory_fault_setup(0); + if (setup) return setup; + struct memory_case kernel = cases[row]; + if (kernel.bytes > page || (kernel.bytes ? kernel.accessible >= kernel.bytes : kernel.accessible != 0)) return 82; + const uint8_t *input = probe == 0 ? NULL : + guard + page - (probe == 1 ? kernel.accessible : kernel.bytes); + + printf("memory enter %u %u %u\n", row, probe, original); + if (fflush(stdout)) return 83; + if (original) { + const volatile uint8_t *source = input; + volatile uint8_t sink = 0; + for (unsigned i = 0; i < kernel.bytes; ++i) sink ^= source[i]; + } else { + int result = kernel.run(input); + if (result) return result; + } + if (free_guard()) return 84; + puts("memory complete"); + return fflush(stdout) ? 85 : 0; +} +` diff --git a/amd64_minmax_runtime_test.go b/amd64_minmax_runtime_test.go new file mode 100644 index 000000000..bf67dd639 --- /dev/null +++ b/amd64_minmax_runtime_test.go @@ -0,0 +1,218 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86MinMaxMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable values require the native 64-bit host counterpart") + } + // Execute the unchanged complete logical-vector program on the actual + // host. This is not original AVX512 opcode or physical-register evidence. + + var source, declarations strings.Builder + checks := map[string]*strings.Builder{"views": {}, "memory": {}} + sigs := map[string]FuncSig{} + index := 0 + for _, op := range x86MinMaxTestOps { + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + signed, minimum := 0, 0 + if op[5] == 'S' { + signed = 1 + } + if strings.HasPrefix(op, "VPMIN") { + minimum = 1 + } + + for w, width := range []string{"X", "Y", "Z"} { + // Register/memory unmasked, register/memory masked, masked broadcast. + for mode := 0; mode < 5; mode++ { + if mode == 4 && bits < 32 { + continue + } + for _, zero := range []bool{false, true} { + if zero && mode < 2 { + continue + } + for _, raw := range []bool{false, true} { + name := fmt.Sprintf("minmax%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),DI\n") + source.WriteString("VMOVUPS (AX),Z0\n") + + input := "(BX)" + registerInput := mode == 0 || mode == 2 + if registerInput { + source.WriteString("VMOVUPS (BX),Z1\n") + input = width + "1" + } else if raw { + source.WriteString("VMOVUPS (AX),Z1\n") + } + suffix, mask := "", "" + if mode == 4 { + suffix = ".BCST" + } + if zero { + suffix += ".Z" + } + if mode >= 2 { + source.WriteString("KMOVQ mask+24(FP),K1\n") + mask = "K1," + } + + destination, oldB := 0, 0 + if raw { + destination = 1 + if registerInput { + oldB = 1 + } + } + line := fmt.Sprintf("%s%s %s,%s0,%s%s%d", op, suffix, input, width, mask, width, destination) + emitX86GoEncodedTestInstruction(t, &source, line, raw) + fmt.Fprintf(&source, "VMOVUPS Z%d,(DI)\n", destination) + fmt.Fprintf(&source, "VMOVUPS Y%d,64(DI)\n", destination) + fmt.Fprintf(&source, "VMOVUPS X%d,96(DI)\nRET\n", destination) + + sigs[name] = FuncSig{ + Name: name, + Args: []LLVMType{Ptr, Ptr, Ptr, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void*, const void*, void*, uint64_t);\n", name) + kind, zeroing := "views", 0 + if mode >= 2 { + kind = "memory" + } + if zero { + zeroing = 1 + } + fmt.Fprintf(checks[kind], " if (check(%s, %d, %d, %d, %d, %d, %d, %d)) {\n", name, 16<> (n % 32); + uint64_t active = mask & low_mask(lanes); + for (int i = 0; i < lanes; i++) { + uint64_t av = lane(a + i * bytes, bytes); + uint64_t bv = lane(b + (mode == 4 ? 0 : i * bytes), bytes); + uint64_t result; + if (mode < 2 || ((mask >> i) & 1)) { + result = minmax(av, bv, bytes * 8, sign, minimum); + } else { + result = zero ? 0 : old_b ? lane(b + i * bytes, bytes) : av; + } + memcpy(want + i * bytes, &result, bytes); + } + memcpy(want + 64, want, 32); + memcpy(want + 96, want, 16); + + fn(a, mode >= 3 && !active ? NULL : b, out, mask); + if (memcmp(out, want, sizeof(out))) { + fprintf(stderr, "width=%d bytes=%d sign=%d min=%d mode=%d zero=%d n=%d\n", + width, bytes, sign, minimum, mode, zero, n); + return 1; + } + } + + if (mode >= 3) { + for (int enabled = 1; enabled <= lanes; enabled++) { + uint8_t a[64], out[112], want[112] = {0}; + memset(a, 255, sizeof(a)); + int readable = mode == 4 ? bytes : enabled * bytes; + uint8_t *b = guard + page - readable; + memset(b, 255, (size_t)readable); + + for (int i = 0; i < lanes; i++) { + if (i < enabled || !zero) { + memset(want + i * bytes, 255, (size_t)bytes); + } + } + memcpy(want + 64, want, 32); + memcpy(want + 96, want, 16); + + fn(a, b, out, low_mask(enabled)); + if (memcmp(out, want, sizeof(out))) { + return 1; + } + } + } + return 0; +} +` diff --git a/amd64_minpos_runtime_test.go b/amd64_minpos_runtime_test.go new file mode 100644 index 000000000..75dbe6fa9 --- /dev/null +++ b/amd64_minpos_runtime_test.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86MinimumPositionMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for vex, op := range []string{"PHMINPOSUW", "VPHMINPOSUW"} { + for _, operand := range []string{"(BX)", "X1", "X0"} { + name := fmt.Sprintf("minpos%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\n", name) + source.WriteString("MOVQ old+0(FP),AX\nMOVQ input+8(FP),BX\nMOVQ out+16(FP),DI\n") + source.WriteString("VMOVUPS (AX),Z0\nVMOVUPS (AX),Z1\n") + if operand != "(BX)" { + fmt.Fprintf(&source, "MOVOU (BX),%s\n", operand) + } + emitX86GoEncodedTestInstruction(t, &source, op+" "+operand+",X0", raw) + source.WriteString("VMOVUPS Z0,(DI)\nVMOVUPS Y0,64(DI)\nVMOVUPS X0,96(DI)\nRET\n") + sigs[name] = FuncSig{ + Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void*, const void*, void*);\n", name) + fmt.Fprintf(&checks, " if (check(%s, %d)) {\n", name, vex) + fmt.Fprintf(&checks, " fprintf(stderr, \"%s failed\\n\");\n return 1;\n }\n", name) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source.String()) + // Original legacy m128 has an alignment exception, whereas this + // unchanged C observes unaligned mathematics. Keep those owners + // explicit and preserve the legacy output's upper aliases. + file = x86BlendMinposPrivateMathCompanion(t, source.String(), 6, 1, 4) + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + ir, err = x86BlendMinposMathIR(file, ir) + if err != nil { + t.Fatal("private mathematical companion lost exact source/IR ownership", err) + } + body := "#include \n#include \n#include \n" + + x86TestGuardPagesC + declarations.String() + x86MinimumPositionOracleC + + "\nint main(void) {\n if (setup_guard()) return 2;\n" + + checks.String() + " return free_guard();\n}\n" + x86ConflictFunnelPortableRuntime(t, llc, clang, "minpos-private-math", triple, ir, body) + }) + } +} + +const x86MinimumPositionOracleC = ` +static int check(void (*fn)(const void*, const void*, void*), int vex) { + const uint16_t values[] = {0, 1, 0x7fff, 0x8000, 0x8001, 0xfffe, 0xffff}; + for (int n = 0; n < 256; n++) { + uint8_t old[65], storage[17], out[112], want[112] = {0}; + uint8_t *input = storage + 1; + for (int i = 0; i < 65; i++) old[i] = (uint8_t)(i * 3 + 31); + + uint16_t minimum = 0xffff, position = 0; + for (int i = 0; i < 8; i++) { + uint16_t value = n < 8 ? (i == n ? 0 : 0xffff) : + values[(n + i * (n % 5)) % 7]; + memcpy(input + i * 2, &value, 2); + if (value < minimum) { + minimum = value; + position = (uint16_t)i; + } + } + memcpy(want, &minimum, 2); + memcpy(want + 2, &position, 2); + if (!vex) memcpy(want + 16, old + 1 + 16, 48); + memcpy(want + 64, want, 32); + memcpy(want + 96, want, 16); + + for (int edge = 0; edge < 2; edge++) { + const uint8_t *source = input; + if (edge) { + source = guard + page - 16; + memcpy(guard + page - 16, input, 16); + } + memset(out, 0xa5, sizeof(out)); + fn(old + 1, source, out); + if (memcmp(out, want, sizeof(out))) { + fprintf(stderr, "vex=%d n=%d edge=%d\n", vex, n, edge); + return 1; + } + } + } + return 0; +} +` diff --git a/amd64_mmx_native_export_test.go b/amd64_mmx_native_export_test.go new file mode 100644 index 000000000..d5a0524d8 --- /dev/null +++ b/amd64_mmx_native_export_test.go @@ -0,0 +1,57 @@ +package plan9asm + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" +) + +// An explicit diagnostic export retains the exact compiled fixture for a +// second execution transport. The original test still runs and reports its +// own outcome; export never converts an emulator failure into a native pass. +func x86MMXExportNativeFixture(t *testing.T, executable, source, ir, level, triple string, rows int) { + t.Helper() + directory := os.Getenv("PLAN9ASM_MMX_NATIVE_EXPORT_DIR") + if directory == "" { + return + } + if !filepath.IsAbs(directory) || level != "-O0" && level != "-O2" { + t.Fatal("native MMX export needs an explicit absolute directory and original optimization axis") + } + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + binary, err := os.ReadFile(executable) + if err != nil { + t.Fatal(err) + } + name := "mmx-native-" + level[1:] + write := func(name string, data []byte, mode os.FileMode) { + t.Helper() + file, err := os.OpenFile(filepath.Join(directory, name), os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + t.Fatal(err) + } + _, writeErr := file.Write(data) + closeErr := file.Close() + if writeErr != nil || closeErr != nil { + t.Fatal("incomplete native MMX export", writeErr, closeErr) + } + } + write(name, binary, 0700) + manifest, err := json.MarshalIndent(map[string]interface{}{ + "executable": name, "sha256": fmt.Sprintf("%x", sha256.Sum256(binary)), + "assembly_sha256": fmt.Sprintf("%x", sha256.Sum256([]byte(source))), + "ir_sha256": fmt.Sprintf("%x", sha256.Sum256([]byte(ir))), + "sdk": runtime.Version(), "triple": triple, "backend_level": level, "rows": rows, + "scope": "Compiled diagnostic artifact only; the original and second execution outcomes remain independent.", + }, "", " ") + if err != nil { + t.Fatal(err) + } + write(name+".json", append(manifest, '\n'), 0600) +} diff --git a/amd64_mmx_word_forms.go b/amd64_mmx_word_forms.go new file mode 100644 index 000000000..58e0b729c --- /dev/null +++ b/amd64_mmx_word_forms.go @@ -0,0 +1,96 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// These are source-selected bank/width facts, not initialized register, +// native-memory, x87 state or entry permissions. In particular Go's ordered +// ymovq Ymm/Ymr row reads a GP for the spelling MOVQ M0,M1. +type x86MMXWordForm struct { + source, destination Operand + bytes int64 + sourceM, targetM bool + sourceX, targetX bool + gpSpelling bool + nonTemporal bool +} + +func x86MMXWordProperties(ins Instr, arch string) (x86MMXWordForm, bool, error) { + if !x86ResultNamedPredicateInstruction(ins) { + return x86MMXWordForm{}, false, nil // Raw bytes have their own physical-bank contract. + } + name, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + streaming := name == "MOVNTQ" + if name != "MOVL" && name != "MOVQ" && name != "MOVD" && name != "MOVDQ2Q" && name != "MOVQOZX" && !streaming { + return x86MMXWordForm{}, false, nil + } + mmx := false + for _, operand := range ins.Args { + mmx = mmx || isAMD64MOVDMRegister(operand) + } + if !mmx && !streaming { + return x86MMXWordForm{}, false, nil + } + fail := func() (x86MMXWordForm, bool, error) { + return x86MMXWordForm{}, true, fmt.Errorf("%s %s outside complete Go MMX word rows: %q", arch, ins.Op, ins.Raw) + } + if dotted || len(ins.Args) != 2 || arch != "amd64" && arch != "386" { + return fail() + } + form := x86MMXWordForm{source: ins.Args[0], destination: ins.Args[1], bytes: 8} + form.sourceM, form.targetM = isAMD64MOVDMRegister(form.source), isAMD64MOVDMRegister(form.destination) + form.sourceX, form.targetX = isAMD64MOVDXRegister(form.source), isAMD64MOVDXRegister(form.destination) + vector := func(operand Operand) bool { + index, valid := amd64ParseXReg(operand.Reg) + return operand.Kind == OpReg && valid && index < 16 && (arch != "386" || index < 8) + } + memory := func(operand Operand) bool { + return isAMD64MemoryOperand(operand) && x86PackedShiftMemoryValid(operand, arch) + } + gp := func(operand Operand) bool { + return operand.Kind == OpReg && isX86YrlRegisterForArch(operand.Reg, arch) + } + if streaming { + if !form.sourceM || !memory(form.destination) { + return fail() + } + form.nonTemporal = true + return form, true, nil // ymr_ml: one MMX source and an m64 destination. + } + if name == "MOVQOZX" { + if !form.sourceM || !vector(form.destination) { + return fail() + } + return form, true, nil // ymrxr: MMX -> X low64, with the X high64 cleared. + } + if name == "MOVL" { + form.bytes = 4 + } + if form.targetM { + if form.sourceM { + if form.bytes != 8 || arch != "amd64" { + return fail() + } + index, _ := amd64ParseMReg(form.source.Reg) + form.source.Reg = amd64MMXEncodingGP(index) + form.sourceM, form.gpSpelling = false, true + } else if form.sourceX { + if form.bytes != 8 || !vector(form.source) { + return fail() + } + } else if gp(form.source) { + if form.bytes == 8 && arch == "386" { + return fail() + } + } else if !memory(form.source) { + return fail() + } + return form, true, nil + } + if form.sourceM && (memory(form.destination) || gp(form.destination) && (form.bytes == 4 || arch == "amd64")) { + return form, true, nil + } + return fail() +} diff --git a/amd64_mmx_word_forms_test.go b/amd64_mmx_word_forms_test.go new file mode 100644 index 000000000..9304c55d9 --- /dev/null +++ b/amd64_mmx_word_forms_test.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "testing" +) + +func TestX86MMXWordGoBankMatchesOriginalBytes(t *testing.T) { + for _, spelling := range []string{"MOVQ", "MOVD", "MOVDQ2Q"} { + for index := 0; index < 8; index++ { + instruction := fmt.Sprintf("%s M%d,M1", spelling, index) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT bank(SB),4,$0-0\n"+instruction+"\nRET\n") + want := []byte{0x48, 0x0f, 0x6e, byte(0xc8 + index), 0xc3} + if !reflect.DeepEqual(code, want) { + t.Fatal("ordered Go Ymm/Ymr word row changed", instruction, code, want) + } + ins := x86ScalarLaneInstruction(t, instruction) + form, known, err := x86MMXWordProperties(ins, "amd64") + gp := []Reg{AX, CX, DX, BX, SP, BP, SI, DI}[index] + if !known || err != nil || form.sourceM || !form.targetM || !form.gpSpelling || + form.source.Reg != gp || form.bytes != 8 || !reflect.DeepEqual(ins, x86ScalarLaneInstruction(t, instruction)) { + t.Fatal("source spelling became the wrong physical bank", instruction, form, err) + } + } + } +} + +func TestX86MMXWordCompleteSourceRows(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range []struct { + source string + bytes int64 + valid bool + }{ + {"MOVL AX,M0", 4, true}, {"MOVL M0,AX", 4, true}, + {"MOVL 1(AX),M0", 4, true}, {"MOVL M0,1(AX)", 4, true}, + {"MOVQ 1(AX),M0", 8, true}, {"MOVQ M0,1(AX)", 8, true}, + {"MOVQ X0,M0", 8, true}, {"MOVQOZX M0,X0", 8, true}, + {"MOVQ AX,M0", 8, arch == "amd64"}, {"MOVQ M0,AX", 8, arch == "amd64"}, + {"MOVQ M0,M1", 8, arch == "amd64"}, + {"MOVL M0,M1", 4, false}, {"MOVL X0,M0", 4, false}, + {"MOVQ M0,X0", 8, false}, {"MOVQOZX X0,M0", 8, false}, + {"MOVQ $7,M0", 8, false}, {"MOVQ.Z M0,AX", 8, false}, + {"MOVNTQ M0,1(AX)", 8, true}, {"MOVNTQ M7,1(AX)(CX*2)", 8, true}, + {"MOVNTQ M0,M1", 8, false}, {"MOVNTQ X0,1(AX)", 8, false}, + {"MOVNTQ AX,1(AX)", 8, false}, {"MOVNTQ.Z M0,1(AX)", 8, false}, + } { + t.Run(arch+"/"+row.source, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, row.source) + before := ins + form, known, err := x86MMXWordProperties(ins, arch) + if !known || (err == nil) != row.valid || row.valid && form.bytes != row.bytes || !reflect.DeepEqual(before, ins) { + t.Fatal("complete MMX word grammar changed", form, known, err, row.valid) + } + goAssemblerOracleOutput(t, arch, "TEXT bank(SB),4,$0-0\n"+row.source+"\nRET\n", "", row.valid) + }) + } + } +} diff --git a/amd64_mode32_object_source_test.go b/amd64_mode32_object_source_test.go new file mode 100644 index 000000000..6375c1a54 --- /dev/null +++ b/amd64_mode32_object_source_test.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "errors" + "regexp" + "strconv" + "strings" + "testing" +) + +var x86Mode32ObjectHighBank = regexp.MustCompile(`\b[XY](?:[89]|[12][0-9]|3[01])\b`) + +// Keep the complete original source acceptance and its missing physical +// binding observable. The returned low-bank fixture tests object generation, +// not equivalence to the original high-bank machine code or runtime behavior. +func x86PhysicalMode32ObjectSource(t *testing.T, source, arch string) string { + t.Helper() + if arch != "386" { + return source + } + goSource := strings.ReplaceAll(source, ",NOSPLIT", ",4") + requireX86GoAssemblerResult(t, arch, goSource, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + highRows := 0 + for _, function := range file.Funcs { + for _, instruction := range function.Instrs { + if !errors.Is(x86Mode32VectorRegisterContext(instruction, arch), ErrProbeNeedsContext) { + continue + } + highRows++ + one, err := Parse(ArchAMD64, "TEXT highbank(SB),4,$0-0\n"+instruction.Raw+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ir, err := Translate(one, Options{ + Goarch: arch, TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"highbank": {Name: "highbank", Ret: Void}}, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("original high bank acquired an unproved physical binding: %s: %v", instruction.Raw, err) + } + } + } + if highRows == 0 { + t.Fatal("mode32 fixture lost its original high-bank source coverage") + } + physical := x86Mode32ObjectHighBank.ReplaceAllStringFunc(source, func(register string) string { + index, err := strconv.Atoi(register[1:]) + if err != nil { + t.Fatal(err) + } + return register[:1] + strconv.Itoa(index%8) + }) + requireX86GoAssemblerResult(t, arch, strings.ReplaceAll(physical, ",NOSPLIT", ",4"), true) + return physical +} diff --git a/amd64_mode32_vector_bindings_test.go b/amd64_mode32_vector_bindings_test.go new file mode 100644 index 000000000..b1255e01a --- /dev/null +++ b/amd64_mode32_vector_bindings_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "errors" + "fmt" + "sort" + "strings" + "testing" +) + +type mode32VectorFamilyCase struct { + name string + rows []string + properties func(Instr, string) (bool, error) + lower func(*amd64Ctx, Instr) (bool, bool, error) +} + +// Existing SDK-table completeness tests independently bind these families to +// Go. The actual Go object here observes syntax acceptance, not register state. +func mode32VectorFamilyCases() []mode32VectorFamilyCase { + logical := mode32VectorFamilyCase{ + name: "logical", + properties: func(ins Instr, arch string) (bool, error) { + _, recognized, err := x86PackedLogicalProperties(ins, arch) + return recognized, err + }, + lower: func(ctx *amd64Ctx, ins Instr) (bool, bool, error) { + return ctx.lowerPackedLogicalFamily(ins.Op, ins) + }, + } + for _, opcode := range x86LogicalOpcodeNames() { + if x86PackedLogicalSpecs[opcode].encoding > x86PackedLogicalLegacyInteger { + logical.rows = append(logical.rows, x86LogicalAcceptedForms(opcode, "386")...) + } + } + move := mode32VectorFamilyCase{ + name: "move", + properties: func(ins Instr, arch string) (bool, error) { + _, recognized, err := amd64PackedVectorMoveProperties(ins, arch) + return recognized, err + }, + lower: func(ctx *amd64Ctx, ins Instr) (bool, bool, error) { + return ctx.lowerPackedVectorMove(ins.Op, ins) + }, + } + var moves []string + for opcode := range amd64PackedVectorMoveSpecs { + moves = append(moves, string(opcode)) + } + sort.Strings(moves) + for _, opcode := range moves { + spec := amd64PackedVectorMoveSpecs[Op(opcode)] + high := 31 + if spec.encoding == amd64PackedMoveVEX { + high = 15 + } + for _, bank := range []string{"X", "Y"} { + reg := fmt.Sprintf("%s%d", bank, high) + for _, args := range []string{"(AX)," + reg, reg + ",(AX)", reg + "," + bank + "0"} { + move.rows = append(move.rows, opcode+" "+args) + } + if spec.encoding != amd64PackedMoveVEX { + for _, args := range []string{"(AX),K7," + reg, reg + ",K7,(AX)", reg + ",K7," + bank + "0"} { + move.rows = append(move.rows, opcode+" "+args, opcode+".Z "+args) + } + } + } + } + fma := mode32VectorFamilyCase{ + name: "FMA3", + properties: func(ins Instr, arch string) (bool, error) { + _, recognized, err := amd64FMA3Properties(ins, arch) + return recognized, err + }, + lower: func(ctx *amd64Ctx, ins Instr) (bool, bool, error) { + return ctx.lowerFMA3(ins.Op, ins) + }, + } + // Scalar rounding, memory and packed broadcast axes remain in the source. + // Z0-Z7 are physically available; this regression targets the X/Y banks. + for _, row := range x86FMA3NamedFormats("386") { + changed := strings.NewReplacer("X2", "X16", "Y2", "Y16").Replace(row) + if changed != row { + fma.rows = append(fma.rows, changed) + } + } + return []mode32VectorFamilyCase{logical, move, fma} +} + +func TestX86Mode32VectorFamiliesRequirePhysicalBindings(t *testing.T) { + checked := 0 + for _, family := range mode32VectorFamilyCases() { + for _, row := range family.rows { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + high := false + for _, operand := range instruction.Args { + width := amd64VectorByteWidth(operand.Reg) + index, valid := amd64VectorRegisterIndex(operand.Reg, width) + high = high || operand.Kind == OpReg && valid && index >= 8 + } + if !high { + continue + } + checked++ + t.Run(family.name+"/"+strings.ReplaceAll(row, " ", "_"), func(t *testing.T) { + requireX86GoAssemblerResult(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", true) + if recognized, err := family.properties(instruction, "386"); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go-accepted high bank became a physical mode32 binding: recognized=%v err=%v", recognized, err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "386" + before := output.String() + if recognized, _, err := family.lower(ctx, instruction); !recognized || + !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("unproved high-bank source emitted IR: recognized=%v err=%v", recognized, err) + } + }) + } + } + if checked != 664 { + t.Fatalf("Go-accepted high-bank inventory changed: checked %d rows, want 664", checked) + } +} + +// This checks only the shared identity predicate. Family grammar tests above +// independently prove source acceptance, and the object tests retain low-bank +// success; this helper cannot manufacture a supported source operand. +func TestX86Mode32VectorRegisterContextBindsEveryOperand(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, bank := range []string{"X", "Y", "Z"} { + for _, index := range []int{0, 7, 8, 15, 16, 31} { + for position := 0; position < 4; position++ { + name := fmt.Sprintf("%s/%s%d/operand%d", arch, bank, index, position) + t.Run(name, func(t *testing.T) { + args := make([]Operand, 4) + for i := range args { + args[i] = Operand{Kind: OpReg, Reg: Reg(bank + "0")} + } + args[position].Reg = Reg(fmt.Sprintf("%s%d", bank, index)) + instruction := Instr{Op: "VECTOR_IDENTITY", Args: args} + err := x86Mode32VectorRegisterContext(instruction, arch) + if arch == "386" && index >= 8 { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved physical identity accepted: %v", err) + } + } else if err != nil { + t.Fatalf("physical register bank rejected: %v", err) + } + }) + } + } + } + } +} diff --git a/amd64_mov_extension_caller_test.go b/amd64_mov_extension_caller_test.go new file mode 100644 index 000000000..9e9cd6d7c --- /dev/null +++ b/amd64_mov_extension_caller_test.go @@ -0,0 +1,610 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "sort" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" +) + +var x86ExtensionTestOperations = []string{ + "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", "MOVBQSX", "MOVBQZX", + "MOVWLSX", "MOVWLZX", "MOVWQSX", "MOVWQZX", "MOVLQSX", "MOVLQZX", + "MOVSWW", "MOVZWW", +} + +func TestX86ExtensionCompletePrimaryTables(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86", "asm6.go") + bytes, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + table := string(bytes) + for _, name := range []string{"ymb_rl", "yml_rl"} { + pattern := regexp.MustCompile(`(?s)var ` + name + ` = \[\]ytab\{\s*\{(Zmb_r|Zm_r), 1, argList\{(Ymb|Yml), Yrl\}\},\s*\}`) + if !pattern.MatchString(table) { + t.Fatalf("actual Go table changed: %s", name) + } + } + rows := regexp.MustCompile(`\{A(MOV[A-Z]+), (ymb_rl|yml_rl), (Pm|Pw|Pq|Pe|Px),`).FindAllStringSubmatch(table, -1) + var names []string + for _, row := range rows { + names = append(names, row[1]) + } + want := append([]string(nil), x86ExtensionTestOperations...) + sort.Strings(names) + sort.Strings(want) + if strings.Join(names, ",") != strings.Join(want, ",") { + t.Fatalf("complete extension rows changed: got %v, want %v", names, want) + } +} + +func x86ExtensionGoListing(t *testing.T, arch, instruction string) (string, error) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "extension.s") + source := "TEXT ·extension(SB),4,$0-0\n" + instruction + "\nRET\n" + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "extension", + "-o", filepath.Join(dir, "extension.o"), path) + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + output, err := runtimeOracleCombinedOutput(ctx, command) + return string(output), err +} + +func TestX86ExtensionActualGoAliasAudit(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, instruction := range []string{ + "MOVSWW AL,AX", "MOVSWW AH,AX", "MOVSWW BP,AX", + "MOVZWW AL,AX", "MOVZWW AH,AX", "MOVZWW BP,AX", + "MOVBLSX AH,AX", "MOVBLSX BP,AX", "MOVLQZX AX,BX", + } { + t.Run(arch+"/"+strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + listing, err := x86ExtensionGoListing(t, arch, instruction) + if err != nil { + t.Fatalf("actual %s assembler: %v\n%s", runtime.Version(), err, listing) + } + t.Log(listing) + }) + } + } +} + +func TestX86ExtensionGoPhysicalRegisterViews(t *testing.T) { + for _, test := range []struct { + instruction, bytes string + source Reg + }{ + {"MOVSWW AL,AX", "66 0f bf c0 c3", AX}, + {"MOVSWW AH,AX", "66 0f bf c4 c3", SP}, + {"MOVZWW CH,AX", "66 0f b7 c5 c3", BP}, + {"MOVSWW DH,AX", "66 0f bf c6 c3", SI}, + {"MOVZWW BH,AX", "66 0f b7 c7 c3", DI}, + {"MOVBLSX AH,AX", "0f be c4 c3", AH}, + {"MOVBLSX AH,R8", "44 0f be c4 c3", SP}, + {"MOVBQSX BH,AX", "48 0f be c7 c3", DI}, + } { + t.Run(test.instruction, func(t *testing.T) { + listing, err := x86ExtensionGoListing(t, "amd64", test.instruction) + if err != nil || !strings.Contains(listing, test.bytes) { + t.Fatalf("actual Go machine view: %v\n%s", err, listing) + } + file, err := Parse(ArchAMD64, "TEXT v(SB),4,$0-0\n"+test.instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := append([]Operand(nil), ins.Args...) + form, recognized, err := x86ScalarExtensionProperties(ins.Op, ins, "amd64") + if !recognized || err != nil || form.source.Reg != test.source || !reflect.DeepEqual(before, ins.Args) { + t.Fatalf("Go ModRM view/source mutation: %+v recognized=%t err=%v", form, recognized, err) + } + }) + } +} + +func TestX86ExtensionCompleteFormatsThreeAPIFiveObjects(t *testing.T) { + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var body strings.Builder + for _, instruction := range x86ExtensionOriginalFormatRows(t, target.arch) { + body.WriteString(instruction + "\n") + } + body.WriteString("@MOV CX,ret+@WORD(FP)\nRET") + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, body.String()) + source += "GLOBL ·payload(SB),8,$16\nDATA ·payload+0(SB)/8,$0x1122334455667788\n" + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + // Keep the complete original mixed source. It has no producers + // for AX/BX/SI or its physical SP bytes; Go format acceptance + // alone cannot supply those inputs to any public API. + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewPackageAPI(t, source, pkg, target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("original mixed source acquired missing entry inputs", api, err) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + ir := x86ScalarFormatObjectUnit(t, file, map[string]FuncSig{"Value": sig}, target.arch, target.triple, 0) + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + }) + } +} + +func x86ExtensionOriginalFormatRows(t *testing.T, arch string) []string { + t.Helper() + var rows []string + for _, operation := range x86ExtensionTestOperations { + if arch == "386" && (operation == "MOVBQSX" || operation == "MOVBQZX" || + operation == "MOVWQSX" || operation == "MOVWQZX" || operation == "MOVLQSX") { + continue + } + for _, source := range []string{"AX", "8(BX)", "-1(BX)(SI*2)", "x+0(FP)", "@WORD(SP)", "·payload(SB)"} { + rows = append(rows, fmt.Sprintf("%s %s,CX", operation, source)) + } + if strings.HasPrefix(operation, "MOVB") || operation == "MOVSWW" || operation == "MOVZWW" { + rows = append(rows, operation+" BL,CX") + if arch == "386" { + rows = append(rows, operation+" DI,CX") + } + } + } + want := 92 + if arch == "386" { + want = 66 + } + if len(rows) != want { + t.Fatal("complete original extension format inventory changed", arch, len(rows), want) + } + return rows +} + +func TestX86ExtensionOriginalOrdinaryFormsWithRealInputs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + word, move, typ := 8, "MOVQ", "uint64" + if target.arch == "386" { + word, move, typ = 4, "MOVL", "uint32" + } + var ordinary []string + physical := 0 + for _, row := range x86ExtensionOriginalFormatRows(t, target.arch) { + if strings.Contains(row, "@WORD(SP)") { + physical++ + continue + } + ordinary = append(ordinary, row) + } + want, wantPhysical := 78, 14 + if target.arch == "386" { + want, wantPhysical = 57, 9 + } + if len(ordinary) != want || physical != wantPhysical { + t.Fatal("original ordinary/physical source partition changed", len(ordinary), physical) + } + for start := 0; start < len(ordinary); start += 32 { + end := start + 32 + if end > len(ordinary) { + end = len(ordinary) + } + t.Run(fmt.Sprintf("batch%d", start/32), func(t *testing.T) { + var source, declarations strings.Builder + source.WriteString("GLOBL ·payload(SB),8,$16\nDATA ·payload+0(SB)/8,$0x1122334455667788\n") + for ordinal, row := range ordinary[start:end] { + name := fmt.Sprintf("Extension%d", ordinal) + fmt.Fprintf(&declarations, "func %s(x %s, p *[32]byte) %s\n", name, typ, typ) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s p+%d(FP),BX\n%s x+0(FP),AX\nMOVL $1,SI\nMOVL $2,DI\n%s $0,CX\n", + name, 3*word, move, word, move, move) + if strings.Contains(row, " BL,") { + source.WriteString("MOVL $3,BX\n") + } + fmt.Fprintf(&source, "%s\n%s CX,ret+%d(FP)\nRET\n", row, move, 2*word) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("original ordinary tuple with real new input producers", api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + } + }) + } + }) + } +} + +func TestX86ExtensionOriginalPhysicalSPWithCapturedAMD64Frame(t *testing.T) { + goAMD64FramePrepareOwnedToolCache(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue // The AMD64 frame producer cannot certify a 386 frame. + } + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, row := range x86ExtensionOriginalFormatRows(t, target.arch) { + if !strings.Contains(row, "@WORD(SP)") { + continue + } + row = strings.Replace(row, "@WORD", "8", 1) + name := fmt.Sprintf("Physical%d", count) + fmt.Fprintf(&declarations, "func %s(x uint64) uint64\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$32-16\nMOVQ x+0(FP),AX\nMOVQ AX,8(SP)\nMOVQ $0,CX\n%s\nMOVQ CX,ret+8(FP)\nRET\n", name, row) + count++ + } + if count != 14 { + t.Fatal("complete original AMD64 physical-SP format set changed", count) + } + goos := "linux" + if strings.Contains(target.triple, "apple") { + goos = "darwin" + } else if strings.Contains(target.triple, "windows") { + goos = "windows" + } + pkg, request, _, options := fixedLocalTransportFixture(t, source.String(), declarations.String(), goos+"/amd64", target.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, + func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("same physical 8(SP) tuple with actual captured frame and numerical stores", err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + }) + }) + } +} + +func TestX86ExtensionOriginal386PhysicalSPStillNeedsSourceOwner(t *testing.T) { + for _, target := range x86RawReturnTargets { + if target.arch != "386" { + continue + } + t.Run(target.triple, func(t *testing.T) { + count := 0 + for _, row := range x86ExtensionOriginalFormatRows(t, target.arch) { + if !strings.Contains(row, "@WORD(SP)") { + continue + } + source, _, pkg := x86SourceStackNumericalFixture(target.arch, + "MOVL $0,CX\n"+row+"\n@MOV CX,ret+@WORD(FP)\nRET\n") + requireX86GoAssemblerResult(t, target.arch, source, true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewPackageAPI(t, source, pkg, target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("unowned physical 4(SP) bytes acquired numerical authority", row, api, err) + } + } + count++ + } + if count != 9 { + t.Fatal("complete original 386 physical-SP source-role boundary changed", count) + } + t.Log("all nine original physical-SP formats retain pending public source ownership; Go/private format objects are checked separately") + }) + } +} + +func TestX86ExtensionSourceAndPhysicalDecoderShareEffects(t *testing.T) { + for _, operation := range x86ExtensionTestOperations { + for _, body := range []string{operation + " BX,AX", x86ExtensionNumericalRawBytes(operation)} { + for _, defined := range []bool{false, true} { + prefix := "" + if defined { + prefix = "MOVQ $0xff80,BX\n" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", prefix+body+"\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if defined && err != nil || !defined && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("source/physical input availability defined=%t: %v\n%s", defined, err, body) + } + } + } + } +} + +func TestX86ExtensionSharedFormIsOrderedAndImmutable(t *testing.T) { + for _, operation := range x86ExtensionTestOperations { + file, err := Parse(ArchAMD64, "TEXT v(SB),4,$0-0\n"+operation+" AX,BX\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + for _, spelling := range []Op{original.Op, "", "MOVQ"} { + instruction := original + instruction.Op = spelling + before := append([]Operand(nil), instruction.Args...) + _, recognized, err := x86ScalarExtensionProperties(original.Op, instruction, "amd64") + if !recognized || (err == nil) != (spelling != "MOVQ") || !reflect.DeepEqual(before, instruction.Args) { + t.Fatalf("%s spelling %s: recognized=%t err=%v source changed=%t", operation, spelling, recognized, err, !reflect.DeepEqual(before, instruction.Args)) + } + } + } +} + +func TestX86ExtensionActualGoRejectedFormats(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, operation := range x86ExtensionTestOperations { + if target.arch == "386" && amd64ScalarExtensionMoveSpecs[Op(operation)].rejectedOn386 { + requireX86GoAssemblerResult(t, target.arch, "TEXT ·extension(SB),4,$0-0\n"+operation+" AX,BX\nRET\n", false) + continue + } + for _, operands := range []string{"$1,AX", "AX,(BX)", "X1,AX", "AX,AL", "8(BAD),AX"} { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, operation+" "+operands+"\nRET") + requireX86GoAssemblerResult(t, target.arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue // The public parser itself rejected the original format. + } + options := x86SourceStackOptions(target.arch, target.triple, sig) + if _, err := Translate(file, options); err == nil { + t.Fatalf("Translate accepted Go-rejected %s %s", operation, operands) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if err == nil { + t.Fatalf("TranslateModule accepted Go-rejected %s %s", operation, operands) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if err == nil { + t.Fatalf("TranslateGoModule accepted Go-rejected %s %s", operation, operands) + } + } + } + } +} + +func TestX86ExtensionRawMOVSXDRequiresKnownNativeWidth(t *testing.T) { + // Register and default-32 memory widths have independent native contracts. + // The 66 memory footprint still differs across vendor descriptions. + for _, encoding := range []string{"BYTE $0x66\nWORD $0x0763", "BYTE $0x66\nWORD $0x0f63"} { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "MOVQ $0x80000000,BX\n"+encoding+"\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown raw MOVSXD width Translate: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown raw MOVSXD width module: %v", err) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown raw MOVSXD width Go module: %v", err) + } + } + } +} + +func TestX86GoCallerRegionExtensionSourceInputs(t *testing.T) { + for _, operation := range x86ExtensionTestOperations { + for _, input := range []string{"register", "memory"} { + t.Run(operation+"/"+input, func(t *testing.T) { + operand := "AX" + if input == "memory" { + operand = "(DI)" + } + body := fmt.Sprintf("%s %s,AX\nMOVL AX,(DX)", operation, operand) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestX86GoCallerRegionExtensionDefinedBytesAndPointerNegatives(t *testing.T) { + for _, test := range []struct { + name, body string + valid bool + }{ + {"one initialized low byte", "MOVB $128,BX\nMOVBLZX BL,AX", true}, + {"one initialized high byte", "MOVB $128,BH\nMOVBLSX BH,AX", true}, + {"complete low word via aliases", "MOVB $128,BL\nMOVB $255,BH\nMOVSWW BL,AX", true}, + {"word low byte incomplete", "MOVB $128,BL\nMOVSWW BL,AX", false}, + {"unknown source", "MOVLQSX BX,AX", false}, + {"partial source long", "MOVW $7,BX\nMOVLQSX BX,AX", false}, + {"caller pointer sign extension", "MOVLQSX DI,AX", false}, + {"caller pointer byte extension", "MOVBQZX DI,AX", false}, + {"caller pointer apparent high-byte REX", "MOVBQZX BH,AX", false}, + {"implicit SP source", "MOVSWW AH,AX", false}, + {"implicit BP source", "MOVZWW CH,AX", false}, + {"partial pointer destination", "MOVQ DI,BX\nMOVBWZX AL,BX", false}, + {"partial pointer replaced completely", "MOVQ DI,BX\nMOVBLZX AL,BX", true}, + {"unaligned pointer slot", "MOVLQSX a+1(FP),AX", false}, + {"partial pointer FP slot", "MOVLQSX a+0(FP),AX", false}, + {"return PC load", "MOVLQSX 8(BP),AX", false}, + {"saved BP load", "MOVWLZX 0(BP),AX", false}, + {"different defined bytes join", "CMPQ AX,$0\nJE other\nMOVB $7,BL\nother:\nMOVBLZX BL,AX", false}, + } { + t.Run(test.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", test.body+"\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if test.valid && err != nil || !test.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("valid=%t: %v", test.valid, err) + } + }) + } +} + +func TestX86GoCallerRegionExtensionThreeAPIs(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "MOVLQSX (DI),AX\nMOVL AX,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "extension.ll", "extension.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "extension-module.ll", "extension-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86SourceStackExtensionRequiresDefinedLowBytes(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, operation := range x86ExtensionTestOperations { + if target.arch == "386" && amd64ScalarExtensionMoveSpecs[Op(operation)].rejectedOn386 { + continue // Covered separately as actual Go-rejected formats, not Context forms. + } + for _, prefix := range []string{"", "MOVB $7,BX\n", "@MOV x+0(FP),BX\n"} { + body := prefix + operation + " BX,AX\n@MOV AX,ret+@WORD(FP)\nBYTE $0xc3" + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := x86SourceStackOptions(target.arch, target.triple, sig) + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller source acquired extension proof: %v\n%s", err, source) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller source acquired module proof: %v\n%s", err, source) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller source acquired Go module proof: %v\n%s", err, source) + } + } + } + } +} + +func TestX86ExtensionCannotBorrowStaleCallerMemoryEffects(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "MOVLQSX (DI),AX\nMOVL AX,(DX)", 1) + for _, change := range []string{"width", "opcode", "source", "missing caller"} { + t.Run(change, func(t *testing.T) { + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs, ResolveSym: resolve} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + for index := range normalized.Funcs[0].Instrs { + instruction := &normalized.Funcs[0].Instrs[index] + if instruction.Op != "MOVLQSX" { + continue + } + switch change { + case "width": + access := instruction.x86CallerMemory[0] + access.width = 8 // Actual read width is 4, never the result width. + instruction.x86CallerMemory[0] = access + case "opcode": + instruction.Op = "MOVWQSX" + case "source": + instruction.Args[0].Mem.Base = DX + } + } + if change == "missing caller" { + sig := sigs["caller.kernel"] + sig.x86CallerRegions = nil + sigs["caller.kernel"] = sig + } + if change == "width" { + if x86RawReturnMemoryBound(normalized.Funcs[0], normalized, options, true) { + t.Fatal("extension accepted a stale destination-width memory certificate") + } + return // A final consumer legitimately recomputes unchanged source effects. + } + if _, err := Translate(normalized, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed %s borrowed an extension caller proof: %v", change, err) + } + }) + } +} diff --git a/amd64_mov_extension_effects.go b/amd64_mov_extension_effects.go new file mode 100644 index 000000000..24f06cf47 --- /dev/null +++ b/amd64_mov_extension_effects.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ScalarExtensionMoveSpec struct { + sourceBits int + destinationBits int + signed bool + rejectedOn386 bool + byteClass bool +} + +// These fourteen rows use Go's ymb_rl or yml_rl tables. MOVSWW/MOVZWW +// read a word despite using ymb_rl and its byte-register ModRM spellings. +var amd64ScalarExtensionMoveSpecs = map[Op]amd64ScalarExtensionMoveSpec{ + "MOVBWSX": {sourceBits: 8, destinationBits: 16, signed: true, byteClass: true}, + "MOVBWZX": {sourceBits: 8, destinationBits: 16, byteClass: true}, + "MOVBLSX": {sourceBits: 8, destinationBits: 32, signed: true, byteClass: true}, + "MOVBLZX": {sourceBits: 8, destinationBits: 32, byteClass: true}, + "MOVBQSX": {sourceBits: 8, destinationBits: 64, signed: true, rejectedOn386: true, byteClass: true}, + "MOVBQZX": {sourceBits: 8, destinationBits: 64, rejectedOn386: true, byteClass: true}, + "MOVWLSX": {sourceBits: 16, destinationBits: 32, signed: true}, + "MOVWLZX": {sourceBits: 16, destinationBits: 32}, + "MOVWQSX": {sourceBits: 16, destinationBits: 64, signed: true, rejectedOn386: true}, + "MOVWQZX": {sourceBits: 16, destinationBits: 64, rejectedOn386: true}, + "MOVLQSX": {sourceBits: 32, destinationBits: 64, signed: true, rejectedOn386: true}, + "MOVLQZX": {sourceBits: 32, destinationBits: 64}, + "MOVSWW": {sourceBits: 16, destinationBits: 16, signed: true, byteClass: true}, + "MOVZWW": {sourceBits: 16, destinationBits: 16, byteClass: true}, +} + +type x86ScalarExtensionForm struct { + spec amd64ScalarExtensionMoveSpec + source Operand + result Reg +} + +func x86ScalarExtensionProperties(op Op, ins Instr, arch string) (x86ScalarExtensionForm, bool, error) { + op = Op(strings.ToUpper(string(op))) + spec, recognized := amd64ScalarExtensionMoveSpecs[op] + if !recognized { + return x86ScalarExtensionForm{}, false, nil + } + fail := func(reason string) (x86ScalarExtensionForm, bool, error) { + return x86ScalarExtensionForm{}, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || len(ins.Args) != 2 { + return fail("requires matching opcode and two operands") + } + if arch == "386" && spec.rejectedOn386 { + return fail("is outside Go's 386 extension tables") + } + if arch == "386" && op == "MOVLQZX" { + // Px is an ordinary MOVL in 386 mode, not a 64-bit write. + spec.destinationBits = 32 + } + source, destination := ins.Args[0], ins.Args[1] + if projected, segment, err := x86NamedSegmentMemorySource(ins, source, arch); segment { + if err != nil { + return x86ScalarExtensionForm{}, true, err + } + source = projected + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, arch) { + return fail("expects a Yrl destination register") + } + if source.Kind == OpReg { + valid := isX86YrlRegisterForArch(source.Reg, arch) + if spec.byteClass { + valid = isGoYmbRegisterForArch(source.Reg, arch) + } + if !valid { + return fail("source register is outside its Go class") + } + if spec.byteClass && spec.sourceBits == 16 { + source.Reg = amd64YmbEffectiveRegister(source.Reg, 16) + } else if spec.sourceBits == 8 && arch != "386" && + (spec.destinationBits == 64 || x86ExtensionHighRegister(destination.Reg)) { + // A REX prefix changes ModRM 4..7 from AH/CH/DH/BH to the + // low bytes of SP/BP/SI/DI. Go emits those actual bytes too. + switch source.Reg { + case AH, CH, DH, BH: + source.Reg = amd64YmbEffectiveRegister(source.Reg, 16) + } + } + } else if !isAMD64MemoryOperand(source) || + source.Kind == OpSym && !x86PackedShiftMemoryValid(source, arch) { + return fail("expects a register or memory source") + } + if source.Kind == OpMem && !x86MemoryRegistersValidForArch(source.Mem, arch) { + return fail("memory source has an invalid address register") + } + return x86ScalarExtensionForm{spec: spec, source: source, result: destination.Reg}, true, nil +} + +func x86ExtensionHighRegister(register Reg) bool { + switch register { + case "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15": + return true + default: + return false + } +} + +func x86ScalarMoveMemoryBytes(ins Instr, operand int, arch string) int64 { + if form, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch); known { + if err == nil && operand == 0 && isAMD64MemoryOperand(form.source) { + return int64(form.bits / 8) + } + return 0 + } + op := Op(normalizeInstructionOpcode(x86SourceMoveInstruction(ins, arch).Op)) + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); recognized { + if err == nil && operand == 0 { + return 1 + } + return 0 + } + if width := x86RawReturnScalarMoveBytes(op); width != 0 { + return width + } + form, recognized, err := x86ScalarExtensionProperties(op, ins, arch) + if !recognized || err != nil || operand != 0 { + return 0 + } + return int64(form.spec.sourceBits / 8) +} + +func x86CallerExtensionStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (bool, bool) { + form, recognized, err := x86ScalarExtensionProperties(ins.Op, *ins, "amd64") + if !recognized || err != nil { + return recognized, false + } + width := int64(form.spec.sourceBits / 8) + if !x86CallerMemoryStep(ins, state, proof, width) { + return true, false + } + if form.source.Kind != OpMem && !x86CallerNumericRead(form.source, width, state) { + return true, false + } + return true, x86CallerNumericWrite(form.result, int64(form.spec.destinationBits/8), state) +} diff --git a/amd64_mov_extension_identity_test.go b/amd64_mov_extension_identity_test.go new file mode 100644 index 000000000..04aedfe87 --- /dev/null +++ b/amd64_mov_extension_identity_test.go @@ -0,0 +1,42 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestX86ExtensionHelperAcceptsMatchingCaseAndEmptyIdentity(t *testing.T) { + for _, operation := range x86ExtensionTestOperations { + for _, identity := range []string{"", operation, strings.ToLower(operation), "Mov" + strings.ToLower(operation[3:])} { + for _, dispatch := range []Op{Op(operation), Op(strings.ToLower(operation))} { + instruction := Instr{Op: Op(identity), Raw: operation + " AX,BX", Args: []Operand{ + {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}, + }} + before := append([]Operand(nil), instruction.Args...) + form, recognized, err := x86ScalarExtensionProperties(dispatch, instruction, "amd64") + if !recognized || err != nil || form.result != BX || !reflect.DeepEqual(before, instruction.Args) || instruction.Op != Op(identity) { + t.Fatalf("%s identity %q: form=%+v recognized=%t err=%v", operation, identity, form, recognized, err) + } + ctx, _ := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + if err := ctx.lowerScalarExtensionMove(dispatch, instruction); err != nil { + t.Fatalf("matching dispatch %s / %q: %v", operation, identity, err) + } + } + } + } +} + +func TestX86ExtensionHelperRejectsUnknownOrConflictingDispatchBeforeIR(t *testing.T) { + for _, operation := range []Op{"MOVLQSX", "UNKNOWN_EXTENSION", "MOVQ"} { + instruction := Instr{Op: "MOVWQSX", Raw: "MOVWQSX AX,BX", Args: []Operand{ + {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}, + }} + before := append([]Operand(nil), instruction.Args...) + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + length := output.Len() + if err := ctx.lowerScalarExtensionMove(operation, instruction); err == nil || output.Len() != length || !reflect.DeepEqual(before, instruction.Args) { + t.Fatalf("dispatch %s: err=%v emitted=%d mutated=%t", operation, err, output.Len()-length, !reflect.DeepEqual(before, instruction.Args)) + } + } +} diff --git a/amd64_mov_extension_runtime_test.go b/amd64_mov_extension_runtime_test.go new file mode 100644 index 000000000..d4ced629f --- /dev/null +++ b/amd64_mov_extension_runtime_test.go @@ -0,0 +1,291 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86ExtensionNumericalCase struct { + op string + source, result int + signed bool + mode, name string +} + +func x86ExtensionNumericalCases() []x86ExtensionNumericalCase { + var cases []x86ExtensionNumericalCase + for _, operation := range []x86ExtensionNumericalCase{ + {op: "MOVBWSX", source: 8, result: 16, signed: true}, + {op: "MOVBWZX", source: 8, result: 16}, + {op: "MOVBLSX", source: 8, result: 32, signed: true}, + {op: "MOVBLZX", source: 8, result: 32}, + {op: "MOVBQSX", source: 8, result: 64, signed: true}, + {op: "MOVBQZX", source: 8, result: 64}, + {op: "MOVWLSX", source: 16, result: 32, signed: true}, + {op: "MOVWLZX", source: 16, result: 32}, + {op: "MOVWQSX", source: 16, result: 64, signed: true}, + {op: "MOVWQZX", source: 16, result: 64}, + {op: "MOVLQSX", source: 32, result: 64, signed: true}, + {op: "MOVLQZX", source: 32, result: 64}, + {op: "MOVSWW", source: 16, result: 16, signed: true}, + {op: "MOVZWW", source: 16, result: 16}, + } { + for _, mode := range []string{"Register", "Memory", "LowAlias", "InPlace", "PhysicalBytes"} { + if mode == "LowAlias" && operation.source == 32 || + mode == "LowAlias" && operation.source == 16 && operation.result != 16 { + continue + } + operation.mode = mode + operation.name = "extension" + operation.op + mode + cases = append(cases, operation) + } + } + // REX.W changes BH to DI's low byte. The source DI is explicitly numeric + // here, not the imported input pointer or an unknown entry register. + for _, operation := range []x86ExtensionNumericalCase{ + {op: "MOVBQZX", source: 8, result: 64}, + {op: "MOVBQSX", source: 8, result: 64, signed: true}, + {op: "MOVSWW", source: 16, result: 16, signed: true}, + {op: "MOVZWW", source: 16, result: 16}, + } { + operation.mode, operation.name = "PhysicalAlias", "extension"+operation.op+"PhysicalAlias" + cases = append(cases, operation) + } + return cases +} + +// Independent fixed encodings for the BX-to-AX forms. In particular LQSX +// requires REX.W; the ambiguous no-REX 63 form is a separate Context test. +func x86ExtensionNumericalRawBytes(operation string) string { + encodings := map[string][]byte{ + "MOVBWSX": {0x66, 0x0f, 0xbe, 0xc3}, + "MOVBWZX": {0x66, 0x0f, 0xb6, 0xc3}, + "MOVBLSX": {0x0f, 0xbe, 0xc3}, + "MOVBLZX": {0x0f, 0xb6, 0xc3}, + "MOVBQSX": {0x48, 0x0f, 0xbe, 0xc3}, + "MOVBQZX": {0x48, 0x0f, 0xb6, 0xc3}, + "MOVWLSX": {0x0f, 0xbf, 0xc3}, + "MOVWLZX": {0x0f, 0xb7, 0xc3}, + "MOVWQSX": {0x48, 0x0f, 0xbf, 0xc3}, + "MOVWQZX": {0x48, 0x0f, 0xb7, 0xc3}, + "MOVLQSX": {0x48, 0x63, 0xc3}, + "MOVLQZX": {0x8b, 0xc3}, + "MOVSWW": {0x66, 0x0f, 0xbf, 0xc3}, + "MOVZWW": {0x66, 0x0f, 0xb7, 0xc3}, + } + var source strings.Builder + for _, value := range encodings[operation] { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + return source.String() +} + +func x86ExtensionNumericalSource() (string, string) { + var declarations, assembly strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + for _, test := range x86ExtensionNumericalCases() { + declaration := strings.ReplaceAll(x86CallerFixtureSource, "float32", "uint64") + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", test.name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+test.name+"(", 1) + declarations.WriteString(declaration) + operand := "BX" + body := "MOVQ (DI),BX\nMOVQ $0x5a6b7c8d9eaf1021,AX\n" + switch test.mode { + case "Memory": + operand = "(DI)" + case "LowAlias": + operand = "BL" + case "InPlace": + body += "MOVQ BX,AX\n" + operand = "AX" + case "PhysicalAlias": + body += "MOVQ BX,DI\n" + operand = "BH" + } + body += "CMPQ BX,$0\n" + if test.mode == "PhysicalBytes" { + body += x86ExtensionNumericalRawBytes(test.op) + } else { + body += fmt.Sprintf("%s %s,AX\n", test.op, operand) + } + body += "JE done\nADDQ $7,AX\ndone:\nMOVQ AX,(DX)" + machine := strings.ReplaceAll(x86CallerFixtureAssembly, "·kernel", "·"+test.name) + machine = strings.Replace(machine, "MOVL $7,AX\nMOVL AX,(DX)", body, 1) + assembly.WriteString(machine) + } + return declarations.String(), assembly.String() +} + +func x86ExtensionActualGoNumerical(t *testing.T, declaration, assembly string, runner []string) { + t.Helper() + var tests strings.Builder + tests.WriteString(`package caller +import "testing" +func reference(input uint64, source, result uint, signed, inPlace bool) uint64 { + mask := uint64(1)< +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +` + declarations.String() + ` +static uint64_t reference(uint64_t input, unsigned source, unsigned result, bool sign, bool in_place) { + uint64_t mask = (UINT64_C(1)<= 0 { + operand := ins.Args[form.memory] + if !x86PackedShiftMemoryValid(operand, arch) { + return fail() + } + if err := x86NamedMultiplyMemoryError(operand, arch); err != nil { + return form, true, err + } + } + return form, true, nil +} + +func x86TypedMoveHalfByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86MoveHalfProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + effect := x86TypedByteEffect{requiresResultProof: true} + if form.spec.registerOnly { + effect.vector.upperWrite = x86TypedVectorUpperPreserve + if form.spec.vector { + effect.vector.upperWrite = x86TypedVectorUpperZero + } + } + for index, operand := range ins.Args { + width := int64(16) + if index == form.memory { + width = 8 + } + entry := x86TypedByteOperand{ + index: index, operand: operand, width: width, + read: index == form.source || index == form.base, write: index == form.destination, + } + if form.spec.registerOnly { + if index == form.source { + entry.requiredBytes = uint64(0xff) << uint(form.sourceLane*8) + } + if index == form.base { + entry.requiredBytes = uint64(0xff) << uint((1-form.outputLane)*8) + } + } + effect.operands = append(effect.operands, entry) + } + return effect, true, nil +} diff --git a/amd64_movemask_grammar_test.go b/amd64_movemask_grammar_test.go new file mode 100644 index 000000000..e0fff3dbb --- /dev/null +++ b/amd64_movemask_grammar_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86MoveMaskCompleteGoEncodingRows(t *testing.T) { + for filename, rows := range map[string][]string{ + "asm6.go": { + "{APMOVMSKB, ymskb, Px, opBytes{Pe, 0xd7, 0xd7}}", + "{AMOVMSKPD, yxrrl, Pq, opBytes{0x50}}", + "{AMOVMSKPS, yxrrl, Pm, opBytes{0x50}}", + "var yxrrl = []ytab{ {Zm_r, 1, argList{Yxr, Yrl}}, }", + "var ymskb = []ytab{ {Zm_r_xm, 2, argList{Yxr, Yrl}}, {Zm_r_xm, 1, argList{Ymr, Yrl}}, }", + }, + "avx_optabs.go": { + "var _yvmovmskpd = []ytab{ {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxr, Yrl}}, {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yyr, Yrl}}, }", + "{as: AVMOVMSKPD, ytab: _yvmovmskpd, prefix: Pavx, op: opBytes{ avxEscape | vex128 | vex66 | vex0F | vexW0, 0x50, avxEscape | vex256 | vex66 | vex0F | vexW0, 0x50, }}", + "{as: AVMOVMSKPS, ytab: _yvmovmskpd, prefix: Pavx, op: opBytes{ avxEscape | vex128 | vex0F | vexW0, 0x50, avxEscape | vex256 | vex0F | vexW0, 0x50, }}", + "{as: AVPMOVMSKB, ytab: _yvmovmskpd, prefix: Pavx, op: opBytes{ avxEscape | vex128 | vex66 | vex0F | vexW0, 0xD7, avxEscape | vex256 | vex66 | vex0F | vexW0, 0xD7, }}", + }, + } { + contents, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86", filename)) + if err != nil { + t.Fatal(err) + } + text := strings.Join(strings.Fields(string(contents)), " ") + for _, row := range rows { + if !strings.Contains(text, row) { + t.Fatalf("complete original Go encoding row changed in %s: %s", filename, row) + } + } + } + if len(amd64MoveMaskSpecs) != 6 { + t.Fatal("incomplete six-name move-mask family") + } +} + +func TestX86MoveMaskIntrinsicFeatureIsInstructionLocal(t *testing.T) { + for _, line := range []string{"PMOVMSKB X0,AX", "VPMOVMSKB X0,AX", "VPMOVMSKB Y0,AX"} { + file, err := Parse(ArchAMD64, "TEXT original(SB),4,$0-0\n"+line+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + features := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "386", Func{Instrs: []Instr{instruction}}) + if features != "+sse2" { + t.Fatalf("isolated %s cannot borrow a sibling's SSE2 feature: %q", line, features) + } + } +} + +func TestX86MoveMaskSharedGrammarMatchesActualGo(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, operation := range []string{"PMOVMSKB", "MOVMSKPS", "MOVMSKPD", "VPMOVMSKB", "VMOVMSKPS", "VMOVMSKPD"} { + for _, source := range []string{"X0", "X7", "X8", "X15", "X16", "Y0", "Y15", "Y16", "Z0", "M0", "M7", "AX", "(AX)", "$1"} { + for _, destination := range []string{"AX", "R8", "AL", "X1"} { + line := operation + " " + source + "," + destination + assembly := "TEXT original(SB),4,$0-0\n" + line + "\nRET\n" + vex := strings.HasPrefix(operation, "V") + validSource := source == "X0" || source == "X7" || + (source == "X8" || source == "X15") && (arch == "amd64" || vex) || + vex && (source == "Y0" || source == "Y15") || + operation == "PMOVMSKB" && (source == "M0" || source == "M7") + valid := validSource && (destination == "AX" || destination == "R8" && arch == "amd64") + goAssemblerOracleOutput(t, arch, assembly, "", valid) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + _, known, err := x86MoveMaskProperties(Op(operation), file.Funcs[0].Instrs[1], arch) + if !known || (err == nil) != valid { + t.Fatalf("%s %s shared grammar: known=%t err=%v", arch, line, known, err) + } + } + } + } + } +} + +func TestX86MoveMaskResultFiveTargetThreeAPIObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86OrderingHintTargets { + size, typ, move := 8, "uint64", "MOVQ" + if target.arch == "386" { + size, typ, move = 4, "uint32", "MOVL" + } + var source, declarations strings.Builder + declarations.WriteString("package masks\n") + for index, instruction := range x86ResultMoveMaskForms { + name := fmt.Sprintf("mask%d", index) + fmt.Fprintf(&declarations, "func %s() %s\n", name, typ) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\nVZEROALL\n%s\n%s AX,ret0+0(FP)\nRET\n", name, size, instruction, move) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + dir := t.TempDir() + input, object := filepath.Join(dir, "mask.ll"), filepath.Join(dir, "mask.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+target.triple, "-filetype=obj", input, "-o", object)) + } + }) + } + } +} diff --git a/amd64_movnti_forms.go b/amd64_movnti_forms.go new file mode 100644 index 000000000..a400d1187 --- /dev/null +++ b/amd64_movnti_forms.go @@ -0,0 +1,72 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's complete scalar non-temporal family shares one yrl_ml/Zr_m row. +// Its Yml compatibility closure includes register destinations whose physical +// ModRM form faults; those forms never consume a source value or an address. +var x86MOVNTIWidths = map[Op]int{ + "MOVNTIL": 32, + "MOVNTIQ": 64, +} + +type x86MOVNTIForm struct { + bits int + source, destination Operand + trap bool +} + +func x86MOVNTIProperties(op Op, ins Instr, arch string) (x86MOVNTIForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + bits, known := x86MOVNTIWidths[Op(name)] + form := x86MOVNTIForm{bits: bits} + if !known { + return form, false, nil + } + fail := func() (x86MOVNTIForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside Go yrl_ml scalar non-temporal grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || arch == "386" && bits == 64 || dotted || + len(ins.Args) != 2 || ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return fail() + } + form.source, form.destination = ins.Args[0], ins.Args[1] + if form.source.Kind != OpReg || !isX86YrlRegisterForArch(form.source.Reg, arch) { + return fail() + } + if form.destination.Kind == OpReg { + if !isX86YrlRegisterForArch(form.destination.Reg, arch) { + return fail() + } + form.trap = true + return form, true, nil + } + if !isAMD64MemoryOperand(form.destination) { + return fail() + } + memory := form.destination + if projected, segment, err := x86NamedSegmentMemorySource(ins, memory, arch); segment { + if err != nil { + return form, true, err + } + memory, form.destination = projected, projected + } + if memory.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(memory.Mem, arch); err != nil { + return form, true, err + } + if memory.Mem.OffRaw != "" || memory.Mem.IndexExt != "" { + return fail() + } + if arch == "386" && strings.HasSuffix(memory.Mem.Sym, "(SB)") { + memory.Mem.Base = "" // Go discards this named annotation, not Index. + } + } + if !x86ScalarCompareMemoryValid(memory, arch) { + return fail() + } + return form, true, nil +} diff --git a/amd64_movnti_grammar_test.go b/amd64_movnti_grammar_test.go new file mode 100644 index 000000000..f75c46950 --- /dev/null +++ b/amd64_movnti_grammar_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86MOVNTICompleteIndependentGoEncoderRows(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + text := string(data) + table := regexp.MustCompile(`(?s)var yrl_ml = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(text) + if len(table) != 2 || strings.Join(strings.Fields(table[1]), "") != + "{Zr_m,1,argList{Yrl,Yml}}," { + t.Fatal("complete Go MOVNTI grammar, zcase or encoding offset changed") + } + rows := regexp.MustCompile(`\{A(MOVNTI[\w]*), yrl_ml, (P\w+), opBytes\{([^}]+)\}\}`).FindAllStringSubmatch(text, -1) + if len(rows) != 2 { + t.Fatal("complete Go scalar non-temporal family changed", rows) + } + if len(x86MOVNTIWidths) != len(rows) { + t.Fatal("typed scalar non-temporal directory is not the complete independent Go family") + } + for _, row := range rows { + prefix, encoding, bits := "Pm", "0xc3", 32 + if row[1] == "MOVNTIQ" { + prefix, encoding, bits = "Pw", "0x0f,0xc3", 64 + } else if row[1] != "MOVNTIL" { + t.Fatal("unmodeled Go scalar non-temporal member", row[1]) + } + if row[2] != prefix || strings.Join(strings.Fields(row[3]), "") != encoding || x86MOVNTIWidths[Op(row[1])] != bits { + t.Fatal("complete Go MOVNTI physical encoding changed", row) + } + } +} + +func TestX86MOVNTICompleteGoRegisterAndAddressClasses(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + registers := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} + if arch == "amd64" { + registers = append(registers, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + } + destinations := append([]string(nil), registers...) + destinations = append(destinations, + "(AX)", "-1(AX)(CX*2)", "128(BX)(SI*8)", "tmp-4(SP)", + "ret+0(FP)", "·cell(SB)", "17", "8(AX)(GS)", "8(AX)(FS)") + var source strings.Builder + source.WriteString("DATA ·cell+0(SB)/8,$0\nGLOBL ·cell(SB),$8\n") + count := 0 + for _, name := range []string{"MOVNTIL", "MOVNTIQ"} { + if arch == "386" && name == "MOVNTIQ" { + continue + } + for _, register := range registers { + for _, destination := range destinations { + text := name + " " + register + "," + destination + ins := x86YMMReadSource(t, text) + form, known, err := x86MOVNTIProperties(ins.Op, ins, arch) + if !known || err != nil || form.trap != (ins.Args[1].Kind == OpReg) || !reflect.DeepEqual(form.source, ins.Args[0]) { + t.Fatal("complete Go Yrl/Yml row lost source shape or trap priority", text, err) + } + fmt.Fprintf(&source, "TEXT ·Format%d(SB),4,$0-8\n%s\nRET\n", count, text) + count++ + } + } + } + want := 136 + if arch == "amd64" { + want = 800 + } + if count != want { + t.Fatal("independent complete scalar non-temporal source/address inventory changed", count) + } + requireX86GoAssemblerResult(t, arch, source.String(), true) + }) + } +} + +func TestX86MOVNTIRejectsCompleteGrammarBeforeIR(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range []string{"MOVNTIL", "MOVNTIQ"} { + if arch == "386" && name == "MOVNTIQ" { + continue // The original Go-mode rejection matrix retains this axis. + } + original := x86YMMReadSource(t, name+" AX,(BX)") + for _, control := range []struct { + name string + change func(*Instr) + }{ + {"dispatch", func(i *Instr) { i.Op = "MOVL" }}, + {"suffix", func(i *Instr) { i.Op += ".Z" }}, + {"arity", func(i *Instr) { i.Args = i.Args[:1] }}, + {"immediate-source", func(i *Instr) { i.Args[0] = Operand{Kind: OpImm, Imm: 1} }}, + {"vector-source", func(i *Instr) { i.Args[0] = Operand{Kind: OpReg, Reg: "X0"} }}, + {"vector-destination", func(i *Instr) { i.Args[1] = Operand{Kind: OpReg, Reg: "X0"} }}, + {"address-immediate", func(i *Instr) { i.Args[1] = Operand{Kind: OpSym, Sym: "$16(BX)"} }}, + } { + t.Run(arch+"/"+name+"/"+control.name, func(t *testing.T) { + ins := original + ins.Args = append([]Operand(nil), original.Args...) + control.change(&ins) + var ir strings.Builder + ctx := &amd64Ctx{b: &ir, goarch: arch} + known, terminated, err := ctx.lowerMOVNTI(Op(name), ins) + if !known || terminated || err == nil || ir.Len() != 0 || ctx.tmp != 0 { + t.Fatal("incomplete MOVNTI grammar emitted before rejection", known, terminated, err, ir.String()) + } + }) + } + } + } +} + +func TestX86MOVNTIRegisterDestinationFaultsBeforeInputs(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range []string{"MOVNTIL", "MOVNTIQ"} { + if arch == "386" && name == "MOVNTIQ" { + continue + } + t.Run(arch+"/"+name, func(t *testing.T) { + // Go's Yml compatibility closure admits mod=3, but MOVNTI has + // no ISA register destination. No source byte or memory input + // may be read before this unconditional architectural #UD. + ins := x86YMMReadSource(t, name+" AX,BX") + var ir strings.Builder + ctx := &amd64Ctx{b: &ir, goarch: arch} + known, terminated, err := ctx.lowerMOVNTI(ins.Op, ins) + if !known || !terminated || err != nil || ctx.tmp != 0 || + ir.String() != " call void asm sideeffect \"ud2\", \"~{memory}\"()\n unreachable\n" { + t.Fatal("register MOVNTI did not fault before inputs", known, terminated, err, ir.String()) + } + }) + } + } +} + +func TestX86MOVNTILogicalStorageDoesNotSupplyNativeBacking(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, source := range []string{ + "TEXT ·Value(SB),4,$0-4\nMOVL $1,AX\nMOVNTIL AX,ret+0(FP)\nRET\n", + "TEXT ·Value(SB),4,$16-4\nMOVL $1,AX\nMOVNTIL AX,tmp-4(SP)\nMOVL $1,ret+0(FP)\nRET\n", + } { + t.Run(target.triple+"/"+strings.Split(source, "\n")[0], func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value() uint32", target.arch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("logical numerical bytes impersonated MOVNTI native storage", api, err) + } + } + }) + } + } +} diff --git a/amd64_movnti_native_test.go b/amd64_movnti_native_test.go new file mode 100644 index 000000000..1842eec17 --- /dev/null +++ b/amd64_movnti_native_test.go @@ -0,0 +1,83 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86MOVNTIOriginalNativeStoresFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + for _, width := range []struct { + suffix, typ, move string + bytes int + }{ + {"L", "uint32", "MOVL", 4}, {"Q", "uint64", "MOVQ", 8}, + } { + if target.arch == "386" && width.bytes == 8 { + continue // Stock Go rejects Q in mode32; the separate rejection matrix owns it. + } + t.Run(target.triple+"/"+width.suffix, func(t *testing.T) { + declaration := fmt.Sprintf("func Value(out *byte,value %s)\n", width.typ) + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\n%s out+0(FP),DX\n%s value+%d(FP),AX\nMOVL $1,CX\nMOVNTI%s AX,-1(DX)(CX*2)\nMOVNTI%s AX,-1(DX)(CX*2)\nSFENCE\nMOVB $0,-1(DX)(CX*2)\nRET\n", + word+width.bytes, move, width.move, word, width.suffix, width.suffix) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("initialized original MOVNTI source rejected", err) + } + for _, input := range []string{ir, x86DwordProductIROptimize(t, ir)} { + if strings.Count(input, `asm sideeffect "movnti`) != 2 || strings.Contains(input, "!nontemporal") { + t.Fatal("native cache policy/dead writes became optional metadata or disappeared") + } + compileLLVMToObject(t, llc, target.triple, "movnti-native.ll", "movnti-native.o", input) + } + }) + } + }) + } + } +} + +func TestX86MOVNTIDeadStoresStillRequireValueAndAddress(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + for _, suffix := range []string{"L", "Q"} { + if target.arch == "386" && suffix == "Q" { + continue + } + for _, unused := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/unused-bool=%t", target.triple, suffix, unused), func(t *testing.T) { + declaration, extent := "func Value(out *byte)", word + if unused { + declaration, extent = "func Value(out *byte,unused bool)", word+1 + } + for _, setup := range []string{move + " out+0(FP),DX\n", "MOVL $1,AX\n"} { + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\n%sMOVNTI%s AX,(DX)\nRET\n", extent, setup, suffix) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Void or unused bool invented MOVNTI inputs", api, err) + } + } + } + }) + } + } + } +} diff --git a/amd64_movnti_runtime_test.go b/amd64_movnti_runtime_test.go new file mode 100644 index 000000000..00f868934 --- /dev/null +++ b/amd64_movnti_runtime_test.go @@ -0,0 +1,169 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86MOVNTIOriginalFlagsAndFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, prototypes, checks, goChecks strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, row := range []struct { + suffix string + bytes int + }{{"L", 4}, {"Q", 8}} { + for _, offset := range []int{0, 1} { + name := fmt.Sprintf("Store%s%d", row.suffix, offset) + fmt.Fprintf(&declarations, "func %s(out *byte,value uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-16\n"+ + "MOVQ out+0(FP),DX\nMOVQ value+8(FP),AX\n"+ + "MOVL $0x7fffffff,R8\nADDL $1,R8\nSTC\nMOVL $%d,CX\n"+ + "MOVNTI%s AX,(DX)(CX*1)\nSFENCE\n"+ + "SETCS 24(DX)\nSETOS 25(DX)\nSETEQ 26(DX)\nSETMI 27(DX)\nRET\n", + name, offset, row.suffix) + fmt.Fprintf(&prototypes, "extern void %s(uint8_t *,uint64_t);\n", name) + fmt.Fprintf(&checks, "if (check_value(%s,%d,%d,value)) return 1; count++;\n", name, row.bytes, offset) + fmt.Fprintf(&goChecks, "check(t,%s,%d,%d,value); count++\n", name, row.bytes, offset) + count++ + } + fmt.Fprintf(&declarations, "func Dead%s(out *byte)\n", row.suffix) + fmt.Fprintf(&assembly, "TEXT ·Dead%s(SB),4,$0-8\nMOVQ out+0(FP),DX\nMOVQ $42,AX\n"+ + "MOVNTI%s AX,(DX)\nMOVNTI%s AX,(DX)\nSFENCE\nRET\n", row.suffix, row.suffix, row.suffix) + } + if count != 4 { + t.Fatal("complete scalar non-temporal value/canary directory changed", count) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + for _, pipeline := range []struct{ name, level, ir string }{ + {"backend_O0", "-O0", ir}, + {"backend_O2", "-O2", ir}, + {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + if strings.Count(pipeline.ir, `asm sideeffect "movnti`) != 8 || strings.Contains(pipeline.ir, "!nontemporal") { + t.Fatal("original/dead native stores became optional cache metadata") + } + output := execution.compileAndRunLLVM(t, "movnti-values-flags", pipeline.ir, + fmt.Sprintf(x86MOVNTIValuesC, prototypes.String(), checks.String()), pipeline.level) + if string(output) != "MOVNTI values/FLAGS/canary outcomes 32 failures 0\n" { + t.Fatal("independent exact native value/FLAGS/canary observations", string(output)) + } + executable := execution.compileLLVM(t, "movnti-dead-faults", pipeline.ir, x86MOVNTIFaultsC, pipeline.level) + for row := 0; row < 2; row++ { + for probe := 0; probe < 4; probe++ { + want := x86ObserveMemoryFault(t, execution, executable, row, probe, 1) + if probe == 2 { + if want != "return" { + t.Fatal("complete accessible original native MOVNTI store faulted", row, want) + } + } else if want != "segv" && want != "bus" { + t.Fatal("NULL, partial-page or readonly original native store did not fault", row, probe, want) + } + if got := x86ObserveMemoryFault(t, execution, executable, row, probe, 0); got != want { + t.Fatal("dead MOVNTI lost exact native fault behavior", row, probe, got, want) + } + } + } + t.Log("32 value/FLAGS/full-buffer outcomes; both widths retain dead NULL, partial-page, complete-page and readonly native writes") + }) + } + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), + fmt.Sprintf(x86MOVNTIOriginalGo, goChecks.String())) +} + +const x86MOVNTIValuesC = ` +#include +#include +#include +%s +static int check_value(void (*run)(uint8_t *,uint64_t),unsigned bytes,unsigned offset,uint64_t value) { + uint8_t actual[48], expected[48]; + memset(actual,0xc3,sizeof(actual)); + memset(expected,0xc3,sizeof(expected)); + for (unsigned i=0;i>(i*8)); + expected[25]=1; + expected[26]=1; + expected[27]=0; + expected[28]=1; + run(actual+1,value); + if (memcmp(actual,expected,sizeof(actual))) { + fprintf(stderr,"MOVNTI value/FLAGS/canary mismatch bytes=%%u offset=%%u value=%%llx\n",bytes,offset,(unsigned long long)value); + return 1; + } + return 0; +} +int main(void) { + const uint64_t values[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000), + UINT64_C(0x1122334455667788),UINT64_C(0x8877665544332211), + UINT64_C(0xaaaaaaaa55555555),UINT64_C(0x00000000ffffffff)}; + unsigned count=0; + for (unsigned i=0;i>uint(i*8))} + expected[25],expected[26],expected[27],expected[28]=1,1,0,1 + run(&actual[1],value) + if actual!=expected {t.Fatalf("original MOVNTI bytes=%%d offset=%%d value=%%x got=%%x want=%%x",bytes,offset,value,actual,expected)} +} +func TestOriginal(t *testing.T) { + values:=[]uint64{0,1,^uint64(0),0x8000000000000000,0x1122334455667788, + 0x8877665544332211,0xaaaaaaaa55555555,0x00000000ffffffff} + count:=0 + for _,value:=range values { + %s + } + if count!=32 {t.Fatal(count)} + t.Log("original Go MOVNTI values/FLAGS/canary outcomes 32 failures 0") +} +` + +const x86MOVNTIFaultsC = x86MemoryFaultSupportC + ` +extern void DeadL(uint8_t *); +extern void DeadQ(uint8_t *); +int main(int argc,char **argv) { + if (argc!=4) return 80; + unsigned row,probe,original; + char tail; + if (sscanf(argv[1],"%u%c",&row,&tail)!=1 || + sscanf(argv[2],"%u%c",&probe,&tail)!=1 || + sscanf(argv[3],"%u%c",&original,&tail)!=1 || row>1 || probe>3 || original>1) return 80; + if (memory_fault_setup(0)) return 81; + unsigned bytes=row?8:4; + uint8_t *input=probe==0?NULL:guard+page-bytes+(probe==1?1:0); + if (probe==3) { +#ifdef _WIN32 + DWORD old; + if (!VirtualProtect(guard,page,PAGE_READONLY,&old)) return 82; +#else + if (mprotect(guard,page,PROT_READ)) return 82; +#endif + } + printf("memory enter %u %u %u\n",row,probe,original); + if (fflush(stdout)) return 83; + if (original) { + if (row) __asm__ volatile("movnti %1,(%0); movnti %1,(%0); sfence"::"r"(input),"r"((uint64_t)42):"memory"); + else __asm__ volatile("movnti %1,(%0); movnti %1,(%0); sfence"::"r"(input),"r"((uint32_t)42):"memory"); + } else if (row) DeadQ(input); + else DeadL(input); + if (free_guard()) return 84; + puts("memory complete"); + return fflush(stdout)?85:0; +} +` diff --git a/amd64_movq_address_alias_test.go b/amd64_movq_address_alias_test.go new file mode 100644 index 000000000..cd3187d25 --- /dev/null +++ b/amd64_movq_address_alias_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "bytes" + "testing" +) + +func TestX86MOVQAliasesRetainCompleteGoAddressRewrite(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, operand := range []string{ + "$·value(SB)", "$value<>+3(SB)", "$value<>-1(SB)", + "$slot+0(FP)", "$slot-8(SP)", "$8(BX)", "$8(BX)(CX*4)", + } { + for _, alias := range x86OriginalMOVQAliases(t) { + t.Run(arch+"/"+alias+"/"+operand, func(t *testing.T) { + source := "TEXT address(SB),4,$0-16\n" + alias + " " + operand + ",AX\nRET\n" + accepted := arch == "amd64" + goAssemblerOracleOutput(t, arch, source, "", accepted) + context := amd64Ctx{goarch: arch} + instruction := x86PureGPUnknownInstruction(t, alias+" "+operand+",AX") + if err := context.validateMOVDAliasForm(instruction); (err == nil) != accepted { + t.Fatal("source alias lost AMOVQ TYPE_ADDR progedit", err) + } + if accepted { + canonical := "TEXT address(SB),4,$0-16\nMOVQ " + operand + ",AX\nRET\n" + if !bytes.Equal(x86OrderingHintGoBytes(t, arch, source), x86OrderingHintGoBytes(t, arch, canonical)) { + t.Fatal("alias address bytes differ from original MOVQ") + } + } + }) + } + } + } +} + +func TestX86MOVQAliasAddressClassDoesNotGrantAnotherDestination(t *testing.T) { + for _, alias := range x86OriginalMOVQAliases(t) { + for _, destination := range []string{"X0", "M0", "8(BX)"} { + t.Run(alias+"/"+destination, func(t *testing.T) { + instruction := x86PureGPUnknownInstruction(t, alias+" $value<>(SB),"+destination) + context := amd64Ctx{goarch: "amd64"} + if err := context.validateMOVDAliasForm(instruction); err == nil { + t.Fatal("address source became a memory, MMX or X immediate") + } + }) + } + } +} diff --git a/amd64_movq_source_alias.go b/amd64_movq_source_alias.go new file mode 100644 index 000000000..10a91b813 --- /dev/null +++ b/amd64_movq_source_alias.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// These are the complete archX86 frontend registrations for AMOVQ. They are +// source spellings, not physical Intel opcode identities or MMX permissions. +var x86GoMOVQSourceAliases = map[Op]Op{ + "MOVD": "MOVQ", + "MOVDQ2Q": "MOVQ", +} + +func x86GoMOVQSourceInstruction(ins Instr) Instr { + if !ins.x86Encoded && ins.x86SourceRaw == nil { + if canonical, present := x86GoMOVQSourceAliases[ins.Op]; present { + ins.Op = canonical + } + } + return ins +} + +// Go progedit rewrites AMOVQ TYPE_ADDR to ALEAQ before matching ymovq. +// Source aliases must retain that complete source class, not mistake it for +// a memory read or a SIMD immediate. This validates syntax only; actual FP, +// AUTO, symbol and numerical address roles remain separate consumers. +func x86GoMOVQAddressSource(operand Operand, arch string) bool { + if arch != "386" && operand.Kind == OpFPAddr { + address := Operand{Kind: OpFP, FPOffset: operand.FPOffset} + _, err := x86LEAOperandWidth(Instr{Op: "LEAQ", Args: []Operand{address, {Kind: OpReg, Reg: AX}}}, arch) + return err == nil + } + if arch == "386" || operand.Kind != OpSym { + return false + } + source := strings.TrimSpace(operand.Sym) + if !strings.HasPrefix(source, "$") { + return false + } + source = strings.TrimSpace(strings.TrimPrefix(source, "$")) + address := Operand{Kind: OpSym, Sym: source} + if memory, matched, err := x86UnnamedSourceAddressInMode(operand, arch); matched || err != nil { + if err != nil { + return false + } + address = Operand{Kind: OpMem, Mem: memory} + } else if memory, matched, err := parseX86SymbolIndexedMemory(source); matched { + if err != nil { + return false + } + address = Operand{Kind: OpMem, Mem: memory} + } else if _, offset, valid := parseFP(source); valid { + address = Operand{Kind: OpFP, FPOffset: offset} + } else if memory, valid := parseMem(source); valid { + address = Operand{Kind: OpMem, Mem: memory} + } + _, err := x86LEAOperandWidth(Instr{Op: "LEAQ", Args: []Operand{address, {Kind: OpReg, Reg: AX}}}, arch) + return err == nil +} + +// Do not normalize the actual source: caller/raw/frame certificates bind its +// entire original Op and Raw. Consumers canonicalize only local dispatch copies. +func validateX86MoveSourceAliases(file *File, arch string) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + if arch == "" { + arch = "amd64" + } + for _, function := range file.Funcs { + for _, ins := range function.Instrs { + if _, _, err := x86ScalarMoveAliasProperties(ins.Op, ins, arch); err != nil { + return err + } + if ins.x86Encoded || ins.x86SourceRaw != nil { + continue + } + base, _, dotted := strings.Cut(string(ins.Op), ".") + if _, alias := x86GoMOVQSourceAliases[Op(base)]; !alias && base != "MOVQ" { + continue + } + if dotted || len(ins.Args) != 2 || arch != "amd64" && arch != "386" { + return fmt.Errorf("%s %s has an invalid source mode, suffix or operand count: %q", arch, ins.Op, ins.Raw) + } + for _, operand := range ins.Args { + if operand.Kind == OpImm && operand.ImmIsFloat { + return fmt.Errorf("%s %s has no original Go FLOAT-immediate row: %q", arch, ins.Op, ins.Raw) + } + } + if arch == "386" { + // These aliases retain MOVQ's five pre-Pw8 memory/MMX/X rows, + // never its mode64-only GP, immediate or MMX-register copies. + _, _, sourceMachine := x86MachineRegisterOperand(ins.Args[0]) + _, _, destinationMachine := x86MachineRegisterOperand(ins.Args[1]) + if !sourceMachine && !destinationMachine { + context := amd64Ctx{goarch: arch} + if err := context.validateMOVDAliasForm(ins); err != nil { + return err + } + } + } + } + } + return nil +} diff --git a/amd64_movq_source_alias_provenance_test.go b/amd64_movq_source_alias_provenance_test.go new file mode 100644 index 000000000..e72bca393 --- /dev/null +++ b/amd64_movq_source_alias_provenance_test.go @@ -0,0 +1,323 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86MOVQSourceAliasesCountedNumericalOrigins(t *testing.T) { + before := x86SourceStackState{callers: map[Reg]x86CallerPointerValue{ + CX: {argument: 0}, SI: {argument: 1}, + }} + proof := &x86GoCallerRegionProof{arguments: []x86CallerArgumentRegion{ + {kind: x86CallerLocalLength, width: 8}, {kind: x86CallerSliceInput, width: 4}, + }} + for _, alias := range x86OriginalMOVQAliases(t) { + for _, row := range []string{"$0,AX", "$4294967296,AX", "$-1,AX", "R9,AX", "R9,R9", "R12,AX", "(CX),AX", "(SI),AX"} { + t.Run(alias+"/"+row, func(t *testing.T) { + instruction := x86PureGPUnknownInstruction(t, alias+" "+row) + original := instruction + goAssemblerOracleOutput(t, "amd64", "TEXT original(SB),4,$0-0\n"+instruction.Raw+"\nRET\n", "", true) + canonical := instruction + canonical.Op = "MOVQ" + want := x86CallerCountedNumbers{regs: map[Reg]x86CallerCountedNumber{ + "R9": {source: 7, mask: 3, value: 2, known: true, version: 4}, + }, source: 7, sequence: 8, flags: x86CallerCountedFlags{known: true, equal: true}} + got := want.clone() + want.step(9, 2, canonical, before, proof) + got.step(9, 2, instruction, before, proof) + if !reflect.DeepEqual(got, want) || !reflect.DeepEqual(instruction, original) { + t.Fatalf("source alias changed complete numeric origin/value/version/flags or original identity: got=%+v want=%+v", got, want) + } + if row == "$4294967296,AX" && (!got.regs[AX].known || got.regs[AX].value != 1<<32) { + t.Fatal("qword alias truncated its high half") + } + if row == "R12,AX" || row == "(SI),AX" { + if _, present := got.regs[AX]; present { + t.Fatal("unknown/non-length source acquired a numerical origin") + } + } + for _, raw := range []string{"encoded", "source raw"} { + physical := instruction + if raw == "encoded" { + physical.x86Encoded = true + } else { + physical.x86SourceRaw = &x86SourceRawInstruction{op: instruction.Op, args: instruction.Args} + } + state := x86CallerCountedNumbers{regs: map[Reg]x86CallerCountedNumber{}, source: -1} + state.step(9, 2, physical, before, proof) + if len(state.regs) != 0 || state.source != -1 { + t.Fatal("raw origin borrowed named source alias semantics", raw, state) + } + } + }) + } + } +} + +func x86SourceAliasCountedAssembly(original, alias string) string { + // Exercise full-word bound copies and zero counter definitions as well as + // source length/pointer imports. Actual raw guard/latch instructions stay raw. + assembly := strings.NewReplacer("MOVL R9,AX", "MOVQ R9,AX", "MOVL R9,R13", "MOVQ R9,R13", + "XORL CX,CX", "MOVQ $0,CX", "XORL R12,R12", "MOVQ $0,R12").Replace(original) + return x86SourceMOVQSpelling(assembly, alias) +} + +func x86SourceMOVQSpelling(assembly, alias string) string { + lines := strings.Split(assembly, "\n") + for index, line := range lines { + fields := strings.Fields(line) + if len(fields) != 0 && fields[0] == "MOVQ" { + start := strings.Index(line, "MOVQ") + lines[index] = line[:start] + alias + line[start+len("MOVQ"):] + } + } + return strings.Join(lines, "\n") +} + +func TestX86MOVQSourceAliasesCountedCompletePartitionsAndObjects(t *testing.T) { + for _, alias := range append([]string{"MOVQ"}, x86OriginalMOVQAliases(t)...) { + for _, raw := range []bool{false, true} { + for _, shape := range []string{"single", "separate exit", "serial", "serial diamond"} { + t.Run(fmt.Sprintf("%s/raw=%t/%s", alias, raw, shape), func(t *testing.T) { + original := x86CallerCountedLoopScalarAssembly(raw) + iterations := [4]int{0, 1, 2, 3} + if shape == "separate exit" { + original = x86CallerCountedLoopSeparateExitAssembly(raw) + } else if strings.HasPrefix(shape, "serial") { + original = x86CallerCountedSCCAssembly(raw, shape == "serial diamond") + iterations = [4]int{0, 2, 4, 6} + if shape == "serial diamond" { + iterations = [4]int{0, 3, 6, 9} // Both feasible diamond paths replay the second SCC. + } + } + assembly := x86SourceAliasCountedAssembly(original, alias) + canonical := x86SourceAliasCountedAssembly(original, "MOVQ") + if !bytes.Equal(x86OrderingHintGoBytes(t, "amd64", assembly), x86OrderingHintGoBytes(t, "amd64", canonical)) { + t.Fatal("actual original Go source aliases differ from complete MOVQ body bytes") + } + pkg, file := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + fingerprint, err := x86CallerSourceFingerprint(file.Funcs[0]) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil || normalized.Funcs[0].x86SourceStack == nil { + t.Fatalf("complete alias counted provenance missing: %v", err) + } + proof := normalized.Funcs[0].x86SourceStack + if proof.countedIterations != iterations || proof.caller.source != fingerprint { + t.Fatal("complete partition inventory or original source identity changed", proof.countedIterations) + } + if actual, err := x86CallerSourceFingerprint(normalized.Funcs[0]); err != nil || actual != proof.normalized { + t.Fatal("normalized source identity was not sealed") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + x86CallerCountedSCCObjects(t, x86CallerCountedLoopScalarSource(), assembly, triple) + } + if after, err := x86CallerSourceFingerprint(file.Funcs[0]); err != nil || after != fingerprint { + t.Fatal("alias replay rewrote original Instr.Op/Raw") + } + for _, change := range []string{"original", "normalized", "missing caller", "access width"} { + t.Run(change, func(t *testing.T) { + _, fresh := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + copy, err := normalizeX86RawFile(fresh, "amd64", options) + if err != nil || copy.Funcs[0].x86SourceStack == nil { + t.Fatal("fresh proof missing", err) + } + if change == "original" { + fresh.Funcs[0].Instrs[1].Raw += " " + decoded, err := decodeX86RawDirectives(fresh.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + failed := normalizeX86RawReturnSourceStack(decoded, options, true, fresh.Funcs[0]) + if failed.x86SourceStack != nil { + t.Fatal("changed original Raw borrowed its alias proof") + } + return + } + if change == "missing caller" { + options.Sigs = map[string]FuncSig{} + for name, sig := range sigs { + sig.x86CallerRegions = nil + options.Sigs[name] = sig + } + defer func() { options.Sigs = sigs }() + } else { + instruction := ©.Funcs[0].Instrs[1] + if change == "normalized" { + instruction.Raw += " " + if x86RawReturnMemoryBound(copy.Funcs[0], copy, options, true) { + t.Fatal("changed normalized alias Raw retained its bound") + } + } else { + access := instruction.x86CallerMemory[0] + access.width++ + instruction.x86CallerMemory[0] = access + if x86CallerMemoryCertified(*instruction, 0) || x86RawReturnMemoryBound(copy.Funcs[0], copy, options, true) { + t.Fatal("changed alias access width retained authority") + } + return + } + } + if _, err := Translate(copy, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed sealed alias/caller source translated", change, err) + } + }) + } + }) + } + } + } +} + +func TestX86MOVQSourceAliasesCountedAllExistingRejectedPaths(t *testing.T) { + for _, alias := range x86OriginalMOVQAliases(t) { + for _, serial := range []bool{false, true} { + original, changes := x86CallerCountedLoopScalarAssembly(false), x86CallerCountedLoopRejectedChanges() + if serial { + original, changes = x86CallerCountedSCCAssembly(false, true), x86CallerCountedSCCRejectedChanges() + } + for name, change := range changes { + t.Run(fmt.Sprintf("%s/serial=%t/%s", alias, serial, name), func(t *testing.T) { + assembly := strings.Replace(original, change[0], change[1], 1) + if assembly == original { + t.Fatal("negative did not mutate the original complete source") + } + assembly = x86SourceMOVQSpelling(assembly, alias) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("alias broadened a rejected loop/partition/mutation", err) + } + if serial { + x86CallerCountedSCCRejectThreeAPIs(t, assembly) + } + }) + } + } + } +} + +func TestX86MOVQSourceAliasesOtherScalarKnowledgeConsumers(t *testing.T) { + for _, alias := range x86OriginalMOVQAliases(t) { + for _, row := range []string{"$0,AX", "$4294967296,AX", "AX,BX", "AX,AX", "R12,AX"} { + t.Run(alias+"/"+row, func(t *testing.T) { + instruction := x86PureGPUnknownInstruction(t, alias+" "+row) + original, canonical := instruction, instruction + canonical.Op = "MOVQ" + state := x86FixedLocalState{pointers: map[Reg]int64{}, opaque: map[Reg]x86FixedLocalOrigins{}, + booleans: map[Reg]uint8{AX: 0xaa}, constants: map[Reg]int64{AX: 1<<32 + 1}, known: map[Reg]uint8{AX: 0xff}} + got, want := cloneFixedLocalState(state), cloneFixedLocalState(state) + err := fixedLocalStep(&instruction, &got, FuncSig{}, nil, 0) + wantErr := fixedLocalStep(&canonical, &want, FuncSig{}, nil, 0) + if (err == nil) != (wantErr == nil) || !reflect.DeepEqual(got, want) || + fixedLocalInstructionShape(instruction) != fixedLocalInstructionShape(canonical) { + t.Errorf("fixed-local scalar numeric/boolean knowledge differs: %v / %v", err, wantErr) + } + carry := x86LockCarryState{carry: true} + carry.gp[0] = 0xff + context := amd64Ctx{goarch: "amd64"} + if got, want := context.lockCarryTransfer(instruction, carry.clone()), context.lockCarryTransfer(canonical, carry.clone()); !reflect.DeepEqual(got, want) { + t.Errorf("carry initialization differs: %+v / %+v", got, want) + } + if !reflect.DeepEqual(instruction, original) { + t.Fatal("scalar observer changed source identity") + } + }) + } + for _, row := range []string{"(AX),BX", "BX,(AX)"} { + ins := x86PureGPUnknownInstruction(t, alias+" "+row) + canonical := ins + canonical.Op = "MOVQ" + for index := 0; index < 2; index++ { + width, write, known := x86InstructionMemoryShape(ins, index, "amd64") + wantWidth, wantWrite, wantKnown := x86InstructionMemoryShape(canonical, index, "amd64") + if width != wantWidth || write != wantWrite || known != wantKnown { + t.Errorf("alias native memory-width observation differs: %s index%d", row, index) + } + } + } + } +} + +func TestX86MOVQSourceAliasesFixedLocalActualObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + var assembly, declarations strings.Builder + for index, alias := range append([]string{"MOVQ"}, x86OriginalMOVQAliases(t)...) { + fmt.Fprintf(&declarations, "func Value%d(x uint64) uint64\n", index) + fmt.Fprintf(&assembly, "TEXT ·Value%d(SB),4,$16-16\n%s x+0(FP),AX\n%s $4294967296,DX\n%s AX,0(SP)\n%s 0(SP),CX\n%s CX,ret+8(FP)\nRET\n", + index, alias, alias, alias, alias, alias) + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, file, options := fixedLocalTransportFixture(t, assembly.String(), declarations.String(), target.name, target.triple) + observed := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + fingerprints := map[string][32]byte{} + for _, function := range file.Funcs { + fingerprint, err := x86CallerSourceFingerprint(function) + if err != nil { + t.Fatal(err) + } + fingerprints[function.Sym] = fingerprint + } + bound, err := observed.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("source-proved fixed-local MOVQ aliases lost scalar knowledge", err) + } + normalized, err := normalizeX86RawFile(file, "amd64", bound) + if err != nil { + t.Fatal(err) + } + for _, function := range normalized.Funcs { + reads, writes := 0, 0 + for _, instruction := range function.Instrs { + for index, access := range instruction.x86FixedLocalMemory { + context := amd64Ctx{fixedLocal: access.owner} + if access.width != 8 || !context.fixedLocalMemoryCertified(instruction, index) { + t.Fatal("alias lost exact original eight-byte certificate", instruction.Raw) + } + if access.write { + writes++ + } else { + reads++ + } + } + } + if reads != 1 || writes != 1 { + t.Fatal("alias local memory inventory changed", function.Sym, reads, writes) + } + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil || ir == "" { + t.Fatal("bound original source alias failed public consumer", err) + } + compileLLVMToObject(t, llc, target.triple, "source-alias-local.ll", "source-alias-local.o", ir) + }) + for _, function := range file.Funcs { + if after, err := x86CallerSourceFingerprint(function); err != nil || after != fingerprints[function.Sym] { + t.Fatal("fixed-local alias consumer rewrote sealed original source") + } + } + }) + } +} diff --git a/amd64_movq_source_alias_runtime_linux_amd64_test.go b/amd64_movq_source_alias_runtime_linux_amd64_test.go new file mode 100644 index 000000000..8abe2a756 --- /dev/null +++ b/amd64_movq_source_alias_runtime_linux_amd64_test.go @@ -0,0 +1,114 @@ +//go:build linux && amd64 + +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +// Native Linux AMD64 owns execution. Other hosts run the portable source, +// original-object and LLVM-object tests; no host skip is runtime evidence. +func TestCrossLinuxRuntimeMatrixX86MOVQSourceAliasesOriginalAndLLVM(t *testing.T) { + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 native runtime tools required") + } + execution := x86FloatCompareExecution{llc: llc, triple: "x86_64-unknown-linux-gnu", + compiler: []string{clang, "-target", "x86_64-unknown-linux-gnu"}} + var source, declarations, prototypes, originals, checks strings.Builder + count := 0 + for _, alias := range append([]string{"MOVQ"}, x86OriginalMOVQAliases(t)...) { + for _, operation := range []string{"gp", "gp-x", "fp-x-clear", "native-gp", "native-x", "address"} { + name, original, core := fmt.Sprintf("F%d", count), fmt.Sprintf("Original%d", count), fmt.Sprintf("OriginalBody%d", count) + declaration := "func Value(x uint64) uint64" + body := alias + " x+0(FP),CX\n" + alias + " CX,DX\n" + alias + " DX,ret+8(FP)\nRET" + switch operation { + case "gp-x": + body = alias + " x+0(FP),CX\n" + alias + " CX,X0\n" + alias + " X0,DX\n" + alias + " DX,ret+8(FP)\nRET" + case "fp-x-clear": + body = alias + " x+0(FP),X0\n" + alias + " X0,X1\nPSRLO $8,X1\n" + alias + " X1,ret+8(FP)\nRET" + case "native-gp", "native-x": + declaration = "func Value(p *[8]byte) uint64" + register := "DX" + if operation == "native-x" { + register = "X0" + } + body = "MOVQ p+0(FP),AX\nMOVL $1,CX\n" + alias + " -1(AX)(CX*1)," + register + "\n" + alias + " " + register + ",ret+8(FP)\nRET" + case "address": + body = alias + " x+0(FP),CX\n" + alias + " $8(CX),DX\n" + alias + " DX,ret+8(FP)\nRET" + } + _, _, _, assembly := x86ResultInitializationFixture(t, "amd64", declaration, body) + encoded := x86OrderingHintGoBytes(t, "amd64", assembly) + var literal []string + for _, value := range encoded { + literal = append(literal, fmt.Sprintf("0x%02x", value)) + } + // Independently transport the SysV input into real Go ABI0 cells. + // Execute complete bytes assembled from this exact original body. + shim := ".text\n.globl " + core + "\n" + core + ":\n.byte " + strings.Join(literal, ",") + "\n" + + ".globl " + original + "\n" + original + ":\nsubq $24,%rsp\nmovq %rdi,0(%rsp)\ncallq " + core + + "\nmovq 8(%rsp),%rax\naddq $24,%rsp\nretq\n" + fmt.Fprintf(&originals, "__asm__(%q);\n", shim) + source.WriteString(strings.ReplaceAll(assembly, "·Value", "·"+name)) + declarations.WriteString(strings.Replace(declaration, "Value", name, 1) + "\n") + argument, expected := "pattern", "pattern" + parameter := "uint64_t" + if strings.HasPrefix(operation, "native-") { + parameter, argument = "const uint64_t*", "&backing[1]" + } + if operation == "fp-x-clear" { + expected = "0" + } else if operation == "address" { + expected = "pattern+UINT64_C(8)" + } + fmt.Fprintf(&prototypes, "extern uint64_t %s(%s);\nextern uint64_t %s(%s);\n", name, parameter, original, parameter) + fmt.Fprintf(&checks, "{uint64_t native=%s(%s),translated=%s(%s),expected=%s;\n"+ + "if(native!=expected || translated!=native || backing[0]!=canary || backing[1]!=pattern || backing[2]!=canary) {\n"+ + "fprintf(stderr,\"alias=%s op=%s sample=%%u native=%%llx LLVM=%%llx expected=%%llx\\n\",sample,"+ + "(unsigned long long)native,(unsigned long long)translated,(unsigned long long)expected);return 1;}}\n", + original, argument, name, argument, expected, alias, operation) + count++ + } + } + if count != 18 { + t.Fatal("complete original source-alias runtime inventory changed", count) + } + driver := `#include +#include +` + originals.String() + prototypes.String() + ` +int main(void) { + const uint64_t patterns[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000), + UINT64_C(0xffffffff00000000),UINT64_C(0x0123456789abcdef),UINT64_C(0xfedcba9876543210)}; + const uint64_t canary=UINT64_C(0x9876543210abcdef); + for(unsigned sample=0;sample<7;sample++) { + uint64_t pattern=patterns[sample],backing[3]={canary,pattern,canary}; +` + checks.String() + ` + } + puts("126 original Go/LLVM MOVQ source-alias cases"); + return 0; +} + +` + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, level := range []string{"-O0", "-O2"} { + executable := execution.compileLLVM(t, "source-movq-alias-"+api, ir, driver, level) + output := x86FloatCompareCaptureCommand(t, exec.Command(executable)) + if string(output) != "126 original Go/LLVM MOVQ source-alias cases\n" { + t.Fatalf("incomplete original/LLVM source alias runtime receipt: %s", output) + } + } + } +} + +func TestCrossLinuxRuntimeMatrixX86MOVQSourceAliasesCountedProvenance(t *testing.T) { + aliases := append([]string{"MOVQ"}, x86OriginalMOVQAliases(t)...) + t.Run("original Go", func(t *testing.T) { runX86CallerCountedLoopOriginalValues(t, aliases) }) + t.Run("LLVM", func(t *testing.T) { runX86CallerCountedLoopLLVMValues(t, aliases) }) +} diff --git a/amd64_movq_source_alias_test.go b/amd64_movq_source_alias_test.go new file mode 100644 index 000000000..7a87dded7 --- /dev/null +++ b/amd64_movq_source_alias_test.go @@ -0,0 +1,396 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Read the complete actual frontend registration, not an Intel mnemonic list. +func x86OriginalMOVQAliases(t *testing.T) []string { + t.Helper() + file, err := parser.ParseFile(token.NewFileSet(), filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go"), nil, 0) + if err != nil { + t.Fatal(err) + } + var aliases []string + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if !ok || function.Name.Name != "archX86" { + continue + } + ast.Inspect(function.Body, func(node ast.Node) bool { + assignment, ok := node.(*ast.AssignStmt) + if !ok || len(assignment.Lhs) != 1 || len(assignment.Rhs) != 1 { + return true + } + index, indexed := assignment.Lhs[0].(*ast.IndexExpr) + selector, selected := assignment.Rhs[0].(*ast.SelectorExpr) + if !indexed || !selected || selector.Sel.Name != "AMOVQ" { + return true + } + bank, registered := index.X.(*ast.Ident) + owner, original := selector.X.(*ast.Ident) + literal, named := index.Index.(*ast.BasicLit) + if !registered || bank.Name != "instructions" || !original || owner.Name != "x86" || !named || literal.Kind != token.STRING { + t.Fatal("unknown original MOVQ alias registration") + } + name, err := strconv.Unquote(literal.Value) + if err != nil { + t.Fatal(err) + } + aliases = append(aliases, name) + return true + }) + } + if !reflect.DeepEqual(aliases, []string{"MOVD", "MOVDQ2Q"}) { + t.Fatalf("complete original MOVQ aliases changed: %v", aliases) + } + return aliases +} + +func TestX86MOVQSourceAliasesCompleteOriginalRegistrationAndTLS(t *testing.T) { + if !reflect.DeepEqual(x86GoMOVQSourceAliases, map[Op]Op{"MOVD": "MOVQ", "MOVDQ2Q": "MOVQ"}) { + t.Fatal("source alias normalization differs from the complete original frontend") + } + for _, alias := range x86OriginalMOVQAliases(t) { + for _, arch := range []string{"386", "amd64"} { + for _, operands := range []string{"TLS,AX", "TLS,R15", "8(TLS),X0", "X7,0(CX)(TLS*4)", "8(TLS),M0"} { + t.Run(arch+"/"+alias+"/"+operands, func(t *testing.T) { + ins := x86PureGPUnknownInstruction(t, alias+" "+operands) + before := ins + if err := x86SourceTLSMoveError(ins, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("complete actual MOVQ alias lost its source TLS boundary: %v", err) + } + if !reflect.DeepEqual(ins, before) { + t.Fatal("source alias validator changed original source") + } + }) + } + } + } +} + +func TestX86MOVQSourceAliasesOriginalFormsAndByteParity(t *testing.T) { + // Every ymovq row, both overlapping register/memory axes and all immediate + // classes are represented. The full historical/current table is pinned by + // TestX86GPSIMDTransferCompleteLegacyOriginalTables independently. + for _, arch := range []string{"amd64", "386"} { + for _, operands := range []string{ + "8(BX),M0", "M0,8(BX)", "X0,M0", "8(BX),X0", "X0,X1", "X0,8(BX)", + "AX,BX", "AX,8(BX)", "8(BX),AX", "$0,AX", "$-1,AX", "$4294967296,AX", "$1,8(BX)", + "AX,M0", "M0,M1", "M0,AX", "AX,X0", "X0,AX", "$slot+0(FP),AX", "$slot+0(SP),AX", + } { + accepted := arch == "amd64" || operands == "8(BX),M0" || operands == "M0,8(BX)" || + operands == "X0,M0" || operands == "8(BX),X0" || operands == "X0,X1" || operands == "X0,8(BX)" + for _, alias := range append([]string{"MOVQ"}, x86OriginalMOVQAliases(t)...) { + t.Run(arch+"/"+alias+"/"+operands, func(t *testing.T) { + source := "TEXT native(SB),4,$0-16\n" + alias + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", accepted) + if accepted { + canonical := strings.Replace(source, alias+" ", "MOVQ ", 1) + first, second := x86OrderingHintGoBytes(t, arch, source), x86OrderingHintGoBytes(t, arch, canonical) + if !bytes.Equal(first, second) { + t.Fatalf("original Go alias bytes differ from MOVQ: %x != %x", first, second) + } + } + }) + } + } + } +} + +func TestX86MOVQSourceAliasesThreeAPIFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + for _, alias := range x86OriginalMOVQAliases(t) { + bodies := []string{alias + " x+0(FP),X0\n" + alias + " X0,X1\n" + alias + " X1,ret+8(FP)\nRET"} + if target.arch == "amd64" { + bodies = append(bodies, + alias+" x+0(FP),CX\n"+alias+" CX,DX\n"+alias+" DX,ret+8(FP)\nRET", + alias+" x+0(FP),CX\n"+alias+" CX,X0\n"+alias+" X0,DX\n"+alias+" DX,ret+8(FP)\nRET") + } + for _, body := range bodies { + t.Run(target.triple+"/"+alias+"/"+body, func(t *testing.T) { + _, _, _, source := x86ResultInitializationFixture(t, target.arch, "func Value(x uint64) uint64", body) + goAssemblerOracleOutput(t, target.arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(x uint64) uint64\n", target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatalf("%s original alias translation: %v", api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, api) + } + }) + } + } + } +} + +func TestX86MOVQSourceAliasesMachineRegisterRows(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + for _, alias := range x86OriginalMOVQAliases(t) { + t.Run(target.triple+"/"+alias, func(t *testing.T) { + var body strings.Builder + for _, machine := range []string{"CR0", "CR2", "CR3", "CR4", "DR0", "DR2", "DR3", "DR6", "DR7"} { + fmt.Fprintf(&body, "%s %s,AX\n%s AX,%s\n", alias, machine, alias, machine) + } + if target.arch == "amd64" { + fmt.Fprintf(&body, "%s CR8,AX\n%s AX,CR8\n", alias, alias) + } + for _, machine := range []string{"GDTR", "IDTR"} { + fmt.Fprintf(&body, "%s 0(BX),%s\n%s %s,16(BX)\n", alias, machine, alias, machine) + } + source := "TEXT machine(SB),4,$0-0\n" + body.String() + "RET\n" + canonical := strings.ReplaceAll(source, alias+" ", "MOVQ ") + if !bytes.Equal(x86OrderingHintGoBytes(t, target.arch, source), x86OrderingHintGoBytes(t, target.arch, canonical)) { + t.Fatal("complete machine-register alias bytes changed") + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func machine()\n", target.arch, target.triple, api) + if err != nil { + t.Fatal("source aliases lost actual Go movtab rows", api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "machine-"+api) + } + }) + } + } +} + +func TestX86MOVQSourceAliasesCompleteOriginalMovtab(t *testing.T) { + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + var actual, expected []string + for _, row := range regexp.MustCompile(`\{AMOVQ,\s*Y\w+,\s*Ynone,[^\n]+`).FindAllString(string(source), -1) { + actual = append(actual, strings.Join(strings.Fields(row), "")) + } + for _, bank := range []struct{ class, opcode string }{{"cr", "20"}, {"dr", "21"}} { + indices := []int{0, 2, 3, 4, 8} + if bank.class == "dr" { + indices = []int{0, 2, 3, 6, 7} + } + for _, direction := range []bool{false, true} { + for _, index := range indices { + from, to, zcase, opcode := fmt.Sprintf("Y%s%d", bank.class, index), "Yrl", "movRegMem2op", bank.opcode + if direction { + from, to, zcase = to, from, "movMemReg2op" + if bank.class == "cr" { + opcode = "22" + } else { + opcode = "23" + } + } + expected = append(expected, fmt.Sprintf("{AMOVQ,%s,Ynone,%s,%s,[4]uint8{0x0f,0x%s,%d,0}},", from, to, zcase, opcode, index)) + } + } + } + for _, class := range []string{"Ygdtr", "Yidtr"} { + digit := 2 + if class == "Yidtr" { + digit = 3 + } + expected = append(expected, + fmt.Sprintf("{AMOVQ,Ym,Ynone,%s,movMemReg2op,[4]uint8{0x0f,0x01,%d,0}},", class, digit), + fmt.Sprintf("{AMOVQ,%s,Ynone,Ym,movRegMem2op,[4]uint8{0x0f,0x01,%d,0}},", class, digit-2)) + } + expected = append(expected, "{AMOVQ,Ytls,Ynone,Yrl,movTLSReg,[4]uint8{0,0,0,0}},") + if len(expected) != 25 || !reflect.DeepEqual(actual, expected) { + t.Fatalf("complete original MOVQ movtab changed: %v", actual) + } +} + +func TestX86MOVQSourceAliasesRejectedFormsThreeAPIs(t *testing.T) { + for _, alias := range append([]string{"MOVQ"}, x86OriginalMOVQAliases(t)...) { + for _, arch := range []string{"amd64", "386"} { + for _, operands := range []string{"TLS,X0", "AX,TLS", "$1.5,X0", "$1.5,AX", "M0,X0", "X0,Y0", "X16,X0", "AX,X0,X1"} { + source := "TEXT Value(SB),4,$0-0\n" + alias + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + for _, api := range []string{"text", "module", "Go"} { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value()\n", arch, triple, api) + if err == nil || errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("%s invalid original alias became Context/success: %s %s: %v", api, alias, operands, err) + } + } + } + } + } +} + +func TestX86MOVQSourceAliasesNumericalEffectsEveryInputByte(t *testing.T) { + for _, alias := range x86OriginalMOVQAliases(t) { + for _, arch := range []string{"amd64", "386"} { + for _, operands := range []string{"AX,BX", "AX,X1", "X0,BX", "X0,X1", "X0,M1"} { + ins := x86PureGPUnknownInstruction(t, alias+" "+operands) + canonical := ins + canonical.Op = "MOVQ" + for missing := -1; missing < 16; missing++ { + for _, consumer := range x86ResultGPSIMDConsumers { + state := x86ResultState{flags: x86ByteFlagsAll} + state.gp[0], state.xmm[0], state.xmm[1] = 0xff, 0xffff, 0x5151 + if missing >= 0 { + state.gp[0] &^= uint8(1) << uint(missing) + state.xmm[0] &^= uint16(1) << uint(missing) + } + want := state.clone() + accepted := consumer.step(ins, &state, nil, 0, arch) + if accepted != consumer.step(canonical, &want, nil, 0, arch) || !reflect.DeepEqual(state, want) { + t.Fatalf("%s numerical alias byte/state mismatch: %s %s arch=%s missing=%d", consumer.name, alias, operands, arch, missing) + } + if state.flags != x86ByteFlagsAll { + t.Fatal("source MOVQ alias changed status knowledge") + } + } + } + } + } + } +} + +func TestX86MOVQSourceAliasesScalarFPWidthsAndOriginalCallerIdentity(t *testing.T) { + for _, alias := range x86OriginalMOVQAliases(t) { + for _, row := range []string{alias + " x+0(FP),AX", alias + " AX,ret+8(FP)"} { + ins := x86PureGPUnknownInstruction(t, row) + index := 0 + if ins.Args[1].Kind == OpFP { + index = 1 + } + if width, write, known := X86FPAccessBytes(ins, index, "amd64"); !known || width != 8 || write != (index == 1) { + t.Errorf("original alias FP width lost: %s: %d %v %v", row, width, write, known) + } + if width := x86ScalarMoveMemoryBytes(ins, index, "amd64"); width != 8 { + t.Errorf("original source-stack width lost: %s: %d", row, width) + } + } + body := alias + " DI,R11\n" + alias + " R11,DI\nMOVL $7,AX\nMOVL AX,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX\nMOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + before, err := x86CallerSourceFingerprint(file.Funcs[0]) + if err != nil { + t.Fatal(err) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Errorf("actual source alias lost complete caller/pointer identity: %s: %v", alias, err) + } + after, err := x86CallerSourceFingerprint(file.Funcs[0]) + if err != nil || before != after { + t.Fatal("alias consumer changed the original sealed caller source") + } + } +} + +func TestX86MOVQSourceAliasesNativeLoadsAndMode32Rejections(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + for _, alias := range x86OriginalMOVQAliases(t) { + for _, destination := range []string{"X0", "DX"} { + if target.arch == "386" && destination == "DX" { + continue + } + t.Run(target.triple+"/"+alias+"/"+destination, func(t *testing.T) { + move, result := "MOVQ", 8 + if target.arch == "386" { + move, result = "MOVL", 4 + } + body := fmt.Sprintf("%s p+0(FP),AX\n%s -1(AX)(CX*1),%s\n%s %s,ret+%d(FP)\nRET", move, alias, destination, alias, destination, result) + body = strings.Replace(body, alias+" -1", "MOVL $1,CX\n"+alias+" -1", 1) + _, _, _, source := x86ResultInitializationFixture(t, target.arch, "func Value(p *[8]byte) uint64", body) + goAssemblerOracleOutput(t, target.arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[8]byte) uint64\n", target.arch, target.triple, api) + if err != nil || !strings.Contains(ir, "load volatile i64") { + t.Fatalf("%s alias lost its exact eight-byte native read: %v\n%s", api, err, ir) + } + x86ResultInitializationObject(t, llc, target.triple, ir, api) + } + }) + } + } + } + for _, alias := range x86OriginalMOVQAliases(t) { + for _, operands := range []string{"AX,BX", "(AX),BX", "AX,(BX)", "AX,X0", "X0,AX", "$1,AX", "M0,M1"} { + source := "TEXT Value(SB),4,$0-0\n" + alias + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", false) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value()\n", "386", "i386-unknown-linux-gnu", api) + if err == nil || errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("%s mode32 alias admitted an actual rejected MOVQ row: %s %s: %v", api, alias, operands, err) + } + } + } + } +} + +func TestX86MOVQSourceAliasesRetainRawIntelAndUnknownState(t *testing.T) { + for _, body := range []string{ + "MOVDQ2Q CX,X0\nMOVQ X0,ret+0(FP)\nRET", + "MOVB $1,CX\nMOVDQ2Q CX,X0\nMOVQ X0,ret+0(FP)\nRET", + "MOVQ $1,CX\nMOVQ CX,X0\nBYTE $0xf2\nBYTE $0x0f\nBYTE $0xd6\nBYTE $0xc8\nMOVQ M1,ret+0(FP)\nRET", + } { + _, _, _, source := x86ResultInitializationFixture(t, "amd64", "func Value() uint64", body) + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value() uint64\n", "amd64", "x86_64-unknown-linux-gnu", api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Errorf("%s source alias acquired initialized/native MMX state: body=%q: %v", api, body, err) + } + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nMOVDQ2Q AX,BX\nBYTE $0xf2\nBYTE $0x0f\nBYTE $0xd6\nBYTE $0xc8\nRET\n") + if err != nil { + t.Fatal(err) + } + normalized, err := NormalizeRawFileForTranslation(file, "amd64") + if err != nil { + t.Fatal(err) + } + raw, err := x86asm.Decode([]byte{0xf2, 0x0f, 0xd6, 0xc8}, 64) + if err != nil || raw.Op != x86asm.MOVDQ2Q { + t.Fatal("independent literal bytes lost physical MOVDQ2Q") + } + physical := normalized.Funcs[0].Instrs[2] + if normalized.Funcs[0].Instrs[1].Op != "MOVDQ2Q" || normalized.Funcs[0].Instrs[1].Raw != "MOVDQ2Q AX,BX" || + physical.Op != "MOVDQ2Q" || physical.x86SourceRaw == nil || physical.Args[1].Reg != "M1" || file.Funcs[0].Instrs[1].Op != "MOVDQ2Q" { + t.Fatalf("named Go alias changed its original spelling or raw Intel physical role: %+v", normalized.Funcs[0].Instrs) + } + for _, alias := range x86OriginalMOVQAliases(t) { + ins := x86PureGPUnknownInstruction(t, alias+" X0,M1") + state := x86ResultState{} + state.xmm[0] = 0xffff + for _, consumer := range x86ResultGPSIMDConsumers { + if consumer.step(ins, &state, nil, 0, "amd64") { + t.Fatal("numerical alias observer manufactured native MMX state", alias, consumer.name) + } + } + } +} diff --git a/amd64_mul_div_round_helpers.go b/amd64_mul_div_round_helpers.go new file mode 100644 index 000000000..e85b6c492 --- /dev/null +++ b/amd64_mul_div_round_helpers.go @@ -0,0 +1,260 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Fixed rounding with SAE must not perform ambient floating arithmetic: LLVM +// may lower constrained operations to libcalls, and even a corrected result +// cannot undo a hardware exception. All arithmetic below is integer-only. +// A finite product is exact in 106 bits. Division generates p+3 quotient bits +// by bounded restoring division and jams the remaining exact remainder. +// The private ABI carries raw i32/i64 bits, never float/double. Otherwise an +// un-inlined mode32 C return would use x87 and could change exception state. +func emitX86FloatMulDivRoundingHelper(b *strings.Builder, bits int) { + fractionBits, bias := 23, 127 + if bits == 64 { + fractionBits, bias = 52, 1023 + } + typ := amd64FMA3LLVMType(1, bits) + infBits := uint64(2*bias+1) << fractionBits + signBit := uint64(1) << (bits - 1) + quietBit := uint64(1) << (fractionBits - 1) + + fmt.Fprintf(b, "define internal i%d @__plan9asm_mul_div_round_f%d(i%d %%left, i%d %%right, i1 %%divide, i32 %%mode, i32 %%mxcsr) {\nentry:\n", bits, bits, bits, bits) + e := amd64FMA3IntegerEmitter{b: b} + daz := e.value("and i32 %%mxcsr, 64") + daz = e.value("icmp ne i32 %s, 0", daz) + leftValue := e.value("bitcast i%d %%left to %s", bits, typ) + rightValue := e.value("bitcast i%d %%right to %s", bits, typ) + left, right := e.input(bits, leftValue, daz), e.input(bits, rightValue, daz) + + // NaNs retain source priority and payload; invalid non-NaN operations + // produce the architecture's negative quiet indefinite value. + a := e.value("bitcast %s %s to i%d", typ, left.floating, bits) + bb := e.value("bitcast %s %s to i%d", typ, right.floating, bits) + aAbs := e.value("and i%d %s, %d", bits, a, signBit-1) + bAbs := e.value("and i%d %s, %d", bits, bb, signBit-1) + aNaN := e.value("icmp ugt i%d %s, %d", bits, aAbs, infBits) + bNaN := e.value("icmp ugt i%d %s, %d", bits, bAbs, infBits) + anyNaN := e.value("or i1 %s, %s", aNaN, bNaN) + selectedNaN := e.value("select i1 %s, i%d %s, i%d %s", aNaN, bits, a, bits, bb) + selectedNaN = e.value("or i%d %s, %d", bits, selectedNaN, quietBit) + aInf := e.value("icmp eq i%d %s, %d", bits, aAbs, infBits) + bInf := e.value("icmp eq i%d %s, %d", bits, bAbs, infBits) + aZero := e.value("icmp eq i128 %s, 0", left.magnitude) + bZero := e.value("icmp eq i128 %s, 0", right.magnitude) + negative := e.value("xor i1 %s, %s", left.negative, right.negative) + sign := e.value("select i1 %s, i%d %d, i%d 0", negative, bits, signBit, bits) + signedInf := e.value("or i%d %s, %d", bits, sign, infBits) + invalidMulA := e.value("and i1 %s, %s", aInf, bZero) + invalidMulB := e.value("and i1 %s, %s", bInf, aZero) + invalidMul := e.value("or i1 %s, %s", invalidMulA, invalidMulB) + invalidDivA := e.value("and i1 %s, %s", aInf, bInf) + invalidDivB := e.value("and i1 %s, %s", aZero, bZero) + invalidDiv := e.value("or i1 %s, %s", invalidDivA, invalidDivB) + invalid := e.value("select i1 %%divide, i1 %s, i1 %s", invalidDiv, invalidMul) + anyInf := e.value("or i1 %s, %s", aInf, bInf) + anyZero := e.value("or i1 %s, %s", aZero, bZero) + divInf := e.value("or i1 %s, %s", aInf, bZero) + resultInf := e.value("select i1 %%divide, i1 %s, i1 %s", divInf, anyInf) + special := e.value("or i1 %s, %s", anyInf, anyZero) + special = e.value("or i1 %s, %s", special, anyNaN) + encoded := e.value("select i1 %s, i%d %s, i%d %s", resultInf, bits, signedInf, bits, sign) + encoded = e.value("select i1 %s, i%d %d, i%d %s", invalid, bits, signBit|infBits|quietBit, bits, encoded) + encoded = e.value("select i1 %s, i%d %s, i%d %s", anyNaN, bits, selectedNaN, bits, encoded) + fmt.Fprintf(b, " br i1 %s, label %%special, label %%finite\nspecial:\n", special) + fmt.Fprintf(b, " ret i%d %s\nfinite:\n br i1 %%divide, label %%division_prepare, label %%multiply\nmultiply:\n", bits, encoded) + product := e.value("mul i128 %s, %s", left.magnitude, right.magnitude) + productUnit := e.value("add i32 %s, %s", left.exponent, right.exponent) + + // Normalize both finite nonzero significands to the same leading bit. + b.WriteString(" br label %round\ndivision_prepare:\n") + aLeading := e.value("call i128 @llvm.ctlz.i128(i128 %s, i1 false)", left.magnitude) + bLeading := e.value("call i128 @llvm.ctlz.i128(i128 %s, i1 false)", right.magnitude) + aLeading = e.value("trunc i128 %s to i32", aLeading) + bLeading = e.value("trunc i128 %s to i32", bLeading) + aHighest := e.value("sub i32 127, %s", aLeading) + bHighest := e.value("sub i32 127, %s", bLeading) + aShift := e.value("sub i32 %d, %s", fractionBits, aHighest) + bShift := e.value("sub i32 %d, %s", fractionBits, bHighest) + aShift = e.value("zext i32 %s to i128", aShift) + bShift = e.value("zext i32 %s to i128", bShift) + numerator := e.value("shl i128 %s, %s", left.magnitude, aShift) + denominator := e.value("shl i128 %s, %s", right.magnitude, bShift) + aTop := e.value("add i32 %s, %s", left.exponent, aHighest) + bTop := e.value("add i32 %s, %s", right.exponent, bHighest) + ratioExponent := e.value("sub i32 %s, %s", aTop, bTop) + belowOne := e.value("icmp ult i128 %s, %s", numerator, denominator) + doubled := e.value("shl i128 %s, 1", numerator) + numerator = e.value("select i1 %s, i128 %s, i128 %s", belowOne, doubled, numerator) + exponentCorrection := e.value("zext i1 %s to i32", belowOne) + ratioExponent = e.value("sub i32 %s, %s", ratioExponent, exponentCorrection) + quotientUnit := e.value("sub i32 %s, %d", ratioExponent, fractionBits+3) + initialRemainder := e.value("sub i128 %s, %s", numerator, denominator) + + // numerator is in [denominator, 2*denominator). Each iteration keeps + // remainder in [0, denominator), emitting exactly one quotient bit. + b.WriteString(" br label %division_loop\ndivision_loop:\n") + b.WriteString(" %quotient = phi i128 [ 1, %division_prepare ], [ %next_quotient, %division_loop ]\n") + fmt.Fprintf(b, " %%remainder = phi i128 [ %s, %%division_prepare ], [ %%next_remainder, %%division_loop ]\n", initialRemainder) + b.WriteString(" %iteration = phi i32 [ 0, %division_prepare ], [ %next_iteration, %division_loop ]\n") + doubledRemainder := e.value("shl i128 %%remainder, 1") + quotientBit := e.value("icmp uge i128 %s, %s", doubledRemainder, denominator) + subtracted := e.value("sub i128 %s, %s", doubledRemainder, denominator) + fmt.Fprintf(b, " %%next_remainder = select i1 %s, i128 %s, i128 %s\n", quotientBit, subtracted, doubledRemainder) + shiftedQuotient := e.value("shl i128 %%quotient, 1") + bit128 := e.value("zext i1 %s to i128", quotientBit) + fmt.Fprintf(b, " %%next_quotient = or i128 %s, %s\n", shiftedQuotient, bit128) + b.WriteString(" %next_iteration = add i32 %iteration, 1\n") + done := e.value("icmp eq i32 %%next_iteration, %d", fractionBits+3) + fmt.Fprintf(b, " br i1 %s, label %%division_done, label %%division_loop\ndivision_done:\n", done) + sticky := e.value("icmp ne i128 %%next_remainder, 0") + sticky = e.value("zext i1 %s to i128", sticky) + jammed := e.value("or i128 %%next_quotient, %s", sticky) + + b.WriteString(" br label %round\nround:\n") + magnitude := e.value("phi i128 [ %s, %%multiply ], [ %s, %%division_done ]", product, jammed) + unit := e.value("phi i32 [ %s, %%multiply ], [ %s, %%division_done ]", productUnit, quotientUnit) + emitX86FloatMulDivRoundPack(&e, bits, magnitude, unit, negative) +} + +// magnitude is nonzero. Multiplication retains its exact integer significand; +// division retains three guard bits plus jammed remainder. Clamp every shift +// before emission, including inactive select arms, so no LLVM poison is used. +func emitX86FloatMulDivRoundPack(e *amd64FMA3IntegerEmitter, bits int, magnitude, unit, negative string) { + fractionBits, bias := 23, 127 + if bits == 64 { + fractionBits, bias = 52, 1023 + } + + leading := e.value("call i128 @llvm.ctlz.i128(i128 %s, i1 false)", magnitude) + leading = e.value("trunc i128 %s to i32", leading) + highest := e.value("sub i32 127, %s", leading) + normalShift := e.value("sub i32 %s, %d", highest, fractionBits) + subnormalShift := e.value("sub i32 %d, %s", 1-bias-fractionBits, unit) + isSubnormal := e.value("icmp sgt i32 %s, %s", subnormalShift, normalShift) + shift := e.value("select i1 %s, i32 %s, i32 %s", isSubnormal, subnormalShift, normalShift) + positiveShift := e.value("icmp sgt i32 %s, 0", shift) + far := e.value("icmp sge i32 %s, 128", shift) + safeRight := e.value("select i1 %s, i32 %s, i32 0", positiveShift, shift) + safeRight = e.value("select i1 %s, i32 127, i32 %s", far, safeRight) + safeRight = e.value("zext i32 %s to i128", safeRight) + right := e.value("lshr i128 %s, %s", magnitude, safeRight) + right = e.value("select i1 %s, i128 0, i128 %s", far, right) + leftCount := e.value("sub i32 0, %s", shift) + leftCount = e.value("select i1 %s, i32 0, i32 %s", positiveShift, leftCount) + leftFar := e.value("icmp ugt i32 %s, 127", leftCount) + leftCount = e.value("select i1 %s, i32 127, i32 %s", leftFar, leftCount) + leftCount = e.value("zext i32 %s to i128", leftCount) + left := e.value("shl i128 %s, %s", magnitude, leftCount) + unrounded := e.value("select i1 %s, i128 %s, i128 %s", positiveShift, right, left) + + mask := e.value("shl i128 1, %s", safeRight) + mask = e.value("sub i128 %s, 1", mask) + remainder := e.value("and i128 %s, %s", magnitude, mask) + remainder = e.value("select i1 %s, i128 %s, i128 %s", far, magnitude, remainder) + inexact := e.value("icmp ne i128 %s, 0", remainder) + halfCount := e.value("sub i32 %s, 1", shift) + halfCount = e.value("select i1 %s, i32 %s, i32 0", positiveShift, halfCount) + halfFar := e.value("icmp ugt i32 %s, 127", halfCount) + halfCount = e.value("select i1 %s, i32 127, i32 %s", halfFar, halfCount) + halfCount = e.value("zext i32 %s to i128", halfCount) + half := e.value("shl i128 1, %s", halfCount) + above := e.value("icmp ugt i128 %s, %s", remainder, half) + tied := e.value("icmp eq i128 %s, %s", remainder, half) + odd := e.value("and i128 %s, 1", unrounded) + odd = e.value("icmp ne i128 %s, 0", odd) + tied = e.value("and i1 %s, %s", tied, odd) + nearest := e.value("or i1 %s, %s", above, tied) + validHalf := e.value("xor i1 %s, true", halfFar) + validHalf = e.value("and i1 %s, %s", validHalf, positiveShift) + nearest = e.value("and i1 %s, %s", nearest, validHalf) + downward := e.value("icmp eq i32 %%mode, 1") + upward := e.value("icmp eq i32 %%mode, 2") + positive := e.value("xor i1 %s, true", negative) + directed := e.value("and i1 %s, %s", downward, negative) + upwardPositive := e.value("and i1 %s, %s", upward, positive) + directed = e.value("or i1 %s, %s", directed, upwardPositive) + awayFromZero := directed + directed = e.value("and i1 %s, %s", directed, inexact) + isNearest := e.value("icmp eq i32 %%mode, 0") + increment := e.value("select i1 %s, i1 %s, i1 %s", isNearest, nearest, directed) + increment = e.value("zext i1 %s to i128", increment) + rounded := e.value("add i128 %s, %s", unrounded, increment) + + carry := e.value("icmp uge i128 %s, %d", rounded, uint64(1)<<(fractionBits+1)) + carried := e.value("lshr i128 %s, 1", rounded) + rounded = e.value("select i1 %s, i128 %s, i128 %s", carry, carried, rounded) + carry32 := e.value("zext i1 %s to i32", carry) + exponent := e.value("add i32 %s, %s", unit, shift) + exponent = e.value("add i32 %s, %d", exponent, fractionBits+bias) + exponent = e.value("add i32 %s, %s", exponent, carry32) + stillSubnormal := e.value("icmp ult i128 %s, %d", rounded, uint64(1)<", llvm.TargetMachine{}, options); err != nil { + t.Fatal(err) + } + return module.String() +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivIntegerRoundingExtremes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + ir := x86MulDivRoundHelperFixtureIR() + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2-%t", optimized), func(t *testing.T) { + source := ir + if optimized { + source = x86MulDivRoundOptimize(t, source) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "integer-mul-div-round", triple, source, x86MulDivRoundSSEOracle, prefix) + }) + } +} + +func TestX86FloatMulDivIntegerRoundingOriginalSSEObjects(t *testing.T) { + for _, row := range []struct { + opcode string + prefix byte + code byte + }{ + {"MULSS", 0xf3, 0x59}, {"MULSD", 0xf2, 0x59}, + {"DIVSS", 0xf3, 0x5e}, {"DIVSD", 0xf2, 0x5e}, + } { + listing := x86InsertExtractGoListing(t, "amd64", "TEXT f(SB),4,$0-0\n"+row.opcode+" X1,X0\nRET\n") + var actual []byte + for _, line := range strings.Split(string(listing), "\n") { + actual = append(actual, x86InsertExtractListedBytes(line)...) + } + want := []byte{row.prefix, 0x0f, row.code, 0xc1, 0xc3} + if !bytes.Equal(actual, want) { + t.Fatalf("actual Go %s object %x, want independent native SSE oracle %x", row.opcode, actual, want) + } + } +} + +func TestX86FloatMulDivIntegerRoundingOnlyIntegerArithmeticAndBoundedDivision(t *testing.T) { + for _, optimized := range []bool{false, true} { + ir := x86MulDivRoundHelperFixtureIR() + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + for _, forbidden := range []string{ + `\b(fadd|fsub|fmul|fdiv|frem|fcmp|fptosi|fptoui|sitofp|uitofp)\b`, + `\b(udiv|sdiv|urem|srem)\b`, `@llvm\.(fma|experimental\.constrained)`, + } { + if regexp.MustCompile(forbidden).MatchString(ir) { + t.Fatalf("O2=%t: fixed-round SAE helper contains forbidden arithmetic %s", optimized, forbidden) + } + } + if !optimized { + for _, limit := range []string{"icmp eq i32 %next_iteration, 26", "icmp eq i32 %next_iteration, 55"} { + if !strings.Contains(ir, limit) { + t.Fatalf("missing exact restoring-division bound %s", limit) + } + } + } + } +} + +func TestX86FloatMulDivIntegerRoundingFiveLLVM22Targets(t *testing.T) { + llc, nm := findLLVM22Tool("llc"), findLLVM22Tool("llvm-nm") + if llc == "" || nm == "" { + t.Fatal("LLVM22 llc/llvm-nm required") + } + for _, optimized := range []bool{false, true} { + ir := x86MulDivRoundHelperFixtureIR() + backendLevel := "-O0" + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + backendLevel = "-O2" + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + "i386-unknown-linux-gnu", "i686-pc-windows-msvc", + } { + t.Run(fmt.Sprintf("O2-%t/%s", optimized, triple), func(t *testing.T) { + dir := t.TempDir() + source, object := filepath.Join(dir, "integer-round.ll"), filepath.Join(dir, "integer-round.o") + assembly := filepath.Join(dir, "integer-round.s") + if err := os.WriteFile(source, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + out, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, llc, backendLevel, "-mtriple="+triple, "-filetype=obj", source, "-o", object)) + cancel() + if err != nil { + t.Fatalf("LLVM22 object: %v\n%s", err, out) + } + ctx, cancel = runtimeOracleStageContext(t) + out, err = runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, nm, "--undefined-only", object)) + cancel() + if err != nil { + t.Fatalf("LLVM22 nm: %v\n%s", err, out) + } + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + // COFF's CRT floating-point marker has an extra C-name + // prefix in mode32. It is not an arithmetic libcall. + marker := "_fltused" + if strings.HasPrefix(triple, "i686-") { + marker = "__fltused" + } + if !strings.Contains(triple, "windows") || len(fields) != 2 || fields[0] != "U" || fields[1] != marker { + t.Fatalf("integer helper has unresolved arithmetic dependency: %s", line) + } + } + ctx, cancel = runtimeOracleStageContext(t) + out, err = runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, llc, backendLevel, "-mtriple="+triple, "-filetype=asm", source, "-o", assembly)) + cancel() + if err != nil { + t.Fatalf("LLVM22 machine assembly: %v\n%s", err, out) + } + machine, err := os.ReadFile(assembly) + if err != nil { + t.Fatal(err) + } + // Includes every helper and its actual call sites. An integer + // IR body alone does not close a float-return x87 C ABI. + floating := regexp.MustCompile(`(?m)^[\t ]+(f[a-z]+|v?(add|sub|mul|div|sqrt|comi|ucomi)(ss|sd|ps|pd))(\s|$)`) + if instruction := floating.Find(machine); instruction != nil { + t.Fatalf("SAE bit carrier emits floating instruction %q", instruction) + } + }) + } + } +} + +const x86MulDivRoundSSEOracle = `#include +#include +#include +extern uint32_t mul_div32(uint32_t,uint32_t,unsigned,unsigned,unsigned); +extern uint64_t mul_div64(uint64_t,uint64_t,unsigned,unsigned,unsigned); +static unsigned mismatches; +static uint64_t state=UINT64_C(0x372bd0ae71d94285); +static uint64_t random_bits(void){state^=state<<13;state^=state>>7;state^=state<<17;return state;} + +/* Independent physical SSE. Language arithmetic, production classification, + integer helpers and AVX512 are not used to compute expected result bits. */ +__attribute__((noinline)) static uint64_t native_sse(unsigned width,uint64_t a,uint64_t b,unsigned divide,unsigned csr){ + unsigned saved;__asm__ volatile("stmxcsr %0":"=m"(saved)::"memory"); + __asm__ volatile("ldmxcsr %0"::"m"(csr):"memory"); + uint64_t result=0; + if(width==32){ + uint32_t aa=a,bb=b,out;float left,right;memcpy(&left,&aa,4);memcpy(&right,&bb,4); + if(divide)__asm__ volatile("divss %1,%0":"+x"(left):"x"(right)); + else __asm__ volatile("mulss %1,%0":"+x"(left):"x"(right)); + memcpy(&out,&left,4);result=out; + }else{ + double left,right;memcpy(&left,&a,8);memcpy(&right,&b,8); + if(divide)__asm__ volatile("divsd %1,%0":"+x"(left):"x"(right)); + else __asm__ volatile("mulsd %1,%0":"+x"(left):"x"(right)); + memcpy(&result,&left,8); + } + __asm__ volatile("ldmxcsr %0"::"m"(saved):"memory");return result; +} + +static int check(unsigned width,uint64_t a,uint64_t b,unsigned divide,unsigned mode,unsigned ambient,unsigned control){ + unsigned actualCSR=control|(ambient<<13),referenceCSR=control|(mode<<13),saved,after; + __asm__ volatile("stmxcsr %0":"=m"(saved)::"memory"); + __asm__ volatile("ldmxcsr %0"::"m"(actualCSR):"memory"); + uint64_t want=native_sse(width,a,b,divide,referenceCSR); + uint64_t got=width==32?mul_div32(a,b,divide,mode,actualCSR):mul_div64(a,b,divide,mode,actualCSR); + __asm__ volatile("stmxcsr %0":"=m"(after)::"memory"); + __asm__ volatile("ldmxcsr %0"::"m"(saved):"memory"); + if(got!=want||after!=actualCSR){ + if(mismatches++<8)fprintf(stderr,"integer MUL/DIV width/divide/mode/ambient/control=%u/%u/%u/%u/%x a=%llx b=%llx got=%llx want=%llx CSR=%x want=%x\n",width,divide,mode,ambient,control,(unsigned long long)a,(unsigned long long)b,(unsigned long long)got,(unsigned long long)want,after,actualCSR); + return 1; + } + return 0; +} + +static int matrix(unsigned width,uint64_t a,uint64_t b){ + const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0}; + int failed=0; + for(unsigned divide=0;divide<2;divide++)for(unsigned mode=0;mode<4;mode++) + for(unsigned ambient=0;ambient<4;ambient++)for(unsigned c=0;c<6;c++) + failed|=check(width,a,b,divide,mode,ambient,controls[c]); + return failed; +} + +int main(void){ + int failed=0; + /* The first focused RED independently observes RU, not ambient RN. */ + failed|=check(32,0x3f800001,0x3f800001,0,2,0,0x1f80); + failed|=check(64,UINT64_C(0x3ff0000000000000),UINT64_C(0x4008000000000000),1,2,0,0x1f80); + /* Tiny before rounding, even though RU would produce minimum normal. */ + failed|=check(32,0x00800000,0x3f800001,1,2,0,0x9f80); + failed|=check(64,UINT64_C(0x0010000000000000),UINT64_C(0x3ff0000000000001),1,2,0,0x9f80); + const uint64_t edges32[]={0,0x80000000,1,0x80000001,0x7fffff,0x807fffff,0x800000,0x80800000, + 0x800001,0x7f7fffff,0xff7fffff,0x3f000000,0xbf000000,0x3f800000,0xbf800000,0x3f800001, + 0xbf800001,0x3f800002,0x40400000,0xc0400000,0x7f800000,0xff800000,0x7fc12345,0xffc45678, + 0x7f812345,0xff845678,0x00800000,0x40000000}; + const uint64_t edges64[]={0,UINT64_C(0x8000000000000000),1,UINT64_C(0x8000000000000001), + UINT64_C(0xfffffffffffff),UINT64_C(0x800fffffffffffff),UINT64_C(0x10000000000000),UINT64_C(0x8010000000000000), + UINT64_C(0x10000000000001),UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff), + UINT64_C(0x3fe0000000000000),UINT64_C(0xbfe0000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000), + UINT64_C(0x3ff0000000000001),UINT64_C(0xbff0000000000001),UINT64_C(0x3ff0000000000002), + UINT64_C(0x4008000000000000),UINT64_C(0xc008000000000000),UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000), + UINT64_C(0x7ff8123456789abc),UINT64_C(0xfff8fedcba987654),UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff0fedcba987654), + UINT64_C(0x10000000000000),UINT64_C(0x4000000000000000)}; + for(unsigned i=0;i(SB)", "0(SP)"} { + ins := x86ScalarLaneInstruction(t, op+" "+operand) + state := x86ResultState{flags: x86ByteFlagsAll, direction: x86ResultDirectionForward} + state.gp[7], state.pointerGP[7] = 0xff, true + if x86ResultTransfer(ins, &state, nil, 0, arch) { + t.Fatal("an m32 descriptor granted pointee, private data or physical stack bytes", arch, op, operand) + } + } + ins := x86ScalarLaneInstruction(t, op+" result+0(FP)") + for missing := -1; missing < 4; missing++ { + state := x86ResultState{fp: []bool{true, true, true, true}, flags: x86ByteFlagsAll, direction: x86ResultDirectionForward} + if missing >= 0 { + state.fp[missing] = false + } + write := strings.Contains(op, "STMXCSR") + if got := x86ResultTransfer(ins, &state, []bool{true, true, true, true}, 0, arch); got != (write || missing < 0) { + t.Fatal("MXCSR load skipped an input byte or store read its old bytes", arch, op, missing, got) + } + if state.flags != x86ByteFlagsAll || state.direction != x86ResultDirectionForward { + t.Fatal("MXCSR changed integer status or direction knowledge") + } + } + } + } +} diff --git a/amd64_mxcsr_native_pointer_test.go b/amd64_mxcsr_native_pointer_test.go new file mode 100644 index 000000000..81d413c6f --- /dev/null +++ b/amd64_mxcsr_native_pointer_test.go @@ -0,0 +1,153 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86NativeMXCSRPointerFamilyFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + word, move := 8, "MOVQ" + if target.arch == "386" { + word, move = 4, "MOVL" + } + var source, declarations strings.Builder + for _, op := range []string{"LDMXCSR", "VLDMXCSR", "STMXCSR", "VSTMXCSR"} { + for _, result := range []bool{false, true} { + name, args := fmt.Sprintf("%s%v", op, result), word + declaration, finish := "", "" + if result { + declaration, args = " uint32", word+4 + finish = fmt.Sprintf("MOVL $7,ret+%d(FP)\n", word) + } + fmt.Fprintf(&declarations, "func %s(p *[8]byte)%s\n", name, declaration) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),AX\n%s 1(AX)\n%sRET\n", + name, args, move, op, finish) + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("complete original native m32 control operations lost their paired emitter", api, err) + } + for _, op := range []string{"ldmxcsr", "vldmxcsr", "stmxcsr", "vstmxcsr"} { + if strings.Count(ir, "asm sideeffect \""+op+" $0\"") != 2 { + t.Fatal("original native opcode/fault identity was not retained", api, op) + } + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + } + }) + } +} + +func TestX86NativeMXCSRPointerFamilyRetainsCompleteInputObligation(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + move, triple, word := "MOVL", "i386-unknown-linux-gnu", 4 + if arch == "amd64" { + move, triple, word = "MOVQ", "x86_64-unknown-linux-gnu", 8 + } + for _, op := range []string{"LDMXCSR", "VLDMXCSR", "STMXCSR", "VSTMXCSR"} { + for _, body := range []string{ + op + " (AX)", + "MOVW p+0(FP),AX\n" + op + " (AX)", + move + " p+0(FP),AX\n" + op + " (AX)(CX*1)", + } { + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+op+"/"+body+"/"+api, func(t *testing.T) { + source := fmt.Sprintf("TEXT ·State(SB),4,$0-%d\n%s\nRET\n", word, body) + ir, err := x86ScalarFPBitViewAPI(t, source, "func State(p *uint32)", arch, triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("native MXCSR operation borrowed undefined address/index bytes", err) + } + }) + } + } + } + } +} + +func TestX86LockRuntimeCompleteSourcesTranslateAllAPIs(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + declarations, source, _, _, count := x86LockScalarRuntimeFixture(arch) + want, triple := 174, "x86_64-unknown-linux-gnu" + if arch == "386" { + want, triple = 129, "i386-unknown-linux-gnu" + } + if count != want { + t.Fatal("complete native LOCK source denominator changed", count) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+api, func(t *testing.T) { + if _, err := x86OrderingHintTranslateAPI(t, api, source, declarations, arch, triple); err != nil { + t.Fatal("original complete native LOCK source rejected", err) + } + }) + } + } +} + +func TestX86LockRuntimeSavedStatusNeedsOriginalSourceRefinement(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + declarations, source, _, _, count := x86LockScalarRuntimeFixture(arch) + file, err := Parse(ArchAMD64, source) + if err != nil || len(file.Funcs) != count { + t.Fatal("complete original LOCK observation inventory changed", err) + } + rows := x86LockScalarRuntimeCases(arch) + for index, function := range file.Funcs { + observations := 0 + for offset, ins := range function.Instrs { + if ins.Op != "PUSHFQ" && ins.Op != "PUSHFL" { + continue + } + if offset+2 >= len(function.Instrs) { + t.Fatal("saved status has no complete following refinement") + } + mask := function.Instrs[offset+2] + if (mask.Op != "ANDQ" && mask.Op != "ANDL") || len(mask.Args) != 2 || + mask.Args[0].Kind != OpImm || mask.Args[0].Imm != int64(x86ScalarRuntimeStatusMask(rows[index].stem)) || + mask.Args[1].Kind != OpReg || mask.Args[1].Reg != AX { + t.Fatal("LOCK status output borrowed undefined architecture bits", function.Sym, mask) + } + observations++ + } + if observations != 1 { + t.Fatal("complete LOCK kernel lost its one saved-status observation", function.Sym) + } + } + // Preserve the original complete source as a negative control. Logical + // and bit-string status bits cannot be materialized merely because a + // separate reference would discard them later. + original := source + for _, mask := range []int{0x41, 0x8c5, 0x8d5} { + for _, suffix := range []string{"L", "Q"} { + original = strings.ReplaceAll(original, fmt.Sprintf("AND%s $%d,AX\n", suffix, mask), "") + } + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, original, declarations, arch, triple) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("the original unrefined LOCK source gained undefined numerical bits", err) + } + }) + } + } +} diff --git a/amd64_mxcsr_source_roles_objects_test.go b/amd64_mxcsr_source_roles_objects_test.go new file mode 100644 index 000000000..107f29f46 --- /dev/null +++ b/amd64_mxcsr_source_roles_objects_test.go @@ -0,0 +1,53 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86MXCSRBoundMemoryFormsOriginalGoAndLLVMFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc is required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + for index, row := range x86MXCSRBoundMemoryForms(t, target.arch) { + name := fmt.Sprintf("mxcsrBound%d", index) + original := strings.Replace(row.source, "·view(SB)", "·"+name+"(SB)", 1) + // Each whole original function retains its own DATA definition, + // without overlapping another function's control word. + original = strings.ReplaceAll(original, "control<>", fmt.Sprintf("control%d<>", index)) + source.WriteString(original) + declarations.WriteString(strings.Replace(row.declaration, "func view(", "func "+name+"(", 1)) + declarations.WriteByte('\n') + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil || len(file.Funcs) != 10 || len(file.Data) != 2 || len(file.Globl) != 2 { + t.Fatal("complete ten-function/two-DATA original batch differs", err) + } + receipt := x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + t.Logf("complete original bound MXCSR source: TEXTs=10 DATA=2 source=%x object=%x SDK=%s", + receipt.Source, receipt.Object, receipt.GoVersion) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil || strings.Count(ir, "define ") != 10 { + t.Fatalf("complete typed MXCSR batch %s: %v", api, err) + } + for _, irLevel := range []string{"0", "2"} { + objectIR := ir + if irLevel == "2" { + objectIR = x86DwordProductIROptimize(t, ir) + } + for _, backendLevel := range []string{"0", "2"} { + t.Run(api+"/IR"+irLevel+"/backend"+backendLevel, func(t *testing.T) { + x86PackedStringCompileObject(t, llc, target.triple, objectIR, "-O"+backendLevel) + }) + } + } + } + }) + } +} diff --git a/amd64_mxcsr_source_roles_test.go b/amd64_mxcsr_source_roles_test.go new file mode 100644 index 000000000..b046a9088 --- /dev/null +++ b/amd64_mxcsr_source_roles_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +type x86MXCSRBoundMemoryForm struct { + name, source, declaration, intrinsic string + local bool +} + +func x86MXCSRBoundMemoryForms(t *testing.T, arch string) []x86MXCSRBoundMemoryForm { + t.Helper() + const declaration = "func view(input uint32) uint32" + var rows []x86MXCSRBoundMemoryForm + for _, op := range []string{"LDMXCSR", "VLDMXCSR", "STMXCSR", "VSTMXCSR"} { + write := strings.Contains(op, "STMXCSR") + intrinsic := "@llvm.x86.sse.ldmxcsr" + if write { + intrinsic = "@llvm.x86.sse.stmxcsr" + } + for _, role := range []string{"AUTO", "FP"} { + body := op + " input+0(FP)\nSTMXCSR ret+RESULT(FP)" + if write { + body = op + " ret+RESULT(FP)" + } + if role == "AUTO" { + body = "MOVL $0x1f80,saved-4(SP)\n" + op + " saved-4(SP)\nSTMXCSR ret+RESULT(FP)" + if write { + body = op + " saved-4(SP)\nMOVL saved-4(SP),AX\nMOVL AX,ret+RESULT(FP)" + } + } + source := x86ResultGPSIMDAdmissionSource(t, arch, declaration, body) + if role == "AUTO" { + source = strings.Replace(source, ",$0-", ",$8-", 1) + } + rows = append(rows, x86MXCSRBoundMemoryForm{op + "/" + role, source, declaration, intrinsic, role == "AUTO"}) + } + } + for _, op := range []string{"LDMXCSR", "VLDMXCSR"} { + const data = "DATA control<>+0(SB)/4,$0x1f80\nGLOBL control<>(SB),8,$4\n" + body := op + " control<>(SB)\nSTMXCSR ret+RESULT(FP)" + source := data + x86ResultGPSIMDAdmissionSource(t, arch, declaration, body) + rows = append(rows, x86MXCSRBoundMemoryForm{op + "/DATA", source, declaration, "@llvm.x86.sse.ldmxcsr", false}) + } + if len(rows) != 10 { + t.Fatal("complete independent MXCSR storage-role denominator changed") + } + return rows +} + +// Preserve the exact former mixed-syntax source. Neither its zero frame, +// undefined GP inputs nor external SB symbol supplies initialized storage. +const x86MXCSRUnboundOriginalSource = ` +TEXT mxcsrstateforms(SB),NOSPLIT,$0-8 + LDMXCSR (AX) + LDMXCSR -8(SP) + LDMXCSR control(SB) + LDMXCSR input+0(FP) + VLDMXCSR (AX) + VLDMXCSR control(SB) + STMXCSR 4(BX) + STMXCSR -8(SP) + STMXCSR control(SB) + STMXCSR output+4(FP) + VSTMXCSR (AX) + VSTMXCSR output+4(FP) + RET +` + +func TestTranslateAMD64MXCSRStateOriginalUnboundSourceRetainsContext(t *testing.T) { + file, err := Parse(ArchAMD64, x86MXCSRUnboundOriginalSource) + if err != nil { + t.Fatal(err) + } + options := Options{TargetTriple: "x86_64-unknown-linux-gnu", Goarch: "amd64", + Sigs: map[string]FuncSig{"mxcsrstateforms": { + Name: "mxcsrstateforms", Args: []LLVMType{I32}, Ret: I32, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 4, Type: I32, Index: 0, Field: -1}}, + }, + }}, + } + if ir, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("original unbound mixed source acquired frame/native memory: %v", err) + } +} + +func TestX86MXCSRStorageRolesCannotBorrowSyntaxAuthority(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + move, triple := "MOVQ", "x86_64-unknown-linux-gnu" + if arch == "386" { + move, triple = "MOVL", "i386-unknown-linux-gnu" + } + for _, op := range []string{"LDMXCSR", "VLDMXCSR", "STMXCSR", "VSTMXCSR"} { + write := strings.Contains(op, "STMXCSR") + rows := []struct{ name, declaration, body, data string }{ + {"unbound GP", "func view() uint32", op + " (AX)\nMOVL $7,ret+RESULT(FP)", ""}, + {"overwritten typed native pointer", "func view(p *[2]uint32) uint32", move + " p+0(FP),AX\nMOVW $0,AX\n" + op + " 4(AX)\nMOVL $7,ret+RESULT(FP)", ""}, + {"undefined SB", "func view() uint32", op + " control(SB)\nMOVL $7,ret+RESULT(FP)", ""}, + } + if write { + rows = append(rows, struct{ name, declaration, body, data string }{ + "defined mutable DATA store", "func view() uint32", op + " control<>+4(SB)\nMOVL $7,ret+RESULT(FP)", + "DATA control<>+0(SB)/4,$0x1f80\nGLOBL control<>(SB),0,$8\n", + }) + } else { + rows = append(rows, struct{ name, declaration, body, data string }{ + "insufficient emitted DATA", "func view() uint32", op + " control<>(SB)\nSTMXCSR ret+RESULT(FP)", + "GLOBL control<>(SB),8,$3\n", + }) + } + for _, row := range rows { + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+op+"/"+row.name+"/"+api, func(t *testing.T) { + source := row.data + x86ResultGPSIMDAdmissionSource(t, arch, row.declaration, row.body) + if ir, err := x86ScalarFPBitViewAPI(t, source, row.declaration, arch, triple, api); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("MXCSR operand syntax granted an unclosed storage role: %v", err) + } + }) + } + } + } + } +} diff --git a/amd64_named_half_forms.go b/amd64_named_half_forms.go new file mode 100644 index 000000000..ef37ea4d7 --- /dev/null +++ b/amd64_named_half_forms.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" +) + +type x86RawNamedHalfEncoding uint8 + +const ( + x86RawNamedHalfAbsent x86RawNamedHalfEncoding = iota + x86RawNamedHalfVEX + x86RawNamedHalfEVEX +) + +// Active input/output bytes differ from both physical register carriers. +// Immediate-first narrowing also writes memory; it is not a source-memory +// conversion with its operands reversed after permissions have been granted. +type x86NamedHalfForm struct { + narrow, evex, sae, zeroing bool + source, destination, mask, memory int + lanes, sourceBytes, destinationBytes, inputBytes, outputBytes int + immediate byte + conversion x86PackedConversionForm +} + +func (f x86NamedHalfForm) undefinedOpcode() bool { + return f.narrow && f.memory == f.destination && f.zeroing +} + +func x86NamedHalfProperties(op Op, ins Instr, arch string) (x86NamedHalfForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if base != "VCVTPH2PS" && base != "VCVTPS2PH" { + return x86NamedHalfForm{}, false, nil + } + f := x86NamedHalfForm{narrow: base == "VCVTPS2PH", destination: len(ins.Args) - 1, mask: -1, memory: -1} + fail := func() (x86NamedHalfForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside complete named half conversion grammar: %q", arch, op, ins.Raw) + } + if arch != "386" && arch != "amd64" || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + properties, err := parseAMD64HalfConversionSuffix(strings.ToUpper(string(op)), base) + if err != nil { + return f, true, err + } + f.sae, f.zeroing = properties.sae, properties.zeroing + ins.Op = op + count := 2 + if f.narrow { + f.source, count = 1, 3 + } + if len(ins.Args) != count && len(ins.Args) != count+1 { + return fail() + } + if len(ins.Args) == count+1 { + f.mask = count - 1 + if !amd64NonzeroKOperand(ins.Args[f.mask]) || f.narrow && arch == "386" && ins.x86NamedHalfEncoding == x86RawNamedHalfAbsent { + return fail() + } + } else if f.zeroing { + return fail() + } + source, destination := ins.Args[f.source], ins.Args[f.destination] + width := amd64VectorByteWidth(destination.Reg) + if f.narrow { + width = amd64VectorByteWidth(source.Reg) + } + if width != 16 && width != 32 && width != 64 { + return fail() + } + f.lanes = width / 4 + f.inputBytes, f.outputBytes = width/2, width + f.sourceBytes, f.destinationBytes = 16, width + if width == 64 { + f.sourceBytes = 32 + } + if f.narrow { + f.inputBytes, f.outputBytes = width, width/2 + f.sourceBytes, f.destinationBytes = width, 16 + if width == 64 { + f.destinationBytes = 32 + } + immediate, valid := x86ImmediateShuffleConstant(ins.Args[0], arch, + ins.x86NamedHalfEncoding != x86RawNamedHalfAbsent) + if !valid || immediate.Imm < -128 || immediate.Imm > 255 { + return fail() + } + f.immediate = byte(immediate.Imm) + } + register := func(arg Operand, bytes int) bool { + return arg.Kind == OpReg && amd64EVEXVectorRegister(arg, bytes) + } + if !register(source, f.sourceBytes) { + if f.narrow || !isAMD64MemoryOperand(source) || !x86PackedShiftMemoryValid(source, arch) { + return fail() + } + f.memory = f.source + } + if !register(destination, f.destinationBytes) { + if !f.narrow || !isAMD64MemoryOperand(destination) || !x86PackedShiftMemoryValid(destination, arch) { + return fail() + } + f.memory = f.destination + } + if f.sae && (width != 64 || f.memory >= 0) { + return fail() // Go's asmevex rejects SAE on either memory ModRM role. + } + f.evex = ins.x86NamedHalfEncoding == x86RawNamedHalfEVEX || width == 64 || f.mask >= 0 || f.sae || + amd64HalfConversionHighRegister(source) || amd64HalfConversionHighRegister(destination) + if f.narrow && f.evex { + immediate, _ := x86ImmediateShuffleConstant(ins.Args[0], arch, + ins.x86NamedHalfEncoding != x86RawNamedHalfAbsent) + if immediate.Imm < 0 { + return fail() + } + } + if err := x86Mode32VectorRegisterContext(ins, arch); err != nil { + return f, true, err + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return f, true, fmt.Errorf("%w: %s needs its original default address mode", ErrProbeNeedsContext, op) + } + // Share active lane/carrier input rules, not a namespace or source role. + f.conversion = x86PackedConversionForm{ + spec: x86PackedConversionSpec{amd64PackedNumericConversionSpec: amd64PackedNumericConversionSpec{ + sourceBits: 16, destinationBits: 32}}, + shape: amd64PackedNumericShape{sourceBytes: f.sourceBytes, destinationBytes: f.destinationBytes, lanes: f.lanes}, + properties: amd64BinaryFloatingSuffix{zeroing: f.zeroing, sae: f.sae}, + inputBytes: f.inputBytes, destination: f.destination - f.source, mask: -1, memory: -1, evex: f.evex, + } + if f.narrow { + f.conversion.spec.sourceBits, f.conversion.spec.destinationBits = 32, 16 + } + if f.mask >= 0 { + f.conversion.mask = f.mask - f.source + } + if f.memory == f.source { + f.conversion.memory, f.conversion.memoryBytes = 0, f.inputBytes + } + return f, true, nil +} + +func x86TypedNamedHalfByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + f, known, err := x86NamedHalfProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + if f.undefinedOpcode() { + return x86TypedByteEffect{}, true, fmt.Errorf("%w: half EVEX.z memory encoding has no ordinary byte continuation", ErrProbeNeedsContext) + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: f.source, operand: ins.Args[f.source], width: int64(f.inputBytes), read: true}, + {index: f.destination, operand: ins.Args[f.destination], width: int64(f.outputBytes), + read: f.mask >= 0 && !f.zeroing && f.memory != f.destination, write: true}, + }} + if f.mask >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{index: f.mask, operand: ins.Args[f.mask], width: 8, read: true}) + } + return effect, true, nil +} + +func validateX86NamedHalfFile(file *File, opt Options) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + var pending error + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + _, known, err := x86NamedHalfProperties(ins.Op, ins, opt.Goarch) + if !known { + continue + } + if err != nil && !errors.Is(err, ErrProbeNeedsContext) { + return fmt.Errorf("%s: %w", fn.Sym, err) + } + if err == nil && !x86ScalarFloatIntegerNativeTarget(opt.Goarch, opt.TargetTriple) { + err = fmt.Errorf("%w: %s needs its matching native x86 MXCSR target", ErrProbeNeedsContext, ins.Op) + } + if pending == nil && err != nil { + pending = fmt.Errorf("%s: %w", fn.Sym, err) + } + } + } + return pending +} diff --git a/amd64_named_half_native.go b/amd64_named_half_native.go new file mode 100644 index 000000000..5071aaeed --- /dev/null +++ b/amd64_named_half_native.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerNamedHalfConversion(op Op, ins Instr) (bool, bool, error) { + f, known, err := x86NamedHalfProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + ins.Op = op + if f.undefinedOpcode() { + return c.lowerX86UndefinedTrap("UD2", Instr{Op: "UD2"}) + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: %s needs its matching native x86 MXCSR target", ErrProbeNeedsContext, op) + } + if c.fpScalarSource.x86SourceStack != nil || c.fpScalarSource.x86FixedLocal != nil { + return true, false, fmt.Errorf("%w: %s retains its independent frame/native source owner", ErrProbeNeedsContext, op) + } + if f.memory >= 0 { + memory := ins.Args[f.memory] + if memory.Kind == OpFP && f.memory == f.source { + if err := c.validateX86PackedShiftFPSpan(memory.FPOffset, f.inputBytes); err != nil { + return true, false, err + } + for at := memory.FPOffset; at < memory.FPOffset+int64(f.inputBytes); at++ { + if slot, _, _ := c.fpVectorSlot(at); slot.Type == Ptr { + return true, false, fmt.Errorf("%w: %s cannot convert opaque pointer bits", ErrProbeNeedsContext, op) + } + } + } else if !c.nativeMemoryOperand(ins, f.memory) { + return true, false, fmt.Errorf("%w: %s requires its original native address", ErrProbeNeedsContext, op) + } + } + if f.memory == f.destination { + return true, false, c.emitNamedHalfNativeStore(ins, f) + } + base, _, _ := strings.Cut(strings.ToLower(string(op)), ".") + var immediate *byte + if f.narrow { + immediate = &f.immediate + } + return true, false, c.emitNativePackedConversion(ins, x86NativePackedConversion{ + byteWidth: f.destinationBytes, sourceBytes: f.sourceBytes, memoryBytes: f.inputBytes, + outputBits: 32, logicalFPBits: 16, lanes: f.lanes, source: f.source, + destination: f.destination, mask: f.mask, evex: f.evex, zeroing: f.zeroing, sae: f.sae, + mnemonic: base, immediate: immediate, + }) +} + +// Narrowing's masked memory opcode is the actual write. A register conversion +// followed by an ordinary store loses inactive-lane faults, FP traps and dead +// effects; an indirect inline-asm output retains all three without metadata. +func (c *amd64Ctx) emitNamedHalfNativeStore(ins Instr, f x86NamedHalfForm) error { + value, err := c.loadPackedCompareBytes(ins.Args[f.source], f.sourceBytes) + if err != nil { + return err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x float>\n", input, f.sourceBytes, value, f.sourceBytes/4) + pointer, pointerType, _, err := c.packedVectorMemoryPointer(ins.Args[f.destination]) + if err != nil { + return err + } + output := "$0" + if pointerType == "ptr addrspace(256)" { + output = "%gs:" + output + } + if pointerType == "ptr addrspace(257)" { + output = "%fs:" + output + } + constraint, mnemonic := "x", "vcvtps2ph" + if f.evex { + constraint, mnemonic = "v", "{evex} "+mnemonic + } + constraints := "=*m," + constraint + arguments := fmt.Sprintf("%s elementtype([%d x i8]) %s, <%d x float> %%%s", + pointerType, f.outputBytes, pointer, f.sourceBytes/4, input) + assembly, prefix := fmt.Sprintf("%s $$%d, $1, %s", mnemonic, f.immediate, output), "" + if f.mask >= 0 { + mask, err := c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return err + } + prefix = "kmovw ${2:k}, %k1; " + constraints += ",r,~{k1}" + arguments += ", i32 " + c.approximateMathMask32(mask) + assembly += " {%k1}" + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s)\n", prefix+assembly, constraints+",~{memory},~{fpsr}", arguments) + return nil +} diff --git a/amd64_named_lock_prefix.go b/amd64_named_lock_prefix.go new file mode 100644 index 000000000..defc12ef3 --- /dev/null +++ b/amd64_named_lock_prefix.go @@ -0,0 +1,76 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) namedLockPrefixBinding(block, instruction int, prefix Instr) error { + context := func(reason string) error { + return fmt.Errorf("%w: %s named LOCK needs a complete source-adjacent atomic operation: %s", ErrProbeNeedsContext, c.goarch, reason) + } + if prefix.Op != "LOCK" || len(prefix.Args) != 0 { + return context("the prefix is not its exact zero-operand form") + } + if block < 0 || block >= len(c.blocks) || instruction < 0 || instruction+1 >= len(c.blocks[block].instrs) { + return context("the body is missing or separated by a source/CFG boundary") + } + body := c.blocks[block].instrs[instruction+1] + op := Op(strings.ToUpper(string(body.Op))) + if form, recognized, err := x86LockScalarRMWProperties(op, body, c.goarch); recognized { + if err != nil { + return err + } + return x86LockScalarNativeMemoryBound(form, body, c.goarch) + } + if form, recognized, err := x86CompareExchangeProperties(op, body, c.goarch); recognized { + if err != nil { + return err + } + if form.destination.Kind != OpMem && form.destination.Kind != OpSym || body.x86UnboundAddressOverride { + return context("the native CMPXCHG memory operand is not bound") + } + return nil + } + spec, known := amd64ExchangeSpecs[op] + if !known { + // A different RMW opcode may not inherit another family's witness. + return context("the whole opcode effect is not independently closed") + } + form, err := parseAMD64ExchangeForm(c.goarch, op, spec, body) + if err != nil { + return context("the complete exchange operand grammar is not bound") + } + if form.other.Kind != OpMem && form.other.Kind != OpSym || body.x86UnboundAddressOverride { + return context("the native memory operand is not bound") + } + if c.goarch == "386" && spec.bits == 8 { + // Go synthesizes unavailable byte-register views through another GP + // register. LOCK must not bind the first instruction of that expansion + // as though it were the final byte memory RMW. + switch form.register.Reg { + case AX, BX, CX, DX, AL, AH, BL, BH, CL, CH, DL, DH: + default: + return context("the source byte-register view needs a Go encoder expansion") + } + } + // These existing memory lowerers emit a complete native XADD with LOCK or + // an implicitly locked XCHG, with read/write memory constraints, no guessed + // alignment and their unchanged byte/flags effects. Address, initialization, + // frame and caller permissions still come from the ordinary consumers. + return nil +} + +func (c *amd64Ctx) verifyNamedLockPrefixes() error { + for block, body := range c.blocks { + for instruction, prefix := range body.instrs { + if !strings.EqualFold(string(prefix.Op), "LOCK") { + continue + } + if err := c.namedLockPrefixBinding(block, instruction, prefix); err != nil { + return err + } + } + } + return c.verifyNamedLockCarry() +} diff --git a/amd64_named_lock_prefix_test.go b/amd64_named_lock_prefix_test.go new file mode 100644 index 000000000..dbae44524 --- /dev/null +++ b/amd64_named_lock_prefix_test.go @@ -0,0 +1,142 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +type x86NamedLockOracleForm struct { + text string + atomic string +} + +// This independent inventory follows Intel's nineteen LOCK RMW opcode stems. +// A nonempty atomic field describes an existing lowerer's emitted operation, +// not a memory, caller or source initialization permission. +func x86NamedLockOracleForms(arch string) []x86NamedLockOracleForm { + widths := "BWL" + if arch == "amd64" { + widths += "Q" + } + var forms []x86NamedLockOracleForm + for _, stem := range []string{"ADD", "ADC", "AND", "OR", "SBB", "SUB", "XOR", "CMPXCHG", "XADD", "XCHG"} { + for _, width := range widths { + register := "CX" + if width == 'B' { + register = "CL" + } + atomic := "" + switch stem { + case "CMPXCHG": + atomic = "asm sideeffect \"lock; cmpxchg" + strings.ToLower(string(width)) + case "XADD": + atomic = "lock; xadd" + case "XCHG": + atomic = "asm sideeffect \"xchg" + default: + atomic = "lock; " + strings.ToLower(stem+string(width)) + } + forms = append(forms, x86NamedLockOracleForm{fmt.Sprintf("%s%c %s,(SI)", stem, width, register), atomic}) + } + } + for _, stem := range []string{"INC", "DEC", "NEG", "NOT"} { + for _, width := range widths { + forms = append(forms, x86NamedLockOracleForm{ + text: fmt.Sprintf("%s%c (SI)", stem, width), atomic: "lock; " + strings.ToLower(stem+string(width)), + }) + } + } + for _, stem := range []string{"BTC", "BTR", "BTS"} { + for _, width := range strings.TrimPrefix(widths, "B") { + forms = append(forms, x86NamedLockOracleForm{ + text: fmt.Sprintf("%s%c CX,(SI)", stem, width), atomic: "lock; " + strings.ToLower(stem+string(width)), + }) + } + } + forms = append(forms, x86NamedLockOracleForm{text: "CMPXCHG8B (SI)", atomic: "asm sideeffect \"lock; cmpxchg8b"}) + if arch == "amd64" { + forms = append(forms, x86NamedLockOracleForm{text: "CMPXCHG16B (SI)", atomic: "asm sideeffect \"lock; cmpxchg16b"}) + } + return forms +} + +func x86NamedLockOracleSource(arch, body string) string { + load, args := "MOVL", 4 + if arch == "amd64" { + load, args = "MOVQ", 8 + } + return fmt.Sprintf(`TEXT ·locked(SB),4,$0-%d + %s p+0(FP),SI + MOVL $2,AX + MOVL $3,BX + MOVL $4,CX + MOVL $5,DX + %s + RET +`, args, load, body) +} + +func TestX86NamedLockCompleteRMWRequiresWholeAtomicBinding(t *testing.T) { + for _, target := range x86OrderingHintTargets { + forms := x86NamedLockOracleForms(target.arch) + want := 49 + if target.arch == "amd64" { + want = 67 + } + if len(forms) != want { + t.Fatal("complete LOCK stem/width denominator changed", target.arch, len(forms)) + } + for _, form := range forms { + t.Run(target.triple+"/"+form.text, func(t *testing.T) { + source := x86NamedLockOracleSource(target.arch, "CMPL AX,AX\n\tLOCK\n\t"+form.text) + actual := x86OrderingHintGoBytes(t, target.arch, source) + if !strings.Contains(string(actual), string([]byte{0xf0})) { + t.Fatalf("actual Go object lost named LOCK prefix: %x", actual) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, "package ordering\nfunc locked(p *[16]byte)\n", target.arch, target.triple) + if form.atomic == "" { + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("actual Go LOCK RMW silently became a non-atomic operation: error=%v atomicrmw=%v cmpxchg=%v nativeLock=%v", err, + strings.Contains(ir, "atomicrmw "), strings.Contains(ir, " = cmpxchg "), strings.Contains(ir, "lock;")) + } + return + } + if err != nil || !strings.Contains(ir, form.atomic) { + t.Fatalf("existing whole atomic operation changed: %v, required emission=%q", err, form.atomic) + } + }) + } + }) + } + } +} + +func TestX86NamedLockDetachedAndSplitPrefixDoesNotBorrowAtomicBody(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, body := range []string{ + "LOCK", + "LOCK\n\tLOCK\n\tXADDL CX,(SI)", + "LOCK\nbody:\n\tXADDL CX,(SI)", + "LOCK\n\tCMPL CX,(SI)", + "LOCK\n\tNOP\n\tXADDL CX,(SI)", + } { + t.Run(arch+"/"+body, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, "text", x86NamedLockOracleSource(arch, body), + "package ordering\nfunc locked(p *[16]byte)\n", arch, testTargetTriple("linux", arch)) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("unbound or split prefix borrowed a following operation", err) + } + }) + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before := output.String() + known, terminated, err := ctx.lowerAtomic("LOCK", Instr{Op: "LOCK", Raw: "LOCK"}) + if !known || terminated || !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatal("standalone helper has no complete ordered instruction to bind", known, terminated, err) + } +} diff --git a/amd64_named_stack_expression_test.go b/amd64_named_stack_expression_test.go new file mode 100644 index 000000000..19c7699c0 --- /dev/null +++ b/amd64_named_stack_expression_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestAMD64NamedStackConstantExpressions(t *testing.T) { + const source = `TEXT namedStackExpressions(SB),$64-0 + MOVL AX, tmpdig-(0*4)(SP) + MOVL BX, tmpdig-(1*4)(SP) + ADDL tmpdig-(0*4)(SP), AX + ADDL tmpdig-(1*4)(SP), BX + RET +` + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if got := file.Funcs[0].Instrs[1].Args[1]; got.Kind != OpMem || got.Mem.Off != 0 { + t.Fatalf("first named stack slot parsed as %+v, want offset 0 memory", got) + } + if got := file.Funcs[0].Instrs[2].Args[1]; got.Kind != OpMem || got.Mem.Off != -4 { + t.Fatalf("second named stack slot parsed as %+v, want offset -4 memory", got) + } + // These original operands establish constant-expression grammar only. + // AX/BX have no inputs, and NAME_AUTO offset zero is not a logical + // four-byte local within [-64,-4]. Keep the complete source as Context. + for _, api := range []string{"text", "module", "Go"} { + t.Run("unbound/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func namedStackExpressions()", + "amd64", "x86_64-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original grammar supplied entry NUM or native frame storage", err) + } + }) + } + + // Preserve the four exact operations and their original offsets. The + // actual source-bound frame and temporary SP relocation put their reads + // and writes inside the 64-byte local; both input values are initialized. + const initialized = `TEXT ·namedStackExpressions(SB),4,$64-24 + MOVQ a+0(FP), AX + MOVQ b+8(FP), BX + MOVQ SP, R11 + ADDQ $-16, SP + MOVL AX, tmpdig-(0*4)(SP) + MOVL BX, tmpdig-(1*4)(SP) + ADDL tmpdig-(0*4)(SP), AX + ADDL tmpdig-(1*4)(SP), BX + MOVQ R11, SP + ADDQ BX, AX + MOVQ AX, ret+16(FP) + RET +` + x86StackExpressionBoundObjects(t, initialized, "func namedStackExpressions(a,b uint64) uint64") +} + +// Observe actual complete Go sources for each API, then compile their emitted +// IR at both optimization levels. A previous API cannot extend the source lease. +func x86StackExpressionBoundObjects(t *testing.T, source, declaration string) { + t.Helper() + for _, api := range []string{"text", "module", "Go"} { + t.Run("initialized-without-binding/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, + "amd64", "x86_64-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("initialized source spelling supplied native frame authority", err) + } + }) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, goos := range []string{"linux", "darwin", "windows"} { + t.Run("bound/"+goos, func(t *testing.T) { + triple := testTargetTriple(goos, "amd64") + pkg, request, _, options := x86CMOVFixedLocalFixture(t, source, declaration, goos+"/amd64", triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "fixed_local_native") { + t.Fatal("initialized exact expression lost its source-bound local frame", err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + }) + }) + } +} + +func TestNamedStackConstantOffsetRejectsUnresolvedExpressions(t *testing.T) { + for _, tc := range []struct { + prefix string + want int64 + ok bool + }{ + {prefix: "tmpdig-(7*4)", want: -28, ok: true}, + {prefix: "tmpdig+(2<<3)", want: 16, ok: true}, + {prefix: "tmpdig-(unknown*4)"}, + {prefix: "tmpdig-(4/0)"}, + {prefix: "tmp-dig-(1*4)"}, + } { + got, ok := parseNamedStackConstantOffset(tc.prefix) + if ok != tc.ok || ok && got != tc.want { + t.Errorf("parseNamedStackConstantOffset(%q) = %d, %v; want %d, %v", tc.prefix, got, ok, tc.want, tc.ok) + } + } +} diff --git a/amd64_native_avx512_qualification_linux_amd64_test.go b/amd64_native_avx512_qualification_linux_amd64_test.go new file mode 100644 index 000000000..33e25676c --- /dev/null +++ b/amd64_native_avx512_qualification_linux_amd64_test.go @@ -0,0 +1,82 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import ( + "os" + "testing" +) + +// Explicit hardware qualification retains the original complete fixtures. +// An absent CPU feature is fatal in each original preflight, never a skip. +func x86RequireNativeAVX512Qualification(t *testing.T) { + t.Helper() + if mode := os.Getenv("PLAN9ASM_AVX512_RUNTIME"); mode != "" && mode != "required" { + t.Fatalf("explicit native AVX512 qualification requires required policy, got %q", mode) + } +} + +func TestAMD64FPClassRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64FPClassRuntimeSemantics(t) +} + +func TestAMD64FixupImmediateRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64FixupImmediateRuntimeSemantics(t) +} + +func TestAMD64FloatingRangeDoubleAndScalarRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64FloatingRangeDoubleAndScalarRuntimeSemantics(t) +} + +func TestAMD64FloatingRangeRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64FloatingRangeRuntimeSemantics(t) +} + +func TestAMD64GetExpRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64GetExpRuntimeSemantics(t) +} + +func TestAMD64GetMantRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64GetMantRuntimeSemantics(t) +} + +func TestAMD64PackedAbsSignRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64PackedAbsSignRuntimeSemantics(t) +} + +func TestAMD64ScalarIntegerFloatRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64ScalarIntegerFloatRuntimeSemantics(t) +} + +func TestAMD64ScalarPrecisionConvertRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64ScalarPrecisionConvertRuntimeSemantics(t) +} + +func TestAMD64ScalarSqrtRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64ScalarSqrtRuntimeSemantics(t) +} + +func TestAMD64ScaleFRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64ScaleFRuntimeSemantics(t) +} + +func TestAMD64ScaledRoundRuntimeSemantics(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyAMD64ScaledRoundRuntimeSemantics(t) +} + +func TestCrossLinuxRuntimeMatrixX86FMA3GoCallerNumericalFamily(t *testing.T) { + x86RequireNativeAVX512Qualification(t) + qualifyCrossLinuxRuntimeMatrixX86FMA3GoCallerNumericalFamily(t) +} diff --git a/amd64_native_floating_operands.go b/amd64_native_floating_operands.go new file mode 100644 index 000000000..b390f1dfc --- /dev/null +++ b/amd64_native_floating_operands.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Floating families share native operand transport, not arithmetic semantics. +// Preserve the actual opcode and memory operand so masking, rounding, +// exceptions and dead accesses survive. Legacy SSE ties its destination. +type x86NativeFloatingOperands struct { + byteWidth, lanes, memoryBytes int + second, destination, mask int + controls amd64FMA3Suffix + sae, readDestination bool + legacy bool + lowBank, forceEVEX bool + logicalFPBits int // Explicit scalar-value projection, not native FP backing. +} + +func (c *amd64Ctx) emitNativeFloatingOperands(ins Instr, f x86NativeFloatingOperands) error { + source := ins.Args[0] + logicalFP := source.Kind == OpFP && (f.logicalFPBits == 32 || f.logicalFPBits == 64) + if source.Kind == OpFP && !logicalFP || ins.x86AddressBits != 0 || source.Kind == OpMem && + !c.nativeMemoryOperand(ins, 0) { + return fmt.Errorf("%w: %s needs original native backing", ErrProbeNeedsContext, ins.Op) + } + typ := fmt.Sprintf("<%d x float>", f.byteWidth/4) // Bit carrier, never arithmetic precision. + load := func(index int) (string, error) { + bytes, err := c.loadPackedCompareBytes(ins.Args[index], f.byteWidth) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", value, f.byteWidth, bytes, typ) + return "%" + value, nil + } + second, err := load(f.second) + if err != nil { + return err + } + old := "zeroinitializer" + if f.legacy { + old = second + } else if f.readDestination || f.mask >= 0 && !f.controls.zeroing { + old, err = load(f.destination) + if err != nil { + return err + } + } + registerConstraint := "v" + if f.legacy || f.lowBank { + registerConstraint = "x" // Legacy/VEX cannot acquire registers 16-31 from an EVEX sibling. + } + inputType, inputConstraint, inputTemplate := typ, registerConstraint, "$3" + var input string + if ins.Args[0].Kind == OpReg { + input, err = load(0) + } else if logicalFP { + // A declared scalar FP field supplies its exact value, not an original + // physical stack address. Scalar MIN/MAX ignores the other source + // lanes; zeros in this private carrier grant no upper-byte facts. + var raw string + raw, err = c.loadFloatBinaryFPBits(source, f.logicalFPBits) + if err == nil { + lanes := 128 / f.logicalFPBits + value, carrier := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> zeroinitializer, i%d %s, i32 0\n", value, lanes, f.logicalFPBits, f.logicalFPBits, raw) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to %s\n", carrier, lanes, f.logicalFPBits, value, typ) + input = "%" + carrier + } + } else { + var pointerType string + input, pointerType, _, err = c.packedVectorMemoryPointer(ins.Args[0]) + inputType = fmt.Sprintf("%s elementtype([%d x i8])", pointerType, f.memoryBytes) + inputConstraint = "*m" + if pointerType == "ptr addrspace(256)" { + inputTemplate = "%gs:" + inputTemplate + } else if pointerType == "ptr addrspace(257)" { + inputTemplate = "%fs:" + inputTemplate + } + } + if err != nil { + return err + } + args := fmt.Sprintf("%s %s, %s %s, %s %s", typ, old, typ, second, inputType, input) + constraints := "=" + registerConstraint + ",0," + registerConstraint + "," + inputConstraint + mnemonic, _, _ := strings.Cut(strings.ToLower(string(ins.Op)), ".") + assembly := mnemonic + " " + if f.forceEVEX { + assembly = "{evex} " + assembly + } + if f.controls.rounding != "" { + assembly += "{" + strings.ToLower(strings.TrimSuffix(f.controls.rounding, "_SAE")) + "-sae}, " + } else if f.sae { + assembly += "{sae}, " + } + assembly += inputTemplate + if f.controls.broadcast { + assembly += fmt.Sprintf("{1to%d}", f.lanes) + } + if f.legacy { + assembly += ", $0" + } else { + assembly += ", $2, $0" + } + if f.mask >= 0 { + mask, err := c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return err + } + maskMove := "kmovw" + if f.lanes > 16 { + maskMove = "kmovd" + } + assembly = maskMove + " ${4:k}, %k1; " + assembly + " {%k1}" + if f.controls.zeroing { + assembly += " {z}" + } + constraints += ",r,~{k1}" + args += ", i32 " + c.approximateMathMask32(mask) + } + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, typ, assembly, constraints+",~{memory},~{fpsr}", args) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, typ, result, f.byteWidth) + if f.legacy { + return c.storeLegacyPackedLogical(ins.Args[f.destination], "%"+bytes) + } + return c.storePackedMoveOperand(ins.Args[f.destination], f.byteWidth, "%"+bytes) +} diff --git a/amd64_native_mmx_movemask_oracle_test.go b/amd64_native_mmx_movemask_oracle_test.go new file mode 100644 index 000000000..0062967b7 --- /dev/null +++ b/amd64_native_mmx_movemask_oracle_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The original Go and C reference independently observe all eight sign bits, +// r32 zero-extension, FLAGS, aliasing, exact m64 faults and real EMMS tags. +func x86MMXMoveMaskOracleSources() (string, string, string, string) { + var declarations, assembly, c, goTest strings.Builder + declarations.WriteString("package oracle\n") + c.WriteString("#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n") + var names []string + for bank := 0; bank < 8; bank++ { + for _, destination := range []string{"BX", "R8"} { + name := fmt.Sprintf("MMXMask%d", len(names)) + names = append(names, name) + fmt.Fprintf(&declarations, "func %s(p *[8]byte,out *[24]byte)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),SI\nMOVQ out+8(FP),DI\nMOVQ $-1,%s\nMOVL $0,AX\nCMPL AX,AX\nMOVQ (SI),M%d\nPMOVMSKB M%d,%s\nEMMS\nMOVQ %s,0(DI)\nSETCS AL\nMOVB AL,8(DI)\nSETPS AL\nMOVB AL,9(DI)\nSETEQ AL\nMOVB AL,10(DI)\nSETMI AL\nMOVB AL,11(DI)\nSETOS AL\nMOVB AL,12(DI)\nLAHF\nMOVB AH,13(DI)\nRET\n", name, destination, bank, bank, destination, destination) + fmt.Fprintf(&c, "extern void %s(const uint8_t*,uint8_t*);\n", name) + } + } + fmt.Fprintf(&c, "static void (*const kernels[])(const uint8_t*,uint8_t*)={%s};\n", strings.Join(names, ",")) + c.WriteString(` +static unsigned reference(const uint8_t *p) { + unsigned result=0; + for(unsigned i=0;i<8;i++) result|=(unsigned)(p[i]>>7)<>j)&1)*128+(low?127:0); + memcpy(before,input,8); memset(out,0xa5,sizeof(out)); + if(check(k,input,out) || memcmp(input,before,8)) return 1; + for(unsigned j=14;j<24;j++) if(out[j]!=0xa5) return 1; + uint8_t alias[32];memset(alias,0xa5,sizeof(alias));memcpy(alias+1,input,8); + if(check(k,alias+1,alias+1) || alias[0]!=0xa5) return 1; + for(unsigned j=15;j<32;j++) if(alias[j]!=0xa5) return 1; + } + long page=sysconf(_SC_PAGESIZE); + uint8_t *mapping=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(mapping==MAP_FAILED || mprotect(mapping+page,page,PROT_NONE)) return 2; + memset(mapping+page-8,0x80,8);uint8_t out[24]; + if(check(k,mapping+page-8,out)) return 1; + for(unsigned fault=0;fault<8;fault++) { + const uint8_t *p=fault?mapping+page-fault:0; + pid_t child=fork();if(child<0)return 2; + if(child==0) { struct rlimit limit={0,0};setrlimit(RLIMIT_CORE,&limit);kernels[k](p,out);_exit(99); } + int status;if(waitpid(child,&status,0)!=child)return 2; + if(!WIFSIGNALED(status) || WTERMSIG(status)!=SIGSEGV) {fprintf(stderr,"missing m64 fault k=%u case=%u status=%x\n",k,fault,status);return 1;} + } + if(munmap(mapping,2*page))return 2; + } + puts("MMX_MASK kernels=16 values=8192 aliases=8192 faults=128 boundary=16 failures=0"); + return 0; +} +`) + fmt.Fprintf(&goTest, "package oracle\nimport(\"encoding/binary\";\"os\";\"runtime/debug\";\"syscall\";\"testing\";\"unsafe\")\nvar kernels=[]func(*[8]byte,*[24]byte){%s}\n", strings.Join(names, ",")) + goTest.WriteString(` +func check(t *testing.T,k int,p *[8]byte,out *[24]byte) { + t.Helper();var want uint64 + for i,b:=range p { want|=uint64(b>>7)<>uint(j))&1)*128+low} + before:=input;for j:=range out {out[j]=0xa5} + check(t,k,&input,&out) + if input!=before {t.Fatal("input changed",k)} + for _,b:=range out[14:] {if b!=0xa5{t.Fatal("output extent",k)}} + var alias [32]byte;for j:=range alias{alias[j]=0xa5};copy(alias[1:9],input[:]) + check(t,k,(*[8]byte)(unsafe.Pointer(&alias[1])),(*[24]byte)(unsafe.Pointer(&alias[1]))) + if alias[0]!=0xa5 {t.Fatal("alias prefix",k)} + for _,b:=range alias[15:] {if b!=0xa5{t.Fatal("alias suffix",k)}} + }} + page:=os.Getpagesize() + mapping,err:=syscall.Mmap(-1,0,2*page,syscall.PROT_READ|syscall.PROT_WRITE,syscall.MAP_PRIVATE|syscall.MAP_ANON) + if err!=nil {t.Fatal(err)} + if err:=syscall.Mprotect(mapping[page:],syscall.PROT_NONE);err!=nil{t.Fatal(err)} + for j:=page-8;j plan.end { + return false, nil + } + if block != 0 || len(c.blocks) != 1 || !plan.matchesInstruction(ordinal, ins) { + return true, fmt.Errorf("%w: native MMX source position changed", ErrProbeNeedsContext) + } + if ordinal != plan.begin { + return true, nil // The complete sealed sequence was emitted at its first instruction. + } + if !plan.matches(c.fpScalarSource, c.fpSourceSig, c.goarch, c.targetTriple) { + return true, fmt.Errorf("%w: native MMX source/declaration binding changed", ErrProbeNeedsContext) + } + return true, c.emitNativeMMXRegion(plan) +} + +// These names differ between Go's operand-width notation and Intel/LLVM's +// spelling. The operand and encoding rows still come from the shared complete +// x86LegacyMMXReadSpecs; this selector grants no additional form or input. +func x86NativeMMXIntegerMnemonic(op Op) string { + switch op { + case "PACKSSLW": + return "packssdw" + case "PADDL": + return "paddd" + case "PCMPEQL": + return "pcmpeqd" + case "PCMPGTL": + return "pcmpgtd" + case "PMADDWL": + return "pmaddwd" + case "PMULULQ": + return "pmuludq" + case "PUNPCKHWL": + return "punpckhwd" + case "PUNPCKHLQ": + return "punpckhdq" + case "PUNPCKLWL": + return "punpcklwd" + case "PUNPCKLLQ": + return "punpckldq" + case "PSLLL": + return "pslld" + case "PSRLL": + return "psrld" + case "PSRAL": + return "psrad" + } + return strings.ToLower(string(op)) +} + +func (c *amd64Ctx) emitNativeMMXRegion(plan *x86NativeMMXRegion) error { + gpBase := (plan.fpExtent + 15) &^ 15 + xBase := gpBase + int64(len(x86NativeMMXGPRegisters))*8 + extent := xBase + for index := range plan.xRead { + if plan.xRead[index] || plan.xWritten[index] { + extent = xBase + 16*int64(index+1) + } + } + image := "%" + c.newTmp() + fmt.Fprintf(c.b, " %s = alloca [%d x i8], align 16\n", image, extent) + pointer := func(offset int64) string { + name := "%" + c.newTmp() + fmt.Fprintf(c.b, " %s = getelementptr i8, ptr %s, i64 %d\n", name, image, offset) + return name + } + // This image gathers only declared logical numeric fields. Its address is + // private to the asm region: source GPs, native pointers and later effects + // cannot acquire it as a frame, contiguous caller storage or a GC root. + var fields []FrameSlot + for _, slots := range [][]FrameSlot{c.sig.Frame.Params, c.sig.Frame.Results} { + for _, slot := range slots { + width := x86FrameScalarBytes(slot.Type, c.goarch) + touched := false + for offset := slot.Offset; offset < slot.Offset+width; offset++ { + touched = touched || plan.fpRead[offset] || plan.fpWritten[offset] + } + if !touched { + continue + } + value, err := c.loadFPVectorBytes(slot.Offset, int(width)) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <%d x i8> %s, ptr %s, align 1\n", width, value, pointer(slot.Offset)) + fields = append(fields, slot) + } + } + for index, reg := range x86NativeMMXGPRegisters { + if !plan.gpRead[index] && !plan.gpWritten[index] { + continue + } + value, err := c.loadReg(reg) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i64 %s, ptr %s, align 1\n", value, pointer(gpBase+8*int64(index))) + } + for index := range plan.xRead { + if !plan.xRead[index] && !plan.xWritten[index] { + continue + } + value, err := c.loadX(Reg(fmt.Sprintf("X%d", index))) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s, align 1\n", value, pointer(xBase+16*int64(index))) + } + assembly, err := x86NativeMMXAssembly(plan, gpBase, xBase) + if err != nil { + return err + } + scratch := []string{"r10", "r11", "xmm15"} + if c.goarch == "386" { + scratch = []string{"eax", "edx", "xmm7"} + } + constraints := []string{"r"} + for _, site := range plan.sites { + if site.maskStore != nil { + constraints = append(constraints, "~{di}") + break + } + } + for _, reg := range scratch { + constraints = append(constraints, "~{"+reg+"}") + } + for index := 0; index < 8; index++ { + constraints = append(constraints, fmt.Sprintf("~{mm%d}", index)) + name := "st" + if index != 0 { + name = fmt.Sprintf("st(%d)", index) + } + constraints = append(constraints, "~{"+name+"}") + } + constraints = append(constraints, "~{fpsr}", "~{memory}") + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(ptr %s)\n", assembly, strings.Join(constraints, ","), image) + for _, slot := range fields { + width := x86FrameScalarBytes(slot.Type, c.goarch) + written := false + for offset := slot.Offset; offset < slot.Offset+width; offset++ { + written = written || plan.fpWritten[offset] + } + if written { + value := "%" + c.newTmp() + fmt.Fprintf(c.b, " %s = load <%d x i8>, ptr %s, align 1\n", value, width, pointer(slot.Offset)) + if err := c.storeFPVectorBytes(slot.Offset, int(width), value, ""); err != nil { + return err + } + } + } + for index, reg := range x86NativeMMXGPRegisters { + if plan.gpWritten[index] { + value := "%" + c.newTmp() + fmt.Fprintf(c.b, " %s = load i64, ptr %s, align 1\n", value, pointer(gpBase+8*int64(index))) + if err := c.storeReg(reg, value); err != nil { + return err + } + } + } + for index, written := range plan.xWritten { + if written { + value := "%" + c.newTmp() + fmt.Fprintf(c.b, " %s = load <16 x i8>, ptr %s, align 1\n", value, pointer(xBase+16*int64(index))) + if err := c.storeLegacyPackedLogical(Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", index))}, value); err != nil { + return err + } + } + } + if !plan.matches(c.fpScalarSource, c.fpSourceSig, c.goarch, c.targetTriple) { + return fmt.Errorf("%w: native MMX source changed while emitting its region", ErrProbeNeedsContext) + } + return nil +} + +func x86NativeMMXAssembly(plan *x86NativeMMXRegion, gpBase, xBase int64) (string, error) { + word, data, address, vector := "movq", "%r10", "%r11", "%xmm15" + if plan.arch == "386" { + word, data, address, vector = "movl", "%eax", "%edx", "%xmm7" + } + var code []string + gp := func(reg Reg) (int64, error) { + index, _, valid := x86ResultGP(reg, x86ResultPointerWord(plan.arch), plan.arch) + if !valid { + return 0, fmt.Errorf("%w: MMX region lost a typed GP identity", ErrProbeNeedsContext) + } + return gpBase + int64(index)*8, nil + } + memory := func(operand Operand) (string, error) { + if operand.Kind == OpFP { + return fmt.Sprintf("%d($0)", operand.FPOffset), nil + } + if operand.Kind != OpMem { + return "", fmt.Errorf("%w: MMX memory lost its source operand class", ErrProbeNeedsContext) + } + base, err := gp(operand.Mem.Base) + if err != nil { + return "", err + } + code = append(code, fmt.Sprintf("%s %d($0), %s", word, base, address)) + if operand.Mem.Index != "" { + index, err := gp(operand.Mem.Index) + if err != nil { + return "", err + } + code = append(code, fmt.Sprintf("%s %d($0), %s", word, index, data), + fmt.Sprintf("lea %d(%s,%s,%d), %s", operand.Mem.Off, address, data, operand.Mem.Scale, address)) + return "(" + address + ")", nil + } + return fmt.Sprintf("%d(%s)", operand.Mem.Off, address), nil + } + for ordinal := plan.begin; ordinal <= plan.end; ordinal++ { + site := plan.sites[ordinal] + if site.instruction.Op == "EMMS" { + code = append(code, "emms") + continue + } + if form := site.word; form != nil { + mnemonic := "movq" + if form.bytes == 4 { + mnemonic = "movd" + } + if form.nonTemporal { + mnemonic = "movntq" + } + if form.targetM { + source := "" + if form.sourceX { + index, _ := amd64ParseXReg(form.source.Reg) + code = append(code, fmt.Sprintf("movdqu %d($0), %s", xBase+int64(index)*16, vector)) + mnemonic, source = "movdq2q", vector + } else if form.source.Kind == OpReg { + offset, err := gp(form.source.Reg) + if err != nil { + return "", err + } + load, reg := word, data + if form.bytes == 4 { + load, reg = "movl", "%eax" + if plan.arch == "amd64" { + reg = "%r10d" + } + } + code = append(code, fmt.Sprintf("%s %d($0), %s", load, offset, reg)) + source = reg + } else { + var err error + source, err = memory(form.source) + if err != nil { + return "", err + } + } + index, _ := amd64ParseMReg(form.destination.Reg) + code = append(code, fmt.Sprintf("%s %s, %%mm%d", mnemonic, source, index)) + continue + } + source, _ := amd64ParseMReg(form.source.Reg) + if form.targetX { + index, _ := amd64ParseXReg(form.destination.Reg) + code = append(code, fmt.Sprintf("movq2dq %%mm%d, %s", source, vector), + fmt.Sprintf("movdqu %s, %d($0)", vector, xBase+int64(index)*16)) + } else if form.destination.Kind == OpReg { + offset, err := gp(form.destination.Reg) + if err != nil { + return "", err + } + reg := data + if form.bytes == 4 && plan.arch == "amd64" { + reg = "%r10d" + } + code = append(code, fmt.Sprintf("%s %%mm%d, %s", mnemonic, source, reg), + fmt.Sprintf("%s %s, %d($0)", word, data, offset)) + if plan.arch == "386" { + code = append(code, fmt.Sprintf("movl $$0, %d($0)", offset+4)) + } + } else { + destination, err := memory(form.destination) + if err != nil { + return "", err + } + code = append(code, fmt.Sprintf("%s %%mm%d, %s", mnemonic, source, destination)) + } + continue + } + if site.moveMask != nil { + offset, err := gp(site.instruction.Args[1].Reg) + if err != nil { + return "", err + } + bank, _ := amd64ParseMReg(site.instruction.Args[0].Reg) + reg := "%r10d" + if plan.arch == "386" { + reg = "%eax" + } + code = append(code, fmt.Sprintf("pmovmskb %%mm%d, %s", bank, reg), + fmt.Sprintf("%s %s, %d($0)", word, data, offset)) + if plan.arch == "386" { + code = append(code, fmt.Sprintf("movl $$0, %d($0)", offset+4)) + } + continue + } + if site.maskStore != nil { + offset, err := gp(DI) + if err != nil { + return "", err + } + destination := "%rdi" + if plan.arch == "386" { + destination = "%edi" + } + mask, _ := amd64ParseMReg(site.instruction.Args[0].Reg) + data, _ := amd64ParseMReg(site.instruction.Args[1].Reg) + code = append(code, fmt.Sprintf("%s %d($0), %s", word, offset, destination), + fmt.Sprintf("maskmovq %%mm%d, %%mm%d", mask, data)) + continue + } + form := site.integer + destination, _ := amd64ParseMReg(site.instruction.Args[form.destination].Reg) + mnemonic := x86NativeMMXIntegerMnemonic(form.canonical) + if form.source.Kind == OpImm { + code = append(code, fmt.Sprintf("%s $$%d, %%mm%d", mnemonic, byte(form.source.Imm), destination)) + continue + } + source := "" + if form.source.Kind == OpReg { + index, _ := amd64ParseMReg(form.source.Reg) + source = fmt.Sprintf("%%mm%d", index) + } else { + var err error + source, err = memory(form.source) + if err != nil { + return "", err + } + } + immediate := "" + if form.spec.table == x86LegacyMMXShuffle { + immediate = fmt.Sprintf("$$%d, ", byte(site.instruction.Args[0].Imm)) + } + code = append(code, fmt.Sprintf("%s %s%s, %%mm%d", mnemonic, immediate, source, destination)) + } + return strings.Join(code, "; "), nil +} diff --git a/amd64_native_mmx_streaming_linux_amd64_test.go b/amd64_native_mmx_streaming_linux_amd64_test.go new file mode 100644 index 000000000..7d32e9c9b --- /dev/null +++ b/amd64_native_mmx_streaming_linux_amd64_test.go @@ -0,0 +1,30 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86MMXStreamingOriginalAndLLVM(t *testing.T) { + declarations, source, c, tests := x86MMXStreamingOracleSources() + t.Run("original-go", func(t *testing.T) { + x86PackedIntegerCompareRunOriginalGo(t, declarations, source, tests) + }) + ir, err := x86ScalarFPBitViewAPI(t, source, declarations[len("package oracle\n"):], + "amd64", "x86_64-unknown-linux-gnu", "Go") + if err != nil { + t.Fatal("complete original streaming source lost its native MMX binding", err) + } + execution := x86FloatCompareExecutionTools(t) + for _, passes := range []string{"default", "O2"} { + text := ir + if passes == "O2" { + text = x86DwordProductIROptimize(t, ir) + } + for _, backend := range []string{"-O0", "-O2"} { + t.Run("llvm-"+passes+backend, func(t *testing.T) { + execution.compileAndRunLLVM(t, "streaming", text, c, backend) + }) + } + } + // Preserve every streaming kernel above, then execute the distinct typed + // MMX-to-GP consumer through its original Go and independent C oracles. + t.Run("move-mask", x86NativeMMXMoveMaskRuntime) +} diff --git a/amd64_native_mmx_streaming_oracle_test.go b/amd64_native_mmx_streaming_oracle_test.go new file mode 100644 index 000000000..65593577d --- /dev/null +++ b/amd64_native_mmx_streaming_oracle_test.go @@ -0,0 +1,85 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86MMXStreamingOracleSources() (string, string, string, string) { + var declarations, assembly, c, goTest strings.Builder + declarations.WriteString("package oracle\n") + c.WriteString("#include \n#include \n#include \n") + var names []string + var offsets []string + for register := 0; register < 8; register++ { + for index, address := range []string{"0(DI)", "0(DI)(DX*8)"} { + name := fmt.Sprintf("Stream%d", len(names)) + names = append(names, name) + offsets = append(offsets, fmt.Sprint(index*8)) + source, declaration := x86NativeMMXNonTemporalFixture("amd64", register, address) + assembly.WriteString(strings.ReplaceAll(source, "Store", name)) + declarations.WriteString(strings.ReplaceAll(declaration, "Store", name) + "\n") + fmt.Fprintf(&c, "extern void %s(uint32_t,uint8_t*);\n", name) + } + } + fmt.Fprintf(&c, "static void (*const kernels[])(uint32_t,uint8_t*) = {%s};\n", strings.Join(names, ",")) + fmt.Fprintf(&c, "static const int offsets[] = {%s};\n", strings.Join(offsets, ",")) + c.WriteString(` +int main(void) { + static const uint32_t values[] = {0,1,0xffffffffu,0x87654321u}; + __asm__ volatile("emms" ::: "memory"); + for (unsigned k=0;k=(unsigned)offsets[k] && byte<(unsigned)offsets[k]+8) { + want=(uint8_t)((uint64_t)values[v] >> (8*(byte-offsets[k]))); + } + if (out[byte]!=want) { + fprintf(stderr,"kernel=%u value=%x byte=%u got=%x want=%x\n", + k,values[v],byte,out[byte],want); + return 1; + } + } + _Alignas(16) uint8_t state[512]; + __asm__ volatile("fxsave %0" : "=m"(state)); + if (state[4]!=0) { + fprintf(stderr,"kernel=%u value=%x x87 tag=%x\n",k,values[v],state[4]); + return 1; + } + } + } + return 0; +} +`) + fmt.Fprintf(&goTest, "package oracle\nimport \"testing\"\nvar kernels=[]func(uint32,*[128]byte){%s}\n", strings.Join(names, ",")) + fmt.Fprintf(&goTest, "var offsets=[]int{%s}\n", strings.Join(offsets, ",")) + goTest.WriteString(` +func TestOriginalStreamingStores(t *testing.T) { + for k,run:=range kernels { + for _,value:=range []uint32{0,1,0xffffffff,0x87654321} { + var aligned struct { word uint64; out [128]byte } + for index:=range aligned.out { aligned.out[index]=0xa5 } + run(value,&aligned.out) + for index,got:=range aligned.out { + want:=byte(0xa5) + if index>=offsets[k] && index> (8*(index-offsets[k]))) + } + if got!=want { t.Fatalf("kernel=%d value=%x byte=%d got=%x want=%x",k,value,index,got,want) } + } + } + } +} +`) + return declarations.String(), assembly.String(), c.String(), goTest.String() +} + +func TestX86NativeMMXStreamingOriginalGoBuildOnly(t *testing.T) { + declarations, source, _, tests := x86MMXStreamingOracleSources() + x86PackedIntegerCompareOriginalGoDriver(t, declarations, source, tests, true) +} diff --git a/amd64_needed.go b/amd64_needed.go index 9a52b4df8..c97a638ab 100644 --- a/amd64_needed.go +++ b/amd64_needed.go @@ -2,18 +2,30 @@ package plan9asm import "strings" -// funcNeedsAMD64CFG decides whether we need the CFG-based amd64 translator. -// The linear prototype cannot handle labels/branches and most vector ops. +// funcNeedsAMD64CFG decides whether the direct LLVM-module prototype can +// represent an amd64 function. Textual translation always uses the complete +// architecture-aware x86 CFG lowerer. func funcNeedsAMD64CFG(fn Func) bool { for _, ins := range fn.Instrs { + if _, known, _ := x86GPSIMDTransferProperties(ins, "amd64"); known { + return true // The scalar prototype has no X-bank or encoding state. + } if ins.Op == OpLABEL { return true } op := strings.ToUpper(string(ins.Op)) + for _, operand := range ins.Args { + if _, _, ok := x86MachineRegisterOperand(operand); ok { + return true + } + if x86SegmentRegisterOperand(operand) { + return true + } + } switch Op(op) { - case "JMP", "JL", "JLT", "JLE", "JG", "JGT", "JGE", + case "JMP", "JCXZW", "JCXZL", "JCXZQ", "JL", "JLT", "JLE", "JG", "JGT", "JGE", "JB", "JLO", "JBE", "JA", "JHI", "JAE", "JHS", - "JZ", "JE", "JEQ", "JNZ", "JNE", "JNC", "JC", "JCC", "JLS", "JNA", "JS", "JNS": + "JZ", "JE", "JEQ", "JNZ", "JNE", "JNC", "JC", "JCS", "JCC", "JLS", "JNA", "JS", "JNS": return true } // A handful of amd64 stdlib asm functions are straight-line, but if we @@ -21,10 +33,10 @@ func funcNeedsAMD64CFG(fn Func) bool { if strings.HasPrefix(op, "MOVO") || strings.HasPrefix(op, "PCLMUL") || strings.HasPrefix(op, "CRC32") || strings.HasPrefix(op, "PXOR") { return true } - // Keep the linear path only for the tiny subset it currently lowers. + // Keep the direct-module path only for the tiny subset it currently lowers. switch Op(op) { case OpTEXT, OpBYTE, OpMOVQ, OpMOVL, OpADDQ, OpSUBQ, OpXORQ, OpCPUID, OpXGETBV: - // For MOVQ/MOVL, linear lowering supports immediate/reg/FP value flow. + // For MOVQ/MOVL, direct lowering supports immediate/reg/FP value flow. // Addressing forms (mem/sym) require CFG lowering. if (op == "MOVQ" || op == "MOVL") && len(ins.Args) == 2 { for _, a := range ins.Args { diff --git a/amd64_new_opfamilies_test.go b/amd64_new_opfamilies_test.go index 096e191de..4559cebbe 100644 --- a/amd64_new_opfamilies_test.go +++ b/amd64_new_opfamilies_test.go @@ -1,9 +1,8 @@ -//go:build !llgo -// +build !llgo - package plan9asm import ( + "errors" + "fmt" "strings" "testing" ) @@ -16,28 +15,20 @@ TEXT notb(SB),NOSPLIT,$0-0 NOTB 3(AX) RET ` - file, err := Parse(ArchAMD64, src) - if err != nil { - t.Fatal(err) - } - ir, err := Translate(file, Options{ - TargetTriple: "x86_64-unknown-linux-gnu", - Sigs: map[string]FuncSig{ - "notb": {Name: "notb", Ret: Void}, - }, - Goarch: "amd64", - }) - if err != nil { - t.Fatal(err) - } + ir := x86UnboundVoidScalarFormatObject(t, src, "notb", "amd64", "x86_64-unknown-linux-gnu") if got := strings.Count(ir, "xor i8"); got != 2 { t.Fatalf("NOTB xor count = %d, want 2:\n%s", got, ir) } - for _, want := range []string{"and i64", "or i64", "load i8", "store i8"} { + for _, want := range []string{"and i64", "or i64", "load volatile i8", "store volatile i8"} { if !strings.Contains(ir, want) { t.Fatalf("NOTB output missing %q:\n%s", want, ir) } } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "not-byte-forms.ll", "not-byte-forms.o", ir) } func TestTranslateAMD64SHA1Family(t *testing.T) { @@ -82,24 +73,17 @@ TEXT crcops(SB),NOSPLIT,$0-0 VZEROUPPER RET ` - file, err := Parse(ArchAMD64, src) - if err != nil { - t.Fatal(err) - } - _, err = Translate(file, Options{ - TargetTriple: "x86_64-unknown-linux-gnu", - Sigs: map[string]FuncSig{ - "crcops": {Name: "crcops", Ret: Void}, - }, - Goarch: "amd64", - }) - if err != nil { - t.Fatal(err) + // This original format sequence does not supply AX, X0 or Z11. + // Its object checks must not invent those public-entry inputs. + ir := x86UnboundVoidScalarFormatObject(t, src, "crcops", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "crc-subset-forms.ll", "crc-subset-forms.o", ir) } -func TestTranslateAMD64AVX512Go126Families(t *testing.T) { - src := ` +const x86Go126FamiliesSource = ` GLOBL mat<>(SB), RODATA, $64 DATA mat<>+0(SB)/8, $0 @@ -116,18 +100,82 @@ TEXT go126ops(SB),NOSPLIT,$0-0 VPCOMPRESSQ Z1, K1, Z2 RET ` - file, err := Parse(ArchAMD64, src) - if err != nil { - t.Fatal(err) + +var x86Go126FamiliesInputs = []string{"Z0", "Z1", "Z2", "Z3", "Z4", "Z5", "Z6", "Z15"} + +func x86Go126FamiliesInitializedSource(omit string) string { + var initialization strings.Builder + for _, reg := range x86Go126FamiliesInputs { + if reg != omit { + fmt.Fprintf(&initialization, "VPXORD %s,%s,%s\n", reg, reg, reg) + } } - _, err = Translate(file, Options{ - TargetTriple: "x86_64-unknown-linux-gnu", - Sigs: map[string]FuncSig{ - "go126ops": {Name: "go126ops", Ret: Void}, - }, - Goarch: "amd64", - }) - if err != nil { - t.Fatal(err) + return strings.Replace(x86Go126FamiliesSource, "\tMOVQ $0xff, AX", initialization.String()+"\tMOVQ $0xff, AX", 1) +} + +func TestTranslateAMD64AVX512Go126Families(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc is required") + } + source := x86Go126FamiliesInitializedSource("") + // Supply Go's textflag constants without changing the original body or + // relying on an ambient include search path in the cross-object helper. + oracleSource := "#define NOSPLIT 4\n#define RODATA 8\n" + source + goAssemblerOracleOutput(t, "amd64", oracleSource, "", true) + for _, target := range dataRelocationTargets { + if target.goarch != "amd64" { + continue + } + t.Run(target.triple, func(t *testing.T) { + dataRelocationGoObject(t, oracleSource, target.goos, target.goarch, true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func go126ops()", target.goarch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("initialized whole instruction sequence rejected", err) + } + x86CompactExpandRequireAllDefinitions(t, ir, []string{"go126ops"}) + for _, optimized := range []bool{false, true} { + objectIR := ir + if optimized { + objectIR = x86DwordProductIROptimize(t, ir) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR-O2=%t/backend%s", optimized, level), func(t *testing.T) { + x86PackedStringCompileObject(t, llc, target.triple, objectIR, level) + }) + } + } + }) + } + }) + } +} + +func TestX86Go126FamiliesCannotBorrowUninitializedVectorInputs(t *testing.T) { + sources := map[string]string{"original": x86Go126FamiliesSource} + for _, reg := range x86Go126FamiliesInputs { + sources["missing-"+reg] = x86Go126FamiliesInitializedSource(reg) + } + sources["missing-mask"] = strings.Replace(x86Go126FamiliesInitializedSource(""), "KMOVQ AX, K1", "NOP", 1) + for name, source := range sources { + t.Run(name, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "#include \"textflag.h\"\n"+source, "", true) + for _, target := range dataRelocationTargets { + if target.goarch != "amd64" { + continue + } + t.Run(target.triple, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + if _, err := x86ScalarFPBitViewAPI(t, source, "func go126ops()", target.goarch, target.triple, api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unknown source acquired implementation-slot bytes", err) + } + }) + } + }) + } + }) } } diff --git a/amd64_nontemporal_vector_forms.go b/amd64_nontemporal_vector_forms.go new file mode 100644 index 000000000..dfeef6539 --- /dev/null +++ b/amd64_nontemporal_vector_forms.go @@ -0,0 +1,119 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86NonTemporalVectorSpec struct { + native string + vector bool + load bool +} + +// Complete Go yxr_ml/ylddqu and _yvmovntdq/_yvmovntdqa rows. MOVNTDQ +// is Go's source alias of MOVNTO, not a distinct physical MMX operation. +var x86NonTemporalVectorSpecs = map[Op]x86NonTemporalVectorSpec{ + "MOVNTO": {"movntdq", false, false}, + "MOVNTDQ": {"movntdq", false, false}, + "MOVNTPD": {"movntpd", false, false}, + "MOVNTPS": {"movntps", false, false}, + "MOVNTDQA": {"movntdqa", false, true}, + "VMOVNTDQ": {"vmovntdq", true, false}, + "VMOVNTPD": {"vmovntpd", true, false}, + "VMOVNTPS": {"vmovntps", true, false}, + "VMOVNTDQA": {"vmovntdqa", true, true}, +} + +type x86NonTemporalVectorForm struct { + spec x86NonTemporalVectorSpec + move amd64PackedVectorMoveForm + trap bool +} + +func x86NonTemporalVectorProperties(op Op, ins Instr, arch string) (x86NonTemporalVectorForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86NonTemporalVectorSpecs[Op(name)] + form := x86NonTemporalVectorForm{spec: spec, move: amd64PackedVectorMoveForm{ + spec: amd64PackedVectorMoveSpec{elementBits: 8, encoding: amd64PackedMoveVEXOrEVEX, aligned: true}, + memory: 1, destination: 1, mask: -1, memoryWrite: !spec.load, + }} + if !known { + return form, false, nil + } + fail := func() (x86NonTemporalVectorForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go non-temporal vector grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != 2 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return fail() + } + vector, memory := ins.Args[0], ins.Args[1] + if spec.load { + vector, memory, form.move.memory = ins.Args[1], ins.Args[0], 0 + } + form.move.byteWidth = amd64VectorByteWidth(vector.Reg) + index, _ := amd64VectorRegisterIndex(vector.Reg, form.move.byteWidth) + if vector.Kind != OpReg || !amd64GoPackedVectorMoveRegister(vector, form.move.byteWidth, arch) || + !spec.vector && (form.move.byteWidth != 16 || !amd64VEXVectorRegister(vector, 16) || + arch == "386" && index >= 8) { + return fail() + } + if !spec.vector && !spec.load && memory.Kind == OpReg && isX86YrlRegisterForArch(memory.Reg, arch) { + form.trap, form.move.memory = true, -1 + return form, true, nil // Go's Yml accepts ModRM.mod=3; the ISA raises #UD before inputs. + } + if !isAMD64MemoryOperand(memory) || memory.Kind == OpSym && strings.HasPrefix(memory.Sym, "$") { + return fail() + } + if memory.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(memory.Mem, arch); err != nil { + return form, true, err + } + if memory.Mem.OffRaw != "" || memory.Mem.IndexExt != "" { + return fail() + } + if arch == "386" && strings.HasSuffix(memory.Mem.Sym, "(SB)") { + memory.Mem.Base = "" // Go ignores Reg on this symbol role, not Index. + } + } + if !x86PackedShiftMemoryValid(memory, arch) { + return fail() + } + return form, true, nil +} + +// Only the paired native instruction supplies a continuation. Standalone +// queries cannot replace native alignment, non-temporal ordering or backing +// with initialized logical FP bytes or an incoming pointer declaration. +func x86ResultNonTemporalVectorStep(ins Instr, state *x86ResultState, owned []bool, + resultOffset int64, arch string, native bool) (bool, bool) { + form, known, err := x86NonTemporalVectorProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + if form.trap { + return true, true // Control-flow checks separately recognize this exact terminal #UD. + } + memory := ins.Args[form.move.memory] + if !native || !x86ResultNamedPredicateInstruction(ins) || + x86Mode32VectorRegisterContext(ins, arch) != nil || memory.Kind != OpMem || memory.Mem.Sym != "" { + return false, true + } + width := int64(form.move.byteWidth) + if !form.spec.load { + index, valid := amd64VectorRegisterIndex(ins.Args[0].Reg, form.move.byteWidth) + mask := x86CallerVectorByteMask(width) + return x86ResultIncomingPointerAddressIdentity(memory, state, arch) && + valid && state.vectorBytes(index)&mask == mask, true + } + if !x86ResultIncomingPointerReadAddressIdentity(memory, state, arch) { + return false, true + } + index, valid := amd64VectorRegisterIndex(ins.Args[1].Reg, form.move.byteWidth) + if !valid { + return false, true + } + state.writeVector(index, form.move.byteWidth, x86CallerVectorByteMask(width), !form.spec.vector) + return true, true +} diff --git a/amd64_operand_forms_test.go b/amd64_operand_forms_test.go index dde7de5cc..d3868edbe 100644 --- a/amd64_operand_forms_test.go +++ b/amd64_operand_forms_test.go @@ -1,9 +1,9 @@ -//go:build !llgo -// +build !llgo - package plan9asm -import "testing" +import ( + "strings" + "testing" +) func TestTranslateAMD64PEXTRDToMemory(t *testing.T) { src := ` @@ -11,20 +11,7 @@ TEXT pextrdmem(SB),NOSPLIT,$0-0 PEXTRD $3, X5, 16(DI) RET ` - file, err := Parse(ArchAMD64, src) - if err != nil { - t.Fatal(err) - } - _, err = Translate(file, Options{ - TargetTriple: "x86_64-unknown-linux-gnu", - Sigs: map[string]FuncSig{ - "pextrdmem": {Name: "pextrdmem", Ret: Void}, - }, - Goarch: "amd64", - }) - if err != nil { - t.Fatal(err) - } + x86UnboundVoidScalarFormatObject(t, src, "pextrdmem", "amd64", "x86_64-unknown-linux-gnu") } func TestTranslateAMD64PEXTRQToMemory(t *testing.T) { @@ -33,20 +20,7 @@ TEXT pextrqmem(SB),NOSPLIT,$0-0 PEXTRQ $1, X5, 16(DI) RET ` - file, err := Parse(ArchAMD64, src) - if err != nil { - t.Fatal(err) - } - _, err = Translate(file, Options{ - TargetTriple: "x86_64-unknown-linux-gnu", - Sigs: map[string]FuncSig{ - "pextrqmem": {Name: "pextrqmem", Ret: Void}, - }, - Goarch: "amd64", - }) - if err != nil { - t.Fatal(err) - } + x86UnboundVoidScalarFormatObject(t, src, "pextrqmem", "amd64", "x86_64-unknown-linux-gnu") } func TestTranslateAMD64VectorBroadcastForms(t *testing.T) { @@ -60,20 +34,12 @@ TEXT broadcastvec(SB),NOSPLIT,$0-0 VGF2P8AFFINEQB $0, Y0, Y1, Y1 RET ` - file, err := Parse(ArchAMD64, src) - if err != nil { - t.Fatal(err) - } - _, err = Translate(file, Options{ - TargetTriple: "x86_64-unknown-linux-gnu", - Sigs: map[string]FuncSig{ - "broadcastvec": {Name: "broadcastvec", Ret: Void}, - }, - Goarch: "amd64", - }) - if err != nil { - t.Fatal(err) + ir := x86UnboundVoidScalarFormatObject(t, src, "broadcastvec", "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "vector-broadcast-forms.ll", "vector-broadcast-forms.o", ir) } func TestAMD64VectorLoweringBranchCoverage(t *testing.T) { @@ -99,36 +65,75 @@ func TestAMD64VectorLoweringBranchCoverage(t *testing.T) { } } - check("gf2-z-source-class", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Z0"), reg("X1"), reg("Z2")}}, false, false) + check("gf2-z-source-class", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Z0"), reg("X1"), reg("Z2")}}, true, true) check("gf2-z-source-error", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Z0"), ident("bad"), reg("Z2")}}, true, true) - check("gf2-y-source-class", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), reg("X1"), reg("Y2")}}, false, false) + check("gf2-y-source-class", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), reg("X1"), reg("Y2")}}, true, true) check("gf2-y-source-error", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), ident("bad"), reg("Y2")}}, true, true) - check("gf2-unsupported-dst", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), reg("Y1"), reg("X2")}}, false, false) + check("gf2-unsupported-dst", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), reg("Y1"), reg("X2")}}, true, true) - check("broadcast-i128-args", "VBROADCASTI128", Instr{Args: []Operand{reg("X0")}}, true, true) - check("broadcast-i128-dst", "VBROADCASTI128", Instr{Args: []Operand{reg("X0"), reg("X1")}}, false, false) - check("broadcast-i128-source", "VBROADCASTI128", Instr{Args: []Operand{ident("bad"), reg("Y1")}}, true, true) check("broadcast-scalar-args", "VBROADCASTSD", Instr{Args: []Operand{imm(1)}}, true, true) check("broadcast-scalar-source", "VBROADCASTSD", Instr{Args: []Operand{ident("bad"), reg("Y1")}}, true, true) - check("broadcast-scalar-immediate", "VBROADCASTSD", Instr{Args: []Operand{imm(1), reg("Y1")}}, false, false) - check("broadcast-scalar-gpr", "VBROADCASTSD", Instr{Args: []Operand{reg("AX"), reg("Y1")}}, false, false) + check("broadcast-scalar-immediate", "VBROADCASTSD", Instr{Args: []Operand{imm(1), reg("Y1")}}, true, true) + check("broadcast-scalar-gpr", "VBROADCASTSD", Instr{Args: []Operand{reg("AX"), reg("Y1")}}, true, true) check("broadcast-scalar-x-source", "VBROADCASTSD", Instr{Args: []Operand{reg("X0"), reg("Y1")}}, false, true) - check("broadcast-scalar-x-dst", "VBROADCASTSD", Instr{Args: []Operand{reg("X0"), reg("X1")}}, false, false) - check("broadcast-scalar-dst", "VBROADCASTSD", Instr{Args: []Operand{mem("AX"), reg("AX")}}, false, false) + check("broadcast-scalar-x-dst", "VBROADCASTSD", Instr{Args: []Operand{reg("X0"), reg("X1")}}, true, true) + check("broadcast-scalar-dst", "VBROADCASTSD", Instr{Args: []Operand{mem("AX"), reg("AX")}}, true, true) check("broadcast-q-x", "VPBROADCASTQ", Instr{Args: []Operand{reg("X0"), reg("X1")}}, false, true) check("broadcast-q-gpr", "VPBROADCASTQ", Instr{Args: []Operand{reg("AX"), reg("X1")}}, false, true) check("broadcast-q-immediate", "VPBROADCASTQ", Instr{Args: []Operand{imm(1), reg("X1")}}, false, false) check("broadcast-q-address", "VPBROADCASTQ", Instr{Args: []Operand{sym("$value(SB)"), reg("X1")}}, false, false) - check("pmululq-args", "PMULULQ", Instr{Args: []Operand{reg("X0")}}, true, true) - check("pmululq-dst", "PMULULQ", Instr{Args: []Operand{reg("X0"), reg("Y1")}}, false, false) - check("pmululq-src", "PMULULQ", Instr{Args: []Operand{reg("Y0"), reg("X1")}}, false, false) - check("pmululq-source-error", "PMULULQ", Instr{Args: []Operand{ident("bad"), reg("X1")}}, true, true) - check("pmululq-memory", "PMULULQ", Instr{Args: []Operand{mem("AX"), reg("X1")}}, false, true) + + checkDwordProduct := func(name string, ins Instr, wantErr bool) { + t.Helper() + typed, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + typed.targetTriple = "x86_64-unknown-linux-gnu" + before, temporary := output.String(), typed.tmp + known, terminated, err := typed.lowerDwordToQwordMultiply("PMULULQ", ins) + if !known || terminated || (err != nil) != wantErr { + t.Fatalf("%s: typed dword product = (%v, %v, %v)", name, known, terminated, err) + } + if wantErr && (output.String() != before || typed.tmp != temporary) { + t.Fatalf("%s: invalid dword product mutated IR or temporary state", name) + } + if !wantErr && !strings.Contains(output.String(), "pmuludq") { + t.Fatalf("%s: original native m128 product opcode was lost", name) + } + } + + checkDwordProduct("pmululq-args", Instr{Args: []Operand{reg("X0")}}, true) + checkDwordProduct("pmululq-dst", Instr{Args: []Operand{reg("X0"), reg("Y1")}}, true) + checkDwordProduct("pmululq-src", Instr{Args: []Operand{reg("Y0"), reg("X1")}}, true) + checkDwordProduct("pmululq-source-error", Instr{Args: []Operand{ident("bad"), reg("X1")}}, true) + checkDwordProduct("pmululq-memory", Instr{Args: []Operand{mem("AX"), reg("X1")}}, false) + check("punpckllq-args", "PUNPCKLLQ", Instr{Args: []Operand{reg("X0")}}, true, true) check("punpckhlq-dst", "PUNPCKHLQ", Instr{Args: []Operand{reg("X0"), reg("Y1")}}, false, false) check("punpckllq-src", "PUNPCKLLQ", Instr{Args: []Operand{reg("Y0"), reg("X1")}}, false, false) check("punpckhlq-source-error", "PUNPCKHLQ", Instr{Args: []Operand{ident("bad"), reg("X1")}}, true, true) check("punpckllq-memory", "PUNPCKLLQ", Instr{Args: []Operand{mem("AX"), reg("X1")}}, false, true) + for _, op := range []Op{"PMINUB", "PMINSB", "PMINUW", "PMINSW", "PMINUD", "PMINSD", "PMAXUB", "PMAXSB", "PMAXUW", "PMAXSW", "PMAXUD", "PMAXSD"} { + check(string(op)+"-args", op, Instr{Args: []Operand{reg("X0")}}, true, true) + invalid := Instr{Args: []Operand{reg("X0"), reg("Y1")}} + rejected, rejectionIR := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before := rejectionIR.String() + known, terminated, err := rejected.lowerVec(op, invalid) + if !known || terminated || err == nil || rejectionIR.String() != before { + t.Fatalf("%s-dst: invalid legacy width must reject before emitting IR: known=%v terminated=%v err=%v", op, known, terminated, err) + } + check(string(op)+"-source", op, Instr{Args: []Operand{ident("bad"), reg("X1")}}, true, true) + // The private format emitter retains the original native m128 opcode; + // this test must supply its actual x86 target, not virtual storage. + previousTarget := c.targetTriple + c.targetTriple = "x86_64-unknown-linux-gnu" + check(string(op)+"-memory", op, Instr{Args: []Operand{mem("AX"), reg("X1")}}, false, true) + c.targetTriple = previousTarget + } + for _, op := range []Op{"PSLLQ", "PSRLQ"} { + check(string(op)+"-args", op, Instr{Args: []Operand{imm(1)}}, true, true) + check(string(op)+"-dst", op, Instr{Args: []Operand{imm(1), reg("Y1")}}, false, false) + check(string(op)+"-count", op, Instr{Args: []Operand{reg("AX"), reg("X1")}}, true, true) + check(string(op)+"-negative", op, Instr{Args: []Operand{imm(-1), reg("X1")}}, true, true) + } check("xorpd-args", "VXORPD", Instr{Args: []Operand{reg("X0")}}, true, true) check("xorpd-z", "VXORPD", Instr{Args: []Operand{reg("Z0"), reg("Z1"), reg("Z2")}}, false, true) @@ -140,11 +145,11 @@ func TestAMD64VectorLoweringBranchCoverage(t *testing.T) { check("xorpd-x", "VXORPD", Instr{Args: []Operand{reg("X0"), reg("X1"), reg("X2")}}, false, true) check("xorpd-x-source1", "VXORPD", Instr{Args: []Operand{ident("bad"), reg("X1"), reg("X2")}}, true, true) check("xorpd-x-source2", "VXORPD", Instr{Args: []Operand{reg("X0"), ident("bad"), reg("X2")}}, true, true) - check("xorpd-unsupported-dst", "VXORPD", Instr{Args: []Operand{reg("X0"), reg("X1"), reg("AX")}}, false, false) + check("xorpd-unsupported-dst", "VXORPD", Instr{Args: []Operand{reg("X0"), reg("X1"), reg("AX")}}, true, true) - check("ternlog-y-source1-class", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("X0"), reg("Y1"), reg("Y2")}}, false, false) - check("ternlog-y-source2-class", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("Y0"), reg("X1"), reg("Y2")}}, false, false) - check("ternlog-y-imm", "VPTERNLOGD", Instr{Args: []Operand{imm(0x95), reg("Y0"), reg("Y1"), reg("Y2")}}, true, true) + check("ternlog-y-source1-class", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("X0"), reg("Y1"), reg("Y2")}}, true, true) + check("ternlog-y-source2-class", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("Y0"), reg("X1"), reg("Y2")}}, true, true) + check("ternlog-y-imm", "VPTERNLOGD", Instr{Args: []Operand{imm(0x95), reg("Y0"), reg("Y1"), reg("Y2")}}, false, true) check("ternlog-y-source1", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), ident("bad"), reg("Y1"), reg("Y2")}}, true, true) check("ternlog-y-source2", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("Y0"), ident("bad"), reg("Y2")}}, true, true) diff --git a/amd64_ordering_hint_family_test.go b/amd64_ordering_hint_family_test.go new file mode 100644 index 000000000..fbec94b34 --- /dev/null +++ b/amd64_ordering_hint_family_test.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "bytes" + "reflect" + "strings" + "testing" +) + +// Independent complete Go ynone rows, not an implementation-spec projection. +var x86OrderingHintOracleRows = []struct { + op Op + registration string + machine []byte +}{ + {"LFENCE", "{ALFENCE, ynone, Pm, opBytes{0xae, 0xe8}}", []byte{0x0f, 0xae, 0xe8}}, + {"MFENCE", "{AMFENCE, ynone, Pm, opBytes{0xae, 0xf0}}", []byte{0x0f, 0xae, 0xf0}}, + {"SFENCE", "{ASFENCE, ynone, Pm, opBytes{0xae, 0xf8}}", []byte{0x0f, 0xae, 0xf8}}, + {"PAUSE", "{APAUSE, ynone, Px, opBytes{0xf3, 0x90}}", []byte{0xf3, 0x90}}, +} + +func TestX86OrderingHintsHaveExactEmptySourceEffects(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86OrderingHintOracleRows { + t.Run(arch+"/"+string(row.op), func(t *testing.T) { + ins := Instr{Op: row.op} + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 0 || effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatalf("zero-operand instruction lacks exact no-output/no-flags source effect: %+v/%t/%v", effect, known, err) + } + move, suffix := Op("MOVQ"), "Q" + if arch == "386" { + move, suffix = "MOVL", "L" + } + if !x86SourceStackInstructionKnown(ins, move, suffix, arch) { + t.Fatal("valid typed instruction has an unregistered source-stack effect") + } + result := x86ResultState{flags: x86ByteFlagsAll} + before := result + if !x86ResultTransfer(ins, &result, nil, 0, arch) || !reflect.DeepEqual(before, result) { + t.Fatal("exact no-output instruction changed result initialization/flags") + } + caller := x86SourceStackState{numericFlags: true, known: map[Reg]uint8{AX: 0xff}} + flags, valid := x86CallerScalarStep(&ins, &caller, nil) + if !valid || flags || !caller.numericFlags || caller.known[AX] != 0xff { + t.Fatal("exact no-output instruction changed caller facts") + } + }) + } + } +} + +func TestX86OrderingHintsActualInstructionsNotNoops(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86OrderingHintOracleRows { + t.Run(arch+"/"+string(row.op), func(t *testing.T) { + source := "TEXT fence(SB),4,$0-0\n" + string(row.op) + "\nRET\n" + actual := x86OrderingHintGoBytes(t, arch, source) + want := append(append([]byte(nil), row.machine...), 0xc3) + if !bytes.Equal(actual, want) { + t.Fatalf("actual Go encoding %x, want %x", actual, want) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + start := output.Len() + known, terminated, err := ctx.lowerArith(row.op, file.Funcs[0].Instrs[1]) + if !known || terminated || err != nil { + t.Fatal("accepted complete zero-operand form", err) + } + if !strings.Contains(output.String()[start:], "asm sideeffect \""+strings.ToLower(string(row.op))+"\"") { + t.Fatal("actual ordering/hint instruction became a software no-op") + } + }) + } + } +} + +func TestX86OrderingHintsIllegalOperandsRejectBeforeEmission(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86OrderingHintOracleRows { + for _, argument := range []string{"AX", "(AX)", "$1", "AX,DX"} { + t.Run(arch+"/"+string(row.op)+"/"+argument, func(t *testing.T) { + source := "TEXT fence(SB),4,$0-0\n" + string(row.op) + " " + argument + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + known, terminated, err := ctx.lowerArith(row.op, file.Funcs[0].Instrs[1]) + if !known || terminated || err == nil || before != output.String() || temporary != ctx.tmp { + t.Fatalf("illegal operand did not fail before emission: %t/%t/%v", known, terminated, err) + } + }) + } + } + } +} diff --git a/amd64_ordering_hint_forms.go b/amd64_ordering_hint_forms.go new file mode 100644 index 000000000..1284bdb0e --- /dev/null +++ b/amd64_ordering_hint_forms.go @@ -0,0 +1,44 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86OrderingHintSpec struct { + native string +} + +// The complete Go ynone axis has no explicit or implicit data operands. +// Hardware ordering/hint execution is not a source-memory permission. +var x86OrderingHintSpecs = map[Op]x86OrderingHintSpec{ + "LFENCE": {native: "lfence"}, + "MFENCE": {native: "mfence"}, + "SFENCE": {native: "sfence"}, + "PAUSE": {native: "pause"}, +} + +func x86OrderingHintProperties(op Op, ins Instr, arch string) (x86OrderingHintSpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86OrderingHintSpecs[Op(name)] + if !known { + return x86OrderingHintSpec{}, false, nil + } + if dotted || arch != "386" && arch != "amd64" || len(ins.Args) != 0 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return spec, true, fmt.Errorf("%s %s requires its exact zero-operand Go ordering/hint form: %q", arch, op, ins.Raw) + } + return spec, true, nil +} + +func (c *amd64Ctx) lowerX86OrderingHint(op Op, ins Instr) (bool, bool, error) { + spec, known, err := x86OrderingHintProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + // Conservative compiler memory scheduling is separate from architectural + // semantics: these instructions have no source data or arithmetic outputs. + // A litmus sample is not a proof of the hardware memory model. + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", spec.native, "~{memory}") + return true, false, nil +} diff --git a/amd64_ordering_hint_linux_amd64_test.go b/amd64_ordering_hint_linux_amd64_test.go new file mode 100644 index 000000000..0b92421ce --- /dev/null +++ b/amd64_ordering_hint_linux_amd64_test.go @@ -0,0 +1,80 @@ +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +// Required Linux/amd64 CI runs the actual static ELF32 objects directly. The +// host/ELF checks reject emulation; no local Rosetta run establishes this axis. +func TestCrossLinuxRuntimeMatrixX86OrderingHintsNative386(t *testing.T) { + declarations, assembly, driver, reference := x86OrderingHintRuntimeFixture(32) + x86OrderingHintNative386(t, declarations, assembly, driver, reference) +} + +func TestCrossLinuxRuntimeMatrixX86RawNOPNative386(t *testing.T) { + declarations, assembly, driver, reference := x86RawNOPRuntimeFixture(32) + x86OrderingHintNative386(t, declarations, assembly, driver, reference) +} + +func TestCrossLinuxRuntimeMatrixX86RawNOPOriginalAMD64(t *testing.T) { + x87IA32RequireNativeHost(t) + declarations, assembly, driver, _ := x86RawNOPRuntimeFixture(64) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "linux", "amd64", declarations, assembly, driver) + if output := x86FloatCompareCaptureCommand(t, exec.Command(original)); string(output) != "original ordering observations 256\n" { + t.Fatalf("actual original Go AMD64 NOP state/no-memory oracle: %q", output) + } + x87IA32RequireNativeHost(t) +} + +func x86OrderingHintNative386(t *testing.T, declarations, assembly, driver, reference string) { + t.Helper() + x87IA32RequireNativeHost(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if err != nil { + t.Fatal("required native IA32 static C compiler:", err) + } + dir := t.TempDir() + original := x86OrderingHintBuildOriginal(t, dir, "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != "original ordering observations 256\n" { + t.Fatalf("unchanged original Go IA32 ordering/status/canary oracle: %q", output) + } + ir, err := x86OrderingHintTranslateAPI(t, "Go", assembly, declarations, "386", "i386-unknown-linux-gnu") + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := filepath.Join(dir, "ordering"+level+".ll") + object := filepath.Join(dir, "ordering"+level+".o") + main, executable := filepath.Join(dir, "main.c"), filepath.Join(dir, "ordering"+level) + for name, contents := range map[string]string{input: ir, main: reference} { + if err := os.WriteFile(name, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, + "-mtriple=i386-unknown-linux-gnu", "-filetype=obj", input, "-o", object)) + x86FloatCompareCaptureCommand(t, exec.Command(compiler, "-static", "-no-pie", + object, main, "-O2", "-o", executable)) + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != "ordering observations 256\n" { + t.Fatalf("LLVM IA32 GP/six-status-flags/MXCSR/canary oracle: %q", output) + } + }) + } + x87IA32RequireNativeHost(t) +} diff --git a/amd64_ordering_hint_rule_consumers_test.go b/amd64_ordering_hint_rule_consumers_test.go new file mode 100644 index 000000000..3117ec89a --- /dev/null +++ b/amd64_ordering_hint_rule_consumers_test.go @@ -0,0 +1,241 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +var x86OrderingHintTargets = []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, +} + +func x86OrderingHintGoBytes(t *testing.T, arch, source string) []byte { + t.Helper() + dir := t.TempDir() + input, object := filepath.Join(dir, "original.s"), filepath.Join(dir, "original.o") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + stage, cancelStage := runtimeOracleStageContext(t) + defer cancelStage() + ctx, cancel := context.WithTimeout(stage, time.Minute) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "asm", "-S", "-p", "ordering", "-o", object, input) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH="+arch, "GOFLAGS=") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), true, arch); failure != nil { + t.Fatalf("actual original Go object: %v\n%s", failure, output) + } + var code []byte + for _, line := range strings.Split(string(output), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") || len(fields[1]) != 2 { + continue + } + offset, encoded, err := goAssemblyListingByteRow(line) + if err != nil || offset != len(code) { + t.Fatalf("noncontiguous original object listing: %s", line) + } + code = append(code, encoded...) + } + if len(code) == 0 { + t.Fatal("actual original object listing lacks complete bytes") + } + return code +} + +func TestX86OrderingHintsCompleteGoRowsAndDispatch(t *testing.T) { + table, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(table), "var ynone = []ytab{\n\t{Zlit, 1, argList{}},\n}") || len(x86OrderingHintSpecs) != 4 { + t.Fatal("complete Go zero-operand family changed") + } + for _, row := range x86OrderingHintOracleRows { + if !strings.Contains(string(table), row.registration) { + t.Fatal("complete Go ordered row changed", row.registration) + } + for _, arch := range []string{"386", "amd64"} { + for _, identity := range []Op{"", row.op, Op(strings.ToLower(string(row.op))), "MOVQ", Op(string(row.op) + ".BAD")} { + t.Run(arch+"/"+string(row.op)+"/identity="+string(identity), func(t *testing.T) { + ins := Instr{Op: identity} + ctx, out := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := out.String(), ctx.tmp + registers := fmt.Sprint(ctx.usedRegs) + known, terminated, err := ctx.lowerX86OrderingHint(row.op, ins) + invalid := identity != "" && !strings.EqualFold(string(identity), string(row.op)) + if !known || terminated || (err != nil) != invalid || invalid && (before != out.String() || temporary != ctx.tmp) { + t.Fatal("identity rejection/emission boundary", identity, err) + } + if fmt.Sprint(ctx.usedRegs) != registers { + t.Fatal("zero-operand source introduced implicit data registers", ctx.usedRegs) + } + }) + } + for _, suffix := range []string{".Z", ".SAE", ".BCST", ".BAD"} { + ins := Instr{Op: Op(string(row.op) + suffix)} + if _, known, err := x86OrderingHintProperties(ins.Op, ins, arch); !known || err == nil { + t.Fatal("invalid suffix was not closed", ins, err) + } + } + } + for _, arch := range []string{"", "arm", "arm64", "wasm"} { + if _, known, err := x86OrderingHintProperties(row.op, Instr{Op: row.op}, arch); !known || err == nil { + t.Fatal("unrelated architecture admitted an x86 form", arch, err) + } + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before := output.String() + if known, terminated, err := ctx.lowerX86OrderingHint("UNKNOWN", Instr{}); known || terminated || err != nil || before != output.String() { + t.Fatal("unknown dispatch emitted a default instruction") + } +} + +func x86OrderingHintTranslateAPI(t *testing.T, api, source, declarations, arch, triple string) (string, error) { + t.Helper() + pkg := mustGoPackage(t, "ordering", declarations) + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + ctx := llvm.NewContext() + defer ctx.Dispose() + if api == "Go" { + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: resolve}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + return "", err + } + return translated.Module.String(), nil + } + file, err := Parse(ArchAMD64, source) + if err != nil { + return "", err + } + sigs, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + return "", err + } + options := Options{Goarch: arch, TargetTriple: triple, ResolveSym: resolve, Sigs: sigs} + if api == "text" { + return Translate(file, options) + } + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + return "", err + } + return module.String(), nil +} + +func TestX86OrderingHintsThreeAPIsFiveTargetNativeObjects(t *testing.T) { + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 llc and llvm-objdump required") + } + for _, target := range x86OrderingHintTargets { + for _, raw := range []bool{false, true} { + var source, declarations strings.Builder + declarations.WriteString("package ordering\n") + for _, row := range x86OrderingHintOracleRows { + name := strings.ToLower(string(row.op)) + fmt.Fprintf(&declarations, "func %s()\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\n", name) + if raw { + for _, value := range row.machine { + fmt.Fprintf(&source, "BYTE $%d\n", value) + } + } else { + fmt.Fprintln(&source, row.op) + } + source.WriteString("RET\n") + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(fmt.Sprintf("%s/raw=%t/%s", target.triple, raw, api), func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + dir := t.TempDir() + input, object := filepath.Join(dir, "ordering.ll"), filepath.Join(dir, "ordering.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+target.triple, "-filetype=obj", "-o", object, input)) + disassembly := x86FloatCompareCaptureCommand(t, exec.Command(disassembler, "--disassemble", "--no-show-raw-insn", object)) + for _, row := range x86OrderingHintOracleRows { + count := 0 + for _, line := range strings.Split(string(disassembly), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && strings.HasSuffix(fields[0], ":") && fields[1] == strings.ToLower(string(row.op)) { + count++ + } + } + if count != 1 { + t.Fatalf("%s object lost or duplicated native %s: count=%d\n%s", level, row.op, count, disassembly) + } + } + } + }) + } + } + } +} + +func TestX86RawPrefixThreeAPIClassification(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for _, row := range []struct { + name string + code []byte + context bool + }{ + {"invalid even with later ignored override", []byte{0xf0, 0x66, 0x0f, 0x31}, false}, + {"valid LOCK atomic effect unbound", []byte{0xf0, 0x01, 0x08}, true}, + {"legal ignored prefix", []byte{0x66, 0x66, 0x0f, 0x31}, false}, + } { + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+row.name+"/"+api, func(t *testing.T) { + source := "TEXT ·prefix(SB),4,$0-0\n" + x86RawDisplacementDirectives(row.code) + "RET\n" + ir, err := x86OrderingHintTranslateAPI(t, api, source, "package ordering\nfunc prefix()\n", target.arch, target.triple) + if row.name == "legal ignored prefix" { + if err != nil || !strings.Contains(ir, "asm sideeffect \"rdtsc\"") { + t.Fatal("legal ignored prefix lost its actual instruction", err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), target.triple, "ignored.ll", "ignored.o", ir) + } else if err == nil || ir != "" || errors.Is(err, ErrProbeNeedsContext) != row.context { + t.Fatalf("invalid versus valid-unbound prefix classification: IR=%q/error=%v", ir, err) + } + }) + } + } + } + physical, err := x86asm.Decode([]byte{0xf0, 0x0f, 0x31}, 64) + if err != nil { + t.Fatal(err) + } + physical.Prefix[0] |= x86asm.PrefixIgnored + if err := decodedX86PrefixError(physical); err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("synthetic Invalid|Ignored metadata tripwire washed invalidity", err) + } +} diff --git a/amd64_ordering_hint_runtime_test.go b/amd64_ordering_hint_runtime_test.go new file mode 100644 index 000000000..3e437284f --- /dev/null +++ b/amd64_ordering_hint_runtime_test.go @@ -0,0 +1,201 @@ +package plan9asm + +import ( + "debug/pe" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86OrderingHintsOriginalWindowsExecutable(t *testing.T) { + declarations, assembly, driver, _ := x86OrderingHintRuntimeFixture(64) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "windows", "amd64", declarations, assembly, driver) + if filepath.Ext(original) != ".exe" { + t.Fatal("Windows original oracle requires its exact executable suffix", original) + } + object, err := pe.Open(original) + if err != nil { + t.Fatal("actual Windows oracle executable", err) + } + defer object.Close() + if object.Machine != pe.IMAGE_FILE_MACHINE_AMD64 || object.Characteristics&pe.IMAGE_FILE_EXECUTABLE_IMAGE == 0 { + t.Fatal("original Windows oracle is not an AMD64 executable") + } +} + +// These observations establish exact GP, six status bits and MXCSR retention, +// not an empirical proof of the architecture's concurrent memory model. +func TestCrossLinuxRuntimeMatrixX86OrderingHintsOriginalAndLLVM(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, assembly, driver, reference := x86OrderingHintRuntimeFixture(64) + dir := t.TempDir() + original := x86OrderingHintBuildOriginal(t, dir, runtime.GOOS, "amd64", declarations, assembly, driver) + if output := execution.run(t, original); string(output) != "original ordering observations 256\n" { + t.Fatalf("actual original Go ordering/status/canary oracle: %q", output) + } + t.Log("256 original Go observations: four instructions, two complementary six-bit seeds, 32 repetitions") + ir := x86NarrowOracleIR(t, assembly, declarations, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "ordering", ir, reference, level) + if string(output) != "ordering observations 256\n" { + t.Fatalf("independent GP/full-status/MXCSR/canary oracle: %q", output) + } + t.Log("256 independent LLVM observations: five GPs, six status flags, MXCSR and canary") + }) + } +} + +func x86OrderingHintBuildOriginal(t *testing.T, dir, goos, arch, declarations, assembly, driver string) string { + t.Helper() + for name, contents := range map[string]string{ + "go.mod": "module ordering\n\ngo 1.20\n", + "decl.go": declarations, "ordering_" + arch + ".s": assembly, + "main.go": driver, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-go") + if goos == "windows" { + executable += ".exe" + } + command := exec.Command(filepath.Join(runtime.GOROOT(), "bin/go"), "build", "-p=2", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+arch, + "CGO_ENABLED=0", "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOEXPERIMENT=") + x86FloatCompareCaptureCommand(t, command) + return executable +} + +func TestX86OrderingHintsOriginal386StaticELFCompileOnly(t *testing.T) { + declarations, assembly, driver, _ := x86OrderingHintRuntimeFixture(32) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go IA32 ELF bytes=%d SHA256=%x; compile only, required native execution is separate", + sealed.size, sealed.hash) +} + +// The independent reference constants never come from the production spec. +// Each sample resets every output byte and both complementary status seeds. +func x86OrderingHintRuntimeFixture(bits int) (declarations, assembly, driver, reference string) { + word, suffix, pop, push := "uint64", "Q", "POPFQ", "PUSHFQ" + values := []string{"0x123456789abcdef0", "0x2468ace013579bdf", "0x3141592653589793", "0x7e6d5c4b3a291807", "0xa1b2c3d4e5f60718"} + canary := "0x13579bdf2468ace0" + if bits == 32 { + word, suffix, pop, push = "uint32", "L", "POPFL", "PUSHFL" + values = []string{"0x9abcdef0", "0x13579bdf", "0x53589793", "0x3a291807", "0xe5f60718"} + canary = "0x2468ace0" + } + var decl, asm, names strings.Builder + decl.WriteString("package main\n") + step := bits / 8 + for _, op := range []string{"LFENCE", "MFENCE", "SFENCE", "PAUSE"} { + for index, seed := range []uint64{0x8c7, 0x12} { + name := fmt.Sprintf("%s_%d", strings.ToLower(op), index) + fmt.Fprintf(&decl, "func %s(out *[8]%s)\n", name, word) + fmt.Fprintf(&names, "%s,\n", name) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$0-%d\nMOV%s out+0(FP),DI\n", name, step, suffix) + for r, reg := range []string{"AX", "BX", "CX", "DX", "SI"} { + fmt.Fprintf(&asm, "MOV%s $%s,%s\n", suffix, values[r], reg) + } + fmt.Fprintf(&asm, "PUSH%s $%d\n%s\n%s\n", suffix, seed, pop, op) + for r, reg := range []string{"AX", "BX", "CX", "DX", "SI"} { + fmt.Fprintf(&asm, "MOV%s %s,%d(DI)\n", suffix, reg, r*step) + } + fmt.Fprintf(&asm, "%s\nPOP%s SI\nMOV%s SI,%d(DI)\nSTMXCSR %d(DI)\nRET\n", push, suffix, suffix, 5*step, 6*step) + } + } + decl.WriteString("func csr() uint32\n") + asm.WriteString("TEXT ·csr(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\n") + driver = fmt.Sprintf(`package main + +import ( + "fmt" + "runtime" +) + +func main() { + runtime.LockOSThread() + samples := []func(*[8]%s){ + %s + } + want := [5]%s{%s} + seeds := [2]%s{0x8c7, 0x12} + observations := 0 + for index, sample := range samples { + for iteration := 0; iteration < 32; iteration++ { + out := [8]%s{0, 0, 0, 0, 0, 0, 0, %s} + before := csr() + sample(&out) + after := csr() + for register := 0; register < 5; register++ { + if out[register] != want[register] { + panic(fmt.Sprintf("original GP kernel=%%d register=%%d out=%%x", index, register, out)) + } + } + if out[5]&0x8d5 != seeds[index%%2]&0x8d5 || + out[6] != %s(before) || after != before || out[7] != %s { + panic(fmt.Sprintf("original status/CSR/canary kernel=%%d out=%%x before=%%x after=%%x", index, out, before, after)) + } + observations++ + } + } + fmt.Printf("original ordering observations %%d\n", observations) +} +`, word, names.String(), word, strings.Join(values, ", "), word, word, canary, word, canary) + var prototypes strings.Builder + for _, op := range []string{"lfence", "mfence", "sfence", "pause"} { + for index := 0; index < 2; index++ { + fmt.Fprintf(&prototypes, "extern void %s_%d(%s_t *);\n", op, index, word) + } + } + reference = fmt.Sprintf(` +#include +#include + +%s +int main(void) { + void (*samples[])(%s_t *) = { + %s + }; + const %s_t want[5] = {%s}; + const %s_t seeds[2] = {0x8c7, 0x12}; + unsigned observations = 0; + for (unsigned index = 0; index < 8; index++) { + for (unsigned iteration = 0; iteration < 32; iteration++) { + %s_t out[8] = {0, 0, 0, 0, 0, 0, 0, %s}; + unsigned before, after; + __asm__ volatile("stmxcsr %%0" : "=m"(before)); + samples[index](out); + __asm__ volatile("stmxcsr %%0" : "=m"(after)); + for (unsigned reg = 0; reg < 5; reg++) { + if (out[reg] != want[reg]) { + fprintf(stderr, "GP kernel=%%u register=%%u got=%%llx\n", + index, reg, (unsigned long long)out[reg]); + return 1; + } + } + if ((out[5] & 0x8d5) != (seeds[index %% 2] & 0x8d5) || + out[6] != before || after != before || out[7] != %s) { + fprintf(stderr, "status/CSR/canary kernel=%%u flags=%%llx CSR=%%llx/%%x/%%x\n", + index, (unsigned long long)out[5], (unsigned long long)out[6], before, after); + return 2; + } + observations++; + } + } + printf("ordering observations %%u\n", observations); + return 0; +} +`, prototypes.String(), word, names.String(), word, strings.Join(values, ", "), word, word, canary, canary) + return decl.String(), asm.String(), driver, reference +} diff --git a/amd64_ordinary_move_address_runtime_test.go b/amd64_ordinary_move_address_runtime_test.go new file mode 100644 index 000000000..c76e6b52f --- /dev/null +++ b/amd64_ordinary_move_address_runtime_test.go @@ -0,0 +1,78 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// The Go assembler owns the prologue and NAME_AUTO bias. A write through the +// formed address, followed by a physical-SP read, distinguishes a misplaced +// address even when an unrelated spill happens to contain the original input. +func TestCrossLinuxRuntimeMatrixGoAMD64NamedLocalAddressBias(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + goAMD64FramePrepareOwnedToolCache(t) + var assembly, declarations, goCalls, cDeclarations, cCalls strings.Builder + prefix := "" + if strings.Contains(execution.triple, "apple") { + prefix = "_" + } + for index, address := range []string{"MOVQ $slot-16(SP),BX", "LEAQ slot-16(SP),BX"} { + name := fmt.Sprintf("Address%d", index) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$16-16\nMOVQ input+0(FP),AX\nMOVQ AX,0(SP)\n%s\nMOVQ $0x13579bdf2468ace0,CX\nMOVQ CX,(BX)\nMOVQ 0(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n", name, address) + fmt.Fprintf(&declarations, "func %s(input uint64) uint64\n", name) + fmt.Fprintf(&goCalls, "check(%d,%s)\n", index, name) + fmt.Fprintf(&cDeclarations, "extern uint64_t address%d(uint64_t) __asm__(%s);\n", index, + runtimeCAsmLabel(prefix+"example.invalid/framefixture."+name)) + fmt.Fprintf(&cCalls, "check(%d,address%d);\n", index, index) + } + original := `package main +import "fmt" +` + declarations.String() + ` +func check(index int, call func(uint64) uint64) { + for sample,input := range []uint64{0,1,0xffffffffffffffff,0x0123456789abcdef} { + got := call(input) + if got != 0x13579bdf2468ace0 { panic("original Go NAME_AUTO address differs from physical SP") } + fmt.Printf("%d %d %016x\n",index,sample,got) + } +} +func main(){ +` + goCalls.String() + "}\n" + driver := `#include +#include +` + runtimeCAsmLabelPreamble + cDeclarations.String() + ` +static void check(int index, uint64_t (*call)(uint64_t)) { + const uint64_t inputs[]={0,1,UINT64_MAX,UINT64_C(0x0123456789abcdef)}; + for(unsigned sample=0; sample<4; sample++) { + printf("%d %u %016llx\n",index,sample,(unsigned long long)call(inputs[sample])); + } +} +int main(void){ +` + cCalls.String() + "return 0;}\n" + want := strings.TrimSpace(string(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly.String(), original))) + if len(strings.Split(want, "\n")) != 8 { + t.Fatal("original Go did not execute both address forms and all four inputs", want) + } + t.Log("original Go executed original MOVQ and LEAQ with its actual native frame/prologue: 8 observations") + pkg, request, _, options := fixedLocalTransportFixture(t, assembly.String(), declarations.String(), runtime.GOOS+"/amd64", execution.triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal("same original source lost its captured private-frame contract", err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + got := strings.TrimSpace(string(execution.compileAndRunLLVM(t, "named-local-address", input, driver, level))) + if got != want { + t.Errorf("original Go/LLVM named-address writeback differs:\nGo:\n%s\nLLVM:\n%s", want, got) + } + }) + } + }) +} diff --git a/amd64_ordinary_move_address_test.go b/amd64_ordinary_move_address_test.go new file mode 100644 index 000000000..561332ff7 --- /dev/null +++ b/amd64_ordinary_move_address_test.go @@ -0,0 +1,180 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "testing" +) + +func TestGoAMD64LocalAddressIgnoresUncertifiedAccessFields(t *testing.T) { + for _, source := range []string{"MOVQ $slot-16(SP),BX", "LEAQ slot-16(SP),BX"} { + // This is an isolated emitter-consumption counterexample, not an + // initialized native entry. The independent runtime owner captures + // and executes the complete actual Go source for both address forms. + emit := func(t *testing.T, stale string) string { + t.Helper() + ins := x86ScalarLaneInstruction(t, source) + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + proof := &x86FixedLocalProof{frame: 16, accesses: map[int]map[int]x86FixedLocalAccess{}} + ctx.fixedLocal = proof + if stale != "" { + shape, err := fixedLocalInstructionDigest(ins) + if err != nil { + t.Fatal(err) + } + access := x86FixedLocalAccess{owner: proof, shape: shape, width: 8, nativeInput: true} + if stale == "mismatched-record" { + expected := access + expected.nativeInput = false + proof.accesses[0] = map[int]x86FixedLocalAccess{0: expected} + } + ins.x86FixedLocalMemory = map[int]x86FixedLocalAccess{0: access} + if _, certified := ctx.fixedLocalMemoryAccess(ins, 0); certified { + t.Fatal("stale private fields unexpectedly authenticated themselves") + } + } + ctx.blocks = []amd64Block{{instrs: []Instr{ins}}} + if terminated, err := ctx.lowerInstr(0, 0, ins, nil, nil); terminated || err != nil { + t.Fatal("address-only emission borrowed an access contract", terminated, err) + } + if ctx.fixedMemRoles != nil { + t.Fatal("original-instruction address projection escaped its lifetime") + } + return output.String() + } + want := emit(t, "") + for _, stale := range []string{"missing-record", "mismatched-record"} { + t.Run(source+"/"+stale, func(t *testing.T) { + if got := emit(t, stale); got != want { + t.Error("unverified nativeInput changed address-only emission") + } + }) + } + } +} + +func TestGoAMD64FixedLocalOriginalMoveAddressDomains(t *testing.T) { + for _, width := range []string{"W", "L", "Q"} { + for _, old := range []uint8{0, 3, 0x55, 0xff} { + t.Run(fmt.Sprintf("%s/old%x", width, old), func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, "MOV"+width+" $slot-16(SP),BX") + original := fmt.Sprintf("%#v", ins) + state := x86FixedLocalState{pointers: map[Reg]int64{SP: 0, BX: 8}, known: map[Reg]uint8{BX: old}, + opaque: map[Reg]x86FixedLocalOrigins{}, constants: map[Reg]int64{BX: 7}, booleans: map[Reg]uint8{}, flags: x86ByteFlagsAll} + proof := &x86FixedLocalProof{frame: 16} + if known, err := fixedLocalPrivateAddressStep(ins, &state, proof); !known || err != nil { + t.Fatal("original source address did not reach its existing private LEA consumer", known, err) + } + if original != fmt.Sprintf("%#v", ins) { + t.Fatal("source-address projection changed the original MOV identity") + } + value, pointer := state.pointers[BX] + want := old &^ 3 + if width == "L" { + want = 0xf0 + } else if width == "Q" { + want = 0 + } + if pointer != (width == "Q") || pointer && value != 0 || state.known[BX] != want || state.flags != x86ByteFlagsAll || + len(state.constants) != 0 || len(state.opaque) != 0 { + t.Fatal("partial address became NUM or retained a full pointer identity", state) + } + }) + } + for _, control := range []string{"missing-proof", "changed-raw", "changed-operand", "physical-origin", "address-size", "SP-output", "unnamed-address", "unnamed-computed-address"} { + t.Run(width+"/"+control, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, "MOV"+width+" $slot-16(SP),BX") + proof := &x86FixedLocalProof{frame: 16} + switch control { + case "missing-proof": + proof = nil + case "changed-raw": + ins.Raw = "MOV" + width + " $other-8(SP),BX" + case "changed-operand": + ins.Args[0].Sym = "$other-8(SP)" + case "physical-origin": + ins.x86Encoded = true + case "address-size": + ins.x86AddressBits = 32 + case "SP-output": + ins = x86ScalarLaneInstruction(t, "MOV"+width+" $slot-16(SP),SP") + case "unnamed-address": + ins = x86ScalarLaneInstruction(t, "MOV"+width+" $8(SP),BX") + case "unnamed-computed-address": + ins = x86ScalarLaneInstruction(t, "MOV"+width+" $(1<<3)(SP),BX") + } + state := x86FixedLocalState{pointers: map[Reg]int64{SP: 0}, known: map[Reg]uint8{}, + opaque: map[Reg]x86FixedLocalOrigins{}, constants: map[Reg]int64{}, booleans: map[Reg]uint8{}} + before := fmt.Sprintf("%#v", state) + known, err := fixedLocalPrivateAddressStep(ins, &state, proof) + if known && !errors.Is(err, ErrProbeNeedsContext) || !reflect.DeepEqual(fmt.Sprintf("%#v", state), before) { + t.Fatal("missing/changed source or physical role supplied address permission", known, err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalOriginalMoveAddressLiveUses(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, width := range []string{"W", "L", "Q"} { + for _, use := range []string{"dereference", "numeric", "escape", "out-of-bounds", "uninitialized-local"} { + t.Run(width+"/"+use, func(t *testing.T) { + address, initialization := "slot-16", "MOVQ AX,0(SP)\n" + tail := "MOVQ (BX),AX\nMOVQ AX,ret+8(FP)\n" + switch use { + case "numeric": + tail = "ADDQ BX,0(SP)\nMOVQ 0(SP),AX\nMOVQ AX,ret+8(FP)\n" + case "escape": + tail = "MOVQ BX,ret+8(FP)\n" + case "out-of-bounds": + address = "slot+8" + case "uninitialized-local": + initialization = "" + } + source := "TEXT ·AddressUse(SB),4,$16-16\nMOVQ input+0(FP),AX\nMOVQ $0,BX\n" + initialization + + "MOV" + width + " $" + address + "(SP),BX\n" + tail + "RET\n" + declaration := "func AddressUse(input uint64) uint64\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + pkg, request, _, options := x86CMOVFixedLocalFixture(t, source, declaration, "linux/amd64", "x86_64-unknown-linux-gnu") + if width == "Q" && use == "dereference" { + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil || ir == "" { + t.Fatal("original full-width MOV lost its independently bounded initialized private pointer", err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, "x86_64-unknown-linux-gnu", ir, level) + } + }) + return + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + captured, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := captured.Close(); err != nil { + t.Error(err) + } + }() + expanded, err := captured.preprocess(request.AssemblyFile, []byte(source)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + if _, err := captured.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original named address acquired absent bytes, numeric bits, escaped pointer or out-of-bounds storage", err) + } + }) + } + } +} diff --git a/amd64_ordinary_move_fp_address_test.go b/amd64_ordinary_move_fp_address_test.go new file mode 100644 index 000000000..fb8659fd3 --- /dev/null +++ b/amd64_ordinary_move_fp_address_test.go @@ -0,0 +1,201 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86OrdinaryMoveFPAddressFixture(arch, width, address string) (string, string) { + word, move := goWordSize(arch), "MOVQ" + if word == 4 { + move = "MOVL" + } + return fmt.Sprintf("TEXT ·FPAddress(SB),4,$0-%d\n%s input+0(FP),AX\nMOV%s %s,BX\n"+ + "%s $0,BX\n%s AX,ret+%d(FP)\nRET\n", 2*word, move, width, address, move, move, word), + "func FPAddress(input uintptr) uintptr\n" +} + +func TestX86OrdinaryMoveNamedFPAddressObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + observations := 0 + for _, target := range x86RawReturnTargets { + for _, width := range []string{"W", "L", "Q"} { + if target.arch == "386" && width == "Q" { + continue // Independently rejected by the grammar owner below. + } + t.Run(target.triple+"/"+width, func(t *testing.T) { + source, declaration := x86OrdinaryMoveFPAddressFixture(target.arch, width, "$input+0(FP)") + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("dead address of an independently typed scalar field rejected", api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + } + }) + observations++ + } + } + if observations != 13 { + t.Fatal("complete mode/target FP-address object partition changed", observations) + } +} + +func TestX86OrdinaryMoveNamedFPAddressSourceBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, width := range []string{"W", "L", "Q"} { + for _, control := range []string{"numeric-FP", "punctuated-FP", "register-FP", "outside-field", "changed-Raw", "changed-operand", "changed-name", "physical-origin", "address-as-number"} { + t.Run(arch+"/"+width+"/"+control, func(t *testing.T) { + address := "$input+0(FP)" + if control == "numeric-FP" { + address = "$0(FP)" + } else if control == "punctuated-FP" { + address = "$pkg.值+0(FP)" + } else if control == "register-FP" { + address = "$AX+0(FP)" + } else if control == "outside-field" { + address = "$input+32(FP)" + } + source, declaration := x86OrdinaryMoveFPAddressFixture(arch, width, address) + if control == "address-as-number" { + move, word := "MOVQ", goWordSize(arch) + if word == 4 { + move = "MOVL" + } + source = strings.Replace(source, "MOV"+width+" ", move+" $0,BX\nMOV"+width+" ", 1) + source = strings.Replace(source, "\n"+move+" $0,BX\n"+move+" AX,ret+", "\n"+move+" BX,ret+", 1) + } + invalidName := control == "numeric-FP" || control == "punctuated-FP" || control == "register-FP" + accepted := !invalidName && !(arch == "386" && width == "Q") + observed := source + if arch == "386" && width == "Q" { + // The existing frame-diagnostic observer deliberately models + // one focused instruction and stack steps. Keep this exact + // original MOV in that closed context; Go reports its LEAQ + // rewrite with PARAM bias +4, rather than the source opcode. + observed = "TEXT ·FPAddress(SB),4,$0-8\nMOVQ " + address + ",BX\nRET\n" + } + goAssemblerOracleOutput(t, arch, observed, "", accepted) + ins := x86ScalarLaneInstruction(t, "MOV"+width+" "+address+",BX") + original := fmt.Sprintf("%#v", ins) + switch control { + case "changed-Raw": + ins.Raw = "MOV" + width + " $input+8(FP),BX" + case "changed-operand": + ins.Args[0].FPOffset = 8 + case "changed-name": + ins.Args[0].FPName = "other" + case "physical-origin": + ins.x86Encoded = true + } + view, known, err := x86NamedFrameMoveAddressInstruction(ins, arch) + if control != "outside-field" && control != "address-as-number" && known && err == nil { + t.Error("invalid/changed FP spelling or physical role became a named address", view) + } + if invalidName || control == "outside-field" || control == "address-as-number" || arch == "386" && width == "Q" { + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source, declaration, arch, x86AddressMemoryTargets(arch)[0], api); err == nil || ir != "" { + t.Error("invalid/unmapped/mode32-Q FP address produced public IR", api, len(ir), err) + } + } + } + if control == "outside-field" && original != fmt.Sprintf("%#v", ins) { + t.Fatal("source-only address projection mutated original instruction") + } + }) + } + } + } +} + +func TestX86FixedLocalAddressOnlyBareNamesBatch(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + var source, declarations strings.Builder + count := 0 + for _, op := range []string{"LEAW", "LEAL", "LEAQ", "MOVW", "MOVL", "MOVQ"} { + for _, name := range []string{"slot", "δοκιμή", "槽"} { + symbol := fmt.Sprintf("BareAddress%d", count) + address := name + "(SP)" + if strings.HasPrefix(op, "MOV") { + address = "$" + address + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$16-16\nMOVQ input+0(FP),AX\nMOVQ AX,0(SP)\n"+ + "%s %s,BX\nMOVQ $0,BX\nMOVQ 0(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n", symbol, op, address) + fmt.Fprintf(&declarations, "func %s(input uint64) uint64\n", symbol) + count++ + } + } + if count != 18 { + t.Fatal("complete address-family/width/name inventory changed", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + triple := "x86_64-unknown-linux-gnu" + pkg, request, _, options := x86CMOVFixedLocalFixture(t, source.String(), declarations.String(), "linux/amd64", triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil || ir == "" { + t.Fatal("original bounded bare AUTO addresses were rejected", err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + }) +} + +func TestX86OrdinaryMoveFrameAddressUnicodeNames(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"amd64", "386"} { + for _, name := range []string{"参数", "δοκιμή", "pkg·值"} { + t.Run(arch+"/FP/"+name, func(t *testing.T) { + source, declaration := x86OrdinaryMoveFPAddressFixture(arch, "W", "$"+name+"+0(FP)") + goAssemblerOracleOutput(t, arch, source, "", true) + ins := x86ScalarLaneInstruction(t, "MOVW $"+name+"+0(FP),BX") + original := fmt.Sprintf("%#v", ins) + view, known, err := x86NamedFrameMoveAddressInstruction(ins, arch) + if !known || err != nil || view.Args[0].FPName != name || original != fmt.Sprintf("%#v", ins) { + t.Error("legitimate Go FP symbol or original identity was lost", known, err, view) + } + triple := x86AddressMemoryTargets(arch)[0] + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, arch, triple, api) + if err != nil || ir == "" { + t.Fatal("legitimate Unicode FP address rejected", api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + } + }) + } + } + for _, name := range []string{"槽-16", "δοκιμή"} { + t.Run("amd64/AUTO/"+name, func(t *testing.T) { + source := "TEXT ·UnicodeAuto(SB),4,$16-16\nMOVQ input+0(FP),AX\nMOVQ AX,0(SP)\n" + + "MOVW $" + name + "(SP),BX\nMOVQ $0,BX\nMOVQ 0(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + declaration := "func UnicodeAuto(input uint64) uint64\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + triple := "x86_64-unknown-linux-gnu" + pkg, request, _, options := x86CMOVFixedLocalFixture(t, source, declaration, "linux/amd64", triple) + fixedLocalFreshAPIs(t, pkg, request, options, []string{"Translate", "Module", "GoModule"}, func(t *testing.T, ir string, err error) { + if err != nil || ir == "" { + t.Fatal("legitimate Unicode AUTO address rejected", err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + }) + }) + } +} diff --git a/amd64_original_go_build_only_test.go b/amd64_original_go_build_only_test.go new file mode 100644 index 000000000..3295e709f --- /dev/null +++ b/amd64_original_go_build_only_test.go @@ -0,0 +1,26 @@ +package plan9asm + +import "testing" + +func TestX86PackedIntegerCompareBuildOnlyNeverExecutesTestMain(t *testing.T) { + const declaration = "package oracle\nfunc LinkedAssembly()\n" + const assembly = "TEXT ·LinkedAssembly(SB),4,$0-0\nRET\n" + const source = `package oracle + +import ( + "os" + "testing" +) + +func TestMain(m *testing.M) { + os.Exit(97) +} + +func TestRequiresAssemblyLink(t *testing.T) { + LinkedAssembly() +} +` + // Even -run=^$ executes TestMain and process initialization. A compile/link + // owner must never run either, on the host or through a cross executor. + x86PackedIntegerCompareOriginalGoDriver(t, declaration, assembly, source, true) +} diff --git a/amd64_original_go_executable_test.go b/amd64_original_go_executable_test.go new file mode 100644 index 000000000..57915671c --- /dev/null +++ b/amd64_original_go_executable_test.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "debug/buildinfo" + "debug/elf" + "debug/macho" + "debug/pe" + "errors" + "fmt" + "os" + "runtime" + "testing" +) + +// Go20's test binaries omit BuildSettings. Require the actual executable +// machine header on every SDK, not optional GOARCH string metadata. This reads +// only the linked artifact and never starts its process or initializers. +func x86OriginalGoLinkedAMD64(name string) (amd64 bool, result error) { + file, err := os.Open(name) + if err != nil { + return false, err + } + defer func() { + if err := file.Close(); err != nil { + amd64, result = false, errors.Join(result, err) + } + }() + info, err := buildinfo.Read(file) + if err != nil || info.GoVersion != runtime.Version() { + return false, fmt.Errorf("linked executable SDK differs: %v, %v", info, err) + } + var magic [4]byte + if _, err := file.ReadAt(magic[:], 0); err != nil { + return false, err + } + switch string(magic[:]) { + case "\x7fELF": + image, err := elf.NewFile(file) + if err != nil { + return false, err + } + return image.Machine == elf.EM_X86_64 && image.Class == elf.ELFCLASS64 && + image.Data == elf.ELFDATA2LSB && image.Entry != 0 && + (image.Type == elf.ET_EXEC || image.Type == elf.ET_DYN), nil + case "\xcf\xfa\xed\xfe", "\xce\xfa\xed\xfe": + image, err := macho.NewFile(file) + if err != nil { + return false, err + } + return image.Cpu == macho.CpuAmd64 && image.Type == macho.TypeExec, nil + default: + if string(magic[:2]) != "MZ" { + return false, fmt.Errorf("unknown linked executable magic %x", magic) + } + image, err := pe.NewFile(file) + if err != nil { + return false, err + } + header, wide := image.OptionalHeader.(*pe.OptionalHeader64) + return image.Machine == pe.IMAGE_FILE_MACHINE_AMD64 && wide && header.AddressOfEntryPoint != 0 && + image.Characteristics&pe.IMAGE_FILE_EXECUTABLE_IMAGE != 0 && image.Characteristics&pe.IMAGE_FILE_DLL == 0, nil + } +} + +func TestX86OriginalGoExecutableReadsActualParentMachine(t *testing.T) { + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + amd64, err := x86OriginalGoLinkedAMD64(executable) + if err != nil || amd64 != (runtime.GOARCH == "amd64") { + t.Fatal("actual linked parent architecture differs from its binary header", amd64, err) + } +} + +func TestX86OriginalGoExecutableRejectsMissingAndNonExecutable(t *testing.T) { + for _, name := range []string{"absent", "header-only"} { + path := t.TempDir() + "/" + name + if name == "header-only" { + if err := os.WriteFile(path, []byte("\x7fELF"), 0600); err != nil { + t.Fatal(err) + } + } + if amd64, err := x86OriginalGoLinkedAMD64(path); amd64 || err == nil { + t.Fatal("missing/truncated artifact became a linked target", name, amd64, err) + } + } +} diff --git a/amd64_original_package_observation_red_test.go b/amd64_original_package_observation_red_test.go new file mode 100644 index 000000000..e3eea5461 --- /dev/null +++ b/amd64_original_package_observation_red_test.go @@ -0,0 +1,224 @@ +package plan9asm + +import ( + "bytes" + "encoding/json" + "errors" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +// Ordinary bodies and SDK imports may accompany a sealed object observation, +// but only bodyless assembly declarations enter its function/layout map. +// Observation remains separate from fixed-local/caller/native-memory permission. +type originalPackageObservationVariant struct { + name string + directory string + declaration string + data bool +} + +var originalPackageObservationVariants = []originalPackageObservationVariant{ + {name: "nested", directory: "internal/kernel"}, + {name: "bodyful", declaration: "func Reference(a uint64) uint64 { return a + 1 }\n"}, + {name: "import", declaration: "import _ \"math/bits\"\n"}, + {name: "data", data: true}, +} + +type originalPackageObservationSelected struct { + Dir, ImportPath, Export string + GoFiles, CompiledGoFiles, SFiles, Imports []string + ImportMap map[string]string + Module *struct { + Path, Version, Dir, GoMod, GoVersion string + Main bool + } +} + +func originalPackageObservationFixture(t *testing.T, variant originalPackageObservationVariant) (GoPackage, GoAMD64FrameCaptureRequest) { + t.Helper() + _, request := goAMD64FrameSourceFixture(t) + root := request.Directory + const module = "example.invalid/framefixture" + directory := root + packagePath := module + if variant.directory != "" { + directory = filepath.Join(root, filepath.FromSlash(variant.directory)) + packagePath += "/" + variant.directory + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + } + declaration := "package framefixture\n" + variant.declaration + "func Value(a uint64) uint64\n" + files := map[string]string{ + "value.go": declaration, + "value_amd64.s": "#include \"textflag.h\"\nTEXT ·Value(SB),NOSPLIT,$160-16\n" + + "MOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nMOVQ 16(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n", + } + if variant.data { + files["data_amd64.s"] = "#include \"textflag.h\"\nDATA ·constant<>+0(SB)/8,$1\n" + + "GLOBL ·constant<>(SB),RODATA|NOPTR,$8\n" + } + for name, source := range files { + if err := os.WriteFile(filepath.Join(directory, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + request.Directory = directory + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append([]string(nil), request.Environment...) + env = append(env, "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOFLAGS=", "GOWORK=off", "GOTOOLCHAIN=local", "GOENV=off") + data, err := goasmcmd.Run(ctx, directory, env, request.GoBinary, + "list", "-deps", "-compiled", "-export", "-mod=readonly", "-json", ".") + if err != nil { + t.Fatalf("first-party original Go export failed: %v; %s", err, data) + } + t.Logf("actual original Go selected/export graph (%s):\n%s", variant.name, data) + exports := make(map[string]string) + var selected *originalPackageObservationSelected + decoder := json.NewDecoder(bytes.NewReader(data)) + for { + var pkg originalPackageObservationSelected + if err := decoder.Decode(&pkg); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if pkg.Export != "" { + if _, duplicate := exports[pkg.ImportPath]; duplicate { + t.Fatalf("duplicate actual export: %s", pkg.ImportPath) + } + exports[pkg.ImportPath] = pkg.Export + } + if pkg.ImportPath == packagePath { + if selected != nil { + t.Fatal("duplicate actual original package selection") + } + copy := pkg + selected = © + } + } + if selected == nil || selected.Module == nil || !selected.Module.Main || + selected.Module.Path != module || selected.Module.Version != "" || + len(selected.CompiledGoFiles) == 0 || len(selected.SFiles) == 0 || selected.Export == "" { + t.Fatalf("incomplete actual original main-package export: %+v", selected) + } + actualDir, err := filepath.EvalSymlinks(selected.Dir) + if err != nil { + t.Fatal(err) + } + wantedDir, err := filepath.EvalSymlinks(directory) + if err != nil || actualDir != wantedDir { + t.Fatalf("actual package physical root differs: %s, %s, %v", actualDir, wantedDir, err) + } + if variant.data && len(selected.SFiles) != 2 { + t.Fatalf("original Go omitted the DATA-only sibling: %v", selected.SFiles) + } + if variant.name == "import" && exports["math/bits"] == "" { + t.Fatal("actual standard-library dependency export was not observed") + } + fset := token.NewFileSet() + var syntax []*ast.File + for _, name := range selected.CompiledGoFiles { + if !filepath.IsAbs(name) { + name = filepath.Join(selected.Dir, name) + } + file, err := parser.ParseFile(fset, name, nil, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + syntax = append(syntax, file) + } + actualImporter := importer.ForCompiler(fset, "gc", func(path string) (io.ReadCloser, error) { + if actual, mapped := selected.ImportMap[path]; mapped { + path = actual + } + filename := exports[path] + if filename == "" { + t.Fatalf("missing actual selected export for %s", path) + } + return os.Open(filename) + }) + typed, err := (&types.Config{Importer: actualImporter, Sizes: types.SizesFor("gc", "amd64"), + GoVersion: "go" + selected.Module.GoVersion}).Check(packagePath, fset, syntax, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: packagePath, Types: typed, Syntax: syntax}, request +} + +func TestGoAMD64OriginalPackageObservationKeepsAuthorityBoundaries(t *testing.T) { + for _, variant := range originalPackageObservationVariants { + t.Run(variant.name, func(t *testing.T) { + pkg, request := originalPackageObservationFixture(t, variant) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + observed, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if observed != nil { + defer func() { + if err := observed.Close(); err != nil { + t.Error(err) + } + }() + } + if variant.name == "bodyful" || variant.name == "import" || variant.name == "data" { + if err != nil || observed == nil { + t.Fatalf("ordinary original source lost its sealed observation: %v", err) + } + if len(observed.functions) != 1 || len(observed.declarations) != 1 || + observed.functions[pkg.Path+".Value"].locals != 168 { + t.Fatal("ordinary Go declaration manufactured an assembly frame observation") + } + if err := observed.Verify(ctx); err != nil { + t.Fatal(err) + } + // Even a live real-object observation supplies no aligned native + // stack storage to any translator without its separate Binder. + const assembly = "TEXT ·Value(SB),4,$160-16\nPXOR X0,X0\nMOVOU X0,16(SP)\nPADDQ 16(SP),X0\nMOVQ $1,ret+8(FP)\nRET\n" + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) } + sig := observed.functions[pkg.Path+".Value"].signature + opt := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, + Sigs: map[string]FuncSig{sig.Name: sig}} + if _, err := Translate(file, opt); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("ordinary observation granted native memory to text API: %v", err) + } + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("ordinary observation granted native memory to module API: %v", err) + } + translated, err := translateGoModuleInContext(context, pkg, []byte(assembly), GoModuleOptions{ + GOOS: "linux", GOARCH: "amd64", TargetTriple: opt.TargetTriple, ResolveSym: resolve}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("ordinary observation granted native memory to typed Go API: %v", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || observed != nil { + t.Fatalf("unsupported original source role acquired an observation: %v", err) + } + }) + } +} diff --git a/amd64_original_sysv_test.go b/amd64_original_sysv_test.go new file mode 100644 index 000000000..401b70039 --- /dev/null +++ b/amd64_original_sysv_test.go @@ -0,0 +1,86 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// These test-only byte wrappers explicitly consume SysV registers, even on +// Windows. Both declarations and indirect calls must retain that convention; +// translated functions still use their target's ordinary C ABI. This is not a +// production Go ABI contract. See clang.llvm.org/docs/AttributeReference.html#sysv-abi. +const x86OriginalSysVAttribute = "__attribute__((sysv_abi))" + +func x86OriginalSysVDeclaration(result, name, arguments string) string { + return fmt.Sprintf("extern %s %s %s(%s);\n", result, x86OriginalSysVAttribute, name, arguments) +} + +func x86OriginalSysVPointer(result, name, arguments string) string { + return fmt.Sprintf("typedef %s (%s *%s)(%s);\n", result, x86OriginalSysVAttribute, name, arguments) +} + +func TestX86OriginalSysVDirectAndPointerTransport(t *testing.T) { + clang, llc := findLLVM22Tool("clang"), findLLVM22Tool("llc") + if clang == "" || llc == "" { + t.Fatal("LLVM 22 clang and llc required") + } + var source strings.Builder + source.WriteString("typedef unsigned char uint8_t; typedef unsigned int uint32_t; typedef unsigned long long uint64_t; typedef long long int64_t;\n") + for index, signature := range []struct { + result, parameters, arguments string + }{ + {"uint64_t", "uint32_t k,uint32_t h", "k,h"}, + {"uint64_t", "uint8_t *out,int64_t index,uint64_t value", "out,index,value"}, + {"void", "uint8_t *out,const uint8_t *input,uint64_t old,uint64_t flags", "out,input,old,flags"}, + {"void", "uint8_t *out,float input,uint64_t old,uint64_t flags", "out,input,old,flags"}, + {"void", "uint8_t *out,double input,uint64_t old,uint64_t flags", "out,input,old,flags"}, + {"void", "uint8_t *out,const uint8_t *a,const uint8_t *b,const uint8_t *old,uint64_t flags,uint32_t csr", "out,a,b,old,flags,csr"}, + } { + name, pointer := fmt.Sprintf("Original%d", index), fmt.Sprintf("OriginalProgram%d", index) + source.WriteString(x86OriginalSysVDeclaration(signature.result, name, signature.parameters)) + source.WriteString(x86OriginalSysVPointer(signature.result, pointer, signature.parameters)) + fmt.Fprintf(&source, "extern %s Translated%d(%s);\n", signature.result, index, signature.parameters) + fmt.Fprintf(&source, "void Check%d(%s pointer,%s) { %s(%s); pointer(%s); Translated%d(%s); }\n", + index, pointer, signature.parameters, name, signature.arguments, signature.arguments, index, signature.arguments) + } + for _, target := range []string{"x86_64-pc-windows-msvc", "x86_64-w64-windows-gnu", "x86_64-unknown-linux-gnu", "x86_64-apple-darwin"} { + t.Run(target, func(t *testing.T) { + directory := t.TempDir() + input, output := filepath.Join(directory, "transport.c"), filepath.Join(directory, "transport.ll") + if err := os.WriteFile(input, []byte(source.String()), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, clang, "-target", target, "-nostdinc", "-Werror", + "-O0", "-S", "-emit-llvm", input, "-o", output) + if data, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("typed direct/indirect transport: %v\n%s", err, data) + } + ir, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if strings.Contains(target, "windows") && strings.Count(string(ir), "call x86_64_sysvcc ") != 12 { + t.Fatal("Windows direct or pointer call erased the original SysV ABI") + } + translatedCalls := 0 + for _, line := range strings.Split(string(ir), "\n") { + if strings.Contains(line, "call ") && strings.Contains(line, "@Translated") { + translatedCalls++ + if strings.Contains(line, "x86_64_sysvcc") { + t.Fatal("original byte-wrapper ABI leaked into a translated call") + } + } + } + if translatedCalls != 6 { + t.Fatalf("translated call controls = %d, want 6", translatedCalls) + } + compileLLVMToObject(t, llc, target, "transport.ll", "transport.o", string(ir)) + }) + } +} diff --git a/amd64_packed_abs_sign_forms.go b/amd64_packed_abs_sign_forms.go new file mode 100644 index 000000000..693688b01 --- /dev/null +++ b/amd64_packed_abs_sign_forms.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedAbsSignSpec struct { + x86PackedBinarySpec + laneBits int + sign bool +} + +// Go's legacy yxm_q4 rows are XMM-only, despite Intel's separate NP MMX +// encodings. ABS is unary; SIGN reads its control and data independently. +// The common shape describes encodings, not memory or physical-entry rights. +var x86PackedAbsSignSpecs = map[Op]x86PackedAbsSignSpec{ + "PABSB": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, laneBits: 8}, + "PABSW": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, laneBits: 16}, + "PABSD": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, laneBits: 32}, + "PSIGNB": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, laneBits: 8, sign: true}, + "PSIGNW": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, laneBits: 16, sign: true}, + "PSIGND": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, laneBits: 32, sign: true}, + + "VPABSB": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, laneBits: 8}, + "VPABSW": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, laneBits: 16}, + "VPABSD": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, laneBits: 32}, + "VPABSQ": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, broadcastBytes: 8, zeroing: true}, laneBits: 64}, + "VPSIGNB": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalVEX}, laneBits: 8, sign: true}, + "VPSIGNW": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalVEX}, laneBits: 16, sign: true}, + "VPSIGND": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalVEX}, laneBits: 32, sign: true}, +} + +type x86PackedAbsSignForm struct { + x86PackedBinaryForm + spec x86PackedAbsSignSpec +} + +func (form x86PackedAbsSignForm) evex(ins Instr) bool { + if form.legacy { + return false + } + if ins.x86PackedAbsEVEX || form.spec.encoding == x86PackedLogicalEVEX || + form.byteWidth == 64 || form.mask >= 0 || form.broadcast { + return true + } + for _, arg := range ins.Args { + if index, valid := amd64VectorRegisterIndex(arg.Reg, form.byteWidth); arg.Kind == OpReg && valid && index >= 16 { + return true + } + } + return false +} + +func x86PackedAbsSignProperties(op Op, ins Instr, arch string) (x86PackedAbsSignForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86PackedAbsSignSpecs[Op(base)] + form := x86PackedAbsSignForm{spec: spec} + if !known { + return form, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return form, true, fmt.Errorf("ABS/sign dispatch/source opcode mismatch") + } + if ins.x86PackedAbsEVEX && spec.encoding != x86PackedLogicalVEXOrEVEX && spec.encoding != x86PackedLogicalEVEX { + return form, true, fmt.Errorf("%s %s has no physical EVEX generation", arch, op) + } + local := ins + local.Op = op + unary := !spec.sign && spec.encoding != x86PackedLogicalLegacyFP + if unary { + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return form, true, fmt.Errorf("%s %s requires source, optional K1-K7 and destination", arch, op) + } + destination := ins.Args[len(ins.Args)-1] + local.Args = []Operand{ins.Args[0], destination, destination} + if len(ins.Args) == 3 { + local.Args = []Operand{ins.Args[0], destination, ins.Args[1], destination} + } + } + var err error + form.x86PackedBinaryForm, err = x86ConflictFunnelOperandGrammar(local, arch, spec.x86PackedBinarySpec, unary) + if !spec.sign { + form.second = -1 + } + if unary { + form.destination = len(ins.Args) - 1 + if form.mask >= 0 { + form.mask = 1 + } + } + return form, true, err +} + +func x86TypedPackedAbsSignByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedAbsSignProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width, sourceWidth := int64(form.byteWidth), int64(form.byteWidth) + if form.memory >= 0 { + sourceWidth = int64(form.memoryBytes) + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: sourceWidth, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: width, + read: form.spec.sign && form.legacy || form.mask >= 0 && !form.zeroing, write: true}, + }} + if form.spec.sign && !form.legacy { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.second, operand: ins.Args[form.second], width: width, read: true}) + } + if form.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return effect, true, nil +} diff --git a/amd64_packed_add_fault_runtime_test.go b/amd64_packed_add_fault_runtime_test.go new file mode 100644 index 000000000..7cce0415f --- /dev/null +++ b/amd64_packed_add_fault_runtime_test.go @@ -0,0 +1,124 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedWrappingAddNativeMemoryStatus(t *testing.T) { + x86PackedAddNativeMemoryStatus(t, x86WrappingAddOpcodes, []int{8, 16, 32, 64, 8, 16, 32, 64}) +} + +func TestCrossLinuxRuntimeMatrixX86PackedSaturatingAddNativeMemoryStatus(t *testing.T) { + x86PackedAddNativeMemoryStatus(t, []string{ + "PADDSB", "PADDSW", "PADDUSB", "PADDUSW", + "VPADDSB", "VPADDSW", "VPADDUSB", "VPADDUSW", + }, []int{8, 16, 8, 16, 8, 16, 8, 16}) +} + +func x86PackedAddNativeMemoryStatus(t *testing.T, opcodes []string, laneBits []int) { + t.Helper() + if len(opcodes) != len(laneBits) { + t.Fatal("opcode/lane inventory differs") + } + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, references, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for opcodeIndex, opcode := range opcodes { + legacy := !strings.HasPrefix(opcode, "V") + bits := laneBits[opcodeIndex] + modes := []int{-1} + if !legacy { + modes = []int{-1, 0, 1} + } + for _, mode := range modes { + name := fmt.Sprintf("packedaddfault%d", index) + index++ + row := opcode + " (BX),X0" + if !legacy { + row = opcode + " (BX),Z0,Z1" + if mode >= 0 { + row = opcode + ".Z (BX),Z0,K1,Z1" + } + } + mask := mode + if mask < 0 { + mask = 0 + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-8\nMOVQ input+0(FP),BX\nVPXORD Z0,Z0,Z0\nMOVL $%d,CX\nKMOVQ CX,K1\n%s\nRET\n", name, mask, row) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const uint8_t*);\n", name) + reference := fmt.Sprintf("reference%d", index) + if legacy { + mnemonic := strings.ToLower(opcode) + if mnemonic == "paddl" { + mnemonic = "paddd" + } + fmt.Fprintf(&references, "__attribute__((noinline)) static void %s(const uint8_t*p){typedef int32_t V __attribute__((vector_size(16)));V v={0,0,0,0};__asm__ volatile(\"%s (%%1),%%0\":\"+x\"(v):\"r\"(p):\"memory\");}\n", reference, mnemonic) + } else { + width := 64 + if mode == 0 { + width = 0 + } else if mode == 1 { + width = bits / 8 + } + fmt.Fprintf(&references, "__attribute__((noinline)) static void %s(const uint8_t*p){for(int i=0;i<%d;++i)(void)((const volatile uint8_t*)p)[i];}\n", reference, width) + } + fmt.Fprintf(&checks, "for(int n=0;n<16;++n)if(compare(%s,%s,memory+n))return 1;\n", reference, name) + for _, offset := range []int{0, 1, bits / 8, 16, 64} { + fmt.Fprintf(&checks, "if(compare(%s,%s,boundary-%d))return 1;\n", reference, name, offset) + } + fmt.Fprintf(&checks, "if(compare(%s,%s,0))return 1;\n", reference, name) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + references.String() + ` +typedef struct {int signal;int code;} Status; +static int observe(void(*fn)(const uint8_t*),const uint8_t*p,Status*out){ + pid_t child=fork();if(child<0)return 1; + if(child==0){struct rlimit limit={0,0};if(setrlimit(RLIMIT_CORE,&limit))_exit(3);fn(p);_exit(0);} + int status;if(waitpid(child,&status,0)!=child)return 1; + if(WIFSIGNALED(status)){*out=(Status){WTERMSIG(status),0};return 0;} + if(WIFEXITED(status)){*out=(Status){0,WEXITSTATUS(status)};return 0;}return 1; +} +static int compare(void(*original)(const uint8_t*),void(*translated)(const uint8_t*),const uint8_t*p){ + Status a,b;if(observe(original,p,&a)||observe(translated,p,&b))return 1; + if(a.signal!=b.signal||a.code!=b.code){fprintf(stderr,"pointer=%p mod16=%lu original=(%d,%d) translated=(%d,%d)\n",p,(unsigned long)((uintptr_t)p%16),a.signal,a.code,b.signal,b.code);return 1;}return 0; +} +int main(void){ + long page=sysconf(_SC_PAGESIZE);if(page<=0||page%64)return 1; + uint8_t*memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED)return 1;memset(memory,0x87,(size_t)page); + if(mprotect(memory,(size_t)page,PROT_READ)||mprotect(memory+page,(size_t)page,PROT_NONE))return 1; + uint8_t*boundary=memory+page; +` + checks.String() + "if(munmap(memory,(size_t)page*2))return 1;return 0;}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-add-fault", triple, text, main, prefix) + }) + } + t.Log("all four original SSE opcodes typed-status at every mod16/NULL/RO guard; E4/E4.nb zero/active masks observed against independent volatile reads, not unexecuted AVX512 hardware") +} diff --git a/amd64_packed_add_forms.go b/amd64_packed_add_forms.go new file mode 100644 index 000000000..03443e38f --- /dev/null +++ b/amd64_packed_add_forms.go @@ -0,0 +1,17 @@ +package plan9asm + +type x86PackedAddForm = x86PackedArithmeticForm +type x86PackedIntegerAddForm = x86PackedArithmeticForm + +func x86PackedIntegerAddProperties(ins Instr, arch string) (x86PackedIntegerAddForm, bool, error) { + return x86PackedArithmeticProperties(ins.Op, ins, arch, x86PackedArithmeticAdd) +} + +// Shared syntax does not admit saturating adds into the private wrapping caller. +func x86PackedWrappingAddProperties(ins Instr, arch string) (x86PackedAddForm, bool, error) { + spec, known := x86PackedArithmeticSpecForOp(ins.Op, x86PackedArithmeticAdd) + if !known || spec.mode != x86PackedWrap { + return x86PackedAddForm{}, false, nil + } + return x86PackedArithmeticFormForSpec(ins.Op, ins, arch, spec) +} diff --git a/amd64_packed_add_forms_test.go b/amd64_packed_add_forms_test.go new file mode 100644 index 000000000..8a2089f6e --- /dev/null +++ b/amd64_packed_add_forms_test.go @@ -0,0 +1,331 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "testing" +) + +var x86WrappingAddOpcodes = []string{"PADDB", "PADDW", "PADDL", "PADDQ", "VPADDB", "VPADDW", "VPADDD", "VPADDQ"} + +func x86WrappingAddSpecForTest(op Op) x86PackedArithmeticSpec { + spec, known := x86PackedArithmeticSpecForOp(op, x86PackedArithmeticAdd) + if !known || spec.mode != x86PackedWrap { + panic("invalid wrapping-add fixture opcode") + } + return spec +} + +func x86WrappingAddFormats(arch string) []string { + var rows []string + for _, opcode := range append(append([]string(nil), x86WrappingAddOpcodes...), "PADDD") { + spec := x86WrappingAddSpecForTest(Op(opcode)) + if !strings.HasPrefix(opcode, "V") { + rows = append(rows, opcode+" X0,X7", opcode+" 16(AX)(BX*2),X7") + if arch == "amd64" { + rows = append(rows, opcode+" X15,X0", opcode+" data(SB),X0", opcode+" (FS),X0", opcode+" (GS),X0") + if opcode != "PADDQ" { + rows = append(rows, opcode+" M0,M7", opcode+" (AX),M7", opcode+" source+0(FP),M7") + } + } + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "16(AX)(BX*2)", "source+0(FP)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1,%s7", opcode, source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s,%s31,K1,%s30", opcode, source, bank, bank), fmt.Sprintf("%s.Z %s,%s31,K7,%s30", opcode, source, bank, bank)) + } + } + if spec.broadcastBytes != 0 { + rows = append(rows, fmt.Sprintf("%s.BCST (AX),%s1,%s7", opcode, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s.BCST.Z source+0(FP),%s31,K7,%s30", opcode, bank, bank)) + } + } + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s31,%s30,%s29", opcode, bank, bank, bank)) + } + } + } + return rows +} + +func TestX86PackedWrappingAddActualGoCompleteAttributes(t *testing.T) { + wrapping := 0 + for _, spec := range x86PackedArithmeticSpecs { + if spec.family == x86PackedArithmeticAdd && spec.mode == x86PackedWrap { + wrapping++ + } + } + if wrapping != 9 { + t.Fatal("eight wrapping operations and the actual Go PADDD alias required") + } + for index, opcode := range x86WrappingAddOpcodes { + spec := x86WrappingAddSpecForTest(Op(opcode)) + bits := 8 << uint(index%4) + broadcast := 0 + if index >= 6 { + broadcast = bits / 8 + } + if spec.laneBits != bits || spec.broadcastBytes != broadcast || spec.zeroing != (index >= 4) { + t.Fatalf("%s has wrong independent arithmetic/tuple attributes: %+v", opcode, spec) + } + } + if x86WrappingAddSpecForTest("PADDD") != x86WrappingAddSpecForTest("PADDL") { + t.Fatal("actual Go alias must retain the exact canonical operation") + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil || !strings.Contains(string(aliases), `instructions["PADDD"] = x86.APADDL`) { + t.Fatalf("actual Go PADDD alias registration: %v", err) + } + for index, opcode := range []string{"PADDB", "PADDW", "PADDL", "PADDQ"} { + byteOp := []int{0xfc, 0xfd, 0xfe, 0xd4}[index] + registration := fmt.Sprintf("{A%s, ymm, Py1, opBytes{0x%x, Pe, 0x%x}}", opcode, byteOp, byteOp) + if opcode == "PADDQ" { + registration = "{APADDQ, yxm, Pe, opBytes{0xd4}}" + } + if !strings.Contains(string(legacy), registration) { + t.Fatalf("actual Go legacy registration changed: %s", registration) + } + opcode = []string{"VPADDB", "VPADDW", "VPADDD", "VPADDQ"}[index] + start := strings.Index(string(avx), "{as: A"+opcode+",") + if start < 0 { + t.Fatal("actual Go missing " + opcode) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(row, "ytab: _yvandnpd") || strings.Contains(row, "Sae") || strings.Contains(row, "Rounding") { + t.Fatalf("actual Go operand attributes changed: %s", row) + } + for _, width := range []string{"vex128", "vex256"} { + if !strings.Contains(row, fmt.Sprintf("avxEscape | %s | vex66 | vex0F | vexW0, 0x%X", width, byteOp)) { + t.Fatal("missing actual VEX registration " + opcode) + } + } + for i, width := range []string{"evex128", "evex256", "evex512"} { + w, broadcast := 0, "" + if index >= 2 { + broadcast = fmt.Sprintf(" | evexBcstN%d", 4<<(index-2)) + } + if index == 3 { + w = 1 + } + want := fmt.Sprintf("avxEscape | %s | evex66 | evex0F | evexW%d, evexN%d%s | evexZeroingEnabled, 0x%X", width, w, 16<= 0 { + sig.Frame.Params = append(sig.Frame.Params[:missing], sig.Frame.Params[missing+1:]...) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := out.String() + _, _, err := ctx.lowerPackedIntegerAdd(ins.Op, ins) + if missing < 0 && err != nil || missing >= 0 && (err == nil || out.String() != before) { + t.Fatalf("%s/%s missing FP byte%d: %v", op, bank, missing, err) + } + } + } + } + } + for _, opcode := range []string{"PADDB", "PADDW", "PADDL", "PADDD"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+opcode+" source+0(FP),M1\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + for missing := -1; missing < 8; missing++ { + sig := x86WordMaddFPType(t) + if missing >= 0 { + sig.Frame.Params = append(sig.Frame.Params[:missing], sig.Frame.Params[missing+1:]...) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := out.String() + _, _, err := ctx.lowerPackedIntegerAdd(ins.Op, ins) + if missing < 0 && err != nil || missing >= 0 && (err == nil || out.String() != before) { + t.Fatalf("%s MMX missing FP byte%d: %v", opcode, missing, err) + } + } + } +} + +func TestX86PackedWrappingAdd386HighRegisterActualByteIdentities(t *testing.T) { + for _, opcode := range []string{"VPADDB", "VPADDW", "VPADDD", "VPADDQ"} { + for _, bank := range []string{"X", "Y"} { + for operand := 0; operand < 3; operand++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[operand] = bank + "16" + row := opcode + " " + strings.Join(args, ",") + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil { + t.Fatalf("actual Go mode32 object: %v\n%s", err, listing) + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 7 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + for _, field := range fields[1:7] { + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + t.Fatal(err) + } + code = append(code, byte(value)) + } + break + } + ins, length, recognized, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if len(code) != 6 || !recognized || err != nil || length != 6 || ins.Op != Op(opcode) || len(ins.Args) != 3 { + t.Fatalf("actual EVEX byte identity %s/%x: %+v/%d/%t/%v", row, code, ins, length, recognized, err) + } + for index, argument := range args { + if ins.Args[index].Reg != Reg(argument) { + t.Fatalf("%s/%x operand%d: %s != %s", row, code, index, ins.Args[index].Reg, argument) + } + } + if _, _, recognized, err := decodedX86PackedIntegerArithmeticInstruction(code, 32); !recognized || err == nil { + t.Fatalf("raw mode32 high bank obtained false permission: %s/%x", row, code) + } + t.Logf("%s actual Go bytes=%x; mode64 identities exact, mode32 rejected", row, code) + } + } + } +} diff --git a/amd64_packed_add_go_oracle_test.go b/amd64_packed_add_go_oracle_test.go new file mode 100644 index 000000000..d3167580c --- /dev/null +++ b/amd64_packed_add_go_oracle_test.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedWrappingAddOriginalGoBits(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle +import "testing" +func lane(p []byte,n int)uint64{var value uint64;for j:=0;j>24)}} + for i:=range patterns[0]{patterns[0][i]=0xff;patterns[1][i]=1;patterns[2][i]=0x80;patterns[3][i]=0} +`) + count := 0 + for index, opcode := range []string{"PADDB", "PADDW", "PADDL", "PADDQ"} { + bytes := 1 << uint(index) + for _, bank := range []string{"X", "M"} { + if bank == "M" && opcode == "PADDQ" { + continue // Actual Go PADDQ uses yxm, not Intel's MMX row. + } + forms := []string{"register"} + if bank == "M" { + forms = append(forms, "memory") + } + for _, form := range forms { + name := fmt.Sprintf("packedaddgo%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(a,b,out *byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\nSTMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\n", name) + move, width := "MOVOU", 16 + if bank == "M" { + move, width = "MOVQ", 8 + } + fmt.Fprintf(&assembly, "%s (BX),%s0\n", move, bank) + first := "(AX)" + if form == "register" { + first = bank + "1" + fmt.Fprintf(&assembly, "%s (AX),%s1\n", move, bank) + } + fmt.Fprintf(&assembly, "CMPQ AX,AX\nSTC\n%s %s,%s0\n%s %s0,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\nSTMXCSR 18(CX)\n", opcode, first, bank, move, bank) + if bank == "M" { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("LDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&tests, ` + for _,a:=range patterns{for _,b:=range patterns{for mode:=0;mode<4;mode++{ + var out [24]byte;for i:=range out{out[i]=0xc3};csr:=uint32(0x1f80|mode<<13|mode) + %[1]s(&a[0],&b[0],&out[1],csr) + if out[0]!=0xc3||out[23]!=0xc3||out[17]!=1||out[18]!=0{t.Fatalf("%[1]s flags/canary %%x",out)} + gotCSR:=uint32(out[19])|uint32(out[20])<<8|uint32(out[21])<<16|uint32(out[22])<<24 + if gotCSR!=csr{t.Fatalf("%[1]s CSR %%x != %%x",gotCSR,csr)} + for i:=0;i<%[2]d;i+=%[3]d{ + want:=lane(a[i:],%[3]d)+lane(b[i:],%[3]d) + for j:=0;j<%[3]d;j++{if out[i+j+1]!=byte(want>>uint(8*j)){t.Fatalf("%[1]s byte%%d got%%x want%%x",i+j,out,want)}} + } + for i:=%[2]d;i<16;i++{if out[i+1]!=0xc3{t.Fatal("touched upper MMX canary")}} + }}} +`, name, width, bytes) + } + } + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go modulo integer-add oracle: %v\n%s", err, output) + } + t.Logf("%d actual Go SSE/MMX functions × 256 independent full-lane wrap/MXCSR/flags/canary cases; no x87 tag-state claim", count) +} diff --git a/amd64_packed_add_runtime_test.go b/amd64_packed_add_runtime_test.go new file mode 100644 index 000000000..f6b627252 --- /dev/null +++ b/amd64_packed_add_runtime_test.go @@ -0,0 +1,231 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" + "testing" +) + +const x86PackedAddIndependentOracle = ` +static const uint64_t masks[]={0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; +static uint64_t lane(const uint8_t *p,int bytes) { + uint64_t v=0;for(int j=0;j>(i/bytes))&1); + for(int j=0;j>(j*8)) : mode==2 ? 0 : old[i+j]) : legacy ? b[i+j] : 0; + if(output[i+j+1]!=want) {fprintf(stderr,"%s round=%d mask=%llx byte=%d got=%x want=%x\n",name,round,(unsigned long long)mask,i+j,output[i+j+1],want);return 1;} + } + } + for(int i=0;i<96;++i) { + if((i=shift+64) && aStorage[i]!=0xa7) return 1; + if((i<3 || i>=67) && bStorage[i]!=0xb9) return 1; + } + } + return 0; +} +` + +func x86PackedAddMMXAssembly(name, opcode, input string) string { + return fmt.Sprintf("TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nVMOVUPS (CX),Z0\nVMOVUPS Z0,(AX)\nCMPQ AX,AX\nSTC\nMOVQ (CX),M0\nMOVQ (BX),M1\n%s %s,M0\nMOVQ M0,(AX)\nEMMS\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", name, opcode, input) +} + +func TestCrossLinuxRuntimeMatrixX86PackedWrappingAddCompleteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for index, opcode := range x86WrappingAddOpcodes { + bits := 8 << uint(index%4) + legacy := !strings.HasPrefix(opcode, "V") + widths := []int{16} + if !legacy { + widths = []int{16, 32, 64} + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, kind := range []string{"register", "memory", "broadcast"} { + if kind == "broadcast" && (legacy || bits < 32) { + continue + } + for mode := 0; mode < 3; mode++ { + if legacy && mode != 0 { + continue + } + destination := mode + if legacy { + destination = 0 + } + name := fmt.Sprintf("packedadd%d", count) + count++ + first := bank + "1" + if kind != "register" { + first = "(BX)" + } + row := opcode + " " + first + ",X0" + if !legacy { + suffix := "" + if kind == "broadcast" { + suffix = ".BCST" + } + if mode == 2 { + suffix += ".Z" + } + row = opcode + suffix + " " + first + "," + bank + "0" + if mode != 0 { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", name, row, destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(checkAdd(%s,\"%s\",%d,%d,%d,%d,%d,%d,%d)) return 1;\n", name, name, width, bits/8, mode, x86FloatAddSubBoolInt(legacy), destination, x86FloatAddSubBoolInt(kind != "register"), x86FloatAddSubBoolInt(kind == "broadcast")) + } + } + } + } + for index, opcode := range []string{"PADDB", "PADDW", "PADDL"} { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("packedadd%d", count) + count++ + first := "M1" + if memory { + first = "(BX)" + } + source.WriteString(x86PackedAddMMXAssembly(name, opcode, first)) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(checkAdd(%s,\"%s\",8,%d,0,1,0,%d,0))return 1;\n", name, name, 1<>(i/%[3]d))&1); + for(int j=0;j<%[3]d;++j) { + uint8_t want=active?(uint8_t)(sum>>(8*j)):%[5]d==2?0:before[i+j]; + if(output[i+j+1]!=want) {fprintf(stderr,"%[1]s FP byte=%%d got=%%x want=%%x\n",i+j,output[i+j+1],want);return 1;} + } + } + if(input[0]!=0xa7 || input[65]!=0xa7 || output[0]!=0xc3 || output[65]!=0xc3) return 1; + } +`, name, width, bits/8, x86FloatAddSubBoolInt(broadcast), mode) + } + } + } + } + requireX86GoAssemblerResult(t, "amd64", definitions.String(), true) + file, err := Parse(ArchAMD64, definitions.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + ir += wrappers.String() + main := "#include \n#include \n#include \n" + declarations.String() + x86PackedAddIndependentOracle + "int main(void){\n" + checks.String() + "return 0;}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "wrapping-add-fp", triple, text, main, prefix) + }) + } + t.Logf("%d complete independent-byte FP programs × 64 logical-value cases at O0/O2; no native FP backing or alignment claim", count) +} diff --git a/amd64_packed_align_fault_unix_test.go b/amd64_packed_align_fault_unix_test.go new file mode 100644 index 000000000..4d559392b --- /dev/null +++ b/amd64_packed_align_fault_unix_test.go @@ -0,0 +1,231 @@ +//go:build darwin || linux + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// E4NF access behavior executes independently of AVX512 hardware. The separate +// native legacy root below compares actual original Go opcode bytes. +func TestX86PackedAlignDeadE4NFReadsFaultAtFullTuple(t *testing.T) { + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable guard-page oracle requires its native 64-bit host counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 llc and clang required") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var source, declarations, prototypes, metadata strings.Builder + count := 0 + for _, row := range x86PackedAlignRows()[1:] { + for _, bank := range []string{"X", "Y", "Z"} { + for _, broadcast := range []bool{false, true} { + if broadcast && row.bits == 8 { + continue + } + op, bytes := row.op, amd64VectorByteWidth(Reg(bank+"0")) + if broadcast { + op, bytes = op+".BCST", row.bits/8 + } + name := fmt.Sprint("AlignDead", count) + instruction := x86PackedAlignInstruction(op+".Z", "(AX)", bank, "K1,", 255, 2, false) + fmt.Fprintf(&declarations, "func %s(p *[64]byte,mask uint64)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),AX\nKMOVQ mask+8(FP),K1\n"+ + "VPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\n%s\nRET\n", name, instruction) + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t *,uint64_t);\n", name) + fmt.Fprintf(&metadata, "{%s,%d},\n", name, bytes) + count++ + } + } + } + if count != 15 { + t.Fatal("incomplete native tuple/broadcast inventory", count) + } + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source.String()) + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", triple, "text") + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +#include +#include +#include +#include +` + prototypes.String() + ` +typedef void (*Kernel)(const uint8_t *,uint64_t); +typedef struct {Kernel run;int bytes;} Row; +static const Row rows[]={ +` + metadata.String() + ` +}; +static int fault(Kernel run,const uint8_t *p,uint64_t mask) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(120); + run(p,mask); + _exit(0); + } + int status=0; + if(waitpid(child,&status,0)!=child) return -2; + if(WIFEXITED(status)) return WEXITSTATUS(status)?-3:0; + return WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS)?1:-4; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<64) return 1; + uint8_t *p=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(p==MAP_FAILED || mprotect(p+page,(size_t)page,PROT_NONE)) return 1; + for(long i=0;i +#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + references.String() + ` +typedef void (*Kernel)(uint8_t *,const uint8_t *,const uint8_t *); +typedef struct {Kernel run,original;int count,dead;} Row; +static const Row rows[]={ +` + metadata.String() + ` +}; +static int observe(Kernel run,const uint8_t *p,int count,int dead) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(120); + uint8_t old[16],out[19]; + for(int i=0;i<16;i++) old[i]=(uint8_t)(i*13+71); + memset(out,0xa7,sizeof(out)); + run(out+1,p,old); + if(out[0]!=0xa7 || out[18]!=0xa7) _exit(2); + for(int i=0;i<17;i++) { + uint8_t expected=0xa7; + if(!dead) { + int at=i+count; + expected=i==16?1:at>=32?0:at<16?p[at]:old[at-16]; + } + if(out[i+1]!=expected) _exit(3); + } + _exit(0); + } + int status=0; + if(waitpid(child,&status,0)!=child) return -2; + if(WIFEXITED(status)) return WEXITSTATUS(status)?-3:0; + return WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS)?1:-4; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<64) return 1; + uint8_t *p=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(p==MAP_FAILED || mprotect(p+page,(size_t)page,PROT_NONE)) return 1; + for(long i=0;i=3 && original!=1) || (NATIVE_LINUX && c==2 && original!=1)) { + fprintf(stderr,"legacy alignment row=%u case=%u original=%d translated=%d\n",r,c,original,translated); + return 1; + } + } + } + return munmap(p,(size_t)page*2)?1:0; +} +` + for _, pipeline := range []struct { + name string + optimized bool + backend int + }{ + {name: "backend_O0"}, {name: "backend_O2", backend: 2}, {name: "IR_O2/backend_O2", optimized: true, backend: 2}, + } { + t.Run(pipeline.name, func(t *testing.T) { + text := ir + if pipeline.optimized { + text = x86DwordProductIROptimize(t, ir) + } + compiler := []string{clang} + if triple != testTargetTriple(runtime.GOOS, runtime.GOARCH) { + compiler = append(compiler, "-target", triple) + } + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, "align_legacy_native", triple, text, mainC, prefix, pipeline.backend) + }) + } +} diff --git a/amd64_packed_align_forms.go b/amd64_packed_align_forms.go new file mode 100644 index 000000000..fd9f9fcfd --- /dev/null +++ b/amd64_packed_align_forms.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedAlignSpec struct { + x86PackedBinarySpec + bits int +} + +// The immediate and selector differ; operand widths, masks and original +// memory envelopes use the same complete binary grammar as other families. +var x86PackedAlignSpecs = map[Op]x86PackedAlignSpec{ + "PALIGNR": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, bits: 8}, + "VPALIGNR": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, bits: 8}, + "VALIGND": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, broadcastBytes: 4, zeroing: true}, bits: 32}, + "VALIGNQ": {x86PackedBinarySpec: x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, broadcastBytes: 8, zeroing: true}, bits: 64}, +} + +type x86PackedAlignForm struct { + x86PackedBinaryForm + spec x86PackedAlignSpec + immediate uint8 +} + +func x86PackedAlignProperties(op Op, ins Instr, arch string) (x86PackedAlignForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86PackedAlignSpecs[Op(base)] + result := x86PackedAlignForm{spec: spec} + if !known { + return result, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || + len(ins.Args) < 3 || len(ins.Args) > 5 || arch == "386" && spec.encoding != x86PackedLogicalLegacyFP { + return result, true, fmt.Errorf("%s %s has an unbound dispatch or named operand arity", arch, op) + } + immediate, integer := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !integer || !amd64UnsignedImmediate(immediate, 8) { + return result, true, fmt.Errorf("%s %s requires Go's complete Yu8 immediate class", arch, op) + } + local := ins + local.Op, local.Args = op, ins.Args[1:] + var err error + result.x86PackedBinaryForm, err = x86ConflictFunnelOperandGrammar(local, arch, spec.x86PackedBinarySpec, false) + result.first++ + result.second++ + result.destination++ + if result.memory >= 0 { + result.memory++ + } + if result.mask >= 0 { + result.mask++ + } + result.immediate = uint8(immediate.Imm) + return result, true, err +} + +// Return a byte in the first/second concatenated vectors, or -1 for a known +// zero. PALIGNR is lane-local and saturates; VALIGN wraps its element count +// and crosses the complete physical vector, not each 128-bit lane. +func (f x86PackedAlignForm) sourceByte(output int) int { + if f.spec.bits == 8 { + selected := output%16 + int(f.immediate) + if selected >= 32 { + return -1 + } + if selected < 16 { + return output/16*16 + selected + } + return f.byteWidth + output/16*16 + selected - 16 + } + bytes := f.spec.bits / 8 + count := int(f.immediate) & (f.byteWidth/bytes - 1) + return (output/bytes+count)*bytes + output%bytes +} + +func (f x86PackedAlignForm) definedBytes(first, second, old uint64) uint64 { + var defined uint64 + for output := 0; output < f.byteWidth; output++ { + at, known := f.sourceByte(output), false + if at < 0 { + known = true + } else if at < f.byteWidth { + known = first>>uint(at)&1 != 0 + } else { + known = second>>uint(at-f.byteWidth)&1 != 0 + } + if f.mask >= 0 && !f.zeroing { + known = known && old>>uint(output)&1 != 0 + } + if known { + defined |= uint64(1) << uint(output) + } + } + return defined +} + +func x86TypedPackedAlignByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedAlignProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width, firstWidth := int64(form.byteWidth), int64(form.byteWidth) + if form.memory >= 0 { + firstWidth = int64(form.memoryBytes) + } + effect := x86TypedByteEffect{ + requiresResultProof: x86ResultNamedPredicateInstruction(ins), + operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: firstWidth, read: true}, + {index: form.second, operand: ins.Args[form.second], width: width, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: width, + read: form.mask >= 0 && !form.zeroing, write: true}, + }, + } + if form.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return effect, true, nil +} diff --git a/amd64_packed_align_forms_test.go b/amd64_packed_align_forms_test.go new file mode 100644 index 000000000..532d10a03 --- /dev/null +++ b/amd64_packed_align_forms_test.go @@ -0,0 +1,126 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86PackedAlignCompleteIndependentGoRows(t *testing.T) { + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{"{APALIGNR, ypalignr, Pq, opBytes{0x3a, 0x0f}}", "var ypalignr = []ytab{\n\t{Zibm_r, 2, argList{Yu8, Yxm, Yxr}},\n}"} { + if !strings.Contains(string(legacy), row) { + t.Fatal("complete legacy align table changed", row) + } + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + counts := map[string]int{} + for _, rule := range rules { + if rule.Opcode != "VPALIGNR" && rule.Opcode != "VALIGND" && rule.Opcode != "VALIGNQ" { + continue + } + index := counts[rule.Opcode] + counts[rule.Opcode]++ + classes := []string{"Yu8,YxmEvex,YxrEvex,YxrEvex", "Yu8,YxmEvex,YxrEvex,Yknot0,YxrEvex", + "Yu8,YymEvex,YyrEvex,YyrEvex", "Yu8,YymEvex,YyrEvex,Yknot0,YyrEvex", "Yu8,Yzm,Yzr,Yzr", "Yu8,Yzm,Yzr,Yknot0,Yzr"} + offsets, advances := []int{0, 0, 3, 3, 6, 6}, []int{0, 3, 0, 3, 0, 3} + ytab, opcode, w, zcase := "_yvalignd", "0x03", 0, "Zevex_i_rm_v_r" + if rule.Opcode == "VALIGNQ" { + w = 1 + } + var attributes []string + if rule.Opcode == "VPALIGNR" { + ytab, opcode = "_yvgf2p8affineinvqb", "0x0F" + classes = append([]string{"Yu8,Yxm,Yxr,Yxr", "Yu8,Yym,Yyr,Yyr"}, classes...) + offsets, advances = []int{0, 2, 4, 4, 7, 7, 10, 10}, []int{2, 2, 0, 3, 0, 3, 0, 3} + attributes = []string{"avxEscape | vex128 | vex66 | vex0F3A | vexW0", opcode, + "avxEscape | vex256 | vex66 | vex0F3A | vexW0", opcode} + } + for _, bits := range []int{128, 256, 512} { + memory := fmt.Sprintf("evexN%d", bits/8) + if rule.Opcode != "VPALIGNR" { + memory += fmt.Sprintf(" | evexBcstN%d", 4*(w+1)) + } + memory += " | evexZeroingEnabled" + attributes = append(attributes, fmt.Sprintf("avxEscape | evex%d | evex66 | evex0F3A | evexW%d", bits, w), memory, opcode) + } + if rule.Opcode == "VPALIGNR" && index < 2 { + zcase = "Zvex_i_rm_v_r" + } else if index%2 != 0 { + zcase = "Zevex_i_rm_v_k_r" + } + if index >= len(classes) || rule.RowIndex != index || rule.Ytab != ytab || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != advances[index] || rule.EncodingOffset != offsets[index] || + strings.Join(rule.Classes, ",") != classes[index] || !reflect.DeepEqual(rule.OpBytes, attributes) { + t.Fatal("independent complete align encoder row or attributes changed", rule) + } + } + if !reflect.DeepEqual(counts, map[string]int{"VPALIGNR": 8, "VALIGND": 6, "VALIGNQ": 6}) { + t.Fatal("complete twenty AVX plus one legacy row inventory changed", counts) + } +} + +func TestX86PackedAlignNativeCarrierAndSourceBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + ins := x86ScalarLaneInstruction(t, "PALIGNR $255,(AX),X2") + fn := Func{Instrs: []Instr{ins}} + features := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, fn) + if features != "+sse2,+ssse3" { + t.Fatal("isolated native PALIGNR borrowed another function's features", arch, features) + } + state := x86ResultState{flags: x86ByteFlagsAll, direction: x86ResultDirectionBackward} + state.pointerGP[0] = true + before := state.clone() + if accepted, known := x86ResultPackedAlignStep(ins, &state, nil, arch, false); !known || accepted || !reflect.DeepEqual(state, before) { + t.Fatal("standalone observer acquired a native read") + } + if accepted, known := x86ResultPackedAlignStep(ins, &state, nil, arch, true); !known || !accepted || state.vectorBytes(2)&0xffff != 0xffff { + t.Fatal("continuing past the native opcode lost the large-count known-zero result") + } + for _, mutation := range []string{"pointer", "encoded", "raw", "segment", "address", "raw-return"} { + changed, input := ins, before.clone() + switch mutation { + case "pointer": + input.pointerGP[0] = false + case "encoded": + changed.x86Encoded = true + case "raw": + changed.x86SourceRaw = new(x86SourceRawInstruction) + case "segment": + changed.x86SegmentPrefix = 0x64 + case "address": + changed.x86AddressBits = 32 + case "raw-return": + changed.x86RawNearReturn = new(x86RawNearReturnForm) + } + original := input.clone() + if accepted, _ := x86ResultPackedAlignStep(changed, &input, nil, arch, true); accepted || !reflect.DeepEqual(input, original) { + t.Fatal("native continuation invented an address or physical origin", arch, mutation) + } + } + } +} + +func TestX86PackedAlignMode32ImmediateUsesGoConstantClass(t *testing.T) { + for _, immediate := range []int64{0, 255, 4294967296, 4294967551} { + row := fmt.Sprintf("PALIGNR $%d,X0,X2", immediate) + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86PackedAlignProperties(ins.Op, ins, "386") + if !known || err != nil || form.immediate != byte(immediate) { + t.Fatal("mode32 source class did not preserve Go's actual imm8", immediate, form, err) + } + } +} diff --git a/amd64_packed_align_runtime_test.go b/amd64_packed_align_runtime_test.go new file mode 100644 index 000000000..7e155bfe5 --- /dev/null +++ b/amd64_packed_align_runtime_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86PackedAlignWholeBytes(t *testing.T, bank string) { + t.Helper() + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable whole-byte oracle requires its native 64-bit host counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 llc and clang required") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var source, declarations, prototypes, metadata strings.Builder + count, pending, batches := 0, 0, 0 + flush := func() { + if pending < 1 || pending > 64 { + t.Fatal("invalid bounded runtime batch", pending) + } + t.Run(fmt.Sprint("batch", batches), func(t *testing.T) { + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source.String()) + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", triple, "text") + if err != nil { + t.Fatal("initialized whole-byte source rejected", err) + } + mainC := ` +#include +#include +#include +` + prototypes.String() + ` +typedef void (*Kernel)(uint8_t *,const uint8_t *,const uint8_t *,const uint8_t *,uint64_t); +typedef struct {Kernel run;int width,bits,selector,legacy,mode;} Row; +static const Row rows[]={ +` + metadata.String() + ` +}; +int main(void) { + const uint64_t masks[]={0,UINT64_MAX,UINT64_C(0xa5a5a5a5a5a5a5a5),UINT64_C(0x8000000000000000),1}; + for(unsigned r=0;r>lane)&1)) value=row.mode==2?0:old[output]; + want[output]=value; + } + row.run(out+1,first,second,old,mask); + if(out[0]!=0xa7 || out[258]!=0xa7 || memcmp(out+1,want,sizeof(want))) { + fprintf(stderr,"align row=%u trial=%d bits=%d width=%d selector=%d legacy=%d mode=%d\n",r,trial,row.bits,row.width,row.selector,row.legacy,row.mode); + return 1; + } + } + } + return 0; +} +` + x86ConflictFunnelPortableRuntime(t, llc, clang, "align_whole_"+bank, triple, ir, mainC) + }) + batches++ + pending = 0 + source.Reset() + declarations.Reset() + prototypes.Reset() + metadata.Reset() + } + for _, row := range x86PackedAlignRows() { + if row.legacy && bank != "X" { + continue + } + for mode, suffix := range []string{"", "", ".Z"} { + if row.legacy && mode != 0 { + continue + } + for _, immediate := range []int{0, 1, 15, 16, 17, 31, 32, 255} { + name, mask, legacy := fmt.Sprint("AlignWhole", count), "", 0 + if mode != 0 { + mask = "K1," + } + if row.legacy { + legacy = 1 + } + instruction := x86PackedAlignInstruction(row.op+suffix, bank+"0", bank, mask, immediate, 2, row.legacy) + fmt.Fprintf(&declarations, "func %s(out *[257]byte,a,b,old *[64]byte,mask uint64)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-40\nMOVQ out+0(FP),DI\nMOVQ a+8(FP),AX\nMOVQ b+16(FP),BX\nMOVQ old+24(FP),DX\n"+ + "VMOVUPS (AX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (DX),Z2\nVMOVUPS (DX),Z3\nKMOVQ mask+32(FP),K1\nSTC\n%s\n"+ + "VMOVUPS Z2,0(DI)\nVMOVUPS Z0,64(DI)\nVMOVUPS Z1,128(DI)\nVMOVUPS Z3,192(DI)\nSETCS 256(DI)\nRET\n", name, instruction) + fmt.Fprintf(&prototypes, "extern void %s(uint8_t *,const uint8_t *,const uint8_t *,const uint8_t *,uint64_t);\n", name) + fmt.Fprintf(&metadata, "{%s,%d,%d,%d,%d,%d},\n", name, amd64VectorByteWidth(Reg(bank+"0")), row.bits, immediate, legacy, mode) + count++ + pending++ + if pending == 64 { + flush() + } + } + } + } + want := 72 + if bank == "X" { + want = 80 + } + if count != want { + t.Fatal("incomplete whole-byte family/mask/selector union", count, want) + } + if pending != 0 { + flush() + } + if batches != 2 { + t.Fatal("incomplete bounded runtime union", batches) + } +} + +func TestX86PackedAlignWholeXByteSemantics(t *testing.T) { + x86PackedAlignWholeBytes(t, "X") +} + +func TestX86PackedAlignWholeYByteSemantics(t *testing.T) { + x86PackedAlignWholeBytes(t, "Y") +} + +func TestX86PackedAlignWholeZByteSemantics(t *testing.T) { + x86PackedAlignWholeBytes(t, "Z") +} diff --git a/amd64_packed_arithmetic_allocation_test.go b/amd64_packed_arithmetic_allocation_test.go new file mode 100644 index 000000000..befe35576 --- /dev/null +++ b/amd64_packed_arithmetic_allocation_test.go @@ -0,0 +1,57 @@ +package plan9asm + +import ( + "errors" + "reflect" + "testing" +) + +var x86PackedArithmeticQuerySink x86LegacyIntegerArithmeticForm + +func TestX86LegacyIntegerArithmeticQueryIsReadOnlyAndBounded(t *testing.T) { + rows := []struct { + instruction string + source Reg + allocations float64 + }{ + {"PADDQ X0,X1", "X0", 0}, + {"PSUBL X0,X1", "X0", 0}, + {"PADDB X0,X1", "X0", 1}, + } + for _, bank := range []struct{ input, physical Reg }{ + {AX, "M0"}, {CX, "M1"}, {DX, "M2"}, {BX, "M3"}, + {SP, "M4"}, {BP, "M5"}, {SI, "M6"}, {DI, "M7"}, + {"R8", "M0"}, {"R9", "M1"}, {"R10", "M2"}, {"R11", "M3"}, + {"R12", "M4"}, {"R13", "M5"}, {"R14", "M6"}, {"R15", "M7"}, + } { + rows = append(rows, struct { + instruction string + source Reg + allocations float64 + }{"PADDB " + string(bank.input) + ",M1", bank.physical, 1}) + } + for _, row := range rows { + t.Run(row.instruction, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, row.instruction) + before := ins + before.Args = append([]Operand(nil), ins.Args...) + var known bool + var err error + allocations := testing.AllocsPerRun(100, func() { + x86PackedArithmeticQuerySink, known, err = x86LegacyIntegerArithmeticProperties(ins.Op, ins, "amd64") + }) + if !known || err != nil || x86PackedArithmeticQuerySink.source.Reg != row.source { + t.Fatal("query lost the original physical bank", known, err, x86PackedArithmeticQuerySink.source) + } + if !reflect.DeepEqual(ins, before) { + t.Fatal("read-only shape query changed its original input") + } + if x86PackedArithmeticQuerySink.mmx && !errors.Is(x86LegacyIntegerArithmeticEmitterContext(x86PackedArithmeticQuerySink, ins), ErrProbeNeedsContext) { + t.Fatal("physical bank projection acquired emitter permission") + } + if allocations > row.allocations { + t.Errorf("shape query allocates %.0f times; at most %.0f physical projections needed", allocations, row.allocations) + } + }) + } +} diff --git a/amd64_packed_arithmetic_boundary_test.go b/amd64_packed_arithmetic_boundary_test.go new file mode 100644 index 000000000..320198410 --- /dev/null +++ b/amd64_packed_arithmetic_boundary_test.go @@ -0,0 +1,71 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestX86PackedArithmeticFamilyMissesStayEmpty(t *testing.T) { + for _, row := range []struct { + op Op + families x86PackedArithmeticFamily + }{ + {"MOVQ", 0}, + {"VPADDB", x86PackedArithmeticSubtract}, + {"VPSUBD", x86PackedArithmeticAdd}, + {"VPMULLQ", x86PackedArithmeticAdd | x86PackedArithmeticSubtract}, + {"PADDD", x86PackedArithmeticLowMultiply}, + } { + t.Run(fmt.Sprintf("%s/families%d", row.op, row.families), func(t *testing.T) { + spec, known := x86PackedArithmeticSpecForOp(row.op, row.families) + if known || spec != (x86PackedArithmeticSpec{}) { + t.Fatal("unrecognized family leaked another family's metadata", known, spec) + } + form, known, err := x86PackedArithmeticProperties(row.op, Instr{Op: row.op}, "amd64", row.families) + if known || err != nil || form != (x86PackedArithmeticForm{}) { + t.Fatal("unrecognized family began physical parsing", known, err, form) + } + }) + } +} + +func TestX86PackedSubtractMetadataRetainsBareNameContract(t *testing.T) { + for _, name := range []string{"", "VPADDB", "VPMULLQ", "vpsubd", "VPSUBD.BAD", "VPSUBD.Z", " VPSUBD"} { + t.Run(name, func(t *testing.T) { + bits, mode, vector, known := amd64PackedSubtractProperties(name) + if known || bits != 0 || mode != 0 || vector { + t.Fatal("bare-name query changed its unrecognized result", bits, mode, vector, known) + } + }) + } + bits, mode, vector, known := amd64PackedSubtractProperties("VPSUBD") + if !known || bits != 32 || mode != amd64PackedSubtractWrap || !vector { + t.Fatal("original bare opcode was lost", bits, mode, vector, known) + } +} + +func TestX86PackedArithmeticDispatchKeepsExactSourceIdentity(t *testing.T) { + for _, row := range []struct { + op Op + instruction string + }{ + {"VPADDD", "VPSUBQ Y0,Y1,Y2"}, + {"VPSUBD", "VPADDD Y0,Y1,Y2"}, + {"VPADDUSB", "VPADDB Y0,Y1,Y2"}, + {"PADDD", "PADDL X0,X1"}, + {"VPMULLD", "VPADDD Y0,Y1,Y2"}, + } { + t.Run(string(row.op), func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, row.instruction) + form, known, err := x86PackedArithmeticProperties(row.op, ins, "amd64", 0) + if !known || err == nil || form != (x86PackedArithmeticForm{}) { + t.Fatal("dispatch replaced the original source opcode", known, err, form) + } + }) + } + // Case folding is still allowed; this establishes syntax only, not NUM. + ins := x86ScalarLaneInstruction(t, "VPSUBD Y0,Y1,Y2") + if _, known, err := x86PackedArithmeticProperties("vpsubd", ins, "amd64", 0); !known || err != nil { + t.Fatal("case-equivalent original opcode was lost", known, err) + } +} diff --git a/amd64_packed_arithmetic_forms.go b/amd64_packed_arithmetic_forms.go new file mode 100644 index 000000000..e027f82df --- /dev/null +++ b/amd64_packed_arithmetic_forms.go @@ -0,0 +1,133 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedArithmeticFamily uint8 + +const ( + x86PackedArithmeticAdd x86PackedArithmeticFamily = 1 << iota + x86PackedArithmeticSubtract + x86PackedArithmeticLowMultiply +) + +type x86PackedArithmeticMode uint8 + +const ( + x86PackedWrap x86PackedArithmeticMode = iota + x86PackedSignedSaturating + x86PackedUnsignedSaturating +) + +type x86PackedArithmeticSpec struct { + x86PackedBinarySpec + laneBits int + family x86PackedArithmeticFamily + mode x86PackedArithmeticMode +} + +// One catalog owns all add/sub/DQ-low-product source spellings. Width, encoding, +// saturation and broadcast are independent axes constrained by Go's tables. +// PADDD aliases PADDL; word products retain their separate MMX grammar. +var x86PackedArithmeticSpecs = map[Op]x86PackedArithmeticSpec{ + "PADDB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 8, x86PackedArithmeticAdd, x86PackedWrap}, + "PADDW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16, x86PackedArithmeticAdd, x86PackedWrap}, + "PADDL": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 32, x86PackedArithmeticAdd, x86PackedWrap}, + "PADDD": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 32, x86PackedArithmeticAdd, x86PackedWrap}, + "PADDQ": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 64, x86PackedArithmeticAdd, x86PackedWrap}, + "PADDSB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 8, x86PackedArithmeticAdd, x86PackedSignedSaturating}, + "PADDSW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16, x86PackedArithmeticAdd, x86PackedSignedSaturating}, + "PADDUSB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 8, x86PackedArithmeticAdd, x86PackedUnsignedSaturating}, + "PADDUSW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16, x86PackedArithmeticAdd, x86PackedUnsignedSaturating}, + "VPADDB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 8, x86PackedArithmeticAdd, x86PackedWrap}, + "VPADDW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, x86PackedArithmeticAdd, x86PackedWrap}, + "VPADDD": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, 32, x86PackedArithmeticAdd, x86PackedWrap}, + "VPADDQ": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 8, zeroing: true}, 64, x86PackedArithmeticAdd, x86PackedWrap}, + "VPADDSB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 8, x86PackedArithmeticAdd, x86PackedSignedSaturating}, + "VPADDSW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, x86PackedArithmeticAdd, x86PackedSignedSaturating}, + "VPADDUSB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 8, x86PackedArithmeticAdd, x86PackedUnsignedSaturating}, + "VPADDUSW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, x86PackedArithmeticAdd, x86PackedUnsignedSaturating}, + + "PSUBB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 8, x86PackedArithmeticSubtract, x86PackedWrap}, + "PSUBW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 16, x86PackedArithmeticSubtract, x86PackedWrap}, + "PSUBL": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 32, x86PackedArithmeticSubtract, x86PackedWrap}, + "PSUBQ": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 64, x86PackedArithmeticSubtract, x86PackedWrap}, + "PSUBSB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 8, x86PackedArithmeticSubtract, x86PackedSignedSaturating}, + "PSUBSW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 16, x86PackedArithmeticSubtract, x86PackedSignedSaturating}, + "PSUBUSB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 8, x86PackedArithmeticSubtract, x86PackedUnsignedSaturating}, + "PSUBUSW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 16, x86PackedArithmeticSubtract, x86PackedUnsignedSaturating}, + "VPSUBB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 8, x86PackedArithmeticSubtract, x86PackedWrap}, + "VPSUBW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, x86PackedArithmeticSubtract, x86PackedWrap}, + "VPSUBD": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, 32, x86PackedArithmeticSubtract, x86PackedWrap}, + "VPSUBQ": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 8, zeroing: true}, 64, x86PackedArithmeticSubtract, x86PackedWrap}, + "VPSUBSB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 8, x86PackedArithmeticSubtract, x86PackedSignedSaturating}, + "VPSUBSW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, x86PackedArithmeticSubtract, x86PackedSignedSaturating}, + "VPSUBUSB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 8, x86PackedArithmeticSubtract, x86PackedUnsignedSaturating}, + "VPSUBUSW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, x86PackedArithmeticSubtract, x86PackedUnsignedSaturating}, + + "PMULLD": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 32, x86PackedArithmeticLowMultiply, x86PackedWrap}, + "VPMULLD": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, 32, x86PackedArithmeticLowMultiply, x86PackedWrap}, + "VPMULLQ": {x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, broadcastBytes: 8, zeroing: true}, 64, x86PackedArithmeticLowMultiply, x86PackedWrap}, +} + +type x86PackedArithmeticForm struct { + x86PackedBinaryForm + spec x86PackedArithmeticSpec +} + +func x86PackedArithmeticSpecForOp(op Op, families x86PackedArithmeticFamily) (x86PackedArithmeticSpec, bool) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86PackedArithmeticSpecs[Op(base)] + if !known || families != 0 && spec.family&families == 0 { + return x86PackedArithmeticSpec{}, false + } + return spec, true +} + +func x86PackedArithmeticProperties(op Op, ins Instr, arch string, families x86PackedArithmeticFamily) (x86PackedArithmeticForm, bool, error) { + spec, known := x86PackedArithmeticSpecForOp(op, families) + if !known { + return x86PackedArithmeticForm{}, false, nil + } + return x86PackedArithmeticFormForSpec(op, ins, arch, spec) +} + +// A validated shape supplies no initialized values, storage or caller facts. +// Legacy bank projection and decoder-only mode32 adaptation remain distinct. +func x86PackedArithmeticFormForSpec(op Op, ins Instr, arch string, spec x86PackedArithmeticSpec) (x86PackedArithmeticForm, bool, error) { + low := spec.family == x86PackedArithmeticLowMultiply + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + family := "arithmetic" + if low { + family = "low multiply" + } + return x86PackedArithmeticForm{}, true, fmt.Errorf("packed %s dispatch/source opcode mismatch: %s/%s", family, op, ins.Op) + } + if !low && (spec.encoding == x86PackedLogicalLegacyFP || spec.encoding == x86PackedLogicalLegacyInteger) { + legacy, _, err := x86LegacyIntegerArithmeticProperties(op, ins, arch) + if err == nil && spec.family == x86PackedArithmeticAdd { + err = x86LegacyIntegerArithmeticEmitterContext(legacy, ins) + } + return x86PackedArithmeticForm{legacy.x86PackedBinaryForm, spec}, true, err + } + local := ins + local.Op = op + grammarArch := arch + if low && arch == "386" && local.x86Encoded && spec.encoding != x86PackedLogicalLegacyFP { + // Only the decoder owns this physical identity, not Go's named frontend. + grammarArch = "amd64" + } + binary, err := x86PackedBinaryProperties(local, grammarArch, spec.x86PackedBinarySpec) + if err == nil && grammarArch != arch && binary.memory >= 0 && !x86PackedShiftMemoryValid(local.Args[binary.memory], arch) { + err = fmt.Errorf("386 %s physical source has an unbound address", op) + } + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && low && binary.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: packed low multiply has no bound address-size override", ErrProbeNeedsContext) + } + return x86PackedArithmeticForm{binary, spec}, true, err +} diff --git a/amd64_packed_average_forms.go b/amd64_packed_average_forms.go new file mode 100644 index 000000000..a2039711f --- /dev/null +++ b/amd64_packed_average_forms.go @@ -0,0 +1,31 @@ +package plan9asm + +type x86PackedAverageSpec struct { + x86PackedBinarySpec + laneBits int +} + +// Go's complete unsigned rounded-average family: ymm/Py1 for legacy and +// _yvandnpd for VEX/EVEX. E4.nb has mask suppression but no broadcast or SAE. +var x86PackedAverageSpecs = map[Op]x86PackedAverageSpec{ + "PAVGB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 8}, + "PAVGW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16}, + "VPAVGB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 8}, + "VPAVGW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16}, +} + +type x86PackedAverageForm = x86PackedPhysicalForm + +// A shape/effect observation supplies no input initialization, caller, frame +// or native storage authority. All consumers retain their own proof gates. +func x86PackedAverageProperties(op Op, ins Instr, arch string) (x86PackedAverageForm, bool, error) { + return x86PackedPhysicalProperties(op, ins, arch, x86PackedAverage) +} + +func x86ResultPackedAverageStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, known, err := x86PackedAverageProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + return x86ResultFixedVectorInputs(ins, form.x86PackedBinaryForm, false, state, owned, arch), true +} diff --git a/amd64_packed_average_fp_test.go b/amd64_packed_average_fp_test.go new file mode 100644 index 000000000..c9f8a32fc --- /dev/null +++ b/amd64_packed_average_fp_test.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "go/types" + "os" + "strings" + "testing" +) + +func TestX86PackedAverageFPSpansThreeAPIsFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + t.Run(triple, func(t *testing.T) { + var source, declarations strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, op := range x86AverageNames[2:] { + for _, bank := range []string{"X", "Y", "Z"} { + for _, mode := range []int{0, 1, 2} { + if arch == "386" && mode != 0 { + continue // Actual named Go rejection is tested independently. + } + name := fmt.Sprintf("F%d", count) + count++ + row := op + if mode == 2 { + row += ".Z" + } + row += " input+0(FP)," + bank + "1" + if mode != 0 { + row += ",K1" + } + row += "," + bank + "2" + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-64\n%s\nRET\n", name, row) + fmt.Fprintf(&declarations, "func %s(input [64]byte)\n", name) + } + } + } + if arch == "amd64" { + for _, op := range x86AverageNames[:2] { + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-64\n%s input+0(FP),M1\nRET\n", count, op) + fmt.Fprintf(&declarations, "func F%d(input [64]byte)\n", count) + count++ + } + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewPackageAPI(t, source.String(), pkg, arch, triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original FP format source invented a vector/mask/MMX input", api, err) + } + } + ir := x86AverageFormatObjectUnit(t, file, sigs, arch, triple) + x86ResultInitializationObject(t, llc, triple, ir, "average-fp-original-emitter-only") + + var initialized, completeDeclarations strings.Builder + complete := 0 + for _, function := range file.Funcs { + instruction := function.Instrs[1] + if !strings.HasPrefix(string(instruction.Op), "V") { + continue // Original MMX rows remain object/Context controls above. + } + name := strings.TrimPrefix(function.Sym, "·") + fmt.Fprintf(&initialized, "TEXT ·%s(SB),4,$0-64\nMOVL $7,AX\nKMOVD AX,K1\n"+ + "VPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\n%s\nRET\n", name, instruction.Raw) + fmt.Fprintf(&completeDeclarations, "func %s(input [64]byte)\n", name) + complete++ + } + want := 18 + if arch == "386" { + want = 6 + } + if complete != want { + t.Fatal("complete initialized FP row union changed", complete, want) + } + goAssemblerOracleOutput(t, arch, initialized.String(), "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, initialized.String(), completeDeclarations.String(), arch, triple, api) + if err != nil { + t.Fatal("complete initialized logical FP average rejected", api, err) + } + x86ResultInitializationObject(t, llc, triple, ir, "average-initialized-fp-"+api) + } + }) + } +} + +func TestX86PackedAverageFPHolesCannotBeMaskedAway(t *testing.T) { + for _, op := range x86AverageNames[2:] { + for index, bank := range []string{"X", "Y", "Z"} { + width := 16 << index + for _, start := range []int{0, 1, -1, 16} { + for _, mode := range []string{"", ".Z"} { + row := fmt.Sprintf("%s%s input%+d(FP),%s1,K1,%s2", op, mode, start, bank, bank) + source := "TEXT f(SB),4,$0-8\nKMOVQ AX,K1\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, frame := range []FrameLayout{ + {Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}}, + {Params: []FrameSlot{{Offset: 0, Type: I8, Index: 0, Field: -1}, {Offset: int64(width - 1), Type: I8, Index: 1, Field: -1}}}, + } { + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void, Args: []LLVMType{I64, I64}, Frame: frame}, nil) + before := output.String() + ins := file.Funcs[0].Instrs[2] + if _, _, err := ctx.lowerPackedUnsignedAverage(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("%s incomplete logical byte span emitted IR: %v", row, err) + } + } + } + } + } + } +} + +func TestX86PackedAverageRequiredLinuxOriginalVEXRegistration(t *testing.T) { + const name = "amd64_packed_average_original_linux_test.go" + for _, goos := range []string{"linux", "darwin", "windows"} { + ctx := build.Default + ctx.GOOS, ctx.GOARCH = goos, "amd64" + selected, err := ctx.MatchFile(".", name) + if err != nil || selected != (goos == "linux") { + t.Fatal("required original AVX axis has wrong actual Go selection", goos, selected, err) + } + } + file, err := parser.ParseFile(token.NewFileSet(), name, nil, 0) + if err != nil { + t.Fatal(err) + } + if len(file.Decls) != 2 { + t.Fatal("required wrapper declaration changed") + } + fn, ok := file.Decls[1].(*ast.FuncDecl) + if !ok || fn.Name.Name != "TestCrossLinuxRuntimeMatrixX86PackedAverageOriginalVEXAliases" || fn.Body == nil || len(fn.Body.List) != 1 { + t.Fatal("required original AVX wrapper omitted or bypassed") + } + expression, ok := fn.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("required wrapper is not an actual unconditional call") + } + call, ok := expression.X.(*ast.CallExpr) + if !ok || len(call.Args) != 2 { + t.Fatal("required wrapper lost its original Go helper") + } + callee, ok := call.Fun.(*ast.Ident) + wide, wideOK := call.Args[1].(*ast.Ident) + if !ok || callee.Name != "x86AverageOriginalGo" || !wideOK || wide.Name != "true" { + t.Fatal("required original wide alias scope was narrowed") + } + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil || !strings.Contains(string(workflow), "-ci-selector '^(TestCrossLinuxRuntimeMatrix|") { + t.Fatal("required Linux CI selector does not select this family", err) + } +} diff --git a/amd64_packed_average_original_go_test.go b/amd64_packed_average_original_go_test.go new file mode 100644 index 000000000..799b1d34f --- /dev/null +++ b/amd64_packed_average_original_go_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedAverageOriginalGo(t *testing.T) { + x86AverageOriginalGo(t, false) +} + +func x86AverageOriginalGo(t *testing.T, wide bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + if wide { + execution.requireAVX(t) + x86AverageRequireAVX2(t, execution) + } + var assembly, declarations, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle + +import ( + "testing" + "unsafe" + "runtime" + "encoding/binary" +) + +func aligned(storage *[192]byte) (*[64]byte,int) { + offset:=64+int((-uintptr(unsafe.Pointer(&storage[64])))&63) + return (*[64]byte)(unsafe.Pointer(&storage[offset])),offset +} + +func TestOriginal(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + for sample:=0;sample<257;sample++ { + for status:=0;status<2;status++ { + var a,b,out [192]byte + ap,_:=aligned(&a) + bp,_:=aligned(&b) + op,offset:=aligned(&out) + for i:=range a { + a[i]=0xa7 + b[i]=0xb9 + out[i]=0xc3 + } + + for i:=0;i<64;i++ { + ap[i]=byte(sample*71+i*37) + bp[i]=byte(sample*19-i*13) + if sample<4 { + ap[i]=[4]byte{255,0,128,255}[sample] + bp[i]=[4]byte{255,1,128,0}[sample] + } + } + beforeA,beforeB:=a,b + flags:=uint64(2) + if status!=0 { + flags|=0x8d5 + } + csr:=uint32(0x1f80|((sample%4)<<13)|(sample%64)) +`) + count := 0 + for _, row := range x86AverageNumericCases() { + if row.width == 64 || row.masking != 0 || !wide && strings.HasPrefix(row.op, "V") { + continue // Native EVEX is a distinct, unexecuted axis here. + } + move, bank, observedWidth := "MOVUPS", "X", 16 + if wide { + move, bank, observedWidth = "VMOVUPS", "Y", 32 + } + name := fmt.Sprintf("Average%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$8-36 +STMXCSR saved-4(SP) +LDMXCSR csr+32(FP) +MOVQ out+0(FP),AX +MOVQ a+8(FP),BX +MOVQ b+16(FP),CX +`, name) + fmt.Fprintf(&assembly, "%s (CX),%s0\n%s (CX),%s2\n", move, bank, move, bank) + if !row.memory || row.destination == 1 { + fmt.Fprintf(&assembly, "%s (BX),%s1\n", move, bank) + } + if row.mmx { + fmt.Fprintf(&assembly, "MOVQ (CX),M0\nMOVQ (BX),M1\n%s %s0,(AX)\n", move, bank) + } + assembly.WriteString("PUSHQ flags+24(FP)\nPOPFQ\n" + row.instruction() + "\n") + if row.mmx { + assembly.WriteString("MOVQ M0,(AX)\nEMMS\n") + } else { + fmt.Fprintf(&assembly, "%s %s%d,(AX)\n", move, bank, row.destination) + } + assembly.WriteString("PUSHFQ\nPOPQ R10\nMOVQ R10,32(AX)\nSTMXCSR 40(AX)\nLDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&tests, `{ + for i:=range out { + out[i]=0xc3 + } + %s(op,ap,bp,flags,csr) + if binary.LittleEndian.Uint64(op[32:40])&0x8d5!=flags&0x8d5 || binary.LittleEndian.Uint32(op[40:44])!=csr || a!=beforeA || b!=beforeB { + t.Fatal("original flags/CSR/input canary") + } + for i:=0;i<%d;i++ { + start:=i/%d*%d + x,y:=uint32(ap[start]),uint32(bp[start]) + if %d==2 { + x|=uint32(ap[start+1])<<8 + y|=uint32(bp[start+1])<<8 + } + value:=(x+y+1)/2 + want:=byte(value>>uint(8*(i%%%d))) + if i>=%d { + want=0 + if %t { + want=bp[i] + } + } + if op[i]!=want { + t.Fatalf("%s sample%%d byte%%d got%%x want%%x",sample,i,op[i],want) + } + } + for i:=range out { + if (i=offset+44||i>=offset+%d&&i +int main(void) { + __builtin_cpu_init(); + int avx2 = !!__builtin_cpu_supports("avx2"); + printf("required original VPAVG Y-width AVX2=%d\n", avx2); + return avx2 ? 0 : 1; +} +` + if err := os.WriteFile(source, []byte(preflight), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + t.Logf("actual original VPAVG capability: %s", strings.TrimSpace(string(execution.run(t, executable)))) +} diff --git a/amd64_packed_average_original_linux_test.go b/amd64_packed_average_original_linux_test.go new file mode 100644 index 000000000..069969777 --- /dev/null +++ b/amd64_packed_average_original_linux_test.go @@ -0,0 +1,11 @@ +//go:build linux + +package plan9asm + +import "testing" + +// Required by the existing Linux cross-matrix selector. Its actual AVX +// preflight and pinned guest transport must fail, not skip, if unavailable. +func TestCrossLinuxRuntimeMatrixX86PackedAverageOriginalVEXAliases(t *testing.T) { + x86AverageOriginalGo(t, true) +} diff --git a/amd64_packed_average_physical_test.go b/amd64_packed_average_physical_test.go new file mode 100644 index 000000000..3ad04fdde --- /dev/null +++ b/amd64_packed_average_physical_test.go @@ -0,0 +1,298 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +var x86AverageNames = []string{"PAVGB", "PAVGW", "VPAVGB", "VPAVGW"} + +// Independent of the production grammar. Named mode32 MMX is excluded by +// Go's Py1 selector, while low X/Y/Z operands have physical mode32 identities. +func x86AveragePhysicalRows(arch string) []string { + var rows []string + for _, name := range x86AverageNames[:2] { + for _, source := range []string{"X0", "X7", "(AX)", "16(AX)(CX*4)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+",X1") + } + if arch == "amd64" { + rows = append(rows, name+" X15,X0", name+" M0,M7", name+" (AX),M7") + } + } + for _, name := range x86AverageNames[2:] { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", bank + "7", "(AX)", "16(AX)(CX*4)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1,%s2", name, source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s,%s31,K1,%s30", name, source, bank, bank), + fmt.Sprintf("%s.Z %s,%s31,K7,%s30", name, source, bank, bank)) + } + } + } + } + return rows +} + +func TestX86PackedAverageThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + var source, declarations strings.Builder + declarations.WriteString("package oracle\n") + for index, row := range x86AveragePhysicalRows(arch) { + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", index, row) + fmt.Fprintf(&declarations, "func F%d()\n", index) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewPackageAPI(t, source.String(), pkg, arch, triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("complete original format source invented input or memory state", api, err) + } + } + ir := x86AverageFormatObjectUnit(t, file, sigs, arch, triple) + x86ResultInitializationObject(t, llc, triple, ir, "average-original-emitter-only") + x86ResultInitializationObject(t, llc, triple, x86MulDivRoundOptimize(t, ir), "average-O2") + }) + } +} + +func TestX86PackedAverageShapesEffectsDoNotGrantCallerPermissions(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range append(x86AveragePhysicalRows(arch), "VPAVGB.Z value+0(FP),X1,K1,X2", "VPAVGW value+0(FP),Z1,Z2") { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, known, err := x86PackedAverageProperties(ins.Op, ins, arch) + if arch == "386" && strings.Contains(row, "K1") { + if err == nil { + t.Fatal("named mode32 mask became a physical encoding") + } + continue + } + if !known || err != nil || before != fmt.Sprintf("%#v", ins) || x86CallerVectorMoveKnown(ins) { + t.Fatalf("shape changed input or acquired caller permission: %s: %v", row, err) + } + for index := range ins.Args { + width, write, valid := x86InstructionMemoryShape(ins, index, arch) + if valid != (index == form.memory) || valid && (width != int64(form.memoryBytes) || write) { + t.Fatal("memory read shape differs from native width", row, width, write, valid) + } + } + effect, observed, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if form.legacy && (!observed || err != nil || len(effect.operands) != 2 || + effect.flagsDefined != 0 || effect.flagsKilled != 0 || + !effect.operands[0].read || effect.operands[0].write || + !effect.operands[1].read || !effect.operands[1].write || + effect.operands[0].width != int64(form.byteWidth) || effect.operands[1].width != int64(form.byteWidth)) { + t.Fatal("legacy full input/output byte effect changed", row, effect, err) + } + // All three dispatch identities are equivalent, but a contradictory + // source never emits IR. Keep the existing empty direct-helper API. + for _, identity := range []Op{"", ins.Op, Op(strings.ToLower(string(ins.Op))), "PADDW"} { + local := ins + local.Op = identity + copy := local + _, _, err := x86PackedAverageProperties(ins.Op, local, arch) + bad := identity == "PADDW" + if (err != nil) != bad || !reflect.DeepEqual(copy, local) { + t.Fatal("dispatch identity mismatch", ins.Op, identity, err) + } + } + } + } +} + +func TestX86PackedAverageActualGoCompleteTables(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APAVGB, ymm, Py1, opBytes{0xe0, Pe, 0xe0}}", + "{APAVGW, ymm, Py1, opBytes{0xe3, Pe, 0xe3}}", + } { + if strings.Count(string(legacy), row) != 1 { + t.Fatalf("actual Go legacy registration changed: %s", row) + } + } + start := strings.Index(string(legacy), "var ymm =") + if start < 0 { + t.Fatal("missing actual ymm table") + } + ymm := strings.SplitN(string(legacy)[start:], "\n}", 2)[0] + for _, row := range []string{"{Zm_r_xm, 1, argList{Ymm, Ymr}}", "{Zm_r_xm, 2, argList{Yxm, Yxr}}"} { + if strings.Count(ymm, "argList{") != 2 || strings.Count(ymm, row) != 1 { + t.Fatal("actual ymm class/selector table changed", ymm) + } + } + for index, name := range x86AverageNames[2:] { + opcode := []string{"E0", "E3"}[index] + expected := fmt.Sprintf(`{as: A%s, ytab: _yvandnpd, prefix: Pavx, op: opBytes{ + avxEscape | vex128 | vex66 | vex0F | vexW0, 0x%s, + avxEscape | vex256 | vex66 | vex0F | vexW0, 0x%s, + avxEscape | evex128 | evex66 | evex0F | evexW0, evexN16 | evexZeroingEnabled, 0x%s, + avxEscape | evex256 | evex66 | evex0F | evexW0, evexN32 | evexZeroingEnabled, 0x%s, + avxEscape | evex512 | evex66 | evex0F | evexW0, evexN64 | evexZeroingEnabled, 0x%s, + }},`, name, opcode, opcode, opcode, opcode, opcode) + if err := x86SignedPackedFloatValidateGoOpcodeRows(string(avx), name, expected); err != nil { + t.Fatal(err) + } + } + start = strings.Index(string(avx), "var _yvandnpd =") + if start < 0 { + t.Fatal("missing actual ytab") + } + table := strings.SplitN(string(avx)[start:], "\n}", 2)[0] + "\n}" + if err := x86AverageValidateGoYtab(table); err != nil { + t.Fatal(err) + } +} + +const x86AverageGoYtab = `var _yvandnpd = []ytab{ + {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr, Yxr}}, + {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yym, Yyr, Yyr}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{YxmEvex, YxrEvex, YxrEvex}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{YxmEvex, YxrEvex, Yknot0, YxrEvex}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{YymEvex, YyrEvex, YyrEvex}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{YymEvex, YyrEvex, Yknot0, YyrEvex}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{Yzm, Yzr, Yzr}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{Yzm, Yzr, Yknot0, Yzr}}, +}` + +func x86AverageValidateGoYtab(table string) error { + actual, err := x86SignedPackedFloatGoOpcodeTokens(table) + if err != nil { + return err + } + expected, err := x86SignedPackedFloatGoOpcodeTokens(x86AverageGoYtab) + if err != nil { + return err + } + if !reflect.DeepEqual(actual, expected) { + return fmt.Errorf("complete actual Go packed-average selector table changed:\n%s", table) + } + return nil +} + +func TestX86PackedAverageGoSelectorMutationsAreDetected(t *testing.T) { + if err := x86AverageValidateGoYtab(x86AverageGoYtab); err != nil { + t.Fatal(err) + } + for _, mutation := range []struct{ old, replacement string }{ + {"zoffset: 2", "zoffset: 3"}, + {"Zvex_rm_v_r", "Zevex_rm_v_r"}, + {"Yknot0", "Yk"}, + {"YxmEvex", "YymEvex"}, + {"{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr, Yxr}},", ""}, + } { + changed := strings.Replace(x86AverageGoYtab, mutation.old, mutation.replacement, 1) + if err := x86AverageValidateGoYtab(changed); err == nil { + t.Fatalf("selector mutation retained source authority: %#v", mutation) + } + } +} + +func TestX86PackedAverageActualGoModeAndPhysicalBoundary(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + for _, row := range x86AveragePhysicalRows(arch) { + source.WriteString(row + "\n") + } + source.WriteString("RET\n") + goAssemblerOracleOutput(t, arch, source.String(), "", true) + } + for _, name := range x86AverageNames[:2] { + for _, source := range []string{"M0", "(AX)", "AX"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+name+" "+source+",M1\nRET\n", "", false) + } + } + for _, name := range x86AverageNames[2:] { + for _, bank := range []string{"X", "Y"} { + for position := 0; position < 3; position++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[position] = bank + "20" + assembly := "TEXT f(SB),4,$0-0\n" + name + " " + strings.Join(args, ",") + "\nRET\n" + goAssemblerOracleOutput(t, "386", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "386" + before := out.String() + known, _, err := ctx.lowerPackedUnsignedAverage(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]) + if !known || !errors.Is(err, ErrProbeNeedsContext) || out.String() != before { + t.Errorf("unproved physical high bank emitted IR: %s: %v", assembly, err) + } + } + } + } +} + +func TestX86PackedAverageNativeStorageAndIdentityBoundaries(t *testing.T) { + for _, name := range x86AverageNames[:2] { + for _, row := range []string{name + " (SP),X0", name + " (BP),X0", name + " (AX)(BP*2),X0", name + " value+0(FP),X0"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], x86WordMaddFPType(t), nil) + before := out.String() + _, _, err = ctx.lowerPackedUnsignedAverage(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]) + if !errors.Is(err, ErrProbeNeedsContext) || out.String() != before { + t.Errorf("virtual storage gained native m128 alignment: %s: %v", row, err) + } + } + } + for _, name := range x86AverageNames { + row := name + " X0,X1" + if strings.HasPrefix(name, "V") { + row += ",X2" + } + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + ins.Op = "PADDW" + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + if _, _, err := ctx.lowerPackedUnsignedAverage(Op(name), ins); err == nil || out.String() != before { + t.Errorf("contradictory opcode emitted PAVG IR: %s: %v", name, err) + } + } +} diff --git a/amd64_packed_average_raw_test.go b/amd64_packed_average_raw_test.go new file mode 100644 index 000000000..0d6d86ebd --- /dev/null +++ b/amd64_packed_average_raw_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "encoding/json" + "errors" + "fmt" + "os/exec" + "strings" + "testing" +) + +func TestX86PackedAverageActualRawPhysicalModes(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + llc := findLLVM22Tool("llc") + if mc == "" || llc == "" { + t.Fatal("LLVM22 llvm-mc and llc required") + } + for _, op := range x86AverageNames[2:] { + for _, bank := range []string{"X", "Y", "Z"} { + for _, memory := range []bool{false, true} { + for mode := 0; mode < 3; mode++ { + first, name := bank+"1", op + if memory { + first = "16(AX)" + } + if mode == 2 { + name += ".Z" + } + args := first + "," + bank + "0" + if mode != 0 { + args += ",K1" + } + row := name + " " + args + "," + bank + "2" + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if len(code) < 2 || code[len(code)-1] != 0xc3 { + t.Fatal("original object lacks exact RET", row, code) + } + code = code[:len(code)-1] + for _, arch := range []string{"amd64", "386"} { + bits, triple := 64, "x86_64-unknown-linux-gnu" + if arch == "386" { + bits, triple = 32, "i386-unknown-linux-gnu" + var bytes, source strings.Builder + for _, value := range code { + fmt.Fprintf(&bytes, "0x%02x ", value) + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + goAssemblerOracleOutput(t, arch, "TEXT raw(SB),4,$0-0\n"+source.String()+"RET\n", "", true) + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "-triple="+triple, "-disassemble", "-mattr=+avx2,+avx512f,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(bytes.String()) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || !strings.Contains(string(output), strings.ToLower(op)) || strings.Contains(string(output), "warning") { + t.Fatalf("actual mode32 physical instruction invalid: %x: %v\n%s", code, err, output) + } + } + ins, length, known, err := decodedX86PackedAverageInstruction(code, bits) + if !known || err != nil || length != len(code) || !ins.x86Encoded { + t.Fatalf("complete physical bytes not bound: %x: %v", code, err) + } + if _, known, err := x86PackedAverageProperties(ins.Op, ins, arch); !known || err != nil { + t.Fatal("physical source rejected", row, arch, err) + } + encoded, err := json.Marshal(ins) + if err != nil { + t.Fatal(err) + } + var copied Instr + if err := json.Unmarshal(encoded, &copied); err != nil || copied.x86Encoded { + t.Fatal("private marker survived portable JSON", err) + } + if arch == "386" && mode != 0 { + if _, _, err := x86PackedAverageProperties(copied.Op, copied, arch); err == nil { + t.Fatal("named/JSON four operands forged physical mode32 authority") + } + } + var source strings.Builder + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + file, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+source.String()+"RET\n") + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"raw": {Name: "raw", Ret: Void}}}) + if err != nil { + t.Fatal("complete raw source failed translation", row, arch, err) + } + x86ResultInitializationObject(t, llc, triple, ir, "average-raw") + } + } + } + } + } +} + +func TestX86PackedAverageRawReservedAndPrefixBoundaries(t *testing.T) { + valid := []byte{0x62, 0xf1, 0x7d, 0x89, 0xe0, 0xd1} + for _, mutation := range []struct { + index int + value byte + }{{1, 0xf2}, {2, 0x7c}, {3, 0x99}, {3, 0xe9}, {3, 0x88}} { + code := append([]byte(nil), valid...) + code[mutation.index] = mutation.value + _, _, known, err := decodedX86PackedAverageInstruction(code, 64) + if known && err == nil { + t.Fatalf("reserved encoding acquired semantics: %x", code) + } + } + ins, _, known, err := decodedX86PackedAverageInstruction(valid, 32) + if !known || err != nil { + t.Fatal("valid physical mask rejected", err) + } + ins.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Base: AX}} + for _, bits := range []int{16, 32, 64} { + local := ins + local.x86AddressBits = bits + if _, _, err := x86PackedAverageProperties(local.Op, local, "386"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed address prefix retained a source shape", bits, err) + } + } +} diff --git a/amd64_packed_average_runtime_test.go b/amd64_packed_average_runtime_test.go new file mode 100644 index 000000000..cc21cd719 --- /dev/null +++ b/amd64_packed_average_runtime_test.go @@ -0,0 +1,302 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86AverageNumericCase struct { + op string + width, bits int + masking, destination int + memory, mmx bool +} + +func x86AverageNumericCases() []x86AverageNumericCase { + var cases []x86AverageNumericCase + for index, op := range x86AverageNames { + widths := []int{8, 16} + if strings.HasPrefix(op, "V") { + widths = []int{16, 32, 64} + } + for _, width := range widths { + for _, memory := range []bool{false, true} { + modes, destinations := 1, 1 + if strings.HasPrefix(op, "V") { + modes, destinations = 3, 3 + } + for mode := 0; mode < modes; mode++ { + for destination := 0; destination < destinations; destination++ { + cases = append(cases, x86AverageNumericCase{op, width, 8 << uint(index%2), mode, destination, memory, width == 8}) + } + } + } + } + } + return cases +} + +func (row x86AverageNumericCase) instruction() string { + bank := map[int]string{8: "M", 16: "X", 32: "Y", 64: "Z"}[row.width] + first := bank + "1" + if row.memory { + first = "(BX)" + } + if !strings.HasPrefix(row.op, "V") { + return row.op + " " + first + "," + bank + "0" + } + op := row.op + if row.masking == 2 { + op += ".Z" + } + args := first + "," + bank + "0" + if row.masking != 0 { + args += ",K1" + } + return fmt.Sprintf("%s %s,%s%d", op, args, bank, row.destination) +} + +func x86AverageNumericFixture(t *testing.T, cases []x86AverageNumericCase) (string, map[string]FuncSig, string) { + t.Helper() + var source, declarations, table strings.Builder + sigs := make(map[string]FuncSig) + for index, row := range cases { + name := fmt.Sprintf("average%d", index) + fmt.Fprintf(&source, `TEXT %s(SB),4,$0-40 +MOVQ out+0(FP),AX +MOVQ a+8(FP),BX +MOVQ b+16(FP),CX +MOVQ mask+24(FP),DX +VMOVUPS (CX),Z0 +VMOVUPS (CX),Z2 +KMOVQ DX,K1 +`, name) + if !row.memory || row.destination == 1 { + source.WriteString("VMOVUPS (BX),Z1\n") + } + source.WriteString("PUSHQ flags+32(FP)\nPOPFQ\n") + if row.mmx { + source.WriteString("VMOVUPS Z0,(AX)\nMOVQ (CX),M0\nMOVQ (BX),M1\n") + } + source.WriteString(row.instruction() + "\n") + if row.mmx { + source.WriteString("MOVQ M0,(AX)\nEMMS\n") + } else { + fmt.Fprintf(&source, "VMOVUPS Z%d,(AX)\n", row.destination) + } + source.WriteString("PUSHFQ\nPOPQ R10\nMOVQ R10,64(AX)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*,uint64_t,uint64_t);\n", name) + fmt.Fprintf(&table, "{%s,%d,%d,%d,%d,%d},\n", name, row.width, row.bits/8, + row.masking, row.destination, x86FloatAddSubBoolInt(!strings.HasPrefix(row.op, "V"))) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := declarations.String() + "static const struct {Kernel run;int width,bytes,masking,destination,legacy;} cases[]={\n" + table.String() + "};\n" + return source.String(), sigs, driver +} + +func x86AverageNumericProgram(t *testing.T, triple string, cases []x86AverageNumericCase) (string, string) { + t.Helper() + source, sigs, driver := x86AverageNumericFixture(t, cases) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{TargetTriple: triple, Goarch: "amd64", Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + return ir, driver +} + +// Byte arithmetic is independent of the lowering specification. W lanes use +// explicit little-endian assembly and a 17-bit sum, including 65535+65535. +const x86AverageIndependentC = `#include +#include +#include +#include +typedef void (*Kernel)(void *, const void *, const void *, uint64_t, uint64_t); +static unsigned lane(const uint8_t *p, int bytes) { + unsigned value = p[0]; + if (bytes == 2) { + value |= (unsigned)p[1] << 8; + } + return value; +} +static const uint64_t masks[] = {0, 1, UINT64_C(0xaaaaaaaa55555555), UINT64_MAX}; +` + +const x86AverageCheckC = `int main(void) { + for (unsigned row = 0; row < sizeof(cases) / sizeof(cases[0]); row++) { + for (int sample = 0; sample < 257; sample++) { + for (int status = 0; status < 2; status++) { + for (int m = 0; m < 4; m++) { + _Alignas(64) uint8_t a[192], b[192], out[192]; + memset(a, 0xa7, sizeof(a)); + memset(b, 0xb9, sizeof(b)); + memset(out, 0xc3, sizeof(out)); + + for (int i = 0; i < 64; i++) { + a[64 + i] = (uint8_t)(sample * 71 + i * 37); + b[64 + i] = (uint8_t)(sample * 19 - i * 13); + if (sample < 4) { + a[64 + i] = (uint8_t[]){255, 0, 128, 255}[sample]; + b[64 + i] = (uint8_t[]){255, 1, 128, 0}[sample]; + } + } + uint8_t beforeA[192], beforeB[192]; + memcpy(beforeA, a, sizeof(a)); + memcpy(beforeB, b, sizeof(b)); + + uint32_t saved = _mm_getcsr(), + profile = (uint32_t)(0x1f80 | ((sample % 4) << 13) | (sample % 64)); + _mm_setcsr(profile); + uint64_t seed = status ? UINT64_C(0x8d5) : 0; + cases[row].run(out + 64, a + 64, b + 64, masks[m], seed); + uint32_t after = _mm_getcsr(); + _mm_setcsr(saved); + uint64_t flags; + memcpy(&flags, out + 128, 8); + if (after != profile || (flags & UINT64_C(0x8d5)) != seed || + memcmp(a, beforeA, sizeof(a)) || memcmp(b, beforeB, sizeof(b))) { + fprintf(stderr, "row%u sample%d CSR/flags/input\n", row, sample); + return 1; + } + + for (int i = 0; i < 64; i++) { + int start = i / cases[row].bytes * cases[row].bytes; + unsigned value = (lane(a + 64 + start, cases[row].bytes) + + lane(b + 64 + start, cases[row].bytes) + 1) / + 2; + int active = + !cases[row].masking || ((masks[m] >> (i / cases[row].bytes)) & 1); + uint8_t old = cases[row].destination == 1 ? a[64 + i] : b[64 + i]; + uint8_t want = + i < cases[row].width + ? (active ? (uint8_t)(value >> (8 * (i % cases[row].bytes))) + : cases[row].masking == 2 ? 0 + : old) + : cases[row].legacy ? b[64 + i] + : 0; + if (out[64 + i] != want) { + fprintf(stderr, "row%u sample%d mask%llx byte%d got%x want%x\n", row, + sample, (unsigned long long)masks[m], i, out[64 + i], want); + return 2; + } + } + for (int i = 0; i < 192; i++) { + if ((i < 64 || i >= 136) && out[i] != 0xc3) { + return 3; + } + } + } + } + } + } + puts("complete average bytes/flags/MXCSR/input/output canaries"); + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedAverageIndependentBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + cases := x86AverageNumericCases() + ir, table := x86AverageNumericProgram(t, execution.triple, cases) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "average-independent", ir, + x86AverageIndependentC+table+x86AverageCheckC, level) + if !strings.Contains(string(output), "complete average bytes") { + t.Fatal("missing full numerical receipt", string(output)) + } + }) + } + t.Logf("%d programs × 257 patterns × 2 flags × 4 masks, full 64-byte views, O0/O2; translated wide shadow views, not original EVEX runtime", len(cases)) +} + +func TestCrossLinuxRuntimeMatrixX86PackedAverageInactiveMemory(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var cases []x86AverageNumericCase + for _, op := range x86AverageNames[2:] { + bits := 8 + if strings.HasSuffix(op, "W") { + bits = 16 + } + for _, width := range []int{16, 32, 64} { + for _, mode := range []int{1, 2} { + cases = append(cases, x86AverageNumericCase{op, width, bits, mode, 0, true, false}) + } + } + } + ir, table := x86AverageNumericProgram(t, execution.triple, cases) + // Unlike a post-load select, an E4.nb masked read cannot touch the guard + // page. With mask 1 only the first byte/word is accessible; mask 0 has no + // accessible source bytes. Full virtual FP spans are tested separately. + const main = x86TestGuardPagesC + ` +#ifndef _WIN32 +#include +#endif +int main(void) { +#ifndef _WIN32 + struct rlimit core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &core)) { + return 9; + } +#endif + if (setup_guard()) { + return 11; + } + + _Alignas(64) uint8_t b[64], out[80]; + memset(b, 0x91, 64); + for (unsigned row = 0; row < sizeof(cases) / sizeof(cases[0]); row++) { + for (int active = 0; active < 2; active++) { + memset(out, 0xc3, 80); + uint8_t *a = guard + page - (active ? cases[row].bytes : 0); + if (active) { + memset(a, 0xff, (size_t)cases[row].bytes); + } + cases[row].run(out + 1, a, b, (uint64_t)active, 0); + for (int i = 0; i < 64; i++) { + unsigned value = (cases[row].bytes == 1 ? 255 : 65535) + + (cases[row].bytes == 1 ? 0x91 : 0x9191) + 1; + value /= 2; + uint8_t want = i < cases[row].width + ? (active && i < cases[row].bytes + ? (uint8_t)(value >> (8 * (i % cases[row].bytes))) + : cases[row].masking == 2 ? 0 + : b[i]) + : 0; + if (out[i + 1] != want) { + return 12; + } + } + for (int i = 0; i < 80; i++) { + if ((i == 0 || i >= 73) && out[i] != 0xc3) { + return 13; + } + } + } + } + if (free_guard()) { + return 14; + } + puts("inactive guard-page reads suppressed"); + return 0; +} +` + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "average-inactive", ir, x86AverageIndependentC+table+main, level) + if !strings.Contains(string(output), "inactive guard-page reads suppressed") { + t.Fatal("missing masked memory receipt", string(output)) + } + }) + } +} diff --git a/amd64_packed_binary_effects.go b/amd64_packed_binary_effects.go new file mode 100644 index 000000000..663cd8193 --- /dev/null +++ b/amd64_packed_binary_effects.go @@ -0,0 +1,79 @@ +package plan9asm + +import "strings" + +// Validated legacy and non-masked VEX forms share source-width and dependency +// axes, not permission. The consumer selects its admitted generation first; +// legacy reads its old destination, VEX independently writes and clears it. +// A physical MMX source is supplied by the parser, never reinterpreted here. +func x86PackedBinaryDependencyEffect(ins Instr, form x86PackedBinaryForm, source Operand) x86TypedByteEffect { + width := int64(form.byteWidth) + sourceWidth := width + if form.memory >= 0 { + sourceWidth = int64(form.memoryBytes) + } + count := 2 + if !form.legacy { + count = 3 + } + effect := x86TypedByteEffect{operands: make([]x86TypedByteOperand, count)} + effect.operands[0] = x86TypedByteOperand{index: form.first, operand: source, width: sourceWidth, read: true} + effect.operands[1] = x86TypedByteOperand{index: form.second, operand: ins.Args[form.second], width: width, read: true, write: form.legacy} + if form.legacy && !form.mmx { + effect.vector = x86TypedVectorEffect{x86TypedVectorUpperPreserve, width} + } else if !form.legacy { + effect.vector = x86TypedVectorEffect{x86TypedVectorUpperZero, 1} + effect.operands[2] = x86TypedByteOperand{index: form.destination, operand: ins.Args[form.destination], width: width, write: true} + } + return effect +} + +type x86PackedBinaryEffectForm struct { + x86PackedBinaryForm + source Operand // Retains the actual MMX bank of a legacy GP spelling. + vex bool +} + +// Keep the existing family parsers as the only opcode/form authority. This +// grouping adds no VEX, mask, broadcast, initialized input or memory backing. +func x86PackedBinaryByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + if effect, known, err := x86TypedLegacyLogicalByteEffect(op, ins, arch); known { + return effect, true, err + } + if effect, known, err := x86TypedByteShuffleEffect(op, ins, arch); known { + return effect, true, err + } + if effect, known, err := x86TypedImmediateShuffleByteEffect(op, ins, arch); known { + return effect, true, err + } + if effect, known, err := x86TypedLegacyCLMULByteEffect(op, ins, arch); known { + return effect, true, err + } + if effect, known, err := x86TypedLegacyPackedShiftByteEffect(op, ins, arch); known { + return effect, true, err + } + form, known, err := x86PackedBinaryEffectProperties(op, ins, arch) + if !known || err != nil || !form.legacy { + return x86TypedByteEffect{}, known && (err != nil || form.legacy), err + } + return x86PackedBinaryDependencyEffect(ins, form.x86PackedBinaryForm, form.source), true, nil +} + +func x86PackedBinaryEffectProperties(op Op, ins Instr, arch string) (x86PackedBinaryEffectForm, bool, error) { + name := strings.ToUpper(string(op)) + if !strings.HasPrefix(name, "P") && !strings.HasPrefix(name, "VP") { + return x86PackedBinaryEffectForm{}, false, nil // Negative filter only; the specs below remain authoritative. + } + if form, known, err := x86LegacyIntegerArithmeticProperties(op, ins, arch); known { + return x86PackedBinaryEffectForm{form.x86PackedBinaryForm, form.source, false}, true, err + } + if form, known, err := x86PackedLowMultiplyProperties(op, ins, arch); known { + var source Operand + if err == nil { + source = ins.Args[form.first] + } + return x86PackedBinaryEffectForm{form.x86PackedBinaryForm, source, form.spec.encoding != x86PackedLogicalEVEX}, true, err + } + form, known, err := x86PackedPhysicalProperties(op, ins, arch, "") + return x86PackedBinaryEffectForm{form.x86PackedBinaryForm, form.source, true}, known, err +} diff --git a/amd64_packed_binary_physical.go b/amd64_packed_binary_physical.go new file mode 100644 index 000000000..4f3b75a8d --- /dev/null +++ b/amd64_packed_binary_physical.go @@ -0,0 +1,46 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// These families share physical operand normalization, not their numerical +// effects or source permissions. Only the existing MMX adapter copies operands; +// the binary grammar reads the original slice without changing it. The boolean +// preserves the families' zero-form return before binary parsing has begun. +func x86PackedBinaryPhysicalSource(op Op, ins Instr, arch string, spec x86PackedBinarySpec, + family string) (x86PackedBinaryForm, Operand, bool, error) { + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86PackedBinaryForm{}, Operand{}, false, + fmt.Errorf("%s dispatch/source opcode mismatch: %s/%s", family, op, ins.Op) + } + local := ins + local.Op = op + if spec.encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(local, arch) + if err != nil { + return x86PackedBinaryForm{}, Operand{}, false, err + } + } + grammarArch := arch + if arch == "386" && local.x86Encoded && spec.encoding == x86PackedLogicalVEXOrEVEX { + grammarArch = "amd64" // Only the complete physical decoder sets this marker. + } + binary, err := x86PackedBinaryProperties(local, grammarArch, spec) + var source Operand + if err == nil { + source = local.Args[binary.first] + if grammarArch != arch && binary.memory >= 0 && !x86PackedShiftMemoryValid(source, arch) { + err = fmt.Errorf("386 %s physical source has an unbound address", op) + } + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && binary.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: %s has no bound address-size override", ErrProbeNeedsContext, family) + } + } + return binary, source, true, err +} diff --git a/amd64_packed_binary_pipeline_bench_test.go b/amd64_packed_binary_pipeline_bench_test.go new file mode 100644 index 000000000..cfd17fd1a --- /dev/null +++ b/amd64_packed_binary_pipeline_bench_test.go @@ -0,0 +1,43 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +// Keep parsing outside the timer, but include source analysis, lowering and +// LLVM verification. These register-only kernels need no native x86 hardware. +// The independent family tests own memory faults and executable semantics. +func BenchmarkX86PackedBinaryTranslation(b *testing.B) { + for _, row := range []string{ + "PSUBL X0,X1", + "VPSUBD Y0,Y1,Y2", + "VPSUBSW.Z Z0,Z1,K1,Z2", + "PMULLD X0,X1", + "VPMULLD Y0,Y1,Y2", + "VPMULLQ.Z Z0,Z1,K1,Z2", + "PADDL X0,X1", + "VPADDD Y0,Y1,Y2", + "VPADDSW.Z Z0,Z1,K1,Z2", + } { + b.Run(row, func(b *testing.B) { + source := "TEXT binary(SB),4,$0-0\n" + + "VPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\n" + + "MOVQ $5,AX\nKMOVQ AX,K1\n" + + strings.Repeat(row+"\n", 32) + "RET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + b.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"binary": {Name: "binary", Ret: Void}}} + b.ReportAllocs() + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + if _, err := Translate(file, options); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/amd64_packed_binary_pipeline_runtime_test.go b/amd64_packed_binary_pipeline_runtime_test.go new file mode 100644 index 000000000..4821aae76 --- /dev/null +++ b/amd64_packed_binary_pipeline_runtime_test.go @@ -0,0 +1,202 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// This host-portable oracle covers the shared lowering axes, not physical x86 +// ISA/FLAGS or legacy memory alignment. The original native family owners keep +// those independent obligations. Expected values do not use production specs. +func TestX86PackedBinaryPipelinePortableSemantics(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang required") + } + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for index, row := range []struct { + instruction string + width, bytes, mode, mask, broadcast int + legacy, memory bool + }{ + {"PSUBL X0,X2", 16, 4, 0, 0, 0, true, false}, + {"PSUBSB X0,X2", 16, 1, 1, 0, 0, true, false}, + {"VPSUBW Y0,Y1,Y2", 32, 2, 0, 0, 0, false, false}, + {"VPSUBQ.BCST.Z (DI),Z1,K1,Z2", 64, 8, 0, 2, 1, false, true}, + {"VPSUBSW (DI),Z1,K1,Z2", 64, 2, 1, 1, 0, false, true}, + {"VPSUBUSB.Z (DI),X1,K1,X2", 16, 1, 2, 2, 0, false, true}, + {"PMULLD X0,X2", 16, 4, 3, 0, 0, true, false}, + {"VPMULLD Y0,Y1,Y2", 32, 4, 3, 0, 0, false, false}, + {"VPMULLD.BCST.Z (DI),X1,K1,X2", 16, 4, 3, 2, 1, false, true}, + {"VPMULLQ (DI),Z1,K1,Z2", 64, 8, 3, 1, 0, false, true}, + {"PADDL X0,X2", 16, 4, 4, 0, 0, true, false}, + {"VPADDQ Y0,Y1,Y2", 32, 8, 4, 0, 0, false, false}, + {"VPADDSW (DI),Z1,K1,Z2", 64, 2, 5, 1, 0, false, true}, + {"VPADDUSB.Z (DI),X1,K1,X2", 16, 1, 6, 2, 0, false, true}, + {"VPADDD.BCST.Z (DI),Z1,K1,Z2", 64, 4, 4, 2, 1, false, true}, + } { + name := fmt.Sprint("binary", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n"+ + "MOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\n"+ + "MOVQ mask+24(FP),AX\nKMOVQ AX,K1\n"+ + "VMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\n", name) + if !row.memory { + source.WriteString("VMOVUPS (DI),Z0\n") + } + fmt.Fprintf(&source, "%s\nVMOVUPS Z2,(DX)\nRET\n", row.instruction) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if (check(%s,%q,%d,%d,%d,%d,%d,%d,%d)) return 1;\n", + name, row.instruction, row.width, row.bytes, row.mode, row.mask, + row.broadcast, x86FloatAddSubBoolInt(row.legacy), x86FloatAddSubBoolInt(row.memory)) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + main := x86PackedBinaryPipelineReference + declarations.String() + + "int main(void) {\n" + checks.String() + "return 0;\n}\n" + host := testTargetTriple(runtime.GOOS, runtime.GOARCH) + targets := []string{host} + if host != "x86_64-unknown-linux-gnu" { + targets = append(targets, "x86_64-unknown-linux-gnu") + } + for _, triple := range targets { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if x86ScalarFloatIntegerNativeTarget("amd64", triple) || x86ScalarFloatIntegerNativeTarget("386", triple) { + // Verify every function and reject native effects before removing + // exactly its feature attribute. This numerical oracle must not + // require AVX512 merely because its source uses Z registers. + profiles := make(map[string]string, len(file.Funcs)) + for _, function := range file.Funcs { + profiles[function.Sym] = inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", function) + } + ir, err = x86PackedShiftPortableFeatureProjection(ir, profiles) + if err != nil { + t.Fatal(err) + } + } + x86ResultInitializationObject(t, llc, triple, ir, "packed-binary-portable") + if triple != host { + return // Cross object evidence, not cross execution. + } + for _, level := range []int{0, 2} { + t.Run(fmt.Sprint("O", level), func(t *testing.T) { + compileAndRunRuntimeTestWithCompiler(t, llc, []string{clang}, "packed-binary", + triple, ir, main, nil, level) + }) + } + }) + } +} + +const x86PackedBinaryPipelineReference = ` +#include +#include +#include + +static uint64_t lane(const uint8_t *p, int bytes) { + uint64_t value = 0; + for (int i = 0; i < bytes; ++i) { + value |= (uint64_t)p[i] << (8*i); + } + return value; +} + +static uint64_t calculate(uint64_t a, uint64_t b, int bytes, int mode) { + if (mode == 3) { + return a*b; + } + if (mode == 4) { + return a+b; + } + if (mode == 0) { + return b-a; + } + if (mode == 2) { + return b < a ? 0 : b-a; + } + if (mode == 6) { + uint64_t max = (UINT64_C(1) << (bytes*8))-1; + return a+b > max ? max : a+b; + } + + int64_t sign = INT64_C(1) << (bytes*8-1); + int64_t x = (a & sign) ? (int64_t)a - 2*sign : (int64_t)a; + int64_t y = (b & sign) ? (int64_t)b - 2*sign : (int64_t)b; + int64_t value = mode == 5 ? y+x : y-x; + if (value < -sign) { + return -sign; + } + if (value >= sign) { + return sign-1; + } + return value; +} + +static int check(void (*fn)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t), + const char *name, int width, int bytes, int mode, int masking, + int broadcast, int legacy, int memory) { + const uint64_t masks[] = {0, UINT64_MAX, 0x5555, UINT64_C(1)<<63}; + for (int sample = 0; sample < 32; ++sample) { + uint8_t a[64], b[64], out[66], oldA[64], oldB[64]; + for (int i = 0; i < 64; ++i) { + a[i] = (uint8_t)(sample*127 + i*53); + b[i] = (uint8_t)(sample*17 - i*91); + } + if (sample < 4) { + const uint8_t edgeA[] = {0xff, 0x80, 0x7f, 0x80}; + const uint8_t edgeB[] = {1, 0x7f, 0x7f, 0x80}; + memset(a, edgeA[sample], 64); + memset(b, edgeB[sample], 64); + } + memcpy(oldA, a, 64); + memcpy(oldB, b, 64); + + for (unsigned m = 0; m < sizeof(masks)/sizeof(masks[0]); ++m) { + uint64_t mask = masks[m], relevant = UINT64_MAX >> (64-width/bytes); + const uint8_t *input = memory && masking && !(mask & relevant) ? NULL : a; + memset(out, 0xa7, sizeof(out)); + fn(out+1, input, b, mask); + + for (int offset = 0; offset < 64; ++offset) { + uint8_t want = legacy ? b[offset] : 0; + if (offset < width) { + int active = !masking || ((mask >> (offset/bytes)) & 1); + if (active) { + uint64_t first = lane(a+(broadcast ? 0 : offset/bytes*bytes), bytes); + uint64_t second = lane(b+offset/bytes*bytes, bytes); + uint64_t result = calculate(first, second, bytes, mode); + want = (uint8_t)(result >> (8*(offset%bytes))); + } else { + want = masking == 2 ? 0 : b[offset]; + } + } + if (out[offset+1] != want) { + fprintf(stderr, "%s sample=%d mask=%llx byte=%d got=%x want=%x\n", + name, sample, (unsigned long long)mask, offset, out[offset+1], want); + return 1; + } + } + if (out[0] != 0xa7 || out[65] != 0xa7 || memcmp(a, oldA, 64) || memcmp(b, oldB, 64)) { + return 1; + } + } + } + return 0; +} +` diff --git a/amd64_packed_bit_count_forms.go b/amd64_packed_bit_count_forms.go new file mode 100644 index 000000000..691ad34ff --- /dev/null +++ b/amd64_packed_bit_count_forms.go @@ -0,0 +1,67 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedBitCountForm struct { + x86PackedBinaryForm + spec amd64PackedBitCountSpec +} + +// _yvexpandpd has the same EVEX source/width/mask axes as the shared binary +// grammar, without a second input. The local projection never rewrites source. +func x86PackedBitCountProperties(op Op, ins Instr, arch string) (x86PackedBitCountForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64PackedBitCountSpecs[Op(base)] + form := x86PackedBitCountForm{spec: spec} + if !known { + return form, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || len(ins.Args) != 2 && len(ins.Args) != 3 { + return form, true, fmt.Errorf("%s %s has an unbound dispatch or unary operand count: %q", arch, op, ins.Raw) + } + local := ins + local.Op = op + destination := ins.Args[len(ins.Args)-1] + local.Args = []Operand{ins.Args[0], destination, destination} + if len(ins.Args) == 3 { + local.Args = []Operand{ins.Args[0], destination, ins.Args[1], destination} + } + local, contextual := x86VectorValueMemoryInput(local, arch) + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, zeroing: true} + if spec.broadcast { + grammar.broadcastBytes = spec.laneBits / 8 + } + var err error + form.x86PackedBinaryForm, err = x86PackedBinaryOperandGrammar(local, arch, grammar, true) + if err != nil { + return form, true, err + } + form.second, form.destination = -1, len(ins.Args)-1 + if form.mask >= 0 { + form.mask = 1 + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && amd64VectorByteWidth(operand.Reg) != 0 && + !amd64GoPackedVectorMoveRegister(operand, form.byteWidth, arch) { + return form, true, fmt.Errorf("%s %s has an unbound vector bank: %q", arch, op, ins.Raw) + } + } + if form.memory >= 0 && !contextual && !x86PackedShiftMemoryValid(ins.Args[form.memory], arch) { + return form, true, fmt.Errorf("%s %s has an unbound memory operand: %q", arch, op, ins.Raw) + } + if contextual { + return form, true, x86VectorValueMemoryContext(ins) + } + return form, true, nil +} + +func x86ResultPackedBitCountStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, known, err := x86PackedBitCountProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + return x86ResultFixedVectorInputs(ins, form.x86PackedBinaryForm, false, state, owned, arch), true +} diff --git a/amd64_packed_compact_forms.go b/amd64_packed_compact_forms.go new file mode 100644 index 000000000..d3bcaaa61 --- /dev/null +++ b/amd64_packed_compact_forms.go @@ -0,0 +1,99 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Both complete Go tables have X/Y/Z register rows and a matching memory +// alternative. Compression writes its compact memory prefix; expansion reads +// one. Neither family admits broadcast/rounding, K0, or unmasked .Z. +type x86PackedCompactForm struct { + laneBits, byteWidth int + destination, mask, memory int + expand, zeroing bool +} + +// Go accepts EVEX.z on compressed memory destinations, but Intel XED requires +// ZEROING=0 for every such memory pattern. This is an architectural #UD, not +// a store that ignores zeroing or a numerical continuation with unknown inputs. +func (form x86PackedCompactForm) undefinedOpcode() bool { + return !form.expand && form.memory == form.destination && form.zeroing +} + +func x86PackedCompactProperties(op Op, ins Instr, arch string) (x86PackedCompactForm, bool, error) { + name, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + bits, known := amd64PackedCompressLaneBits[Op(name)] + form := x86PackedCompactForm{laneBits: bits, mask: -1, memory: -1} + if !known { + spec, matched := amd64PackedExpandSpecs[Op(name)] + if !matched { + return form, false, nil + } + form.laneBits, form.expand = spec.laneBits, true + } + fail := func(reason string) (x86PackedCompactForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go compact/expand grammar (%s): %q", arch, op, reason, ins.Raw) + } + if arch != "386" && arch != "amd64" || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail("mode or dispatch identity") + } + if dotted && suffix != "Z" { + return fail("only optional .Z is accepted") + } + form.zeroing = dotted + if len(ins.Args) != 2 && len(ins.Args) != 3 || form.zeroing && len(ins.Args) != 3 { + return fail("source, [K1-K7,] destination; .Z requires a mask") + } + form.destination = len(ins.Args) - 1 + if len(ins.Args) == 3 { + mask := ins.Args[1] + index, valid := amd64ParseKReg(mask.Reg) + if mask.Kind != OpReg || !valid || index == 0 { + return fail("masked forms require K1-K7") + } + form.mask = 1 + } + register := ins.Args[0] + if form.expand { + register = ins.Args[form.destination] + } + form.byteWidth = amd64VectorByteWidth(register.Reg) + if !amd64GoPackedVectorMoveRegister(register, form.byteWidth, arch) { + return fail("X/Y/Z register width or mode-bound bank") + } + other := form.destination + if form.expand { + other = 0 + } + operand := ins.Args[other] + if operand.Kind == OpReg { + if !amd64GoPackedVectorMoveRegister(operand, form.byteWidth, arch) { + return fail("register widths or banks do not match") + } + } else if isAMD64MemoryOperand(operand) { + form.memory = other + } else { + return fail("memory alternative is not a Go memory operand") + } + return form, true, nil +} + +func x86TypedCompactExpandByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedCompactProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + if form.undefinedOpcode() { + return x86TypedByteEffect{}, true, fmt.Errorf("%w: compact EVEX.z memory encoding has no ordinary byte continuation", ErrProbeNeedsContext) + } + operands := []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: int64(form.byteWidth), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), + read: form.mask >= 0 && !form.zeroing && form.memory != form.destination, write: true}, + } + if form.mask >= 0 { + operands = append(operands, x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return x86TypedByteEffect{operands: operands, dynamicMemory: form.memory >= 0, requiresResultProof: true}, true, nil +} diff --git a/amd64_packed_conversion_complete_inventory_test.go b/amd64_packed_conversion_complete_inventory_test.go new file mode 100644 index 000000000..ef7106b5f --- /dev/null +++ b/amd64_packed_conversion_complete_inventory_test.go @@ -0,0 +1,191 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86PackedConversionCompleteNativeSemanticInventory(t *testing.T) { + // Independent expected multiplicities: plain register/memory, alias/dead, + // merge/zero/broadcast, explicit rounding/SAE and physical low-bank EVEX. + want := map[string]int{ + "CVTPS2PL": 4, "CVTTPS2PL": 4, "CVTPD2PL": 4, "CVTTPD2PL": 4, + "VCVTPS2DQ": 28, "VCVTTPS2DQ": 25, + "VCVTPD2DQX": 8, "VCVTPD2DQY": 8, "VCVTPD2DQ": 12, + "VCVTTPD2DQX": 8, "VCVTTPD2DQY": 8, "VCVTTPD2DQ": 9, + "VCVTPD2QQ": 28, "VCVTPD2UQQ": 28, "VCVTTPD2QQ": 25, "VCVTTPD2UQQ": 25, + "VCVTPS2UDQ": 28, "VCVTTPS2UDQ": 25, "VCVTUDQ2PS": 28, + "VCVTQQ2PD": 28, "VCVTUQQ2PD": 28, + "VCVTPS2QQ": 28, "VCVTPS2UQQ": 28, "VCVTTPS2QQ": 25, "VCVTTPS2UQQ": 25, + "VCVTUDQ2PD": 24, + "VCVTPD2UDQX": 8, "VCVTPD2UDQY": 8, "VCVTPD2UDQ": 12, + "VCVTTPD2UDQX": 8, "VCVTTPD2UDQY": 8, "VCVTTPD2UDQ": 9, + "VCVTQQ2PSX": 8, "VCVTQQ2PSY": 8, "VCVTQQ2PS": 12, + "VCVTUQQ2PSX": 8, "VCVTUQQ2PSY": 8, "VCVTUQQ2PS": 12, + "CVTPL2PS": 4, "CVTPL2PD": 4, "VCVTDQ2PS": 46, "VCVTDQ2PD": 34, + } + seen := make(map[string]int) + physical := 0 + rows := x86PackedConversionCompleteRuntimeRows() + for _, row := range rows { + member := strings.Split(strings.Fields(row.instruction)[0], ".")[0] + seen[member]++ + if !row.wide || row.inputBits != 32 && row.inputBits != 64 { + t.Fatal("complete native fixture lost full alias storage or input bit type", row) + } + if row.rawOverride != "" { + physical++ + if member != "VCVTDQ2PS" && member != "VCVTDQ2PD" { + t.Fatal("unowned physical-generation override", row) + } + } + } + if len(rows) != 692 || len(seen) != 42 || physical != 2 { + t.Fatal("incomplete42 native semantic inventory", len(rows), len(seen), physical) + } + for member, count := range want { + if seen[member] != count { + t.Error("changed native member semantic axes", member, seen[member], count) + } + } + for member := range seen { + if want[member] == 0 { + t.Error("unexpected native member", member) + } + } + assembly, _, goMain, cMain := x86PackedConversionRuntimeFixture(rows[len(rows)-2:]) + for _, required := range []string{"VMOVDQU64 (SI),Z0", "VMOVDQU64 (DI),Z1", "STMXCSR 72(DI)", "SETPC 68(DI)", "LDMXCSR saved-4(SP)"} { + if !strings.Contains(assembly, required) { + t.Fatal("native value oracle lost full inputs, aliases, status or MXCSR restoration", required) + } + } + if !strings.Contains(goMain, "unsafe.Add(address,(-uintptr(address))&15)") || + !strings.Contains(cMain, "_Alignas(16) uint64_t input[8],out[12]") { + t.Fatal("native legacy m128 values rely on accidental input alignment") + } +} + +func TestX86PackedConversionCompleteNativeFaultInventory(t *testing.T) { + rows := x86PackedConversionCompleteFaultRows() + if len(rows) != 222 { + t.Fatal("complete42 memory-fault denominator changed", len(rows)) + } + members := make(map[string]bool) + signed := make(map[string]int) + for _, row := range rows { + member := strings.Split(row.name, ".")[0] + members[member] = true + if row.lanes < 2 || row.lanes > 16 || row.laneBytes != 4 && row.laneBytes != 8 { + t.Fatal("fault mask domain lost active output lanes", row) + } + if row.name == "CVTPL2PD" && (row.width != 8 || row.aligned || row.precision || row.floating) { + t.Fatal("exact signed32-to-double m64 inherited m128/precision semantics", row) + } + if row.name == "CVTPL2PS" && (row.width != 16 || !row.aligned || !row.precision || row.floating) { + t.Fatal("signed32-to-single m128 lost alignment/precision", row) + } + if member == "VCVTDQ2PS" || member == "VCVTDQ2PD" || strings.HasPrefix(member, "CVTPL") { + signed[fmt.Sprintf("%s/%d/%s", member, row.lanes, strings.TrimPrefix(row.name, member))]++ + } + } + if len(members) != 42 || len(signed) != 20 { + t.Fatal("missing complete42 fault members or signed semantic shapes", len(members), len(signed)) + } + for _, op := range []string{"VCVTDQ2PS", "VCVTDQ2PD"} { + lanes := []int{4, 8, 16} + if op == "VCVTDQ2PD" { + lanes = []int{2, 4, 8} + } + for _, count := range lanes { + for _, suffix := range []string{"", ".Z", ".BCST.Z"} { + if signed[fmt.Sprintf("%s/%d/%s", op, count, suffix)] != 1 { + t.Error("missing or duplicated native signed fault shape", op, count, suffix) + } + } + } + } + for _, scope := range []struct{ count, limit int }{{692, 64}, {222, 16}} { + seen := make([]int, scope.count) + for owner := 0; owner < 8; owner++ { + owned := 0 + for batch, start := 0, 0; start < scope.count; batch, start = batch+1, start+scope.limit { + if batch%8 != owner { + continue + } + for index := start; index < start+scope.limit && index < scope.count; index++ { + seen[index]++ + owned++ + } + } + if owned == 0 || owned > 2*scope.limit { + t.Fatal("empty or monolithic native scheduling owner", scope, owner, owned) + } + } + for index, count := range seen { + if count != 1 { + t.Fatal("native scheduling changed complete denominator", scope, index, count) + } + } + } +} + +func TestX86PackedConversionSignedNativeRuntimeFixtureObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + rows := x86PackedConversionCompleteRuntimeRows() + if len(rows) != 692 { + t.Fatal("complete native runtime fixture inventory changed") + } + // The604 older kernels retain their own whole fixture objects. These88 + // new signed/alias/dead/physical kernels compile the exact runtime wrapper, + // not a reduced standalone instruction or a native execution certificate. + for start := 604; start < len(rows); start += 64 { + end := start + 64 + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + batch := append([]x86PackedConversionRuntimeRow(nil), rows[start:end]...) + var instructions []string + for _, row := range batch { + instructions = append(instructions, row.instruction) + } + raw := x86PackedConversionContinuationRawRows(t, "amd64", instructions) + for _, spelling := range []string{"named", "raw"} { + t.Run(spelling, func(t *testing.T) { + if spelling == "raw" { + for index := range batch { + batch[index].instruction = raw[index] + if batch[index].rawOverride != "" { + batch[index].instruction = batch[index].rawOverride + } + } + } + source, declarations, _, _ := x86PackedConversionRuntimeFixture(batch) + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal("new native runtime fixture lost initialized public path", err) + } + optimized := x86DwordProductIROptimize(t, ir) + for _, pipeline := range []struct{ name, ir, backend string }{ + {"backend_O0", ir, "-O0"}, {"backend_O2", ir, "-O2"}, {"IR_O2_backend_O2", optimized, "-O2"}, + } { + t.Run(pipeline.name, func(t *testing.T) { + x86PackedStringCompileObject(t, llc, "x86_64-unknown-linux-gnu", pipeline.ir, pipeline.backend) + }) + } + }) + } + }) + } + }) + } +} diff --git a/amd64_packed_conversion_complete_runtime_policy_test.go b/amd64_packed_conversion_complete_runtime_policy_test.go new file mode 100644 index 000000000..f563698c1 --- /dev/null +++ b/amd64_packed_conversion_complete_runtime_policy_test.go @@ -0,0 +1,199 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "strconv" + "strings" + "testing" +) + +func x86PackedConversionRuntimeFunctions(t *testing.T, path string) map[string]*ast.FuncDecl { + t.Helper() + syntax, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + t.Fatal("missing required complete native conversion owner or driver", path, err) + } + functions := make(map[string]*ast.FuncDecl) + for _, declaration := range syntax.Decls { + if function, ok := declaration.(*ast.FuncDecl); ok { + if functions[function.Name.Name] != nil { + t.Fatal("duplicate complete native conversion function", function.Name.Name) + } + functions[function.Name.Name] = function + } + } + return functions +} + +func x86PackedConversionOriginalCProgramUnchanged(t *testing.T) { + t.Helper() + functions := x86PackedConversionRuntimeFunctions(t, "amd64_lower_packed_numeric_conversion_test.go") + function := functions["amd64PackedNumericConversionNativeRuntimeSemantics"] + if function == nil || function.Body == nil { + t.Fatal("missing original independent native C program") + } + want := map[string]string{ + "source": "35a753aaa54d59683eed39625574d35e24d342cc53a91ceceb998c17a36d7175", + "mainC": "08481b19c5e80e7b2da8c77268fa37e486991839b06c9cd37efc22c6e727d43b", + } + ast.Inspect(function.Body, func(node ast.Node) bool { + spec, ok := node.(*ast.ValueSpec) + if !ok || len(spec.Names) != 1 || len(spec.Values) != 1 { + return true + } + expected, found := want[spec.Names[0].Name] + if !found { + return true + } + literal, ok := spec.Values[0].(*ast.BasicLit) + if !ok || literal.Kind != token.STRING { + t.Fatal("original independent source/reference was replaced", spec.Names[0].Name) + } + value, err := strconv.Unquote(literal.Value) + if err != nil || fmt.Sprintf("%x", sha256.Sum256([]byte(value))) != expected { + t.Fatal("original complete assembly or independent C reference changed", spec.Names[0].Name, err) + } + delete(want, spec.Names[0].Name) + return true + }) + if len(want) != 0 { + t.Fatal("original source/reference literal disappeared", want) + } +} + +func TestX86PackedConversionCompleteNativeRuntimeOwnership(t *testing.T) { + x86PackedConversionOriginalCProgramUnchanged(t) + const path = "amd64_packed_conversion_runtime_linux_amd64_test.go" + for _, qualified := range []bool{false, true} { + for _, target := range []struct{ os, arch string }{ + {"linux", "amd64"}, {"linux", "386"}, {"linux", "arm"}, {"linux", "arm64"}, + {"js", "wasm"}, {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, + } { + t.Run(fmt.Sprintf("selection/qualified=%t/%s-%s", qualified, target.os, target.arch), func(t *testing.T) { + context := ciNativeAVX512BuildContext(target.os, target.arch, qualified) + selected, err := context.MatchFile(".", path) + want := qualified && target.os == "linux" && target.arch == "amd64" + if err != nil || selected != want { + t.Fatal("conversion owner lost its explicit native selection", target, qualified, selected, err) + } + }) + } + } + functions := x86PackedConversionRuntimeFunctions(t, path) + want := make(map[string]bool) + for _, family := range []struct{ root, driver string }{ + {"TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512Status", "x86PackedConversionCompleteNativeValues"}, + {"TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaults", "x86PackedConversionCompleteNativeFaults"}, + {"TestCrossLinuxRuntimeMatrixX86PackedConversionNativeIndependentC", "amd64PackedNumericConversionNativeRuntimeSemantics"}, + } { + parts := 8 + if family.driver == "amd64PackedNumericConversionNativeRuntimeSemantics" { + parts = 1 + } + for part := 0; part < parts; part++ { + name := family.root + if part != 0 { + name += fmt.Sprintf("Part%d", part) + } + want[name] = true + t.Run(name, func(t *testing.T) { + function := functions[name] + if function == nil || function.Body == nil || len(function.Body.List) != 1 { + t.Fatal("missing or bypassed complete native conversion partition", name) + } + statement, ok := function.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("required native partition does not execute its real driver", name) + } + call, ok := statement.X.(*ast.CallExpr) + if !ok { + t.Fatal("native partition lost its complete direct call", name) + } + callee, ok := call.Fun.(*ast.Ident) + arguments := 2 + if parts == 1 { + arguments = 1 + } + if !ok || callee.Name != family.driver || len(call.Args) != arguments { + t.Fatal("native partition did not consume the complete 42-member driver", name) + } + argument, ok := call.Args[0].(*ast.Ident) + if !ok || argument.Name != "t" { + t.Fatal("native partition lost its actual test context", name) + } + if parts != 1 { + literal, ok := call.Args[1].(*ast.BasicLit) + if !ok || literal.Kind != token.INT || literal.Value != strconv.Itoa(part) { + t.Fatal("native partition borrowed another slice or a sampled subset", name) + } + } + }) + } + } + if len(want) != 17 { + t.Fatal("missing complete status/fault/native-C owner inventory") + } + for name := range functions { + if strings.HasPrefix(name, "Test") && !want[name] { + t.Fatal("unregistered native conversion partition", name) + } + } +} + +func TestX86PackedConversionCompleteNativeCPUDriverRequired(t *testing.T) { + const path = "amd64_packed_conversion_native_tools_test.go" + functions := x86PackedConversionRuntimeFunctions(t, path) + function := functions["x86PackedConversionNativeTools"] + if function == nil || function.Body == nil { + t.Fatal("complete conversions require a real F/DQ/BW/VL and native-host owner") + } + calls := make(map[string]int) + ast.Inspect(function.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if callee, ok := call.Fun.(*ast.Ident); ok { + calls[callee.Name]++ + if callee.Name == "x86FloatCompareRuntimeTools" { + if len(call.Args) != 2 { + t.Fatal("native conversion owner lost the real EVEX execution route") + } + axis, ok := call.Args[1].(*ast.Ident) + if !ok || axis.Name != "x86FloatCompareEVEX" { + t.Fatal("native conversion owner substituted an SSE/emulator route") + } + } + } + if callee, ok := call.Fun.(*ast.SelectorExpr); ok { + calls[callee.Sel.Name]++ + if callee.Sel.Name == "Skip" || callee.Sel.Name == "Skipf" || callee.Sel.Name == "SkipNow" { + t.Fatal("missing native hardware cannot become a successful skipped conversion owner") + } + } + return true + }) + for _, required := range []string{"x86PackedConversionRequireNativeHost", "x86FloatCompareRuntimeTools", "compileC", "run"} { + if calls[required] != 1 { + t.Fatal("native conversion owner omitted an actual host/CPU/opcode execution check", required, calls[required]) + } + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, required := range []string{ + "AVX512F=%u AVX512DQ=%u AVX512BW=%u AVX512VL=%u OSXSAVE=%u XCR0=%llx", + "PACKED_CONVERSION_NATIVE_F_DQ_BW_VL_OK", "__get_cpuid_count(7,0", "xgetbv", + "vcvtqq2pd", "vcvtdq2ps", "kmovq", "0xe6", + } { + if !strings.Contains(string(data), required) { + t.Fatal("complete native CPU/opcode preflight missing", required) + } + } +} diff --git a/amd64_packed_conversion_complete_runtime_test.go b/amd64_packed_conversion_complete_runtime_test.go new file mode 100644 index 000000000..9a813ed6e --- /dev/null +++ b/amd64_packed_conversion_complete_runtime_test.go @@ -0,0 +1,192 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +const ( + x86PackedConversionNativeOwners = 8 + x86PackedConversionValueCapture = 64 + x86PackedConversionFaultCapture = 16 +) + +// The full994-row object matrix owns spelling/address axes. Native execution +// keeps every independent semantic shape, rather than duplicating that syntax +// Cartesian product:38 existing families plus signed4, aliases, dead results, +// all signed masks/broadcasts/rounding and the two physical low-bank EVEX forms. +func x86PackedConversionCompleteRuntimeRows() []x86PackedConversionRuntimeRow { + rows := x86PackedConversionRuntimeRows(true) + for _, row := range x86SignedPackedValueRows() { + input := row.input + if strings.Contains(input, "(") { + input = "(SI)" + } + rows = append(rows, x86PackedConversionRuntimeRow{ + instruction: row.op + " " + input + "," + row.mask + row.bank + "1", + wide: true, inputBits: 32, mask: row.mask != "", sae: strings.Contains(row.op, "SAE"), + }) + } + for _, op := range []string{"CVTPL2PS", "CVTPL2PD", "VCVTDQ2PS", "VCVTDQ2PD"} { + banks := []string{"X"} + if strings.HasPrefix(op, "V") { + banks = []string{"X", "Y", "Z"} + } + for _, bank := range banks { + input := bank + if op == "VCVTDQ2PD" { + input = "X" + if bank == "Z" { + input = "Y" + } + } + rows = append(rows, + x86PackedConversionRuntimeRow{instruction: op + " " + input + "1," + bank + "1", wide: true, inputBits: 32}, + x86PackedConversionRuntimeRow{instruction: op + " " + input + "0," + bank + "1", wide: true, dead: true, inputBits: 32}) + } + } + for _, physical := range []struct { + instruction string + code []byte + }{ + {"VCVTDQ2PS X0,X1", []byte{0x62, 0xf1, 0x7c, 0x08, 0x5b, 0xc8}}, + {"VCVTDQ2PD X0,X1", []byte{0x62, 0xf1, 0x7e, 0x08, 0xe6, 0xc8}}, + } { + rows = append(rows, x86PackedConversionRuntimeRow{ + instruction: physical.instruction, rawOverride: strings.TrimSuffix(x86RawDisplacementDirectives(physical.code), "\n"), + wide: true, inputBits: 32, + }) + } + return rows +} + +func x86PackedConversionCompleteFaultRows() []x86PackedConversionFaultRow { + rows := x86PackedConversionFaultRows(true) + for _, member := range []struct { + name string + width, lanes int + legacy, precision bool + }{ + {"CVTPL2PS", 16, 4, true, true}, + {"CVTPL2PD", 8, 2, true, false}, + {"VCVTDQ2PS", 16, 4, false, true}, + {"VCVTDQ2PS", 32, 8, false, true}, + {"VCVTDQ2PS", 64, 16, false, true}, + {"VCVTDQ2PD", 8, 2, false, false}, + {"VCVTDQ2PD", 16, 4, false, false}, + {"VCVTDQ2PD", 32, 8, false, false}, + } { + outputBytes := member.lanes * 4 + if strings.HasSuffix(member.name, "PD") { + outputBytes *= 2 + } + destination := map[int]string{16: "X1", 32: "Y1", 64: "Z1"}[outputBytes] + row := x86PackedConversionFaultRow{ + name: member.name, width: member.width, laneBytes: 4, lanes: member.lanes, + precision: member.precision, + // SSE CVTDQ2PD reads only m64 and has no m128 alignment + // requirement. It must not inherit the other legacy forms'. + aligned: member.legacy && member.width == 16, + conversion: member.name + " (DI)," + destination, + } + rows = append(rows, row) + if !member.legacy { + row.name, row.masked = member.name+".Z", true + row.conversion = row.name + " (DI),K1," + destination + rows = append(rows, row) + row.name, row.width = member.name+".BCST.Z", 4 + row.conversion = row.name + " (DI),K1," + destination + rows = append(rows, row) + } + } + return rows +} + +func x86PackedConversionCompleteNativeFaults(t *testing.T, owner int) { + t.Helper() + if owner < 0 || owner >= x86PackedConversionNativeOwners { + t.Fatal("invalid native conversion fault owner", owner) + } + execution := x86PackedConversionNativeTools(t) + rows := x86PackedConversionCompleteFaultRows() + if len(rows) != 222 { + t.Fatal("complete42 fault denominator changed", len(rows)) + } + x86PackedConversionRunFaultRows(t, execution, rows, x86PackedConversionFaultCapture, owner, x86PackedConversionNativeOwners, true, true) +} + +func x86PackedConversionCompleteNativeValues(t *testing.T, owner int) { + t.Helper() + if owner < 0 || owner >= x86PackedConversionNativeOwners { + t.Fatal("invalid native conversion owner", owner) + } + execution := x86PackedConversionNativeTools(t) + rows := x86PackedConversionCompleteRuntimeRows() + if len(rows) != 692 { + t.Fatal("complete42 native semantic denominator changed", len(rows)) + } + consumed := 0 + for batch, start := 0, 0; start < len(rows); batch, start = batch+1, start+x86PackedConversionValueCapture { + if batch%x86PackedConversionNativeOwners != owner { + continue + } + end := start + x86PackedConversionValueCapture + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + current := rows[start:end] + assembly, declarations, main, driver := x86PackedConversionRuntimeFixture(current) + var want []byte + t.Run("original_go_named", func(t *testing.T) { + want = x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, main) + if strings.Count(string(want), "\n") != len(current) { + t.Fatal("incomplete original Go named runtime receipts") + } + }) + var instructions []string + for _, row := range current { + instructions = append(instructions, row.instruction) + } + raw := x86PackedConversionContinuationRawRows(t, "amd64", instructions) + rawRows := append([]x86PackedConversionRuntimeRow(nil), current...) + for index := range rawRows { + rawRows[index].instruction = raw[index] + if current[index].rawOverride != "" { + rawRows[index].instruction = current[index].rawOverride + } + } + rawAssembly, _, rawMain, _ := x86PackedConversionRuntimeFixture(rawRows) + t.Run("original_go_raw", func(t *testing.T) { + got := x86ScalarFloatIntegerOriginalOutput(t, execution, rawAssembly, rawMain) + if string(got) != string(want) || len(want) == 0 { + t.Fatal("original named/raw Go runtime diverged or lost receipts") + } + }) + if len(want) == 0 { + t.Fatal("missing original native Go execution") + } + for _, spelling := range []struct{ name, source string }{{"named", assembly}, {"raw", rawAssembly}} { + t.Run(spelling.name, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, spelling.source, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal("initialized complete native conversion source rejected", err) + } + x86PackedConversionNativePipelines(t, execution, ir, driver, want) + }) + } + }) + } + consumed += len(current) + }) + } + if consumed == 0 { + t.Fatal("empty native value owner") + } + t.Logf("%d/692 native kernels: actual named/raw Go,3APIs,3LLVM paths; full MXCSR, masks, aliases and dead results", consumed) +} diff --git a/amd64_packed_conversion_effects.go b/amd64_packed_conversion_effects.go new file mode 100644 index 000000000..0b7603bf5 --- /dev/null +++ b/amd64_packed_conversion_effects.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func x86TypedPackedConversionByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86PackedConversionSpecification(Op(base)); !known { + return x86TypedByteEffect{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(op), string(ins.Op)) { + return x86TypedByteEffect{}, true, fmt.Errorf("packed conversion effect/source mismatch") + } + ins.Op = op + f, known, err := x86PackedConversionProperties(ins, arch) + if err != nil { + return x86TypedByteEffect{}, known, err + } + width := f.inputBytes + if f.memory >= 0 { + width = f.memoryBytes + } + effect := x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: int64(width), read: true}, + {index: f.destination, operand: ins.Args[f.destination], width: int64(f.shape.destinationBytes), write: true}, + }} + if f.mask >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{index: f.mask, operand: ins.Args[f.mask], width: 8, read: true}) + if !f.properties.zeroing { + effect.operands = append(effect.operands, x86TypedByteOperand{index: f.destination, operand: ins.Args[f.destination], + width: int64(f.shape.lanes * f.spec.destinationBits / 8), read: true}) + } + } + return effect, true, nil // Integer FLAGS do not change; MXCSR remains native. +} + +func x86ResultPackedConversionStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + return x86ResultPackedConversionStepWithRead(ins, state, owned, arch, false) +} + +// The native conversion emitter retains the complete opcode's access and +// MXCSR effects. This continuation grants no caller or pointee facts. +func x86ResultPackedConversionStepWithRead(ins Instr, state *x86ResultState, owned []bool, + arch string, nativeRead bool) (bool, bool) { + f, known, err := x86PackedConversionProperties(ins, arch) + if !known { + return false, false + } + if err != nil || f.mmx || !x86ResultNamedPredicateInstruction(ins) { + return false, true + } + return x86ResultPackedConversionInputs(ins, f, state, owned, arch, nativeRead), true +} + +// The caller selects and authenticates its own family before sharing these +// lane/input dependencies. No source or native-read capability follows. +func x86ResultPackedConversionInputs(ins Instr, f x86PackedConversionForm, state *x86ResultState, owned []bool, + arch string, nativeRead bool) bool { + if f.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[f.mask].Reg) + if !valid || !state.masks[index] { + return false + } + } + if f.memory >= 0 { + if !x86ResultReadWithNativeContinuation(ins, ins.Args[0], int64(f.memoryBytes), state, owned, arch, nativeRead) { + return false + } + } else { + index, _ := amd64VectorRegisterIndex(ins.Args[0].Reg, f.shape.sourceBytes) + input := x86CallerVectorByteMask(int64(f.inputBytes)) + if state.vectorBytes(index)&input != input { + return false + } + } + index, _ := amd64VectorRegisterIndex(ins.Args[f.destination].Reg, f.shape.destinationBytes) + active := x86CallerVectorByteMask(int64(f.shape.lanes * f.spec.destinationBits / 8)) + if f.mask >= 0 && !f.properties.zeroing && state.vectorBytes(index)&active != active { + return false + } + state.writeVector(index, f.shape.destinationBytes, x86CallerVectorByteMask(int64(f.shape.destinationBytes)), f.spec.legacy) + return true +} + +func x86CallerPackedConversionStep(ins *Instr, f x86PackedConversionForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if f.mmx || f.mask >= 0 && !state.masks[ins.Args[f.mask].Reg] { + return false + } + if f.memory >= 0 { + source := ins.Args[0] + if source.Kind == OpFP || source.Kind == OpMem && + (source.Mem.Base == SP || source.Mem.Base == BP || source.Mem.Index == BP) { + return false + } + if !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.memoryBytes), false) { + return false + } + } else { + input := x86CallerVectorByteMask(int64(f.inputBytes)) + if state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)]&input != input { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + active := x86CallerVectorByteMask(int64(f.shape.lanes * f.spec.destinationBits / 8)) + if f.mask >= 0 && !f.properties.zeroing && state.vectors[destination]&active != active { + return false + } + if f.spec.legacy { + state.vectors[destination] |= x86CallerVectorByteMask(int64(f.shape.destinationBytes)) + } else { + state.vectors[destination] = ^uint64(0) + } + return true +} diff --git a/amd64_packed_conversion_effects_test.go b/amd64_packed_conversion_effects_test.go new file mode 100644 index 000000000..0ef051b7a --- /dev/null +++ b/amd64_packed_conversion_effects_test.go @@ -0,0 +1,240 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86PackedConversionIndependentEveryByteEffects(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, spec := range x86PackedConversionOracleSpecs() { + for _, shape := range spec.shapes() { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + for _, masking := range []string{"", "merge", "zero"} { + if spec.legacy && masking != "" { + continue + } + op, mask := spec.name, "" + if masking != "" { + mask = "K1," + } + if masking == "zero" { + op += ".Z" + } + ins := x86ScalarLaneInstruction(t, op+" "+bank[shape.source]+"0,"+mask+bank[shape.destination]+"1") + f, known, err := x86PackedConversionProperties(ins, arch) + if err != nil || !known || f.inputBytes != shape.lanes*spec.input/8 || f.shape.destinationBytes != shape.destination { + t.Fatal(ins.Raw, f, err) + } + for missing := -1; missing < 64; missing++ { + for _, missingDestination := range []bool{false, true} { + state := x86ResultState{flags: x86ByteFlagsAll} + state.masks[1] = true + state.writeVector(0, 64, ^uint64(0), false) + state.writeVector(1, 64, ^uint64(0), false) + index := 0 + if missingDestination { + index = 1 + } + if missing >= 0 { + state.writeVector(index, 64, ^(uint64(1) << uint(missing)), false) + } + want := missing < 0 || !missingDestination && missing >= shape.lanes*spec.input/8 || + missingDestination && (masking != "merge" || missing >= shape.lanes*spec.output/8) + accepted, recognized := x86ResultPackedConversionStep(ins, &state, nil, arch) + if !recognized || accepted != want { + t.Fatal("required byte mismatch", ins.Raw, arch, missing, missingDestination, accepted, want) + } + if accepted { + expected := ^uint64(0) + if spec.legacy && missingDestination && missing >= 16 { + expected &^= uint64(1) << uint(missing) + } + if state.vectorBytes(1) != expected || state.flags != x86ByteFlagsAll { + t.Fatal("upper alias or integer FLAGS changed", ins.Raw, state.vectorBytes(1), expected) + } + } + } + } + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if err != nil || !known || effect.flagsRead != 0 || effect.flagsKilled != 0 || effect.flagsDefined != 0 { + t.Fatal("typed family effect/FLAGS disagreement", ins.Raw, effect, err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0)}, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedConversionStep(&ins, f, &state, nil) { + t.Fatal("initialized register grammar missing caller effect", ins.Raw) + } + } + } + } + } +} + +func TestX86PackedConversionResultAndStorageBoundaries(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + shape := spec.shapes()[0] + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + for _, initialized := range []bool{false, true} { + setup := "" + if initialized { + setup = "XORPS X0,X0\n" + if !spec.legacy { + setup = "VPXORD Z0,Z0,Z0\n" + } + } + assembly := fmt.Sprintf("TEXT ·f(SB),4,$0-4\n%s%s %s0,%s1\nPEXTRD $0,X1,AX\nMOVL AX,ret+0(FP)\nRET\n", setup, spec.name, bank[shape.source], bank[shape.destination]) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, assembly, "func f() uint32", "amd64", "x86_64-unknown-linux-gnu", api) + if !initialized { + x86VectorScratchCodegenControl(t, assembly, "func f() uint32", "amd64", "x86_64-unknown-linux-gnu", ir, err) + continue + } + if err != nil { + t.Fatal("uninitialized conversion result boundary", spec.name, initialized, api, err) + } + } + } + for _, operand := range []string{"data+0(FP)", "local-16(SP)", "(SP)", "(BP)"} { + assembly := "TEXT ·f(SB),4,$64-64\n" + spec.name + " " + operand + "," + bank[shape.destination] + "1\nRET\n" + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": { + Name: "f", Ret: Void, Args: []LLVMType{I64, I64, I64, I64, I64, I64, I64, I64}, + }}}) + if err == nil { + t.Fatal("logical slots acquired packed native backing", spec.name, operand) + } + } + } +} + +func TestX86PackedConversionPhysicalBroadcastTuple(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + if spec.legacy { + continue + } + for _, shape := range spec.shapes() { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[shape.destination] + row := spec.name + ".BCST (AX)," + bank + "1" + x86UnboundVoidScalarFormatObject(t, "TEXT ·f(SB),4,$0-0\n"+row+"\nRET\n", "f", "amd64", "x86_64-unknown-linux-gnu") + initialized := "TEXT ·f(SB),4,$0-8\nMOVQ p+0(FP),AX\n" + row + "\nRET\n" + ir, err := x86ScalarFPBitViewAPI(t, initialized, "func f(p *byte)", "amd64", "x86_64-unknown-linux-gnu", "text") + if err != nil { + t.Fatal(row, err) + } + if !strings.Contains(ir, fmt.Sprintf("{1to%d}", shape.lanes)) || !strings.Contains(ir, fmt.Sprintf("elementtype([%d x i8])", spec.input/8)) { + t.Fatal("physical broadcast confused carrier with active lanes", row, ir) + } + } + } +} + +func TestX86PackedConversionMalformedTypedGrammar(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + shape := spec.shapes()[0] + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + source, destination := bank[shape.source]+"0", bank[shape.destination]+"1" + invalid := []string{ + spec.name + " " + source, + spec.name + " $1," + destination, + spec.name + " " + source + ",(AX)", + spec.name + " " + source + ",K0," + destination, + spec.name + " " + source + ",K8," + destination, + spec.name + ".Z " + source + "," + destination, + spec.name + ".BCST " + source + "," + destination, + spec.name + ".RN_SAE (AX)," + destination, + spec.name + ".SAE (AX)," + destination, + spec.name + ".RN_SAE.SAE " + source + "," + destination, + spec.name + " " + source + ",Y32", + } + if spec.legacy { + invalid = append(invalid, spec.name+".BCST (AX),"+destination, + spec.name+" "+source+",K1,"+destination) + } + for _, row := range invalid { + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86PackedConversionProperties(ins, "amd64"); !known || err == nil { + t.Fatal("malformed complete-family form accepted", row, err) + } + } + for _, position := range []int{0, 1} { + ins := x86ScalarLaneInstruction(t, spec.name+" "+source+","+destination) + ins.Args[position].Reg = Reg(bank[amd64VectorByteWidth(ins.Args[position].Reg)] + "8") + if _, known, err := x86PackedConversionProperties(ins, "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("mode32 invented physical vector register", ins, err) + } + } + } + context := &amd64Ctx{goarch: "amd64"} + if known, _, err := context.lowerPackedNumericConversion("UNKNOWN", Instr{Op: "OTHER"}); known || err != nil { + t.Fatal("conversion lowerer claimed another instruction family", known, err) + } + valid := x86ScalarLaneInstruction(t, "CVTPS2PL X0,X1") + if known, _, err := context.lowerPackedNumericConversion("OTHER", valid); known || err != nil { + t.Fatal("unknown dispatch claimed a packed source spelling", known, err) + } + if known, _, err := context.lowerPackedNumericConversion("CVTPS2PL", Instr{Op: "OTHER"}); !known || err == nil { + t.Fatal("known dispatch lost its mismatched source identity", known, err) + } +} + +func TestX86PackedConversionCompleteCallerConsumer(t *testing.T) { + var originals strings.Builder + count := 0 + for _, spec := range x86PackedConversionOracleSpecs() { + for _, shape := range spec.shapes() { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + row := spec.name + " " + bank[shape.source] + "0," + bank[shape.destination] + "1" + body := "VMOVUPS (DI),Z0\n" + row + "\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + originals.WriteString(strings.Replace(assembly, "TEXT ·kernel", fmt.Sprintf("TEXT ·kernel%d", count), 1)) + count++ + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("complete initialized conversion caller", row, err) + } + for _, operand := range []string{"(BX)", "(DX)", "(SP)", "(BP)", "a+0(FP)"} { + body := spec.name + " " + operand + "," + bank[shape.destination] + "1\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("conversion acquired unproved address/envelope/backing", row, operand, err) + } + } + } + } + goAssemblerOracleOutput(t, "amd64", originals.String(), "", true) + t.Logf("%d complete caller shapes, five independent address/backing negatives each", count) +} + +func TestX86PackedConversionWholeFileSyntaxBeforeContext(t *testing.T) { + for _, context := range []struct{ name, arch, triple, instruction string }{ + {"target", "amd64", "", "CVTPS2PL X0,X1"}, + {"foreign target", "amd64", "aarch64-unknown-linux-gnu", "CVTPS2PL X0,X1"}, + {"physical MMX", "amd64", "x86_64-unknown-linux-gnu", "CVTPS2PL X0,M0"}, + {"physical mode32", "386", "i386-unknown-linux-gnu", "VCVTPS2DQ X8,X0"}, + } { + for _, api := range []string{"text", "module", "Go"} { + source := "TEXT ·f(SB),4,$0-0\n" + context.instruction + "\nRET\n" + _, err := x86ScalarFPBitViewAPI(t, source, "func f()", context.arch, context.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original physical/target boundary lost", context.name, api, err) + } + for _, malformed := range []string{"VCVTPS2DQ.Z X0,X1", "CVTPS2PL X0,(AX)"} { + for _, suffix := range []struct{ assembly, declaration string }{ + {malformed + "\n", "func f()"}, + {"TEXT ·g(SB),4,$0-0\nRET\n" + malformed + "\n", "func f();func g()"}, + } { + _, err := x86ScalarFPBitViewAPI(t, source+suffix.assembly, suffix.declaration, context.arch, context.triple, api) + if err == nil || errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "packed numeric conversion grammar") { + t.Fatal("earlier Context hid dead malformed source", context.name, api, malformed, err) + } + } + } + } + } +} diff --git a/amd64_packed_conversion_fault_runtime_test.go b/amd64_packed_conversion_fault_runtime_test.go new file mode 100644 index 000000000..5752a88cf --- /dev/null +++ b/amd64_packed_conversion_fault_runtime_test.go @@ -0,0 +1,311 @@ +//go:build unix + +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +type x86PackedConversionFaultRow struct { + name string + width int + laneBytes int + lanes int + masked bool + floating bool + precision bool + aligned bool + conversion string +} + +func x86PackedConversionFaultRows(wide bool) []x86PackedConversionFaultRow { + var rows []x86PackedConversionFaultRow + for _, spec := range x86PackedConversionOracleSpecs() { + if !wide && !spec.legacy { + continue + } + for _, shape := range spec.shapes() { + destination := map[int]string{16: "X1", 32: "Y1", 64: "Z1"}[shape.destination] + floating := strings.Contains(strings.Split(spec.name, "2")[0], "PS") || strings.Contains(strings.Split(spec.name, "2")[0], "PD") + precision := floating || spec.input > map[int]int{32: 24, 64: 53}[spec.output] + rows = append(rows, x86PackedConversionFaultRow{name: spec.name, width: shape.lanes * spec.input / 8, laneBytes: spec.input / 8, lanes: shape.lanes, + floating: floating, precision: precision, aligned: spec.legacy, conversion: spec.name + " (DI)," + destination}) + if !spec.legacy { + rows = append(rows, + x86PackedConversionFaultRow{name: spec.name + ".Z", width: shape.lanes * spec.input / 8, laneBytes: spec.input / 8, lanes: shape.lanes, masked: true, + floating: floating, precision: precision, conversion: spec.name + ".Z (DI),K1," + destination}, + x86PackedConversionFaultRow{name: spec.name + ".BCST.Z", width: spec.input / 8, laneBytes: spec.input / 8, lanes: shape.lanes, masked: true, + floating: floating, precision: precision, conversion: spec.name + ".BCST.Z (DI),K1," + destination}) + } + } + } + return rows +} + +// One actual Go TEXT holds every independent straight-line kernel. Split at +// decoded RET boundaries, never a byte search (0xc3 can occur in displacement). +// This creates no production native/source authorization. +func x86PackedConversionFaultOriginalBytes(t *testing.T, rows []x86PackedConversionFaultRow) [][]byte { + t.Helper() + var source strings.Builder + source.WriteString("TEXT original(SB),4,$0-0\n") + for _, row := range rows { + source.WriteString("LDMXCSR (SI)\n") + if row.masked { + source.WriteString("KMOVW DX,K1\n") + } + source.WriteString(row.conversion + "\nRET\n") + } + code := x86PackedShiftActualGoBytes(t, "amd64", source.String()) + var result [][]byte + offset := 0 + for _, row := range rows { + start := offset + header := []byte{0x0f, 0xae, 0x16} // Actual Go LDMXCSR (SI). + if row.masked { + header = append(header, 0xc5, 0xf8, 0x92, 0xca) // KMOVW DX,K1. + } + if len(code)-offset < len(header) || !bytes.Equal(code[offset:offset+len(header)], header) { + t.Fatal("actual original header changed", row.name, offset) + } + offset += len(header) + _, length, known, err := decodedX86PackedNumericConversionInstruction(code[offset:], 64) + if !known { + _, length, known, err = decodedX86SameWidthConversionInstruction(code[offset:], 64) + } + if !known { + _, length, known, err = decodedX86PackedDoubleDwordInstruction(code[offset:], 64) + } + if !known { + native, decodeErr := x86asm.Decode(code[offset:], 64) + err, length = decodeErr, native.Len + } + if err != nil || length <= 0 || offset+length >= len(code) || code[offset+length] != 0xc3 { + t.Fatal("whole original conversion/RET boundary", row.name, offset, err) + } + offset += length + 1 + result = append(result, append([]byte(nil), code[start:offset]...)) + } + if offset != len(code) { + t.Fatal("unconsumed original Go instruction bytes") + } + return result +} + +func x86PackedConversionDeadFaults(t *testing.T, wide, nativeTraps bool) { + t.Helper() + axis := x86FloatCompareSSE + if wide || nativeTraps { + axis = x86FloatCompareEVEX + } + execution := x86FloatCompareRuntimeTools(t, axis) + rows := x86PackedConversionFaultRows(wide) + x86PackedConversionRunFaultRows(t, execution, rows, 96, 0, 1, nativeTraps, false) +} + +func x86PackedConversionRunFaultRows(t *testing.T, execution x86FloatCompareExecution, rows []x86PackedConversionFaultRow, limit, owner, owners int, nativeTraps, complete bool) { + t.Helper() + consumed := 0 + for batchNumber, start := 0, 0; start < len(rows); batchNumber, start = batchNumber+1, start+limit { + if batchNumber%owners != owner { + continue + } + end := start + limit + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + batch := rows[start:end] + var original [][]byte + observe := func(t *testing.T) { + original = x86PackedConversionFaultOriginalBytes(t, batch) + if len(original) != len(batch) { + t.Fatal("incomplete original Go opcode reference") + } + } + if complete { + t.Run("original_go_opcode_reference", observe) + } else { + observe(t) + } + if len(original) != len(batch) { + t.Fatal("missing original Go opcode reference") + } + var source, decl, native, prototypes, pairs strings.Builder + for index, row := range batch { + name, reference := fmt.Sprintf("fault%d", index), fmt.Sprintf("original%d", index) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-24\nMOVQ p+0(FP),DI\nMOVQ csr+8(FP),SI\nLDMXCSR (SI)\n", name) + if row.masked { + source.WriteString("MOVL mask+16(FP),DX\nKMOVW DX,K1\n") + } + source.WriteString(row.conversion + "\nRET\n") + fmt.Fprintf(&decl, "func %s(p *uint64,csr *uint32,mask uint32)\n", name) + var octets []string + for _, value := range original[index] { + octets = append(octets, fmt.Sprintf("0x%02x", value)) + } + symbol := reference + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(octets, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint64_t*,uint32_t*,uint32_t),%s(uint64_t*,uint32_t*,uint32_t);\n", name, reference) + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d,%d,%d,%d,%d},\n", reference, name, row.width, row.laneBytes, row.lanes, + x86FloatAddSubBoolInt(row.masked), x86FloatAddSubBoolInt(row.floating), + x86FloatAddSubBoolInt(row.precision), x86FloatAddSubBoolInt(row.aligned)) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf("#define NATIVE_TRAPS %d\n#define COMPLETE_NATIVE %d\n", x86FloatAddSubBoolInt(nativeTraps), x86FloatAddSubBoolInt(complete)) + `#include +#include +#include +#include +#include +#include +#include +#include +` + native.String() + prototypes.String() + ` +typedef void (*program)(uint64_t*,uint32_t*,uint32_t); +static const struct {program original,translated;int width,lane,lanes,masked,floating,precision,aligned;} pairs[]={ +` + pairs.String() + `}; +static int outcome(program fn,uint8_t *p,uint32_t csr,uint32_t mask) { + pid_t child=fork();if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0};if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + fn((uint64_t*)p,&csr,mask);_exit(0); + } + int status;if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status)&&WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status)) {int sig=WTERMSIG(status);if(sig==SIGSEGV||sig==SIGBUS)return 1;if(sig==SIGFPE)return 2;} + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + uint8_t *p=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(p==MAP_FAILED||mprotect(p+page,page,PROT_NONE))return 2; + memset(p,0,page); + for(unsigned i=0;i len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + original := x86PackedConversionFaultOriginalBytes(t, rows[start:end]) + if len(original) != end-start { + t.Fatal("original native fault fixture lost a conversion") + } + consumed += len(original) + }) + } + if consumed != len(rows) { + t.Fatal("native fault batches overlap or omit rows") + } + t.Logf("%d original-Go complete memory/masked/broadcast fault kernels; assembly only, not native execution", consumed) +} diff --git a/amd64_packed_conversion_forms.go b/amd64_packed_conversion_forms.go new file mode 100644 index 000000000..5c02f97b5 --- /dev/null +++ b/amd64_packed_conversion_forms.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" +) + +// Element types, active lanes and physical registers are separate axes. In +// particular X <- X can read 8 or 16 bytes and can write only 8 numeric bytes +// followed by eight architectural zero bytes. +type x86PackedConversionSpec struct { + amd64PackedNumericConversionSpec + legacy, vex bool +} + +func x86PackedConversionSpecification(op Op) (x86PackedConversionSpec, bool) { + if spec, ok := amd64PackedNumericConversionSpecs[op]; ok { + return x86PackedConversionSpec{amd64PackedNumericConversionSpec: spec}, true + } + base := string(op) + legacy := !strings.HasPrefix(base, "V") + spec := amd64PackedNumericConversionSpec{ + sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, + sourceBits: 32, destinationBits: 32, truncate: strings.HasPrefix(base, "CVTT") || strings.HasPrefix(base, "VCVTT"), + } + switch base { + case "CVTPS2PL", "CVTTPS2PL": + spec.shapes = amd64PackedNumericSame32[:1] + case "VCVTPS2DQ", "VCVTTPS2DQ": + spec.shapes, spec.explicit = amd64PackedNumericSame32, "round" + case "CVTPD2PL", "CVTTPD2PL", "VCVTPD2DQX", "VCVTTPD2DQX": + spec.sourceBits = 64 + spec.shapes = []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}} + case "VCVTPD2DQY", "VCVTTPD2DQY": + spec.sourceBits = 64 + spec.shapes = []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}} + case "VCVTPD2DQ", "VCVTTPD2DQ": + spec.sourceBits, spec.explicit = 64, "round" + spec.shapes = []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}} + default: + return x86PackedConversionSpec{}, false + } + if spec.explicit != "" && spec.truncate { + spec.explicit = "sae" + } + return x86PackedConversionSpec{amd64PackedNumericConversionSpec: spec, legacy: legacy, vex: !legacy}, true +} + +type x86PackedConversionForm struct { + spec x86PackedConversionSpec + shape amd64PackedNumericShape + properties amd64BinaryFloatingSuffix + inputBytes, memoryBytes int + destination, memory, mask int + mmx, evex bool +} + +// This grammar supplies widths, not a native frame, caller, pointee or MMX/x87 +// certificate. Memory consumers must independently retain those boundaries. +func x86PackedConversionProperties(ins Instr, arch string) (x86PackedConversionForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86PackedConversionSpecification(Op(base)) + if !known { + return x86PackedConversionForm{}, false, nil + } + return x86PackedConversionOperandProperties(ins, arch, spec) +} + +// Share operand axes, not a family namespace, source proof or caller registry. +func x86PackedConversionOperandProperties(ins Instr, arch string, spec x86PackedConversionSpec) (x86PackedConversionForm, bool, error) { + _, suffix, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + f := x86PackedConversionForm{spec: spec, destination: len(ins.Args) - 1, memory: -1, mask: -1} + fail := func() (x86PackedConversionForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside packed numeric conversion grammar: %q", arch, ins.Op, ins.Raw) + } + var valid bool + f.properties, valid = parseAMD64BinaryFloatingSuffix(suffix) + if !valid || arch != "amd64" && arch != "386" || len(ins.Args) != 2 && len(ins.Args) != 3 || + spec.legacy && (suffix != "" || len(ins.Args) != 2) || + f.properties.rounding != "" && spec.explicit != "round" || f.properties.sae && spec.explicit != "sae" { + return fail() + } + if len(ins.Args) == 3 { + if !amd64NonzeroKOperand(ins.Args[1]) { + return fail() + } + f.mask = 1 + } else if f.properties.zeroing { + return fail() + } + destination := ins.Args[f.destination] + _, f.mmx = amd64ParseMReg(destination.Reg) + if destination.Kind != OpReg || f.mmx && !spec.legacy { + return fail() + } + width := amd64VectorByteWidth(destination.Reg) + if f.mmx { + f.shape = amd64PackedNumericShape{sourceBytes: 16, destinationBytes: 8, lanes: 2} + } else { + for _, shape := range spec.shapes { + if shape.destinationBytes == width { + f.shape = shape + break + } + } + if f.shape.lanes == 0 || !amd64EVEXVectorRegister(destination, width) || + spec.legacy && !amd64VEXVectorRegister(destination, 16) { + return fail() + } + } + f.inputBytes = f.shape.lanes * spec.sourceBits / 8 + source := ins.Args[0] + if source.Kind == OpReg { + if f.properties.broadcast || !amd64EVEXVectorRegister(source, f.shape.sourceBytes) || + spec.legacy && !amd64VEXVectorRegister(source, 16) { + return fail() + } + } else if isAMD64MemoryOperand(source) && x86SignedPackedFloatMemoryOperand(source, arch) { + f.memory, f.memoryBytes = 0, f.inputBytes + if f.properties.broadcast { + f.memoryBytes = spec.sourceBits / 8 + } + } else { + return fail() + } + if (f.properties.rounding != "" || f.properties.sae) && + (source.Kind != OpReg || f.shape != spec.shapes[len(spec.shapes)-1]) { + return fail() + } + f.evex = !spec.legacy && (!spec.vex || ins.x86PackedNumericEVEX || f.mask >= 0 || + f.properties.broadcast || f.properties.rounding != "" || f.properties.sae || + f.shape.sourceBytes == 64 || f.shape.destinationBytes == 64) + for index, operand := range []Operand{source, destination} { + if operand.Kind != OpReg || index == 1 && f.mmx { + continue + } + index, _ := amd64VectorRegisterIndex(operand.Reg, amd64VectorByteWidth(operand.Reg)) + if arch == "386" && index >= 8 { + return f, true, fmt.Errorf("%w: %s register %s has no physical mode32 binding", ErrProbeNeedsContext, ins.Op, operand.Reg) + } + f.evex = f.evex || index >= 16 + } + return f, true, nil +} + +func validateX86PackedConversionFile(file *File, opt Options) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + var pending error + deferContext := func(err error) { + if pending == nil { + pending = err + } + } + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + form, known, err := x86PackedConversionProperties(ins, opt.Goarch) + if !known { + continue + } + if err != nil { + // Physical mode32 binding is not a grammar error. Retain its + // refusal, but still validate every later/dead source format. + if errors.Is(err, ErrProbeNeedsContext) { + deferContext(fmt.Errorf("%s: %w", fn.Sym, err)) + continue + } + return fmt.Errorf("%s: %w", fn.Sym, err) + } + if form.mmx { + deferContext(fmt.Errorf("%w: %s has no complete physical MMX/x87 caller contract", ErrProbeNeedsContext, ins.Op)) + } + if !x86ScalarFloatIntegerNativeTarget(opt.Goarch, opt.TargetTriple) { + deferContext(fmt.Errorf("%w: %s needs its matching native x86 MXCSR target", ErrProbeNeedsContext, ins.Op)) + } + } + } + return pending +} diff --git a/amd64_packed_conversion_matrix_test.go b/amd64_packed_conversion_matrix_test.go new file mode 100644 index 000000000..20460314b --- /dev/null +++ b/amd64_packed_conversion_matrix_test.go @@ -0,0 +1,320 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// Independent Go/Intel encoding inventory. No expected class, tuple, opcode +// or semantic width is obtained from the production conversion tables. +type x86PackedConversionOracle struct { + name, shape, control, pp, opcode string + input, output, w int + legacy, vex bool +} + +func x86PackedConversionOracleSpecs() []x86PackedConversionOracle { + return []x86PackedConversionOracle{ + {"CVTPS2PL", "single", "", "Pe", "0x5b", 32, 32, 0, true, false}, + {"CVTTPS2PL", "single", "", "Pf3", "0x5b", 32, 32, 0, true, false}, + {"CVTPD2PL", "narrow16", "", "Pf2", "0xe6", 64, 32, 0, true, false}, + {"CVTTPD2PL", "narrow16", "", "Pe", "0xe6", 64, 32, 0, true, false}, + {"VCVTPS2DQ", "same", "round", "66", "0x5B", 32, 32, 0, false, true}, + {"VCVTTPS2DQ", "same", "sae", "F3", "0x5B", 32, 32, 0, false, true}, + {"VCVTPD2DQX", "narrow16", "", "F2", "0xE6", 64, 32, 1, false, true}, + {"VCVTPD2DQY", "narrow32", "", "F2", "0xE6", 64, 32, 1, false, true}, + {"VCVTPD2DQ", "narrow64", "round", "F2", "0xE6", 64, 32, 1, false, false}, + {"VCVTTPD2DQX", "narrow16", "", "66", "0xE6", 64, 32, 1, false, true}, + {"VCVTTPD2DQY", "narrow32", "", "66", "0xE6", 64, 32, 1, false, true}, + {"VCVTTPD2DQ", "narrow64", "sae", "66", "0xE6", 64, 32, 1, false, false}, + {"VCVTPD2QQ", "same", "round", "66", "0x7B", 64, 64, 1, false, false}, + {"VCVTPD2UQQ", "same", "round", "66", "0x79", 64, 64, 1, false, false}, + {"VCVTTPD2QQ", "same", "sae", "66", "0x7A", 64, 64, 1, false, false}, + {"VCVTTPD2UQQ", "same", "sae", "66", "0x78", 64, 64, 1, false, false}, + {"VCVTPS2UDQ", "same", "round", "", "0x79", 32, 32, 0, false, false}, + {"VCVTTPS2UDQ", "same", "sae", "", "0x78", 32, 32, 0, false, false}, + {"VCVTUDQ2PS", "same", "round", "F2", "0x7A", 32, 32, 0, false, false}, + {"VCVTQQ2PD", "same", "round", "F3", "0xE6", 64, 64, 1, false, false}, + {"VCVTUQQ2PD", "same", "round", "F3", "0x7A", 64, 64, 1, false, false}, + {"VCVTPS2QQ", "wide", "round", "66", "0x7B", 32, 64, 0, false, false}, + {"VCVTPS2UQQ", "wide", "round", "66", "0x79", 32, 64, 0, false, false}, + {"VCVTTPS2QQ", "wide", "sae", "66", "0x7A", 32, 64, 0, false, false}, + {"VCVTTPS2UQQ", "wide", "sae", "66", "0x78", 32, 64, 0, false, false}, + {"VCVTUDQ2PD", "wide", "", "F3", "0x7A", 32, 64, 0, false, false}, + {"VCVTPD2UDQX", "narrow16", "", "", "0x79", 64, 32, 1, false, false}, + {"VCVTPD2UDQY", "narrow32", "", "", "0x79", 64, 32, 1, false, false}, + {"VCVTPD2UDQ", "narrow64", "round", "", "0x79", 64, 32, 1, false, false}, + {"VCVTTPD2UDQX", "narrow16", "", "", "0x78", 64, 32, 1, false, false}, + {"VCVTTPD2UDQY", "narrow32", "", "", "0x78", 64, 32, 1, false, false}, + {"VCVTTPD2UDQ", "narrow64", "sae", "", "0x78", 64, 32, 1, false, false}, + {"VCVTQQ2PSX", "narrow16", "", "", "0x5B", 64, 32, 1, false, false}, + {"VCVTQQ2PSY", "narrow32", "", "", "0x5B", 64, 32, 1, false, false}, + {"VCVTQQ2PS", "narrow64", "round", "", "0x5B", 64, 32, 1, false, false}, + {"VCVTUQQ2PSX", "narrow16", "", "F2", "0x7A", 64, 32, 1, false, false}, + {"VCVTUQQ2PSY", "narrow32", "", "F2", "0x7A", 64, 32, 1, false, false}, + {"VCVTUQQ2PS", "narrow64", "round", "F2", "0x7A", 64, 32, 1, false, false}, + } +} + +type x86PackedOracleShape struct{ source, destination, lanes int } + +func (spec x86PackedConversionOracle) shapes() []x86PackedOracleShape { + var widths []int + switch spec.shape { + case "single", "narrow16": + widths = []int{16} + case "narrow32": + widths = []int{32} + case "narrow64": + widths = []int{64} + default: + widths = []int{16, 32, 64} + } + var result []x86PackedOracleShape + for _, width := range widths { + source, destination := width, width + lanes := width * 8 / spec.input + if spec.shape == "wide" { + source, lanes = width/2, width*8/spec.output + } else if strings.HasPrefix(spec.shape, "narrow") { + destination = width / 2 + } + if source < 16 { + source = 16 + } + if destination < 16 { + destination = 16 + } + result = append(result, x86PackedOracleShape{source, destination, lanes}) + } + return result +} + +func TestX86PackedConversionIndependentCompleteGoRows(t *testing.T) { + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(legacy), "var yxcvm1 = []ytab{\n\t{Zm_r_xm, 2, argList{Yxm, Yxr}},\n\t{Zm_r_xm, 2, argList{Yxm, Ymr}},\n}") { + t.Fatal("complete legacy operand table changed") + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, spec := range x86PackedConversionOracleSpecs() { + if spec.legacy { + mmxPrefix, mmxOpcode := "Pm", "0x2d" + if spec.input == 64 { + mmxPrefix = "Pe" + } + if strings.Contains(spec.name, "CVTT") { + mmxOpcode = "0x2c" + } + want := fmt.Sprintf("{A%s, yxcvm1, Px, opBytes{%s, %s, %s, %s}}", spec.name, spec.pp, spec.opcode, mmxPrefix, mmxOpcode) + if !strings.Contains(string(legacy), want) { + t.Fatal("legacy opcode attributes changed", want) + } + count += 2 + continue + } + shapes := spec.shapes() + table := "_yvcvtpd2qq" + switch { + case spec.shape == "same" && spec.vex: + table = "_yvcvtdq2ps" + case spec.shape == "wide": + table = "_yvcvtps2qq" + if spec.name == "VCVTUDQ2PD" { + table = "_yvcvtudq2pd" + } + case spec.shape == "narrow64": + table = "_yvcvtpd2dq" + case spec.shape == "narrow16", spec.shape == "narrow32": + table = "_yvcvtpd2udq" + strings.ToLower(spec.name[len(spec.name)-1:]) + if spec.vex { + table = strings.Replace(table, "udq", "dq", 1) + } + } + type row struct { + classes []string + zcase string + offset int + } + var rows []row + var opbytes []string + prefix := map[int]string{16: "x", 32: "y", 64: "z"} + appendShape := func(shape x86PackedOracleShape, evex bool) { + encoding, word, extra := "vex", 0, "" + if evex { + encoding, word, extra = "evex", spec.w, "Evex" + } + src := "Y" + prefix[shape.source] + "m" + extra + dst := "Y" + prefix[shape.destination] + "r" + extra + src, dst = strings.ReplaceAll(src, "zmEvex", "zm"), strings.ReplaceAll(dst, "zrEvex", "zr") + bits := shape.source * 8 + if spec.shape == "wide" { + bits = shape.destination * 8 + } + attributes := fmt.Sprintf("avxEscape | %s%d", encoding, bits) + if spec.pp != "" { + attributes += " | " + encoding + spec.pp + } + attributes += fmt.Sprintf(" | %s0F | %sW%d", encoding, encoding, word) + opbytes = append(opbytes, attributes) + if evex { + attributes = fmt.Sprintf("evexN%d | evexBcstN%d", shape.lanes*spec.input/8, spec.input/8) + if bits == 512 && spec.control != "" { + attributes += map[string]string{"round": " | evexRoundingEnabled", "sae": " | evexSaeEnabled"}[spec.control] + } + opbytes = append(opbytes, attributes+" | evexZeroingEnabled") + rows = append(rows, row{[]string{src, dst}, "Zevex_rm_v_r", 0}, row{[]string{src, "Yknot0", dst}, "Zevex_rm_k_r", 3}) + } else { + rows = append(rows, row{[]string{src, dst}, "Zvex_rm_v_r", 2}) + } + opbytes = append(opbytes, spec.opcode) + } + if spec.vex { + for _, shape := range shapes { + if shape.source < 64 && shape.destination < 64 { + appendShape(shape, false) + } + } + } + if len(shapes) == 3 && spec.name != "VCVTUDQ2PD" { + shapes = []x86PackedOracleShape{shapes[2], shapes[0], shapes[1]} + } + for _, shape := range shapes { + appendShape(shape, true) + } + index, offset := 0, 0 + for _, actual := range rules { + if actual.Opcode != spec.name { + continue + } + if index >= len(rows) { + t.Fatal("unexpected extra Go encoder row", spec.name) + } + want := rows[index] + if actual.Ytab != table || actual.Prefix != "Pavx" || actual.RowIndex != index || actual.EncodingOffset != offset || + actual.ZCase != want.zcase || actual.ZOffset != want.offset || !reflect.DeepEqual(actual.Classes, want.classes) || !reflect.DeepEqual(actual.OpBytes, opbytes) { + t.Fatalf("%s row%d changed: %+v\nwant %s %+v %v", spec.name, index, actual, table, want, opbytes) + } + index, offset, count = index+1, offset+want.offset, count+1 + } + if index != len(rows) { + t.Fatal("missing encoder rows", spec.name, index, len(rows)) + } + } + t.Logf("38 independent names, %d complete Go encoder rows", count) +} + +func x86PackedConversionOracleRows(arch string) []string { + var rows []string + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + for _, spec := range x86PackedConversionOracleSpecs() { + for _, shape := range spec.shapes() { + destination := bank[shape.destination] + "7" + for _, source := range []string{bank[shape.source] + "0", "(AX)", "3(BX)(CX*4)"} { + rows = append(rows, spec.name+" "+source+","+destination) + if !spec.legacy { + rows = append(rows, spec.name+" "+source+",K1,"+destination, spec.name+".Z "+source+",K7,"+destination) + } + } + if !spec.legacy { + rows = append(rows, spec.name+".BCST (AX),"+destination, spec.name+".BCST.Z 3(BX)(CX*4),K7,"+destination) + if arch == "amd64" { + rows = append(rows, spec.name+" "+bank[shape.source]+"31,"+bank[shape.destination]+"31") + } + } + if (shape.source == 64 || shape.destination == 64) && spec.control != "" { + suffixes := []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} + if spec.control == "sae" { + suffixes = []string{"SAE"} + } + for _, suffix := range suffixes { + rows = append(rows, spec.name+"."+suffix+" "+bank[shape.source]+"0,"+destination, + spec.name+"."+suffix+".Z "+bank[shape.source]+"0,K7,"+destination) + } + } + } + } + return rows +} + +func TestX86PackedConversionFiveTargetsThreeAPIObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + rows := x86PackedConversionOracleRows(target.arch) + seen := make(map[string]bool) + const batchSize = 96 + for start := 0; start < len(rows); start += batchSize { + end := start + batchSize + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + var source strings.Builder + for index, row := range rows[start:end] { + if seen[row] { + t.Fatal("duplicate matrix row", row) + } + seen[row] = true + fmt.Fprintf(&source, "TEXT ·f%d(SB),4,$0-0\n%s\nRET\n", index, row) + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + file, signatures := x86UnboundVoidFormatSources(t, source.String(), target.arch, target.triple) + ir := x86ScalarFormatObjectUnit(t, file, signatures, target.arch, target.triple, 0) + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, input) + } + compileLLVMToObject(t, llc, target.triple, "packed-conversion.ll", "packed-conversion.o", input) + } + }) + } + if len(seen) != len(rows) { + t.Fatal("incomplete batch union") + } + t.Logf("%d unchanged unique formats, <=96 functions/module, 3 public Context APIs and 2 emitter IR pipelines; initialized public objects are separate", len(rows)) + }) + } +} + +func TestX86PackedConversionMMXAndTargetRemainContext(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + shape := spec.shapes()[0] + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + row := spec.name + " " + bank[shape.source] + "0," + bank[shape.destination] + "1" + for _, target := range []string{"", "aarch64-unknown-linux-gnu", "i386-unknown-linux-gnu"} { + _, err := x86ScalarFPBitViewAPI(t, "TEXT ·f(SB),4,$0-0\nRET\n"+row+"\n", "func f()", "amd64", target, "text") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("dead instruction acquired a foreign/missing target", row, target, err) + } + } + if spec.legacy { + for _, destination := range []string{"M0", "M7"} { + row := spec.name + " X0," + destination + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + _, err := x86ScalarFPBitViewAPI(t, "TEXT ·f(SB),4,$0-0\n"+row+"\nRET\n", "func f()", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("virtual MMX destination granted physical x87 state", row, err) + } + } + } + } +} diff --git a/amd64_packed_conversion_native_address_test.go b/amd64_packed_conversion_native_address_test.go new file mode 100644 index 000000000..9fcb193e2 --- /dev/null +++ b/amd64_packed_conversion_native_address_test.go @@ -0,0 +1,96 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Independent named memory inventory from Go's legacy and AVX encoder rows. +// This address-role witness does not borrow the production spec or assert that +// a Context form was translated; the separate complete family matrix retains +// all widths, masks, rounding and raw encodings. +func x86PackedNativeAddressRows() []struct{ name, destination string } { + var rows []struct{ name, destination string } + for _, name := range []string{ + "CVTPS2PL", "CVTTPS2PL", "CVTPD2PL", "CVTTPD2PL", + "VCVTPS2DQ", "VCVTTPS2DQ", "VCVTPD2DQX", "VCVTPD2DQY", "VCVTTPD2DQX", "VCVTTPD2DQY", + "VCVTPD2QQ", "VCVTPD2UQQ", "VCVTTPD2QQ", "VCVTTPD2UQQ", + "VCVTPS2UDQ", "VCVTTPS2UDQ", "VCVTUDQ2PS", "VCVTQQ2PD", "VCVTUQQ2PD", + "VCVTPS2QQ", "VCVTPS2UQQ", "VCVTTPS2QQ", "VCVTTPS2UQQ", "VCVTUDQ2PD", + "VCVTPD2UDQX", "VCVTPD2UDQY", "VCVTTPD2UDQX", "VCVTTPD2UDQY", + "VCVTQQ2PSX", "VCVTQQ2PSY", "VCVTUQQ2PSX", "VCVTUQQ2PSY", + } { + rows = append(rows, struct{ name, destination string }{name, "X1"}) + } + for _, name := range []string{"VCVTPD2DQ", "VCVTTPD2DQ", "VCVTPD2UDQ", "VCVTTPD2UDQ", "VCVTQQ2PS", "VCVTUQQ2PS"} { + rows = append(rows, struct{ name, destination string }{name, "Y1"}) + } + return rows +} + +func TestX86PackedConversionOriginalNativeAddressRequired(t *testing.T) { + rows := x86PackedNativeAddressRows() + if len(rows) != 38 { + t.Fatal("complete native conversion memory-name inventory changed", len(rows)) + } + for _, arch := range []string{"386", "amd64"} { + var original strings.Builder + for index, row := range rows { + for _, address := range []string{"(SP)", "32(SP)", "(BP)", "-17(BP)", "32(SP)(CX*2)", "32(AX)(BP*2)"} { + name := fmt.Sprintf("original%d_%d", index, original.Len()) + original.WriteString(x86OrdinaryBPNativeSource(arch, name, row.name+" "+address+","+row.destination, address, 128)) + } + } + goAssemblerOracleOutput(t, arch, original.String(), "", true) + } + for _, target := range x86RawReturnTargets { + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + for _, row := range rows { + for _, address := range []string{"(SP)", "32(SP)", "(BP)", "-17(BP)", "32(SP)(CX*2)", "32(AX)(BP*2)"} { + source := x86OrdinaryBPNativeSource(target.arch, "view", row.name+" "+address+","+row.destination, address, 128) + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *byte)", target.arch, target.triple, api) + x86RequireOrdinaryBPNativeOutcome(t, target.arch, address, ir, err) + } + } + }) + } + } +} + +func TestX86PackedConversionOrdinaryNativeAddressControls(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + move, frame := "MOVQ", 8 + if target.arch == "386" { + move, frame = "MOVL", 4 + } + var source, declarations strings.Builder + for index, row := range x86PackedNativeAddressRows() { + for variant, address := range []string{"(AX)", "16(AX)(CX*2)"} { + name := fmt.Sprintf("control%d_%d", index, variant) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),AX\nMOVL $0,CX\n%s %s,%s\nRET\n", name, frame, move, row.name, address, row.destination) + fmt.Fprintf(&declarations, "func %s(p *[128]byte)\n", name) + } + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("ordinary GP source control failed", err) + } + if strings.Count(ir, "asm sideeffect") != 76 { + t.Fatal("dead conversions lost whole native effects") + } + compileLLVMToObject(t, llc, target.triple, "native-address.ll", "native-address.o", x86MulDivRoundOptimize(t, ir)) + }) + } + }) + } +} diff --git a/amd64_packed_conversion_native_host_linux_amd64_test.go b/amd64_packed_conversion_native_host_linux_amd64_test.go new file mode 100644 index 000000000..374fd2180 --- /dev/null +++ b/amd64_packed_conversion_native_host_linux_amd64_test.go @@ -0,0 +1,8 @@ +package plan9asm + +import "testing" + +func x86PackedConversionRequireNativeHost(t *testing.T) { + t.Helper() + x87IA32RequireNativeHost(t) +} diff --git a/amd64_packed_conversion_native_host_other_test.go b/amd64_packed_conversion_native_host_other_test.go new file mode 100644 index 000000000..cbed8caa2 --- /dev/null +++ b/amd64_packed_conversion_native_host_other_test.go @@ -0,0 +1,10 @@ +//go:build !linux || !amd64 + +package plan9asm + +import "testing" + +func x86PackedConversionRequireNativeHost(t *testing.T) { + t.Helper() + t.Fatal("required complete packed conversion runtime needs actual native Linux/amd64") +} diff --git a/amd64_packed_conversion_native_tools_test.go b/amd64_packed_conversion_native_tools_test.go new file mode 100644 index 000000000..6c993b7a6 --- /dev/null +++ b/amd64_packed_conversion_native_tools_test.go @@ -0,0 +1,92 @@ +package plan9asm + +import ( + "os" + "path/filepath" + "testing" +) + +func x86PackedConversionNativeTools(t *testing.T) x86FloatCompareExecution { + t.Helper() + x86PackedConversionRequireNativeHost(t) + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + directory := t.TempDir() + source := filepath.Join(directory, "packed-conversion-cpu.c") + executable := filepath.Join(directory, "packed-conversion-cpu") + if err := os.WriteFile(source, []byte(x86PackedConversionNativePreflightC), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + t.Logf("actual native packed conversion CPU/opcode preflight: %s", execution.run(t, executable)) + return execution +} + +func x86PackedConversionNativePipelines(t *testing.T, execution x86FloatCompareExecution, ir, driver string, want []byte) { + t.Helper() + for _, pipeline := range []struct { + name, backend string + optimize bool + }{ + {"backend_O0", "-O0", false}, + {"backend_O2", "-O2", false}, + {"IR_O2_backend_O2", "-O2", true}, + } { + t.Run(pipeline.name, func(t *testing.T) { + input := ir + if pipeline.optimize { + input = x86DwordProductIROptimize(t, ir) + } + got := execution.compileAndRunLLVM(t, "packed-conversion", input, driver, pipeline.backend) + if string(got) != string(want) { + t.Fatalf("original native/LLVM receipt differs\noriginal:\n%sLLVM:\n%s", want, got) + } + }) + } +} + +// This preflight executes original whole opcodes only after both CPUID and +// actual OS register state pass. A missing feature or a failed opcode is an +// infrastructure failure, never a skipped or portable runtime certificate. +const x86PackedConversionNativePreflightC = `#include +#include +#include + +__attribute__((target("avx512f,avx512dq,avx512bw,avx512vl"))) +static int original_conversion_opcodes(void) { + _Alignas(64) uint64_t qwords[8]={0}; + _Alignas(16) uint32_t dwords[4]={0}; + uint64_t mask; + __asm__ volatile("vpxord %%zmm0,%%zmm0,%%zmm0; " + "vcvtqq2pd %%zmm0,%%zmm1; vmovdqu64 %%zmm1,%0; " + "kxnorq %%k1,%%k1,%%k1; kmovq %%k1,%2; " + "vcvtdq2ps %%xmm0,%%xmm1%{%%k1%}%{z%}; vmovdqu32 %%xmm1,%1" + : "=m"(qwords),"=m"(dwords),"=&r"(mask) + :: "zmm0","zmm1","k1","memory"); + for (unsigned i=0;i<8;i++) if (qwords[i]) return 0; + for (unsigned i=0;i<4;i++) if (dwords[i]) return 0; + return mask==UINT64_MAX; +} + +int main(void) { + unsigned a,b,c,d,f=0,dq=0,bw=0,vl=0,osxsave=0; + uint64_t xcr0=0; + if (__get_cpuid(1,&a,&b,&c,&d)) { + osxsave=(c>>27)&1; + if (osxsave) { + unsigned low,high; + __asm__ volatile("xgetbv" : "=a"(low),"=d"(high) : "c"(0)); + xcr0=((uint64_t)high<<32)|low; + } + } + if (__get_cpuid_count(7,0,&a,&b,&c,&d)) { + f=(b>>16)&1; dq=(b>>17)&1; bw=(b>>30)&1; vl=(b>>31)&1; + } + printf("AVX512F=%u AVX512DQ=%u AVX512BW=%u AVX512VL=%u OSXSAVE=%u XCR0=%llx\n", + f,dq,bw,vl,osxsave,(unsigned long long)xcr0); + fflush(stdout); + if (!f || !dq || !bw || !vl || !osxsave || (xcr0&0xe6)!=0xe6) return 1; + if (!original_conversion_opcodes()) return 2; + puts("PACKED_CONVERSION_NATIVE_F_DQ_BW_VL_OK"); + return 0; +} +` diff --git a/amd64_packed_conversion_raw_test.go b/amd64_packed_conversion_raw_test.go new file mode 100644 index 000000000..1aae6d7a4 --- /dev/null +++ b/amd64_packed_conversion_raw_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Assemble a bounded batch once and partition every byte by real instruction +// boundaries. Runtime raw cases do not use our mnemonic-to-opcode tables. +func x86PackedConversionOriginalRawRows(t *testing.T, arch string, rows []string) []string { + t.Helper() + code := x86PackedShiftActualGoBytes(t, arch, "TEXT raw(SB),4,$0-0\n"+strings.Join(rows, "\n")+"\n") + mode := 64 + if arch == "386" { + mode = 32 + } + var raw []string + position := 0 + for _, row := range rows { + _, length, known, err := decodedX86PackedNumericConversionInstruction(code[position:], mode) + if !known { + instruction, decodeErr := x86asm.Decode(code[position:], mode) + length, err = instruction.Len, decodeErr + } + if err != nil || length < 1 || length > len(code)-position { + t.Fatal("original conversion boundary", row, err) + } + var directives strings.Builder + for _, value := range code[position : position+length] { + fmt.Fprintf(&directives, "BYTE $%d\n", value) + } + raw = append(raw, strings.TrimSuffix(directives.String(), "\n")) + position += length + } + if position != len(code) { + t.Fatal("original conversion batch has unconsumed bytes") + } + return raw +} + +func TestX86PackedConversionRawCompleteOriginalGoForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + rows := x86PackedConversionOracleRows(arch) + var source strings.Builder + source.WriteString("TEXT raw(SB),4,$0-0\n") + for _, row := range rows { + source.WriteString(row + "\n") + } + source.WriteString("RET\n") + code := x86PackedShiftActualGoBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(arch, err) + } + if len(decoded.Instrs) != len(rows)+1 { + t.Fatal("whole original Go byte stream lost instructions", arch, len(decoded.Instrs), len(rows)+1) + } + for index, row := range rows { + want, got := x86ScalarLaneInstruction(t, row), decoded.Instrs[index] + if want.Op != got.Op || len(want.Args) != len(got.Args) { + t.Fatal("raw opcode/mask mismatch", arch, row, got) + } + for operand := range want.Args { + if want.Args[operand].String() != got.Args[operand].String() { + t.Fatal("raw physical operand mismatch", arch, row, got) + } + } + } + t.Logf("%s: %d actual-Go forms, complete raw byte/typed-operand agreement", arch, len(rows)) + } +} + +func TestX86PackedConversionRawPrefixAndTruncationBoundaries(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + if spec.legacy { + continue + } + for _, shape := range spec.shapes() { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + row := spec.name + ".Z 64(AX)(CX*4),K1," + bank[shape.destination] + "7" + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT raw(SB),4,$0-0\n"+row+"\n") + ins, length, known, err := decodedX86PackedNumericConversionInstruction(code, 64) + if err != nil || !known || length != len(code) || ins.Op != Op(spec.name+".Z") { + t.Fatal(row, fmt.Sprintf("%x", code), ins, length, err) + } + for end := 0; end < len(code); end++ { + if _, _, known, err := decodedX86PackedNumericConversionInstruction(code[:end], 64); known && err == nil { + t.Fatal("truncated instruction accepted", row, end) + } + } + for _, mutation := range []struct { + index int + xor byte + }{{1, 0x08}, {2, 0x04}, {2, 0x08}, {3, 0x08}, {3, 0x01}} { + bad := append([]byte(nil), code...) + bad[mutation.index] ^= mutation.xor + if _, _, known, err := decodedX86PackedNumericConversionInstruction(bad, 64); known && err == nil { + t.Fatal("invalid fixed/vvvv/zero field accepted", row, fmt.Sprintf("%x", bad)) + } + } + // Exact high source/destination registers are unavailable in mode32; + // a legal K1 mask by itself is not a high-register binding. + if _, _, known, err := decodedX86PackedNumericConversionInstruction(code, 32); !known || err != nil { + t.Fatal("legal mode32 masked conversion rejected", row, err) + } + } + } + for _, encoding := range []struct { + pp, opcode byte + name string + }{ + {1, 0x5b, "VCVTPS2DQ"}, {2, 0x5b, "VCVTTPS2DQ"}, + {3, 0xe6, "VCVTPD2DQX"}, {1, 0xe6, "VCVTTPD2DQX"}, + } { + for _, mode := range []int{32, 64} { + for _, w := range []byte{0, 0x80} { + code := []byte{0xc4, 0xe1, 0x78 | encoding.pp | w, encoding.opcode, 0xc1} + ins, n, known, err := decodedX86PackedNumericConversionInstruction(code, mode) + if err != nil || !known || n != len(code) || string(ins.Op) != encoding.name { + t.Fatal("Intel VEX.WIG must not become W0-only", mode, fmt.Sprintf("%x", code), ins, err) + } + } + } + } +} diff --git a/amd64_packed_conversion_runtime_linux_amd64_test.go b/amd64_packed_conversion_runtime_linux_amd64_test.go new file mode 100644 index 000000000..1962c724b --- /dev/null +++ b/amd64_packed_conversion_runtime_linux_amd64_test.go @@ -0,0 +1,75 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import "testing" + +// This required owner is separate from Rosetta SSE and object-only evidence. +// CPUID/OS-state and original-opcode preflight must execute successfully. +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512Status(t *testing.T) { + x86PackedConversionCompleteNativeValues(t, 0) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeIndependentC(t *testing.T) { + amd64PackedNumericConversionNativeRuntimeSemantics(t) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaults(t *testing.T) { + x86PackedConversionCompleteNativeFaults(t, 0) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512StatusPart1(t *testing.T) { + x86PackedConversionCompleteNativeValues(t, 1) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512StatusPart2(t *testing.T) { + x86PackedConversionCompleteNativeValues(t, 2) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512StatusPart3(t *testing.T) { + x86PackedConversionCompleteNativeValues(t, 3) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512StatusPart4(t *testing.T) { + x86PackedConversionCompleteNativeValues(t, 4) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512StatusPart5(t *testing.T) { + x86PackedConversionCompleteNativeValues(t, 5) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512StatusPart6(t *testing.T) { + x86PackedConversionCompleteNativeValues(t, 6) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512StatusPart7(t *testing.T) { + x86PackedConversionCompleteNativeValues(t, 7) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaultsPart1(t *testing.T) { + x86PackedConversionCompleteNativeFaults(t, 1) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaultsPart2(t *testing.T) { + x86PackedConversionCompleteNativeFaults(t, 2) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaultsPart3(t *testing.T) { + x86PackedConversionCompleteNativeFaults(t, 3) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaultsPart4(t *testing.T) { + x86PackedConversionCompleteNativeFaults(t, 4) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaultsPart5(t *testing.T) { + x86PackedConversionCompleteNativeFaults(t, 5) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaultsPart6(t *testing.T) { + x86PackedConversionCompleteNativeFaults(t, 6) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaultsPart7(t *testing.T) { + x86PackedConversionCompleteNativeFaults(t, 7) +} diff --git a/amd64_packed_conversion_runtime_selection_test.go b/amd64_packed_conversion_runtime_selection_test.go new file mode 100644 index 000000000..a24fdd9f6 --- /dev/null +++ b/amd64_packed_conversion_runtime_selection_test.go @@ -0,0 +1,17 @@ +package plan9asm + +import ( + "os" + "strings" + "testing" +) + +func TestX86PackedConversionRequiredNativeOwners(t *testing.T) { + // The complete policy binds all 17 unconditional owners to the explicit + // native tag. Ordinary conversion counterparts retain the default CI selector. + TestX86PackedConversionCompleteNativeRuntimeOwnership(t) + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil || !strings.Contains(string(workflow), "-ci-selector '^(TestCrossLinuxRuntimeMatrix|") { + t.Fatal("ordinary conversion runtime counterparts not selected by CI", err) + } +} diff --git a/amd64_packed_conversion_runtime_test.go b/amd64_packed_conversion_runtime_test.go new file mode 100644 index 000000000..8b02d6e9a --- /dev/null +++ b/amd64_packed_conversion_runtime_test.go @@ -0,0 +1,267 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86PackedConversionRuntimeRow struct { + instruction string + rawOverride string + wide, dead bool + inputBits int + mask bool + sae bool +} + +func x86PackedConversionRuntimeRows(wide bool) []x86PackedConversionRuntimeRow { + var rows []x86PackedConversionRuntimeRow + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + for _, spec := range x86PackedConversionOracleSpecs() { + if !wide && !spec.legacy { + continue + } + for _, shape := range spec.shapes() { + destination := bank[shape.destination] + "1" + for _, input := range []string{bank[shape.source] + "0", "(SI)"} { + rows = append(rows, x86PackedConversionRuntimeRow{instruction: spec.name + " " + input + "," + destination, wide: wide, inputBits: spec.input}) + } + rows = append(rows, x86PackedConversionRuntimeRow{instruction: spec.name + " " + bank[shape.source] + "1," + destination, wide: wide, inputBits: spec.input}, + x86PackedConversionRuntimeRow{instruction: spec.name + " " + bank[shape.source] + "0," + destination, wide: wide, dead: true, inputBits: spec.input}) + if !spec.legacy { + for _, spelling := range []string{ + spec.name + " " + bank[shape.source] + "0,K1," + destination, + spec.name + ".Z (SI),K1," + destination, + spec.name + ".BCST (SI),K1," + destination, + spec.name + ".BCST.Z (SI),K1," + destination, + } { + rows = append(rows, x86PackedConversionRuntimeRow{instruction: spelling, wide: wide, mask: true, inputBits: spec.input}) + } + } + if (shape.source == 64 || shape.destination == 64) && spec.control != "" { + controls := []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} + if spec.control == "sae" { + controls = []string{"SAE"} + } + for _, control := range controls { + rows = append(rows, x86PackedConversionRuntimeRow{ + instruction: spec.name + "." + control + ".Z " + bank[shape.source] + "0,K1," + destination, + wide: wide, mask: true, sae: true, inputBits: spec.input, + }) + } + } + } + } + return rows +} + +func x86PackedConversionRuntimeFixture(rows []x86PackedConversionRuntimeRow) (assembly, declarations, original, driver string) { + var asm, decl, goCalls, cDecl, cCalls strings.Builder + for index, row := range rows { + name := fmt.Sprintf("packed%d", index) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$8-24\nSTMXCSR saved-4(SP)\nMOVQ out+0(FP),DI\nMOVQ in+8(FP),SI\n", name) + if row.wide { + asm.WriteString("VMOVDQU64 (SI),Z0\nVMOVDQU64 (DI),Z1\nMOVL mask+20(FP),CX\nKMOVW CX,K1\n") + } else { + asm.WriteString("MOVOU (SI),X0\nMOVOU (DI),X1\n") + } + asm.WriteString("LDMXCSR csr+16(FP)\nMOVQ $-1,DX\nCMPQ DX,$-1\n") + asm.WriteString(row.instruction + "\n") + if !row.dead { + if row.wide { + asm.WriteString("VMOVDQU64 Z1,(DI)\n") + } else { + asm.WriteString("MOVOU X1,(DI)\n") + } + } + asm.WriteString("STMXCSR 72(DI)\nSETEQ 64(DI)\nSETCS 65(DI)\nSETOS 66(DI)\nSETMI 67(DI)\nSETPC 68(DI)\nLDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&decl, "func %s(out *[12]uint64,in *[8]uint64,csr,mask uint32)\n", name) + masks, sae := 1, 0 + if row.mask { + masks = 4 + } + if row.sae { + sae = 1 + } + fmt.Fprintf(&goCalls, "check(%d,%d,%d,%d,%s)\n", index, row.inputBits, masks, sae, name) + fmt.Fprintf(&cDecl, "extern void %s(uint64_t*,const uint64_t*,uint32_t,uint32_t);\n", name) + fmt.Fprintf(&cCalls, "check(%d,%d,%d,%d,%s);\n", index, row.inputBits, masks, sae, name) + } + var gValues, cValues strings.Builder + values := append(append([]uint64{}, x86ScalarFloatIntegerSSInputs...), x86ScalarFloatIntegerSDInputs...) + values = append(values, 0xffffff, 0x1000001, 0xffffffff, 0x1fffffffffffff, 0x20000000000001, 0x7fffffffffffffff, 0xffffffffffffffff) + for _, value := range values { + fmt.Fprintf(&gValues, "%#x,", value) + fmt.Fprintf(&cValues, "UINT64_C(%#x),", value) + } + original = `package main +import ("fmt";"unsafe") +var values=[]uint64{` + gValues.String() + `} +var masks=[]uint32{0xffff,0,1,0xaaaa} +func check(index,bits,maskCount,sae int,fn func(*[12]uint64,*[8]uint64,uint32,uint32)) { + hash:=uint64(1469598103934665603); count:=0 + for n:=range values { for rc:=0;rc<4;rc++ { for controls:=0;controls<8;controls++ { for m:=0;m>shift)))*1099511628211} } + count++ + }}}} + fmt.Printf("%d %d %016x\n",index,count,hash) +} +` + decl.String() + "func main(){\n" + goCalls.String() + "}\n" + driver = `#include +#include +static const uint64_t values[]={` + cValues.String() + `}; +static const uint32_t masks[]={0xffff,0,1,0xaaaa}; +static void check(unsigned index,unsigned bits,unsigned maskCount,unsigned sae,void (*fn)(uint64_t*,const uint64_t*,uint32_t,uint32_t)) { + uint64_t hash=UINT64_C(1469598103934665603);unsigned count=0,N=sizeof(values)/sizeof(*values); + for(unsigned n=0;n>shift))*UINT64_C(1099511628211); + count++; + } + printf("%u %u %016llx\n",index,count,(unsigned long long)hash); +} +` + cDecl.String() + "int main(void){\n" + cCalls.String() + "return 0;}\n" + return asm.String(), decl.String(), original, driver +} + +func x86PackedConversionRuntimeMatrix(t *testing.T, wide bool) { + t.Helper() + axis := x86FloatCompareSSE + if wide { + axis = x86FloatCompareEVEX + } + execution := x86FloatCompareRuntimeTools(t, axis) + rows := x86PackedConversionRuntimeRows(wide) + const limit = 96 + consumed := 0 + for start := 0; start < len(rows); start += limit { + end := start + limit + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + assembly, declarations, main, driver := x86PackedConversionRuntimeFixture(rows[start:end]) + want := x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, main) + if strings.Count(string(want), "\n") != end-start { + t.Fatal("incomplete original Go function/case receipts") + } + var originals []string + for _, row := range rows[start:end] { + originals = append(originals, row.instruction) + } + raw := x86PackedConversionOriginalRawRows(t, "amd64", originals) + rawRows := append([]x86PackedConversionRuntimeRow(nil), rows[start:end]...) + for index := range rawRows { + rawRows[index].instruction = raw[index] + } + rawAssembly, _, _, _ := x86PackedConversionRuntimeFixture(rawRows) + for _, spelling := range []struct{ name, source string }{{"named", assembly}, {"raw", rawAssembly}} { + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, spelling.source, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal(start, api, err) + } + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, ir) + } + for _, level := range []string{"-O0", "-O2"} { + got := execution.compileAndRunLLVM(t, "packed-conversion-matrix", input, driver, level) + if string(got) != string(want) { + t.Fatalf("batch %d %s/%s optimized=%v %s\nGo:\n%sLLVM:\n%s", start, spelling.name, api, optimized, level, want, got) + } + } + } + } + } + consumed += end - start + }) + } + if consumed != len(rows) { + t.Fatal("runtime batch partition lost rows") + } + t.Logf("%d independent conversion kernels, 63 bit-pattern seeds x 4 rounding x 4 DAZ/FTZ x 2 clean/sticky status; masked kernels also 4 masks; named/raw x 3 APIs x 4 optimization axes", len(rows)) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionLegacyIEEEStatusFlags(t *testing.T) { + x86PackedConversionRuntimeMatrix(t, false) +} + +// Object receipts for the actual wide runtime fixture are not execution +// receipts. They close its source/API/lifecycle path before a native owner +// attempts the same kernels, without executing unsupported host instructions. +func TestX86PackedConversionNativeRuntimeFixtureObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + rows := x86PackedConversionRuntimeRows(true) + const limit = 96 + consumed := 0 + for start := 0; start < len(rows); start += limit { + end := start + limit + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + batch := append([]x86PackedConversionRuntimeRow(nil), rows[start:end]...) + var instructions []string + for _, row := range batch { + instructions = append(instructions, row.instruction) + } + raw := x86PackedConversionOriginalRawRows(t, "amd64", instructions) + for _, spelling := range []string{"named", "raw"} { + if spelling == "raw" { + for index := range batch { + batch[index].instruction = raw[index] + } + } + source, declarations, _, _ := x86PackedConversionRuntimeFixture(batch) + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal(spelling, api, err) + } + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, ir) + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "native-runtime.ll", "native-runtime.o", input) + } + } + } + consumed += end - start + }) + } + if consumed != len(rows) { + t.Fatal("native runtime fixture partition lost rows") + } + t.Logf("%d complete wide runtime kernels, named/raw x 3 APIs x 2 IR pipelines; object-only", consumed) +} diff --git a/amd64_packed_conversion_status_runtime_test.go b/amd64_packed_conversion_status_runtime_test.go new file mode 100644 index 000000000..eff5f8183 --- /dev/null +++ b/amd64_packed_conversion_status_runtime_test.go @@ -0,0 +1,108 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The four legacy Yxm/Yxr rows in Go's yxcvm1 table are an independently +// executable status witness for packed float-to-dword conversion. MMX rows +// and AVX/EVEX forms need their own physical-state execution owners. +func x86PackedConversionConstantStatusOracle() (assembly, declarations, original, driver string) { + var asm, decl, goCalls, cDecl, cCalls strings.Builder + index := 0 + for _, name := range []string{"CVTPS2PL", "CVTTPS2PL", "CVTPD2PL", "CVTTPD2PL"} { + values := []uint64{0x7fc12345, 0x7f800000, 0xff800000, 0x4f000000, 0x3fc00000, 1} + if strings.Contains(name, "PD") { + values = []uint64{0x7ff8123456789abc, 0x7ff0000000000000, 0xfff0000000000000, 0x41e0000000000000, 0x3ff8000000000000, 1} + } + for _, bits := range values { + if strings.Contains(name, "PS") { + bits |= bits << 32 + } + for _, live := range []bool{false, true} { + function := fmt.Sprintf("packedProbe%d", index) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$8-12\n", function) + asm.WriteString("STMXCSR saved-4(SP)\nMOVQ out+0(FP),DI\n") + fmt.Fprintf(&asm, "MOVQ $%#x,AX\nMOVQ AX,X0\nPUNPCKLQDQ X0,X0\n", bits) + fmt.Fprintf(&asm, "LDMXCSR csr+8(FP)\n%s X0,X1\n", name) + if live { + asm.WriteString("MOVOU X1,(DI)\n") + } + asm.WriteString("STMXCSR 16(DI)\nLDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&decl, "func %s(out *[4]uint64, csr uint32)\n", function) + fmt.Fprintf(&goCalls, "check(%d, %s)\n", index, function) + fmt.Fprintf(&cDecl, "extern void %s(uint64_t*, uint32_t);\n", function) + fmt.Fprintf(&cCalls, "check(%d, %s);\n", index, function) + index++ + } + } + } + original = `package main +import "fmt" +func check(index int, fn func(*[4]uint64, uint32)) { + for mode := 0; mode < 4; mode++ { + out := [4]uint64{0x1111111111111111, 0x2222222222222222, 0x3333333333333333, 0x4444444444444444} + fn(&out, 0x1f80|uint32(mode)<<13) + fmt.Printf("%d %d %016x %016x %016x %016x\n", index, mode, out[0], out[1], out[2], out[3]) + } +} +` + decl.String() + "func main() {\n" + goCalls.String() + "}\n" + driver = `#include +#include +static void check(int index, void (*fn)(uint64_t*, uint32_t)) { + for (unsigned mode = 0; mode < 4; mode++) { + uint64_t out[] = {UINT64_C(0x1111111111111111), UINT64_C(0x2222222222222222), UINT64_C(0x3333333333333333), UINT64_C(0x4444444444444444)}; + fn(out, 0x1f80 | (mode << 13)); + printf("%d %u %016llx %016llx %016llx %016llx\n", index, mode, + (unsigned long long)out[0], (unsigned long long)out[1], + (unsigned long long)out[2], (unsigned long long)out[3]); + } +} +` + cDecl.String() + "int main(void) {\n" + cCalls.String() + "return 0;\n}\n" + return asm.String(), decl.String(), original, driver +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionOriginalConstantsAndStatus(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + assembly, declarations, main, driver := x86PackedConversionConstantStatusOracle() + original := x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, main) + want := strings.Split(strings.TrimSpace(string(original)), "\n") + if len(want) != 192 { + t.Fatalf("original Go produced %d rows, want 4 instructions x 6 inputs x 2 live/dead x 4 MXCSR modes", len(want)) + } + ir, err := x86ScalarFPBitViewAPI(t, assembly, declarations, "amd64", execution.triple, "text") + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, ir) + } + for _, backend := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR_O2=%v/backend%s", optimized, backend), func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "packed-conversion-status", input, driver, backend) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM row count=%d original=%d", len(got), len(want)) + } + mismatches := 0 + for index := range want { + if got[index] != want[index] { + if mismatches == 0 { + t.Logf("first mismatch: original %s; LLVM %s", want[index], got[index]) + } + mismatches++ + } + } + if mismatches != 0 { + t.Errorf("original Go packed value/MXCSR/canary mismatches=%d/%d", mismatches, len(want)) + } + }) + } + } +} diff --git a/amd64_packed_dot_forms.go b/amd64_packed_dot_forms.go new file mode 100644 index 000000000..89167bd4f --- /dev/null +++ b/amd64_packed_dot_forms.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedDotForm struct { + x86PackedBinaryForm + spec amd64PackedDotProductSpec +} + +// The four integer operations share Go's complete _yvblendmpd grammar. +// Raw VEX is a distinct register/full-memory X/Y shape, not a named Go row; +// BF16 never inherits this integer grammar or its numerical continuation. +func x86PackedDotProperties(op Op, ins Instr, arch string) (x86PackedDotForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64PackedDotProductSpecs[Op(base)] + if !known || spec.bfloat { + return x86PackedDotForm{}, false, nil + } + result := x86PackedDotForm{spec: spec} + if ins.Op != "" && !strings.EqualFold(string(op), string(ins.Op)) { + return result, true, fmt.Errorf("integer dot-product dispatch/source opcode mismatch") + } + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, broadcastBytes: 4, zeroing: true} + if ins.x86PackedDotEncoding == x86RawPackedDotVEX { + grammar = x86PackedBinarySpec{encoding: x86PackedLogicalVEX} + } + local := ins + local.Op = op + var err error + result.x86PackedBinaryForm, err = x86PackedBinaryOperandGrammar(local, arch, grammar, ins.x86Encoded) + if err != nil || result.memory < 0 { + return result, true, err + } + source := ins.Args[result.memory] + if !x86PackedShiftMemoryValid(source, arch) || source.Kind == OpMem && + (!x86ConditionalMoveOffsetKnown(source.Mem) || source.Mem.IndexExt != "" || + source.Mem.Segment != "" && source.Mem.Segment != FS && source.Mem.Segment != GS) { + return result, true, fmt.Errorf("%s has an unresolved or foreign memory source", op) + } + if source.Kind == OpSym { + if address, parseErr := parseInt(strings.TrimSpace(source.Sym)); parseErr == nil && + address != int64(int32(address)) && !(arch == "386" && address == int64(uint32(address))) { + err = fmt.Errorf("%s absolute source exceeds its disp32 extent", op) + } + } else if source.Kind == OpMem { + err = x86SymbolIndexedMemoryError(source.Mem, arch) + } + if err == nil && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: %s needs its original default address mode", ErrProbeNeedsContext, op) + } + return result, true, err +} + +func x86TypedPackedDotByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedDotProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width, firstWidth := int64(form.byteWidth), int64(form.byteWidth) + if form.memory >= 0 { + firstWidth = int64(form.memoryBytes) + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: firstWidth, read: true}, + {index: form.second, operand: ins.Args[form.second], width: width, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: width, read: true, write: true}, + }} + if form.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return effect, true, nil +} + +// All three inputs contribute the corresponding dword, including the old +// destination with zero masking. Neither a mask nor an implementation zero +// initializes an arithmetic input, including dead results in Void functions. +func (form x86PackedDotForm) inputsDefined(first, second, old uint64) bool { + output := x86CallerVectorByteMask(int64(form.byteWidth)) + input := output + if form.broadcast { + input = 15 + } + return first&input == input && second&output == output && old&output == output +} diff --git a/amd64_packed_double_dword_native_linux_amd64_test.go b/amd64_packed_double_dword_native_linux_amd64_test.go new file mode 100644 index 000000000..bd5ad5e7b --- /dev/null +++ b/amd64_packed_double_dword_native_linux_amd64_test.go @@ -0,0 +1,11 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixAMD64PackedDoubleDwordNativeAVX512(t *testing.T) { + execution := x86PackedDoubleDwordNativeTools(t) + x86PackedDoubleDwordOriginalGo(t, execution) + amd64PackedDoubleDwordRuntime(t, execution, true) +} diff --git a/amd64_packed_double_dword_native_test.go b/amd64_packed_double_dword_native_test.go new file mode 100644 index 000000000..7ecc9a584 --- /dev/null +++ b/amd64_packed_double_dword_native_test.go @@ -0,0 +1,236 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func x86PackedDoubleDwordGoAssemblerSource(source string) string { + // Preserve the original NOSPLIT spelling and let this actual driver's + // original textflag.h define it, rather than silently rewriting TEXT. + return fmt.Sprintf("#include %q\n", filepath.Join(runtime.GOROOT(), "pkg", "include", "textflag.h")) + source +} + +const x86PackedDoubleDwordMaskPreflightC = ` +#undef main +__attribute__((target("avx512f,avx512bw"))) static int original_mask64(void) { + uint64_t input=0xa5,output; + __asm__ volatile("kmovq %1,%%k1; kmovq %%k1,%0" : "=r"(output) : "r"(input) : "k1"); + return output==input; +} +static int mask64_required_state(unsigned bw) { return bw!=0; } +int main(void) { + int failure=compare_preflight_main(); + if (failure) return failure; + unsigned a,b,c,d,bw=0; + if (__get_cpuid_count(7,0,&a,&b,&c,&d)) bw=(b>>30)&1; + printf("original double/dword mask transport AVX512BW=%u\n",bw); + fflush(stdout); + if (!mask64_required_state(bw)) return 83; + return original_mask64() ? 0 : 84; +} +` + +func x86PackedDoubleDwordNativeTools(t *testing.T) x86FloatCompareExecution { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + dir := t.TempDir() + source, binary := filepath.Join(dir, "mask-preflight.c"), filepath.Join(dir, "mask-preflight") + program := "#define REQUIRED_AXIS 3\n#define main compare_preflight_main\n" + x86FloatCompareOpcodePreflightC + x86PackedDoubleDwordMaskPreflightC + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, binary) + t.Logf("actual source KMOVQ preflight: %s", execution.run(t, binary)) + return execution +} + +func TestX86PackedDoubleDwordNativeMaskPreflight(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, binary := filepath.Join(dir, "mask-capabilities.c"), filepath.Join(dir, "mask-capabilities") + program := "#define REQUIRED_AXIS 3\n#define main compare_preflight_main\n" + x86FloatCompareOpcodePreflightC + + strings.Replace(x86PackedDoubleDwordMaskPreflightC, "int main(void)", "int actual_mask_main(void)", 1) + ` +int main(void) { return !mask64_required_state(1) || mask64_required_state(0); } +` + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, binary) + execution.run(t, binary) +} + +func TestX86PackedDoubleDwordRuntimeRequiresNativeUnion(t *testing.T) { + full, local := x86PackedDoubleDwordOriginalAssembly(), x86PackedDoubleDwordSSEAssembly(t) + for _, opcode := range []string{"VCVTDQ2PD (CX), K1, Z1", "VCVTTPD2DQY (BX), X2", "VCVTPD2DQ.RD_SAE.Z Z0, K1, Y3", "CVTPD2PL X4, X0"} { + if strings.Count(full, opcode) != 1 { + t.Fatal("lost or repeated original conversion", opcode) + } + } + parsed, err := Parse(ArchAMD64, local) + if err != nil { + t.Fatal(err) + } + for _, instruction := range parsed.Funcs[0].Instrs { + if strings.HasPrefix(string(instruction.Op), "V") || strings.HasPrefix(string(instruction.Op), "KMOV") { + t.Fatal("portable SSE counterpart borrowed a wide opcode", instruction.Op) + } + } + if strings.Count(local, "CVTPD2PL X4, X0") != 1 { + t.Fatal("portable SSE counterpart borrowed a wide ISA or lost its original legacy conversion") + } + file, err := parser.ParseFile(token.NewFileSet(), "amd64_packed_double_dword_native_linux_amd64_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + var body *ast.BlockStmt + for _, declaration := range file.Decls { + if fn, ok := declaration.(*ast.FuncDecl); ok && fn.Name.Name == "TestCrossLinuxRuntimeMatrixAMD64PackedDoubleDwordNativeAVX512" { + body = fn.Body + } + } + if body == nil || len(body.List) != 3 { + t.Fatal("complete original Go and named/raw LLVM runtime must consume native transport") + } + for index, name := range []string{"x86PackedDoubleDwordNativeTools", "x86PackedDoubleDwordOriginalGo", "amd64PackedDoubleDwordRuntime"} { + var call *ast.CallExpr + if index == 0 { + if assignment, ok := body.List[index].(*ast.AssignStmt); ok && len(assignment.Rhs) == 1 { + call, _ = assignment.Rhs[0].(*ast.CallExpr) + } + } else if statement, ok := body.List[index].(*ast.ExprStmt); ok { + call, _ = statement.X.(*ast.CallExpr) + } + if call == nil { + t.Fatal("required conversion consumer cannot skip or return", name) + } + callee, ok := call.Fun.(*ast.Ident) + if !ok || callee.Name != name { + t.Fatal("required conversion consumer missing", name) + } + if index > 0 { + route, ok := call.Args[1].(*ast.Ident) + if !ok || route.Name != "execution" { + t.Fatal("conversion consumer lost its exact preflight route") + } + } + if index == 2 { + if len(call.Args) != 3 { + t.Fatal("complete wide conversion execution was omitted") + } + wide, ok := call.Args[2].(*ast.Ident) + if !ok || wide.Name != "true" { + t.Fatal("complete wide conversion assertions cannot be disabled") + } + } + } +} + +func x86PackedDoubleDwordOriginalGo(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + if execution.triple != "x86_64-unknown-linux-gnu" || len(execution.prefix) != 0 || execution.preserveArgv0 { + t.Fatal("original packed double/dword requires actual native Linux AMD64") + } + dir := x86PackedDoubleDwordOriginalPackage(t) + command := exec.Command("go", "test", "-p=2", "-count=1", "-run", "^TestOriginalDoubleDword$", "-v", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + t.Logf("actual original Go named/raw double/dword: %s", x86FloatCompareCaptureCommand(t, command)) +} + +func TestX86PackedDoubleDwordCompleteOriginalGoCompiles(t *testing.T) { + dir := x86PackedDoubleDwordOriginalPackage(t) + command := exec.Command("go", "test", "-p=2", "-c", "-o", filepath.Join(dir, "original"), ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + x86FloatCompareCaptureCommand(t, command) +} + +func x86PackedDoubleDwordOriginalPackage(t *testing.T) string { + t.Helper() + dir := t.TempDir() + named := strings.Replace(x86PackedDoubleDwordOriginalAssembly(), "packeddoubledword(SB)", "·named(SB)", 1) + raw := strings.Replace(x86PackedDoubleDwordRawAssembly(x86PackedDoubleDwordOriginalAssembly()), "packeddoubledword(SB)", "·raw(SB)", 1) + assembly := "#include \"textflag.h\"\n" + named + raw + "TEXT ·setCSR(SB),4,$0-4\nLDMXCSR csr+0(FP)\nRET\nTEXT ·getCSR(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\n" + for name, contents := range map[string]string{ + "go.mod": "module originaldoubledword\n\ngo 1.20\n", "original_amd64.s": assembly, + "original_test.go": x86PackedDoubleDwordOriginalGoTest, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + return dir +} + +// This is output-only numerical evidence, not an MMX/x87 TAG/TOS identity +// claim. The original fixture's MM0 conversion remains unchanged. +const x86PackedDoubleDwordOriginalGoTest = `package original + +import ( + "encoding/binary" + "math" + "runtime" + "testing" +) + +func named(out *[120]byte, doubles *[8]float64, dwords *[8]int32, old *[8]float64, mask uint64) +func raw(out *[120]byte, doubles *[8]float64, dwords *[8]int32, old *[8]float64, mask uint64) +func setCSR(csr uint32) +func getCSR() uint32 + +func TestOriginalDoubleDword(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + defer setCSR(getCSR()) + + doubles := [8]float64{ + 1.75, -2.75, 2147483648.0, math.Float64frombits(0x7ff8000000000000), + -2147483648.0, -2147483649.0, 3.0, math.Float64frombits(0x8000000000000000), + } + dwords := [8]int32{0, 1, -1, 2147483647, -2147483648, 123456789, -987654321, 42} + old := [8]float64{100.5, 101.5, 102.5, 103.5, 104.5, 105.5, 106.5, 107.5} + truncated := [4]int32{1, -2, -2147483648, -2147483648} + down := [8]int32{1, 0, -2147483648, 0, 0, -2147483648, 0, 0} + + for index, fn := range []func(*[120]byte, *[8]float64, *[8]int32, *[8]float64, uint64){named, raw} { + var guarded [122]byte + for i := range guarded { + guarded[i] = 0xcc + } + setCSR(0x1f80) + fn((*[120]byte)(guarded[1:121]), &doubles, &dwords, &old, 0xa5) + output := guarded[1:121] + if guarded[0] != 0xcc || guarded[121] != 0xcc { + t.Fatal("original conversion output canary", index) + } + + for i := 0; i < 8; i++ { + want := old[i] + if 0xa5&(1< 5 || arch == "386" && vector { + return f, true, fmt.Errorf("%s %s has an unbound dispatch or Go operand arity", arch, op) + } + immediate, integer := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !integer || !amd64UnsignedImmediate(immediate, 8) { + return f, true, fmt.Errorf("%s %s requires Go's resolved Yu8 immediate", arch, op) + } + spec := x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP} + if vector { + spec = x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: bits / 8, zeroing: true} + } + local := ins + local.Op, local.Args = op, ins.Args[1:] + var err error + f.x86PackedBinaryForm, err = x86ConflictFunnelOperandGrammar(local, arch, spec, false) + f.first++ + f.second++ + f.destination++ + if f.memory >= 0 { + f.memory++ + } + if f.mask >= 0 { + f.mask++ + } + f.selector = uint8(immediate.Imm) + return f, true, err +} + +// Index the concatenation of Go's first and second sources. PS selects its +// low pair from the second source; PD alternates second/first within each +// 128-bit lane, using a distinct immediate bit for every output qword. +func (f x86PackedFloatShuffleForm) sourceByte(output int) int { + bytes := f.bits / 8 + lane := output / bytes + first := lane%4 >= 2 + selected := int(f.selector>>uint(2*(lane%4))) & 3 + if f.bits == 64 { + first = lane%2 != 0 + selected = int(f.selector>>uint(lane)) & 1 + } + at := output/16*16 + selected*bytes + output%bytes + if first { + if f.broadcast { + at %= bytes + } + return at + } + return f.byteWidth + at +} + +func (f x86PackedFloatShuffleForm) definedBytes(first, second, old uint64) uint64 { + var defined uint64 + for output := 0; output < f.byteWidth; output++ { + at := f.sourceByte(output) + input := first + if at >= f.byteWidth { + input, at = second, at-f.byteWidth + } + known := input>>uint(at)&1 != 0 + if f.mask >= 0 && !f.zeroing { + known = known && old>>uint(output)&1 != 0 + } + if known { + defined |= uint64(1) << uint(output) + } + } + return defined +} + +func x86TypedPackedFloatShuffleByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + f, known, err := x86PackedFloatShuffleProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width, firstWidth := int64(f.byteWidth), int64(f.byteWidth) + if f.memory >= 0 { + firstWidth = int64(f.memoryBytes) + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: f.first, operand: ins.Args[f.first], width: firstWidth, read: true}, + {index: f.second, operand: ins.Args[f.second], width: width, read: true}, + }} + if f.legacy { + effect.operands[1].write = true + } else { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: f.destination, operand: ins.Args[f.destination], width: width, write: true, + read: f.mask >= 0 && !f.zeroing, + }) + } + if f.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: f.mask, operand: ins.Args[f.mask], width: 8, read: true}) + } + return effect, true, nil +} diff --git a/amd64_packed_function_lifecycle_test.go b/amd64_packed_function_lifecycle_test.go new file mode 100644 index 000000000..073c46adf --- /dev/null +++ b/amd64_packed_function_lifecycle_test.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +// These expectations describe the emitted native memory carrier, not the +// existence of an opcode in the implementation's family tables. +func TestX86PackedFunctionFeatureViews(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + instruction string + features string + }{ + {"PAVGB (AX),X1", "+sse2"}, + {"PAVGW (AX),X1", "+sse2"}, + {"PMULLW (AX),X1", "+sse2"}, + {"PMULHW (AX),X1", "+sse2"}, + {"PMULHUW (AX),X1", "+sse2"}, + {"PMULHRSW (AX),X1", "+sse2,+ssse3"}, + {"PMADDWL (AX),X1", "+sse2"}, + {"PMADDUBSW (AX),X1", "+sse2,+ssse3"}, + {"PMULULQ (AX),X1", "+sse2"}, + {"PMULDQ (AX),X1", "+sse2,+sse4.1"}, + } { + t.Run(arch+"/"+row.instruction, func(t *testing.T) { + function := Func{Instrs: []Instr{x86ScalarLaneInstruction(t, row.instruction)}} + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, function); got != row.features { + t.Fatalf("native memory features = %q, want %q", got, row.features) + } + // A new function representation must not reuse a preceding + // source's physical memory form, even when its slice is reused. + function.Instrs[0].Args[0] = Operand{Kind: OpFP, FPOffset: 0} + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, function); got != "" { + t.Fatalf("logical FP inherited native memory features: %q", got) + } + }) + } + } +} + +func TestX86PackedFunctionRegisterViews(t *testing.T) { + for _, row := range []struct { + source, physical Reg + }{ + {AX, "M0"}, + {SP, "M4"}, + } { + function := Func{Instrs: []Instr{ + {Op: "PAVGB", Args: []Operand{{Kind: OpReg, Reg: row.source}, {Kind: OpReg, Reg: "M1"}}}, + }} + for _, arch := range []string{"amd64", "386"} { + consumer := newX86Ctx(new(strings.Builder), function, FuncSig{}, nil, nil, arch, "", false) + consumer.scanUsedRegs() + if consumer.usedRegs[row.physical] != (arch == "amd64") { + t.Fatalf("%s %s: physical bank discovery = %v", arch, row.source, consumer.usedRegs) + } + if function.Instrs[0].Args[0].Reg != row.source { + t.Fatal("register discovery mutated the original GP spelling") + } + } + } +} + +func TestX86PackedFunctionLoweringAcrossLabels(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := "TEXT binary(SB),4,$0-0\n" + + "PXOR X0,X0\nPXOR X1,X1\n" + + "first:\nPAVGB X0,X1\nPMULHW X0,X1\nJMP second\n" + + "second:\nPMADDWL X0,X1\nPMULULQ X0,X1\nPAVGB X0,X1\nRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + options := Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"binary": {Name: "binary", Ret: Void}}} + if _, err := Translate(file, options); err != nil { + t.Fatal("complete initialized four-family function", arch, err) + } + // Reusing the parsed file in a later request must observe the new + // invalid form rather than a preceding request's valid instruction. + for index := range file.Funcs[0].Instrs { + if file.Funcs[0].Instrs[index].Op == "PMULHW" { + file.Funcs[0].Instrs[index].Args[0].Reg = "Y0" + break + } + } + if _, err := Translate(file, options); err == nil { + t.Fatal("changed source retained a preceding valid form", arch) + } + } +} + +func TestX86PackedFunctionErrorPrecedence(t *testing.T) { + for _, row := range []struct { + name, first, want string + }{ + {"unknown_before_form", "NOT_AN_INSTRUCTION", "NOT_AN_INSTRUCTION"}, + {"lock_before_form", "LOCK", "LOCK"}, + {"repeat_before_form", "REP", "REP"}, + } { + t.Run(row.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row.first+"\nVPAVGB X0,Y1,X2\nRET\n") + if err != nil { + t.Fatal(err) + } + consumer := newX86Ctx(new(strings.Builder), file.Funcs[0], FuncSig{Ret: Void}, nil, nil, "amd64", "", false) + if err := consumer.lowerBlocks(); err == nil || !strings.Contains(err.Error(), row.want) { + t.Fatalf("earlier source failure was displaced by a later form: %v", err) + } + }) + } +} diff --git a/amd64_packed_integer_compare_fault_runtime_test.go b/amd64_packed_integer_compare_fault_runtime_test.go new file mode 100644 index 000000000..a5634278b --- /dev/null +++ b/amd64_packed_integer_compare_fault_runtime_test.go @@ -0,0 +1,328 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedIntegerCompareNativeMemoryStatus(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, prototypes, references, checks strings.Builder + count := 0 + for opIndex, op := range x86PackedIntegerCompareTestNames { + bits := []int{8, 16, 32, 64}[opIndex%4] + widths := []int{16} + if opIndex >= 8 { + widths = []int{16, 32, 64} + } + for _, width := range widths { + variants := []int{0} + if opIndex >= 8 { + variants = []int{1, 2} + if width < 64 { + variants = append(variants, 0) + } + if bits >= 32 { + variants = append(variants, 3, 4) + } + } + for _, variant := range variants { + masked, broadcast := variant == 2 || variant == 4, variant >= 3 + name := fmt.Sprintf("faultcompare%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(input *byte,mask uint64)\n", name) + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ input+0(FP),BX\nMOVQ mask+8(FP),AX\nKMOVQ AX,K1\nVPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\n", name) + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + operands := "(BX),X1" + if opIndex >= 8 { + mask, destination := "", bank+"0" + if variant != 0 { + destination = "K2" + } + if masked { + mask = "K1," + } + operands = fmt.Sprintf("(BX),%s1,%s%s", bank, mask, destination) + } + instruction := op + if broadcast { + instruction += ".BCST" + } + fmt.Fprintf(&source, "%s %s\nRET\n", instruction, operands) + fmt.Fprintf(&references, "__attribute__((noinline)) static void reference%d(const uint8_t *p,uint64_t mask) {\n", count-1) + if opIndex < 8 { + mnemonic := strings.ToLower(strings.ReplaceAll(op, "L", "D")) + references.WriteString(" typedef int32_t V __attribute__((vector_size(16)));\n V value = {0,0,0,0};\n") + fmt.Fprintf(&references, " __asm__ volatile(\"%s (%%1),%%0\" : \"+x\"(value) : \"r\"(p) : \"memory\");\n", mnemonic) + } else { + bytes := bits / 8 + fmt.Fprintf(&references, " for (int lane = 0; lane < %d; ++lane) {\n", width/bytes) + if masked { + references.WriteString(" if (!((mask>>lane)&1)) continue;\n") + } + offset := "lane*" + fmt.Sprint(bytes) + if broadcast { + offset = "0" + } + fmt.Fprintf(&references, " for (int j = 0; j < %d; ++j) {\n (void)((const volatile uint8_t*)p)[%s+j];\n }\n", bytes, offset) + references.WriteString(" }\n") + } + references.WriteString("}\n") + lanes := width * 8 / bits + masks := []uint64{0} + if masked { + masks = []uint64{0, 1, uint64(1) << uint(lanes-1), ^uint64(0)} + } + for _, mask := range masks { + expectedReadable := "0" + if opIndex < 8 { + expectedReadable = "offset == 0 ? 0 : 2" + } + fmt.Fprintf(&checks, " for (int offset = 0; offset < 16; ++offset) {\n if (compare(reference%d,%s,memory+offset,UINT64_C(%d),%s)) return 1;\n }\n", count-1, name, mask, expectedReadable) + for _, offset := range []int{0, 1, bits / 8, 16, 32, 64} { + expected := x86PackedIntegerCompareMemoryExpectation(opIndex < 8, width, bits/8, masked, broadcast, mask, offset, offset%16 == 0) + fmt.Fprintf(&checks, " if (compare(reference%d,%s,boundary-%d,UINT64_C(%d),%d)) return 1;\n", count-1, name, offset, mask, expected) + } + expectedNull := x86PackedIntegerCompareMemoryExpectation(opIndex < 8, width, bits/8, masked, broadcast, mask, 0, true) + fmt.Fprintf(&checks, " if (compare(reference%d,%s,0,UINT64_C(%d),%d)) return 1;\n", count-1, name, mask, expectedNull) + } + } + } + } + x86LEAActualTargetGoObject(t, "amd64", triple, source.String()) + ir, err := x86PackedIntegerCompareAPI(t, source.String(), declarations.String(), "amd64", triple, "Go") + if err != nil { + t.Fatal(err) + } + program := `#include +#include +#include +#include +#include +#include +#include +#include +#include +` + prototypes.String() + references.String() + x86PackedIntegerCompareFaultStatusOracle + ` +int main(void) { + struct rlimit noCore = {0,0}; + if (setrlimit(RLIMIT_CORE,&noCore)) return 76; + unsigned a,b,c,d; + if (!__get_cpuid(1,&a,&b,&c,&d) || !(c&(1u<<19)) || !(c&(1u<<20))) { + fprintf(stderr,"packed compare fault root requires actual SSE4.1 and SSE4.2\n"); + return 77; + } + __asm__ volatile("pxor %%xmm0,%%xmm0\n\t" + "pcmpeqq %%xmm0,%%xmm0\n\t" + "pcmpgtq %%xmm0,%%xmm0" : : : "xmm0"); + printf("packed compare fault preflight: CPUID SSE4.1=1 SSE4.2=1; actual PCMPEQQ/PCMPGTQ executed\n"); + long page = sysconf(_SC_PAGESIZE); + if (page <= 0 || page%64) return 1; + uint8_t *memory = mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANON,-1,0); + if (memory == MAP_FAILED) return 1; + memset(memory,0x87,(size_t)page); + if (mprotect(memory,(size_t)page,PROT_READ) || + mprotect(memory+page,(size_t)page,PROT_NONE)) return 1; + uint8_t *boundary = memory+page; +` + checks.String() + ` + if (munmap(memory,(size_t)page*2)) return 1; + return 0; +} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-compare-fault", triple, text, program, prefix) + }) + } + t.Logf("%d dead-result programs at mod16/NULL/read-only guard pages, exact native legacy and independent E4 lane-read status at O0/O2; not native EVEX execution", count) +} + +// Expected status comes from the independently constructed fixture accesses, +// not a production descriptor. 0 requires normal exit0; 1 requires a memory +// fault. Only readable, unaligned legacy sources use 2: preserve the original +// machine's exact alignment behavior without guessing a vendor's fault policy. +func x86PackedIntegerCompareMemoryExpectation(legacy bool, width, laneBytes int, masked, broadcast bool, mask uint64, available int, aligned bool) int { + if legacy { + if available < 16 { + return 1 + } + if !aligned { + return 2 + } + return 0 + } + readEnd := 0 + for lane := 0; lane < width/laneBytes; lane++ { + if masked && mask&(uint64(1)< readEnd { + readEnd = end + } + } + if readEnd > available { + return 1 + } + return 0 +} + +const x86PackedIntegerCompareFaultStatusOracle = ` +typedef struct { int signal; int code; } Status; + +static int observe(void (*fn)(const uint8_t*,uint64_t),const uint8_t *p, + uint64_t mask,Status *out) { + pid_t child = fork(); + if (child < 0) return 1; + if (child == 0) { + struct rlimit limit = {0,0}; + if (setrlimit(RLIMIT_CORE,&limit)) _exit(3); + fn(p,mask); + _exit(0); + } + int status; + if (waitpid(child,&status,0) != child) return 1; + if (WIFSIGNALED(status)) { + *out = (Status){WTERMSIG(status),0}; + return 0; + } + if (WIFEXITED(status)) { + *out = (Status){0,WEXITSTATUS(status)}; + return 0; + } + return 1; +} + +static int compare(void (*original)(const uint8_t*,uint64_t), + void (*translated)(const uint8_t*,uint64_t), + const uint8_t *p,uint64_t mask,int expectation) { + Status a,b; + if (observe(original,p,mask,&a) || observe(translated,p,mask,&b)) return 1; + int allowedA = a.code == 0 && (a.signal == 0 || a.signal == SIGSEGV || a.signal == SIGBUS); + int allowedB = b.code == 0 && (b.signal == 0 || b.signal == SIGSEGV || b.signal == SIGBUS); + if (!allowedA || !allowedB || expectation < 0 || expectation > 2 || + (expectation == 0 && (a.signal || b.signal)) || + (expectation == 1 && (!a.signal || !b.signal)) || + a.signal != b.signal || a.code != b.code) { + fprintf(stderr,"mod16=%lu mask=%llx original=(%d,%d) translated=(%d,%d)\n", + (unsigned long)((uintptr_t)p%16),(unsigned long long)mask, + a.signal,a.code,b.signal,b.code); + return 1; + } + return 0; +} +` + +func TestX86PackedIntegerCompareFaultStatusRejectsSymmetricUnexpectedExit(t *testing.T) { + execution := x86ScalarCountHostExecution(t) + // The child observations are real, but these deliberately raised signals + // test the oracle's status domain, not a memory fault or native opcode. + // Run on the actual host; this C-only control has no x86 ISA requirement. + program := `#include +#include +#include +#include +#include +#include +` + x86PackedIntegerCompareFaultStatusOracle + ` +static int selectedSignal; + +static void unexpected(const uint8_t *p,uint64_t mask) { + if (!selectedSignal) { + _exit(7); + } + raise(selectedSignal); +} + +static void complete(const uint8_t *p,uint64_t mask) {} + +int main(void) { + const int rejected[] = {SIGILL,SIGABRT,SIGFPE,SIGTRAP,0}; + for (unsigned i = 0; i < sizeof(rejected)/sizeof(rejected[0]); ++i) { + selectedSignal = rejected[i]; + if (!compare(unexpected,unexpected,0,0,2)) { + fprintf(stderr,"symmetric unexpected status accepted: signal=%d\n",selectedSignal); + return 1; + } + } + return compare(complete,complete,0,0,0); +} +` + ir := "target triple = \"" + execution.triple + "\"\ndefine void @status_tripwire_object() { ret void }\n" + execution.compileAndRunLLVM(t, "packed-compare-status-tripwire", ir, program, "-O2") +} + +func TestX86PackedIntegerCompareFaultStatusRequiresSuccessfulReadableControl(t *testing.T) { + execution := x86ScalarCountHostExecution(t) + // Deliberately raised memory signals are classification tripwires. They do + // not replace the independently mapped native-width fixtures above. + program := `#include +#include +#include +#include +#include +#include +` + x86PackedIntegerCompareFaultStatusOracle + ` +static void unexpectedMemoryFault(const uint8_t *p,uint64_t mask) { + raise(SIGSEGV); +} + +static void complete(const uint8_t *p,uint64_t mask) {} + +int main(void) { + if (!compare(unexpectedMemoryFault,unexpectedMemoryFault,0,0,0)) { + fprintf(stderr,"symmetric memory fault replaced required successful control\n"); + return 1; + } + if (compare(unexpectedMemoryFault,unexpectedMemoryFault,0,0,1)) return 2; + return compare(complete,complete,0,0,0); +} +` + ir := "target triple = \"" + execution.triple + "\"\ndefine void @readable_tripwire_object() { ret void }\n" + execution.compileAndRunLLVM(t, "packed-compare-readable-tripwire", ir, program, "-O2") +} + +func TestX86PackedIntegerCompareFaultExpectedAccessMatrix(t *testing.T) { + for _, test := range []struct { + name string + legacy, masked, broadcast bool + width, bytes, available int + mask uint64 + aligned bool + want int + }{ + {"legacy-aligned-readable", true, false, false, 16, 8, 16, 0, true, 0}, + {"legacy-unaligned-readable", true, false, false, 16, 8, 16, 0, false, 2}, + {"legacy-guard-overlap", true, false, false, 16, 8, 15, 0, true, 1}, + {"vex-full-readable-unaligned", false, false, false, 32, 4, 32, 0, false, 0}, + {"vex-full-guard-overlap", false, false, false, 32, 4, 31, 0, false, 1}, + {"evex-zero-mask-null", false, true, false, 64, 1, 0, 0, true, 0}, + {"evex-low-lane-readable", false, true, false, 64, 1, 1, 1, false, 0}, + {"evex-high-lane-guard", false, true, false, 64, 1, 63, uint64(1) << 63, false, 1}, + {"evex-high-lane-readable", false, true, false, 64, 1, 64, uint64(1) << 63, false, 0}, + {"broadcast-zero-mask-null", false, true, true, 64, 8, 0, 0, false, 0}, + {"broadcast-readable", false, true, true, 64, 8, 8, 0xff, false, 0}, + {"broadcast-guard-overlap", false, true, true, 64, 8, 7, 0xff, false, 1}, + {"broadcast-irrelevant-mask-null", false, true, true, 16, 8, 0, 4, false, 0}, + } { + t.Run(test.name, func(t *testing.T) { + got := x86PackedIntegerCompareMemoryExpectation(test.legacy, test.width, test.bytes, + test.masked, test.broadcast, test.mask, test.available, test.aligned) + if got != test.want { + t.Fatalf("independent expected access status=%d, want=%d", got, test.want) + } + }) + } +} diff --git a/amd64_packed_integer_compare_forms.go b/amd64_packed_integer_compare_forms.go new file mode 100644 index 000000000..115fdf467 --- /dev/null +++ b/amd64_packed_integer_compare_forms.go @@ -0,0 +1,193 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedIntegerCompareSpec struct { + laneBits int + greater bool + legacy bool + mmx bool +} + +var x86PackedIntegerCompareSpecs = map[Op]x86PackedIntegerCompareSpec{ + "PCMPEQB": {8, false, true, true}, + "PCMPEQW": {16, false, true, true}, + "PCMPEQL": {32, false, true, true}, + "PCMPEQQ": {64, false, true, false}, + "PCMPGTB": {8, true, true, true}, + "PCMPGTW": {16, true, true, true}, + "PCMPGTL": {32, true, true, true}, + "PCMPGTQ": {64, true, true, false}, + "VPCMPEQB": {8, false, false, false}, + "VPCMPEQW": {16, false, false, false}, + "VPCMPEQD": {32, false, false, false}, + "VPCMPEQQ": {64, false, false, false}, + "VPCMPGTB": {8, true, false, false}, + "VPCMPGTW": {16, true, false, false}, + "VPCMPGTD": {32, true, false, false}, + "VPCMPGTQ": {64, true, false, false}, +} + +type x86PackedIntegerCompareForm struct { + x86PackedBinaryForm + spec x86PackedIntegerCompareSpec + source Operand + maskResult bool +} + +// Integer relations of an identical physical vector view are independent of +// its bits, including all eight immediate predicates. This is only a numerical +// dependency rule after full grammar validation. Mask inputs remain required; +// floating comparisons, memory, raw roles and the native MMX bank stay separate. +func (form x86PackedIntegerCompareForm) identicalVectorInputs(ins Instr) bool { + return x86ResultNamedPredicateInstruction(ins) && !form.mmx && form.memory < 0 && + form.source.Kind == OpReg && ins.Args[form.second].Kind == OpReg && + form.source.Reg == ins.Args[form.second].Reg +} + +// Properties describe the Go operand rows and physical mode, not initialized +// inputs, native backing, memory bounds, or caller authority. The raw decoder's +// eight-entry encoding registry remains distinct from these sixteen Go names. +func x86PackedIntegerCompareProperties(op Op, ins Instr, arch string) (x86PackedIntegerCompareForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86PackedIntegerCompareSpecs[Op(base)] + if !known { + return x86PackedIntegerCompareForm{}, false, nil + } + form := x86PackedIntegerCompareForm{spec: spec} + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return form, true, fmt.Errorf("packed integer compare dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + local.Args = append([]Operand(nil), ins.Args...) + fail := func() (x86PackedIntegerCompareForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go packed integer compare grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" { + return fail() + } + if spec.legacy || len(local.Args) == 3 && local.Args[2].Kind == OpReg && !amd64IsKOperand(local.Args[2]) { + encoding := x86PackedLogicalVEX + if spec.legacy { + encoding = x86PackedLogicalLegacyFP + if spec.mmx { + encoding = x86PackedLogicalLegacyInteger + var err error + local, err = x86LegacyMMXReadInstruction(local, arch) + if err != nil { + return form, true, err + } + } + } + binary, err := x86PackedBinaryProperties(local, arch, x86PackedBinarySpec{encoding: encoding}) + form.x86PackedBinaryForm = binary + if err != nil { + return form, true, err + } + } else { + // _yvpcmpeqb EVEX rows produce K0-7, not an X/Y/Z destination; + // their optional input mask is K1-7. There is no .Z form. + form.maskResult = true + binary, valid := x86PackedMaskCompareProperties(local, arch, 0, spec.laneBits, dotted, suffix) + if !valid { + return fail() + } + form.x86PackedBinaryForm = binary + } + form.source = local.Args[form.first] + if err := x86Mode32VectorRegisterContext(local, arch); err != nil { + return form, true, err + } + if form.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + return form, true, fmt.Errorf("%w: packed integer compare has no bound address-size override", ErrProbeNeedsContext) + } + return form, true, nil +} + +// The EVEX K-result rows of _yvpcmpeqb and _yvpcmpb have the same source, +// width, mask and broadcast axes. Only the latter has a leading immediate. +// This descriptor supplies operand roles, never initialized data or backing. +func x86PackedMaskCompareProperties(ins Instr, arch string, first, laneBits int, dotted bool, suffix string) (x86PackedBinaryForm, bool) { + form := x86PackedBinaryForm{first: first, second: first + 1, destination: len(ins.Args) - 1, mask: -1, memory: -1} + if len(ins.Args) != first+3 && len(ins.Args) != first+4 || dotted && suffix != "BCST" { + return form, false + } + if !amd64IsKOperand(ins.Args[form.destination]) { + return form, false + } + width, valid := amd64PackedCompareWidth(ins.Args[form.second]) + if !valid || !amd64GoPackedVectorMoveRegister(ins.Args[form.second], width, "amd64") { + return form, false + } + form.byteWidth = width + source := ins.Args[first] + if source.Kind == OpReg { + if !amd64GoPackedVectorMoveRegister(source, width, "amd64") { + return form, false + } + } else if !x86PackedShiftMemoryValid(source, arch) { + return form, false + } else { + form.memory, form.memoryBytes = first, width + } + if dotted { + if laneBits < 32 || form.memory < 0 { + return form, false + } + form.broadcast, form.memoryBytes = true, laneBits/8 + } + if len(ins.Args) == first+4 { + mask := first + 2 + if !amd64NonzeroKOperand(ins.Args[mask]) || arch == "386" && !ins.x86Encoded && ins.x86SourceRaw == nil { + return form, false + } + form.mask = mask + } + return form, true +} + +func x86TypedPackedIntegerCompareByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedIntegerCompareProperties(op, ins, arch) + immediateFamily := false + if !known { + immediate, recognized, immediateErr := x86ImmediatePackedCompareProperties(op, ins, arch) + form, known, err = immediate.x86PackedIntegerCompareForm, recognized, immediateErr + immediateFamily = recognized + } + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + firstWidth := form.byteWidth + if form.memory >= 0 { + firstWidth = form.memoryBytes + } + readInputs := !form.identicalVectorInputs(ins) + operands := []x86TypedByteOperand{ + {index: form.first, operand: form.source, width: int64(firstWidth), read: readInputs}, + {index: form.second, operand: ins.Args[form.second], width: int64(form.byteWidth), read: readInputs}, + } + if form.mask >= 0 { + operands = append(operands, x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + width := form.byteWidth + if form.maskResult { + width = 8 + } + operands = append(operands, x86TypedByteOperand{ + index: form.destination, operand: ins.Args[form.destination], width: int64(width), write: true, + }) + effect := x86TypedByteEffect{operands: operands, requiresResultProof: immediateFamily} + if !form.mmx && !form.maskResult { + effect.vector.upperWrite = x86TypedVectorUpperZero + effect.vector.memoryAlignment = 1 + if form.spec.legacy { + effect.vector.upperWrite = x86TypedVectorUpperPreserve + effect.vector.memoryAlignment = 16 + } + } + return effect, true, nil +} diff --git a/amd64_packed_integer_compare_forms_test.go b/amd64_packed_integer_compare_forms_test.go new file mode 100644 index 000000000..fb366d3b2 --- /dev/null +++ b/amd64_packed_integer_compare_forms_test.go @@ -0,0 +1,348 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86PackedIntegerCompareCompleteActualGoRowsAndAttributes(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + read := func(name string) string { + data, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + return strings.Join(strings.Fields(string(data)), "") + } + legacy, vector := read("asm6.go"), read("avx_optabs.go") + for _, definition := range []string{ + "var ymm = []ytab{ {Zm_r_xm, 1, argList{Ymm, Ymr}}, {Zm_r_xm, 2, argList{Yxm, Yxr}}, }", + "var yxm_q4 = []ytab{ {Zm_r, 1, argList{Yxm, Yxr}}, }", + } { + if !strings.Contains(legacy, strings.Join(strings.Fields(definition), "")) { + t.Fatal("actual selected SDK legacy operand rows changed", definition) + } + } + rows := []string{ + "{zcase:Zvex_rm_v_r,zoffset:2,args:argList{Yxm,Yxr,Yxr}}", + "{zcase:Zvex_rm_v_r,zoffset:2,args:argList{Yym,Yyr,Yyr}}", + "{zcase:Zevex_rm_v_r,zoffset:0,args:argList{YxmEvex,YxrEvex,Yk}}", + "{zcase:Zevex_rm_v_k_r,zoffset:3,args:argList{YxmEvex,YxrEvex,Yknot0,Yk}}", + "{zcase:Zevex_rm_v_r,zoffset:0,args:argList{YymEvex,YyrEvex,Yk}}", + "{zcase:Zevex_rm_v_k_r,zoffset:3,args:argList{YymEvex,YyrEvex,Yknot0,Yk}}", + "{zcase:Zevex_rm_v_r,zoffset:0,args:argList{Yzm,Yzr,Yk}}", + "{zcase:Zevex_rm_v_k_r,zoffset:3,args:argList{Yzm,Yzr,Yknot0,Yk}}", + } + if !strings.Contains(vector, "var_yvpcmpeqb=[]ytab{"+strings.Join(rows, ",")+",}") { + t.Fatal("actual selected SDK ordered eight-row compare table changed") + } + opcodes := []int{0x74, 0x75, 0x76, 0x29, 0x64, 0x65, 0x66, 0x37} + bits := []int{8, 16, 32, 64, 8, 16, 32, 64} + for index, op := range x86PackedIntegerCompareTestNames[:8] { + table, prefix, code := "ymm", "Py1", fmt.Sprintf("0x%x,Pe,0x%x", opcodes[index], opcodes[index]) + if bits[index] == 64 { + table, prefix, code = "yxm_q4", "Pq4", fmt.Sprintf("0x%x", opcodes[index]) + } + if !strings.Contains(legacy, fmt.Sprintf("{A%s,%s,%s,opBytes{%s}}", op, table, prefix, code)) { + t.Fatal("actual selected SDK legacy opcode attributes changed", op) + } + } + for index, op := range x86PackedIntegerCompareTestNames[8:] { + mapName, evexW, broadcast := "0F", "W0", "" + if bits[index] == 64 { + mapName, evexW, broadcast = "0F38", "W1", "|evexBcstN8" + } else if bits[index] == 32 { + broadcast = "|evexBcstN4" + } + var attributes strings.Builder + for _, width := range []int{128, 256} { + fmt.Fprintf(&attributes, "avxEscape|vex%d|vex66|vex%s|vexW0,0x%X,", width, mapName, opcodes[index]) + } + for _, width := range []int{128, 256, 512} { + fmt.Fprintf(&attributes, "avxEscape|evex%d|evex66|evex%s|evex%s,evexN%d%s,0x%X,", width, mapName, evexW, width/8, broadcast, opcodes[index]) + } + if !strings.Contains(vector, "{as:A"+op+",ytab:_yvpcmpeqb,prefix:Pavx,op:opBytes{"+attributes.String()+"}},") { + t.Fatal("actual selected SDK complete five encoding attributes changed", op) + } + } + actual := map[string]int{} + for _, source := range []string{legacy, vector} { + for _, match := range regexp.MustCompile(`\{(?:as:)?A(V?PCMP(?:EQ|GT)[BWLQD]),`).FindAllStringSubmatch(source, -1) { + actual[match[1]]++ + } + } + if len(actual) != 16 || len(x86PackedIntegerCompareSpecs) != 16 { + t.Fatal("complete Go named-family denominator changed", actual) + } + for index, op := range x86PackedIntegerCompareTestNames { + spec := x86PackedIntegerCompareSpecs[Op(op)] + if actual[op] != 1 || spec.laneBits != bits[index%8] || spec.greater != (index%8 >= 4) || spec.legacy != (index < 8) || spec.mmx != (index < 8 && bits[index%8] < 64) { + t.Fatal("Go registration and typed semantics differ", op, spec) + } + } +} + +func TestX86PackedIntegerCompareLogicalFPSpanRequiresEveryByteEvenMaskZero(t *testing.T) { + for _, op := range x86PackedIntegerCompareTestNames[8:] { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + if broadcast && bits < 32 { + continue + } + span, instruction := width, op + if broadcast { + span, instruction = bits/8, op+".BCST" + } + t.Run(fmt.Sprintf("%s/%d/bcst=%t", op, width, broadcast), func(t *testing.T) { + var parameters strings.Builder + for index := 0; index < span; index++ { + if index != 0 { + parameters.WriteByte(',') + } + fmt.Fprintf(¶meters, "a%d uint8", index) + } + pkg := mustGoPackage(t, "compare", "package compare\nfunc query("+parameters.String()+")\n") + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + source := fmt.Sprintf("TEXT ·query(SB),4,$0-%d\nMOVQ $0,AX\nKMOVQ AX,K1\n%s a0+0(FP),%s1,K1,K2\nRET\n", span, instruction, bank) + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + ir, err := Translate(file, options) + x86VectorScratchCodegenControl(t, source, "func query("+parameters.String()+")", "amd64", "x86_64-unknown-linux-gnu", ir, err) + // Keep explicit initialization as a separate positive control; + // every logical FP byte remains independently required. + initialized := strings.Replace(source, "KMOVQ AX,K1\n", "KMOVQ AX,K1\nVPXORD Z1,Z1,Z1\n", 1) + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", initialized) + file, err = Parse(ArchAMD64, initialized) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, options); err != nil { + t.Fatal("complete logical FP fields", err) + } + for missing := 0; missing < span; missing++ { + t.Run(fmt.Sprint(missing), func(t *testing.T) { + sig := sigs["query"] + sig.Frame.Params = append([]FrameSlot(nil), sig.Frame.Params...) + sig.Frame.Params = append(sig.Frame.Params[:missing], sig.Frame.Params[missing+1:]...) + mutated := options + mutated.Sigs = map[string]FuncSig{"query": sig} + if ir, err := Translate(file, mutated); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("mask0 borrowed absent logical FP byte%d: %v", missing, err) + } + }) + } + }) + } + } + } +} + +func TestX86PackedIntegerCompareMMXActualEncodedBankAndDefinedness(t *testing.T) { + for opIndex, op := range x86PackedIntegerCompareTestNames[:8] { + if opIndex%4 == 3 { + continue + } + opcode := []byte{0x74, 0x75, 0x76, 0x29, 0x64, 0x65, 0x66, 0x37}[opIndex] + for index, spelling := range []string{"M0", "M1", "M2", "M3", "M4", "M5", "M6", "M7", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + t.Run(op+"/"+spelling, func(t *testing.T) { + source := "TEXT probe(SB),4,$0-0\n" + op + " " + spelling + ",M7\nRET\n" + want := []byte{0x0f, opcode, 0xf8 | byte(index&7), 0xc3} + if index >= 16 { + want = append([]byte{0x41}, want...) + } + if got := x86PackedShiftActualGoBytes(t, "amd64", source); !bytes.Equal(got, want) { + t.Fatalf("actual Go GP spelling changed native MMX identity: %x != %x", got, want) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || effect.operands[0].operand.Reg != Reg(fmt.Sprintf("M%d", index&7)) || effect.operands[0].width != 8 { + t.Fatal("pure query confused GP initialization with MMX", effect, err) + } + state := x86ResultState{flags: x86ByteFlagsAll} + for i := range state.gp { + state.gp[i] = 0xff + } + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("numeric GP bytes supplied unproved MMX state") + } + }) + } + } + for _, op := range x86PackedIntegerCompareTestNames[:8] { + ins := Instr{Op: Op(op), Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}} + for input := 0; input < 2; input++ { + for missing := 0; missing < 16; missing++ { + t.Run(fmt.Sprintf("%s/input%d/byte%d", op, input, missing), func(t *testing.T) { + state := x86ResultState{flags: x86ByteFlagsAll} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + state.xmm[1-input] &^= uint16(1) << uint(missing) + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("compare borrowed undefined input byte") + } + }) + } + } + } +} + +type x86PackedIntegerCompareCase struct { + op, operands string + width, bits int + legacy, mmx bool + mask, memory bool + broadcast bool + first, second, destination int +} + +func x86PackedIntegerCompareCases(arch string) []x86PackedIntegerCompareCase { + var cases []x86PackedIntegerCompareCase + for index, op := range x86PackedIntegerCompareTestNames { + bits := []int{8, 16, 32, 64}[index%4] + if index < 8 { + for _, row := range []struct { + operands string + first, second int + memory bool + }{ + {"X0,X1", 0, 1, false}, {"X1,X0", 1, 0, false}, {"X0,X0", 0, 0, false}, {"(BX),X1", 0, 1, true}, + } { + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: row.operands, width: 16, bits: bits, legacy: true, memory: row.memory, first: row.first, second: row.second, destination: row.second}) + } + if arch == "amd64" && bits < 64 { + for _, memory := range []bool{false, true} { + operand := "M0,M1" + if memory { + operand = "(BX),M1" + } + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: operand, width: 8, bits: bits, legacy: true, mmx: true, memory: memory, first: 0, second: 1, destination: 1}) + } + } + continue + } + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + if width < 64 { + for _, alias := range [][3]int{{0, 1, 2}, {0, 1, 0}, {0, 1, 1}, {0, 0, 0}} { + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: fmt.Sprintf("%s%d,%s%d,%s%d", bank, alias[0], bank, alias[1], bank, alias[2]), width: width, bits: bits, first: alias[0], second: alias[1], destination: alias[2]}) + } + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: fmt.Sprintf("(BX),%s1,%s2", bank, bank), width: width, bits: bits, memory: true, first: 0, second: 1, destination: 2}) + } + for _, masked := range []bool{false, true} { + if arch == "386" && masked { + continue + } + for _, memory := range []bool{false, true} { + for _, alias := range [][3]int{{0, 1, 0}, {0, 1, 2}, {1, 0, 1}, {0, 0, 7}} { + first := fmt.Sprintf("%s%d", bank, alias[0]) + if memory { + first = "(BX)" + } + mask := "" + if masked { + mask = "K2," + } + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: fmt.Sprintf("%s,%s%d,%sK%d", first, bank, alias[1], mask, alias[2]), width: width, bits: bits, mask: masked, memory: memory, first: alias[0], second: alias[1], destination: alias[2]}) + } + if memory && bits >= 32 { + mask := "" + if masked { + mask = "K2," + } + cases = append(cases, x86PackedIntegerCompareCase{op: op + ".BCST", operands: fmt.Sprintf("(BX),%s1,%sK2", bank, mask), width: width, bits: bits, mask: masked, memory: true, broadcast: true, first: 0, second: 1, destination: 2}) + } + } + } + } + } + return cases +} + +func TestX86PackedIntegerCompareThreeAPIsFiveTargetsO0O2Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc is required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + cases := x86PackedIntegerCompareCases(target.arch) + for index, row := range cases { + fmt.Fprintf(&source, "TEXT ·compare%d(SB),4,$0-0\n%s %s\nRET\n", index, row.op, row.operands) + } + file, signatures := x86UnboundVoidFormatSources(t, source.String(), target.arch, target.triple) + ir := x86InstructionFormatObjectUnit(t, file, signatures, target.arch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerPackedIntegerCompare(ins.Op, ins) + }) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + t.Logf("%d original Go forms, three-API Context, emitter-only IR/backend O0/O2 objects; initialized public inputs have their own owner", len(cases)) + }) + } +} + +func TestX86PackedIntegerCompareDescriptorPermissionsAndMalformedForms(t *testing.T) { + for _, row := range x86PackedIntegerCompareCases("amd64") { + file, err := Parse(ArchAMD64, "TEXT query(SB),4,$0-0\n"+row.op+" "+row.operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, known, err := x86PackedIntegerCompareProperties(ins.Op, ins, "amd64") + if !known || err != nil || form.byteWidth != row.width || form.spec.laneBits != row.bits || form.mmx != row.mmx || form.broadcast != row.broadcast || (form.mask >= 0) != row.mask || (form.memory >= 0) != row.memory { + t.Fatal("complete form lost an independent axis", row, form, err) + } + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatal("pure descriptor lost unchanged flags", effect, err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0xffff, "Z1": 0xff}, masks: map[Reg]bool{"K2": true}} + before := state + before.vectors, before.masks = x86CloneCallerVectorState(state.vectors, state.masks) + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, nil) || x86SourceStackInstructionKnown(ins, OpMOVQ, "Q", "amd64") || !reflect.DeepEqual(state, before) || ins.x86CallerMemory != nil { + t.Fatal("query descriptor invented caller input/permission", row) + } + for _, mutate := range []func(*Instr){ + func(i *Instr) { i.Op += ".Z" }, + func(i *Instr) { i.Args = append(i.Args, Operand{Kind: OpReg, Reg: "X0"}) }, + func(i *Instr) { i.Args[len(i.Args)-1] = Operand{Kind: OpReg, Reg: "AX"} }, + func(i *Instr) { i.Args[0] = Operand{Kind: OpImm, Imm: 3} }, + } { + local := ins + local.Args = append([]Operand(nil), ins.Args...) + mutate(&local) + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = "amd64" + before, temporary := output.String(), ctx.tmp + known, _, err := ctx.lowerPackedIntegerCompare(local.Op, local) + if !known || err == nil || output.String() != before || ctx.tmp != temporary { + t.Fatal("malformed form emitted before rejection", row, local, err) + } + } + } +} diff --git a/amd64_packed_integer_compare_go_oracle_test.go b/amd64_packed_integer_compare_go_oracle_test.go new file mode 100644 index 000000000..54dd1babe --- /dev/null +++ b/amd64_packed_integer_compare_go_oracle_test.go @@ -0,0 +1,288 @@ +package plan9asm + +import ( + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func x86PackedIntegerCompareRunOriginalGo(t *testing.T, declaration, assembly, testSource string) { + x86PackedIntegerCompareOriginalGoDriver(t, declaration, assembly, testSource, false) +} + +func x86PackedIntegerCompareOriginalGoDriver(t *testing.T, declaration, assembly, testSource string, buildOnly bool) { + t.Helper() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declaration, + "value_amd64.s": assembly, "value_test.go": testSource, + } { + data := []byte(source) + if strings.HasSuffix(name, ".go") { + var err error + data, err = format.Source(data) + if err != nil { + t.Fatalf("first-party original fixture syntax: %v", err) + } + } + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + t.Fatal(err) + } + } + driver, _ := x86LEAActualSDKDriver(t) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + arguments := []string{"test", "-p=2", "-vet=off", "-count=1"} + executable := filepath.Join(dir, "original-compare.test") + if buildOnly { + arguments = append(arguments, "-c", "-o", executable) + } + arguments = append(arguments, ".") + command := exec.CommandContext(ctx, driver, arguments...) + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if buildOnly { + // These native Linux fixtures may contain Unix fault probes. Compile + // their actual target independently of the host; never execute them. + command.Env = append(command.Env, "GOOS=linux") + } + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go packed compare driver (build-only=%t): %v\n%s", buildOnly, err, output) + } + if buildOnly { + if amd64, err := x86OriginalGoLinkedAMD64(executable); err != nil || !amd64 { + t.Fatal("build-only driver lost its actual linked SDK/AMD64 executable", err) + } + } +} + +func x86PackedIntegerCompareOriginalWideSources() (string, string, string) { + rows := x86PackedIntegerCompareCases("amd64") + declaration, assembly, _ := x86PackedIntegerCompareRuntimeFixture(rows) + var checks strings.Builder + for index, row := range rows { + maskResult := !row.legacy && strings.Contains(row.operands, "K") + fmt.Fprintf(&checks, " checkWide(t,compare%d,%q,%d,%d,%t,%t,%t,%t,%t,%t,%t,%d,%d,%d)\n", + index, fmt.Sprintf("compare%d", index), row.width, row.bits/8, strings.Contains(row.op, "GT"), row.legacy, row.mmx, + maskResult, row.mask, row.memory, row.broadcast, row.first, row.second, row.destination) + } + tests := `package oracle + +import ( + "encoding/binary" + "testing" + "unsafe" +) + +func wideLane(p []byte, bytes int) uint64 { + var result uint64 + for j := 0; j < bytes; j++ { result |= uint64(p[j]) << uint(8*j) } + return result +} + +func checkWide(t *testing.T,fn func(*byte,*byte,*byte,uint64,uint32,uint64),name string, + width,bytes int,greater,legacy,mmx,maskResult,masked,memory,broadcast bool, + firstBank,secondBank,destination int) { + t.Helper() + masks := [4]uint64{0,1,0x55aa55aa55aa55aa,^uint64(0)} + for round := 0; round < 12; round++ { + for _,mask := range masks { + var aStorage,bStorage [96]byte + aOffset := int((16-uintptr(unsafe.Pointer(&aStorage[0]))%16)%16) + bOffset := int((16-uintptr(unsafe.Pointer(&bStorage[0]))%16)%16) + for i := range aStorage { aStorage[i],bStorage[i] = 0xa7,0xb9 } + a,b := aStorage[aOffset:aOffset+64],bStorage[bOffset:bOffset+64] + for i := range a { a[i],b[i] = byte(i*17+round*31),byte(i*23+round*53) } + if round < 4 { + patterns := [4][2]byte{{0xff,1},{0x80,0x7f},{1,0},{0,0xff}} + for i := range a { a[i],b[i] = patterns[round][0],patterns[round][1] } + } + originalA,originalB := aStorage,bStorage + var out [424]byte + for i := range out { out[i] = 0xc3 } + csr := uint32(0x1f80 | (round%4)<<13 | round) + expectedFlags := uint64(0x895) + if mask&3 == 0 { expectedFlags = 0 } + if mask&3 == 1 { expectedFlags = 0x55 } + fn(&out[1],&a[0],&b[0],mask,csr,2|expectedFlags) + if binary.LittleEndian.Uint64(out[409:417])&0x8d5 != expectedFlags || out[417] != byte(2|(expectedFlags&0xd5)) || + binary.LittleEndian.Uint32(out[418:422]) != csr || out[0] != 0xc3 || + out[422] != 0xc3 || out[423] != 0xc3 || originalA != aStorage || originalB != bStorage { + t.Fatalf("%s flags/CSR/full input/output canaries: %x",name,out) + } + var expected [3][64]byte + copy(expected[0][:],a) + copy(expected[1][:],b) + copy(expected[2][:],b) + first,second := a,b + if !memory && firstBank != 0 { first = b } + if secondBank == 0 { second = a } + var result [64]byte + var resultMask uint64 + for i := 0; i < width; i += bytes { + offset := i + if broadcast { offset = 0 } + left,right := wideLane(first[offset:],bytes),wideLane(second[i:],bytes) + selected := left == right + if greater { + sign := uint64(1) << uint(bytes*8-1) + selected = (right^sign) > (left^sign) + } + if selected { + for j := 0; j < bytes; j++ { result[i+j] = 0xff } + if !masked || (mask>>uint(i/bytes))&1 != 0 { resultMask |= uint64(1)< (left^sign) + } + want := byte(0) + if selected { want = 0xff } + for j := 0; j < bytes; j++ { + if out[i+j] != want { + t.Fatalf("%s round%d byte%d got%x want%x",name,round,i+j,out[i+j],want) + } + } + } + for i := width; i < 16; i++ { + if out[i] != 0xc3 { t.Fatalf("%s narrow write changed canary",name) } + } + } +} + +func TestOriginalCompare(t *testing.T) { +` + checks.String() + "}\n" + x86PackedIntegerCompareRunOriginalGo(t, declarations.String(), assembly.String(), tests) + t.Logf("%d actual original Go SSE/MMX kernels ×12 initialized numerical/6flag/LAHF/MXCSR/full-buffer cases; MMX bit results do not certify x87 native state", count) +} diff --git a/amd64_packed_integer_compare_linux_amd64_test.go b/amd64_packed_integer_compare_linux_amd64_test.go new file mode 100644 index 000000000..f421afe62 --- /dev/null +++ b/amd64_packed_integer_compare_linux_amd64_test.go @@ -0,0 +1,43 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedIntegerCompareOriginalNativeWideBits(t *testing.T) { + // No emulator, Rosetta, host-inapplicable Skip, or object-only substitute. + // The shared preflight binds native AMD64 and enabled AVX512F/VL state. + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + dir := t.TempDir() + input, output := filepath.Join(dir, "compare-isa.c"), filepath.Join(dir, "compare-isa") + const program = `#include +#include +#include + +int main(void) { + struct rlimit noCore = {0,0}; + if (setrlimit(RLIMIT_CORE,&noCore)) return 77; + unsigned a,b,c,d; + if (!__get_cpuid_count(7,0,&a,&b,&c,&d)) return 78; + printf("AVX2=%u AVX512BW=%u\n",(b>>5)&1,(b>>30)&1); + return (b&(1u<<5)) && (b&(1u<<30)) ? 0 : 79; +} +` + if err := os.WriteFile(input, []byte(program), 0600); err != nil { + t.Fatal(err) + } + compile := append([]string(nil), execution.compiler...) + compile = append(compile, "-O2", input, "-o", output) + x86FloatCompareCaptureCommand(t, exec.Command(compile[0], compile[1:]...)) + run := execution.command(output) + t.Logf("native packed integer ISA supplement: %s", x86FloatCompareCaptureCommand(t, exec.Command(run[0], run[1:]...))) + declaration, assembly, tests := x86PackedIntegerCompareOriginalWideSources() + x86LEAActualTargetGoObject(t, "amd64", execution.triple, assembly) + x86PackedIntegerCompareRunOriginalGo(t, declaration, assembly, tests) + t.Log("532 native original Go forms ×48 independent scalar lane/full-bank/K/flags/CSR/full-buffer observations; execution requires native Linux AMD64 AVX2+AVX512F/VL/BW") +} diff --git a/amd64_packed_integer_compare_runtime_test.go b/amd64_packed_integer_compare_runtime_test.go new file mode 100644 index 000000000..e610c0060 --- /dev/null +++ b/amd64_packed_integer_compare_runtime_test.go @@ -0,0 +1,224 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Every input is initialized by this first-party fixture. Its declared pointer +// transport is not a caller-region or native-frame permission certificate. +func x86PackedIntegerCompareRuntimeFixture(rows []x86PackedIntegerCompareCase) (string, string, string) { + var declarations, source, prototypes strings.Builder + for index, row := range rows { + name := fmt.Sprintf("compare%d", index) + fmt.Fprintf(&declarations, "func %s(out,a,b *byte, mask uint64, csr uint32, flags uint64)\n", name) + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t,uint64_t);\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-48\nMOVQ out+0(FP),DI\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nLDMXCSR csr+32(FP)\n", name) + source.WriteString("VMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\n") + for mask := 0; mask < 8; mask++ { + fmt.Fprintf(&source, "KMOVQ DX,K%d\n", mask) + } + // Seed the independently selected status before the physical MMX + // region. No CFG or unrelated effect can split that region's state. + source.WriteString("PUSHQ flags+40(FP)\nPOPFQ\n") + if row.mmx { + source.WriteString("MOVQ (BX),M0\nMOVQ (CX),M1\n") + } + fmt.Fprintf(&source, "%s %s\n", row.op, row.operands) + if row.mmx { + source.WriteString("MOVQ M1,400(DI)\nEMMS\n") + } else { + source.WriteString("MOVQ $0,400(DI)\n") + } + for bank := 0; bank < 3; bank++ { + fmt.Fprintf(&source, "MOVOU X%d,%d(DI)\nVMOVUPS Y%d,%d(DI)\nVMOVUPS Z%d,%d(DI)\n", bank, bank*112, bank, bank*112+16, bank, bank*112+48) + } + for mask := 0; mask < 8; mask++ { + fmt.Fprintf(&source, "KMOVQ K%d,%d(DI)\n", mask, 336+mask*8) + } + // Observe LAHF before masking the popped word: AND changes FLAGS. + // The whole RFLAGS word includes undefined/unmodeled control bits; + // only the six independently seeded status bits are numerical here. + source.WriteString("LAHF\nMOVB AH,416(DI)\nPUSHFQ\nPOPQ DX\n" + + "ANDQ $0x8d5,DX\nMOVQ DX,408(DI)\nSTMXCSR 417(DI)\nRET\n") + } + return declarations.String(), source.String(), prototypes.String() +} + +const x86PackedIntegerCompareScalarOracle = ` +static uint64_t compareLane(const uint8_t *p, int bytes) { + uint64_t value = 0; + for (int j = 0; j < bytes; ++j) { + value |= (uint64_t)p[j] << (8*j); + } + return value; +} + +static int compareValue(uint64_t first, uint64_t second, int bytes, int greater) { + if (!greater) { + return first == second; + } + uint64_t sign = UINT64_C(1) << (bytes*8-1); + return (second ^ sign) > (first ^ sign); +} + +__attribute__((noinline)) static int checkCompare( + void (*fn)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t,uint64_t), + const char *name, int width, int bytes, int greater, int legacy, int mmx, + int maskResult, int masked, int memory, int broadcast, + int firstBank, int secondBank, int destination) { + const uint64_t masks[] = {0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; + for (int round = 0; round < 12; ++round) { + for (int m = 0; m < 4; ++m) { + _Alignas(64) uint8_t aStorage[96], bStorage[96], output[424]; + uint8_t *a = aStorage+16; + uint8_t *b = bStorage+16; + memset(aStorage,0xa7,sizeof(aStorage)); + memset(bStorage,0xb9,sizeof(bStorage)); + memset(output,0xc3,sizeof(output)); + for (int i = 0; i < 64; ++i) { + a[i] = (uint8_t)(i*17+round*31); + b[i] = (uint8_t)(i*23+round*53); + } + if (round < 4) { + memset(a,round == 0 ? 0xff : round == 1 ? 0x80 : round == 2 ? 1 : 0,64); + memset(b,round == 0 ? 1 : round == 1 ? 0x7f : round == 2 ? 0 : 0xff,64); + } + uint8_t originalA[96], originalB[96]; + memcpy(originalA,aStorage,96); + memcpy(originalB,bStorage,96); + uint64_t mask = masks[m]; + uint32_t saved, csr = 0x1f80 | ((round%4)<<13) | round, after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(csr)); + uint64_t expectedFlags = (mask&3) == 0 ? 0 : (mask&3) == 1 ? 0x55 : 0x895; + fn(output+1,a,b,mask,csr,2|expectedFlags); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + if (after != csr || compareLane(output+418,4) != csr || + compareLane(output+409,8) != expectedFlags || output[417] != (2|(expectedFlags&0xd5)) || + output[0] != 0xc3 || output[422] != 0xc3 || output[423] != 0xc3 || + memcmp(originalA,aStorage,96) || memcmp(originalB,bStorage,96)) { + fprintf(stderr,"%s CSR/flags/full-input/output canary mismatch\n",name); + return 1; + } + uint8_t expected[3][64]; + memcpy(expected[0],a,64); + memcpy(expected[1],b,64); + memcpy(expected[2],b,64); + const uint8_t *first = memory || firstBank == 0 ? a : b; + const uint8_t *second = secondBank == 0 ? a : b; + uint64_t resultMask = 0; + uint8_t result[64] = {0}; + for (int i = 0; i < width; i += bytes) { + int selected = compareValue(compareLane(first+(broadcast ? 0 : i),bytes), + compareLane(second+i,bytes),bytes,greater); + if (selected && (!masked || ((mask>>(i/bytes))&1))) { + resultMask |= UINT64_C(1) << (i/bytes); + } + memset(result+i,selected ? 0xff : 0,bytes); + } + if (!maskResult && !mmx) { + memcpy(expected[destination],result,width); + if (!legacy) { + memset(expected[destination]+width,0,64-width); + } + } + for (int bank = 0; bank < 3; ++bank) { + for (int view = 0; view < 3; ++view) { + int offset = bank*112+(view == 0 ? 0 : view == 1 ? 16 : 48); + int length = view == 0 ? 16 : view == 1 ? 32 : 64; + if (memcmp(output+1+offset,expected[bank],length)) { + fprintf(stderr,"%s round%d mask%llx bank%d view%d mismatch\n", + name,round,(unsigned long long)mask,bank,view); + return 1; + } + } + } + for (int k = 0; k < 8; ++k) { + uint64_t want = maskResult && k == destination ? resultMask : mask; + if (compareLane(output+337+k*8,8) != want) { + fprintf(stderr,"%s K%d mismatch\n",name,k); + return 1; + } + } + for (int i = 0; i < 8; ++i) { + if (output[401+i] != (mmx ? result[i] : 0)) { + fprintf(stderr,"%s MMX bit-result mismatch\n",name); + return 1; + } + } + } + } + return 0; +} +` + +func TestX86PackedIntegerCompareStatusObserverThreeTargetObjects(t *testing.T) { + rows := []x86PackedIntegerCompareCase{{ + op: "PCMPEQB", operands: "X0,X1", width: 16, bits: 8, legacy: true, + }} + for index, op := range []string{"PCMPEQB", "PCMPEQW", "PCMPEQL", "PCMPGTB", "PCMPGTW", "PCMPGTL"} { + for _, operands := range []string{"M0,M1", "M1,M1", "(BX),M1"} { + rows = append(rows, x86PackedIntegerCompareCase{ + op: op, operands: operands, width: 8, bits: 8 << uint(index%3), legacy: true, mmx: true, + }) + } + } + declarations, source, _ := x86PackedIntegerCompareRuntimeFixture(rows) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, "amd64", triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, api) + if err != nil || ir == "" { + t.Fatal("defined-status observer lost complete source effects", api, err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + x86PackedStringCompileObject(t, llc, triple, text, "-O0") + x86PackedStringCompileObject(t, llc, triple, text, "-O2") + } + } + }) + } +} + +func TestCrossLinuxRuntimeMatrixX86PackedIntegerCompareCompleteScalarSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + rows := x86PackedIntegerCompareCases("amd64") + declarations, source, prototypes := x86PackedIntegerCompareRuntimeFixture(rows) + x86LEAActualTargetGoObject(t, "amd64", triple, source) + ir, err := x86PackedIntegerCompareAPI(t, source, declarations, "amd64", triple, "Go") + if err != nil { + t.Fatal(err) + } + var checks strings.Builder + for index, row := range rows { + maskResult := !row.legacy && strings.Contains(row.operands, "K") + fmt.Fprintf(&checks, " if (checkCompare(compare%d,\"compare%d\",%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)) return 1;\n", + index, index, row.width, row.bits/8, x86FloatAddSubBoolInt(strings.Contains(row.op, "GT")), + x86FloatAddSubBoolInt(row.legacy), x86FloatAddSubBoolInt(row.mmx), x86FloatAddSubBoolInt(maskResult), + x86FloatAddSubBoolInt(row.mask), x86FloatAddSubBoolInt(row.memory), x86FloatAddSubBoolInt(row.broadcast), + row.first, row.second, row.destination) + } + program := "#include \n#include \n#include \n" + + "#if !defined(_WIN32)\n#include \n#endif\n" + + prototypes + x86PackedIntegerCompareScalarOracle + "\nint main(void) {\n" + + "#if !defined(_WIN32)\n struct rlimit noCore = {0,0};\n if (setrlimit(RLIMIT_CORE,&noCore)) return 3;\n#endif\n" + + checks.String() + " return 0;\n}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-integer-compare", triple, text, program, prefix) + }) + } + t.Logf("%d forms ×48 independent signed/equal/mask/full-bank/full flags/CSR/input+output canaries at O0/O2; portable VEX/EVEX, not native AVX512", len(rows)) +} diff --git a/amd64_packed_integer_compare_typed_test.go b/amd64_packed_integer_compare_typed_test.go new file mode 100644 index 000000000..c9bdac0bc --- /dev/null +++ b/amd64_packed_integer_compare_typed_test.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86PackedIntegerCompareTestNames = []string{ + "PCMPEQB", "PCMPEQW", "PCMPEQL", "PCMPEQQ", + "PCMPGTB", "PCMPGTW", "PCMPGTL", "PCMPGTQ", + "VPCMPEQB", "VPCMPEQW", "VPCMPEQD", "VPCMPEQQ", + "VPCMPGTB", "VPCMPGTW", "VPCMPGTD", "VPCMPGTQ", +} + +func x86PackedIntegerCompareAPI(t *testing.T, source, declaration, arch, triple, api string) (string, error) { + t.Helper() + pkg := mustGoPackage(t, "compare", "package compare\n"+declaration) + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + ctx := llvm.NewContext() + defer ctx.Dispose() + if api == "Go" { + module, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: arch, TargetTriple: triple, ResolveSym: resolve, + }) + if module != nil && !module.Module.IsNil() { + defer module.Module.Dispose() + return module.Module.String(), err + } + return "", err + } + file, err := Parse(ArchAMD64, source) + if err != nil { + return "", err + } + signatures, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + return "", err + } + options := Options{Goarch: arch, TargetTriple: triple, ResolveSym: resolve, Sigs: signatures} + if api == "text" { + return Translate(file, options) + } + if api != "module" { + t.Fatal("unknown packed comparison consumer", api) + } + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + return module.String(), err + } + return "", err +} + +func TestX86PackedIntegerCompareMode32HighBankActualSourceNeedsContext(t *testing.T) { + for _, op := range x86PackedIntegerCompareTestNames[8:] { + for _, operands := range []string{ + "X8,X1,X2", "X1,X8,X2", "X1,X2,X15", + "Y15,Y1,Y2", "Y1,Y15,Y2", "Y1,Y2,Y8", + "X20,X1,K2", "X1,X20,K2", "Y20,Y1,K2", "Y1,Y20,K2", + } { + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(op+"/"+operands+"/"+triple, func(t *testing.T) { + source := "TEXT ·query(SB),4,$0-0\n" + op + " " + operands + "\nRET\n" + x86LEAActualTargetGoObject(t, "386", triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86PackedIntegerCompareAPI(t, source, "func query()\n", "386", triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("actual Go acceptance fabricated a physical high bank: IR=%d err=%v", len(ir), err) + } + }) + } + }) + } + } + } +} + +func TestX86PackedIntegerCompareDispatchIdentityBeforeEmission(t *testing.T) { + for _, op := range x86PackedIntegerCompareTestNames { + operands := []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}} + if strings.HasPrefix(op, "V") { + operands = append(operands, Operand{Kind: OpReg, Reg: "X2"}) + } + for _, identity := range []Op{"", Op(op), Op(strings.ToLower(op)), "MOVOU", "UNKNOWN"} { + t.Run(op+"/"+string(identity), func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = "amd64" + ins := Instr{Op: identity, Args: operands} + before, temporary := output.String(), ctx.tmp + original := fmt.Sprintf("%#v", ins) + known, terminated, err := ctx.lowerPackedIntegerCompare(Op(op), ins) + invalid := identity != "" && !strings.EqualFold(string(identity), op) + if !known || terminated || (err != nil) != invalid || + invalid && (before != output.String() || temporary != ctx.tmp) || original != fmt.Sprintf("%#v", ins) { + t.Fatalf("dispatch identity changed source or emitted before rejection: %v", err) + } + }) + } + } +} + +func TestX86PackedIntegerCompareLegacyNativeOpcodeAndVirtualBoundaries(t *testing.T) { + for _, op := range x86PackedIntegerCompareTestNames[:8] { + for _, target := range x86RawReturnTargets { + t.Run(op+"/"+target.triple, func(t *testing.T) { + source := "TEXT ·query(SB),4,$0-0\n" + op + " (AX),X1\nRET\n" + x86UnboundVoidFormatSource(t, source, "query", target.arch, target.triple) + word, move := int(goWordSize(target.arch)), "MOVQ" + if word == 4 { + move = "MOVL" + } + // Keep the exact original m128 instruction. Only its actual + // pointer and destructive X input are established here; the + // complete native-object owner covers the family separately. + initialized := fmt.Sprintf("TEXT ·query(SB),4,$0-%d\n%s p+0(FP),AX\nPXOR X1,X1\n%s (AX),X1\nRET\n", word, move, op) + x86LEAActualTargetGoObject(t, target.arch, target.triple, initialized) + mnemonic := strings.ToLower(strings.ReplaceAll(op, "L", "D")) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86PackedIntegerCompareAPI(t, initialized, "func query(p *[16]byte)\n", target.arch, target.triple, api) + if err != nil { + t.Fatal("initialized original legacy source rejected", api, err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + if !strings.Contains(text, "asm sideeffect \""+mnemonic+" ($2), $0\"") { + t.Fatal("legacy m128 erased original opcode/fault semantics", api) + } + } + } + }) + } + for _, operand := range []string{"a+0(FP)", "16(SP)", "16(BP)"} { + t.Run(op+"/virtual/"+operand, func(t *testing.T) { + source := "TEXT ·query(SB),4,$64-16\n" + op + " " + operand + ",X1\nRET\n" + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source) + ir, err := x86PackedIntegerCompareAPI(t, source, "func query(a,b uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("virtual fields acquired native legacy m128 alignment: IR=%d err=%v", len(ir), err) + } + }) + } + } +} diff --git a/amd64_packed_logical_forms.go b/amd64_packed_logical_forms.go new file mode 100644 index 000000000..db8b6847f --- /dev/null +++ b/amd64_packed_logical_forms.go @@ -0,0 +1,292 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedLogicalEncoding uint8 + +const ( + x86PackedLogicalLegacyFP x86PackedLogicalEncoding = iota + x86PackedLogicalLegacyInteger + x86PackedLogicalVEX + x86PackedLogicalVEXOrEVEX + x86PackedLogicalEVEX +) + +type x86PackedLogicalSpec struct { + laneBits int + mode amd64PackedLogicalMode + encoding x86PackedLogicalEncoding + // These are opcode attributes, not inferred from a shared ytab. Go's + // avx_optabs supplies N4 for PS/D and N8 for PD/Q, and zeroing for each + // EVEX row. VEX-only integer and legacy forms have neither attribute. + broadcastBytes int + zeroing bool +} + +var x86PackedLogicalSpecs = map[Op]x86PackedLogicalSpec{ + "ANDPS": {32, amd64PackedLogicalAnd, x86PackedLogicalLegacyFP, 0, false}, + "ANDPD": {64, amd64PackedLogicalAnd, x86PackedLogicalLegacyFP, 0, false}, + "ANDNPS": {32, amd64PackedLogicalAndNot, x86PackedLogicalLegacyFP, 0, false}, + "ANDNPD": {64, amd64PackedLogicalAndNot, x86PackedLogicalLegacyFP, 0, false}, + "ORPS": {32, amd64PackedLogicalOr, x86PackedLogicalLegacyFP, 0, false}, + "ORPD": {64, amd64PackedLogicalOr, x86PackedLogicalLegacyFP, 0, false}, + "XORPS": {32, amd64PackedLogicalXor, x86PackedLogicalLegacyFP, 0, false}, + "XORPD": {64, amd64PackedLogicalXor, x86PackedLogicalLegacyFP, 0, false}, + "PAND": {8, amd64PackedLogicalAnd, x86PackedLogicalLegacyInteger, 0, false}, + "PANDN": {8, amd64PackedLogicalAndNot, x86PackedLogicalLegacyInteger, 0, false}, + "POR": {8, amd64PackedLogicalOr, x86PackedLogicalLegacyInteger, 0, false}, + "PXOR": {8, amd64PackedLogicalXor, x86PackedLogicalLegacyInteger, 0, false}, + "VANDPS": {32, amd64PackedLogicalAnd, x86PackedLogicalVEXOrEVEX, 4, true}, + "VANDPD": {64, amd64PackedLogicalAnd, x86PackedLogicalVEXOrEVEX, 8, true}, + "VANDNPS": {32, amd64PackedLogicalAndNot, x86PackedLogicalVEXOrEVEX, 4, true}, + "VANDNPD": {64, amd64PackedLogicalAndNot, x86PackedLogicalVEXOrEVEX, 8, true}, + "VORPS": {32, amd64PackedLogicalOr, x86PackedLogicalVEXOrEVEX, 4, true}, + "VORPD": {64, amd64PackedLogicalOr, x86PackedLogicalVEXOrEVEX, 8, true}, + "VXORPS": {32, amd64PackedLogicalXor, x86PackedLogicalVEXOrEVEX, 4, true}, + "VXORPD": {64, amd64PackedLogicalXor, x86PackedLogicalVEXOrEVEX, 8, true}, + "VPAND": {8, amd64PackedLogicalAnd, x86PackedLogicalVEX, 0, false}, + "VPANDN": {8, amd64PackedLogicalAndNot, x86PackedLogicalVEX, 0, false}, + "VPOR": {8, amd64PackedLogicalOr, x86PackedLogicalVEX, 0, false}, + "VPXOR": {8, amd64PackedLogicalXor, x86PackedLogicalVEX, 0, false}, + "VPANDD": {32, amd64PackedLogicalAnd, x86PackedLogicalEVEX, 4, true}, + "VPANDQ": {64, amd64PackedLogicalAnd, x86PackedLogicalEVEX, 8, true}, + "VPANDND": {32, amd64PackedLogicalAndNot, x86PackedLogicalEVEX, 4, true}, + "VPANDNQ": {64, amd64PackedLogicalAndNot, x86PackedLogicalEVEX, 8, true}, + "VPORD": {32, amd64PackedLogicalOr, x86PackedLogicalEVEX, 4, true}, + "VPORQ": {64, amd64PackedLogicalOr, x86PackedLogicalEVEX, 8, true}, + "VPXORD": {32, amd64PackedLogicalXor, x86PackedLogicalEVEX, 4, true}, + "VPXORQ": {64, amd64PackedLogicalXor, x86PackedLogicalEVEX, 8, true}, +} + +type x86PackedLogicalForm struct { + x86PackedBinaryForm + spec x86PackedLogicalSpec + source Operand +} + +// The first input is known; these bits still depend on the second input. +// ANDNOT is first & ^second in Go operand order, so it has AND's dependency. +// This numerical projection neither removes the first read nor grants storage. +func x86PackedLogicalSecondBits(mode amd64PackedLogicalMode, first byte) byte { + switch mode { + case amd64PackedLogicalAnd, amd64PackedLogicalAndNot: + return first + case amd64PackedLogicalOr: + return ^first + default: + return 255 // XOR cannot remove an unknown input bit. + } +} + +// This is a pure numerical-byte dependency, shared by independent consumers. +// A validated form and initialized mask are prerequisites; the result grants +// no memory, frame, source-selection or caller permission. Higher physical +// bytes are updated separately according to the consumer's encoding model. +func x86PackedLogicalDefinedBytes(ins Instr, form x86PackedLogicalForm, first, second, old uint64) uint64 { + width := x86CallerVectorByteMask(int64(form.byteWidth)) + known := first & second & width + if form.memory < 0 && ins.Args[form.first].Reg == ins.Args[form.second].Reg && + (form.spec.mode == amd64PackedLogicalXor || form.spec.mode == amd64PackedLogicalAndNot) { + known = width + } + if form.mask >= 0 && !form.zeroing { + known &= old // A defined mask is not necessarily all ones. + } + return known +} + +// Reuse the complete logical grammar and its numerical zero identity. A +// legacy XMM operation preserves upper aliases and requires aligned m128; +// neither the descriptor nor a same-register zero grants memory authority. +func x86TypedLegacyLogicalByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + local := ins + local.Op = op + form, known, err := x86PackedLogicalProperties(local, arch) + if known && ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86TypedByteEffect{}, true, fmt.Errorf("packed logical dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + if !known || err != nil || !form.legacy || form.mmx { + return x86TypedByteEffect{}, known && (err != nil || form.legacy && !form.mmx), err + } + effect := x86PackedBinaryDependencyEffect(ins, form.x86PackedBinaryForm, form.source) + return x86PackedLogicalZeroDependencies(ins, form, effect), true, nil +} + +// Apply the shared numerical identity only after the consumer has validated +// its encoding and storage roles. A memory operand never becomes a zero idiom. +func x86PackedLogicalZeroDependencies(ins Instr, form x86PackedLogicalForm, effect x86TypedByteEffect) x86TypedByteEffect { + if x86PackedLogicalDefinedBytes(ins, form, 0, 0, 0) != 0 { + for index := range effect.operands { + effect.operands[index].read = false + } + } + return effect +} + +// These fields describe only an operand grammar. Arithmetic, interleave and +// bitwise effects remain in their own typed operation specs and consumers. +type x86PackedBinarySpec struct { + encoding x86PackedLogicalEncoding + broadcastBytes int + zeroing bool +} + +type x86PackedBinaryForm struct { + byteWidth, first, second, destination int + mask, memory, memoryBytes int + legacy, mmx, broadcast, zeroing bool +} + +// Go accepts some X/Y high-bank spellings in mode32 without proving their +// physical register identity. Apply this only after the family's syntax has +// been validated; it grants no initialized values, ABI or memory permissions. +func x86Mode32VectorRegisterContext(ins Instr, goarch string) error { + if goarch != "386" { + return nil + } + for _, operand := range ins.Args { + index, valid := amd64VectorRegisterIndex(operand.Reg, amd64VectorByteWidth(operand.Reg)) + if operand.Kind == OpReg && valid && index >= 8 { + return fmt.Errorf("%w: 386 %s register %s has no proved physical mode32 binding", ErrProbeNeedsContext, ins.Op, operand.Reg) + } + } + return nil +} + +// This grammar supplies no initialized values, address bounds or caller +// privileges. The emitter and the private source/CFG proof share only forms. +func x86PackedLogicalProperties(ins Instr, goarch string) (x86PackedLogicalForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86PackedLogicalSpecs[Op(base)] + if !recognized { + return x86PackedLogicalForm{}, false, nil + } + local := ins + if spec.encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(ins, goarch) + if err != nil { + return x86PackedLogicalForm{}, true, err + } + } + form, err := x86PackedBinaryProperties(local, goarch, x86PackedBinarySpec{ + encoding: spec.encoding, broadcastBytes: spec.broadcastBytes, zeroing: spec.zeroing, + }) + if err == nil { + err = x86Mode32VectorRegisterContext(ins, goarch) + } + result := x86PackedLogicalForm{x86PackedBinaryForm: form, spec: spec} + if err == nil { + result.source = local.Args[form.first] + } + return result, true, err +} + +func x86PackedBinaryProperties(ins Instr, goarch string, spec x86PackedBinarySpec) (x86PackedBinaryForm, error) { + return x86PackedBinaryOperandGrammar(ins, goarch, spec, false) +} + +// Only a validated unary projection or physical decoder may avoid the named +// four-operand frontend cap. Every other grammar axis keeps the actual GOARCH. +func x86PackedBinaryOperandGrammar(ins Instr, goarch string, spec x86PackedBinarySpec, mode32MaskOperands bool) (x86PackedBinaryForm, error) { + _, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + form := x86PackedBinaryForm{first: 0, second: 1, destination: len(ins.Args) - 1, mask: -1, memory: -1} + fail := func() (x86PackedBinaryForm, error) { + return form, fmt.Errorf("%s %s operands/suffix outside Go packed binary grammar: %q", goarch, ins.Op, ins.Raw) + } + if goarch != "386" && goarch != "amd64" { + return fail() + } + form.legacy = spec.encoding == x86PackedLogicalLegacyFP || spec.encoding == x86PackedLogicalLegacyInteger + if form.legacy { + if dotted || len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return fail() + } + form.byteWidth = 16 + _, form.mmx = amd64ParseMReg(ins.Args[1].Reg) + if form.mmx { + if spec.encoding != x86PackedLogicalLegacyInteger || goarch != "amd64" { + return fail() + } + form.byteWidth = 8 + } + } else { + switch suffix { + case "": + if dotted { + return fail() + } + case "Z": + form.zeroing = true + case "BCST": + form.broadcast = true + case "BCST.Z": + form.broadcast, form.zeroing = true, true + default: + return fail() + } + if len(ins.Args) != 3 && len(ins.Args) != 4 || form.broadcast && spec.broadcastBytes == 0 || form.zeroing && !spec.zeroing { + return fail() + } + if len(ins.Args) == 4 { + if goarch == "386" && !mode32MaskOperands || spec.encoding == x86PackedLogicalVEX || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + form.mask = 2 + } else if form.zeroing { + return fail() + } + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + } + register := func(operand Operand) bool { + if form.mmx { + _, valid := amd64ParseMReg(operand.Reg) + return operand.Kind == OpReg && valid + } + if form.legacy { + index, valid := amd64ParseXReg(operand.Reg) + return operand.Kind == OpReg && valid && (goarch == "amd64" && index < 16 || goarch == "386" && index < 8) + } + if spec.encoding == x86PackedLogicalVEX { + return (form.byteWidth == 16 || form.byteWidth == 32) && amd64VEXVectorRegister(operand, form.byteWidth) + } + return amd64GoPackedVectorMoveRegister(operand, form.byteWidth, goarch) + } + if !register(ins.Args[form.destination]) || !register(ins.Args[form.second]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) { + if first.Kind == OpSym { + symbol := strings.TrimSpace(first.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if first.Kind == OpMem { + // Go asmandsz requires signed disp32 in amd64 mode. Its historical + // 386 rule additionally admits uint32 spellings for the same bits; + // packed logical rows have no LEAL-only overflow exception. + displacement := first.Mem.Off + if displacement != int64(int32(displacement)) && + (goarch != "386" || displacement != int64(uint32(displacement))) { + return fail() + } + if !x86MemoryRegistersValidForArch(first.Mem, goarch) || first.Mem.Index != "" && + (first.Mem.Index == SP || first.Mem.Scale != 1 && first.Mem.Scale != 2 && first.Mem.Scale != 4 && first.Mem.Scale != 8) { + return fail() + } + } + form.memory, form.memoryBytes = form.first, form.byteWidth + if form.broadcast { + form.memoryBytes = spec.broadcastBytes + } + } else if form.broadcast || !register(first) { + return fail() + } + return form, nil +} diff --git a/amd64_packed_logical_forms_test.go b/amd64_packed_logical_forms_test.go new file mode 100644 index 000000000..0993b128d --- /dev/null +++ b/amd64_packed_logical_forms_test.go @@ -0,0 +1,344 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" +) + +func x86LogicalOpcodeNames() []Op { + names := make([]Op, 0, len(x86PackedLogicalSpecs)) + for opcode := range x86PackedLogicalSpecs { + names = append(names, opcode) + } + sort.Slice(names, func(i, j int) bool { return names[i] < names[j] }) + return names +} + +func TestX86PackedLogicalSpecsBindEveryGoOpcodeAttribute(t *testing.T) { + if len(x86PackedLogicalSpecs) != 32 { + t.Fatal("incomplete packed logical family") + } + if !goToolchainAtLeast(runtime.Version(), 1, 27) { + return + } + dir := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86") + avx, err := os.ReadFile(filepath.Join(dir, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + legacy, err := os.ReadFile(filepath.Join(dir, "asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, opcode := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[opcode] + if spec.encoding <= x86PackedLogicalLegacyInteger { + row := "{A" + string(opcode) + ", " + start := strings.Index(string(legacy), row) + if start < 0 { + t.Fatalf("Go omits %s", opcode) + } + line := strings.SplitN(string(legacy)[start:], "\n", 2)[0] + table := "yxm" + if spec.encoding == x86PackedLogicalLegacyInteger { + table = "ymm" + } + if !strings.Contains(line, ", "+table+",") { + t.Fatalf("Go legacy row differs: %s", line) + } + continue + } + prefix := "{as: A" + string(opcode) + ", ytab: " + start := strings.Index(string(avx), prefix) + if start < 0 { + t.Fatalf("Go omits %s", opcode) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + table := "_yvandnpd" + if spec.encoding == x86PackedLogicalVEX { + table = "_yvaddsubpd" + } + if spec.encoding == x86PackedLogicalEVEX { + table = "_yvblendmpd" + } + if !strings.HasPrefix(row, prefix+table+",") { + t.Fatalf("Go ytab changed for %s", opcode) + } + if spec.encoding == x86PackedLogicalVEX { + if strings.Contains(row, "evex") || spec.broadcastBytes != 0 || spec.zeroing { + t.Fatalf("VEX attributes changed: %s", row) + } + continue + } + for _, line := range strings.Split(row, "\n") { + if !strings.Contains(line, "avxEscape | evex") { + continue + } + if !strings.Contains(line, fmt.Sprintf("evexBcstN%d", spec.broadcastBytes)) || !strings.Contains(line, "evexZeroingEnabled") || + strings.Contains(line, "evexSae") || strings.Contains(line, "evexRounding") { + t.Fatalf("%s opcode attributes disagree with Go: %s", opcode, line) + } + } + } +} + +// Rows come from the actual yxm/ymm, _yvaddsubpd, _yvandnpd and +// _yvblendmpd tables; memory and all opcode-specific suffix axes are included. +func x86LogicalAcceptedForms(opcode Op, arch string) []string { + spec := x86PackedLogicalSpecs[opcode] + if spec.encoding <= x86PackedLogicalLegacyInteger { + last := 15 + if arch == "386" { + last = 7 + } + rows := []string{fmt.Sprintf("%s X0,X%d", opcode, last), fmt.Sprintf("%s 8(AX),X%d", opcode, last)} + if spec.encoding == x86PackedLogicalLegacyInteger && arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s M0,M7", opcode), fmt.Sprintf("%s 8(AX),M7", opcode)) + } + return rows + } + var rows []string + widths := []string{"X", "Y", "Z"} + if spec.encoding == x86PackedLogicalVEX { + widths = widths[:2] + } + for _, width := range widths { + last := 31 + if spec.encoding == x86PackedLogicalVEX { + last = 15 + } + if arch == "386" && width == "Z" { + last = 7 + } + dest := fmt.Sprintf("%s%d", width, last) + rows = append(rows, fmt.Sprintf("%s %s0,%s1,%s", opcode, width, width, dest), fmt.Sprintf("%s 8(AX),%s1,%s", opcode, width, dest)) + if spec.broadcastBytes == 0 { + continue + } + rows = append(rows, fmt.Sprintf("%s.BCST 8(AX),%s1,%s", opcode, width, dest)) + if arch == "386" { + continue + } + rows = append(rows, + fmt.Sprintf("%s %s0,%s1,K7,%s", opcode, width, width, dest), + fmt.Sprintf("%s.Z %s0,%s1,K1,%s", opcode, width, width, dest), + fmt.Sprintf("%s 8(AX),%s1,K7,%s", opcode, width, dest), + fmt.Sprintf("%s.Z 8(AX),%s1,K1,%s", opcode, width, dest), + fmt.Sprintf("%s.BCST 8(AX),%s1,K7,%s", opcode, width, dest), + fmt.Sprintf("%s.BCST.Z 8(AX),%s1,K1,%s", opcode, width, dest)) + } + return rows +} + +func TestX86PackedLogicalCompleteGoRowsAndLLVMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source, goSource strings.Builder + source.WriteString("TEXT logicalforms(SB),$0-0\n") + goSource.WriteString("TEXT logicalforms(SB),$0-0\n") + for _, op := range x86LogicalOpcodeNames() { + for _, row := range x86LogicalAcceptedForms(op, target.arch) { + fmt.Fprintln(&goSource, row) + if target.arch == "386" { + // Preserve Go's full source grammar independently. LLVM + // object generation requires physically bound registers. + row = strings.NewReplacer("X31", "X7", "Y31", "Y7", "X15", "X7", "Y15", "Y7").Replace(row) + } + fmt.Fprintln(&source, row) + parsed, err := Parse(ArchAMD64, "TEXT one(SB),$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, recognized, err := x86PackedLogicalProperties(parsed.Funcs[0].Instrs[1], target.arch); !recognized || err != nil { + t.Fatalf("%s: %v", row, err) + } + } + } + source.WriteString("RET\n") + goSource.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, goSource.String(), true) + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "logicalforms", target.arch, target.triple) + compileLLVMToObject(t, llc, target.triple, "logical.ll", "logical.o", ir) + }) + } +} + +func TestX86PackedLogicalRejectsInvalidFormsBeforeEmission(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, opcode := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[opcode] + args := "X0,X1,X2" + if spec.encoding <= x86PackedLogicalLegacyInteger { + args = "X0,X1" + } + rows := []string{string(opcode) + ".SAE " + args, string(opcode) + ".RN_SAE " + args, string(opcode) + ".Z.Z " + args, + string(opcode) + " $3," + args, string(opcode) + " X0,(AX)", string(opcode) + " X0,Y1", string(opcode) + " X0,X1,X2,X3"} + memoryTail := ",X0,X1" + if spec.encoding <= x86PackedLogicalLegacyInteger { + memoryTail = ",X0" + } + rows = append(rows, string(opcode)+" (AX)(CX*3)"+memoryTail, string(opcode)+" (AX)(SP*1)"+memoryTail) + if spec.encoding <= x86PackedLogicalLegacyInteger { + rows = append(rows, string(opcode)+" X16,X0", string(opcode)+".BCST 8(AX),X0") + if spec.encoding == x86PackedLogicalLegacyFP { + rows = append(rows, string(opcode)+" M0,M1") + } + } else { + rows = append(rows, string(opcode)+".Z X0,X1,X2", string(opcode)+" X0,X1,K0,X2", string(opcode)+".BCST X0,X1,X2", + string(opcode)+".Z.BCST (AX),X0,K1,X1") + if spec.encoding == x86PackedLogicalVEX { + rows = append(rows, string(opcode)+" Z0,Z1,Z2", string(opcode)+" X16,X1,X2", string(opcode)+".BCST (AX),X0,X1", string(opcode)+" X0,X1,K1,X2") + } + } + if arch == "386" { + rows = append(rows, string(opcode)+" 8(R9),"+args) + } + for _, row := range rows { + t.Run(arch+"/"+row, func(t *testing.T) { + assembly := "TEXT bad(SB),$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, arch, assembly, false) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + return + } + instruction := file.Funcs[0].Instrs[1] + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := out.String() + ok, term, err := ctx.lowerPackedLogicalFamily(instruction.Op, instruction) + if !ok || term || err == nil || out.String() != before { + t.Fatalf("invalid form emitted (%v,%v,%v)", ok, term, err) + } + }) + } + } + } +} + +func TestX86PackedLogicalDispatchIdentityBeforeEmission(t *testing.T) { + for _, opcode := range x86LogicalOpcodeNames() { + for _, sourceOp := range []Op{"", opcode, Op(strings.ToLower(string(opcode))), "UNKNOWN", "MOVQ"} { + row := x86LogicalAcceptedForms(opcode, "amd64")[0] + file, err := Parse(ArchAMD64, "TEXT one(SB),$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + instruction.Op = sourceOp + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + ok, term, err := ctx.lowerPackedLogicalFamily(opcode, instruction) + if sourceOp == "UNKNOWN" || sourceOp == "MOVQ" { + if !ok || term || err == nil || out.String() != before { + t.Fatalf("%s/%s conflicting helper identity emitted", opcode, sourceOp) + } + } else if !ok || term || err != nil { + t.Fatalf("%s/%s helper compatibility lost: %v", opcode, sourceOp, err) + } + if instruction.Op != sourceOp { + t.Fatal("helper rewrote source identity") + } + } + } +} + +func TestX86PackedLogicalSymbolAddressIsNotMemory(t *testing.T) { + for _, opcode := range x86LogicalOpcodeNames() { + t.Run(string(opcode), func(t *testing.T) { + args := "X0,X1" + if x86PackedLogicalSpecs[opcode].encoding <= x86PackedLogicalLegacyInteger { + args = "X0" + } + source := "TEXT bad(SB),$0-0\n" + string(opcode) + " $symbol(SB)," + args + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + instruction := file.Funcs[0].Instrs[1] + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + ok, term, err := ctx.lowerPackedLogicalFamily(instruction.Op, instruction) + if !ok || term || err == nil || out.String() != before { + t.Fatalf("symbol address acquired memory load/emission: (%v,%v,%v)", ok, term, err) + } + }) + } +} + +func TestX86PackedLogicalMemoryDisplacementMatchesGoEncoding(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, opcode := range x86LogicalOpcodeNames() { + args := "X0,X1" + if x86PackedLogicalSpecs[opcode].encoding <= x86PackedLogicalLegacyInteger { + args = "X0" + } + invalid := []int64{-2147483649, 4294967296} + if arch == "amd64" { + invalid = append(invalid, 2147483648, 4294967295) + } + for _, displacement := range invalid { + t.Run(fmt.Sprintf("%s/%s/%d", arch, opcode, displacement), func(t *testing.T) { + source := fmt.Sprintf("TEXT bad(SB),$0-0\n%s %d(AX)(CX*2),%s\nRET\n", opcode, displacement, args) + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := out.String() + ok, term, err := ctx.lowerPackedLogicalFamily(instruction.Op, instruction) + if !ok || term || err == nil || out.String() != before { + t.Fatalf("unencodable displacement emitted: (%v,%v,%v)", ok, term, err) + } + }) + } + } + } +} + +func TestX86PackedLogicalBoundaryDisplacementsGoAndLLVMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT boundaries(SB),$0-0\n") + displacements := []int64{-2147483648, 2147483647} + if target.arch == "386" { + displacements = append(displacements, 2147483648, 4294967295) + } + for _, opcode := range x86LogicalOpcodeNames() { + args := "X0,X1" + if x86PackedLogicalSpecs[opcode].encoding <= x86PackedLogicalLegacyInteger { + args = "X0" + } + for _, displacement := range displacements { + fmt.Fprintf(&source, "%s %d(AX)(CX*2),%s\n", opcode, displacement, args) + } + } + source.WriteString("RET\n") + ir := x86UnboundVoidScalarFormatObject(t, source.String(), "boundaries", target.arch, target.triple) + compileLLVMToObject(t, llc, target.triple, "boundary.ll", "boundary.o", ir) + }) + } +} diff --git a/amd64_packed_logical_oracle_test.go b/amd64_packed_logical_oracle_test.go new file mode 100644 index 000000000..5c81c0f6d --- /dev/null +++ b/amd64_packed_logical_oracle_test.go @@ -0,0 +1,73 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestX86LogicalRuntimeBatchesRetainCompleteInventory(t *testing.T) { + fixtures := x86LogicalRuntimeFixtures(t) + if len(fixtures) != 472 { + t.Fatalf("complete opcode/width/memory/mask inventory: got %d, want 472", len(fixtures)) + } + seen := make(map[string]bool, len(fixtures)) + for index, batch := range x86LogicalRuntimeBatches(fixtures) { + if len(batch) == 0 || len(batch) > 64 { + t.Errorf("batch %d has %d fixtures, require 1..64", index, len(batch)) + } + for _, fixture := range batch { + if seen[fixture.name] { + t.Fatalf("duplicate fixture %s", fixture.name) + } + seen[fixture.name] = true + } + } + if len(seen) != len(fixtures) { + t.Fatalf("batch union lost fixtures: got %d, want %d", len(seen), len(fixtures)) + } + for _, fixture := range fixtures { + if !seen[fixture.name] { + t.Errorf("missing fixture %s", fixture.name) + } + } +} + +// This is the real numerical oracle with a deliberately wrong implementation, +// not an instruction-semantics pass. Its former return code was 256, which a +// Unix parent observes as success. A large case ID must remain diagnostic data. +func TestX86LogicalRuntimeOracleCannotWrapFailureToSuccess(t *testing.T) { + fixture := x86LogicalRuntimeFixtures(t)[245] + program := x86LogicalRuntimeC([]x86LogicalRuntimeFixture{fixture}) + fmt.Sprintf(` +void %s(uint8_t *out, const uint8_t *a, const uint8_t *b, uint64_t mask) { + memset(out, 0, 65); +} +`, fixture.name) + requireX86GeneratedOracleFailure(t, program, fixture.name+" numerical/canary failure") +} + +func requireX86GeneratedOracleFailure(t *testing.T, program, diagnostic string) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, executable := filepath.Join(dir, "mutation.c"), filepath.Join(dir, "mutation") + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + arguments := execution.command(executable) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, arguments[0], arguments[1:]...)) + var failure *exec.ExitError + if !errors.As(err, &failure) || failure.ExitCode() != 1 { + t.Fatalf("wrong result must terminate with failure 1, got %v; output=%s", err, output) + } + if !strings.Contains(string(output), diagnostic) { + t.Fatalf("failure diagnostic lost the exact fixture: %s", output) + } +} diff --git a/amd64_packed_logical_runtime_test.go b/amd64_packed_logical_runtime_test.go new file mode 100644 index 000000000..f5949c65c --- /dev/null +++ b/amd64_packed_logical_runtime_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// The numerical oracle derives semantics from the source mnemonic, not the +// emitter's mode field; a wrong opcode-to-mode registration must still fail. +func x86LogicalIndependentCOperator(t *testing.T, opcode Op) string { + t.Helper() + name := string(opcode) + switch { + case strings.Contains(name, "ANDN"): + return "aByte & (uint8_t)~b[i]" + case strings.Contains(name, "AND"): + return "aByte & b[i]" + case strings.Contains(name, "XOR"): + return "aByte ^ b[i]" + case strings.Contains(name, "OR"): + return "aByte | b[i]" + default: + t.Fatalf("unknown numerical oracle opcode %s", opcode) + return "" + } +} + +func TestCrossLinuxRuntimeMatrixX86PackedLogicalCompleteSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires Linux/amd64 runtime counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 llc/clang missing") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + fixtures := x86LogicalRuntimeFixtures(t) + for index, batch := range x86LogicalRuntimeBatches(fixtures) { + t.Run(fmt.Sprintf("batch-%02d", index), func(t *testing.T) { + var assembly strings.Builder + sigs := make(map[string]FuncSig, len(batch)) + for _, fixture := range batch { + assembly.WriteString(fixture.assembly) + sigs[fixture.name] = fixture.signature + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "logical_complete_semantics", + triple, ir, x86LogicalRuntimeC(batch), prefix) + }) + } + t.Logf("%d exact opcode/width/memory/mask programs; 16 independently expected numerical/canary inputs each", len(fixtures)) +} + +type x86LogicalRuntimeFixture struct { + name, assembly, declaration, checks string + signature FuncSig +} + +func x86LogicalRuntimeBatches(fixtures []x86LogicalRuntimeFixture) [][]x86LogicalRuntimeFixture { + const batchSize = 64 + var batches [][]x86LogicalRuntimeFixture + for start := 0; start < len(fixtures); start += batchSize { + end := start + batchSize + if end > len(fixtures) { + end = len(fixtures) + } + batches = append(batches, fixtures[start:end]) + } + return batches +} + +func x86LogicalRuntimeC(fixtures []x86LogicalRuntimeFixture) string { + var declarations, checks, calls strings.Builder + for _, fixture := range fixtures { + declarations.WriteString(fixture.declaration) + // Keep independent reference loops out of one giant optimizer graph. + // noinline also prevents -O2 from reconstructing that graph in main. + fmt.Fprintf(&checks, "static __attribute__((noinline)) int check_%s(void) {\n", fixture.name) + checks.WriteString("const uint64_t masks[]={0,1,0x55aa55aa55aa55aaULL,~0ULL};\n") + checks.WriteString(fixture.checks) + checks.WriteString("return 0;\n}\n") + // Case IDs belong in diagnostics, not process exit codes: Unix keeps + // only eight bits and would report return 256 as successful execution. + fmt.Fprintf(&calls, "if (check_%s()) { fprintf(stderr, \"%s numerical/canary failure\\n\"); return 1; }\n", + fixture.name, fixture.name) + } + return "#include \n#include \n#include \n" + + declarations.String() + checks.String() + + "int main(void) {\n" + calls.String() + "return 0;\n}\n" +} + +func x86LogicalRuntimeFixtures(t *testing.T) []x86LogicalRuntimeFixture { + t.Helper() + var fixtures []x86LogicalRuntimeFixture + count := 0 + for _, opcode := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[opcode] + widths := []int{16, 32, 64} + if spec.encoding <= x86PackedLogicalLegacyInteger { + widths = widths[:1] + } + if spec.encoding == x86PackedLogicalVEX { + widths = widths[:2] + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, memory := range []string{"register", "memory", "broadcast"} { + if memory == "broadcast" && spec.broadcastBytes == 0 { + continue + } + for _, mode := range []string{"unmasked", "merge", "zero"} { + if mode != "unmasked" && spec.broadcastBytes == 0 { + continue + } + var assembly, declarations, checks strings.Builder + name := fmt.Sprintf("logical%d", count) + count++ + legacy := spec.encoding <= x86PackedLogicalLegacyInteger + first := bank + "1" + if memory != "register" { + first = "(BX)" + } + operation := string(opcode) + if memory == "broadcast" { + operation += ".BCST" + } + if mode == "zero" { + operation += ".Z" + } + row := operation + " " + first + "," + bank + "0" + destination := 2 + if legacy { + destination = 0 + } else { + if mode != "unmasked" { + row += ",K1" + } + row += "," + bank + "2" + } + fmt.Fprintf(&assembly, "TEXT %s(SB),$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (CX),Z2\n", name) + if memory == "register" { + assembly.WriteString("VMOVUPS (BX),Z1\n") + } + assembly.WriteString("KMOVQ DX,K1\nSTC\n" + row + "\n") + fmt.Fprintf(&assembly, "VMOVUPS Z%d,(AX)\nSETCS 64(AX)\nRET\n", destination) + signature := FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + operator := x86LogicalIndependentCOperator(t, opcode) + maskExpression := "1" + if mode != "unmasked" { + maskExpression = fmt.Sprintf("((mask>>(i/%d))&1)", spec.laneBits/8) + } + inactive := "b[i]" + if mode == "zero" { + inactive = "0" + } + upper := "0" + if legacy { + upper = "b[i]" + } + index := "i" + if memory == "broadcast" { + index = fmt.Sprintf("i%%%d", spec.broadcastBytes) + } + aOffset := 1 + if legacy && memory == "memory" { + // The original legacy m128 instruction requires 16-byte + // alignment. Keep other source forms deliberately unaligned; + // the independent native fault root checks legacy misalignment. + aOffset = 16 + } + fmt.Fprintf(&checks, ` + for(int round=0;round<4;++round) for(int m=0;m<4;++m) { + uint64_t mask=masks[m]; _Alignas(16) uint8_t aStorage[96]; uint8_t bStorage[68],output[67]; uint8_t *a=aStorage+%[9]d,*b=bStorage+3; + memset(aStorage,0xa7,96); memset(bStorage,0xb9,68); memset(output,0xc3,67); + for(int i=0;i<64;++i) { a[i]=(uint8_t)(i*19+round*83+7); b[i]=(uint8_t)(i*31+round*59+13); } + %[1]s(output+1,a,b,mask); + if(output[0]!=0xc3 || output[66]!=0xc3 || output[65]!=1) return %[2]d; + for(int i=0;i<64;++i) { + uint8_t aByte=a[%[3]s]; uint8_t want=i<%[4]d ? ((%[5]s) ? (%[6]s) : (%[7]s)) : (%[8]s); + if(output[i+1]!=want || a[i]!=(uint8_t)(i*19+round*83+7) || b[i]!=(uint8_t)(i*31+round*59+13)) return %[2]d; + } + for(int i=0;i<96;++i) if((i<%[9]d || i>=%[9]d+64) && aStorage[i]!=0xa7) return %[2]d; + if(bStorage[2]!=0xb9 || bStorage[67]!=0xb9) return %[2]d; + } +`, name, count+10, index, width, maskExpression, operator, inactive, upper, aOffset) + if mode != "unmasked" && memory != "register" { + // A zero relevant mask suppresses every memory read, + // including broadcast. NULL is never dereferenced. + fmt.Fprintf(&checks, `{ uint8_t b[64],out[65]; memset(b,0x5a,64); %[1]s(out,0,b,0); + for(int i=0;i<64;++i) { uint8_t want=i<%[2]d ? (%[3]s) : 0; if(out[i]!=want) return %[4]d; } } +`, name, width, map[string]string{"merge": "0x5a", "zero": "0"}[mode], count+10) + } + fixtures = append(fixtures, x86LogicalRuntimeFixture{ + name: name, assembly: assembly.String(), declaration: declarations.String(), + checks: checks.String(), signature: signature, + }) + } + } + } + } + return fixtures +} + +func TestX86PackedLogicalXWritesClearWiderAliases(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires amd64 or Rosetta; Linux counterpart executes the same oracle") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple, prefix = "x86_64-apple-darwin", []string{"/usr/bin/arch", "-x86_64"} + } + for _, opcode := range []string{"VXORPS", "VPXOR", "VPXORD"} { + t.Run(opcode, func(t *testing.T) { + source := "TEXT logicalalias(SB),$0-16\nMOVQ out+0(FP),AX\nMOVQ in+8(FP),BX\nVMOVUPS (BX),Z0\n" + opcode + " X0,X0,X0\nVMOVUPS Z0,(AX)\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{ + "logicalalias": {Name: "logicalalias", Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}}, + }}) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void logicalalias(uint8_t *, const uint8_t *); +int main(void) { + uint8_t in[66], out[66]; + for (int k=0; k<66; ++k) { in[k]=(uint8_t)(k*19+7); out[k]=0xa5; } + logicalalias(out+1,in+1); + if(out[0]!=0xa5 || out[65]!=0xa5) return 2; + for(int k=1; k<65; ++k) if(out[k]!=0) return k+10; + for(int k=0; k<66; ++k) if(in[k]!=(uint8_t)(k*19+7)) return 3; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, strings.ToLower(opcode)+"_alias", triple, ir, mainC, prefix) + }) + } +} diff --git a/amd64_packed_low_multiply_forms.go b/amd64_packed_low_multiply_forms.go new file mode 100644 index 000000000..e3a6c5dc4 --- /dev/null +++ b/amd64_packed_low_multiply_forms.go @@ -0,0 +1,7 @@ +package plan9asm + +type x86PackedLowMultiplyForm = x86PackedArithmeticForm + +func x86PackedLowMultiplyProperties(op Op, ins Instr, arch string) (x86PackedLowMultiplyForm, bool, error) { + return x86PackedArithmeticProperties(op, ins, arch, x86PackedArithmeticLowMultiply) +} diff --git a/amd64_packed_low_multiply_forms_test.go b/amd64_packed_low_multiply_forms_test.go new file mode 100644 index 000000000..5903e23bf --- /dev/null +++ b/amd64_packed_low_multiply_forms_test.go @@ -0,0 +1,259 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86PackedLowMultiplyCompleteGoEncoderRows(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APMULLD, yxm_q4, Pq4, opBytes{0x40}}", + "{Zm_r, 1, argList{Yxm, Yxr}}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatal("actual legacy Go row changed", row) + } + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + lowCount := 0 + for _, spec := range x86PackedArithmeticSpecs { + if spec.family == x86PackedArithmeticLowMultiply { + lowCount++ + } + } + if lowCount != 3 { + t.Fatal("incomplete D/Q low-product inventory") + } + for _, name := range []string{"VPMULLD", "VPMULLQ"} { + table, w, broadcast, rows := "_yvandnpd", 0, 4, 5 + if name == "VPMULLQ" { + table, w, broadcast, rows = "_yvblendmpd", 1, 8, 3 + } + start := strings.Index(string(avx), "{as: A"+name+",") + if start < 0 { + t.Fatal("missing Go opcode", name) + } + record := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(record, "ytab: "+table+", prefix: Pavx") || strings.Count(record, "avxEscape") != rows { + t.Fatal("Go opcode row inventory changed", record) + } + if name == "VPMULLD" { + for _, width := range []int{128, 256} { + want := fmt.Sprintf("avxEscape | vex%d | vex66 | vex0F38 | vexW0, 0x40,", width) + if !strings.Contains(record, want) { + t.Fatal("Go VEX attributes changed", want) + } + } + } + for _, width := range []int{128, 256, 512} { + want := fmt.Sprintf("avxEscape | evex%d | evex66 | evex0F38 | evexW%d, evexN%d | evexBcstN%d | evexZeroingEnabled, 0x40,", width, w, width/8, broadcast) + if !strings.Contains(record, want) { + t.Fatal("Go EVEX attributes changed", want) + } + } + start = strings.Index(string(avx), "var "+table+" =") + if start < 0 { + t.Fatal("missing Go operand table", table) + } + operands := strings.SplitN(string(avx)[start:], "\n}", 2)[0] + var expected []string + if name == "VPMULLD" { + expected = append(expected, + "{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr, Yxr}}", + "{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yym, Yyr, Yyr}}") + } + for _, classes := range [][2]string{{"YxmEvex", "YxrEvex"}, {"YymEvex", "YyrEvex"}, {"Yzm", "Yzr"}} { + expected = append(expected, + fmt.Sprintf("{zcase: Zevex_rm_v_r, zoffset: 0, args: argList{%s, %s, %s}}", classes[0], classes[1], classes[1]), + fmt.Sprintf("{zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{%s, %s, Yknot0, %s}}", classes[0], classes[1], classes[1])) + } + if strings.Count(operands, "argList{") != len(expected) { + t.Fatal("Go operand row inventory changed", operands) + } + for _, row := range expected { + if !strings.Contains(operands, row) { + t.Fatal("Go operand classes/encoding changed", row) + } + } + } +} + +func x86LowMultiplyFormatRows(arch string) []string { + var rows []string + for _, source := range []string{"X0", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-16(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, "PMULLD "+source+",X1") + } + for _, op := range []string{"VPMULLD", "VPMULLQ"} { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + for _, broadcast := range []bool{false, true} { + if broadcast && source == bank+"0" { + continue + } + for mode := 0; mode < 3; mode++ { + if arch == "386" && mode != 0 { + continue + } + name, args := op, source+","+bank+"1" + if broadcast { + name += ".BCST" + } + if mode != 0 { + args += ",K7" + } + if mode == 2 { + name += ".Z" + } + rows = append(rows, name+" "+args+","+bank+"2") + } + } + } + } + } + return rows +} + +func TestX86PackedLowMultiplyGoFormatsAndConsumerShapes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86LowMultiplyFormatRows(arch) { + t.Run(arch+"/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$64-64\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch) + if !known || err != nil { + t.Fatal("valid Go shape lost", known, err) + } + if !x86CallerVectorMoveKnown(ins) { + t.Fatal("caller consumer lost valid family") + } + if form.memory >= 0 { + width, write, known := x86InstructionMemoryShape(ins, form.memory, arch) + want := form.byteWidth + if strings.Contains(row, ".BCST") { + want = 4 + if strings.HasPrefix(row, "VPMULLQ") { + want = 8 + } + } + if !known || write || width != int64(want) { + t.Fatal("read tuple differs from Go", width, write, known, want) + } + } + }) + } + } +} + +func TestX86PackedLowMultiplyMode32HighBanksRemainContext(t *testing.T) { + for _, name := range []string{"VPMULLD", "VPMULLQ"} { + for _, bank := range []string{"X", "Y"} { + for _, register := range []int{8, 15, 16, 31} { + row := fmt.Sprintf("%s %s%d,%s1,%s2", name, bank, register, bank, bank) + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + for _, target := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "386", TargetTriple: target, Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go syntax invented a physical high bank", row, target, err) + } + } + } + } + } +} + +func TestX86PackedLowMultiplyResultBytes(t *testing.T) { + for _, row := range []string{"PMULLW X0,X1", "VPMULLW X0,X1,X2", "PMULLD X0,X1", "VPMULLD X0,X1,X2", "VPMULLQ X0,X1,X2"} { + for _, arch := range []string{"amd64", "386"} { + for missing := -1; missing < 16; missing++ { + ins := x86ScalarLaneInstruction(t, row) + state := x86ResultState{flags: 0x3f} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + state.vectorUpper[1], state.vectorUpper[2] = 0xffff0000, 0xff000000 + if missing >= 0 { + state.xmm[0] &^= 1 << uint(missing) + } + if !x86ResultTransfer(ins, &state, nil, 0, arch) { + t.Fatalf("registered low multiply result effect missing: %s/%s", arch, row) + } + laneBytes := 4 + if strings.Contains(row, "LLW") { + laneBytes = 2 + } else if strings.Contains(row, "LLQ") { + laneBytes = 8 + } + want := uint64(0xffff) + if missing >= 0 { + want &^= (uint64(1)< len(rows) { + end = len(rows) + } + t.Run(fmt.Sprint(begin), func(t *testing.T) { + var source, declarations strings.Builder + for index, row := range rows[begin:end] { + if strings.HasPrefix(row, "PMULLD ") && (strings.Contains(row, "(FP)") || strings.Contains(row, "(SP)")) { + // These original-Go forms have an explicit rejection + // regression below, not a successful object claim. + continue + } + fmt.Fprintf(&declarations, "func Low%d(value [64]byte)\n", index) + fmt.Fprintf(&source, "TEXT ·Low%d(SB),4,$64-64\n%s\nRET\n", index, row) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "low-multiply-"+api) + x86ResultInitializationObject(t, llc, target.triple, x86MulDivRoundOptimize(t, ir), "low-multiply-"+api+"-optimized") + } + }) + } + }) + } +} + +func TestX86PackedLowMultiplyVirtualLegacyStorageRemainsContext(t *testing.T) { + for _, row := range []string{"PMULLD value+0(FP),X1", "PMULLD local-16(SP),X1", "PMULLD (SP),X1", "PMULLD (BP),X1"} { + for _, target := range x86RawReturnTargets { + source := "TEXT ·Low(SB),4,$64-64\nPXOR X1,X1\n" + row + "\nRET\n" + declaration := "func Low(value [64]byte)" + address := strings.Fields(row)[1] + address = strings.Split(address, ",")[0] + if address == "(BP)" || address == "(SP)" { + source = x86OrdinaryBPNativeSource(target.arch, "Low", row, address, 64) + declaration = "func Low(p *byte)" + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+row+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + x86RequireOrdinaryBPNativeOutcome(t, target.arch, address, ir, err) + }) + } + } + } +} + +func TestX86PackedLowMultiplyResultMemoryAndMasks(t *testing.T) { + for _, op := range []string{"VPMULLW", "VPMULLD", "VPMULLQ"} { + for _, broadcast := range []bool{false, true} { + if broadcast && op == "VPMULLW" { + continue + } + row, width := op, 16 + if broadcast { + row, width = row+".BCST", 4 + if op == "VPMULLQ" { + width = 8 + } + } + row += ".Z value+0(FP),X1,K1,X2" + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-16\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + for missing := -1; missing < 16; missing++ { + for _, maskKnown := range []bool{false, true} { + state := x86ResultState{fp: make([]bool, 16)} + owned := make([]bool, 16) + for i := range owned { + owned[i], state.fp[i] = true, i != missing + } + state.xmm[1], state.masks[1] = 0xffff, maskKnown + accepted := x86ResultTransfer(ins, &state, owned, 16, "amd64") + want := maskKnown && (missing < 0 || missing >= width) + if accepted != want || accepted && state.vectorBytes(2) != ^uint64(0) || len(ins.x86CallerMemory) != 0 { + t.Fatal("mask/tuple observation invented bytes or caller permission", row, missing, maskKnown, accepted) + } + } + } + } + } +} + +func TestX86PackedLowMultiplyResultThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var source, declarations strings.Builder + for index, row := range []string{"PMULLW X0,X1", "VPMULLW X0,X1,X1", "PMULLD X0,X1", "VPMULLD X0,X1,X1", "VPMULLQ X0,X1,X1"} { + fmt.Fprintf(&declarations, "func Low%d() uint32\n", index) + fmt.Fprintf(&source, "TEXT ·Low%d(SB),4,$0-4\nMOVL $0x87654321,AX\n", index) + for _, reg := range []string{"X0", "X1"} { + for lane := 0; lane < 4; lane++ { + fmt.Fprintf(&source, "PINSRD $%d,AX,%s\n", lane, reg) + } + } + fmt.Fprintf(&source, "%s\nPEXTRD $0,X1,AX\nMOVL AX,ret0+0(FP)\nRET\n", row) + } + for _, target := range x86RawReturnTargets { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(target, api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "low-multiply-result-"+api) + } + } +} diff --git a/amd64_packed_low_multiply_raw_test.go b/amd64_packed_low_multiply_raw_test.go new file mode 100644 index 000000000..55135061d --- /dev/null +++ b/amd64_packed_low_multiply_raw_test.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86PackedLowMultiplyRawWIGAndPhysicalMasks(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, length := range []byte{0, 4} { + for _, w := range []byte{0, 0x80} { + code := []byte{0xc4, 0xe2, 0x71 | length | w, 0x40, 0xd0} + ins, size, known, err := decodedX86PackedMultiplyLowInstruction(code, mode) + if !known || err != nil || size != len(code) || ins.Op != "VPMULLD" { + t.Fatalf("VEX.WIG does not select PMULLQ: mode=%d code=%x known=%v size=%d err=%v", mode, code, known, size, err) + } + } + } + for _, qword := range []bool{false, true} { + for length := byte(0); length < 3; length++ { + for mask := byte(1); mask < 8; mask++ { + for _, zero := range []bool{false, true} { + code := encodeX86EVEXPackedMultiplyLow(qword, length, int(mask), zero, 2, 1, 0) + ins, size, known, err := decodedX86PackedMultiplyLowInstruction(code, mode) + if !known || err != nil || size != len(code) || !ins.x86Encoded { + t.Fatalf("physical EVEX must not inherit named operand limit: mode=%d code=%x known=%v size=%d err=%v", mode, code, known, size, err) + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + if _, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch); !known || err != nil { + t.Fatal("typed consumer lost decoded physical form", ins.Raw, err) + } + ins.x86Encoded = false + if _, _, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch); mode == 32 && err == nil { + t.Fatal("named mode32 instruction borrowed decoder identity") + } + } + } + } + } + } +} + +func TestX86PackedLowMultiplyRawThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var encodings [][]byte + for _, length := range []byte{0, 4} { + for _, w := range []byte{0, 0x80} { + encodings = append(encodings, []byte{0xc4, 0xe2, 0x71 | length | w, 0x40, 0xd0}) + } + } + for _, qword := range []bool{false, true} { + for length := byte(0); length < 3; length++ { + for _, mask := range []int{1, 7} { + for _, zero := range []bool{false, true} { + encodings = append(encodings, encodeX86EVEXPackedMultiplyLow(qword, length, mask, zero, 2, 1, 0)) + } + } + } + } + var source, declarations strings.Builder + for i, code := range encodings { + fmt.Fprintf(&declarations, "func RawLow%d()\n", i) + fmt.Fprintf(&source, "TEXT ·RawLow%d(SB),4,$0-0\n", i) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("RET\n") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "raw-low-multiply-"+api) + } + }) + } +} diff --git a/amd64_packed_low_multiply_result.go b/amd64_packed_low_multiply_result.go new file mode 100644 index 000000000..db5c5c48d --- /dev/null +++ b/amd64_packed_low_multiply_result.go @@ -0,0 +1,66 @@ +package plan9asm + +// This numerical observer shares the complete W/D/Q operand grammars. It +// supplies no caller region, native backing, pointer or raw-return permission. +func x86ResultPackedLowMultiplyStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + low, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch) + form, bits := low.x86PackedBinaryForm, low.spec.laneBits + if !known { + word, wordKnown, wordErr := x86PackedWordMultiplyProperties(ins.Op, ins, arch) + if !wordKnown || word.spec.mode != amd64PackedWordMultiplyLow { + return false, false + } + form, bits, err = word.x86PackedBinaryForm, 16, wordErr + } + if err != nil || form.mmx { + return false, true + } + return x86ResultPackedLaneArithmeticStep(ins, form, bits, state, owned, arch), true +} + +// A lane is numerical only when every byte of both input lanes is known. +// This dependency rule is shared by low products and wrapping/saturating +// arithmetic, not their operations or their independent memory permissions. +func x86ResultPackedLaneArithmeticStep(ins Instr, form x86PackedBinaryForm, bits int, state *x86ResultState, owned []bool, arch string) bool { + return x86ResultPackedLaneArithmeticStepWithRead(ins, form, bits, state, owned, arch, false) +} + +func x86ResultPackedLaneArithmeticStepWithRead(ins Instr, form x86PackedBinaryForm, bits int, + state *x86ResultState, owned []bool, arch string, nativeRead bool) bool { + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false + } + } + if form.memory >= 0 { + memory := ins.Args[form.memory] + // Logical FP/AUTO bytes can establish numerical dependence, not the + // original native alignment. Legacy lowering checks that separately. + if !x86ResultReadWithNativeContinuation(ins, memory, int64(form.memoryBytes), state, owned, arch, nativeRead) { + return false + } + } + bytes := func(index int) uint64 { + if index == form.memory { + return x86CallerVectorByteMask(int64(form.byteWidth)) + } + register, _ := amd64VectorRegisterIndex(ins.Args[index].Reg, form.byteWidth) + return state.vectorBytes(register) + } + first, second := bytes(form.first), bytes(form.second) + var initialized uint64 + laneBytes := bits / 8 + for index := 0; index < form.byteWidth; index += laneBytes { + lane := x86CallerVectorByteMask(int64(laneBytes)) << uint(index) + if first&lane == lane && second&lane == lane { + initialized |= lane + } + } + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + if form.mask >= 0 && !form.zeroing { + initialized &= state.vectorBytes(destination) + } + state.writeVector(destination, form.byteWidth, initialized, form.legacy) + return true +} diff --git a/amd64_packed_low_multiply_runtime_test.go b/amd64_packed_low_multiply_runtime_test.go new file mode 100644 index 000000000..f025042e0 --- /dev/null +++ b/amd64_packed_low_multiply_runtime_test.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +const x86LowMultiplyReference = ` +#include +#include +#include +#include + +typedef void (*kernel)(uint8_t *, const uint8_t *, const uint8_t *, + uint64_t *, uint32_t *, uint64_t, uint64_t); + +static uint64_t little(const uint8_t *p, int bytes) { + uint64_t value = 0; + for (int i = 0; i < bytes; ++i) { + value |= (uint64_t)p[i] << (8 * i); + } + return value; +} + +static int check(kernel fn, const char *name, int width, int bytes, + int legacy, int masking, int broadcast, int memory, int dst) { + _Alignas(64) uint8_t a[64], b[64], out[192], oldA[64], oldB[64]; + const uint64_t masks[] = {0, UINT64_MAX, 0x5555, UINT64_C(1) << 63}; + for (int sample = 0; sample < 65; ++sample) { + for (int i = 0; i < 64; ++i) { + a[i] = (uint8_t)(sample * 127 + i * 53); + b[i] = (uint8_t)(sample * 17 - i * 91); + } + memcpy(oldA, a, 64); + memcpy(oldB, b, 64); + for (int seedIndex = 0; seedIndex < 2; ++seedIndex) { + uint64_t seed = seedIndex ? 0x8d5 : 0; + for (unsigned mi = 0; mi < sizeof(masks)/sizeof(masks[0]); ++mi) { + uint64_t mask = masks[mi]; + uint64_t relevant = mask & ((UINT64_C(1) << (width / bytes)) - 1); + const uint8_t *input = memory && masking && !relevant ? NULL : a; + struct { uint64_t before, value, after; } flags = {11, 0, 13}; + struct { uint32_t before, value, after; } csr = {17, 0, 19}; + uint32_t initialCSR = _mm_getcsr(); + memset(out, 0xa7, sizeof(out)); + fn(out + 64, input, b, &flags.value, &csr.value, seed, mask); + if ((flags.value & 0x8d5) != seed || flags.before != 11 || flags.after != 13 || + csr.value != initialCSR || csr.before != 17 || csr.after != 19 || + memcmp(a, oldA, 64) || memcmp(b, oldB, 64)) { + fprintf(stderr, "%s state/input/canary mismatch\n", name); + return 1; + } + for (int i = 0; i < 192; ++i) { + unsigned want = 0xa7; + int offset = i - 64; + if (offset >= 0 && offset < 64) { + const uint8_t *old = dst == 0 ? a : b; + want = legacy ? old[offset] : 0; + if (offset < width) { + int lane = offset / bytes; + int active = !masking || ((mask >> lane) & 1); + if (active) { + uint64_t x = little(a + (broadcast ? 0 : lane * bytes), bytes); + uint64_t y = little(b + lane * bytes, bytes); + want = (uint8_t)((x * y) >> (8 * (offset % bytes))); + } else { + want = masking == 2 ? 0 : old[offset]; + } + } + } + if (out[i] != want) { + fprintf(stderr, "%s sample=%d mask=%llx byte=%d got=%x want=%x\n", + name, sample, (unsigned long long)mask, i, out[i], want); + return 1; + } + } + } + } + } + return 0; +} +` + +// This executes the portable LLVM result, not original AVX512 machine code. +// Independent full-byte observations catch stale upper aliases and merging; +// NULL with inactive/high-only masks catches eager E4 memory accesses. +func TestCrossLinuxRuntimeMatrixX86PackedLowMultiplySoftware(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + for _, destination := range []int{0, 1} { + t.Run(fmt.Sprint("destination-", destination), func(t *testing.T) { + var declarations, assembly, reference, checks strings.Builder + declarations.WriteString("package oracle\n") + reference.WriteString(x86LowMultiplyReference) + count := 0 + for _, name := range []string{"PMULLD", "VPMULLD", "VPMULLQ"} { + legacy, bits := 0, 32 + widths := []int{16, 32, 64} + if name == "PMULLD" { + legacy, widths = 1, []int{16} + } + if name == "VPMULLQ" { + bits = 64 + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for source := 0; source < 3; source++ { + for masking := 0; masking < 3; masking++ { + if legacy != 0 && (masking != 0 || source == 2 || destination == 0) { + continue + } + fn := fmt.Sprint("Low", count) + count++ + row, first, broadcast, memory := name, bank+"0", 0, 0 + if source != 0 { + first, memory = "(DI)", 1 + } + if source == 2 { + row, broadcast = row+".BCST", 1 + } + if masking == 2 { + row += ".Z" + } + row += " " + first + "," + bank + "1" + if masking != 0 { + row += ",K1" + } + if legacy == 0 { + row += fmt.Sprintf(",%s%d", bank, destination) + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed,mask uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-56\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\nVMOVUPS (SI),Z1\n", fn) + if source == 0 || destination == 0 { + // For memory forms Z0's merge source uses B; do not + // read the possibly inactive source address early. + base := "DI" + if source != 0 { + base = "SI" + } + fmt.Fprintf(&assembly, "VMOVUPS (%s),Z0\n", base) + } + assembly.WriteString("MOVQ mask+48(FP),AX\nKMOVQ AX,K1\nPUSHQ seed+40(FP)\nPOPFQ\n") + fmt.Fprintf(&assembly, "%s\nVMOVUPS Z%d,(DX)\nPUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n", row, destination) + fmt.Fprintf(&reference, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t);\n", fn) + old := destination + if memory != 0 { + old = 1 + } + fmt.Fprintf(&checks, " if (check(%s,%q,%d,%d,%d,%d,%d,%d,%d)) return 1;\n", fn, row, width, bits/8, legacy, masking, broadcast, memory, old) + } + } + } + } + if count > 96 { + t.Fatal("runtime batch exceeds its resource bound", count) + } + reference.WriteString("int main(void) {\n" + checks.String() + " return 0;\n}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sigs[name] = x86WordMultiplySignature(t, pkg, name) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, + Sigs: sigs, ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }}) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "low-multiply", ir, reference.String(), level) + }) + } + t.Logf("%d kernels, 65 inputs, two FLAGS seeds, four masks; full aliases, canaries and inactive NULL sources", count) + }) + } +} diff --git a/amd64_packed_move_zero_store_test.go b/amd64_packed_move_zero_store_test.go new file mode 100644 index 000000000..de6c05408 --- /dev/null +++ b/amd64_packed_move_zero_store_test.go @@ -0,0 +1,403 @@ +package plan9asm + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +// Independent EVEX opcodes whose Go-accepted .Z memory destination requires +// architectural #UD. Intel XED's memory patterns require ZEROING=0 for both +// packed moves and all six compression opcodes. VMOVDQA/DQU are VEX-only. +var x86PackedZeroStoreOpcodes = []string{ + "VMOVAPD", "VMOVAPS", "VMOVUPD", "VMOVUPS", + "VMOVDQA32", "VMOVDQA64", "VMOVDQU8", "VMOVDQU16", "VMOVDQU32", "VMOVDQU64", + "VCOMPRESSPS", "VCOMPRESSPD", "VPCOMPRESSB", "VPCOMPRESSW", "VPCOMPRESSD", "VPCOMPRESSQ", +} + +func x86PackedZeroStoreInstruction(name, instruction string) string { + return fmt.Sprintf("TEXT %s(SB),4,$0-0\n%s\nRET\n", name, instruction) +} + +func x86PackedZeroStoreSource() (string, map[string]FuncSig, []string) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var names []string + for _, opcode := range x86PackedZeroStoreOpcodes { + for _, bank := range []string{"X", "Y", "Z"} { + name := "fault" + opcode + bank + source.WriteString(x86PackedZeroStoreInstruction(name, fmt.Sprintf("%s.Z %s0,K1,(AX)", opcode, bank))) + sigs[name] = FuncSig{Name: name, Ret: Void} + names = append(names, name) + } + } + return source.String(), sigs, names +} + +func TestX86PackedZeroMemoryStorePreservesUndefinedOpcode(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + source, sigs, names := x86PackedZeroStoreSource() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.arch, source, true) + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + trap := `call void asm sideeffect "ud2", "~{memory}"()` + if strings.Count(ir, trap) != len(names) || strings.Contains(ir, "call void @llvm.masked.store.") { + t.Fatalf("Go-accepted EVEX.z memory stores must deliver #UD before reading registers or memory; traps=%d want=%d", strings.Count(ir, trap), len(names)) + } + compileLLVMToObject(t, llc, target.triple, "zero-store.ll", "zero-store.o", ir) + }) + } +} + +func TestX86PackedZeroMemoryStoreFaultPrecedesFPBoundsAndMaskReads(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, opcode := range x86PackedZeroStoreOpcodes { + t.Run(arch+"/"+opcode, func(t *testing.T) { + instruction := Instr{Op: Op(opcode + ".Z"), Args: []Operand{ + {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "K1"}, + {Kind: OpFP, FPOffset: 1 << 20}, + }} + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := output.Len() + lower := ctx.lowerPackedVectorMove + if strings.Contains(opcode, "COMPRESS") { + lower = ctx.lowerPackedCompress + } + recognized, terminated, err := lower(instruction.Op, instruction) + if !recognized || !terminated || err != nil || output.String()[before:] != " call void asm sideeffect \"ud2\", \"~{memory}\"()\n unreachable\n" { + t.Fatalf("%s invalid encoding evaluated nonexistent FP/mask/register data before #UD: %t/%t/%v", arch, recognized, terminated, err) + } + }) + } + } +} + +func x86PackedFirstGoInstructionBytes(t *testing.T, arch, instruction string) []byte { + t.Helper() + listing, err := x86ScalarLiteralGoListing(t, arch, "TEXT probe(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatalf("actual Go packed-move object: %v\n%s", err, listing) + } + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 7 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + bytes := make([]byte, 6) + for index := range bytes { + value, err := strconv.ParseUint(fields[index+1], 16, 8) + if err != nil { + t.Fatalf("actual Go object byte: %v\n%s", err, listing) + } + bytes[index] = byte(value) + } + return bytes + } + t.Fatalf("actual Go object lacks packed-move bytes:\n%s", listing) + return nil +} + +func TestX86PackedZeroMemoryStoreActualGoAndLLVM22Encoding(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM 22 llvm-mc not found") + } + for _, arch := range []string{"amd64", "386"} { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + for _, opcode := range x86PackedZeroStoreOpcodes { + for _, bank := range []string{"X", "Y", "Z"} { + instruction := fmt.Sprintf("%s %s0,K1,(AX)", opcode, bank) + normal := x86PackedFirstGoInstructionBytes(t, arch, instruction) + zero := x86PackedFirstGoInstructionBytes(t, arch, strings.Replace(instruction, " ", ".Z ", 1)) + if normal[0] != 0x62 || normal[3]&0x80 != 0 || zero[3]&0x80 == 0 { + t.Fatalf("%s %s lacks independent EVEX.z witness: normal=%x zero=%x", arch, instruction, normal, zero) + } + wantW := strings.HasSuffix(opcode, "64") || opcode == "VMOVDQU16" || strings.HasSuffix(opcode, "PD") || + opcode == "VPCOMPRESSW" || opcode == "VPCOMPRESSQ" + if (normal[2]&0x80 != 0) != wantW { + t.Fatalf("actual Go object changed the native element selector: %s %s bytes=%x", arch, instruction, normal) + } + for index := range normal { + wanted := normal[index] + if index == 3 { + wanted |= 0x80 + } + if zero[index] != wanted { + t.Fatalf(".Z changed more than the native EVEX.z bit: %s %s", arch, instruction) + } + } + for _, test := range []struct { + bytes []byte + invalid bool + }{{normal, false}, {zero, true}} { + var input strings.Builder + for _, value := range test.bytes { + fmt.Fprintf(&input, "0x%02x ", value) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple="+triple, + "--mattr=+avx512f,+avx512bw,+avx512vl,+avx512vbmi2") + command.Stdin = strings.NewReader(input.String()) + command.WaitDelay = 2 * time.Second + output, err := command.CombinedOutput() + cancel() + // llvm-mc can return zero while diagnosing rejected bytes. + invalid := strings.Contains(string(output), "invalid instruction encoding") + family := "VMOV" + if strings.Contains(opcode, "COMPRESS") { + family = "COMPRESS" + } + if err != nil || invalid != test.invalid || !test.invalid && !strings.Contains(strings.ToUpper(string(output)), family) { + t.Fatalf("%s %s invalid=%t: LLVM 22 diagnostic mismatch: %v\n%s", arch, instruction, test.invalid, err, output) + } + // LLVM 22's 32-bit decoder aliases some W1 move spellings + // to W0 names. Verify the independent W byte above and the + // exact opcode in 64-bit decode, not the aliased 32-bit name. + if !test.invalid { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=x86_64-unknown-linux-gnu", + "--mattr=+avx512f,+avx512bw,+avx512vl,+avx512vbmi2") + command.Stdin = strings.NewReader(input.String()) + command.WaitDelay = 2 * time.Second + output, err := command.CombinedOutput() + cancel() + if err != nil || !strings.Contains(strings.ToUpper(string(output)), opcode) || strings.Contains(string(output), "invalid instruction encoding") { + t.Fatalf("actual Go object does not decode as the exact opcode: %s %s: %v\n%s", arch, instruction, err, output) + } + } + } + } + } + } +} + +func TestCrossLinuxRuntimeMatrixX86PackedZeroMemoryStoreFault(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the separately required Linux/amd64 packed-move fault oracle") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + source, sigs, names := x86PackedZeroStoreSource() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, "amd64") + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + var oracle strings.Builder + oracle.WriteString(`#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#include +#endif + +`) + for _, name := range names { + fmt.Fprintf(&oracle, "extern void %s(void);\n", name) + } + oracle.WriteString("\nstatic void (*fault_functions[])(void) = {\n") + for _, name := range names { + fmt.Fprintf(&oracle, " %s,\n", name) + } + oracle.WriteString(`}; + +int main(int argc, char **argv) { +#ifdef _WIN32 + SetErrorMode(SEM_NOGPFAULTERRORBOX | SEM_FAILCRITICALERRORS); + if (argc == 2) { + unsigned index = (unsigned)strtoul(argv[1], 0, 10); + if (index >= sizeof(fault_functions)/sizeof(fault_functions[0])) return 6; + fault_functions[index](); + return 3; + } + for (unsigned i = 0; i < sizeof(fault_functions)/sizeof(fault_functions[0]); i++) { + char index[16]; + snprintf(index, sizeof(index), "%u", i); + intptr_t status = _spawnl(_P_WAIT, argv[0], argv[0], index, NULL); + if ((uint32_t)status != UINT32_C(0xc000001d)) return 5; + } +#else + (void)argc; + (void)argv; + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) return 1; + for (unsigned i = 0; i < sizeof(fault_functions)/sizeof(fault_functions[0]); i++) { + pid_t child = fork(); + if (child < 0) return 2; + if (!child) { fault_functions[i](); _exit(3); } + int status; + if (waitpid(child, &status, 0) != child) return 4; + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGILL) return 5; + } +#endif + return 0; +} +`) + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-zero-store-fault", triple, ir, oracle.String(), runner) + + // No preceding valid AVX512 instruction is executed. EVEX.z=1 with a + // memory destination is always #UD, irrespective of mask/address values. + var mainGo strings.Builder + mainGo.WriteString("package main\nimport \"os\"\n") + for _, name := range names { + fmt.Fprintf(&mainGo, "func %s()\n", name) + } + mainGo.WriteString("func main() { switch os.Args[1] {\n") + for _, name := range names { + fmt.Fprintf(&mainGo, "case %q: %s()\n", name, name) + } + mainGo.WriteString("} }\n") + dir := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module example.com/packed-zero-store\n\ngo 1.20\n", + "main.go": mainGo.String(), + "fault_amd64.s": strings.ReplaceAll(source, "TEXT fault", "TEXT ·fault"), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-packed-zero-store") + if runtime.GOOS == "windows" { + executable += ".exe" + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + build := exec.CommandContext(ctx, "go", "build", "-o", executable, ".") + build.Dir, build.Env = dir, append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0", "GOARCH=amd64") + build.WaitDelay = 2 * time.Second + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("original Go packed-store build: %v\n%s", err, output) + } + for _, name := range names { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + args := append(append([]string(nil), runner...), executable, name) + command := exec.CommandContext(ctx, args[0], args[1:]...) + command.Env = append(os.Environ(), "GOTRACEBACK=single") + command.WaitDelay = 2 * time.Second + output, err := command.CombinedOutput() + cancel() + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 2 || !x86GoPackedStoreFaultDiagnostic(runtime.GOOS, name, string(output)) { + t.Fatalf("original Go %s did not deliver SIGILL: %v\n%s", name, err, output) + } + } +} + +var x86GoPackedStoreFatalFault = regexp.MustCompile(`^fatal error: fault\n\n?\[signal 0xc000001d code=0x[0-9a-f]+ addr=0x[0-9a-f]+ pc=0x[0-9a-f]+\]\n`) + +func x86GoPackedStoreFaultDiagnostic(goos, name, output string) bool { + output = strings.ReplaceAll(output, "\r\n", "\n") + if strings.Contains(output, "\r") { + return false + } + matched := strings.HasPrefix(output, "SIGILL: illegal instruction\n") + if goos == "windows" { + // Current Go converts the native exception to sigpanic; older SDKs + // print the unhandled exception directly. Both must identify #UD, + // not an ordinary panic, access violation or just an exit status 2. + // Go's fatal header can have either one newline or a separating + // blank line before the complete native signal record. + matched = strings.HasPrefix(output, "Exception 0xc000001d ") || x86GoPackedStoreFatalFault.MatchString(output) + } + if !matched { + return false + } + lines := strings.Split(output, "\n") + for index, line := range lines { + if line == "main."+name+"()" && index+1 < len(lines) { + location := strings.ReplaceAll(lines[index+1], "\\", "/") + _, position, found := strings.Cut(location, "/fault_amd64.s:") + fields := strings.Fields(position) + if !found || len(fields) == 0 { + continue + } + lineNumber, err := strconv.ParseUint(fields[0], 10, 31) + if err == nil && lineNumber > 0 { + return true + } + } + } + return false +} + +func TestX86GoPackedStoreFaultDiagnostic(t *testing.T) { + const stack = "\nmain.faultVMOVAPDX()\n\tC:/fixture/fault_amd64.s:2 fp=0x123 pc=0x456\n" + const modern = "fatal error: fault\n\n[signal 0xc000001d code=0x0 addr=0x0 pc=0x456]\n" + for _, test := range []struct { + name, goos, output string + valid bool + }{ + {"Unix", "linux", "SIGILL: illegal instruction\nPC=0x456\n" + stack, true}, + {"Windows exception", "windows", "Exception 0xc000001d 0x0 0x0 0x456\n" + stack, true}, + {"Windows sigpanic", "windows", modern + stack, true}, + {"Windows native sigpanic", "windows", strings.Replace(modern, "fault\n\n", "fault\n", 1) + stack, true}, + {"Windows native newlines", "windows", strings.ReplaceAll(modern+stack, "\n", "\r\n"), true}, + {"Windows native sigpanic CRLF", "windows", strings.ReplaceAll(strings.Replace(modern, "fault\n\n", "fault\n", 1)+stack, "\n", "\r\n"), true}, + {"extra fatal blank line", "windows", strings.Replace(modern, "fault\n\n", "fault\n\n\n", 1) + stack, false}, + {"malformed signal record", "windows", strings.Replace(modern, " pc=0x456]", " pc=unobserved]", 1) + stack, false}, + {"unterminated signal record", "windows", strings.Replace(modern, "0x456]", "0x456", 1) + stack, false}, + {"missing fault address", "windows", strings.Replace(modern, " addr=0x0", "", 1) + stack, false}, + {"ordinary panic", "windows", "panic: Exception 0xc000001d\n" + stack, false}, + {"different signal", "windows", strings.ReplaceAll(modern+stack, "0xc000001d", "0xc0000005"), false}, + {"partial exception code", "windows", "Exception 0xc000001d0\n" + stack, false}, + {"missing tested frame", "windows", modern, false}, + {"wrong tested frame", "windows", modern + strings.ReplaceAll(stack, "faultVMOVAPDX", "faultOther"), false}, + {"wrong source", "windows", modern + strings.ReplaceAll(stack, "fault_amd64.s", "main.go"), false}, + {"malformed source position", "windows", modern + strings.ReplaceAll(stack, ".s:2 ", ".s:2invalid "), false}, + {"zero source position", "windows", modern + strings.ReplaceAll(stack, ".s:2 ", ".s:0 "), false}, + {"bare native carriage return", "windows", modern + strings.ReplaceAll(stack, "fp=", "\rfp="), false}, + } { + t.Run(test.name, func(t *testing.T) { + if got := x86GoPackedStoreFaultDiagnostic(test.goos, "faultVMOVAPDX", test.output); got != test.valid { + t.Fatalf("actual Go illegal-instruction diagnostic: got %t, want %t", got, test.valid) + } + }) + } +} diff --git a/amd64_packed_narrow_forms.go b/amd64_packed_narrow_forms.go new file mode 100644 index 000000000..d90a86be2 --- /dev/null +++ b/amd64_packed_narrow_forms.go @@ -0,0 +1,79 @@ +package plan9asm + +import "strings" + +type x86PackedNarrowSpec struct { + x86PackedBinarySpec + inputBits, outputBits int + mode amd64PackedNarrowMode +} + +// Go's ymm rows admit MMX only in amd64 mode; PACKUSDW is yxm_q4. +// All four V rows use _yvandnpd, with N4 broadcast only for dword inputs. +// Memory exception behavior is E4NF (SSWB/USWB: E4NF.nb), independently +// of their output-lane writemask. No initialized bank or ABI is granted here. +var x86PackedNarrowSpecs = map[Op]x86PackedNarrowSpec{ + "PACKSSLW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 32, 16, amd64PackedNarrowSigned}, + "PACKSSWB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16, 8, amd64PackedNarrowSigned}, + "PACKUSDW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 32, 16, amd64PackedNarrowUnsigned}, + "PACKUSWB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16, 8, amd64PackedNarrowUnsigned}, + "VPACKSSDW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, 32, 16, amd64PackedNarrowSigned}, + "VPACKSSWB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, 8, amd64PackedNarrowSigned}, + "VPACKUSDW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, 32, 16, amd64PackedNarrowUnsigned}, + "VPACKUSWB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, 8, amd64PackedNarrowUnsigned}, +} + +type x86PackedNarrowForm struct { + x86PackedBinaryForm + spec x86PackedNarrowSpec + source Operand +} + +// Numerical dependencies only. Each narrowed element needs its complete +// signed input, independently within each source half of a 128-bit chunk. +// This never shrinks the native memory tuple or supplies a mask/pointer role. +func x86PackedNarrowDefinedBytes(form x86PackedNarrowForm, first, second, old uint64) uint64 { + inputBytes, outputBytes := form.spec.inputBits/8, form.spec.outputBits/8 + chunkBytes := 16 + if form.byteWidth < chunkBytes { + chunkBytes = form.byteWidth + } + half := chunkBytes / 2 + var known uint64 + for output := 0; output < form.byteWidth; output += outputBytes { + chunk, local := output/chunkBytes*chunkBytes, output%chunkBytes + input, source := chunk+(local%half)/outputBytes*inputBytes, second + if local >= half { + source = first + } + needed := x86CallerVectorByteMask(int64(inputBytes)) << uint(input) + if source&needed == needed { + known |= x86CallerVectorByteMask(int64(outputBytes)) << uint(output) + } + } + if form.mask >= 0 && !form.zeroing { + known &= old + } + return known +} + +func x86PackedNarrowProperties(ins Instr, goarch string) (x86PackedNarrowForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86PackedNarrowSpecs[Op(base)] + if !known { + return x86PackedNarrowForm{}, false, nil + } + binary, source, started, err := x86PackedBinaryPhysicalSource(ins.Op, ins, goarch, spec.x86PackedBinarySpec, "packed narrow") + if !started { + return x86PackedNarrowForm{}, true, err + } + if err != nil && source.Kind != OpInvalid { + // Keep the prior mode32 Context precedence when both a register and + // physical address are invalid. Rejected forms never expose a source. + if modeErr := x86Mode32VectorRegisterContext(ins, goarch); modeErr != nil { + err = modeErr + } + source = Operand{} + } + return x86PackedNarrowForm{binary, spec, source}, true, err +} diff --git a/amd64_packed_narrow_independent_runtime_test.go b/amd64_packed_narrow_independent_runtime_test.go new file mode 100644 index 000000000..c213acd31 --- /dev/null +++ b/amd64_packed_narrow_independent_runtime_test.go @@ -0,0 +1,468 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +// The independent oracle describes the ISA, not production properties. Go's +// reversed operands put its second source in the low half of each 128-bit lane. +type x86NarrowOracleFamily struct { + legacy, vector string + inputBytes int + unsigned bool +} + +type x86NarrowOracleRow struct { + family x86NarrowOracleFamily + bank string + width, source, alias int + masking int // 0: none, 1: merge, 2: zero + legacy bool +} + +func x86NarrowOracleRows() []x86NarrowOracleRow { + var rows []x86NarrowOracleRow + for _, family := range x86NarrowOracleFamilies { + for _, bank := range []string{"X", "M"} { + if bank == "M" && family.legacy == "PACKUSDW" { + continue + } + width := 16 + if bank == "M" { + width = 8 + } + for source := 0; source < 2; source++ { + for alias := 0; alias < 2; alias++ { + rows = append(rows, x86NarrowOracleRow{family, bank, width, source, alias, 0, true}) + } + } + } + for widthIndex, bank := range []string{"X", "Y", "Z"} { + for source := 0; source < 3; source++ { + if source == 2 && family.inputBytes != 4 { + continue + } + for alias := 0; alias < 3; alias++ { + for mask := 0; mask < 3; mask++ { + rows = append(rows, x86NarrowOracleRow{family, bank, 16 << widthIndex, source, alias, mask, false}) + } + } + } + } + } + return rows +} + +func (row x86NarrowOracleRow) instruction() (string, string) { + first, destination := row.bank+"0", row.bank+fmt.Sprint(row.alias) + if row.legacy { + destination = row.bank + "1" + if row.alias == 1 { + first = destination + } + } + if row.source != 0 { + first = "(CX)" + if row.legacy && row.alias == 1 { + first = "(DX)" + } + } + if row.legacy { + return row.family.legacy + " " + first + "," + destination, destination + } + op := row.family.vector + if row.source == 2 { + op += ".BCST" + } + if row.masking == 2 { + op += ".Z" + } + mask := "" + if row.masking != 0 { + mask = "K1," + } + return op + " " + first + "," + row.bank + "1," + mask + destination, destination +} + +// Each scalar is signed before clamping, including the unsigned output forms. +// This reference does not call or reproduce production's vector/typed helpers. +const x86NarrowIndependentCReference = ` +static const int32_t edges[]={ + INT32_MIN,-65537,-32769,-32768,-32767,-257,-256,-129,-128,-127,-1,0,1, + 126,127,128,129,254,255,256,257,32766,32767,32768,32769, + 65534,65535,65536,65537,INT32_MAX-1,INT32_MAX}; +static void fill(uint8_t *out,int bytes,unsigned sample,unsigned salt) { + for(int lane=0;lane<64/bytes;++lane) { + int32_t value=edges[(sample+salt+lane*7)%(sizeof(edges)/sizeof(edges[0]))]; + if(bytes==4) memcpy(out+lane*bytes,&value,4); + else {int16_t small=(int16_t)value;memcpy(out+lane*bytes,&small,2);} + } +} +static void independent_pack(uint8_t *out,const uint8_t *first,const uint8_t *second, + int width,int inputBytes,int unsignedOutput,int broadcast) { + int chunk=width==8?8:16,outputBytes=inputBytes/2; + int64_t minimum=unsignedOutput?0:-(INT64_C(1)<<(outputBytes*8-1)); + int64_t maximum=unsignedOutput?(INT64_C(1)<<(outputBytes*8))-1:(INT64_C(1)<<(outputBytes*8-1))-1; + for(int start=0;startmaximum) value=maximum; + uint32_t result=(uint32_t)value; + int destination=start+source*(chunk/2)+lane*outputBytes; + for(int byte=0;byte>(byte*8)); + } + } +} +` + +func x86NarrowIndependentAssembly(name string, row x86NarrowOracleRow) string { + var assembly strings.Builder + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-52\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVQ mask+32(FP),R9\n", name) + // Prepare status before entering the uninterrupted physical MMX region. + assembly.WriteString("LDMXCSR csr+48(FP)\nPUSHQ flags+40(FP)\nPOPFQ\n") + if row.bank == "M" { + assembly.WriteString("MOVQ (CX),M0\nMOVQ (DX),M1\n") + } else { + assembly.WriteString("VMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nKMOVQ R9,K1\n") + } + instruction, destination := row.instruction() + fmt.Fprintln(&assembly, instruction) + if row.bank == "M" { + fmt.Fprintf(&assembly, "MOVQ %s,(AX)\nEMMS\n", destination) + } else { + fmt.Fprintf(&assembly, "VMOVUPS Z%s,(AX)\n", destination[1:]) + } + assembly.WriteString("PUSHFQ\nPOPQ R10\nMOVQ R10,64(AX)\nSTMXCSR 72(AX)\nRET\n") + return assembly.String() +} + +func TestCrossLinuxRuntimeMatrixX86PackedNarrowIndependentWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + rows := x86NarrowOracleRows() + var assembly, declarations, table, main strings.Builder + declarations.WriteString("package narroworacle\n") + main.WriteString(`#include +#include +#include +typedef void (*kernel)(void*,const void*,const void*,const void*,uint64_t,uint64_t,uint32_t); +`) + for index, row := range rows { + name := fmt.Sprintf("numeric%d", index) + fmt.Fprintf(&declarations, "func %s(out *[80]byte,a,b,old *[64]byte,mask,flags uint64,csr uint32)\n", name) + assembly.WriteString(x86NarrowIndependentAssembly(name, row)) + fmt.Fprintf(&main, "extern void %s(void*,const void*,const void*,const void*,uint64_t,uint64_t,uint32_t);\n", name) + fmt.Fprintf(&table, "{%s,%d,%d,%t,%d,%d,%d,%t},\n", name, row.width, row.family.inputBytes, + row.family.unsigned, row.source, row.alias, row.masking, row.legacy) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + main.WriteString("static const struct {kernel run;int width,bytes,uns,source,alias,masking,legacy;} cases[]={\n" + table.String() + "};\n") + main.WriteString(x86NarrowIndependentCReference) + main.WriteString(` +int main(void) { + unsigned saved;__asm__ volatile("stmxcsr %0":"=m"(saved)); + const uint32_t profiles[]={0x1f80,0x3fc1,0x5f82,0x9fe5}; + unsigned long long observations=0; + for(unsigned row=0;row>(i/(cases[row].bytes/2))&1)) + value=cases[row].masking==2?0:previous[i]; + } + want[1+i]=value; + } + uint64_t observedFlags;uint32_t observedCSR; + memcpy(&observedFlags,output+65,8);memcpy(want+65,&observedFlags,8); + memcpy(&observedCSR,output+73,4);memcpy(want+73,&observedCSR,4); + ++observations; + if((observedFlags&0x8d5)!=(flags&0x8d5)||observedCSR!=profiles[profile]|| + memcmp(output,want,sizeof(output))||memcmp(a,beforeA,64)||memcmp(b,beforeB,64)||memcmp(old,beforeOld,64)) { + fprintf(stderr,"PACK row=%u width=%d inputBytes=%d unsigned=%d source=%d alias=%d masking=%d legacy=%d sample=%u mask=%llx flags=%llx csr=%x\n", + row,cases[row].width,cases[row].bytes,cases[row].uns,cases[row].source,cases[row].alias,cases[row].masking, + cases[row].legacy,sample,(unsigned long long)mask,(unsigned long long)observedFlags,observedCSR); + for(unsigned i=0;imaximum {value=maximum} + for byteIndex:=0;byteIndex>(byteIndex*8)) + } + } + } + return out +} +func TestOriginal(t *testing.T) { + runtime.LockOSThread();defer runtime.UnlockOSThread() + profiles:=[]uint32{0x1f80,0x3fc1,0x5f82,0x9fe5} + var aa,bb [128]byte + a,b:=aligned(&aa),aligned(&bb) + observations:=0 + for row,entry:=range cases {for sample:=0;sample<32;sample++ {for _,csr:=range profiles {for seed:=0;seed<2;seed++ { + for i:=range aa {aa[i]=0xa7;bb[i]=0xb9} + fill(a,entry.bytes,sample,0);fill(b,entry.bytes,sample,13) + beforeA,beforeB:=aa,bb + var output,want [34]byte + for i:=range output {output[i]=0xc3;want[i]=0xc3} + flags:=uint64(2);if seed!=0 {flags|=0x8d5} + entry.run((*[32]byte)(unsafe.Pointer(&output[1])),a,b,flags,csr) + first:=a;if entry.alias!=0 {first=b} + packed:=reference(first,b,entry.width,entry.bytes,entry.unsigned) + copy(want[1:1+entry.width],packed[:entry.width]) + observed:=binary.LittleEndian.Uint64(output[17:25]);copy(want[17:29],output[17:29]) + actualCSR:=binary.LittleEndian.Uint32(output[25:29]) + observations++ + if output!=want||aa!=beforeA||bb!=beforeB||observed&0x8d5!=flags&0x8d5||actualCSR!=csr { + t.Fatalf("original PACK row%d sample%d width%d signedInputBytes%d flags%x csr%x got%x want%x",row,sample,entry.width,entry.bytes,observed,actualCSR,output,want) + } + }}}} + t.Logf("original SSE/MMX kernels=%d observations=%d sixFlags/fullCSR/input-output-canaries",len(cases),observations) +} +` + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module narroworacle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": tests, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, gotoolchain.HostExecutable("original")) + command := exec.Command("go", "test", "-c", "-p=2", "-vet=off", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH=amd64", "CGO_ENABLED=0", + "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOMAXPROCS=2") + if output := x86FloatCompareCaptureCommand(t, command); len(output) != 0 { + t.Logf("actual original Go build diagnostic (unchanged bytes):\n%s", output) + } + output := execution.run(t, executable, "-test.v", "-test.run=^TestOriginal$", "-test.count=1") + want := fmt.Sprintf("original SSE/MMX kernels=%d observations=%d sixFlags/fullCSR/input-output-canaries", count, count*32*4*2) + if !strings.Contains(string(output), want) || !strings.Contains(string(output), "--- PASS: TestOriginal") || + strings.Contains(string(output), "SKIP") { + t.Fatalf("incomplete original-Go oracle denominator: %q", output) + } + t.Log(strings.TrimSpace(string(output))) + t.Logf("%d actual stock Go SSE/MMX kernels; no VEX/EVEX original code executed", count) +} + +var x86NarrowOracleFamilies = []x86NarrowOracleFamily{ + {"PACKSSLW", "VPACKSSDW", 4, false}, + {"PACKSSWB", "VPACKSSWB", 2, false}, + {"PACKUSDW", "VPACKUSDW", 4, true}, + {"PACKUSWB", "VPACKUSWB", 2, true}, +} + +func x86NarrowOracleIR(t *testing.T, source, declarations, triple string) string { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "narroworacle", declarations) + sigs := make(map[string]FuncSig) + for _, function := range file.Funcs { + name := strings.TrimPrefix(function.Sym, "·") + object, ok := pkg.Types.Scope().Lookup(name).(*types.Func) + if !ok { + t.Fatal("independent source lacks its declaration", name) + } + sig, err := DeriveGoFuncABI0Signature(name, object, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, Options{ + Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + return module.String() +} + +// Intel SDM Vol. 2A, Table 2-50 and the PACK chapters classify these complete +// memory tuples as E4NF/E4NF.nb: a zero output mask does not suppress the read. +// No AVX/AVX512 original instruction is executed by this software LLVM oracle. +func TestCrossLinuxRuntimeMatrixX86PackedNarrowIndependentE4NF(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, table strings.Builder + declarations.WriteString("package narroworacle\n") + count := 0 + for _, family := range x86NarrowOracleFamilies { + for widthIndex, bank := range []string{"X", "Y", "Z"} { + for _, broadcast := range []bool{false, true} { + if broadcast && family.inputBytes != 4 { + continue + } + for _, zero := range []bool{false, true} { + name := fmt.Sprintf("fault%d", count) + count++ + op := family.vector + bytes := 16 << widthIndex + if broadcast { + op += ".BCST" + bytes = 4 + } + if zero { + op += ".Z" + } + fmt.Fprintf(&declarations, "func %s(input *byte)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-8\nMOVQ input+0(FP),AX\nVPXORD %s0,%s0,%s0\nMOVL $0,CX\nKMOVQ CX,K1\n%s (AX),%s0,K1,%s1\nRET\n", + name, bank, bank, bank, op, bank, bank) + fmt.Fprintf(&table, "{observe%d, %d, %d},\n", count-1, bytes, bytes-1) + } + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + var main strings.Builder + for index := 0; index < count; index++ { + fmt.Fprintf(&main, "extern void fault%d(const uint8_t*);\n", index) + fmt.Fprintf(&main, `static int observe%d(const uint8_t *input) { + fault%d(input); + return 0; +} +`, index, index) + } + driver := x86MemoryFaultDriver(main.String(), table.String()) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := execution.compileLLVM(t, "narrow-independent-e4nf", ir, driver, level) + x86CheckMemoryFaultMatrix(t, execution, executable, count) + }) + } +} diff --git a/amd64_packed_narrow_official_forms_test.go b/amd64_packed_narrow_official_forms_test.go new file mode 100644 index 000000000..ffafa3cb4 --- /dev/null +++ b/amd64_packed_narrow_official_forms_test.go @@ -0,0 +1,286 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "os/exec" + "reflect" + "strings" + "testing" +) + +// Select the semantic family by name, but derive its entire vector syntax +// denominator from Go's independent encoder reader, never the lowerer spec. +func x86OfficialPackedNarrowRows(t *testing.T, arch string) []string { + t.Helper() + byOpcode := x86OfficialPackedBinaryWitnesses(t) + var rows []string + for _, opcode := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + witnesses := byOpcode[opcode] + if len(witnesses) == 0 { + t.Fatal("missing independent official pack family", opcode) + } + for _, witness := range witnesses { + if arch == "386" && strings.Count(witness, ",") == 3 { + continue // Actual mode32 frontend refusals are a separate gate. + } + rows = append(rows, witness) + } + } + return rows +} + +func TestX86PackedNarrowOfficialRowsThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, raw := range []bool{false, true} { + var rawRows []string + if raw { + rawRows = x86OfficialPackedNarrowRawRows(t) + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(fmt.Sprintf("raw=%t/%s", raw, target.triple), func(t *testing.T) { + var source, declarations strings.Builder + declarations.WriteString("package narrow\n") + rows := x86OfficialPackedNarrowRows(t, target.arch) + if raw { + // Physical mode32 masks are legal even though its named Go + // frontend rejects them; bytes come from the actual amd64 Go + // encoder with low physical register/address banks throughout. + rows = rawRows + } + for index, row := range rows { + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", index, row) + fmt.Fprintf(&declarations, "func F%d()\n", index) + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + pkg := mustGoPackage(t, "example.invalid/narrow", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for index := range rows { + name := fmt.Sprintf("F%d", index) + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), + target.arch, types.SizesFor("gc", target.arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + before := fmt.Sprintf("%#v %#v", file, sigs) + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewPackageAPI(t, source.String(), pkg, target.arch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original named/raw format batch invented native entry inputs", api, err) + } + } + units := *file + units.Funcs = append([]Func(nil), file.Funcs...) + if raw { + for index, fn := range units.Funcs { + decoded, err := decodeX86RawDirectives(fn, target.arch) + if err != nil { + t.Fatal("actual physical format unit could not be decoded", err) + } + units.Funcs[index] = decoded + } + } + // Keep every original form, but release each bounded object + // before the next batch. Units are not initialized entries. + for start := 0; start < len(units.Funcs); start += 64 { + end := start + 64 + if end > len(units.Funcs) { + end = len(units.Funcs) + } + batch := units + batch.Funcs = units.Funcs[start:end] + ir := x86InstructionFormatObjectUnit(t, &batch, sigs, target.arch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerPackedSaturatingNarrow(ins.Op, ins) + }) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + } + if before != fmt.Sprintf("%#v %#v", file, sigs) { + t.Fatal("object coverage changed original source/signatures") + } + t.Logf("all %d original named/physical Go forms retain emitter-only LLVM objects and three-API Context; independent initialized sources own public/runtime coverage", len(rows)) + }) + } + } +} + +func x86OfficialPackedNarrowRawRows(t *testing.T) []string { + t.Helper() + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM 22 llvm-mc required") + } + var rows []string + var disassembly strings.Builder + var equivalentPairs [][2]int + for _, row := range x86OfficialPackedNarrowRows(t, "amd64") { + code := assembleX87ControlBytes(t, "amd64", "TEXT rawpack(SB),4,$0-0\n"+row+"\nRET\n") + code = code[:len(code)-1] + appendBytes := func(bytes []byte) { + var source strings.Builder + for _, value := range bytes { + fmt.Fprintf(&source, "BYTE $%d\n", value) + fmt.Fprintf(&disassembly, "0x%02x ", value) + } + disassembly.WriteByte('\n') + rows = append(rows, source.String()) + } + original := len(rows) + appendBytes(code) + opcode, _, _ := strings.Cut(strings.Fields(row)[0], ".") + // WIG is physical grammar beyond Go's chosen W0 encoding. EVEX + // dword-to-word is W0-only; every VEX and EVEX word-to-byte row + // must have an independently disassembled equivalent W1 object. + if code[0] == 0x62 && opcode != "VPACKSSWB" && opcode != "VPACKUSWB" { + continue + } + wig := append([]byte(nil), code...) + if wig[0] == 0xc5 { + wig = append([]byte{0xc4, wig[1]&0x80 | 0x61, wig[1] & 0x7f}, wig[2:]...) + } + if (wig[0] != 0xc4 && wig[0] != 0x62) || wig[2]&0x80 != 0 { + t.Fatalf("unexpected original Go pack prefix: %s %x", row, code) + } + wig[2] |= 0x80 + equivalentPairs = append(equivalentPairs, [2]int{original, len(rows)}) + appendBytes(wig) + } + for _, triple := range []string{"x86_64", "i386"} { + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple="+triple, + "--mattr=+avx2,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(disassembly.String()) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || strings.Contains(string(output), "warning:") { + t.Fatalf("independent pack physical grammar: %s %v\n%s", triple, err, output) + } + var instructions []string + for _, line := range strings.Split(string(output), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "vpack") { + instructions = append(instructions, line) + } + } + if len(instructions) != len(rows) { + t.Fatalf("physical pack disassembly lost forms: %s got %d, want %d", triple, len(instructions), len(rows)) + } + for _, pair := range equivalentPairs { + if instructions[pair[0]] != instructions[pair[1]] { + t.Fatalf("WIG changed physical instruction: %s %q != %q", triple, instructions[pair[0]], instructions[pair[1]]) + } + } + } + t.Logf("%d actual Go W0 rows plus %d independently equivalent W1 rows", len(rows)-len(equivalentPairs), len(equivalentPairs)) + return rows +} + +func TestX86PackedNarrowMode32ActualGoRawForms(t *testing.T) { + rows := x86OfficialPackedNarrowRows(t, "386") + source := "TEXT rawpack(SB),4,$0-0\n" + strings.Join(rows, "\n") + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", true) + code := assembleX87ControlBytes(t, "386", source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "386") + if err != nil { + t.Fatal("actual Go mode32 pack bytes lost physical decoding", err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatal("actual byte decoder lost instructions", len(decoded.Instrs), len(want)) + } + for index, instruction := range decoded.Instrs { + if instruction.Op != want[index].Op || !reflect.DeepEqual(instruction.Args, want[index].Args) { + t.Fatalf("mode32 instruction %d: %+v != %+v", index, instruction, want[index]) + } + } +} + +func TestX86PackedNarrowE4NFMemoryRemainsObservable(t *testing.T) { + for _, opcode := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for _, bank := range []string{"X", "Y", "Z"} { + rows := []string{opcode + ".Z (AX)," + bank + "0,K1," + bank + "1"} + if opcode == "VPACKSSDW" || opcode == "VPACKUSDW" { + rows = append(rows, opcode+".BCST.Z (AX),"+bank+"0,K1,"+bank+"1") + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + source := "TEXT pack(SB),4,$0-0\nMOVL $0,CX\nKMOVQ CX,K1\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + initialized := strings.Replace(source, "$0-0\n", "$0-8\nMOVQ p+0(FP),AX\nVPXORD Z0,Z0,Z0\n", 1) + goAssemblerOracleOutput(t, "amd64", initialized, "", true) + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source, "func pack()", "amd64", "x86_64-unknown-linux-gnu", api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original missing-address source invented native input", api, err) + } + ir, err := x86ScalarFPBitViewAPI(t, initialized, "func pack(p *[64]byte)", "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "load volatile") || + !strings.Contains(x86MulDivRoundOptimize(t, ir), "load volatile") { + t.Fatal("E4NF memory read disappeared for a zero mask/dead output") + } + } + }) + } + } + } +} + +func TestX86PackedNarrowLegacyGoModeAndNativeStorageBoundaries(t *testing.T) { + for _, opcode := range []string{"PACKSSLW", "PACKSSWB", "PACKUSWB"} { + for _, source := range []string{"M0", "(AX)"} { + t.Run("386/"+opcode+"/"+source, func(t *testing.T) { + assembly := "TEXT pack(SB),4,$0-0\n" + opcode + " " + source + ",M1\nRET\n" + goAssemblerOracleOutput(t, "386", assembly, "", false) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "386", + Sigs: map[string]FuncSig{"pack": {Name: "pack", Ret: Void}}}); err == nil { + t.Fatal("Go-rejected mode32 MMX row acquired successful lowering") + } + }) + } + } + for _, opcode := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB"} { + t.Run("virtual-memory/"+opcode, func(t *testing.T) { + assembly := "TEXT pack(SB),4,$64-64\n" + opcode + " value+0(FP),X0\nRET\n" + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + sig := x86WordMaddFPType(t) + sig.Name = "pack" + if _, err := Translate(file, Options{Goarch: "amd64", + Sigs: map[string]FuncSig{"pack": sig}}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("virtual independent FP slots invented native legacy alignment", err) + } + }) + } +} diff --git a/amd64_packed_narrow_rules_test.go b/amd64_packed_narrow_rules_test.go new file mode 100644 index 000000000..b0e26920e --- /dev/null +++ b/amd64_packed_narrow_rules_test.go @@ -0,0 +1,140 @@ +package plan9asm + +import ( + "encoding/json" + "fmt" + "os/exec" + "reflect" + "strings" + "testing" +) + +func TestX86PackedNarrowMemoryShapeConsumesTheCompleteGrammar(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + rows := x86OfficialPackedNarrowRows(t, arch) + for _, opcode := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB"} { + rows = append(rows, opcode+" (AX),X1") + if arch == "amd64" && opcode != "PACKUSDW" { + rows = append(rows, opcode+" (AX),M1") + } + } + for _, row := range rows { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + if instruction.Args[0].Kind != OpMem { + continue + } + t.Run(arch+"/"+row, func(t *testing.T) { + want := int64(amd64VectorByteWidth(instruction.Args[len(instruction.Args)-1].Reg)) + if _, mmx := amd64ParseMReg(instruction.Args[len(instruction.Args)-1].Reg); mmx { + want = 8 + } + if strings.Contains(row, ".BCST") { + want = 4 + } + width, write, valid := x86InstructionMemoryShape(instruction, 0, arch) + if !valid || write || width != want { + t.Fatalf("memory-shape consumer diverged from Go's pack grammar: %d/%t/%t, want %d/false/true", width, write, valid, want) + } + for index := 1; index < len(instruction.Args); index++ { + if _, _, valid := x86InstructionMemoryShape(instruction, index, arch); valid { + t.Fatal("non-memory operand received a memory envelope") + } + } + }) + } + } +} + +func TestX86PackedNarrowRawWIGAttributesAndPhysicalMode32Masks(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM 22 llvm-mc required") + } + // Intel SDM Vol.2B PACK tables: VEX W is ignored; EVEX word-to-byte + // ignores W, but EVEX dword-to-word requires W0. This independent table + // deliberately does not copy the implementation's encoder attributes. + for _, opcode := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for _, width := range []string{"X", "Y", "Z"} { + for _, evex := range []bool{false, true} { + if !evex && width == "Z" { + continue + } + row := opcode + " " + width + "1," + width + "2," + if evex { + row += "K1," + } + row += width + "3" + original := assembleX87ControlBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + original = original[:len(original)-1] + if evex && original[0] != 0x62 || !evex && original[0] != 0xc4 && original[0] != 0xc5 { + t.Fatal("actual Go encoder selected a different generation", row, original) + } + code := append([]byte(nil), original...) + if code[0] == 0xc5 { + // Expand the actual C5 prefix to C4 before changing its W bit. + code = append([]byte{0xc4, code[1]&0x80 | 0x61, code[1] & 0x7f}, code[2:]...) + } + code[2] |= 0x80 + validW1 := !evex || opcode == "VPACKSSWB" || opcode == "VPACKUSWB" + for _, arch := range []string{"amd64", "386"} { + t.Run(arch+"/"+row, func(t *testing.T) { + if validW1 { + var encoded strings.Builder + for _, value := range code { + fmt.Fprintf(&encoded, "0x%02x ", value) + } + triple := "x86_64" + if arch == "386" { + triple = "i386" + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple="+triple, "--mattr=+avx2,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(encoded.String() + "\n") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || !strings.Contains(string(output), strings.ToLower(opcode)) || strings.Contains(string(output), "warning:") { + t.Fatalf("independent LLVM22 WIG identity: %x %v\n%s", code, err, output) + } + } + got, size, matched, err := decodedX86VectorPackedSaturatingNarrowInstruction(code, map[string]int{"amd64": 64, "386": 32}[arch]) + if !matched || (err == nil) != validW1 || validW1 && size != len(code) { + t.Fatalf("WIG/W0 family contract: %x known=%t bytes=%d error=%v valid=%t", code, matched, size, err, validW1) + } + if !validW1 { + return + } + before := fmt.Sprintf("%#v", got) + form, known, err := x86PackedNarrowProperties(got, arch) + if !known || err != nil || form.byteWidth != amd64VectorByteWidth(Reg(width+"0")) { + t.Fatal("physical raw input lost typed grammar", got, form, err) + } + copy := got + if form2, _, err := x86PackedNarrowProperties(copy, arch); err != nil || !reflect.DeepEqual(form, form2) { + t.Fatal("ordinary value copy lost private physical provenance", err) + } + if arch == "386" && evex { + data, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + var detached Instr + if err := json.Unmarshal(data, &detached); err != nil { + t.Fatal(err) + } + if _, _, err := x86PackedNarrowProperties(detached, arch); err == nil { + t.Fatal("JSON fabricated Go-rejected mode32 source-mask authority") + } + } + if before != fmt.Sprintf("%#v", got) { + t.Fatal("grammar changed original decoded input") + } + }) + } + } + } + } +} diff --git a/amd64_packed_physical_forms.go b/amd64_packed_physical_forms.go new file mode 100644 index 000000000..3a3ec3a0d --- /dev/null +++ b/amd64_packed_physical_forms.go @@ -0,0 +1,96 @@ +package plan9asm + +import "strings" + +type x86PackedPhysicalFamily string + +const ( + x86PackedAverage x86PackedPhysicalFamily = "packed average" + x86PackedWordProduct x86PackedPhysicalFamily = "packed word multiply" + x86PackedMultiplyAdd x86PackedPhysicalFamily = "packed multiply-add" + x86PackedDwordProduct x86PackedPhysicalFamily = "dword product" +) + +// A common shape does not make arithmetic lanes and memory accesses equal. +// Dword products read complete qwords before selecting their low dwords; +// multiply-add reads the full tuple even when all result lanes are masked off. +type x86PackedPhysicalSpec struct { + x86PackedBinarySpec + family x86PackedPhysicalFamily + inputBits, outputBits, memoryBits int // memoryBits == 0 means an E4NF full-tuple read. + mode amd64PackedWordMultiplyMode + signed bool + nativeOpcode string +} + +type x86PackedPhysicalForm struct { + x86PackedBinaryForm + spec x86PackedPhysicalSpec + source Operand // Actual physical bank, not an initialized value or storage permission. +} + +// The existing four family tables remain the only opcode membership authority. +// Normalize the name once, then share the physical grammar and its typed result +// among consumers; do not cache it on mutable instructions or across analyses. +func x86PackedPhysicalSpecForOp(op Op, family x86PackedPhysicalFamily) (x86PackedPhysicalSpec, bool) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if spec, known := x86PackedAverageSpecs[Op(base)]; known && (family == "" || family == x86PackedAverage) { + return x86PackedPhysicalSpec{ + x86PackedBinarySpec: spec.x86PackedBinarySpec, family: x86PackedAverage, + inputBits: spec.laneBits, outputBits: spec.laneBits, memoryBits: spec.laneBits, + }, true + } + if spec, known := x86PackedWordMultiplySpecs[Op(base)]; known && (family == "" || family == x86PackedWordProduct) { + return x86PackedPhysicalSpec{ + x86PackedBinarySpec: x86PackedBinarySpec{encoding: spec.encoding, zeroing: spec.encoding == x86PackedLogicalVEXOrEVEX}, + family: x86PackedWordProduct, inputBits: 16, outputBits: 16, memoryBits: 16, + mode: spec.mode, + }, true + } + if spec, known := x86PackedMultiplyAddSpecs[Op(base)]; known && (family == "" || family == x86PackedMultiplyAdd) { + return x86PackedPhysicalSpec{ + x86PackedBinarySpec: spec.x86PackedBinarySpec, family: x86PackedMultiplyAdd, + inputBits: spec.inputBits, outputBits: spec.outputBits, nativeOpcode: spec.nativeOpcode, + }, true + } + if spec, known := x86DwordProductSpecs[Op(base)]; known && (family == "" || family == x86PackedDwordProduct) { + return x86PackedPhysicalSpec{ + x86PackedBinarySpec: spec.x86PackedBinarySpec, family: x86PackedDwordProduct, + inputBits: 32, outputBits: 64, memoryBits: 64, + signed: spec.signed, nativeOpcode: spec.nativeOpcode, + }, true + } + return x86PackedPhysicalSpec{}, false +} + +// An empty family accepts this whole group. A family-specific consumer retains +// its original boundary, including a zero form before physical parsing starts. +func x86PackedPhysicalProperties(op Op, ins Instr, arch string, family x86PackedPhysicalFamily) (x86PackedPhysicalForm, bool, error) { + spec, known := x86PackedPhysicalSpecForOp(op, family) + if !known { + return x86PackedPhysicalForm{}, false, nil + } + binary, source, started, err := x86PackedBinaryPhysicalSource(op, ins, arch, spec.x86PackedBinarySpec, string(spec.family)) + if !started { + return x86PackedPhysicalForm{}, true, err + } + return x86PackedPhysicalForm{binary, spec, source}, true, err +} + +func (c *amd64Ctx) lowerPackedPhysical(op Op, ins Instr, family x86PackedPhysicalFamily) (bool, bool, error) { + form, known, err := x86PackedPhysicalProperties(op, ins, c.goarch, family) + if !known || err != nil { + return known, false, err + } + switch form.spec.family { + case x86PackedAverage: + return c.lowerPackedAverageForm(op, ins, form) + case x86PackedWordProduct: + return c.lowerPackedWordMultiplyForm(op, ins, form) + case x86PackedMultiplyAdd: + return c.lowerWordMultiplyAddForm(op, ins, form) + case x86PackedDwordProduct: + return c.lowerDwordToQwordMultiplyForm(op, ins, form) + } + panic("unregistered packed physical family") +} diff --git a/amd64_packed_physical_pipeline_bench_test.go b/amd64_packed_physical_pipeline_bench_test.go new file mode 100644 index 000000000..867110c11 --- /dev/null +++ b/amd64_packed_physical_pipeline_bench_test.go @@ -0,0 +1,41 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +// Measure complete analysis, lowering and LLVM verification for the four +// physical-source families. Parsing stays outside the timer; independent +// family tests cover numerical values, native faults and source permissions. +func BenchmarkX86PackedPhysicalTranslation(b *testing.B) { + for _, family := range x86PackedPhysicalTestFamilies() { + for _, name := range strings.Fields(family.names) { + b.Run(name, func(b *testing.B) { + operands := "X0,X1" + if strings.HasPrefix(name, "V") { + operands = "Y0,Y1,Y2" + } + source := "TEXT binary(SB),4,$0-0\n" + + "VPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\n" + + strings.Repeat(name+" "+operands+"\n", 32) + "RET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + b.Fatal(err) + } + options := Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"binary": {Name: "binary", Ret: Void}}, + } + b.ReportAllocs() + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + if _, err := Translate(file, options); err != nil { + b.Fatal(err) + } + } + }) + } + } +} diff --git a/amd64_packed_physical_properties_test.go b/amd64_packed_physical_properties_test.go new file mode 100644 index 000000000..854bc567a --- /dev/null +++ b/amd64_packed_physical_properties_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +type x86PackedPhysicalTestFamily struct { + names string + parse func(Op, Instr, string) (x86PackedBinaryForm, Operand, bool, error) + decode func([]byte, int) (Instr, int, bool, error) + rawOp string +} + +func x86PackedPhysicalTestFamilies() []x86PackedPhysicalTestFamily { + return []x86PackedPhysicalTestFamily{ + {"PAVGB PAVGW VPAVGB VPAVGW", func(op Op, ins Instr, arch string) (x86PackedBinaryForm, Operand, bool, error) { + form, known, err := x86PackedAverageProperties(op, ins, arch) + return form.x86PackedBinaryForm, form.source, known, err + }, decodedX86PackedAverageInstruction, "VPAVGB"}, + {"PMULLW PMULHW PMULHUW PMULHRSW VPMULLW VPMULHW VPMULHUW VPMULHRSW", func(op Op, ins Instr, arch string) (x86PackedBinaryForm, Operand, bool, error) { + form, known, err := x86PackedWordMultiplyProperties(op, ins, arch) + return form.x86PackedBinaryForm, form.source, known, err + }, decodedX86PackedMultiplyInstruction, "VPMULLW"}, + {"PMULDQ PMULULQ VPMULDQ VPMULUDQ", func(op Op, ins Instr, arch string) (x86PackedBinaryForm, Operand, bool, error) { + form, known, err := x86DwordProductProperties(op, ins, arch) + return form.x86PackedBinaryForm, form.source, known, err + }, decodedX86DwordToQwordMultiplyInstruction, "VPMULDQ"}, + {"PMADDWL PMADDUBSW VPMADDWD VPMADDUBSW", func(op Op, ins Instr, arch string) (x86PackedBinaryForm, Operand, bool, error) { + form, known, err := x86PackedMultiplyAddProperties(op, ins, arch) + return form.x86PackedBinaryForm, form.source, known, err + }, decodedX86EVEXPackedMADDInstruction, "VPMADDWD"}, + } +} + +func x86PackedPhysicalTestInstruction(name string, memory bool) Instr { + ins := Instr{Op: Op(name), Args: []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}}} + if strings.HasPrefix(name, "V") { + ins.Args = append(ins.Args, Operand{Kind: OpReg, Reg: "X2"}) + } + if memory { + ins.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Base: AX, Off: 16}} + } + return ins +} + +func x86PackedPhysicalTestMMX(name string) bool { + return strings.Contains(" PAVGB PAVGW PMULLW PMULHW PMULHUW PMULULQ PMADDWL ", " "+name+" ") +} + +func TestX86PackedPhysicalPropertiesPreserveOperands(t *testing.T) { + for _, family := range x86PackedPhysicalTestFamilies() { + for _, name := range strings.Fields(family.names) { + t.Run(name, func(t *testing.T) { + for _, memory := range []bool{false, true} { + ins := x86PackedPhysicalTestInstruction(name, memory) + before := append([]Operand(nil), ins.Args...) + form, source, known, err := family.parse(ins.Op, ins, "amd64") + if !known || err != nil || form.byteWidth != 16 || !reflect.DeepEqual(source, before[0]) { + t.Fatal("complete original form changed", form, source, err) + } + if !reflect.DeepEqual(ins.Args, before) { + t.Fatal("successful grammar observation mutated source operands") + } + if _, _, known, err := family.parse(ins.Op, ins, "arm64"); !known || err == nil { + t.Fatal("foreign source mode became a matching form", err) + } + changed := ins + changed.Op = "NOP" + if _, _, known, err := family.parse(ins.Op, changed, "amd64"); !known || err == nil { + t.Fatal("dispatch/source mismatch acquired a form", err) + } + if !reflect.DeepEqual(ins.Args, before) { + t.Fatal("rejected grammar observation mutated source operands") + } + } + if !x86PackedPhysicalTestMMX(name) { + return + } + for _, alias := range []struct{ source, physical Reg }{{AX, "M0"}, {SP, "M4"}} { + ins := Instr{Op: Op(name), Args: []Operand{{Kind: OpReg, Reg: alias.source}, {Kind: OpReg, Reg: "M1"}}} + before := append([]Operand(nil), ins.Args...) + form, source, known, err := family.parse(ins.Op, ins, "amd64") + if !known || err != nil || !form.mmx || form.byteWidth != 8 || source.Reg != alias.physical { + t.Fatal("original GP spelling lost its distinct MMX bank", form, source, err) + } + if !reflect.DeepEqual(ins.Args, before) { + t.Fatal("MMX projection overwrote the original GP spelling") + } + if _, _, known, err := family.parse(ins.Op, ins, "386"); !known || err == nil { + t.Fatal("named mode32 source acquired an MMX row", err) + } + } + }) + } + } +} + +func TestX86PackedPhysicalPropertiesRetainDecoderBoundaries(t *testing.T) { + for _, family := range x86PackedPhysicalTestFamilies() { + t.Run(family.rawOp, func(t *testing.T) { + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+ + family.rawOp+" 16(AX),X1,K1,X2\nRET\n") + if len(code) < 2 || code[len(code)-1] != 0xc3 { + t.Fatal("original instruction bytes lack the terminal RET", code) + } + code = code[:len(code)-1] + ins, length, known, err := family.decode(code, 32) + if !known || err != nil || length != len(code) || !ins.x86Encoded { + t.Fatal("actual mode32 physical decode failed", ins, length, err) + } + before := append([]Operand(nil), ins.Args...) + form, source, known, err := family.parse(ins.Op, ins, "386") + if !known || err != nil || form.mask < 0 || form.memory != 0 || source.Mem.Base != AX { + t.Fatal("physical masked form lost its original decoder identity", form, source, err) + } + text := ins + text.x86Encoded = false + if _, _, _, err := family.parse(text.Op, text, "386"); err == nil { + t.Fatal("named four-operand source borrowed decoder admission") + } + for _, bits := range []int{16, 32, 64} { + changed := ins + changed.x86AddressBits = bits + if _, _, _, err := family.parse(changed.Op, changed, "386"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("address-size override lost its independent context requirement", bits, err) + } + } + changed := ins + changed.x86UnboundAddressOverride = true + if _, _, _, err := family.parse(changed.Op, changed, "386"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound address override acquired a grammar-only authority", err) + } + if !reflect.DeepEqual(ins.Args, before) { + t.Fatal("physical or rejected observations modified original operands") + } + }) + } +} + +func TestX86PackedPhysicalPropertiesAvoidRedundantCopies(t *testing.T) { + for _, family := range x86PackedPhysicalTestFamilies() { + for _, name := range strings.Fields(family.names) { + t.Run(name, func(t *testing.T) { + for _, memory := range []bool{false, true} { + ins := x86PackedPhysicalTestInstruction(name, memory) + _, _, known, err := family.parse(ins.Op, ins, "amd64") + if !known || err != nil { + t.Fatal("allocation fixture must be accepted", err) + } + allocations := testing.AllocsPerRun(100, func() { + _, _, known, err = family.parse(ins.Op, ins, "amd64") + }) + limit := 0.0 + if x86PackedPhysicalTestMMX(name) { + limit = 1 // Existing MMX bank adapter owns its source projection. + } + if !known || err != nil || allocations > limit { + t.Errorf("memory=%v: %.0f allocations, want at most %.0f; known=%v err=%v", memory, allocations, limit, known, err) + } + } + }) + } + } +} + +func BenchmarkX86PackedPhysicalProperties(b *testing.B) { + for _, family := range x86PackedPhysicalTestFamilies() { + for _, name := range strings.Fields(family.names) { + for _, memory := range []bool{false, true} { + kind := "register" + if memory { + kind = "memory" + } + ins := x86PackedPhysicalTestInstruction(name, memory) + b.Run(name+"/"+kind, func(b *testing.B) { + b.ReportAllocs() + for n := 0; n < b.N; n++ { + if _, _, known, err := family.parse(ins.Op, ins, "amd64"); !known || err != nil { + b.Fatal("allocation benchmark rejected its unchanged input", err) + } + } + }) + } + } + } +} + +const x86PackedNarrowPhysicalNames = "PACKSSLW PACKSSWB PACKUSDW PACKUSWB VPACKSSDW VPACKSSWB VPACKUSDW VPACKUSWB" + +func TestX86PackedNarrowPhysicalRejectionContract(t *testing.T) { + for _, row := range []struct { + instruction string + context bool + width int + }{ + {"VPACKSSDW (R8),X1,X8", true, 16}, + {"VPACKSSDW (AX),X1,X8", true, 16}, + {"VPACKSSDW (R8),X1,X2", false, 16}, + {"PACKSSWB X0,M1", false, 0}, + } { + t.Run(row.instruction, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row.instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + ins.x86Encoded = true + before := append([]Operand(nil), ins.Args...) + form, known, err := x86PackedNarrowProperties(ins, "386") + if !known || err == nil || errors.Is(err, ErrProbeNeedsContext) != row.context || + form.byteWidth != row.width || !reflect.DeepEqual(form.source, Operand{}) { + t.Fatal("rejected physical form changed its classification or retained a source", form, known, err) + } + if row.width == 0 && !reflect.DeepEqual(form, x86PackedNarrowForm{}) { + t.Fatal("early MMX failure retained a partial form") + } + if !reflect.DeepEqual(ins.Args, before) { + t.Fatal("rejected physical source was modified") + } + }) + } +} + +func TestX86PackedNarrowPropertiesAvoidRedundantCopies(t *testing.T) { + for _, name := range strings.Fields(x86PackedNarrowPhysicalNames) { + t.Run(name, func(t *testing.T) { + for _, memory := range []bool{false, true} { + ins := x86PackedPhysicalTestInstruction(name, memory) + _, known, err := x86PackedNarrowProperties(ins, "amd64") + if !known || err != nil { + t.Fatal("allocation fixture must be accepted", err) + } + allocations := testing.AllocsPerRun(100, func() { + _, known, err = x86PackedNarrowProperties(ins, "amd64") + }) + limit := 0.0 + if !strings.HasPrefix(name, "V") && name != "PACKUSDW" { + limit = 1 // Existing MMX bank adapter owns its projection. + } + if !known || err != nil || allocations > limit { + t.Errorf("memory=%v: %.0f allocations, want at most %.0f; known=%v err=%v", memory, allocations, limit, known, err) + } + } + }) + } +} + +func BenchmarkX86PackedNarrowProperties(b *testing.B) { + for _, name := range strings.Fields(x86PackedNarrowPhysicalNames) { + for _, memory := range []bool{false, true} { + kind := "register" + if memory { + kind = "memory" + } + ins := x86PackedPhysicalTestInstruction(name, memory) + b.Run(name+"/"+kind, func(b *testing.B) { + b.ReportAllocs() + for n := 0; n < b.N; n++ { + if _, known, err := x86PackedNarrowProperties(ins, "amd64"); !known || err != nil { + b.Fatal("allocation benchmark rejected its input", err) + } + } + }) + } + } +} diff --git a/amd64_packed_precision_forms.go b/amd64_packed_precision_forms.go new file mode 100644 index 000000000..909cbd67d --- /dev/null +++ b/amd64_packed_precision_forms.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" +) + +// Reuse pure lane/carrier axes from packed conversion, not its caller or +// numeric-family authority. Narrowing's active low bytes differ from its X/Y +// carrier; the remaining carrier bytes are architectural zeros. +var x86PackedPrecisionSpecs = map[Op]x86PackedConversionSpec{ + "CVTPS2PD": {amd64PackedNumericConversionSpec: amd64PackedNumericConversionSpec{ + sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericFloat, sourceBits: 32, destinationBits: 64, + shapes: amd64PackedNumericWiden32To64[:1]}, legacy: true}, + "CVTPD2PS": {amd64PackedNumericConversionSpec: amd64PackedNumericConversionSpec{ + sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, + shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, legacy: true}, + "VCVTPS2PD": {amd64PackedNumericConversionSpec: amd64PackedNumericConversionSpec{ + sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericFloat, sourceBits: 32, destinationBits: 64, + shapes: amd64PackedNumericWiden32To64, explicit: "sae"}, vex: true}, + "VCVTPD2PSX": {amd64PackedNumericConversionSpec: amd64PackedNumericConversionSpec{ + sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, + shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, vex: true}, + "VCVTPD2PSY": {amd64PackedNumericConversionSpec: amd64PackedNumericConversionSpec{ + sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, + shapes: []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}}}, vex: true}, + "VCVTPD2PS": {amd64PackedNumericConversionSpec: amd64PackedNumericConversionSpec{ + sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, + shapes: []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}}, explicit: "round"}, vex: true}, +} + +func x86PackedPrecisionProperties(op Op, ins Instr, arch string) (x86PackedConversionForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86PackedPrecisionSpecs[Op(base)] + if !known { + return x86PackedConversionForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(op), string(ins.Op)) || dotted && suffix == "" { + return x86PackedConversionForm{}, true, fmt.Errorf("%s packed precision opcode/control identity mismatch: %s/%s", arch, op, ins.Op) + } + ins.Op = op + local, contextual := x86VectorValueMemoryInput(ins, arch) + form, _, err := x86PackedConversionOperandProperties(local, arch, spec) + if err != nil { + return form, true, err + } + if form.memory >= 0 { + if contextual { + return form, true, x86VectorValueMemoryContext(ins) + } + if err := x86NamedMultiplyMemoryError(ins.Args[0], arch); err != nil { + return form, true, err + } + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return form, true, fmt.Errorf("%w: %s needs its original default address mode", ErrProbeNeedsContext, op) + } + form.evex = form.evex || !spec.legacy && ins.x86PackedPrecisionEVEX + return form, true, nil +} + +func x86TypedPackedPrecisionByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedPrecisionProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := form.inputBytes + if form.memory >= 0 { + width = form.memoryBytes + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: int64(width), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.shape.destinationBytes), + read: form.mask >= 0 && !form.properties.zeroing, write: true}, + }} + if form.mask >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return effect, true, nil +} + +func validateX86PackedPrecisionFile(file *File, opt Options) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + var pending error + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + _, known, err := x86PackedPrecisionProperties(ins.Op, ins, opt.Goarch) + if !known { + continue + } + if err != nil && !errors.Is(err, ErrProbeNeedsContext) { + return fmt.Errorf("%s: %w", fn.Sym, err) + } + if err == nil && !x86ScalarFloatIntegerNativeTarget(opt.Goarch, opt.TargetTriple) { + err = fmt.Errorf("%w: %s needs its matching native x86 MXCSR target", ErrProbeNeedsContext, ins.Op) + } + if pending == nil && err != nil { + pending = fmt.Errorf("%s: %w", fn.Sym, err) + } + } + } + return pending +} diff --git a/amd64_packed_reciprocal_forms.go b/amd64_packed_reciprocal_forms.go new file mode 100644 index 000000000..c51a74d15 --- /dev/null +++ b/amd64_packed_reciprocal_forms.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedReciprocalForm struct { + op Op + byteWidth int + legacy bool +} + +// Go's yxm and _yvptest tables have only X/m128 and VEX X/Y packed +// estimates. Scalar estimates, EVEX, masks and suffixes are separate grammars. +func x86PackedReciprocalProperties(op Op, ins Instr, arch string) (x86PackedReciprocalForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + form := x86PackedReciprocalForm{op: Op(name)} + switch name { + case "RCPPS", "RSQRTPS": + form.legacy = true + case "VRCPPS", "VRSQRTPS": + default: + return form, false, nil + } + if arch == "" { + arch = "amd64" + } + fail := func() (x86PackedReciprocalForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go packed reciprocal grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != 2 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + form.byteWidth = amd64VectorByteWidth(ins.Args[1].Reg) + vector := func(operand Operand) bool { + if !form.legacy { + return amd64VEXVectorRegister(operand, form.byteWidth) + } + index, valid := amd64ParseXReg(operand.Reg) + limit := 16 + if arch == "386" { + limit = 8 + } + return operand.Kind == OpReg && valid && index < limit + } + if form.byteWidth != 16 && (form.legacy || form.byteWidth != 32) || !vector(ins.Args[1]) { + return fail() + } + source := ins.Args[0] + if source.Kind == OpReg { + if !vector(source) { + return fail() + } + } else if !form.legacy && x86Mode32UnboundVEXMemorySource(source, arch) { + return form, true, fmt.Errorf("%w: 386 %s extended source base has no physical address binding", ErrProbeNeedsContext, op) + } else if !isAMD64MemoryOperand(source) || !x86PackedShiftMemoryValid(source, arch) { + return fail() + } else if err := x86NamedMultiplyMemoryError(source, arch); err != nil { + return form, true, err + } + return form, true, x86Mode32VectorRegisterContext(ins, arch) +} + +func x86TypedPackedReciprocalByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedReciprocalProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + vector := x86TypedVectorEffect{upperWrite: x86TypedVectorUpperZero, memoryAlignment: 1} + if form.legacy { + vector.upperWrite, vector.memoryAlignment = x86TypedVectorUpperPreserve, 16 + } + width := int64(form.byteWidth) + return x86TypedByteEffect{requiresResultProof: true, vector: vector, operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: width, read: true}, + {index: 1, operand: ins.Args[1], width: width, write: true}, + }}, true, nil +} + +func x86ResultPackedReciprocalStep(ins Instr, state *x86ResultState, owned []bool, arch string, native bool) (bool, bool) { + form, known, err := x86PackedReciprocalProperties(ins.Op, ins, arch) + if !known || err != nil || !x86ResultNamedPredicateInstruction(ins) { + return false, known + } + source, mask := ins.Args[0], x86CallerVectorByteMask(int64(form.byteWidth)) + defined := mask + if source.Kind == OpReg { + index, _ := amd64VectorRegisterIndex(source.Reg, form.byteWidth) + if state.vectorBytes(index)&mask != mask { + defined = 0 // Pure dead estimates may retain an unknown numerical value. + } + } else { + // Virtual FP bytes are not an aligned native m128 operand. The + // paired original opcode, rather than storage permission, owns #GP. + if form.legacy { + if _, static := x86StaticSBReadAddress(source, arch); !static && + (!native || !x86OrdinaryNativeMemoryOperand(source, arch)) { + return false, true + } + } + if !x86ResultReadWithNativeContinuation(ins, source, int64(form.byteWidth), state, owned, arch, native) { + return false, true + } + } + destination, _ := amd64VectorRegisterIndex(ins.Args[1].Reg, form.byteWidth) + state.writeVector(destination, form.byteWidth, defined, form.legacy) + return true, true // No FLAGS, DF, MXCSR or storage facts change. +} diff --git a/amd64_packed_reduction_boundary_test.go b/amd64_packed_reduction_boundary_test.go new file mode 100644 index 000000000..437718fb9 --- /dev/null +++ b/amd64_packed_reduction_boundary_test.go @@ -0,0 +1,108 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86PackedReductionDotCompleteOfficialGoAttributes(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, opcode := range []string{"DPPD", "DPPS", "VDPPD", "VDPPS"} { + code := "0x41" + if strings.HasSuffix(opcode, "PS") { + code = "0x40" + } + if opcode[0] != 'V' { + if !strings.Contains(string(legacy), "{A"+opcode+", yxshuf, Pq, opBytes{0x3a, "+code+", 0}}") || + !strings.Contains(string(legacy), "{Zibm_r, 2, argList{Yu8, Yxm, Yxr}}") { + t.Fatal("complete dot legacy table/opcode attributes changed", opcode) + } + continue + } + table, widths := "_yvdppd", []string{"vex128"} + if opcode == "VDPPS" { + table, widths = "_yvblendpd", []string{"vex128", "vex256"} + } + begin := strings.Index(string(avx), "{as: A"+opcode+",") + if begin < 0 { + t.Fatal("official Go omitted dot member", opcode) + } + row := strings.SplitN(string(avx)[begin:], "\n\t}},", 2)[0] + if !strings.Contains(row, "ytab: "+table+",") || strings.Contains(row, "evex") { + t.Fatal("dot borrowed an unsupported width/mask/suffix generation", row) + } + for _, width := range widths { + if !strings.Contains(row, "avxEscape | "+width+" | vex66 | vex0F3A | vexW0, "+code) { + t.Fatal("complete dot VEX opcode attributes changed", row) + } + } + } +} + +func TestX86PackedReductionVoidRetainsMeaningfulInputsAndNativeBoundaries(t *testing.T) { + for _, row := range x86PackedReductionOracleRows() { + destination := 1 + if !row.legacy { + destination = 2 + } + source, declaration := x86PackedReductionInitializedFixture("Value", "amd64", row.row, destination) + variants := map[string]string{} + if strings.Contains(row.row, "(BX)") { + variants["numeric-is-not-a-pointer"] = strings.Replace(source, "MOVQ p+0(FP),BX", "MOVQ $42,BX", 1) + variants["unknown-pointer"] = strings.Replace(source, "MOVQ p+0(FP),BX\n", "", 1) + variants["native-SP"] = strings.Replace(source, row.row, strings.Replace(row.row, "(BX)", "(SP)", 1), 1) + } else { + variants["missing-first-data"] = strings.Replace(source, "VPXORD Z0,Z0,Z0\n", "", 1) + } + variants["missing-second-or-legacy-old-data"] = strings.Replace(source, "VPXORD Z1,Z1,Z1\n", "", 1) + for name, changed := range variants { + t.Run(fmt.Sprintf("%s/%s", row.row, name), func(t *testing.T) { + if name == "numeric-is-not-a-pointer" { + x86RequireNumericNativeSourceContext(t, changed, declaration, "amd64") + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", changed) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, changed, declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if name == "numeric-is-not-a-pointer" { + if err != nil || !strings.Contains(ir, fmt.Sprintf("call <%d x float> asm sideeffect \"%s ", row.width/4, row.mnemonic)) || + !strings.Contains(ir, "~{memory}") || !strings.Contains(ir, "~{fpsr}") { + t.Fatal("complete NUM address lost its original reduction width or effects", api, err) + } + x86NumericNativeAddressObject(t, ir, "x86_64-unknown-linux-gnu") + continue + } + if strings.HasPrefix(name, "missing-") { + x86VectorScratchCodegenControl(t, changed, declaration, "amd64", "x86_64-unknown-linux-gnu", ir, err) + continue + } + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Error("Void invented a meaningful arithmetic input or native storage permission", api, err) + } + } + }) + } + if !row.legacy { + t.Run(row.row+"/VEX-write-only-destination", func(t *testing.T) { + withoutOld := strings.Replace(source, "VPXORD Z2,Z2,Z2\n", "", 1) + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, withoutOld, declaration, "amd64", "x86_64-unknown-linux-gnu", api); err != nil { + t.Fatal("VEX reduction invented a merging destination dependency", api, err) + } + } + }) + } + } +} diff --git a/amd64_packed_reduction_bytes_test.go b/amd64_packed_reduction_bytes_test.go new file mode 100644 index 000000000..f9cc858c4 --- /dev/null +++ b/amd64_packed_reduction_bytes_test.go @@ -0,0 +1,313 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "regexp" + "strconv" + "strings" + "testing" +) + +func x86PackedReductionOriginalRawRows(t *testing.T, rows []x86PackedReductionOracleRow) []string { + t.Helper() + var instructions []string + for _, row := range rows { + instructions = append(instructions, row.row) + } + listing, err := x86ExtensionGoListing(t, "amd64", strings.Join(instructions, "\n")) + if err != nil { + t.Fatalf("actual original reduction listing: %v\n%s", err, listing) + } + raw, err := x86PackedReductionRawRowsFromGoListing(listing, rows) + if err != nil { + t.Fatalf("complete original reduction Go PC/byte boundaries: %v\n%s", err, listing) + } + return raw +} + +var x86PackedReductionListingInstruction = regexp.MustCompile(`^\s*0x([0-9a-f]+)\s+([0-9]+)\s+\(.*:([0-9]+)\)\s+([A-Z0-9]+)(?:\s|$)`) + +// The fixture's boundaries come from the actual Go listing, not our decoder. +// In particular x86asm may accept just a VEX prefix as a shorter instruction. +// Bind every source line and PC, then independently check all original bytes, +// including the terminal RET. Neither padding nor an unexplained suffix is +// silently dropped from the source used by the raw continuation tests. +func x86PackedReductionRawRowsFromGoListing(listing string, rows []x86PackedReductionOracleRow) ([]string, error) { + var expected [][]byte + for _, row := range rows { + expected = append(expected, x86PackedReductionIndependentFixtureBytes(row)) + } + return x86PackedReductionRawRowsFromGoListingWithExpected(listing, rows, expected) +} + +func x86PackedReductionRawRowsFromGoListingWithExpected(listing string, rows []x86PackedReductionOracleRow, expected [][]byte) ([]string, error) { + if len(expected) != len(rows) { + return nil, fmt.Errorf("incomplete independent original byte inventory") + } + var code []byte + var pcs []int + for _, line := range strings.Split(listing, "\n") { + if encoded := x86InsertExtractListedBytes(line); len(encoded) != 0 { + fields := strings.Fields(line) + pc, err := strconv.ParseUint(strings.TrimPrefix(fields[0], "0x"), 16, 32) + if err != nil || int(pc) != len(code) { + return nil, fmt.Errorf("noncontiguous Go byte listing at %q", line) + } + code = append(code, encoded...) + continue + } + match := x86PackedReductionListingInstruction.FindStringSubmatch(line) + if match == nil { + continue + } + pc, pcErr := strconv.ParseUint(match[1], 16, 32) + decimalPC, decimalErr := strconv.ParseUint(match[2], 10, 32) + sourceLine, sourceErr := strconv.Atoi(match[3]) + if pcErr != nil || decimalErr != nil || sourceErr != nil || pc != decimalPC { + return nil, fmt.Errorf("invalid Go PC/source row %q", line) + } + if sourceLine == 1 && pc == 0 && (match[4] == "TEXT" || match[4] == "FUNCDATA") { + continue + } + index := len(pcs) + mnemonic := "RET" + if index < len(rows) { + mnemonic = strings.ToUpper(rows[index].mnemonic) + } + if index > len(rows) || sourceLine != index+2 || match[4] != mnemonic || + index == 0 && pc != 0 || index > 0 && int(pc) <= pcs[index-1] { + return nil, fmt.Errorf("missing, duplicate or changed original Go instruction %q", line) + } + pcs = append(pcs, int(pc)) + } + if len(pcs) != len(rows)+1 || len(code) != pcs[len(pcs)-1]+1 || code[len(code)-1] != 0xc3 { + return nil, fmt.Errorf("incomplete original Go program: PCs=%v bytes=%x", pcs, code) + } + var raw []string + for index, row := range rows { + start, end := pcs[index], pcs[index+1] + if end > len(code) { + return nil, fmt.Errorf("original Go PC outside byte listing: %s at %d:%d", row.row, start, end) + } + want := expected[index] + if !bytes.Equal(code[start:end], want) { + return nil, fmt.Errorf("original Go instruction bytes changed: %s at %d:%d got=%x want=%x", row.row, start, end, code[start:end], want) + } + raw = append(raw, strings.TrimSuffix(x86RawDisplacementDirectives(code[start:end]), "\n")) + } + return raw, nil +} + +// These independent oracle rows use only X/Y0, X/Y1, X/Y2 and bare (BX). +// Check each actual Go encoding's opcode, pp/L, register fields and selector; +// the production parser, typed spec and decoder do not supply these bytes. +func x86PackedReductionIndependentFixtureBytes(row x86PackedReductionOracleRow) []byte { + memory := strings.Contains(row.row, "(BX)") + modRM := byte(0xc8) + if memory { + modRM = 0x0b + } + if !row.legacy { + modRM += 8 + } + opcode := byte(0x7c) + if strings.Contains(row.mnemonic, "hsub") { + opcode = 0x7d + } + if row.dot { + opcode = 0x40 + if strings.HasSuffix(row.mnemonic, "pd") { + opcode = 0x41 + } + if row.legacy { + return []byte{0x66, 0x0f, 0x3a, opcode, modRM, 0xff} + } + control := byte(0x71) + if row.width == 32 { + control |= 4 + } + return []byte{0xc4, 0xe3, control, opcode, modRM, 0xff} + } + prefix := byte(0xf2) + if strings.HasSuffix(row.mnemonic, "pd") { + prefix = 0x66 + } + if row.legacy { + return []byte{prefix, 0x0f, opcode, modRM} + } + control := byte(0xf3) + if prefix == 0x66 { + control = 0xf1 + } + if row.width == 32 { + control |= 4 + } + return []byte{0xc5, control, opcode, modRM} +} + +func TestX86PackedReductionGoListingKeepsWholeBoundaries(t *testing.T) { + rows := []x86PackedReductionOracleRow{ + {row: "VDPPS $255,X0,X1,X2", mnemonic: "vdpps", width: 16, dot: true}, + {row: "VHADDPD Y0,Y1,Y2", mnemonic: "vhaddpd", width: 32}, + } + const listing = `extension.extension STEXT nosplit asm size=11 align=0x0 args=0x0 locals=0x0 + 0x0000 00000 (extension.s:1) TEXT extension.extension(SB), NOSPLIT|NOFRAME, $0-0 + 0x0000 00000 (extension.s:1) FUNCDATA $0, gclocals(SB) + 0x0000 00000 (extension.s:2) VDPPS $255, X0, X1, X2 + 0x0006 00006 (extension.s:3) VHADDPD Y0, Y1, Y2 + 0x000a 00010 (extension.s:4) RET + 0x0000 c4 e3 71 40 d0 ff c5 f5 7c d0 c3 ..q@....|.. +` + raw, err := x86PackedReductionRawRowsFromGoListing(listing, rows) + if err != nil || len(raw) != 2 { + t.Fatal("independent complete VEX/RET listing rejected", raw, err) + } + for name, changed := range map[string]string{ + "partial VEX boundary": strings.Replace(listing, "0x0006 00006", "0x0002 00002", 1), + "source row changed": strings.Replace(listing, "extension.s:3", "extension.s:5", 1), + "opcode changed": strings.Replace(listing, ") VHADDPD", ") VHSUBPD", 1), + "physical opcode": strings.Replace(strings.Replace(listing, "71 40 d0", "71 41 d0", 1), "..q@", "..qA", 1), + "physical vector L": strings.Replace(listing, "c5 f5", "c5 f1", 1), + "missing RET": strings.Replace(listing, " 0x000a 00010 (extension.s:4) RET\n", "", 1), + "byte gap": strings.Replace(listing, "0x0000 c4", "0x0001 c4", 1), + "trailing bytes": strings.Replace(listing, "c3 ..q@....|..", "c3 90 ..q@....|...", 1), + "abbreviated bytes": strings.Replace(listing, " ..q@....|..", "", 1), + } { + t.Run(name, func(t *testing.T) { + if _, err := x86PackedReductionRawRowsFromGoListing(changed, rows); err == nil { + t.Fatal("incomplete or changed whole-instruction listing accepted") + } + }) + } +} + +func x86PackedReductionOracleState(width int) x86ResultState { + state := x86ResultState{flags: x86ByteFlagsAll, direction: x86ResultDirectionBackward} + for index := 0; index < 3; index++ { + state.writeVector(index, 64, uint64(0xa5a55a5affff0000)|x86CallerVectorByteMask(int64(width)), true) + } + state.gp[3], state.pointerGP[3] = 0xff, true + return state +} + +func TestX86PackedReductionEverySelectorMeaningfulByteAndUpperAlias(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86PackedReductionOracleRows() { + if strings.Contains(row.row, "(BX)") || arch == "386" && row.dot && !row.legacy { + continue + } + t.Run(arch+"/"+row.row, func(t *testing.T) { + selectors := 1 + if row.dot { + selectors = 256 + } + for selector := 0; selector < selectors; selector++ { + spelling := strings.Replace(row.row, "$255", fmt.Sprintf("$%d", selector), 1) + ins := x86ScalarLaneInstruction(t, spelling) + initial := x86PackedReductionOracleState(row.width) + destination, _ := amd64VectorRegisterIndex(ins.Args[row.destination].Reg, row.width) + want := initial.clone() + want.writeVector(destination, row.width, x86CallerVectorByteMask(int64(row.width)), row.legacy) + state := initial.clone() + if !x86ResultTransfer(ins, &state, nil, 0, arch) || !reflect.DeepEqual(state, want) { + t.Fatal("complete initialized reduction changed unrelated input/FLAGS/DF/upper bytes", spelling) + } + for _, input := range []int{row.first, row.second} { + source, _ := amd64VectorRegisterIndex(ins.Args[input].Reg, row.width) + for missing := 0; missing < row.width; missing++ { + required := true + if row.dot { + laneBytes := 4 + if strings.HasSuffix(row.mnemonic, "pd") { + laneBytes = 8 + } + required = selector&(1< len(code) || !bytes.Equal(code[offset:end], want) { + t.Fatal("original native reduction fault bytes/whole boundaries changed", row.instruction, offset) + } + result = append(result, append([]byte(nil), code[offset:end]...)) + offset = end + } + if offset != len(code) { + t.Fatal("unconsumed original native reduction fault bytes", offset, len(code)) + } + return result +} + +func x86PackedReductionFaultFixture(rows []x86PackedReductionRuntimeRow, original [][]byte) (assembly, declarations, driver string) { + var source, decl, native, prototypes, pairs strings.Builder + for index, row := range rows { + name, reference := fmt.Sprintf("reductionfault%d", index), fmt.Sprintf("original%d", index) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-24\nMOVQ p+0(FP),DI\nMOVQ csr+8(FP),SI\nMOVQ b+16(FP),DX\n"+ + "LDMXCSR (SI)\nVMOVDQU (DX),Y1\n%s\nRET\n", name, row.instruction) + fmt.Fprintf(&decl, "func %s(p *byte,csr *uint32,b *[4]uint64)\n", name) + var octets []string + for _, value := range original[index] { + octets = append(octets, fmt.Sprintf("0x%02x", value)) + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+reference+"\n"+reference+":\n.byte "+strings.Join(octets, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,uint32_t*,uint64_t*),%s(uint8_t*,uint32_t*,uint64_t*);\n", name, reference) + bits := 32 + if strings.HasSuffix(row.oracle.mnemonic, "pd") { + bits = 64 + } + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d,%d,%d,%d},\n", reference, name, row.oracle.width, bits, + x86FloatAddSubBoolInt(row.oracle.legacy), x86FloatAddSubBoolInt(row.oracle.dot), row.selector, + x86FloatAddSubBoolInt(strings.Contains(row.oracle.mnemonic, "hsub"))) + } + driver = `#include +#include +#include +#include +#include +#include +#include +#include +` + native.String() + prototypes.String() + ` +typedef void (*program)(uint8_t*,uint32_t*,uint64_t*); +static const struct {program original,translated;unsigned width,bits,aligned,dot,selector,subtract;} pairs[]={ +` + pairs.String() + `}; +static int outcome(program fn,uint8_t *p,uint32_t csr,uint64_t *second) { + pid_t child=fork();if(child<0)return -1; + if(!child) { + struct rlimit no_core={0,0};if(setrlimit(RLIMIT_CORE,&no_core))_exit(120); + fn(p,&csr,second);_exit(0); + } + int status;if(waitpid(child,&status,0)!=child)return -1; + if(WIFEXITED(status)&&WEXITSTATUS(status)==0)return 0; + if(WIFSIGNALED(status)) { + int sig=WTERMSIG(status); + if(sig==SIGSEGV||sig==SIGBUS)return 1; + if(sig==SIGFPE)return 2; + } + return -1; +} +static void fill(uint8_t *p,unsigned bits,uint64_t even,uint64_t odd) { + for(unsigned lane=0;lane<32/(bits/8);lane++) { + uint64_t value=lane&1?odd:even; + memcpy(p+lane*(bits/8),&value,bits/8); + } +} +int main(void) { + long page=sysconf(_SC_PAGESIZE);if(page<64)return 1; + uint8_t *p=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(p==MAP_FAILED||mprotect(p+page,page,PROT_NONE))return 2; + _Alignas(16) uint64_t second[4]={0}; + memset(p,0,page); + for(unsigned i=0;i= x86PackedReductionNativeOwners { + t.Fatal("invalid complete native reduction fault owner", owner) + } + execution := x86PackedReductionNativeTools(t) + rows := x86PackedReductionNativeRows(true) + if len(rows) != 37 { + t.Fatal("complete native reduction fault denominator changed", len(rows)) + } + consumed := 0 + for batch, start := 0, 0; start < len(rows); batch, start = batch+1, start+x86PackedReductionFaultCapture { + if batch%x86PackedReductionNativeOwners != owner { + continue + } + end := start + x86PackedReductionFaultCapture + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + current := rows[start:end] + var original [][]byte + t.Run("original_go_opcode_reference", func(t *testing.T) { + original = x86PackedReductionFaultOriginalBytes(t, current) + if len(original) != len(current) { + t.Fatal("incomplete actual Go-assembled native reference kernels") + } + }) + if len(original) != len(current) { + t.Fatal("missing actual Go-assembled native reference kernels") + } + assembly, declarations, driver := x86PackedReductionFaultFixture(current, original) + raw := x86PackedReductionRuntimeRawRows(t, current, true) + rawRows := append([]x86PackedReductionRuntimeRow(nil), current...) + for index := range rawRows { + rawRows[index].instruction = raw[index] + } + rawAssembly, _, _ := x86PackedReductionFaultFixture(rawRows, original) + for _, spelling := range []struct{ name, source string }{{"named", assembly}, {"raw", rawAssembly}} { + t.Run(spelling.name, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", spelling.source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, spelling.source, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal("initialized native reduction dead source rejected", err) + } + x86PackedReductionNativePipelines(t, execution, ir, driver, + []byte("complete original Go-opcode/LLVM native reduction dead-fault receipt\n")) + }) + } + }) + } + consumed += len(current) + }) + } + if consumed == 0 { + t.Fatal("empty native reduction dead-fault owner") + } + t.Logf("%d/37 original-Go-opcode native memory/alignment and actual unmasked invalid/denormal/overflow/precision/dot-underflow kernels", consumed) +} diff --git a/amd64_packed_reduction_forms.go b/amd64_packed_reduction_forms.go new file mode 100644 index 000000000..9347ef736 --- /dev/null +++ b/amd64_packed_reduction_forms.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// These two arithmetic families share a two-source native reduction envelope, +// not an arithmetic formula. Horizontal pairs consume every input byte; dot +// products consume only imm-selected lanes while still reading the full native +// memory tuple and retaining the opcode's complete exception/MXCSR effects. +type x86PackedReductionForm struct { + first, second, destination, memory int + byteWidth, laneBits int + dot, legacy bool + immediate uint8 +} + +func x86PackedReductionProperties(ins Instr, arch string) (x86PackedReductionForm, bool, error) { + if horizontal, known, err := x86HorizontalFloatProperties(ins, arch); known { + return x86PackedReductionForm{ + first: horizontal.first, second: horizontal.second, destination: horizontal.destination, + memory: horizontal.memory, byteWidth: horizontal.byteWidth, laneBits: horizontal.spec.laneBits, + legacy: !horizontal.spec.vector, + }, true, err + } + base, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := amd64PackedFloatingDotSpecs[Op(base)] + form := x86PackedReductionForm{ + first: 1, second: 2, destination: 2, memory: -1, + byteWidth: 16, laneBits: spec.laneBits, dot: true, legacy: spec.form == amd64PackedFloatingDotLegacy, + } + if !known { + return form, false, nil + } + fail := func() (x86PackedReductionForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go dot-product grammar: %q", arch, ins.Op, ins.Raw) + } + if arch != "386" && arch != "amd64" || dotted { + return fail() + } + count := 3 + if !form.legacy { + count, form.destination = 4, 3 + } + if len(ins.Args) != count || arch == "386" && !form.legacy && !ins.x86Encoded && ins.x86SourceRaw == nil { + return fail() + } + immediate, valid := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !valid || !amd64UnsignedImmediate(immediate, 8) { + return fail() + } + form.immediate = uint8(immediate.Imm) + if !form.legacy { + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + } + if form.byteWidth != 16 && (spec.form != amd64PackedFloatingDotVEX128Or256 || form.byteWidth != 32) { + return fail() + } + register := func(operand Operand) bool { + index, ok := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + return operand.Kind == OpReg && ok && index < 16 && (arch != "386" || index < 8) + } + if !register(ins.Args[form.second]) || !register(ins.Args[form.destination]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) { + if !x86PackedShiftMemoryValid(first, arch) { + return fail() + } + if first.Kind == OpMem && (x86SymbolIndexedMemoryError(first.Mem, arch) != nil || first.Mem.OffRaw != "" || first.Mem.IndexExt != "") { + return fail() + } + form.memory = form.first + } else if !register(first) { + return fail() + } + return form, true, nil +} + +func (form x86PackedReductionForm) inputBytes() uint64 { + if !form.dot { + return x86CallerVectorByteMask(int64(form.byteWidth)) + } + var result uint64 + bytes := form.laneBits / 8 + for block := 0; block < form.byteWidth; block += 16 { + for lane := 0; lane < 16/bytes; lane++ { + if form.immediate&(1<= 0 && (ins.x86AddressBits != 0 || first.Kind == OpMem && + !c.nativeMemoryOperand(ins, form.first) && !x86CallerMemoryCertified(ins, form.first) && !c.fixedLocalMemoryCertified(ins, form.first)) { + return true, false, fmt.Errorf("%w: %s needs original native source backing", ErrProbeNeedsContext, op) + } + // Float carriers are bitcasts only. The exact original packed instruction + // performs every arithmetic operation, including MXCSR and NaN priority; + // no pure intrinsic, scalar expansion or load-folding assumption replaces it. + typ := amd64FloatingVectorType(form.byteWidth/4, 32) + load := func(operand Operand) (string, error) { + bytes, err := c.loadPackedCompareBytes(operand, form.byteWidth) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", value, form.byteWidth, bytes, typ) + return "%" + value, nil + } + second, err := load(ins.Args[form.second]) + if err != nil { + return true, false, err + } + args, constraints := typ+" "+second, "=x,x" + if form.legacy { + constraints = "=x,0" + } + inputTemplate := "$2" + if form.memory >= 0 && first.Kind != OpFP { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(first) + if err != nil { + return true, false, err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + inputTemplate = segment + "($2)" + args += ", " + pointerType + " " + pointer + constraints += ",r" + } else { + input, err := load(first) + if err != nil { + return true, false, err + } + args += ", " + typ + " " + input + constraints += ",x" + } + assembly := strings.ToLower(string(ins.Op)) + " " + if form.dot { + assembly += fmt.Sprintf("$$%d, ", form.immediate) + } + assembly += inputTemplate + if !form.legacy { + assembly += ", $1" + } + assembly += ", $0" + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, typ, assembly, constraints+",~{memory},~{fpsr}", args) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, typ, result, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(ins.Args[form.destination], "%"+bytes) + } + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, "%"+bytes) +} diff --git a/amd64_packed_reduction_native_host_other_test.go b/amd64_packed_reduction_native_host_other_test.go new file mode 100644 index 000000000..a2ca21eb8 --- /dev/null +++ b/amd64_packed_reduction_native_host_other_test.go @@ -0,0 +1,10 @@ +//go:build !linux || !amd64 + +package plan9asm + +import "testing" + +func x86PackedReductionRequireNativeHost(t *testing.T) { + t.Helper() + t.Fatal("required native reduction owners execute only on actual Linux/amd64; other hosts retain their object and portable counterparts") +} diff --git a/amd64_packed_reduction_native_policy_test.go b/amd64_packed_reduction_native_policy_test.go new file mode 100644 index 000000000..baf1320d6 --- /dev/null +++ b/amd64_packed_reduction_native_policy_test.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "os" + "strconv" + "strings" + "testing" +) + +func x86PackedReductionNativeFunctions(path string) (map[string]*ast.FuncDecl, error) { + syntax, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + return nil, err + } + functions := make(map[string]*ast.FuncDecl) + for _, declaration := range syntax.Decls { + if function, ok := declaration.(*ast.FuncDecl); ok { + if functions[function.Name.Name] != nil { + return nil, fmt.Errorf("duplicate required native reduction function %s", function.Name.Name) + } + functions[function.Name.Name] = function + } + } + return functions, nil +} + +func TestX86PackedReductionCompleteNativeOwners(t *testing.T) { + x86PackedReductionOriginalRuntimeProgramsUnchanged(t) + const path = "amd64_packed_reduction_runtime_linux_amd64_test.go" + functions, err := x86PackedReductionNativeFunctions(path) + if err != nil { + t.Log("required native reduction owners are missing", err) + } + want := make(map[string]bool) + for _, family := range []struct{ root, driver string }{ + {"TestCrossLinuxRuntimeMatrixX86PackedReductionNativeValues", "x86PackedReductionNativeValues"}, + {"TestCrossLinuxRuntimeMatrixX86PackedReductionNativeDeadFaults", "x86PackedReductionNativeFaults"}, + } { + for part := 0; part < 4; part++ { + name := family.root + fmt.Sprintf("Part%d", part) + want[name] = true + t.Run(name, func(t *testing.T) { + function := functions[name] + if function == nil || function.Body == nil || len(function.Body.List) != 1 { + t.Fatal("missing or bypassed complete native reduction owner", name) + } + statement, ok := function.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("native reduction owner must execute the complete direct driver", name) + } + call, ok := statement.X.(*ast.CallExpr) + if !ok { + t.Fatal("native reduction owner omitted its execution call", name) + } + callee, ok := call.Fun.(*ast.Ident) + if !ok || callee.Name != family.driver || len(call.Args) != 2 { + t.Fatal("native reduction owner substituted a sampled or object-only driver", name) + } + context, ok := call.Args[0].(*ast.Ident) + if !ok || context.Name != "t" { + t.Fatal("native reduction owner lost the real test context", name) + } + partition, ok := call.Args[1].(*ast.BasicLit) + if !ok || partition.Kind != token.INT || partition.Value != strconv.Itoa(part) { + t.Fatal("native reduction owner borrowed another partition", name) + } + }) + } + } + for name := range functions { + if strings.HasPrefix(name, "Test") && !want[name] { + t.Error("unregistered native reduction owner", name) + } + } + t.Run("actual_native_host_selection", func(t *testing.T) { + for _, target := range []struct{ os, arch string }{ + {"linux", "amd64"}, {"linux", "386"}, {"linux", "arm"}, {"linux", "arm64"}, + {"js", "wasm"}, {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, + } { + context := build.Default + context.GOOS, context.GOARCH = target.os, target.arch + selected, err := context.MatchFile(".", path) + if err != nil || selected != (target.os == "linux" && target.arch == "amd64") { + t.Fatal("native reduction owner lost its required matching-host selection", target, selected, err) + } + } + }) +} + +func TestX86PackedReductionCompleteNativeDriverRequired(t *testing.T) { + const path = "amd64_packed_reduction_native_tools_test.go" + functions, err := x86PackedReductionNativeFunctions(path) + if err != nil { + t.Fatal("missing real SSE3/SSE4.1/AVX reduction execution driver", err) + } + for _, required := range []string{"x86PackedReductionNativeTools", "x86PackedReductionNativePipelines"} { + if function := functions[required]; function == nil || function.Body == nil { + t.Fatal("missing complete native reduction driver", required) + } + } + function := functions["x86PackedReductionNativeTools"] + calls := make(map[string]int) + ast.Inspect(function.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if callee, ok := call.Fun.(*ast.Ident); ok { + calls[callee.Name]++ + if callee.Name == "x86FloatCompareRuntimeTools" { + if len(call.Args) != 2 { + t.Fatal("native reductions lost the actual VEX execution route") + } + axis, ok := call.Args[1].(*ast.Ident) + if !ok || axis.Name != "x86FloatCompareVEX" { + t.Fatal("SSE3/SSE4.1/AVX reductions must not require AVX512 or borrow a weaker SSE axis") + } + } + } + if callee, ok := call.Fun.(*ast.SelectorExpr); ok { + calls[callee.Sel.Name]++ + if callee.Sel.Name == "Skip" || callee.Sel.Name == "Skipf" || callee.Sel.Name == "SkipNow" { + t.Fatal("missing tools or CPU cannot skip a required native reduction owner") + } + } + return true + }) + for _, required := range []string{"x86PackedReductionRequireNativeHost", "x86FloatCompareRuntimeTools", "compileC", "run"} { + if calls[required] != 1 { + t.Fatal("native reduction driver omitted actual host/CPU/opcode execution", required, calls[required]) + } + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, required := range []string{ + "SSE3=%u SSE41=%u AVX=%u OSXSAVE=%u XCR0=%llx", + "PACKED_REDUCTION_NATIVE_SSE3_SSE41_AVX_OK", "__get_cpuid(1", "xgetbv", "(xcr0&6)!=6", + "dppd", "dpps", "vdppd", "vdpps", "haddps", "haddpd", "hsubps", "hsubpd", + "vhaddps", "vhaddpd", "vhsubps", "vhsubpd", "execution.prefix", "execution.preserveArgv0", + "backend_O0", "backend_O2", "IR_O2_backend_O2", "x86DwordProductIROptimize", + "compileAndRunLLVM", "string(got) != string(want)", + } { + if !strings.Contains(string(data), required) { + t.Fatal("native reductions omitted a CPU/opcode/pipeline obligation", required) + } + } + for _, forbidden := range []string{"x86FloatCompareEVEX", "avx512", "zmm", "0xe6"} { + if strings.Contains(string(data), forbidden) { + t.Fatal("X/Y native reduction owner improperly depends on AVX512/Z state", forbidden) + } + } +} diff --git a/amd64_packed_reduction_native_regression_test.go b/amd64_packed_reduction_native_regression_test.go new file mode 100644 index 000000000..0f356b7a2 --- /dev/null +++ b/amd64_packed_reduction_native_regression_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "strings" + "testing" +) + +type x86PackedReductionOracleRow struct { + row, mnemonic string + width, first, second, destination int + dot, legacy bool +} + +// Independent complete yxshuf/_yvdppd/_yvblendpd and yxm/_yvaddsubpd +// families: five dot shapes and twelve horizontal shapes, register/memory. +// Selectors and every byte dependency are separate inexpensive unit axes. +func x86PackedReductionOracleRows() []x86PackedReductionOracleRow { + var rows []x86PackedReductionOracleRow + for _, op := range []string{"DPPD", "DPPS", "VDPPD", "VDPPS"} { + banks := []string{"X"} + if op == "VDPPS" { + banks = append(banks, "Y") + } + for _, bank := range banks { + for _, memory := range []bool{false, true} { + first := bank + "0" + if memory { + first = "(BX)" + } + row := x86PackedReductionOracleRow{ + row: op + " $255," + first + "," + bank + "1", mnemonic: strings.ToLower(op), + width: 16, first: 1, second: 2, destination: 2, dot: true, legacy: !strings.HasPrefix(op, "V"), + } + if bank == "Y" { + row.width = 32 + } + if !row.legacy { + row.row += "," + bank + "2" + row.destination = 3 + } + rows = append(rows, row) + } + } + } + for _, op := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD", "VHADDPS", "VHADDPD", "VHSUBPS", "VHSUBPD"} { + banks := []string{"X"} + if strings.HasPrefix(op, "V") { + banks = append(banks, "Y") + } + for _, bank := range banks { + for _, memory := range []bool{false, true} { + first := bank + "0" + if memory { + first = "(BX)" + } + row := x86PackedReductionOracleRow{ + row: op + " " + first + "," + bank + "1", mnemonic: strings.ToLower(op), + width: 16, first: 0, second: 1, destination: 1, legacy: !strings.HasPrefix(op, "V"), + } + if bank == "Y" { + row.width = 32 + } + if !row.legacy { + row.row += "," + bank + "2" + row.destination = 2 + } + rows = append(rows, row) + } + } + } + return rows +} + +func x86PackedReductionOriginalRuntimeProgramsUnchanged(t *testing.T) { + t.Helper() + for _, original := range []struct{ path, name, hash string }{ + {"amd64_lower_dot_float_test.go", "TestAMD64PackedFloatingDotRuntimeSemantics", "b928e394bd039f7f23a0fcbded8d22a0fbdb343d9d61b98c11d48bc95dcd11e5"}, + {"amd64_horizontal_float_numerical_test.go", "TestCrossLinuxRuntimeMatrixX86HorizontalFloatCompletePhysicalSemantics", "8195e2105b0760edd32d4661d4af4909e3cbbd52819b3bdf9a62554f2b1ed076"}, + } { + data, err := os.ReadFile(original.path) + if err != nil { + t.Fatal(err) + } + set := token.NewFileSet() + syntax, err := parser.ParseFile(set, original.path, data, 0) + if err != nil { + t.Fatal(err) + } + found := false + for _, declaration := range syntax.Decls { + function, ok := declaration.(*ast.FuncDecl) + if !ok || function.Name.Name != original.name { + continue + } + start, end := set.Position(function.Pos()).Offset, set.Position(function.End()).Offset + if fmt.Sprintf("%x", sha256.Sum256(data[start:end])) != original.hash { + t.Fatal("original initialized runtime source/reference/observations changed", original.name) + } + found = true + } + if !found { + t.Fatal("original runtime disappeared", original.name) + } + } +} + +func TestX86PackedReductionOriginalInitializedNativePrograms(t *testing.T) { + x86PackedReductionOriginalRuntimeProgramsUnchanged(t) + for _, original := range []struct{ name, source, declaration, opcode string }{ + {"dot", `TEXT floatdotsemantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ a+8(FP), CX + MOVQ b+16(FP), DX + MOVUPS 0(CX), X0 + MOVUPS 0(DX), X1 + STC + DPPS $0xf1, X0, X1 + MOVUPS X1, 0(AX) + SETCS 16(AX) + RET +`, "func floatdotsemantics(out,a,b *byte)", "dpps"}, + {"horizontal", "TEXT horizontal0(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nSTC\nHADDPS X0,X1\nVMOVUPS Z1,(AX)\nSETCS 64(AX)\nRET\n", + "func horizontal0(out,a,b *byte)", "haddps"}, + } { + t.Run(original.name, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", original.source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, original.source, original.declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal("original initialized input/FLAGS/output program needs its missing native numerical continuation", err) + } + var native, reductions []string + moves := 0 + for _, line := range strings.Split(ir, "\n") { + if !strings.Contains(line, "asm sideeffect") { + continue + } + native = append(native, strings.TrimSpace(line)) + if strings.Contains(line, `asm sideeffect "`+original.opcode+" ") { + reductions = append(reductions, line) + } + if strings.Contains(line, `asm sideeffect "movups `) { + moves++ + } + } + if len(reductions) != 1 || !strings.Contains(reductions[0], "~{fpsr}") || !strings.Contains(reductions[0], "~{memory}") || + original.name == "dot" && moves != 3 { + t.Fatalf("original reduction/native moves lost exact effects: opcode=%s reductions=%d MOVUPS=%d\n%s", original.opcode, len(reductions), moves, strings.Join(native, "\n")) + } + t.Logf("original %s: %d whole reduction and %d native MOVUPS; exact IR effects:\n%s", original.opcode, len(reductions), moves, strings.Join(native, "\n")) + }) + } + }) + } +} + +func TestX86PackedReductionCompleteTypedClassRegistration(t *testing.T) { + rows := x86PackedReductionOracleRows() + if len(rows) != 34 { + t.Fatal("complete12-member/17-shape/34-format reduction inventory changed", len(rows)) + } + for _, arch := range []string{"386", "amd64"} { + for _, row := range rows { + if arch == "386" && row.dot && !row.legacy { + continue // Go's named frontend cannot represent four operands. + } + t.Run(arch+"/"+row.row, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, row.row) + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || !effect.requiresResultProof { + t.Fatal("complete numerical reduction class is absent from typed effects", known, err) + } + reads, writes := make(map[int]bool), make(map[int]bool) + for _, operand := range effect.operands { + if operand.width != int64(row.width) { + t.Fatal("reduction borrowed another tuple or vector width", operand.width) + } + reads[operand.index] = reads[operand.index] || operand.read + writes[operand.index] = writes[operand.index] || operand.write + } + if !reads[row.first] || !reads[row.second] || !writes[row.destination] || + !row.legacy && reads[row.destination] { + t.Fatal("missing source/legacy old-destination dependency or invented VEX merge", effect) + } + }) + } + } +} diff --git a/amd64_packed_reduction_native_tools_test.go b/amd64_packed_reduction_native_tools_test.go new file mode 100644 index 000000000..d55798415 --- /dev/null +++ b/amd64_packed_reduction_native_tools_test.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "os" + "path/filepath" + "testing" +) + +func x86PackedReductionNativeTools(t *testing.T) x86FloatCompareExecution { + t.Helper() + x86PackedReductionRequireNativeHost(t) + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareVEX) + if execution.triple != "x86_64-unknown-linux-gnu" || len(execution.prefix) != 0 || execution.preserveArgv0 { + t.Fatal("required native reductions cannot borrow guest or foreign execution") + } + directory := t.TempDir() + source, executable := filepath.Join(directory, "reduction-cpu.c"), filepath.Join(directory, "reduction-cpu") + if err := os.WriteFile(source, []byte(x86PackedReductionNativePreflightC), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + t.Logf("actual native reduction CPU and all12 original opcodes: %s", execution.run(t, executable)) + return execution +} + +func x86PackedReductionNativePipelines(t *testing.T, execution x86FloatCompareExecution, ir, driver string, want []byte) { + t.Helper() + if len(want) == 0 { + t.Fatal("native reduction pipelines require original executed receipts") + } + for _, pipeline := range []struct { + name, backend string + optimize bool + }{ + {"backend_O0", "-O0", false}, + {"backend_O2", "-O2", false}, + {"IR_O2_backend_O2", "-O2", true}, + } { + t.Run(pipeline.name, func(t *testing.T) { + input := ir + if pipeline.optimize { + input = x86DwordProductIROptimize(t, ir) + } + got := execution.compileAndRunLLVM(t, "packed-reduction", input, driver, pipeline.backend) + if string(got) != string(want) { + t.Fatalf("original Go/native LLVM reductions differ\noriginal:\n%sLLVM:\n%s", want, got) + } + }) + } +} + +const x86PackedReductionNativePreflightC = `#include +#include +#include + +__attribute__((target("sse3,sse4.1,avx"))) +static unsigned original_members(void) { + unsigned members=0; + unsigned csr=0x1f80; + __asm__ volatile("ldmxcsr %0" :: "m"(csr) : "memory"); +#define CHECK(OP,DST,BIT) do { \ + uint64_t out[4]; \ + __asm__ volatile("vxorps %%ymm0,%%ymm0,%%ymm0; " \ + "vxorps %%ymm1,%%ymm1,%%ymm1; " \ + "vxorps %%ymm2,%%ymm2,%%ymm2; " \ + OP "; vmovups %%" DST ",%0" \ + : "=m"(out) :: "ymm0","ymm1","ymm2","memory"); \ + for (unsigned i=0;i<4;i++) if(out[i]) return 0; \ + members|=1u<<(BIT); \ +} while(0) + CHECK("dppd $255,%%xmm0,%%xmm1","ymm1",0); + CHECK("dpps $255,%%xmm0,%%xmm1","ymm1",1); + CHECK("vdppd $255,%%xmm0,%%xmm1,%%xmm2","ymm2",2); + CHECK("vdpps $255,%%ymm0,%%ymm1,%%ymm2","ymm2",3); + CHECK("haddps %%xmm0,%%xmm1","ymm1",4); + CHECK("haddpd %%xmm0,%%xmm1","ymm1",5); + CHECK("hsubps %%xmm0,%%xmm1","ymm1",6); + CHECK("hsubpd %%xmm0,%%xmm1","ymm1",7); + CHECK("vhaddps %%ymm0,%%ymm1,%%ymm2","ymm2",8); + CHECK("vhaddpd %%ymm0,%%ymm1,%%ymm2","ymm2",9); + CHECK("vhsubps %%ymm0,%%ymm1,%%ymm2","ymm2",10); + CHECK("vhsubpd %%ymm0,%%ymm1,%%ymm2","ymm2",11); +#undef CHECK + return members; +} + +int main(void) { + unsigned a,b,c,d,sse3=0,sse41=0,avx=0,osxsave=0; + uint64_t xcr0=0; + if (__get_cpuid(1,&a,&b,&c,&d)) { + sse3=c&1; sse41=(c>>19)&1; avx=(c>>28)&1; osxsave=(c>>27)&1; + if (osxsave) { + unsigned low,high; + __asm__ volatile("xgetbv" : "=a"(low),"=d"(high) : "c"(0)); + xcr0=((uint64_t)high<<32)|low; + } + } + printf("SSE3=%u SSE41=%u AVX=%u OSXSAVE=%u XCR0=%llx\n", + sse3,sse41,avx,osxsave,(unsigned long long)xcr0); + fflush(stdout); + if (!sse3 || !sse41 || !avx || !osxsave || (xcr0&6)!=6) return 1; + if (original_members()!=0xfff) return 2; + puts("PACKED_REDUCTION_NATIVE_SSE3_SSE41_AVX_OK members=fff"); + return 0; +} +` diff --git a/amd64_packed_reduction_objects_test.go b/amd64_packed_reduction_objects_test.go new file mode 100644 index 000000000..5b5ec3e6e --- /dev/null +++ b/amd64_packed_reduction_objects_test.go @@ -0,0 +1,78 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86PackedReductionInitializedFixture(name, arch, instruction string, destination int) (string, string) { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + return fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),AX\n%s p+0(FP),BX\n"+ + "VPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\nSTC\n%s\nVMOVDQU64 Z%d,(AX)\nSETCS 64(AX)\nRET\n", + name, word, move, move, instruction, destination), "func " + name + "(p *byte)\n" +} + +func TestX86PackedReductionCompleteInitializedNamedRawFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + rows := x86PackedReductionOracleRows() + raw := x86PackedReductionOriginalRawRows(t, rows) + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + for _, spelling := range []string{"named", "raw"} { + t.Run(spelling, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for index, row := range rows { + if target.arch == "386" && spelling == "named" && row.dot && !row.legacy { + continue + } + instruction := row.row + if spelling == "raw" { + instruction = raw[index] + } + destination := 1 + if !row.legacy { + destination = 2 + } + body, declaration := x86PackedReductionInitializedFixture(fmt.Sprintf("Reduction%d", index), target.arch, instruction, destination) + source.WriteString(body) + declarations.WriteString(declaration) + count++ + } + if count != 34 && !(target.arch == "386" && spelling == "named" && count == 28) { + t.Fatal("complete reduction source matrix lost rows", count) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("complete initialized reduction source rejected", err) + } + optimized := x86DwordProductIROptimize(t, ir) + for _, pipeline := range []struct{ name, ir, backend string }{ + {"backend_O0", ir, "-O0"}, {"backend_O2", ir, "-O2"}, {"IR_O2_backend_O2", optimized, "-O2"}, + } { + t.Run(pipeline.name, func(t *testing.T) { + if strings.Count(pipeline.ir, "asm sideeffect") != count || strings.Count(pipeline.ir, "~{fpsr}") != count || + strings.Contains(pipeline.ir, "call <4 x float> @llvm.x86.sse41.dpps") || + strings.Contains(pipeline.ir, "@__plan9asm_add_sub_special") { + t.Fatal("reduction lost whole native arithmetic/MXCSR effects or retained replaced software helpers") + } + x86PackedStringCompileObject(t, llc, target.triple, pipeline.ir, pipeline.backend) + }) + } + }) + } + }) + } + }) + } +} diff --git a/amd64_packed_reduction_runtime_fixture_test.go b/amd64_packed_reduction_runtime_fixture_test.go new file mode 100644 index 000000000..6af40d12c --- /dev/null +++ b/amd64_packed_reduction_runtime_fixture_test.go @@ -0,0 +1,262 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +const ( + x86PackedReductionNativeOwners = 4 + x86PackedReductionValueCapture = 64 + x86PackedReductionFaultCapture = 8 +) + +type x86PackedReductionRuntimeRow struct { + oracle x86PackedReductionOracleRow + instruction string + destination int + selector byte + memory bool + self bool +} + +func x86PackedReductionNativeRows(faults bool) []x86PackedReductionRuntimeRow { + var rows []x86PackedReductionRuntimeRow + for _, oracle := range x86PackedReductionOracleRows() { + if strings.Contains(oracle.row, "(BX)") { + continue + } + selectors := []byte{0} + if oracle.dot { + selectors = []byte{0, 0x0f, 0x10, 0x11, 0x5a, 0x7f, 0xf0, 0xf1, 0xff} + if strings.HasSuffix(oracle.mnemonic, "pd") { + selectors = []byte{0, 0x0f, 0x10, 0x11, 0x33, 0xcc, 0xff} + } + if faults { + selectors = []byte{0, 0x0f, 0xf0, 0xf1, 0xff} + } + } + for _, selector := range selectors { + variants := 6 + if oracle.legacy { + variants = 3 + } + if faults { + variants = 1 + } + for variant := 0; variant < variants; variant++ { + row := x86PackedReductionRuntimeRow{oracle: oracle, selector: selector} + row.destination, row.memory = variant/2, variant%2 == 1 + if oracle.legacy { + row.destination, row.self = 1, variant == 2 + } + if faults { + row.memory, row.destination = true, 2 + if oracle.legacy { + row.destination = 1 + } + } + bank := "X" + if oracle.width == 32 { + bank = "Y" + } + first := bank + "0" + if row.self { + first = bank + "1" + } + if row.memory { + first = "(SI)" + if faults { + first = "(DI)" + } + } + row.instruction = strings.ToUpper(oracle.mnemonic) + " " + if oracle.dot { + row.instruction += fmt.Sprintf("$%d,", selector) + } + row.instruction += first + "," + bank + "1" + if !oracle.legacy { + row.instruction += fmt.Sprintf(",%s%d", bank, row.destination) + } + rows = append(rows, row) + } + } + } + return rows +} + +func x86PackedReductionRuntimeExpectedBytes(row x86PackedReductionRuntimeRow, faults bool) []byte { + code := x86PackedReductionIndependentFixtureBytes(row.oracle) + modRM := byte(0xc0 | row.destination<<3) + if row.self { + modRM |= 1 + } + if row.memory { + modRM = byte(row.destination<<3 | 6) // Runtime's actual (SI). + if faults { + modRM = byte(row.destination<<3 | 7) // Dead native source is (DI). + } + } + index := len(code) - 1 + if row.oracle.dot { + index-- + code[index+1] = row.selector + } + code[index] = modRM + return code +} + +func x86PackedReductionRuntimeRawRows(t *testing.T, rows []x86PackedReductionRuntimeRow, faults bool) []string { + t.Helper() + var instructions []string + var oracle []x86PackedReductionOracleRow + var expected [][]byte + for _, row := range rows { + instructions = append(instructions, row.instruction) + item := row.oracle + item.row = row.instruction + oracle = append(oracle, item) + expected = append(expected, x86PackedReductionRuntimeExpectedBytes(row, faults)) + } + listing, err := x86ExtensionGoListing(t, "amd64", strings.Join(instructions, "\n")) + if err != nil { + t.Fatalf("actual runtime instruction Go listing: %v\n%s", err, listing) + } + raw, err := x86PackedReductionRawRowsFromGoListingWithExpected(listing, oracle, expected) + if err != nil { + t.Fatalf("independent runtime original opcode/PC boundaries: %v\n%s", err, listing) + } + return raw +} + +func x86PackedReductionRuntimeFixture(rows []x86PackedReductionRuntimeRow) (assembly, declarations, original, driver string) { + var asm, decl, goCalls, cDecl, cCalls strings.Builder + for index, row := range rows { + name := fmt.Sprintf("reduction%d", index) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$8-36\nSTMXCSR saved-4(SP)\n"+ + "MOVQ out+0(FP),DI\nMOVQ a+8(FP),SI\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),CX\n"+ + "VMOVDQU (SI),Y0\nVMOVDQU (DX),Y1\nVMOVDQU (CX),Y2\n"+ + "LDMXCSR csr+32(FP)\nMOVQ $-1,R8\nCMPQ R8,$-1\n%s\n"+ + "VMOVDQU Y%d,(DI)\nSTMXCSR 40(DI)\nSETEQ 32(DI)\nSETCS 33(DI)\n"+ + "SETOS 34(DI)\nSETMI 35(DI)\nSETPC 36(DI)\nLDMXCSR saved-4(SP)\nRET\n", + name, row.instruction, row.destination) + fmt.Fprintf(&decl, "func %s(out *[8]uint64,a,b,old *[4]uint64,csr uint32)\n", name) + bits := 32 + if strings.HasSuffix(row.oracle.mnemonic, "pd") { + bits = 64 + } + fmt.Fprintf(&goCalls, "check(%d,%d,%s)\n", index, bits, name) + fmt.Fprintf(&cDecl, "extern void %s(uint64_t*,const uint64_t*,const uint64_t*,const uint64_t*,uint32_t);\n", name) + fmt.Fprintf(&cCalls, "check(%d,%d,%s);\n", index, bits, name) + } + var gValues, cValues strings.Builder + values := append(append([]uint64(nil), x86ScalarFloatIntegerSSInputs...), x86ScalarFloatIntegerSDInputs...) + for _, value := range values { + fmt.Fprintf(&gValues, "%#x,", value) + fmt.Fprintf(&cValues, "UINT64_C(%#x),", value) + } + original = `package main +import ("fmt";"unsafe") +var values=[]uint64{` + gValues.String() + `} +func check(index,bits int,fn func(*[8]uint64,*[4]uint64,*[4]uint64,*[4]uint64,uint32)) { + hash:=uint64(1469598103934665603);count:=0 + for n:=range values {for rc:=0;rc<4;rc++ {for controls:=0;controls<8;controls++ { + var backing [3][6]uint64 + var inputs [3]*[4]uint64 + for source:=range inputs { + p:=unsafe.Pointer(&backing[source][0]) + inputs[source]=(*[4]uint64)(unsafe.Add(p,(-uintptr(p))&15)) + for lane:=range inputs[source] { + start:=n+2*lane+7*source + value:=values[start%len(values)] + if bits==32 {value=uint64(uint32(value))|uint64(uint32(values[(start+1)%len(values)]))<<32} + inputs[source][lane]=value + } + } + var out [8]uint64 + for lane:=range out {out[lane]=uint64(lane+1)*0x0101010101010101} + csr:=uint32(0x1f80|rc<<13) + if controls&1!=0 {csr|=0x40};if controls&2!=0 {csr|=0x8000};if controls&4!=0 {csr|=0x21} + fn(&out,inputs[0],inputs[1],inputs[2],csr) + for _,word:=range out {for shift:=0;shift<64;shift+=8 {hash=(hash^uint64(byte(word>>shift)))*1099511628211}} + count++ + }}} + fmt.Printf("%d %d %016x\n",index,count,hash) +} +` + decl.String() + "func main(){\n" + goCalls.String() + "}\n" + driver = `#include +#include +static const uint64_t values[]={` + cValues.String() + `}; +static void check(unsigned index,unsigned bits,void (*fn)(uint64_t*,const uint64_t*,const uint64_t*,const uint64_t*,uint32_t)) { + uint64_t hash=UINT64_C(1469598103934665603);unsigned count=0,N=sizeof(values)/sizeof(*values); + for(unsigned n=0;n>shift))*UINT64_C(1099511628211); + count++; + } + printf("%u %u %016llx\n",index,count,(unsigned long long)hash); +} +` + cDecl.String() + "int main(void){\n" + cCalls.String() + "return 0;}\n" + return asm.String(), decl.String(), original, driver +} + +func TestX86PackedReductionCompleteNativeFixtureInventory(t *testing.T) { + for _, faults := range []bool{false, true} { + rows := x86PackedReductionNativeRows(faults) + want := 258 + if faults { + want = 37 + } + if len(rows) != want { + t.Fatal("complete semantic class lost value/fault shapes", faults, len(rows)) + } + members := make(map[string]map[int]bool) + owners := [4]int{} + limit := x86PackedReductionValueCapture + if faults { + limit = x86PackedReductionFaultCapture + } + for index, row := range rows { + if members[row.oracle.mnemonic] == nil { + members[row.oracle.mnemonic] = make(map[int]bool) + } + members[row.oracle.mnemonic][row.oracle.width] = true + owners[index/limit%4]++ + if strings.Contains(row.instruction, "Z") || row.oracle.width > 32 { + t.Fatal("native X/Y fixture acquired physical Z state", row.instruction) + } + } + for _, name := range []string{"dppd", "dpps", "vdppd", "vdpps", "haddps", "haddpd", "hsubps", "hsubpd", "vhaddps", "vhaddpd", "vhsubps", "vhsubpd"} { + widths := members[name] + want := map[int]bool{16: true} + if name == "vdpps" || strings.HasPrefix(name, "vh") { + want[32] = true + } + if !reflect.DeepEqual(widths, want) { + t.Fatal("native member omitted an independent physical width", faults, name, widths) + } + } + if len(members) != 12 { + t.Fatal("native semantic class lost or invented members", members) + } + for owner, count := range owners { + if count == 0 { + t.Fatal("empty native semantic owner", faults, owner) + } + } + } +} diff --git a/amd64_packed_reduction_runtime_linux_amd64_test.go b/amd64_packed_reduction_runtime_linux_amd64_test.go new file mode 100644 index 000000000..061eb9431 --- /dev/null +++ b/amd64_packed_reduction_runtime_linux_amd64_test.go @@ -0,0 +1,39 @@ +package plan9asm + +import "testing" + +func x86PackedReductionRequireNativeHost(t *testing.T) { + t.Helper() +} + +func TestCrossLinuxRuntimeMatrixX86PackedReductionNativeValuesPart0(t *testing.T) { + x86PackedReductionNativeValues(t, 0) +} + +func TestCrossLinuxRuntimeMatrixX86PackedReductionNativeValuesPart1(t *testing.T) { + x86PackedReductionNativeValues(t, 1) +} + +func TestCrossLinuxRuntimeMatrixX86PackedReductionNativeValuesPart2(t *testing.T) { + x86PackedReductionNativeValues(t, 2) +} + +func TestCrossLinuxRuntimeMatrixX86PackedReductionNativeValuesPart3(t *testing.T) { + x86PackedReductionNativeValues(t, 3) +} + +func TestCrossLinuxRuntimeMatrixX86PackedReductionNativeDeadFaultsPart0(t *testing.T) { + x86PackedReductionNativeFaults(t, 0) +} + +func TestCrossLinuxRuntimeMatrixX86PackedReductionNativeDeadFaultsPart1(t *testing.T) { + x86PackedReductionNativeFaults(t, 1) +} + +func TestCrossLinuxRuntimeMatrixX86PackedReductionNativeDeadFaultsPart2(t *testing.T) { + x86PackedReductionNativeFaults(t, 2) +} + +func TestCrossLinuxRuntimeMatrixX86PackedReductionNativeDeadFaultsPart3(t *testing.T) { + x86PackedReductionNativeFaults(t, 3) +} diff --git a/amd64_packed_reduction_runtime_objects_test.go b/amd64_packed_reduction_runtime_objects_test.go new file mode 100644 index 000000000..26a06ef49 --- /dev/null +++ b/amd64_packed_reduction_runtime_objects_test.go @@ -0,0 +1,97 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86PackedReductionNativeFixtureObjects(t *testing.T, faults bool) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + rows := x86PackedReductionNativeRows(faults) + limit := x86PackedReductionValueCapture + if faults { + limit = x86PackedReductionFaultCapture + } + consumed := 0 + for start := 0; start < len(rows); start += limit { + end := start + limit + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + current := rows[start:end] + var original [][]byte + if faults { + original = x86PackedReductionFaultOriginalBytes(t, current) + } + fixture := func(rows []x86PackedReductionRuntimeRow) (string, string) { + if faults { + assembly, declarations, _ := x86PackedReductionFaultFixture(rows, original) + return assembly, declarations + } + assembly, declarations, _, _ := x86PackedReductionRuntimeFixture(rows) + return assembly, declarations + } + assembly, declarations := fixture(current) + raw := x86PackedReductionRuntimeRawRows(t, current, faults) + rawRows := append([]x86PackedReductionRuntimeRow(nil), current...) + for index := range rawRows { + rawRows[index].instruction = raw[index] + } + rawAssembly, _ := fixture(rawRows) + for _, spelling := range []struct{ name, source string }{{"named", assembly}, {"raw", rawAssembly}} { + t.Run(spelling.name, func(t *testing.T) { + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", spelling.source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, spelling.source, declarations, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal("initialized native runtime fixture rejected before execution", err) + } + optimized := x86DwordProductIROptimize(t, ir) + for _, pipeline := range []struct{ name, ir, backend string }{ + {"backend_O0", ir, "-O0"}, {"backend_O2", ir, "-O2"}, {"IR_O2_backend_O2", optimized, "-O2"}, + } { + t.Run(pipeline.name, func(t *testing.T) { + for index, row := range current { + name := fmt.Sprintf("reduction%d", index) + if faults { + name = fmt.Sprintf("reductionfault%d", index) + } + _, body, found := strings.Cut(pipeline.ir, "define void @"+name+"(") + body, _, ended := strings.Cut(body, "\n}") + if !found || !ended || strings.Count(body, `asm sideeffect "`+row.oracle.mnemonic+` `) != 1 || + !strings.Contains(body, "~{fpsr}") || !strings.Contains(body, "~{memory}") { + t.Fatal("actual runtime fixture lost the original whole arithmetic/exception effect", row.instruction) + } + } + x86PackedStringCompileObject(t, llc, "x86_64-unknown-linux-gnu", pipeline.ir, pipeline.backend) + }) + } + }) + } + }) + } + consumed += len(current) + }) + } + if consumed != len(rows) { + t.Fatal("native runtime object partition lost initialized rows", consumed, len(rows)) + } + t.Logf("all%d actual native runtime kernels have initialized Go/public3API/3pipeline objects; this is not native execution", consumed) +} + +func TestX86PackedReductionNativeValueFixtureObjects(t *testing.T) { + x86PackedReductionNativeFixtureObjects(t, false) +} + +func TestX86PackedReductionNativeFaultFixtureObjects(t *testing.T) { + x86PackedReductionNativeFixtureObjects(t, true) +} diff --git a/amd64_packed_reduction_runtime_test.go b/amd64_packed_reduction_runtime_test.go new file mode 100644 index 000000000..4f2eb1717 --- /dev/null +++ b/amd64_packed_reduction_runtime_test.go @@ -0,0 +1,75 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86PackedReductionNativeValues(t *testing.T, owner int) { + t.Helper() + if owner < 0 || owner >= x86PackedReductionNativeOwners { + t.Fatal("invalid complete native reduction value owner", owner) + } + execution := x86PackedReductionNativeTools(t) + rows := x86PackedReductionNativeRows(false) + if len(rows) != 258 { + t.Fatal("complete native reduction value denominator changed", len(rows)) + } + consumed := 0 + for batch, start := 0, 0; start < len(rows); batch, start = batch+1, start+x86PackedReductionValueCapture { + if batch%x86PackedReductionNativeOwners != owner { + continue + } + end := start + x86PackedReductionValueCapture + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + current := rows[start:end] + assembly, declarations, original, driver := x86PackedReductionRuntimeFixture(current) + var want []byte + t.Run("original_go_named", func(t *testing.T) { + want = x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, original) + if strings.Count(string(want), "\n") != len(current) { + t.Fatal("incomplete original named Go reduction execution receipts") + } + }) + raw := x86PackedReductionRuntimeRawRows(t, current, false) + rawRows := append([]x86PackedReductionRuntimeRow(nil), current...) + for index := range rawRows { + rawRows[index].instruction = raw[index] + } + rawAssembly, _, rawOriginal, _ := x86PackedReductionRuntimeFixture(rawRows) + t.Run("original_go_raw", func(t *testing.T) { + got := x86ScalarFloatIntegerOriginalOutput(t, execution, rawAssembly, rawOriginal) + if string(got) != string(want) || len(want) == 0 { + t.Fatal("original named/raw Go reduction execution differs or lost receipts") + } + }) + if len(want) == 0 { + t.Fatal("missing actual original Go reduction execution") + } + for _, spelling := range []struct{ name, source string }{{"named", assembly}, {"raw", rawAssembly}} { + t.Run(spelling.name, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, spelling.source, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal("initialized native reduction value source rejected", err) + } + x86PackedReductionNativePipelines(t, execution, ir, driver, want) + }) + } + }) + } + consumed += len(current) + }) + } + if consumed == 0 { + t.Fatal("empty complete native reduction value owner") + } + t.Logf("%d/258 actual named/raw Go kernels, full physical Y32 aliases/FLAGS/MXCSR,3 APIs and3 LLVM execution paths", consumed) +} diff --git a/amd64_packed_rotate_fault_runtime_unix_test.go b/amd64_packed_rotate_fault_runtime_unix_test.go new file mode 100644 index 000000000..cfd7d8652 --- /dev/null +++ b/amd64_packed_rotate_fault_runtime_unix_test.go @@ -0,0 +1,67 @@ +//go:build darwin || linux + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The output is deliberately dead. Active native reads must still fault, +// while inactive E4 elements and broadcasts must not read their address. +func TestX86PackedRotateDeadE4MemoryFaults(t *testing.T) { + execution := x86ScalarCountHostExecution(t) + var source, declarations, prototypes, rows strings.Builder + count := 0 + for _, family := range x86PackedRotateTestFamilies { + for _, bank := range []string{"X", "Y", "Z"} { + for _, broadcast := range []bool{false, true} { + for _, zeroing := range []bool{false, true} { + name := fmt.Sprint("RotateFault", count) + count++ + width := amd64VectorByteWidth(Reg(bank + "0")) + extent, suffix := width, "" + if broadcast { + extent, suffix = family.bits/8, ".BCST" + } + if zeroing { + suffix += ".Z" + } + operands := fmt.Sprintf("$255,(AX),K1,%s2", bank) + if family.variable { + operands = fmt.Sprintf("(AX),%s1,K1,%s2", bank, bank) + } + fmt.Fprintf(&declarations, "func %s(p *[64]byte,mask uint64)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),AX\n"+ + "KMOVQ mask+8(FP),K1\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\n%s%s %s\nRET\n", + name, family.op, suffix, operands) + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t *,uint64_t);\n", name) + fmt.Fprintf(&rows, "{%s,%d,%d,%d},\n", name, extent, family.bits/8, width*8/family.bits) + } + } + } + } + if count != 96 { + t.Fatal("incomplete rotate width/broadcast/merge memory inventory", count) + } + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source.String()) + driver := x86E4DeadReadFaultOracle(prototypes.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal("complete dead rotate source rejected", err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + execution.compileAndRunLLVM(t, "rotate-e4-faults", input, driver, level) + }) + } + }) + } +} diff --git a/amd64_packed_rotate_format_test.go b/amd64_packed_rotate_format_test.go new file mode 100644 index 000000000..ec5bc7753 --- /dev/null +++ b/amd64_packed_rotate_format_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "errors" + "testing" +) + +// Preserve the original Go object and three public Context outcomes before +// classifying physical emitter rows. A format object supplies no entry values. +func x86PackedRotateUnboundFormatObject(t *testing.T, source, arch, triple string, wantPhysical, wantContext int) string { + t.Helper() + file, signatures := x86UnboundVoidFormatSources(t, source, arch, triple) + normalized, err := normalizeX86RawFile(file, arch) + if err != nil { + t.Fatal("original rotate formats lost their decoded source", err) + } + units := *normalized + units.Funcs = nil + physical, contextual := 0, 0 + for _, function := range normalized.Funcs { + unit := function + unit.Instrs = nil + for _, instruction := range function.Instrs { + if instruction.Op != OpTEXT && instruction.Op != OpRET { + _, known, err := x86PackedRotateProperties(instruction.Op, instruction, arch) + if known && errors.Is(err, ErrProbeNeedsContext) && arch == "386" && + x86Mode32VectorRegisterContext(instruction, arch) != nil { + contextual++ + continue // Retained by the complete original Go/public Context controls. + } + if !known || err != nil { + t.Fatal("original rotate row lost its complete typed grammar", instruction.Raw, err) + } + physical++ + } + unit.Instrs = append(unit.Instrs, instruction) + } + units.Funcs = append(units.Funcs, unit) + } + if physical != wantPhysical || contextual != wantContext { + t.Fatal("original rotate format classification changed", physical, contextual, wantPhysical, wantContext) + } + t.Logf("%d physical emitter formats; %d Go-accepted mode32 high-bank Context rows; no initialized/native claim", physical, contextual) + return x86InstructionFormatObjectUnit(t, &units, signatures, arch, triple, 0, + func(c *amd64Ctx, instruction Instr) (bool, bool, error) { + return c.lowerPackedRotate(instruction.Op, instruction) + }) +} diff --git a/amd64_packed_rotate_forms.go b/amd64_packed_rotate_forms.go new file mode 100644 index 000000000..59311e073 --- /dev/null +++ b/amd64_packed_rotate_forms.go @@ -0,0 +1,78 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedRotateForm struct { + x86PackedBinaryForm + spec amd64PackedRotateSpec + immediate uint8 +} + +// Go's _yvprold and _yvblendmpd differ only in the count/data role. Both +// retain the shared EVEX width, register, mask and scalar-broadcast grammar. +func x86PackedRotateProperties(op Op, ins Instr, arch string) (x86PackedRotateForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64PackedRotateSpecs[Op(base)] + form := x86PackedRotateForm{spec: spec} + if !known { + return form, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || len(ins.Args) != 3 && len(ins.Args) != 4 { + return form, true, fmt.Errorf("%s %s has an unbound dispatch or operand count", arch, op) + } + local := ins + local.Op = op + if !spec.variable { + if !amd64UnsignedImmediate(ins.Args[0], 8) || ins.Args[0].ImmIsFloat || ins.Args[0].ImmRaw != "" { + return form, true, fmt.Errorf("%s %s requires the original unsigned imm8", arch, op) + } + form.immediate = uint8(ins.Args[0].Imm) + destination := ins.Args[len(ins.Args)-1] + // The extra destination exists only for unary shape validation. It + // never becomes a numerical input to the immediate operation. + local.Args = []Operand{ins.Args[1], destination, destination} + if len(ins.Args) == 4 { + local.Args = []Operand{ins.Args[1], destination, ins.Args[2], destination} + } + } + var err error + form.x86PackedBinaryForm, err = x86ConflictFunnelOperandGrammar(local, arch, + x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, broadcastBytes: spec.laneBits / 8, zeroing: true}, false) + if !spec.variable { + form.first, form.second, form.destination = 1, -1, len(ins.Args)-1 + if form.memory >= 0 { + form.memory = 1 + } + } + return form, true, err +} + +func x86TypedPackedRotateByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedRotateProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width, firstWidth := int64(form.byteWidth), int64(form.byteWidth) + if form.memory >= 0 { + firstWidth = int64(form.memoryBytes) + } + effect := x86TypedByteEffect{requiresResultProof: true, + vector: x86TypedVectorEffect{x86TypedVectorUpperZero, 1}, + operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: firstWidth, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: width, + read: form.mask >= 0 && !form.zeroing, write: true}, + }} + if form.second >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.second, operand: ins.Args[form.second], width: width, read: true}) + } + if form.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return effect, true, nil +} diff --git a/amd64_packed_rotate_inputs_test.go b/amd64_packed_rotate_inputs_test.go new file mode 100644 index 000000000..f2248e256 --- /dev/null +++ b/amd64_packed_rotate_inputs_test.go @@ -0,0 +1,205 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86PackedRotateCompleteIndependentGoRows(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + for _, family := range x86PackedRotateTestFamilies { + classes := []string{"YxmEvex,YxrEvex", "YxmEvex,Yknot0,YxrEvex", + "YymEvex,YyrEvex", "YymEvex,Yknot0,YyrEvex", "Yzm,Yzr", "Yzm,Yknot0,Yzr"} + table, plain, masked, stride := "_yvprold", "Zevex_i_rm_vo", "Zevex_i_rm_k_vo", 4 + for index, class := range classes { + if family.variable { + parts := strings.Split(class, ",") + classes[index] = parts[0] + "," + parts[len(parts)-1] + "," + strings.Join(parts[1:], ",") + } else { + classes[index] = "Yu8," + class + } + } + if family.variable { + table, plain, masked, stride = "_yvblendmpd", "Zevex_rm_v_r", "Zevex_rm_v_k_r", 3 + } + var attributes []string + for _, width := range []int{128, 256, 512} { + mapName, opcode := "evex0F", "0x72" + if family.variable { + mapName, opcode = "evex0F38", "0x14" + if family.left { + opcode = "0x15" + } + } + attributes = append(attributes, + fmt.Sprintf("avxEscape | evex%d | evex66 | %s | evexW%d", width, mapName, family.bits/32-1), + fmt.Sprintf("evexN%d | evexBcstN%d | evexZeroingEnabled", width/8, family.bits/8), opcode) + if !family.variable { + direction := "00" + if family.left { + direction = "01" + } + attributes = append(attributes, direction) + } + } + count := 0 + for _, rule := range rules { + if rule.Opcode != family.op { + continue + } + zcase := plain + if count%2 != 0 { + zcase = masked + } + if count >= 6 || rule.RowIndex != count || rule.Ytab != table || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != count%2*stride || rule.EncodingOffset != count/2*stride || + strings.Join(rule.Classes, ",") != classes[count] || !reflect.DeepEqual(rule.OpBytes, attributes) { + t.Fatal("independent rotate Go encoding row changed", family.op, rule) + } + count++ + } + if count != 6 { + t.Fatal("incomplete rotate Go encoding inventory", family.op, count) + } + } +} + +func TestX86PackedRotateExactNumericalInputs(t *testing.T) { + for _, family := range x86PackedRotateTestFamilies { + for _, bank := range []string{"X", "Y", "Z"} { + width := amd64VectorByteWidth(Reg(bank + "0")) + for mode := 0; mode < 3; mode++ { + op, mask := family.op, "" + if mode != 0 { + mask = "K1," + } + if mode == 2 { + op += ".Z" + } + args := "$7," + bank + "1," + if family.variable { + args = bank + "0," + bank + "1," + } + ins := x86ScalarLaneInstruction(t, op+" "+args+mask+bank+"2") + for missingRegister := 0; missingRegister < 4; missingRegister++ { + for missingByte := -1; missingByte < 64; missingByte++ { + state := x86ResultState{flags: x86ByteFlagsAll, direction: x86ResultDirectionBackward} + for register := 0; register < 4; register++ { + bytes := ^uint64(0) + if register == missingRegister && missingByte >= 0 { + bytes &^= 1 << uint(missingByte) + } + state.writeVector(register, 64, bytes, false) + } + state.masks[1] = missingRegister != 3 || missingByte != 0 + before, want := state.clone(), state.clone() + required := missingRegister == 1 || missingRegister == 0 && family.variable || missingRegister == 2 && mode == 1 + accepted := !(required && missingByte >= 0 && missingByte < width || mode != 0 && !state.masks[1]) + if accepted { + want.writeVector(2, width, x86CallerVectorByteMask(int64(width)), false) + } + if got := x86ResultTransfer(ins, &state, nil, 0, "amd64"); got != accepted || !reflect.DeepEqual(state, want) { + t.Fatal("rotate fabricated input bytes, lost upper zeroing or changed unrelated state", ins.Raw, missingRegister, missingByte, got, accepted, before, state) + } + } + } + } + } + } +} + +func TestX86PackedRotateRejectsUnresolvedShapesBeforeIR(t *testing.T) { + for _, family := range x86PackedRotateTestFamilies { + for _, mutation := range []string{"suffix", "offset", "extension", "segment", "index", "absolute", "dispatch", "address", "float", "unresolved"} { + if family.variable && (mutation == "float" || mutation == "unresolved") { + continue + } + t.Run(family.op+"/"+mutation, func(t *testing.T) { + args, source := "$7,(AX),X2", 1 + if family.variable { + args, source = "(AX),X1,X2", 0 + } + ins := x86ScalarLaneInstruction(t, family.op+" "+args) + op := ins.Op + switch mutation { + case "suffix": + op += ".SAE" + ins.Op = op + case "offset": + ins.Args[source].Mem.OffRaw = "UNKNOWN" + case "extension": + ins.Args[source].Mem.IndexExt = "UXTW" + case "segment": + ins.Args[source].Mem.Segment = "DS" + case "index": + ins.Args[source].Mem.Index, ins.Args[source].Mem.Scale = SP, 2 + case "absolute": + ins.Args[source] = Operand{Kind: OpSym, Sym: "4294967296"} + case "dispatch": + ins.Op = "VPXORD" + case "address": + ins.x86AddressBits = 32 + case "float": + ins.Args[0].ImmIsFloat = true + case "unresolved": + ins.Args[0].ImmRaw = "UNKNOWN" + } + var ir strings.Builder + consumer := &amd64Ctx{b: &ir, goarch: "amd64"} + known, _, err := consumer.lowerPackedRotate(op, ins) + if !known || err == nil || ir.Len() != 0 { + t.Fatal("invalid rotate shape reached IR", known, err, ir.String()) + } + }) + } + } +} + +func TestX86PackedRotateNativeReadKeepsOwnership(t *testing.T) { + for _, family := range x86PackedRotateTestFamilies { + args := "$7,-1(SI)(BX*2),X2" + if family.variable { + args = "-1(SI)(BX*2),X1,X2" + } + ins := x86ScalarLaneInstruction(t, family.op+" "+args) + for _, missing := range []string{"none", "pointer", "index", "pointer index", "raw", "address"} { + t.Run(fmt.Sprint(family.op, "/", missing), func(t *testing.T) { + state, changed := x86ResultState{}, ins + state.writeVector(1, 64, ^uint64(0), false) + state.pointerGP[6], state.gp[3] = true, 0xff + switch missing { + case "pointer": + state.pointerGP[6] = false + case "index": + state.gp[3] = 1 + case "pointer index": + state.pointerGP[3], state.gp[3] = true, 0 + case "raw": + changed.x86Encoded = true + case "address": + changed.x86AddressBits = 32 + } + before := state.clone() + if x86ResultTransfer(changed, &state, nil, 0, "amd64") || !reflect.DeepEqual(before, state) { + t.Fatal("standalone transfer invented a native memory contract") + } + accepted, _ := x86LoweredFixedVectorNativeRead(changed, &state, nil, "amd64") + want := before.clone() + if missing == "none" { + want.writeVector(2, 16, 0xffff, false) + } + if accepted != (missing == "none") || !reflect.DeepEqual(state, want) { + t.Fatal("paired rotate read changed pointer, NUM or independent state", accepted, state, want) + } + }) + } + } +} diff --git a/amd64_packed_rotate_writeback_runtime_test.go b/amd64_packed_rotate_writeback_runtime_test.go new file mode 100644 index 000000000..fa734dba6 --- /dev/null +++ b/amd64_packed_rotate_writeback_runtime_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// These original EVEX functions execute portable LLVM on the actual host. +// This checks the complete physical byte bank, not native AVX512 hardware. +func TestX86PackedRotatePortableValuesAndUpperWriteback(t *testing.T) { + x86PackedRotateWritebackRuntime(t, false, false) +} + +func TestX86PackedRotatePortableMemoryValuesAndUpperWriteback(t *testing.T) { + x86PackedRotateWritebackRuntime(t, false, true) +} + +// Isolate the emitter's upper-byte semantics from source-effect registration. +// All inputs are explicitly initialized, but this is not public API evidence. +func TestX86PackedRotateEmitterPortableUpperWriteback(t *testing.T) { + x86PackedRotateWritebackRuntime(t, true, false) +} + +var x86PackedRotateTestFamilies = []struct { + op string + bits int + left bool + variable bool +}{ + {"VPROLD", 32, true, false}, {"VPROLQ", 64, true, false}, + {"VPRORD", 32, false, false}, {"VPRORQ", 64, false, false}, + {"VPROLVD", 32, true, true}, {"VPROLVQ", 64, true, true}, + {"VPRORVD", 32, false, true}, {"VPRORVQ", 64, false, true}, +} + +func x86PackedRotateWritebackRuntime(t *testing.T, emitter, memory bool) { + t.Helper() + execution := x86ScalarCountHostExecution(t) + widths, apis := []int{16, 32, 64}, []string{"text", "module", "Go"} + if emitter { + widths, apis = []int{16}, []string{"emitter"} + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + t.Run(bank, func(t *testing.T) { + var source, declarations, prototypes, rows strings.Builder + count := 0 + for _, family := range x86PackedRotateTestFamilies { + for mode := 0; mode < 3; mode++ { + for destination := 0; destination < 3; destination++ { + name := fmt.Sprint("Rotate", count) + count++ + immediate := []int{0, 7, 255}[destination] + instruction := family.op + broadcast := memory && destination == 0 + if broadcast { + immediate = 7 + instruction += ".BCST" + } + if mode == 2 { + instruction += ".Z" + } + if family.variable { + first := bank + "0" + if memory { + first = "(CX)" + } + instruction += " " + first + "," + bank + "1," + } else { + data := bank + "1" + if memory { + data = "(BX)" + } + instruction += fmt.Sprintf(" $%d,%s,", immediate, data) + } + if mode != 0 { + instruction += "K1," + } + instruction += fmt.Sprint(bank, destination) + fmt.Fprintf(&declarations, "func %s(out *[260]byte,data,counts,old *[64]byte,mask uint64)\n", name) + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-40\nMOVQ out+0(FP),DI\nMOVQ data+8(FP),BX\nMOVQ counts+16(FP),CX\nMOVQ old+24(FP),SI\n", name) + source.WriteString("VMOVUPS (CX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (SI),Z2\nVMOVUPS (SI),Z3\nKMOVQ mask+32(FP),K1\nMOVL $5,R8\nCMPL R8,$3\n") + fmt.Fprintf(&source, "%s\nVMOVUPS Z0,0(DI)\nVMOVUPS Z1,64(DI)\nVMOVUPS Z2,128(DI)\nVMOVUPS Z3,192(DI)\nSETCS 256(DI)\nSETOS 257(DI)\nRET\n", instruction) + fmt.Fprintf(&rows, "{%s,%q,%d,%d,%t,%t,%t,%d,%d,%d},\n", name, instruction, + width, family.bits, family.left, family.variable, broadcast, mode, destination, immediate) + } + } + } + if count != 72 { + t.Fatal("complete rotate family/mask/alias inventory changed", count) + } + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source.String()) + driver := strings.NewReplacer("PROTOTYPES", prototypes.String(), "ROWS", rows.String()).Replace(x86PackedRotateWritebackDriver) + for _, api := range apis { + t.Run(api, func(t *testing.T) { + var ir string + var err error + if emitter { + file, parseErr := Parse(ArchAMD64, source.String()) + if parseErr != nil { + t.Fatal(parseErr) + } + pkg := mustGoPackage(t, "rotate", "package rotate\n"+declarations.String()) + resolve := func(name string) string { return strings.TrimPrefix(name, "·") } + sigs, sigErr := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if sigErr != nil { + t.Fatal(sigErr) + } + ir = x86ScalarFormatObjectUnit(t, file, sigs, "amd64", execution.triple, 0) + driver = strings.ReplaceAll(driver, "Rotate", "InstructionFormat") + } else { + ir, err = x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + } + if err != nil || ir == "" { + t.Fatal("initialized rotate source rejected", err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + execution.compileAndRunLLVM(t, "packed-rotate-writeback", input, driver, level) + }) + } + }) + } + }) + } +} + +const x86PackedRotateWritebackDriver = ` +#include +#include +#include +#include +PROTOTYPES +struct row { + void (*run)(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t); + const char *name; + unsigned width,bits; + bool left,variable,broadcast; + unsigned mode,destination,immediate; +}; +static const struct row rows[] = { ROWS }; + +static uint64_t rotate(uint64_t value, unsigned count, unsigned bits, bool left) { + const uint64_t mask = bits == 64 ? UINT64_MAX : UINT32_MAX; + value &= mask; + count %= bits; + if (!count) return value; + return (left ? (value << count) | (value >> (bits-count)) : + (value >> count) | (value << (bits-count))) & mask; +} + +int main(void) { + const uint64_t masks[] = {0,1,0x55,0xaa,UINT64_MAX,UINT64_C(1)<<63}; + for (unsigned r=0;rbits/8; + for (unsigned sample=0;sample<8;++sample) { + uint8_t data[64],counts[64],old[64],data_before[64],counts_before[64],old_before[64]; + for (unsigned byte=0;byte<64;++byte) { + data[byte]=(uint8_t)(byte*37+sample*53+1); + old[byte]=(uint8_t)(byte*11+sample*67+129); + } + for (unsigned lane=0;lane<64/lane_bytes;++lane) { + uint64_t value=UINT64_C(0xfedcba9876543200)+sample*row->bits+lane*5+sample; + memcpy(counts+lane*lane_bytes,&value,lane_bytes); + } + memcpy(data_before,data,64); + memcpy(counts_before,counts,64); + memcpy(old_before,old,64); + for (unsigned m=0;mdestination*64; + for (unsigned lane=0;lanewidth/lane_bytes;++lane) { + if (row->mode && !(masks[m]>>lane&1)) { + if (row->mode==2) memset(destination+lane*lane_bytes,0,lane_bytes); + continue; + } + uint64_t value=0,count=row->immediate; + unsigned source_lane=row->broadcast?0:lane; + memcpy(&value,data+(row->variable?lane:source_lane)*lane_bytes,lane_bytes); + if (row->variable) { + count=0; + memcpy(&count,counts+source_lane*lane_bytes,lane_bytes); + } + value=rotate(value,(unsigned)(count&(row->bits-1)),row->bits,row->left); + memcpy(destination+lane*lane_bytes,&value,lane_bytes); + } + memset(destination+row->width,0,64-row->width); + expected[257]=expected[258]=0; + row->run(output+1,data,counts,old,masks[m]); + if (memcmp(output,expected,sizeof(output)) || memcmp(data,data_before,64) || + memcmp(counts,counts_before,64) || memcmp(old,old_before,64)) { + for (unsigned byte=0;bytename,sample,(unsigned long long)masks[m],byte,output[byte],expected[byte]); + break; + } + } + return 1; + } + } + } + } + return 0; +} +` diff --git a/amd64_packed_sad_forms.go b/amd64_packed_sad_forms.go new file mode 100644 index 000000000..5adb54cdd --- /dev/null +++ b/amd64_packed_sad_forms.go @@ -0,0 +1,137 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedSADForm struct { + x86PackedBinaryForm + spec amd64PackedSADSpec + immediate uint8 +} + +// The ordered r/m, vvvv (or old destination), mask and destination views +// are shared by lowering, caller availability and native read envelopes. +// This is a grammar, not an address, initialized-value or ABI permission. +func x86PackedSADProperties(ins Instr, goarch string) (x86PackedSADForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := amd64PackedSADSpecs[Op(base)] + if !known { + return x86PackedSADForm{}, false, nil + } + result := x86PackedSADForm{spec: spec} + fail := func() (x86PackedSADForm, bool, error) { + return result, true, fmt.Errorf("%s %s operands outside Go packed SAD grammar: %q", goarch, ins.Op, ins.Raw) + } + if goarch != "amd64" && goarch != "386" { + return fail() + } + if goarch == "386" && len(ins.Args) > 3 && !ins.x86Encoded { + return fail() + } + local := ins + local.Args = append([]Operand(nil), ins.Args...) + shift := 0 + if spec.kind != amd64PackedSADBlockSum { + if len(local.Args) == 0 || !amd64UnsignedImmediate(local.Args[0], 8) { + return fail() + } + result.immediate = uint8(local.Args[0].Imm) + local.Args = local.Args[1:] + shift = 1 + } + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP} + if spec.form == amd64PackedSADVEXXY { + grammar.encoding = x86PackedLogicalVEX + } else if spec.form == amd64PackedSADEVEXXYZ { + grammar.encoding = x86PackedLogicalVEXOrEVEX + } + if spec.kind == amd64PackedSADDoubleBlock { + grammar.encoding, grammar.zeroing = x86PackedLogicalEVEX, true + } + // Go's shared table accepts high vector spellings on 386, but does not + // establish their physical bank. Validate the table first and return + // Context below, rather than misclassifying accepted syntax as invalid. + // Address register/disp32 checks retain the actual mode independently. + for i, operand := range local.Args { + if operand.Kind != OpMem { + continue + } + if !x86MemoryRegistersValidForArch(operand.Mem, goarch) { + return fail() + } + if goarch == "386" && operand.Mem.Off == int64(uint32(operand.Mem.Off)) { + local.Args[i].Mem.Off = int64(int32(operand.Mem.Off)) + } + } + form, err := x86PackedBinaryProperties(local, "amd64", grammar) + form.first += shift + form.second += shift + form.destination += shift + if form.memory >= 0 { + form.memory += shift + } + if form.mask >= 0 { + form.mask += shift + } + result.x86PackedBinaryForm = form + if err == nil && goarch == "386" { + for _, operand := range ins.Args { + index, valid := amd64VectorRegisterIndex(operand.Reg, amd64VectorByteWidth(operand.Reg)) + if operand.Kind == OpReg && valid && index >= 8 && (form.legacy || amd64VectorByteWidth(operand.Reg) == 64) { + return fail() + } + } + } + if err == nil && spec.kind != amd64PackedSADDoubleBlock && form.mask >= 0 { + return fail() + } + if err == nil { + err = x86Mode32VectorRegisterContext(ins, goarch) + } + if err == nil && form.memory >= 0 && ins.x86AddressBits != 0 { + err = fmt.Errorf("%w: %s lacks its original native address-size binding", ErrProbeNeedsContext, ins.Op) + } + return result, true, err +} + +// Only block SAD compares corresponding bytes. Sliding and shuffled SAD do +// not become constant merely because both operands name the same register. +// Call this only after validating the complete instruction grammar. +func (form x86PackedSADForm) constantZero(ins Instr) bool { + return form.spec.kind == amd64PackedSADBlockSum && form.memory < 0 && + ins.Args[form.first].Reg == ins.Args[form.second].Reg +} + +// Required byte masks for one output word. PSADBW's other six bytes in +// each qword are independent known zero; MPS/DBPS compute all output words. +func (form x86PackedSADForm) wordInputs(outputByte int) (first, second uint64) { + switch form.spec.kind { + case amd64PackedSADBlockSum: + if outputByte%8 != 0 { + return 0, 0 + } + return uint64(255) << uint(outputByte), uint64(255) << uint(outputByte) + case amd64PackedSADMultiple: + lane := outputByte / 16 * 16 + control := form.immediate + if lane != 0 { + control >>= 3 + } + firstOffset := lane + int(control&3)*4 + secondOffset := lane + int(control>>2&1)*4 + outputByte%16/2 + return uint64(15) << uint(firstOffset), uint64(15) << uint(secondOffset) + default: + block := outputByte / 8 * 8 + word := outputByte % 8 / 2 + secondOffset := block + word/2*4 + for i := 0; i < 4; i++ { + shuffled := block + word + i + lane := shuffled / 16 * 16 + selector := form.immediate >> uint(2*(shuffled%16/4)) & 3 + first |= uint64(1) << uint(lane+int(selector)*4+shuffled%4) + } + return first, uint64(15) << uint(secondOffset) + } +} diff --git a/amd64_packed_sad_forms_test.go b/amd64_packed_sad_forms_test.go new file mode 100644 index 000000000..5e0cfe523 --- /dev/null +++ b/amd64_packed_sad_forms_test.go @@ -0,0 +1,319 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func x86SADRows(arch string) []string { + rows := []string{"PSADBW X0,X7", "PSADBW (AX),X7", "MPSADBW $0,X0,X7", "MPSADBW $255,16(AX)(BX*2),X7"} + if arch == "amd64" { + rows = append(rows, "PSADBW X15,X0", "PSADBW data(SB),X1", "PSADBW (FS),X1", "MPSADBW $128,(GS),X15") + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "16(AX)(BX*2)"} { + rows = append(rows, fmt.Sprintf("VPSADBW %s,%s1,%s7", source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("VPSADBW %s,%s31,%s30", source, bank, bank)) + for _, imm := range []int{0, 1, 7, 8, 37, 228, 255} { + rows = append(rows, + fmt.Sprintf("VDBPSADBW $%d,%s,%s31,%s30", imm, source, bank, bank), + fmt.Sprintf("VDBPSADBW $%d,%s,%s31,K1,%s30", imm, source, bank, bank), + fmt.Sprintf("VDBPSADBW.Z $%d,%s,%s31,K7,%s30", imm, source, bank, bank)) + } + if bank != "Z" { + rows = append(rows, fmt.Sprintf("VMPSADBW $37,%s,%s1,%s15", source, bank, bank)) + } + } + } + } + return rows +} + +func x86SADGoPackage(t *testing.T, arch, declaration string) GoPackage { + t.Helper() + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "sad.go", "package sad\nfunc sad"+declaration+"\n", 0) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("sad", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: "sad", Types: typed, Syntax: []*ast.File{syntax}} +} + +func TestX86PackedSADActualGoCompleteOptabs(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{"{APSADBW, yxm, Pq, opBytes{0xf6}}", "{AMPSADBW, yxshuf, Pq, opBytes{0x3a, 0x42, 0}}"} { + if !strings.Contains(string(legacy), row) { + t.Fatal("actual Go legacy registration changed: " + row) + } + } + for name, expected := range map[string][]string{ + "yxm": {"Yxm,Yxr"}, + "yxshuf": {"Yu8,Yxm,Yxr"}, + "_yvaesdec": {"Yxm,Yxr,Yxr", "Yym,Yyr,Yyr", "YxmEvex,YxrEvex,YxrEvex", "YymEvex,YyrEvex,YyrEvex", "Yzm,Yzr,Yzr"}, + "_yvblendpd": {"Yu8,Yxm,Yxr,Yxr", "Yu8,Yym,Yyr,Yyr"}, + "_yvalignd": {"Yu8,YxmEvex,YxrEvex,YxrEvex", "Yu8,YxmEvex,YxrEvex,Yknot0,YxrEvex", "Yu8,YymEvex,YyrEvex,YyrEvex", "Yu8,YymEvex,YyrEvex,Yknot0,YyrEvex", "Yu8,Yzm,Yzr,Yzr", "Yu8,Yzm,Yzr,Yknot0,Yzr"}, + } { + text := string(avx) + if !strings.HasPrefix(name, "_") { + text = string(legacy) + } + section := regexp.MustCompile(`(?s)var ` + name + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(text) + if len(section) != 2 { + t.Fatal("actual table missing " + name) + } + var got []string + for _, row := range regexp.MustCompile(`argList\{([^}]*)\}`).FindAllStringSubmatch(section[1], -1) { + got = append(got, strings.ReplaceAll(row[1], " ", "")) + } + if !reflect.DeepEqual(got, expected) { + t.Fatalf("actual %s rows %q != %q", name, got, expected) + } + } + for _, row := range []struct { + op, table string + count int + }{{"VPSADBW", "_yvaesdec", 5}, {"VMPSADBW", "_yvblendpd", 2}, {"VDBPSADBW", "_yvalignd", 3}} { + start := strings.Index(string(avx), "{as: A"+row.op+",") + if start < 0 { + t.Fatal("missing actual " + row.op) + } + entry := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(entry, "ytab: "+row.table) || strings.Count(entry, "avxEscape") != row.count || strings.Contains(entry, "Bcst") || strings.Contains(entry, "Sae") || strings.Contains(entry, "Rounding") { + t.Fatalf("actual SAD registration changed: %s", entry) + } + if strings.Contains(entry, "ZeroingEnabled") != (row.op == "VDBPSADBW") { + t.Fatal("actual SAD zeroing attribute changed " + row.op) + } + } +} + +func TestX86PackedSADCompleteRowsThreeAPIsFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + source := "TEXT sad(SB),4,$0-0\n" + strings.Join(x86SADRows(arch), "\n") + "\nRET\n" + // Retain all original operands as actual-Go/public-Context/emitter + // formats. Separate complete-input programs own public objects. + ir := x86UnboundVoidScalarFormatObject(t, source, "sad", arch, triple) + compileLLVMToObject(t, llc, triple, "sad-text.ll", "sad-text.o", ir) + compileLLVMToObject(t, llc, triple, "sad-O2.ll", "sad-O2.o", x86MulDivRoundOptimize(t, ir)) + }) + } +} + +func TestX86PackedSADDispatchMalformedAndPhysicalBounds(t *testing.T) { + for _, row := range []string{"PSADBW X0,X1", "VPSADBW Z0,Z1,Z2", "MPSADBW $5,X0,X1", "VMPSADBW $37,Y0,Y1,Y2", "VDBPSADBW.Z $228,Z0,Z1,K1,Z2"} { + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", ins.Op, Op(strings.ToLower(string(ins.Op))), "UNKNOWN", "PSADBW.Z"} { + local := ins + local.Op = identity + original := local + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerPackedSumAbsoluteDifferences(ins.Op, local) + bad := identity != "" && !strings.EqualFold(string(identity), string(ins.Op)) + if !known || term || (err != nil) != bad || bad && output.String() != before || !reflect.DeepEqual(local, original) { + t.Fatalf("dispatch %s/%s: %v", ins.Op, identity, err) + } + } + } + for _, row := range []string{"PSADBW M0,M1", "PSADBW Y0,Y1", "PSADBW $1,X0", "VPSADBW X0,Y1,Y2", "VPSADBW X0,X1,K1,X2", "VPSADBW.Z X0,X1,X2", "VPSADBW.BCST (AX),Z0,Z1", "MPSADBW $-1,X0,X1", "MPSADBW $256,X0,X1", "VMPSADBW $1,Z0,Z1,Z2", "VMPSADBW $1,Y16,Y1,Y2", "VDBPSADBW $1,X0,X1,K0,X2", "VDBPSADBW.Z $1,X0,X1,X2", "VDBPSADBW.BCST $1,(AX),X0,X1", "PSADBW (AX)(SP*2),X0", "VPSADBW 2147483648(AX),Z0,Z1", "MPSADBW $1,$data(SB),X0"} { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + if row == "VDBPSADBW.Z $1,X0,X1,X2" { + x86SADReservedZeroMaskGoObservation(t, source) + } else { + goAssemblerOracleOutput(t, "amd64", source, "", false) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + if known, _, err := ctx.lowerPackedSumAbsoluteDifferences(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]); !known || err == nil || output.String() != before { + t.Fatalf("malformed SAD emitted IR %s: %v", row, err) + } + } + for _, row := range []string{"PSADBW X8,X0", "MPSADBW $1,X0,X15", "VPSADBW Y0,Y1,Y20", "VPSADBW Z0,Z1,Z31"} { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + rejected := !strings.HasPrefix(row, "V") || strings.Contains(row, "Z31") + goAssemblerOracleOutput(t, "386", source, "", !rejected) + file, _ := Parse(ArchAMD64, source) + _, _, err := x86PackedSADProperties(file.Funcs[0].Instrs[1], "386") + if rejected && err == nil || !rejected && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("386 accepted spelling acquired physical register %s: %v", row, err) + } + } +} + +func x86SADReservedZeroMaskGoObservation(t *testing.T, source string) { + t.Helper() + dir := t.TempDir() + asm, object := filepath.Join(dir, "reserved.s"), filepath.Join(dir, "reserved.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "sad", "-o", object, asm) + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), err == nil, "amd64"); failure != nil { + t.Fatalf("actual reserved zero-mask observation: %v\n%s", failure, output) + } + if err != nil { + if !bytes.Contains(output, []byte("mask register must be specified for .Z instructions")) { + t.Fatalf("unexpected original-Go diagnostic: %s", output) + } + t.Log("actual Go rejected unchanged source without its nonzero mask") + return + } + // Intel SDM Vol. 2A Table 2-40 defines EVEX.z=1/aaa=0 as #UD. + // Older Go accepts this source and emits precisely that reserved pattern; + // acceptance is an observation, not authorization to execute or translate it. + if !bytes.Contains(output, []byte("62 f3 75 88 42 d0 01 c3")) { + t.Fatalf("actual Go accepted reserved-body bytes changed: %s", output) + } + code := []byte{0x62, 0xf3, 0x75, 0x88, 0x42, 0xd0, 0x01} + if _, _, known, err := decodedX86RawPackedSADInstruction(code, 64); !known || err == nil { + t.Fatal("actual reserved bytes acquired valid raw semantics") + } + t.Log("actual Go accepted reserved EVEX.z/aaa=0 bytes; named and raw validators still reject them") +} + +func TestX86PackedSADFPSpanAndNativeMemoryBoundary(t *testing.T) { + for _, row := range []string{"VPSADBW source+0(FP),Z0,Z1", "VMPSADBW $37,source+0(FP),Y0,Y1", "VDBPSADBW.Z $228,source+0(FP),Z0,K1,Z1"} { + source := "TEXT sad(SB),4,$0-64\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, _ := Parse(ArchAMD64, source) + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedSADProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, missing := range []int{-1, 0, form.memoryBytes - 1} { + sig := x86WordMaddFPType(t) + if missing >= 0 { + for i, slot := range sig.Frame.Params { + if slot.Offset == int64(missing) { + sig.Frame.Params = append(sig.Frame.Params[:i], sig.Frame.Params[i+1:]...) + break + } + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + _, _, err := ctx.lowerPackedSumAbsoluteDifferences(ins.Op, ins) + if (err != nil) != (missing >= 0) || missing >= 0 && output.String() != before { + t.Fatalf("FP complete native-width span %s missing%d: %v", row, missing, err) + } + } + } + for _, row := range []string{"PSADBW source+0(FP),X0", "MPSADBW $1,(SP),X0", "PSADBW 8(BP),X0", "MPSADBW $1,(AX)(BP*2),X0"} { + source := "TEXT f(SB),4,$0-64\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, _ := Parse(ArchAMD64, source) + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], x86WordMaddFPType(t), nil) + before := output.String() + if _, _, err := ctx.lowerPackedSumAbsoluteDifferences(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("virtual slots became native storage %s: %v", row, err) + } + } +} + +func TestX86PackedSADLowerEffectAndFullReadClassificationAgree(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86SADRows(arch) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, known, err := x86PackedSADProperties(ins, arch) + if !known || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatalf("lowered accepted SAD lacks registered source effect %s: %v", row, err) + } + for index := range ins.Args { + width, write, valid := x86CallerMoveMemoryShape(ins, index) + if valid != (index == form.memory) || valid && (write || width != int64(form.byteWidth)) { + t.Fatalf("source/native read classifications disagree %s operand%d: %d/%t/%t", row, index, width, write, valid) + } + } + } + } +} + +func TestX86PackedSADDecoderPhysicalRowsDoNotGrantSourcePermissions(t *testing.T) { + for _, encoded := range []Op{"VPSADBW", "VPSADBW.EVEX", "VMPSADBW", "VDBPSADBW"} { + for width := 0; width < 3; width++ { + if width == 2 && (encoded == "VMPSADBW" || encoded == "VPSADBW") { + continue + } + code := encodeX86RawPackedSAD(encoded, width, 0, 1, 2, 0, false, false) + for _, mode := range []int{32, 64} { + ins, length, known, err := decodedX86RawPackedSADInstruction(code, mode) + if !known || err != nil || length != len(code) { + t.Fatalf("physical %x mode%d: %v", code, mode, err) + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + if _, known, err := x86PackedSADProperties(ins, arch); !known || err != nil { + t.Fatalf("actual physical decoder row %x: %v", code, err) + } + if mode == 32 && len(ins.Args) > 3 { + local := ins + local.x86Encoded = false + if _, _, err := x86PackedSADProperties(local, arch); err == nil { + t.Fatal("named/public source borrowed physical frontend permission") + } + } + // Byte identity is not initialized input or pointer permission. + body := x86SourceStackState{vectors: make(map[Reg]uint64), masks: make(map[Reg]bool)} + form, _, _ := x86PackedSADProperties(ins, "amd64") + if !x86CallerSADStep(&ins, form, &body, nil) { + t.Fatal("pure register effect unexpectedly failed") + } + knownBytes := body.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)] + if knownBytes&3 != 0 { + t.Fatal("raw producer invented initialized SAD input bytes") + } + } + } + } +} diff --git a/amd64_packed_sad_reference_test.go b/amd64_packed_sad_reference_test.go new file mode 100644 index 000000000..0e6c18a33 --- /dev/null +++ b/amd64_packed_sad_reference_test.go @@ -0,0 +1,47 @@ +package plan9asm + +// Test-only unsigned byte mathematics. It knows no producer/parser/LLVM +// helper and is also used to construct the separate original-Go oracle. +func x86SADReference(op string, width, control int, first, second []byte) []uint16 { + result := make([]uint16, width/2) + difference := func(a, b byte) uint16 { + if a >= b { + return uint16(a) - uint16(b) + } + return uint16(b) - uint16(a) + } + if op == "PSADBW" || op == "VPSADBW" { + for byteIndex := 0; byteIndex < width; byteIndex++ { + result[byteIndex/8*4] += difference(first[byteIndex], second[byteIndex]) + } + return result + } + if op == "MPSADBW" || op == "VMPSADBW" { + for lane := 0; lane < width/16; lane++ { + selector := control >> (lane * 3) + block := lane*16 + (selector&3)*4 + window := lane*16 + (selector>>2&1)*4 + for word := 0; word < 8; word++ { + for byteIndex := 0; byteIndex < 4; byteIndex++ { + result[lane*8+word] += difference(first[block+byteIndex], second[window+word+byteIndex]) + } + } + } + return result + } + for lane := 0; lane < width/16; lane++ { + var permuted [16]byte + for word := 0; word < 4; word++ { + copy(permuted[word*4:word*4+4], first[lane*16+(control>>(2*word)&3)*4:]) + } + for word := 0; word < 8; word++ { + block := word / 4 * 8 + window := block + word%4 + other := lane*16 + block + (word%4/2)*4 + for byteIndex := 0; byteIndex < 4; byteIndex++ { + result[lane*8+word] += difference(permuted[window+byteIndex], second[other+byteIndex]) + } + } + } + return result +} diff --git a/amd64_packed_sad_runtime_test.go b/amd64_packed_sad_runtime_test.go new file mode 100644 index 000000000..7902d92e8 --- /dev/null +++ b/amd64_packed_sad_runtime_test.go @@ -0,0 +1,324 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86SADNumericCase struct { + op, bank string + width, imm, masking int + memory bool +} + +func x86SADNumericCases() []x86SADNumericCase { + var cases []x86SADNumericCase + for _, op := range []string{"PSADBW", "VPSADBW", "MPSADBW", "VMPSADBW", "VDBPSADBW"} { + for widthIndex, bank := range []string{"X", "Y", "Z"} { + if !strings.HasPrefix(op, "V") && bank != "X" || op == "VMPSADBW" && bank == "Z" { + continue + } + controls := []int{0} + if strings.Contains(op, "MPS") || op == "VDBPSADBW" { + controls = []int{0, 1, 2, 3, 4, 5, 6, 7, 27, 37, 85, 170, 228, 255} + } + for _, control := range controls { + for _, memory := range []bool{false, true} { + for mask := 0; mask < 3; mask++ { + if mask != 0 && op != "VDBPSADBW" { + continue + } + cases = append(cases, x86SADNumericCase{op, bank, 16 << widthIndex, control, mask, memory}) + } + } + } + } + } + return cases +} + +func (row x86SADNumericCase) instruction() string { + first := row.bank + "0" + if row.memory { + first = "(CX)" + } + op := row.op + if row.masking == 2 { + op += ".Z" + } + if op == "PSADBW" { + return op + " " + first + ",X2" + } + if op == "MPSADBW" { + return fmt.Sprintf("%s $%d,%s,X2", op, row.imm, first) + } + if op == "VPSADBW" { + return fmt.Sprintf("%s %s,%s1,%s2", op, first, row.bank, row.bank) + } + mask := "" + if row.masking != 0 { + mask = "K1," + } + return fmt.Sprintf("%s $%d,%s,%s1,%s%s2", op, row.imm, first, row.bank, mask, row.bank) +} + +func x86SADOriginalLegacyOracle(t *testing.T) { + t.Helper() + var declarations, assembly, tests strings.Builder + declarations.WriteString("package sad\n") + tests.WriteString(`package sad +import("testing";"encoding/binary") +func TestOriginal(t *testing.T) { + var a,b [64]byte + for sample:=0;sample<32;sample++ { + for i:=range a {a[i]=byte(i*17+sample*31);b[i]=byte(i*73-sample*19)} +`) + // Actual stock Go executes every legacy imm8, in both register and + // m128 forms. It does not execute AVX512 or the Z-alias observation program. + for _, op := range []string{"PSADBW", "MPSADBW"} { + controls := 1 + if op == "MPSADBW" { + controls = 256 + } + for imm := 0; imm < controls; imm++ { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("original%d", strings.Count(declarations.String(), "func ")) + fmt.Fprintf(&declarations, "func %s(out *[24]byte,a,b *[64]byte,flags uint64)\n", name) + row := x86SADNumericCase{op: op, bank: "X", width: 16, imm: imm, memory: memory} + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVUPS (CX),X0\nMOVUPS (DX),X2\nPUSHQ flags+24(FP)\nPOPFQ\n%s\nMOVUPS X2,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,16(AX)\nRET\n", name, row.instruction()) + fmt.Fprintf(&tests, `{ + var out [24]byte + beforeA,beforeB:=a,b + flags:=uint64(2) + if sample&1!=0 {flags|=0x8d5} + %s(&out,&a,&b,flags) + want:=[8]uint16{} +`, name) + if op == "PSADBW" { + tests.WriteString("for i:=0;i<16;i++ {d:=int(a[i])-int(b[i]);if d<0 {d=-d};want[i/8*4]+=uint16(d)}\n") + } else { + fmt.Fprintf(&tests, "for word:=0;word<8;word++ {for i:=0;i<4;i++ {d:=int(a[%d+i])-int(b[%d+word+i]);if d<0 {d=-d};want[word]+=uint16(d)}}\n", (imm&3)*4, (imm>>2&1)*4) + } + fmt.Fprintf(&tests, "for i,w:=range want {if g:=binary.LittleEndian.Uint16(out[i*2:]);g!=w {t.Fatalf(\"%s sample%%d word%%d got%%d want%%d\",sample,i,g,w)}}\n", name) + tests.WriteString("if binary.LittleEndian.Uint64(out[16:])&0x8d5!=flags&0x8d5 || a!=beforeA || b!=beforeB {t.Fatal(\"flags/input canary\")}\n}\n") + } + } + } + tests.WriteString("}\n}\n") + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module sad\n\ngo 1.20\n", "sad.go": declarations.String(), "sad_amd64.s": assembly.String(), "sad_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + output, err := runtimeOracleCombinedOutput(ctx, command) + if err != nil { + t.Fatalf("actual original legacy SAD oracle: %v\n%s", err, output) + } + t.Logf("actual original SSE %d kernels x 32 initialized samples; full status flags and unchanged input canaries", strings.Count(declarations.String(), "func ")) +} + +func TestCrossLinuxRuntimeMatrixX86PackedSADOriginalVEXUpperAliases(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package sad\n") + tests.WriteString(`package sad +import("testing";"encoding/binary") +func TestOriginal(t *testing.T) { + var a,b,old [64]byte + for sample:=0;sample<32;sample++ { + for i:=range a {a[i]=byte(i*17+sample*31);b[i]=byte(i*73-sample*19);old[i]=byte(0x91+i*11)} +`) + kernels := 0 + for _, op := range []string{"PSADBW", "MPSADBW", "VPSADBW", "VMPSADBW"} { + controls := []int{0} + if strings.Contains(op, "MPS") { + controls = []int{0, 1, 2, 3, 4, 5, 6, 7, 27, 37, 85, 170, 228, 255} + } + for _, imm := range controls { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("alias%d", kernels) + kernels++ + row := x86SADNumericCase{op: op, bank: "X", width: 16, imm: imm, memory: memory} + fmt.Fprintf(&declarations, "func %s(out *[48]byte,a,b,old *[64]byte,flags uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-40\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVUPS (CX),X0\nMOVUPS (DX),X1\nVMOVUPS (R8),Y2\n", name) + legacy := !strings.HasPrefix(op, "V") + if legacy { + assembly.WriteString("MOVUPS (DX),X2\n") + } + fmt.Fprintf(&assembly, "PUSHQ flags+32(FP)\nPOPFQ\n%s\nVMOVUPS Y2,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,32(AX)\nRET\n", row.instruction()) + fmt.Fprintf(&tests, "{\n var out,want [48]byte\n for i:=range out {out[i]=0xc3;want[i]=0xc3}\n beforeA,beforeB,beforeOld:=a,b,old\n flags:=uint64(2)\n if sample&1!=0 {flags|=0x8d5}\n %s(&out,&a,&b,&old,flags)\n sums:=[8]uint16{}\n", name) + if strings.Contains(op, "MPS") { + fmt.Fprintf(&tests, "for word:=0;word<8;word++ {for i:=0;i<4;i++ {d:=int(a[%d+i])-int(b[%d+word+i]);if d<0 {d=-d};sums[word]+=uint16(d)}}\n", (imm&3)*4, (imm>>2&1)*4) + } else { + tests.WriteString("for i:=0;i<16;i++ {d:=int(a[i])-int(b[i]);if d<0 {d=-d};sums[i/8*4]+=uint16(d)}\n") + } + tests.WriteString("for i,w:=range sums {binary.LittleEndian.PutUint16(want[2*i:],w)}\n") + if legacy { + tests.WriteString("copy(want[16:32],old[16:32])\n") + } else { + tests.WriteString("for i:=16;i<32;i++ {want[i]=0}\n") + } + tests.WriteString("observed:=binary.LittleEndian.Uint64(out[32:40])\n binary.LittleEndian.PutUint64(want[32:40],observed)\n") + fmt.Fprintf(&tests, "if out!=want || observed&0x8d5!=flags&0x8d5 || a!=beforeA || b!=beforeB || old!=beforeOld {t.Fatalf(\"%s sample%%d actual%%x want%%x flags%%x\",sample,out,want,observed)}\n}\n", name) + } + } + } + tests.WriteString("}\n}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module sad\n\ngo 1.20\n", "sad.go": declarations.String(), "sad_amd64.s": assembly.String(), "sad_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + output, err := runtimeOracleCombinedOutput(ctx, command) + if err != nil { + t.Fatalf("actual original SSE/VEX upper-alias oracle: %v\n%s", err, output) + } + t.Logf("actual original SSE/VEX %d kernels x 32 initialized samples; all 32 output bytes, six status flags and canaries; no EVEX instruction executed", kernels) +} + +func TestCrossLinuxRuntimeMatrixX86PackedSADIndependentWholeBytes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + x86SADOriginalLegacyOracle(t) + var declarations, assembly, table strings.Builder + declarations.WriteString("package sad\n") + cases := x86SADNumericCases() + for index, row := range cases { + name := fmt.Sprintf("sad%d", index) + fmt.Fprintf(&declarations, "func %s(out *[72]byte,a,b,old *[64]byte,mask,flags uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-48\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVQ mask+32(FP),R9\nVMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nKMOVQ R9,K1\n", name) + if !strings.HasPrefix(row.op, "V") { + assembly.WriteString("MOVUPS (DX),X2\n") + } + // Isolate the SAD preservation contract from arithmetic flag producers. + // Both seeds still demand all six defined status bits, including AF. + fmt.Fprintf(&assembly, "PUSHQ flags+40(FP)\nPOPFQ\n%s\nVMOVUPS Z2,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,64(AX)\nRET\n", row.instruction()) + kind := 0 + if strings.Contains(row.op, "MPS") { + kind = 1 + } else if row.op == "VDBPSADBW" { + kind = 2 + } + legacy := 0 + if !strings.HasPrefix(row.op, "V") { + legacy = 1 + } + fmt.Fprintf(&table, "{sad%d,%d,%d,%d,%d,%d},\n", index, kind, row.width, row.imm, row.masking, legacy) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + pkg := mustGoPackage(t, "sad", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for index := range cases { + name := fmt.Sprintf("sad%d", index) + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, Options{ + Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + var main strings.Builder + main.WriteString(`#include +#include +#include +typedef void (*kernel)(void*,const void*,const void*,const void*,uint64_t,uint64_t); +`) + for index := range cases { + fmt.Fprintf(&main, "extern void sad%d(void*,const void*,const void*,const void*,uint64_t,uint64_t);\n", index) + } + main.WriteString("static const struct {kernel run;int kind,width,imm,masking,legacy;} cases[]={\n" + table.String() + "};\n") + main.WriteString(` +static unsigned diff(unsigned a,unsigned b) {return a>b ? a-b : b-a;} +static void reference(uint8_t *out,int kind,int width,int imm,const uint8_t *a,const uint8_t *b) { + memset(out,0,width); + uint16_t sums[32]={0}; + if(kind==0) { + for(int i=0;i>(3*lane),block=lane*16+(select&3)*4,window=lane*16+((select>>2)&1)*4; + for(int word=0;word<8;++word) for(int i=0;i<4;++i) sums[lane*8+word]+=diff(a[block+i],b[window+word+i]); + } + } else { + for(int lane=0;lane>(2*d))&3)*4,4); + for(int word=0;word<8;++word) for(int i=0;i<4;++i) + sums[lane*8+word]+=diff(shuffled[(word/4)*8+word%4+i],b[lane*16+(word/4)*8+(word%4/2)*4+i]); + } + } + for(int word=0;word>8;} +} +int main(void) { + for(unsigned row=0;row>(i/2)&1)) value=cases[row].masking==2 ? 0 : old[i]; + want[1+i]=value; + } + uint64_t flags;memcpy(&flags,out+65,8);memcpy(want+65,&flags,8); + if((flags&0x8d5)!=(seededFlags&0x8d5) || memcmp(out,want,sizeof(out)) || memcmp(aa,beforeA,sizeof(aa)) || memcmp(bb,beforeB,sizeof(bb))) { + fprintf(stderr,"SAD row=%u kind=%d width=%d imm=%d masking=%d legacy=%d sample=%u flags=%llx\n",row,cases[row].kind,cases[row].width,cases[row].imm,cases[row].masking,cases[row].legacy,sample,(unsigned long long)flags); + for(int i=0;i<80;++i) if(out[i]!=want[i]) fprintf(stderr,"byte%d got=%u want=%u\n",i,out[i],want[i]); + return 1; + } + } + return 0; +} +`) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + ir := module.String() + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "sad-independent-whole-bytes", triple, ir, main.String(), prefix) + t.Logf("%d initialized kernels x 32 inputs; all 64 alias bytes, merge/zero masks, six status flags and canaries", len(cases)) + }) + } + // These are LLVM software-lowering numerical tests plus actual original + // SSE execution. Z source acceptance/objects do not prove native AVX512. +} diff --git a/amd64_packed_shift_aligned_count_test.go b/amd64_packed_shift_aligned_count_test.go new file mode 100644 index 000000000..fd42023d4 --- /dev/null +++ b/amd64_packed_shift_aligned_count_test.go @@ -0,0 +1,162 @@ +package plan9asm + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +var x86LegacyShiftMemoryOpcodes = []struct { + name string + code byte +}{ + {"PSLLW", 0xf1}, {"PSLLL", 0xf2}, {"PSLLQ", 0xf3}, + {"PSRLW", 0xd1}, {"PSRLL", 0xd2}, {"PSRLQ", 0xd3}, +} + +func TestX86PackedShiftLegacyMemoryActualGoEncoding(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, operation := range x86LegacyShiftMemoryOpcodes { + source := "TEXT shifts(SB),4,$0-0\n" + operation.name + " (CX),X0\nRET\n" + dir := t.TempDir() + input, object := filepath.Join(dir, "original.s"), filepath.Join(dir, "original.o") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-o", object, input) + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch, "GOTOOLCHAIN=local") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("actual Go %s %s: %v\n%s", arch, operation.name, err, output) + } + var code []byte + for _, line := range regexp.MustCompile(`(?m)^\s*0x[[:xdigit:]]+\s+((?:[[:xdigit:]]{2} )+)`).FindAllSubmatch(output, -1) { + part, err := hex.DecodeString(strings.ReplaceAll(string(line[1]), " ", "")) + if err != nil { + t.Fatal(err) + } + code = append(code, part...) + } + want := []byte{0x66, 0x0f, operation.code, 0x01, 0xc3} + if !bytes.Equal(code, want) { + t.Fatalf("actual Go %s %s memory encoding=%x, want=%x\n%s", arch, operation.name, code, want, output) + } + } + } +} + +func TestX86PackedShiftNativeCountThreeAPIsFiveO2Objects(t *testing.T) { + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 llc and llvm-objdump required") + } + var source strings.Builder + source.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, operation := range x86LegacyShiftMemoryOpcodes { + for _, memory := range []string{"(AX)", "8(AX)(FS)", "8(AX)(GS)"} { + fmt.Fprintf(&source, "%s %s,X0\n", operation.name, memory) + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewPackageAPI(t, source.String(), x86PackedShiftGoPackage(t, target.arch), target.arch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original native/segment count source invented entry inputs", api, err) + } + } + ir := x86InstructionFormatObjectUnit(t, file, map[string]FuncSig{"shifts": {Name: "shifts", Ret: Void}}, + target.arch, target.triple, 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerX86PackedShift(ins.Op, ins) + }) + for axis, body := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, mnemonic := range []string{"psllw", "pslld", "psllq", "psrlw", "psrld", "psrlq"} { + for _, segment := range []string{"", "%fs:", "%gs:"} { + if strings.Count(body, `asm sideeffect "`+mnemonic+` `+segment+`($2), $0", "=x,0,r,~{memory}"`) != 1 { + t.Fatalf("%s IR%d lost original %s %s m128 count operation", target.triple, axis, mnemonic, segment) + } + } + } + if strings.Contains(body, `asm sideeffect "movdqa`) { + t.Fatal("count access was replaced by a stronger MOVDQA access") + } + dir := t.TempDir() + input, object := filepath.Join(dir, "native-count.ll"), filepath.Join(dir, "native-count.o") + if err := os.WriteFile(input, []byte(body), 0600); err != nil { + t.Fatal(err) + } + stage, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(stage, llc, "-O2", "-mtriple="+target.triple, "-filetype=obj", input, "-o", object) + output, err := runtimeOracleCombinedOutput(stage, command) + cancel() + if err != nil { + t.Fatalf("O2 %s IR%d: %v\n%s", target.triple, axis, err, output) + } + stage, cancel = runtimeOracleStageContext(t) + command = exec.CommandContext(stage, disassembler, "-d", object) + output, err = runtimeOracleCombinedOutput(stage, command) + cancel() + if err != nil { + t.Fatalf("O2 disassembly %s IR%d: %v\n%s", target.triple, axis, err, output) + } + for _, mnemonic := range []string{"psllw", "pslld", "psllq", "psrlw", "psrld", "psrlq"} { + pattern := regexp.MustCompile(`(?m)\b` + mnemonic + `\s+[^\n]*\([^\n]*\)`) + if len(pattern.FindAll(output, -1)) != 3 { + t.Fatalf("O2 actual original %s m128 count %s IR%d:\n%s", mnemonic, target.triple, axis, output) + } + } + for _, segment := range []string{"%fs:", "%gs:"} { + if strings.Count(string(output), segment) != 6 { + t.Fatalf("O2 lost six native %s count overrides %s IR%d:\n%s", segment, target.triple, axis, output) + } + } + } + } +} + +func TestX86PackedShiftLegacyNativeBackingAndCallerPermissionsRemainContext(t *testing.T) { + for _, operation := range x86LegacyShiftMemoryOpcodes { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+operation.name+" count+0(FP),X0\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || output.Len() != before { + t.Fatalf("unproved native FP backing emitted IR: %s %v", operation.name, err) + } + } + // Initialized vectors do not supply native backing or memory permission. + for _, operation := range x86LegacyShiftMemoryOpcodes { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+operation.name+" (CX),X0\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !recognized || err != nil { + t.Fatal(err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0)}} + if x86CallerPackedShiftStep(&ins, form, &state, nil) { + t.Fatal("initialized vector granted legacy native-memory permission") + } + } +} diff --git a/amd64_packed_shift_alignment_linux_test.go b/amd64_packed_shift_alignment_linux_test.go new file mode 100644 index 000000000..3410b242d --- /dev/null +++ b/amd64_packed_shift_alignment_linux_test.go @@ -0,0 +1,317 @@ +//go:build linux && (amd64 || arm64) + +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" +) + +func x86PackedShiftLinuxExecution(t *testing.T) []string { + t.Helper() + if runtime.GOARCH == "amd64" { + return nil // Required native Linux/amd64 counterpart, not Darwin Rosetta. + } + qemu, err := exec.LookPath("qemu-x86_64") + if err != nil { + t.Fatal("required explicit QEMU x86 execution:", err) + } + ctx, cancel := runtimeOracleStageContext(t) + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, qemu, "--version")) + cancel() + if err != nil || !strings.HasPrefix(string(output), "qemu-x86_64 version 10.2.3 ") { + t.Fatalf("required QEMU10.2.3: %v\n%s", err, output) + } + t.Logf("explicit executable transport: %s -L /usr/x86_64-linux-gnu", qemu) + return []string{qemu, "-L", "/usr/x86_64-linux-gnu"} +} + +const x86PackedShiftLegacyMemoryGoDriver = `package main +import ( + "fmt" + "os" + "strconv" + "unsafe" +) +func numeric(functions []func(unsafe.Pointer,unsafe.Pointer,unsafe.Pointer,uint64)) { + limits:=[]uint64{0,1,7,15,16,17,31,32,33,63,64,65,255,256,1<<32,^uint64(0)} + flags:=[3][5]byte{{1,0,1,0,1},{0,1,0,1,1},{0,0,0,0,0}} + for kernel,fn:=range functions { + for round,count:=range limits { + for seed:=0;seed<3;seed++ { + var data [18]byte + var storage [96]byte + var out [23]byte + for i:=range data { data[i]=byte(i*71+round*97+13) } + for i:=range storage { storage[i]=byte(i*31+7) } + for i:=range out { out[i]=0xc3 } + start:=uintptr(unsafe.Pointer(&storage[0])) + offset:=int((16-start%16)%16) + for i:=0;i<8;i++ { storage[offset+i]=byte(count>>(8*i)) } + beforeData,beforeStorage:=data,storage + fn(unsafe.Pointer(&out[1]),unsafe.Pointer(&data[1]),unsafe.Pointer(&storage[offset]),uint64(seed)) + if out[0]!=0xc3 || out[22]!=0xc3 || data!=beforeData || storage!=beforeStorage { + panic("canary/input mutation") + } + for i:=0;i<5;i++ { + if out[17+i]!=flags[seed][i] { panic("flags") } + } + bytes:=[]int{2,4,8,2,4,8}[kernel] + for i:=0;i<16;i+=bytes { + var value,want uint64 + for b:=0;b>count } + } + for b:=0;b>(8*b)) { panic("numeric") } + } + } + } + } + } +} +func main() { + functions:=[]func(unsafe.Pointer,unsafe.Pointer,unsafe.Pointer,uint64){legacycount0,legacycount1,legacycount2,legacycount3,legacycount4,legacycount5} + if os.Args[1]=="numeric" { numeric(functions); return } + index,err:=strconv.Atoi(os.Args[1]) + if err!=nil { panic(err) } + var data,count [64]byte + var out [23]byte + for i:=range data { + data[i]=byte(i*71+13) + count[i]=byte(i*31+7) + } + for i:=range out { out[i]=0xc3 } + start:=uintptr(unsafe.Pointer(&count[0])) + offset:=(16-start%16)%16+1 + beforeData,beforeCount:=data,count + fmt.Printf("original Go count mod16=%d\n",uintptr(unsafe.Pointer(&count[offset]))%16) + functions[index](unsafe.Pointer(&out[1]),unsafe.Pointer(&data[1]),unsafe.Pointer(&count[offset]),0) + if out[0]!=0xc3 || out[22]!=0xc3 || data!=beforeData || count!=beforeCount { panic("canary/input mutation") } + flags:=[5]byte{1,0,1,0,1} + for i:=0;i<5;i++ { if out[17+i]!=flags[i] { panic("flags") } } + /* The initialized low count qword exceeds every lane width. */ + for i:=1;i<=16;i++ { if out[i]!=0 { panic("numeric") } } + os.Exit(90) +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedShiftOriginalLegacyMemory(t *testing.T) { + prefix := x86PackedShiftLinuxExecution(t) + x86PackedShiftVerifyGuestArguments(t, prefix) + _, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + observer := x86PackedShiftBuildOriginalCountObserver(t, x86PackedShiftLinuxCCompiler(t, clang)) + assembly, _ := x86PackedShiftLegacyMemoryAssembly() + var declarations strings.Builder + for i := range x86LegacyShiftMemoryOpcodes { + fmt.Fprintf(&declarations, "func legacycount%d(out,data,count unsafe.Pointer,seed uint64)\n", i) + } + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module originalcount\n\ngo 1.20\n", + "main.go": x86PackedShiftLegacyMemoryGoDriver + declarations.String(), + "original_amd64.s": strings.ReplaceAll(assembly, "TEXT legacycount", "TEXT ·legacycount"), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + env := append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local") + executable := filepath.Join(dir, "original") + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, "go", "build", "-p=2", "-o", executable, ".") + command.Dir, command.Env = dir, env + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("original Go body/link: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + argv := x86PackedShiftGuestExecution(prefix, executable, "numeric") + command = exec.CommandContext(ctx, argv[0], argv[1:]...) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("original Go aligned numeric/flags: %v\n%s", err, output) + } + for i := range x86LegacyShiftMemoryOpcodes { + ctx, cancel = runtimeOracleStageContext(t) + argv = x86PackedShiftGuestExecution(prefix, observer, strconv.Itoa(i)) + command = exec.CommandContext(ctx, argv[0], argv[1:]...) + reference, referenceErr := runtimeOracleCombinedOutput(ctx, command) + cancel() + status, observationErr := x86PackedShiftCountObservation(i, reference, referenceErr) + if observationErr != nil { + t.Fatalf("original native opcode observer %d: %v\n%s", i, observationErr, reference) + } + ctx, cancel = runtimeOracleStageContext(t) + argv := x86PackedShiftGuestExecution(prefix, executable, strconv.Itoa(i)) + command = exec.CommandContext(ctx, argv[0], argv[1:]...) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + exit, pureExit := err.(*exec.ExitError) + text := string(output) + if !strings.Contains(text, "original Go count mod16=1") || !pureExit { + t.Fatalf("original Go unaligned m128 count identity %d: %v\n%s", i, err, output) + } + if status == "normal" { + if exit.ExitCode() != 90 { + t.Fatalf("original Go differs from normal native opcode %d: %v\n%s", i, err, output) + } + } else if exit.ExitCode() != 2 || !strings.Contains(text, "main.legacycount") || + (!strings.Contains(text, "unexpected fault address") && !strings.Contains(text, "invalid memory address")) { + t.Fatalf("original Go differs from native opcode memory fault %d: %v\n%s", i, err, output) + } + t.Logf("original Go/native opcode %s mod16=1 status=%s", x86LegacyShiftMemoryOpcodes[i].name, status) + } + t.Log("6 original Go m128 kernels ×48 numeric/flags/canary inputs and 6 independent native opcode unaligned status comparisons") +} + +func x86PackedShiftVerifyGuestArguments(t *testing.T, prefix []string) { + t.Helper() + dir := t.TempDir() + const source = `package main +import ( + "fmt" + "os" +) +func main() { fmt.Printf("%q\n", os.Args) } +` + for name, contents := range map[string]string{ + "go.mod": "module argumenttransport\n\ngo 1.20\n", "main.go": source, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "argument-transport") + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, "go", "build", "-p=2", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", + "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("original Go argument witness body/link: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + argv := append(append([]string(nil), prefix...), executable) + command = exec.CommandContext(ctx, argv[0], argv[1:]...) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || string(output) != fmt.Sprintf("%q\n", []string{executable}) { + t.Fatalf("guest no-argument witness: %v\n%s", err, output) + } + t.Logf("guest no-argument witness: %s", strings.TrimSpace(string(output))) + arguments := []string{"first", "with space"} + for _, repaired := range []bool{false, true} { + argv := append(append([]string(nil), prefix...), executable) + argv = append(argv, arguments...) + want := append([]string{executable}, arguments...) + if len(prefix) != 0 && !repaired { + // The authenticated binfmt release treats the first argument as + // original argv[0] even for ordinary explicit execution. + want = arguments + } + if repaired { + argv = x86PackedShiftGuestExecution(prefix, executable, arguments...) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, argv[0], argv[1:]...) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || string(output) != fmt.Sprintf("%q\n", want) { + t.Fatalf("guest argument witness repaired=%t command=%q: %v\n%s; want %q", + repaired, argv, err, output, fmt.Sprintf("%q\n", want)) + } + t.Logf("guest argument witness repaired=%t: %s", repaired, strings.TrimSpace(string(output))) + } +} + +func TestCrossLinuxRuntimeMatrixX86PackedShiftLLVMNativeCount(t *testing.T) { + prefix := x86PackedShiftLinuxExecution(t) + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + assembly, sigs := x86PackedShiftLegacyMemoryAssembly() + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + const triple = "x86_64-unknown-linux-gnu" + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + driver := x86PackedShiftLegacyCountCHeader + x86PackedShiftLegacyCountDeclarations() + x86PackedShiftOriginalCountC() + + x86PackedShiftCountCChecks + x86PackedShiftLegacyNativeCountCMain + compileAndRunRuntimeTestWithCompiler(t, llc, x86PackedShiftLinuxCCompiler(t, clang), "legacy_count_contract", triple, ir, driver, prefix) + t.Log("6 original opcode/LLVM m128 kernels ×48 numeric/flags/canary inputs, 90 unaligned status comparisons and 6 original/LLVM full128 guard-page faults") +} + +func x86PackedShiftLinuxCCompiler(t *testing.T, clang string) []string { + t.Helper() + if runtime.GOARCH == "amd64" { + return []string{clang} + } + gcc, err := exec.LookPath("x86_64-linux-gnu-gcc") + if err != nil { + t.Fatal("required AMD64 C/link driver:", err) + } + return []string{gcc, "-no-pie"} +} + +func TestCrossLinuxRuntimeMatrixX86PackedShiftCompleteNumerical(t *testing.T) { + prefix := x86PackedShiftLinuxExecution(t) + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + cases := x86PackedShiftNumericalCases() + assembly, sigs := x86PackedShiftNumericalAssembly(cases) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + const triple = "x86_64-unknown-linux-gnu" + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + var driver strings.Builder + driver.WriteString(x86PackedShiftIndependentC) + for _, c := range cases { + fmt.Fprintf(&driver, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint64_t);\n", c.name) + } + driver.WriteString("static const struct shift_case cases[]={\n") + for _, c := range cases { + fmt.Fprintf(&driver, "{%s,%q,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d},\n", c.name, c.row, + c.width, c.bits, c.countMode, c.immediate, c.memory, c.maskMode, c.destination, + x86ShiftBoolean(c.left), x86ShiftBoolean(c.legacy), x86ShiftBoolean(c.broadcast)) + } + driver.WriteString(`}; +int main(void) { + for(unsigned i=0;i= 6 || len(output) > 256 || !strings.HasSuffix(string(output), "\n") { + return "", fmt.Errorf("unbound original count observation") + } + lines := strings.Split(strings.TrimSuffix(string(output), "\n"), "\n") + diagnostic := false + if len(lines) == 2 && lines[0] == "qemu: uncaught target signal 11 (Segmentation fault) - core dumped" { + diagnostic = true + lines = lines[1:] + } + if len(lines) != 1 { + return "", fmt.Errorf("unexpected original count observer output: %q", output) + } + prefix := fmt.Sprintf("COUNT_NATIVE_STATUS kernel=%d mod16=1 ", kernel) + switch lines[0] { + case prefix + "wait=0 outcome=normal": + if !diagnostic { + return "normal", nil + } + case prefix + "wait=11 outcome=fault", prefix + "wait=139 outcome=fault": + return "fault", nil + } + return "", fmt.Errorf("invalid original count status/kernel/address: %q", output) +} diff --git a/amd64_packed_shift_dependency_test.go b/amd64_packed_shift_dependency_test.go new file mode 100644 index 000000000..876316bf3 --- /dev/null +++ b/amd64_packed_shift_dependency_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +// An independent bit-by-bit relation, not the production interval arithmetic. +// Every possible in-range dynamic count is enumerated; all other counts emit +// zero. Operand bytes are known only when every possible selected bit is known. +func x86ShiftIndependentBitInputs(bits, width, count int, left, byteShift, dynamic, perLane bool) [][2]uint64 { + if byteShift { + bits = 128 + } + masks := make([][2]uint64, width) + for output := 0; output < width*8; output++ { + first, end := int(uint8(count)), int(uint8(count))+1 + if byteShift { + first, end = first*8, first*8+1 + } + if dynamic { + first, end = 0, bits + } + for shift := first; shift < end; shift++ { + input := output%bits + shift + if left { + input = output%bits - shift + } + if input >= 0 && input < bits { + input += output / bits * bits + masks[output/8][0] |= uint64(1) << uint(input/8) + } + } + if dynamic { + start, end := 0, 8 + if perLane { + start, end = output/bits*(bits/8), (output/bits+1)*(bits/8) + } + for at := start; at < end; at++ { + masks[output/8][1] |= uint64(1) << uint(at) + } + } + } + return masks +} + +func TestX86PackedShiftResultEveryByteDependency(t *testing.T) { + for _, bank := range []struct { + name string + width int + }{{"X", 16}, {"Y", 32}, {"Z", 64}} { + for _, bits := range []int{16, 32, 64, 128} { + for _, left := range []bool{false, true} { + for mode := 0; mode < 3; mode++ { + byteShift, dynamic, perLane := bits == 128, mode != 0, mode == 2 + if byteShift && dynamic { + continue + } + name := "VPSRL" + if left { + name = "VPSLL" + } + if perLane { + name += "V" + } + name += map[int]string{16: "W", 32: "D", 64: "Q", 128: "DQ"}[bits] + last := 255 + if dynamic { + last = 0 + } + for immediate := 0; immediate <= last; immediate++ { + count := fmt.Sprintf("$%d", immediate) + if dynamic { + count = "X0" + if perLane { + count = bank.name + "0" + } + } + row := fmt.Sprintf("%s %s,%s1,%s2", name, count, bank.name, bank.name) + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !known || err != nil { + t.Fatal(row, err) + } + want := x86ShiftIndependentBitInputs(bits, bank.width, immediate, left, byteShift, dynamic, perLane) + for output, masks := range want { + data, counts := x86PackedShiftByteInputs(form, output) + if data != masks[0] || counts != masks[1] { + t.Fatalf("%s byte=%d data=%x/%x counts=%x/%x", row, output, data, masks[0], counts, masks[1]) + } + } + for missing := -1; missing < bank.width; missing++ { + for operand := 0; operand < 2; operand++ { + inputs := [2]uint64{^uint64(0), ^uint64(0)} + if missing >= 0 { + inputs[operand] &^= uint64(1) << uint(missing) + } + var expected uint64 + for output, masks := range want { + if inputs[0]&masks[0] == masks[0] && inputs[1]&masks[1] == masks[1] { + expected |= uint64(1) << uint(output) + } + } + if got := x86PackedShiftDefinedBytes(form, inputs[0], inputs[1], 0); got != expected { + t.Fatal(row, missing, operand, got, expected) + } + state := x86ResultState{flags: x86ByteFlagsAll} + state.writeVector(0, 64, inputs[1], false) + state.writeVector(1, 64, inputs[0], false) + physical := expected | ^x86CallerVectorByteMask(int64(bank.width)) + if !x86ResultTransfer(ins, &state, nil, 0, "amd64") || state.vectorBytes(2) != physical || state.flags != x86ByteFlagsAll { + t.Fatal("complete result byte state differs", row, missing, operand) + } + } + } + } + } + } + } + } +} diff --git a/amd64_packed_shift_exec_test.go b/amd64_packed_shift_exec_test.go new file mode 100644 index 000000000..e321197e1 --- /dev/null +++ b/amd64_packed_shift_exec_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "reflect" + "testing" +) + +func x86PackedShiftGuestExecution(prefix []string, executable string, arguments ...string) []string { + command := append([]string(nil), prefix...) + command = append(command, executable) + if len(prefix) != 0 { + // The pinned binfmt QEMU preserves a separate original argv[0]. + // Native execution must not duplicate it. The Linux runtime oracle + // independently verifies this pinned transport before any ASM runs. + command = append(command, executable) + } + return append(command, arguments...) +} + +func TestX86PackedShiftGuestArgumentTransport(t *testing.T) { + for _, c := range []struct { + name string + prefix []string + arguments []string + want []string + }{ + {"native", nil, []string{"numeric"}, []string{"/guest", "numeric"}}, + {"native_empty", nil, nil, []string{"/guest"}}, + {"pinned_empty", []string{"/qemu", "-L", "/sysroot"}, nil, + []string{"/qemu", "-L", "/sysroot", "/guest", "/guest"}}, + {"pinned_numeric", []string{"/qemu", "-L", "/sysroot"}, []string{"numeric"}, + []string{"/qemu", "-L", "/sysroot", "/guest", "/guest", "numeric"}}, + {"pinned_fault", []string{"/qemu", "-L", "/sysroot"}, []string{"3", "with space"}, + []string{"/qemu", "-L", "/sysroot", "/guest", "/guest", "3", "with space"}}, + } { + t.Run(c.name, func(t *testing.T) { + beforePrefix := append([]string(nil), c.prefix...) + beforeArguments := append([]string(nil), c.arguments...) + got := x86PackedShiftGuestExecution(c.prefix, "/guest", c.arguments...) + if !reflect.DeepEqual(got, c.want) { + t.Fatalf("guest command: got %q, want %q", got, c.want) + } + if !reflect.DeepEqual(c.prefix, beforePrefix) || !reflect.DeepEqual(c.arguments, beforeArguments) { + t.Fatal("argument construction mutated its inputs") + } + }) + } +} diff --git a/amd64_packed_shift_fault_test.go b/amd64_packed_shift_fault_test.go new file mode 100644 index 000000000..100b924c0 --- /dev/null +++ b/amd64_packed_shift_fault_test.go @@ -0,0 +1,85 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86PackedShiftFaultProgram() (string, map[string]FuncSig, string, string, []bool) { + var source, declarations, rows strings.Builder + sigs := map[string]FuncSig{} + var required []bool + for i, row := range []struct { + instruction string + bytes, accessible int + }{ + {"VPSRLW.Z (CX),Y0,K1,Y1", 16, 8}, + {"VPSRLD.Z (CX),Y0,K1,Y1", 16, 8}, + {"VPSRLQ.Z (CX),Y0,K1,Y1", 16, 8}, + {"VPSRLW.Z $1,(CX),K1,Y1", 0, 0}, + {"VPSRLDQ $1,(CX),Y1", 32, 8}, + {"VPSRLD.Z $1,(CX),K1,Y1", 0, 0}, + {"VPSRLQ.BCST.Z $1,(CX),K1,Y1", 0, 0}, + {"VPSRLVW.Z (CX),Y0,K1,Y1", 0, 0}, + {"VPSRLVD.BCST.Z (CX),Y0,K1,Y1", 0, 0}, + {"VPSRLD $255,(CX),Y1", 32, 8}, + {"VPSRLVD (CX),Y0,Y1", 32, 8}, + {"VPSRLVD.Z (CX),Y0,K1,Y1", 4, 3}, + } { + name := fmt.Sprintf("faultshift%d", i) + mask := 0 + if i == 11 { + mask = 1 + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ data+8(FP),BX\nMOVQ count+16(FP),CX\nVMOVUPS (BX),Y0\nVPXOR Y0,Y0,Y0\nMOVQ $%d,DX\nKMOVQ DX,K1\n%s\nVMOVUPS Y1,(AX)\nRET\n", name, mask, row.instruction) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*);\n", name) + want := "0" + if i == 4 { + // VPSRLDQ shifts each independent 128-bit memory lane. The + // newly added readable control also checks its actual bytes. + want = "i % 16 == 15 ? 0 : input[i + 1]" + } + fmt.Fprintf(&declarations, `static int observe%d(const uint8_t *input) { + uint8_t data[32], output[34]; + memset(data, 0x71, sizeof(data)); + memset(output, 0xa7, sizeof(output)); + %s(output + 1, data, input); + if (output[0] != 0xa7 || output[33] != 0xa7) return 3; + for (unsigned i = 0; i < 32; ++i) { + uint8_t want = %s; + if (output[i + 1] != want || data[i] != 0x71) return 4; + } + return 0; +} +`, i, name, want) + fmt.Fprintf(&rows, "{observe%d, %d, %d},\n", i, row.bytes, row.accessible) + required = append(required, row.bytes != 0) + } + return source.String(), sigs, declarations.String(), rows.String(), required +} + +func TestCrossLinuxRuntimeMatrixX86PackedShiftNativeReadFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + source, sigs, declarations, rows, required := x86PackedShiftFaultProgram() + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + driver := x86MemoryFaultDriver(declarations, rows) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := execution.compileLLVM(t, "packed-shift-memory-faults", ir, driver, level) + x86CheckMemoryAccessMatrix(t, execution, executable, required) + }) + } + t.Log("7 full/active native reads and 5 lane-mask suppression cases, including readable controls") +} diff --git a/amd64_packed_shift_forms.go b/amd64_packed_shift_forms.go new file mode 100644 index 000000000..46cb8911b --- /dev/null +++ b/amd64_packed_shift_forms.go @@ -0,0 +1,208 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedShiftCount uint8 + +const ( + x86PackedShiftUniform x86PackedShiftCount = iota + x86PackedShiftPerLane + x86PackedShiftBytes +) + +type x86PackedShiftForm struct { + countMode x86PackedShiftCount + laneBits, byteWidth int + count, data, destination, mask, memory int + memoryBytes int + left, legacy, mmx, broadcast, zeroing bool + faultSuppressed bool + arithmetic bool + countOperand Operand +} + +func x86TypedLegacyPackedShiftByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedShiftProperties(op, ins, arch) + if !known || err != nil || !form.legacy || form.mmx { + return x86TypedByteEffect{}, known && (err != nil || form.legacy && !form.mmx), err + } + return x86PackedShiftDependencyEffect(ins, form), true, nil +} + +// Properties establish encoding and access shape only, never initialized +// data, pointer bounds, alignment or permission to return through raw bytes. +func x86PackedShiftProperties(op Op, ins Instr, goarch string) (x86PackedShiftForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + uniform, recognized := amd64PackedLogicalShiftSpecs[Op(base)] + arithmetic, arithmeticUniform := amd64PackedArithmeticShiftSpecs[Op(base)] + if arithmeticUniform { + uniform, recognized = arithmetic, true + } + perLane, variable := amd64PerLaneVariableShiftSpecs[base] + byteShift := base == "PSLLO" || base == "PSLLDQ" || base == "PSRLO" || base == "PSRLDQ" || base == "VPSLLDQ" || base == "VPSRLDQ" + if !recognized && !variable && !byteShift { + return x86PackedShiftForm{}, false, nil + } + f := x86PackedShiftForm{count: 0, data: 1, destination: len(ins.Args) - 1, mask: -1, memory: -1, + arithmetic: arithmeticUniform || variable && perLane.arithmetic} + fail := func() (x86PackedShiftForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside Go packed shift grammar: %q", goarch, op, ins.Raw) + } + if goarch != "amd64" && goarch != "386" || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + local := ins + local.Op = op + if recognized && !uniform.vector { + var err error + local, err = x86LegacyMMXReadInstruction(local, goarch) + if err != nil { + return f, true, err + } + } + f.countMode, f.laneBits, f.left, f.legacy = x86PackedShiftUniform, uniform.laneBits, uniform.left, !uniform.vector + if variable { + f.countMode, f.laneBits, f.left, f.legacy = x86PackedShiftPerLane, perLane.laneBits, perLane.operation == "shl", false + } + if byteShift { + f.countMode, f.laneBits, f.left, f.legacy = x86PackedShiftBytes, 8, strings.Contains(base, "SLL"), !strings.HasPrefix(base, "V") + } + if f.legacy { + if dotted || len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return fail() + } + f.data = f.destination + f.byteWidth = 16 + _, f.mmx = amd64ParseMReg(ins.Args[f.destination].Reg) + if f.mmx { + if byteShift || goarch == "386" { + return fail() + } + f.byteWidth = 8 + } + } else { + switch suffix { + case "": + if dotted { + return fail() + } + case "Z": + f.zeroing = true + case "BCST": + f.broadcast = true + case "BCST.Z": + f.broadcast, f.zeroing = true, true + default: + return fail() + } + if len(ins.Args) != 3 && len(ins.Args) != 4 || byteShift && dotted || f.broadcast && f.laneBits == 16 { + return fail() + } + if len(ins.Args) == 4 { + if byteShift || goarch == "386" && !ins.x86Encoded || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + f.mask = 2 + } else if f.zeroing { + return fail() + } + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + } + register := func(operand Operand, width int) bool { + if f.mmx { + _, valid := amd64ParseMReg(operand.Reg) + return operand.Kind == OpReg && valid + } + if f.legacy { + index, valid := amd64ParseXReg(operand.Reg) + return operand.Kind == OpReg && valid && (goarch == "amd64" && index < 16 || goarch == "386" && index < 8) + } + return amd64GoPackedVectorMoveRegister(operand, width, goarch) + } + if !register(ins.Args[f.destination], f.byteWidth) { + return fail() + } + count, data := local.Args[f.count], local.Args[f.data] + if count.Kind == OpImm { + var valid bool + count, valid = x86ImmediateShuffleConstant(count, goarch, ins.x86Encoded) + if !valid { + return fail() + } + if variable || count.Imm < -128 || count.Imm > 255 || f.legacy && count.Imm > 127 { + return fail() + } + if count.Imm < 0 && (base == "VPSRAQ" || f.mask >= 0 || + !f.legacy && amd64PackedByteShiftRequiresEVEX(data, ins.Args[f.destination], f.byteWidth)) { + return fail() + } + if isAMD64MemoryOperand(data) { + f.memory, f.memoryBytes = f.data, f.byteWidth + if f.broadcast { + f.memoryBytes = f.laneBits / 8 + } + } else if f.broadcast || !register(data, f.byteWidth) { + return fail() + } + // All W/D/Q immediate Full tuples use E4/E4.nb. Byte shifts + // retain E4NF.nb; uniform Mem128 count rows are handled below. + f.faultSuppressed = !f.legacy && !byteShift + } else { + if byteShift || f.broadcast && !variable || !register(data, f.byteWidth) { + return fail() + } + countWidth := f.byteWidth + if !variable && !f.mmx { + countWidth = 16 + } + if isAMD64MemoryOperand(count) { + f.memory, f.memoryBytes = f.count, countWidth + if f.broadcast { + f.memoryBytes = f.laneBits / 8 + } + } else if f.broadcast || !register(count, countWidth) { + return fail() + } + f.faultSuppressed = variable + } + f.countOperand = count + if f.memory >= 0 && !x86PackedShiftMemoryValid(ins.Args[f.memory], goarch) { + return fail() + } + if f.memory >= 0 && ins.Args[f.memory].Kind == OpMem { + memory := ins.Args[f.memory].Mem + if err := x86SymbolIndexedMemoryError(memory, goarch); err != nil { + return f, true, err + } + if memory.OffRaw != "" || memory.IndexExt != "" { + return fail() + } + } + if goarch == "386" { + for _, operand := range ins.Args { + width := amd64VectorByteWidth(operand.Reg) + if index, valid := amd64VectorRegisterIndex(operand.Reg, width); operand.Kind == OpReg && valid && index >= 8 { + return f, true, fmt.Errorf("mode32 packed shift has no physical SIMD register %s: %w", operand.Reg, ErrProbeNeedsContext) + } + } + } + return f, true, nil +} + +func x86PackedShiftMemoryValid(operand Operand, goarch string) bool { + if operand.Kind == OpSym { + symbol := strings.TrimSpace(operand.Sym) + _, integer := parseInt(symbol) + return !strings.HasPrefix(symbol, "$") && (strings.HasSuffix(symbol, "(SB)") || integer == nil) + } + if operand.Kind != OpMem { + return operand.Kind == OpFP + } + memory := operand.Mem + return (memory.Off == int64(int32(memory.Off)) || goarch == "386" && memory.Off == int64(uint32(memory.Off))) && + x86MemoryRegistersValidForArch(memory, goarch) && (memory.Index == "" || memory.Index != SP && + (memory.Scale == 1 || memory.Scale == 2 || memory.Scale == 4 || memory.Scale == 8)) +} diff --git a/amd64_packed_shift_forms_test.go b/amd64_packed_shift_forms_test.go new file mode 100644 index 000000000..39cd1904d --- /dev/null +++ b/amd64_packed_shift_forms_test.go @@ -0,0 +1,354 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "sort" + "strings" + "testing" +) + +func x86PackedShiftNames() []string { + var names []string + for op := range amd64PackedLogicalShiftSpecs { + names = append(names, string(op)) + } + for op, spec := range amd64PerLaneVariableShiftSpecs { + if !spec.arithmetic { + names = append(names, op) + } + } + names = append(names, "PSLLO", "PSLLDQ", "PSRLO", "PSRLDQ", "VPSLLDQ", "VPSRLDQ") + sort.Strings(names) + return names +} + +func x86PackedShiftRows(op, arch string) []string { + legacy := !strings.HasPrefix(op, "V") + bytes := strings.HasSuffix(op, "DQ") || op == "PSLLO" || op == "PSRLO" + perLane := strings.Contains(op, "LLV") || strings.Contains(op, "RLV") + if legacy { + rows := []string{op + " $-128,X0", op + " $127,X1"} + if !bytes { + rows = append(rows, op+" X0,X1", op+" (AX),X1") + if arch == "amd64" { + rows = append(rows, op+" $1,M0", op+" M0,M1", op+" (AX),M1") + } + } + return rows + } + var rows []string + for _, bank := range []string{"X", "Y", "Z"} { + if !perLane { + rows = append(rows, fmt.Sprintf("%s $0,%s0,%s2", op, bank, bank), fmt.Sprintf("%s $255,(AX),%s2", op, bank)) + if bank != "Z" { + rows = append(rows, fmt.Sprintf("%s $-128,%s0,%s2", op, bank, bank)) + } + } + if bytes { + continue + } + countBank := "X" + if perLane { + countBank = bank + } + rows = append(rows, fmt.Sprintf("%s %s0,%s1,%s2", op, countBank, bank, bank), fmt.Sprintf("%s (AX),%s1,%s2", op, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s0,%s1,K1,%s2", op, countBank, bank, bank), fmt.Sprintf("%s.Z (AX),%s1,K7,%s2", op, bank, bank)) + if !perLane { + rows = append(rows, fmt.Sprintf("%s.Z $3,(AX),K1,%s2", op, bank)) + } + } + if !strings.HasSuffix(op, "W") { + input := "$3,(AX)," + if perLane { + input = "(AX)," + bank + "1," + } + rows = append(rows, fmt.Sprintf("%s.BCST %s%s2", op, input, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s.BCST.Z %sK1,%s2", op, input, bank)) + } + } + } + return rows +} + +func TestX86PackedShiftEveryGoOpcodeAndAttribute(t *testing.T) { + if len(x86PackedShiftNames()) != 24 { + t.Fatal("incomplete logical-shift family") + } + dir := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + avx, err := os.ReadFile(filepath.Join(dir, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + legacy, err := os.ReadFile(filepath.Join(dir, "asm6.go")) + if err != nil { + t.Fatal(err) + } + re := regexp.MustCompile(`\{as: A(VPS(?:LL|RL)(?:[WDQ]|V[WDQ]|DQ)), ytab: ([^,]+),`) + rows := re.FindAllStringSubmatch(string(avx), -1) + if len(rows) != 14 { + t.Fatalf("Go AVX family inventory changed: %d", len(rows)) + } + for _, row := range rows { + op, table := row[1], "_yvpslld" + if strings.HasSuffix(op, "DQ") { + table = "_yvpslldq" + } else if strings.Contains(op, "LLV") || strings.Contains(op, "RLV") { + table = "_yvandnpd" + if strings.HasSuffix(op, "W") { + table = "_yvblendmpd" + } + } + if row[2] != table { + t.Fatalf("%s Go table %s != %s", op, row[2], table) + } + start := strings.Index(string(avx), row[0]) + block := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + for _, line := range strings.Split(block, "\n") { + if !strings.Contains(line, "avxEscape | evex") { + continue + } + byteShift := strings.HasSuffix(op, "DQ") + uniformCount := !strings.Contains(op, "LLV") && !strings.Contains(op, "RLV") && !strings.Contains(line, "0x7") + broadcast := !byteShift && !strings.HasSuffix(op, "W") && !uniformCount + if strings.Contains(line, "evexZeroingEnabled") == byteShift || strings.Contains(line, "evexBcstN") != broadcast || + strings.Contains(line, "evexRounding") || strings.Contains(line, "evexSae") { + t.Fatalf("%s attribute disagreement: %s", op, line) + } + } + } + for _, op := range []string{"PSLLW", "PSLLL", "PSLLQ", "PSRLW", "PSRLL", "PSRLQ"} { + if !strings.Contains(string(legacy), "{A"+op+", yps,") { + t.Fatalf("Go yps member changed: %s", op) + } + } + // Bind the complete operand-class inventory, not just the table names. + // These four AVX tables are shared with unrelated instructions; checking + // their actual rows detects a changed immediate, mask or memory class. + tables := map[string][]string{ + "_yvpslld": { + "Yu8,Yxr,Yxr", "Yi8,Yxr,Yxr", "Yu8,Yyr,Yyr", "Yi8,Yyr,Yyr", + "Yxm,Yxr,Yxr", "Yxm,Yyr,Yyr", + "Yu8,YxmEvex,YxrEvex", "Yu8,YxmEvex,Yknot0,YxrEvex", + "Yu8,YymEvex,YyrEvex", "Yu8,YymEvex,Yknot0,YyrEvex", + "Yu8,Yzm,Yzr", "Yu8,Yzm,Yknot0,Yzr", + "YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", + "YxmEvex,YyrEvex,YyrEvex", "YxmEvex,YyrEvex,Yknot0,YyrEvex", + "YxmEvex,Yzr,Yzr", "YxmEvex,Yzr,Yknot0,Yzr", + }, + "_yvpslldq": { + "Yu8,Yxr,Yxr", "Yi8,Yxr,Yxr", "Yu8,Yyr,Yyr", "Yi8,Yyr,Yyr", + "Yu8,YxmEvex,YxrEvex", "Yu8,YymEvex,YyrEvex", "Yu8,Yzm,Yzr", + }, + "_yvandnpd": { + "Yxm,Yxr,Yxr", "Yym,Yyr,Yyr", + "YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", + "YymEvex,YyrEvex,YyrEvex", "YymEvex,YyrEvex,Yknot0,YyrEvex", + "Yzm,Yzr,Yzr", "Yzm,Yzr,Yknot0,Yzr", + }, + "_yvblendmpd": { + "YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", + "YymEvex,YyrEvex,YyrEvex", "YymEvex,YyrEvex,Yknot0,YyrEvex", + "Yzm,Yzr,Yzr", "Yzm,Yzr,Yknot0,Yzr", + }, + } + assertRows := func(source, name string, want []string) { + t.Helper() + block := regexp.MustCompile(`(?s)var ` + regexp.QuoteMeta(name) + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(source) + if len(block) != 2 { + t.Fatalf("missing Go table %s", name) + } + var got []string + for _, row := range regexp.MustCompile(`argList\{([^}]+)\}`).FindAllStringSubmatch(block[1], -1) { + got = append(got, strings.ReplaceAll(row[1], " ", "")) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("%s operand classes changed:\ngot %q\nwant %q", name, got, want) + } + } + for name, want := range tables { + assertRows(string(avx), name, want) + } + assertRows(string(legacy), "yps", []string{"Ymm,Ymr", "Yi8,Ymr", "Yxm,Yxr", "Yi8,Yxr"}) + assertRows(string(legacy), "ypsdq", []string{"Yi8,Yxr"}) + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil { + t.Fatal(err) + } + for alias, canonical := range map[string]string{"PSLLDQ": "APSLLO", "PSRLDQ": "APSRLO"} { + if !regexp.MustCompile(`instructions\["` + alias + `"\]\s*=\s*x86\.` + canonical).Match(aliases) { + t.Fatalf("Go byte-shift alias changed: %s", alias) + } + } +} + +func TestX86PackedShiftCompleteGoRowsThreeAPIsFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, op := range x86PackedShiftNames() { + for _, row := range x86PackedShiftRows(op, target.arch) { + fmt.Fprintln(&source, row) + } + } + source.WriteString("RET\n") + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + pkg := x86PackedShiftGoPackage(t, target.arch) + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewPackageAPI(t, source.String(), pkg, target.arch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original format source invented numerical/native entry inputs", api, err) + } + } + ir := x86InstructionFormatObjectUnit(t, file, map[string]FuncSig{"shifts": {Name: "shifts", Ret: Void}}, + target.arch, target.triple, 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerX86PackedShift(ins.Op, ins) + }) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + }) + } +} + +func TestX86PackedShiftImmediateAnd386RejectedGoRows(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []string{ + "PSLLW $128,X0", "PSRLO $128,X0", "VPSRLW $-1,X20,X21", + "VPSRLW $-1,(AX),X1", "VPSRLQ.Z $-1,X0,K1,X1", "VPSLLDQ $-1,Z0,Z1", + "VPSRLD $256,X0,X1", "VPSRLVW $1,X0,X1", "VPSRLW.BCST $1,(AX),X1", + } { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + assertX86PackedLogicalShiftRejected(t, arch, testTargetTriple("linux", arch), row) + } + } + for _, op := range []string{"PSLLW", "PSLLL", "PSLLQ", "PSRLW", "PSRLL", "PSRLQ"} { + for _, row := range []string{op + " $1,M0", op + " M0,M1", op + " (AX),M1"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + assertX86PackedLogicalShiftRejected(t, "386", "i386-unknown-linux-gnu", row) + } + } +} + +func TestX86PackedShiftIdentityAndInvalidFormsHaveNoIRSideEffects(t *testing.T) { + for _, op := range x86PackedShiftNames() { + row := x86PackedShiftRows(op, "amd64")[0] + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", Op(strings.ToLower(op)), original.Op} { + ins := original + ins.Op = identity + ctx, _ := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + if recognized, _, err := ctx.lowerX86PackedShift(Op(strings.ToLower(op)), ins); !recognized || err != nil { + t.Fatalf("matching identity %s/%s: %v", op, identity, err) + } + } + for _, dispatch := range []Op{"UNKNOWN_SHIFT", "PSRAD", "PSLLL"} { + if dispatch == original.Op { + continue + } + ins := original + before := append([]Operand(nil), ins.Args...) + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + length := output.Len() + recognized, _, err := ctx.lowerX86PackedShift(dispatch, ins) + if recognized && err == nil || output.Len() != length || !reflect.DeepEqual(before, ins.Args) || ins.Op != original.Op { + t.Fatalf("conflicting/unknown %s/%s mutated or emitted IR", op, dispatch) + } + } + } + for _, row := range []string{ + "VPSRLW $8,X0,Y1", "VPSLLDQ.Z $1,X0,K1,X1", "VPSLLW $1,X0,K0,X1", + "VPSLLD.BCST X0,X1,X2", "VPSRLD.RN_SAE $1,X0,X1", "VPSRLW $1,8(BAD),X1", + "VPSRLW $1,8(AX)(SP*2),X1", "VPSRLW $1,2147483648(AX),X1", + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + continue + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + length := output.Len() + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); err == nil || output.Len() != length { + t.Fatalf("invalid row %s emitted %d bytes: %v", row, output.Len()-length, err) + } + } +} + +func TestX86PackedShiftFullNativeMemoryReadAndSuppression(t *testing.T) { + for _, test := range []struct { + row string + bytes int + suppressed bool + }{ + {"VPSRLW (AX),Y0,K1,Y1", 16, false}, + {"VPSRLD (AX),Z0,K1,Z1", 16, false}, + {"VPSRLQ (AX),Z0,K1,Z1", 16, false}, + {"VPSRLW.Z $1,(AX),K1,Z1", 64, true}, + {"VPSRLD.Z $1,(AX),K1,Z1", 64, true}, + {"VPSRLQ.BCST.Z $1,(AX),K1,Z1", 8, true}, + {"VPSRLVW.Z (AX),Z0,K1,Z1", 64, true}, + {"VPSRLVD.BCST.Z (AX),Z0,K1,Z1", 4, true}, + {"VPSRLDQ $1,(AX),Z1", 64, false}, + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+test.row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !recognized || err != nil || form.memoryBytes != test.bytes || form.faultSuppressed != test.suppressed { + t.Fatalf("native operand mismatch %s: %+v/%v", test.row, form, err) + } + if !test.suppressed { + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), fmt.Sprintf("load volatile <%d x i8>", test.bytes)) { + t.Fatalf("lost full native read %s:\n%s", test.row, output.String()) + } + } + } +} + +func x86PackedShiftGoPackage(t *testing.T, arch string) GoPackage { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "shifts.go", "package shifts\nfunc shifts()\n", 0) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("shifts", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: "shifts", Types: typed, Syntax: []*ast.File{file}} +} diff --git a/amd64_packed_shift_fp.go b/amd64_packed_shift_fp.go new file mode 100644 index 000000000..167f8ae01 --- /dev/null +++ b/amd64_packed_shift_fp.go @@ -0,0 +1,25 @@ +package plan9asm + +import ( + "fmt" + "math" +) + +// A logical FP span is usable only when every byte belongs to a validated +// scalar ABI0 slot. Padding and unknown native caller-frame bytes are not +// synthesized. This is a value/span proof, not a physical alignment proof. +func (c *amd64Ctx) validateX86PackedShiftFPSpan(off int64, width int) error { + if width <= 0 || off < 0 || off > math.MaxInt64-int64(width) { + return fmt.Errorf("packed shift FP span is not bounded: %w", ErrProbeNeedsContext) + } + if err := c.validateFPVectorSpan(off, width); err != nil { + return fmt.Errorf("packed shift needs complete FP bytes: %v: %w", err, ErrProbeNeedsContext) + } + for byteOffset := off; byteOffset < off+int64(width); byteOffset++ { + slot, _, _ := c.fpVectorSlot(byteOffset) + if x86FrameScalarBytes(slot.Type, c.goarch) == 0 { + return fmt.Errorf("packed shift FP type %s lacks scalar byte storage: %w", slot.Type, ErrProbeNeedsContext) + } + } + return nil +} diff --git a/amd64_packed_shift_fp_context_test.go b/amd64_packed_shift_fp_context_test.go new file mode 100644 index 000000000..c9546ad8b --- /dev/null +++ b/amd64_packed_shift_fp_context_test.go @@ -0,0 +1,202 @@ +package plan9asm + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86PackedShiftCompleteFPSpanThreeAPIsFiveObjects(t *testing.T) { + // Even a fully declared logical ABI0 span comprises independent LLVM + // allocas. Read every bound byte without using a pointer to just the first + // alloca, and retain mask-controlled reads. BCST at +4 crosses two slots. + rows := []struct { + instruction string + masked bool + }{ + {"VPSRLW a+0(FP),Y1,Y2", false}, + {"VPSRLQ a+0(FP),Z1,Z2", false}, + {"VPSRLVW a+0(FP),Z1,Z2", false}, + {"VPSRLVD.BCST a+4(FP),Z1,Z2", false}, + {"VPSRLVQ.BCST a+4(FP),Z1,Z2", false}, + {"VPSRLD $1,a+0(FP),Z2", false}, + {"VPSRLQ.BCST $1,a+4(FP),Z2", false}, + {"VPSRLDQ $1,a+0(FP),Z2", false}, + {"VPSRLVW.Z a+0(FP),Z1,K1,Z2", true}, + {"VPSRLVD.BCST.Z a+4(FP),Z1,K1,Z2", true}, + {"VPSRLVQ.BCST.Z a+4(FP),Z1,K1,Z2", true}, + {"VPSRLQ.BCST.Z $1,a+4(FP),K1,Z2", true}, + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "shifts.go", "package shifts\nfunc shifts(a,b,c,d,e,f,g,h uint64)\n", 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", target.arch) + typed, err := (&types.Config{Sizes: sizes}).Check("shifts", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + pkg := GoPackage{Path: "shifts", Types: typed, Syntax: []*ast.File{file}} + sig, err := DeriveGoFuncABI0Signature("shifts", typed.Scope().Lookup("shifts").(*types.Func), target.arch, sizes) + if err != nil { + t.Fatal(err) + } + for _, row := range rows { + if row.masked && target.arch == "386" { + continue // Go's 386 source frontend rejects this explicit mask row. + } + source := "TEXT shifts(SB),4,$0-64\n" + row.instruction + "\nRET\n" + goAssemblerOracleOutput(t, target.arch, source, "", true) + parsed, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"shifts": sig}} + original := parsed + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + if row.masked { + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewPackageAPI(t, source, pkg, target.arch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original logical FP source invented the incoming mask", row.instruction, api, err) + } + } + format := x86InstructionFormatObjectUnit(t, original, options.Sigs, target.arch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerX86PackedShift(ins.Op, ins) + }) + compileLLVMToObject(t, llc, target.triple, "fp-original-format.ll", "fp-original-format.o", format) + // The public positive source obtains its mask from an actual + // typed input. Independent FP slots still have no contiguous + // backing; the shift must gather precisely its complete span. + source = strings.Replace(source, "\n"+row.instruction, + "\nMOVQ a+0(FP),AX\nKMOVQ AX,K1\n"+row.instruction, 1) + goAssemblerOracleOutput(t, target.arch, source, "", true) + parsed, err = Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + } + ir, err := Translate(parsed, options) + if err != nil { + t.Fatalf("%s original FP read %s: Translate=%v", target.triple, row.instruction, err) + } + compileLLVMToObject(t, llc, target.triple, "fp-shift.ll", "fp-shift.o", ir) + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, parsed, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatalf("%s original FP read %s: Module=%v", target.triple, row.instruction, err) + } + compileLLVMToObject(t, llc, target.triple, "fp-module.ll", "fp-module.o", module.String()) + result, err := translateGoModuleInContext(context, pkg, []byte(strings.Replace(source, "shifts(SB)", "·shifts(SB)", 1)), + GoModuleOptions{GOARCH: target.arch, Sizes: sizes, TargetTriple: target.triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatalf("%s original FP read %s: GoModule=%v", target.triple, row.instruction, err) + } + compileLLVMToObject(t, llc, target.triple, "fp-go-module.ll", "fp-go-module.o", result.Module.String()) + }() + instruction := original.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, original.Funcs[0], sig, nil) + if _, _, err := ctx.lowerX86PackedShift(instruction.Op, instruction); err != nil { + t.Fatalf("%s complete FP span: %v", row.instruction, err) + } + if row.masked && !strings.Contains(output.String(), "br i1") { + t.Fatalf("%s lost conditional FP reads", row.instruction) + } + } + } +} + +func TestX86PackedShiftIncompleteFPSpansFailBeforeIR(t *testing.T) { + for _, instruction := range []string{ + "VPSRLW a+8(FP),Y1,Y2", "VPSRLQ.BCST $1,a+12(FP),Z2", + "VPSRLVD.BCST.Z a+14(FP),Z1,K1,Z2", "VPSRLDQ $1,a+0(FP),Z2", + "VPSRLVW.Z a+0(FP),Z1,K1,Z2", "VPSRLQ.BCST $1,a-1(FP),Z2", + } { + source := "TEXT shifts(SB),4,$0-16\n" + instruction + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, frame := range []FrameLayout{ + {Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}}, + {Params: []FrameSlot{{Offset: 0, Type: I8, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}}, + } { + sig := FuncSig{Ret: Void, Args: []LLVMType{frame.Params[0].Type, frame.Params[1].Type}, Frame: frame} + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.Len() + ins := file.Funcs[0].Instrs[1] + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || before != output.Len() { + t.Fatalf("%s incomplete/padded FP span emitted IR: %v", instruction, err) + } + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + if target.arch == "386" && strings.Contains(instruction, "K1") { + continue + } + fset := token.NewFileSet() + declaration, err := parser.ParseFile(fset, "shifts.go", "package shifts\nfunc shifts(a,b uint64)\n", 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", target.arch) + typed, err := (&types.Config{Sizes: sizes}).Check("shifts", fset, []*ast.File{declaration}, nil) + if err != nil { + t.Fatal(err) + } + sig, err := DeriveGoFuncABI0Signature("shifts", typed.Scope().Lookup("shifts").(*types.Func), target.arch, sizes) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"shifts": sig}} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s original unbound FP span %s: Translate=%v", target.triple, instruction, err) + } + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s original unbound FP span %s: Module=%v", target.triple, instruction, err) + } + pkg := GoPackage{Path: "shifts", Types: typed, Syntax: []*ast.File{declaration}} + result, err := translateGoModuleInContext(context, pkg, []byte(strings.Replace(source, "shifts(SB)", "·shifts(SB)", 1)), + GoModuleOptions{GOARCH: target.arch, Sizes: sizes, TargetTriple: target.triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s original unbound FP span %s: GoModule=%v", target.triple, instruction, err) + } + }() + } + } +} diff --git a/amd64_packed_shift_fp_runtime_test.go b/amd64_packed_shift_fp_runtime_test.go new file mode 100644 index 000000000..b38497dbd --- /dev/null +++ b/amd64_packed_shift_fp_runtime_test.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "os/exec" + "runtime" + "strings" + "testing" +) + +func TestX86PackedShiftFPSpanNumerical(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + var prefix []string + flags := []string{"-no-pie"} + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + if !rosettaAvailable() { + t.Fatal("Darwin x86 diagnostic transport required") + } + prefix = []string{"/usr/bin/arch", "-x86_64"} + flags = []string{"-arch", "x86_64"} + } else if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { + qemu, err := exec.LookPath("qemu-x86_64") + if err != nil { + t.Fatal("explicit Linux x86 runtime transport required:", err) + } + clang, err = exec.LookPath("x86_64-linux-gnu-gcc") + if err != nil { + t.Fatal("x86 cross C compiler required:", err) + } + prefix = []string{qemu, "-L", "/usr/x86_64-linux-gnu"} + } else if runtime.GOARCH != "amd64" { + t.Skip("required Linux/amd64 counterpart supplies this numerical axis") + } + const source = `TEXT fpbytes(SB),4,$0-16 +VPSRLD.BCST $0,a+0(FP),X0 +MOVQ X0,AX +MOVQ AX,r+8(FP) +RET +TEXT fpcross(SB),4,$0-24 +VPSRLQ.BCST $0,a+4(FP),X0 +MOVQ X0,AX +MOVQ AX,r+16(FP) +RET +TEXT fpcount(SB),4,$0-24 +MOVQ $0x89abcdef76543210,AX +MOVQ AX,X0 +VPSRLD a+0(FP),X0,X1 +MOVQ X1,AX +MOVQ AX,r+16(FP) +RET +` + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + bytesSig := FuncSig{Name: "fpbytes", Args: []LLVMType{I8, I8, I8, I8}, Ret: I64, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I8, Index: 0, Field: -1}, {Offset: 1, Type: I8, Index: 1, Field: -1}, + {Offset: 2, Type: I8, Index: 2, Field: -1}, {Offset: 3, Type: I8, Index: 3, Field: -1}, + }, Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}}} + wideSig := FuncSig{Args: []LLVMType{I64, I64}, Ret: I64, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}, + }, Results: []FrameSlot{{Offset: 16, Type: I64, Index: 0, Field: -1}}}} + triple := testTargetTriple(runtime.GOOS, "amd64") + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"fpbytes": bytesSig, "fpcross": wideSig, "fpcount": wideSig}}) + if err != nil { + t.Fatal(err) + } + const driver = ` +#include +#include +extern uint64_t fpbytes(uint8_t,uint8_t,uint8_t,uint8_t); +extern uint64_t fpcross(uint64_t,uint64_t); +extern uint64_t fpcount(uint64_t,uint64_t); +int main(void) { + for(unsigned i=0;i<96;++i) { + uint8_t a=(uint8_t)(i*71+13),b=(uint8_t)(i*97+11); + uint8_t c=(uint8_t)(i*53+17),d=(uint8_t)(i*19+7); + uint64_t word=(uint32_t)a|((uint32_t)b<<8)|((uint32_t)c<<16)|((uint32_t)d<<24); + if(fpbytes(a,b,c,d)!=(word|(word<<32))) { fprintf(stderr,"four byte FP BCST mismatch %u\n",i); return 1; } + uint64_t first=UINT64_C(0xabcdef1276543210)^(UINT64_C(0x12345678)*i); + uint64_t second=UINT64_C(0x179bdeff23056789)^(UINT64_C(0x87654321)*i); + if(fpcross(first,second)!=((first>>32)|(second<<32))) { fprintf(stderr,"cross slot FP BCST mismatch %u\n",i); return 2; } + uint64_t count=i<40?i:UINT64_C(0x100000000)+i; + uint64_t expected=count<32?((uint64_t)(UINT32_C(0x89abcdef)>>count)<<32)|(UINT32_C(0x76543210)>>count):0; + if(fpcount(count,second)!=expected) { fprintf(stderr,"full m128 FP count mismatch %u\n",i); return 3; } + } + return 0; +} +` + compileAndRunRuntimeTestWithCompiler(t, llc, append([]string{clang}, flags...), "packed_shift_fp_span", triple, ir, driver, prefix) + if strings.Contains(ir, "inttoptr i64 0") { + t.Fatal("FP bytes were replaced by an unbound zero pointer") + } + t.Logf("three initialized FP span programs ×96 numerical inputs; original Go encoding accepted, LLVM execution transport=%v", prefix) +} diff --git a/amd64_packed_shift_literal.go b/amd64_packed_shift_literal.go new file mode 100644 index 000000000..692c95cbc --- /dev/null +++ b/amd64_packed_shift_literal.go @@ -0,0 +1,164 @@ +package plan9asm + +// One adjacent immediate logical-right shift retains bit dependencies which +// cannot be represented by the ordinary whole-byte vector domain. This is not +// a vector value, a memory permission or a fact about any other instruction. +type x86ResultShiftBits struct { + input uint64 + register, width, lane, count uint8 +} + +func x86ResultShiftBitSource(ins Instr, form x86PackedShiftForm, input uint64, destination int) x86ResultShiftBits { + if !x86ResultNamedPredicateInstruction(ins) || form.left || form.arithmetic || form.mmx || + form.mask >= 0 || form.countMode != x86PackedShiftUniform || form.countOperand.Kind != OpImm || + form.laneBits != 16 && form.laneBits != 32 && form.laneBits != 64 { + return x86ResultShiftBits{} + } + return x86ResultShiftBits{input: input, register: uint8(destination), + width: uint8(form.byteWidth), lane: uint8(form.laneBits), count: uint8(form.countOperand.Imm)} +} + +// Only emitted, immutable same-module bytes may remove an input dependency. +// Extent and the actual full native read remain independently checked by the +// ordinary logical transfer before this projection is used. +type x86ShiftLiteral struct { + bytes [64]byte + width uint8 +} + +func (pending x86ResultShiftBits) definedBytes(ins Instr, form x86PackedLogicalForm, literals map[x86StaticSBAddress]x86ShiftLiteral, arch string) uint64 { + if pending.width == 0 || form.byteWidth != int(pending.width) || form.mask >= 0 || form.memoryBytes <= 0 || form.memory != form.first || + !x86ResultNamedPredicateInstruction(ins) || form.spec.mode == amd64PackedLogicalXor { + return 0 + } + second, valid := amd64VectorRegisterIndex(ins.Args[form.second].Reg, form.byteWidth) + address, static := x86StaticSBReadAddress(form.source, arch) + literal := literals[address] + if !valid || second != int(pending.register) || !static || int(literal.width) < form.memoryBytes { + return 0 + } + var known uint64 + for at := 0; at < form.byteWidth; at++ { + mask := x86PackedLogicalSecondBits(form.spec.mode, literal.bytes[at%form.memoryBytes]) + defined := true + for bit := 0; bit < 8; bit++ { + if mask&(1<= int(pending.lane) { + continue // Logical right shifts supply zero above the lane. + } + input := at*8/int(pending.lane)*int(pending.lane)/8 + inLane/8 + if pending.input&(uint64(1)< 64 { + continue + } + address, valid := x86StaticSBReadAddress(form.source, arch) + if !valid || address.offset > maxDataGlobalSize-int64(form.memoryBytes) { + continue + } + if requests[address.symbol] == nil { + requests[address.symbol] = make(map[int64]int) + } + if requests[address.symbol][address.offset] < form.memoryBytes { + requests[address.symbol][address.offset] = form.memoryBytes + } + } + } + return requests +} + +// Snapshot only requested tuples (at most one Z register), never a large global +// allocation. The existing static directory rejects coalesced DATA definitions, +// overlapping ranges and code/signature namespaces. A unique valid readonly +// GLOBL and a relocation-free object are additional literal requirements. +func x86LoweredShiftLiterals(file *File, resolve func(string) string, values *x86StaticAddressValues, arch string) map[x86StaticSBAddress]x86ShiftLiteral { + if file == nil || values == nil || resolve == nil { + return nil + } + needed := false + for _, function := range file.Funcs { + for _, ins := range function.Instrs { + form, known, err := x86PackedShiftProperties(ins.Op, ins, arch) + needed = needed || known && err == nil && x86ResultShiftBitSource(ins, form, 0, 0).width != 0 + } + } + if !needed { + return nil + } + type object struct { + globals int + literal bool + data []DataStmt + } + objects := make(map[string]*object) + objectFor := func(symbol string) *object { + name := resolveAssemblyDataSymbol(symbol, resolve) + if objects[name] == nil { + objects[name] = &object{literal: values.objects[name] && !values.writable[name]} + } + return objects[name] + } + for _, global := range file.Globl { + current := objectFor(global.Sym) + readonly, err := globlReadOnly(global.Flags) + current.globals++ + current.literal = current.literal && err == nil && readonly + } + for _, datum := range file.Data { + current := objectFor(datum.Sym) + current.literal = current.literal && datum.Addr == "" && int64(len(datum.Payload)) <= datum.Width + current.data = append(current.data, datum) + } + literals := make(map[x86StaticSBAddress]x86ShiftLiteral) + for symbol, offsets := range x86PackedLogicalLiteralRequests(file, arch) { + current := objects[resolveAssemblyDataSymbol(symbol, resolve)] + if current == nil || !current.literal || current.globals != 1 { + continue + } + for offset, width := range offsets { + address := x86StaticSBAddress{symbol, offset} + literal := x86ShiftLiteral{width: uint8(width)} + for _, datum := range current.data { + for at := 0; at < width; at++ { + offset := address.offset + int64(at) - datum.Off + if offset < 0 || offset >= datum.Width { + continue + } + // Same little-endian, sign-extended numeric/padded string + // bytes as dataStmtPayload, without allocating datum.Width. + if datum.Payload != nil { + if offset < int64(len(datum.Payload)) { + literal.bytes[at] = datum.Payload[offset] + } + } else if offset < 8 { + literal.bytes[at] = byte(datum.Value >> uint(offset*8)) + } else if datum.Value < 0 { + literal.bytes[at] = 0xff + } + } + } + literals[address] = literal + } + } + return literals +} diff --git a/amd64_packed_shift_literal_binding_test.go b/amd64_packed_shift_literal_binding_test.go new file mode 100644 index 000000000..fced86772 --- /dev/null +++ b/amd64_packed_shift_literal_binding_test.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86PackedShiftLiteralBindingAndStateLifetime(t *testing.T) { + for _, mutation := range []string{"control", "missing-global", "mutable", "duplicate-global", + "relocation", "overlap", "payload-too-long", "code-collision", "resolved-alias"} { + t.Run(mutation, func(t *testing.T) { + file, err := Parse(ArchAMD64, x86ShiftLiteralMaskSource(x86ShiftLiteralMaskCases()[0], "PAND", 4, 0x0f, 0)) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { + symbol = strings.TrimPrefix(symbol, "·") + if symbol == "alias" { + return "mask<>" + } + return symbol + } + switch mutation { + case "missing-global": + file.Globl = nil + case "mutable": + file.Globl[0].Flags = "0" + case "duplicate-global": + file.Globl = append(file.Globl, file.Globl[0]) + case "relocation": + file.Data[0].Addr = "·other" + case "overlap": + file.Data = append(file.Data, file.Data[0]) + case "payload-too-long": + file.Data[0].Payload = make([]byte, 9) + case "code-collision": + file.Funcs = append(file.Funcs, Func{Sym: "mask<>"}) + case "resolved-alias": + file.Data[1].Sym = "alias" + } + values := x86LoweredStaticAddressValues(file, resolve, nil, "amd64") + literals := x86LoweredShiftLiterals(file, resolve, values, "amd64") + if mutation != "control" { + if len(literals) != 0 { + t.Fatal("unbound, mutable or aliased object supplied literal bits", literals) + } + return + } + literal := literals[x86StaticSBAddress{symbol: "mask<>"}] + if len(literals) != 1 || literal.width != 16 { + t.Fatal("complete actual readonly tuple was lost", literals) + } + for _, value := range literal.bytes[:literal.width] { + if value != 0x0f { + t.Fatal("literal bits disagree with actual emitted DATA", literal) + } + } + }) + } + pending := x86ResultShiftBits{input: 1, register: 3, width: 64, lane: 16, count: 4} + state := x86ResultState{shiftBits: pending} + if state.clone().shiftBits != pending || state.intersect(state).shiftBits != pending { + t.Fatal("identical adjacent dependency was not value-copied") + } + other := state.clone() + other.shiftBits.input = 3 + if state.intersect(other).shiftBits.width != 0 || state.intersect(x86ResultState{}).shiftBits.width != 0 { + t.Fatal("different predecessor shifts were joined as one fact") + } +} + +func TestX86PackedShiftRawKeepsNamedBitDependenciesSeparate(t *testing.T) { + pending := x86ResultShiftBits{input: 1, register: 3, width: 64, lane: 16, count: 4} + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86PackedShiftRawCases() { + t.Run(arch+"/"+row.name(), func(t *testing.T) { + ins := x86RawMoveObservedInstruction(t, arch, row.code(false)) + for _, knownBytes := range []uint64{1, ^uint64(0)} { + state := x86ResultState{shiftBits: pending} + for index := 0; index < 3; index++ { + state.writeVector(index, 64, knownBytes, false) + } + state.pointerGP[3], state.masks[1] = true, true + accepted, known := x86LoweredRawPackedShiftStep(ins, &state, nil, arch) + if !known || !accepted || state.shiftBits != pending { + t.Fatal("raw projection created or replaced a named-only bit dependency", accepted, known) + } + } + }) + } + } + + state := x86ResultState{} + state.writeVector(1, 16, 1, true) + ins := x86ScalarLaneInstruction(t, "VPSRLW $4,X1,X0") + accepted, known := x86ResultPackedShiftStepWithRead(ins, &state, nil, "amd64", true) + want := x86ResultShiftBits{input: 1, register: 0, width: 16, lane: 16, count: 4} + if !accepted || !known || state.shiftBits != want { + t.Fatal("genuine named input lost its adjacent bit dependency", accepted, known, state.shiftBits) + } +} diff --git a/amd64_packed_shift_literal_mask_test.go b/amd64_packed_shift_literal_mask_test.go new file mode 100644 index 000000000..6b0cc1cd7 --- /dev/null +++ b/amd64_packed_shift_literal_mask_test.go @@ -0,0 +1,213 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +type x86ShiftLiteralMaskCase struct { + shift, bank string + width int + vector bool +} + +// Independent of the production shift registry: all W/D/Q logical-right +// families, with legacy X and VEX/EVEX X/Y/Z carriers. +func x86ShiftLiteralMaskCases() []x86ShiftLiteralMaskCase { + var rows []x86ShiftLiteralMaskCase + for _, pair := range [][2]string{{"PSRLW", "VPSRLW"}, {"PSRLL", "VPSRLD"}, {"PSRLQ", "VPSRLQ"}} { + rows = append(rows, x86ShiftLiteralMaskCase{shift: pair[0], bank: "X", width: 16}) + for _, bank := range []struct { + name string + width int + }{{"X", 16}, {"Y", 32}, {"Z", 64}} { + rows = append(rows, x86ShiftLiteralMaskCase{shift: pair[1], bank: bank.name, width: bank.width, vector: true}) + } + } + return rows +} + +func x86ShiftLiteralMaskSource(row x86ShiftLiteralMaskCase, logical string, shift int, mask byte, extract int) string { + var source strings.Builder + value := uint64(mask) * 0x0101010101010101 + for at := 0; at < row.width; at += 8 { + fmt.Fprintf(&source, "DATA mask<>+%d(SB)/8,$%#x\n", at, value) + } + fmt.Fprintf(&source, "GLOBL mask<>(SB),8,$%d\n", row.width) + source.WriteString("TEXT ·view(SB),4,$0-12\nMOVQ input+0(FP),SI\nPINSRB $0,(SI),X0\n") + // Deliberately leave every other byte of X0/Y0/Z0 unknown. Initializing + // the whole bank would hide the missing bit-to-byte dependency. + destination := "X0" + if row.vector { + destination = "X1" + fmt.Fprintf(&source, "%s $%d,%s0,%s1\n", row.shift, shift, row.bank, row.bank) + logical = "V" + logical + if row.bank == "Z" { + logical += "D" + } + fmt.Fprintf(&source, "%s mask<>(SB),%s1,%s1\n", logical, row.bank, row.bank) + } else { + fmt.Fprintf(&source, "%s $%d,X0\n%s mask<>(SB),X0\n", row.shift, shift, logical) + } + fmt.Fprintf(&source, "PEXTRB $%d,%s,AX\nMOVL AX,ret+8(FP)\nRET\n", extract, destination) + return source.String() +} + +func TestX86PackedLogicalRightShiftLiteralMaskDefinedBits(t *testing.T) { + for _, row := range x86ShiftLiteralMaskCases() { + for _, shift := range []int{0, 1, 4, 7} { + for _, logical := range []string{"PAND", "PANDN", "POR"} { + mask := byte(255 >> uint(shift)) + if logical == "POR" { + mask = ^mask // OR's known ones remove the same unknown bits. + } + source := x86ShiftLiteralMaskSource(row, logical, shift, mask, 0) + for _, api := range []string{"text", "module", "Go"} { + t.Run(fmt.Sprintf("%s/%s/%s/%d/%s", row.shift, row.bank, logical, shift, api), func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(input *byte) uint32\n", + "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil || ir == "" { + t.Fatalf("defined selected bits lost after literal mask: output=%t err=%v", ir != "", err) + } + }) + } + } + } + } +} + +func TestX86PackedShiftLiteralMaskKeepsUnknownBits(t *testing.T) { + for _, row := range x86ShiftLiteralMaskCases() { + for _, mutation := range []struct { + name, logical string + shift, extract int + mask byte + }{ + {"one-unknown-bit", "PAND", 4, 0, 0x1f}, + {"unmasked-unknown-bits", "PAND", 4, 0, 0xff}, + {"shift-needs-next-byte", "PAND", 5, 0, 0x0f}, + {"unused-byte-stays-unknown", "PAND", 4, 1, 0x0f}, + {"xor-does-not-kill-dependencies", "PXOR", 4, 0, 0x0f}, + {"or-leaves-unknown-bit", "POR", 4, 0, 0xe0}, + } { + source := x86ShiftLiteralMaskSource(row, mutation.logical, mutation.shift, mutation.mask, mutation.extract) + for _, api := range []string{"text", "module", "Go"} { + t.Run(row.shift+"/"+row.bank+"/"+mutation.name+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(input *byte) uint32\n", + "amd64", "x86_64-unknown-linux-gnu", api) + x86VectorScratchCodegenControl(t, source, "func view(input *byte) uint32\n", + "amd64", "x86_64-unknown-linux-gnu", ir, err) + }) + } + } + } +} + +func TestX86PackedShiftLiteralMaskKeepsSourceAndMemoryBoundaries(t *testing.T) { + for _, row := range x86ShiftLiteralMaskCases() { + source := x86ShiftLiteralMaskSource(row, "PAND", 4, 0x0f, 0) + logical := "PAND" + if row.vector { + logical = "VPAND" + if row.bank == "Z" { + logical += "D" + } + } + maskLine := logical + " mask<>(SB)," + for _, mutation := range []struct{ name, source string }{ + {"unknown-native-base", strings.Replace(source, "MOVQ input+0(FP),SI\n", "", 1)}, + {"unknown-selected-byte", strings.Replace(source, "PINSRB $0,(SI),X0\n", "", 1)}, + {"unknown-index", strings.Replace(source, "(SI),X0", "(SI)(CX*1),X0", 1)}, + // The original mutation first renames GLOBL, leaving the DATA + // and the actual read at mask<>. Keep that source/case intact. + {"unknown-literal", strings.Replace(source, " mask<>(SB),", " external(SB),", 1)}, + {"unbound-read-symbol", strings.Replace(source, maskLine, logical+" external(SB),", 1)}, + {"mutable-mask", strings.Replace(source, "GLOBL mask<>(SB),8,", "GLOBL mask<>(SB),0,", 1)}, + {"short-literal", strings.Replace(source, maskLine, logical+" mask<>+1(SB),", 1)}, + {"intervening-nop", strings.Replace(source, maskLine, "NOP\n"+maskLine, 1)}, + {"intervening-other-register", strings.Replace(source, maskLine, "PINSRB $0,(SI),X2\n"+maskLine, 1)}, + {"intervening-branch", strings.Replace(source, maskLine, "JMP masked\nmasked:\n"+maskLine, 1)}, + } { + for _, api := range []string{"text", "module", "Go"} { + t.Run(row.shift+"/"+row.bank+"/"+mutation.name+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, mutation.source, "func view(input *byte) uint32\n", + "amd64", "x86_64-unknown-linux-gnu", api) + if mutation.name == "unbound-read-symbol" { + prefix, _, _ := strings.Cut(source, "TEXT ") + changedPrefix, _, _ := strings.Cut(mutation.source, "TEXT ") + if prefix != changedPrefix || strings.Count(mutation.source, logical+" external(SB),") != 1 { + t.Fatal("unbound read must change exactly the instruction, not DATA/GLOBL") + } + } + switch mutation.name { + case "unknown-literal", "mutable-mask": + x86ShiftMutableMaskCodegenControl(t, mutation.source, row.width, api, ir, err) + return + case "unknown-selected-byte", "intervening-nop", "intervening-other-register", "intervening-branch": + x86VectorScratchCodegenControl(t, mutation.source, "func view(input *byte) uint32\n", + "amd64", "x86_64-unknown-linux-gnu", ir, err) + return + } + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("missing source/readonly/address role was manufactured: output=%t err=%v", ir != "", err) + } + }) + } + } + } +} + +// An actual read of complete mutable DATA is separate from a literal that may +// eliminate an input dependency. Keep native effects through O2; an unwritten +// private object's ordinary load may be folded by the optimizer. +func x86ShiftMutableMaskCodegenControl(t *testing.T, source string, width int, api, ir string, codegenErr error) { + t.Helper() + const declaration = "func view(input *byte) uint32\n" + const triple = "x86_64-unknown-linux-gnu" + x86VectorScratchCodegenControl(t, source, declaration, "amd64", triple, ir, codegenErr) + + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := testResolveSym("scratch") + values := x86LoweredStaticAddressValues(file, resolve, nil, "amd64") + literals := x86LoweredShiftLiterals(file, resolve, values, "amd64") + if literal := literals[x86StaticSBAddress{symbol: "mask<>"}]; literal.width != 0 { + t.Fatal("mutable or undeclared GLOBL acquired immutable mask bytes") + } + + const load = "MOVQ input+0(FP),SI\n" + if strings.Count(source, load) != 1 { + t.Fatal("the native-read control lost its one input initialization") + } + initialized := strings.Replace(source, load, "VPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\n"+load, 1) + before, err := x86ScalarFPBitViewAPI(t, initialized, declaration, "amd64", triple, api) + if err != nil || before == "" { + t.Fatal("explicit-vector native-read control failed", err) + } + wantGlobal := fmt.Sprintf("@mask = internal global [%d x i8] c\"%s\"", width, strings.Repeat("\\0F", width)) + if !strings.Contains(ir, wantGlobal) { + t.Fatal("the read mask lost its complete mutable DATA initializer", width) + } + legacyRead := strings.Contains(ir, "asm sideeffect \"pand (") + if legacyRead { + if width != 16 || !strings.Contains(ir, "~{memory}") { + t.Fatal("legacy mask lost its original m128 PAND") + } + } else if strings.Count(ir, fmt.Sprintf("load <%d x i8>, ptr @mask, align 1", width)) != 1 { + t.Fatal("the owned mask lost its complete ordinary read", width) + } + if x86NumericNativeVolatileReadBytes(t, ir) != 1 { + t.Fatal("the independent PINSRB byte read was lost") + } + x86RequireSameNativeMemoryEffects(t, before, ir) + optimized := x86DwordProductIROptimize(t, ir) + if x86NumericNativeVolatileReadBytes(t, optimized) != 1 { + t.Fatal("optimization lost the independent PINSRB byte read") + } + x86RequireSameNativeMemoryEffects(t, before, optimized) + x86NumericNativeAddressObject(t, optimized, triple) +} diff --git a/amd64_packed_shift_literal_objects_test.go b/amd64_packed_shift_literal_objects_test.go new file mode 100644 index 000000000..634125ab6 --- /dev/null +++ b/amd64_packed_shift_literal_objects_test.go @@ -0,0 +1,67 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86ShiftLiteralKernel struct { + name string + shift, mode int + mask byte +} + +// Keep every form in one complete source, with distinct function and DATA +// identities. The legacy subset is the same source used by the native oracle. +func x86ShiftLiteralMaskProgram(legacyOnly bool) (string, string, []x86ShiftLiteralKernel) { + var source, declarations strings.Builder + var kernels []x86ShiftLiteralKernel + for _, row := range x86ShiftLiteralMaskCases() { + if legacyOnly && row.vector { + continue + } + for _, shift := range []int{0, 1, 4, 7} { + for mode, logical := range []string{"PAND", "PANDN", "POR"} { + name := fmt.Sprint("shiftMask", len(kernels)) + mask := byte(255 >> uint(shift)) + if mode == 2 { + mask = ^mask + } + assembly := x86ShiftLiteralMaskSource(row, logical, shift, mask, 0) + assembly = strings.ReplaceAll(assembly, "mask<>", name+"Mask<>") + assembly = strings.ReplaceAll(assembly, "·view", "·"+name) + source.WriteString(assembly) + fmt.Fprintf(&declarations, "func %s(input *byte) uint32\n", name) + kernels = append(kernels, x86ShiftLiteralKernel{name, shift, mode, mask}) + } + } + } + return source.String(), declarations.String(), kernels +} + +func TestX86PackedShiftLiteralMaskCompleteThreeAPIObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + source, declarations, kernels := x86ShiftLiteralMaskProgram(false) + if len(kernels) != 144 { + t.Fatal("incomplete W/D/Q × legacy/VEX/EVEX × shift × logical union", len(kernels)) + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, "amd64", triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "shift-mask.ll", "shift-mask.o", ir) + }) + } + }) + } + t.Log("144 complete functions × 3 APIs × Linux/Darwin/Windows LLVM22 objects; no physical AVX512 execution claim") +} diff --git a/amd64_packed_shift_literal_runtime_test.go b/amd64_packed_shift_literal_runtime_test.go new file mode 100644 index 000000000..9fab7ad6a --- /dev/null +++ b/amd64_packed_shift_literal_runtime_test.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "fmt" + "go/format" + "os" + "path/filepath" + "strings" + "testing" +) + +// Native legacy SSE is the independent counterpart. Virtual Y/Z object checks +// above do not stand in for executing hardware AVX512. The existing execution +// route owns explicit Linux QEMU or a truthful host-inapplicable result. +func TestCrossLinuxRuntimeMatrixX86PackedShiftLiteralMasksOriginalGoAndLLVM(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + source, declarations, kernels := x86ShiftLiteralMaskProgram(true) + if len(kernels) != 36 { + t.Fatal("incomplete original legacy W/D/Q × 3 logical × 4 shift union", len(kernels)) + } + var goCases, cDeclarations, cCases strings.Builder + for _, kernel := range kernels { + fmt.Fprintf(&goCases, "{%q,%s,%d,%d,%d},\n", kernel.name, kernel.name, kernel.shift, kernel.mode, kernel.mask) + fmt.Fprintf(&cDeclarations, "extern uint32_t %s(const uint8_t *);\n", kernel.name) + fmt.Fprintf(&cCases, "{%q,%s,%d,%d,%d},\n", kernel.name, kernel.name, kernel.shift, kernel.mode, kernel.mask) + } + program := `package main + +import "fmt" + +` + declarations + ` +type kernel struct { + name string + call func(*byte) uint32 + shift uint + mode int + mask byte +} + +var kernels = []kernel{ +` + goCases.String() + `} + +func main() { + for _, test := range kernels { + for sample := 0; sample < 256; sample++ { + input := [3]byte{0xa5, byte(sample), 0x5a} + before := input + want := byte(sample) >> test.shift + switch test.mode { + case 0: + want &= test.mask + case 1: + want = ^want & test.mask + case 2: + want |= test.mask + } + got := test.call(&input[1]) + if got != uint32(want) || input != before { + panic(fmt.Sprintf("%s sample=%d got=%x want=%x input=%x", test.name, sample, got, want, input)) + } + } + } + fmt.Println("original legacy shift masks: 36x256 PASS") +} +` + formatted, err := format.Source([]byte(program)) + if err != nil { + t.Fatal(err) + } + directory := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module original_shift_masks\n\ngo 1.20\n", + "main.go": string(formatted), "masks_amd64.s": source, + } { + if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + goos := "linux" + if strings.Contains(execution.triple, "apple") { + goos = "darwin" + } else if strings.Contains(execution.triple, "windows") { + goos = "windows" + } + original := filepath.Join(directory, "original") + if goos == "windows" { + original += ".exe" + } + environment := []string{"GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "CGO_ENABLED=0", "GOARCH=amd64", "GOOS=" + goos} + dataGlobalAlignmentCommand(t, directory, environment, "go", "build", "-p=2", "-o", original, ".") + if output := string(execution.run(t, original)); strings.TrimSpace(output) != "original legacy shift masks: 36x256 PASS" { + t.Fatal("missing actual original-Go oracle completion", output) + } + main := `#include +#include +#include +` + cDeclarations.String() + ` +struct Case { + const char *name; + uint32_t (*call)(const uint8_t *); + unsigned shift, mode; + uint8_t mask; +}; +static const struct Case cases[] = { +` + cCases.String() + `}; + +int main(void) { + for (unsigned c = 0; c < sizeof(cases)/sizeof(cases[0]); c++) { + const struct Case *test = cases + c; + for (unsigned sample = 0; sample < 256; sample++) { + uint8_t input[3] = {0xa5, (uint8_t)sample, 0x5a}; + uint8_t before[3]; + memcpy(before, input, sizeof(input)); + uint8_t want = (uint8_t)sample >> test->shift; + if (test->mode == 0) want &= test->mask; + if (test->mode == 1) want = (uint8_t)~want & test->mask; + if (test->mode == 2) want |= test->mask; + uint32_t got = test->call(input + 1); + if (got != want || memcmp(input, before, sizeof(input))) { + fprintf(stderr, "%s sample=%u got=%x want=%x\n", test->name, sample, got, (unsigned)want); + return 1; + } + } + } + puts("LLVM legacy shift masks: 36x256 PASS"); + return 0; +} +` + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(api+"/"+level, func(t *testing.T) { + if output := string(execution.compileAndRunLLVM(t, "shift-literal-masks", ir, main, level)); strings.TrimSpace(output) != "LLVM legacy shift masks: 36x256 PASS" { + t.Fatal("missing actual LLVM oracle completion", output) + } + }) + } + } + t.Log("36 original Go legacy SSE kernels × all 256 byte values; independent C byte oracle; 3 APIs × llc O0/O2; other vector bytes intentionally not initialized") +} diff --git a/amd64_packed_shift_mode32_test.go b/amd64_packed_shift_mode32_test.go new file mode 100644 index 000000000..076e5343f --- /dev/null +++ b/amd64_packed_shift_mode32_test.go @@ -0,0 +1,97 @@ +package plan9asm + +import ( + "encoding/hex" + "errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86PackedShiftActualGoBytes(t *testing.T, arch, source string) []byte { + t.Helper() + dir := t.TempDir() + input := filepath.Join(dir, "original.s") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-o", filepath.Join(dir, "original.o"), input) + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch, "GOTOOLCHAIN=local") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("actual Go %s original source: %v\n%s", arch, err, output) + } + var code []byte + for _, line := range regexp.MustCompile(`(?m)^\s*0x[[:xdigit:]]+\s+((?:[[:xdigit:]]{2} )+)`).FindAllSubmatch(output, -1) { + part, err := hex.DecodeString(strings.ReplaceAll(string(line[1]), " ", "")) + if err != nil { + t.Fatal(err) + } + code = append(code, part...) + } + return code +} + +func TestX86PackedShiftMode32NamedPhysicalBanks(t *testing.T) { + for _, row := range []string{ + "VPSRLW $1,X20,X21", "VPSRLD X0,Y20,Y21", "VPSRLQ (AX),X20,X21", + "VPSRLVW X1,X20,X21", "VPSRLVD.BCST (AX),Y20,Y21", "VPSRLVQ X0,X20,X21", + "VPSLLDQ $1,X16,X31", "VPSRLDQ $1,Y20,Y21", + } { + source := "TEXT shifts(SB),4,$0-0\n" + row + "\nRET\n" + code := x86PackedShiftActualGoBytes(t, "386", source) + _, _, recognized, err := decodedX86PackedUniformShiftInstruction(code, 32) + if !recognized { + _, _, recognized, err = decodedX86PerLaneVariableShiftInstruction(code, 32) + } + if !recognized { + _, _, recognized, err = decodedX86RawVectorByteShiftInstruction(code, 32) + } + if !recognized || err == nil { + t.Fatalf("mode32 raw bytes %x for %s did not reject unavailable physical bank: %v", code, row, err) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + options := Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"shifts": {Ret: Void}}} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s named unavailable SIMD bank %s: %v", triple, row, err) + } + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s module unavailable SIMD bank %s: %v", triple, row, err) + } + result, err := translateGoModuleInContext(context, x86PackedShiftGoPackage(t, "386"), + []byte(strings.Replace(source, "shifts(SB)", "·shifts(SB)", 1)), GoModuleOptions{GOARCH: "386", TargetTriple: triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s Go module unavailable SIMD bank %s: %v", triple, row, err) + } + }() + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "386" + before := output.Len() + ins := file.Funcs[0].Instrs[1] + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || output.Len() != before { + t.Fatalf("unavailable mode32 SIMD bank emitted IR: %s %v", row, err) + } + } + } +} diff --git a/amd64_packed_shift_native_count.go b/amd64_packed_shift_native_count.go new file mode 100644 index 000000000..d6188aee7 --- /dev/null +++ b/amd64_packed_shift_native_count.go @@ -0,0 +1,35 @@ +package plan9asm + +import "fmt" + +func x86PackedShiftHasNativeCount(form x86PackedShiftForm) bool { + return form.legacy && !form.mmx && form.memory == form.count +} + +// Preserve the original SSE2 m128 operation, including its complete native +// read and fault behavior. Substituting MOVDQA would impose its alignment +// contract on a different instruction. The tied data input, vector output +// and memory clobber keep the original operation observable without granting +// the source address any permission or claiming an alignment guarantee. +func (c *amd64Ctx) emitX86PackedShiftNativeCount(form x86PackedShiftForm, source Operand, data string) (string, error) { + if !x86PackedShiftHasNativeCount(form) || form.byteWidth != 16 || form.countMode != x86PackedShiftUniform { + return "", fmt.Errorf("expected legacy SSE m128 packed count form") + } + name := "psrl" + if form.left { + name = "psll" + } else if form.arithmetic { + name = "psra" + } + switch form.laneBits { + case 16: + name += "w" + case 32: + name += "d" + case 64: + name += "q" + default: + return "", fmt.Errorf("unsupported legacy packed count lane width %d", form.laneBits) + } + return c.emitPackedBinaryNativeMemory(source, form.laneBits, data, name) +} diff --git a/amd64_packed_shift_native_count_linux_test.go b/amd64_packed_shift_native_count_linux_test.go new file mode 100644 index 000000000..6eef222c6 --- /dev/null +++ b/amd64_packed_shift_native_count_linux_test.go @@ -0,0 +1,46 @@ +//go:build linux && (amd64 || arm64) + +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func x86PackedShiftBuildOriginalCountObserver(t *testing.T, compiler []string) string { + t.Helper() + const main = ` +#include +int main(int argc,char **argv) { + if(argc!=2) return 97; + int kernel=atoi(argv[1]); + if(kernel<0 || kernel>=6) return 98; + _Alignas(16) uint8_t storage[96]; + for(int i=0;i<96;i++) storage[i]=(uint8_t)(i*31+7); + const uint8_t *count=storage+1; + if((uintptr_t)count%16!=1) return 99; + int status=observe_count(originals[kernel],kernel,0,0,count,0); + if(WIFEXITED(status) && WEXITSTATUS(status)==0) + printf("COUNT_NATIVE_STATUS kernel=%d mod16=%llu wait=%d outcome=normal\n",kernel,(unsigned long long)((uintptr_t)count%16),status); + else if(WIFSIGNALED(status) && WTERMSIG(status)==SIGSEGV) + printf("COUNT_NATIVE_STATUS kernel=%d mod16=%llu wait=%d outcome=fault\n",kernel,(unsigned long long)((uintptr_t)count%16),status); + else { fprintf(stderr,"original count status=%x\n",status); return 100; } + return 0; +} +` + dir := t.TempDir() + input, executable := filepath.Join(dir, "original.c"), filepath.Join(dir, "original-count-observer") + if err := os.WriteFile(input, []byte(x86PackedShiftLegacyCountCHeader+x86PackedShiftOriginalCountC()+x86PackedShiftCountCChecks+main), 0600); err != nil { + t.Fatal(err) + } + arguments := append(append([]string(nil), compiler[1:]...), "-O2", input, "-o", executable) + ctx, cancel := runtimeOracleStageContext(t) + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, compiler[0], arguments...)) + cancel() + if err != nil { + t.Fatalf("original opcode status observer: %v\n%s", err, output) + } + return executable +} diff --git a/amd64_packed_shift_native_count_runtime_test.go b/amd64_packed_shift_native_count_runtime_test.go new file mode 100644 index 000000000..900588393 --- /dev/null +++ b/amd64_packed_shift_native_count_runtime_test.go @@ -0,0 +1,232 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86PackedShiftLegacyNativeStatusSemantics(t *testing.T) { + var prefix []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + if !rosettaAvailable() { + t.Fatal("Darwin x86 diagnostic transport required") + } + prefix = []string{"/usr/bin/arch", "-x86_64"} + } else if runtime.GOARCH != "amd64" || runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("required Linux original/LLVM native-count counterpart supplies this execution axis") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + assembly, sigs := x86PackedShiftLegacyMemoryAssembly() + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, "amd64") + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + driver := x86PackedShiftLegacyCountCHeader + x86PackedShiftLegacyCountDeclarations() + + x86PackedShiftOriginalCountC() + x86PackedShiftCountCChecks + x86PackedShiftLegacyNativeCountCMain + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy_count_status", triple, ir, driver, prefix) + t.Logf("6 original-opcode/LLVM kernels ×48 numerical inputs, 90 unaligned status comparisons, 6 full128 guard-page fault pairs; transport=%v", prefix) +} + +func x86PackedShiftLegacyCountDeclarations() string { + var source strings.Builder + for i := range x86LegacyShiftMemoryOpcodes { + fmt.Fprintf(&source, "extern void legacycount%d(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", i) + } + source.WriteString("static legacy_fn kernels[]={legacycount0,legacycount1,legacycount2,legacycount3,legacycount4,legacycount5};\n") + return source.String() +} + +func x86PackedShiftLegacyMemoryAssembly() (string, map[string]FuncSig) { + var source strings.Builder + sigs := map[string]FuncSig{} + for i, operation := range x86LegacyShiftMemoryOpcodes { + name := fmt.Sprintf("legacycount%d", i) + fmt.Fprintf(&source, `TEXT %s(SB),4,$0-32 +MOVQ out+0(FP),AX +MOVQ data+8(FP),BX +MOVQ count+16(FP),CX +MOVQ seed+24(FP),DX +MOVOU (BX),X0 +CMPQ DX,$0 +JE carry +CMPQ DX,$1 +JE overflow +MOVQ $1,DX +CMPQ DX,$0 +JMP shifted +carry: +MOVQ $-1,DX +ADDQ $1,DX +JMP shifted +overflow: +MOVQ $0x7fffffffffffffff,DX +ADDQ $1,DX +shifted: +%s (CX),X0 +MOVOU X0,(AX) +SETCS 16(AX) +SETOS 17(AX) +SETEQ 18(AX) +SETMI 19(AX) +SETPS 20(AX) +RET +`, name, operation.name) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + } + return source.String(), sigs +} + +const x86PackedShiftLegacyCountCHeader = ` +#include +#include +#include +#include +#include +#include +#include +#include +typedef void (*legacy_fn)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t); +` + +// This independent reference executes the original six physical opcodes. The +// Go object oracle binds each corresponding mnemonic to its original bytes; +// neither this reference nor the math oracle substitutes a different load. +func x86PackedShiftOriginalCountC() string { + var source strings.Builder + for i, mnemonic := range []string{"psllw", "pslld", "psllq", "psrlw", "psrld", "psrlq"} { + fmt.Fprintf(&source, ` +static void original%d(uint8_t *out,const uint8_t *data,const uint8_t *count,uint64_t seed) { + __asm__ volatile( + "movdqu (%%[data]),%%%%xmm0\n\t" + "cmpq $0,%%[seed]\n\tje 1f\n\t" + "cmpq $1,%%[seed]\n\tje 2f\n\t" + "movq $1,%%%%rax\n\tcmpq $0,%%%%rax\n\tjmp 3f\n\t" + "1: movq $-1,%%%%rax\n\taddq $1,%%%%rax\n\tjmp 3f\n\t" + "2: movq $0x7fffffffffffffff,%%%%rax\n\taddq $1,%%%%rax\n\t" + "3: %s (%%[count]),%%%%xmm0\n\t" + "movdqu %%%%xmm0,(%%[out])\n\t" + "setb 16(%%[out])\n\tseto 17(%%[out])\n\tsete 18(%%[out])\n\t" + "sets 19(%%[out])\n\tsetp 20(%%[out])\n\t" + : : [out] "r"(out),[data] "r"(data),[count] "r"(count),[seed] "r"(seed) + : "rax","xmm0","cc","memory"); +} +`, i, mnemonic) + } + source.WriteString("static legacy_fn originals[]={original0,original1,original2,original3,original4,original5};\n") + return source.String() +} + +// Status observation is independent of the translated implementation. A +// successful child must still satisfy all numerical, flag and canary checks. +const x86PackedShiftCountCChecks = ` +static uint64_t read_le(const uint8_t *p,int bytes) { + uint64_t value=0; + for(int i=0;i>shift; + for(int b=0;b>(8*b))) return 20+kernel; + } + return 0; +} +static int observe_count(legacy_fn fn,int kernel,int round,int seed,const uint8_t *count,int guard) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + + if(guard) { + uint8_t data[18],out[23]; + memset(data,13,18); memset(out,0xc3,23); + fn(out+1,data+1,count,0); + _exit(90); + } + _exit(check_count(fn,kernel,round,seed,count)); + } + int status=0; + if(waitpid(child,&status,0)!=child) return -1; + return status; +} +` + +const x86PackedShiftLegacyNativeCountCMain = ` +int main(void) { + const uint64_t counts[]={0,1,7,15,16,17,31,32,33,63,64,65,255,256,UINT64_C(1)<<32,UINT64_MAX}; + for(int kernel=0;kernel<6;kernel++) { + for(int round=0;round<16;round++) for(int seed=0;seed<3;seed++) { + _Alignas(16) uint8_t count[32]; + for(int i=0;i<32;i++) count[i]=(uint8_t)(i*31+7); + for(int i=0;i<8;i++) count[i]=(uint8_t)(counts[round]>>(8*i)); + int original=check_count(originals[kernel],kernel,round,seed,count); + int translated=check_count(kernels[kernel],kernel,round,seed,count); + if(original || translated) return 10+kernel; + } + for(int offset=1;offset<16;offset++) { + _Alignas(16) uint8_t count[48]; + memset(count,0x79,sizeof(count)); + memset(count+offset,0,8); count[offset]=1; + int original=observe_count(originals[kernel],kernel,offset,offset%3,count+offset,0); + int translated=observe_count(kernels[kernel],kernel,offset,offset%3,count+offset,0); + int normal=WIFEXITED(original) && WEXITSTATUS(original)==0; + int fault=count_memory_fault(original); + if(original<0 || translated<0 || (!normal && !fault) || original!=translated) { + fprintf(stderr,"legacy kernel=%d mod16=%d original=%x translated=%x\n",kernel,offset,original,translated); + return 30+kernel; + } + } + } + size_t page=(size_t)sysconf(_SC_PAGESIZE); + uint8_t *area=mmap(0,page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(area==MAP_FAILED || mprotect(area+page,page,PROT_NONE)) return 40; + memset(area,7,page); + for(int kernel=0;kernel<6;kernel++) { + int original=observe_count(originals[kernel],kernel,0,0,area+page-8,1); + int translated=observe_count(kernels[kernel],kernel,0,0,area+page-8,1); + if(original<0 || translated<0 || !count_memory_fault(original) || original!=translated) { + fprintf(stderr,"legacy full128 kernel=%d original=%x translated=%x\n",kernel,original,translated); + return 50+kernel; + } + } + if(munmap(area,page*2)) return 60; + return 0; +} +` diff --git a/amd64_packed_shift_raw_format_test.go b/amd64_packed_shift_raw_format_test.go new file mode 100644 index 000000000..413eda71c --- /dev/null +++ b/amd64_packed_shift_raw_format_test.go @@ -0,0 +1,23 @@ +package plan9asm + +import ( + "errors" + "testing" +) + +// Preserve original bytes, return and literal DATA. These isolated operand +// objects do not supply native entry inputs, caller memory or a return proof. +func x86PackedShiftRawFormatObject(t *testing.T, file *File, options Options) string { + t.Helper() + if ir, err := Translate(file, options); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original raw shift acquired unbound native entry/mask/return inputs", err) + } + normalized, err := normalizeX86RawFile(file, options.Goarch) + if err != nil { + t.Fatal("original raw shift formats could not be decoded", err) + } + return x86InstructionFormatObjectUnit(t, normalized, options.Sigs, options.Goarch, options.TargetTriple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerX86PackedShift(ins.Op, ins) + }) +} diff --git a/amd64_packed_shift_result.go b/amd64_packed_shift_result.go new file mode 100644 index 000000000..9e65f704f --- /dev/null +++ b/amd64_packed_shift_result.go @@ -0,0 +1,137 @@ +package plan9asm + +// One output byte can span two input bytes for an immediate bit shift. With an +// initialized but unknown count, it instead requires the union of every input +// bit that could reach it. Oversized logical counts contribute only zeros. +// The masks describe numerical dependence, never the native memory envelope. +func x86PackedShiftByteInputs(form x86PackedShiftForm, output int) (data, count uint64) { + laneBytes := form.laneBits / 8 + if form.countMode == x86PackedShiftBytes { + laneBytes = 16 + } + lane, at := output/laneBytes*laneBytes, output%laneBytes + first, last := at*8, at*8+7 + if form.countOperand.Kind == OpImm { + shift := int(uint8(form.countOperand.Imm)) + if form.countMode == x86PackedShiftBytes { + shift *= 8 + } + if form.left { + first, last = first-shift, last-shift + } else { + first, last = first+shift, last+shift + } + } else { + if form.left { + first = 0 + } else { + last = laneBytes*8 - 1 + } + count = 255 // The upper qword of a uniform register count is ignored. + if form.countMode == x86PackedShiftPerLane { + count = x86CallerVectorByteMask(int64(laneBytes)) << uint(lane) + } + } + if first < 0 { + first = 0 + } + if form.arithmetic && first >= laneBytes*8 { + first = laneBytes*8 - 1 // Every shifted-out bit reads the sign bit. + } + if last >= laneBytes*8 { + last = laneBytes*8 - 1 + } + if first <= last { + data = x86CallerVectorByteMask(int64(last/8-first/8+1)) << uint(lane+first/8) + } + return data, count +} + +func x86PackedShiftDefinedBytes(form x86PackedShiftForm, data, counts, old uint64) uint64 { + var known uint64 + for output := 0; output < form.byteWidth; output++ { + dataMask, countMask := x86PackedShiftByteInputs(form, output) + if data&dataMask == dataMask && counts&countMask == countMask { + known |= uint64(1) << uint(output) + } + } + // A defined mask is not a known active mask. Every inactive merging byte + // still needs its old destination; inactive zeroing bytes are constants. + if form.mask >= 0 && !form.zeroing { + known &= old + } + return known +} + +func x86ResultPackedShiftStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + return x86ResultPackedShiftStepWithRead(ins, state, owned, arch, false) +} + +func x86ResultPackedShiftStepWithRead(ins Instr, state *x86ResultState, owned []bool, arch string, nativeRead bool) (bool, bool) { + form, known, err := x86PackedShiftProperties(ins.Op, ins, arch) + if !known { + return false, false + } + literal := x86RawByteShiftLiteralForm(ins, arch) + if err != nil || form.mmx || !literal && (!x86ResultNamedPredicateInstruction(ins) || + ins.x86PackedShiftEncoding != x86RawPackedShiftAbsent || ins.x86PackedShiftW) { + return false, true // Raw identity and physical MMX have separate owners. + } + // This is the existing independent same-source DATA role, not ordinary + // native continuation. Its full tuple still needs loweredStaticReads; + // leaving the raw origin intact prevents an incoming-pointer fallback. + var pending x86ResultShiftBits + if nativeRead && ins.Args[form.data].Kind == OpReg { + input, _ := amd64VectorRegisterIndex(ins.Args[form.data].Reg, form.byteWidth) + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + pending = x86ResultShiftBitSource(ins, form, state.vectorBytes(input), destination) + } + if !x86ResultPackedShiftInputs(ins, form, state, owned, arch, nativeRead) { + return false, true + } + if nativeRead { + // Record only at the original named entry. The raw consumer shares + // byte effects through a local projection, not this named-only fact. + state.shiftBits = pending + } + return true, true +} + +func x86ResultPackedShiftInputs(ins Instr, form x86PackedShiftForm, state *x86ResultState, owned []bool, arch string, nativeRead bool) bool { + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false + } + } + if form.memory >= 0 { + // Even a saturating count or zero mask cannot create memory authority. + // Legacy m128 counts additionally need original native backing. Only + // lowering's ephemeral same-file DATA directory can provide an SB + // input here; logical FP/AUTO storage still cannot stand in for it. + memory := ins.Args[form.memory] + if _, static := x86StaticSBReadAddress(memory, arch); form.legacy && !static && + (!nativeRead || !x86OrdinaryNativeMemoryOperand(memory, arch)) { + return false + } + if !x86ResultReadWithNativeContinuation(ins, memory, int64(form.memoryBytes), state, owned, arch, nativeRead) { + return false + } + } + bytes := func(index int) uint64 { + if index == form.memory { + return x86CallerVectorByteMask(int64(form.byteWidth)) + } + operand := ins.Args[index] + register, _ := amd64VectorRegisterIndex(operand.Reg, amd64VectorByteWidth(operand.Reg)) + return state.vectorBytes(register) + } + data, counts := bytes(form.data), ^uint64(0) + if form.countOperand.Kind != OpImm { + counts = bytes(form.count) + } + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + state.writeVector(destination, form.byteWidth, + x86PackedShiftDefinedBytes(form, data, counts, state.vectorBytes(destination)), form.legacy) + return true +} diff --git a/amd64_packed_shift_result_grammar_test.go b/amd64_packed_shift_result_grammar_test.go new file mode 100644 index 000000000..b5e5fae9a --- /dev/null +++ b/amd64_packed_shift_result_grammar_test.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "sort" + "strings" + "testing" +) + +func TestX86PackedShiftResultIndependentGoGrammarAxes(t *testing.T) { + inventory := make(map[string]bool) + for _, row := range x86PackedShiftResultRows() { + inventory[strings.Fields(row.row)[0]] = true + } + var names []string + for name := range inventory { + names = append(names, name) + } + sort.Strings(names) + if !reflect.DeepEqual(names, x86PackedShiftNames()) { + t.Fatal("independent whole-family denominator changed", names) + } + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var rows []string + for _, name := range names { + vector := strings.HasPrefix(name, "V") + perLane := strings.Contains(name, "LLV") || strings.Contains(name, "RLV") + byteShift := strings.HasSuffix(name, "DQ") || strings.HasSuffix(name, "O") + banks := []string{"X"} + if vector { + banks = []string{"X", "Y", "Z"} + } + for _, bank := range banks { + if !perLane { + first, last := -128, 127 + if vector { + last = 255 + } + if bank == "Z" { + first = 0 + } + for immediate := first; immediate <= last; immediate++ { + row := fmt.Sprintf("%s $%d,%s1", name, immediate, bank) + if vector { + row += "," + bank + "2" + } + rows = append(rows, row) + } + } + limit := 32 + if arch == "386" && bank == "Z" { + // cmd/asm accepts high named X/Y banks in mode32, but + // rejects high Z. Neither supplies a physical register. + limit = 8 + } + if !vector { + limit = 16 + if arch == "386" { + limit = 8 + } + } + for index := 0; index < limit; index++ { + data := fmt.Sprintf("%s%d", bank, index) + if !vector { + rows = append(rows, name+" $7,"+data) + if !byteShift { + rows = append(rows, name+" "+data+",X1") + } + continue + } + if !perLane { + rows = append(rows, name+" $7,"+data+","+bank+"2", name+" $7,"+bank+"1,"+data) + } + if !byteShift { + countBank := "X" + if perLane { + countBank = bank + } + rows = append(rows, fmt.Sprintf("%s %s%d,%s1,%s2", name, countBank, index, bank, bank), + name+" "+countBank+"0,"+data+","+bank+"2", name+" "+countBank+"0,"+bank+"1,"+data) + } + } + } + } + if arch == "386" { + rows = append(rows, "PSRLQ $4294967297,X1", "PSRLDQ $-4294967297,X1", "VPSRLQ $4294967551,X1,X2", "VPSRLDQ $4294967551,X1,X2") + } + source := "TEXT f(SB),4,$0-0\n" + strings.Join(rows, "\n") + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for index, row := range rows { + ins := file.Funcs[0].Instrs[index+1] + originalCount := ins.Args[0] + context := false + if arch == "386" { + for _, arg := range ins.Args { + reg, valid := amd64VectorRegisterIndex(arg.Reg, amd64VectorByteWidth(arg.Reg)) + context = context || valid && reg >= 8 + } + } + form, known, err := x86PackedShiftProperties(ins.Op, ins, arch) + if !known || context && !errors.Is(err, ErrProbeNeedsContext) || !context && err != nil { + t.Fatal(row, err) + } + if !context && ins.Args[0].Kind == OpImm && uint8(form.countOperand.Imm) != uint8(ins.Args[0].Imm) { + t.Fatal("classified immediate changed encoded count", row) + } + if !reflect.DeepEqual(originalCount, ins.Args[0]) { + t.Fatal("classification rewrote the retained source", row) + } + } + t.Logf("%d actual Go forms, with physical mode32 context kept separate", len(rows)) + }) + } +} + +func TestX86PackedShiftResultMode32ClassifiedImmediateObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + source := "TEXT ·view(SB),4,$0-4\nMOVL $7,AX\n" + for dword := 0; dword < 4; dword++ { + source += fmt.Sprintf("PINSRD $%d,AX,X1\n", dword) + } + source += "PSRLQ $4294967297,X1\nPSRLDQ $-4294967297,X1\nVPSRLQ $4294967551,X1,X2\nVPSRLDQ $4294967551,X2,X1\nPEXTRD $0,X1,AX\nMOVL AX,ret+0(FP)\nRET\n" + for _, target := range x86RawReturnTargets { + if target.arch != "386" { + continue + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32\n", target.arch, target.triple, api) + if err != nil { + t.Fatal(target.triple, api, err) + } + compileLLVMToObject(t, llc, target.triple, "classified-shift.ll", "classified-shift.o", ir) + } + } +} + +func TestX86PackedShiftResultMode32HighZGoRejections(t *testing.T) { + seen := make(map[string]bool) + for _, candidate := range x86PackedShiftResultRows() { + name := strings.Fields(candidate.row)[0] + if !strings.HasPrefix(name, "V") || seen[name] { + continue + } + seen[name] = true + for index := 8; index < 32; index++ { + count := "$7" + if strings.Contains(name, "LLV") || strings.Contains(name, "RLV") { + count = "Z0" + } + row := fmt.Sprintf("%s %s,Z%d,Z2", name, count, index) + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86PackedShiftProperties(ins.Op, ins, "386"); !known || err == nil { + t.Fatal("mode32 physical Z register admitted", row) + } + } + } +} + +func TestX86PackedShiftResultUnresolvedFormsCannotEmit(t *testing.T) { + for _, row := range []string{"PSLLW $0,X1", "PSRLDQ $0,X1", "VPSRLQ $0,X1,X2", "VPSRLDQ $0,X1,X2"} { + for _, mutation := range []string{"floating", "unresolved"} { + ins := x86ScalarLaneInstruction(t, row) + if mutation == "floating" { + ins.Args[0].ImmIsFloat = true + } else { + ins.Args[0].ImmRaw = "missing" + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + before := output.Len() + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); err == nil || output.Len() != before { + t.Fatal("noninteger immediate emitted IR", row, mutation, err) + } + } + } + for _, mutation := range []string{"displacement", "index"} { + ins := x86ScalarLaneInstruction(t, "VPSRLQ (AX),X1,X2") + if mutation == "displacement" { + ins.Args[0].Mem.OffRaw = "missing" + } else { + ins.Args[0].Mem.IndexExt = "unknown" + } + if _, known, err := x86PackedShiftProperties(ins.Op, ins, "amd64"); !known || err == nil { + t.Fatal("unresolved memory accepted", mutation) + } + } +} diff --git a/amd64_packed_shift_result_runtime_test.go b/amd64_packed_shift_result_runtime_test.go new file mode 100644 index 000000000..1fe4f3119 --- /dev/null +++ b/amd64_packed_shift_result_runtime_test.go @@ -0,0 +1,152 @@ +package plan9asm + +import ( + "fmt" + "strconv" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedShiftPartialResults(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, native, cases strings.Builder + for index, row := range x86PackedShiftResultRows() { + name := fmt.Sprint("partialShift", index) + fmt.Fprintf(&declarations, "func %s(data,count uint32) uint32\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-12\nMOVL data+0(FP),AX\nMOVL count+4(FP),BX\nMOVL $0,CX\n", name) + for dword := 0; dword < 4; dword++ { + if row.data&(15< +#include +#include +` + native.String() + ` +struct Case { + uint32_t (*translated)(uint32_t,uint32_t); + void (*original)(const uint8_t*,const uint8_t*,uint8_t*); + const char *row; + unsigned bits; + int immediate,left,perlane; + uint16_t dataKnown,countKnown; + unsigned lane; +}; +static const struct Case cases[]={ +` + cases.String() + `}; +static uint64_t read_shift_bytes(const uint8_t *bytes,unsigned width) { + uint64_t value=0; + for(unsigned i=0;idataKnown&(1u<>(i%4*8)); + if(test->countKnown&(1u<>(i*8)):0; + } + for(unsigned output=0;output<128;output++) { + unsigned base=output/test->bits*test->bits; + uint64_t shift=test->immediate>=0?(unsigned)test->immediate: + test->perlane?read_shift_bytes(count+base/8,test->bits/8):read_shift_bytes(count,8); + if(shift>=test->bits) continue; + int from=(int)(output%test->bits)+(test->left?-(int)shift:(int)shift); + if(from<0 || from>=(int)test->bits) continue; + unsigned input=base+(unsigned)from; + want[output/8]|=((data[input/8]>>(input%8))&1)<<(output%8); + } + uint32_t expected=(uint32_t)read_shift_bytes(want+test->lane*4,4); + uint32_t got=test->translated(inputs[sample],counts[n]); + if(got!=expected) { + fprintf(stderr,"%s sample=%u count=%u LLVM=%x want=%x\n",test->row,sample,counts[n],got,expected); + return 1; + } + if(test->original) { + test->original(count,data,original); + if(read_shift_bytes(original+test->lane*4,4)!=expected) { + fprintf(stderr,"%s original Go selected output differs\n",test->row); + return 2; + } + } + } + return 0; +} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "shift-partial-result", triple, text, oracle, prefix) + }) + } + t.Log("44 partial-state scalar results x 96 inputs; original Go legacy opcodes plus independent bit oracle; no native AVX512 claim") +} diff --git a/amd64_packed_shift_result_test.go b/amd64_packed_shift_result_test.go new file mode 100644 index 000000000..24f65d243 --- /dev/null +++ b/amd64_packed_shift_result_test.go @@ -0,0 +1,220 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// This inventory is independent of the production opcode maps. Initialized +// dwords are deliberately narrower than X wherever the selected output permits. +type x86PackedShiftResultCase struct { + row string + data uint16 + count uint16 + lane int +} + +func x86PackedShiftResultRows() []x86PackedShiftResultCase { + var rows []x86PackedShiftResultCase + add := func(row string, data, count uint16, lane int) { + rows = append(rows, x86PackedShiftResultCase{row, data, count, lane}) + } + for _, op := range []string{"PSLLW", "PSLLL", "PSLLQ", "PSRLW", "PSRLL", "PSRLQ"} { + add(op+" $7,X1", 0xffff, 0, 0) + add(op+" X0,X1", 0xffff, 0x00ff, 0) + } + for _, op := range []string{"PSLLO", "PSLLDQ"} { + add(op+" $12,X1", 0x000f, 0, 3) + } + for _, op := range []string{"PSRLO", "PSRLDQ"} { + add(op+" $4,X1", 0x00f0, 0, 0) + } + for _, op := range []string{"VPSLLW", "VPSLLD", "VPSLLQ", "VPSRLW", "VPSRLD", "VPSRLQ"} { + add(op+" $7,X1,X2", 0xffff, 0, 0) + add(op+" X0,X1,X2", 0xffff, 0x00ff, 0) + } + for _, op := range []string{"VPSLLVW", "VPSLLVD", "VPSLLVQ", "VPSRLVW", "VPSRLVD", "VPSRLVQ"} { + add(op+" X0,X1,X2", 0xffff, 0xffff, 0) + } + add("VPSLLDQ $12,X1,X2", 0x000f, 0, 3) + add("VPSRLDQ $4,X1,X2", 0x00f0, 0, 0) + add("PSRLQ $32,X1", 0x00f0, 0, 0) + add("PSLLQ $32,X1", 0x000f, 0, 1) + add("PSLLQ X0,X1", 0x000f, 0x00ff, 0) + add("PSRLQ X0,X1", 0x00f0, 0x00ff, 1) + add("VPSLLVQ X0,X1,X2", 0x000f, 0x00ff, 0) + add("VPSRLVQ X0,X1,X2", 0x00f0, 0x00ff, 1) + add("PSRLDQ $16,X1", 0, 0, 0) + add("PSLLQ $64,X1", 0, 0, 0) + return rows +} + +func TestX86PackedShiftResultThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + var source, declarations strings.Builder + for index, row := range x86PackedShiftResultRows() { + name := fmt.Sprint("shiftResult", index) + fmt.Fprintf(&declarations, "func %s() uint32\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-4\n%s\n", name, + x86PackedShiftResultBody(row.row, row.data, row.count, row.lane, "ret+0(FP)")) + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + compileLLVMToObject(t, llc, target.triple, "shift-result.ll", "shift-result.o", ir) + } + }) + } +} + +func TestX86PackedShiftResultMemoryAndMaskBoundaries(t *testing.T) { + for _, row := range []string{ + "PSRLQ (AX),X1", "PSRLQ a+0(FP),X1", "PSRLQ local-16(SP),X1", + "VPSRLQ $255,(AX),X2", "VPSRLDQ $16,·data(SB),X2", + "VPSRLVQ (AX),X1,X2", "VPSRLQ.BCST $64,(AX),X2", + "PSRLQ $7,M1", + } { + ins := x86ScalarLaneInstruction(t, row) + state := x86ResultState{fp: make([]bool, 16), auto: make([]bool, 16)} + for i := range state.fp { + state.fp[i], state.auto[i] = true, true + } + for i := range state.xmm { + state.xmm[i] = 0xffff + } + owned := make([]bool, 16) + for i := range owned { + owned[i] = true + } + if x86ResultTransfer(ins, &state, owned, 16, "amd64") || len(ins.x86CallerMemory) != 0 { + t.Fatal("numerical effect borrowed native/MMX permission", row) + } + } + for _, row := range []string{"VPSRLQ $7,X1,K3,X2", "VPSRLQ.Z $7,X1,K3,X2"} { + ins := x86ScalarLaneInstruction(t, row) + for _, initialized := range []bool{false, true} { + state := x86ResultState{} + state.xmm[1], state.xmm[2], state.masks[3] = 0xffff, 0x5a5a, initialized + accepted := x86ResultTransfer(ins, &state, nil, 0, "amd64") + if accepted != initialized { + t.Fatal("uninitialized mask was ignored", row) + } + if initialized { + want := uint16(0x5a5a) + if strings.Contains(row, ".Z") { + want = 0xffff + } + if state.xmm[2] != want { + t.Fatal("merge/zero effect", row, state.xmm[2], want) + } + } + } + } + for _, body := range []string{ + "MOVL $7,AX\nPINSRD $0,AX,X1\nPSRLQ $1,X1\nPEXTRD $0,X1,AX\n", + "MOVL $7,AX\nPINSRD $1,AX,X1\nPSRLDQ $3,X1\nPEXTRD $0,X1,AX\n", + "MOVL $7,AX\nPINSRD $0,AX,X0\nPINSRD $0,AX,X1\nPSLLQ X0,X1\nPEXTRD $0,X1,AX\n", + } { + _, file, sig, source := x86ResultInitializationFixture(t, "amd64", "func Value() uint32", body+"MOVL AX,ret0+0(FP)\nRET") + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("one missing dependency acquired a result", err) + } + } + } +} + +func TestX86PackedShiftResultOwnedMemoryStillNeedsFullNativeTuple(t *testing.T) { + for _, fixture := range []struct { + row string + width int + }{ + {"VPSRLQ a+0(FP),X1,X2", 16}, + {"VPSRLVQ a+0(FP),X1,X2", 16}, + {"VPSRLVQ.BCST a+0(FP),X1,X2", 8}, + {"VPSRLQ $64,a+0(FP),X2", 16}, + {"VPSRLQ.BCST $64,a+0(FP),X2", 8}, + {"VPSRLDQ $16,a+0(FP),X2", 16}, + } { + for _, auto := range []bool{false, true} { + row := fixture.row + if auto { + row = strings.ReplaceAll(row, "a+0(FP)", "local-16(SP)") + } + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$16-16\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + for missing := -1; missing < 16; missing++ { + state := x86ResultState{fp: make([]bool, 16), auto: make([]bool, 16)} + owned := make([]bool, 16) + for i := range state.fp { + state.fp[i], state.auto[i], owned[i] = i != missing, i != missing, true + } + state.xmm[1] = 0xffff + accepted := x86ResultTransfer(ins, &state, owned, 16, "amd64") + want := missing < 0 || missing >= fixture.width + if accepted != want || accepted && state.xmm[2] != 0xffff || len(ins.x86CallerMemory) != 0 { + t.Fatal("logical tuple/memory authority mismatch", row, missing, accepted, want) + } + } + } + } +} + +func x86PackedShiftResultBody(row string, data, count uint16, lane int, result string) string { + var body strings.Builder + body.WriteString("MOVL $7,AX\n") + for register, bytes := range []uint16{count, data} { + for dword := 0; dword < 4; dword++ { + if bytes&(15<= 0 { + continue + } + if legacy && immediate == 255 { + continue + } + if immediate < 0 && memory == 2 && !perLane { + continue + } + for _, destination := range []int{0, 1, 2} { + if legacy && destination != 0 { + continue + } + // Boundary immediates need not multiply every mask/alias + // combination; dynamic counts cover all boundaries below. + if !legacy && !perLane && immediate != 1 && immediate != -1 && (mask != 0 || destination != 2) { + continue + } + name := fmt.Sprintf("shift%d", len(cases)) + operation := op + if memory == 2 { + operation += ".BCST" + } + if mask == 2 { + operation += ".Z" + } + count := "$" + fmt.Sprint(immediate) + data := bank + "0" + mode := 0 + if immediate < 0 { + count = "X1" + if perLane { + count, mode = bank+"1", 1 + } + if memory != 0 { + count = "(CX)" + } + } else if memory != 0 { + data = "(BX)" + } + if byteShift { + mode = 2 + } + row := operation + " " + count + "," + data + if !legacy { + if mask != 0 { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + } + cases = append(cases, x86PackedShiftNumericalCase{ + name: name, row: row, width: width, bits: bits, countMode: mode, + immediate: immediate, memory: memory, maskMode: mask, destination: destination, + left: strings.Contains(op, "SLL"), legacy: legacy, broadcast: memory == 2, + }) + } + } + } + } + } + } + return cases +} + +func x86PackedShiftNumericalAssembly(cases []x86PackedShiftNumericalCase) (string, map[string]FuncSig) { + var assembly strings.Builder + sigs := map[string]FuncSig{} + for _, test := range cases { + fmt.Fprintf(&assembly, `TEXT %s(SB),4,$0-40 +MOVQ out+0(FP),AX +MOVQ data+8(FP),BX +MOVQ count+16(FP),CX +MOVQ mask+24(FP),DX +MOVQ seed+32(FP),SI +VMOVUPS (BX),Z0 +VMOVUPS (CX),Z1 +VMOVUPS (CX),Z2 +KMOVQ DX,K1 +CMPQ SI,$0 +JE carry +CMPQ SI,$1 +JE overflow +MOVQ $1,SI +CMPQ SI,$0 +JMP shifted +carry: +MOVQ $-1,SI +ADDQ $1,SI +JMP shifted +overflow: +MOVQ $0x7fffffffffffffff,SI +ADDQ $1,SI +shifted: +%s +VMOVUPS Z%d,(AX) +SETCS 64(AX) +SETOS 65(AX) +SETEQ 66(AX) +SETMI 67(AX) +SETPS 68(AX) +RET +`, test.name, test.row, test.destination) + sigs[test.name] = FuncSig{Name: test.name, Args: []LLVMType{Ptr, Ptr, Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}} + } + return assembly.String(), sigs +} + +const x86PackedShiftIndependentC = ` +#include +#include +#include +typedef void (*shift_fn)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint64_t); +struct shift_case { shift_fn fn; const char *row; int width,bits,mode,immediate,memory,mask,destination,left,legacy,broadcast; }; +static uint64_t read_le(const uint8_t *p,int bytes) { + uint64_t value=0; + for(int i=0;idestination==0?a:counts; + memcpy(out,old,64); + if(!c->legacy) memset(out+c->width,0,64-c->width); + if(c->mode==2) { + for(int i=0;iwidth;++i) { + int offset=i%16, from=c->left ? offset-c->immediate : offset+c->immediate; + out[i]=from>=0 && from<16 ? a[i-offset+from] : 0; + } + return; + } + int bytes=c->bits/8; + for(int i=0;iwidth;i+=bytes) { + if(c->mask && !((mask>>(i/bytes))&1)) { + if(c->mask==2) memset(out+i,0,bytes); + continue; + } + uint64_t count=c->immediate>=0?(uint64_t)c->immediate:read_le(counts+(c->mode==1 && !c->broadcast?i:0),c->mode==1?bytes:8); + uint64_t value=read_le(a+(c->broadcast && c->immediate>=0?0:i),bytes); + uint64_t result=0; + if(count<(uint64_t)c->bits) result=c->left?value<>count; + for(int b=0;b>(8*b)); + } +} +static int check_case(const struct shift_case *c,int index) { + const uint64_t limits[]={0,1,7,15,16,17,31,32,33,63,64,65,255,256,UINT64_C(1)<<32,UINT64_MAX}; + const uint64_t masks[]={0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; + const uint8_t flags[3][5]={{1,0,1,0,1},{0,1,0,1,1},{0,0,0,0,0}}; + for(unsigned round=0;round<16;++round) for(unsigned m=0;m<4;++m) for(unsigned seed=0;seed<3;++seed) { + uint8_t aStorage[66],storage[71],beforeA[66],want[64]; + _Alignas(16) uint8_t countStorage[96],beforeCount[96]; + uint8_t *a=aStorage+1,*counts=countStorage+(c->legacy?16:3),*out=storage+1; + memset(aStorage,0xa7,sizeof(aStorage)); memset(countStorage,0xb9,sizeof(countStorage)); memset(storage,0xc3,sizeof(storage)); + for(int i=0;i<64;++i) { a[i]=(uint8_t)(i*71+round*97+13); counts[i]=(uint8_t)(i*43+19); } + int bytes=c->bits/8; + for(int i=0;i<64;i+=bytes) { + uint64_t count=limits[(round+i/bytes)%16]; + if(c->mode!=1) count=limits[round]; + for(int b=0;b>(8*b)); + } + if(c->mode==0) for(int b=0;b<8;++b) counts[b]=(uint8_t)(limits[round]>>(8*b)); + memcpy(beforeA,aStorage,sizeof(beforeA)); memcpy(beforeCount,countStorage,sizeof(beforeCount)); + expected(want,a,counts,masks[m],c); + c->fn(out,a,counts,masks[m],seed); + if(memcmp(want,out,64) || memcmp(flags[seed],out+64,5) || storage[0]!=0xc3 || storage[70]!=0xc3 || + memcmp(aStorage,beforeA,sizeof(beforeA)) || memcmp(countStorage,beforeCount,sizeof(beforeCount))) { + for(int i=0;i<69;++i) if(out[i]!=(i<64?want[i]:flags[seed][i-64])) { + fprintf(stderr,"case=%d %s round=%u mask=%u seed=%u byte=%d actual=%02x expected=%02x\n",index,c->row,round,m,seed,i,out[i],i<64?want[i]:flags[seed][i-64]); break; + } + return 1; + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedLogicalShiftCompleteSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires Linux/amd64 runtime counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + cases := x86PackedShiftNumericalCases() + assembly, sigs := x86PackedShiftNumericalAssembly(cases) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, "amd64") + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + var driver strings.Builder + driver.WriteString(x86PackedShiftIndependentC) + for _, c := range cases { + fmt.Fprintf(&driver, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint64_t);\n", c.name) + } + driver.WriteString("static const struct shift_case cases[]={\n") + for _, c := range cases { + fmt.Fprintf(&driver, "{%s,%q,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d},\n", c.name, c.row, c.width, c.bits, c.countMode, c.immediate, c.memory, c.maskMode, c.destination, x86ShiftBoolean(c.left), x86ShiftBoolean(c.legacy), x86ShiftBoolean(c.broadcast)) + } + driver.WriteString("};\nint main(void) { for(unsigned i=0;i>(8*i)) } + %s(unsafe.Pointer(&out[1]),unsafe.Pointer(&data[1]),unsafe.Pointer(&countData[1]),uint64(seed)) + if out[0]!=0xc3 || out[22]!=0xc3 { t.Fatal("canary") } + for i:=0;i<5;i++ { if out[17+i]!=flags[seed][i] { t.Fatalf("flags seed=%%d bit=%%d got=%%d",seed,i,out[17+i]) } } + for i:=0;i<16;i++ { + var want byte +`, name) + if byteShift { + from := "i-7" + if strings.Contains(op, "SRL") { + from = "i+7" + } + fmt.Fprintf(&tests, "from:=%s\nif from>=0 && from<16 { want=data[from+1] }\n", from) + } else { + operator := "<<" + if strings.Contains(op, "SRL") { + operator = ">>" + } + fmt.Fprintf(&tests, "offset:=(i/%d)*%d\nvar value uint64\nfor b:=0;b<%d;b++ { value|=uint64(data[offset+b+1])<<(8*b) }\nif count<%d { want=byte((value %s count)>>(8*(i%%%d))) }\n", bits/8, bits/8, bits/8, bits, operator, bits/8) + } + tests.WriteString("if out[i+1]!=want { t.Fatalf(\"byte %d got=%x want=%x\",i,out[i+1],want) }\n}\n}\n}\n") + } + tests.WriteString("}\n") + dir := t.TempDir() + for name, bytes := range map[string]string{"go.mod": "module original\n\ngo 1.20\n", "original.go": source.String(), "original_amd64.s": assembly.String(), "original_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(bytes), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOARCH=amd64", "CGO_ENABLED=0", "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go legacy shift: %v\n%s", err, output) + } + t.Logf("10 original Go SSE2 mnemonic/alias programs ×48 initialized/flags/canary inputs; Rosetta=%t", crossRosetta) +} diff --git a/amd64_packed_sign_extend_forms.go b/amd64_packed_sign_extend_forms.go new file mode 100644 index 000000000..e48629ee7 --- /dev/null +++ b/amd64_packed_sign_extend_forms.go @@ -0,0 +1,28 @@ +package plan9asm + +import "strings" + +// Keep all twelve signed members separate from the pre-existing unsigned +// inventory: sharing pure grammar must not extend its raw-source identity. +var x86PackedSignExtendSpecs = map[Op]x86PackedExtendSpec{ + "PMOVSXBW": {8, 16, false, 0x20}, + "PMOVSXBD": {8, 32, false, 0x21}, + "PMOVSXBQ": {8, 64, false, 0x22}, + "PMOVSXWD": {16, 32, false, 0x23}, + "PMOVSXWQ": {16, 64, false, 0x24}, + "PMOVSXDQ": {32, 64, false, 0x25}, + "VPMOVSXBW": {8, 16, true, 0x20}, + "VPMOVSXBD": {8, 32, true, 0x21}, + "VPMOVSXBQ": {8, 64, true, 0x22}, + "VPMOVSXWD": {16, 32, true, 0x23}, + "VPMOVSXWQ": {16, 64, true, 0x24}, + "VPMOVSXDQ": {32, 64, true, 0x25}, +} + +func x86PackedSignExtendProperties(ins Instr, arch string) (x86PackedExtendForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + if _, known := x86PackedSignExtendSpecs[Op(base)]; !known { + return x86PackedExtendForm{}, false, nil + } + return x86PackedExtendProperties(ins, arch) +} diff --git a/amd64_packed_string_fault_linux_amd64_test.go b/amd64_packed_string_fault_linux_amd64_test.go new file mode 100644 index 000000000..cc1483b2c --- /dev/null +++ b/amd64_packed_string_fault_linux_amd64_test.go @@ -0,0 +1,17 @@ +//go:build linux && amd64 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86PackedStringOriginalNativeMemoryFaults(t *testing.T) { + x87IA32RequireNativeHost(t) + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("native packed-string fault owner requires LLVM22 llc/clang") + } + execution := x86FloatCompareExecution{llc: llc, triple: "x86_64-unknown-linux-gnu", + compiler: []string{clang, "-target", "x86_64-unknown-linux-gnu", "-mno-red-zone"}} + x86PackedStringRunMemoryFaults(t, execution, true) + x87IA32RequireNativeHost(t) +} diff --git a/amd64_packed_string_fault_policy_crlf_test.go b/amd64_packed_string_fault_policy_crlf_test.go new file mode 100644 index 000000000..473b58018 --- /dev/null +++ b/amd64_packed_string_fault_policy_crlf_test.go @@ -0,0 +1,45 @@ +package plan9asm + +import ( + "bytes" + "os" + "strings" + "testing" +) + +func TestX86PackedStringFaultNativePolicyKeepsCRLFAndMutations(t *testing.T) { + source, err := os.ReadFile("amd64_packed_string_fault_linux_amd64_test.go") + if err != nil { + t.Fatal(err) + } + lf := string(normalizeNativeOwnerPolicySource(source)) + for _, ending := range []struct{ name, newline string }{{"LF", "\n"}, {"CRLF", "\r\n"}} { + t.Run(ending.name, func(t *testing.T) { + encode := func(text string) []byte { + return []byte(strings.ReplaceAll(text, "\n", ending.newline)) + } + original := encode(lf) + before := append([]byte(nil), original...) + t.Run("original", func(t *testing.T) { + if err := x86PackedStringFaultNativePolicy(original); err != nil { + t.Fatal("Go-equivalent line endings changed native fault ownership", err) + } + if !bytes.Equal(original, before) { + t.Fatal("policy parsing rewrote original native source") + } + }) + mutations := x86PackedStringFaultNativeMutations(lf) + if len(mutations) != 9 { + t.Fatal("complete native policy mutation denominator changed") + } + for name, mutation := range mutations { + t.Run(name, func(t *testing.T) { + changed := encode(mutation) + if mutation == lf || bytes.Equal(changed, original) || x86PackedStringFaultNativePolicy(changed) == nil { + t.Fatal("line-ending adaptation hid a native policy mutation") + } + }) + } + }) + } +} diff --git a/amd64_packed_string_fault_runtime_test.go b/amd64_packed_string_fault_runtime_test.go new file mode 100644 index 000000000..cd4292ef3 --- /dev/null +++ b/amd64_packed_string_fault_runtime_test.go @@ -0,0 +1,207 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// This portable counterpart executes original legacy Go bytes and translated +// forms. Only the separately required Linux owner executes original VEX bytes. +func TestX86PackedStringOriginalLegacyAndLLVMReadFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + execution = x86PackedStringFaultCompiler(t, execution) + x86PackedStringRunMemoryFaults(t, execution, false) +} + +func x86PackedStringFaultCompiler(t *testing.T, execution x86FloatCompareExecution) x86FloatCompareExecution { + t.Helper() + if runtime.GOOS == "darwin" { + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + // Reuse the actual selected SDK/no-default-config helper. This only + // configures the test's native C driver, never the Go source profile. + command := runtimeOracleNativeCCommand(t, ctx, execution.compiler[0], execution.compiler[1:]...) + execution.compiler = append([]string(nil), command.Args...) + } + return execution +} + +func TestX86PackedStringFaultOriginalByteDriverObjects(t *testing.T) { + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("original PCMP byte/driver objects require LLVM22") + } + triple := "x86_64-unknown-linux-gnu" + if runtime.GOOS == "darwin" { + triple = "x86_64-apple-darwin" + } + execution := x86FloatCompareExecution{llc: llc, triple: triple, + compiler: []string{clang, "-target", triple, "-mno-red-zone"}} + if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { + // The already authenticated matching cross compiler supplies its SDK; + // this object-only owner does not execute its emulator transport. + execution = x86FloatCompareExecutionTools(t) + } + execution = x86PackedStringFaultCompiler(t, execution) + cases := x86PackedStringFaultCases() + source, declarations := x86PackedStringFaultFixture(cases, "amd64") + x86LEAActualTargetGoObject(t, "amd64", triple, source) + driver := x86PackedStringFaultDriver(t, triple, cases, true) + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, "Go") + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, ir) + } + execution.compileLLVM(t, "original-pcmp-fault-driver", input, driver, level) + } + t.Log("complete original eight-name Go-byte/LLVM/C driver compiled; no faulting executable was launched") +} + +func x86PackedStringRunMemoryFaults(t *testing.T, execution x86FloatCompareExecution, native bool) { + t.Helper() + if native { + if len(execution.prefix) != 0 || execution.preserveArgv0 || execution.triple != "x86_64-unknown-linux-gnu" { + t.Fatal("native PCMP fault owner cannot use a guest transport or another target") + } + execution.requireAVX(t) + } + cases := x86PackedStringFaultCases() + source, declarations := x86PackedStringFaultFixture(cases, "amd64") + // This independently checks the complete original Go ABI0 source, not only + // the physical-register byte shim consumed by the C fault observer below. + x86LEAActualTargetGoObject(t, "amd64", execution.triple, source) + driver := x86PackedStringFaultDriver(t, execution.triple, cases, native) + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", execution.triple, "Go") + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + executable := execution.compileLLVM(t, "packed-string-read-faults", input, driver, level) + probes := []int{0, 1, 2, 4} // NULL, 15-byte tail, full tail, readable mod16=1. + if native { + probes = nil + for probe := 0; probe < 34; probe++ { + probes = append(probes, probe) + } + } + for row := range cases { + for _, probe := range probes { + original := x86ObserveMemoryFault(t, execution, executable, row, probe, 1) + translated := x86ObserveMemoryFault(t, execution, executable, row, probe, 0) + fault := probe == 0 || probe == 1 || probe >= 19 + if original != translated || fault && original != "segv" && original != "bus" || !fault && original != "return" { + t.Fatalf("PCMP row=%d/%s zero=%t dead=%t probe=%d original=%s LLVM=%s", + row, cases[row].op, cases[row].zero, cases[row].dead, probe, original, translated) + } + } + } + t.Logf("complete32-case PCMP read/fault matrix: original native VEX=%t, probes=%d, LLVM%s; readable values and full input-page canaries checked", native, len(probes), level) + }) + } +} + +func x86PackedStringFaultDriver(t *testing.T, triple string, cases []x86PackedStringFaultCase, native bool) string { + t.Helper() + var declarations, records strings.Builder + declarations.WriteString(x86OriginalSysVPointer("uint32_t", "original_program", "const uint8_t*")) + for index, row := range cases { + originalRow := row + if !native { + // A legacy opcode is only the portable memory-fault counterpart. + // It cannot certify the named original VEX opcode's execution. + originalRow.op = strings.TrimPrefix(originalRow.op, "V") + } + original := fmt.Sprintf("original_stringfault%d", index) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+x86PackedStringFaultBody(originalRow)+"RET\n") + if len(code) < 8 || code[len(code)-1] != 0xc3 { + t.Fatal("original Go PCMP byte program lost its complete ordinary RET", originalRow, code) + } + var bytes []string + for _, value := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&declarations, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + declarations.WriteString(x86OriginalSysVDeclaration("uint32_t", original, "const uint8_t*")) + fmt.Fprintf(&declarations, "extern uint32_t stringfault%d(const uint8_t*);\n", index) + fmt.Fprintf(&records, " {%s,stringfault%d,%d,%d},\n", original, index, + x86FloatAddSubBoolInt(row.zero), x86FloatAddSubBoolInt(row.dead)) + } + return x86MemoryFaultSupportC + declarations.String() + ` +struct string_fault_case { + original_program original; + uint32_t (*translated)(const uint8_t*); + int zero,dead; +}; + +static const struct string_fault_case cases[] = { +` + records.String() + "};\n" + x86PackedStringFaultMainC +} + +const x86PackedStringFaultMainC = ` +int main(int argc, char **argv) { + if (argc != 4) return 80; + unsigned row,probe,original; + char tail; + if (sscanf(argv[1],"%u%c",&row,&tail) != 1 || + sscanf(argv[2],"%u%c",&probe,&tail) != 1 || + sscanf(argv[3],"%u%c",&original,&tail) != 1 || + row >= sizeof(cases)/sizeof(cases[0]) || probe >= 34 || original > 1) return 80; + + __builtin_cpu_init(); + if (!__builtin_cpu_supports("sse4.2")) return 86; + int setup = memory_fault_setup(0); + if (setup) return setup; + const struct string_fault_case *kernel = &cases[row]; + uint8_t fill = kernel->zero ? 0 : 255; + memset(guard,fill,page); + if (mprotect(guard,page,PROT_READ)) return 87; + + const uint8_t *input; + if (probe == 0) { + input = NULL; + } else if (probe == 1) { + input = guard+page-15; + } else if (probe == 2) { + input = guard+page-16; + } else if (probe < 19) { + input = guard+32+(probe-3); + } else { + input = guard+page-(probe-19); + } + + printf("memory enter %u %u %u\n",row,probe,original); + if (fflush(stdout)) return 83; + uint32_t value = original ? kernel->original(input) : kernel->translated(input); + + // A missing primary read must fail immediately. Calling the reference + // here could otherwise supply the fault that the translated code lost. + if (probe == 0 || probe == 1 || probe >= 19) return 93; + + // Values are compared only after the primary instruction returned. The + // original opcode, not a C string scan or a guessed length, is the oracle. + if (value != kernel->original(input) || (kernel->dead && value != 7)) return 91; + for (size_t i = 0; i < page; ++i) { + if (guard[i] != fill) return 92; + } + if (free_guard()) return 84; + puts("memory complete"); + return fflush(stdout) ? 85 : 0; +} +` diff --git a/amd64_packed_string_fault_test.go b/amd64_packed_string_fault_test.go new file mode 100644 index 000000000..addb42a81 --- /dev/null +++ b/amd64_packed_string_fault_test.go @@ -0,0 +1,327 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "strconv" + "strings" + "testing" +) + +func TestX86PackedStringFaultRequiredNativeOwner(t *testing.T) { + source, err := os.ReadFile("amd64_packed_string_fault_linux_amd64_test.go") + if err != nil { + t.Fatal("complete original-Go/LLVM packed-string fault owner absent", err) + } + if err := x86PackedStringFaultNativePolicy(source); err != nil { + t.Fatal(err) + } +} + +type x86PackedStringFaultCase struct { + op string + zero, dead bool +} + +// Independent source axes, not the production descriptor or a shortened SDK +// function. Each original spelling retains both live/dead and empty/full cases. +func x86PackedStringFaultCases() []x86PackedStringFaultCase { + var cases []x86PackedStringFaultCase + for _, op := range []string{"PCMPESTRI", "PCMPESTRM", "PCMPISTRI", "PCMPISTRM", + "VPCMPESTRI", "VPCMPESTRM", "VPCMPISTRI", "VPCMPISTRM"} { + for _, zero := range []bool{false, true} { + for _, dead := range []bool{false, true} { + cases = append(cases, x86PackedStringFaultCase{op, zero, dead}) + } + } + } + return cases +} + +func x86PackedStringFaultBody(row x86PackedStringFaultCase) string { + body := "PXOR X1,X1\n" + if !row.zero { + body += "PCMPEQB X1,X1\n" + } + if strings.Contains(row.op, "E") { + length := 16 + if row.zero { + length = 0 + } + body += fmt.Sprintf("MOVL $%d,AX\nMOVL $%d,DX\n", length, length) + } + body += row.op + " $12,(DI),X1\n" + if row.dead { + body += "MOVL $7,AX\n" + } else if strings.HasSuffix(row.op, "M") { + body += "MOVL X0,AX\n" + } else { + body += "MOVL CX,AX\n" + } + return body +} + +func x86PackedStringFaultFixture(cases []x86PackedStringFaultCase, arch string) (string, string) { + move, word := "MOVQ", 8 + if arch == "386" { + move, word = "MOVL", 4 + } + var source, declarations strings.Builder + for index, row := range cases { + name := fmt.Sprintf("stringfault%d", index) + fmt.Fprintf(&declarations, "func %s(p *byte) uint32\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),DI\n%sMOVL AX,ret+%d(FP)\nRET\n", + name, word+4, move, x86PackedStringFaultBody(row), word) + } + return source.String(), declarations.String() +} + +func TestX86PackedStringFaultCompleteSourceReadBeforeDeadResults(t *testing.T) { + cases := x86PackedStringFaultCases() + if len(cases) != 32 { + t.Fatal("eight complete spellings require independent empty/full and live/dead cases") + } + for _, arch := range []string{"amd64", "386"} { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + source, declarations := x86PackedStringFaultFixture(cases, arch) + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, arch, triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86UndefinedTrapOptimizedIR(t, ir) + } + if strings.Count(text, "load volatile <16 x i8>") != len(cases) || + strings.Count(text, "define ") != len(cases) { + t.Fatal("empty lengths, EOS or dead results erased a complete observable m128 read", optimized) + } + } + }) + } + } +} + +func TestX86PackedStringFaultReadOracleRejectsLostVolatility(t *testing.T) { + source, declarations := x86PackedStringFaultFixture(x86PackedStringFaultCases(), "amd64") + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", "x86_64-unknown-linux-gnu", "Go") + if err != nil { + t.Fatal(err) + } + mutation := strings.ReplaceAll(ir, "load volatile <16 x i8>", "load <16 x i8>") + if mutation == ir || strings.Count(x86UndefinedTrapOptimizedIR(t, mutation), "load <16 x i8>") == 32 { + t.Fatal("read oracle cannot detect historical dead/zero-length nonvolatile access loss") + } +} + +func x86PackedStringFaultMainSource(t *testing.T) string { + t.Helper() + file, err := parser.ParseFile(token.NewFileSet(), "amd64_packed_string_fault_runtime_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + var source string + ast.Inspect(file, func(node ast.Node) bool { + value, ok := node.(*ast.ValueSpec) + if !ok || len(value.Names) != 1 || value.Names[0].Name != "x86PackedStringFaultMainC" || len(value.Values) != 1 { + return true + } + literal, ok := value.Values[0].(*ast.BasicLit) + if !ok { + t.Fatal("fault observer must retain its complete registered C template") + } + source, err = strconv.Unquote(literal.Value) + if err != nil { + t.Fatal(err) + } + return false + }) + if source == "" { + t.Fatal("actual fault observer template absent") + } + return source +} + +// This checks the fixed, registered test template, not arbitrary C programs. +// Primary failure must precede any reference call or success/canary receipt. +func x86PackedStringFaultObserverPolicy(source string) error { + previous := -1 + for _, component := range []string{ + "printf(\"memory enter %u %u %u\\n\",row,probe,original);", + "uint32_t value = original ? kernel->original(input) : kernel->translated(input);", + "if (probe == 0 || probe == 1 || probe >= 19) return 93;", + "if (value != kernel->original(input) || (kernel->dead && value != 7)) return 91;", + "if (guard[i] != fill) return 92;", + "if (free_guard()) return 84;", + "puts(\"memory complete\");", + } { + position := strings.Index(source, component) + if position <= previous || strings.Count(source, component) != 1 { + return fmt.Errorf("primary fault, independent reference and completion order changed") + } + previous = position + } + return nil +} + +func TestX86PackedStringFaultObserverCannotBorrowReferenceFault(t *testing.T) { + source := x86PackedStringFaultMainSource(t) + if err := x86PackedStringFaultObserverPolicy(source); err != nil { + t.Fatal(err) + } + const guard = "if (probe == 0 || probe == 1 || probe >= 19) return 93;" + const reference = "if (value != kernel->original(input) || (kernel->dead && value != 7)) return 91;" + for name, mutation := range map[string]string{ + "no primary failure": strings.Replace(source, guard, "", 1), + "reference supplies fault": strings.Replace(strings.Replace(source, guard, "", 1), reference, reference+"\n"+guard, 1), + "NULL lost": strings.Replace(source, guard, "if (probe == 1 || probe >= 19) return 93;", 1), + "tail lost": strings.Replace(source, guard, "if (probe == 0 || probe >= 19) return 93;", 1), + "partial matrix lost": strings.Replace(source, guard, "if (probe == 0 || probe == 1) return 93;", 1), + "success on missing read": strings.Replace(source, guard, strings.Replace(guard, "93", "0", 1), 1), + } { + t.Run(name, func(t *testing.T) { + if mutation == source || x86PackedStringFaultObserverPolicy(mutation) == nil { + t.Fatal("missing primary access could borrow a reference fault") + } + }) + } +} + +func TestX86PackedStringFaultCompleteFiveTargetOriginalGoAndLLVMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("complete packed-string fault objects require LLVM22") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + source, declarations := x86PackedStringFaultFixture(x86PackedStringFaultCases(), target.arch) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + x86PackedStringCompileObject(t, llc, target.triple, input, level) + } + } + }) + } +} + +func x86PackedStringFaultNativePolicy(source []byte) error { + source = normalizeNativeOwnerPolicySource(source) + if !strings.HasPrefix(string(source), "//go:build linux && amd64\n") { + return fmt.Errorf("packed-string faults need the required native Linux/AMD64 owner") + } + file, err := parser.ParseFile(token.NewFileSet(), "native.go", source, 0) + if err != nil { + return err + } + roots := 0 + for _, declaration := range file.Decls { + fn, ok := declaration.(*ast.FuncDecl) + if !ok || fn.Name.Name != "TestCrossLinuxRuntimeMatrixX86PackedStringOriginalNativeMemoryFaults" { + continue + } + roots++ + var gates []token.Pos + var run token.Pos + runs := 0 + top := map[*ast.CallExpr]bool{} + for _, statement := range fn.Body.List { + if expr, ok := statement.(*ast.ExprStmt); ok { + if call, ok := expr.X.(*ast.CallExpr); ok { + top[call] = true + } + } + } + var failure error + ast.Inspect(fn.Body, func(node ast.Node) bool { + switch node := node.(type) { + case *ast.CallExpr: + if selector, ok := node.Fun.(*ast.SelectorExpr); ok && strings.HasPrefix(selector.Sel.Name, "Skip") { + failure = fmt.Errorf("required execution cannot skip") + } + if name, ok := node.Fun.(*ast.Ident); ok { + if name.Name == "x87IA32RequireNativeHost" { + gates = append(gates, node.Pos()) + if !top[node] { + failure = fmt.Errorf("native gate must be unconditional") + } + } + if name.Name == "x86PackedStringRunMemoryFaults" { + runs++ + run = node.Pos() + if !top[node] || len(node.Args) != 3 { + failure = fmt.Errorf("whole fault execution must be unconditional") + } else if value, ok := node.Args[2].(*ast.Ident); !ok || value.Name != "true" { + failure = fmt.Errorf("all original native spellings must execute") + } + } + } + case *ast.KeyValueExpr: + if name, ok := node.Key.(*ast.Ident); ok && (name.Name == "prefix" || name.Name == "preserveArgv0") { + failure = fmt.Errorf("required native execution cannot use a guest prefix") + } + case *ast.ReturnStmt: + failure = fmt.Errorf("native owner cannot return early") + } + return true + }) + if failure != nil { + return failure + } + if len(gates) != 2 || runs != 1 || !(gates[0] < run && run < gates[1]) { + return fmt.Errorf("native observations must surround exactly one complete execution") + } + } + if roots != 1 { + return fmt.Errorf("native root missing or duplicated") + } + return nil +} + +func x86PackedStringFaultNativeMutations(text string) map[string]string { + const run = "\tx86PackedStringRunMemoryFaults(t, execution, true)" + const gate = "\tx87IA32RequireNativeHost(t)" + return map[string]string{ + "missing after": strings.Replace(text, run+"\n"+gate, run, 1), + "both before": strings.Replace(text, run+"\n"+gate, gate+"\n"+run, 1), + "conditional run": strings.Replace(text, run, "\tif false {\n"+run+"\n\t}", 1), + "skip": strings.Replace(text, run, "\tt.Skip(\"tool\")\n"+run, 1), + "early return": strings.Replace(text, run, "\treturn\n"+run, 1), + "guest prefix": strings.Replace(text, "{llc: llc,", "{prefix: []string{\"guest\"}, llc: llc,", 1), + "legacy only": strings.Replace(text, run, strings.Replace(run, "true", "false", 1), 1), + "wrong host": strings.Replace(text, "linux && amd64", "darwin && amd64", 1), + "missing root": strings.Replace(text, "TestCrossLinuxRuntimeMatrixX86PackedStringOriginalNativeMemoryFaults", "NoNativeOwner", 1), + } +} + +func TestX86PackedStringFaultNativePolicyRejectsMutation(t *testing.T) { + source, err := os.ReadFile("amd64_packed_string_fault_linux_amd64_test.go") + if err != nil { + t.Fatal(err) + } + text := string(normalizeNativeOwnerPolicySource(source)) + for name, mutation := range x86PackedStringFaultNativeMutations(text) { + t.Run(name, func(t *testing.T) { + if mutation == text || x86PackedStringFaultNativePolicy([]byte(mutation)) == nil { + t.Fatal("native lifecycle mutation not rejected") + } + }) + } +} diff --git a/amd64_packed_string_status_observation_test.go b/amd64_packed_string_status_observation_test.go new file mode 100644 index 000000000..42d577634 --- /dev/null +++ b/amd64_packed_string_status_observation_test.go @@ -0,0 +1,28 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86PackedStringFullFlagsObservationNeedsIndependentDFAndID(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range strings.Fields("PCMPESTRI PCMPESTRM PCMPISTRI PCMPISTRM VPCMPESTRI VPCMPESTRM VPCMPISTRI VPCMPISTRM") { + t.Run(arch+"/"+op, func(t *testing.T) { + declaration, source := x86PackedStringVectorFixture([]x86PackedStringVectorCase{{op, 0, 1, false, 0}}, arch) + original := strings.Replace(source, x86PackedStringStatusObservation(arch, true), x86PackedStringStatusObservation(arch, false), 1) + if original == source { + t.Fatal("original unrefined FLAGS observation was not retained") + } + x86LEAActualTargetGoObject(t, arch, x86AddressMemoryTargets(arch)[0], original) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, original, declaration, arch, x86AddressMemoryTargets(arch)[0], api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("six arithmetic flags supplied unknown incoming DF/ID bits", api, err) + } + } + }) + } + } +} diff --git a/amd64_packed_string_vector_linux_amd64_test.go b/amd64_packed_string_vector_linux_amd64_test.go new file mode 100644 index 000000000..8b60d50a0 --- /dev/null +++ b/amd64_packed_string_vector_linux_amd64_test.go @@ -0,0 +1,41 @@ +//go:build linux && amd64 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The portable byte oracle does not execute a VEX opcode. This required native +// counterpart executes actual Go ABI0 AVX sources and both LLVM levels, with +// complete Y aliases, six flags/LAHF/index and independent input canaries. +func TestCrossLinuxRuntimeMatrixX86PackedStringImplicitVectorNativeAVX(t *testing.T) { + x87IA32RequireNativeHost(t) + execution := x86FloatCompareExecutionTools(t) + execution.compiler = append(execution.compiler, "-mno-red-zone") + execution.requireAVX(t) + cases, declarations, source := x86PackedStringNativeYFixture() + original := x86DefinedStatusBuildOriginal(t, execution, declarations, source, x86PackedStringNativeGoDriver(cases)) + output := execution.run(t, original) + want := fmt.Sprintf("original native AVX packed-string observations %d failures 0\n", len(cases)*4) + if string(output) != want { + t.Fatalf("actual original Go AVX byte oracle: %s", output) + } + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, "amd64", execution.triple, "Go") + main := strings.Replace(x86PackedStringVectorCOracle(cases), "view<3", "view<2", 1) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + output := execution.compileAndRunLLVM(t, "native-packed-string-vector", input, main, level) + want := fmt.Sprintf("portable packed-string byte observations %d failures 0\n", len(cases)*4) + if string(output) != want { + t.Fatalf("native LLVM AVX byte oracle: %s", output) + } + }) + } +} diff --git a/amd64_packed_string_vector_runtime_test.go b/amd64_packed_string_vector_runtime_test.go new file mode 100644 index 000000000..f8573a706 --- /dev/null +++ b/amd64_packed_string_vector_runtime_test.go @@ -0,0 +1,319 @@ +//go:build unix + +package plan9asm + +import ( + "crypto/sha256" + "debug/elf" + "fmt" + "os" + "regexp" + "strings" + "testing" +) + +func x86PackedStringVectorCOracle(cases []x86PackedStringVectorCase) string { + var functions, registrations strings.Builder + for index, row := range cases { + name := fmt.Sprintf("stringvector%d", index) + fmt.Fprintf(&functions, "extern void %s(uint8_t *,const uint8_t *);\n", name) + // Native legacy SSE supplies an independent low-mask/index and flags + // oracle. VEX upper-zeroing is checked separately against the ISA rule; + // this portable test never claims execution of a native VEX instruction. + fmt.Fprintf(&functions, `static void reference%d(const uint8_t *left,const uint8_t *right,uint8_t *mask,uint64_t *flags,uint64_t *index) { + uint64_t a=5,d=9,c=73,f; + __asm__ volatile( + "movdqu (%%[left]),%%%%xmm1\n\t" + "movdqu (%%[right]),%%%%xmm2\n\t" + "%s $%d,%%%%xmm2,%%%%xmm1\n\t" + "pushfq\n\t" + "popq %%[flags]\n\t" + "movdqu %%%%xmm0,(%%[mask])\n\t" + : "+a"(a),"+d"(d),"+c"(c),[flags]"=&r"(f) + : [left]"r"(left),[right]"r"(right),[mask]"r"(mask) + : "xmm0","xmm1","xmm2","cc","memory"); + *flags=f&UINT64_C(0x8d5); + *index=c; +} + +`, index, strings.ToLower(strings.TrimPrefix(row.op, "V")), row.imm) + fmt.Fprintf(®istrations, "{%s,reference%d,%d,%d,%d,%d,%q},\n", name, index, + row.source, row.dst, x86FloatAddSubBoolInt(strings.HasSuffix(row.op, "M")), + x86FloatAddSubBoolInt(strings.HasPrefix(row.op, "V")), row.op) + } + return `#include +#include +#include +` + functions.String() + ` +struct program { + void (*run)(uint8_t *,const uint8_t *); + void (*reference)(const uint8_t *,const uint8_t *,uint8_t *,uint64_t *,uint64_t *); + int source,destination,mask,vector; + const char *name; +}; +static const struct program programs[]={ +` + registrations.String() + `}; + +int main(void) { + unsigned observations=0; + for (unsigned program=0;programreference(input+1+p->destination*64,input+1+p->source*64,mask,&flags,&index); + p->run(output+1,input+1); + __asm__ volatile("stmxcsr %0":"=m"(after)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + memcpy(&gotFlags,output+1+336,8); + memcpy(&gotIndex,output+1+352,8); + uint8_t lahf=(uint8_t)((flags&UINT64_C(0xd5))|2); + if (gotFlags!=flags || output[1+344]!=lahf || gotIndex!=index || + after!=before || output[0]!=0xc3 || output[361]!=0xc3 || memcmp(input,original,sizeof(input))) { + fprintf(stderr,"%s/%u flags/index/input/canary mismatch\n",p->name,program); + return 1; + } + const unsigned widths[]={16,32,64},offsets[]={0,16,48}; + for (unsigned reg=0;reg<3;++reg) { + for (unsigned view=0;view<3;++view) { + for (unsigned byte=0;bytemask && reg==0) { + if (byte<16) { + want=mask[byte]; + } else if (p->vector) { + want=0; + } + } + uint8_t got=output[1+reg*112+offsets[view]+byte]; + if (got!=want) { + fprintf(stderr,"%s/%u round=%u bank=%u width=%u byte=%u got=%u want=%u\n", + p->name,program,round,reg,widths[view],byte,got,want); + return 1; + } + } + } + } + ++observations; + } + } + printf("portable packed-string byte observations %u failures 0\n",observations); + return 0; +} +` +} + +func TestX86PackedStringImplicitVectorByteOracle(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + execution.compiler = append(execution.compiler, "-mno-red-zone") + cases := x86PackedStringVectorCases() + declarations, source := x86PackedStringVectorFixture(cases, "amd64") + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, "amd64", execution.triple, "Go") + // Only the execution target-feature request changes: all source operations + // and LLVM byte arithmetic stay intact. This makes the wide virtual bank + // oracle executable using SSE4.2, without asserting native AVX/AVX512. + features := regexp.MustCompile(`"target-features"="([^"]*)"`) + ir = features.ReplaceAllStringFunc(ir, func(attribute string) string { + values := features.FindStringSubmatch(attribute)[1] + for _, value := range strings.Split(values, ",") { + switch value { + case "+avx", "+avx512f", "+avx512vl", "+sse4.2": + default: + t.Fatalf("unreviewed portable feature request: %q", value) + } + } + return `"target-features"="+sse4.2"` + }) + if strings.Contains(ir, `"target-features"="+avx`) { + t.Fatal("portable oracle retained an unreviewed native AVX feature request") + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + output := execution.compileAndRunLLVM(t, "packed-string-vector-bytes", input, x86PackedStringVectorCOracle(cases), level) + want := fmt.Sprintf("portable packed-string byte observations %d failures 0\n", len(cases)*4) + if string(output) != want { + t.Fatalf("incomplete byte oracle: %s", output) + } + }) + } +} + +func x86PackedStringNativeYFixture() ([]x86PackedStringVectorCase, string, string) { + var cases []x86PackedStringVectorCase + for _, row := range x86PackedStringVectorCases() { + if row.imm == 8 { + cases = append(cases, row) + } + } + declarations, source := x86PackedStringVectorFixture(cases, "amd64") + // Actual native AVX has 32-byte banks. Z-bank execution is a distinct + // AVX512 requirement and is not silently inferred from this counterpart. + for reg := 0; reg < 3; reg++ { + source = strings.ReplaceAll(source, fmt.Sprintf("VMOVUPS %d(SI),Z%d\n", reg*64, reg), + fmt.Sprintf("VMOVUPS %d(SI),Y%d\n", reg*64, reg)) + source = strings.ReplaceAll(source, fmt.Sprintf("VMOVUPS Z%d,%d(DI)\n", reg, reg*112+48), "") + } + return cases, declarations, source +} + +func x86PackedStringNativeGoDriver(cases []x86PackedStringVectorCase) string { + var registrations strings.Builder + for index, row := range cases { + fmt.Fprintf(®istrations, "{stringvector%d,%d,%d,%t,%t,%t,%q},\n", index, row.source, row.dst, + strings.HasSuffix(row.op, "M"), strings.HasPrefix(row.op, "V"), + strings.Contains(row.op, "ESTR"), row.op) + } + return `package main + +import ( + "encoding/binary" + "fmt" + "os" + "unsafe" +) + +type program struct { + run func(*[360]byte,*[192]byte) + source,destination int + mask,vector,explicit bool + name string +} + +var programs=[]program{ +` + registrations.String() + `} + +func fail(name string,round int,detail string) { + fmt.Fprintf(os.Stderr,"%s round=%d %s\n",name,round,detail) + os.Exit(1) +} + +func main() { + observations:=0 + for _,p:=range programs { + for round:=0;round<4;round++ { + input:=[194]byte{} + output:=[362]byte{} + for i:=range input { + input[i]=byte(i*73+round*19+1) + } + input[1+3]=0 + input[1+64+7]=0 + input[1+128+11]=0 + original:=input + for i:=range output { + output[i]=0xc3 + } + p.run((*[360]byte)(unsafe.Pointer(&output[1])),(*[192]byte)(unsafe.Pointer(&input[1]))) + if input!=original || output[0]!=0xc3 || output[361]!=0xc3 { + fail(p.name,round,"input/canary mismatch") + } + left:=input[1+p.destination*64:1+p.destination*64+16] + right:=input[1+p.source*64:1+p.source*64+16] + la,lb:=5,9 + if !p.explicit { + la,lb=16,16 + for i:=0;i<16;i++ { + if left[i]==0 && la==16 { + la=i + } + if right[i]==0 && lb==16 { + lb=i + } + } + } + var result uint16 + for i:=0;i<16;i++ { + if i>=la && i>=lb || i>uint(b*8)) + } + } else if p.vector { + want=0 + } + } + if got:=output[1+reg*112+offsets[view]+b];got!=want { + fail(p.name,round,fmt.Sprintf("bank=%d width=%d byte=%d got=%d want=%d",reg,width,b,got,want)) + } + } + } + } + observations++ + } + } + fmt.Printf("original native AVX packed-string observations %d failures 0\n",observations) +} +` +} + +func TestX86PackedStringNativeAVXCounterpartOriginalObject(t *testing.T) { + cases, declarations, source := x86PackedStringNativeYFixture() + execution := x86FloatCompareExecution{triple: "x86_64-unknown-linux-gnu"} + executable := x86DefinedStatusBuildOriginal(t, execution, declarations, source, x86PackedStringNativeGoDriver(cases)) + object, err := elf.Open(executable) + if err != nil { + t.Fatal(err) + } + defer object.Close() + if object.Class != elf.ELFCLASS64 || object.Machine != elf.EM_X86_64 || object.Data != elf.ELFDATA2LSB { + t.Fatal("native AVX counterpart is not an actual Linux/amd64 ELF") + } + bytes, err := os.ReadFile(executable) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go Linux/amd64 ELF bytes=%d sha=%x; compile-only, native execution remains required", len(bytes), sha256.Sum256(bytes)) +} diff --git a/amd64_packed_string_vector_test.go b/amd64_packed_string_vector_test.go new file mode 100644 index 000000000..e24b128b8 --- /dev/null +++ b/amd64_packed_string_vector_test.go @@ -0,0 +1,441 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86PackedStringCompleteActualGoTableAttributes(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + read := func(name string) string { + data, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + return strings.Join(strings.Fields(string(data)), "") + } + legacy, vector := read("asm6.go"), read("avx_optabs.go") + for _, table := range []struct{ source, definition string }{ + {legacy, "var yxshuf = []ytab{ {Zibm_r, 2, argList{Yu8, Yxm, Yxr}}, }"}, + {vector, "var _yvaeskeygenassist = []ytab{ {zcase: Zvex_i_rm_r, zoffset: 0, args: argList{Yu8, Yxm, Yxr}}, {zcase: Zvex_i_rm_r, zoffset: 2, args: argList{Yi8, Yxm, Yxr}}, }"}, + } { + if !strings.Contains(table.source, strings.Join(strings.Fields(table.definition), "")) { + t.Fatal("actual selected SDK packed-string complete operand table changed", table.definition) + } + } + for _, row := range []struct { + op, opcode, width string + }{ + {"PCMPESTRI", "61", "WIG"}, {"PCMPESTRM", "60", "WIG"}, + {"PCMPISTRI", "63", "WIG"}, {"PCMPISTRM", "62", "W0"}, + } { + legacyRecord := fmt.Sprintf("{A%s,yxshuf,Pq,opBytes{0x3a,0x%s,0}}", row.op, row.opcode) + vectorRecord := fmt.Sprintf("{as:AV%s,ytab:_yvaeskeygenassist,prefix:Pavx,op:opBytes{avxEscape|vex128|vex66|vex0F3A|vex%s,0x%s,}},", row.op, row.width, row.opcode) + if !strings.Contains(legacy, legacyRecord) || !strings.Contains(vector, vectorRecord) { + t.Fatalf("actual selected SDK opcode/prefix/width attributes changed: %s", row.op) + } + } + // Count actual family registrations as well as checking each expected row. + // A shared ytab alone is not evidence of equal WIG or prefix attributes. + pattern := regexp.MustCompile(`\{(?:as:)?A(V?PCMP[EI]STR[[:alnum:]]+),`) + actual := make(map[string]int) + for _, source := range []string{legacy, vector} { + for _, match := range pattern.FindAllStringSubmatch(source, -1) { + actual[match[1]]++ + } + } + if len(actual) != len(amd64PackedStringCompareSpecs) { + t.Fatal("actual selected SDK packed-string family denominator changed", actual) + } + for opcode := range amd64PackedStringCompareSpecs { + if actual[string(opcode)] != 1 { + t.Fatalf("actual selected SDK family registration is missing/duplicate: %s", opcode) + } + } +} + +type x86PackedStringVectorCase struct { + op string + source, dst int + memory bool + imm int +} + +func x86PackedStringVectorCases() []x86PackedStringVectorCase { + var cases []x86PackedStringVectorCase + for _, op := range []string{ + "PCMPESTRI", "PCMPESTRM", "PCMPISTRI", "PCMPISTRM", + "VPCMPESTRI", "VPCMPESTRM", "VPCMPISTRI", "VPCMPISTRM", + } { + for _, operands := range []struct { + source, dst int + memory bool + }{{0, 1, false}, {1, 0, false}, {0, 0, false}, {1, 1, false}, {2, 1, false}, {2, 0, true}, {2, 1, true}} { + for _, immediate := range []int{0, 8, 64} { + cases = append(cases, x86PackedStringVectorCase{op, operands.source, operands.dst, operands.memory, immediate}) + } + } + } + return cases +} + +func x86PackedStringVectorFixture(cases []x86PackedStringVectorCase, arch string) (string, string) { + var declarations, source strings.Builder + move, pointer, args := "MOVQ", 8, 16 + if arch == "386" { + move, pointer, args = "MOVL", 4, 8 + } + for index, row := range cases { + name := fmt.Sprintf("stringvector%d", index) + fmt.Fprintf(&declarations, "func %s(out *[360]byte, in *[192]byte)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s out+0(FP),DI\n%s in+%d(FP),SI\n", name, args, move, move, pointer) + for register := 0; register < 3; register++ { + fmt.Fprintf(&source, "VMOVUPS %d(SI),Z%d\n", register*64, register) + } + source.WriteString("MOVL $5,AX\nMOVL $9,DX\nMOVL $73,CX\n") + operand := fmt.Sprintf("X%d", row.source) + if row.memory { + operand = fmt.Sprintf("%d(SI)", row.source*64) + } + fmt.Fprintf(&source, "%s $%d,%s,X%d\n", row.op, row.imm, operand, row.dst) + for register := 0; register < 3; register++ { + for _, view := range []struct { + bank string + offset int + }{{"X", 0}, {"Y", 16}, {"Z", 48}} { + fmt.Fprintf(&source, "VMOVUPS %s%d,%d(DI)\n", view.bank, register, register*112+view.offset) + } + } + source.WriteString(x86PackedStringStatusObservation(arch, true)) + } + return declarations.String(), source.String() +} + +func x86PackedStringStatusObservation(arch string, refined bool) string { + suffix := "Q" + if arch == "386" { + suffix = "L" + } + if !refined { + return "PUSHF" + suffix + "\nPOP" + suffix + " AX\nMOV" + suffix + + " AX,336(DI)\nLAHF\nMOVB AH,344(DI)\nMOV" + suffix + " CX,352(DI)\nRET\n" + } + // PCMP defines all six arithmetic flags, not incoming DF/ID. Capture LAHF + // first, then refine the popped FLAGS immediately, before any other effect + // can invalidate its transient bit knowledge or AND can change FLAGS. + return "LAHF\nMOVB AH,344(DI)\nPUSHF" + suffix + "\nPOP" + suffix + " BX\nAND" + + suffix + " $0x8d5,BX\nMOV" + suffix + " BX,336(DI)\nMOV" + suffix + " CX,352(DI)\nRET\n" +} + +func x86PackedStringObjectBatches(rows []x86PackedStringVectorCase) [][]x86PackedStringVectorCase { + const maxFunctions = 64 + var batches [][]x86PackedStringVectorCase + for len(rows) != 0 { + count := len(rows) + if count > maxFunctions { + count = maxFunctions + } + batches = append(batches, rows[:count]) + rows = rows[count:] + } + return batches +} + +func TestX86PackedStringObjectBatchInventory(t *testing.T) { + original := x86PackedStringVectorCases() + var joined []x86PackedStringVectorCase + seen := make(map[x86PackedStringVectorCase]bool) + for _, batch := range x86PackedStringObjectBatches(original) { + if len(batch) == 0 || len(batch) > 64 { + t.Fatal("empty or oversized packed-string object batch", len(batch)) + } + for _, row := range batch { + if seen[row] { + t.Fatal("duplicate original packed-string form", row) + } + seen[row] = true + } + joined = append(joined, batch...) + } + if len(original) != 8*7*3 || !reflect.DeepEqual(joined, original) { + t.Fatal("bounded batches changed the complete ordered original form inventory") + } + owners := map[string]string{ + "i386-unknown-linux-gnu": "386", + "i686-pc-windows-msvc": "386", + "x86_64-apple-darwin": "amd64", + "x86_64-unknown-linux-gnu": "amd64", + "x86_64-pc-windows-msvc": "amd64", + } + if len(x86RawReturnTargets) != len(owners) { + t.Fatal("new object targets need their own complete packed-string owner") + } + for _, target := range x86RawReturnTargets { + if owners[target.triple] != target.arch { + t.Fatal("missing or duplicate complete target owner", target) + } + delete(owners, target.triple) + } + if len(owners) != 0 { + t.Fatal("orphaned packed-string object owner", owners) + } +} + +// Each target is independently schedulable, and each bounded LLVM unit is +// released before the next batch. Partitioning retains all 168 original rows, +// all five targets and the original three-API/two-optimization matrix. +func x86PackedStringTargetObjects(t *testing.T, arch, triple string) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc is required") + } + for index, batch := range x86PackedStringObjectBatches(x86PackedStringVectorCases()) { + t.Run(fmt.Sprintf("batch-%d", index), func(t *testing.T) { + declarations, source := x86PackedStringVectorFixture(batch, arch) + x86LEAActualTargetGoObject(t, arch, triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, arch, triple, api) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + x86PackedStringCompileObject(t, llc, triple, input, level) + }) + } + }) + } + }) + } +} + +func TestX86PackedStringObjectsLinux386(t *testing.T) { + x86PackedStringTargetObjects(t, "386", "i386-unknown-linux-gnu") +} + +func TestX86PackedStringObjectsWindows386(t *testing.T) { + x86PackedStringTargetObjects(t, "386", "i686-pc-windows-msvc") +} + +func TestX86PackedStringObjectsDarwinAMD64(t *testing.T) { + x86PackedStringTargetObjects(t, "amd64", "x86_64-apple-darwin") +} + +func TestX86PackedStringObjectsLinuxAMD64(t *testing.T) { + x86PackedStringTargetObjects(t, "amd64", "x86_64-unknown-linux-gnu") +} + +func TestX86PackedStringObjectsWindowsAMD64(t *testing.T) { + x86PackedStringTargetObjects(t, "amd64", "x86_64-pc-windows-msvc") +} + +func TestX86PackedStringResultEffectsAndPreEmissionIdentity(t *testing.T) { + for _, row := range go127PackedStringCompareSpecs() { + spec := amd64PackedStringCompareSpecs[Op(row.op)] + reg, width, zeroUpper := spec.resultEffect() + wantReg, wantWidth := CX, 4 + if row.mask { + wantReg, wantWidth = "X0", 16 + } + if reg != wantReg || width != wantWidth || zeroUpper != (row.vector && row.mask) { + t.Fatalf("%s implicit writer effect = %s/%d/%t", row.op, reg, width, zeroUpper) + } + for _, identity := range []Op{"", Op(row.op), Op(strings.ToLower(row.op)), "MOVOU", "UNKNOWN"} { + for _, arch := range []string{"amd64", "386"} { + t.Run(row.op+"/"+arch+"/"+string(identity), func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + instruction := Instr{Op: identity, Args: []Operand{ + {Kind: OpImm, Imm: 8}, {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, + }} + before, temporary := output.String(), ctx.tmp + original := fmt.Sprintf("%#v", instruction) + known, terminated, err := ctx.lowerPackedStringCompare(Op(row.op), instruction) + invalid := identity != "" && !strings.EqualFold(string(identity), row.op) + if !known || terminated || (err != nil) != invalid || invalid && (before != output.String() || temporary != ctx.tmp) || + original != fmt.Sprintf("%#v", instruction) { + t.Fatalf("implicit writer bypassed source identity or emitted before rejection: %v", err) + } + }) + } + } + } +} + +func TestX86PackedStringImmediateCompleteByteDomainOriginalObjects(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT ·query(SB),4,$0-0\n") + for _, row := range go127PackedStringCompareSpecs() { + first := 0 + if row.vector { + first = -128 + } + for immediate := first; immediate <= 255; immediate++ { + fmt.Fprintf(&source, "%s $%d,X0,X1\n", row.op, immediate) + } + } + source.WriteString("RET\n") + x86LEAActualTargetGoObject(t, arch, testTargetTriple("linux", arch), source.String()) + }) + } +} + +func TestX86PackedStringStrictMalformedOperands(t *testing.T) { + for _, row := range go127PackedStringCompareSpecs() { + for _, operands := range []string{ + "$256,X0,X1", "$-129,X0,X1", "$0.0,X0,X1", "$8,Y0,X1", + "$8,X0,X16", "$8,X0,Y1", "$8,X0,(AX)", "$8,X0,X1,X2", + } { + t.Run(row.op+"/"+operands, func(t *testing.T) { + source := "TEXT ·query(SB),4,$0-0\n" + row.op + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: func(symbol string) string { + return strings.TrimPrefix(symbol, "·") + }, Sigs: map[string]FuncSig{"query": {Name: "query", Ret: Void}}}) + if err == nil || ir != "" { + t.Fatal("malformed implicit vector writer returned IR", err) + } + }) + } + } +} + +func TestX86PackedStringMode32HighBankNeedsPhysicalContext(t *testing.T) { + for _, op := range []string{"VPCMPESTRI", "VPCMPESTRM", "VPCMPISTRI", "VPCMPISTRM"} { + for _, operands := range []string{"X8,X1", "X1,X8", "X15,X15"} { + t.Run(op+"/"+operands, func(t *testing.T) { + source := fmt.Sprintf("TEXT ·query(SB),4,$0-0\n%s $8,%s\nRET\n", op, operands) + goAssemblerOracleOutput(t, "386", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "query", "package query\nfunc query()\n") + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "386", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("frontend high spelling acquired a physical mode32 vector binding: %v", err) + } + }) + } + } +} + +func TestX86PackedStringDoesNotSupplyCallerDefinedness(t *testing.T) { + for _, op := range []string{ + "PCMPESTRI", "PCMPESTRM", "PCMPISTRI", "PCMPISTRM", + "VPCMPESTRI", "VPCMPESTRM", "VPCMPISTRI", "VPCMPISTRM", + } { + for _, operand := range []string{"X0", "(DI)", "(DX)"} { + t.Run(op+"/"+operand, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", op+" $8,"+operand+",X1\nMOVL AX,(DX)", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unregistered implicit vector inputs/writers acquired caller permission: %v", err) + } + }) + } + } +} + +func TestX86PackedStringDescriptorDoesNotInventInitializedBytes(t *testing.T) { + for _, row := range go127PackedStringCompareSpecs() { + instruction := Instr{Op: Op(row.op), Args: []Operand{ + {Kind: OpImm, Imm: 8}, {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, + }} + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0xffff, "Z1": 0xff}, masks: map[Reg]bool{"K1": true}} + before := state + before.vectors, before.masks = x86CloneCallerVectorState(state.vectors, state.masks) + if x86CallerVectorMoveKnown(instruction) || x86CallerVectorMoveStep(&instruction, &state, nil) || + x86SourceStackInstructionKnown(instruction, OpMOVQ, "Q", "amd64") { + t.Fatalf("%s descriptor granted an unregistered caller effect", row.op) + } + if !reflect.DeepEqual(state, before) || instruction.x86CallerMemory != nil { + t.Fatalf("%s failed caller effect mutated byte knowledge or native certificates", row.op) + } + } +} + +func TestX86PackedStringIndependentFPFieldsRequireEveryByte(t *testing.T) { + var parameters strings.Builder + for index := 0; index < 16; index++ { + if index != 0 { + parameters.WriteString(",") + } + fmt.Fprintf(¶meters, "a%d uint8", index) + } + pkg := mustGoPackage(t, "strings", "package strings\nfunc query("+parameters.String()+")\n") + for _, row := range go127PackedStringCompareSpecs() { + t.Run(row.op, func(t *testing.T) { + source := "TEXT ·query(SB),4,$0-16\n" + row.op + " $8,a0+0(FP),X1\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + if _, err := Translate(file, options); err != nil { + t.Fatalf("complete independent logical FP fields: %v", err) + } + for missing := 0; missing < 16; missing++ { + t.Run(fmt.Sprint(missing), func(t *testing.T) { + // Each mutation starts from the original complete declaration. + // Logical FP completeness is not native address backing or an + // initialized caller-region witness. + signature := sigs["query"] + signature.Frame.Params = append([]FrameSlot(nil), signature.Frame.Params...) + signature.Frame.Params = append(signature.Frame.Params[:missing], signature.Frame.Params[missing+1:]...) + mutated := options + mutated.Sigs = map[string]FuncSig{"query": signature} + if ir, err := Translate(file, mutated); err == nil || ir != "" { + t.Fatalf("missing logical FP byte%d acquired a complete m128 source: %v", missing, err) + } + }) + } + }) + } +} + +func x86PackedStringCompileObject(t *testing.T, llc, triple, ir, level string) { + t.Helper() + if level != "-O0" && level != "-O2" { + t.Fatal("unexpected packed-string object level", level) + } + dir := t.TempDir() + input, output := filepath.Join(dir, "strings.ll"), filepath.Join(dir, "strings.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+triple, "-filetype=obj", input, "-o", output)) + info, err := os.Lstat(output) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatalf("packed-string object is missing/nonregular/empty: %v", err) + } +} diff --git a/amd64_packed_subtract_fault_runtime_test.go b/amd64_packed_subtract_fault_runtime_test.go new file mode 100644 index 000000000..032d6c1de --- /dev/null +++ b/amd64_packed_subtract_fault_runtime_test.go @@ -0,0 +1,151 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedSubtractNativeMemoryStatus(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, references, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for _, original := range x86PackedSubtractCallerRows() { + parsed, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+original+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := parsed.Funcs[0].Instrs[1] + form, _, err := x86PackedSubtractProperties(instruction, "amd64") + if err != nil { + t.Fatal(err) + } + if form.memory < 0 || form.mask >= 0 && !form.zeroing { + continue + } + masks := []int{0} + if form.mask >= 0 { + masks = []int{0, 1} + } + for _, mask := range masks { + name := fmt.Sprintf("packedsubtractfault%d", index) + reference := fmt.Sprintf("subtractReference%d", index) + index++ + row := strings.ReplaceAll(original, "(DI)", "(BX)") + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-8\nMOVQ input+0(FP),BX\n"+ + "VPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\n"+ + "VPXORD Z15,Z15,Z15\nVPXORD Z21,Z21,Z21\nVPXORD Z22,Z22,Z22\n"+ + "MOVL $%d,CX\nKMOVQ CX,K1\n%s\nRET\n", name, mask, row) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const uint8_t*);\n", name) + if form.legacy { + mnemonic := strings.ToLower(string(instruction.Op)) + if mnemonic == "psubl" { + mnemonic = "psubd" + } + fmt.Fprintf(&references, "__attribute__((noinline)) static void %s(const uint8_t *p) {\n"+ + " typedef int32_t V __attribute__((vector_size(16)));\n V value = {0,0,0,0};\n"+ + " __asm__ volatile(\"%s (%%1),%%0\" : \"+x\"(value) : \"r\"(p) : \"memory\");\n}\n", + reference, mnemonic) + } else { + bytes := form.memoryBytes + if form.mask >= 0 { + bytes = 0 + if mask != 0 { + bytes = form.laneBits / 8 + } + } + fmt.Fprintf(&references, "__attribute__((noinline)) static void %s(const uint8_t *p) {\n"+ + " for (int i = 0; i < %d; ++i) {\n (void)((const volatile uint8_t*)p)[i];\n }\n}\n", + reference, bytes) + } + fmt.Fprintf(&checks, "for(int n=0;n<16;++n) {\n if(compare(%s,%s,memory+n)) return 1;\n}\n", reference, name) + for _, offset := range []int{0, 1, form.laneBits / 8, 16, 32, 64} { + fmt.Fprintf(&checks, "if(compare(%s,%s,boundary-%d)) return 1;\n", reference, name, offset) + } + fmt.Fprintf(&checks, "if(compare(%s,%s,0)) return 1;\n", reference, name) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + references.String() + ` +typedef struct { int signal; int code; } Status; + +static int observe(void (*fn)(const uint8_t*), const uint8_t *p, Status *out) { + pid_t child = fork(); + if (child < 0) return 1; + if (child == 0) { + struct rlimit limit = {0,0}; + if (setrlimit(RLIMIT_CORE,&limit)) _exit(3); + fn(p); + _exit(0); + } + int status; + if (waitpid(child,&status,0) != child) return 1; + if (WIFSIGNALED(status)) { + *out = (Status){WTERMSIG(status),0}; + return 0; + } + if (WIFEXITED(status)) { + *out = (Status){0,WEXITSTATUS(status)}; + return 0; + } + return 1; +} + +static int compare(void (*original)(const uint8_t*), + void (*translated)(const uint8_t*), const uint8_t *p) { + Status a, b; + if (observe(original,p,&a) || observe(translated,p,&b)) return 1; + if (a.signal != b.signal || a.code != b.code) { + fprintf(stderr,"pointer=%p mod16=%lu original=(%d,%d) translated=(%d,%d)\n", + p,(unsigned long)((uintptr_t)p%16),a.signal,a.code,b.signal,b.code); + return 1; + } + return 0; +} + +int main(void) { + long page = sysconf(_SC_PAGESIZE); + if (page <= 0 || page%64) return 1; + uint8_t *memory = mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANON,-1,0); + if (memory == MAP_FAILED) return 1; + memset(memory,0x87,(size_t)page); + if (mprotect(memory,(size_t)page,PROT_READ) || + mprotect(memory+page,(size_t)page,PROT_NONE)) return 1; + uint8_t *boundary = memory+page; +` + checks.String() + ` + if (munmap(memory,(size_t)page*2)) return 1; + return 0; +} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-subtract-fault", triple, text, main, prefix) + }) + } + t.Logf("%d dead-output memory programs at every mod16/NULL/RO guard; native original SSE and independent E4 reads at O0/O2, not AVX512 hardware", index) +} diff --git a/amd64_packed_subtract_forms.go b/amd64_packed_subtract_forms.go new file mode 100644 index 000000000..e33c1ab33 --- /dev/null +++ b/amd64_packed_subtract_forms.go @@ -0,0 +1,12 @@ +package plan9asm + +type x86PackedSubtractForm struct { + x86PackedBinaryForm + laneBits int + mode amd64PackedSubtractMode +} + +func x86PackedSubtractProperties(ins Instr, arch string) (x86PackedSubtractForm, bool, error) { + form, known, err := x86PackedArithmeticProperties(ins.Op, ins, arch, x86PackedArithmeticSubtract) + return x86PackedSubtractForm{form.x86PackedBinaryForm, form.spec.laneBits, form.spec.mode}, known, err +} diff --git a/amd64_packed_subtract_go_oracle_test.go b/amd64_packed_subtract_go_oracle_test.go new file mode 100644 index 000000000..a5b54e7d8 --- /dev/null +++ b/amd64_packed_subtract_go_oracle_test.go @@ -0,0 +1,133 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedSubtractOriginalGoBits(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle + +import "testing" + +func lane(p []byte, n int) uint64 { + var value uint64 + for j := 0; j < n; j++ { + value |= uint64(p[j]) << uint(8*j) + } + return value +} + +func difference(a, b uint64, n, mode int) uint64 { + if mode == 0 { + return b-a + } + if mode == 2 { + if b < a { + return 0 + } + return b-a + } + sign := int64(1) << uint(n*8-1) + left, right := int64(a), int64(b) + if a&uint64(sign) != 0 { + left -= sign*2 + } + if b&uint64(sign) != 0 { + right -= sign*2 + } + result := right-left + if result < -sign { + result = -sign + } + if result > sign-1 { + result = sign-1 + } + return uint64(result) +} + +func TestOriginalSubtract(t *testing.T) { + var patterns [8][16]byte + state := uint32(0x12345678) + for p := range patterns { + for i := range patterns[p] { + state = state*1664525+1013904223 + patterns[p][i] = byte(state>>24) + } + } + for i := range patterns[0] { + patterns[0][i] = 0xff + patterns[1][i] = 1 + patterns[2][i] = 0x80 + patterns[3][i] = 0 + } +`) + opcodes := []string{"PSUBB", "PSUBW", "PSUBL", "PSUBQ", "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW"} + for index, opcode := range opcodes { + bits, mode, _, _ := amd64PackedSubtractProperties(opcode) + name := fmt.Sprintf("packedsubtractgo%d", index) + fmt.Fprintf(&declarations, "func %s(a,b,out *byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\n"+ + "STMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\n"+ + "MOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\n"+ + "MOVOU (AX),X1\nMOVOU (BX),X0\nCMPQ AX,AX\nSTC\n"+ + "%s X1,X0\nMOVOU X0,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\n"+ + "STMXCSR 18(CX)\nLDMXCSR saved-4(SP)\nRET\n", name, opcode) + fmt.Fprintf(&tests, ` + for _, a := range patterns { + for _, b := range patterns { + for csrMode := 0; csrMode < 4; csrMode++ { + var out [24]byte + for i := range out { + out[i] = 0xc3 + } + csr := uint32(0x1f80|csrMode<<13|csrMode) + %[1]s(&a[0],&b[0],&out[1],csr) + if out[0] != 0xc3 || out[23] != 0xc3 || out[17] != 1 || out[18] != 0 { + t.Fatalf("%[1]s flags/canary %%x",out) + } + gotCSR := uint32(out[19]) | uint32(out[20])<<8 | uint32(out[21])<<16 | uint32(out[22])<<24 + if gotCSR != csr { + t.Fatalf("%[1]s CSR %%x != %%x",gotCSR,csr) + } + for i := 0; i < 16; i += %[2]d { + want := difference(lane(a[i:],%[2]d),lane(b[i:],%[2]d),%[2]d,%[3]d) + for j := 0; j < %[2]d; j++ { + if out[i+j+1] != byte(want >> uint(8*j)) { + t.Fatalf("%[1]s byte%%d got%%x want%%x",i+j,out,want) + } + } + } + } + } + } +`, name, bits/8, mode) + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go packed subtract oracle: %v\n%s", err, output) + } + t.Logf("%d actual Go SSE functions × 256 independent full-lane wrapping/saturation/MXCSR/flags/canary cases", len(opcodes)) +} diff --git a/amd64_packed_subtract_low_z_test.go b/amd64_packed_subtract_low_z_test.go new file mode 100644 index 000000000..e3edbcb08 --- /dev/null +++ b/amd64_packed_subtract_low_z_test.go @@ -0,0 +1,96 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86PackedSubtractLowZThreeAPIFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var parameters []string + for _, prefix := range []string{"a", "b"} { + for index := 0; index < 8; index++ { + parameters = append(parameters, fmt.Sprintf("%s%d uint64", prefix, index)) + } + } + declaration := "func Value(" + strings.Join(parameters, ",") + ")" + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + for _, opcode := range []string{"VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW"} { + forms := []struct{ name, instruction string }{ + {"register", opcode + " Z0,Z1,Z2"}, + {"alias", opcode + " Z1,Z1,Z2"}, + {"memory", opcode + " a0+0(FP),Z1,Z2"}, + } + if opcode == "VPSUBD" || opcode == "VPSUBQ" { + forms = append(forms, struct{ name, instruction string }{ + "broadcast", opcode + ".BCST a0+0(FP),Z1,Z2", + }) + } + for _, form := range forms { + t.Run(target.triple+"/"+opcode+"/"+form.name, func(t *testing.T) { + // This fixture establishes form translation and LLVM objects, + // not an initialized numeric return or a caller-region grant. + pkg := mustGoPackage(t, "example.invalid/subtract", "package subtract\n"+declaration+"\n") + sig, err := DeriveGoFuncABI0Signature("Value", pkg.Types.Scope().Lookup("Value").(*types.Func), + target.arch, types.SizesFor("gc", target.arch)) + if err != nil { + t.Fatal(err) + } + body := "VMOVUPS a0+0(FP),Z0\nVMOVUPS b0+64(FP),Z1\n" + + form.instruction + "\nRET" + source := "TEXT ·Value(SB),4,$0-128\n" + body + "\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + goAssemblerOracleOutput(t, target.arch, source, "", true) + options := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Value": sig}, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + before := fmt.Sprintf("%#v %#v", file, sig) + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "text") + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, module.String(), "module") + translated, err := translateGoModuleInContext(context, pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: options.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, translated.Module.String(), "typed-go") + if before != fmt.Sprintf("%#v %#v", file, sig) { + t.Fatal("translation mutated original typed inputs") + } + }) + } + } + } +} diff --git a/amd64_packed_subtract_mode32_test.go b/amd64_packed_subtract_mode32_test.go new file mode 100644 index 000000000..a689f48e1 --- /dev/null +++ b/amd64_packed_subtract_mode32_test.go @@ -0,0 +1,91 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86PackedSubtract386HighRegisterDoesNotAcquirePhysicalBinding(t *testing.T) { + for _, opcode := range []string{"VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW"} { + for _, bank := range []string{"X", "Y"} { + for position := 0; position < 3; position++ { + for _, high := range []int{16, 31} { + args := []string{bank + "0", bank + "1", bank + "2"} + args[position] = fmt.Sprintf("%s%d", bank, high) + row := opcode + " " + strings.Join(args, ",") + t.Run(strings.ReplaceAll(row, " ", "_"), func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + if _, recognized, err := x86PackedSubtractProperties(instruction, "386"); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go-accepted mode32 high bank became a physical-register permission: recognized=%v err=%v", recognized, err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "386" + before := output.String() + if recognized, _, err := ctx.lowerPackedIntegerSubtract(instruction.Op, instruction); !recognized || + !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("unproved mode32 high bank emitted IR: recognized=%v err=%v", recognized, err) + } + }) + } + } + } + } +} + +func TestX86PackedSubtract386HighRegisterActualByteIdentities(t *testing.T) { + for _, opcode := range []string{"VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW"} { + for _, bank := range []string{"X", "Y"} { + for operand := 0; operand < 3; operand++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[operand] = bank + "16" + row := opcode + " " + strings.Join(args, ",") + t.Run(strings.ReplaceAll(row, " ", "_"), func(t *testing.T) { + code := x86PackedFirstGoInstructionBytes(t, "386", row) + ins, length, recognized, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if len(code) != 6 || !recognized || err != nil || length != 6 || ins.Op != Op(opcode) || len(ins.Args) != 3 { + t.Fatalf("actual EVEX byte identity %s/%x: %+v/%d/%t/%v", row, code, ins, length, recognized, err) + } + for index, argument := range args { + if ins.Args[index].Reg != Reg(argument) { + t.Fatalf("%s/%x operand%d: %s != %s", row, code, index, ins.Args[index].Reg, argument) + } + } + if _, _, recognized, err := decodedX86PackedIntegerArithmeticInstruction(code, 32); !recognized || err == nil { + t.Fatalf("raw mode32 high bank obtained false permission: %s/%x", row, code) + } + }) + } + } + } +} + +func TestX86PackedSubtract386BroadcastHighRegisterNeedsContext(t *testing.T) { + for _, opcode := range []string{"VPSUBD", "VPSUBQ"} { + for _, bank := range []string{"X", "Y"} { + for position := 1; position < 3; position++ { + args := []string{"(AX)", bank + "1", bank + "2"} + args[position] = bank + "16" + row := opcode + ".BCST " + strings.Join(args, ",") + t.Run(strings.ReplaceAll(row, " ", "_"), func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, known, err := x86PackedSubtractProperties(file.Funcs[0].Instrs[1], "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("broadcast high bank became a mode32 binding: %v", err) + } + }) + } + } + } +} diff --git a/amd64_packed_subtract_runtime_test.go b/amd64_packed_subtract_runtime_test.go new file mode 100644 index 000000000..75a1e1070 --- /dev/null +++ b/amd64_packed_subtract_runtime_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Only legacy SSE is executed as native instruction evidence. VEX/EVEX +// numerical expectations come from independent scalar lane arithmetic. +const x86PackedSubtractIndependentOracle = ` +static const uint64_t subtractMasks[] = { + 0, 1, UINT64_C(0x55aa55aa55aa55aa), UINT64_MAX +}; + +static uint64_t subtractLane(const uint8_t *p, int bytes) { + uint64_t value = 0; + for (int j = 0; j < bytes; ++j) { + value |= (uint64_t)p[j] << (8*j); + } + return value; +} + +static uint64_t subtractValue(uint64_t a, uint64_t b, int bytes, int mode) { + if (mode == 0) { + return b-a; + } + if (mode == 2) { + return b < a ? 0 : b-a; + } + int bits = bytes*8; + int64_t sign = INT64_C(1) << (bits-1); + int64_t left = (a & sign) ? (int64_t)a - (sign*2) : (int64_t)a; + int64_t right = (b & sign) ? (int64_t)b - (sign*2) : (int64_t)b; + int64_t result = right-left; + if (result < -sign) { + result = -sign; + } + if (result > sign-1) { + result = sign-1; + } + return (uint64_t)result; +} + +__attribute__((noinline)) static int checkSubtract( + void (*fn)(uint8_t*, const uint8_t*, const uint8_t*, uint64_t), + const char *name, int width, int bytes, int arithmetic, + int maskMode, int legacy, int memory, int broadcast) { + for (int round = 0; round < 12; ++round) { + for (int m = 0; m < 4; ++m) { + _Alignas(64) uint8_t aStorage[96], bStorage[96], output[68]; + int shift = legacy && memory ? 16 : 1; + uint8_t *a = aStorage+shift; + uint8_t *b = bStorage+3; + memset(aStorage, 0xa7, sizeof(aStorage)); + memset(bStorage, 0xb9, sizeof(bStorage)); + memset(output, 0xc3, sizeof(output)); + for (int i = 0; i < 64; ++i) { + a[i] = (uint8_t)(i*17+round*31); + b[i] = (uint8_t)(i*23+round*53); + } + if (round < 4) { + memset(a, round == 0 ? 0xff : round == 1 ? 0x80 : round == 2 ? 1 : 0, 64); + memset(b, round == 0 ? 1 : round == 1 ? 0x7f : round == 2 ? 0 : 0xff, 64); + } + uint64_t mask = subtractMasks[m]; + uint32_t saved, before = 0x1f80 | ((round%4)<<13) | round, after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + fn(output+1, a, b, mask); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + if (after != before || output[0] != 0xc3 || output[67] != 0xc3 || + output[65] != 1 || output[66] != 0) { + fprintf(stderr, "%s changed MXCSR, flags or output canaries\n", name); + return 1; + } + for (int i = 0; i < 64; i += bytes) { + uint64_t result = subtractValue(subtractLane(a+(broadcast ? 0 : i), bytes), + subtractLane(b+i, bytes), bytes, arithmetic); + int active = maskMode == 0 || ((mask >> (i/bytes)) & 1); + for (int j = 0; j < bytes; ++j) { + uint8_t want = i < width ? + (active ? (uint8_t)(result >> (8*j)) : maskMode == 2 ? 0 : b[i+j]) : + legacy ? b[i+j] : 0; + if (output[i+j+1] != want) { + fprintf(stderr, "%s round=%d mask=%llx byte=%d got=%x want=%x\n", + name, round, (unsigned long long)mask, i+j, output[i+j+1], want); + return 1; + } + } + } + for (int i = 0; i < 96; ++i) { + if ((i < shift || i >= shift+64) && aStorage[i] != 0xa7) { + return 1; + } + if ((i < 3 || i >= 67) && bStorage[i] != 0xb9) { + return 1; + } + } + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedSubtractCompleteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + rows := x86PackedSubtractCallerRows() + for index, row := range rows { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + form, _, err := x86PackedSubtractProperties(instruction, "amd64") + if err != nil { + t.Fatal(err) + } + mode := 0 + if form.mask >= 0 { + mode = 1 + if form.zeroing { + mode = 2 + } + } + name := fmt.Sprintf("packedsubtract%d", index) + row = strings.ReplaceAll(row, "(DI)", "(BX)") + register, valid := amd64VectorRegisterIndex(instruction.Args[form.destination].Reg, form.byteWidth) + if !valid { + t.Fatal("missing actual vector destination") + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n"+ + "MOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\n"+ + "VMOVUPS (BX),Z0\nVMOVUPS (BX),Z20\n"+ + "VMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nVMOVUPS (CX),Z15\n"+ + "VMOVUPS (CX),Z21\nVMOVUPS (CX),Z22\nKMOVQ DX,K1\n"+ + "CMPQ AX,AX\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", + name, row, register) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(checkSubtract(%s,\"%s\",%d,%d,%d,%d,%d,%d,%d)) return 1;\n", + name, name, form.byteWidth, form.laneBits/8, form.mode, mode, + x86FloatAddSubBoolInt(form.legacy), x86FloatAddSubBoolInt(form.memory >= 0), + x86FloatAddSubBoolInt(form.broadcast)) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := "#include \n#include \n#include \n" + + declarations.String() + x86PackedSubtractIndependentOracle + + "int main(void){\n" + checks.String() + "return 0;}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-subtract-complete", triple, text, main, prefix) + }) + } + t.Logf("%d programs × 48 independent wrapping/saturating/mask/upper-view/flags/MXCSR/canary cases at O0/O2; not native AVX512 execution", len(rows)) +} diff --git a/amd64_packed_testmask_forms.go b/amd64_packed_testmask_forms.go new file mode 100644 index 000000000..20db9ec47 --- /dev/null +++ b/amd64_packed_testmask_forms.go @@ -0,0 +1,100 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedTestMaskForm struct { + x86PackedBinaryForm + spec amd64PackedTestMaskSpec + source Operand +} + +// Source vectors, optional K input and K result are disjoint roles. The +// shared mask-result grammar grants no numerical bytes or memory backing. +func x86PackedTestMaskProperties(op Op, ins Instr, arch string) (x86PackedTestMaskForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := amd64PackedTestMaskSpecs[base] + f := x86PackedTestMaskForm{spec: spec} + if !known { + return f, false, nil + } + fail := func() (x86PackedTestMaskForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside packed integer test-to-mask grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + local := ins + local.Op = op + roles, valid := x86PackedMaskCompareProperties(local, arch, 0, spec.laneBits, dotted, suffix) + if !valid { + return fail() + } + f.x86PackedBinaryForm, f.source = roles, ins.Args[roles.first] + for _, index := range []int{roles.first, roles.second} { + arg := ins.Args[index] + if arg.Kind == OpReg && !amd64GoPackedVectorMoveRegister(arg, roles.byteWidth, arch) { + return fail() + } + } + if roles.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + return f, true, fmt.Errorf("%w: test-to-mask has no bound address-size override", ErrProbeNeedsContext) + } + return f, true, nil +} + +func x86TypedPackedTestMaskByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + f, known, err := x86PackedTestMaskProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + firstWidth := f.byteWidth + if f.memory >= 0 { + firstWidth = f.memoryBytes + } + effect := x86TypedByteEffect{requiresResultProof: true, + vector: x86TypedVectorEffect{memoryAlignment: 1}, + operands: []x86TypedByteOperand{ + {index: f.first, operand: f.source, width: int64(firstWidth), read: true}, + {index: f.second, operand: ins.Args[f.second], width: int64(f.byteWidth), read: true}, + {index: f.destination, operand: ins.Args[f.destination], width: 8, write: true}, + }} + if f.mask >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{index: f.mask, operand: ins.Args[f.mask], width: 8, read: true}) + } + return effect, true, nil +} + +func x86ResultPackedTestMaskStep(ins Instr, state *x86ResultState, owned []bool, arch string, native bool) (bool, bool) { + f, known, err := x86PackedTestMaskProperties(ins.Op, ins, arch) + if !known || err != nil || !x86ResultNamedPredicateInstruction(ins) { + return false, known + } + if f.mask >= 0 { + mask, _ := amd64ParseKReg(ins.Args[f.mask].Reg) + if !state.masks[mask] { + return false, true + } + } + for _, index := range []int{f.first, f.second} { + operand := ins.Args[index] + if index == f.memory { + if !x86ResultReadWithNativeContinuation(ins, operand, int64(f.memoryBytes), state, owned, arch, native) { + return false, true + } + continue + } + vector, valid := amd64VectorRegisterIndex(operand.Reg, f.byteWidth) + required := x86CallerVectorByteMask(int64(f.byteWidth)) + if !valid || state.vectorBytes(vector)&required != required { + return false, true + } + } + // Even equal source registers depend on their actual intersection bits. + // Disabled and high K bits are written zero, never merged from old K. + destination, _ := amd64ParseKReg(ins.Args[f.destination].Reg) + state.masks[destination] = true + return true, true +} diff --git a/amd64_packed_unpack_fault_runtime_test.go b/amd64_packed_unpack_fault_runtime_test.go new file mode 100644 index 000000000..81fecb232 --- /dev/null +++ b/amd64_packed_unpack_fault_runtime_test.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86PackedUnpackFaultProgram() (string, map[string]FuncSig, string, string) { + var assembly, declarations, rows strings.Builder + sigs := map[string]FuncSig{} + for index, row := range []struct { + instruction string + bytes int + }{ + {"VPUNPCKLBW.Z (BX),Z0,K1,Z1", 64}, + {"VPUNPCKHWD.Z (BX),Y0,K1,Y1", 32}, + {"VUNPCKHPS.BCST.Z (BX),X0,K1,X1", 4}, + {"VUNPCKHPD.BCST.Z (BX),Z0,K1,Z1", 8}, + } { + name := fmt.Sprintf("unpackfault%d", index) + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-16\nMOVQ input+0(FP),BX\nMOVQ out+8(FP),AX\nVPXORD Z0,Z0,Z0\nMOVL $0,CX\nKMOVQ CX,K1\n%s\nVMOVUPS Z1,(AX)\nRET\n", name, row.instruction) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(const uint8_t*,uint8_t*);\n", name) + fmt.Fprintf(&declarations, `static int observe%d(const uint8_t *input) { + uint8_t output[66]; + memset(output, 0xc3, sizeof(output)); + %s(input, output + 1); + if (output[0] != 0xc3 || output[65] != 0xc3) return 1; + for (unsigned i = 1; i <= 64; ++i) if (output[i]) return 2; + return 0; +} +`, index, name) + fmt.Fprintf(&rows, "{observe%d, %d, %d},\n", index, row.bytes, row.bytes-1) + } + return assembly.String(), sigs, declarations.String(), rows.String() +} + +func TestCrossLinuxRuntimeMatrixX86PackedUnpackZeroMaskStillFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + source, sigs, declarations, rows := x86PackedUnpackFaultProgram() + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := x86MemoryFaultDriver(declarations, rows) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := execution.compileLLVM(t, "unpack-e4nf-fault", ir, main, level) + x86CheckMemoryFaultMatrix(t, execution, executable, 4) + }) + } +} diff --git a/amd64_packed_unpack_forms.go b/amd64_packed_unpack_forms.go new file mode 100644 index 000000000..8f454983c --- /dev/null +++ b/amd64_packed_unpack_forms.go @@ -0,0 +1,106 @@ +package plan9asm + +import "strings" + +type x86PackedUnpackSpec struct { + laneBits int + high bool + grammar x86PackedBinarySpec +} + +// Go's six ymm rows include MMX. Its other six legacy rows use yxm. +// All twelve vector rows use _yvandnpd, but only D/Q/PS/PD permit BCST. +// Their EVEX memory exception class is E4NF: masking does not suppress faults. +var x86PackedUnpackSpecs = map[Op]x86PackedUnpackSpec{ + "PUNPCKLBW": {8, false, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKHBW": {8, true, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKLWL": {16, false, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKHWL": {16, true, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKLLQ": {32, false, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKHLQ": {32, true, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKLQDQ": {64, false, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "PUNPCKHQDQ": {64, true, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "UNPCKLPS": {32, false, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "UNPCKHPS": {32, true, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "UNPCKLPD": {64, false, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "UNPCKHPD": {64, true, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "VPUNPCKLBW": {8, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 0, true}}, + "VPUNPCKHBW": {8, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 0, true}}, + "VPUNPCKLWD": {16, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 0, true}}, + "VPUNPCKHWD": {16, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 0, true}}, + "VPUNPCKLDQ": {32, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 4, true}}, + "VPUNPCKHDQ": {32, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 4, true}}, + "VPUNPCKLQDQ": {64, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 8, true}}, + "VPUNPCKHQDQ": {64, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 8, true}}, + "VUNPCKLPS": {32, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 4, true}}, + "VUNPCKHPS": {32, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 4, true}}, + "VUNPCKLPD": {64, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 8, true}}, + "VUNPCKHPD": {64, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 8, true}}, +} + +type x86PackedUnpackForm struct { + x86PackedBinaryForm + spec x86PackedUnpackSpec + source Operand +} + +// Pure byte dependencies shared by independent caller and result observers. +// Memory access, register mode, mask and alignment roles remain prerequisites +// of each consumer; no dependency mask shrinks a native memory operation. +func x86PackedUnpackDefinedBytes(form x86PackedUnpackForm, first, second, old uint64) uint64 { + laneBytes := form.spec.laneBits / 8 + groupBytes := 16 + if form.byteWidth < groupBytes { + groupBytes = form.byteWidth + } + half := groupBytes / 2 + start := 0 + if form.spec.high { + start = half + } + var known uint64 + for group := 0; group < form.byteWidth; group += groupBytes { + for offset := 0; offset < half; offset += laneBytes { + for byteIndex := 0; byteIndex < laneBytes; byteIndex++ { + source := uint(group + start + offset + byteIndex) + destination := uint(group + 2*offset + byteIndex) + known |= (second >> source & 1) << destination + known |= (first >> source & 1) << (destination + uint(laneBytes)) + } + } + } + if form.mask >= 0 && !form.zeroing { + known &= old + } + return known +} + +func x86PackedUnpackProperties(ins Instr, goarch string) (x86PackedUnpackForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86PackedUnpackSpecs[Op(base)] + if !recognized { + return x86PackedUnpackForm{}, false, nil + } + local := ins + if spec.grammar.encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(ins, goarch) + if err != nil { + return x86PackedUnpackForm{}, true, err + } + } + form, err := x86PackedBinaryProperties(local, goarch, spec.grammar) + // Intel's three MMX low-unpack rows are mm/m32, unlike the high + // rows' mm/m64. The unused high source half is not a memory access. + if err == nil && form.mmx && form.memory >= 0 && !spec.high { + form.memoryBytes = 4 + } + if err == nil { + err = x86Mode32VectorRegisterContext(ins, goarch) + } + result := x86PackedUnpackForm{x86PackedBinaryForm: form, spec: spec} + if err == nil { + result.source = local.Args[form.first] + } + return result, true, err +} diff --git a/amd64_packed_unpack_forms_test.go b/amd64_packed_unpack_forms_test.go new file mode 100644 index 000000000..f2c403582 --- /dev/null +++ b/amd64_packed_unpack_forms_test.go @@ -0,0 +1,387 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" +) + +func x86UnpackTestLaneBytes(opcode string) int { + switch { + case strings.HasSuffix(opcode, "BW"): + return 1 + case strings.HasSuffix(opcode, "WL"), strings.HasSuffix(opcode, "WD"): + return 2 + case strings.HasSuffix(opcode, "LQ"), strings.HasSuffix(opcode, "DQ") && !strings.HasSuffix(opcode, "QDQ"), strings.HasSuffix(opcode, "PS"): + return 4 + default: + return 8 + } +} + +func TestX86PackedUnpack386EVEXHighRegistersActualBytesNeedContext(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + if !strings.HasPrefix(opcode, "V") { + continue + } + decode := decodedX86FloatingUnpackInstruction + if strings.HasPrefix(opcode, "VPUNPCK") { + decode = decodedX86EVEXPackedUnpackInstruction + } + for _, bank := range []string{"X", "Y"} { + for _, high := range []int{16, 31} { + for index := 0; index < 3; index++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[index] = fmt.Sprintf("%s%d", bank, high) + row := opcode + " " + strings.Join(args, ",") + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil { + t.Fatalf("actual Go mode32 object: %v\n%s", err, listing) + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 7 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + for _, field := range fields[1:7] { + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + t.Fatalf("actual Go object byte %q: %v", field, err) + } + code = append(code, byte(value)) + } + break + } + if len(code) != 6 || code[0] != 0x62 { + t.Fatalf("actual Go lost EVEX encoding: %s\n%s", row, listing) + } + ins, length, recognized, err := decode(code, 64) + if !recognized || err != nil || length != len(code) || ins.Op != Op(opcode) || len(ins.Args) != 3 { + t.Fatalf("mode64 actual byte identity: %s/%x: %+v/%d/%t/%v", row, code, ins, length, recognized, err) + } + for index, argument := range args { + if ins.Args[index].Reg != Reg(argument) { + t.Fatalf("actual Go %x decoded argument %d as %s want %s", code, index, ins.Args[index].Reg, argument) + } + } + if _, _, recognized, err := decode(code, 32); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s/%x invented raw mode32 high registers: %t/%v", row, code, recognized, err) + } + } + } + } + } +} + +func TestX86PackedUnpack386NamedHighRegistersKeepPhysicalContext(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + if !strings.HasPrefix(opcode, "V") { + continue + } + for _, bank := range []string{"X", "Y"} { + for _, high := range []int{8, 15, 16, 31} { + for index := 0; index < 3; index++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[index] = fmt.Sprintf("%s%d", bank, high) + row := opcode + " " + strings.Join(args, ",") + // This invokes the actual Go tool on both compatibility + // versions. Syntax acceptance alone is not a native binding. + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil || !strings.Contains(listing, row[:len(opcode)]) { + t.Fatalf("actual Go lost accepted mode32 source %s: %v\n%s", row, err, listing) + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if _, err := Translate(file, Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"f": {Ret: Void}}}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s named high register acquired a physical mode32 binding: %v", row, err) + } + } + } + } + } + } +} + +func x86UnpackTestHasMMX(opcode string) bool { + return strings.HasPrefix(opcode, "PUNPCK") && !strings.HasSuffix(opcode, "QDQ") +} + +func x86UnpackAcceptedForms(opcode, arch string) []string { + if !strings.HasPrefix(opcode, "V") { + last := 15 + if arch == "386" { + last = 7 + } + rows := []string{fmt.Sprintf("%s X0,X%d", opcode, last), fmt.Sprintf("%s 8(AX),X%d", opcode, last)} + if arch == "amd64" && x86UnpackTestHasMMX(opcode) { + rows = append(rows, opcode+" M0,M7", opcode+" 8(AX),M7") + } + return rows + } + var rows []string + for _, bank := range []string{"X", "Y", "Z"} { + last := 31 + if arch == "386" { + last = 7 + } + for _, first := range []string{bank + "0", "8(AX)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1,%s%d", opcode, first, bank, bank, last)) + if arch == "amd64" { + rows = append(rows, + fmt.Sprintf("%s %s,%s1,K7,%s%d", opcode, first, bank, bank, last), + fmt.Sprintf("%s.Z %s,%s1,K1,%s%d", opcode, first, bank, bank, last)) + } + } + if x86UnpackTestLaneBytes(opcode) >= 4 { + rows = append(rows, fmt.Sprintf("%s.BCST 8(AX),%s1,%s%d", opcode, bank, bank, last)) + if arch == "amd64" { + rows = append(rows, + fmt.Sprintf("%s.BCST 8(AX),%s1,K7,%s%d", opcode, bank, bank, last), + fmt.Sprintf("%s.BCST.Z 8(AX),%s1,K1,%s%d", opcode, bank, bank, last)) + } + } + } + return rows +} + +func TestX86PackedUnpackSpecsBindEveryGoOpcodeAttribute(t *testing.T) { + if len(x86PackedUnpackSpecs) != len(x86CallerUnpackOpcodes()) { + t.Fatal("incomplete interleave family") + } + dir := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86") + legacy, err := os.ReadFile(filepath.Join(dir, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(dir, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, opcode := range x86CallerUnpackOpcodes() { + spec, ok := x86PackedUnpackSpecs[Op(opcode)] + if !ok || spec.laneBits != 8*x86UnpackTestLaneBytes(opcode) { + t.Fatalf("missing or wrong source-defined lane width: %s", opcode) + } + if !strings.HasPrefix(opcode, "V") { + start := strings.Index(string(legacy), "{A"+opcode+", ") + if start < 0 { + t.Fatalf("actual Go omits %s", opcode) + } + line := strings.SplitN(string(legacy)[start:], "\n", 2)[0] + table := "yxm" + if x86UnpackTestHasMMX(opcode) { + table = "ymm" + } + if !strings.Contains(line, ", "+table+",") { + t.Fatalf("actual Go table differs for %s: %s", opcode, line) + } + continue + } + prefix := "{as: A" + opcode + ", ytab: _yvandnpd," + start := strings.Index(string(avx), prefix) + if start < 0 { + t.Fatalf("actual Go omits %s family table", opcode) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if strings.Count(row, "avxEscape | vex") != 2 || strings.Count(row, "avxEscape | evex") != 3 { + t.Fatalf("actual Go encoding generations changed: %s", row) + } + for _, line := range strings.Split(row, "\n") { + if !strings.Contains(line, "avxEscape | evex") { + continue + } + broadcast := "" + if x86UnpackTestLaneBytes(opcode) >= 4 { + broadcast = fmt.Sprintf("evexBcstN%d", x86UnpackTestLaneBytes(opcode)) + } + if !strings.Contains(line, "evexZeroingEnabled") || strings.Contains(line, "evexSae") || strings.Contains(line, "evexRounding") || + broadcast == "" && strings.Contains(line, "evexBcst") || broadcast != "" && !strings.Contains(line, broadcast) { + t.Fatalf("actual opcode attributes differ: %s", line) + } + } + } +} + +func TestX86PackedUnpackCompleteGoRowsAndFiveLLVMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT unpackforms(SB),4,$0-0\n") + count := 0 + for _, opcode := range x86CallerUnpackOpcodes() { + for _, row := range x86UnpackAcceptedForms(opcode, target.arch) { + count++ + fmt.Fprintln(&source, row) + } + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, ins := range file.Funcs[0].Instrs[1 : len(file.Funcs[0].Instrs)-1] { + if _, recognized, err := x86PackedUnpackProperties(ins, target.arch); !recognized || err != nil { + t.Fatalf("actual Go accepted %s but grammar rejected: %v", ins.Raw, err) + } + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func unpackforms()", target.arch, target.triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original unpack formats invented missing numerical/native inputs", api, err) + } + } + ir := x86InstructionFormatObjectUnit(t, file, map[string]FuncSig{"unpackforms": {Name: "unpackforms", Ret: Void}}, + target.arch, target.triple, 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerPackedUnpack(ins.Op, ins) + }) + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + t.Logf("all %d original Go forms retain emitter-only LLVM22 objects and three-API Context controls", count) + }) + } +} + +func TestX86PackedUnpackDispatchIdentityBeforeEmission(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nVPUNPCKLBW X0,X1,X2\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, opcode := range []Op{"VPUNPCKLBW", "vpunpcklbw"} { + for _, sourceOp := range []Op{"", "VPUNPCKLBW", "vpunpcklbw", "VPUNPCKHBW", "UNKNOWN"} { + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + ins := file.Funcs[0].Instrs[1] + ins.Op = sourceOp + _, _, err := ctx.lowerPackedUnpack(opcode, ins) + wantError := sourceOp == "VPUNPCKHBW" || sourceOp == "UNKNOWN" + if (err != nil) != wantError || wantError && output.String() != before || !wantError && output.String() == before { + t.Fatalf("dispatch identity %s/%s: %v, %d emitted bytes", opcode, sourceOp, err, output.Len()) + } + } + } +} + +func TestX86PackedUnpackRejectsMalformedGoFormsBeforeEmission(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, opcode := range x86CallerUnpackOpcodes() { + rows := []string{opcode + " X0,(AX)", opcode + ".SAE X0,X1"} + if strings.HasPrefix(opcode, "V") { + rows = []string{ + opcode + " X0,X1,(AX)", + opcode + " X0,Y1,Y2", + opcode + " X0,X1,K0,X2", + opcode + ".Z X0,X1,X2", + opcode + ".SAE X0,X1,X2", + opcode + ".BCST X0,X1,X2", + } + if x86UnpackTestLaneBytes(opcode) < 4 { + rows = append(rows, opcode+".BCST (AX),X1,X2") + } + } else { + rows = append(rows, opcode+" X0,Y1") + if arch == "386" { + rows = append(rows, opcode+" X8,X1") + } + } + for _, row := range rows { + t.Run(arch+"/"+row, func(t *testing.T) { + source := "TEXT bad(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := output.String() + ins := file.Funcs[0].Instrs[1] + known, _, err := ctx.lowerPackedUnpack(ins.Op, ins) + if !known || err == nil || output.String() != before { + t.Fatalf("malformed source emitted IR: known=%t err=%v", known, err) + } + }) + } + } + } +} + +func TestX86PackedUnpackMMXMemoryNativeExtent(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + if !x86UnpackTestHasMMX(opcode) { + continue + } + source := "TEXT f(SB),4,$0-0\n" + opcode + " (AX),M0\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + form, recognized, err := x86PackedUnpackProperties(file.Funcs[0].Instrs[1], "amd64") + want := 4 + if strings.Contains(opcode, "UNPCKH") { + want = 8 + } + if !recognized || err != nil || !form.mmx || form.memoryBytes != want { + t.Fatalf("%s native memory extent=%d want=%d: %v", opcode, form.memoryBytes, want, err) + } + } +} + +func TestX86PackedUnpackTypedFPMemoryExtentBeforeEmission(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + for _, row := range x86UnpackAcceptedForms(opcode, "amd64") { + if !strings.Contains(row, "8(AX)") { + continue + } + row = strings.Replace(row, "8(AX)", "a+0(FP)", 1) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedUnpackProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, complete := range []bool{false, true} { + sig := FuncSig{Ret: Void} + for offset := 0; offset < form.memoryBytes; offset += 4 { + if !complete && offset+4 == form.memoryBytes { + break + } + index := len(sig.Args) + sig.Args = append(sig.Args, I32) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(offset), Type: I32, Index: index, Field: -1}) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + known, _, err := ctx.lowerPackedUnpack(ins.Op, ins) + if form.legacy && !form.mmx { + if !known || !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatal("logical FP bytes invented original legacy native alignment", row, err) + } + continue + } + if !known || complete && err != nil || !complete && (err == nil || output.String() != before) { + t.Fatalf("%s complete=%t native bytes=%d: %v", row, complete, form.memoryBytes, err) + } + } + } + } +} diff --git a/amd64_packed_unpack_go_oracle_test.go b/amd64_packed_unpack_go_oracle_test.go new file mode 100644 index 000000000..2e90d3993 --- /dev/null +++ b/amd64_packed_unpack_go_oracle_test.go @@ -0,0 +1,108 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedUnpackOriginalGoBitOracle(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle +import "testing" +func TestOriginalGoInterleaveBits(t *testing.T) { + patterns:=[][16]byte{ + {0x45,0x23,0x81,0x7f,0x78,0x56,0xc4,0xff,0xbc,0x9a,0x78,0x56,0x34,0x12,0xf0,0x7f}, + {0x54,0x76,0x98,0xba,0xdc,0xfe,0xf8,0xff,0x45,0x23,0xc1,0x7f,0,0,0,0x80}, + {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}, + {0xff,0xfe,0xfd,0xfc,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf5,0xf4,0xf3,0xf2,0xf1,0xf0}, + } +`) + count := 0 + for _, opcode := range x86CallerUnpackOpcodes() { + if strings.HasPrefix(opcode, "V") { + continue // Native VEX/EVEX execution needs separate ISA witnesses. + } + banks := []string{"X"} + if x86UnpackTestHasMMX(opcode) { + banks = append(banks, "M") + } + for _, bank := range banks { + forms := []string{"register"} + if bank == "M" { + forms = append(forms, "memory") + } + for _, form := range forms { + name := fmt.Sprintf("unpack%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(a,b,out *byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\nSTMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\n", name) + move, bytes := "MOVOU", 16 + if bank == "M" { + move, bytes = "MOVQ", 8 + } + fmt.Fprintf(&assembly, "%s (BX),%s0\n", move, bank) + first := bank + "1" + if form == "register" { + fmt.Fprintf(&assembly, "%s (AX),%s1\n", move, bank) + } else { + first = "(AX)" + } + fmt.Fprintf(&assembly, "CMPQ AX,AX\nSTC\n%s %s,%s0\n%s %s0,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\nSTMXCSR 18(CX)\n", opcode, first, bank, move, bank) + if bank == "M" { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("LDMXCSR saved-4(SP)\nRET\n") + high := 0 + if strings.Contains(opcode, "UNPCKH") { + high = bytes / 2 + } + fmt.Fprintf(&tests, ` + for _,a:=range patterns { for _,b:=range patterns { for mode:=0;mode<4;mode++ { + var out [24]byte + for i:=range out {out[i]=0xc3} + csr:=uint32(0x1f80|mode<<13|mode) + %[1]s(&a[0],&b[0],&out[1],csr) + if out[0]!=0xc3 || out[23]!=0xc3 || out[17]!=1 || out[18]!=0 {t.Fatalf("%[1]s canary/flags: %%x",out)} + gotCSR:=uint32(out[19])|uint32(out[20])<<8|uint32(out[21])<<16|uint32(out[22])<<24 + if gotCSR!=csr {t.Fatalf("%[1]s changed MXCSR %%x want %%x",gotCSR,csr)} + for i:=0;i<16;i++ { + want:=byte(0xc3) + if i<%[2]d { + source:=%[3]d+(i/(2*%[4]d))*%[4]d+i%%%[4]d + want=b[source] + if i/%[4]d%%2!=0 {want=a[source]} + } + if out[i+1]!=want {t.Fatalf("%[1]s byte %%d got %%x want %%x",i,out[i+1],want)} + } + } } } +`, name, bytes, high, x86UnpackTestLaneBytes(opcode)) + } + } + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", + "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go interleave bit oracle: %v\n%s", err, output) + } + t.Logf("%d original SSE/MMX Go functions; 64 independent bitwise/MXCSR/flag/canary inputs each", count) +} diff --git a/amd64_packed_unpack_mmx_runtime_test.go b/amd64_packed_unpack_mmx_runtime_test.go new file mode 100644 index 000000000..c94e9295c --- /dev/null +++ b/amd64_packed_unpack_mmx_runtime_test.go @@ -0,0 +1,180 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86PackedUnpackMMXNativeProgram() (string, map[string]FuncSig, string, string, int) { + var assembly, declarations, rows strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, opcode := range x86CallerUnpackOpcodes() { + if !x86UnpackTestHasMMX(opcode) { + continue + } + name, fpName := fmt.Sprintf("unpackmmx%d", count), fmt.Sprintf("unpackmmxfp%d", count) + count++ + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-24\nMOVQ input+0(FP),AX\nMOVQ second+8(FP),M0\n%s (AX),M0\nMOVQ M0,ret+16(FP)\nEMMS\nRET\n", name, opcode) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, I64}, Ret: I64, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}, + Results: []FrameSlot{{Offset: 16, Type: I64, Index: 0, Field: -1}}, + }} + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-16\nMOVQ $0,AX\nMOVQ AX,M0\n%s a+0(FP),M0\nMOVQ M0,ret+8(FP)\nEMMS\nRET\n", fpName, opcode) + sigs[fpName] = FuncSig{Name: fpName, Args: []LLVMType{I32, I32}, Ret: I64, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}, {Offset: 4, Type: I32, Index: 1, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }} + fmt.Fprintf(&declarations, "extern uint64_t %s(const uint8_t*,uint64_t);\nextern uint64_t %s(uint32_t,uint32_t);\n", name, fpName) + high, extent := 0, 4 + if strings.Contains(opcode, "UNPCKH") { + high, extent = 4, 8 + } + fmt.Fprintf(&rows, "{%s, %s, %d, %d, %d},\n", name, fpName, x86UnpackTestLaneBytes(opcode), high, extent) + } + return assembly.String(), sigs, declarations.String(), rows.String(), count +} + +func TestCrossLinuxRuntimeMatrixX86PackedUnpackMMXNativeMemoryAndFP(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + assembly, sigs, declarations, rows, count := x86PackedUnpackMMXNativeProgram() + requireX86GoAssemblerResult(t, "amd64", assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := x86MemoryFaultSupportC + declarations + ` +static const struct { + uint64_t (*run)(const uint8_t*,uint64_t); + uint64_t (*fp)(uint32_t,uint32_t); + int lane, high, extent; +} cases[] = { +` + rows + `}; +static uint64_t expectedMMX(const uint8_t *first,uint64_t second,int lane,int high) { + uint8_t old[8],out[8]; + memcpy(old,&second,8); + for(int i=0;i<8;++i) { + int source=high+(i/(2*lane))*lane+i%lane; + out[i]=(i/lane)%2 ? first[source] : old[source]; + } + uint64_t value; + memcpy(&value,out,8); + return value; +} +struct MMXModeImage { _Alignas(16) uint8_t bytes[512]; }; +static void beginMMXMode(uint16_t *oldControl,uint32_t *oldCSR) { + const uint16_t control=0x077f; + const uint32_t csr=0x5f80; + __asm__ volatile("fnstcw %0; stmxcsr %1; fninit; fldcw %2; ldmxcsr %3; " + "pcmpeqd %%mm0,%%mm0; pcmpeqd %%mm6,%%mm6; emms; fld1" + : "=m"(*oldControl),"=m"(*oldCSR) : "m"(control),"m"(csr) + : "mm0","mm6","st","memory"); +} +static int endMMXMode(uint64_t result,uint16_t control,uint32_t csr) { + struct MMXModeImage image; + __asm__ volatile("fxsave64 %0; fninit; fldcw %1; ldmxcsr %2" + : "=m"(image) : "m"(control),"m"(csr) : "st","memory"); + uint16_t actualControl,status; + uint32_t actualCSR; + uint64_t actualResult,canary; + memcpy(&actualControl,image.bytes,2); + memcpy(&status,image.bytes+2,2); + memcpy(&actualCSR,image.bytes+24,4); + memcpy(&actualResult,image.bytes+32,8); + memcpy(&canary,image.bytes+32+6*16,8); + return actualControl==0x077f && (status&0x3800)==0 && image.bytes[4]==0 && + actualCSR==0x5f80 && actualResult==result && canary==UINT64_MAX; +} +static int checkMMXModeObserver(void) { + for(int mutation=0;mutation<3;mutation++) { + uint16_t control; + uint32_t csr; + beginMMXMode(&control,&csr); + if(mutation==0) { + __asm__ volatile("emms" : : : "st","memory"); + } else if(mutation==1) { + __asm__ volatile("pxor %%mm0,%%mm0" : : : "mm0","st","memory"); + } else { + __asm__ volatile("pxor %%mm0,%%mm0; emms" : : : "mm0","st","memory"); + } + if(endMMXMode(0,control,csr)!=(mutation==2)) return 1; + } + return 0; +} +static int checkMMX(uint64_t (*program)(const uint8_t*,uint64_t), + uint64_t (*fp)(uint32_t,uint32_t),int lane,int high,int extent,uint8_t *end) { + for(int seed=0;seed<16;++seed) { + uint8_t *first=end-extent,source[8]; + for(int i=0;i<8;++i) source[i]=(uint8_t)(seed*73+i*19+7); + memcpy(first,source,extent); + uint64_t second=UINT64_C(0xcafe11223344ba5e)^((uint64_t)seed*UINT64_C(0x1234567890)); + uint16_t control; + uint32_t csr; + beginMMXMode(&control,&csr); + uint64_t result=program(first,second); + if(!endMMXMode(result,control,csr)) return 1; + if(result!=expectedMMX(source,second,lane,high) || memcmp(first,source,extent)) return 1; + uint32_t a,b; + memcpy(&a,source,4); + memcpy(&b,source+4,4); + beginMMXMode(&control,&csr); + result=fp(a,b); + if(!endMMXMode(result,control,csr) || result!=expectedMMX(source,0,lane,high)) return 1; + } + return 0; +} +int main(int argc, char **argv) { + if (argc != 4) return 80; + if (checkMMXModeObserver()) return 92; + unsigned row = (unsigned)atoi(argv[1]); + unsigned probe = (unsigned)atoi(argv[2]); + unsigned original = (unsigned)atoi(argv[3]); + if (row >= sizeof(cases)/sizeof(cases[0]) || probe > 3 || original > 1) return 80; + int setup = memory_fault_setup(1); + if (setup) return setup; + uint8_t *end = guard + page; + const uint8_t *input = NULL; + if (probe == 1) input = end - cases[row].extent + 1; + if (probe == 2) input = end - cases[row].extent; + if (probe == 3) input = cases[row].high ? guard - 4 : end - 3; + + printf("memory enter %u %u %u\n", row, probe, original); + if (fflush(stdout)) return 83; + if (original) { + const volatile uint8_t *source = input; + volatile uint8_t sink = 0; + for (int i = 0; i < cases[row].extent; ++i) sink ^= source[i]; + } else if (probe == 2) { + if (checkMMX(cases[row].run, cases[row].fp, cases[row].lane, + cases[row].high, cases[row].extent, end)) return 91; + } else { + volatile uint64_t observed = cases[row].run(input, UINT64_C(0xabcdef1234567890)); + (void)observed; + } + if (free_guard()) return 84; + puts("memory complete"); + return fflush(stdout) ? 85 : 0; +} +` + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := execution.compileLLVM(t, "unpack-mmx-native-memory-fp", ir, main, level) + x86MMXExportNativeFixture(t, executable, assembly, ir, level, execution.triple, count) + x86CheckMemoryFaultMatrix(t, execution, executable, count) + for row := 0; row < count; row++ { + want := x86ObserveMemoryFault(t, execution, executable, row, 3, 1) + if want != "segv" && want != "bus" { + t.Fatalf("row %d: independent leading/short read did not fault: %s", row, want) + } + if got := x86ObserveMemoryFault(t, execution, executable, row, 3, 0); got != want { + t.Fatalf("row %d: translated leading/short read %s, want %s", row, got, want) + } + } + }) + } +} diff --git a/amd64_packed_unpack_runtime_test.go b/amd64_packed_unpack_runtime_test.go new file mode 100644 index 000000000..230412238 --- /dev/null +++ b/amd64_packed_unpack_runtime_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// One independent reference is shared by all format programs. Keeping this +// out of main avoids compiling hundreds of copies of the same canary loops. +const x86UnpackBitwiseRuntimeOracle = ` +static uint8_t patterns[4][64]; +static const uint64_t masks[]={0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; +__attribute__((noinline)) static int checkUnpack( + void (*program)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t), + const char *name,int width,int lane,int high,int mode,int broadcast,int legacy,int destination,int shift) { + for(int round=0;round<4;++round) for(int m=0;m<4;++m) { + _Alignas(64) uint8_t aStorage[96],bStorage[96],output[68]; + uint8_t *a=aStorage+shift,*b=bStorage+3; + memset(aStorage,0xa7,sizeof(aStorage)); + memset(bStorage,0xb9,sizeof(bStorage)); + memset(output,0xc3,sizeof(output)); + memcpy(a,patterns[round],64); + memcpy(b,patterns[(round+1)%4],64); + uint64_t mask=masks[m]; + uint32_t saved,before=0x1f80|(round<<13)|round,after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + program(output+1,a,b,mask); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + if(after!=before || output[0]!=0xc3 || output[67]!=0xc3 || output[65]!=1 || output[66]!=0) { + fprintf(stderr,"%s round=%d mask=%llx CSR/flags/canary failure\n",name,round,(unsigned long long)mask); + return 1; + } + for(int i=0;i<64;++i) { + int source=(i/16)*16+high+((i%16)/(2*lane))*lane+i%lane; + uint8_t selected=(i/lane)%2 ? a[broadcast ? source%lane : source] : b[source]; + const uint8_t *old=destination==1 ? a : b; + int active=mode==0 || ((mask>>(i/lane))&1); + uint8_t want=i=shift+64) && aStorage[i]!=0xa7) return 1; + if((i<3 || i>=67) && bStorage[i]!=0xb9) return 1; + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedUnpackCompleteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, opcode := range x86CallerUnpackOpcodes() { + legacy := !strings.HasPrefix(opcode, "V") + widths := []int{16} + if !legacy { + widths = []int{16, 32, 64} + } + lane := x86UnpackTestLaneBytes(opcode) + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, memory := range []string{"register", "memory", "broadcast"} { + if memory == "broadcast" && (legacy || lane < 4) { + continue + } + for _, mode := range []string{"unmasked", "merge", "zero"} { + if legacy && mode != "unmasked" { + continue + } + for _, destination := range []int{0, 1, 2} { + if legacy && destination != 0 { + continue + } + name := fmt.Sprintf("unpack%d", count) + count++ + first := bank + "1" + if memory != "register" { + first = "(BX)" + } + operation := opcode + if memory == "broadcast" { + operation += ".BCST" + } + if mode == "zero" { + operation += ".Z" + } + row := operation + " " + first + "," + bank + "0" + if !legacy { + if mode != "unmasked" { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + } + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", name, row, destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + high := 0 + if strings.Contains(opcode, "UNPCKH") { + high = 8 + } + // Legacy m128 requires aligned storage. All other forms + // exercise unaligned input/output pointers and canaries. + shift := 1 + if legacy && memory == "memory" { + shift = 16 + } + modeValue := map[string]int{"unmasked": 0, "merge": 1, "zero": 2}[mode] + fmt.Fprintf(&checks, "if(checkUnpack(%s,\"%s\",%d,%d,%d,%d,%d,%d,%d,%d)) return 1;\n", name, name, + width, lane, high, modeValue, x86FloatAddSubBoolInt(memory == "broadcast"), x86FloatAddSubBoolInt(legacy), destination, shift) + } + } + } + } + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +` + declarations.String() + x86UnpackBitwiseRuntimeOracle + ` +int main(void) { + for(int p=0;p<4;++p) for(int i=0;i<64;++i) patterns[p][i]=(uint8_t)(i*19+p*83+7); + // These include float32/float64 signaling/quiet NaN payloads. An unpack + // must copy their exact bits and leave even pre-existing MXCSR flags alone. + const uint64_t specials[]={UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff8fedcba987654),UINT64_C(0x7f812345ffc45678),UINT64_C(0x800000007fc12345)}; + memcpy(patterns[2],specials,sizeof(specials)); + memcpy(patterns[3]+32,specials,sizeof(specials)); +` + checks.String() + "return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "unpack-complete", triple, ir, main, prefix) + t.Logf("%d opcode/width/memory/mask/alias programs, 16 independent bitwise/MXCSR/flag/canary inputs each", count) +} diff --git a/amd64_packed_vector_move_aligned.go b/amd64_packed_vector_move_aligned.go new file mode 100644 index 000000000..2cb1a24cf --- /dev/null +++ b/amd64_packed_vector_move_aligned.go @@ -0,0 +1,99 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Alignment faults are part of the original instruction, including its exact +// mask and zeroing behavior. Neither LLVM align metadata nor a masked load can +// replace this operation: the former asserts alignment instead of checking it, +// and the latter can remove a dead read. Pass an opaque address in a register, +// without granting the optimizer any pointee, extent or alignment assumption. +// Intel SDM Vol. 2A Table 2-46 keeps E1 alignment separate from page-fault +// suppression; the runtime oracle compares both against original Go opcodes. +func (c *amd64Ctx) lowerAlignedPackedMoveMemory(ins Instr, form amd64PackedVectorMoveForm) error { + return c.lowerNativeAlignedVectorMemory(ins, form, false) +} + +func (c *amd64Ctx) lowerNativeAlignedVectorMemory(ins Instr, form amd64PackedVectorMoveForm, legacy bool) error { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(ins.Args[form.memory]) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + name, _, _ := strings.Cut(strings.ToLower(string(ins.Op)), ".") + constraint, prefix := "x", "" + if form.nativeEVEX(ins) { + constraint = "v" + // Retain EVEX for high source registers even if LLVM allocates a low + // carrier. APD/APS otherwise also have a VEX encoding at X/Y widths. + prefix = "{evex} " + } + vectorType := fmt.Sprintf("<%d x float>", form.byteWidth/4) + vector := func(operand Operand) (string, error) { + value, err := c.loadPackedCompareBytes(operand, form.byteWidth) + if err != nil { + return "", err + } + carrier := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", carrier, form.byteWidth, value, vectorType) + return "%" + carrier, nil + } + mask := "" + if form.mask >= 0 { + value, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", narrow, value) + mask = "%" + narrow + } + if form.memoryWrite { + value, err := vector(ins.Args[0]) + if err != nil { + return err + } + assembly := prefix + name + " $0, " + segment + "($1)" + constraints := constraint + ",r" + arguments := fmt.Sprintf("%s %s, %s %s", vectorType, value, pointerType, pointer) + if mask != "" { + assembly = "kmovw ${2:k}, %k1; " + assembly + " {%k1}" + constraints += ",r,~{k1}" + arguments += ", i32 " + mask + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s)\n", assembly, constraints+",~{memory}", arguments) + return nil + } + assembly := prefix + name + " " + segment + "($1), $0" + constraints := "=" + constraint + ",r" + arguments := pointerType + " " + pointer + if mask != "" { + if form.zeroing { + assembly = "kmovw ${2:k}, %k1; " + assembly + " {%k1} {z}" + } else { + old, err := vector(ins.Args[form.destination]) + if err != nil { + return err + } + assembly = "kmovw ${3:k}, %k1; " + prefix + name + " " + segment + "($2), $0 {%k1}" + constraints = "=" + constraint + ",0,r" + arguments = vectorType + " " + old + ", " + arguments + } + constraints += ",r,~{k1}" + arguments += ", i32 " + mask + } + loaded, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", loaded, vectorType, assembly, constraints+",~{memory}", arguments) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, vectorType, loaded, form.byteWidth) + if legacy { + return c.storeLegacyPackedLogical(ins.Args[form.destination], "%"+bytes) + } + return c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, "%"+bytes) +} diff --git a/amd64_packed_vector_move_aligned_oracle_unix_test.go b/amd64_packed_vector_move_aligned_oracle_unix_test.go new file mode 100644 index 000000000..a7475b5a3 --- /dev/null +++ b/amd64_packed_vector_move_aligned_oracle_unix_test.go @@ -0,0 +1,27 @@ +//go:build unix + +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +// Every host compiles the actual Go-byte original/LLVM oracle. Only the required +// Linux/amd64 runtime root executes it; this test is not native ISA evidence. +func TestX86PackedMoveAlignedOriginalGoOracleCompileOnly(t *testing.T) { + execution := fixedLocalPackedCompileOnlyTools(t) + ir, main, count := x86PackedMoveObservableFixture(t, true, true, execution.triple) + compileLLVMToObject(t, execution.llc, execution.triple, "aligned-oracle.ll", "aligned-oracle.o", ir) + dir := t.TempDir() + source, object := filepath.Join(dir, "oracle.c"), filepath.Join(dir, "oracle.o") + if err := os.WriteFile(source, []byte(main), 0600); err != nil { + t.Fatal(err) + } + command := append([]string(nil), execution.compiler...) + command = append(command, "-O2", "-c", source, "-o", object) + x86FloatCompareCaptureCommand(t, exec.Command(command[0], command[1:]...)) + t.Logf("compiled %d original-Go aligned cases; native numerical/fault execution NOT performed by this test", count) +} diff --git a/amd64_packed_vector_move_aligned_runtime_linux_amd64_test.go b/amd64_packed_vector_move_aligned_runtime_linux_amd64_test.go new file mode 100644 index 000000000..a3efcedfb --- /dev/null +++ b/amd64_packed_vector_move_aligned_runtime_linux_amd64_test.go @@ -0,0 +1,17 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedMaskedMemory(t *testing.T) { + x86PackedMoveMaskedMemoryRuntime(t, true) +} + +func TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedRegisterViews(t *testing.T) { + x86PackedMoveRegisterViewsRuntime(t, true) +} + +func TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedOriginalGo(t *testing.T) { + x86PackedMoveObservableRuntime(t, true, true) +} diff --git a/amd64_packed_vector_move_aligned_test.go b/amd64_packed_vector_move_aligned_test.go new file mode 100644 index 000000000..16800ab3f --- /dev/null +++ b/amd64_packed_vector_move_aligned_test.go @@ -0,0 +1,292 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86PackedMoveAlignedNativeOpcodeAndFeatures(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + for _, mode := range []string{"load", "store", "merge", "zero", "masked-store"} { + masked := mode != "load" && mode != "store" + if row.vexOnly && masked { + continue + } + t.Run(row.name+"/"+bank+"/"+mode, func(t *testing.T) { + op, mask := row.name, "" + if masked { + mask = "K7," + } + if mode == "zero" { + op += ".Z" + } + operation := fmt.Sprintf("%s (AX),%s%s0", op, mask, bank) + if strings.HasSuffix(mode, "store") { + operation = fmt.Sprintf("%s %s0,%s(AX)", op, bank, mask) + } + source := "TEXT ·view(SB),4,$0-8\nMOVQ p+0(FP),AX\nMOVL $3,CX\nKMOVD CX,K7\nVPXORQ Z0,Z0,Z0\n" + operation + "\nRET\n" + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "asm sideeffect") || !strings.Contains(ir, strings.ToLower(row.name)+" ") { + t.Fatal("aligned memory lost its original observable opcode", operation) + } + feature := "+avx" + if masked || bank == "Z" || strings.HasPrefix(row.name, "VMOVDQA3") || row.name == "VMOVDQA64" { + feature = "+avx512f" + } + if !strings.Contains(ir, feature) { + t.Fatal("missing required native feature", feature) + } + }) + } + } + } +} + +func TestX86PackedMoveAlignedVirtualMemoryNeedsContext(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + for _, memory := range []string{"a+0(FP)", "local-64(SP)", "(BP)"} { + for _, operands := range []string{memory + "," + bank + "0", bank + "0," + memory} { + source := fmt.Sprintf("TEXT ·view(SB),4,$64-64\n%s %s\nRET\n", row.name, operands) + _, err := x86ScalarFPBitViewAPI(t, source, "func view(a,b,c,d,e,f,g,h uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s %s acquired native alignment/backing: %v", row.name, operands, err) + } + } + } + } + } +} + +func TestX86PackedMoveAlignedResultFiveTargetsThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + word, move := 8, "MOVQ" + if target.arch == "386" { + word, move = 4, "MOVL" + } + var source, declarations strings.Builder + count := 0 + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + for _, mode := range []string{"plain", "merge", "zero"} { + if row.vexOnly && mode != "plain" { + continue + } + name, mask := row.name, "" + if mode != "plain" { + mask = "K1," + } + if mode == "zero" { + name += ".Z" + } + fmt.Fprintf(&declarations, "func view%d(p *[64]byte) uint32\n", count) + fmt.Fprintf(&source, "TEXT ·view%d(SB),4,$0-%d\n%s p+0(FP),SI\nMOVL $1,AX\nKMOVD AX,K1\nVPXORQ Z1,Z1,Z1\n%s (SI),%s%s1\nPEXTRD $3,X1,AX\nMOVL AX,ret+%d(FP)\nRET\n", count, word+4, move, name, mask, bank, word) + count++ + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + compileLLVMToObject(t, llc, target.triple, "aligned-result.ll", "aligned-result.o", ir) + } + }) + } +} + +func TestX86PackedMoveAlignedNativeCompleteMemoryAxes(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("GLOBL table<>(SB),8,$128\n") + count := 0 + add := func(instruction string) { + name := fmt.Sprintf("aligned%d", count) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, instruction) + count++ + } + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + // Every mask register, both directions, merge and zero. + for mask := 0; mask <= 7; mask++ { + if row.vexOnly && mask != 0 { + continue + } + predicate := "" + if mask != 0 { + predicate = fmt.Sprintf("K%d,", mask) + } + add(fmt.Sprintf("%s (AX),%s%s7", row.name, predicate, bank)) + add(fmt.Sprintf("%s %s7,%s(AX)", row.name, bank, predicate)) + if mask != 0 { + add(fmt.Sprintf("%s.Z (AX),%s%s7", row.name, predicate, bank)) + } + } + // Orthogonal address-space and high-bank encoding classes. + indexes := []int{0, 7} + if target.arch == "amd64" { + indexes = append(indexes, 8, 15) + if !row.vexOnly { + indexes = append(indexes, 16, 31) + } + } + for _, index := range indexes { + for _, address := range []string{"16(AX)(CX*2)", "table<>+16(SB)", "16(FS)", "16(GS)"} { + add(fmt.Sprintf("%s %s,%s%d", row.name, address, bank, index)) + add(fmt.Sprintf("%s %s%d,%s", row.name, bank, index, address)) + } + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + ir := x86UnboundVoidVectorMoveFormatObject(t, source.String(), target.arch, target.triple) + for _, segment := range []string{"%fs:", "%gs:", "ptr addrspace(256)", "ptr addrspace(257)"} { + if !strings.Contains(ir, segment) { + t.Fatal("native address-space lost", segment) + } + } + if got := strings.Count(ir, "asm sideeffect"); got != count { + t.Fatalf("native memory witness count got %d want %d", got, count) + } + compileLLVMToObject(t, llc, target.triple, "aligned-memory.ll", "aligned-memory.o", ir) + t.Logf("%d actual-Go/LLVM aligned memory forms", count) + }) + } +} + +func TestX86PackedMoveAlignedNativeAddressSizeBoundary(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, addressBits := range []int{16, 32, 64} { + ins := x86ScalarLaneInstruction(t, row.name+" (AX),X0") + ins.x86AddressBits = addressBits + ctx := &amd64Ctx{goarch: "amd64"} + if _, _, err := ctx.lowerPackedVectorMove(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound original address width borrowed ordinary memory", row.name, addressBits, err) + } + } + } +} + +func TestX86PackedMoveAlignedDATAReadIsNotWriteAuthority(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, masked := range []bool{false, true} { + if row.vexOnly && masked { + continue + } + predicate := "" + if masked { + predicate = "K1," + } + source := "DATA table<>+0(SB)/8,$1\nGLOBL table<>(SB),8,$64\nTEXT ·view(SB),4,$0-4\nMOVL $1,AX\nKMOVD AX,K1\n" + + row.name + " table<>(SB),X1\n" + row.name + " X1," + predicate + "table<>(SB)\nPEXTRD $0,X1,AX\nMOVL AX,ret+0(FP)\nRET\n" + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32\n", "amd64", "x86_64-unknown-linux-gnu", api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("readable DATA gained write authority", row.name, masked, api, err) + } + } + } + } +} + +func TestX86PackedMoveAlignedRawSourceCannotBorrowConditionalRead(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[64]byte) uint32", + "MOVQ p+0(FP),AX\n"+row.name+" (AX),X1\nPEXTRD $0,X1,CX\nMOVL CX,ret0+8(FP)\nRET") + for _, mutation := range []string{"decoded", "address size", "unbound address", "raw text", "source stack", "fixed local", "segment", "symbol"} { + fn := file.Funcs[0] + fn.Instrs = append([]Instr(nil), fn.Instrs...) + fn.Instrs[2].Args = append([]Operand(nil), fn.Instrs[2].Args...) + switch mutation { + case "decoded": + fn.Instrs[2].x86Encoded = true + case "address size": + fn.Instrs[2].x86AddressBits = 32 + case "unbound address": + fn.Instrs[2].x86UnboundAddressOverride = true + case "raw text": + fn.X86RawText = []byte{0xc3} + case "source stack": + fn.x86SourceStack = new(x86SourceStackProof) + case "fixed local": + fn.x86FixedLocal = new(x86FixedLocalProof) + case "segment": + fn.Instrs[2].Args[0].Mem.Segment = "FS" + case "symbol": + fn.Instrs[2].Args[0].Mem.Sym = "table<>(SB)" + } + ctx := &amd64Ctx{fpScalarSource: fn, sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("aligned read bypassed source identity", row.name, mutation, err) + } + } + } +} + +func TestX86PackedMoveAlignedRegisterCopyRemainsPortable(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + ins := x86ScalarLaneInstruction(t, row.name+" "+bank+"0,"+bank+"1") + if features := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", Func{Instrs: []Instr{ins}}); features != "" { + t.Fatal("register bit-copy acquired physical memory ISA", ins, features) + } + } + } +} diff --git a/amd64_packed_vector_move_complete_grammar_test.go b/amd64_packed_vector_move_complete_grammar_test.go new file mode 100644 index 000000000..00810eb8f --- /dev/null +++ b/amd64_packed_vector_move_complete_grammar_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86PackedMoveCompleteIndependentEncoderRows(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + for _, expected := range []struct { + name, table, pp, evexW, store, load string + vex, evex bool + }{ + {"VMOVDQA", "_yvmovdqa", "66", "", "0x7F", "0x6F", true, false}, + {"VMOVDQU", "_yvmovdqa", "F3", "", "0x7F", "0x6F", true, false}, + {"VMOVAPD", "_yvmovapd", "66", "1", "0x29", "0x28", true, true}, + {"VMOVAPS", "_yvmovapd", "", "0", "0x29", "0x28", true, true}, + {"VMOVUPD", "_yvmovapd", "66", "1", "0x11", "0x10", true, true}, + {"VMOVUPS", "_yvmovapd", "", "0", "0x11", "0x10", true, true}, + {"VMOVDQA32", "_yvmovdqa32", "66", "0", "0x7F", "0x6F", false, true}, + {"VMOVDQA64", "_yvmovdqa32", "66", "1", "0x7F", "0x6F", false, true}, + {"VMOVDQU8", "_yvmovdqa32", "F2", "0", "0x7F", "0x6F", false, true}, + {"VMOVDQU16", "_yvmovdqa32", "F2", "1", "0x7F", "0x6F", false, true}, + {"VMOVDQU32", "_yvmovdqa32", "F3", "0", "0x7F", "0x6F", false, true}, + {"VMOVDQU64", "_yvmovdqa32", "F3", "1", "0x7F", "0x6F", false, true}, + } { + var classes, zcases, opbytes []string + var offsets []int + for _, encoding := range []string{"vex", "evex"} { + if encoding == "vex" && !expected.vex || encoding == "evex" && !expected.evex { + continue + } + for direction, opcode := range []string{expected.store, expected.load} { + widths := []int{128, 256} + if encoding == "evex" { + widths = append(widths, 512) + } + for _, width := range widths { + bank := map[int]string{128: "x", 256: "y", 512: "z"}[width] + register, memory := "Y"+bank+"r", "Y"+bank+"m" + word := "0" + if encoding == "evex" { + word = expected.evexW + if width != 512 { + register, memory = register+"Evex", memory+"Evex" + } + } + prefix := fmt.Sprintf("avxEscape | %s%d", encoding, width) + if expected.pp != "" { + prefix += " | " + encoding + expected.pp + } + prefix += " | " + encoding + "0F | " + encoding + "W" + word + opbytes = append(opbytes, prefix) + if encoding == "evex" { + opbytes = append(opbytes, fmt.Sprintf("evexN%d | evexZeroingEnabled", width/8)) + } + opbytes = append(opbytes, opcode) + for _, masked := range []bool{false, true} { + if masked && encoding == "vex" { + continue + } + first, last, zcase := register, memory, "r_v_rm" + if direction == 1 { + first, last, zcase = memory, register, "rm_v_r" + } + args, offset := first+","+last, 2 + if encoding == "evex" { + offset = 0 + if masked { + args, offset = first+",Yknot0,"+last, 3 + zcase = strings.Replace(zcase, "_v_", "_k_", 1) + } + } + classes, zcases, offsets = append(classes, args), append(zcases, "Z"+encoding+"_"+zcase), append(offsets, offset) + } + } + } + } + count, encodingOffset := 0, 0 + for _, rule := range rules { + if rule.Opcode != expected.name { + continue + } + if count >= len(classes) || rule.RowIndex != count || rule.Ytab != expected.table || rule.Prefix != "Pavx" || + strings.Join(rule.Classes, ",") != classes[count] || rule.ZCase != zcases[count] || rule.ZOffset != offsets[count] || + rule.EncodingOffset != encodingOffset || !reflect.DeepEqual(rule.OpBytes, opbytes) { + t.Fatalf("%s complete Go row/attributes changed: %+v", expected.name, rule) + } + encodingOffset += offsets[count] + count++ + } + if count != len(classes) { + t.Fatal("missing independent encoder rows", expected.name, count, len(classes)) + } + } +} + +func TestX86PackedMoveCompleteRegisterMaskAndAddressAxes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + var rows []string + for _, row := range x86PackedMoveResultOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + limit := 32 + if row.vexOnly { + limit = 16 + } + if arch == "386" && bank == "Z" { + limit = 8 + } + for index := 0; index < limit; index++ { + rows = append(rows, fmt.Sprintf("%s %s%d,%s1", row.name, bank, index, bank), fmt.Sprintf("%s %s0,%s%d", row.name, bank, bank, index)) + } + if !row.vexOnly { + for mask := 1; mask <= 7; mask++ { + for _, suffix := range []string{"", ".Z"} { + rows = append(rows, fmt.Sprintf("%s%s %s0,K%d,%s1", row.name, suffix, bank, mask, bank), + fmt.Sprintf("%s%s (AX),K%d,%s1", row.name, suffix, mask, bank), + fmt.Sprintf("%s%s %s0,K%d,(AX)", row.name, suffix, bank, mask)) + } + } + } + for _, address := range []string{"(AX)", "-16(BX)(CX*2)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1", row.name, address, bank), fmt.Sprintf("%s %s1,%s", row.name, bank, address)) + } + } + } + var source strings.Builder + for index, row := range rows { + fmt.Fprintf(&source, "TEXT f%d(SB),4,$64-64\n%s\nRET\n", index, row) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for index, fn := range file.Funcs { + ins := fn.Instrs[1] + _, known, err := amd64PackedVectorMoveProperties(ins, arch) + physicalContext := false + for _, arg := range ins.Args { + register, vector := amd64VectorRegisterIndex(arg.Reg, amd64VectorByteWidth(arg.Reg)) + physicalContext = physicalContext || arch == "386" && vector && register >= 8 + } + if !known || physicalContext && !errors.Is(err, ErrProbeNeedsContext) || !physicalContext && err != nil { + t.Fatal("independent Go family grammar mismatch", arch, rows[index], err) + } + } + t.Logf("%s: %d independent actual-Go accepted form axes", arch, len(rows)) + } +} + +func TestX86PackedMoveConstructedMemoryCannotBypassGrammar(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for _, arch := range []string{"amd64", "386"} { + for _, change := range []string{"unknown offset", "unknown index", "bad scale", "stack index", "vector base", "vector index", "unknown register", "address literal"} { + ins := x86ScalarLaneInstruction(t, row.name+" (AX),X1") + switch change { + case "unknown offset": + ins.Args[0].Mem.OffRaw = "missing" + case "unknown index": + ins.Args[0].Mem.IndexExt = "missing" + case "bad scale": + ins.Args[0].Mem.Index, ins.Args[0].Mem.Scale = CX, 3 + case "stack index": + ins.Args[0].Mem.Index, ins.Args[0].Mem.Scale = SP, 1 + case "vector base": + ins.Args[0].Mem.Base = "X0" + case "vector index": + ins.Args[0].Mem.Index, ins.Args[0].Mem.Scale = "X0", 1 + case "unknown register": + ins.Args[0].Mem.Base = "UNKNOWN" + case "address literal": + ins.Args[0] = Operand{Kind: OpSym, Sym: "$table<>(SB)"} + } + if _, known, err := amd64PackedVectorMoveProperties(ins, arch); !known || err == nil { + t.Fatal("constructed memory bypassed grammar", row.name, arch, change, err) + } + if x86ScalarFPResultTransfer(ins, &x86ResultState{}, nil, 0, arch) { + t.Fatal("invalid memory acquired a result effect", row.name, arch, change) + } + } + } + } +} diff --git a/amd64_packed_vector_move_forms.go b/amd64_packed_vector_move_forms.go new file mode 100644 index 000000000..84e08b701 --- /dev/null +++ b/amd64_packed_vector_move_forms.go @@ -0,0 +1,178 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedMoveEncoding uint8 + +const ( + amd64PackedMoveVEX amd64PackedMoveEncoding = iota + amd64PackedMoveVEXOrEVEX + amd64PackedMoveEVEX +) + +type amd64PackedVectorMoveSpec struct { + elementBits int + encoding amd64PackedMoveEncoding + aligned bool +} + +// This is the complete _yvmovdqa, _yvmovapd and _yvmovdqa32 family. +// Encoding generation controls widths/registers/masks; elementBits controls +// mask granularity, not the native memory access's complete byte envelope. +var amd64PackedVectorMoveSpecs = map[Op]amd64PackedVectorMoveSpec{ + "VMOVDQA": {8, amd64PackedMoveVEX, true}, + "VMOVDQU": {8, amd64PackedMoveVEX, false}, + "VMOVAPD": {64, amd64PackedMoveVEXOrEVEX, true}, + "VMOVAPS": {32, amd64PackedMoveVEXOrEVEX, true}, + "VMOVUPD": {64, amd64PackedMoveVEXOrEVEX, false}, + "VMOVUPS": {32, amd64PackedMoveVEXOrEVEX, false}, + "VMOVDQA32": {32, amd64PackedMoveEVEX, true}, + "VMOVDQA64": {64, amd64PackedMoveEVEX, true}, + "VMOVDQU8": {8, amd64PackedMoveEVEX, false}, + "VMOVDQU16": {16, amd64PackedMoveEVEX, false}, + "VMOVDQU32": {32, amd64PackedMoveEVEX, false}, + "VMOVDQU64": {64, amd64PackedMoveEVEX, false}, +} + +type amd64PackedVectorMoveForm struct { + spec amd64PackedVectorMoveSpec + byteWidth int + destination int + mask int // -1 for unmasked forms + memory int // -1 for register-to-register forms + memoryWrite bool + zeroing bool // native EVEX.z; memory destinations raise #UD + physicalEVEX bool // Exact decoded generation, including unmasked low X/Y registers. +} + +// Go's frontend accepts .Z memory stores and sets EVEX.z in their objects, +// although the native instruction requires EVEX.z=0 for a memory destination. +// Keep syntax acceptance separate from its terminal exception effect. +func (form amd64PackedVectorMoveForm) undefinedOpcode() bool { + return form.zeroing && form.memoryWrite +} + +// Only the written vector bytes are described here. Callers independently +// supply numerical source/merge bytes and validate every mask and native +// memory role. VEX/EVEX upper-zeroing is applied by the state consumer. +func (form amd64PackedVectorMoveForm) definedActiveBytes(source, previous uint64) uint64 { + known := source & x86CallerVectorByteMask(int64(form.byteWidth)) + if form.mask >= 0 && !form.zeroing { + known &= previous + } + return known +} + +// Both the lowerer and caller-effect proof consume this grammar. It supplies +// no memory bounds, alignment, initialized bytes, mask value or continuation. +func amd64PackedVectorMoveProperties(ins Instr, goarch string) (amd64PackedVectorMoveForm, bool, error) { + name := strings.ToUpper(string(ins.Op)) + base, suffix, dotted := strings.Cut(name, ".") + spec, recognized := amd64PackedVectorMoveSpecs[Op(base)] + if !recognized { + return amd64PackedVectorMoveForm{}, false, nil + } + form := amd64PackedVectorMoveForm{ + spec: spec, destination: len(ins.Args) - 1, + mask: -1, memory: -1, zeroing: suffix == "Z", + physicalEVEX: ins.x86PackedVectorMoveEVEX, + } + fail := func() (amd64PackedVectorMoveForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix are outside the packed vector-move grammar: %q", goarch, base, ins.Raw) + } + if goarch != "amd64" && goarch != "386" || dotted && suffix != "Z" || + form.physicalEVEX && spec.encoding == amd64PackedMoveVEX || + len(ins.Args) != 2 && len(ins.Args) != 3 { + return fail() + } + if len(ins.Args) == 3 { + if spec.encoding == amd64PackedMoveVEX || !amd64NonzeroKOperand(ins.Args[1]) { + return fail() + } + form.mask = 1 + } else if form.zeroing { + return fail() + } + vector := func(operand Operand, width int) bool { + if !amd64GoPackedVectorMoveRegister(operand, width, goarch) { + return false + } + if spec.encoding == amd64PackedMoveVEX { + return width != 64 && amd64VEXVectorRegister(operand, width) + } + return true + } + source, destination := ins.Args[0], ins.Args[form.destination] + switch { + case source.Kind == OpReg: + form.byteWidth = amd64VectorByteWidth(source.Reg) + if !vector(source, form.byteWidth) { + return fail() + } + if isAMD64MemoryOperand(destination) { + form.memory, form.memoryWrite = form.destination, true + } else if !vector(destination, form.byteWidth) { + return fail() + } + case isAMD64MemoryOperand(source): + form.byteWidth = amd64VectorByteWidth(destination.Reg) + if !vector(destination, form.byteWidth) { + return fail() + } + form.memory = 0 + default: + return fail() + } + if form.memory >= 0 { + memory := ins.Args[form.memory] + if !x86PackedShiftMemoryValid(memory, goarch) { + return fail() + } + if memory.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(memory.Mem, goarch); err != nil { + return form, true, err + } + if memory.Mem.OffRaw != "" || memory.Mem.IndexExt != "" { + return fail() + } + } + } + return form, true, x86Mode32VectorRegisterContext(ins, goarch) +} + +// This selects actual ordinary memory, not an extent or a continuation proof. +// The lowerer and private conditional result reader must select the same path. +func (form amd64PackedVectorMoveForm) unalignedOrdinaryMemory(ins Instr, arch string) bool { + return !form.spec.aligned && form.ordinaryMemory(ins, arch) +} + +func (form amd64PackedVectorMoveForm) ordinaryMemory(ins Instr, arch string) bool { + return form.memory >= 0 && + x86OrdinaryNativeMemoryOperand(ins.Args[form.memory], arch) +} + +func (form amd64PackedVectorMoveForm) nativeEVEX(ins Instr) bool { + if form.physicalEVEX || form.spec.encoding == amd64PackedMoveEVEX || form.mask >= 0 || form.byteWidth == 64 { + return true + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && amd64VectorByteWidth(operand.Reg) != 0 { + index, ok := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + if ok && index >= 16 { + return true + } + } + } + return false +} + +func amd64GoPackedVectorMoveRegister(operand Operand, width int, goarch string) bool { + if !amd64EVEXVectorRegister(operand, width) { + return false + } + index, _ := amd64VectorRegisterIndex(operand.Reg, width) + return goarch != "386" || width != 64 || index < 8 +} diff --git a/amd64_packed_vector_move_forms_test.go b/amd64_packed_vector_move_forms_test.go new file mode 100644 index 000000000..d9c68638b --- /dev/null +++ b/amd64_packed_vector_move_forms_test.go @@ -0,0 +1,222 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +type x86PackedMoveExpectedAxes struct { + operands string + memory, mask int + write, zero bool +} + +func TestX86PackedMoveGrammarCoversEveryCurrentGoFamilyMember(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go"), nil, 0) + if err != nil { + t.Fatal(err) + } + tables := map[string]amd64PackedMoveEncoding{ + "_yvmovdqa": amd64PackedMoveVEX, "_yvmovapd": amd64PackedMoveVEXOrEVEX, + "_yvmovdqa32": amd64PackedMoveEVEX, + } + wanted := make(map[Op]amd64PackedMoveEncoding) + rows := make(map[string]int) + ast.Inspect(file, func(node ast.Node) bool { + if value, ok := node.(*ast.ValueSpec); ok && len(value.Names) == 1 && len(value.Values) == 1 { + name := value.Names[0].Name + if _, family := tables[name]; family { + literal, ok := value.Values[0].(*ast.CompositeLit) + if !ok { + t.Fatalf("Go packed-move table is no longer literal: %s", name) + } + rows[name] = len(literal.Elts) + } + } + literal, ok := node.(*ast.CompositeLit) + if !ok { + return true + } + var opcode, table string + for _, element := range literal.Elts { + field, ok := element.(*ast.KeyValueExpr) + if !ok { + continue + } + key, keyOK := field.Key.(*ast.Ident) + value, valueOK := field.Value.(*ast.Ident) + if !keyOK || !valueOK { + continue + } + switch key.Name { + case "as": + opcode = value.Name + case "ytab": + table = value.Name + } + } + if encoding, family := tables[table]; family { + if !strings.HasPrefix(opcode, "AVMOV") { + t.Fatalf("unregistered spelling in shared Go move family: %s", opcode) + } + wanted[Op(strings.TrimPrefix(opcode, "A"))] = encoding + } + return true + }) + if !reflect.DeepEqual(rows, map[string]int{"_yvmovdqa": 4, "_yvmovapd": 16, "_yvmovdqa32": 12}) || len(wanted) != len(amd64PackedVectorMoveSpecs) { + t.Fatalf("current Go family changed: rows=%v opcodes=%v", rows, wanted) + } + for opcode, encoding := range wanted { + spec, present := amd64PackedVectorMoveSpecs[opcode] + if !present || spec.encoding != encoding { + t.Fatalf("Go opcode/encoding omitted or changed: %s/%d", opcode, encoding) + } + } +} + +func TestX86PackedMoveGrammarAxesAndFiveLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var opcodes []string + for opcode := range amd64PackedVectorMoveSpecs { + opcodes = append(opcodes, string(opcode)) + } + sort.Strings(opcodes) + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + sigs := make(map[string]FuncSig) + for _, opcode := range opcodes { + spec := amd64PackedVectorMoveSpecs[Op(opcode)] + for index, bank := range []string{"X", "Y", "Z"} { + if bank == "Z" && spec.encoding == amd64PackedMoveVEX { + continue + } + width, high := 16< %s, %s %s, align 1\n", width, value, pointerType, pointer) + return nil + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, %s %s, align 1\n", value, width, pointerType, pointer) + return c.storePackedMoveOperand(ins.Args[form.destination], width, "%"+value) + } + + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + bits := form.spec.elementBits + lanes := width * 8 / bits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, bits) + value := "zeroinitializer" + if form.memoryWrite { + value, err = c.loadPackedCompareLanes(ins.Args[0], width, bits, false) + } else if !form.zeroing { + value, err = c.loadPackedCompareLanes(ins.Args[form.destination], width, bits, false) + } + if err != nil { + return err + } + + for lane := 0; lane < lanes; lane++ { + condition := amd64MaskBitI1(c, mask, lane) + address := c.newTmp() + // No inbounds GEP: inactive lanes, including a null base, need no + // allocation. Only the guarded branch accesses the selected bytes. + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %d\n", address, pointerType, pointer, lane*bits/8) + if form.memoryWrite { + element := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", element, vectorType, value, lane) + storeLabel, joinLabel := c.newTmp()+"_packed_store", c.newTmp()+"_packed_join" + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n%s:\n", condition, storeLabel, joinLabel, storeLabel) + fmt.Fprintf(c.b, " store volatile i%d %%%s, %s %%%s, align 1\n", bits, element, pointerType, address) + fmt.Fprintf(c.b, " br label %%%s\n%s:\n", joinLabel, joinLabel) + continue + } + loaded, err := c.loadX86ScalarIf(bits, condition, func() (string, error) { + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %%%s, align 1\n", loaded, bits, pointerType, address) + return "%" + loaded, nil + }) + if err != nil { + return err + } + if !form.zeroing { + old, selected := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", old, vectorType, value, lane) + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %s, i%d %%%s\n", selected, condition, bits, loaded, bits, old) + loaded = "%" + selected + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, value, bits, loaded, lane) + value = "%" + inserted + } + if form.memoryWrite { + return nil + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytes, vectorType, value, width) + return c.storePackedMoveOperand(ins.Args[form.destination], width, "%"+bytes) +} diff --git a/amd64_packed_vector_move_observable_runtime_linux_amd64_test.go b/amd64_packed_vector_move_observable_runtime_linux_amd64_test.go new file mode 100644 index 000000000..aeccd0ca4 --- /dev/null +++ b/amd64_packed_vector_move_observable_runtime_linux_amd64_test.go @@ -0,0 +1,9 @@ +//go:build linux && amd64 && plan9asm_native_avx512 + +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86PackedMoveUnalignedOriginalGo(t *testing.T) { + x86PackedMoveUnalignedRuntime(t, true) +} diff --git a/amd64_packed_vector_move_observable_runtime_unix_test.go b/amd64_packed_vector_move_observable_runtime_unix_test.go new file mode 100644 index 000000000..0e6eaadca --- /dev/null +++ b/amd64_packed_vector_move_observable_runtime_unix_test.go @@ -0,0 +1,256 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedMoveUnalignedPortableFaults(t *testing.T) { + x86PackedMoveUnalignedRuntime(t, false) +} + +func x86PackedMoveUnalignedRuntime(t *testing.T, native bool) { + x86PackedMoveObservableRuntime(t, native, false) +} + +func x86PackedMoveObservableRuntime(t *testing.T, native, aligned bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + if native { + execution = x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + } + ir, main, count := x86PackedMoveObservableFixture(t, native, aligned, execution.triple) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + execution.compileAndRunLLVM(t, "packed-move-observable", text, main, "-O2") + }) + } + t.Logf("%d independent move cases; original native Go execution=%t; aligned native semantics=%t", count, native, aligned) +} + +func x86PackedMoveObservableFixture(t *testing.T, native, aligned bool, triple string) (string, string, int) { + t.Helper() + if aligned && !native { + t.Fatal("aligned fault semantics require the original native counterpart") + } + var source, declarations, originals, pairs strings.Builder + sigs := make(map[string]FuncSig) + count := 0 + for _, row := range x86PackedMoveResultOracle { + if row.aligned != aligned { + continue + } + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + width := 16 << axis + for _, mode := range []string{"plain-live", "plain-dead", "plain-store", "merge-live", "zero-live", "zero-dead", "mask-store"} { + masked := strings.HasPrefix(mode, "merge") || strings.HasPrefix(mode, "zero") || mode == "mask-store" + if row.vexOnly && masked { + continue + } + store, dead, zero := strings.HasSuffix(mode, "store"), strings.HasSuffix(mode, "dead"), strings.HasPrefix(mode, "zero") + op, mask := row.name, "" + if masked { + mask = "K1," + } + if zero { + op += ".Z" + } + operation := fmt.Sprintf("%s (AX),%s%s0\n", op, mask, bank) + if store { + operation = fmt.Sprintf("%s %s0,%s(AX)\n", op, bank, mask) + } + name := fmt.Sprintf("packed_observable%d", count) + body := "KMOVQ CX,K1\nVMOVUPS (DI),Z0\nSTC\n" + operation + if !store && !dead { + body += "VMOVUPS Z0,(DI)\n" + } + body += "SETCS 64(DI)\n" + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ p+0(FP),AX\nMOVQ out+8(FP),DI\nMOVQ mask+16(FP),CX\n%sRET\n", name, body) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,uint8_t*,uint64_t);\n", name) + original := "0" + if native { + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+body+"RET\n") + if len(code) < 2 || code[len(code)-1] != 0xc3 { + t.Fatal("missing original Go opcode and RET", name, code) + } + var bytes []string + for _, value := range code[:len(code)-1] { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + original = "original_" + name + fmt.Fprintf(&originals, `__attribute__((target("avx512f,avx512bw,avx512vl"))) +static void %s(uint8_t *p,uint8_t *out,uint64_t mask) { + __asm__ volatile(".byte %s" : : "a"(p),"D"(out),"c"(mask) : "zmm0","k1","memory","cc"); +} +`, original, strings.Join(bytes, ",")) + } + flag := func(value bool) int { + if value { + return 1 + } + return 0 + } + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d,%d,%d,%d},\n", original, name, width, row.bits/8, flag(masked), flag(store), flag(dead), flag(zero)) + count++ + } + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + alignment := "0" + if aligned { + alignment = "1" + } + main := "#define ALIGNED " + alignment + "\n" + x86PackedMoveObservableRuntimeC + declarations.String() + originals.String() + "static const struct pair pairs[]={\n" + pairs.String() + `}; +int main(void) { + if (pairs[0].original) { + unsigned a,b,c,d; + if (!__get_cpuid_count(7,0,&a,&b,&c,&d) || !(b&(1u<<30))) return 78; + } + long page=sysconf(_SC_PAGESIZE); + if(page<256) return 1; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(region==MAP_FAILED || mprotect(region+page,page,PROT_NONE)) return 2; + for(unsigned op=0;op +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef void (*operation)(uint8_t*,uint8_t*,uint64_t); +struct pair { operation original,translated; int width,lane,masked,store,dead,zero; }; +static int active(struct pair pair,uint64_t mask,int byte) { + return !pair.masked || ((mask>>(byte/pair.lane))&1); +} +static int expected_fault(struct pair pair,uint64_t mask,int extent) { + for(int byte=extent;byte=pair.width?0:active(pair,masks[m],i)?a[in+i]:pair.zero?0:x[out+i]; + } + } + want_x[out+64]=1; + unsigned saved=_mm_getcsr(),csr=0x1f80|((sample&3)<<13)|(sample&63); + _mm_setcsr(csr); pair.translated(a+in,x+out,masks[m]); unsigned after=_mm_getcsr(); + _mm_setcsr(saved); + if(after!=csr || memcmp(a,want_a,256) || memcmp(x,want_x,256)) return 1; + if(pair.original) { + _mm_setcsr(csr); pair.original(b+in,y+out,masks[m]); after=_mm_getcsr(); + _mm_setcsr(saved); + if(after!=csr || memcmp(a,b,256) || memcmp(x,y,256)) return 2; + } + } + return 0; +} +` + +func TestX86PackedMoveObservableFaultChildrenDisableCore(t *testing.T) { + if err := runtimeOracleFaultCorePolicy(runtimeOracleFaultC(t, "amd64_packed_vector_move_observable_runtime_unix_test.go")); err != nil { + t.Fatal(err) + } +} diff --git a/amd64_packed_vector_move_result.go b/amd64_packed_vector_move_result.go new file mode 100644 index 000000000..7ca7f5e08 --- /dev/null +++ b/amd64_packed_vector_move_result.go @@ -0,0 +1,60 @@ +package plan9asm + +// Memory availability is separate from the register's defined-byte transfer. +// conditionalRead is only supplied by the lowering-only, source-wide observer +// paired with an exact native aligned or unaligned volatile read. It is never a +// public observer, initialized-pointee, extent, alignment or write permission. +func x86ResultPackedVectorMoveTransfer(ins Instr, form amd64PackedVectorMoveForm, state *x86ResultState, owned []bool, resultOffset int64, arch string, conditionalRead bool) bool { + if form.undefinedOpcode() { + return false + } + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false + } + } + width := int64(form.byteWidth) + widthMask := x86CallerVectorByteMask(width) + known := widthMask + if form.memory >= 0 && form.spec.aligned && !form.ordinaryMemory(ins, arch) { + // Virtual FP/AUTO storage is not the original aligned memory. Ordinary + // memory instead retains its actual opcode: continuation defines its + // active bytes without assuming that its native address was aligned. + return false + } + if form.memory == 0 { + if !conditionalRead && !x86ResultRead(ins.Args[0], width, state, owned, arch) { + return false + } + } else { + index, valid := amd64VectorRegisterIndex(ins.Args[0].Reg, form.byteWidth) + if !valid { + return false + } + known = state.vectorBytes(index) & widthMask + } + if form.memoryWrite { + if known != widthMask { + return false + } + destination := ins.Args[form.destination] + if conditionalRead && form.ordinaryMemory(ins, arch) && + x86ResultIncomingPointerAddressIdentity(destination, state, arch) { + // The paired native store does not read inactive destination lanes + // and establishes no logical storage, bounds or future read facts. + return true + } + if form.mask >= 0 && !x86ResultRead(destination, width, state, owned, arch) { + return false // An unknown mask may retain any old destination byte. + } + return x86ResultStore(destination, width, state, owned, resultOffset, arch) + } + destination, valid := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + if !valid { + return false + } + known = form.definedActiveBytes(known, state.vectorBytes(destination)) + state.writeVector(destination, form.byteWidth, known, false) + return true +} diff --git a/amd64_packed_vector_move_result_test.go b/amd64_packed_vector_move_result_test.go new file mode 100644 index 000000000..aeed03cb2 --- /dev/null +++ b/amd64_packed_vector_move_result_test.go @@ -0,0 +1,450 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// Independent family inventory: do not derive expected permissions or widths +// from the production registry whose completeness these tests check. +var x86PackedMoveResultOracle = []struct { + name string + bits int + aligned, vexOnly bool +}{ + {"VMOVDQA", 8, true, true}, {"VMOVDQU", 8, false, true}, + {"VMOVAPD", 64, true, false}, {"VMOVAPS", 32, true, false}, + {"VMOVUPD", 64, false, false}, {"VMOVUPS", 32, false, false}, + {"VMOVDQA32", 32, true, false}, {"VMOVDQA64", 64, true, false}, + {"VMOVDQU8", 8, false, false}, {"VMOVDQU16", 16, false, false}, + {"VMOVDQU32", 32, false, false}, {"VMOVDQU64", 64, false, false}, +} + +func TestX86PackedMoveResultEveryByteAndWriteEncoding(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue // The independent Go table has no VEX.512 row. + } + width := 16 << axis + for _, mode := range []string{"plain", "merge", "zero"} { + if row.vexOnly && mode != "plain" { + continue + } + name, mask := row.name, "" + if mode != "plain" { + mask = ",K1" + } + if mode == "zero" { + name += ".Z" + } + instruction := fmt.Sprintf("%s %s0%s,%s1", name, bank, mask, bank) + ins := x86ScalarLaneInstruction(t, instruction) + for missing := 0; missing < 64; missing++ { + for source := 0; source < 2; source++ { + state := x86ResultState{flags: x86ByteFlagsAll, masks: [8]bool{1: true}} + state.writeVector(0, 64, ^uint64(0), false) + state.writeVector(1, 64, ^uint64(0), false) + state.writeVector(source, 64, ^(uint64(1) << uint(missing)), false) + before := state.clone() + if !x86ScalarFPResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatalf("typed packed move is missing: %s", instruction) + } + want := ^uint64(0) + if missing < width && (source == 0 || mode == "merge") { + want &^= uint64(1) << uint(missing) + } + if state.vectorBytes(1) != want || state.vectorBytes(0) != before.vectorBytes(0) || state.flags != before.flags { + t.Fatalf("%s source%d missing%d: got %016x want %016x", instruction, source, missing, state.vectorBytes(1), want) + } + } + } + } + } + } +} + +func TestX86PackedMoveResultMemoryAndMaskBoundaries(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + width := 16 << axis + for _, memory := range []string{"in+0(FP)", fmt.Sprintf("local-%d(SP)", width)} { + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s %s,%s1", row.name, memory, bank)) + owned := make([]bool, width) + for i := range owned { + owned[i] = true + } + for missing := -1; missing < width; missing++ { + state := x86ResultState{fp: append([]bool(nil), owned...), auto: append([]bool(nil), owned...)} + if missing >= 0 { + state.fp[missing], state.auto[missing] = false, false + } + got := x86ScalarFPResultTransfer(ins, &state, owned, 0, "amd64") + if got != (!row.aligned && missing < 0) { + t.Fatalf("%s %s missing%d: accepted=%t", row.name, memory, missing, got) + } + } + } + if row.vexOnly { + continue + } + for _, suffix := range []string{"", ".Z"} { + state := x86ResultState{} + state.writeVector(0, 64, ^uint64(0), false) + state.writeVector(1, 64, ^uint64(0), false) + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s%s %s0,K1,%s1", row.name, suffix, bank, bank)) + if x86ScalarFPResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("undefined mask acquired a numerical value", ins) + } + } + } + } +} + +func TestX86PackedMoveResultFiveTargetsThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + word, pointerMove := 8, "MOVQ" + if target.arch == "386" { + word, pointerMove = 4, "MOVL" + } + var source, declarations strings.Builder + count := 0 + for _, row := range x86PackedMoveResultOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + for _, memory := range []bool{false, true} { + for _, mode := range []string{"plain", "merge", "zero"} { + if row.vexOnly && mode != "plain" { + continue + } + name, mask, input := row.name, "", bank+"0" + if memory { + input = "(SI)" + } + if mode != "plain" { + mask = ",K1" + } + if mode == "zero" { + name += ".Z" + } + fmt.Fprintf(&declarations, "func view%d(p *[64]byte) uint32\n", count) + fmt.Fprintf(&source, "TEXT ·view%d(SB),4,$0-%d\n%s p+0(FP),SI\nMOVL $1,AX\nKMOVD AX,K1\nVPXORQ %s0,%s0,%s0\nVPXORQ %s1,%s1,%s1\n%s %s%s,%s1\nPEXTRD $3,X1,AX\nMOVL AX,ret+%d(FP)\nRET\n", + count, word+4, pointerMove, bank, bank, bank, bank, bank, bank, name, input, mask, bank, word) + count++ + } + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + compileLLVMToObject(t, llc, target.triple, "packed-result.ll", "packed-result.o", ir) + } + }) + } +} + +func TestX86PackedMoveResultPointerReadIsLoweringOnly(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for _, arch := range []string{"amd64", "386"} { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + body := fmt.Sprintf("%s p+0(FP),AX\n%s (AX),X1\nPEXTRD $0,X1,CX\nMOVL CX,ret0+%d(FP)\nRET", move, row.name, word) + _, file, sig, _ := x86ResultInitializationFixture(t, arch, "func Value(p *[64]byte) uint32", body) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("declaration-only observer acquired a physical read", row.name, arch, err) + } + } + } + } +} + +func TestX86PackedMoveUnalignedDeadReadIsObservable(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if row.aligned { + continue + } + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + width := 16 << axis + source := fmt.Sprintf("TEXT ·view(SB),4,$0-8\nMOVQ p+0(FP),AX\n%s (AX),%s0\nRET\n", row.name, bank) + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, fmt.Sprintf("load volatile <%d x i8>", width)) { + t.Fatal("dead full-tuple native read is removable", row.name, bank) + } + } + } +} + +func TestX86PackedMoveMaskedUnalignedReadIsObservable(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if row.aligned || row.vexOnly { + continue + } + for axis, bank := range []string{"X", "Y", "Z"} { + width := 16 << axis + source := fmt.Sprintf("TEXT ·view(SB),4,$0-16\nMOVQ p+0(FP),AX\nMOVQ mask+8(FP),CX\nKMOVQ CX,K1\n%s.Z (AX),K1,%s0\nRET\n", row.name, bank) + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte, mask uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if err != nil { + t.Fatal(err) + } + if got := strings.Count(ir, fmt.Sprintf("load volatile i%d,", row.bits)); got != width*8/row.bits { + t.Fatalf("%s %s masked dead read has %d observable lanes; want %d", row.name, bank, got, width*8/row.bits) + } + } + } +} + +func TestX86PackedMoveResultStoreAndStaticExtentBoundaries(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + width := 16 << axis + for _, destination := range []string{"ret+0(FP)", fmt.Sprintf("local-%d(SP)", width)} { + for _, masked := range []bool{false, true} { + if row.vexOnly && masked { + continue + } + mask := "" + if masked { + mask = "K1," + } + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s %s0,%s%s", row.name, bank, mask, destination)) + for _, oldKnown := range []bool{false, true} { + for missing := -1; missing < width; missing++ { + owned := make([]bool, width) + state := x86ResultState{fp: make([]bool, width), auto: make([]bool, width), masks: [8]bool{1: true}} + for i := range owned { + owned[i], state.fp[i], state.auto[i] = true, oldKnown, oldKnown + } + known := ^uint64(0) + if missing >= 0 { + known &^= uint64(1) << uint(missing) + } + state.writeVector(0, 64, known, false) + want := !row.aligned && missing < 0 && (!masked || oldKnown) + if got := x86ScalarFPResultTransfer(ins, &state, owned, 0, "amd64"); got != want { + t.Fatal("store borrowed source or inactive destination bytes", ins, oldKnown, missing, got, want) + } + } + } + } + } + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s table<>+16(SB),%s1", row.name, bank)) + for _, available := range []int64{int64(width - 1), int64(width)} { + state := x86ResultState{loweredStaticReads: map[x86StaticSBAddress]int64{{symbol: "table<>", offset: 16}: available}} + if got := x86ScalarFPResultTransfer(ins, &state, nil, 0, "amd64"); got != (available == int64(width)) { + t.Fatal("static read ignored complete extent", row.name, bank, available, got) + } + } + } + } +} + +func TestX86PackedMoveResultSourceBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word, move, triple := 8, "MOVQ", "x86_64-unknown-linux-gnu" + if arch == "386" { + word, move, triple = 4, "MOVL", "i386-unknown-linux-gnu" + } + for _, row := range x86PackedMoveResultOracle { + load := move + " p+0(FP),AX\n" + read := row.name + " (AX),X1\n" + bodies := map[string]string{ + "unknown base": read, + "numeric base": "MOVL $1,AX\n" + read, + "lost identity": load + "MOVL $1,AX\n" + read, + "unknown index": load + row.name + " (AX)(DI*1),X1\n", + "pointer index": load + row.name + " (AX)(AX*1),X1\n", + "unknown store source": load + row.name + " X1,(AX)\n", + "unknown store base": load + read + row.name + " X1,(DI)\n", + "pointer store index": load + read + row.name + " X1,(AX)(AX*1)\n", + "unknown static": row.name + " ·foreign(SB),X1\n", + "unknown path": load + "MOVL $1,CX\nCMPL CX,$0\nJE done\n" + read + "done:\n", + } + if !row.vexOnly { + bodies["undefined mask"] = load + row.name + ".Z (AX),K1,X1\n" + bodies["undefined merge"] = load + "MOVL $1,CX\nKMOVD CX,K1\n" + row.name + " (AX),K1,X1\n" + bodies["undefined store mask"] = load + read + row.name + " X1,K1,(AX)\n" + } + for name, body := range bodies { + source := fmt.Sprintf("TEXT ·view(SB),4,$0-%d\n%sPEXTRD $0,X1,CX\nMOVL CX,ret+%d(FP)\nRET\n", word+4, body, word) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte) uint32\n", arch, triple, api) + if arch == "amd64" && (name == "unknown store source" || name == "unknown path" || name == "undefined merge") { + x86VectorScratchCodegenControl(t, source, "func view(p *[64]byte) uint32\n", arch, triple, ir, err) + continue + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("source closure or incoming identity bypass", arch, row.name, name, api, err) + } + } + } + } + } +} + +func x86PackedMoveInitializedInstruction(name, arch, initialization, instruction, observation string) (string, string) { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + source := fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),AX\nMOVL $7,CX\nKMOVD CX,K1\n%s%s\n"+ + "%s\nMOVL CX,ret+%d(FP)\nRET\n", name, word+4, move, initialization, instruction, observation, word) + return source, fmt.Sprintf("func %s(p *[64]byte) uint32\n", name) +} + +func TestX86PackedMoveInitializedNativeStoresWholeFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc is required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + var names []string + for _, row := range x86PackedMoveResultOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + for _, masked := range []bool{false, true} { + if row.vexOnly && masked { + continue + } + predicate := "" + if masked { + predicate = "K1," + } + name := fmt.Sprintf("view%d", len(names)) + names = append(names, name) + // Keep the original initialized load/store pair, including + // native alignment effects. Its result is independent of the + // store and cannot be supplied by a fresh logical allocation. + body, declaration := x86PackedMoveInitializedInstruction(name, target.arch, + fmt.Sprintf("%s (AX),%s1\n", row.name, bank), fmt.Sprintf("%s %s1,%s(AX)", row.name, bank, predicate), "PEXTRD $0,X1,CX") + source.WriteString(body) + declarations.WriteString(declaration) + } + } + } + if len(names) != 64 { + t.Fatal("complete initialized native-store denominator changed", len(names)) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("complete initialized native-store sequence rejected", err) + } + x86CompactExpandRequireAllDefinitions(t, ir, names) + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86DwordProductIROptimize(t, ir) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR-O2=%t/backend%s", optimized, level), func(t *testing.T) { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + }) + } + } + }) + } + }) + } +} + +func TestX86PackedMoveResultRawOriginBoundaries(t *testing.T) { + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[64]byte) uint32", + "MOVQ p+0(FP),AX\nVMOVDQU64 (AX),Z1\nPEXTRD $0,X1,CX\nMOVL CX,ret0+8(FP)\nRET") + for _, mutation := range []string{"decoded", "address size", "raw text", "source stack", "fixed local", "segment", "symbol"} { + fn := file.Funcs[0] + fn.Instrs = append([]Instr(nil), fn.Instrs...) + fn.Instrs[2].Args = append([]Operand(nil), fn.Instrs[2].Args...) + switch mutation { + case "decoded": + fn.Instrs[2].x86Encoded = true + case "address size": + fn.Instrs[2].x86AddressBits = 32 + case "raw text": + fn.X86RawText = []byte{0xc3} + case "source stack": + fn.x86SourceStack = new(x86SourceStackProof) + case "fixed local": + fn.x86FixedLocal = new(x86FixedLocalProof) + case "segment": + fn.Instrs[2].Args[0].Mem.Segment = "FS" + case "symbol": + fn.Instrs[2].Args[0].Mem.Sym = "table<>(SB)" + } + ctx := &amd64Ctx{fpScalarSource: fn, sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("raw/source role acquired ordinary conditional byte availability", mutation, err) + } + } +} + +func TestX86PackedMoveResultStaticSourceThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + var source, declarations strings.Builder + source.WriteString("DATA table<>+0(SB)/8,$0x76543210abcdef01\nGLOBL table<>(SB),8,$64\n") + count := 0 + for _, row := range x86PackedMoveResultOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + addresses := []string{"table<>(SB)"} + if target.arch == "386" { + for _, annotation := range []string{"AX", "BX", "CX", "DX", "SI", "DI", "SP", "BP"} { + addresses = append(addresses, "table<>(SB)("+annotation+")") + } + } + for _, address := range addresses { + fmt.Fprintf(&declarations, "func view%d() uint32\n", count) + fmt.Fprintf(&source, "TEXT ·view%d(SB),4,$0-4\n%s %s,%s1\nPEXTRD $0,X1,AX\nMOVL AX,ret+0(FP)\nRET\n", count, row.name, address, bank) + count++ + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("complete DATA directory did not compose with physical reads", target, api, err) + } + compileLLVMToObject(t, llc, target.triple, "packed-data.ll", "packed-data.o", ir) + } + } +} diff --git a/amd64_packed_vector_move_runtime_partition_test.go b/amd64_packed_vector_move_runtime_partition_test.go new file mode 100644 index 000000000..df54ab59a --- /dev/null +++ b/amd64_packed_vector_move_runtime_partition_test.go @@ -0,0 +1,109 @@ +package plan9asm + +import ( + "go/ast" + "os" + "strings" + "testing" +) + +var x86PackedMoveRuntimeRows = []struct { + op string + bits int +}{ + {"VMOVAPS", 32}, {"VMOVAPD", 64}, {"VMOVUPS", 32}, {"VMOVUPD", 64}, + {"VMOVDQA32", 32}, {"VMOVDQA64", 64}, {"VMOVDQU8", 8}, {"VMOVDQU16", 16}, {"VMOVDQU32", 32}, {"VMOVDQU64", 64}, +} + +// Test-only ISA partition. An aligned register copy stays portable; only its +// original physical memory instruction requires the native counterpart. +func x86PackedMoveRuntimeNativeRow(op string, memory bool) bool { + if !memory { + return false + } + switch op { + case "VMOVDQA", "VMOVAPS", "VMOVAPD", "VMOVDQA32", "VMOVDQA64": + return true + } + return false +} + +func TestX86PackedMoveRuntimePartitionIsComplete(t *testing.T) { + frameFunctions := fixedLocalLeaseFunctions(t, "amd64_lower_packed_move_frame_test.go") + for name, execute := range map[string]string{ + "TestX86PackedMoveFrameSlotsObjects": "false", "TestX86PackedMoveFrameSlotsRuntime": "true", + } { + function := frameFunctions[name] + if function == nil || fixedLocalLeaseCalls(function.Body)["x86PackedMoveFrameSlots"] != 1 { + t.Fatal("missing original frame objects or native runtime owner", name) + } + ast.Inspect(function.Body, func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok && fixedLocalLeaseCallName(call) == "x86PackedMoveFrameSlots" { + if len(call.Args) != 2 || !fixedLocalLeaseIdent(call.Args[1], execute) { + t.Error("frame object compilation changed its role or weakened native execution", name) + } + } + return true + }) + } + // These are the exact historical rows, independent of the implementation + // registry. Every old row must have one owner, never an execution skip. + oldRows := map[string]int{"VMOVAPS": 32, "VMOVAPD": 64, "VMOVUPS": 32, "VMOVUPD": 64, "VMOVDQA32": 32, "VMOVDQA64": 64, "VMOVDQU8": 8, "VMOVDQU16": 16, "VMOVDQU32": 32, "VMOVDQU64": 64} + var masked, views, frame [2]int + seen := make(map[string]bool) + for _, row := range x86PackedMoveRuntimeRows { + name := row.op + if seen[name] || oldRows[name] != row.bits { + t.Fatal("unknown or duplicate historical runtime row", row) + } + seen[name] = true + aligned := name == "VMOVAPS" || name == "VMOVAPD" || name == "VMOVDQA32" || name == "VMOVDQA64" + if aligned { + frame[1]++ // Actual-Go accepted; virtual FP must report Context. + } else { + frame[0]++ // Numeric portable FP oracle. + } + for width := 0; width < 3; width++ { + owner := 0 + if x86PackedMoveRuntimeNativeRow(name, true) { + owner = 1 + } + if (owner == 1) != aligned { + t.Fatal("incorrect historical masked-memory owner", name, width) + } + masked[owner]++ + for form := 0; form < 6; form++ { + owner = 0 + if x86PackedMoveRuntimeNativeRow(name, form < 3) { + owner = 1 + } + if (owner == 1) != (aligned && form < 3) { + t.Fatal("incorrect historical view owner", name, width, form) + } + views[owner]++ + } + } + } + if len(seen) != len(oldRows) || masked != [2]int{18, 12} || views != [2]int{144, 36} || frame != [2]int{6, 4} { + t.Fatal("lost old rows", masked, views, frame) + } + for file, roots := range map[string][]string{ + "amd64_lower_packed_move_memory_test.go": {"TestX86PackedMoveMaskedMemoryRuntime", "TestX86PackedMoveRegisterViewsRuntime"}, + "amd64_packed_vector_move_aligned_runtime_linux_amd64_test.go": { + "TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedMaskedMemory", "TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedRegisterViews", "TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedOriginalGo"}, + } { + source, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(source), "t.Skip") { + t.Fatal("runtime partition silently skips rows", file) + } + for _, root := range roots { + if !strings.Contains(string(source), "func "+root+"(t *testing.T)") { + t.Fatal("missing required runtime counterpart", root) + } + } + } + t.Log("historical memory rows: portable=18 native=12; alias rows: portable=144 native=36; FP rows: portable=6 explicit Context=4; all native roots are required cross-runtime tests") +} diff --git a/amd64_packed_word_multiply_forms.go b/amd64_packed_word_multiply_forms.go new file mode 100644 index 000000000..b93f89346 --- /dev/null +++ b/amd64_packed_word_multiply_forms.go @@ -0,0 +1,41 @@ +package plan9asm + +type x86PackedWordMultiplySpec struct { + mode amd64PackedWordMultiplyMode + encoding x86PackedLogicalEncoding +} + +var x86PackedWordMultiplySpecs = map[Op]x86PackedWordMultiplySpec{ + "PMULLW": {amd64PackedWordMultiplyLow, x86PackedLogicalLegacyInteger}, + "PMULHW": {amd64PackedWordMultiplySignedHigh, x86PackedLogicalLegacyInteger}, + "PMULHUW": {amd64PackedWordMultiplyUnsignedHigh, x86PackedLogicalLegacyInteger}, + "PMULHRSW": {amd64PackedWordMultiplyRoundedSignedHigh, x86PackedLogicalLegacyFP}, + "VPMULLW": {amd64PackedWordMultiplyLow, x86PackedLogicalVEXOrEVEX}, + "VPMULHW": {amd64PackedWordMultiplySignedHigh, x86PackedLogicalVEXOrEVEX}, + "VPMULHUW": {amd64PackedWordMultiplyUnsignedHigh, x86PackedLogicalVEXOrEVEX}, + "VPMULHRSW": {amd64PackedWordMultiplyRoundedSignedHigh, + x86PackedLogicalVEXOrEVEX}, +} + +type x86PackedWordMultiplyForm = x86PackedPhysicalForm + +// This shared grammar records the real native read/modify/write shape only. +// It grants neither input initialization nor native backing/caller authority. +func x86PackedWordMultiplyProperties(op Op, ins Instr, arch string) (x86PackedWordMultiplyForm, bool, error) { + return x86PackedPhysicalProperties(op, ins, arch, x86PackedWordProduct) +} + +func x86ResultPackedWordMultiplyStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, known, err := x86PackedWordMultiplyProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + if !form.mmx && form.spec.mode == amd64PackedWordMultiplyLow { + // W/D/Q low products share per-lane numerical dependencies. Missing + // source bytes leave only their dependent result lanes unknown; they + // never become initialized implementation-slot values. Whole-source + // word input obligations remain in the separate scalar-FP consumer. + return x86ResultPackedLaneArithmeticStep(ins, form.x86PackedBinaryForm, 16, state, owned, arch), true + } + return x86ResultFixedVectorInputs(ins, form.x86PackedBinaryForm, false, state, owned, arch), true +} diff --git a/amd64_packed_word_multiply_forms_test.go b/amd64_packed_word_multiply_forms_test.go new file mode 100644 index 000000000..54004eeac --- /dev/null +++ b/amd64_packed_word_multiply_forms_test.go @@ -0,0 +1,540 @@ +package plan9asm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +var x86WordMultiplyNames = []string{ + "PMULLW", "PMULHW", "PMULHUW", "PMULHRSW", + "VPMULLW", "VPMULHW", "VPMULHUW", "VPMULHRSW", +} + +// This catalog is derived from the actual Go ymm, yxm_q4 and _yvandnpd +// rows, independently of the production specification. It includes valid +// syntax which requires a different memory or physical input contract. +func x86WordMultiplyForms(arch string) []string { + var rows []string + for _, name := range x86WordMultiplyNames[:4] { + for _, source := range []string{"X0", "X7", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-16(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+",X1") + } + if arch == "amd64" { + rows = append(rows, name+" X15,X0") + if name != "PMULHRSW" { + for _, source := range []string{"M0", "M7", "(AX)", "value+0(FP)", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + rows = append(rows, name+" "+source+",M1") + } + } + } + } + for _, name := range x86WordMultiplyNames[4:] { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", bank + "7", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1,%s2", name, source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s,%s31,K1,%s30", name, source, bank, bank), + fmt.Sprintf("%s.Z %s,%s31,K7,%s30", name, source, bank, bank)) + } + } + if arch == "amd64" || bank != "Z" { + rows = append(rows, fmt.Sprintf("%s %s20,%s21,%s22", name, bank, bank, bank)) + } + } + } + return rows +} + +func TestX86PackedWordMultiplyCompleteActualGoAttributes(t *testing.T) { + if len(x86PackedWordMultiplySpecs) != 8 { + t.Fatal("complete named packed-word multiply family must contain eight names") + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APMULLW, ymm, Py1, opBytes{0xd5, Pe, 0xd5}}", + "{APMULHW, ymm, Py1, opBytes{0xe5, Pe, 0xe5}}", + "{APMULHUW, ymm, Py1, opBytes{0xe4, Pe, 0xe4}}", + "{APMULHRSW, yxm_q4, Pq4, opBytes{0x0b}}", + "{Zm_r_xm, 1, argList{Ymm, Ymr}}", "{Zm_r_xm, 2, argList{Yxm, Yxr}}", + "{Zm_r, 1, argList{Yxm, Yxr}}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatalf("actual Go complete legacy row changed: %s", row) + } + } + for _, class := range []string{"Yax", "Ycx", "Yrx", "Yrl", "Yrl32", "Ym", "Ymr"} { + if !strings.Contains(string(legacy), "ycover["+class+"*Ymax+Ymm] = 1") { + t.Fatal("actual Go MMX source class changed", class) + } + } + for _, classes := range []string{ + "Yxm, Yxr, Yxr", "Yym, Yyr, Yyr", + "YxmEvex, YxrEvex, YxrEvex", "YxmEvex, YxrEvex, Yknot0, YxrEvex", + "YymEvex, YyrEvex, YyrEvex", "YymEvex, YyrEvex, Yknot0, YyrEvex", + "Yzm, Yzr, Yzr", "Yzm, Yzr, Yknot0, Yzr", + } { + start := strings.Index(string(avx), "var _yvandnpd =") + table := strings.SplitN(string(avx)[start:], "\n}", 2)[0] + if !strings.Contains(table, "argList{"+classes+"}") || strings.Count(table, "argList{") != 8 { + t.Fatal("actual Go complete VEX/EVEX class rows changed", classes) + } + } + for _, row := range x86PackedWordMultiplyGoRows { + start := strings.Index(string(avx), "{as: A"+string(row.op)+",") + if start < 0 { + t.Fatal("actual Go omitted opcode", row.op) + } + record := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(record, "ytab: _yvandnpd, prefix: Pavx") || + strings.Contains(record, "Bcst") || strings.Contains(record, "Sae") { + t.Fatal("actual Go opcode-specific attribute changed", record) + } + mapName := "0F" + if row.mapID == 2 { + mapName = "0F38" + } + for _, width := range []string{"128", "256"} { + want := fmt.Sprintf("avxEscape | vex%s | vex66 | vex%s | vexW0, 0x%02X", width, mapName, row.opcode) + if !strings.Contains(record, want) { + t.Fatal("actual Go VEX encoding changed", want) + } + } + for index, width := range []string{"128", "256", "512"} { + want := fmt.Sprintf("avxEscape | evex%s | evex66 | evex%s | evexW0, evexN%d | evexZeroingEnabled, 0x%02X", width, mapName, 16<= 3: + registerIndex = (index - 3) & 7 + } + if len(code) == 5 && code[0] == 0x41 { + code = code[1:] // Actual Go's ignored MMX REX.B annotation. + } + want := []byte{0x0f, row.opcode, byte(0xc8 | registerIndex), 0xc3} + if !bytes.Equal(code, want) { + t.Fatalf("%s actual encoded source bank=%x want=%x", source, code, want) + } + } + } +} + +func TestX86PackedWordMultiplyMalformedIdentityAndBanksBeforeIR(t *testing.T) { + for _, name := range x86WordMultiplyNames { + vector := strings.HasPrefix(name, "V") + operands := "X0,X1" + if vector { + operands = "X0,X1,X2" + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+name+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", original.Op, Op(strings.ToLower(name)), "MOVQ", Op(name + ".BAD")} { + ins := original + ins.Op = identity + copy := ins + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before, temporary := out.String(), ctx.tmp + known, term, err := ctx.lowerPackedWordMultiply(original.Op, ins) + bad := identity != "" && !strings.EqualFold(string(identity), name) + if !known || term || (err != nil) != bad || bad && (out.String() != before || ctx.tmp != temporary) || !reflect.DeepEqual(ins, copy) { + t.Fatal("dispatch mismatch/clone contract", name, identity, known, term, err) + } + } + bad := []string{"$1,X1", "Y0,Y1", "X16,X0", "X0,(AX)", "(AX),(BX)", "(AX)(SP*2),X1"} + if vector { + bad = []string{"$1,X1,X2", "X0,Y1,Y2", "X0,X1,(AX)", "X0,X1,K0,X2", "(AX)(SP*2),X1,X2", "2147483648(AX),X1,X2"} + } + for _, operands := range bad { + source := "TEXT f(SB),4,$0-0\n" + name + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + ins := file.Funcs[0].Instrs[1] + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + if known, _, err := ctx.lowerPackedWordMultiply(ins.Op, ins); !known || err == nil || out.String() != before { + t.Fatal("malformed form emitted IR", ins, err) + } + } + if vector { + for _, suffix := range []string{".BCST", ".SAE", ".RN_SAE", ".Z"} { + source := "TEXT f(SB),4,$0-0\n" + name + suffix + " (AX),X1,X2\nRET\n" + if suffix == ".Z" { + x86WordMultiplyReservedZeroMaskGoObservation(t, source, name) + } else { + goAssemblerOracleOutput(t, "amd64", source, "", false) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if _, _, err := x86PackedWordMultiplyProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], "amd64"); err == nil { + t.Fatal("opcode attributes leaked from shared ytab", source) + } + } + } + } +} + +func TestX86PackedWordMultiplyThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + t.Run(triple, func(t *testing.T) { + var source, declarations strings.Builder + declarations.WriteString("package oracle\n") + for i, row := range x86WordMultiplyForms(arch) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$64-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedWordMultiplyProperties(ins.Op, ins, arch) + if errors.Is(err, ErrProbeNeedsContext) || form.mmx { + continue + } + if err != nil { + t.Fatal(err) + } + if form.legacy && form.memory >= 0 && (form.source.Kind == OpFP || + form.source.Kind == OpMem && form.source.Mem.Base == SP) { + continue + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$64-64\n%s\nRET\n", i, row) + fmt.Fprintf(&declarations, "func F%d(value [64]byte)\n", i) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewPackageAPI(t, source.String(), pkg, arch, triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original word-multiply formats invented register or native inputs", api, err) + } + } + // Keep every old format/function in its original Go object and in + // emitter-only objects. Initialized public positives live in the + // complete-family value matrix, not these uninitialized probes. + ir := x86WordMultiplyFormatObjectUnit(t, file, sigs, arch, triple, 64) + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, text, level) + } + } + }) + } +} + +func TestX86PackedWordMultiplyMode32PhysicalAndSourceBoundaries(t *testing.T) { + for _, name := range x86WordMultiplyNames[4:] { + for _, bank := range []string{"X", "Y"} { + for _, index := range []int{8, 15, 16, 31} { + for operand := 0; operand < 3; operand++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[operand] = fmt.Sprintf("%s%d", bank, index) + source := "TEXT f(SB),4,$0-0\n" + name + " " + strings.Join(args, ",") + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + if _, _, err := x86PackedWordMultiplyProperties(ins.Op, ins, "386"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("named high bank lost physical Context", ins, err) + } + } + } + } + for _, source := range []string{ + name + " Z8,Z1,Z2", name + " Z0,Z8,Z2", name + " Z0,Z1,Z8", + name + " X0,X1,K1,X2", name + ".Z Z0,Z1,K1,Z2", + } { + assembly := "TEXT f(SB),4,$0-0\n" + source + "\nRET\n" + goAssemblerOracleOutput(t, "386", assembly, "", false) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + continue + } + ins := file.Funcs[0].Instrs[1] + if _, _, err := x86PackedWordMultiplyProperties(ins.Op, ins, "386"); err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go rejected a source format, not merely an unknown physical identity", source, err) + } + } + } + for _, row := range x86PackedWordMultiplyGoRows { + for _, width := range []int{0, 1, 2} { + for _, zero := range []bool{false, true} { + code := encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, width, 1, false, zero, 2, 1, 0) + ins, n, known, err := decodedX86PackedMultiplyInstruction(code, 32) + if !known || err != nil || n != len(code) || !ins.x86Encoded { + t.Fatal("actual low physical mode32 EVEX decode lost its origin", ins, err) + } + if _, _, err := x86PackedWordMultiplyProperties(ins.Op, ins, "386"); err != nil { + t.Fatal("physical grammar borrowed textual operand limit", err) + } + textual := ins + textual.x86Encoded = false + textual.Raw = fmt.Sprintf("BYTE %x genuine-looking text is not a decoder", code) + if _, _, err := x86PackedWordMultiplyProperties(textual.Op, textual, "386"); err == nil { + t.Fatal("raw text manufactured a decoder identity") + } + encoded, err := json.Marshal(ins) + if err != nil { + t.Fatal(err) + } + var restored Instr + if err := json.Unmarshal(encoded, &restored); err != nil { + t.Fatal(err) + } + if restored.x86Encoded { + t.Fatal("public JSON manufactured a decoder marker") + } + if _, _, err := x86PackedWordMultiplyProperties(restored.Op, restored, "386"); err == nil { + t.Fatal("JSON-restored source bypassed Go's named limit") + } + } + } + } +} + +func TestX86PackedWordMultiplyLegacyResultBytesAndUnknownInputs(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86WordMultiplyNames[:4] { + for _, first := range []string{"X0", "a+0(FP)", "value-16(SP)"} { + for _, complete := range []bool{true, false} { + move := "MOVOU" + if !complete { + move = "MOVSS" + } + body := move + " a+0(FP),X0\nMOVOU b+16(FP),X1\n" + move + " X0,value-16(SP)\n" + name + " " + first + ",X1\nMOVSD X1,ret0+32(FP)\nRET" + if !complete { + // Memory MOVSS zeros X[127:32]; its register form instead + // preserves the destination's genuinely undefined bytes. + body = strings.Replace(body, "MOVSS a+0(FP),X0", "MOVSS a+0(FP),X2\nMOVSS X2,X0", 1) + } + _, _, sig, source := x86ResultInitializationFixture(t, arch, "func Value(a,b [16]byte) uint64", body) + source = strings.Replace(source, ",$0-", ",$32-", 1) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + goAssemblerOracleOutput(t, arch, source, "", true) + err = CheckX86ABI0ResultInitialization(file.Funcs[0], sig, arch) + wantContext := !complete && first != "a+0(FP)" + if wantContext && !errors.Is(err, ErrProbeNeedsContext) || !wantContext && err != nil { + t.Fatal("legacy result byte proof invented unknown word input", name, first, complete, err) + } + } + } + } + } +} + +func TestX86PackedWordMultiplyVirtualLegacyMemoryRemainsContext(t *testing.T) { + for _, name := range x86WordMultiplyNames[:4] { + for _, address := range []string{"value+0(FP)", "(SP)", "(BP)", "(AX)(BP*2)"} { + for _, target := range x86RawReturnTargets { + source := "TEXT ·f(SB),4,$64-64\nPXOR X0,X0\n" + name + " " + address + ",X0\nRET\n" + declaration := "func f(value [64]byte)" + if address != "value+0(FP)" { + source = x86OrdinaryBPNativeSource(target.arch, "f", name+" "+address+",X0", address, 64) + declaration = "func f(p *byte)" + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+name+"/"+address+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + x86RequireOrdinaryBPNativeOutcome(t, target.arch, address, ir, err) + }) + } + } + } + } +} + +func TestX86PackedWordMultiplyMode32LowZActualGo(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, name := range x86WordMultiplyNames[4:] { + for _, first := range []string{"Z0", "(AX)"} { + row := name + " " + first + ", Z1, Z2" + t.Run(row, func(t *testing.T) { + source := "TEXT wordmul(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source, "func wordmul()", "386", "i386-unknown-linux-gnu", api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original low-Z format invented native or register inputs", api, err) + } + } + ir := x86WordMultiplyFormatObjectUnit(t, file, map[string]FuncSig{ + "wordmul": {Name: "wordmul", Ret: Void}, + }, "386", "i386-unknown-linux-gnu", 0) + x86ResultInitializationObject(t, llc, "i386-unknown-linux-gnu", ir, "word-multiply-low-z") + + initialized := "TEXT wordmul(SB),4,$0-4\nMOVL p+0(FP),AX\nVPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "386", initialized, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, initialized, "func wordmul(p *byte)", "386", "i386-unknown-linux-gnu", api) + if err != nil { + t.Fatal("initialized low-bank Z source rejected", api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, "i386-unknown-linux-gnu", ir, level) + } + } + }) + } + } +} + +func TestX86PackedWordMultiplyLegacyByteEffectActualGo(t *testing.T) { + for _, name := range x86WordMultiplyNames[:4] { + for _, arch := range []string{"386", "amd64"} { + row := name + " X0, X1" + t.Run(arch+"/"+name, func(t *testing.T) { + source := "TEXT wordmul(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 2 { + t.Fatalf("accepted legacy format lacks complete typed byte effect: known=%t err=%v effect=%+v", known, err, effect) + } + if !effect.operands[0].read || effect.operands[0].write || !effect.operands[1].read || + !effect.operands[1].write || effect.operands[0].width != 16 || effect.operands[1].width != 16 || + effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatal(fmt.Sprintf("incorrect read/modify/write/flags effect: %+v", effect)) + } + }) + } + } +} diff --git a/amd64_packed_word_multiply_native_linux_amd64_test.go b/amd64_packed_word_multiply_native_linux_amd64_test.go new file mode 100644 index 000000000..39ff70e3a --- /dev/null +++ b/amd64_packed_word_multiply_native_linux_amd64_test.go @@ -0,0 +1,80 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// This is a required Linux/amd64 runtime axis selected by the cross-runtime +// TestCrossLinuxRuntimeMatrix prefix. Missing AVX/AVX2 is a failure, not a skip. +// Darwin/Rosetta object/software results cannot satisfy this original-Go axis. +func TestCrossLinuxRuntimeMatrixX86PackedWordMultiplyOriginalVEX(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + execution.requireAVX(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\nfunc ReadCSR() uint32\nfunc SetCSR(value uint32)\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\nTEXT ·SetCSR(SB),4,$0-4\nLDMXCSR value+0(FP)\nRET\n") + tests.WriteString(x86WordMultiplyOriginalGoTestHeader) + count := 0 + for _, name := range x86WordMultiplyNames[4:] { + for _, width := range []int{16, 32} { + bank := map[int]string{16: "X", 32: "Y"}[width] + for _, memory := range []bool{false, true} { + for destination := 0; destination < 3; destination++ { + fn := fmt.Sprintf("V%d", count) + count++ + source := bank + "0" + if memory { + source = "(DI)" + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed uint64,profile uint32)\n", fn) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$0-56 +MOVQ out+0(FP),DX +MOVQ a+8(FP),DI +MOVQ b+16(FP),SI +MOVQ flags+24(FP),CX +MOVQ csr+32(FP),BX +VMOVDQU (DI),Y0 +VMOVDQU (SI),Y1 +VMOVDQU (SI),Y2 +LDMXCSR profile+48(FP) +PUSHQ seed+40(FP) +POPFQ +%s %s,%s1,%s%d +VMOVDQU Y%d,(DX) +PUSHFQ +POPQ AX +MOVQ AX,(CX) +STMXCSR (BX) +VZEROUPPER +RET +`, fn, name, source, bank, bank, destination, destination) + fmt.Fprintf(&tests, `{ + for i:=range out {out[i]=0xc3} + flags:=struct{before,value,after uint64}{0x1122334455667788,0,0x8877665544332211} + csr:=struct{before,value,after uint32}{0x11223344,0,0x44332211} + %s(op,ap,bp,&flags.value,&csr.value,seed,profile) + if flags.value&0x8d5!=seed||flags.before!=0x1122334455667788||flags.after!=0x8877665544332211||csr.value!=profile||csr.before!=0x11223344||csr.after!=0x44332211||ReadCSR()!=profile||a!=originalA||b!=originalB {t.Fatalf("%s state/canary sample%%d",sample)} + for i,value:=range out { + want:=byte(0xc3) + if i>=offset&&i>uint((index&1)*8)) + } + } + if value!=want {t.Fatalf("%s full-byte/upper-zero/canary sample%%d byte%%d got%%x want%%x",sample,i,value,want)} + } +} +`, fn, fn, width, x86WordMultiplyReferenceMode(name), fn) + } + } + } + } + tests.WriteString("}}\n}\n") + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), tests.String()) + t.Logf("%d actual original VEX X/Y kernels ×257 samples ×2 six-flag seeds; register/memory and all destination aliases, full Y upper-zero/MXCSR/input/canaries. No original EVEX execution claim.", count) +} diff --git a/amd64_packed_word_multiply_runtime_test.go b/amd64_packed_word_multiply_runtime_test.go new file mode 100644 index 000000000..02f7ca788 --- /dev/null +++ b/amd64_packed_word_multiply_runtime_test.go @@ -0,0 +1,451 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func x86WordMultiplyReservedZeroMaskGoObservation(t *testing.T, source, name string) { + t.Helper() + dir := t.TempDir() + asm, object := filepath.Join(dir, "reserved.s"), filepath.Join(dir, "reserved.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "oracle", "-o", object, asm) + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), err == nil, "amd64"); failure != nil { + t.Fatalf("actual zero-mask source observation: %v\n%s", failure, output) + } + if err != nil { + if !bytes.Contains(output, []byte("mask register must be specified for .Z instructions")) { + t.Fatalf("unknown original-Go rejection: %s", output) + } + return + } + // Go1.20 can emit the reserved EVEX.z=1/aaa=0 pattern. This is an + // accepted object observation, never authority to execute or translate it. + opcode := map[string]byte{"VPMULLW": 0xd5, "VPMULHW": 0xe5, "VPMULHUW": 0xe4, "VPMULHRSW": 0x0b}[name] + mapByte := byte(0xf1) + if name == "VPMULHRSW" { + mapByte = 0xf2 + } + code := []byte{0x62, mapByte, 0x75, 0x88, opcode, 0x10} + listing := make([]string, 0, len(code)+1) + for _, value := range append(append([]byte(nil), code...), 0xc3) { + listing = append(listing, fmt.Sprintf("%02x", value)) + } + if !bytes.Contains(output, []byte(strings.Join(listing, " "))) { + t.Fatalf("actual accepted reserved bytes changed: %s", output) + } + if _, _, recognized, err := decodedX86PackedMultiplyInstruction(code, 64); !recognized || err == nil { + t.Fatal("reserved physical bytes acquired valid semantics") + } + t.Log("actual Go accepts reserved zero-mask bytes; named/raw validation still rejects, and no #UD code is executed") +} + +// Independent integer arithmetic: neither reference reads a production mode, +// emitter, descriptor or LLVM intrinsic. Rounded signed high is deliberately +// wrapping, including (-32768)*(-32768) -> 0x8000, not saturation. +const x86WordMultiplyGoReference = `func product(a,b uint16,mode int) uint16 { + if mode==0 {return uint16(uint32(a)*uint32(b))} + if mode==2 {return uint16((uint32(a)*uint32(b))>>16)} + x,y:=int64(int16(a)),int64(int16(b)) + value,shift:=x*y,uint(16) + if mode==3 {value+=16384;shift=15} + return uint16(value>>shift) +} +` + +const x86WordMultiplyOriginalGoTestHeader = `package oracle +import ("testing";"unsafe";"runtime") +` + x86WordMultiplyGoReference + ` +func aligned(value *[160]byte) (*[64]byte,int) { + offset:=16+int((-uintptr(unsafe.Pointer(&value[16])))&15) + return (*[64]byte)(unsafe.Pointer(&value[offset])),offset +} +func TestOriginal(t *testing.T) { + runtime.LockOSThread();defer runtime.UnlockOSThread() + edges:=[...]uint16{0,1,2,0x3fff,0x4000,0x4001,0x7fff,0x8000,0x8001,0xbfff,0xc000,0xfffe,0xffff} + saved:=ReadCSR();defer SetCSR(saved) + for sample:=0;sample<257;sample++ {for _,seed:=range []uint64{0,0x8d5} { + var a,b,out [160]byte + ap,_:=aligned(&a);bp,_:=aligned(&b);op,offset:=aligned(&out) + for i:=range a {a[i]=0xa7;b[i]=0xb9} + for i:=0;i<64;i+=2 { + x,y:=edges[(sample+i/2)%13],edges[(sample/13+i/2*3)%13] + if sample>169 {x=uint16(sample*71+i*7919);y=uint16(sample*13+i*37)} + ap[i],ap[i+1]=byte(x),byte(x>>8);bp[i],bp[i+1]=byte(y),byte(y>>8) + } + originalA,originalB:=a,b + profile:=uint32(0x1f80|((sample%4)<<13)|(sample%64)) + SetCSR(profile) +` + +const x86WordMultiplyCReference = `#include +#include +#include +#include +static uint16_t word(const uint8_t *p) {return (uint16_t)p[0]|(uint16_t)p[1]<<8;} +static uint16_t product(uint16_t a,uint16_t b,int mode) { + if(mode==0) return (uint16_t)((uint32_t)a*(uint32_t)b); + if(mode==2) return (uint16_t)(((uint32_t)a*(uint32_t)b)>>16); + int64_t x=a<32768?a:(int64_t)a-65536,y=b<32768?b:(int64_t)b-65536; + int64_t value=x*y,divisor=65536; + if(mode==3) {value+=16384;divisor=32768;} + // Mathematical floor division, independent of C's signed-shift policy. + return (uint16_t)(value<0?-((-value+divisor-1)/divisor):value/divisor); +} +static const uint16_t edge[]={0,1,2,0x3fff,0x4000,0x4001,0x7fff,0x8000,0x8001,0xbfff,0xc000,0xfffe,0xffff}; +static const uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaa55555555),UINT64_MAX}; +struct Aligned {_Alignas(64) uint8_t before[64],value[64],after[64];}; +typedef void (*Program)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t); +static int check(Program program,const char *name,int width,int legacy,int alias,int mode,int masking) { + for(int sample=0;sample<257;sample++) for(int status=0;status<2;status++) for(int m=0;m<4;m++) { + struct Aligned a,b,out; + memset(&a,0xa7,sizeof(a));memset(&b,0xb9,sizeof(b));memset(&out,0xc3,sizeof(out)); + for(int i=0;i<64;i+=2) { + uint16_t x=edge[(sample+i/2)%13],y=edge[(sample/13+i/2*3)%13]; + if(sample>169) {x=(uint16_t)(sample*71+i*7919);y=(uint16_t)(sample*13+i*37);} + a.value[i]=(uint8_t)x;a.value[i+1]=(uint8_t)(x>>8); + b.value[i]=(uint8_t)y;b.value[i+1]=(uint8_t)(y>>8); + } + struct Aligned originalA=a,originalB=b; + struct {uint64_t before,value,after;} flags={UINT64_C(0x1122334455667788),0,UINT64_C(0x8877665544332211)}; + struct {uint32_t before,value,after;} csr={0x11223344,0,0x44332211}; + uint32_t saved=_mm_getcsr(),profile=(uint32_t)(0x1f80|((sample%4)<<13)|(sample%64)); + _mm_setcsr(profile); + uint64_t seed=status?UINT64_C(0x8d5):0; + program(out.value,a.value,b.value,&flags.value,&csr.value,seed,masks[m]); + _mm_setcsr(saved); + if((flags.value&UINT64_C(0x8d5))!=seed||flags.before!=UINT64_C(0x1122334455667788)||flags.after!=UINT64_C(0x8877665544332211)|| + csr.value!=profile||csr.before!=0x11223344||csr.after!=0x44332211||memcmp(&a,&originalA,sizeof(a))||memcmp(&b,&originalB,sizeof(b))) { + fprintf(stderr,"%s state sample%d flags%llx CSR%x/%x\n",name,sample,(unsigned long long)flags.value,csr.value,profile);return 1; + } + for(int i=0;i<64;i++) { + uint16_t value=product(word((alias?b.value:a.value)+(i&~1)),word(b.value+(i&~1)),mode); + int active=!masking||((masks[m]>>(i/2))&1); + uint8_t want=i>((i&1)*8)):(masking==2?0:b.value[i])):(legacy?b.value[i]:0); + if(out.value[i]!=want||out.before[i]!=0xc3||out.after[i]!=0xc3) { + fprintf(stderr,"%s sample%d mask%llx byte%d got%x want%x\n",name,sample,(unsigned long long)masks[m],i,out.value[i],want);return 2; + } + } + } + return 0; +} +` + +func x86WordMultiplyReferenceMode(name string) int { + switch strings.TrimPrefix(name, "V") { + case "PMULLW": + return 0 + case "PMULHW": + return 1 + case "PMULHUW": + return 2 + case "PMULHRSW": + return 3 + default: + panic("unknown independent word multiply reference name") + } +} + +func x86WordMultiplySignature(t *testing.T, pkg GoPackage, name string) FuncSig { + t.Helper() + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + return sig +} + +func TestCrossLinuxRuntimeMatrixX86PackedWordMultiplyOriginalGo(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\nfunc ReadCSR() uint32\nfunc SetCSR(value uint32)\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\nTEXT ·SetCSR(SB),4,$0-4\nLDMXCSR value+0(FP)\nRET\n") + tests.WriteString(x86WordMultiplyOriginalGoTestHeader) + count := 0 + for _, name := range x86WordMultiplyNames[:4] { + for _, bank := range []string{"X", "M"} { + if bank == "M" && name == "PMULHRSW" { + continue + } + kinds := []string{"Register", "Alias", "Memory"} + if bank == "M" { + kinds = append(kinds, "AXBank", "R14Bank") + } + for _, kind := range kinds { + fn := fmt.Sprintf("F%d", count) + count++ + move, width, source := "MOVOU", 16, bank+"0" + if bank == "M" { + move, width = "MOVQ", 8 + } + alias := 0 + if kind == "Alias" { + source, alias = bank+"1", 1 + } + if kind == "Memory" { + source = "(DI)" + } + if kind == "AXBank" { + source = "AX" + } + if kind == "R14Bank" { + source = "R14" + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed uint64,profile uint32)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-56\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\n%s (DI),%s0\n%s (SI),%s1\n", fn, move, bank, move, bank) + if kind == "R14Bank" { + assembly.WriteString("MOVQ (DI),M6\n") + } + fmt.Fprintf(&assembly, "LDMXCSR profile+48(FP)\nPUSHQ seed+40(FP)\nPOPFQ\n%s %s,%s1\n%s %s1,(DX)\n", name, source, bank, move, bank) + if bank == "M" { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("PUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n") + fmt.Fprintf(&tests, `{ + for i:=range out {out[i]=0xc3} + flags:=struct{before,value,after uint64}{0x1122334455667788,0,0x8877665544332211} + csr:=struct{before,value,after uint32}{0x11223344,0,0x44332211} + %s(op,ap,bp,&flags.value,&csr.value,seed,profile) + if flags.value&0x8d5!=seed||flags.before!=0x1122334455667788||flags.after!=0x8877665544332211||csr.value!=profile||csr.before!=0x11223344||csr.after!=0x44332211||ReadCSR()!=profile||a!=originalA||b!=originalB {t.Fatalf("%s flags/CSR/input/canary sample%%d flags%%x csr%%x/%%x",sample,flags.value,csr.value,profile)} + for i,value:=range out { + want:=byte(0xc3) + if i>=offset&&i>uint((index&1)*8)) + } + if value!=want {t.Fatalf("%s full-byte/canary sample%%d byte%%d got%%x want%%x",sample,i,value,want)} + } +} +`, fn, fn, width, alias, x86WordMultiplyReferenceMode(name), fn) + } + } + } + tests.WriteString("}}\n}\n") + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), tests.String()) + t.Logf("%d actual original SSE/MMX kernels ×257 initialized samples ×2 all-six-flag seeds, full MXCSR, bytes/input/canaries; MMX GP spellings observe the physical bank only, not a new LLVM permission. Rosetta is not native AMD64 evidence.", count) +} + +func x86WordMultiplyRunOriginalGo(t *testing.T, execution x86FloatCompareExecution, declarations, assembly, tests string) { + t.Helper() + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + dir := t.TempDir() + for name, contents := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations, "value_amd64.s": assembly, "value_test.go": tests} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + binary := filepath.Join(dir, "original.test") + command := exec.Command("go", "test", "-p=2", "-vet=off", "-c", "-o", binary, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + x86FloatCompareCaptureCommand(t, command) + output := execution.run(t, binary, "-test.run=^TestOriginal$", "-test.count=1", "-test.v") + if !strings.Contains(string(output), "--- PASS: TestOriginal") || !strings.Contains(string(output), "PASS\n") { + t.Fatal("original Go had no successful terminal", string(output)) + } +} + +func TestCrossLinuxRuntimeMatrixX86PackedWordMultiplySoftwareAllWidths(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, reference, checks strings.Builder + declarations.WriteString("package oracle\n") + reference.WriteString(x86WordMultiplyCReference) + count := 0 + for _, name := range x86WordMultiplyNames { + widths := []int{16} + if strings.HasPrefix(name, "V") { + widths = append(widths, 32, 64) + } + for _, width := range widths { + for _, memory := range []bool{false, true} { + for _, alias := range []bool{false, true} { + for _, masking := range []int{0, 1, 2} { + if !strings.HasPrefix(name, "V") && masking != 0 { + continue + } + fn := fmt.Sprintf("W%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + source := bank + "0" + if alias { + source = bank + "1" + } + if memory { + source = "(DI)" + } + inputAlias := alias && !memory + op, args := name, source+","+bank+"1" + if strings.HasPrefix(name, "V") { + if masking != 0 { + args += ",K1" + } + if masking == 2 { + op += ".Z" + } + args += "," + bank + "1" + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed,mask uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-56\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\nVMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\n", fn) + if masking != 0 { + assembly.WriteString("MOVQ mask+48(FP),AX\nKMOVQ AX,K1\n") + } + assembly.WriteString("PUSHQ seed+40(FP)\nPOPFQ\n") + fmt.Fprintf(&assembly, "%s %s\nVMOVUPS Z1,(DX)\nPUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n", op, args) + fmt.Fprintf(&reference, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t);\n", fn) + legacy, a := 0, 0 + if !strings.HasPrefix(name, "V") { + legacy = 1 + } + if inputAlias { + a = 1 + } + fmt.Fprintf(&checks, "if(check(%s,%q,%d,%d,%d,%d,%d)) return 1;\n", fn, name+" "+args, width, legacy, a, x86WordMultiplyReferenceMode(name), masking) + } + } + } + } + } + reference.WriteString("int main(void) {\n" + checks.String() + "return 0;}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sigs[name] = x86WordMultiplySignature(t, pkg, name) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs, ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }}) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "word-multiply-all-widths", ir, reference.String(), level) + }) + } + t.Logf("%d initialized LLVM software kernels ×257 samples ×2 seeds ×4 masks; full64-byte aliases, six flags, MXCSR/input/canaries. Original-Go accepted source/object is separate; this does NOT execute original AVX/AVX512 instructions or prove native EVEX capability.", count) +} + +func TestCrossLinuxRuntimeMatrixX86PackedWordMultiplySoftwareMaskedMemory(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, name := range x86WordMultiplyNames[4:] { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, zero := range []bool{false, true} { + fn := fmt.Sprintf("M%d", count) + count++ + op, zeroInt := name, 0 + if zero { + op, zeroInt = op+".Z", 1 + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed,mask uint64)\n", fn) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$0-56 +MOVQ out+0(FP),DX +MOVQ a+8(FP),DI +MOVQ b+16(FP),SI +MOVQ flags+24(FP),CX +MOVQ csr+32(FP),BX +MOVQ mask+48(FP),AX +KMOVQ AX,K1 +VMOVUPS (SI),Z1 +PUSHQ seed+40(FP) +POPFQ +%s (DI),%s1,K1,%s1 +VMOVUPS Z1,(DX) +PUSHFQ +POPQ AX +MOVQ AX,(CX) +STMXCSR (BX) +RET +`, fn, op, bank, bank) + fmt.Fprintf(&checks, "if(checkMasked(%s,%q,%d,%d,%d,p)) return 1;\n", fn, op, width, zeroInt, x86WordMultiplyReferenceMode(name)) + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + var external strings.Builder + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sigs[name] = x86WordMultiplySignature(t, pkg, name) + fmt.Fprintf(&external, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t);\n", name) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }}) + if err != nil { + t.Fatal(err) + } + const maskedReference = ` +static int checkMasked(Program program,const char *name,int width,int zero,int mode,const uint8_t *pageTail) { + const uint64_t mask[]={0,1,UINT64_C(1)<<32}; + for(int state=0;state<2;state++) for(int m=0;m<3;m++) { + struct Aligned b,out; + memset(&b,0xb9,sizeof(b));memset(&out,0xc3,sizeof(out)); + struct Aligned before=b; + struct {uint64_t before,value,after;} flags={0x11223344,0,0x44332211}; + struct {uint32_t before,value,after;} csr={0x55667788,0,0x88776655}; + uint64_t seed=state?UINT64_C(0x8d5):0; + uint32_t saved=_mm_getcsr(),profile=0x1fbf; + _mm_setcsr(profile); + program(out.value,m==1?pageTail:NULL,b.value,&flags.value,&csr.value,seed,mask[m]); + _mm_setcsr(saved); + if((flags.value&UINT64_C(0x8d5))!=seed||flags.before!=0x11223344||flags.after!=0x44332211|| + csr.value!=profile||csr.before!=0x55667788||csr.after!=0x88776655||memcmp(&b,&before,sizeof(b))) return 2; + for(int i=0;i<64;i++) { + uint8_t want=0; + if(i>(i*8)); + else if(!zero) want=b.value[i]; + } + if(out.value[i]!=want||out.before[i]!=0xc3||out.after[i]!=0xc3) { + fprintf(stderr,"%s masked read byte%d mask%llx got%x want%x\n",name,i,(unsigned long long)mask[m],out.value[i],want);return 3; + } + } + } + return 0; +} +` + main := x86WordMultiplyCReference + external.String() + x86TestGuardPagesC + maskedReference + ` +int main(void) { + if (setup_guard() || page < 64) return 5; + uint8_t *p = guard + page - 2; + p[0] = 0; + p[1] = 0x80; +` + checks.String() + ` + if (free_guard()) return 6; + return 0; +} +` + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "word-multiply-masked-memory", ir, main, level) + }) + } + t.Logf("%d software EVEX kernels: inactive NULL, guard-page first word, irrelevant high mask bits, two flag seeds/full bytes/MXCSR/canaries. These semantics do not grant source/caller memory permission or native EVEX execution.", count) +} diff --git a/amd64_packed_zero_extend_contract_test.go b/amd64_packed_zero_extend_contract_test.go new file mode 100644 index 000000000..40b06b305 --- /dev/null +++ b/amd64_packed_zero_extend_contract_test.go @@ -0,0 +1,244 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86PackedZeroExtendDispatchIdentityBeforeEmission(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + opcode := prefix + "MOVZX" + ratio.suffix + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+opcode+" X0,X1\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, identity := range []Op{"", Op(opcode), Op(strings.ToLower(opcode)), "PMOVSXBW", "VPMOVZXBW.Z"} { + ins := file.Funcs[0].Instrs[1] + ins.Op = identity + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerPackedExtendMove(Op(opcode), ins) + conflict := identity != "" && !strings.EqualFold(string(identity), opcode) + if !known || term || (err != nil) != conflict || conflict && output.String() != before { + t.Fatalf("dispatch %s/%s: known=%t term=%t err=%v", opcode, identity, known, term, err) + } + } + } + } +} + +func TestX86PackedZeroExtendFPBytesBindActualGoDeclaration(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, bank := range []string{"X", "Y", "Z"} { + if prefix == "P" && bank != "X" { + continue + } + width := map[string]int{"X": 16, "Y": 32, "Z": 64}[bank] * ratio.input / ratio.output + for _, layout := range []string{"complete", "last byte absent", "alignment hole"} { + var parameters []string + for i := 0; i < width; i++ { + if layout == "last byte absent" && i == width-1 { + continue + } + if layout == "alignment hole" && i >= 1 { + parameters = append(parameters, fmt.Sprintf("a%d uint64", i)) + } else { + parameters = append(parameters, fmt.Sprintf("a%d byte", i)) + } + } + fset := token.NewFileSet() + declaration, err := parser.ParseFile(fset, "decl.go", "package bound\nfunc f("+strings.Join(parameters, ",")+")\n", 0) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check("bound", fset, []*ast.File{declaration}, nil) + if err != nil { + t.Fatal(err) + } + row := prefix + "MOVZX" + ratio.suffix + " a0+0(FP)," + bank + "1" + params := typed.Scope().Lookup("f").Type().(*types.Signature).Params() + var fields []*types.Var + for i := 0; i < params.Len(); i++ { + fields = append(fields, params.At(i)) + } + sizes := types.SizesFor("gc", "amd64") + offsets := sizes.Offsetsof(fields) + end := offsets[len(offsets)-1] + sizes.Sizeof(fields[len(fields)-1].Type()) + allocation := (end + 7) &^ 7 // Actual ABI0 word-aligned scalar allocation. + assembly := fmt.Sprintf("TEXT f(SB),4,$0-%d\n%s\nRET\n", allocation, row) + if layout == "complete" { + requireX86GoAssemblerResult(t, "amd64", assembly, true) + } + ctx := llvm.NewContext() + bound, err := translateGoModuleInContext(ctx, GoPackage{Path: "bound", Types: typed, Syntax: []*ast.File{declaration}}, []byte(assembly), GoModuleOptions{GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu"}) + if bound != nil { + bound.Module.Dispose() + } + ctx.Dispose() + if (err != nil) != (layout != "complete") { + t.Fatalf("%s/%s ignored actual Go field layout: %v", row, layout, err) + } + } + } + } + } +} + +func TestX86PackedZeroExtendEveryDeclaredFPByteBeforeEmission(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, row := range x86ZeroExtendAcceptedForms(prefix+"MOVZX"+ratio.suffix, "amd64") { + if !strings.Contains(row, "8(AX)") { + continue + } + row = strings.Replace(row, "8(AX)", "a+1(FP)", 1) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedZeroExtendProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for missing := -1; missing < form.memoryBytes; missing++ { + sig := FuncSig{Name: "f", Ret: Void} + for i := 0; i < form.memoryBytes; i++ { + if i != missing { + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(i + 1), Type: I8, Index: len(sig.Args), Field: -1}) + sig.Args = append(sig.Args, I8) + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + known, term, err := ctx.lowerPackedExtendMove(ins.Op, ins) + if !known || term || (err != nil) != (missing >= 0) || missing >= 0 && output.String() != before { + t.Fatalf("%s missing byte %d emitted/accepted undeclared FP bytes: %v", row, missing, err) + } + } + } + } + } +} + +func TestX86GoCallerPackedZeroExtendMemoryCertificatesBindNativeSource(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, original := range x86ZeroExtendAcceptedForms(prefix+"MOVZX"+ratio.suffix, "amd64") { + if !strings.Contains(original, "8(AX)") { + continue + } + row := strings.NewReplacer("8(AX)", "8(DI)", "X15", "X2", "X31", "X2", "Y31", "Y2", "Z31", "Z2").Replace(original) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVUPS (SI),Z2\nKMOVQ AX,K1\nKMOVQ AX,K7\n"+row+"\nVMOVSS X2,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, recognized, err := x86PackedZeroExtendProperties(ins, "amd64") + if !recognized || err != nil || form.memory < 0 { + continue + } + found = true + index := form.memory + access, exists := ins.x86CallerMemory[index] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(ins, index) { + t.Fatalf("%s lacks its exact shrinking native read certificate", row) + } + for _, change := range []string{"opcode", "width", "address", "index", "destination", "direction", "mask", "address-bits"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VPMOVSXBD" + case "width": + access.width++ + mutated.x86CallerMemory[index] = access + case "address": + mutated.Args[index].Mem.Off++ + case "index": + mutated.Args[index].Mem.Index, mutated.Args[index].Mem.Scale = BX, 4 + case "destination": + mutated.Args[form.destination].Reg = "X3" + case "direction": + access.write = true + mutated.x86CallerMemory[index] = access + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K2" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K2"}) + } + case "address-bits": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s mutation retained %s source certificate", change, row) + } + } + } + if !found { + t.Fatal("no widening source instruction consumed its caller certificate") + } + } + } + } +} + +func TestX86PackedZeroExtendInvalidMemoryAndMaskGrammar(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []string{ + "VPMOVZXBD (AX)(SP*1),X1", "VPMOVZXBD (AX)(CX*3),X1", + "VPMOVZXBD $glob(SB),X1", "VPMOVZXBD X0,K0,X1", + "VPMOVZXBD.SAE X0,X1", "VPMOVZXBD.RN_SAE X0,X1", + "VPMOVZXBD. X0,X1", "PMOVZXBD X0,K1,X1", + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + continue + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := output.String() + ins := file.Funcs[0].Instrs[1] + known, term, err := ctx.lowerPackedExtendMove(ins.Op, ins) + if !known || term || err == nil || output.String() != before { + t.Fatalf("%s/%s emitted invalid grammar: %v", arch, row, err) + } + } + } + // A full signed displacement must not silently wrap in native amd64. + for _, off := range []int64{1 << 31, -1<<31 - 1, 1 << 32} { + ins := Instr{Op: "VPMOVZXBD", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX, Off: off}}, {Kind: OpReg, Reg: "X1"}}} + if _, recognized, err := x86PackedZeroExtendProperties(ins, "amd64"); !recognized || err == nil { + t.Fatalf("out-of-range native displacement %d accepted", off) + } + } + for _, scale := range []int64{1, 2, 4, 8} { + row := fmt.Sprintf("VPMOVZXBD 3(AX)(CX*%d),X1", scale) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, _, err := x86PackedZeroExtendProperties(file.Funcs[0].Instrs[1], "amd64"); err != nil { + t.Fatal(row, err) + } + } +} diff --git a/amd64_packed_zero_extend_forms.go b/amd64_packed_zero_extend_forms.go new file mode 100644 index 000000000..51e22cd15 --- /dev/null +++ b/amd64_packed_zero_extend_forms.go @@ -0,0 +1,163 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedExtendSpec struct { + inputBits int + outputBits int + vector bool + opcode byte +} + +type x86PackedZeroExtendSpec = x86PackedExtendSpec + +// Legacy yxm_q4 and the two AVX tables have different physical source widths: +// ratio two uses _yvcvtdq2pd (Y/m256 to Z); the other ratios _yvbroadcastss. +// None permits broadcast, SAE or rounding. EVEX masking is per widened lane. +var x86PackedZeroExtendSpecs = map[Op]x86PackedZeroExtendSpec{ + "PMOVZXBW": {8, 16, false, 0x30}, + "PMOVZXBD": {8, 32, false, 0x31}, + "PMOVZXBQ": {8, 64, false, 0x32}, + "PMOVZXWD": {16, 32, false, 0x33}, + "PMOVZXWQ": {16, 64, false, 0x34}, + "PMOVZXDQ": {32, 64, false, 0x35}, + "VPMOVZXBW": {8, 16, true, 0x30}, + "VPMOVZXBD": {8, 32, true, 0x31}, + "VPMOVZXBQ": {8, 64, true, 0x32}, + "VPMOVZXWD": {16, 32, true, 0x33}, + "VPMOVZXWQ": {16, 64, true, 0x34}, + "VPMOVZXDQ": {32, 64, true, 0x35}, +} + +type x86PackedExtendForm struct { + spec x86PackedExtendSpec + byteWidth int + sourceRegisterBytes int + memoryBytes int + destination int + memory int + mask int + zeroing bool + signed bool +} + +type x86PackedZeroExtendForm = x86PackedExtendForm + +// This specifies encoding and byte effects, never caller bounds or permission. +func x86PackedZeroExtendProperties(ins Instr, goarch string) (x86PackedZeroExtendForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + if _, known := x86PackedZeroExtendSpecs[Op(base)]; !known { + return x86PackedExtendForm{}, false, nil + } + return x86PackedExtendProperties(ins, goarch) +} + +// Sign and zero extension share operand shape, shrinking tuples and masked +// access rules, not membership, sign-byte dependencies or raw authority. +func x86PackedExtendProperties(ins Instr, goarch string) (x86PackedExtendForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86PackedZeroExtendSpecs[Op(base)] + signed := false + if !recognized { + spec, recognized = x86PackedSignExtendSpecs[Op(base)] + signed = true + } + if !recognized { + return x86PackedExtendForm{}, false, nil + } + form := x86PackedExtendForm{ + spec: spec, destination: len(ins.Args) - 1, memory: -1, mask: -1, + zeroing: suffix == "Z", sourceRegisterBytes: 16, signed: signed, + } + fail := func() (x86PackedExtendForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go packed widening grammar: %q", goarch, ins.Op, ins.Raw) + } + if goarch != "amd64" && goarch != "386" || len(ins.Args) != 2 && len(ins.Args) != 3 || + dotted && (!spec.vector || suffix != "Z") { + return fail() + } + if len(ins.Args) == 3 { + if !spec.vector || !amd64NonzeroKOperand(ins.Args[1]) { + return fail() + } + form.mask = 1 + } else if form.zeroing { + return fail() + } + destination := ins.Args[form.destination] + form.byteWidth = amd64VectorByteWidth(destination.Reg) + if !spec.vector { + index, valid := amd64ParseXReg(destination.Reg) + if destination.Kind != OpReg || !valid || index >= 16 || goarch == "386" && index >= 8 { + return fail() + } + } else if !amd64GoPackedVectorMoveRegister(destination, form.byteWidth, goarch) { + return fail() + } + form.memoryBytes = form.byteWidth * spec.inputBits / spec.outputBits + if form.memoryBytes > 16 { + form.sourceRegisterBytes = 32 + } + source := ins.Args[0] + if source.Kind == OpReg { + if !amd64EVEXVectorRegister(source, form.sourceRegisterBytes) { + return fail() + } + index, _ := amd64VectorRegisterIndex(source.Reg, form.sourceRegisterBytes) + if !spec.vector && (index >= 16 || goarch == "386" && index >= 8) { + return fail() + } + } else if isAMD64MemoryOperand(source) { + if source.Kind == OpSym { + symbol := strings.TrimSpace(source.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if source.Kind == OpMem { + displacement := source.Mem.Off + if displacement != int64(int32(displacement)) && + (goarch != "386" || displacement != int64(uint32(displacement))) || + !x86MemoryRegistersValidForArch(source.Mem, goarch) || source.Mem.Index != "" && + (source.Mem.Index == SP || source.Mem.Scale != 1 && source.Mem.Scale != 2 && source.Mem.Scale != 4 && source.Mem.Scale != 8) { + return fail() + } + } + form.memory = 0 + } else { + return fail() + } + return form, true, x86Mode32VectorRegisterContext(ins, goarch) +} + +func x86TypedPackedZeroExtendByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + if _, known, err := x86PackedZeroExtendProperties(ins, arch); !known || err != nil { + return x86TypedByteEffect{}, known, err + } + return x86TypedPackedExtendByteEffect(op, ins, arch) +} + +func x86TypedPackedExtendByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedExtendProperties(ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + if op != ins.Op { + return x86TypedByteEffect{}, true, fmt.Errorf("packed widening effect/source opcode mismatch") + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: int64(form.memoryBytes), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), + read: form.mask >= 0 && !form.zeroing, write: true}, + }} + if form.mask >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: form.mask, operand: ins.Args[form.mask], width: 8, read: true, + }) + } + return effect, true, nil +} diff --git a/amd64_packed_zero_extend_forms_test.go b/amd64_packed_zero_extend_forms_test.go new file mode 100644 index 000000000..ebcfca1a0 --- /dev/null +++ b/amd64_packed_zero_extend_forms_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "testing" +) + +// Independent byte ratios, not values taken from the production spec. +var x86ZeroExtendTestRatios = []struct { + suffix string + input int + output int +}{ + {"BW", 1, 2}, {"BD", 1, 4}, {"BQ", 1, 8}, + {"WD", 2, 4}, {"WQ", 2, 8}, {"DQ", 4, 8}, +} + +func x86ZeroExtendAcceptedForms(opcode, arch string) []string { + if !strings.HasPrefix(opcode, "V") { + last := 15 + if arch == "386" { + last = 7 + } + return []string{fmt.Sprintf("%s X0,X%d", opcode, last), fmt.Sprintf("%s 8(AX),X%d", opcode, last)} + } + var rows []string + for _, bank := range []string{"X", "Y", "Z"} { + source, last := "X", 31 + if bank == "Z" && (strings.HasSuffix(opcode, "BW") || strings.HasSuffix(opcode, "WD") || strings.HasSuffix(opcode, "DQ")) { + source = "Y" + } + if arch == "386" { + last = 7 + } + for _, input := range []string{source + "0", "8(AX)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s%d", opcode, input, bank, last)) + rows = append(rows, fmt.Sprintf("%s %s,K7,%s%d", opcode, input, bank, last), + fmt.Sprintf("%s.Z %s,K1,%s%d", opcode, input, bank, last)) + } + } + // These select the two VEX optab rows rather than their EVEX aliases. + rows = append(rows, opcode+" X0,X1", opcode+" X0,Y1") + return rows +} + +func TestX86PackedZeroExtendSpecsBindCompleteGoTables(t *testing.T) { + dir := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86") + legacy, err := os.ReadFile(filepath.Join(dir, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(dir, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + if len(x86PackedZeroExtendSpecs) != 12 { + t.Fatalf("unsigned family has %d names, want 12", len(x86PackedZeroExtendSpecs)) + } + // Bind every operand class row, independently of the lowerer's accepted + // form generator. Memory width attributes belong to each opcode below. + legacyTable := "var yxm_q4 = []ytab{\n\t{Zm_r, 1, argList{Yxm, Yxr}},\n}" + if !strings.Contains(string(legacy), legacyTable) { + t.Fatal("Go changed the complete legacy register/memory operand table") + } + for _, table := range []string{"_yvbroadcastss", "_yvcvtdq2pd"} { + want := "var " + table + " = []ytab{\n" + want += "\t{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr}},\n" + want += "\t{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yyr}},\n" + for _, destination := range []string{"YxrEvex", "YyrEvex", "Yzr"} { + source := "YxmEvex" + if table == "_yvcvtdq2pd" && destination == "Yzr" { + source = "YymEvex" + } + want += fmt.Sprintf("\t{zcase: Zevex_rm_v_r, zoffset: 0, args: argList{%s, %s}},\n", source, destination) + want += fmt.Sprintf("\t{zcase: Zevex_rm_k_r, zoffset: 3, args: argList{%s, Yknot0, %s}},\n", source, destination) + } + want += "}" + if !strings.Contains(string(avx), want) { + t.Fatalf("Go changed %s's complete VEX/EVEX operand table", table) + } + } + for index, ratio := range x86ZeroExtendTestRatios { + opcode := byte(0x30 + index) + for _, prefix := range []string{"P", "VP"} { + name := prefix + "MOVZX" + ratio.suffix + spec, present := x86PackedZeroExtendSpecs[Op(name)] + if !present || spec.inputBits != ratio.input*8 || spec.outputBits != ratio.output*8 || spec.vector != (prefix == "VP") || spec.opcode != opcode { + t.Fatalf("independent unsigned ratio/encoding mismatch: %s %+v", name, spec) + } + if prefix == "P" { + want := fmt.Sprintf("{A%s, yxm_q4, Pq4, opBytes{0x%02x}}", name, opcode) + if !strings.Contains(string(legacy), want) { + t.Fatalf("missing exact Go legacy row: %s", want) + } + continue + } + table := "_yvbroadcastss" + if ratio.output == 2*ratio.input { + table = "_yvcvtdq2pd" + } + marker := "{as: A" + name + ", ytab: " + table + ", prefix: Pavx, op: opBytes{" + start := strings.Index(string(avx), marker) + if start < 0 { + t.Fatalf("missing exact Go AVX family row: %s", marker) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + want := marker + for _, width := range []int{128, 256} { + want += fmt.Sprintf("\n\t\tavxEscape | vex%d | vex66 | vex0F38 | vexW0, 0x%02x,", width, opcode) + } + for _, width := range []int{128, 256, 512} { + want += fmt.Sprintf("\n\t\tavxEscape | evex%d | evex66 | evex0F38 | evexW0, evexN%d | evexZeroingEnabled, 0x%02x,", width, width/8*ratio.input/ratio.output, opcode) + } + if row != want { + t.Fatalf("changed complete Go prefix/opcode/width/exception attributes:\n%s\nwant:\n%s", row, want) + } + } + } + for _, source := range []string{string(legacy), string(avx)} { + for _, match := range regexp.MustCompile(`\{(?:as: )?A(V?PMOVZX[A-Z]+),`).FindAllStringSubmatch(source, -1) { + if _, exists := x86PackedZeroExtendSpecs[Op(match[1])]; !exists { + t.Fatalf("Go added an unregistered unsigned family name %s", match[1]) + } + } + } +} + +func TestX86PackedZeroExtendNamedMode32RetainsActualPhysicalContext(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, bank := range []string{"X", "Y"} { + for _, high := range []int{8, 16, 31} { + for _, args := range []string{fmt.Sprintf("X%d,%s0", high, bank), fmt.Sprintf("X0,%s%d", bank, high)} { + row := "VPMOVZX" + ratio.suffix + " " + args + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil { + t.Fatalf("actual mode32 Go source acceptance: %s: %v\n%s", row, err, listing) + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, target := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if _, err := Translate(file, Options{Goarch: "386", TargetTriple: target, Sigs: map[string]FuncSig{"f": {Ret: Void}}}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("named high register acquired physical mode32 binding: %s: %v", row, err) + } + } + } + } + } + } +} + +func TestX86PackedZeroExtendActualGoBytesBindRawWidths(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, row := range []string{"X0,X1", "X0,Y1", "8(AX),Y1", "X20,K3,X21", "X20,K3,Y21"} { + instruction := "VPMOVZX" + ratio.suffix + " " + row + listing, err := x86ExtensionGoListing(t, "amd64", instruction) + if err != nil { + t.Fatal(err, listing) + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 3 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + break + } + code = append(code, byte(value)) + } + break + } + decoded, length, recognized, err := decodedX86PackedExtendInstruction(code, 64) + if !recognized || err != nil || length <= 0 || decoded.Op != Op("VPMOVZX"+ratio.suffix) { + t.Fatalf("actual Go object did not decode as source family: %s/%x: %+v %v", instruction, code, decoded, err) + } + form, recognized, err := x86PackedZeroExtendProperties(decoded, "amd64") + if !recognized || err != nil || form.memoryBytes != form.byteWidth*ratio.input/ratio.output { + t.Fatalf("actual raw width lost its independent source ratio: %+v %v", form, err) + } + } + } +} diff --git a/amd64_packed_zero_extend_mode32_initialized_test.go b/amd64_packed_zero_extend_mode32_initialized_test.go new file mode 100644 index 000000000..d257fc97c --- /dev/null +++ b/amd64_packed_zero_extend_mode32_initialized_test.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// Retain the original six ratios, three banks, register/native source and +// merging/zeroing directory in the same order. These rows alone supply no +// numerical bytes, pointer, old destination or mask. +func x86ZeroExtendLowMaskedMode32Rows(t *testing.T) []string { + t.Helper() + var rows []string + for _, ratio := range x86ZeroExtendTestRatios { + for _, bank := range []string{"X", "Y", "Z"} { + source := "X0" + if bank == "Z" && ratio.output == 2*ratio.input { + source = "Y0" + } + for _, input := range []string{source, "(AX)"} { + for _, suffix := range []string{"", ".Z"} { + rows = append(rows, fmt.Sprintf("VPMOVZX%s%s %s,K7,%s7", ratio.suffix, suffix, input, bank)) + } + } + } + } + if len(rows) != 72 { + t.Fatal("complete original low-bank K7 format directory changed", len(rows)) + } + return rows +} + +func x86ZeroExtendLowMaskedInitialized(row, name string) (string, string) { + var source strings.Builder + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-12\nMOVL p+0(FP),AX\nVMOVUPS (AX),Z7\n", name) + if !strings.Contains(row, "(AX)") { + source.WriteString("VMOVUPS (AX),Y0\n") + } + source.WriteString("KMOVD mask+4(FP),K7\n" + row + "\nPEXTRD $0,X7,AX\nMOVL AX,ret+8(FP)\nRET\n") + return source.String(), fmt.Sprintf("func %s(p *[64]byte,mask uint32) uint32\n", name) +} + +func TestX86PackedZeroExtendLowMaskedMode32InitializedThreeAPIObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + rows := x86ZeroExtendLowMaskedMode32Rows(t) + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + for batch := 0; batch < 2; batch++ { + t.Run(fmt.Sprintf("%s/batch%d", triple, batch), func(t *testing.T) { + var source, declarations strings.Builder + for index, row := range rows[batch*36 : (batch+1)*36] { + program, declaration := x86ZeroExtendLowMaskedInitialized(row, fmt.Sprint("LowMasked", index)) + source.WriteString(program) + declarations.WriteString(declaration) + } + x86LEAActualTargetGoObject(t, "386", triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "386", triple, api) + if err != nil || ir == "" { + t.Fatal("actual initialized low-bank K7 program lost public translation", err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, text, level) + } + } + }) + } + }) + } + } +} + +func TestX86PackedZeroExtendLowMaskedMode32RetainsActualInputs(t *testing.T) { + for _, row := range x86ZeroExtendLowMaskedMode32Rows(t) { + t.Run(row, func(t *testing.T) { + source, declaration := x86ZeroExtendLowMaskedInitialized(row, "LowMasked") + mutations := map[string]string{ + "missing-address": "MOVL p+0(FP),AX\n", + "missing-mask": "KMOVD mask+4(FP),K7\n", + } + if !strings.Contains(row, "(AX)") { + mutations["missing-register-source"] = "VMOVUPS (AX),Y0\n" + } + if !strings.Contains(row, ".Z ") { + mutations["missing-merge-bytes"] = "VMOVUPS (AX),Z7\n" + } + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, source, declaration, "386", "i386-unknown-linux-gnu", api); err != nil || ir == "" { + t.Fatal("initialized control failed before input mutations", api, err) + } + for name, removed := range mutations { + changed := strings.Replace(source, removed, "", 1) + if changed == source { + t.Fatal("input mutation did not apply", name) + } + if ir, err := x86ScalarFPBitViewAPI(t, changed, declaration, "386", "i386-unknown-linux-gnu", api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("complete widening borrowed a missing input", api, name, err) + } + } + if strings.Contains(row, ".Z ") { + withoutOld := strings.Replace(source, "VMOVUPS (AX),Z7\n", "", 1) + if ir, err := x86ScalarFPBitViewAPI(t, withoutOld, declaration, "386", "i386-unknown-linux-gnu", api); err != nil || ir == "" { + t.Fatal("zeroing register widening incorrectly requires old merge bytes", api, err) + } + } + } + }) + } +} diff --git a/amd64_packed_zero_extend_native_runtime_unix_test.go b/amd64_packed_zero_extend_native_runtime_unix_test.go new file mode 100644 index 000000000..9bde099a7 --- /dev/null +++ b/amd64_packed_zero_extend_native_runtime_unix_test.go @@ -0,0 +1,108 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Legacy and VEX-X live/dead kernels use actual original legacy Go bytes as +// their reference. Masked X/Y/Z dead reads use an independent volatile byte +// reference: portable LLVM execution does not certify native AVX512 execution. +// The complete-byte runtime separately checks masks, aliases, FLAGS and MXCSR. +func TestCrossLinuxRuntimeMatrixX86PackedZeroExtendNativeResultsAndFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + const ratiosPerBatch = 3 + for batch := 0; batch < 2; batch++ { + t.Run(fmt.Sprint("batch", batch), func(t *testing.T) { + var source, declarations, prototypes, originals, rows strings.Builder + count := 0 + for _, ratio := range x86ZeroExtendTestRatios[batch*ratiosPerBatch : (batch+1)*ratiosPerBatch] { + add := func(instruction string, width, discarded int, mask int64, native bool) { + name := fmt.Sprintf("Widen%d", count) + original := fmt.Sprintf("Original%d", count) + count++ + body := "MOVQ p+0(FP),DX\n" + if strings.Contains(instruction, "K1") { + body += fmt.Sprintf("MOVQ $%d,AX\nKMOVQ AX,K1\n", mask) + } + body += instruction + "\nPEXTRQ $0,X1,AX\n" + if discarded != 0 { + body += "MOVQ $0x87654321fedcba98,AX\n" + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\n%sMOVQ AX,ret+8(FP)\nRET\n", name, body) + fmt.Fprintf(&declarations, "func %s(p *[32]byte) uint64\n", name) + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\n", name) + if native { + nativeBody := "PMOVZX" + ratio.suffix + " (DI),X0\nPEXTRQ $0,X0,AX\n" + if discarded != 0 { + nativeBody += "MOVQ $0x87654321fedcba98,AX\n" + } + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+nativeBody+"RET\n") + var bytes []string + for _, value := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\n", original) + } else { + fmt.Fprintf(&originals, "static uint64_t %s(const uint8_t *p) {\n", original) + fmt.Fprintf(&originals, " for (unsigned b = 0; b < %d; b++) {\n", width) + originals.WriteString(" volatile uint8_t value = ((volatile const uint8_t*)p)[b];\n (void)value;\n }\n") + originals.WriteString(" return UINT64_C(0x87654321fedcba98);\n}\n") + } + fmt.Fprintf(&rows, "{%s,%s,%d,%d,%d,%d},\n", name, original, width, discarded, ratio.input, ratio.output) + } + for _, prefix := range []string{"P", "VP"} { + for discarded := 0; discarded < 2; discarded++ { + add(prefix+"MOVZX"+ratio.suffix+" (DX),X1", 16*ratio.input/ratio.output, discarded, 0, true) + } + } + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, mask := range []int64{0, 1, -1} { + readWidth := width * ratio.input / ratio.output + if mask == 0 { + readWidth = 0 + } else if mask == 1 { + readWidth = ratio.input + } + add("VPMOVZX"+ratio.suffix+".Z (DX),K1,"+bank+"1", readWidth, 1, mask, false) + } + } + } + if count != 39 || count > 64 { + t.Fatal("incomplete or oversized widening read matrix", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf(x86NativeScalarReadRuntimeC, prototypes.String(), originals.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprint("O2=", optimized), func(t *testing.T) { + text, level := ir, "-O0" + if optimized { + text, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + output := execution.compileAndRunLLVM(t, "widen-native", text, driver, level) + if string(output) != "39 native scalar kernels: numerical 4992, faults 468, tripwires 2; failures 0\n" { + t.Fatal("widening numerical/dead/native/masked reads differ:", string(output)) + } + t.Log("Original legacy Go / portable masked-read reference:", strings.TrimSpace(string(output))) + }) + } + }) + } + }) + } +} diff --git a/amd64_packed_zero_extend_raw_axes_test.go b/amd64_packed_zero_extend_raw_axes_test.go new file mode 100644 index 000000000..fb5fe334c --- /dev/null +++ b/amd64_packed_zero_extend_raw_axes_test.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os/exec" + "strings" + "testing" +) + +func TestX86PackedExtendRawWBitMatchesIndependentLLVMDecoder(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM22 llvm-mc required") + } + for _, spec := range rawPackedExtendTestSpecs { + for _, evex := range []bool{false, true} { + for width := 0; width < 3; width++ { + if !evex && width == 2 { + continue + } + code := encodeX86VEXPackedExtend(spec.opcode, width != 0, 1, 0) + code[2] |= 0x80 // All VEX forms are WIG, not W0. + if evex { + code = encodeX86EVEXPackedExtend(spec.opcode, byte(width), 1, true, 1, 0) + code[2] |= 0x80 // EVEX is WIG except signed/unsigned DQ. + } + var bytes strings.Builder + for _, value := range code { + fmt.Fprintf(&bytes, "0x%02x ", value) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=x86_64", "--mattr=+avx,+avx2,+avx512f,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(bytes.String() + "\n") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + invalid := evex && spec.inputBits == 32 + if invalid { + if !strings.Contains(string(output), "invalid instruction encoding") { + t.Fatalf("LLVM22 did not reject EVEX DQ W1 %x: %v\n%s", code, err, output) + } + } else if err != nil || !strings.Contains(strings.ToUpper(string(output)), string(spec.op)) { + t.Fatalf("LLVM22 independent WIG decode %x: %v\n%s", code, err, output) + } + for _, mode := range []int{32, 64} { + ins, length, recognized, err := decodedX86PackedExtendInstruction(code, mode) + if invalid { + if !recognized || err == nil { + t.Fatalf("mode%d illegal EVEX DQ W1 acquired widening semantics: %x %+v/%v", mode, code, ins, err) + } + } else if !recognized || err != nil || length != len(code) || strings.TrimSuffix(string(ins.Op), ".Z") != string(spec.op) { + t.Fatalf("mode%d legal WIG %x lost exact family semantics: %+v/%v", mode, code, ins, err) + } + } + } + } + } +} + +func TestX86PackedZeroExtendActualGoLowMaskedMode32(t *testing.T) { + rows := x86ZeroExtendLowMaskedMode32Rows(t) + for _, row := range rows { + t.Run(row, func(t *testing.T) { + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil { + t.Fatalf("actual mode32 Go acceptance: %s: %v\n%s", row, err, listing) + } + // Keep the complete original uninitialized source. The Go object + // proves this syntax, not its missing source, K7, merge or address. + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func f()\n", "386", "i386-unknown-linux-gnu", api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized original mask form invented numerical inputs", row, api, err) + } + } + }) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required for all original private formats") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + for batch := 0; batch < 2; batch++ { + t.Run(fmt.Sprintf("private-formats/%s/batch%d", triple, batch), func(t *testing.T) { + var source strings.Builder + for index, row := range rows[batch*36 : (batch+1)*36] { + fmt.Fprintf(&source, "TEXT Format%d(SB),4,$0-0\n%s\nRET\n", index, row) + } + file, signatures := x86UnboundVoidFormatSources(t, source.String(), "386", triple) + ir := x86InstructionFormatObjectUnit(t, file, signatures, "386", triple, 0, + func(ctx *amd64Ctx, ins Instr) (bool, bool, error) { + return ctx.lowerPackedZeroExtendMove(ins.Op, ins) + }) + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + }) + } + } +} diff --git a/amd64_packed_zero_extend_result.go b/amd64_packed_zero_extend_result.go new file mode 100644 index 000000000..61c394f4e --- /dev/null +++ b/amd64_packed_zero_extend_result.go @@ -0,0 +1,50 @@ +package plan9asm + +func x86ResultPackedZeroExtendStep(ins Instr, state *x86ResultState, owned []bool, arch string, conditionalNativeRead bool) (bool, bool) { + form, known, err := x86PackedZeroExtendProperties(ins, arch) + if !known || err != nil { + return false, known + } + if !x86ResultNamedPredicateInstruction(ins) { + if !conditionalNativeRead || !x86RawZeroExtendSource(ins, arch) { + return false, true + } + ins.x86Encoded, ins.x86SourceRaw = false, nil + } + return x86ResultPackedExtendTransfer(ins, form, state, owned, arch, conditionalNativeRead), true +} + +func x86ResultPackedSignExtendStep(ins Instr, state *x86ResultState, owned []bool, arch string, native bool) (bool, bool) { + form, known, err := x86PackedSignExtendProperties(ins, arch) + if !known || err != nil || !x86ResultNamedPredicateInstruction(ins) { + return false, known // No raw SX role follows from the separate ZX identity. + } + return x86ResultPackedExtendTransfer(ins, form, state, owned, arch, native), true +} + +func x86ResultPackedExtendTransfer(ins Instr, form x86PackedExtendForm, state *x86ResultState, + owned []bool, arch string, conditionalNativeRead bool) bool { + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false + } + } + var input uint64 + if form.memory >= 0 { + // Only the lowering observer may use the paired native E5 read. + // The full potential tuple remains required for FP/AUTO/caller proofs; + // a numerical mask cannot manufacture a smaller owned envelope. + if !x86ResultReadWithNativeContinuation(ins, ins.Args[0], int64(form.memoryBytes), state, owned, arch, conditionalNativeRead) { + return false + } + input = x86CallerVectorByteMask(int64(form.memoryBytes)) + } else { + index, _ := amd64VectorRegisterIndex(ins.Args[0].Reg, form.sourceRegisterBytes) + input = state.vectorBytes(index) + } + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + initialized := x86PackedExtendDefinedBytes(form, input, state.vectorBytes(destination)) + state.writeVector(destination, form.byteWidth, initialized, !form.spec.vector) + return true +} diff --git a/amd64_packed_zero_extend_result_test.go b/amd64_packed_zero_extend_result_test.go new file mode 100644 index 000000000..2ac7c2ea7 --- /dev/null +++ b/amd64_packed_zero_extend_result_test.go @@ -0,0 +1,248 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86PackedZeroExtendResultConsumers(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + for _, native := range []bool{false, true} { + t.Run(fmt.Sprint(target.triple, "/native=", native), func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, ratio := range x86ZeroExtendTestRatios { + for _, vector := range []bool{false, true} { + for _, width := range []int{16, 32, 64} { + if !vector && width != 16 { + continue + } + for masking := 0; masking < 3; masking++ { + if !vector && masking != 0 { + continue + } + name := fmt.Sprint("Widen", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + input := "X0" + if width*ratio.input/ratio.output > 16 { + input = "Y0" + } + offset, load, declaration := 8, "MOVQ value+0(FP),X0", "value uint64" + if native { + input, load, declaration = "(DX)", "MOVQ value+0(FP),DX", "value *[32]byte" + if target.arch == "386" { + offset, load = 4, "MOVL value+0(FP),DX" + } + } + row := "PMOVZX" + ratio.suffix + if vector { + row = "V" + row + } + if masking == 2 { + row += ".Z" + } + row += " " + input + "," + if masking != 0 { + row += "K1," + } + row += bank + "1" + fmt.Fprintf(&declarations, "func %s(%s) uint32\n", name, declaration) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\nVZEROALL\n%s\nMOVL $1,AX\nKMOVW AX,K1\n%s\nPEXTRD $0,X1,AX\nMOVL AX,ret+%d(FP)\nRET\n", name, offset+4, load, row, offset) + } + } + } + } + if count != 60 || count > 64 { + t.Fatal("incomplete or oversized result matrix", count) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("widening result lacks its numerical consumer", api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "widen-result-"+api) + x86ResultInitializationObject(t, llc, target.triple, x86MulDivRoundOptimize(t, ir), "widen-result-"+api+"-O2") + } + }) + } + } +} + +func TestX86PackedZeroExtendNativeResultBoundaries(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + name := prefix + "MOVZX" + ratio.suffix + t.Run(name, func(t *testing.T) { + body := "MOVQ p+0(FP),DX\n" + name + " (DX),X1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+8(FP)\nRET" + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[8]byte) uint32", body) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("standalone result observer acquired native memory", err) + } + } + ins := x86ScalarLaneInstruction(t, name+" -1(DX)(BX*1),X1") + state := x86ResultState{flags: x86ByteFlagsAll} + base, _, _ := x86ResultGP(DX, 8, "amd64") + index, _, _ := x86ResultGP(BX, 8, "amd64") + state.pointerGP[base], state.gp[index] = true, 0xff + state.writeVector(1, 64, 0xa5a5000000000000, false) + before := state.clone() + if accepted, known := x86ResultPackedZeroExtendStep(ins, &state, nil, "amd64", true); !accepted || !known { + t.Fatal("paired native read has no conditional numerical bytes") + } + want := uint64(0xa5a500000000ffff) + if prefix == "VP" { + want = ^uint64(0) + } + if state.vectorBytes(1) != want || state.flags != before.flags || state.gp != before.gp || state.pointerGP != before.pointerGP { + t.Fatal("native widening changed another state domain") + } + for _, mutation := range []string{"decoded", "raw", "raw return", "address", "unbound address", "segment", "unknown base", "unknown index", "pointer index"} { + changed, rejected := ins, before.clone() + switch mutation { + case "decoded": + changed.x86Encoded = true + case "raw": + changed.x86SourceRaw = new(x86SourceRawInstruction) + case "raw return": + changed.x86RawNearReturn = new(x86RawNearReturnForm) + case "address": + changed.x86AddressBits = 32 + case "unbound address": + changed.x86UnboundAddressOverride = true + case "segment": + changed.x86SegmentPrefix = 0x64 + case "unknown base": + rejected.pointerGP[base] = false + case "unknown index": + rejected.gp[index] = 0 + case "pointer index": + // A reaching pointer import revokes numerical bytes. + // Only the source-wide shape pass carries their union. + rejected.pointerGP[index], rejected.gp[index] = true, 0 + } + expected := rejected.clone() + if accepted, _ := x86ResultPackedZeroExtendStep(changed, &rejected, nil, "amd64", true); accepted || !reflect.DeepEqual(rejected, expected) { + t.Fatal("unbound origin/address acquired native result bytes", mutation) + } + } + }) + } + } +} + +func TestX86PackedZeroExtendResultTupleAndMaskBoundaries(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + ins := x86ScalarLaneInstruction(t, "VPMOVZX"+ratio.suffix+".Z value+0(FP),K1,"+bank+"1") + for missing := -1; missing < 32; missing++ { + for _, maskKnown := range []bool{false, true} { + state := x86ResultState{fp: make([]bool, 32)} + owned := make([]bool, 32) + for index := range owned { + owned[index], state.fp[index] = true, index != missing + } + state.masks[1] = maskKnown + accepted := x86ResultTransfer(ins, &state, owned, 32, "amd64") + want := maskKnown && (missing < 0 || missing >= width*ratio.input/ratio.output) + if accepted != want || accepted && state.vectorBytes(1) != ^uint64(0) || len(ins.x86CallerMemory) != 0 { + t.Fatal("mask/tuple invented source bytes or caller permission", ratio.suffix, width, missing, maskKnown) + } + } + } + } + } +} + +func TestX86PackedZeroExtendNativeResultsKeepWholeSourceBoundaries(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + name := prefix + "MOVZX" + ratio.suffix + t.Run(name, func(t *testing.T) { + load := name + " (DX),X1\n" + for mutation, body := range map[string]string{ + "pointer index": name + " (DX)(DX*1),X1\n", + "lost pointer": "MOVL $1,DX\n" + load, + "missing path": "MOVL $0,BX\nCMPL BX,$0\nJE after\n" + load + "after:\n", + "unknown store input": load + "MOVL BX,(DX)\n", + "input write": load + "MOVQ $1,p+0(FP)\n", + } { + source := "TEXT ·Value(SB),4,$0-12\nMOVQ p+0(FP),DX\n" + body + "PEXTRD $0,X1,AX\nMOVL AX,ret+8(FP)\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + if mutation == "lost pointer" { + x86RequireNumericNativeSourceContext(t, source, "func Value(p *[32]byte) uint32", "amd64") + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[32]byte) uint32", "amd64", "x86_64-unknown-linux-gnu", api) + if mutation == "lost pointer" { + read := fmt.Sprintf("load volatile <%d x i8>", 16*ratio.input/ratio.output) + if err != nil || !strings.Contains(ir, read) { + t.Fatal("complete NUM address lost its shrinking source tuple", api, err) + } + x86NumericNativeAddressObject(t, ir, "x86_64-unknown-linux-gnu") + continue + } + if mutation == "missing path" { + x86VectorScratchCodegenControl(t, source, "func Value(p *[32]byte) uint32", "amd64", "x86_64-unknown-linux-gnu", ir, err) + continue + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("widening result borrowed another source contract", mutation, api, err) + } + } + } + body := "MOVQ p+0(FP),DX\n" + load + "PEXTRD $0,X1,AX\nMOVL AX,ret0+8(FP)\nRET" + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[32]byte) uint32", body) + for _, origin := range []string{"raw TEXT", "source stack", "fixed local"} { + fn := file.Funcs[0] + switch origin { + case "raw TEXT": + fn.X86RawText = []byte{0xc3} + case "source stack": + fn.x86SourceStack = new(x86SourceStackProof) + case "fixed local": + fn.x86FixedLocal = new(x86FixedLocalProof) + } + ctx := &amd64Ctx{fpScalarSource: fn, sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("widening result bypassed original entry boundary", origin, err) + } + } + }) + } + } +} + +func TestX86PackedZeroExtendDeadNativeReads(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, row := range []string{ + "PMOVZX" + ratio.suffix + " (DX),X1", + "VPMOVZX" + ratio.suffix + " (DX),Z1", + "VPMOVZX" + ratio.suffix + ".Z (DX),K1,Z1", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT ·Read(SB),4,$0-16\nMOVQ p+0(FP),DX\nMOVQ mask+8(FP),AX\nKMOVQ AX,K1\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + ir, err := x86ScalarFPBitViewAPI(t, source, "func Read(p *[32]byte,mask uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "Go") + if err != nil { + t.Fatal(err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + if !strings.Contains(text, "load volatile") || strings.Contains(row, ".Z") && !strings.Contains(text, "br i1") { + t.Fatal("dead widening discarded native reads or masked fault suppression") + } + } + }) + } + } +} diff --git a/amd64_packed_zero_extend_runtime_test.go b/amd64_packed_zero_extend_runtime_test.go new file mode 100644 index 000000000..f27de7dab --- /dev/null +++ b/amd64_packed_zero_extend_runtime_test.go @@ -0,0 +1,315 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// The byte reference uses the source spelling's mathematical ratio, not a +// production spec or emitted instruction. Wider stores observe every alias. +const x86ZeroExtendByteOracle = ` +static int checkExtend( + void (*fn)(uint8_t *, const uint8_t *, const uint8_t *, uint64_t), + const char *name, unsigned width, unsigned input, unsigned output, + int mode, int legacy, int alias, int memory +) { + const uint64_t masks[] = {0, 1, UINT64_C(0x55aa55aa55aa55aa), UINT64_MAX}; + for (unsigned round = 0; round < 4; round++) { + for (unsigned m = 0; m < 4; m++) { + uint8_t aStorage[96], bStorage[96], result[74]; + uint8_t *a = aStorage + 1, *b = bStorage + 3; + memset(aStorage, 0xa7, sizeof(aStorage)); + memset(bStorage, 0xb9, sizeof(bStorage)); + memset(result, 0xc3, sizeof(result)); + for (unsigned i = 0; i < 64; i++) { + a[i] = (uint8_t)(i * 19 + round * 83 + 129); + b[i] = (uint8_t)(i * 31 + round * 59 + 13); + } + uint32_t saved, before = 0x1f80 | (round << 13) | round, after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + fn(result + 1, a, b, masks[m]); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + if (after != before || result[0] != 0xc3 || result[73] != 0xc3 || + result[65] != 1 || result[66] != 0) { + fprintf(stderr, "%s CSR/flags/canary failure\n", name); + return 1; + } + const uint8_t *old = alias ? a : b; + for (unsigned i = 0; i < 64; i++) { + const unsigned part = i % output; + const unsigned source = i / output * input + part; + const int active = mode == 0 || ((masks[m] >> (i / output)) & 1); + uint8_t want; + if (i >= width) { + want = legacy ? old[i] : 0; + } else if (!active) { + want = mode == 2 ? 0 : old[i]; + } else { + want = part < input ? a[source] : 0; + } + if (result[i + 1] != want || a[i] != (uint8_t)(i * 19 + round * 83 + 129) || + b[i] != (uint8_t)(i * 31 + round * 59 + 13)) { + fprintf(stderr, "%s byte=%u got=%x want=%x\n", name, i, result[i + 1], want); + return 1; + } + } + for (unsigned i = 0; i < 96; i++) { + if ((i < 1 || i >= 65) && aStorage[i] != 0xa7) return 1; + if ((i < 3 || i >= 67) && bStorage[i] != 0xb9) return 1; + } + } + } + if (memory) { + uint8_t b[64], result[74]; + memset(b, 0x59, sizeof(b)); + memset(result, 0xc3, sizeof(result)); + if (mode != 0) { + // Irrelevant high mask bits cannot access an entirely inactive NULL. + fn(result + 1, NULL, b, UINT64_C(1) << 63); + for (unsigned i = 0; i < 64; i++) { + uint8_t want = i < width && mode == 1 ? 0x59 : 0; + if (result[i + 1] != want) return 1; + } + if (result[0] != 0xc3 || result[73] != 0xc3 || result[65] != 1 || result[66] != 0) return 1; + } + if (setup_guard()) return 1; + // An unmasked source uses its shrinking native tuple width, not a + // complete X/Y allocation. A masked source needs one active element. + unsigned sourceBytes = mode == 0 ? width * input / output : input; + uint8_t *last = guard + page - sourceBytes; + for (unsigned i = 0; i < sourceBytes; i++) ((volatile uint8_t *)last)[i] = (uint8_t)(0xd0 + i); + fn(result + 1, last, b, 1); + int failed = result[0] != 0xc3 || result[73] != 0xc3 || result[65] != 1 || result[66] != 0; + for (unsigned i = 0; i < 64; i++) { + unsigned part = i % output; + int active = mode == 0 || i < output; + uint8_t want = i >= width ? (legacy ? 0x59 : 0) : + active ? (part < input ? (uint8_t)(0xd0 + i / output * input + part) : 0) : + mode == 1 ? 0x59 : 0; + if (result[i + 1] != want || b[i] != 0x59) failed = 1; + } + if (free_guard()) return 1; + if (failed) return 1; + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedZeroExtendCompleteBytes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, ratio := range x86ZeroExtendTestRatios { + for _, vector := range []bool{false, true} { + for _, width := range []int{16, 32, 64} { + if !vector && width != 16 { + continue + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + sourceBank := "X" + if width*ratio.input/ratio.output > 16 { + sourceBank = "Y" + } + for _, memory := range []bool{false, true} { + for mode := 0; mode < 3; mode++ { + if !vector && mode != 0 { + continue + } + for _, destination := range []int{0, 1} { + if memory && destination == 0 { + continue // Memory has no register-source alias. + } + name := fmt.Sprintf("extend%d", count) + count++ + opcode := "PMOVZX" + ratio.suffix + if vector { + opcode = "V" + opcode + } + input, mask := sourceBank+"0", "" + if memory { + input = "(BX)" + } + if mode != 0 { + mask = "K1," + if mode == 2 { + opcode += ".Z" + } + } + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z1\n", name) + if !memory { + assembly.WriteString("VMOVUPS (BX),Z0\n") + } + fmt.Fprintf(&assembly, "KMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s %s,%s%s%d\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", opcode, input, mask, bank, destination, destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, " if (checkExtend(%s,\"%s\",%d,%d,%d,%d,%d,%d,%d)) return 1;\n", name, name, width, ratio.input, ratio.output, mode, x86FloatAddSubBoolInt(!vector), x86FloatAddSubBoolInt(destination == 0), x86FloatAddSubBoolInt(memory)) + } + } + } + } + } + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + oracle := "#include \n#include \n#include \n" + + declarations.String() + x86TestGuardPagesC + x86ZeroExtendByteOracle + + "\nint main(void) {\n" + checks.String() + " return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "zero-extend-bytes", triple, ir, oracle, prefix) + t.Logf("%d ratio/width/register-memory/mask/self-alias programs, 16 complete-byte flags/MXCSR/canary inputs each; inactive NULL and page-boundary reads", count) +} + +func TestCrossLinuxRuntimeMatrixX86PackedZeroExtendOriginalGoBytes(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package oracle\n") + checks.WriteString(`package oracle +import "testing" +func TestOriginalGoUnsignedBits(t *testing.T) { + for round := 0; round < 4; round++ { + var input, old [64]byte + for i := range input { + input[i] = byte(i*19 + round*83 + 129) + old[i] = byte(i*31 + round*59 + 13) + } +`) + count := 0 + for _, ratio := range x86ZeroExtendTestRatios { + for _, memory := range []bool{false, true} { + for _, alias := range []bool{false, true} { + if memory && alias { + continue + } + name := fmt.Sprintf("original%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(a,b,out *byte)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\nMOVOU (BX),X1\n", name) + source, destination := "(AX)", "X1" + if !memory { + assembly.WriteString("MOVOU (AX),X0\n") + source = "X0" + if alias { + destination = "X0" + } + } + fmt.Fprintf(&assembly, "CMPQ AX,AX\nSTC\nPMOVZX%s %s,%s\nMOVOU %s,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\nRET\n", ratio.suffix, source, destination, destination) + fmt.Fprintf(&checks, ` + { + var result [20]byte + for i := range result { result[i] = 0xc3 } + %[1]s(&input[0], &old[0], &result[1]) + if result[0] != 0xc3 || result[19] != 0xc3 || result[17] != 1 || result[18] != 0 { + t.Fatalf("%[1]s flags/canary failure: %%x", result) + } + for i := 0; i < 16; i++ { + want := byte(0) + if i%%%[2]d < %[3]d { want = input[i/%[2]d*%[3]d+i%%%[2]d] } + if result[i+1] != want { t.Fatalf("%[1]s byte %%d got %%x want %%x", i, result[i+1], want) } + } + } +`, name, ratio.output, ratio.input) + } + } + } + checks.WriteString(" }\n}\n") + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": checks.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=1", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go unsigned byte oracle: %v\n%s", err, output) + } + t.Logf("%d original legacy SSE4.1 register/memory/self-alias functions, 4 complete-byte/flags/canary inputs each; not native VEX/EVEX execution", count) +} + +func TestCrossLinuxRuntimeMatrixX86PackedZeroExtendIndependentFPBytes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, ratio := range x86ZeroExtendTestRatios { + for _, width := range []int{16, 32, 64} { + for _, legacy := range []bool{false, true} { + if legacy && width != 16 { + continue + } + name := fmt.Sprintf("fpExtend%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + opcode := "VPMOVZX" + ratio.suffix + if legacy { + opcode = "PMOVZX" + ratio.suffix + } + var cArgs, cInputs []string + sig := FuncSig{Name: name, Ret: Void} + for i := 0; i < 32; i++ { + cArgs = append(cArgs, "uint8_t") + cInputs = append(cInputs, fmt.Sprintf("input[%d]", i)) + sig.Args = append(sig.Args, I8) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(i), Type: I8, Index: i, Field: -1}) + } + sig.Args = append(sig.Args, Ptr) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: 32, Type: Ptr, Index: 32, Field: -1}) + sigs[name] = sig + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-40\nMOVQ out+32(FP),AX\nMOVL $89,BX\nMOVD BX,X1\nVPBROADCASTB X1,Z1\n%s a+0(FP),%s1\nVMOVUPS Z1,(AX)\nRET\n", name, opcode, bank) + fmt.Fprintf(&declarations, "extern void %s(%s, uint8_t *);\n", name, strings.Join(cArgs, ", ")) + fmt.Fprintf(&checks, " memset(result, 0xc3, sizeof(result));\n %s(%s, result + 1);\n", name, strings.Join(cInputs, ", ")) + fmt.Fprintf(&checks, ` for (unsigned i = 0; i < 64; i++) { + unsigned part = i %% %d; + uint8_t want = i >= %d ? %d : part < %d ? input[i / %d * %d + part] : 0; + if (result[i + 1] != want) { + fprintf(stderr, "%s independent FP byte %%u: %%x != %%x\n", i, result[i + 1], want); + return 1; + } + } + if (result[0] != 0xc3 || result[65] != 0xc3) return 1; +`, ratio.output, width, map[bool]int{false: 0, true: 89}[legacy], ratio.input, ratio.output, ratio.input, name) + } + } + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + oracle := "#include \n#include \n#include \n" + declarations.String() + ` +int main(void) { + for (unsigned round = 0; round < 4; round++) { + uint8_t input[32], result[66]; + for (unsigned i = 0; i < 32; i++) input[i] = (uint8_t)(round * 83 + i * 19 + 129); +` + checks.String() + " }\n return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "zero-extend-fp-bytes", triple, ir, oracle, prefix) + t.Logf("%d widening programs, 32 independently passed byte fields, 4 full 64-byte alias/canary inputs each", count) +} diff --git a/amd64_parallel_bits_cross_test.go b/amd64_parallel_bits_cross_test.go new file mode 100644 index 000000000..48b7a6f25 --- /dev/null +++ b/amd64_parallel_bits_cross_test.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "testing" +) + +// This name is intentionally included by the required cross-runtime CI regex. +func TestCrossLinuxRuntimeMatrixParallelBits386(t *testing.T) { + if os.Getenv("PLAN9ASM_CROSS_EXEC") != "1" { + t.Skip("set PLAN9ASM_CROSS_EXEC=1 for the required Linux cross-runtime matrix") + } + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("cross-runtime driver requires linux/amd64") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, tool := range []string{"i686-linux-gnu-gcc", "qemu-i386"} { + if _, err := exec.LookPath(tool); err != nil { + t.Fatalf("required cross-runtime tool %s: %v", tool, err) + } + } + var source, decl, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + for _, raw := range []bool{false, true} { + name := fmt.Sprintf("parallel386_%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-12\nMOVL value+0(FP),AX\nMOVL mask+4(FP),BX\n", name) + line := op + " (BX),AX,CX\n" + if raw { + code := assembleX87ControlBytes(t, "386", "TEXT probe(SB),4,$0-0\n"+line+"RET\n") + for _, b := range code[:len(code)-1] { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + } else { + source.WriteString(line) + } + source.WriteString("MOVL CX,ret+8(FP)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{I32, Ptr}, Ret: I32, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}, {Offset: 4, Type: Ptr, Index: 1, Field: -1}}, Results: []FrameSlot{{Offset: 8, Type: I32, Index: 0, Field: -1}}}} + fmt.Fprintf(&decl, "extern uint32_t %s(uint32_t,const uint32_t *);\n", name) + deposit := 0 + if strings.HasPrefix(op, "PDEP") { + deposit = 1 + } + fmt.Fprintf(&checks, "if(check(%s,mask,%d)) return %d;\n", name, deposit, index) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := `#include +#include +#include +#include +#include +` + decl.String() + ` +static void memory_fault(int signal_number) { (void)signal_number; _exit(98); } +static uint32_t reference(uint32_t value,uint32_t mask,int deposit) { + uint32_t result=0,cursor=1; + while(mask) {uint32_t low=mask&(-mask);if(deposit ? (value&cursor)!=0 : (value&low)!=0) result|=deposit?low:cursor;mask&=mask-1;cursor<<=1;} + return result; +} +static int check(uint32_t(*fn)(uint32_t,const uint32_t *),uint32_t *mask,int deposit) { + uint32_t state=0x9e3779b9; + for(int i=0;i<256;i++) { + state^=state<<13;state^=state>>17;state^=state<<5;uint32_t value=state; + state^=state<<13;state^=state>>17;state^=state<<5;*mask=state; + if(i==0)*mask=0;if(i==1)*mask=0xffffffffU;if(i>=2&&i<34)*mask=1U<<(i-2); + if(fn(value,mask)!=reference(value,*mask,deposit)) {fprintf(stderr,"386 parallel bits mismatch case=%d deposit=%d\n",i,deposit);return 1;} + } + return 0; +} +int main(void) { + if(signal(SIGSEGV,memory_fault)==SIG_ERR)return 92; + long page=sysconf(_SC_PAGESIZE);if(page<4)return 90; + uint8_t *memory=mmap(0,page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(memory==MAP_FAILED||mprotect(memory+page,page,PROT_NONE))return 91; + uint32_t *mask=(uint32_t *)(memory+page-4); +` + checks.String() + "return munmap(memory,page*2)!=0;\n}\n" + compileAndRunRuntimeTestWithCompiler(t, llc, []string{"i686-linux-gnu-gcc"}, "parallel_bits_386", "i386-unknown-linux-gnu", ir, mainC, []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"}) +} diff --git a/amd64_permute128_forms.go b/amd64_permute128_forms.go new file mode 100644 index 000000000..6471ef837 --- /dev/null +++ b/amd64_permute128_forms.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Both _yvperm2f128 members have the same fixed VEX.256.W0 grammar and +// bit-selection semantics. Their different native CPUID requirements are not +// numerical input, memory or source permissions. +var x86Permute128Specs = map[Op]byte{ + "VPERM2F128": 0x06, + "VPERM2I128": 0x46, +} + +type x86Permute128Form struct { + x86PackedBinaryForm + immediate uint8 +} + +func x86Permute128Properties(op Op, ins Instr, arch string) (x86Permute128Form, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + _, known := x86Permute128Specs[Op(base)] + result := x86Permute128Form{} + if !known { + return result, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || len(ins.Args) != 4 || + arch == "386" && !(ins.x86Encoded && ins.x86Permute128VEX) { + return result, true, fmt.Errorf("%s %s has an unbound dispatch or named four-operand role", arch, op) + } + immediate, integer := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !integer || !amd64UnsignedImmediate(immediate, 8) { + return result, true, fmt.Errorf("%s requires Go's complete Yu8 immediate class", op) + } + local := ins + local.Op, local.Args = op, ins.Args[1:] + form, err := x86ConflictFunnelOperandGrammar(local, arch, + x86PackedBinarySpec{encoding: x86PackedLogicalVEX}, false) + form.first++ + form.second++ + form.destination++ + if form.memory >= 0 { + form.memory++ + } + result.x86PackedBinaryForm, result.immediate = form, uint8(immediate.Imm) + if err == nil && form.byteWidth != 32 { + err = fmt.Errorf("%s requires exactly VEX.256 operands", op) + } + return result, true, err +} + +// Return one byte in the two Plan9 sources, or -1 for an immediate zero. +// Intel SRC1 is Plan9 source two. Selector bits 2 and 6 have no effect. +func (form x86Permute128Form) sourceByte(output int) int { + control := form.immediate >> uint(output/16*4) + if control&8 != 0 { + return -1 + } + selected := int(control & 3) + source := 32 + if selected >= 2 { + source = 0 + } + return source + selected%2*16 + output%16 +} + +func (form x86Permute128Form) inputsDefined(first, second uint64) bool { + for output := 0; output < 32; output++ { + selected := form.sourceByte(output) + if selected < 0 { + continue + } + known := first + if selected >= 32 { + known, selected = second, selected-32 + } + if known>>uint(selected)&1 == 0 { + return false + } + } + return true +} + +func x86TypedPermute128ByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86Permute128Properties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + return x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: 32, read: true}, + {index: form.second, operand: ins.Args[form.second], width: 32, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: 32, write: true}, + }}, true, nil +} diff --git a/amd64_permute_inlane_forms.go b/amd64_permute_inlane_forms.go new file mode 100644 index 000000000..e0510d656 --- /dev/null +++ b/amd64_permute_inlane_forms.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Both members share Go's complete eighteen-row _yvpermilpd grammar. The +// immediate selects data bytes; variable control reads a separate vector. +var x86InLanePermuteBits = map[Op]int{"VPERMILPS": 32, "VPERMILPD": 64} + +// Both permute families share their operand, mask and E4NF envelopes. Their +// selector topology and which variable operand is data remain distinct. +type x86ElementPermuteForm struct { + bits, byteWidth, source, data, destination, mask, memory, memoryBytes int + immediate, broadcast, zeroing, wholeVector bool + selector uint8 +} + +func x86InLanePermuteProperties(op Op, ins Instr, arch string) (x86ElementPermuteForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + bits, known := x86InLanePermuteBits[Op(base)] + if !known { + return x86ElementPermuteForm{}, false, nil + } + return x86ElementPermuteFormProperties(op, ins, arch, bits, false) +} + +func x86ElementPermuteProperties(op Op, ins Instr, arch string) (x86ElementPermuteForm, bool, error) { + if form, known, err := x86InLanePermuteProperties(op, ins, arch); known { + return form, true, err + } + if form, known, err := x86QwordPermuteProperties(op, ins, arch); known { + return form, true, err + } + return x86VariableDwordPermuteProperties(op, ins, arch) +} + +func x86VariableDwordPermuteProperties(op Op, ins Instr, arch string) (x86ElementPermuteForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + switch base { + case "VPERMD", "VPERMPS": + default: + return x86ElementPermuteForm{}, false, nil + } + form, known, err := x86ElementPermuteFormProperties(op, ins, arch, 32, true) + if err == nil && form.immediate { + err = fmt.Errorf("%s requires a vector index source, not an immediate", op) + } + return form, known, err +} + +func x86ElementPermuteFormProperties(op Op, ins Instr, arch string, bits int, wholeVector bool) (x86ElementPermuteForm, bool, error) { + _, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + f := x86ElementPermuteForm{bits: bits, wholeVector: wholeVector, source: 0, data: 1, + destination: len(ins.Args) - 1, mask: -1, memory: -1} + fail := func() (x86ElementPermuteForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside Go's complete element-permute grammar: %q", arch, op, ins.Raw) + } + if arch == "" { + arch = "amd64" + } + if arch != "amd64" && arch != "386" || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || + len(ins.Args) != 3 && len(ins.Args) != 4 { + return fail() + } + switch suffix { + case "": + if dotted { + return fail() + } + case "Z": + f.zeroing = true + case "BCST": + f.broadcast = true + case "BCST.Z": + f.broadcast, f.zeroing = true, true + default: + return fail() + } + if len(ins.Args) == 4 { + if arch == "386" || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() // Named mode32's three-operand limit is not a physical EVEX restriction. + } + f.mask = 2 + } else if f.zeroing { + return fail() + } + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + register := func(operand Operand) bool { + if !amd64EVEXVectorRegister(operand, f.byteWidth) { + return false + } + index, _ := amd64VectorRegisterIndex(operand.Reg, f.byteWidth) + return arch != "386" || f.byteWidth != 64 || index < 8 + } + if f.byteWidth != 16 && f.byteWidth != 32 && f.byteWidth != 64 || wholeVector && f.byteWidth == 16 || + !register(ins.Args[f.destination]) { + return fail() + } + f.immediate = ins.Args[0].Kind == OpImm + if f.immediate { + constant, integer := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !integer || constant.Imm < -128 || constant.Imm > 255 { + return fail() + } + f.source, f.data, f.selector = 1, -1, uint8(constant.Imm) + if constant.Imm < 0 && (f.broadcast || amd64PackedWordShuffleRequiresEVEX( + ins.Args[f.source], ins.Args[f.destination], f.byteWidth, f.mask >= 0)) { + return fail() + } + } else if !register(ins.Args[f.data]) { + return fail() + } + source := ins.Args[f.source] + if isAMD64MemoryOperand(source) && x86PackedShiftMemoryValid(source, arch) { + f.memory, f.memoryBytes = f.source, f.byteWidth + if f.broadcast { + f.memoryBytes = bits / 8 + } + } else if f.broadcast || !register(source) { + return fail() + } + return f, true, x86Mode32VectorRegisterContext(ins, arch) +} + +func (f x86ElementPermuteForm) sourceByte(output int) int { + if f.broadcast { + return output % (f.bits / 8) + } + bytes := f.bits / 8 + lane := output / bytes + if f.wholeVector { + return output/32*32 + int(f.selector>>uint(2*(lane%4))&3)*8 + output%8 + } + control, mask := uint(lane), uint8(1) + if f.bits == 32 { + control, mask = uint(2*(lane%4)), 3 + } + return output/16*16 + int(f.selector>>control&mask)*bytes + output%bytes +} + +func x86TypedElementPermuteByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + f, known, err := x86ElementPermuteProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := f.byteWidth + if f.memory >= 0 { + width = f.memoryBytes + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: f.source, operand: ins.Args[f.source], width: int64(width), read: true}, + {index: f.destination, operand: ins.Args[f.destination], width: int64(f.byteWidth), write: true}, + }} + if f.data >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{ + index: f.data, operand: ins.Args[f.data], width: int64(f.byteWidth), read: true}) + } + if f.mask >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{index: f.mask, operand: ins.Args[f.mask], width: 8, read: true}) + if !f.zeroing { + effect.operands[1].read = true + } + } + return effect, true, nil +} diff --git a/amd64_permute_qword_forms.go b/amd64_permute_qword_forms.go new file mode 100644 index 000000000..7656cceb8 --- /dev/null +++ b/amd64_permute_qword_forms.go @@ -0,0 +1,16 @@ +package plan9asm + +import "strings" + +// Both members share all ten Go _yvpermq rows. Immediate control repeats +// within each 256-bit block; variable control selects from the full vector. +var x86QwordPermuteBits = map[Op]int{"VPERMQ": 64, "VPERMPD": 64} + +func x86QwordPermuteProperties(op Op, ins Instr, arch string) (x86ElementPermuteForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + bits, known := x86QwordPermuteBits[Op(base)] + if !known { + return x86ElementPermuteForm{}, false, nil + } + return x86ElementPermuteFormProperties(op, ins, arch, bits, true) +} diff --git a/amd64_polynomial_fold_composition_test.go b/amd64_polynomial_fold_composition_test.go new file mode 100644 index 000000000..e90633208 --- /dev/null +++ b/amd64_polynomial_fold_composition_test.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +const x86PolynomialFoldDeclaration = "func fold(p *[128]byte, n, seed uint32) uint32\n" + +// A first-party polynomial fold, not a copied CRC implementation. Its complete +// result path crosses native pointer reads, private aligned DATA, both CLMUL +// qword selectors, byte/lane shifts, a loop backedge and a two-arm value join. +func x86PolynomialFoldSource(arch string) string { + word, move, add := 8, "MOVQ", "ADDQ" + if arch == "386" { + word, move, add = 4, "MOVL", "ADDL" + } + return fmt.Sprintf(`DATA foldPoly<>+0(SB)/8,$0x1f123bb5a7c9d001 +DATA foldPoly<>+8(SB)/8,$0xa7c1e305f1b692d3 +GLOBL foldPoly<>(SB),8,$16 + +TEXT ·fold(SB),4,$0-%d + %s p+0(FP),SI + MOVL n+%d(FP),CX + MOVL seed+%d(FP),AX + PXOR X1,X1 + PINSRD $0,AX,X1 + MOVOA foldPoly<>(SB),X3 + CMPL CX,$0 + JE finish +loop: + MOVOU (SI),X0 + PXOR X0,X1 + MOVOU X1,X2 + PCLMULQDQ $0,X3,X1 + PCLMULQDQ $17,X3,X2 + PSRLDQ $8,X2 + PXOR X2,X1 + TESTL $1,CX + JE even + MOVOU X1,X4 + PSRLQ $7,X4 + PXOR X4,X1 + JMP joined +even: + MOVOU X1,X4 + PSLLQ $3,X4 + PXOR X4,X1 +joined: + %s $16,SI + SUBL $1,CX + JNE loop +finish: + MOVOU X1,X2 + PSRLO $8,X2 + PXOR X2,X1 + PSRLQ $11,X1 + PEXTRD $0,X1,AX + MOVL AX,ret+%d(FP) + RET +`, word+12, move, word, word+4, add, word+8) +} + +func TestX86PolynomialFoldCompositionThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + source := x86PolynomialFoldSource(target.arch) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, x86PolynomialFoldDeclaration, target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, native := range []string{"movdqu", "movdqa"} { + if !strings.Contains(ir, "asm sideeffect \""+native+" ($1), $0\"") { + t.Fatal(api, "conditional result lost its original native load", native) + } + } + compileLLVMToObject(t, llc, target.triple, "polynomial-fold.ll", "polynomial-fold.o", ir) + } + }) + } +} + +func TestX86PolynomialFoldCompositionDoesNotCreateStandalonePermissions(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := x86PolynomialFoldSource(arch) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "fold", "package fold\n"+x86PolynomialFoldDeclaration) + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + fn := file.Funcs[0] + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(fn, sigs["fold"], arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("linked-data/native-load composition leaked declaration-only authority", arch, err) + } + } + } +} + +func TestX86PolynomialFoldCompositionRejectsOneMissingProof(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for name, change := range map[string][2]string{ + "external data": {"MOVOA foldPoly<>(SB),X3", "MOVOA otherPoly<>(SB),X3"}, + "short data": {"DATA foldPoly<>+8(SB)/8,$0xa7c1e305f1b692d3\nGLOBL foldPoly<>(SB),8,$16", "GLOBL foldPoly<>(SB),8,$8"}, + "relocated data": {"$0xa7c1e305f1b692d3", "$·external(SB)"}, + "missing pointer": {"MOVOU (SI),X0", "MOVOU (DI),X0"}, + "missing joined value": {"even:\n\tMOVOU X1,X4", "even:\n\tMOVOU X5,X4"}, + "unknown data input": {"PCLMULQDQ $0,X3,X1", "PCLMULQDQ $0,X5,X1"}, + } { + t.Run(arch+"/"+name, func(t *testing.T) { + source := strings.Replace(x86PolynomialFoldSource(arch), change[0], change[1], 1) + if arch == "386" && name == "relocated data" { + source = strings.Replace(source, "+8(SB)/8,$·external(SB)", "+8(SB)/4,$·external(SB)", 1) + } + goAssemblerOracleOutput(t, arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + triple := testTargetTriple("linux", arch) + ir, err := x86ScalarFPBitViewAPI(t, source, x86PolynomialFoldDeclaration, arch, triple, api) + if arch == "amd64" && (name == "missing joined value" || name == "unknown data input") { + x86VectorScratchCodegenControl(t, source, x86PolynomialFoldDeclaration, arch, triple, ir, err) + continue + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal(api, "incomplete composition acquired a result", err) + } + } + }) + } + } +} diff --git a/amd64_polynomial_fold_runtime_unix_test.go b/amd64_polynomial_fold_runtime_unix_test.go new file mode 100644 index 000000000..403324540 --- /dev/null +++ b/amd64_polynomial_fold_runtime_unix_test.go @@ -0,0 +1,148 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +const x86PolynomialFoldOriginalGo = `package main +import ("encoding/binary"; "fmt") + +func multiply(a,b uint64) (lo,hi uint64) { + for bit:=uint(0);bit<64;bit++ { + if b>>bit&1==0 { continue } + lo^=a<>(64-bit) } + } + return +} +func reference(p *[128]byte,n,seed uint32) uint32 { + low,high:=uint64(seed),uint64(0) + for block:=uint32(0);block>7 + high^=high>>7 + } else { + low^=low<<3 + high^=high<<3 + } + } + return uint32((low^high)>>11) +} +func main() { + seeds:=[]uint32{0,1,0xffffffff,0x89abcdef} + for sample:=0;sample<8;sample++ { + var storage [144]byte + for i:=range storage { storage[i]=byte(i*71+sample*97+(i*sample)%251) } + before:=storage + for offset:=0;offset<4;offset++ { + p:=(*[128]byte)(storage[offset:offset+128]) + for n:=uint32(0);n<=8;n++ { + for _,seed:=range seeds { + got,want:=fold(p,n,seed),reference(p,n,seed) + if got!=want || storage!=before { + panic(fmt.Sprintf("original Go sample=%d offset=%d n=%d seed=%x got=%x want=%x",sample,offset,n,seed,got,want)) + } + fmt.Println(got) + } + } + } + } + for _,seed:=range seeds { + got,want:=fold(nil,0,seed),reference(nil,0,seed) + if got!=want { panic("nil/zero-length control") } + fmt.Println(got) + } +} +` + +func TestCrossLinuxRuntimeMatrixX86PolynomialFoldOriginalGoThreeAPIs(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + goos := "linux" + if strings.Contains(execution.triple, "apple") { + goos = "darwin" + } + source := x86PolynomialFoldSource("amd64") + directory := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module polynomial_fold_oracle\n\ngo 1.20\n", + "main.go": x86PolynomialFoldOriginalGo + x86PolynomialFoldDeclaration, + "fold.s": source, + } { + if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + original := filepath.Join(directory, "original") + environment := []string{"GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0", "GOARCH=amd64", "GOOS=" + goos} + dataGlobalAlignmentCommand(t, directory, environment, "go", "build", "-p=2", "-o", original, ".") + results := strings.Fields(string(execution.run(t, original))) + const samples = 8*4*9*4 + 4 + if len(results) != samples { + t.Fatalf("actual linked Go executed %d results, want %d", len(results), samples) + } + var values strings.Builder + for _, result := range results { + value, err := strconv.ParseUint(result, 10, 32) + if err != nil { + t.Fatal("original Go result is not uint32", err) + } + fmt.Fprintf(&values, "UINT32_C(%d),\n", value) + } + main := `#include +#include +#include +extern uint32_t fold(const uint8_t*,uint32_t,uint32_t); +static const uint32_t original[]={ +` + values.String() + `}; +int main(void) { + const uint32_t seeds[]={0,1,0xffffffff,0x89abcdef}; + unsigned index=0; + for(unsigned sample=0;sample<8;sample++) { + uint8_t storage[144],before[144]; + for(unsigned i=0;i<144;i++) storage[i]=(uint8_t)(i*71+sample*97+(i*sample)%251); + memcpy(before,storage,sizeof(storage)); + for(unsigned offset=0;offset<4;offset++) + for(unsigned n=0;n<=8;n++) + for(unsigned seed=0;seed<4;seed++) { + uint32_t got=fold(storage+offset,n,seeds[seed]); + if(got!=original[index] || memcmp(storage,before,sizeof(storage))) { + fprintf(stderr,"LLVM sample=%u offset=%u n=%u seed=%x got=%x original=%x\n", + sample,offset,n,seeds[seed],got,original[index]); + return 1; + } + index++; + } + } + for(unsigned seed=0;seed<4;seed++) if(fold(0,0,seeds[seed])!=original[index++]) return 2; + return index!=sizeof(original)/sizeof(original[0]); +} +` + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, x86PolynomialFoldDeclaration, "amd64", execution.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(api+"/"+level, func(t *testing.T) { + text := ir + if level == "-O2" { + text = x86MulDivRoundOptimize(t, text) + } + execution.compileAndRunLLVM(t, "polynomial-fold", text, main, level) + }) + } + } + t.Logf("%d actual Go linked loop/join results, independently checked polynomial reference, three LLVM APIs O0/O2; source identical, inputs unchanged", samples) +} diff --git a/amd64_prelude_demand.go b/amd64_prelude_demand.go new file mode 100644 index 000000000..585187ae8 --- /dev/null +++ b/amd64_prelude_demand.go @@ -0,0 +1,57 @@ +package plan9asm + +import ( + "strings" + "sync" +) + +// Only immutable compiler declarations are shared. Source-dependent helpers +// and all source/caller permissions remain local to the current translation. +var amd64DemandPreludeCatalog [2]struct { + once sync.Once + declarations map[string]string + ordinary string +} + +func amd64PreludeForGOARCH(goarch string) (map[string]string, string) { + index := 0 + if goarch == "386" { + index = 1 + } + catalog := &amd64DemandPreludeCatalog[index] + catalog.once.Do(func() { + var full strings.Builder + // The original emitter inspects file.Funcs for SHA/string/rounding + // helpers. An empty file selects only its fixed declarations. + emitAMD64Prelude(&full, goarch, &File{Arch: ArchAMD64}) + text := full.String() + catalog.declarations = llvmPreludeDeclarations(text) + var ordinary strings.Builder + for _, line := range strings.Split(text, "\n") { + if strings.HasPrefix(line, "declare ") && !strings.Contains(line, "@llvm.") { + ordinary.WriteString(line) + ordinary.WriteByte('\n') + } + } + for name := range catalog.declarations { + if !strings.HasPrefix(name, "llvm.") { + delete(catalog.declarations, name) + } + } + catalog.ordinary = ordinary.String() + }) + return catalog.declarations, catalog.ordinary +} + +func emitAMD64SourcePrelude(b *strings.Builder, file *File, goarch string) { + _, ordinary := amd64PreludeForGOARCH(goarch) + emitX86SHADeclarations(b, file) + b.WriteString(ordinary) + emitX86StringHelpers(b, file) + emitAMD64FMA3RoundingHelpers(b, file, goarch) +} + +func emitUsedAMD64Prelude(b *strings.Builder, goarch string) { + declarations, _ := amd64PreludeForGOARCH(goarch) + emitUsedLLVMPrelude(b, declarations) +} diff --git a/amd64_prelude_demand_bench_test.go b/amd64_prelude_demand_bench_test.go new file mode 100644 index 000000000..7f97edc75 --- /dev/null +++ b/amd64_prelude_demand_bench_test.go @@ -0,0 +1,37 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +// Use identical initialized scalar input to isolate the text-fallback cost. +// The direct case must gain no demand scan or catalog lookup. Existing packed +// binary/physical and baseline-GP benchmarks retain their original case IDs. +func BenchmarkX86PreludeTranslation(b *testing.B) { + for _, annotate := range []bool{false, true} { + name := "direct" + if annotate { + name = "annotated-fallback" + } + b.Run(name, func(b *testing.B) { + source := "TEXT scalar(SB),4,$0-0\n" + strings.Repeat("MOVL $7,AX\n", 64) + "RET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + b.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", AnnotateSource: annotate, + Sigs: map[string]FuncSig{"scalar": {Name: "scalar", Ret: Void}}} + if _, err := Translate(file, options); err != nil { + b.Fatal("benchmark must complete before timing", err) + } + b.ReportAllocs() + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + if _, err := Translate(file, options); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/amd64_prelude_demand_test.go b/amd64_prelude_demand_test.go new file mode 100644 index 000000000..fb9514cb7 --- /dev/null +++ b/amd64_prelude_demand_test.go @@ -0,0 +1,146 @@ +package plan9asm + +import ( + "sort" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func requireX86DemandDeclarations(t *testing.T, ir string, want ...string) { + t.Helper() + var got []string + ordinary := make(map[string]bool) + for _, line := range strings.Split(ir, "\n") { + if !strings.HasPrefix(line, "declare ") { + continue + } + for name := range llvmTextGlobalNames(line) { + if strings.HasPrefix(name, "llvm.") { + got = append(got, name) + } else { + ordinary[name] = true + } + } + } + sort.Strings(got) + want = append([]string(nil), want...) + sort.Strings(want) + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("LLVM declarations = %v, want only actual uses %v (IR: %d bytes)", got, want, len(ir)) + } + if len(ordinary) != 6 { + t.Errorf("ordinary declarations = %v, want the six original external declarations", ordinary) + } + for _, name := range []string{"syscall", "cliteErrno", "atan2", "fmod", "remainder", "tan"} { + if !ordinary[name] { + t.Errorf("lost original ordinary declaration %s", name) + } + } +} + +func TestX86DemandPreludeContainsOnlyUsedDeclarations(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + name, body string + want []string + }{ + {name: "scalar", body: "MOVL $7,AX\nRET"}, + {name: "annotated-comment", body: "MOVL $7,AX // @llvm.bswap.i64( is only a comment\nRET"}, + {name: "scalar-intrinsic", body: "MOVL $7,AX\nPOPCNTL AX,BX\nRET", want: []string{"llvm.ctpop.i32"}}, + {name: "vector-intrinsic", body: "PXOR X0,X0\nPXOR X1,X1\nPSHUFB X0,X1\nRET", want: []string{"llvm.x86.ssse3.pshuf.b.128"}}, + {name: "source-sha", body: "PXOR X1,X1\nSHA1MSG1 X1,X1\nRET", want: []string{"llvm.x86.sha1msg1"}}, + } { + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+row.name+"/"+api, func(t *testing.T) { + ir, err := targetBackendRequest(t, api, testTargetTriple("linux", arch), targetBackendCase{ + arch: ArchAMD64, goarch: arch, body: row.body, + }) + if err != nil { + t.Fatal("initialized source must translate before checking declaration demand", err) + } + requireX86DemandDeclarations(t, ir, row.want...) + }) + } + } + } +} + +func TestX86DemandPreludeRetainsSourceHelperClosure(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, arch := range []string{"amd64", "386"} { + for _, api := range []string{"text", "module", "Go"} { + for _, row := range []struct { + kind, bits string + }{{"SS", "32"}, {"SD", "64"}} { + t.Run(arch+"/rounded-fma"+row.bits+"/"+api, func(t *testing.T) { + // The emitted software helper itself references both intrinsics, + // even when a matching native entry uses exact inline assembly. + ir, err := targetBackendRequest(t, api, testTargetTriple("linux", arch), targetBackendCase{ + arch: ArchAMD64, goarch: arch, + body: "PXOR X0,X0\nPXOR X1,X1\nPXOR X2,X2\nVFMADD132" + row.kind + ".RD_SAE X0,X1,X2\nRET", + }) + if err != nil { + t.Fatal("initialized FMA source must translate before checking helper closure", err) + } + helper := x86StringMemoryFunction(t, ir, "__plan9asm_fma3_round_f"+row.bits) + for _, call := range []string{"@llvm.fma.f" + row.bits + "(", "@llvm.ctlz.i128("} { + if !strings.Contains(helper, call) { + t.Fatal("source-dependent FMA helper lost its intrinsic reference", call) + } + } + requireX86DemandDeclarations(t, ir, "llvm.fma.f"+row.bits, "llvm.ctlz.i128") + compileLLVMToObject(t, llc, testTargetTriple("linux", arch), "demand-fma.ll", "demand-fma.o", ir) + }) + } + t.Run(arch+"/string-helper/"+api, func(t *testing.T) { + // Reuse the initialized native-pointer source owner; selecting + // declarations must not rebuild its memory or runtime matrix. + source, declarations := x86StringRepeatCFGFixture( + x86StringMemoryRow{op: "MOVSB", kind: "movs", width: 1, reads: 1, writes: 1}, + arch, "REP", "adjacent") + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, arch, testTargetTriple("linux", arch), api) + if err != nil { + t.Fatal("original string source must translate before checking helper preservation", err) + } + helper := x86StringMemoryFunction(t, ir, "__plan9asm_movsb") + if err := x86StringMemoryIRAccessError(helper, 1, 1, 1); err != nil { + t.Fatal(err) + } + requireX86DemandDeclarations(t, ir) + compileLLVMToObject(t, llc, testTargetTriple("linux", arch), "demand-string.ll", "demand-string.o", ir) + }) + } + } +} + +func TestX86DemandPreludeLeavesDirectModuleIndependent(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT scalar(SB),4,$0-0\nMOVL $7,AX\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"scalar": {Name: "scalar", Ret: Void}}} + for _, translate := range []func(llvm.Context, *File, Options) (llvm.Module, error){ + translateModuleDirectInContext, TranslateModuleInContext, + } { + module, err := translate(ctx, file, options) + if err != nil || module.IsNil() { + if !module.IsNil() { + module.Dispose() + } + t.Fatal("ordinary direct-module source must remain independent of textual prelude", err) + } + ir := module.String() + module.Dispose() + if strings.Contains(ir, "declare ") { + t.Fatal("ordinary direct-module source acquired textual prelude declarations") + } + } +} diff --git a/amd64_private_aligned_frame_audit_test.go b/amd64_private_aligned_frame_audit_test.go new file mode 100644 index 000000000..8c1fe0527 --- /dev/null +++ b/amd64_private_aligned_frame_audit_test.go @@ -0,0 +1,291 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// These are source/translation observations, not execution tests. In +// particular, neither missing initialization nor a missing SP restoration +// fixture is executed. A typed ABI0 declaration is not a private-frame proof. +func x86PrivateAlignedFrameAuditFixture(t *testing.T, operation string) (string, FuncSig, GoPackage) { + t.Helper() + const goSource = `package frameaudit +func Value(a, b, c, d uint64) uint64 +` + return x86PrivateAlignedFrameAuditTypedFixture(t, operation, goSource) +} + +func x86PrivateAlignedFrameAuditTypedFixture(t *testing.T, operation, goSource string) (string, FuncSig, GoPackage) { + t.Helper() + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "frameaudit.go", goSource, 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + typed, err := (&types.Config{Sizes: sizes}).Check("frameaudit", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + declaration, ok := typed.Scope().Lookup("Value").(*types.Func) + if !ok { + t.Fatal("missing actual type-checked first-party declaration") + } + sig, err := DeriveGoFuncABI0Signature("frameaudit.Value", declaration, "amd64", sizes) + if err != nil { + t.Fatal(err) + } + source := fmt.Sprintf(`TEXT ·Value(SB),4,$160-40 +MOVQ a+0(FP),AX +MOVQ b+8(FP),CX +MOVQ c+16(FP),DX +MOVQ d+24(FP),BX +MOVQ SP,R11 +MOVQ $31,R9 +NOTQ R9 +ANDQ R9,SP +ADDQ $32,SP +MOVQ R11,96(SP) +MOVQ AX,16(SP) +MOVQ CX,24(SP) +MOVQ DX,48(SP) +MOVQ BX,56(SP) +MOVOU 48(SP),X0 +%s +MOVQ X0,AX +MOVQ 96(SP),R11 +MOVQ R11,SP +MOVQ AX,ret+32(FP) +RET +`, operation) + return source, sig, GoPackage{Path: "frameaudit", Types: typed, Syntax: []*ast.File{syntax}} +} + +func x86PrivateAlignedFrameAuditAPIs(t *testing.T, source string, sig FuncSig, pkg GoPackage, check func(*testing.T, error)) { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { + return strings.Replace(symbol, "·", "frameaudit.", 1) + } + sigs := map[string]FuncSig{sig.Name: sig} + if declaration, ok := pkg.Types.Scope().Lookup("callee").(*types.Func); ok { + callee, err := DeriveGoFuncABI0Signature("frameaudit.callee", declaration, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[callee.Name] = callee + } + opt := Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: resolve, Sigs: sigs, + } + t.Run("Translate", func(t *testing.T) { + _, err := Translate(file, opt) + check(t, err) + }) + t.Run("TranslateModuleInContext", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + check(t, err) + }) + t.Run("TranslateGoModule", func(t *testing.T) { + translation, err := TranslateGoModule(pkg, []byte(source), GoModuleOptions{ + GOOS: "linux", GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: resolve, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + check(t, err) + }) +} + +func TestX86PrivateAlignedFrameAuditOriginalGoM128Source(t *testing.T) { + // Full legacy opcode membership of the bridges inspected in this audit. + // The count instructions have their own exact-opcode fault contract; this + // list does not impose an alignment requirement on them. + operations := []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", + "ADDPS 16(SP),X0", "ADDPD 16(SP),X0", "SUBPS 16(SP),X0", "SUBPD 16(SP),X0", + "MULPS 16(SP),X0", "MULPD 16(SP),X0", "DIVPS 16(SP),X0", "DIVPD 16(SP),X0", + "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", + "PSLLW 16(SP),X0", "PSLLL 16(SP),X0", "PSLLQ 16(SP),X0", + "PSRLW 16(SP),X0", "PSRLL 16(SP),X0", "PSRLQ 16(SP),X0", + } + if len(operations) != 21 { + t.Fatal("incomplete audited legacy bridge membership") + } + for _, operation := range operations { + t.Run(strings.Fields(operation)[0], func(t *testing.T) { + source, _, _ := x86PrivateAlignedFrameAuditFixture(t, operation) + listing := string(x86InsertExtractGoListing(t, "amd64", source)) + for _, required := range []string{ + "locals=0xa8", + "MOVQ\ta+176(FP), AX", "MOVQ\tb+184(FP), CX", + "MOVQ\tc+192(FP), DX", "MOVQ\td+200(FP), BX", "MOVQ\tAX, ret+208(FP)", + "ANDQ\tR9, SP", "ADDQ\t$32, SP", "MOVQ\tR11, SP", + "RET", + } { + if !strings.Contains(listing, required) { + t.Fatalf("actual Go source frame/prologue/restoration missing %q:\n%s", required, listing) + } + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + if len(code) == 0 { + t.Fatal("original Go listing lacks physical instruction bytes") + } + // Identify complete observed machine sequences, never a version + // string or autosize alone. Older Go saves BP in the 160(SP) + // slot; current Go pushes BP. Both reserve 168 bytes, but their + // producer and restoration instructions are not interchangeable. + physicalForm := "" + for _, form := range []struct { + name string + prefix, suffix []byte + instructions []string + }{ + { + name: "pushed_BP", + prefix: []byte{0x55, 0x48, 0x89, 0xe5, 0x48, 0x81, 0xec, 0xa0, 0, 0, 0}, + suffix: []byte{0x48, 0x81, 0xc4, 0xa0, 0, 0, 0, 0x5d, 0xc3}, + instructions: []string{ + "PUSHQ\tBP", "MOVQ\tSP, BP", "SUBQ\t$160, SP", "ADDQ\t$160, SP", "POPQ\tBP", + }, + }, + { + name: "slot_saved_BP", + prefix: []byte{0x48, 0x81, 0xec, 0xa8, 0, 0, 0, 0x48, 0x89, 0xac, 0x24, 0xa0, 0, 0, 0, 0x48, 0x8d, 0xac, 0x24, 0xa0, 0, 0, 0}, + suffix: []byte{0x48, 0x8b, 0xac, 0x24, 0xa0, 0, 0, 0, 0x48, 0x81, 0xc4, 0xa8, 0, 0, 0, 0xc3}, + instructions: []string{ + "SUBQ\t$168, SP", "MOVQ\tBP, 160(SP)", "LEAQ\t160(SP), BP", "MOVQ\t160(SP), BP", "ADDQ\t$168, SP", + }, + }, + } { + if !bytes.HasPrefix(code, form.prefix) || !bytes.HasSuffix(code, form.suffix) { + continue + } + for _, instruction := range form.instructions { + if !strings.Contains(listing, instruction) { + t.Fatalf("actual %s object lacks its paired listing instruction %q", form.name, instruction) + } + } + physicalForm = form.name + } + if physicalForm == "" { + t.Fatalf("actual Go physical prologue/epilogue lacks an observed complete pairing: %x", code) + } + t.Logf("actual Go original accepted; TEXT160 + savedBP8; physicalForm=%s operation=%s sourceSHA=%x codeSHA=%x code=%x", physicalForm, operation, sha256.Sum256([]byte(source)), sha256.Sum256(code), code) + if strings.HasPrefix(operation, "PADDB ") { + t.Logf("independent original Go frame/slot listing:\n%s", listing) + } + }) + } +} + +func TestX86PrivateAlignedFrameAuditBoundedSourceStillNeedsContract(t *testing.T) { + for _, operation := range []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", + } { + t.Run(strings.Fields(operation)[0], func(t *testing.T) { + source, sig, pkg := x86PrivateAlignedFrameAuditFixture(t, operation) + x86PrivateAlignedFrameAuditAPIs(t, source, sig, pkg, func(t *testing.T, err error) { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("typed scalar declaration alone granted native private-frame memory: %v", err) + } + t.Logf("bounded initialized source remains Context pending its private-frame producer: %v", err) + }) + }) + } +} + +// RED checkpoint: the legacy arithmetic bridge currently admits these +// unproved virtual-SP inputs. This is a missing source contract observation, +// not a claim that any third-party function has been executed incorrectly. +func TestX86PrivateAlignedFrameAuditUnprovedNativeFloatingMemory(t *testing.T) { + mutations := []struct { + name string + edit func(string) string + goSource string + }{ + {"unaligned source residue", func(source string) string { + return strings.Replace(source, "ADDQ $32,SP", "ADDQ $33,SP", 1) + }, ""}, + {"uninitialized source bytes", func(source string) string { + return strings.Replace(source, "MOVQ CX,24(SP)\n", "", 1) + }, ""}, + {"source TEXT extent", func(source string) string { + return strings.Replace(source, "16(SP),X0", "152(SP),X0", 1) + }, ""}, + {"missing source SP restoration", func(source string) string { + return strings.Replace(source, "MOVQ R11,SP\n", "", 1) + }, ""}, + {"unknown alignment mask", func(source string) string { + return strings.Replace(source, "MOVQ $31,R9\nNOTQ R9", "MOVQ AX,R9", 1) + }, ""}, + {"private pointer result escape", func(source string) string { + return strings.Replace(source, "MOVQ AX,ret+32(FP)", "MOVQ R11,ret+32(FP)", 1) + }, ""}, + {"overwritten saved SP", func(source string) string { + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ $0,96(SP)", 1) + }, ""}, + {"inconsistent restore join", func(source string) string { + return strings.Replace(source, "MOVQ R11,SP", "CMPQ AX,$0\nJE afterRestore\nMOVQ R11,SP\nafterRestore:", 1) + }, ""}, + {"private pointer global escape", func(source string) string { + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ R11,96(SP)\nMOVQ SP,framePointer(SB)", 1) + "GLOBL framePointer(SB),8,$8\n" + }, ""}, + {"unknown incoming pointer alias", func(source string) string { + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ R11,96(SP)\nMOVQ SP,(BX)", 1) + }, "package frameaudit\nfunc Value(a, b, c uint64, d *uint64) uint64\n"}, + {"unproved call", func(source string) string { + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ R11,96(SP)\nCALL ·callee(SB)", 1) + }, "package frameaudit\nfunc Value(a, b, c, d uint64) uint64\nfunc callee()\n"}, + {"raw SP overwrite", func(source string) string { + // MOVQ AX,SP is encoded explicitly, not described by a forged + // decoder marker or an assumed neutral raw effect. + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ R11,96(SP)\nBYTE $0x48\nBYTE $0x89\nBYTE $0xc4", 1) + }, ""}, + } + for _, op := range []string{"ADDPS", "ADDPD", "SUBPS", "SUBPD", "MULPS", "MULPD", "DIVPS", "DIVPD"} { + for _, mutation := range mutations { + t.Run(op+"/"+mutation.name, func(t *testing.T) { + source, sig, pkg := x86PrivateAlignedFrameAuditFixture(t, op+" 16(SP),X0") + if mutation.goSource != "" { + source, sig, pkg = x86PrivateAlignedFrameAuditTypedFixture(t, op+" 16(SP),X0", mutation.goSource) + } + source = mutation.edit(source) + // Object acceptance cannot authorize execution of these negative + // fixtures or stand in for source-SP/result/no-escape semantics. + x86InsertExtractGoListing(t, "amd64", source) + x86PrivateAlignedFrameAuditAPIs(t, source, sig, pkg, func(t *testing.T, err error) { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source private-frame backing admitted by native bridge: %v", err) + } + }) + }) + } + } +} diff --git a/amd64_private_legacy_read_family_test.go b/amd64_private_legacy_read_family_test.go new file mode 100644 index 000000000..5f8a92a8b --- /dev/null +++ b/amd64_private_legacy_read_family_test.go @@ -0,0 +1,327 @@ +package plan9asm + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +// Independent yxmov whole128 directory; MOVOA is cmd/asm's MOVO alias. +// The complete original opcode/prefix/zcase oracle is separately retained by +// TestX86GoCallerLegacyMoveCompleteGoRows, not derived from production's map. +var x86PrivateLegacyReadNames = []string{ + "MOVOU", "MOVOA", "MOVO", "MOVUPS", "MOVAPS", "MOVUPD", "MOVAPD", +} + +func x86PrivateLegacyReadOriginal(t *testing.T, body string) (*File, FuncSig, Options) { + t.Helper() + assembly := "TEXT ·kernel(SB),4,$0-32\nMOVQ a+0(FP),DI\n" + body + "\nBYTE $0xc3\n" + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + sig := sigs["caller.kernel"] + if sig.Ret != Void || len(sig.Frame.Results) != 0 || len(sig.Args) != 4 || + sig.x86CallerRegions == nil || !x86CallerRegionSourceMatches(sig, file.Funcs[0], "amd64") { + t.Fatal("actual source and same pointer-only Void signature prerequisite is missing") + } + for _, argument := range sig.Args { + if argument != Ptr { + t.Fatal("fixture changed the existing four-pointer caller domain") + } + } + return file, sig, Options{Goarch: "amd64", ResolveSym: resolve, Sigs: sigs} +} + +func x86PrivateLegacyReadProved(t *testing.T, body string, width int64) (Func, FuncSig) { + t.Helper() + file, sig, options := x86PrivateLegacyReadOriginal(t, body) + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + if len(normalized.Funcs) != 1 || !x86CallerNormalizedSourceMatches(normalized.Funcs[0], sig) { + t.Fatal("whole normalized source does not match its exact caller signature") + } + fn := normalized.Funcs[0] + found := 0 + for ordinal, ins := range fn.Instrs { + // Native pointer imports have their own exact FP/physical-stack + // certificate. Select the actual DI-to-X numerical source role first. + if len(ins.Args) != 2 || ins.Args[0].Kind != OpMem || ins.Args[0].Mem.Base != DI || + ins.Args[1].Kind != OpReg || !isAMD64XReg(ins.Args[1].Reg) { + continue + } + access, present := ins.x86CallerMemory[0] + if !present || access.write || access.width != width || !x86CallerMemoryCertified(ins, 0) { + t.Fatalf("instruction %d lacks its exact numerical-source read%d certificate", ordinal, width) + } + found++ + } + if found != 1 { + t.Fatalf("source has %d certified reads, want exactly one", found) + } + return fn, sig +} + +func TestX86PrivateLegacyReadObserverCompleteSourceForms(t *testing.T) { + // Input is the existing numerical slice-origin/nonalias contract, not an + // allocation or native-pointer extent. The independent producer must pass + // before these leaves can diagnose the private observer. + for _, op := range x86PrivateLegacyReadNames { + for _, destination := range []string{"X1", "X0"} { + t.Run(op+"/register_"+destination, func(t *testing.T) { + body := "MOVOU (DI),X0\n" + op + " X0," + destination + "\nPEXTRD $3," + destination + ",AX" + fn, sig := x86PrivateLegacyReadProved(t, body, 16) + if err := CheckX86ABI0ResultInitialization(fn, sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("public declaration-only Void query acquired caller memory: %v", err) + } + if err := checkX86ScalarFPResultInitialization(fn, sig, "amd64"); err != nil { + t.Errorf("private observer lost a same-source exact read16 before %s register transfer: %v", op, err) + } + }) + } + } + for _, op := range []string{"MOVOU", "MOVUPS", "MOVUPD"} { + for _, address := range []string{"(DI)", "4(DI)", "4(DI)(BX*2)"} { + t.Run(op+"/read_"+address, func(t *testing.T) { + body := "MOVQ $1,BX\n" + op + " " + address + ",X0\nPEXTRD $3,X0,AX" + fn, sig := x86PrivateLegacyReadProved(t, body, 16) + if err := checkX86ScalarFPResultInitialization(fn, sig, "amd64"); err != nil { + t.Errorf("private observer discarded a proved full-width source read: %v", err) + } + }) + } + } +} + +func TestX86PrivateLegacyReadObserverExistingByteCounterparts(t *testing.T) { + // These are ordinary logical byte-effect models, never caller producers. + // In particular, logical FP/AUTO bytes do not prove native alignment/backing. + forms := []string{ + "X0,X1", "X0,X0", "a+0(FP),X1", "X0,ret0+16(FP)", + "value-32(SP),X1", "X0,value-16(SP)", + } + for _, arch := range []string{"amd64", "386"} { + for _, op := range x86PrivateLegacyReadNames { + for _, operands := range forms { + t.Run(arch+"/"+op+"/"+operands, func(t *testing.T) { + ins := x86CallerLegacyMoveInstruction(t, op+" "+operands) + effect, known, err := x86TypedLegacyPackedMoveByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 2 || effect.flagsRead != 0 || + effect.flagsDefined != 0 || effect.flagsKilled != 0 || effect.dynamicMemory { + t.Fatalf("whole128 descriptor differs from the independent move rows: %+v/%v", effect, err) + } + for index, operand := range effect.operands { + if operand.index != index || operand.width != 16 || operand.read != (index == 0) || + operand.write != (index == 1) || !reflect.DeepEqual(operand.operand, ins.Args[index]) { + t.Fatal("typed effect changed source/destination, direction or exact width") + } + } + owned := make([]bool, 32) + state := x86ResultState{fp: make([]bool, 32), auto: make([]bool, 32), flags: x86ByteFlagCF | x86ByteFlagOF} + for offset := range owned { + owned[offset] = true + state.fp[offset], state.auto[offset] = offset < 16, offset < 16 + } + state.xmm[0] = 0xffff + for _, transfer := range []func(Instr, *x86ResultState, []bool, int64, string) bool{x86ResultTransfer, x86ScalarFPResultTransfer} { + copy := state.clone() + if !transfer(ins, ©, owned, 16, arch) || copy.flags != state.flags { + t.Fatal("existing full-byte transition failed or changed numerical flags") + } + if !x86ResultRead(ins.Args[1], 16, ©, owned, arch) { + t.Fatal("complete move failed to define exactly its observed destination") + } + } + }) + } + for missing := 0; missing < 16; missing++ { + t.Run(fmt.Sprintf("%s/%s/missing-byte%d", arch, op, missing), func(t *testing.T) { + ins := x86CallerLegacyMoveInstruction(t, op+" X0,X1") + state := x86ResultState{flags: x86ByteFlagCF | x86ByteFlagOF} + state.xmm[0], state.xmm[1] = 0xffff&^(uint16(1)<= 0 { + source := ins.Args[form.memory] + if source.Kind == OpMem { + if !x86ConditionalMoveOffsetKnown(source.Mem) || source.Mem.IndexExt != "" || + source.Mem.Segment != "" && source.Mem.Segment != FS && source.Mem.Segment != GS { + err = fmt.Errorf("qword lookup source has an unresolved or foreign address") + } else { + err = x86SymbolIndexedMemoryError(source.Mem, arch) + } + } + if err == nil && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: qword lookup needs its original default address mode", ErrProbeNeedsContext) + } + } + return form, true, err +} + +func x86TypedQwordLookupByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86QwordLookupProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width, firstWidth := int64(form.byteWidth), int64(form.byteWidth) + if form.memory >= 0 { + firstWidth = int64(form.memoryBytes) + } + outputWidth := width + if form.spec.output == amd64QwordBitLookupMask { + outputWidth = 8 + } + effect := x86TypedByteEffect{requiresResultProof: true, operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: firstWidth, read: true}, + {index: form.second, operand: ins.Args[form.second], width: width, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: outputWidth, + read: form.spec.output == amd64QwordBitLookupBytes && form.mask >= 0 && !form.zeroing, write: true}, + }} + if form.mask >= 0 { + effect.operands = append(effect.operands, + x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + return effect, true, nil +} diff --git a/amd64_qword_lookup_forms_test.go b/amd64_qword_lookup_forms_test.go new file mode 100644 index 000000000..d04f375e7 --- /dev/null +++ b/amd64_qword_lookup_forms_test.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86QwordLookupCompleteIndependentGoRows(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + counts := map[string]int{} + for _, rule := range rules { + if rule.Opcode != "VPMULTISHIFTQB" && rule.Opcode != "VPSHUFBITQMB" { + continue + } + index := counts[rule.Opcode] + counts[rule.Opcode]++ + classes := []string{"YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", + "YymEvex,YyrEvex,YyrEvex", "YymEvex,YyrEvex,Yknot0,YyrEvex", "Yzm,Yzr,Yzr", "Yzm,Yzr,Yknot0,Yzr"} + ytab, opcode, w := "_yvblendmpd", "0x83", 1 + if rule.Opcode == "VPSHUFBITQMB" { + ytab, opcode, w = "_yvpshufbitqmb", "0x8F", 0 + classes = []string{"YxmEvex,YxrEvex,Yk", "YxmEvex,YxrEvex,Yknot0,Yk", + "YymEvex,YyrEvex,Yk", "YymEvex,YyrEvex,Yknot0,Yk", "Yzm,Yzr,Yk", "Yzm,Yzr,Yknot0,Yk"} + } + var bytes []string + for _, width := range []int{128, 256, 512} { + attributes := fmt.Sprintf("evexN%d", width/8) + if rule.Opcode == "VPMULTISHIFTQB" { + attributes += " | evexBcstN8 | evexZeroingEnabled" + } + bytes = append(bytes, fmt.Sprintf("avxEscape | evex%d | evex66 | evex0F38 | evexW%d", width, w), attributes, opcode) + } + zcase, offset := "Zevex_rm_v_r", 0 + if index%2 != 0 { + zcase, offset = "Zevex_rm_v_k_r", 3 + } + if index >= 6 || rule.RowIndex != index || rule.Ytab != ytab || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != offset || rule.EncodingOffset != index/2*3 || + strings.Join(rule.Classes, ",") != classes[index] || !reflect.DeepEqual(rule.OpBytes, bytes) { + t.Fatal("independent complete qword lookup encoder row/attributes changed", rule) + } + } + if len(counts) != 2 || counts["VPMULTISHIFTQB"] != 6 || counts["VPSHUFBITQMB"] != 6 { + t.Fatal("complete qword lookup row inventory changed", counts) + } +} + +func TestX86QwordLookupEveryInputByteAndIndependentState(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, name := range []string{"VPMULTISHIFTQB", "VPSHUFBITQMB"} { + for _, bank := range []string{"X", "Y", "Z"} { + destination := bank + "2" + if name == "VPSHUFBITQMB" { + destination = "K3" + } + ins := x86ScalarLaneInstruction(t, name+" "+bank+"0,"+bank+"1,"+destination) + width := amd64VectorByteWidth(ins.Args[0].Reg) + for input := 0; input < 2; input++ { + for missing := -1; missing < width; missing++ { + state := x86ResultState{flags: x86ByteFlagsAll, direction: x86ResultDirectionEither} + for register := 0; register < 3; register++ { + state.writeVector(register, 64, ^uint64(0), false) + } + state.pointerGP[6], state.gp[3], state.masks[1] = true, 0x55, true + if missing >= 0 { + state.writeVector(input, 64, state.vectorBytes(input)&^(uint64(1)< 0 && isX86RawDirective(fn.Instrs[start-1]) { + start-- + } + return start > 0 && start < len(fn.Instrs) && fn.Instrs[start-1].Op == OpRET +} + +func x86RawTailLayoutObserved(file *File, symbol string) bool { + target := strings.TrimSuffix(symbol, "<>") + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + for _, operand := range ins.Args { + if fn.Sym == symbol && operand.Kind == OpMem && operand.Mem.Base == PC { + return true + } + + ref := "" + switch operand.Kind { + case OpSym: + ref = operand.Sym + case OpMem: + ref = operand.Mem.Sym + } + if ref == "" { + continue + } + ref = strings.TrimSpace(strings.TrimPrefix(ref, "*")) + base, offset, ok := parseSBRef(ref) + if !ok || strings.TrimSuffix(strings.TrimPrefix(base, "$"), "<>") != target { + continue + } + if offset != 0 || !isX86DirectControlTransfer(ins.Op) { + return true + } + } + } + } + for _, datum := range file.Data { + base, _, ok := parseSBRef(datum.Addr) + if ok && strings.TrimSuffix(base, "<>") == target { + return true + } + } + return false +} + +func isX86DirectControlTransfer(op Op) bool { + return op == OpCALL || op == OpJMP || op == OpRET || isAMD64ConditionalBranch(op) +} + +// NormalizeRawFileForTranslation performs the whole-file raw-byte analysis +// before a compile-only caller partitions functions into bounded modules. +// Address-sensitive TEXT bodies must be identified while all references are +// still visible. TranslateModuleInContext may safely normalize the result +// again after partitioning. This preparation API has no function signatures: +// final typed FP return-contract validation runs when translation consumes its +// existing Options.Sigs. Preparation does not prove a callable return ABI. +func NormalizeRawFileForTranslation(file *File, goarch string) (*File, error) { + return normalizeX86RawFile(file, goarch) +} + +// preserveAddressSensitiveX86RawText identifies raw-only TEXT bodies whose +// exact byte layout is observed elsewhere in the same source file. Typical +// examples are boot trampolines patched through symbol+offset references and +// TEXT-backed descriptor/data tables. Decoding such a body into LLVM +// operations would make the referenced byte offsets false even when the +// instruction semantics happened to match. +// +// Raw-only bodies that are merely called or jumped to are deliberately not +// selected: those remain on the normal semantic decoder path. +func preserveAddressSensitiveX86RawText(file *File) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + candidates := make(map[string]int) + encoded := make(map[int][]byte) + alignments := make(map[int]int64) + for index := range file.Funcs { + fn := &file.Funcs[index] + candidateName := strings.TrimSuffix(fn.Sym, "<>") + if fn.X86RawText != nil { + candidates[candidateName] = index + encoded[index] = append([]byte(nil), fn.X86RawText...) + alignments[index] = fn.X86RawAlign + continue + } + var code []byte + hasRaw := false + rawOnly := true + var alignment int64 + for _, ins := range fn.Instrs { + if ins.Op == OpTEXT { + continue + } + if form, known, err := x86DirectiveProperties(ins); known && form.effect == x86DirectiveLayout && !hasRaw { + if err != nil { + rawOnly = false + break + } + if form.alignment > alignment { + alignment = form.alignment + } + continue + } + if !isX86RawDirective(ins) { + rawOnly = false + break + } + part, err := x86RawDirectiveBytes(ins) + if err != nil { + return fmt.Errorf("%s: %w", fn.Sym, err) + } + code = append(code, part...) + hasRaw = true + } + if !rawOnly || !hasRaw { + continue + } + candidates[candidateName] = index + encoded[index] = code + alignments[index] = alignment + } + if len(candidates) == 0 { + return nil + } + + addressed := make(map[int]bool) + markSymbol := func(symbol string) { + base, _, ok := parseSBRef(strings.TrimSpace(strings.TrimPrefix(symbol, "*"))) + if !ok { + return + } + base = strings.TrimSuffix(base, "<>") + if index, exists := candidates[base]; exists { + addressed[index] = true + } + } + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + op := Op(strings.ToUpper(string(ins.Op))) + if op == OpCALL || op == OpJMP || op == OpRET || isAMD64ConditionalBranch(op) { + continue + } + for _, operand := range ins.Args { + switch operand.Kind { + case OpSym: + markSymbol(operand.Sym) + case OpMem: + if operand.Mem.Sym != "" { + markSymbol(operand.Mem.Sym) + } + } + } + } + } + for index := range addressed { + file.Funcs[index].X86RawText = append([]byte(nil), encoded[index]...) + file.Funcs[index].X86RawAlign = alignments[index] + } + return nil +} + +// decodeX86RawDirectives replaces consecutive BYTE/WORD/LONG/QUAD directives +// with the Plan 9 instructions represented by those bytes. The decoder and +// syntax printer come from Go's architecture tables; the resulting instruction +// still has to pass the ordinary plan9asm operand-form and semantic lowerers. +// This is deliberately not an opaque inline-assembly escape hatch. +func decodeX86RawDirectives(fn Func, goarch string) (Func, error) { + mode := 64 + if goarch == "386" { + mode = 32 + } else if goarch != "" && goarch != "amd64" { + return fn, fmt.Errorf("raw x86 directives require GOARCH 386 or amd64, got %q", goarch) + } + + knownLabels := make(map[string]bool) + for _, ins := range fn.Instrs { + if ins.Op == OpLABEL && len(ins.Args) == 1 && ins.Args[0].Kind == OpLabel { + knownLabels[ins.Args[0].Sym] = true + } + } + + var normalized []Instr + for i := 0; i < len(fn.Instrs); { + if !isX86RawDirective(fn.Instrs[i]) { + normalized = append(normalized, fn.Instrs[i]) + i++ + continue + } + + start := i + var code []byte + var rawLines []string + opaque := make(map[int]x86RawDecodedInstruction) + for i < len(fn.Instrs) { + if isX86RawDirective(fn.Instrs[i]) { + encoded, err := x86RawDirectiveBytes(fn.Instrs[i]) + if err != nil { + return fn, err + } + code = append(code, encoded...) + rawLines = append(rawLines, fn.Instrs[i].Raw) + i++ + continue + } + // Keep a run of named SSE constant reads in the same physical + // layout only when raw bytes resume after it. Go's encoder lengths + // are checked against the official assembler in tests. + end := i + for end < len(fn.Instrs) { + if _, ok := x86RawMixedSSESymbolReadSize(fn.Instrs[end]); !ok { + break + } + end++ + } + if end == i || end == len(fn.Instrs) || !isX86RawDirective(fn.Instrs[end]) { + break + } + for i < end { + ins := fn.Instrs[i] + length, _ := x86RawMixedSSESymbolReadSize(ins) + opaque[len(code)] = x86RawDecodedInstruction{length: length, instrs: []Instr{ins}} + code = append(code, make([]byte, length)...) + rawLines = append(rawLines, ins.Raw) + i++ + } + } + // Go appends BYTE/WORD/LONG/QUAD after a final RET to the previous + // TEXT, even when GLOBL declarations intervene. Without a following + // source label, the tail has no control-flow entry and is not an + // executable instruction stream. Generated tables can therefore + // contain arbitrary bytes, including apparent branches. Keep labeled + // tails on the normal decoder path so reachable code never disappears. + if i == len(fn.Instrs) && start > 0 && fn.Instrs[start-1].Op == OpRET { + continue + } + rawGroup := strings.Join(rawLines, "; ") + if mode == 64 { + if target, ok := x86RawRetpolineTarget(code); ok { + normalized = append(normalized, Instr{ + Op: OpJMP, + Args: []Operand{{Kind: OpReg, Reg: target}}, + Raw: fmt.Sprintf("JMP %s /* decoded retpoline from %s */", target, rawGroup), + }) + continue + } + } + decoded, err := decodeX86RawDirectiveGroupWithOpaque(code, mode, start, rawGroup, knownLabels, opaque) + if err != nil { + return fn, err + } + for index := range decoded { + scalarSegmentLoad := x86ScalarSegmentLoadRawIdentity(decoded[index], goarch) + if scalarSegmentLoad.arch != "" { + decoded[index].x86Encoded = true + } + decoded[index].x86SourceRaw = &x86SourceRawInstruction{ + op: decoded[index].Op, + args: append([]Operand(nil), decoded[index].Args...), + vectorBytes: decoded[index].x86VectorBytes, + addressBits: decoded[index].x86AddressBits, + packedConversion: x86PackedConversionRawIdentity(decoded[index], goarch), + packedMove: x86PackedMoveRawIdentity(decoded[index], goarch), + packedReduction: x86PackedReductionRawIdentity(decoded[index], goarch), + ternaryLogic: x86TernaryLogicRawIdentity(decoded[index], goarch), + legacyFloatCompare: x86LegacyFloatCompareRawIdentity(decoded[index], goarch), + comi: x86COMIRawIdentity(decoded[index], goarch), + floatMinMax: x86FloatMinMaxRawIdentity(decoded[index], goarch), + scalarPrecision: x86ScalarPrecisionRawIdentity(decoded[index], goarch), + scalarFloatInteger: x86ScalarFloatIntegerRawIdentity(decoded[index], goarch), + packedPrecision: x86PackedPrecisionRawIdentity(decoded[index], goarch), + namedHalf: x86NamedHalfRawIdentity(decoded[index], goarch), + fma3: x86FMA3RawIdentity(decoded[index], goarch), + explicitMaskMove: x86ExplicitMaskMoveRawIdentity(decoded[index], goarch), + packedShift: x86PackedShiftRawIdentity(decoded[index], goarch), + packedAbs: x86PackedAbsRawIdentity(decoded[index], goarch), + packedDot: x86PackedDotRawIdentity(decoded[index], goarch), + permute128: x86Permute128RawIdentity(decoded[index], goarch), + zeroExtend: x86ZeroExtendRawIdentity(decoded[index], goarch), + scalarSegmentLoad: scalarSegmentLoad, + } + } + normalized = append(normalized, decoded...) + } + fn.Instrs = normalized + return fn, nil +} + +type x86RawDecodedInstruction struct { + length int + instrs []Instr +} + +// decodeX86RawDirectiveGroup follows every reachable path through one raw +// directive group. Internal relative JMP/Jcc targets become synthetic source +// labels, so backward edges and both short and near conditional branches keep +// their control-flow semantics. Bytes skipped by all reachable paths remain +// uninterpreted, which permits the embedded signatures and constant data used +// by generated assembly. Any edge outside the group or into an instruction is +// still rejected rather than being guessed from final linked layout. +func decodeX86RawDirectiveGroup(code []byte, mode, start int, rawGroup string, knownLabels map[string]bool) ([]Instr, error) { + return decodeX86RawDirectiveGroupWithOpaque(code, mode, start, rawGroup, knownLabels, nil) +} + +func decodeX86RawDirectiveGroupWithOpaque( + code []byte, mode, start int, rawGroup string, + knownLabels map[string]bool, opaque map[int]x86RawDecodedInstruction, +) ([]Instr, error) { + decodedByOffset := make(map[int]x86RawDecodedInstruction) + labels := make(map[int]string) + addressTargets := make(map[int]int) + var literalRanges []x86RawLiteralRange + recordLiteral := func(offset int, literal x86RawLiteralRange) { + literal.source = offset + literalRanges = append(literalRanges, literal) + } + owners := make([]int, len(code)) + for i := range owners { + owners[i] = -1 + } + for offset, named := range opaque { + if named.length <= 0 || offset < 0 || offset+named.length > len(code) { + return nil, fmt.Errorf("invalid named x86 source span at byte %d: %q", offset, rawGroup) + } + for current := offset; current < offset+named.length; current++ { + if owners[current] != -1 { + return nil, fmt.Errorf("overlapping named x86 source spans at byte %d: %q", current, rawGroup) + } + owners[current] = offset + } + } + queue := []int{0} + queued := map[int]bool{0: true} + + labelFor := func(target int) string { + if label := labels[target]; label != "" { + return label + } + base := fmt.Sprintf("raw_jcc_%d_%d", start, target) + label := base + for suffix := 2; knownLabels[label]; suffix++ { + label = fmt.Sprintf("%s_%d", base, suffix) + } + knownLabels[label] = true + labels[target] = label + return label + } + enqueue := func(target int) error { + if target < 0 || target > len(code) { + return fmt.Errorf("raw x86 PC-relative target byte %d at instruction %d is outside directive group: %q", target, start, rawGroup) + } + if target < len(code) && owners[target] >= 0 && owners[target] != target { + return fmt.Errorf("raw x86 PC-relative target byte %d at instruction %d is not an instruction boundary: %q", target, start, rawGroup) + } + labelFor(target) + if !queued[target] { + queued[target] = true + queue = append(queue, target) + } + return nil + } + markInstruction := func(offset, length int) error { + if length <= 0 || offset+length > len(code) { + return fmt.Errorf("raw x86 instruction at instruction %d byte %d has invalid length %d: %q", start, offset, length, rawGroup) + } + for current := offset; current < offset+length; current++ { + if owners[current] >= 0 && owners[current] != offset { + return fmt.Errorf("raw x86 instruction at instruction %d byte %d overlaps instruction at byte %d: %q", start, offset, owners[current], rawGroup) + } + } + for current := offset; current < offset+length; current++ { + owners[current] = offset + } + return nil + } + annotate := func(instrs []Instr) []Instr { + for i := range instrs { + instrs[i].Raw = fmt.Sprintf("%s /* decoded from %s */", instrs[i].Raw, rawGroup) + } + return instrs + } + + for len(queue) != 0 { + blockStart := queue[0] + queue = queue[1:] + if blockStart == len(code) { + continue + } + if owners[blockStart] >= 0 && owners[blockStart] != blockStart { + return nil, fmt.Errorf("raw x86 PC-relative target byte %d at instruction %d is not an instruction boundary: %q", blockStart, start, rawGroup) + } + for offset := blockStart; offset < len(code); { + if _, exists := decodedByOffset[offset]; exists { + break + } + if owners[offset] >= 0 && owners[offset] != offset { + return nil, fmt.Errorf("raw x86 reachable byte %d at instruction %d is inside instruction at byte %d: %q", offset, start, owners[offset], rawGroup) + } + if named, ok := opaque[offset]; ok { + decodedByOffset[offset] = named + offset += named.length + continue + } + if instruction, length, ok, err := decodedX86VectorZeroInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector zero at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawADX(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 ADX at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + if literal.last > literal.first { + recordLiteral(offset, literal) + } + offset += length + continue + } + if instruction, length, ok, err := decodedX86ExtendedPrefetchInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 extended prefetch at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86EnqueueInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 enqueue at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86WaitPackageInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 WAITPKG at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok := decodedX86EndBranchInstruction(code[offset:]); ok { + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok := decodedX86SystemManagementInstruction(code[offset:]); ok { + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + + if instruction, length, ok, err := decodedX86SegmentMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 segment-register move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if syntax, length, ok := decodedX86RandomInstruction(code[offset:], mode); ok { + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instrs, err := parseDecodedX86Instruction(syntax) + if err != nil { + return nil, fmt.Errorf("parse decoded raw x86 instruction %q at instruction %d byte %d: %w", syntax, start, offset, err) + } + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: annotate(instrs)} + offset += length + continue + } + if instruction, length, ok := decodedX86OrderingFenceInstruction(code[offset:], mode); ok { + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Op, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86CacheLineWritebackInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 cache-line writeback at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawBlockBroadcastRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX block broadcast literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawBlockBroadcastInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX block broadcast at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVBROADCAST128RIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VBROADCAST128 literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VBROADCAST128Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VBROADCAST128 at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVMOVIntegerLiteral(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VMOVD/VMOVQ literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VMOVQInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VMOVQ at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86MaskMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 KMOV at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86MaskVectorMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask/vector move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawEVEXPackedIntegerMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed integer move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedIntegerMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed integer move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXUnalignedLoadInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 unaligned load at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawGatherInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 gather at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawScatterInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scatter at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawQwordPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 qword permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawPackedSignInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed sign at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ExplicitMaskMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 explicit mask move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScalarMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawDuplicateMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 duplicate move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86DuplicateMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 duplicate move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawFloatLogicalRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 floating logical literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawEVEXFloatLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX floating logical instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXPackedFloatLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX packed floating logical instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86FloatingUnpackInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 floating unpack at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 in-lane floating permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawMaskBlendRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask blend literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawMaskBlendInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask blend at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVariableBlendRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 variable blend literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VariableBlendInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 variable blend at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawImmediatePackedBlendRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 immediate packed blend literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawCLMULRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 carryless multiply literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawCLMULInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 carryless multiply at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ImmediatePackedBlendInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 immediate packed blend at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawBinaryFloatRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 binary floating literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXBinaryFloatInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX binary floating instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedWidenConversionRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed widening conversion literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedWidenConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed widening conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarFloatToIntegerInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar float-to-integer instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarIntegerFloatInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar integer-to-float instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar precision conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawQQToFloatRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 QWORD-to-float literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawQQToFloatInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 QWORD-to-float conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedNumericConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw packed numeric conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86SameWidthConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 same-width packed conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedDoubleDwordInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed double/dword conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedPrecisionConvertInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed precision conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScalarHalfConversionRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar FP16 literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarHalfConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar FP16 conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawFP16ConversionRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 FP16 conversion literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedHalfConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed half conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawBF16RIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BF16 literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawBF16Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BF16 at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86MinimumPositionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 minimum-position instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + // F3/0F38/3A is K-data MW2D, not 66/0F38/3A VPMINUW. + // Match the complete prefix key before the older min/max matcher. + if instruction, length, ok, err := decodedX86EVEXMaskDataBroadcastInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask-data broadcast at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawMinMaxRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer min/max literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedIntegerMinMaxInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer min/max at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86SingleSourceNarrowInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 single-source narrowing at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVariableShiftRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 per-lane variable shift literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PerLaneVariableShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 per-lane shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawMaskCompareRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask compare literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawMaskCompareInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask compare at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedCompareRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer compare literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedIntegerCompareInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer compare at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVectorBitCountInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector bit count at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawScalarReciprocalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar reciprocal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVectorHighLowMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector high/low move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVectorHalfMemoryInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector half memory move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawKLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 K-register operation at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawMaskShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawPackedTestMaskInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed test mask at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawPackedSADInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed SAD at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScalarFlagCompareRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar flag compare literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarFlagCompareInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar flag compare at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXVectorTestInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX vector test at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXMoveMaskInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX move mask at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVectorFloatCompareRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector floating compare literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VectorFloatCompareInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector floating compare instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXHorizontalIntegerInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 horizontal integer at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawHorizontalFloatRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 horizontal floating literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXPackedDotInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX packed dot at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXHorizontalFloatInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX horizontal floating instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedLogicalRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed logical literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXPackedIntegerLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX packed integer logical instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed logical instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedMADDRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX packed multiply-add literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedMADDInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed multiply-add at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXPackedMADDInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX packed multiply-add instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawFMA3RIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 FMA3 literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXFMA3Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX FMA3 instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXTRACT128Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEXTRACT128 instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXNonTemporalMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX non-temporal move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86InsertPSInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 INSERTPS/VINSERTPS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedScalarExtractInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed scalar extract at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VPERMQInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPERMQ at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VINSERT128Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VINSERT128 at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedInsertInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed insert at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScalarFloatBroadcastLiteral(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar float broadcast literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarBroadcastInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar broadcast at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedBroadcastLiteral(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed broadcast literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed broadcast at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedAlignRightInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed align-right at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86Permute128Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 128-bit lane permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VSHUFPSInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VSHUFPS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedShuffleInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed shuffle at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86Shuffle128BitBlocksInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VSHUFF/VSHUFI block shuffle at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVPSHUFBRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPSHUFB literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VPSHUFBInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPSHUFB at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedExtendInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed extending move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedArithmeticRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer arithmetic literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer arithmetic at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVariableRotateRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 variable rotate literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVariableRotateInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 variable rotate at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedImmediateRotateInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed immediate rotate at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawImmediateThreeVectorRIPData( + code, offset, mode, 0x03, decodedX86VectorAlignInstruction, "vector align", + ); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector align literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VectorAlignInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VALIGN at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawImmediateThreeVectorRIPData( + code, offset, mode, 0x25, decodedX86TernaryLogicInstruction, "ternary logic", + ); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 ternary logic literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86TernaryLogicInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 ternary logic at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86FunnelShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 funnel shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedUniformShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed uniform shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawIndexedPermuteRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 indexed permute literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86IndexedPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 indexed permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedCompressInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed compress at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedExpandInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed expand at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86QwordLookupInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 qword lookup at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86IFMAInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 IFMA at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86GFNIInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 GFNI at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VPINSRInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPINSR at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXVPINSRInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX VPINSR at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawEVEXUnpackRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed unpack literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedUnpackInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed unpack at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VPUNPCKInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPUNPCK at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VNNIInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VNNI at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VariableDwordPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPERMD/VPERMPS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawEVEXVariableDwordPermuteRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX variable dword permute literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXVariableDwordPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX variable dword permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86SHAInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 SHA at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawBZHIInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BZHI at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86BMI2ShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BMI2 variable shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ParallelBitsInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 parallel bits at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86BLSInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BLS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ANDNInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 ANDN at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86MULXInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 MULX at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RORXInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 RORX at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86AESInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 AES at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86DwordToQwordMultiplyInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 dword-to-qword multiply at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedAbsRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPABS literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScaledRoundRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scaled round literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVEXRoundRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VROUND literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVectorByteShiftRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector byte shift literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVectorByteShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector byte shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawScaledRoundInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scaled round at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVEXRoundInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VROUND at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawPackedAbsInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPABS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VectorPackedSaturatingNarrowInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed saturating narrow at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedMultiplyRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed multiply literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedAverageInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed average at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedMultiplyInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed multiply at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + + if instruction, length, ok, err := decodedX86ScalarPortIOInstruction(code[offset:]); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar port I/O at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + + if instruction, length, literal, ok, err := decodeX86RawLegacySIMDMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 legacy SIMD move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + + inst, err := x86asm.Decode(code[offset:], mode) + if err != nil && mode == 64 { + if nop, ok := decodedX86REXWNOP(code[offset:]); ok { + inst, err = nop, nil + } + } + if err != nil || inst.Len <= 0 || inst.Op == 0 { + if err == nil { + if inst.Len <= 0 { + err = fmt.Errorf("decoder consumed zero bytes") + } else { + err = fmt.Errorf("incomplete or unknown instruction") + } + } + end := offset + 16 + if end > len(code) { + end = len(code) + } + return nil, fmt.Errorf("decode raw x86 directive group at instruction %d byte %d (%x): %w: %q", start, offset, code[offset:end], err, rawGroup) + } + if err := markInstruction(offset, inst.Len); err != nil { + return nil, err + } + syntax, err := decodedX86GoSyntax(inst, code[offset:offset+inst.Len]) + if err != nil { + return nil, fmt.Errorf("convert decoded raw x86 instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if inst.Op == x86asm.NOP { + // Prefix/encoding checks above still apply. A multi-byte NOP's + // ModRM/SIB fields encode its length, not a memory access or + // PC-relative reference. Keep that length for branch boundaries, + // but do not send inert fields to the physical memory handoff. + instruction := Instr{Op: "NOP", Raw: fmt.Sprintf("NOP /* decoded from %s */", rawGroup)} + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: []Instr{instruction}} + offset += inst.Len + continue + } + opName := strings.Fields(syntax)[0] + op := Op(strings.ToUpper(opName)) + if inst.PCRel != 0 { + if instruction, literal, ok, err := decodeX86RawLegacyScalarRIPData(code, offset, inst, syntax); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 legacy scalar literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += inst.Len + continue + } + if mode == 64 && inst.Op == x86asm.LEA { + mem, ok := inst.Args[1].(x86asm.Mem) + if ok && mem.Base == x86asm.RIP { + // x86asm exposes disp32 as an unsigned magnitude here. + // RIP-relative LEA always sign-extends that 32-bit field. + target := offset + inst.Len + int(int32(mem.Disp)) + if target < 0 || target >= len(code) { + return nil, fmt.Errorf("raw x86 local address target byte %d at instruction %d is outside directive group: %q", target, start, rawGroup) + } + instrs, err := parseDecodedX86PhysicalInstruction(inst, syntax) + if err != nil || len(instrs) != 1 || len(instrs[0].Args) != 2 { + return nil, fmt.Errorf("parse decoded raw x86 local address %q at instruction %d byte %d: %v", syntax, start, offset, err) + } + instrs[0].Args[0] = Operand{Kind: OpSym, Sym: "·__plan9asm_raw_address_pending(SB)"} + instrs[0].Raw = fmt.Sprintf("%s ·__plan9asm_raw_address_pending(SB), %s /* decoded from %s */", op, instrs[0].Args[1], rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: instrs} + addressTargets[offset] = target + offset += inst.Len + continue + } + } + target, ok := x86RawRelativeTarget(inst, offset) + isJump := op == OpJMP + isConditional := isAMD64ConditionalBranch(op) + if !ok || (!isJump && !isConditional) { + return nil, fmt.Errorf("raw x86 PC-relative instruction %q at instruction %d byte %d cannot be mapped safely to source labels: %q", syntax, start, offset, rawGroup) + } + if err := enqueue(target); err != nil { + return nil, fmt.Errorf("raw x86 branch at byte %d: %w", offset, err) + } + label := labelFor(target) + branch := Instr{Op: op, Args: []Operand{{Kind: OpIdent, Ident: label}}, Raw: fmt.Sprintf("%s %s /* decoded from %s */", op, label, rawGroup)} + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: []Instr{branch}} + if isConditional { + if err := enqueue(offset + inst.Len); err != nil { + return nil, fmt.Errorf("raw x86 conditional fallthrough at byte %d: %w", offset, err) + } + } + break + } + if inst.Op == x86asm.JMP { + return nil, fmt.Errorf("raw x86 indirect jump at instruction %d byte %d cannot be mapped safely: %q", start, offset, rawGroup) + } + instrs, err := parseDecodedX86PhysicalInstruction(inst, syntax) + if err != nil { + return nil, fmt.Errorf("parse decoded raw x86 instruction %q at instruction %d byte %d: %w", syntax, start, offset, err) + } + if inst.Op == x86asm.LEA && len(instrs) == 1 && len(instrs[0].Args) == 2 { + // Bare absolute SIB/no-base addresses print as numbers, but + // the Go parser treats those as symbols. Retain the actual + // typed decoder memory operand, not a fabricated SB name. + physical := normalizeDecodedX86LEASegments(normalizeDecodedX86MemoryDisplacements(inst)) + if memory, ok := physical.Args[1].(x86asm.Mem); ok && memory.Base == 0 && memory.Index == 0 && memory.Segment == 0 { + instrs[0].Args[0] = Operand{Kind: OpMem, Mem: MemRef{Off: memory.Disp}} + } + } + if inst.AddrSize != mode { + physicalMemory := false + for _, operand := range inst.Args { + if _, ok := operand.(x86asm.Mem); ok { + physicalMemory = true + } + } + switch inst.Op { + case x86asm.MASKMOVQ, x86asm.MASKMOVDQU, x86asm.MONITOR: + // These decoded forms address DI/AX implicitly. x/arch + // exposes only register (or no) operands, not a Mem. + physicalMemory = true + } + for i := range instrs { + if physicalMemory { + instrs[i].x86AddressBits = inst.AddrSize + // Generic Go syntax erases address-size prefixes. Only + // LEA's complete typed form consumes that width here; + // ordinary memory must not borrow a full-width address. + instrs[i].x86UnboundAddressOverride = inst.Op != x86asm.LEA + } + } + } + if inst.Op == x86asm.MOVSXD && inst.DataSize != 64 { + for i := range instrs { + instrs[i].x86RawMOVSXDNeedsCaller = true + } + } + if inst.Op == x86asm.CVTPI2PS || inst.Op == x86asm.CVTPI2PD { + for i := range instrs { + instrs[i].x86MMXSourceConversion = true + } + } + if inst.Op == x86asm.RET { + form := x86RawNearReturnEncoding(code[offset : offset+inst.Len]) + for i := range instrs { + if instrs[i].Op != OpRET { + continue + } + instrs[i].x86RawNearReturn = &form + if form.nativeWidth && form.cleanup == 0 { + // Keep the encoded semantics in private metadata. Removing the + // zero operand does not validate the containing source frame. + instrs[i].Args = nil + } + } + } + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: annotate(instrs)} + offset += inst.Len + if inst.Op == x86asm.RET { + break + } + } + } + + for _, literal := range literalRanges { + for offset := literal.first; offset < literal.last; offset++ { + if owners[offset] >= 0 { + return nil, fmt.Errorf("RIP-relative literal byte %d overlaps reachable instruction at byte %d: %q", offset, owners[offset], rawGroup) + } + } + } + if len(addressTargets) != 0 { + first := len(code) + for _, target := range addressTargets { + if target < first { + first = target + } + } + // Only a fully unreachable suffix after a decoded RET can be moved to + // a shared global. Addressing executable bytes or a separately folded + // literal would lose source-relative pointer identity. + retBeforeSuffix := false + for offset, decoded := range decodedByOffset { + if offset >= first { + continue + } + for _, ins := range decoded.instrs { + if ins.Op == OpRET { + retBeforeSuffix = true + } + } + } + if !retBeforeSuffix { + return nil, fmt.Errorf("raw x86 local address data at byte %d has no preceding RET: %q", first, rawGroup) + } + for offset := first; offset < len(code); offset++ { + if owners[offset] >= 0 { + return nil, fmt.Errorf("raw x86 local address byte %d overlaps reachable instruction: %q", offset, rawGroup) + } + } + for _, literal := range literalRanges { + if literal.last <= first { + continue + } + if literal.first < first { + return nil, fmt.Errorf("raw x86 local address overlaps the middle of a folded literal: %q", rawGroup) + } + decoded := decodedByOffset[literal.source] + for index := range decoded.instrs { + ins := &decoded.instrs[index] + for arg := range ins.Args { + if ins.Args[arg].Kind != OpSym || ins.Args[arg].Sym != "·__plan9asm_raw_literal_pending(SB)" { + continue + } + if err := rebindX86RawStaticRead(ins, arg, "·__plan9asm_raw_address_pending(SB)"); err != nil { + return nil, err + } + ins.Raw = strings.Replace(ins.Raw, "·__plan9asm_raw_literal_pending(SB)", ins.Args[arg].Sym, 1) + // An address escape may permit writes to this pool. Force a + // runtime load rather than specializing from its initial bytes. + ins.x86RIPLiteralData = nil + // Address escape forbids constant specialization, not a + // read of this exact allocated static object. Retain the + // physical read range and bind the shared whole suffix. + ins.x86RIPMemoryRead.offset = int64(literal.first - first) + ins.x86RIPMemoryRead.object = append([]byte(nil), code[first:]...) + ins.x86RIPAddressData = code[first:] + ins.x86RIPAddressOff = literal.first - first + ins.x86RIPAddressGroup = start + } + } + decodedByOffset[literal.source] = decoded + } + for offset, target := range addressTargets { + decoded := decodedByOffset[offset] + decoded.instrs[0].x86RIPLiteral = true + decoded.instrs[0].x86RIPAddressData = code[first:] + decoded.instrs[0].x86RIPAddressOff = target - first + decoded.instrs[0].x86RIPAddressGroup = start + decodedByOffset[offset] = decoded + } + } + + offsets := make([]int, 0, len(decodedByOffset)) + for offset := range decodedByOffset { + offsets = append(offsets, offset) + } + sort.Ints(offsets) + result := make([]Instr, 0, len(offsets)+len(labels)) + for _, offset := range offsets { + if label := labels[offset]; label != "" { + result = append(result, Instr{Op: OpLABEL, Args: []Operand{{Kind: OpLabel, Sym: label}}, Raw: label + ":"}) + } + result = append(result, decodedByOffset[offset].instrs...) + } + if label := labels[len(code)]; label != "" { + result = append(result, Instr{Op: OpLABEL, Args: []Operand{{Kind: OpLabel, Sym: label}}, Raw: label + ":"}) + } + return result, nil +} + +// decodedX86SegmentMoveInstruction recognizes the complete 8C/8E segment +// register move family represented by Go 1.27's ymovtab. Go spells these +// operations MOVW even though x86 disassemblers often print register forms as +// MOVL or MOVQ according to the encoded general-register destination width. +func decodedX86SegmentMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segmentOverride := Reg("") + addressOverride := false + rex := byte(0) + for i < len(code) { + switch code[i] { + case 0x64: + rex = 0 + segmentOverride = FS + i++ + case 0x65: + rex = 0 + segmentOverride = GS + i++ + case 0x66: + rex = 0 + i++ + case 0x67: + rex = 0 + addressOverride = true + i++ + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + continue + } + goto opcode + } + } + +opcode: + if i >= len(code) || (code[i] != 0x8c && code[i] != 0x8e) || len(code) < i+2 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRM := code[i+1] + segmentNumber := int(modRM>>3&7) + int(rex>>2&1)*8 + segmentNames := [...]Reg{ES, CS, SS, DS, FS, GS} + if segmentNumber >= len(segmentNames) { + return Instr{}, 0, true, fmt.Errorf("reserved segment-register encoding %d", segmentNumber) + } + bExt := int(rex & 1) + xExt := int(rex>>1) & 1 + other, consumed, decodeErr := decodedX86RMSource(code[i+1:], mode, bExt, xExt, segmentOverride, 1) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + segment := Operand{Kind: OpReg, Reg: segmentNames[segmentNumber]} + args := []Operand{segment, other} + if code[i] == 0x8e { + args[0], args[1] = args[1], args[0] + } + return Instr{ + Op: "MOVW", + Args: args, + Raw: fmt.Sprintf("MOVW %s, %s", args[0].String(), args[1].String()), + }, i + 1 + consumed, true, nil +} + +// decodedX86MaskMoveInstruction recognizes Go 1.27's complete VEX-encoded +// KMOVB/W/D/Q family. x/arch v0.14 does not decode opmask registers and can +// split these bytes into unrelated legacy instructions, so recover all four +// _ykmovb operand rows before consulting the generic decoder. +func decodedX86MaskMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86MaskMoveInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86MaskMoveInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + opcodeIndex := i + vexLength + if len(code) <= opcodeIndex || code[opcodeIndex] < 0x90 || code[opcodeIndex] > 0x93 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must be zero") + } + if vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be 1111") + } + + opcode := code[opcodeIndex] + pp := vexBits & 3 + width64 := vexLength == 3 && vexBits&0x80 != 0 + width := "" + if opcode == 0x90 || opcode == 0x91 { + switch { + case !width64 && pp == 1: + width = "B" + case !width64 && pp == 0: + width = "W" + case width64 && pp == 1: + width = "D" + case width64 && pp == 0: + width = "Q" + } + } else { + switch { + case !width64 && pp == 1: + width = "B" + case !width64 && pp == 0: + width = "W" + case !width64 && pp == 3: + width = "D" + case width64 && pp == 3: + width = "Q" + } + } + if width == "" { + return Instr{}, 0, true, fmt.Errorf("invalid VEX.W/pp combination") + } + + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + mod := modRM >> 6 + regNumber := int(modRM>>3&7) + rExt*8 + rmNumber := int(modRM&7) + bExt*8 + kReg := func(number int) (Operand, error) { + if number < 0 || number > 7 { + return Operand{}, fmt.Errorf("invalid opmask register K%d", number) + } + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", number))}, nil + } + gpReg := func(number int) (Operand, error) { + reg, valid := decodedX86GeneralRegister(number) + if !valid || mode == 32 && number >= 8 { + return Operand{}, fmt.Errorf("invalid general register %d", number) + } + return Operand{Kind: OpReg, Reg: reg}, nil + } + + var src, dst Operand + consumed := 1 + switch opcode { + case 0x90: // K-or-memory -> K + dst, err = kReg(regNumber) + if err != nil { + return Instr{}, 0, true, err + } + if mod == 3 { + src, err = kReg(rmNumber) + } else { + src, consumed, err = memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + } + case 0x91: // K -> memory + if mod == 3 { + return Instr{}, 0, true, fmt.Errorf("opcode 91 requires a memory destination") + } + src, err = kReg(regNumber) + if err == nil { + dst, consumed, err = memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + } + case 0x92: // GP -> K + if mod != 3 { + return Instr{}, 0, true, fmt.Errorf("opcode 92 requires a general-register source") + } + dst, err = kReg(regNumber) + if err == nil { + src, err = gpReg(rmNumber) + } + case 0x93: // K -> GP + if mod != 3 { + return Instr{}, 0, true, fmt.Errorf("opcode 93 requires a general-register destination") + } + src, err = kReg(rmNumber) + if err == nil { + dst, err = gpReg(regNumber) + } + } + if err != nil { + return Instr{}, 0, true, err + } + op := Op("KMOV" + width) + instruction = Instr{ + Op: op, + Args: []Operand{src, dst}, + Raw: fmt.Sprintf("%s %s, %s", op, src.String(), dst.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86EVEXPackedIntegerMoveInstruction recognizes the complete EVEX +// family behind Go 1.27's shared _yvmovdqa32 table. x/arch v0.14 cannot +// decode these encodings, including the raw VMOVDQU32 loads emitted by +// github.com/minio/sha256-simd. Decode all six opcodes and all twelve table +// rows before consulting the generic decoder. +func decodedX86EVEXPackedIntegerMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedIntegerMoveInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXPackedIntegerMoveInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedMoveInstruction(code, mode, memory, "packed integer move", func(pp byte, width64 bool, opcode byte) (Op, bool, bool) { + store := opcode == 0x7f + if opcode != 0x6f && !store { + return "", false, false + } + switch { + case pp == 1 && !width64: + return "VMOVDQA32", store, true + case pp == 1 && width64: + return "VMOVDQA64", store, true + case pp == 3 && !width64: + return "VMOVDQU8", store, true + case pp == 3 && width64: + return "VMOVDQU16", store, true + case pp == 2 && !width64: + return "VMOVDQU32", store, true + case pp == 2 && width64: + return "VMOVDQU64", store, true + default: + return "", false, false + } + }) +} + +type decodedX86EVEXMoveMatcher func(pp byte, width64 bool, opcode byte) (op Op, store bool, ok bool) + +func decodedX86EVEXPackedMoveInstruction(code []byte, mode int, memory x86RawMemoryDecoder, family string, match decodedX86EVEXMoveMatcher) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 1 || p1&0x04 == 0 || p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, false, nil + } + pp := p1 & 3 + width64 := p1&0x80 != 0 + op, store, matched := match(pp, width64, opcode) + if !matched { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is not valid for %s", family) + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + disp8Scale := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if zeroing && store && code[i+5]>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing is invalid for a memory store") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + regNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + reg := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, regNumber))} + rm, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + + args := []Operand{rm, reg} + if store { + args[0], args[1] = reg, rm + } + if maskNumber != 0 { + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))} + args = []Operand{args[0], mask, args[1]} + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + instruction = Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))} + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXRMOperand(code []byte, mode, bExt, xExt int, segment Reg, vectorPrefix string, disp8Scale int) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if code[0]>>6 == 3 { + number := int(code[0]&7) + bExt*8 + xExt*16 + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, number))}, 1, nil + } + return decodedX86RMSource(code, mode, bExt, xExt, segment, disp8Scale) +} + +func x86RawRelativeTarget(inst x86asm.Inst, offset int) (int, bool) { + for _, arg := range inst.Args { + if relative, ok := arg.(x86asm.Rel); ok { + return offset + inst.Len + int(relative), true + } + } + return 0, false +} + +// decodedX86VMOVQInstruction recognizes the complete Go 1.27 _yvmovd and +// _yvmovq families. VMOVD has the X-to-GP/memory and reverse forms; VMOVQ +// additionally has two X-to-X encodings. Recover every VEX and EVEX form +// before consulting the generic decoder, which can split the raw VMOVD bytes +// emitted by Weaviate into a relative branch. Raw AVX512-FP16 VMOVW uses the +// same operand grammar with a two-byte transfer and a distinct opcode map. +func decodedX86VMOVQInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VMOVQInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VMOVQInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXVMOVDQInstructionWithMemory(code, i, mode, segment, addressOverride, memory) + } + if len(code) < i+3 { + return Instr{}, 0, false, nil + } + + var rExt, xExt, bExt, vexLength int + var vexBits byte + switch code[i] { + case 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case 0xc4: + if len(code) < i+4 || code[i+1]&0x1f != 1 { + return Instr{}, 0, false, nil + } + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + opcodeIndex := i + vexLength + if len(code) <= opcodeIndex { + return Instr{}, 0, false, nil + } + + opcode := code[opcodeIndex] + pp := vexBits & 3 + width64 := vexLength == 3 && vexBits&0x80 != 0 + type vmovqDirection int + const ( + xToGeneral vmovqDirection = iota + xToVector + generalToX + vectorToX + ) + op := Op("") + direction := xToGeneral + switch { + case opcode == 0x7e && pp == 1 && !width64: + op = "VMOVD" + direction = xToGeneral + case opcode == 0x6e && pp == 1 && !width64: + op = "VMOVD" + direction = generalToX + case opcode == 0x7e && pp == 1 && width64: + op = "VMOVQ" + direction = xToGeneral + case opcode == 0xd6 && pp == 1 && !width64: + op = "VMOVQ" + direction = xToVector + case opcode == 0x6e && pp == 1 && width64: + op = "VMOVQ" + direction = generalToX + case opcode == 0x7e && pp == 2 && !width64: + op = "VMOVQ" + direction = vectorToX + default: + return Instr{}, 0, false, nil + } + ok = true + if vexBits&0x7c != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be 1111 and VEX.L must be zero") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + vectorNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || vectorNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + vector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", vectorNumber))} + rmVector := direction == xToVector || direction == vectorToX + rm, consumed, decodeErr := memory.vexTransfer(code[modRMIndex:], mode, bExt, xExt, segment, rmVector) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + if mode == 32 && width64 && rm.Kind == OpReg { + return Instr{}, 0, true, fmt.Errorf("64-bit general register is unavailable in 32-bit mode") + } + + args := []Operand{vector, rm} + if direction == generalToX || direction == vectorToX { + args[0], args[1] = rm, vector + } + instruction = Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s", op, args[0].String(), args[1].String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXVMOVDQInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXVMOVDQInstructionWithMemory(code, i, mode, segment, addressOverride, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXVMOVDQInstructionWithMemory(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + wordMove := p0&0x0f == 5 && p1&3 == 1 && (opcode == 0x6e || opcode == 0x7e) + if p0&0x0f != 1 && !wordMove { + return Instr{}, 0, false, nil + } + if p1&0x04 == 0 || p1&0x78 != 0x78 || p2 != 0x08 { + if wordMove { + return Instr{}, 0, true, fmt.Errorf("VMOVW requires reserved vvvv, fixed 128-bit length, and no mask or broadcast") + } + return Instr{}, 0, false, nil + } + pp := p1 & 3 + width64 := p1&0x80 != 0 + type vmovdqDirection int + const ( + xToGeneral vmovdqDirection = iota + xToVector + generalToX + vectorToX + ) + op := Op("") + direction := xToGeneral + switch { + case wordMove: + op = "VMOVW" + if opcode == 0x6e { + direction = generalToX + } + case opcode == 0x7e && pp == 1 && !width64: + op = "VMOVD" + direction = xToGeneral + case opcode == 0x6e && pp == 1 && !width64: + op = "VMOVD" + direction = generalToX + case opcode == 0x7e && pp == 1 && width64: + op = "VMOVQ" + direction = xToGeneral + case opcode == 0xd6 && pp == 1 && width64: + op = "VMOVQ" + direction = xToVector + case opcode == 0x6e && pp == 1 && width64: + op = "VMOVQ" + direction = generalToX + case opcode == 0x7e && pp == 2 && width64: + op = "VMOVQ" + direction = vectorToX + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + vectorNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + rmVector := direction == xToVector || direction == vectorToX + if mode == 32 && (vectorNumber >= 8 || rmVector && modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + if modRM>>6 == 3 && !rmVector && xExt != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.X cannot extend a general register") + } + vector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", vectorNumber))} + transferBytes := 4 + if wordMove { + transferBytes = 2 + } else if op == "VMOVQ" { + transferBytes = 8 + } + var rm Operand + var consumed int + var decodeErr error + if rmVector { + rm, consumed, decodeErr = memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, "X", transferBytes) + } else { + rm, consumed, decodeErr = memory.scalar(code[modRMIndex:], mode, bExt, xExt, segment, transferBytes) + } + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + if mode == 32 && op == "VMOVQ" && rm.Kind == OpReg && !rmVector { + return Instr{}, 0, true, fmt.Errorf("64-bit general register is unavailable in 32-bit mode") + } + args := []Operand{vector, rm} + if direction == generalToX || direction == vectorToX { + args[0], args[1] = rm, vector + } + instruction = Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s", op, args[0].String(), args[1].String()), + x86Encoded: wordMove, + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86VEXRMOperand(code []byte, mode, bExt, xExt int, segment Reg, vectorRegister bool) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if code[0]>>6 != 3 || !vectorRegister { + return decodedX86VEXRMSource(code, mode, bExt, xExt, segment) + } + number := int(code[0]&7) + bExt*8 + if mode == 32 && number >= 8 { + return Operand{}, 0, fmt.Errorf("extended vector register in 32-bit mode") + } + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", number))}, 1, nil +} + +// The complete VEX/EVEX non-temporal move family shares the physical prefix +// decoder. The historical VEX entry name is retained for its existing callers. +// These boundaries grant no raw translation, memory or entry-state permission. +func decodedX86VEXNonTemporalMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86VEXNonTemporalMoveInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXNonTemporalMoveInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, known := decodeX86RawVectorEncoding(code) + if !known { + return Instr{}, 0, false, nil + } + var op Op + load := false + switch { + case p.mapNumber == 1 && p.opcode == 0xe7 && p.pp == 1: + op = "VMOVNTDQ" + case p.mapNumber == 1 && p.opcode == 0x2b && p.pp == 1: + op = "VMOVNTPD" + case p.mapNumber == 1 && p.opcode == 0x2b && p.pp == 0: + op = "VMOVNTPS" + case p.mapNumber == 2 && p.opcode == 0x2a && p.pp == 1: + op, load = "VMOVNTDQA", true + default: + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("non-temporal vector move: %s", message) + } + if mode != 32 && mode != 64 || p.addressOverride { + return fail("unknown mode or unbound address-size override") + } + if p.upper != 0 || p.mask != 0 || p.zero || p.broadcast || p.vectorLength > 2 { + return fail("reserved input, mask, zeroing, broadcast or length") + } + // VEX.W is ignored; EVEX.W distinguishes PD from the W0 rows. + if p.evex && (!p.fixed || p.w != (op == "VMOVNTPD")) { + return fail("invalid EVEX fixed or width bit") + } + if len(code) <= p.modRM || code[p.modRM]>>6 == 3 { + return fail("missing native memory ModRM") + } + register := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (register >= 8 || p.x != 0 || p.b != 0) { + return fail("unbound extended physical register in mode32") + } + bank := []string{"X", "Y", "Z"}[p.vectorLength] + vector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprint(bank, register))} + var operand Operand + var consumed int + var err error + if p.evex { + operand, consumed, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, bank, 16<>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + firstSource, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86EVEXPackedInsertInstruction recognizes every EVEX row in Go +// 1.27's _yvinsertf32x4 and _yvinsertf32x8 tables. The floating and integer +// spellings share their encodings apart from the opcode, while EVEX.W selects +// 32- or 64-bit lane names. This includes Y/Z output widths, register and +// memory insert sources, and optional merge/zero masks. +func decodedX86EVEXPackedInsertInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedInsertInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXPackedInsertInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) < i+5 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 3 || p1&3 != 1 { + return Instr{}, 0, false, nil + } + integer := opcode == 0x38 || opcode == 0x3a + wideInsert := opcode == 0x1a || opcode == 0x3a + if opcode != 0x18 && opcode != 0x1a && !integer { + return Instr{}, 0, false, nil + } + ok = true + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed vector inserts are unavailable in 32-bit mode") + } + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for packed vector inserts") + } + vectorBits := p2 >> 5 & 3 + if wideInsert { + if vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("wide packed vector insert requires a Z output") + } + } else if vectorBits != 1 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("packed vector insert requires a Y or Z output") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + insertBytes := 16 + sourceName := "X" + laneBits := 32 + if p1&0x80 != 0 { + laneBits = 64 + } + if wideInsert { + insertBytes = 32 + sourceName = "Y" + } + stem := "VINSERTF" + if integer { + stem = "VINSERTI" + } + op := Op(fmt.Sprintf("%s%dX%d", stem, laneBits, insertBytes*8/laneBits)) + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + first, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, sourceName, insertBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{immediate, first, second} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86VEXPackedBroadcastInstruction recognizes the complete VEX and EVEX +// VPBROADCASTB/W/D/Q and K-data broadcast families represented by Go 1.27's +// _yvpbroadcastb and _yvpbroadcastmb2q tables. +// x/arch v0.14 can misclassify the VEX bytes as PC-relative branches, as +// observed in github.com/vivint/infectious, and cannot decode the EVEX +// general-register form used by github.com/JohanLindvall/haste. +func decodedX86VEXPackedBroadcastInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXPackedBroadcastInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXPackedBroadcastInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + // Original native observers use this shared broadcast-family entry. + // The block form keeps the same decoder and memory policy as raw source. + if instruction, length, ok, err = decodedX86VBROADCAST128InstructionWithMemory(code, mode, memory); ok || err != nil { + return + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXPackedBroadcastInstructionWithMemory(code, i, mode, segment, addressOverride, memory) + } + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 { + return Instr{}, 0, false, nil + } + op, recognized := map[byte]Op{ + 0x78: "VPBROADCASTB", + 0x79: "VPBROADCASTW", + 0x58: "VPBROADCASTD", + 0x59: "VPBROADCASTQ", + }[code[i+3]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vex0, vexBits := code[i+1], code[i+2] + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if int(^vexBits>>3)&15 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be 1111") + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXPackedBroadcastInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedBroadcastInstructionWithMemory(code, i, mode, segment, addressOverride, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXPackedBroadcastInstructionWithMemory(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+5 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f == 2 && p1&3 == 2 && (opcode == 0x2a || opcode == 0x3a) { + return decodedX86EVEXMaskDataBroadcastInstruction(code, mode) + } + if p0&0x0f != 2 || p1&3 != 1 { + return Instr{}, 0, false, nil + } + width64 := p1&0x80 != 0 + vectorSource := false + laneBytes := 0 + op := Op("") + switch opcode { + case 0x7a: + op, laneBytes = "VPBROADCASTB", 1 + case 0x7b: + op, laneBytes = "VPBROADCASTW", 2 + case 0x7c: + if width64 { + op, laneBytes = "VPBROADCASTQ", 8 + } else { + op, laneBytes = "VPBROADCASTD", 4 + } + case 0x78: + op, laneBytes, vectorSource = "VPBROADCASTB", 1, true + case 0x79: + op, laneBytes, vectorSource = "VPBROADCASTW", 2, true + case 0x58: + op, laneBytes, vectorSource = "VPBROADCASTD", 4, true + case 0x59: + op, laneBytes, vectorSource = "VPBROADCASTQ", 8, true + default: + return Instr{}, 0, false, nil + } + ok = true + if (!vectorSource || opcode != 0x59) && opcode != 0x7c && width64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be zero for %s", op) + } + if vectorSource && opcode == 0x59 && !width64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be one for %s", op) + } + if p1&0x7c != 0x7c || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for packed scalar broadcasts") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + + var source Operand + consumed := 0 + if vectorSource { + if mode == 32 && modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, err = memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, "X", laneBytes) + } else { + if modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX opcode %#x requires a general-register source", opcode) + } + if xExt != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.X cannot extend a general register") + } + if mode == 32 && opcode == 0x7c && width64 { + // GP-only opcode 7C ignores EVEX.W in non-64-bit mode. + // Retain the original Q/W1 identity, but decode its physical + // r32 input; the shared broadcast grammar uses dword lanes. + laneBytes = 4 + } + source, consumed, err = decodedX86RMSource(code[modRMIndex:], mode, bExt, 0, segment, laneBytes) + } + if err != nil { + return Instr{}, 0, true, err + } + args := []Operand{source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86PackedAlignRightInstruction recognizes every raw PALIGNR and +// VPALIGNR encoding represented by Go 1.27's ypalignr and +// _yvgf2p8affineinvqb tables: legacy X, VEX X/Y, and EVEX X/Y/Z with optional +// merge/zero masks. x/arch v0.14 rejects the VEX bytes used by the historical +// sha256-simd copy in github.com/ipfs/fs-repo-migrations. +func decodedX86PackedAlignRightInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86PackedAlignRightInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedAlignRightInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + hasDataSizePrefix := false + rex := byte(0) + hasREX := false + for i < len(code) { + switch code[i] { + case 0x64: + rex = 0 + hasREX = false + segment = FS + i++ + case 0x65: + rex = 0 + hasREX = false + segment = GS + i++ + case 0x66: + rex = 0 + hasREX = false + hasDataSizePrefix = true + i++ + case 0x67: + rex = 0 + hasREX = false + addressOverride = true + i++ + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + hasREX = true + i++ + continue + } + goto opcode + } + } + +opcode: + if i >= len(code) { + return Instr{}, 0, false, nil + } + if code[i] == 0xc4 { + if hasDataSizePrefix || hasREX || len(code) < i+5 || code[i+1]&0x1f != 3 || code[i+3] != 0x0f { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 VPALIGNR exceeds the Go assembler frontend's operand limit") + } + vex0, vexBits := code[i+1], code[i+2] + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + firstSource, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + args := []Operand{ + {Kind: OpImm, Imm: int64(code[immediateIndex])}, + firstSource, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))}, + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: "VPALIGNR", Args: args, Raw: "VPALIGNR " + strings.Join(rawArgs, ", ")}, immediateIndex + 1, true, nil + } + if code[i] == 0x62 { + if hasDataSizePrefix || hasREX || len(code) < i+6 || code[i+1]&0x0f != 3 || code[i+2]&0x07 != 5 || code[i+4] != 0x0f { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 VPALIGNR exceeds the Go assembler frontend's operand limit") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p1&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be zero") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is absent from Go 1.27's VPALIGNR table") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + firstSource, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, vectorWidth) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + op := Op("VPALIGNR") + args := []Operand{ + {Kind: OpImm, Imm: int64(code[immediateIndex])}, + firstSource, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))}, + } + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))}) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil + } + + if !hasDataSizePrefix || len(code) < i+5 || code[i] != 0x0f || code[i+1] != 0x3a || code[i+2] != 0x0f { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if hasREX && rex&0x08 != 0 { + return Instr{}, 0, true, fmt.Errorf("REX.W is absent from Go 1.27's PALIGNR encoding") + } + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex & 1) + modRMIndex := i + 3 + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (destinationNumber >= 8 || bExt != 0 || xExt != 0) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + instruction = Instr{ + Op: "PALIGNR", + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("PALIGNR %s, %s, %s", immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86Permute128Instruction recognizes every raw VPERM2F128 and +// VPERM2I128 encoding represented by Go 1.27's _yvperm2f128 table: an +// immediate, Y/m256 first source, Y second source, and Y destination. +func decodedX86Permute128Instruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86Permute128InstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86Permute128InstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + hasLegacyPrefix := false + hasREX := false + for i < len(code) { + switch code[i] { + case 0x64: + hasREX = false + segment = FS + i++ + case 0x65: + hasREX = false + segment = GS + i++ + case 0x66: + hasREX = false + hasLegacyPrefix = true + i++ + case 0x67: + hasREX = false + addressOverride = true + i++ + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + hasREX = true + i++ + continue + } + goto opcode + } + } + +opcode: + if i >= len(code) || code[i] != 0xc4 || hasLegacyPrefix || hasREX || len(code) < i+5 { + return Instr{}, 0, false, nil + } + vex0, vexBits, opcode := code[i+1], code[i+2], code[i+3] + if vex0&0x1f != 3 { + return Instr{}, 0, false, nil + } + var op Op + for name, encoding := range x86Permute128Specs { + if encoding == opcode { + op = name + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must select 256-bit vectors") + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + firstSource, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "Y") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in physical mode32 permute128") + } + if immediateIndex+1 > 15 { + return Instr{}, 0, true, fmt.Errorf("permute128 instruction exceeds 15 bytes") + } + args := []Operand{ + {Kind: OpImm, Imm: int64(code[immediateIndex])}, + firstSource, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", secondSourceNumber))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", destinationNumber))}, + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + x86Encoded: true, x86VectorBytes: 32, x86Permute128VEX: true}, immediateIndex + 1, true, nil +} + +// decodedX86InsertPSInstruction recognizes all encodings represented by Go +// 1.27's yxshuf and _yvinsertps tables: legacy INSERTPS plus VEX and EVEX +// VINSERTPS, each with an X-register or ModRM/SIB memory source. x/arch v0.14 +// can consume one byte past VINSERTPS and desynchronize the rest of a raw +// directive group, as observed in github.com/racerxdl/segdsp. +func decodedX86InsertPSInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) <= i { + return Instr{}, 0, false, nil + } + + if code[i] == 0x66 { + j := i + 1 + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) < j+4 || code[j] != 0x0f || code[j+1] != 0x3a || code[j+2] != 0x21 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + modRMIndex := j + 3 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: "INSERTPS", + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("INSERTPS %s, %s, %s", immediate.String(), source.String(), destination.String()), + }, immediateIndex + 1, true, nil + } + + if code[i] == 0xc4 { + if len(code) < i+5 || code[i+1]&0x1f != 3 || code[i+3] != 0x21 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1 := code[i+1], code[i+2] + // The physical VEX form is WIG, independently of Go's W0 encoder. + if p1&0x07 != 0x01 { + return Instr{}, 0, true, fmt.Errorf("VINSERTPS requires VEX.L0.66") + } + modRMIndex := i + 4 + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + baseNumber := int(^p1>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (baseNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VINSERTPS register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + base := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", baseNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: "VINSERTPS", + Args: []Operand{immediate, source, base, destination}, + Raw: fmt.Sprintf("VINSERTPS %s, %s, %s, %s", immediate.String(), source.String(), base.String(), destination.String()), + x86Encoded: true, + }, immediateIndex + 1, true, nil + } + + if code[i] != 0x62 || len(code) < i+6 || code[i+1]&0x0f != 3 || code[i+4] != 0x21 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("EVEX VINSERTPS is unavailable in 32-bit mode") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p1&0x87 != 0x05 || p2&0xf7 != 0 { + return Instr{}, 0, true, fmt.Errorf("VINSERTPS requires EVEX.W0.L0.66 without mask, zeroing, or broadcast") + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + baseNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X", 4) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + base := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", baseNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: "VINSERTPS", + Args: []Operand{immediate, source, base, destination}, + Raw: fmt.Sprintf("VINSERTPS %s, %s, %s, %s", immediate.String(), source.String(), base.String(), destination.String()), + }, immediateIndex + 1, true, nil +} + +// decodedX86PackedScalarExtractInstruction recognizes every VEX and EVEX +// encoding row shared by Go 1.27's _yvextractps and _yvpextrw tables: +// VEXTRACTPS and VPEXTRB/W/D/Q with GP-register or ModRM/SIB memory +// destinations. x/arch v0.14 does not decode VEXTRACTPS and can interpret +// bytes after other scalar extracts as instructions or PC-relative operands, +// so recover the complete family before consulting the generic decoder. +func decodedX86PackedScalarExtractInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86PackedScalarExtractInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedScalarExtractInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) <= i { + return Instr{}, 0, false, nil + } + var ( + mapNumber byte + opcode byte + width64 bool + rExt int + rHighExt int + xExt int + bExt int + modRMIndex int + disp8Scale = 1 + evex bool + twoByteVEX bool + prefixValidated bool + ) + switch code[i] { + case 0xc4: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + mapNumber = code[i+1] & 0x1f + opcode = code[i+3] + width64 = code[i+2]&0x80 != 0 + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + modRMIndex = i + 4 + prefixValidated = code[i+2]&0x7f == 0x79 + case 0xc5: + if len(code) < i+3 { + return Instr{}, 0, false, nil + } + mapNumber = 1 + opcode = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + modRMIndex = i + 3 + twoByteVEX = true + prefixValidated = code[i+1]&0x7f == 0x79 + case 0x62: + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + evex = true + mapNumber = code[i+1] & 0x0f + opcode = code[i+4] + width64 = code[i+2]&0x80 != 0 + rExt = int(^code[i+1]>>7) & 1 + rHighExt = int(^code[i+1]>>4) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + modRMIndex = i + 5 + prefixValidated = code[i+2]&0x7f == 0x7d && code[i+3] == 0x08 + default: + return Instr{}, 0, false, nil + } + + var op Op + laneBytes := 0 + expectedWidth64 := false + switch { + case mapNumber == 3 && opcode == 0x14: + op, laneBytes = "VPEXTRB", 1 + case mapNumber == 3 && opcode == 0x15: + op, laneBytes = "VPEXTRW", 2 + case mapNumber == 3 && opcode == 0x16 && !width64: + op, laneBytes = "VPEXTRD", 4 + case mapNumber == 3 && opcode == 0x16 && width64: + op, laneBytes, expectedWidth64 = "VPEXTRQ", 8, true + case mapNumber == 3 && opcode == 0x17: + op, laneBytes = "VEXTRACTPS", 4 + case mapNumber == 1 && opcode == 0xc5: + op, laneBytes = "VPEXTRW", 2 + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if !prefixValidated { + return Instr{}, 0, true, fmt.Errorf("reserved vvvv, vector length, prefix, or EVEX mask bits are invalid") + } + if width64 != expectedWidth64 { + return Instr{}, 0, true, fmt.Errorf("invalid W bit for %s", op) + } + if evex { + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed scalar extracts are unavailable in 32-bit mode") + } + disp8Scale = laneBytes + } + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + sourceNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && sourceNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended source register in 32-bit mode") + } + destination, consumed, decodeErr := memory.scalar(code[modRMIndex:], mode, bExt, xExt, segment, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + if mapNumber == 1 && destination.Kind != OpReg { + return Instr{}, 0, true, fmt.Errorf("VEX map-1 VPEXTRW requires a GP-register destination") + } + if evex && destination.Kind == OpReg && xExt != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed scalar extract destination is outside the GP-register class") + } + if twoByteVEX && destination.Kind == OpReg && bExt != 0 { + return Instr{}, 0, true, fmt.Errorf("two-byte VEX cannot encode an extended destination register") + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", sourceNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86VPERMQInstruction recognizes the complete VEX.256 immediate +// VPERMQ Y/m256, Y family. x/arch v0.14 can misclassify its trailing immediate +// as a PC-relative operand, so recover register and ModRM/SIB memory forms +// before consulting the generic decoder. EVEX forms are decoded correctly by +// x/arch and continue through the ordinary path. +func decodedX86VPERMQInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 3 || code[i+3] != 0x00 { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + if code[i+2] != 0xfd { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be one, VEX.vvvv must be 1111, VEX.L must be one, and VEX.pp must be 66") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + vex1 := code[i+1] + rExt := int(^vex1>>7) & 1 + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + modRMIndex := i + 4 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + source, consumed, err := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "Y") + if err != nil { + return Instr{}, 0, true, err + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", destinationNumber))} + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + instruction = Instr{ + Op: "VPERMQ", + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("VPERMQ %s, %s, %s", immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86ScalarPrecisionConvertInstruction recognizes the complete +// CVTSS2SD/CVTSD2SS and VCVTSS2SD/VCVTSD2SS family represented by Go 1.27's +// legacy yxm and EVEX-aware _yvaddsd tables. Besides VEX and legacy forms it +// handles EVEX high X registers, memory, masks, zeroing, SAE, and all four +// embedded rounding modes. x/arch v0.14 can split VCVTSD2SS into POP/ROLL. +func decodedX86ScalarPrecisionConvertInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) <= i { + return Instr{}, 0, false, nil + } + + // Legacy F2/F3 0F 5A /r forms overwrite their X destination. + j := i + if code[j] == 0xf2 || code[j] == 0xf3 { + prefix := code[j] + j++ + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) >= j+3 && code[j] == 0x0f && code[j+1] == 0x5a { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + op := Op("CVTSD2SS") + if prefix == 0xf3 { + op = "CVTSS2SD" + } + modRMIndex := j + 2 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: op, Args: []Operand{source, destination}, x86Encoded: true, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + }, modRMIndex + consumed, true, nil + } + } + + if code[i] == 0xc4 || code[i] == 0xc5 { + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch code[i] { + case 0xc5: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case 0xc4: + if len(code) < i+5 || code[i+1]&0x1f != 1 { + return Instr{}, 0, false, nil + } + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + } + if opcode != 0x5a || (vexBits&3 != 2 && vexBits&3 != 3) { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar precision conversion requires VEX.128") + } + op := Op("VCVTSD2SS") + if vexBits&3 == 2 { + op = "VCVTSS2SD" + } + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + modRM := code[modRMIndex] + first, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VEX register in 32-bit mode") + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: op, Args: []Operand{first, second, destination}, x86Encoded: true, + Raw: fmt.Sprintf("%s %s, %s, %s", op, first.String(), second.String(), destination.String()), + }, modRMIndex + consumed, true, nil + } + + if len(code) < i+6 || code[i] != 0x62 || code[i+1]&0x0f != 1 || code[i+4] != 0x5a { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + prefix := p1 & 3 + wBit := p1&0x80 != 0 + op := Op("") + inputBytes := 0 + supportsRounding := false + switch prefix { + case 3: + op = "VCVTSD2SS" + inputBytes = 8 + supportsRounding = true + if !wBit { + return Instr{}, 0, true, fmt.Errorf("VCVTSD2SS requires EVEX.W1") + } + case 2: + op = "VCVTSS2SD" + inputBytes = 4 + if wBit { + return Instr{}, 0, true, fmt.Errorf("VCVTSS2SD requires EVEX.W0") + } + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + embeddedControl := p2&0x10 != 0 + vectorBits := p2 >> 5 & 3 + if !embeddedControl && vectorBits != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar precision conversion requires EVEX.128") + } + if embeddedControl && modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("embedded rounding/SAE requires a register source") + } + if embeddedControl { + if supportsRounding { + op += Op("." + [...]string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"}[vectorBits]) + } else if vectorBits == 0 { + op += ".SAE" + } else { + return Instr{}, 0, true, fmt.Errorf("VCVTSS2SD supports SAE but not explicit rounding") + } + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 scalar precision conversion mask forms exceed the Go assembler frontend's operand limit") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + first, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X", inputBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + args := []Operand{first, second} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{ + Op: op, Args: args, x86Encoded: true, x86ScalarPrecisionEVEX: true, + Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + }, modRMIndex + consumed, true, nil +} + +// decodedX86DuplicateMoveInstruction recognizes every Go 1.27 yxm and +// _yvmovddup encoding for MOVDDUP/MOVSHDUP/MOVSLDUP and their V-prefixed +// forms. This includes legacy X, VEX X/Y, and EVEX X/Y/Z register or memory +// sources with optional merge/zero masks and compressed displacements. +// x/arch v0.14 rejects the VMOVSHDUP bytes generated by segdsp. +func decodedX86DuplicateMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) <= i { + return Instr{}, 0, false, nil + } + + // Legacy duplicate moves use F2/F3 0F opcode /r and the ModRM.reg X + // register as the destination. + j := i + if code[j] == 0xf2 || code[j] == 0xf3 { + prefix := code[j] + j++ + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) >= j+3 && code[j] == 0x0f { + op := Op("") + switch { + case prefix == 0xf2 && code[j+1] == 0x12: + op = "MOVDDUP" + case prefix == 0xf3 && code[j+1] == 0x12: + op = "MOVSLDUP" + case prefix == 0xf3 && code[j+1] == 0x16: + op = "MOVSHDUP" + } + if op != "" { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + modRMIndex := j + 2 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{Op: op, Args: []Operand{source, destination}, Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String())}, modRMIndex + consumed, true, nil + } + } + } + + if code[i] == 0xc4 || code[i] == 0xc5 { + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch code[i] { + case 0xc5: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case 0xc4: + if len(code) < i+5 || code[i+1]&0x1f != 1 { + return Instr{}, 0, false, nil + } + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + } + prefix := vexBits & 3 + op := Op("") + switch { + case prefix == 3 && opcode == 0x12: + op = "VMOVDDUP" + case prefix == 2 && opcode == 0x12: + op = "VMOVSLDUP" + case prefix == 2 && opcode == 0x16: + op = "VMOVSHDUP" + } + if op == "" { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + // All three VEX duplicate-move opcodes are WIG. Unlike the EVEX + // encodings below, either W value denotes the same operation. + if vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be reserved") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VEX register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{Op: op, Args: []Operand{source, destination}, Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String())}, modRMIndex + consumed, true, nil + } + + if len(code) < i+6 || code[i] != 0x62 || code[i+1]&0x0f != 1 { + return Instr{}, 0, false, nil + } + p0, p1, p2, opcode := code[i+1], code[i+2], code[i+3], code[i+4] + prefix := p1 & 3 + op := Op("") + elementBytes := 4 + switch { + case prefix == 3 && opcode == 0x12: + op = "VMOVDDUP" + elementBytes = 8 + case prefix == 2 && opcode == 0x12: + op = "VMOVSLDUP" + case prefix == 2 && opcode == 0x16: + op = "VMOVSHDUP" + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + double := op == "VMOVDDUP" + if double != (p1&0x80 != 0) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match duplicate-move element width") + } + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvvv must be reserved") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is absent from Go 1.27's duplicate-move table") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + disp8Scale := vectorWidth + if double && vectorBits == 0 { + disp8Scale = elementBytes + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + sourceNumber := int(modRM&7) + bExt*8 + xExt*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && vectorName == "Z" { + if modRM>>6 == 3 && sourceNumber >= 8 || destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("high Z register is unavailable in 32-bit mode") + } + } + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86InLaneFloatingPermuteInstruction recognizes every encoding in Go +// 1.27's shared _yvpermilpd table: VPERMILPS/VPERMILPD immediate and +// variable-control VEX X/Y forms plus EVEX X/Y/Z, memory, scalar broadcast, +// merge-mask, and zero-mask forms. x/arch v0.14 can split a VEX immediate form +// into a fake PC-relative branch inside generated segdsp byte streams. +func decodedX86InLaneFloatingPermuteInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) <= i { + return Instr{}, 0, false, nil + } + if code[i] == 0xc4 { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + vex0, vexBits, opcode := code[i+1], code[i+2], code[i+3] + mapNumber := vex0 & 0x1f + immediate := mapNumber == 3 && (opcode == 0x04 || opcode == 0x05) + variable := mapNumber == 2 && (opcode == 0x0c || opcode == 0x0d) + if !immediate && !variable { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if immediate && vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("immediate VPERMIL requires reserved VEX.vvvv") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + secondNumber := int(^vexBits>>3) & 15 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8 || variable && secondNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VEX register in 32-bit mode") + } + first, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + double := opcode&1 != 0 + op := Op("VPERMILPS") + if double { + op = "VPERMILPD" + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := make([]Operand, 0, 3) + end := modRMIndex + consumed + if immediate { + if len(code) <= end { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[end])}, first, destination) + end++ + } else { + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + args = append(args, first, second, destination) + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, end, true, nil + } + + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2, opcode := code[i+1], code[i+2], code[i+3], code[i+4] + mapNumber := p0 & 0x0f + immediate := mapNumber == 3 && (opcode == 0x04 || opcode == 0x05) + variable := mapNumber == 2 && (opcode == 0x0c || opcode == 0x0d) + if !immediate && !variable { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p1&0x07 != 5 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit and pp must select 66") + } + double := opcode&1 != 0 + if double != (p1&0x80 != 0) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match VPERMILPS/PD") + } + if immediate && (p1&0x78 != 0x78 || p2&0x08 == 0) { + return Instr{}, 0, true, fmt.Errorf("immediate VPERMIL requires reserved EVEX.vvvvv") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + laneBytes := 4 + op := Op("VPERMILPS") + if double { + laneBytes = 8 + op = "VPERMILPD" + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 VPERMIL mask forms exceed the Go assembler frontend's operand limit") + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + op += ".BCST" + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + firstNumber := int(modRM&7) + bExt*8 + xExt*16 + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && vectorName == "Z" { + if modRM>>6 == 3 && firstNumber >= 8 || variable && secondNumber >= 8 || destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("high Z register is unavailable in 32-bit mode") + } + } + first, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := make([]Operand, 0, 4) + end := modRMIndex + consumed + if immediate { + if len(code) <= end { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[end])}, first) + end++ + } else { + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + args = append(args, first, second) + } + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, end, true, nil +} + +// decodedX86ImmediatePackedBlendInstruction recognizes the complete Go 1.27 +// immediate packed-blend family: legacy PBLENDW/BLENDPS/BLENDPD X forms and +// VEX VPBLENDW/VPBLENDD/VBLENDPS/VBLENDPD X/Y forms. Every register and +// ModRM/SIB memory source is handled before x/arch's generic decoder, which +// can split VBLENDPD into unrelated scalar instructions in generated code. +func decodedX86ImmediatePackedBlendInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86ImmediatePackedBlendInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86ImmediatePackedBlendInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) <= i { + return Instr{}, 0, false, nil + } + + // Legacy 66 0F 3A /r ib encodings use the ModRM.reg X register as both + // their second source and destination. + j := i + if code[j] == 0x66 { + j++ + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) >= j+4 && code[j] == 0x0f && code[j+1] == 0x3a { + op, recognized := map[byte]Op{0x0c: "BLENDPS", 0x0d: "BLENDPD", 0x0e: "PBLENDW"}[code[j+2]] + if recognized { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + modRMIndex := j + 3 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, immediate.String(), source.String(), destination.String()), + }, immediateIndex + 1, true, nil + } + } + } + + // All vector forms are VEX.128/256.66.0F3A. VPBLENDD is W0; + // VBLENDPS/PD and VPBLENDW are WIG. Go386's named arity limit does + // not invalidate these independently decoded low-bank physical forms. + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 3 { + return Instr{}, 0, false, nil + } + op, recognized := map[byte]Op{ + 0x02: "VPBLENDD", + 0x0c: "VBLENDPS", + 0x0d: "VBLENDPD", + 0x0e: "VPBLENDW", + }[code[i+3]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vex0, vexBits := code[i+1], code[i+2] + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 && op == "VPBLENDD" { + return Instr{}, 0, true, fmt.Errorf("VPBLENDD VEX.W must be zero") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + if mode == 32 { + if rExt != 0 || xExt != 0 { + return Instr{}, 0, true, fmt.Errorf("legacy LES encoding is not a 32-bit VEX instruction") + } + bExt = 0 // The physical high B and vvvv bits are ignored outside 64-bit mode. + } + modRMIndex := i + 4 + modRM := code[modRMIndex] + first, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondNumber := int(^vexBits>>3) & 15 + if mode == 32 { + secondNumber &= 7 + } + destinationNumber := int(modRM>>3&7) + rExt*8 + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: op, Args: []Operand{immediate, first, second, destination}, x86Encoded: true, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), first.String(), second.String(), destination.String()), + }, immediateIndex + 1, true, nil +} + +// decodedX86FloatingUnpackInstruction recognizes every legacy yxm and +// VEX/EVEX _yvandnpd row used by UNPCKLPS/UNPCKHPS/UNPCKLPD/UNPCKHPD and +// their V-prefixed forms. This includes X/Y/Z widths, register and ModRM/SIB +// memory sources, EVEX broadcast, merge masks, zero masks, and compressed +// displacements. x/arch v0.14 can overrun VUNPCK by one byte and desynchronize +// a raw directive stream, as observed in github.com/racerxdl/segdsp. +func decodedX86FloatingUnpackInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) <= i { + return Instr{}, 0, false, nil + } + + legacyPrefix := byte(0) + j := i + if code[j] == 0x66 { + legacyPrefix = 1 + j++ + } + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) >= j+3 && code[j] == 0x0f && (code[j+1] == 0x14 || code[j+1] == 0x15) { + opcode := code[j+1] + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + stem := "UNPCKL" + if opcode == 0x15 { + stem = "UNPCKH" + } + op := Op(stem + map[byte]string{0: "PS", 1: "PD"}[legacyPrefix]) + modRMIndex := j + 2 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + }, modRMIndex + consumed, true, nil + } + + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + } + if vexLength != 0 && (opcode == 0x14 || opcode == 0x15) { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&3 > 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp does not select PS or PD floating unpack") + } + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("floating unpack requires VEX.W0") + } + stem := "VUNPCKL" + if opcode == 0x15 { + stem = "VUNPCKH" + } + op := Op(stem + map[byte]string{0: "PS", 1: "PD"}[vexBits&3]) + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + secondNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VEX register in 32-bit mode") + } + first, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: op, + Args: []Operand{first, second, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, first.String(), second.String(), destination.String()), + }, modRMIndex + consumed, true, nil + } + + if code[i] != 0x62 || len(code) < i+6 || code[i+1]&0x0f != 1 || (code[i+4] != 0x14 && code[i+4] != 0x15) { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + if p1&3 > 1 { + return Instr{}, 0, true, fmt.Errorf("EVEX.pp does not select PS or PD floating unpack") + } + double := p1&3 == 1 + if double != (p1&0x80 != 0) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W and pp disagree for floating unpack") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorBytes := [...]int{16, 32, 64}[vectorBits] + laneBytes := 4 + suffix := "PS" + if double { + laneBytes = 8 + suffix = "PD" + } + stem := "VUNPCKL" + if code[i+4] == 0x15 { + stem = "VUNPCKH" + } + op := Op(stem + suffix) + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX masking is unavailable for 386 floating unpack forms") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorBytes + if broadcast { + disp8Scale = laneBytes + op += ".BCST" + } + evexRExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + evexXExt := int(^p0>>6) & 1 + evexBExt := int(^p0>>5) & 1 + firstNumber := int(modRM&7) + evexBExt*8 + evexXExt*16 + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + evexRExt*8 + rHighExt*16 + if mode == 32 { + if modRM>>6 == 3 && firstNumber >= 8 || secondNumber >= 8 || destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("%w: EVEX floating unpack high register has no proved physical mode32 binding", ErrProbeNeedsContext) + } + } + first, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, evexBExt, evexXExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{first, second} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86VEXPackedFloatLogicalInstruction recognizes the complete +// VEX.128/VEX.256 VANDPS/PD, VANDNPS/PD, VORPS/PD, and VXORPS/PD family. +// x/arch v0.14 can split or reject valid VEX encodings, including the raw +// VXORPS emitted by Weaviate, so recover every register and ModRM/SIB memory +// form before the generic decoder. VEX.W is ignored by these opcode rows. +func decodedX86VEXPackedFloatLogicalInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXPackedFloatLogicalInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXPackedFloatLogicalInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + stem, recognized := map[byte]string{ + 0x54: "VAND", + 0x55: "VANDN", + 0x56: "VOR", + 0x57: "VXOR", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + suffix := "" + switch vexBits & 3 { + case 0: + suffix = "PS" + case 1: + suffix = "PD" + default: + return Instr{}, 0, true, fmt.Errorf("VEX.pp does not select a packed floating logical instruction") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, err := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if err != nil { + return Instr{}, 0, true, err + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + op := Op(stem + suffix) + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// K is a data register here, not an optional writemask. Recognizing these +// physical register shapes grants no initialized-mask or memory authority. +func decodedX86EVEXMaskDataBroadcastInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 2 || p.opcode != 0x2a && p.opcode != 0x3a { + return Instr{}, 0, false, nil + } + op := Op("VPBROADCASTMW2D") + if p.opcode == 0x2a { + op = "VPBROADCASTMB2Q" + } + fail := func(reason string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("%s %s", op, reason) + } + if mode != 32 && mode != 64 { + return fail("has an unsupported x86 mode") + } + if p.w != (p.opcode == 0x2a) { + return fail("has a reserved EVEX.W value") + } + if !p.fixed || p.upper != 0 { + return fail("requires reserved EVEX.vvvv and the fixed prefix bit") + } + if p.segment != "" || p.addressOverride { + return fail("does not admit segment or address-size overrides") + } + if p.mask != 0 || p.zero || p.broadcast { + return fail("does not admit writemasking, zeroing, or EVEX.b") + } + if p.vectorLength == 3 { + return fail("has a reserved EVEX vector length") + } + if len(code) <= p.modRM { + return fail("is missing its ModRM byte") + } + modRM := code[p.modRM] + if modRM>>6 != 3 || p.b != 0 || p.x != 0 { + return fail("requires an unextended K-register source") + } + destination := int(modRM>>3&7) + p.r*8 + if mode == 32 && destination >= 8 { + return fail("has an extended vector register in 32-bit mode") + } + src := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM&7))} + dst := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", [...]string{"X", "Y", "Z"}[p.vectorLength], destination))} + return Instr{Op: op, Args: []Operand{src, dst}, Raw: fmt.Sprintf("%s %s, %s", op, src.String(), dst.String())}, p.modRM + 1, true, nil +} + +// decodedX86VEXPackedIntegerLogicalInstruction recognizes the complete +// VEX.128/VEX.256 VPAND, VPANDN, VPOR, and VPXOR family described by Go +// 1.27's shared _yvaddsubpd operand table. Recover both VEX.WIG values and +// every register and ModRM/SIB memory form before the generic decoder, which +// can split or reject raw encodings emitted by ecosystem assembly. +func decodedX86VEXPackedIntegerLogicalInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXPackedIntegerLogicalInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXPackedIntegerLogicalInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + op, recognized := map[byte]Op{ + 0xdb: "VPAND", + 0xdf: "VPANDN", + 0xeb: "VPOR", + 0xef: "VPXOR", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, err := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if err != nil { + return Instr{}, 0, true, err + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86EVEXPackedLogicalInstruction recognizes every EVEX register, +// memory, broadcast, merge-mask, and zero-mask row shared by Go 1.27's +// VPANDD/Q, VPANDND/Q, VPORD/Q, and VPXORD/Q _yvblendmpd table. +func decodedX86EVEXPackedLogicalInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedLogicalInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXPackedLogicalInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) < i+5 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 1 || p1&3 != 1 { + return Instr{}, 0, false, nil + } + stem, recognized := map[byte]string{0xdb: "VPAND", 0xdf: "VPANDN", 0xeb: "VPOR", 0xef: "VPXOR"}[code[i+4]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorBytes := [...]int{16, 32, 64}[vectorBits] + width := "D" + laneBytes := 4 + if p1&0x80 != 0 { + width = "Q" + laneBytes = 8 + } + op := Op(stem + width) + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX masking is unavailable for 386 packed logical forms") + } + + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorBytes + if broadcast { + disp8Scale = laneBytes + op += ".BCST" + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + firstNumber := int(modRM&7) + bExt*8 + xExt*16 + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && vectorName == "Z" { + if modRM>>6 == 3 && firstNumber >= 8 || secondNumber >= 8 || destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("high Z register is unavailable in 32-bit mode") + } + } + first, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{first, second} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86VEXPackedMADDInstruction recognizes the complete VEX.128 and +// VEX.256 VPMADDWD and VPMADDUBSW family in Go 1.27's opcode tables. +// VPMADDWD is available through both two- and three-byte VEX map-1 forms; +// VPMADDUBSW uses the three-byte VEX map-2 form. VEX.W is ignored. Recover +// every register and ModRM/SIB memory form before the generic decoder, which +// can split raw VPMADDWD bytes emitted by ecosystem assembly. +func decodedX86VEXPackedMADDInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXPackedMADDInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXPackedMADDInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, opcodeMap, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + opcodeMap = 1 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4: + vexLength = 3 + opcodeMap = int(code[i+1] & 0x1f) + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + op, recognized := map[[2]int]Op{ + {1, 0xf5}: "VPMADDWD", + {2, 0x04}: "VPMADDUBSW", + }[[2]int{opcodeMap, int(opcode)}] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + x86Encoded: true, + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86VEXBinaryFloatInstruction recognizes the complete VEX and EVEX +// VADD, VMUL, VSUB, VMIN, VDIV, and VMAX family for PS/PD/SS/SD, plus +// VSQRTSS/VSQRTSD, plus raw-only packed/scalar FP16 MAP5 binary forms. Packed SQRT +// has a distinct single-source grammar and is decoded by +// decodedX86SameWidthConversionInstruction. Scalar VEX forms require VEX.128; +// packed VEX forms accept both vector widths. EVEX adds X/Y/Z widths, masking, +// zeroing, packed broadcasts, rounding, and SAE. +// Recover every register and ModRM/SIB memory form before the generic decoder. +func decodedX86VEXBinaryFloatInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXBinaryFloatInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXBinaryFloatInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXBinaryFloatInstructionWithMemory(code, i, mode, segment, addressOverride, memory) + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + stem, recognized := map[byte]string{ + 0x51: "VSQRT", + 0x58: "VADD", + 0x59: "VMUL", + 0x5c: "VSUB", + 0x5d: "VMIN", + 0x5e: "VDIV", + 0x5f: "VMAX", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + pp := vexBits & 3 + if opcode == 0x51 && pp < 2 { + return Instr{}, 0, false, nil + } + suffix := [...]string{"PS", "PD", "SS", "SD"}[pp] + scalar := pp >= 2 + if scalar && vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar binary floating instruction requires VEX.128") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + op := Op(stem + suffix) + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXBinaryFloatInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXBinaryFloatInstructionWithMemory(code, i, mode, segment, addressOverride, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXBinaryFloatInstructionWithMemory(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+6 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + stem, recognized := map[byte]string{ + 0x51: "VSQRT", + 0x58: "VADD", + 0x59: "VMUL", + 0x5c: "VSUB", + 0x5d: "VMIN", + 0x5e: "VDIV", + 0x5f: "VMAX", + }[opcode] + mapNumber := p0 & 0x0f + rawHalf := mapNumber == 5 && (p1&3 == 0 || p1&3 == 2) + rawHalfScalar := rawHalf && p1&3 == 2 + rawSqrt := stem == "VSQRT" && !rawHalf + if !recognized || mapNumber != 1 && !rawHalf { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + pp := p1 & 3 + if opcode == 0x51 && pp < 2 { + return Instr{}, 0, false, nil + } + width64 := p1&0x80 != 0 + wantWidth64 := pp == 1 || pp == 3 + if rawHalf { + wantWidth64 = false + } + if width64 != wantWidth64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match floating element type") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + scalar := pp >= 2 + evexB := p2&0x10 != 0 + registerSource := modRM>>6 == 3 + embeddedControl := evexB && registerSource + broadcast := evexB && !registerSource + if broadcast && scalar { + return Instr{}, 0, true, fmt.Errorf("scalar binary floating instruction does not support broadcast") + } + if !embeddedControl && vectorBits == 3 && !rawHalfScalar && !rawSqrt { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + vectorPrefix := "X" + vectorWidth := 16 + switch { + case embeddedControl && !scalar: + vectorPrefix = "Z" + vectorWidth = 64 + case !scalar: + vectorPrefix = [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth = [...]int{16, 32, 64}[vectorBits] + } + laneBytes := 4 + if rawHalf { + laneBytes = 2 + } else if pp == 1 || pp == 3 { + laneBytes = 8 + } + disp8Scale := vectorWidth + if scalar || broadcast { + disp8Scale = laneBytes + } + + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (!rawHalf && !rawSqrt && maskNumber != 0 || rExt != 0 || rHighExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register or mask in 32-bit mode") + } + firstSource, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + + suffix := [...]string{"PS", "PD", "SS", "SD"}[pp] + if rawHalf { + suffix = "PH" + if rawHalfScalar { + suffix = "SH" + } + } + op := Op(stem + suffix) + if broadcast { + op += ".BCST" + } + if embeddedControl { + if stem == "VMIN" || stem == "VMAX" { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[vectorBits] + } + } + if zeroing { + op += ".Z" + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{ + Op: op, Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + x86Encoded: rawHalf || rawSqrt, x86SqrtEVEX: rawSqrt, + x86FloatMinMaxEVEX: !rawHalf && (stem == "VMIN" || stem == "VMAX"), + }, modRMIndex + consumed, true, nil +} + +// decodedX86ScalarIntegerFloatInstruction recognizes Go 1.27's complete +// signed and unsigned scalar integer-to-floating-point family. VEX supports +// the four signed forms; EVEX additionally supports unsigned forms, ignored +// scalar L'L encodings, and register-only embedded rounding where enabled. +func decodedX86ScalarIntegerFloatInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXScalarIntegerFloatInstruction(code, i, mode, segment, addressOverride) + } + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case code[i] == 0xc4 && len(code) >= i+5 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + opcodeIndex := i + vexLength + if len(code) <= opcodeIndex || code[opcodeIndex] != 0x2a { + return Instr{}, 0, false, nil + } + ok = true + pp := vexBits & 3 + if pp != 2 && pp != 3 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be F3 or F2") + } + if vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar integer-to-float instruction requires VEX.128") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + passthroughNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || passthroughNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86RMSource(code[modRMIndex:], mode, bExt, xExt, segment, 1) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + width64 := vexLength == 3 && vexBits&0x80 != 0 + op := scalarIntegerFloatRawOperation(false, pp, width64) + passthrough := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", passthroughNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + args := []Operand{source, passthrough, destination} + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s, %s, %s", op, source.String(), passthrough.String(), destination.String())}, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXScalarIntegerFloatInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+6 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 1 || opcode != 0x2a && opcode != 0x7b { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + pp := p1 & 3 + if pp != 2 && pp != 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX.pp must be F3 or F2") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x87 != 0 { + return Instr{}, 0, true, fmt.Errorf("masking and zeroing are absent from the scalar integer-to-float table") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + embeddedRounding := p2&0x10 != 0 + if !embeddedRounding && vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + if embeddedRounding && modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("embedded rounding requires a register source") + } + width64 := p1&0x80 != 0 + roundingEnabled := pp == 2 || width64 + if embeddedRounding && !roundingEnabled { + return Instr{}, 0, true, fmt.Errorf("explicit rounding is absent from VCVTSI2SDL/VCVTUSI2SDL") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + passthroughNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if modRM>>6 == 3 && xExt != 0 { + return Instr{}, 0, true, fmt.Errorf("general register source cannot use the EVEX high vector extension") + } + if mode == 32 && (rExt != 0 || rHighExt != 0 || xExt != 0 || bExt != 0 || passthroughNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + sourceBytes := 4 + if width64 && mode == 64 { + // Signed and unsigned scalar conversions ignore W1 in mode32, + // including the independently observed m32 compressed tuple. + sourceBytes = 8 + } + source, consumed, decodeErr := decodedX86RMSource(code[modRMIndex:], mode, bExt, xExt, segment, sourceBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + op := scalarIntegerFloatRawOperation(opcode == 0x7b, pp, width64) + if embeddedRounding { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[vectorBits] + } + passthrough := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", passthroughNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + args := []Operand{source, passthrough, destination} + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s, %s, %s", op, source.String(), passthrough.String(), destination.String())}, modRMIndex + consumed, true, nil +} + +func scalarIntegerFloatRawOperation(unsigned bool, pp byte, width64 bool) Op { + prefix := "VCVTSI2" + if unsigned { + prefix = "VCVTUSI2" + } + result := "SS" + if pp == 3 { + result = "SD" + } + source := "L" + if width64 { + source = "Q" + } + return Op(prefix + result + source) +} + +// decodedX86VectorFloatCompareInstruction recognizes the complete VEX and +// EVEX VCMPPD/PS/SD/SS family. VEX writes X/Y vector results. EVEX writes K +// results and adds an optional K write mask, packed broadcast, and SAE. +func decodedX86VectorFloatCompareInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXVectorFloatCompareInstruction(code, i, mode, segment, addressOverride) + } + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case code[i] == 0xc4 && len(code) >= i+6 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + opcodeIndex := i + vexLength + if len(code) <= opcodeIndex || code[opcodeIndex] != 0xc2 { + return Instr{}, 0, false, nil + } + ok = true + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + pp := vexBits & 3 + scalar := pp >= 2 + if scalar && vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar vector floating compare requires VEX.128") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("%w: raw mode32 VEX compare high-bank identity", ErrProbeNeedsContext) + } + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + op := vectorFloatCompareRawOperation(pp) + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{immediate, firstSource, secondSource, destination} + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), firstSource.String(), secondSource.String(), destination.String()), x86Encoded: true}, immediateIndex + 1, true, nil +} + +func decodedX86EVEXVectorFloatCompareInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+7 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 1 || code[i+4] != 0xc2 { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + pp := p1 & 3 + width64 := p1&0x80 != 0 + wantWidth64 := pp == 1 || pp == 3 + if width64 != wantWidth64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match floating element type") + } + if p2&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing is absent from vector floating compare") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + if rExt != 0 || rHighExt != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX destination is an unextended K register") + } + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + scalar := pp >= 2 + evexB := p2&0x10 != 0 + registerSource := modRM>>6 == 3 + sae := evexB && registerSource + broadcast := evexB && !registerSource + if broadcast && scalar { + return Instr{}, 0, true, fmt.Errorf("scalar vector floating compare does not support broadcast") + } + if !sae && vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + vectorPrefix := "X" + vectorWidth := 16 + switch { + case sae && !scalar: + vectorPrefix = "Z" + vectorWidth = 64 + case !scalar: + vectorPrefix = [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth = [...]int{16, 32, 64}[vectorBits] + } + laneBytes := 4 + if width64 { + laneBytes = 8 + } + disp8Scale := vectorWidth + if scalar || broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + + op := vectorFloatCompareRawOperation(pp) + if broadcast { + op += ".BCST" + } + if sae { + op += ".SAE" + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + if mode == 32 && (xExt != 0 || bExt != 0 || secondSourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("%w: raw mode32 EVEX compare high-bank identity", ErrProbeNeedsContext) + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM>>3&7))} + args := []Operand{immediate, firstSource, secondSource} + if writeMask := int(p2 & 7); writeMask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", writeMask))}) + } + args = append(args, destination) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), x86Encoded: true}, immediateIndex + 1, true, nil +} + +func vectorFloatCompareRawOperation(pp byte) Op { + return [...]Op{"VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"}[pp] +} + +// decodedX86VEXHorizontalFloatInstruction recognizes the complete VEX.128 and +// VEX.256 VHADD/VHSUB/VADDSUB PS/PD family in Go 1.27's _yvaddsubpd table. +// VEX.W is ignored. Recover every register and ModRM/SIB memory form before +// the generic decoder, which can misdecode these bytes as relative branches. +func decodedX86VEXHorizontalFloatInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + stem, recognized := map[byte]string{0x7c: "VHADD", 0x7d: "VHSUB", 0xd0: "VADDSUB"}[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + suffix := "" + switch vexBits & 3 { + case 1: + suffix = "PD" + case 3: + suffix = "PS" + default: + return Instr{}, 0, true, fmt.Errorf("VEX.pp does not select a horizontal floating instruction") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + op := Op(stem + suffix) + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +type decodedX86VEXFMA3Spec struct { + stem string + scalar bool +} + +func (spec decodedX86VEXFMA3Spec) op(width64 bool) Op { + suffix := "PS" + if spec.scalar { + suffix = "SS" + } + if width64 { + if spec.scalar { + suffix = "SD" + } else { + suffix = "PD" + } + } + return Op(spec.stem + suffix) +} + +// decodedX86VEXFMA3Ops covers all 36 packed and 24 scalar FMA3 names in Go +// 1.27's _yvaddpd and _yvaddsd operand tables. VEX.W selects PS/PD or SS/SD. +var decodedX86VEXFMA3Ops = map[byte]decodedX86VEXFMA3Spec{ + 0x96: {stem: "VFMADDSUB132"}, + 0x97: {stem: "VFMSUBADD132"}, + 0x98: {stem: "VFMADD132"}, + 0x99: {stem: "VFMADD132", scalar: true}, + 0x9a: {stem: "VFMSUB132"}, + 0x9b: {stem: "VFMSUB132", scalar: true}, + 0x9c: {stem: "VFNMADD132"}, + 0x9d: {stem: "VFNMADD132", scalar: true}, + 0x9e: {stem: "VFNMSUB132"}, + 0x9f: {stem: "VFNMSUB132", scalar: true}, + 0xa6: {stem: "VFMADDSUB213"}, + 0xa7: {stem: "VFMSUBADD213"}, + 0xa8: {stem: "VFMADD213"}, + 0xa9: {stem: "VFMADD213", scalar: true}, + 0xaa: {stem: "VFMSUB213"}, + 0xab: {stem: "VFMSUB213", scalar: true}, + 0xac: {stem: "VFNMADD213"}, + 0xad: {stem: "VFNMADD213", scalar: true}, + 0xae: {stem: "VFNMSUB213"}, + 0xaf: {stem: "VFNMSUB213", scalar: true}, + 0xb6: {stem: "VFMADDSUB231"}, + 0xb7: {stem: "VFMSUBADD231"}, + 0xb8: {stem: "VFMADD231"}, + 0xb9: {stem: "VFMADD231", scalar: true}, + 0xba: {stem: "VFMSUB231"}, + 0xbb: {stem: "VFMSUB231", scalar: true}, + 0xbc: {stem: "VFNMADD231"}, + 0xbd: {stem: "VFNMADD231", scalar: true}, + 0xbe: {stem: "VFNMSUB231"}, + 0xbf: {stem: "VFNMSUB231", scalar: true}, +} + +// decodedX86VEXFMA3Instruction recognizes every VEX and EVEX FMA3 form behind +// Go 1.27's complete FMA3 table. EVEX adds X/Y/Z widths, masking, zeroing, +// packed broadcasts, and embedded rounding. x/arch v0.14 can split or reject +// raw FMA encodings emitted by generated ecosystem assembly, so recover the +// family before the generic decoder. +func decodedX86VEXFMA3Instruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXFMA3InstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXFMA3InstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXFMA3InstructionWithMemory(code, i, mode, segment, addressOverride, memory) + } + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 { + return Instr{}, 0, false, nil + } + vex0, vexBits, opcode := code[i+1], code[i+2], code[i+3] + spec, recognized := decodedX86VEXFMA3Ops[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if spec.scalar && vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar FMA3 requires VEX.128") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + op := spec.op(vexBits&0x80 != 0) + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + x86FMA3Encoding: x86RawFMA3VEX, + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXFMA3Instruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXFMA3InstructionWithMemory(code, i, mode, segment, addressOverride, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXFMA3InstructionWithMemory(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+6 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + spec, recognized := decodedX86VEXFMA3Ops[opcode] + half := p0&0x0f == 6 + if p0&0x0f != 2 && !half || !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 || p1&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("EVEX.pp must be 66") + } + if half && p1&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("FP16 FMA3 requires EVEX.W0") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + evexB := p2&0x10 != 0 + registerSource := modRM>>6 == 3 + embeddedRounding := evexB && registerSource + broadcast := evexB && !registerSource + if broadcast && spec.scalar { + return Instr{}, 0, true, fmt.Errorf("scalar FMA3 does not support broadcast") + } + if !embeddedRounding && vectorBits == 3 && !(half && spec.scalar) { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + vectorPrefix := "X" + vectorWidth := 16 + switch { + case embeddedRounding && !spec.scalar: + vectorPrefix = "Z" + vectorWidth = 64 + case !spec.scalar: + vectorPrefix = [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth = [...]int{16, 32, 64}[vectorBits] + } + laneBytes := 4 + width64 := p1&0x80 != 0 + if width64 { + laneBytes = 8 + } + if half { + laneBytes = 2 + } + disp8Scale := vectorWidth + if spec.scalar || broadcast { + disp8Scale = laneBytes + } + + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (rExt != 0 || rHighExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + + op := spec.op(width64) + if half { + op = Op(spec.stem + "PH") + if spec.scalar { + op = Op(spec.stem + "SH") + } + } + if broadcast { + op += ".BCST" + } + if embeddedRounding { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[vectorBits] + } + if zeroing { + op += ".Z" + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + encoding := x86RawFMA3EVEX + if half { + encoding = x86RawFMA3Absent // The existing FP16 owner remains distinct. + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + x86Encoded: half, x86FMA3Encoding: encoding}, modRMIndex + consumed, true, nil +} + +// decodedX86VEXTRACT128Instruction recognizes the complete Go 1.27 vector +// lane-extract family: the VEX F128/I128 forms and all EVEX F/I 32/64-bit +// 128/256-bit block forms. x/arch v0.14 can misdecode these raw bytes as +// PC-relative control flow or reject the EVEX encodings outright. +func decodedX86VEXTRACT128Instruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXTRACT128InstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXTRACT128InstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXTRACTInstructionWithMemory(code, i, mode, segment, addressOverride, memory) + } + if len(code) < i+6 || code[i] != 0xc4 || code[i+1]&0x1f != 3 { + return Instr{}, 0, false, nil + } + vex0, vexBits, opcode := code[i+1], code[i+2], code[i+3] + op, recognized := map[byte]Op{0x19: "VEXTRACTF128", 0x39: "VEXTRACTI128"}[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("VEXTRACT128 requires VEX.256") + } + if int(^vexBits>>3)&15 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be 1111") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + sourceNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || sourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + destination, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", sourceNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +func decodedX86EVEXTRACTInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXTRACTInstructionWithMemory(code, i, mode, segment, addressOverride, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXTRACTInstructionWithMemory(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 3 || p1&0x07 != 5 { + return Instr{}, 0, false, nil + } + opcode := code[i+4] + width64 := p1&0x80 != 0 + outputBytes := 0 + op := Op("") + switch opcode { + case 0x19: + outputBytes = 16 + op = "VEXTRACTF32X4" + if width64 { + op = "VEXTRACTF64X2" + } + case 0x1b: + outputBytes = 32 + op = "VEXTRACTF32X8" + if width64 { + op = "VEXTRACTF64X4" + } + case 0x39: + outputBytes = 16 + op = "VEXTRACTI32X4" + if width64 { + op = "VEXTRACTI64X2" + } + case 0x3b: + outputBytes = 32 + op = "VEXTRACTI32X8" + if width64 { + op = "VEXTRACTI64X4" + } + default: + return Instr{}, 0, false, nil + } + ok = true + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for vector extract") + } + vectorBits := (p2 >> 5) & 3 + if outputBytes == 16 && vectorBits != 1 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("128-bit extract requires a 256- or 512-bit source") + } + if outputBytes == 32 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("256-bit extract requires a 512-bit source") + } + sourceName := [...]string{"", "Y", "Z"}[vectorBits] + destinationName := "X" + if outputBytes == 32 { + destinationName = "Y" + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + if zeroing && modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing is invalid for a memory destination") + } + sourceNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + destinationNumber := int(modRM&7) + bExt*8 + xExt*16 + if mode == 32 && (sourceNumber >= 8 || modRM>>6 == 3 && destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + destination, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, destinationName, outputBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", sourceName, sourceNumber))} + args := []Operand{immediate, source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86VSHUFPSInstruction recognizes Go 1.27's complete shared +// VSHUFPS/VSHUFPD operand table. x/arch v0.14 rejects some extended VEX +// encodings and all EVEX forms used by generated ecosystem assembly. +func decodedX86VSHUFPSInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXPackedFloatShuffleInstruction(code, i, mode, segment, addressOverride) + } + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case len(code) >= i+3 && code[i] == 0xc5 && code[i+2] == 0xc6: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1 && code[i+3] == 0xc6: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + ok = true + op := Op("") + switch vexBits & 3 { + case 0: + op = "VSHUFPS" + case 1: + op = "VSHUFPD" + default: + return Instr{}, 0, true, fmt.Errorf("VEX.pp does not select VSHUFPS or VSHUFPD") + } + // Both VEX members are WIG in the ISA, independently of Go's W0 + // encoder choice. EVEX retains its separate PS W0 / PD W1 grammar. + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, err := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if err != nil { + return Instr{}, 0, true, err + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +func decodedX86EVEXPackedFloatShuffleInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+7 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 1 || p1&0x04 == 0 || code[i+4] != 0xc6 { + return Instr{}, 0, false, nil + } + op := Op("") + switch { + case p1&3 == 0 && p1&0x80 == 0: + op = "VSHUFPS" + case p1&3 == 1 && p1&0x80 != 0: + op = "VSHUFPD" + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = 4 + if op == "VSHUFPD" { + disp8Scale = 8 + } + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{immediate, firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86PackedShuffleInstruction recognizes the complete Go 1.27 +// _yvpshufd family: VPSHUFD, VPSHUFHW, and VPSHUFLW across VEX.128, +// VEX.256, and EVEX.128/256/512, including memory, mask, zeroing, and the +// VPSHUFD broadcast forms. x/arch v0.14 can split the VEX bytes emitted by +// Weaviate into a relative branch, so recover the family before consulting +// the generic decoder. +func decodedX86PackedShuffleInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86PackedShuffleInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedShuffleInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXPackedShuffleInstruction(code, i, mode, segment, addressOverride, memory) + } + + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case len(code) >= i+3 && code[i] == 0xc5 && code[i+2] == 0x70: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1 && code[i+3] == 0x70: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + op := map[byte]Op{1: "VPSHUFD", 2: "VPSHUFHW", 3: "VPSHUFLW"}[vexBits&3] + if op == "" { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + source, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +func decodedX86EVEXPackedShuffleInstruction(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 1 || p1&0x04 == 0 || code[i+4] != 0x70 { + return Instr{}, 0, false, nil + } + op := map[byte]Op{1: "VPSHUFD", 2: "VPSHUFHW", 3: "VPSHUFLW"}[p1&3] + if op == "" || op == "VPSHUFD" && p1&0x80 != 0 { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 packed shuffle mask form exceeds the Go assembler frontend's operand limit") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && op != "VPSHUFD" { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is only available for VPSHUFD") + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = 4 + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{immediate, source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86Shuffle128BitBlocksInstruction recognizes the complete EVEX-only +// Go 1.27 _yvshuff32x4 family: VSHUFF32X4, VSHUFF64X2, VSHUFI32X4, and +// VSHUFI64X2. x/arch v0.14 rejects these raw encodings, including the +// VSHUFF64X2 sequence in github.com/minio/sha256-simd. +func decodedX86Shuffle128BitBlocksInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+7 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 3 || p1&0x07 != 5 || opcode != 0x23 && opcode != 0x43 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits != 1 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("VSHUFF/VSHUFI block shuffle requires a 256- or 512-bit vector") + } + vectorPrefix := [...]string{"", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{0, 32, 64}[vectorBits] + width64 := p1&0x80 != 0 + op := Op("VSHUFF32X4") + laneBytes := 4 + if opcode == 0x43 { + op = "VSHUFI32X4" + } + if width64 { + laneBytes = 8 + if opcode == 0x23 { + op = "VSHUFF64X2" + } else { + op = "VSHUFI64X2" + } + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{immediate, firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86VPSHUFBInstruction recognizes every VEX and EVEX encoding behind +// Go 1.27's _yvandnpd table. x/arch v0.14 rejects the EVEX forms used by +// github.com/minio/sha256-simd. +func decodedX86VPSHUFBInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VPSHUFBInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VPSHUFBInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + if code[i] == 0xc4 && code[i+1]&0x1f == 2 && code[i+3] == 0x00 { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vex0, vex1 := code[i+1], code[i+2] + if vex1&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + // Intel PSHUFB specifies WIG for both VEX and EVEX. Go emits W0, + // but that encoding choice cannot reject independently decoded W1. + if len(code) <= i+4 { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + vectorName := "X" + if vex1&0x04 != 0 { + vectorName = "Y" + } + modRMIndex := i + 4 + modRM := code[modRMIndex] + dataNumber := int(^vex1>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (dataNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + control, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, dataNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: "VPSHUFB", + Args: []Operand{control, data, destination}, + Raw: fmt.Sprintf("VPSHUFB %s, %s, %s", control.String(), data.String(), destination.String()), + x86Encoded: true, + }, modRMIndex + consumed, true, nil + } + + if len(code) < i+5 || code[i] != 0x62 || code[i+1]&0x0f != 2 || code[i+2]&0x07 != 5 || code[i+4] != 0x00 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for VPSHUFB") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + dataNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (dataNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + control, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, vectorWidth) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, dataNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{control, data} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := Op("VPSHUFB") + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), x86Encoded: true}, modRMIndex + consumed, true, nil +} + +type decodedX86PackedExtendProperties struct { + op Op + inputBits int + outputBits int +} + +func decodedX86PackedExtendOpcode(opcode byte) (decodedX86PackedExtendProperties, bool) { + properties := map[byte]decodedX86PackedExtendProperties{ + 0x20: {op: "VPMOVSXBW", inputBits: 8, outputBits: 16}, + 0x21: {op: "VPMOVSXBD", inputBits: 8, outputBits: 32}, + 0x22: {op: "VPMOVSXBQ", inputBits: 8, outputBits: 64}, + 0x23: {op: "VPMOVSXWD", inputBits: 16, outputBits: 32}, + 0x24: {op: "VPMOVSXWQ", inputBits: 16, outputBits: 64}, + 0x25: {op: "VPMOVSXDQ", inputBits: 32, outputBits: 64}, + 0x30: {op: "VPMOVZXBW", inputBits: 8, outputBits: 16}, + 0x31: {op: "VPMOVZXBD", inputBits: 8, outputBits: 32}, + 0x32: {op: "VPMOVZXBQ", inputBits: 8, outputBits: 64}, + 0x33: {op: "VPMOVZXWD", inputBits: 16, outputBits: 32}, + 0x34: {op: "VPMOVZXWQ", inputBits: 16, outputBits: 64}, + 0x35: {op: "VPMOVZXDQ", inputBits: 32, outputBits: 64}, + }[opcode] + return properties, properties.op != "" +} + +// decodedX86PackedExtendInstruction recognizes all six signed and all six +// unsigned packed extending moves in Go 1.27. It covers their VEX.128/256 and +// EVEX.128/256/512 register and memory forms, including high registers, +// writemasks, zeroing, and compressed displacements. EVEX.b is deliberately +// rejected because no broadcast form exists in the Go operand tables. +func decodedX86PackedExtendInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXPackedExtendInstruction(code, i, mode, segment, addressOverride) + } + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 2 { + return Instr{}, 0, false, nil + } + vex0, vex1 := code[i+1], code[i+2] + properties, recognized := decodedX86PackedExtendOpcode(code[i+3]) + // Intel SDM PMOVSX/PMOVZX: VEX.W is ignored for all six ratios. + // Go chooses W0, but raw W1 is not an invalid instruction. + if !recognized || vex1&3 != 1 { + return Instr{}, 0, false, nil + } + ok = true + if vex1&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRMIndex := i + 4 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (destinationNumber >= 8 || code[modRMIndex]>>6 == 3 && int(code[modRMIndex]&7)+bExt*8 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + destinationName := "X" + if vex1&0x04 != 0 { + destinationName = "Y" + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationName, destinationNumber))} + instruction = Instr{ + Op: properties.op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", properties.op, source.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXPackedExtendInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 2 || p1&0x07 != 5 { + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86PackedExtendOpcode(code[i+4]) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + // The EVEX DQ forms alone require W0; the other five ratios are WIG. + // Keep this physical constraint distinct from Go's W0 encoder choice. + if properties.inputBits == 32 && p1&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed dword-to-qword extension requires W0") + } + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for packed extending moves") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + destinationName := [...]string{"X", "Y", "Z"}[vectorBits] + destinationBytes := [...]int{16, 32, 64}[vectorBits] + sourceBytes := destinationBytes * properties.inputBits / properties.outputBits + sourceName := "X" + if sourceBytes > 16 { + sourceName = "Y" + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + sourceNumber := int(modRM&7) + bExt*8 + xExt*16 + if mode == 32 && (destinationNumber >= 8 || modRM>>6 == 3 && sourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, sourceName, sourceBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationName, destinationNumber))} + args := []Operand{source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := properties.op + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +type decodedX86PackedUniformShiftProperties struct { + op Op + laneBytes int + immediate bool +} + +func decodedX86PackedUniformShiftOpcode(opcode, modRM byte, evex, w bool) (decodedX86PackedUniformShiftProperties, bool) { + laneBytes := map[byte]int{0x71: 2, 0x72: 4, 0x73: 8}[opcode] + if laneBytes != 0 { + direction := modRM >> 3 & 7 + if direction != 2 && direction != 4 && direction != 6 { + return decodedX86PackedUniformShiftProperties{}, false + } + stem := "VPSRL" + switch direction { + case 4: + if laneBytes == 8 { + return decodedX86PackedUniformShiftProperties{}, false + } + stem = "VPSRA" + if evex && w && laneBytes == 4 { + laneBytes = 8 + } + case 6: + stem = "VPSLL" + } + suffix := map[int]string{2: "W", 4: "D", 8: "Q"}[laneBytes] + return decodedX86PackedUniformShiftProperties{op: Op(stem + suffix), laneBytes: laneBytes, immediate: true}, true + } + variable := map[byte]struct { + op Op + laneBytes int + }{ + 0xf1: {op: "VPSLLW", laneBytes: 2}, + 0xf2: {op: "VPSLLD", laneBytes: 4}, + 0xf3: {op: "VPSLLQ", laneBytes: 8}, + 0xd1: {op: "VPSRLW", laneBytes: 2}, + 0xd2: {op: "VPSRLD", laneBytes: 4}, + 0xd3: {op: "VPSRLQ", laneBytes: 8}, + 0xe1: {op: "VPSRAW", laneBytes: 2}, + 0xe2: {op: "VPSRAD", laneBytes: 4}, + } + properties, ok := variable[opcode] + if !ok { + return decodedX86PackedUniformShiftProperties{}, false + } + if evex && w && opcode == 0xe2 { + properties.op, properties.laneBytes = "VPSRAQ", 8 + } + return decodedX86PackedUniformShiftProperties{op: properties.op, laneBytes: properties.laneBytes}, true +} + +// decodedX86PackedUniformShiftInstruction recognizes the complete Go 1.27 +// immediate and XMM/m128-count grammar: six logical shifts and three +// arithmetic right shifts across VEX/EVEX X/Y/Z widths. x/arch v0.14 +// misdecodes several VEX/EVEX forms used by external assembly. +func decodedX86PackedUniformShiftInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86PackedUniformShiftInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedUniformShiftInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + + var vexBits byte + var opcodeIndex, modRMIndex, rExt, xExt, bExt int + isVEX := false + vexHasW := false + switch { + case code[i] == 0xc5: + vexBits = code[i+1] + opcodeIndex = i + 2 + modRMIndex = i + 3 + rExt = int(^vexBits>>7) & 1 + isVEX = true + case len(code) >= i+5 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vex0 := code[i+1] + vexBits = code[i+2] + opcodeIndex = i + 3 + modRMIndex = i + 4 + rExt = int(^vex0>>7) & 1 + xExt = int(^vex0>>6) & 1 + bExt = int(^vex0>>5) & 1 + isVEX = true + vexHasW = true + } + if isVEX { + properties, recognized := decodedX86PackedUniformShiftOpcode(code[opcodeIndex], code[modRMIndex], false, vexHasW && vexBits&0x80 != 0) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + modRM := code[modRMIndex] + vvvvNumber := int(^vexBits>>3) & 15 + if properties.immediate { + if modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("VEX immediate packed shift requires a register source") + } + if rExt != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.R must not extend the shift opcode selector") + } + if mode == 32 && vvvvNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, vvvvNumber))} + return Instr{ + Op: properties.op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", properties.op, immediate.String(), source.String(), destination.String()), + x86Encoded: true, + x86PackedShiftEncoding: x86RawPackedShiftVEX, + x86PackedShiftW: vexHasW && vexBits&0x80 != 0, + }, immediateIndex + 1, true, nil + } + + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (vvvvNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + count, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, vvvvNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: properties.op, + Args: []Operand{count, data, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", properties.op, count.String(), data.String(), destination.String()), + x86Encoded: true, + x86PackedShiftEncoding: x86RawPackedShiftVEX, + x86PackedShiftW: vexHasW && vexBits&0x80 != 0, + }, modRMIndex + consumed, true, nil + } + + if len(code) < i+6 || code[i] != 0x62 || code[i+1]&0x0f != 1 || code[i+2]&0x07 != 5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + modRMIndex = i + 5 + modRM := code[modRMIndex] + properties, recognized := decodedX86PackedUniformShiftOpcode(code[i+4], modRM, true, p1&0x80 != 0) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if properties.laneBytes != 2 && (p1&0x80 != 0) != (properties.laneBytes == 8) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match %s", properties.op) + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + rExt = int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt = int(^p0>>6) & 1 + bExt = int(^p0>>5) & 1 + vvvvNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + broadcast := p2&0x10 != 0 + if properties.immediate { + if p0&0x90 != 0x90 { + return Instr{}, 0, true, fmt.Errorf("EVEX.R and EVEX.R' must not extend the shift opcode selector") + } + if broadcast && properties.laneBytes == 2 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for %s", properties.op) + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + if mode == 32 && (vvvvNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = properties.laneBytes + } + source, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, vvvvNumber))} + args := []Operand{immediate, source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := properties.op + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{ + Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + x86Encoded: true, x86PackedShiftEncoding: x86RawPackedShiftEVEX, x86PackedShiftW: p1&0x80 != 0, + }, immediateIndex + 1, true, nil + } + + if broadcast { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for the uniform-count form of %s", properties.op) + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (vvvvNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + count, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, "X", 16) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, vvvvNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{count, data} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := properties.op + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{ + Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + x86Encoded: true, x86PackedShiftEncoding: x86RawPackedShiftEVEX, x86PackedShiftW: p1&0x80 != 0, + }, modRMIndex + consumed, true, nil +} + +// decodedX86TernaryLogicInstruction recognizes VPTERNLOGD/Q across the +// complete Go 1.27 _yvalignd table. x/arch v0.14 rejects the EVEX encodings +// used by github.com/minio/sha256-simd. +func decodedX86TernaryLogicInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86TernaryLogicInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86TernaryLogicInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + instruction, length, ok, err = decodedX86EVEXImmediateThreeVectorPhysical(code, mode, + 0x25, "VPTERNLOGD", "VPTERNLOGQ", "VPTERNLOG", memory, true) + if ok && err == nil { + instruction.x86Encoded = true + instruction.x86VectorBytes = amd64VectorByteWidth(instruction.Args[len(instruction.Args)-1].Reg) + if !memory.physicalPC { + arch := "amd64" + if mode == 32 { + arch = "386" + } + _, _, err = x86TernaryLogicProperties(instruction.Op, instruction, arch) + } + } + return instruction, length, ok, err +} + +// decodedX86VectorAlignInstruction recognizes VALIGND/Q across the complete +// Go 1.27 _yvalignd table. +func decodedX86VectorAlignInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXImmediateThreeVectorInstruction(code, mode, 0x03, "VALIGND", "VALIGNQ", "VALIGN") +} + +// decodedX86FunnelShiftInstruction recognizes the complete Go 1.27 VBMI2 +// funnel-shift family: immediate/variable, left/right, and W/D/Q lanes. +// x/arch v0.14 rejects all of these EVEX encodings. +func decodedX86FunnelShiftInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 || code[i+2]&0x07 != 5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + mapNumber := p0 & 0x0f + variable := false + switch mapNumber { + case 2: + variable = true + case 3: + default: + return Instr{}, 0, false, nil + } + opcode := code[i+4] + wBit := p1&0x80 != 0 + laneBits := 0 + left := false + switch opcode { + case 0x70: + left, laneBits = true, 16 + case 0x71: + left = true + if wBit { + laneBits = 64 + } else { + laneBits = 32 + } + case 0x72: + laneBits = 16 + case 0x73: + if wBit { + laneBits = 64 + } else { + laneBits = 32 + } + default: + return Instr{}, 0, false, nil + } + ok = true + if laneBits == 16 && !wBit { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be one for word funnel shifts") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if !variable && mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 immediate funnel shifts exceed the Go assembler frontend's operand limit") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if variable && mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 variable funnel-shift mask form exceeds the Go assembler frontend's operand limit") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && laneBits == 16 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for word funnel shifts") + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBits / 8 + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + opDirection := "R" + if left { + opDirection = "L" + } + laneSuffix := map[int]string{16: "W", 32: "D", 64: "Q"}[laneBits] + op := Op("VPSH" + opDirection + "D") + if variable { + op += "V" + } + op += Op(laneSuffix) + args := make([]Operand, 0, 5) + end := modRMIndex + consumed + if !variable { + if len(code) <= end { + return Instr{}, 0, false, nil + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[end])}) + end++ + } + args = append(args, firstSource, secondSource) + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, end, true, nil +} + +// decodedX86EVEXImmediateThreeVectorInstruction implements the EVEX immediate, +// r/m, vvvv, destination shape shared by Go 1.27's _yvalignd families. +func decodedX86EVEXImmediateThreeVectorInstruction(code []byte, mode int, opcode byte, dwordOp, qwordOp Op, family string) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXImmediateThreeVectorInstructionWithMemory(code, mode, opcode, dwordOp, qwordOp, family, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXImmediateThreeVectorInstructionWithMemory(code []byte, mode int, opcode byte, dwordOp, qwordOp Op, family string, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXImmediateThreeVectorPhysical(code, mode, opcode, dwordOp, qwordOp, family, memory, false) +} + +func decodedX86EVEXImmediateThreeVectorPhysical(code []byte, mode int, opcode byte, dwordOp, qwordOp Op, family string, memory x86RawMemoryDecoder, physicalMode32 bool) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+7 || code[i] != 0x62 || code[i+1]&0x0f != 3 || code[i+2]&0x07 != 5 || code[i+4] != opcode { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && !physicalMode32 { + return Instr{}, 0, true, fmt.Errorf("386 %s is rejected by the Go assembler frontend's operand limit", family) + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + laneBytes := 4 + op := dwordOp + if p1&0x80 != 0 { + laneBytes = 8 + op = qwordOp + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + if mode == 32 { + firstNumber, firstRegister := amd64VectorRegisterIndex(firstSource.Reg, vectorWidth) + if destinationNumber >= 8 || secondSourceNumber >= 8 || firstRegister && firstNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit %s", family) + } + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{immediate, firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86PackedImmediateRotateInstruction recognizes all four EVEX +// immediate rotates in Go 1.27's _yvprold table. x/arch v0.14 rejects these +// raw encodings, including VPRORD in github.com/minio/sha256-simd. +func decodedX86PackedImmediateRotateInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+7 || code[i] != 0x62 || code[i+1]&0x0f != 1 || code[i+2]&0x07 != 5 || code[i+4] != 0x72 { + return Instr{}, 0, false, nil + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + selector := modRM >> 3 & 7 + if selector != 0 && selector != 1 { + // Opcode 72 is a grouped encoding. Selectors /2 and /6 are the + // VPSRL/VPSLL logical-shift family, and other selectors may belong to + // decoders added later. This decoder only owns the rotate selectors. + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x90 != 0x90 { + return Instr{}, 0, true, fmt.Errorf("EVEX.R and EVEX.R' must not extend the rotate opcode selector") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + laneBytes := 4 + left := false + switch selector { + case 0: + case 1: + left = true + } + op := Op("VPRORD") + if left { + op = "VPROLD" + } + if p1&0x80 != 0 { + laneBytes = 8 + if left { + op = "VPROLQ" + } else { + op = "VPRORQ" + } + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 %s mask form exceeds the Go assembler frontend's operand limit", op) + } + + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + destinationNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + if mode == 32 && (destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{immediate, source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +var decodedX86PackedIntegerArithmeticOps = map[byte]struct { + op Op + laneBytes int +}{ + 0xfc: {op: "VPADDB", laneBytes: 1}, + 0xfd: {op: "VPADDW", laneBytes: 2}, + 0xfe: {op: "VPADDD", laneBytes: 4}, + 0xd4: {op: "VPADDQ", laneBytes: 8}, + 0xec: {op: "VPADDSB", laneBytes: 1}, + 0xed: {op: "VPADDSW", laneBytes: 2}, + 0xdc: {op: "VPADDUSB", laneBytes: 1}, + 0xdd: {op: "VPADDUSW", laneBytes: 2}, + 0xf8: {op: "VPSUBB", laneBytes: 1}, + 0xf9: {op: "VPSUBW", laneBytes: 2}, + 0xfa: {op: "VPSUBD", laneBytes: 4}, + 0xfb: {op: "VPSUBQ", laneBytes: 8}, + 0xe8: {op: "VPSUBSB", laneBytes: 1}, + 0xe9: {op: "VPSUBSW", laneBytes: 2}, + 0xd8: {op: "VPSUBUSB", laneBytes: 1}, + 0xd9: {op: "VPSUBUSW", laneBytes: 2}, +} + +// decodedX86PackedIntegerArithmeticInstruction recognizes the complete +// VEX/EVEX VPADD and VPSUB families behind Go 1.27's shared _yvandnpd table. +// x/arch v0.14 rejects EVEX forms and can split VEX forms used by ecosystem +// assembly. +func decodedX86PackedIntegerArithmeticInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86PackedIntegerArithmeticInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedIntegerArithmeticInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+3 { + return Instr{}, 0, false, nil + } + + var vexBits byte + var opcodeIndex, modRMIndex, rExt, xExt, bExt int + isVEX := false + switch { + case code[i] == 0xc5: + vexBits = code[i+1] + opcodeIndex = i + 2 + modRMIndex = i + 3 + rExt = int(^vexBits>>7) & 1 + isVEX = true + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vex0 := code[i+1] + vexBits = code[i+2] + opcodeIndex = i + 3 + modRMIndex = i + 4 + rExt = int(^vex0>>7) & 1 + xExt = int(^vex0>>6) & 1 + bExt = int(^vex0>>5) & 1 + isVEX = true + } + if isVEX { + properties, recognized := decodedX86PackedIntegerArithmeticOps[code[opcodeIndex]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + modRM := code[modRMIndex] + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + firstSource, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: properties.op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", properties.op, firstSource.String(), secondSource.String(), destination.String()), + }, modRMIndex + consumed, true, nil + } + + if len(code) < i+5 || code[i] != 0x62 || code[i+1]&0x0f != 1 || code[i+2]&0x07 != 5 { + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86PackedIntegerArithmeticOps[code[i+4]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + wBit := p1&0x80 != 0 + if wBit != (properties.laneBytes == 8) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match %s", properties.op) + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 %s mask form exceeds the Go assembler frontend's operand limit", properties.op) + } + + rExt = int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt = int(^p0>>6) & 1 + bExt = int(^p0>>5) & 1 + modRMIndex = i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && properties.laneBytes != 4 && properties.laneBytes != 8 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for %s", properties.op) + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = properties.laneBytes + } + firstSource, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := properties.op + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86IndexedPermuteInstruction recognizes the complete EVEX-only +// VPERMI2 and VPERMT2 families behind Go 1.27's shared _yvblendmpd table. +func decodedX86IndexedPermuteInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 2 || p1&0x07 != 5 || + (opcode < 0x75 || opcode > 0x77) && (opcode < 0x7d || opcode > 0x7f) { + return Instr{}, 0, false, nil + } + width64 := p1&0x80 != 0 + op := Op("") + laneBytes := 0 + switch opcode { + case 0x75: + if width64 { + op, laneBytes = "VPERMI2W", 2 + } else { + op, laneBytes = "VPERMI2B", 1 + } + case 0x76: + if width64 { + op, laneBytes = "VPERMI2Q", 8 + } else { + op, laneBytes = "VPERMI2D", 4 + } + case 0x77: + if width64 { + op, laneBytes = "VPERMI2PD", 8 + } else { + op, laneBytes = "VPERMI2PS", 4 + } + case 0x7d: + if width64 { + op, laneBytes = "VPERMT2W", 2 + } else { + op, laneBytes = "VPERMT2B", 1 + } + case 0x7e: + if width64 { + op, laneBytes = "VPERMT2Q", 8 + } else { + op, laneBytes = "VPERMT2D", 4 + } + case 0x7f: + if width64 { + op, laneBytes = "VPERMT2PD", 8 + } else { + op, laneBytes = "VPERMT2PS", 4 + } + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 indexed permute mask form exceeds the Go assembler operand limit") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && laneBytes < 4 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for byte and word VPERMI2") + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + if mode == 32 && vectorBits == 2 && + (destinationNumber >= 8 || secondSourceNumber >= 8 || + modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("386 Z register exceeds the Go assembler frontend's register class") + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// Go 1.27's _yvcompresspd and _yvexpandpd families share the same typed +// register/memory grammar, with the ModRM roles reversed for stores. +func decodedX86PackedCompressInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedCompactInstruction(code, mode, true) +} + +func decodedX86PackedExpandInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedCompactInstruction(code, mode, false) +} + +func decodedX86PackedCompactInstruction(code []byte, mode int, store bool) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 2 || p1&0x07 != 5 { + return Instr{}, 0, false, nil + } + if store && opcode != 0x63 && opcode != 0x8a && opcode != 0x8b || + !store && opcode != 0x62 && opcode != 0x88 && opcode != 0x89 { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed compact reserves the vvvv register field") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for packed compact") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + width64 := p1&0x80 != 0 + op := Op("") + laneBytes := 0 + switch opcode { + case 0x63: + if width64 { + op, laneBytes = "VPCOMPRESSW", 2 + } else { + op, laneBytes = "VPCOMPRESSB", 1 + } + case 0x8a: + if width64 { + op, laneBytes = "VCOMPRESSPD", 8 + } else { + op, laneBytes = "VCOMPRESSPS", 4 + } + case 0x8b: + if width64 { + op, laneBytes = "VPCOMPRESSQ", 8 + } else { + op, laneBytes = "VPCOMPRESSD", 4 + } + case 0x62: + if width64 { + op, laneBytes = "VPEXPANDW", 2 + } else { + op, laneBytes = "VPEXPANDB", 1 + } + case 0x88: + if width64 { + op, laneBytes = "VEXPANDPD", 8 + } else { + op, laneBytes = "VEXPANDPS", 4 + } + case 0x89: + if width64 { + op, laneBytes = "VPEXPANDQ", 8 + } else { + op, laneBytes = "VPEXPANDD", 4 + } + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + if mode == 32 && + (int(modRM>>3&7)+rExt*8+rHighExt*16 >= 8 || + modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register has no physical 32-bit binding") + } + regNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + reg := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, regNumber))} + rm, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, laneBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + args := []Operand{reg} + if !store { + args = []Operand{rm} + } + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + if store { + args = append(args, rm) + } else { + args = append(args, reg) + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86IFMAInstruction recognizes the complete EVEX-only +// VPMADD52HUQ/LUQ family behind Go 1.27's _yvblendmpd table. x/arch v0.14 +// rejects these encodings. +func decodedX86IFMAInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86EVEXQwordBinaryInstruction(code, mode, false) +} + +func decodedX86QwordLookupInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86EVEXQwordBinaryInstruction(code, mode, true) +} + +// decodedX86VPINSRInstruction recognizes the complete VEX.128 VPINSR +// byte/word/dword/qword family, including two- and three-byte VEX encodings and +// every ModRM/SIB source. x/arch v0.14 rejects some extended-register VPINSRQ +// forms used by generated crypto assembly. EVEX forms continue through x/arch. +func decodedX86VPINSRInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VPINSRInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VPINSRInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits byte + op := Op("") + switch { + case len(code) >= i+3 && code[i] == 0xc5 && code[i+2] == 0xc4: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + op = "VPINSRW" + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1 && code[i+3] == 0xc4: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + op = "VPINSRW" + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 3 && code[i+3] == 0x20: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + op = "VPINSRB" + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 3 && code[i+3] == 0x22: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + op = "VPINSRD" + if vexBits&0x80 != 0 { + op = "VPINSRQ" + } + default: + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must be zero") + } + if (op == "VPINSRB" || op == "VPINSRW") && vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero for %s", op) + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + firstVectorNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (op == "VPINSRQ" || rExt != 0 || xExt != 0 || bExt != 0 || firstVectorNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("64-bit width or extended register in 32-bit mode") + } + scalar, consumed, err := memory.scalar(code[modRMIndex:], mode, bExt, xExt, segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + firstVector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", firstVectorNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, scalar, firstVector, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), scalar.String(), firstVector.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86VPUNPCKInstruction recognizes the complete VEX.128/VEX.256 +// low/high byte, word, dword, and qword unpack family. x/arch v0.14 can split +// or reject VEX encodings with extended registers, so recover every VEX +// register and ModRM/SIB memory form before the generic decoder. EVEX forms +// use the separate decodedX86EVEXPackedUnpackInstruction grammar. +func decodedX86VPUNPCKInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + op, recognized := map[byte]Op{ + 0x60: "VPUNPCKLBW", + 0x61: "VPUNPCKLWD", + 0x62: "VPUNPCKLDQ", + 0x6c: "VPUNPCKLQDQ", + 0x68: "VPUNPCKHBW", + 0x69: "VPUNPCKHWD", + 0x6a: "VPUNPCKHDQ", + 0x6d: "VPUNPCKHQDQ", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, err := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if err != nil { + return Instr{}, 0, true, err + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86VEXVectorRMOperand(code []byte, mode, bExt, xExt int, segment Reg, vectorPrefix string) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if code[0]>>6 != 3 { + return decodedX86VEXRMSource(code, mode, bExt, xExt, segment) + } + number := int(code[0]&7) + bExt*8 + if mode == 32 && number >= 8 { + return Operand{}, 0, fmt.Errorf("extended vector register in 32-bit mode") + } + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, number))}, 1, nil +} + +// Preserve both physical integer-dot generations before generic decoding. +// The equivalent Go-named EVEX row cannot identify an original VEX opcode. +func decodedX86VNNIInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86IntegerPackedDotInstruction(code, mode) +} + +// decodedX86VariableDwordPermuteInstruction recognizes the complete VEX.256 +// VPERMD/VPERMPS family shared by Go 1.27's _yvpermd table. x/arch v0.14 can +// reject or split these encodings, so recover every register and ModRM/SIB +// memory source before consulting the generic decoder. EVEX forms continue +// through x/arch. +func decodedX86VariableDwordPermuteInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 2 { + return Instr{}, 0, false, nil + } + vexBits := code[i+2] + opcode := code[i+3] + op, recognized := map[byte]Op{ + 0x36: "VPERMD", + 0x16: "VPERMPS", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must be one") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + vex1 := code[i+1] + rExt := int(^vex1>>7) & 1 + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + modRMIndex := i + 4 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + indicesNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || indicesNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + data, consumed, err := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "Y") + if err != nil { + return Instr{}, 0, true, err + } + indices := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", indicesNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{data, indices, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, data.String(), indices.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86EVEXVariableDwordPermuteInstruction covers the Y/Z EVEX forms of +// Go 1.27's shared VPERMD/VPERMPS _yvpermd table. +func decodedX86EVEXVariableDwordPermuteInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 2 || p1&0x07 != 5 || (opcode != 0x36 && opcode != 0x16) { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be zero for VPERMD/VPERMPS") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits != 1 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("VPERMD/VPERMPS EVEX form requires Y or Z width") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 VPERMD/VPERMPS mask form exceeds the Go assembler operand limit") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = 4 + } + data, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + indicesNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && vectorBits == 2 && + (indicesNumber >= 8 || destinationNumber >= 8 || + modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("386 Z register exceeds the Go assembler frontend's register class") + } + indices := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, indicesNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{data, indices} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := Op("VPERMD") + if opcode == 0x16 { + op = "VPERMPS" + } + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86RandomInstruction recognizes the complete register-only +// RDRAND/RDSEED/RDPID encoding family. RDPID shares RDSEED's ModRM extension +// but requires F3 and has only a 32-bit destination. x/arch v0.14 drops the +// operand from 64-bit RDRAND and cannot decode RDSEED or RDPID, so recover +// these encodings before consulting the generic decoder. +func decodedX86RandomInstruction(code []byte, mode int) (syntax string, length int, ok bool) { + i := 0 + rdpid := len(code) > 0 && code[0] == 0xf3 + if rdpid { + i++ + } + bits := 32 + if len(code) > i && code[i] == 0x66 { + bits = 16 + i++ + } + rex := byte(0) + if mode == 64 && len(code) > i && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + if rex&0x08 != 0 { + bits = 64 + } + } + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0xc7 { + return "", 0, false + } + modRM := code[i+2] + if modRM>>6 != 3 { + return "", 0, false + } + family := "" + switch modRM >> 3 & 7 { + case 6: + if rdpid { + return "", 0, false + } + family = "RDRAND" + case 7: + if rdpid { + if bits != 32 { + return "", 0, false + } + family = "RDPID" + } else { + family = "RDSEED" + } + default: + return "", 0, false + } + register := int(modRM & 7) + if mode == 64 && rex&0x01 != 0 { + register += 8 + } + registerNames := [...]string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + if rdpid { + return fmt.Sprintf("RDPID %s", registerNames[register]), i + 3, true + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + return fmt.Sprintf("%s%s %s", family, width, registerNames[register]), i + 3, true +} + +// decodedX86VBROADCAST128Instruction recognizes the complete AVX2 +// VBROADCASTF128/I128 m128, Y family. x/arch v0.14 misdecodes these VEX +// opcodes as legacy scalar instructions, so recover every ModRM/SIB memory +// form before consulting the generic decoder. +func decodedX86VBROADCAST128Instruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VBROADCAST128InstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VBROADCAST128InstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + // VEX.256.66.0F38.W0 1A/5A /r. The unused encoded vvvv field must be + // 1111, so the complete third VEX byte is 0x7d. + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 || code[i+2] != 0x7d || + (code[i+3] != 0x1a && code[i+3] != 0x5a) { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + vex1 := code[i+1] + rExt := int(^vex1>>7) & 1 + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + destinationNumber := int(code[i+4]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + source, consumed, err := memory.scalar(code[i+4:], mode, bExt, xExt, segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + if source.Kind != OpMem { + return Instr{}, 0, true, fmt.Errorf("source must be m128 memory") + } + destination := Reg(fmt.Sprintf("Y%d", destinationNumber)) + op := Op("VBROADCASTI128") + if code[i+3] == 0x1a { + op = "VBROADCASTF128" + } + instruction = Instr{ + Op: op, + Args: []Operand{source, {Kind: OpReg, Reg: destination}}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination), + } + return instruction, i + 4 + consumed, true, nil +} + +// decodedX86SHAInstruction recognizes the complete Go 1.27 Intel SHA table: +// SHA1NEXTE, SHA1MSG1/2, SHA1RNDS4, SHA256MSG1/2, and SHA256RNDS2. Older +// x/arch decoders reject some raw 0F38 encodings used by minio/sha256-simd. +func decodedX86SHAInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86SHAInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86SHAInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := x86RawSegmentPrefixes(code) + rexPrefix := byte(0) + if mode == 64 && i < len(code) && code[i]&0xf0 == 0x40 { + rexPrefix = code[i] + i++ + } + if len(code) < i+4 || code[i] != 0x0f { + return Instr{}, 0, false, nil + } + + op := Op("") + encoding := uint16(code[i+1])<<8 | uint16(code[i+2]) + if encoding < 0x38c8 || encoding > 0x38cd && encoding != 0x3acc { + return Instr{}, 0, false, nil // Negative filter; the shared table owns recognition. + } + for name, candidate := range x86SHAEncodings { + if encoding == candidate { + op = name + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + immediate := encoding == 0x3acc + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := i + 3 + modRM := code[modRMIndex] + rExt := int(rexPrefix>>2) & 1 + xExt := int(rexPrefix>>1) & 1 + bExt := int(rexPrefix) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + + var source Operand + consumed := 0 + if modRM>>6 == 3 { + sourceNumber := int(modRM&7) + bExt*8 + if mode == 32 && sourceNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source = Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", sourceNumber))} + consumed = 1 + } else { + var decodeErr error + source, consumed, decodeErr = memory.scalar(code[modRMIndex:], mode, bExt, xExt, segment, 1) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + } + + args := []Operand{source, destination} + end := modRMIndex + consumed + if immediate { + if len(code) <= end { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + // SHA1RNDS4 consumes only the low two bits. Normalize the raw byte to + // Go's Yu2 operand class so the recovered instruction remains valid. + args = []Operand{{Kind: OpImm, Imm: int64(code[end] & 3)}, source, destination} + end++ + } else if op == "SHA256RNDS2" { + args = []Operand{{Kind: OpReg, Reg: Reg("X0")}, source, destination} + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, end, true, nil +} + +// decodedX86BLSInstruction recognizes the complete BMI1 BLSI/BLSMSK/BLSR +// register and ModRM/SIB memory families. x/arch v0.14 reports these VEX.NDD +// encodings as unknown AVX opcodes, so recover them before generic decoding. +func decodedX86BLSInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86BLSInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86BLSInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 || code[i+2]&0x07 != 0 || code[i+3] != 0xf3 { + return Instr{}, 0, false, nil + } + ok = true + vex1, vex2 := code[i+1], code[i+2] + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + width64 := vex2&0x80 != 0 + destinationNumber := int(^vex2>>3) & 15 + modRM := code[i+4] + family := "" + switch modRM >> 3 & 7 { + case 1: + family = "BLSR" + case 2: + family = "BLSMSK" + case 3: + family = "BLSI" + default: + return Instr{}, 0, true, fmt.Errorf("invalid opcode extension /%d", modRM>>3&7) + } + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + if mode == 32 && (xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + source, consumed, err := memory.scalar(code[i+4:], mode, bExt, xExt, segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + destination, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid destination register %d", destinationNumber) + } + op := Op(family + "L") + if width64 && mode == 64 { + op = Op(family + "Q") + } + instruction = Instr{ + Op: op, + Args: []Operand{source, {Kind: OpReg, Reg: destination}}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination), + } + return instruction, i + 4 + consumed, true, nil +} + +// decodedX86MULXInstruction recognizes the complete BMI2 MULX register and +// ModRM/SIB memory family. x/arch rejects this VEX.NDD encoding even though +// Go's assembler and the semantic lowerer support MULXL/MULXQ. +func decodedX86MULXInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86MULXInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86MULXInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i, segment, addressOverride := x86RawSegmentPrefixes(code) + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 || code[i+2]&0x07 != 3 || code[i+3] != 0xf6 { + return Instr{}, 0, false, nil + } + ok = true + vex1, vex2 := code[i+1], code[i+2] + if vex2&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must be zero") + } + rExt := int(^vex1>>7) & 1 + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + width64 := vex2&0x80 != 0 + lowRegister := int(^vex2>>3) & 15 + modRM := code[i+4] + highRegister := int(modRM>>3&7) + rExt*8 + // Native-PC observation ignores W outside long mode, as the actual ISA + // does. The zero-value ordinary raw decoder retains its old role/width + // restriction; a byte boundary cannot broaden that source contract. + if mode == 32 && (width64 && !memory.physicalPC || rExt != 0 || xExt != 0 || bExt != 0 || lowRegister >= 8 || highRegister >= 8) { + return Instr{}, 0, true, fmt.Errorf("64-bit width or extended register in 32-bit mode") + } + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + source, consumed, err := memory.scalar(code[i+4:], mode, bExt, xExt, segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + low, valid := decodedX86GeneralRegister(lowRegister) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid low destination register %d", lowRegister) + } + high, valid := decodedX86GeneralRegister(highRegister) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid high destination register %d", highRegister) + } + op := Op("MULXL") + if width64 && mode == 64 { + op = "MULXQ" + } + instruction = Instr{ + Op: op, + Args: []Operand{source, {Kind: OpReg, Reg: low}, {Kind: OpReg, Reg: high}}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, source.String(), low, high), + } + return instruction, i + 4 + consumed, true, nil +} + +func decodedX86VEXRMSource(code []byte, mode, bExt, xExt int, segment Reg) (Operand, int, error) { + return decodedX86RMSource(code, mode, bExt, xExt, segment, 1) +} + +func decodedX86RMSource(code []byte, mode, bExt, xExt int, segment Reg, disp8Scale int) (Operand, int, error) { + return decodedX86RMSourceWithPC(code, mode, bExt, xExt, segment, disp8Scale, false) +} + +func decodedX86RMSourceWithPC(code []byte, mode, bExt, xExt int, segment Reg, disp8Scale int, physicalPC bool) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if disp8Scale <= 0 { + return Operand{}, 0, fmt.Errorf("invalid disp8 scale %d", disp8Scale) + } + modRM := code[0] + mod := int(modRM >> 6) + rm := int(modRM & 7) + if mod == 3 { + register, ok := decodedX86GeneralRegister(rm + bExt*8) + if !ok || mode == 32 && rm+bExt*8 >= 8 { + return Operand{}, 0, fmt.Errorf("invalid source register") + } + return Operand{Kind: OpReg, Reg: register}, 1, nil + } + + mem := MemRef{Segment: segment} + consumed := 1 + readDisp8 := func() error { + if len(code) < consumed+1 { + return fmt.Errorf("truncated disp8") + } + mem.Off = int64(int8(code[consumed])) * int64(disp8Scale) + consumed++ + return nil + } + readDisp32 := func() error { + if len(code) < consumed+4 { + return fmt.Errorf("truncated disp32") + } + mem.Off = int64(int32(binary.LittleEndian.Uint32(code[consumed : consumed+4]))) + consumed += 4 + return nil + } + + if rm == 4 { + if len(code) < consumed+1 { + return Operand{}, 0, fmt.Errorf("truncated SIB byte") + } + sib := code[consumed] + consumed++ + indexNumber := int(sib>>3&7) + xExt*8 + if int(sib>>3&7) != 4 || xExt != 0 { + index, ok := decodedX86GeneralRegister(indexNumber) + if !ok || mode == 32 && indexNumber >= 8 { + return Operand{}, 0, fmt.Errorf("invalid SIB index register") + } + mem.Index = index + mem.Scale = int64(1 << (sib >> 6)) + } + baseBits := int(sib & 7) + if mod == 0 && baseBits == 5 { + if err := readDisp32(); err != nil { + return Operand{}, 0, err + } + } else { + baseNumber := baseBits + bExt*8 + base, ok := decodedX86GeneralRegister(baseNumber) + if !ok || mode == 32 && baseNumber >= 8 { + return Operand{}, 0, fmt.Errorf("invalid SIB base register") + } + mem.Base = base + } + } else if mod == 0 && rm == 5 { + if mode == 64 { + if !physicalPC { + return Operand{}, 0, fmt.Errorf("RIP-relative source is not source-layout safe") + } + mem.Base = "RIP" + } + if err := readDisp32(); err != nil { + return Operand{}, 0, err + } + } else { + baseNumber := rm + bExt*8 + base, ok := decodedX86GeneralRegister(baseNumber) + if !ok || mode == 32 && baseNumber >= 8 { + return Operand{}, 0, fmt.Errorf("invalid base register") + } + mem.Base = base + } + + switch mod { + case 1: + if err := readDisp8(); err != nil { + return Operand{}, 0, err + } + case 2: + if err := readDisp32(); err != nil { + return Operand{}, 0, err + } + } + return Operand{Kind: OpMem, Mem: mem}, consumed, nil +} + +func decodedX86GeneralRegister(number int) (Reg, bool) { + names := [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + if number < 0 || number >= len(names) { + return "", false + } + return names[number], true +} + +// x86RawRetpolineTarget recognizes the complete amd64 RETPOLINE(reg) byte +// family emitted by the Go runtime. Its local CALL and backward PAUSE loop are +// a speculative-execution barrier; architecturally the sequence replaces the +// temporary return address with reg and returns to it, which is an indirect +// tail jump. Requiring every fixed byte prevents arbitrary PC-relative raw +// control flow from being accepted through this path. +func x86RawRetpolineTarget(code []byte) (Reg, bool) { + if len(code) != 14 || + code[0] != 0xe8 || code[1] != 0x04 || code[2] != 0 || code[3] != 0 || code[4] != 0 || + code[5] != 0xf3 || code[6] != 0x90 || code[7] != 0xeb || code[8] != 0xfc || + (code[9] != 0x48 && code[9] != 0x4c) || code[10] != 0x89 || code[11]&0xc7 != 0x04 || + code[12] != 0x24 || code[13] != 0xc3 { + return "", false + } + register := int(code[11]>>3&7) + int(code[9]>>2&1)*8 + if register == 4 { + return "", false + } + names := [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + return names[register], true +} + +// Go's assembler omits Intel's N for these no-wait x87 encodings. Keep the +// entire control/environment alias family on the existing named lowerers. +var x86RawX87ControlAliases = map[x86asm.Op]Op{ + x86asm.FNCLEX: "FCLEX", + x86asm.FNINIT: "FINIT", + x86asm.FNSAVE: "FSAVE", + x86asm.FNSTCW: "FSTCW", + x86asm.FNSTENV: "FSTENV", + x86asm.FNSTSW: "FSTSW", +} + +func decodedX86GoSyntax(inst x86asm.Inst, encoding []byte) (string, error) { + if err := decodedX86PrefixError(inst); err != nil { + return "", err + } + for _, argument := range inst.Args { + if memory, ok := argument.(x86asm.Mem); ok && memory.Base == x86asm.EIP && inst.Op != x86asm.NOP { + // The pinned decoder leaves PCRel unset for 67-prefixed EIP + // addressing. Its wrapped native instruction address is not a + // numerical GP input or a proved source-group RIP relocation. + return "", fmt.Errorf("%w: raw x86 EIP-relative memory requires its native instruction address", ErrProbeNeedsContext) + } + } + inst = normalizeDecodedX86MemoryDisplacements(inst) + inst = normalizeDecodedX86LEASegments(inst) + for _, prefix := range inst.Prefix { + if !prefix.IsVEX() { + continue + } + // The pinned x/arch decoder recognizes only these VEX operations. + // For other VEX bytes it can incorrectly return an ordinary legacy + // opcode with a successful status and the wrong instruction length. + // Family-specific decoders above must handle those encodings; never + // let generic fallback turn a missing family into valid but wrong IR. + switch inst.Op { + case x86asm.VMOVDQA, x86asm.VMOVDQU, x86asm.VMOVNTDQ, x86asm.VMOVNTDQA: + default: + return "", fmt.Errorf("unrecognized VEX instruction decoded as legacy %s", inst.Op) + } + break + } + if syntax, recognized, err := decodedX86FarReturn(inst); recognized { + return syntax, err + } + syntax := x86asm.GoSyntax(inst, 0, nil) + replaceOp := func(op string) { + // A repeater is a separate pseudo-instruction; an actual segment + // prefix can precede the mnemonic. Never replace either prefix + // with an alias and move the real opcode into its operand list. + start := strings.LastIndexByte(syntax, ';') + 1 + for start < len(syntax) && syntax[start] == ' ' { + start++ + } + for _, prefix := range inst.Prefix { + switch prefix & 0xff { + case x86asm.PrefixES, x86asm.PrefixCS, x86asm.PrefixSS, x86asm.PrefixDS, x86asm.PrefixFS, x86asm.PrefixGS: + if strings.HasPrefix(syntax[start:], prefix.String()+" ") { + start += len(prefix.String()) + 1 + } + } + } + end := strings.IndexByte(syntax[start:], ' ') + if end < 0 { + end = len(syntax) + } else { + end += start + } + syntax = syntax[:start] + op + syntax[end:] + } + if op, ok := decodedX86CacheLineWriteback(encoding); ok { + replaceOp(op) + return syntax, nil + } + if op, ok := x86RawX87ControlAliases[inst.Op]; ok { + // The Go memory mnemonics encode the 32-bit environment layout. + // Mapping a 66-prefixed 14/94-byte legacy layout to them would change + // the memory footprint. Keep that unsupported format explicit. + if inst.DataSize == 16 && (inst.Op == x86asm.FNSAVE || inst.Op == x86asm.FNSTENV) { + return "", fmt.Errorf("raw %s uses a 16-bit environment layout not represented by Go %s", inst.Op, op) + } + replaceOp(string(op)) + return syntax, nil + } + if op, recognized, err := decodedX86X87EnvironmentLoad(inst, encoding); recognized { + if err != nil { + return "", err + } + replaceOp(string(op)) + return syntax, nil + } + + switch inst.Op { + case x86asm.RDFSBASE, x86asm.RDGSBASE, x86asm.WRFSBASE, x86asm.WRGSBASE: + // x/arch prints widthless Intel names. Reuse the complete named + // FSGSBASE grammar, selecting L/Q from the actual GPR: an ignored + // 66 prefix can set DataSize=16 while the operand remains 32-bit. + register, ok := inst.Args[0].(x86asm.Reg) + bits := decodedX86RegisterBits(register) + if !ok || inst.Mode != 64 || (bits != 32 && bits != 64) { + return "", fmt.Errorf("raw %s requires a 32/64-bit GPR in 64-bit mode", inst.Op) + } + for _, extra := range inst.Args[1:] { + if extra != nil { + return "", fmt.Errorf("raw %s requires exactly one register", inst.Op) + } + } + for _, prefix := range inst.Prefix { + if prefix&0xff == x86asm.PrefixLOCK || prefix&x86asm.PrefixInvalid != 0 { + return "", fmt.Errorf("raw %s has an invalid prefix %s", inst.Op, prefix) + } + } + // Segment/address/operand overrides do not affect this register-only + // family. Clear them only after rejecting invalid (not ignored) ones. + inst.Prefix = x86asm.Prefixes{} + syntax = x86asm.GoSyntax(inst, 0, nil) + width := "L" + if bits == 64 { + width = "Q" + } + replaceOp(inst.Op.String() + width) + case x86asm.INC, x86asm.DEC: + bits := 0 + switch destination := inst.Args[0].(type) { + case x86asm.Reg: + bits = decodedX86RegisterBits(destination) + case x86asm.Mem: + bits = inst.MemBytes * 8 + } + width := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("raw %s has unsupported destination width %d", inst.Op, bits) + } + replaceOp(inst.Op.String() + width) + case x86asm.SLDT, x86asm.STR, x86asm.SMSW: + // x/arch prints the unsuffixed Intel names, while Go's descriptor + // grammar selects the destination width in the mnemonic. Memory + // destinations are always the architectural 16-bit selector/status. + bits := 0 + switch destination := inst.Args[0].(type) { + case x86asm.Mem: + bits = 16 + case x86asm.Reg: + bits = decodedX86RegisterBits(destination) + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("raw %s has unsupported destination width %d", inst.Op, bits) + } + replaceOp(inst.Op.String() + width) + case x86asm.PSHUFW: + // Physical imm8 bits have no signedness, but Go's complete ymshuf + // source table requires Yi8. Render that signed representative in + // both modes; the 32-bit generic printer otherwise masks it unsigned. + selector, ok := inst.Args[2].(x86asm.Imm) + if !ok || selector < 0 || selector > 255 || inst.Args[3] != nil { + return "", fmt.Errorf("raw PSHUFW expected an 8-bit selector") + } + firstSpace := strings.IndexByte(syntax, ' ') + firstComma := strings.IndexByte(syntax, ',') + if firstSpace < 0 || firstComma < firstSpace { + return "", fmt.Errorf("raw PSHUFW has malformed operands") + } + syntax = syntax[:firstSpace+1] + fmt.Sprintf("$%d", int8(selector)) + syntax[firstComma:] + case x86asm.SHLD, x86asm.SHRD: + // Go encodes double shifts with the three-operand SHL/SHR form, + // whose immediate class is signed Yi8 (single shifts use Yu8). + if count, ok := inst.Args[2].(x86asm.Imm); ok { + firstSpace := strings.IndexByte(syntax, ' ') + firstComma := strings.IndexByte(syntax, ',') + if firstSpace < 0 || firstComma < firstSpace { + return "", fmt.Errorf("raw %s has malformed double-shift operands", inst.Op) + } + syntax = syntax[:firstSpace+1] + fmt.Sprintf("$%d", int8(count)) + syntax[firstComma:] + } + case x86asm.CMPPD, x86asm.CMPPS, x86asm.CMPSD_XMM, x86asm.CMPSS: + predicate, ok := inst.Args[2].(x86asm.Imm) + if !ok || predicate < 0 || predicate > 255 || inst.Args[3] != nil { + return "", fmt.Errorf("raw %s expected an unsigned 8-bit predicate", inst.Op) + } + // Go's legacy yxcmpi table places the signed predicate last, unlike + // both Intel syntax and the vector VCMP family. Reorder typed args + // before rendering, preserving register and segment/address details. + inst.Args[0], inst.Args[1], inst.Args[2] = x86asm.Imm(int8(predicate)), inst.Args[0], inst.Args[1] + syntax = x86asm.GoSyntax(inst, 0, nil) + // The 32-bit printer masks negative immediates to uint32. The Go + // encoder requires a signed byte, not that unsigned spelling. + syntax = syntax[:strings.LastIndexByte(syntax, ',')+1] + fmt.Sprintf(" $%d", int8(predicate)) + if inst.Op == x86asm.CMPSD_XMM { + replaceOp("CMPSD") + } + return syntax, nil + case x86asm.NOP: + // The decoder prints NOPL/NOPW according to the encoded data size. + // The address operand of a multi-byte x86 NOP is not read, so the + // semantic instruction is the operand-free Plan 9 NOP. + return "NOP", nil + case x86asm.MOVSXD: + return decodedX86MOVSXD(inst, encoding) + case x86asm.MOVD: + // Intel MOVD transfers 32 bits. Go's MOVD spelling instead aliases + // AMOVQ and transfers 64 bits for XMM/memory forms. + replaceOp("MOVL") + case x86asm.MOVDQA: + replaceOp("MOVO") + case x86asm.MOVDQU: + replaceOp("MOVOU") + case x86asm.MOVNTDQ: + // Go's assembler calls the legacy 128-bit non-temporal store MOVNTO. + // MOVNTDQ is reserved for the VEX/EVEX VMOVNTDQ spelling. + replaceOp("MOVNTO") + case x86asm.MOVSD_XMM: + replaceOp("MOVSD") + case x86asm.CVTDQ2PS, x86asm.CVTPI2PS: + // Go's yxcvm2 table shares CVTPL2PS across SSE X and MMX M sources. + replaceOp("CVTPL2PS") + case x86asm.CVTPS2DQ, x86asm.CVTPS2PI: + // Go's yxcvm1 table likewise names both destinations CVTPS2PL. + replaceOp("CVTPS2PL") + case x86asm.CVTTPS2DQ, x86asm.CVTTPS2PI: + replaceOp("CVTTPS2PL") + case x86asm.CVTDQ2PD, x86asm.CVTPI2PD: + replaceOp("CVTPL2PD") + case x86asm.CVTPD2DQ, x86asm.CVTPD2PI: + replaceOp("CVTPD2PL") + case x86asm.CVTTPD2DQ, x86asm.CVTTPD2PI: + replaceOp("CVTTPD2PL") + case x86asm.MUL, x86asm.IMUL, x86asm.DIV, x86asm.IDIV: + argumentCount := 0 + for _, argument := range inst.Args { + if argument != nil { + argumentCount++ + } + } + if argumentCount == 1 { + bits := inst.MemBytes * 8 + if source, ok := inst.Args[0].(x86asm.Reg); ok { + bits = decodedX86RegisterBits(source) + } + width := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("%s has unsupported source width %d", inst.Op, bits) + } + replaceOp(inst.Op.String() + width) + } else if inst.Op == x86asm.IMUL && argumentCount == 3 { + destination, ok := inst.Args[0].(x86asm.Reg) + if !ok { + return "", fmt.Errorf("three-operand IMUL destination is not a register") + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[decodedX86RegisterBits(destination)] + if width == "" { + return "", fmt.Errorf("three-operand IMUL has unsupported destination width %d", decodedX86RegisterBits(destination)) + } + replaceOp("IMUL3" + width) + } + case x86asm.BSWAP: + destination, ok := inst.Args[0].(x86asm.Reg) + if !ok { + return "", fmt.Errorf("BSWAP destination is not a register") + } + width := map[int]string{32: "L", 64: "Q"}[decodedX86RegisterBits(destination)] + if width == "" { + return "", fmt.Errorf("BSWAP has unsupported destination width %d", decodedX86RegisterBits(destination)) + } + replaceOp("BSWAP" + width) + case x86asm.MOVBE: + bits := inst.MemBytes * 8 + for _, operand := range inst.Args { + if register, ok := operand.(x86asm.Reg); ok { + bits = decodedX86RegisterBits(register) + break + } + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("MOVBE has unsupported operand width %d", bits) + } + replaceOp("MOVBE" + width) + case x86asm.LZCNT, x86asm.TZCNT: + // Go names each bit-count width explicitly. x/arch prints the + // widthless Intel mnemonic, even for 16- and 64-bit encodings. + destination, ok := inst.Args[0].(x86asm.Reg) + if !ok { + return "", fmt.Errorf("%s destination is not a register", inst.Op) + } + bits := decodedX86RegisterBits(destination) + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("%s has unsupported destination width %d", inst.Op, bits) + } + replaceOp(inst.Op.String() + width) + case x86asm.CVTSI2SS, x86asm.CVTSI2SD: + sourceBits := inst.MemBytes * 8 + if source, ok := inst.Args[1].(x86asm.Reg); ok { + sourceBits = decodedX86RegisterBits(source) + } + sourceWidth := map[int]string{32: "L", 64: "Q"}[sourceBits] + if sourceWidth == "" { + return "", fmt.Errorf("%s has unsupported decoded source width %d", inst.Op, sourceBits) + } + resultWidth := "SS" + if inst.Op == x86asm.CVTSI2SD { + resultWidth = "SD" + } + replaceOp("CVTS" + sourceWidth + "2" + resultWidth) + case x86asm.CVTSS2SI, x86asm.CVTSD2SI, x86asm.CVTTSS2SI, x86asm.CVTTSD2SI: + // x/arch prints Intel's SI destination stem (and, in some memory + // cases, derives the wrong Plan 9 width). Go's assembler names this + // legacy family CVT{T}{SS,SD}2S{L,Q}. Recover the destination width + // from REX.W, which is the architectural selector for all four ops. + outputWidth := "L" + if inst.Mode == 64 { + for _, prefix := range inst.Prefix { + if prefix.IsREX() && prefix&x86asm.PrefixREXW != 0 { + outputWidth = "Q" + break + } + } + } + name := map[x86asm.Op]string{ + x86asm.CVTSS2SI: "CVTSS2S", + x86asm.CVTSD2SI: "CVTSD2S", + x86asm.CVTTSS2SI: "CVTTSS2S", + x86asm.CVTTSD2SI: "CVTTSD2S", + }[inst.Op] + replaceOp(name + outputWidth) + case x86asm.PACKSSDW: + replaceOp("PACKSSLW") + case x86asm.PCMPEQD: + replaceOp("PCMPEQL") + case x86asm.PCMPGTD: + replaceOp("PCMPGTL") + case x86asm.PMADDWD: + replaceOp("PMADDWL") + case x86asm.PMULUDQ: + replaceOp("PMULULQ") + case x86asm.PSLLD: + replaceOp("PSLLL") + case x86asm.PSRAD: + replaceOp("PSRAL") + case x86asm.PSRLD: + replaceOp("PSRLL") + case x86asm.PSUBD: + replaceOp("PSUBL") + case x86asm.PUNPCKHDQ: + replaceOp("PUNPCKHLQ") + case x86asm.PUNPCKHWD: + replaceOp("PUNPCKHWL") + case x86asm.PUNPCKLDQ: + replaceOp("PUNPCKLLQ") + case x86asm.PUNPCKLWD: + replaceOp("PUNPCKLWL") + case x86asm.CRC32: + bits := inst.MemBytes * 8 + if source, ok := inst.Args[1].(x86asm.Reg); ok { + bits = decodedX86RegisterBits(source) + } + width := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("CRC32 has unsupported decoded source width %d", bits) + } + replaceOp("CRC32" + width) + case x86asm.MOVSX, x86asm.MOVZX: + dst, ok := inst.Args[0].(x86asm.Reg) + if !ok { + return "", fmt.Errorf("%s destination is not a register", inst.Op) + } + destinationBits := decodedX86RegisterBits(dst) + sourceBits := inst.MemBytes * 8 + if source, ok := inst.Args[1].(x86asm.Reg); ok { + sourceBits = decodedX86RegisterBits(source) + } + widthLetter := func(bits int) string { + switch bits { + case 8: + return "B" + case 16: + return "W" + case 32: + return "L" + case 64: + return "Q" + default: + return "" + } + } + sourceLetter := widthLetter(sourceBits) + destinationLetter := widthLetter(destinationBits) + if sourceLetter == "" || destinationLetter == "" { + return "", fmt.Errorf("%s has unsupported decoded widths %d -> %d", inst.Op, sourceBits, destinationBits) + } + extension := "SX" + if inst.Op == x86asm.MOVZX { + extension = "ZX" + } + if sourceBits == 16 && destinationBits == 16 { + name := "MOVSWW" + if inst.Op == x86asm.MOVZX { + name = "MOVZWW" + } + replaceOp(name) + } else { + replaceOp("MOV" + sourceLetter + destinationLetter + extension) + } + } + normalizeDecodedX86GoOpcode(inst, &syntax, replaceOp) + syntax = normalizeDecodedX86SegmentMemorySyntax(syntax) + return syntax, nil +} + +// The pinned x/arch decoder retains ModR/M/SIB disp32 as an unsigned +// magnitude. Rendering that value as a named Go address changes 64-bit +// arithmetic and violates Go's signed-displacement checks. Preserve the +// physical field's sign before every opcode-specific rendering path. +// MOV A0-A3 instead contains a full absolute moffs address: it must never +// be narrowed or sign-extended as though it were a ModR/M displacement. +func normalizeDecodedX86MemoryDisplacements(inst x86asm.Inst) x86asm.Inst { + if inst.AddrSize != 32 && inst.AddrSize != 64 { + return inst + } + if opcode := byte(inst.Opcode >> 24); opcode >= 0xa0 && opcode <= 0xa3 { + return inst + } + for index, argument := range inst.Args { + if memory, ok := argument.(x86asm.Mem); ok { + memory.Disp = int64(int32(memory.Disp)) + inst.Args[index] = memory + } + } + return inst +} + +// LEA computes only the effective offset. All six segment override prefixes +// are architecturally irrelevant to it, unlike an actual memory load/store. +func normalizeDecodedX86LEASegments(inst x86asm.Inst) x86asm.Inst { + if inst.Op != x86asm.LEA { + return inst + } + for index, argument := range inst.Args { + if memory, ok := argument.(x86asm.Mem); ok { + memory.Segment = 0 + inst.Args[index] = memory + } + } + var prefixes x86asm.Prefixes + next := 0 + for _, prefix := range inst.Prefix { + if prefix == 0 { + break + } + switch byte(prefix) { + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65: + continue + } + prefixes[next] = prefix + next++ + } + inst.Prefix = prefixes + return inst +} + +// x/arch prints a raw segment override as GS:0x30 or GS:0x30(AX). The +// Plan 9 parser represents those same addresses as 48(GS) and 48(AX)(GS). +// Rewrite only FS/GS operands; other segment and RIP-relative forms retain +// their original spelling so unsupported address semantics fail closed. +func normalizeDecodedX86SegmentMemorySyntax(syntax string) string { + space := strings.IndexByte(syntax, ' ') + if space < 0 { + return syntax + } + args := splitTopLevelCSV(syntax[space+1:]) + for i, arg := range args { + arg = strings.TrimSpace(arg) + segment := "" + switch { + case strings.HasPrefix(arg, "FS:"): + segment = "FS" + case strings.HasPrefix(arg, "GS:"): + segment = "GS" + default: + continue + } + address := strings.TrimPrefix(arg, segment+":") + open := strings.IndexByte(address, '(') + offset := address + base := "" + if open >= 0 { + offset = address[:open] + base = address[open:] + } + if offset == "" { + offset = "0" + } + value, err := strconv.ParseInt(offset, 0, 64) + if err != nil || strings.Contains(base, "(IP)") || strings.Contains(base, "(RIP)") { + continue + } + args[i] = fmt.Sprintf("%d%s(%s)", value, base, segment) + } + return syntax[:space+1] + strings.Join(args, ", ") +} + +// normalizeDecodedX86GoOpcode bridges the architectural names emitted by +// x/arch's decoder and the semantic names/classes used by Go 1.27's +// cmd/internal/obj/x86 tables. Go's assembler intentionally uses aliases such +// as SETOS and CMOVQLE, while x/arch prints Intel names such as SETO and +// CMOVLE. x/arch also prints the byte-immediate ADD opcode as ADDL even when +// its destination is an 8-bit register; that spelling is rejected by Go's +// Yml/Ymb operand classes and must be corrected from the decoded operands. +func normalizeDecodedX86GoOpcode(inst x86asm.Inst, syntax *string, replaceOp func(string)) { + op := strings.ToUpper(inst.Op.String()) + setAliases := map[string]string{ + "SETA": "SETHI", "SETAE": "SETCC", "SETB": "SETCS", "SETBE": "SETLS", + "SETE": "SETEQ", "SETG": "SETGT", "SETGE": "SETGE", "SETL": "SETLT", + "SETLE": "SETLE", "SETNE": "SETNE", "SETNO": "SETOC", "SETNP": "SETPC", + "SETNS": "SETPL", "SETO": "SETOS", "SETP": "SETPS", "SETS": "SETMI", + } + if alias, ok := setAliases[op]; ok { + replaceOp(alias) + // GoSyntax deliberately omits the B suffix for extended byte registers + // (for example it prints SETO R9). Reconstruct the decoded register's + // actual byte spelling before reparsing the instruction. + if len(inst.Args) > 0 { + if reg, ok := inst.Args[0].(x86asm.Reg); ok { + *syntax = alias + " " + reg.String() + } + } + return + } + + cmovConditions := map[string]string{ + "CMOVA": "HI", "CMOVAE": "CC", "CMOVB": "CS", "CMOVBE": "LS", + "CMOVE": "EQ", "CMOVG": "GT", "CMOVGE": "GE", "CMOVL": "LT", + "CMOVLE": "LE", "CMOVNE": "NE", "CMOVNO": "OC", "CMOVNP": "PC", + "CMOVNS": "PL", "CMOVO": "OS", "CMOVP": "PS", "CMOVS": "MI", + } + if condition, ok := cmovConditions[op]; ok { + bits := 0 + for i := 0; i < 2 && i < len(inst.Args); i++ { + if reg, ok := inst.Args[i].(x86asm.Reg); ok { + if n := decodedX86RegisterBits(reg); n > bits { + bits = n + } + } + } + if bits == 0 { + bits = inst.MemBytes * 8 + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + if width != "" { + replaceOp("CMOV" + width + condition) + } + return + } + + widthOpcode := decodedX86WidthOpcode(op) + if widthOpcode == "" || len(inst.Args) == 0 { + return + } + dstBits := 0 + if reg, ok := inst.Args[0].(x86asm.Reg); ok { + dstBits = decodedX86RegisterBits(reg) + } else if _, ok := inst.Args[0].(x86asm.Mem); ok { + dstBits = inst.MemBytes * 8 + } + width := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[dstBits] + if width != "" { + replaceOp(widthOpcode + width) + } +} + +func decodedX86WidthOpcode(op string) string { + switch op { + case "ADD", "ADC", "SBB", "SUB", "AND", "OR", "XOR", "CMP", "TEST": + return op + case "ROL", "ROR", "RCL", "RCR", "SHL", "SHR", "SAR": + return op + case "SHLD": + return "SHL" + case "SHRD": + return "SHR" + default: + return "" + } +} + +// Go 1.27 names 0F 18 /0-/3 but not the newer /4, /6 and /7 cache +// hints. Decode their complete ModRM memory forms directly; /5 and register +// operands are not instructions. RIP-relative addresses remain rejected by +// decodedX86RMSource because their byte layout cannot be inferred here. +func decodedX86ExtendedPrefetchInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + rex := byte(0) + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + rex = 0 + case 0x65: + segment = GS + rex = 0 + case 0x66: + rex = 0 + case 0x67: + addressOverride = true + rex = 0 + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + continue + } + goto opcode + } + i++ + } + +opcode: + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0x18 { + return Instr{}, 0, false, nil + } + extension := code[i+2] >> 3 & 7 + op := Op("") + switch extension { + case 4: + op = "PREFETCHRST2" + case 6: + op = "PREFETCHIT1" + case 7: + op = "PREFETCHIT0" + default: + return Instr{}, 0, false, nil + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if code[i+2]>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("%s requires a memory operand", op) + } + address, consumed, err := decodedX86RMSource(code[i+2:], mode, int(rex&1), int(rex>>1&1), segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + return Instr{Op: op, Args: []Operand{address}, Raw: fmt.Sprintf("%s %s", op, address.String())}, i + 2 + consumed, true, nil +} + +// decodedX86CacheLineWritebackInstruction recognizes the complete raw +// CLFLUSH/CLFLUSHOPT/CLWB memory family before x/arch's generic decoder. In +// particular, generated assembly in github.com/wencode/hack places a REX +// prefix before the mandatory 0x66 prefix, which x/arch v0.14 rejects even +// though x86 hardware accepts the prefix sequence. +func decodedX86CacheLineWritebackInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + hasDataSizePrefix := false + addressOverride := false + segment := Reg("") + rex := byte(0) + hasREX := false + for i < len(code) { + switch code[i] { + case 0x64: + // A legacy prefix after REX makes that REX byte ineffective. Only + // the REX prefix immediately preceding the opcode escape applies. + rex = 0 + hasREX = false + segment = FS + i++ + case 0x65: + rex = 0 + hasREX = false + segment = GS + i++ + case 0x66: + rex = 0 + hasREX = false + hasDataSizePrefix = true + i++ + case 0x67: + rex = 0 + hasREX = false + addressOverride = true + i++ + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + hasREX = true + i++ + continue + } + goto opcode + } + } + +opcode: + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0xae { + return Instr{}, 0, false, nil + } + modRM := code[i+2] + extension := modRM >> 3 & 7 + op := Op("") + switch { + case extension == 7 && !hasDataSizePrefix: + op = "CLFLUSH" + case extension == 7 && hasDataSizePrefix: + op = "CLFLUSHOPT" + case extension == 6 && hasDataSizePrefix: + op = "CLWB" + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if hasREX && rex&0x0c != 0 { + return Instr{}, 0, true, fmt.Errorf("REX.W and REX.R are absent from Go 1.27's %s encoding", op) + } + if modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("%s requires a memory operand", op) + } + bExt := int(rex & 1) + xExt := int(rex>>1) & 1 + memory, consumed, decodeErr := decodedX86VEXRMSource(code[i+2:], mode, bExt, xExt, segment) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + instruction = Instr{ + Op: op, + Args: []Operand{memory}, + Raw: fmt.Sprintf("%s %s", op, memory.String()), + } + return instruction, i + 2 + consumed, true, nil +} + +// decodedX86CacheLineWriteback recovers instructions which x/arch v0.14 +// cannot distinguish from their older 0F AE encodings. The explicit 0x66 +// mandatory prefix plus the ModRM extension is the architectural distinction: +// /7 is CLFLUSHOPT and /6 is CLWB. Parse only the actual prefix/opcode region +// so an identical byte sequence inside a displacement cannot be mistaken for +// an opcode. +func decodedX86CacheLineWriteback(encoding []byte) (string, bool) { + hasDataSizePrefix := false + i := 0 + for i < len(encoding) { + switch encoding[i] { + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65, 0x67, 0xf0, 0xf2, 0xf3: + i++ + case 0x66: + hasDataSizePrefix = true + i++ + default: + if encoding[i] >= 0x40 && encoding[i] <= 0x4f { + i++ + continue + } + goto opcode + } + } + +opcode: + if !hasDataSizePrefix || i+2 >= len(encoding) || encoding[i] != 0x0f || encoding[i+1] != 0xae { + return "", false + } + modRM := encoding[i+2] + if modRM>>6 == 3 { + return "", false + } + switch modRM >> 3 & 7 { + case 7: + return "CLFLUSHOPT", true + case 6: + return "CLWB", true + default: + return "", false + } +} + +func decodedX86RegisterBits(reg x86asm.Reg) int { + switch { + case x86asm.AL <= reg && reg <= x86asm.R15B: + return 8 + case x86asm.AX <= reg && reg <= x86asm.R15W: + return 16 + case x86asm.EAX <= reg && reg <= x86asm.R15L: + return 32 + case x86asm.RAX <= reg && reg <= x86asm.R15: + return 64 + default: + return 0 + } +} + +func isX86RawDirective(ins Instr) bool { + switch Op(strings.ToUpper(string(ins.Op))) { + case OpBYTE, OpWORD, "LONG", "QUAD": + return true + default: + return false + } +} + +func x86RawDirectiveBytes(ins Instr) ([]byte, error) { + if len(ins.Args) != 1 || ins.Args[0].Kind != OpImm || ins.Args[0].ImmRaw != "" { + return nil, fmt.Errorf("%s expects exactly one resolved integer constant: %q", ins.Op, ins.Raw) + } + value := uint64(ins.Args[0].Imm) + switch Op(strings.ToUpper(string(ins.Op))) { + case OpBYTE: + return []byte{byte(value)}, nil + case OpWORD: + var buf [2]byte + binary.LittleEndian.PutUint16(buf[:], uint16(value)) + return buf[:], nil + case "LONG": + var buf [4]byte + binary.LittleEndian.PutUint32(buf[:], uint32(value)) + return buf[:], nil + case "QUAD": + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], value) + return buf[:], nil + default: + return nil, fmt.Errorf("not an x86 raw directive: %q", ins.Raw) + } +} + +func parseDecodedX86Instruction(syntax string) ([]Instr, error) { + file, err := Parse(ArchAMD64, "TEXT decoded(SB), NOSPLIT, $0-0\n"+syntax+"\n") + if err != nil { + return nil, err + } + if len(file.Funcs) != 1 || len(file.Funcs[0].Instrs) < 2 { + return nil, fmt.Errorf("decoder syntax produced no instruction") + } + return append([]Instr(nil), file.Funcs[0].Instrs[1:]...), nil +} diff --git a/amd64_raw_abs.go b/amd64_raw_abs.go new file mode 100644 index 000000000..cc55bfb5b --- /dev/null +++ b/amd64_raw_abs.go @@ -0,0 +1,81 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86RawPackedAbsInstruction implements Go's complete VPABS B/W/D/Q +// rows: VEX X/Y for B/W/D, EVEX X/Y/Z with optional K masking, and EVEX +// D/Q memory broadcast. VPABSQ is EVEX-only. VEX B/W/D ignore W; EVEX +// requires W0 for B/W/D and W1 for Q, independently of the named Go encoder. +func decodedX86RawPackedAbsInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || p.pp != 1 || p.opcode < 0x1c || p.opcode > 0x1f { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VPABS: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.evex && !p.fixed || p.vectorLength > 2 || p.upper != 0 { + return fail("invalid vector prefix or reserved vvvv bits") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + laneBytes := [...]int{1, 2, 4, 8}[p.opcode-0x1c] + if p.evex && p.w != (laneBytes == 8) || !p.evex && (laneBytes == 8 || p.vectorLength == 2) { + return fail("invalid W bit or vector length for opcode") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + registerSource := code[p.modRM]>>6 == 3 + if p.broadcast && (!p.evex || laneBytes < 4 || registerSource) { + return fail("broadcast requires EVEX D/Q memory source") + } + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + scale := 16 << p.vectorLength + if p.broadcast { + scale = laneBytes + } + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width, scale) + } else { + source, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + } + if err != nil { + return Instr{}, 0, true, err + } + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (destinationNumber >= 8 || registerSource && (p.b != 0 || p.x != 0)) { + return fail("extended register in 32-bit mode") + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))}) + op := Op("VPABS" + [...]string{"B", "W", "D", "Q"}[p.opcode-0x1c]) + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86PackedAbsEVEX: p.evex}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_abs_test.go b/amd64_raw_abs_test.go new file mode 100644 index 000000000..ffdf568bd --- /dev/null +++ b/amd64_raw_abs_test.go @@ -0,0 +1,229 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawPackedAbsForms = []struct { + opcode byte + op Op + lane int + evex bool +}{ + {0x1c, "VPABSB", 1, false}, + {0x1d, "VPABSW", 2, false}, + {0x1e, "VPABSD", 4, false}, + {0x1f, "VPABSQ", 8, true}, +} + +func encodeX86RawPackedAbs(formIndex, width, mask, destination, source int, zeroing, broadcast bool) []byte { + form := x86RawPackedAbsForms[formIndex] + vectorBits := map[int]int{16: 0, 32: 1, 64: 2}[width] + var code []byte + if width == 64 || mask != 0 || zeroing || broadcast || form.evex { + code = encodeX86EVEXPackedWordMultiply(2, form.opcode, vectorBits, mask, form.evex, zeroing, destination, 0, source) + if broadcast { + code[3] |= 0x10 + } + } else { + code = encodeX86VEXPackedWordMultiply(2, form.opcode, width == 32, false, destination, 0, source) + } + return code +} + +func TestDecodedX86RawPackedAbsCompleteFamily(t *testing.T) { + for formIndex, form := range x86RawPackedAbsForms { + for _, width := range []int{16, 32, 64} { + for _, masked := range []bool{false, true} { + for _, zeroing := range []bool{false, true} { + if !masked && zeroing { + continue + } + for _, source := range []int{0, 7, 16, 31} { + if !masked && width != 64 && !form.evex && source >= 16 { + continue + } + mask := 0 + if masked { + mask = 3 + } + name := fmt.Sprintf("%s/%d/k%d/z%t/src%d", form.op, width, mask, zeroing, source) + t.Run(name, func(t *testing.T) { + code := encodeX86RawPackedAbs(formIndex, width, mask, 19, source, zeroing, false) + if width != 64 && !masked && !form.evex { + code = encodeX86RawPackedAbs(formIndex, width, mask, 11, source, zeroing, false) + } + got, length, ok, err := decodedX86RawPackedAbsInstruction(code, 64) + if err != nil || !ok || length != len(code) || !strings.HasPrefix(string(got.Op), string(form.op)) || + got.Args[0].String() != fmt.Sprintf("%s%d", map[int]string{16: "X", 32: "Y", 64: "Z"}[width], source) { + t.Fatalf("decode %x = %+v length=%d ok=%t err=%v", code, got, length, ok, err) + } + }) + } + } + } + } + } + // Actual bytes from simd@v1.21.1 arith_avx2_amd64.s. + real := []byte{0xc4, 0xa2, 0x7d, 0x1e, 0x04, 0x86} + got, length, ok, err := decodedX86RawPackedAbsInstruction(real, 64) + if err != nil || !ok || length != len(real) || got.Op != "VPABSD" || + got.Args[0].String() != "0(SI)(R8*4)" || got.Args[1].String() != "Y0" { + t.Fatalf("decode real VPABSD %x = %+v length=%d ok=%t err=%v", real, got, length, ok, err) + } +} + +func TestDecodedX86RawPackedAbsRejectsInvalidForms(t *testing.T) { + valid := encodeX86RawPackedAbs(2, 32, 0, 1, 2, false, false) + for name, edit := range map[string]func([]byte) []byte{ + "reserved vvvv": func(b []byte) []byte { b[2] &^= 0x08; return b }, + "EVEX W1": func([]byte) []byte { + code := encodeX86RawPackedAbs(2, 64, 0, 1, 2, false, false) + code[2] |= 0x80 + return code + }, + "truncated": func(b []byte) []byte { return b[:4] }, + "address override": func(b []byte) []byte { return append([]byte{0x67}, b...) }, + } { + t.Run(name, func(t *testing.T) { + bad := edit(append([]byte(nil), valid...)) + if _, _, matched, err := decodedX86RawPackedAbsInstruction(bad, 64); !matched || err == nil { + t.Fatalf("accepted invalid packed absolute %x: matched=%t err=%v", bad, matched, err) + } + }) + } +} + +func TestTranslateX86RawPackedAbsObjects(t *testing.T) { + const source = `TEXT rawPackedAbs(SB),$0-0 + BYTE $0xc4 + BYTE $0xa2 + BYTE $0x7d + BYTE $0x1e + BYTE $0x04 + BYTE $0x86 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-packed-abs.ll", "raw-packed-abs.o", ir) + }) + } +} + +func x86RawPackedAbsConstant(formIndex, width int, broadcast bool) []byte { + form := x86RawPackedAbsForms[formIndex] + code := encodeX86RawPackedAbs(formIndex, width, 0, 0, 0, false, broadcast) + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = form.lane + } + for i := 0; i < dataWidth; i++ { + code = append(code, byte(i*23+5)) + } + return code +} + +func TestDecodeX86RawPackedAbsConstantCompleteFamily(t *testing.T) { + for formIndex, form := range x86RawPackedAbsForms { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + if broadcast && form.lane < 4 { + continue + } + name := fmt.Sprintf("%s/%d/bcst%t", form.op, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawPackedAbsConstant(formIndex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := form.op + if broadcast { + want += ".BCST" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } +} + +func TestDecodeX86RawPackedAbsConstantRejectsUnsafeSources(t *testing.T) { + valid := x86RawPackedAbsConstant(2, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawPackedAbsConstantObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, form := range []struct { + formIndex, width int + broadcast bool + }{ + {0, 32, false}, {1, 64, false}, {2, 64, true}, {3, 64, true}, + } { + code := x86RawPackedAbsConstant(form.formIndex, form.width, form.broadcast) + name := fmt.Sprintf("rawPackedAbsConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-packed-abs-constant.ll", "raw-packed-abs-constant.o", ir) + }) + } +} diff --git a/amd64_raw_address_override_test.go b/amd64_raw_address_override_test.go new file mode 100644 index 000000000..9d156d637 --- /dev/null +++ b/amd64_raw_address_override_test.go @@ -0,0 +1,753 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawAddressOverrideDoesNotBorrowCallerPointer(t *testing.T) { + for _, access := range []struct { + name, old, replacement string + }{ + {"load", "MOVL $7,AX", "BYTE $0x67\nBYTE $0x8b\nBYTE $0x07"}, + {"store", "MOVL AX,(DX)", "BYTE $0x67\nBYTE $0x89\nBYTE $0x02"}, + } { + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(access.name+"/"+triple, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, access.old, access.replacement, 1) + // These are real stock-Go objects, not executions of a program + // that could dereference a truncated incoming address. + x86LEAActualTargetGoObject(t, "amd64", triple, assembly) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("physical EA32 %s borrowed a full-width caller pointer: %v", access.name, err) + } + }) + } + } +} + +func x86AddressOverrideDecoded(t *testing.T, mode int, code []byte) []Instr { + t.Helper() + instructions, err := x86AddressOverrideDecode(mode, code) + if err != nil { + t.Fatal(err) + } + return instructions +} + +func x86AddressOverrideDecode(mode int, code []byte) ([]Instr, error) { + arch := "amd64" + if mode == 32 { + arch = "386" + } + file, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+x86RawDisplacementDirectives(code)+"RET\n") + if err != nil { + return nil, err + } + function, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + return nil, err + } + return function.Instrs, nil +} + +func x86AddressOverrideNamedBytes(t *testing.T, arch, instruction string) []byte { + t.Helper() + listing, err := x86ScalarLiteralGoListing(t, arch, "TEXT raw(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatalf("actual named implicit opcode: %v\n%s", err, listing) + } + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + var code []byte + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + t.Fatalf("actual named opcode byte: %v\n%s", err, listing) + } + code = append(code, byte(value)) + } + return code + } + t.Fatalf("actual named opcode lacks its byte listing:\n%s", listing) + return nil +} + +func TestX86RawAddressOverrideGenericMemoryAndImplicitStrings(t *testing.T) { + // Cover the generic decoder's explicit read/write/RMW, scalar/vector and + // implicit memory shapes. Specialized prefix parsers are a separate scope. + forms := []struct { + name string + code []byte + }{ + {"byte-load", []byte{0x8a, 0x07}}, {"byte-store", []byte{0x88, 0x07}}, + {"word-load", []byte{0x66, 0x8b, 0x07}}, {"word-store", []byte{0x66, 0x89, 0x07}}, + {"long-load", []byte{0x8b, 0x07}}, {"long-store", []byte{0x89, 0x07}}, + {"add-read", []byte{0x03, 0x07}}, {"add-rmw", []byte{0x01, 0x07}}, + {"compare", []byte{0x3b, 0x07}}, {"test", []byte{0x85, 0x07}}, + {"exchange", []byte{0x87, 0x07}}, {"increment", []byte{0xff, 0x07}}, + {"shift-rmw", []byte{0xc1, 0x27, 0x01}}, + {"movups-load", []byte{0x0f, 0x10, 0x07}}, {"movups-store", []byte{0x0f, 0x11, 0x07}}, + {"addps", []byte{0x0f, 0x58, 0x07}}, + {"movsb", []byte{0xa4}}, {"cmpsb", []byte{0xa6}}, + {"stosb", []byte{0xaa}}, {"lodsb", []byte{0xac}}, {"scasb", []byte{0xae}}, + {"rep-movsb", []byte{0xf3, 0xa4}}, {"repne-scasb", []byte{0xf2, 0xae}}, + {"insb", []byte{0x6c}}, {"outsb", []byte{0x6e}}, {"xlat", []byte{0xd7}}, + {"insw", []byte{0x66, 0x6d}}, {"insl", []byte{0x6d}}, + {"outsw", []byte{0x66, 0x6f}}, {"outsl", []byte{0x6f}}, + {"movsw", []byte{0x66, 0xa5}}, {"movsl", []byte{0xa5}}, + {"cmpsw", []byte{0x66, 0xa7}}, {"cmpsl", []byte{0xa7}}, + {"stosw", []byte{0x66, 0xab}}, {"stosl", []byte{0xab}}, + {"lodsw", []byte{0x66, 0xad}}, {"lodsl", []byte{0xad}}, + {"scasw", []byte{0x66, 0xaf}}, {"scasl", []byte{0xaf}}, + } + for _, mode := range []int{32, 64} { + modeForms := append([]struct { + name string + code []byte + }(nil), forms...) + if mode == 64 { + modeForms = append(modeForms, struct { + name string + code []byte + }{"quad-load", []byte{0x48, 0x8b, 0x07}}, struct { + name string + code []byte + }{"quad-store", []byte{0x48, 0x89, 0x07}}) + for _, form := range []struct { + name string + code []byte + }{ + {"movsq", []byte{0x48, 0xa5}}, {"cmpsq", []byte{0x48, 0xa7}}, + {"stosq", []byte{0x48, 0xab}}, {"lodsq", []byte{0x48, 0xad}}, + {"scasq", []byte{0x48, 0xaf}}, + } { + modeForms = append(modeForms, form) + } + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + var complete []byte + for _, form := range modeForms { + for _, segment := range []byte{0, 0x64, 0x65} { + complete = append(complete, 0x67) + if segment != 0 { + complete = append(complete, segment) + } + complete = append(complete, form.code...) + } + } + x86HandoffOriginalBytes(t, arch, append(complete, 0xc3)) + for _, form := range modeForms { + for _, segment := range []byte{0, 0x64, 0x65} { + t.Run(fmt.Sprintf("mode%d/%s/segment%02x", mode, form.name, segment), func(t *testing.T) { + code := []byte{0x67} + if segment != 0 { + code = append(code, segment) + } + code = append(code, form.code...) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) || physical.AddrSize != mode/2 { + t.Fatalf("actual decoder input: %+v, %v", physical, err) + } + instructions, err := x86AddressOverrideDecode(mode, code) + if strings.Contains(form.name, "stos") || strings.Contains(form.name, "scas") || strings.HasPrefix(form.name, "ins") { + if segment != 0 { + // The decoded destination remains ES, but x/arch + // leaves this unused FS/GS prefix unmarked. Require + // explicit Context, not a GoSyntax parser accident. + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "unbound physical") || instructions != nil { + t.Fatalf("unbound segment did not retain Context: %v", err) + } + return + } + } + if err != nil { + t.Fatal(err) + } + marked := 0 + for _, instruction := range instructions { + if instruction.Op == OpTEXT || instruction.Op == OpRET { + continue + } + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != mode/2 { + t.Fatalf("generic physical memory override lost its restriction: %+v", instruction) + } + marked++ + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + if mode == 32 { + ctx.goarch = "386" + } + before := output.String() + if terminated, err := ctx.lowerInstr(0, 0, instruction, nil, nil); terminated || !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("unbound memory emitted IR: terminated=%t, error=%v", terminated, err) + } + } + if marked == 0 { + t.Fatal("lost complete physical memory instruction") + } + }) + } + } + } +} + +func TestX86RawAddressOverrideAllAddressShapesAndMoffs(t *testing.T) { + for _, mode := range []int{32, 64} { + shapes := [][]byte{} + if mode == 32 { + // All eight EA16 r/m forms, including disp16-only and BP+disp8. + for rm := byte(0); rm < 8; rm++ { + shape := []byte{rm} + if rm == 6 { + shape = append(shape, 0xc0, 0xff) + } + shapes = append(shapes, shape, []byte{0x40 | rm, 0xc0}, []byte{0x80 | rm, 0xc0, 0xff}) + } + } else { + // All base registers and SIB scales; EIP-relative r/m=5 is + // deliberately covered by the existing closed EIP boundary. + for rm := byte(0); rm < 8; rm++ { + if rm != 4 && rm != 5 { + shapes = append(shapes, []byte{rm}) + } + if rm != 4 { + shapes = append(shapes, []byte{0x40 | rm, 0xc0}, []byte{0x80 | rm, 0xc0, 0xff, 0xff, 0xff}) + } + } + for scale := byte(0); scale < 4; scale++ { + shapes = append(shapes, + []byte{0x04, scale<<6 | 0x18}, + []byte{0x04, scale<<6 | 0x1d, 0xc0, 0xff, 0xff, 0xff}) + } + shapes = append(shapes, []byte{0x04, 0x20}, []byte{0x04, 0x25, 0xc0, 0xff, 0xff, 0xff}) + } + for _, opcode := range []byte{0x8b, 0x89} { + for index, shape := range shapes { + t.Run(fmt.Sprintf("mode%d/op%02x/shape%d", mode, opcode, index), func(t *testing.T) { + code := append([]byte{0x67, opcode}, shape...) + found := false + for _, instruction := range x86AddressOverrideDecoded(t, mode, code) { + if instruction.Op == OpMOVL { + found = true + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != mode/2 { + t.Fatalf("physical memory shape lost override: %+v", instruction) + } + } + } + if !found { + t.Fatal("lost actual ModR/M address shape") + } + }) + } + } + for _, opcode := range []byte{0xa0, 0xa1, 0xa2, 0xa3} { + t.Run(fmt.Sprintf("mode%d/moffs%02x", mode, opcode), func(t *testing.T) { + code := []byte{0x67, opcode, 0xc0, 0xff} + if mode == 64 { + code = append(code, 0xff, 0xff) + } + found := false + for _, instruction := range x86AddressOverrideDecoded(t, mode, code) { + if instruction.Op == "MOVB" || instruction.Op == OpMOVL { + found = true + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != mode/2 { + t.Fatalf("absolute address override borrowed default addressing: %+v", instruction) + } + } + } + if !found { + t.Fatal("lost actual moffs operand") + } + }) + } + } +} + +func TestX86RawAddressOverrideEA32ExtendedSIBFieldsStayRestricted(t *testing.T) { + // Physical REX.B/X extend EA32 base/index fields in 64-bit mode. The + // index field 4 without REX.X means no index, not an incoming SP grant. + for base := byte(0); base < 16; base++ { + for index := byte(0); index < 16; index++ { + for scale := byte(0); scale < 4; scale++ { + t.Run(fmt.Sprintf("base%d/index%d/scale%d", base, index, scale), func(t *testing.T) { + rex := byte(0x40) | base>>3 | (index>>3)<<1 + code := []byte{0x67, rex, 0x8b, 0x44, scale<<6 | (index&7)<<3 | base&7, 0x80} + physical, err := x86asm.Decode(code, 64) + if err != nil || physical.Len != len(code) || physical.AddrSize != 32 { + t.Fatalf("physical extended EA32 fields: %+v, %v", physical, err) + } + found := false + for _, instruction := range x86AddressOverrideDecoded(t, 64, code) { + if instruction.Op == OpMOVL { + found = true + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != 32 { + t.Fatalf("extended EA32 acquired native pointer semantics: %+v", instruction) + } + } + } + if !found { + t.Fatal("lost extended physical memory form") + } + }) + } + } + } +} + +func TestX86RawAddressOverrideExistingTypedConsumersRemainBound(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + mode := 64 + if target.arch == "386" { + mode = 32 + } + forms := []struct { + name string + code []byte + }{ + {"ADCXL", []byte{0x67, 0x66, 0x0f, 0x38, 0xf6, 0x18}}, + {"ADOXL", []byte{0x67, 0xf3, 0x0f, 0x38, 0xf6, 0x18}}, + {"ENQCMD", []byte{0x67, 0xf2, 0x0f, 0x38, 0xf8, 0x00}}, + {"ENQCMDS", []byte{0x67, 0xf3, 0x0f, 0x38, 0xf8, 0x00}}, + {"UMONITOR", []byte{0x67, 0xf3, 0x0f, 0xae, 0xf3}}, + } + if mode == 64 { + forms = append(forms, struct { + name string + code []byte + }{"ADCXQ", []byte{0x67, 0x66, 0x48, 0x0f, 0x38, 0xf6, 0x18}}, struct { + name string + code []byte + }{"ADOXQ", []byte{0x67, 0xf3, 0x48, 0x0f, 0x38, 0xf6, 0x18}}) + } + for _, form := range forms { + t.Run(target.triple+"/"+form.name, func(t *testing.T) { + assembly := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(form.code) + "RET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, assembly) + found := false + for _, instruction := range x86AddressOverrideDecoded(t, mode, form.code) { + if instruction.Op != Op(form.name) { + continue + } + found = true + if instruction.x86SourceRaw == nil || instruction.x86AddressBits != mode/2 || instruction.x86UnboundAddressOverride { + t.Fatalf("dedicated typed address receipt changed: %+v", instruction) + } + } + if !found { + t.Fatal("lost dedicated width-consuming instruction") + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + var ir string + if form.name == "UMONITOR" { + // Retain this family's existing physical-width observation; + // it does not establish a monitor extent or runtime result. + ir, err = Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw": {Name: "raw", Ret: Void}}, + }) + } else { + // ADX and enqueue inputs are absent. Original Go/emitter + // objects remain required, while every public API retains + // Context instead of inventing numerical or native authority. + ir = x86UnboundVoidRawScalarFormatObject(t, assembly, target.arch, target.triple) + } + if err != nil || !strings.Contains(ir, fmt.Sprintf("to i%d", mode/2)) { + t.Fatalf("dedicated consumer lost physical truncation: %v", err) + } + compileLLVMToObject(t, llc, target.triple, "address-consumer.ll", "address-consumer.o", ir) + }) + } + } +} + +func TestX86RawAddressOverrideImplicitMaskedStoreAndMonitor(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + for _, form := range []struct { + name, named string + code []byte + }{ + {"MASKMOVQ", "MASKMOVQ M1,M0", []byte{0x67, 0x0f, 0xf7, 0xc1}}, + {"MASKMOVDQU", "MASKMOVDQU X1,X0", []byte{0x67, 0x66, 0x0f, 0xf7, 0xc1}}, + {"MONITOR", "MONITOR", []byte{0x67, 0x0f, 0x01, 0xc8}}, + } { + t.Run(target.triple+"/"+form.name, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, "TEXT raw(SB),4,$0-0\n"+form.named+"\nRET\n") + if encoded := x86AddressOverrideNamedBytes(t, target.arch, form.named); !bytes.HasPrefix(encoded, form.code[1:]) { + t.Fatalf("named Go form has different physical opcode: %x, want prefix %x", encoded, form.code[1:]) + } + assembly := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(form.code) + "RET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, assembly) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw": {Name: "raw", Ret: Void}}, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("implicit physical address override borrowed default addressing: %v", err) + } + + // Keep the complete original default-address byte control. + // Removing 0x67 cannot supply an initialized mask/data/DI or + // a complete MMX/EMMS region to a bare public entry. + control := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(form.code[1:]) + "RET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, control) + if form.name != "MONITOR" { + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, control, "func raw()", target.arch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("default raw mask store invented its native input owner", api, err) + } + } + // The same physical register/opcode control has genuine LLVM + // objects only with independently initialized named inputs. + // This does not grant a raw numerical or implicit-address role. + source, declaration := x86ImplicitMaskMoveOracleRegisters(target.arch, form.name, "Default", 1, 0) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("initialized default mask-store control lost its original opcode", api, err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + body := x86StringMemoryFunction(t, text, "Default") + if !strings.Contains(body, "asm sideeffect") || !strings.Contains(body, strings.ToLower(form.name)) || + !strings.Contains(body, "~{memory}") || strings.Contains(body, "call void @llvm.masked.store") || + form.name == "MASKMOVQ" && !strings.Contains(body, "emms") { + t.Fatal("default opcode lost its native access or MMX cleanup", api) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + } + return + } + file, err = Parse(ArchAMD64, control) + if err != nil { + t.Fatal(err) + } + ir, err = Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw": {Name: "raw", Ret: Void}}, + }) + if err != nil { + t.Fatalf("default-address control regressed: %v", err) + } + compileLLVMToObject(t, llc, target.triple, "implicit-address.ll", "implicit-address.o", ir) + }) + } + } +} + +func TestX86RawAddressOverrideImplicitMaskedStoreAllRegisterFields(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, xmm := range []bool{false, true} { + last := 7 + if xmm && mode == 64 { + last = 15 + } + for source := 0; source <= last; source++ { + for mask := 0; mask <= last; mask++ { + t.Run(fmt.Sprintf("mode%d/xmm%t/source%d/mask%d", mode, xmm, source, mask), func(t *testing.T) { + code := []byte{0x67} + if xmm { + code = append(code, 0x66) + } + if source >= 8 || mask >= 8 { + rex := byte(0x40) + if source >= 8 { + rex |= 4 + } + if mask >= 8 { + rex |= 1 + } + code = append(code, rex) + } + code = append(code, 0x0f, 0xf7, byte(0xc0|(source&7)<<3|(mask&7))) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) || physical.AddrSize != mode/2 { + t.Fatalf("physical implicit store: %+v, %v", physical, err) + } + for _, instruction := range x86AddressOverrideDecoded(t, mode, code) { + if instruction.Op == OpTEXT || instruction.Op == OpRET { + continue + } + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != mode/2 { + t.Fatalf("implicit address restriction lost at register alias: %+v", instruction) + } + } + }) + } + } + } + } +} + +func TestX86RawAddressOverrideUnsupportedImplicitDecoderBoundaries(t *testing.T) { + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + // Go accepts its named VEX spelling, while the pinned x/arch + // generic decoder does not implement this opcode. Do not grant + // a legacy/default address interpretation to its raw bytes. + x86LEAActualTargetGoObject(t, target.arch, target.triple, "TEXT raw(SB),4,$0-0\nVMASKMOVDQU X0,X1\nRET\n") + for _, code := range [][]byte{ + {0xc5, 0xf9, 0xf7, 0xc1}, {0x67, 0xc5, 0xf9, 0xf7, 0xc1}, + {0x0f, 0x01, 0xfa}, {0x67, 0x0f, 0x01, 0xfa}, // MONITORX + } { + mode := 64 + if target.arch == "386" { + mode = 32 + } + if _, err := x86AddressOverrideDecode(mode, code); err == nil { + t.Fatalf("unsupported implicit raw decoder acquired a consumer: %x", code) + } + } + requireLEANativeGoObject(t, target.arch, "TEXT raw(SB),4,$0-0\nMONITORX\nRET\n", false) + }) + } +} + +func TestX86RawAddressOverrideDoesNotInventMemoryForRegisterForms(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, code := range [][]byte{{0x67, 0x8b, 0xc2}, {0x67, 0x89, 0xc2}, {0x67, 0x03, 0xc2}} { + t.Run(fmt.Sprintf("mode%d/%x", mode, code), func(t *testing.T) { + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("physical register control: %+v, %v", physical, err) + } + for _, instruction := range x86AddressOverrideDecoded(t, mode, code) { + if instruction.x86UnboundAddressOverride || instruction.x86AddressBits != 0 { + t.Fatalf("ignored register prefix gained physical memory semantics: %+v", instruction) + } + } + }) + } + } +} + +func TestX86RawAddressOverrideRestrictionSurvivesNormalizationAndMutation(t *testing.T) { + assembly := "TEXT raw(SB),4,$0-0\nBYTE $0x67\nBYTE $0x8b\nBYTE $0x07\nRET\n" + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + for pass := 0; pass < 3; pass++ { + file, err = normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + copy := *file + copy.Funcs = append([]Func(nil), file.Funcs...) + copy.Funcs[0].Instrs = append([]Instr(nil), file.Funcs[0].Instrs...) + file = © + found := false + for _, instruction := range file.Funcs[0].Instrs { + if instruction.Op != OpMOVL { + continue + } + found = true + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != 32 { + t.Fatal("normalization/copy lost restrictive decoder identity") + } + for _, width := range []int{0, 16, 32, 64, 128} { + mutated := instruction + mutated.x86AddressBits = width + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{mutated}}, FuncSig{Ret: Void}, nil) + before := output.String() + if _, err := ctx.lowerInstr(0, 0, mutated, nil, nil); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("width %d laundered decoder restriction: %v", width, err) + } + } + } + if !found { + t.Fatal("normalization/copy lost source instruction") + } + } +} + +func TestX86RawAddressOverrideCannotReuseDefaultCallerCertificate(t *testing.T) { + instruction := Instr{Op: OpMOVL, Args: []Operand{ + {Kind: OpReg, Reg: AX}, {Kind: OpMem, Mem: MemRef{Base: DX}}, + }} + x86CallerRecordMemory(&instruction, 1, 4, true) + if !x86CallerMemoryCertified(instruction, 1) { + t.Fatal("default-width certificate control is invalid") + } + for _, width := range []int{0, 16, 32, 64} { + mutated := instruction + mutated.x86UnboundAddressOverride = true + mutated.x86AddressBits = width + if x86CallerMemoryCertified(mutated, 1) { + t.Fatalf("decoder restriction reused a default-width certificate at width %d", width) + } + // A change to the spelling or operands cannot turn a restrictive + // decoder receipt into a positive LEA/address capability either. + mutated.Op = "LEAQ" + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{mutated}}, FuncSig{Ret: Void}, nil) + before := output.String() + if _, err := ctx.lowerInstr(0, 0, mutated, nil, nil); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("changed opcode bypassed restrictive source identity: %v", err) + } + } +} + +func TestX86RawAddressOverrideDefaultCallerControlsAndPhysicalSegments(t *testing.T) { + for _, replacement := range []string{ + "MOVL $7,AX\nMOVL AX,(DX)", "MOVL (DI),AX\nADDL $7,AX\nMOVL AX,(DX)", + } { + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple+"/"+replacement, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX\nMOVL AX,(DX)", replacement, 1) + x86LEAActualTargetGoObject(t, "amd64", triple, assembly) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + space int + }{{0x64, FS, 257}, {0x65, GS, 256}} { + t.Run(string(segment.reg), func(t *testing.T) { + code := []byte{segment.prefix, 0x8b, 0x07} + found := false + for _, instruction := range x86AddressOverrideDecoded(t, 64, code) { + if instruction.Op == OpMOVL { + found = true + if instruction.x86UnboundAddressOverride || instruction.x86AddressBits != 0 || instruction.Args[0].Mem.Segment != segment.reg { + t.Fatalf("ordinary segment/address identity changed: %+v", instruction) + } + } + } + if !found { + t.Fatal("lost segmented MOVL control") + } + assembly := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(code) + "RET\n" + triple := "x86_64-unknown-linux-gnu" + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + ir := x86UnboundVoidRawScalarFormatObject(t, assembly, "amd64", triple) + x86PackedStringCompileObject(t, llc, triple, ir, "-O0") + initialized := strings.Replace(assembly, x86RawDisplacementDirectives(code), + "MOVQ $0,DI\n"+x86RawDisplacementDirectives(code), 1) + x86LEAActualTargetGoObject(t, "amd64", triple, initialized) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, initialized, "func raw()", "amd64", triple, api) + if err != nil || !strings.Contains(ir, fmt.Sprintf("load volatile i32, ptr addrspace(%d)", segment.space)) { + t.Fatalf("initialized default-width physical segment was erased: %s: %v", api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + } + }) + } +} + +func TestX86RawAddressOverrideFiveTargetsAndThreeAPIs(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, opcode := range []byte{0x8b, 0x89} { + t.Run(fmt.Sprintf("%s/%02x", target.triple, opcode), func(t *testing.T) { + assembly, sig, pkg := x86SourceStackNumericalFixture(target.arch, + "@MOV x+0(FP),DI\n"+x86RawDisplacementDirectives([]byte{0x67, opcode, 0x07})+"@MOV AX,ret+@WORD(FP)\nRET") + x86LEAActualTargetGoObject(t, target.arch, target.triple, assembly) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + ir, err := Translate(file, opt) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("ordinary API acquired overridden pointer: %v", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module API acquired overridden pointer: %v", err) + } + translated, err := TranslateGoModule(pkg, []byte(assembly), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: opt.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("typed API acquired overridden pointer: %v", err) + } + }) + } + } +} + +func TestX86RawAddressOverrideRetainsGenericPhysicalMemoryWidth(t *testing.T) { + for _, mode := range []int{32, 64} { + arch := "amd64" + if mode == 32 { + arch = "386" + } + for _, opcode := range []byte{0x8b, 0x89} { + t.Run(fmt.Sprintf("mode%d/%02x", mode, opcode), func(t *testing.T) { + code := []byte{0x67, opcode, 0x07} + assembly := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(code) + "RET\n" + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + function, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal(err) + } + for _, instruction := range function.Instrs { + if instruction.Op == OpMOVL { + if instruction.x86AddressBits != mode/2 { + t.Fatalf("decoded physical address width lost: got %d, want %d; %+v", instruction.x86AddressBits, mode/2, instruction) + } + return + } + } + t.Fatal("decoded raw source lost its MOVL") + }) + } + } +} diff --git a/amd64_raw_adx.go b/amd64_raw_adx.go new file mode 100644 index 000000000..72c4a0e4f --- /dev/null +++ b/amd64_raw_adx.go @@ -0,0 +1,150 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// Intel SDM Vol. 2A specifies 66/F3 0F 38 F6 /r for ADCX/ADOX. REX.W +// alone selects 64 bits: 0x66 is mandatory, not a 16-bit width override. +// Reuse the decoder's ADD r/m,reg ModRM/SIB grammar, never its flag semantics. +func decodeX86RawADX(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + return decodeX86RawADXWithMemory(code, offset, mode, x86RawMemoryDecoder{}) +} + +func decodeX86RawADXWithMemory(code []byte, offset, mode int, strategy x86RawMemoryDecoder) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 32 && mode != 64 || offset < 0 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + bytes := code[offset:] + i := 0 + var repeat, segment, rex byte + operandOverride, addressOverride, locked := false, false, false + for i < len(bytes) { + switch bytes[i] { + case 0xf2, 0xf3: + repeat, rex = bytes[i], 0 + case 0x66: + operandOverride, rex = true, 0 + case 0x67: + addressOverride, rex = true, 0 + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65: + segment, rex = bytes[i], 0 + case 0xf0: + locked, rex = true, 0 + default: + if mode == 64 && bytes[i] >= 0x40 && bytes[i] <= 0x4f { + rex = bytes[i] + i++ + continue + } + goto opcode + } + i++ + } + +opcode: + prefix := repeat + if prefix == 0 && operandOverride { + prefix = 0x66 + } + if prefix != 0x66 && prefix != 0xf3 || len(bytes) < i+3 || + bytes[i] != 0x0f || bytes[i+1] != 0x38 || bytes[i+2] != 0xf6 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + fail := func(reason string) (Instr, int, x86RawLiteralRange, bool, error) { + return Instr{}, 0, x86RawLiteralRange{}, true, fmt.Errorf("ADX: %s", reason) + } + if locked { + return fail("LOCK is not permitted") + } + bits := 32 + if rex&8 != 0 { + bits = 64 + } + var op Op + for name, spec := range amd64ADXSpecs { + if spec.prefix == prefix && spec.bits == bits { + op = name + break + } + } + var equivalent []byte + if segment == 0x64 || segment == 0x65 { + equivalent = append(equivalent, segment) + } + if addressOverride { + equivalent = append(equivalent, 0x67) + } + if rex != 0 { + equivalent = append(equivalent, rex) + } + equivalent = append(equivalent, 0x03) + header := len(equivalent) + equivalent = append(equivalent, bytes[i+3:]...) + decoded, err := x86asm.Decode(equivalent, mode) + if err != nil || decoded.Op != x86asm.ADD || decoded.Len <= header { + return fail("truncated or invalid ModRM/SIB address") + } + length := i + 3 + decoded.Len - header + if length > 15 { + return fail("encoding exceeds the 15-byte instruction limit") + } + if strategy.physicalPC { + // A whole-object observer needs only the validated instruction's PC + // and length. Do not patch RIP bytes, require source-local literal + // storage, or return operands that could be mistaken for read facts. + return Instr{}, length, x86RawLiteralRange{}, true, nil + } + if decoded.PCRel != 0 { + if decoded.AddrSize != 64 || segment == 0x64 || segment == 0x65 { + return fail("PC-relative source needs an unsegmented source-local literal") + } + patched, data, literal, err := x86RawRIPBytes(code, offset, offset+i+3, bits/8) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + instruction, _, _, matched, err := decodeX86RawADX(patched, 0, mode) + if err != nil || !matched { + return fail("cannot represent patched source-local literal") + } + setX86RawRIPDataOperand(&instruction, 0, data) + return instruction, length, literal, true, nil + } + memory, isMemory := decoded.Args[1].(x86asm.Mem) + if isMemory { + // The textual parser has only two address groups. Attach FS/GS + // structurally so a 16-bit BX+SI address does not gain a third one. + memory.Segment = 0 + decoded.Args[1] = memory + } + syntax, err := decodedX86GoSyntax(decoded, equivalent[:decoded.Len]) + if err != nil { + return fail("cannot represent decoded operands") + } + parsed, err := parseDecodedX86Instruction(syntax) + if err != nil || len(parsed) != 1 || len(parsed[0].Args) != 2 { + return fail("cannot represent decoded register/memory form") + } + instruction := parsed[0] + if isMemory { + if memory.Base == 0 && memory.Index == 0 { + instruction.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Off: memory.Disp}} + } + if instruction.Args[0].Kind != OpMem { + return fail("cannot represent decoded source as memory") + } + switch segment { + case 0x64: + instruction.Args[0].Mem.Segment = FS + case 0x65: + instruction.Args[0].Mem.Segment = GS + } + } + instruction.Op = op + instruction.x86Encoded = true + instruction.x86AddressBits = decoded.AddrSize + instruction.Raw = fmt.Sprintf("%s %s, %s", op, instruction.Args[0], instruction.Args[1]) + return instruction, length, x86RawLiteralRange{}, true, nil +} diff --git a/amd64_raw_adx_test.go b/amd64_raw_adx_test.go new file mode 100644 index 000000000..aa2e97f30 --- /dev/null +++ b/amd64_raw_adx_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86ADXGrammarMatchesCompleteGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + text := string(data) + table := regexp.MustCompile(`(?s)var yml_rl = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(text) + if len(table) != 2 || strings.Count(table[1], "argList{") != 1 || + !regexp.MustCompile(`\{Zm_r,\s*1,\s*argList\{Yml, Yrl\}\}`).MatchString(table[1]) { + t.Fatal("Go ADX register/memory operand grammar changed") + } + rows := regexp.MustCompile(`\{A((?:ADCX|ADOX)[LQ]),\s*yml_rl,\s*(Pq[45]w?),\s*opBytes\{0xf6\}\}`).FindAllStringSubmatch(text, -1) + if len(rows) != 4 || len(amd64ADXSpecs) != len(rows) { + t.Fatal("typed ADX grammar does not match the complete Go encoder family") + } + for _, row := range rows { + spec, ok := amd64ADXSpecs[Op(row[1])] + bits, prefix, carry := 32, byte(0x66), amd64ADXCarryCF + if strings.HasSuffix(row[2], "w") { + bits = 64 + } + if strings.HasPrefix(row[2], "Pq5") { + prefix, carry = 0xf3, amd64ADXCarryOF + } + if !ok || spec.bits != bits || spec.prefix != prefix || spec.carry != carry { + t.Fatalf("%s spec %+v does not match Go encoder row %v", row[1], spec, row) + } + } +} + +func TestX86RawADXEveryRegisterEncoding(t *testing.T) { + for op, spec := range amd64ADXSpecs { + for _, mode := range []int{32, 64} { + if mode == 32 && spec.bits == 64 { + continue + } + registers := 16 + if mode == 32 { + registers = 8 + } + for source := 0; source < registers; source++ { + for destination := 0; destination < registers; destination++ { + code := []byte{spec.prefix} + rex := byte(0x40 | source>>3 | (destination>>3)<<2) + if spec.bits == 64 { + rex |= 8 + } + if rex != 0x40 { + code = append(code, rex) + } + code = append(code, 0x0f, 0x38, 0xf6, byte(0xc0|(destination&7)<<3|source&7)) + got, length, _, matched, err := decodeX86RawADX(code, 0, mode) + wantSource, _ := decodedX86GeneralRegister(source) + wantDestination, _ := decodedX86GeneralRegister(destination) + if err != nil || !matched || length != len(code) || got.Op != op || + got.Args[0].Reg != wantSource || got.Args[1].Reg != wantDestination { + t.Fatalf("decode %x mode=%d: %+v error=%v", code, mode, got, err) + } + } + } + } + } +} + +func TestX86RawADXAddressAndSegmentForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, mode := range []int{32, 64} { + for _, segment := range []byte{0, 0x64, 0x65} { + for _, override := range []bool{false, true} { + for _, prefix := range []byte{0x66, 0xf3} { + code := []byte{prefix} + if segment != 0 { + code = append(code, segment) + } + bits := mode + if override { + code = append(code, 0x67) + bits /= 2 + } + code = append(code, 0x0f, 0x38, 0xf6, 0x18) + ins, length, _, matched, err := decodeX86RawADX(code, 0, mode) + if err != nil || !matched || length != len(code) || ins.x86AddressBits != bits || ins.Args[0].Kind != OpMem { + t.Fatalf("decode %x mode=%d: %+v error=%v", code, mode, ins, err) + } + wantSegment := Reg("") + if segment == 0x64 { + wantSegment = FS + } else if segment == 0x65 { + wantSegment = GS + } + if ins.Args[0].Mem.Segment != wantSegment { + t.Fatalf("segment lost: %+v", ins) + } + var source strings.Builder + source.WriteString("TEXT adxaddress(SB),$0-0\n") + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("RET\n") + arch, triple := "amd64", "x86_64-unknown-linux-gnu" + if mode == 32 { + arch, triple = "386", "i386-unknown-linux-gnu" + } + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), arch, triple) + if override && !strings.Contains(ir, fmt.Sprintf(" to i%d", bits)) { + t.Fatal("address-size wrapping was not lowered") + } + compileLLVMToObject(t, llc, triple, "adx-address.ll", "adx-address.o", ir) + } + } + } + } +} + +func TestX86RawADXSourceLocalRIPAndInvalidEncodings(t *testing.T) { + for op, spec := range amd64ADXSpecs { + code := []byte{spec.prefix} + if spec.bits == 64 { + code = append(code, 0x48) + } + code = append(code, 0x0f, 0x38, 0xf6, 0x1d, 1, 0, 0, 0, 0xc3) + for i := 0; i < spec.bits/8; i++ { + code = append(code, byte(i+1)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, string(op), map[string]bool{}) + if err != nil || len(decoded) != 2 || decoded[0].Op != op || len(decoded[0].x86RIPLiteralData) != spec.bits/8 { + t.Fatalf("source-local literal %x: %+v error=%v", code, decoded, err) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated literal", map[string]bool{}); err == nil { + t.Fatal("accepted incomplete literal") + } + } + for _, code := range [][]byte{ + {0xf0, 0x66, 0x0f, 0x38, 0xf6, 0xc0}, + {0x66, 0x0f, 0x38, 0xf6}, + {0xf3, 0x0f, 0x38, 0xf6, 0x04}, + {0x66, 0x0f, 0x38, 0xf6, 0x80}, + append([]byte{0x66}, append([]byte(strings.Repeat("\x67", 11)), 0x0f, 0x38, 0xf6, 0xc0)...), + } { + if _, _, _, matched, err := decodeX86RawADX(code, 0, 64); !matched || err == nil { + t.Fatalf("accepted invalid ADX %x", code) + } + } +} + +func TestX86RawADXCompleteRegisterAndMemoryForms(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, prefix := range []byte{0x66, 0xf3} { + for _, bits := range []int{32, 64} { + if mode == 32 && bits == 64 { + continue + } + for _, form := range []struct { + name string + modrm []byte + source string + }{ + {"register", []byte{0xd8}, "AX"}, + {"memory", []byte{0x58, 0x20}, "32(AX)"}, + {"indexed", []byte{0x5c, 0x88, 0xf0}, "-16(AX)(CX*4)"}, + } { + name := fmt.Sprintf("mode%d/prefix%x/bits%d/%s", mode, prefix, bits, form.name) + t.Run(name, func(t *testing.T) { + code := []byte{prefix} + if bits == 64 { + code = append(code, 0x48) + } + code = append(code, 0x0f, 0x38, 0xf6) + code = append(code, form.modrm...) + code = append(code, 0xc3) + decoded, err := decodeX86RawDirectiveGroup(code, mode, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + stem, width := "ADCX", "L" + if prefix == 0xf3 { + stem = "ADOX" + } + if bits == 64 { + width = "Q" + } + if len(decoded) != 2 || decoded[0].Op != Op(stem+width) || decoded[1].Op != OpRET || + len(decoded[0].Args) != 2 || decoded[0].Args[0].String() != form.source || decoded[0].Args[1].Reg != BX { + t.Fatalf("decoded %x as %+v, want %s%s %s, BX; RET", code, decoded, stem, width, form.source) + } + }) + } + } + } + } +} + +func TestX86RawADXExtendedRegistersFromEthereum(t *testing.T) { + for _, form := range []struct { + code []byte + op Op + source, destination Reg + }{ + {[]byte{0x66, 0x4c, 0x0f, 0x38, 0xf6, 0xc8}, "ADCXQ", AX, "R9"}, + {[]byte{0xf3, 0x4c, 0x0f, 0x38, 0xf6, 0xd3}, "ADOXQ", BX, "R10"}, + {[]byte{0x66, 0x4d, 0x0f, 0x38, 0xf6, 0xe7}, "ADCXQ", "R15", "R12"}, + } { + code := append(append([]byte(nil), form.code...), 0xc3) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "Ethereum ADX", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != form.op || + decoded[0].Args[0].Reg != form.source || decoded[0].Args[1].Reg != form.destination { + t.Fatalf("decoded %x as %+v", code, decoded) + } + } +} + +func TestX86RawADXObjectsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT adxraw(SB),$0-0\n") + for _, prefix := range []byte{0x66, 0xf3} { + for _, rex := range []byte{0, 0x48} { + if target.arch == "386" && rex != 0 { + continue + } + for _, modrm := range [][]byte{{0xd8}, {0x58, 0x20}, {0x5c, 0x88, 0xf0}} { + code := []byte{prefix} + if rex != 0 { + code = append(code, rex) + } + code = append(code, 0x0f, 0x38, 0xf6) + code = append(code, modrm...) + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.arch, target.triple) + compileLLVMToObject(t, llc, target.triple, "adx-raw.ll", "adx-raw.o", ir) + }) + } +} diff --git a/amd64_raw_aes.go b/amd64_raw_aes.go new file mode 100644 index 000000000..3d38233f1 --- /dev/null +++ b/amd64_raw_aes.go @@ -0,0 +1,162 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// VAES is absent from the pinned generic decoder. Decode its complete actual +// Go VEX/EVEX family without lending another instruction's operand/PC proof. +func decodedX86AESInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86AESInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86AESInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := 0, Reg(""), false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto prefix + } + } +prefix: + if i >= len(code) { + return Instr{}, 0, false, nil + } + var opcodeIndex, modRMIndex, mapSelect, width, rExt, rHigh, xExt, bExt, vRegister, vHigh, rmHigh int + evex := false + switch code[i] { + case 0xc4: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + p0, p1 := code[i+1], code[i+2] + opcodeIndex, modRMIndex, mapSelect = i+3, i+4, int(p0&31) + if p1&3 != 1 || x86AESRawOpcode(mapSelect, code[opcodeIndex]) == "" { + return Instr{}, 0, false, nil + } + rExt, xExt, bExt = int(^p0>>7)&1, int(^p0>>6)&1, int(^p0>>5)&1 + vRegister = int(^p1>>3) & 15 + width = 16 + if p1&4 != 0 { + width = 32 + } + case 0x62: + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcodeIndex, modRMIndex, mapSelect = i+4, i+5, int(p0&3) + if p0&0x0c != 0 || p1&3 != 1 || x86AESRawOpcode(mapSelect, code[opcodeIndex]) == "" { + return Instr{}, 0, false, nil + } + evex = true + if p1&4 != 4 || p2&0x97 != 0 { + return Instr{}, 0, true, fmt.Errorf("AES EVEX has reserved bit/mask/zeroing/broadcast attributes") + } + rExt, rHigh, xExt, bExt = int(^p0>>7)&1, int(^p0>>4)&1, int(^p0>>6)&1, int(^p0>>5)&1 + vRegister, vHigh, rmHigh = int(^p1>>3)&15, int(^p2>>3)&1, xExt + switch p2 >> 5 & 3 { + case 0: + width = 16 + case 1: + width = 32 + case 2: + width = 64 + default: + return Instr{}, 0, true, fmt.Errorf("reserved AES EVEX vector length") + } + default: + return Instr{}, 0, false, nil + } + op := x86AESRawOpcode(mapSelect, code[opcodeIndex]) + unary := op == "VAESIMC" || op == "VAESKEYGENASSIST" + if unary && (evex || width != 16 || vRegister != 0) { + return Instr{}, 0, true, fmt.Errorf("AES unary requires VEX.128 with reserved vvvv") + } + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing AES ModRM") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("%w: AES native address-size override is not bound", ErrProbeNeedsContext) + } + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + rHigh*16 + stateNumber := vRegister + vHigh*16 + if mode == 32 && (destinationNumber >= 8 || stateNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("%w: extended AES vector in physical mode32", ErrProbeNeedsContext) + } + if mode == 32 && code[modRMIndex]>>6 == 3 && int(code[modRMIndex]&7)+bExt*8+rmHigh*16 >= 8 { + return Instr{}, 0, true, fmt.Errorf("%w: extended AES source vector in physical mode32", ErrProbeNeedsContext) + } + destination, valid := decodedX86VectorRegister(destinationNumber, width) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid AES destination") + } + disp8Scale := 1 + if evex { + disp8Scale = width + } + var source Operand + var consumed int + var err error + if code[modRMIndex]>>6 == 3 { + source, consumed, err = decodedX86VectorRMSource(code[modRMIndex:], mode, bExt, xExt, rmHigh, width, segment, disp8Scale) + } else { + source, consumed, err = decodedX86RMSourceWithPC(code[modRMIndex:], mode, bExt, xExt, segment, disp8Scale, memory.physicalPC) + } + if err != nil { + return Instr{}, 0, true, err + } + length := modRMIndex + consumed + args := []Operand{source} + if op == "VAESKEYGENASSIST" { + if len(code) <= length { + return Instr{}, 0, true, fmt.Errorf("missing AES imm8") + } + args = append([]Operand{{Kind: OpImm, Imm: int64(code[length])}}, args...) + length++ + } + if !unary { + state, _ := decodedX86VectorRegister(stateNumber, width) + args = append(args, Operand{Kind: OpReg, Reg: state}) + } + args = append(args, Operand{Kind: OpReg, Reg: destination}) + spelling := make([]string, len(args)) + for index, a := range args { + spelling[index] = a.String() + } + return Instr{Op: op, Args: args, Raw: string(op) + " " + strings.Join(spelling, ", "), x86Encoded: true}, length, true, nil +} + +func x86AESRawOpcode(mapSelect int, opcode byte) Op { + if mapSelect == 3 && opcode == 0xdf { + return "VAESKEYGENASSIST" + } + if mapSelect == 2 { + switch opcode { + case 0xdb: + return "VAESIMC" + case 0xdc: + return "VAESENC" + case 0xdd: + return "VAESENCLAST" + case 0xde: + return "VAESDEC" + case 0xdf: + return "VAESDECLAST" + } + } + return "" +} diff --git a/amd64_raw_alias_test.go b/amd64_raw_alias_test.go new file mode 100644 index 000000000..e09fa4f14 --- /dev/null +++ b/amd64_raw_alias_test.go @@ -0,0 +1,550 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86RawDecoderPreservesGoOpcodeAliasesAndMandatoryPrefixes(t *testing.T) { + tests := []struct { + name string + code []byte + want Op + }{ + {name: "PACKSSDW is Go PACKSSLW", code: []byte{0x66, 0x0f, 0x6b, 0xca}, want: "PACKSSLW"}, + {name: "PCMPEQD is Go PCMPEQL", code: []byte{0x66, 0x0f, 0x76, 0xca}, want: "PCMPEQL"}, + {name: "PCMPGTD is Go PCMPGTL", code: []byte{0x66, 0x0f, 0x66, 0xca}, want: "PCMPGTL"}, + {name: "PMADDWD is Go PMADDWL", code: []byte{0x66, 0x0f, 0xf5, 0xca}, want: "PMADDWL"}, + {name: "PMULUDQ is Go PMULULQ", code: []byte{0x66, 0x0f, 0xf4, 0xca}, want: "PMULULQ"}, + {name: "PSLLD is Go PSLLL", code: []byte{0x66, 0x0f, 0xf2, 0xca}, want: "PSLLL"}, + {name: "PSRAD is Go PSRAL", code: []byte{0x66, 0x0f, 0xe2, 0xca}, want: "PSRAL"}, + {name: "PSRLD is Go PSRLL", code: []byte{0x66, 0x0f, 0xd2, 0xca}, want: "PSRLL"}, + {name: "PSUBD is Go PSUBL", code: []byte{0x66, 0x0f, 0xfa, 0xca}, want: "PSUBL"}, + {name: "PUNPCKHDQ is Go PUNPCKHLQ", code: []byte{0x66, 0x0f, 0x6a, 0xca}, want: "PUNPCKHLQ"}, + {name: "PUNPCKHWD is Go PUNPCKHWL", code: []byte{0x66, 0x0f, 0x69, 0xca}, want: "PUNPCKHWL"}, + {name: "PUNPCKLDQ is Go PUNPCKLLQ", code: []byte{0x66, 0x0f, 0x62, 0xca}, want: "PUNPCKLLQ"}, + {name: "PUNPCKLWD is Go PUNPCKLWL", code: []byte{0x66, 0x0f, 0x61, 0xca}, want: "PUNPCKLWL"}, + {name: "MOVDQA is Go MOVO", code: []byte{0x66, 0x0f, 0x6f, 0xc1}, want: "MOVO"}, + {name: "MOVDQU is Go MOVOU", code: []byte{0xf3, 0x0f, 0x6f, 0xc1}, want: "MOVOU"}, + {name: "MOVSD XMM is Go MOVSD", code: []byte{0xf2, 0x0f, 0x10, 0xc1}, want: "MOVSD"}, + {name: "CLFLUSHOPT mandatory prefix", code: []byte{0x66, 0x0f, 0xae, 0x38}, want: "CLFLUSHOPT"}, + {name: "CLWB mandatory prefix", code: []byte{0x66, 0x0f, 0xae, 0x30}, want: "CLWB"}, + } + for _, goarch := range []string{"386", "amd64"} { + for _, test := range tests { + t.Run(goarch+"/"+test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.want { + t.Fatalf("decoded %#x as %#v, want one %s instruction", test.code, got.Instrs, test.want) + } + }) + } + } +} + +func TestX86RawDecoderVBROADCASTI128CompleteFamily(t *testing.T) { + tests := []struct { + name string + goarch string + code []byte + source MemRef + dest Reg + }{ + {name: "low registers", goarch: "amd64", code: []byte{0xc4, 0xe2, 0x7d, 0x5a, 0x13}, source: MemRef{Base: BX}, dest: "Y2"}, + {name: "extended base", goarch: "amd64", code: []byte{0xc4, 0xc2, 0x7d, 0x5a, 0x13}, source: MemRef{Base: "R11"}, dest: "Y2"}, + {name: "extended destination", goarch: "amd64", code: []byte{0xc4, 0x62, 0x7d, 0x5a, 0x1b}, source: MemRef{Base: BX}, dest: "Y11"}, + {name: "extended base and destination with displacement", goarch: "amd64", code: []byte{0xc4, 0x42, 0x7d, 0x5a, 0x70, 0x10}, source: MemRef{Base: "R8", Off: 16}, dest: "Y14"}, + {name: "extended SIB", goarch: "amd64", code: []byte{0xc4, 0x02, 0x7d, 0x5a, 0x64, 0x88, 0x20}, source: MemRef{Base: "R8", Index: "R9", Scale: 4, Off: 32}, dest: "Y12"}, + {name: "FS segment", goarch: "amd64", code: []byte{0x64, 0xc4, 0xe2, 0x7d, 0x5a, 0x13}, source: MemRef{Segment: FS, Base: BX}, dest: "Y2"}, + {name: "32-bit base", goarch: "386", code: []byte{0xc4, 0xe2, 0x7d, 0x5a, 0x53, 0x7f}, source: MemRef{Base: BX, Off: 127}, dest: "Y2"}, + {name: "32-bit absolute", goarch: "386", code: []byte{0xc4, 0xe2, 0x7d, 0x5a, 0x15, 0x78, 0x56, 0x34, 0x12}, source: MemRef{Off: 0x12345678}, dest: "Y2"}, + } + for _, test := range tests { + t.Run(test.goarch+"/"+test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, test.goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 { + t.Fatalf("decoded %#x as %#v, want one instruction", test.code, got.Instrs) + } + ins := got.Instrs[0] + if ins.Op != "VBROADCASTI128" || len(ins.Args) != 2 || ins.Args[0].Kind != OpMem || ins.Args[0].Mem != test.source || ins.Args[1].Kind != OpReg || ins.Args[1].Reg != test.dest { + t.Fatalf("decoded %#x as %#v, want VBROADCASTI128 %#v, %s", test.code, ins, test.source, test.dest) + } + }) + } +} + +func TestX86RawDecoderNormalizesArchitecturalAliasesToGo127Forms(t *testing.T) { + code := []byte{ + 0x41, 0x0f, 0x90, 0xc1, // SETO R9B -> SETOS R9B + 0x41, 0x0f, 0x92, 0xc1, // SETB R9B -> SETCS R9B + 0x48, 0x0f, 0x4e, 0xc1, // CMOVLE RAX, RCX -> CMOVQLE AX, CX + 0x04, 0x01, // ADD $1, AL -> ADDB $1, AL + } + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + want := []Op{"SETOS", "SETCS", "CMOVQLE", "ADDB"} + if len(got.Instrs) != len(want) { + t.Fatalf("decoded %#x as %#v, want %d instructions", code, got.Instrs, len(want)) + } + for i, op := range want { + if got.Instrs[i].Op != op { + t.Errorf("decoded instruction %d as %s, want %s (%q)", i, got.Instrs[i].Op, op, got.Instrs[i].Raw) + } + } + if got.Instrs[0].Args[0].Reg != R9B || got.Instrs[1].Args[0].Reg != R9B { + t.Errorf("SETcc byte aliases = %s and %s, want R9B", got.Instrs[0].Args[0].Reg, got.Instrs[1].Args[0].Reg) + } +} + +func TestX86RawDecoderNormalizesEveryThreeOperandIMULWidth(t *testing.T) { + tests := []struct { + name string + code []byte + want Op + }{ + {name: "word imm8 register", code: []byte{0x66, 0x6b, 0xf9, 0x07}, want: "IMUL3W"}, + {name: "long imm8 register", code: []byte{0x6b, 0xf9, 0x07}, want: "IMUL3L"}, + {name: "quad imm8 register", code: []byte{0x48, 0x6b, 0xf9, 0x07}, want: "IMUL3Q"}, + {name: "word imm16 memory", code: []byte{0x66, 0x69, 0x78, 0x08, 0x34, 0x12}, want: "IMUL3W"}, + {name: "long imm32 memory", code: []byte{0x69, 0x78, 0x08, 0x78, 0x56, 0x34, 0x12}, want: "IMUL3L"}, + {name: "quad imm32 memory", code: []byte{0x48, 0x69, 0x78, 0x08, 0x78, 0x56, 0x34, 0x12}, want: "IMUL3Q"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.want || len(got.Instrs[0].Args) != 3 { + t.Fatalf("decoded %#x as %#v, want one three-operand %s", test.code, got.Instrs, test.want) + } + }) + } +} + +func TestX86RawDecoderNormalizesCompleteBSWAPFamily(t *testing.T) { + tests := []struct { + name string + code []byte + want Op + reg Reg + }{ + {name: "long low register", code: []byte{0x0f, 0xc8}, want: "BSWAPL", reg: AX}, + {name: "long extended register", code: []byte{0x41, 0x0f, 0xcf}, want: "BSWAPL", reg: Reg("R15")}, + {name: "quad low register", code: []byte{0x48, 0x0f, 0xc8}, want: "BSWAPQ", reg: AX}, + {name: "quad extended register", code: []byte{0x49, 0x0f, 0xcf}, want: "BSWAPQ", reg: Reg("R15")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.want || len(got.Instrs[0].Args) != 1 || got.Instrs[0].Args[0].Reg != test.reg { + t.Fatalf("decoded %#x as %#v, want %s %s", test.code, got.Instrs, test.want, test.reg) + } + }) + } +} + +func TestX86RawDecoderNormalizesCompleteScalarIntegerFloatFamily(t *testing.T) { + tests := []struct { + name string + code []byte + want Op + }{ + {name: "long to float register", code: []byte{0xf3, 0x0f, 0x2a, 0xc8}, want: "CVTSL2SS"}, + {name: "long to float memory", code: []byte{0xf3, 0x0f, 0x2a, 0x48, 0x08}, want: "CVTSL2SS"}, + {name: "quad to float register", code: []byte{0xf3, 0x48, 0x0f, 0x2a, 0xc8}, want: "CVTSQ2SS"}, + {name: "quad to float memory", code: []byte{0xf3, 0x48, 0x0f, 0x2a, 0x48, 0x08}, want: "CVTSQ2SS"}, + {name: "long to double register", code: []byte{0xf2, 0x0f, 0x2a, 0xc8}, want: "CVTSL2SD"}, + {name: "long to double memory", code: []byte{0xf2, 0x0f, 0x2a, 0x48, 0x08}, want: "CVTSL2SD"}, + {name: "quad to double register", code: []byte{0xf2, 0x48, 0x0f, 0x2a, 0xc8}, want: "CVTSQ2SD"}, + {name: "quad to double memory", code: []byte{0xf2, 0x48, 0x0f, 0x2a, 0x48, 0x08}, want: "CVTSQ2SD"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.want || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want one %s", test.code, got.Instrs, test.want) + } + }) + } +} + +func TestX86RawDecoderNormalizesCompleteScalarFloatIntegerFamily(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + for _, precision := range []struct { + name string + prefix byte + }{ + {name: "SS", prefix: 0xf3}, + {name: "SD", prefix: 0xf2}, + } { + for _, conversion := range []struct { + name string + opcode byte + }{ + {name: "CVT", opcode: 0x2d}, + {name: "CVTT", opcode: 0x2c}, + } { + for _, width := range []struct { + name string + rex []byte + }{ + {name: "L"}, + {name: "Q", rex: []byte{0x48}}, + } { + if goarch == "386" && width.name == "Q" { + continue + } + for _, source := range []struct { + name string + modRM byte + disp []byte + }{ + {name: "register", modRM: 0xc1}, + {name: "memory", modRM: 0x43, disp: []byte{0x08}}, + } { + name := goarch + "/" + conversion.name + precision.name + "2S" + width.name + "/" + source.name + t.Run(name, func(t *testing.T) { + code := []byte{precision.prefix} + code = append(code, width.rex...) + code = append(code, 0x0f, conversion.opcode, source.modRM) + code = append(code, source.disp...) + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + want := Op(conversion.name + precision.name + "2S" + width.name) + if len(got.Instrs) != 1 || got.Instrs[0].Op != want || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want one %s", code, got.Instrs, want) + } + }) + } + } + } + } + } +} + +func TestTranslateX86RawScalarFloatIntegerReportedEncoding(t *testing.T) { + const source = `TEXT rawScalarFloatInteger(SB),$0-0 + LONG $0x2c0f48f2 + BYTE $0xc0 // CVTTSD2SQ X0, AX + RET +` + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", triple) + if !strings.Contains(ir, `call i64 asm sideeffect "cvttsd2siq $1, $0"`) || strings.Contains(ir, "fptosi double") { + t.Fatalf("raw CVTTSD2SQ semantics missing:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "raw-scalar-float-integer.ll", "raw-scalar-float-integer.o", ir) + }) + } +} + +func TestX86RawDecoderNormalizesCompleteMOVNTDQFamily(t *testing.T) { + tests := []struct { + name string + code []byte + reg Reg + base Reg + }{ + {name: "low vector and base registers", code: []byte{0x66, 0x0f, 0xe7, 0x13}, reg: Reg("X2"), base: BX}, + {name: "extended vector register", code: []byte{0x66, 0x44, 0x0f, 0xe7, 0x1b}, reg: Reg("X11"), base: BX}, + {name: "extended base register", code: []byte{0x66, 0x41, 0x0f, 0xe7, 0x13}, reg: Reg("X2"), base: Reg("R11")}, + {name: "extended vector and base registers", code: []byte{0x66, 0x45, 0x0f, 0xe7, 0x1b}, reg: Reg("X11"), base: Reg("R11")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != "MOVNTO" || len(got.Instrs[0].Args) != 2 || got.Instrs[0].Args[0].Reg != test.reg || got.Instrs[0].Args[1].Mem.Base != test.base { + t.Fatalf("decoded %#x as %#v, want MOVNTO %s, (%s)", test.code, got.Instrs, test.reg, test.base) + } + }) + } +} + +func TestTranslateAMD64MOVNTOCompleteOperandForm(t *testing.T) { + const source = `TEXT movnto(SB),$0-0 + MOVNTO X2, 32(BX)(R8*1) + MOVNTO X11, sink<>(SB) + RET + +GLOBL sink<>(SB),$16 +` + ir := x86UnboundVoidScalarFormatObject(t, source, "movnto", "amd64", "x86_64-unknown-linux-gnu") + if count := strings.Count(ir, `asm sideeffect "movntdq `); count != 2 { + t.Fatalf("MOVNTO emitted %d native stores, want 2:\n%s", count, ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-movnto.ll", "amd64-movnto.o", ir) +} + +func TestDecodedX86CacheLineWritebackCoversMemoryEncodingsWithoutFalsePositives(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + ok bool + }{ + {name: "CLFLUSHOPT displacement", code: []byte{0x67, 0x66, 0x0f, 0xae, 0x78, 0x08}, want: "CLFLUSHOPT", ok: true}, + {name: "CLWB SIB displacement", code: []byte{0x66, 0x0f, 0xae, 0xb4, 0x24, 0x78, 0x56, 0x34, 0x12}, want: "CLWB", ok: true}, + {name: "CLWB REX base", code: []byte{0x66, 0x41, 0x0f, 0xae, 0x30}, want: "CLWB", ok: true}, + {name: "plain CLFLUSH", code: []byte{0x0f, 0xae, 0x38}}, + {name: "plain XSAVEOPT", code: []byte{0x0f, 0xae, 0x30}}, + {name: "register ModRM", code: []byte{0x66, 0x0f, 0xae, 0xf8}}, + {name: "opcode bytes in displacement", code: []byte{0x66, 0x8b, 0x80, 0x0f, 0xae, 0x38, 0x00}}, + } { + t.Run(test.name, func(t *testing.T) { + got, ok := decodedX86CacheLineWriteback(test.code) + if got != test.want || ok != test.ok { + t.Fatalf("decodedX86CacheLineWriteback(%#x) = %q, %v; want %q, %v", test.code, got, ok, test.want, test.ok) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedWencodeCLFLUSHOPT(t *testing.T) { + code := []byte{0x41, 0x66, 0x0f, 0xae, 0x3c, 0x09} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "wencode CLFLUSHOPT sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "CLFLUSHOPT" || len(decoded[0].Args) != 1 || decoded[0].Args[0].String() != "0(CX)(CX*1)" { + t.Fatalf("decoded %x as %#v, want CLFLUSHOPT 0(CX)(CX*1)", code, decoded) + } +} + +func encodeX86RawCacheLineWriteback(op Op, mode, base, index int) []byte { + extension := byte(7) + mandatory := op != "CLFLUSH" + if op == "CLWB" { + extension = 6 + } + var prefixes []byte + if mode == 64 { + rex := byte(0x40 | base/8 | index/8<<1) + if mandatory { + prefixes = append(prefixes, 0x66) + } + prefixes = append(prefixes, rex) + } else if mandatory { + prefixes = append(prefixes, 0x66) + } + mod := byte(0) + if base&7 == 5 { + mod = 1 + } + modRM := mod<<6 | extension<<3 | 4 + sib := byte(2<<6 | (index&7)<<3 | base&7) + code := append(prefixes, 0x0f, 0xae, modRM, sib) + if mod == 1 { + code = append(code, 0) + } + return code +} + +func TestDecodedX86RawCacheLineWritebackCompleteRegisterAddressFamily(t *testing.T) { + operations := []Op{"CLFLUSH", "CLFLUSHOPT", "CLWB"} + count := 0 + for _, mode := range []int{32, 64} { + registers := 8 + if mode == 64 { + registers = 16 + } + for _, op := range operations { + for base := 0; base < registers; base++ { + for index := 0; index < registers; index++ { + if index == 4 { + continue // SIB index 4 without X is the no-index encoding. + } + code := encodeX86RawCacheLineWriteback(op, mode, base, index) + got, length, ok, err := decodedX86CacheLineWritebackInstruction(code, mode) + baseReg, _ := decodedX86GeneralRegister(base) + indexReg, _ := decodedX86GeneralRegister(index) + wantMemory := MemRef{Base: baseReg, Index: indexReg, Scale: 4} + if err != nil || !ok || length != len(code) || got.Op != op || len(got.Args) != 1 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantMemory { + t.Fatalf("decode mode=%d %x = %+v, length=%d, ok=%v, err=%v; want %s %+v", mode, code, got, length, ok, err, op, wantMemory) + } + count++ + } + } + } + } + if count != 888 { + t.Fatalf("covered %d cache-line register-address encodings, want 888", count) + } +} + +func TestDecodedX86RawCacheLineWritebackMemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + op Op + mem MemRef + }{ + {name: "CLFLUSH base", code: []byte{0x0f, 0xae, 0x3b}, mode: 64, op: "CLFLUSH", mem: MemRef{Base: BX}}, + {name: "CLFLUSHOPT disp8", code: []byte{0x66, 0x0f, 0xae, 0x7e, 0x7f}, mode: 64, op: "CLFLUSHOPT", mem: MemRef{Base: SI, Off: 127}}, + {name: "CLWB disp32", code: []byte{0x66, 0x0f, 0xae, 0xb7, 0x78, 0x56, 0x34, 0x12}, mode: 64, op: "CLWB", mem: MemRef{Base: DI, Off: 0x12345678}}, + {name: "CLFLUSHOPT ineffective REX before 66", code: []byte{0x41, 0x66, 0x0f, 0xae, 0x3c, 0x09}, mode: 64, op: "CLFLUSHOPT", mem: MemRef{Base: CX, Index: CX, Scale: 1}}, + {name: "CLFLUSHOPT effective REX after 66", code: []byte{0x66, 0x41, 0x0f, 0xae, 0x3c, 0x09}, mode: 64, op: "CLFLUSHOPT", mem: MemRef{Base: "R9", Index: CX, Scale: 1}}, + {name: "CLWB FS extended SIB", code: []byte{0x64, 0x66, 0x43, 0x0f, 0xae, 0x74, 0x8b, 0x20}, mode: 64, op: "CLWB", mem: MemRef{Segment: FS, Base: "R11", Index: "R9", Scale: 4, Off: 32}}, + {name: "CLFLUSHOPT 386 absolute", code: []byte{0x66, 0x0f, 0xae, 0x3d, 0x78, 0x56, 0x34, 0x12}, mode: 32, op: "CLFLUSHOPT", mem: MemRef{Off: 0x12345678}}, + } { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86CacheLineWritebackInstruction(test.code, test.mode) + if err != nil || !ok || length != len(test.code) || got.Op != test.op || len(got.Args) != 1 || got.Args[0].Kind != OpMem || got.Args[0].Mem != test.mem { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %s %+v", test.code, got, length, ok, err, test.op, test.mem) + } + }) + } + + for _, test := range []struct { + name string + code []byte + mode int + }{ + {name: "address override", code: []byte{0x67, 0x66, 0x0f, 0xae, 0x38}, mode: 64}, + {name: "REX.W", code: []byte{0x66, 0x48, 0x0f, 0xae, 0x38}, mode: 64}, + {name: "REX.R", code: []byte{0x66, 0x44, 0x0f, 0xae, 0x38}, mode: 64}, + {name: "register operand", code: []byte{0x66, 0x0f, 0xae, 0xf8}, mode: 64}, + {name: "RIP relative", code: []byte{0x66, 0x0f, 0xae, 0x3d, 0, 0, 0, 0}, mode: 64}, + {name: "truncated SIB", code: []byte{0x66, 0x0f, 0xae, 0x3c}, mode: 64}, + {name: "truncated disp8", code: []byte{0x66, 0x0f, 0xae, 0x78}, mode: 64}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86CacheLineWritebackInstruction(test.code, test.mode) + if !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", test.code, ok, err) + } + }) + } + for _, code := range [][]byte{ + {0x0f, 0xae, 0x30}, // XSAVEOPT, not CLWB. + {0x66, 0x0f, 0xae, 0x28}, // unrelated ModRM extension. + {0x66, 0x0f, 0xaf, 0x38}, // unrelated opcode. + {0x66, 0x0f}, // truncated before the opcode can be identified. + {0xf2, 0x66, 0x0f, 0xae, 0x38}, // unsupported mandatory prefix combination. + {0x66, 0x0f, 0xae, 0x38, 0x0f, 0xae, 0x38}, // only the first instruction is consumed. + } { + _, length, ok, err := decodedX86CacheLineWritebackInstruction(code, 64) + if len(code) == 7 { + if err != nil || !ok || length != 4 { + t.Fatalf("concatenated encoding %x returned length=%d ok=%v err=%v", code, length, ok, err) + } + continue + } + if ok || err != nil { + t.Fatalf("unrelated encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawAliasesThroughLLVM22(t *testing.T) { + const source = `TEXT rawaliases(SB),$0-0 + BYTE $0x66; BYTE $0x0f; BYTE $0x6b; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x76; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x66; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xf5; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xf4; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xf2; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xe2; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xd2; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xfa; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x6a; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x69; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x62; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x61; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x6f; BYTE $0xc1 + BYTE $0xf3; BYTE $0x0f; BYTE $0x6f; BYTE $0xc1 + BYTE $0xf2; BYTE $0x0f; BYTE $0x10; BYTE $0xc1 + BYTE $0x66; BYTE $0x0f; BYTE $0xae; BYTE $0x38 + BYTE $0x66; BYTE $0x0f; BYTE $0xae; BYTE $0x30 + RET +` + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + // These original bytes supply neither X inputs nor the AX cache + // address. Their complete formats remain objects, not public entries. + ir := x86UnboundVoidRawScalarFormatObject(t, source, target.goarch, target.triple) + for _, want := range []string{"clflushopt $0", "clwb $0", "+clflushopt,+clwb"} { + if !strings.Contains(ir, want) { + t.Fatalf("raw alias lowering omitted %q:\n%s", want, ir) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "x86-raw-aliases.ll", "x86-raw-aliases.o", ir) + }) + } +} diff --git a/amd64_raw_align_right_test.go b/amd64_raw_align_right_test.go new file mode 100644 index 000000000..c9491493e --- /dev/null +++ b/amd64_raw_align_right_test.go @@ -0,0 +1,214 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawPALIGNR(immediate byte, mode, source, destination int) []byte { + code := []byte{0x66} + if mode == 64 { + code = append(code, byte(0x40|destination/8<<2|source/8)) + } + modRM := byte(0xc0 | destination&7<<3 | source&7) + return append(code, 0x0f, 0x3a, 0x0f, modRM, immediate) +} + +func encodeX86RawVEXVPALIGNR(immediate byte, wide bool, first, second, destination int) []byte { + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-first/8)<<5 | 3) + p1 := byte((^second&15)<<3 | 1) + if wide { + p1 |= 4 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0xc4, p0, p1, 0x0f, modRM, immediate} +} + +func encodeX86RawEVEXVPALIGNR(immediate, vectorBits byte, mask int, zeroing bool, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 3 + p1 := byte(^second&15)<<3 | 5 + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0x62, p0, p1, p2, 0x0f, modRM, immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedIPFSVPALIGNR(t *testing.T) { + code := []byte{0xc4, 0xe3, 0x41, 0x0f, 0xc6, 0x04} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "IPFS VPALIGNR sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VPALIGNR $4, X6, X7, X0" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawPALIGNRCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, mode := range []int{32, 64} { + registers := 8 + if mode == 64 { + registers = 16 + } + for source := 0; source < registers; source++ { + for destination := 0; destination < registers; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawPALIGNR(immediate, mode, source, destination) + got, length, ok, err := decodedX86PackedAlignRightInstruction(code, mode) + want := fmt.Sprintf("PALIGNR $%d, X%d, X%d", immediate, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode mode=%d %x = %+v, length=%d, ok=%v, err=%v; want %q", mode, code, got, length, ok, err, want) + } + count++ + } + } + } + if count != 320 { + t.Fatalf("covered %d PALIGNR register encodings, want 320", count) + } +} + +func TestDecodedX86RawVPALIGNRCompleteVEXRegisterFamily(t *testing.T) { + count := 0 + for _, wide := range []bool{false, true} { + vectorName := "X" + if wide { + vectorName = "Y" + } + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(first*16 + destination) + code := encodeX86RawVEXVPALIGNR(immediate, wide, first, second, destination) + got, length, ok, err := decodedX86PackedAlignRightInstruction(code, 64) + want := fmt.Sprintf("VPALIGNR $%d, %s%d, %s%d, %s%d", immediate, vectorName, first, vectorName, second, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d VEX VPALIGNR register encodings, want 8192", count) + } +} + +func TestDecodedX86RawVPALIGNRCompleteEVEXRegisterFamily(t *testing.T) { + count := 0 + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + immediate := byte(first*8 + destination) + code := encodeX86RawEVEXVPALIGNR(immediate, byte(vectorBits), masking.mask, masking.zeroing, first, second, destination) + got, length, ok, err := decodedX86PackedAlignRightInstruction(code, 64) + args := []string{ + fmt.Sprintf("$%d", immediate), + fmt.Sprintf("%s%d", vectorName, first), + fmt.Sprintf("%s%d", vectorName, second), + } + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := "VPALIGNR" + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + if count != 294912 { + t.Fatalf("covered %d EVEX VPALIGNR register encodings, want 294912", count) + } +} + +func TestDecodedX86RawPackedAlignRightMemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + { + name: "legacy extended SIB", + code: []byte{0x64, 0x66, 0x47, 0x0f, 0x3a, 0x0f, 0x64, 0x8b, 0x20, 0xff}, + want: "PALIGNR $255, 32(R11)(R9*4)(FS), X12", + }, + { + name: "VEX extended SIB", + code: []byte{0x65, 0xc4, 0x03, 0x15, 0x0f, 0x64, 0x8b, 0x20, 0xfe}, + want: "VPALIGNR $254, 32(R11)(R9*4)(GS), Y13, Y12", + }, + { + name: "EVEX compressed disp8 zero mask", + code: []byte{0x64, 0x62, 0x03, 0x15, 0xc2, 0x0f, 0x64, 0x8b, 0x7f, 0xfd}, + want: "VPALIGNR.Z $253, 8128(R11)(R9*4)(FS), Z29, K2, Z28", + }, + } { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86PackedAlignRightInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) || got.Raw != test.want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", test.code, got, length, ok, err, test.want) + } + }) + } + + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "VEX 386 operand limit", mode: 32, encoding: encodeX86RawVEXVPALIGNR(1, false, 0, 1, 2), wantOK: true}, + {name: "EVEX 386 operand limit", mode: 32, encoding: encodeX86RawEVEXVPALIGNR(1, 0, 0, false, 0, 1, 2), wantOK: true}, + {name: "VEX wrong pp", mode: 64, encoding: []byte{0xc4, 0xe3, 0x40, 0x0f, 0xc6, 4}, wantOK: true}, + {name: "VEX W", mode: 64, encoding: []byte{0xc4, 0xe3, 0xc1, 0x0f, 0xc6, 4}, wantOK: true}, + {name: "VEX truncated immediate", mode: 64, encoding: []byte{0xc4, 0xe3, 0x41, 0x0f, 0xc6}, wantOK: true}, + {name: "EVEX W", mode: 64, encoding: []byte{0x62, 0xf3, 0x85, 0x08, 0x0f, 0xc2, 4}, wantOK: true}, + {name: "EVEX broadcast", mode: 64, encoding: []byte{0x62, 0xf3, 0x05, 0x18, 0x0f, 0x02, 4}, wantOK: true}, + {name: "EVEX reserved vector length", mode: 64, encoding: []byte{0x62, 0xf3, 0x05, 0x68, 0x0f, 0xc2, 4}, wantOK: true}, + {name: "EVEX zero without mask", mode: 64, encoding: []byte{0x62, 0xf3, 0x05, 0x88, 0x0f, 0xc2, 4}, wantOK: true}, + {name: "EVEX truncated immediate", mode: 64, encoding: []byte{0x62, 0xf3, 0x05, 0x08, 0x0f, 0xc2}, wantOK: true}, + {name: "legacy REX.W", mode: 64, encoding: []byte{0x66, 0x48, 0x0f, 0x3a, 0x0f, 0xc2, 4}, wantOK: true}, + {name: "legacy RIP relative", mode: 64, encoding: []byte{0x66, 0x0f, 0x3a, 0x0f, 0x05, 0, 0, 0, 0, 4}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe3, 0x41, 0x0f, 0xc6, 4}, wantOK: true}, + {name: "wrong opcode", mode: 64, encoding: []byte{0xc4, 0xe3, 0x41, 0x0e, 0xc6, 4}}, + {name: "wrong map", mode: 64, encoding: []byte{0xc4, 0xe2, 0x41, 0x0f, 0xc6, 4}}, + {name: "legacy missing 66", mode: 64, encoding: []byte{0x0f, 0x3a, 0x0f, 0xc2, 4}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86PackedAlignRightInstruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x returned ok=%v err=%v, want ok=%v", test.encoding, ok, err, test.wantOK) + } + if test.wantOK && err == nil { + t.Fatalf("invalid encoding %x was accepted", test.encoding) + } + if !test.wantOK && err != nil { + t.Fatalf("unrelated encoding %x returned err=%v", test.encoding, err) + } + }) + } +} diff --git a/amd64_raw_andn.go b/amd64_raw_andn.go new file mode 100644 index 000000000..07875065d --- /dev/null +++ b/amd64_raw_andn.go @@ -0,0 +1,83 @@ +package plan9asm + +import "fmt" + +// decodedX86ANDNInstruction recognizes the complete BMI1 ANDNL/ANDNQ +// Yml,Yrl,Yrl family. x/arch v0.14 reports this VEX.NDS encoding as an +// unknown AVX opcode, so recover every register and safe ModRM/SIB memory form +// before generic decoding. +func decodedX86ANDNInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86ANDNInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86ANDNInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 2 || code[i+3] != 0xf2 { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + vexExt, vexByte := code[i+1], code[i+2] + if vexByte&7 != 0 { + return Instr{}, 0, true, fmt.Errorf("reserved ANDN VEX prefix") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if len(code) <= i+4 { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + rExt := int(^vexExt>>7) & 1 + xExt := int(^vexExt>>6) & 1 + bExt := int(^vexExt>>5) & 1 + width64 := vexByte&0x80 != 0 + secondNumber := int(^vexByte>>3) & 15 + modRM := code[i+4] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (secondNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended source or destination register in 32-bit mode") + } + second, valid := decodedX86GeneralRegister(secondNumber) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid second-source register %d", secondNumber) + } + destination, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid destination register %d", destinationNumber) + } + first, consumed, err := memory.scalar(code[i+4:], mode, bExt, xExt, segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + op := Op("ANDNL") + if width64 && mode == 64 { + op = "ANDNQ" + } + instruction = Instr{ + Op: op, + Args: []Operand{first, {Kind: OpReg, Reg: second}, {Kind: OpReg, Reg: destination}}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, first.String(), second, destination), + } + return instruction, i + 4 + consumed, true, nil +} diff --git a/amd64_raw_average.go b/amd64_raw_average.go new file mode 100644 index 000000000..7cdd70fed --- /dev/null +++ b/amd64_raw_average.go @@ -0,0 +1,25 @@ +package plan9asm + +import "fmt" + +// The complete VEX/EVEX physical decoder is the only producer of the private +// marker which distinguishes raw mode32 masks from named Go's operand limit. +func decodedX86PackedAverageInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedAverageInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedAverageInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || p.pp != 1 || p.opcode != 0xe0 && p.opcode != 0xe3 { + return Instr{}, 0, false, nil + } + if p.broadcast { + return Instr{}, 0, true, fmt.Errorf("packed average has no broadcast or rounding encoding") + } + op, bits := Op("VPAVGB"), 8 + if p.opcode == 0xe3 { + op, bits = "VPAVGW", 16 + } + // WIG is a hardware property; the named Go encoder always emits W0. + return decodedX86BinaryVectorOperandsWithMemory(code, p, mode, op, bits, memory) +} diff --git a/amd64_raw_bf16_convert.go b/amd64_raw_bf16_convert.go new file mode 100644 index 000000000..ec190bbeb --- /dev/null +++ b/amd64_raw_bf16_convert.go @@ -0,0 +1,112 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27 has no named AVX-512 BF16 encodings. Conversion and dot-product +// instructions share widths, masks and dword broadcasts; only conversion +// narrows its destination and reserves vvvv. +func decodedX86RawBF16Instruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || !p.evex || p.mapNumber != 2 || p.pp != 2 || (p.opcode != 0x72 && p.opcode != 0x52) { + return Instr{}, 0, false, nil + } + dot := p.opcode == 0x52 + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("raw BF16: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.w || !dot && p.upper != 0 || p.vectorLength > 2 { + return fail("reserved EVEX width, source or vector-length field") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero K mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + registerSource := code[p.modRM]>>6 == 3 + if p.broadcast && registerSource { + return fail("broadcast requires a memory source") + } + if mode == 32 && (!registerSource && (p.b != 0 || p.x != 0) || p.r != 0 || p.upper >= 8) { + return fail("extended address or destination register in 32-bit mode") + } + + sourceBytes := 16 << p.vectorLength + sourcePrefix := [...]string{"X", "Y", "Z"}[p.vectorLength] + destinationPrefix := [...]string{"X", "X", "Y"}[p.vectorLength] + if dot { + destinationPrefix = sourcePrefix + } + accessBytes := sourceBytes + if p.broadcast { + accessBytes = 4 + } + source, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix, accessBytes) + if err != nil { + return Instr{}, 0, true, err + } + if mode == 32 && registerSource { + index, _ := amd64VectorRegisterIndex(source.Reg, sourceBytes) + if index > 7 { + return fail("extended source register in 32-bit mode") + } + } + destination := int(code[p.modRM]>>3&7) + p.r*8 + op := [...]Op{"VCVTNEPS2BF16X", "VCVTNEPS2BF16Y", "VCVTNEPS2BF16"}[p.vectorLength] + if dot { + op = "VDPBF16PS" + } + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + args := []Operand{source} + if dot { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", sourcePrefix, p.upper))}) + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationPrefix, destination))}) + parts := make([]string, len(args)) + for i, arg := range args { + parts[i] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(parts, ", ")), + x86Encoded: true, + }, p.modRM + consumed, true, nil +} + +func decodeX86RawBF16RIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, ok := decodeX86RawVectorEncoding(code[offset:]) + if !ok || !p.evex || p.mapNumber != 2 || p.pp != 2 || (p.opcode != 0x52 && p.opcode != 0x72) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRM := offset + p.modRM + if len(code) <= modRM || code[modRM]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + } + return x86RawRIPDataThroughDecoder(code, offset, mode, modRM, width, decodedX86RawBF16Instruction, "BF16") +} diff --git a/amd64_raw_bf16_convert_test.go b/amd64_raw_bf16_convert_test.go new file mode 100644 index 000000000..82eab0a00 --- /dev/null +++ b/amd64_raw_bf16_convert_test.go @@ -0,0 +1,256 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawBF16Convert(vectorBits, source, destination, mask int, zeroing, broadcast, memory bool) []byte { + if memory { + source = 0 + } + p0 := byte((1-destination/8&1)<<7 | (1-source/16)<<6 | + (1-source/8&1)<<5 | (1-destination/16)<<4 | 2) + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | destination&7<<3 | source&7) + if memory { + modRM = byte(0x40 | destination&7<<3) + } + code := []byte{0x62, p0, 0x7e, p2, 0x72, modRM} + if memory { + code = append(code, 7) + } + return code +} + +func TestDecodedX86RawBF16ConvertCompleteLLVM22Forms(t *testing.T) { + for vectorBits, names := range []struct { + op Op + source string + dest string + }{ + {"VCVTNEPS2BF16X", "X", "X"}, + {"VCVTNEPS2BF16Y", "Y", "X"}, + {"VCVTNEPS2BF16", "Z", "Y"}, + } { + for _, memory := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + for _, zeroing := range []bool{false, true} { + if zeroing && !masked { + continue + } + for _, broadcast := range []bool{false, true} { + if broadcast && !memory { + continue + } + code := encodeX86RawBF16Convert(vectorBits, 20, 21, mask, zeroing, broadcast, memory) + got, length, ok, err := decodedX86RawBF16Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := names.op + if broadcast { + wantOp += ".BCST" + } + if zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s21", names.dest) { + t.Fatalf("decode %x = %+v, want %s -> %s21", code, got, wantOp, names.dest) + } + if !memory && got.Args[0].String() != fmt.Sprintf("%s20", names.source) { + t.Fatalf("decode %x source = %s, want %s20", code, got.Args[0], names.source) + } + if memory { + accessBytes := 16 << vectorBits + if broadcast { + accessBytes = 4 + } + if want := fmt.Sprintf("%d(AX)", 7*accessBytes); got.Args[0].String() != want { + t.Fatalf("decode %x source = %s, want exact tuple %s", code, got.Args[0], want) + } + } + if masked && (len(got.Args) != 3 || got.Args[1].String() != "K3") { + t.Fatalf("decode %x lost K3: %+v", code, got) + } + } + } + } + } + } +} + +func TestDecodedX86RawBF16ConvertRejectsReservedFields(t *testing.T) { + valid := encodeX86RawBF16Convert(2, 1, 2, 0, false, false, false) + for _, change := range []struct { + index int + bits byte + }{ + {2, 0x80}, // EVEX.W + {2, 0x08}, // reserved vvvv + {3, 0x20}, // reserved LL + {3, 0x80}, // zeroing without a K mask + {3, 0x10}, // broadcast with register source + } { + bad := append([]byte(nil), valid...) + bad[change.index] ^= change.bits + if _, _, ok, err := decodedX86RawBF16Instruction(bad, 64); !ok || err == nil { + t.Fatalf("accepted reserved BF16 conversion %x: ok=%v err=%v", bad, ok, err) + } + } + if _, _, ok, err := decodedX86RawBF16Instruction(valid[:5], 64); ok && err == nil { + t.Fatal("accepted truncated BF16 conversion") + } +} + +func TestTranslateRawBF16ConvertMaskAndBroadcastTargets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawBF16ConvertForms(SB),$0-0\n") + for vectorBits := 0; vectorBits < 3; vectorBits++ { + for _, memory := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && !memory { + continue + } + code := encodeX86RawBF16Convert(vectorBits, 1, 2, 3, true, broadcast, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "rawBF16ConvertForms": {Name: "rawBF16ConvertForms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + body := x86StringMemoryFunction(t, ir, "rawBF16ConvertForms") + // Count actual instructions inside the function. The original + // register/BCST rows never used a full-vector masked load; an + // unconditional unused declaration formerly hid that gap. + for _, want := range []struct { + text string + count int + }{ + {" = call <8 x bfloat> @llvm.x86.vcvtneps2bf16128(", 3}, + {" = call <8 x bfloat> @llvm.x86.vcvtneps2bf16256(", 3}, + {" = call <16 x bfloat> @llvm.x86.avx512bf16.cvtneps2bf16.512(", 3}, + {" = call <4 x i32> @llvm.masked.load.v4i32.p0(", 1}, + {" = call <8 x i32> @llvm.masked.load.v8i32.p0(", 1}, + {" = call <16 x i32> @llvm.masked.load.v16i32.p0(", 1}, + {" = load volatile i32, ", 3}, + {" = phi i32 ", 3}, + } { + if count := strings.Count(body, want.text); count != want.count { + t.Fatalf("BF16 lowering emitted %d uses of %q, want %d", count, want.text, want.count) + } + } + compileLLVMToObject(t, llc, triple, "amd64-raw-bf16-convert-forms.ll", "amd64-raw-bf16-convert-forms.o", ir) + }) + } +} + +func TestTranslateRawBF16Convert386Targets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawBF16Convert386(SB),$0-0\n") + for vectorBits := 0; vectorBits < 3; vectorBits++ { + code := encodeX86RawBF16Convert(vectorBits, 1, 2, 0, false, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "386", + Sigs: map[string]FuncSig{ + "rawBF16Convert386": {Name: "rawBF16Convert386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "386-raw-bf16-convert.ll", "386-raw-bf16-convert.o", ir) + }) + } +} + +func TestTranslateRawBF16ConvertGoHighwayRegression(t *testing.T) { + const source = ` +TEXT rawBF16Convert(SB),$0-0 + LONG $0x487ef262; WORD $0x0472; BYTE $0x97 // VCVTNEPS2BF16 (DI)(DX*4), Y0 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + triple string + goos string + }{ + {"x86_64-apple-darwin", "darwin"}, + {"x86_64-unknown-linux-gnu", "linux"}, + {"x86_64-pc-windows-msvc", "windows"}, + } { + t.Run(target.goos, func(t *testing.T) { + ir, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "rawBF16Convert": {Name: "rawBF16Convert", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "amd64-raw-bf16-convert.ll", "amd64-raw-bf16-convert.o", ir) + }) + } +} diff --git a/amd64_raw_bf16_dot_test.go b/amd64_raw_bf16_dot_test.go new file mode 100644 index 000000000..af4ca0d1a --- /dev/null +++ b/amd64_raw_bf16_dot_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +func rawBF16Dot(length byte, source1, source2, destination, mask int, memory, broadcast, zero bool) []byte { + code := encodeX86EVEXFMA3(0x52, false, length, broadcast, zero, mask, destination, source1, source2) + code[2] = code[2]&^3 | 2 // EVEX.F3.0F38.W0. + if memory { + code[1] |= 0x60 + code[5] = byte(0x40 | (destination&7)<<3) + code = append(code, 1) + } + return code +} + +func TestDecodeRawBF16DotCompleteFormats(t *testing.T) { + for length := byte(0); length < 3; length++ { + for _, memory := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && !memory { + continue + } + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := rawBF16Dot(length, 20, 21, 22, mask, memory, broadcast, zero) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "BF16 dot", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op("VDPBF16PS") + if broadcast { + want += ".BCST" + } + if zero { + want += ".Z" + } + prefix := [...]string{"X", "Y", "Z"}[length] + if len(got) != 1 || got[0].Op != want || !got[0].x86Encoded || + got[0].Args[1].String() != prefix+"20" || got[0].Args[len(got[0].Args)-1].String() != prefix+"22" { + t.Fatalf("%x: got %+v, want raw %s %s20 -> %s22", code, got, want, prefix, prefix) + } + wantSource := prefix + "21" + if memory { + width := 16 << length + if broadcast { + width = 4 // Broadcast one BF16 pair, not one 16-bit input. + } + wantSource = fmt.Sprintf("%d(AX)", width) + } + if got[0].Args[0].String() != wantSource { + t.Fatalf("%x: source %s, want %s", code, got[0].Args[0], wantSource) + } + } + } + } + } + } +} + +func TestDecodeRawBF16DotInvalidFormats(t *testing.T) { + for _, change := range []struct { + index int + bits byte + }{ + {2, 0x80}, {2, 0x04}, {3, 0x80}, {3, 0x60}, {3, 0x10}, + } { + code := rawBF16Dot(0, 1, 2, 3, 0, false, false, false) + code[change.index] ^= change.bits + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid BF16 dot", map[string]bool{}); err == nil { + t.Fatalf("accepted reserved BF16 dot %x", code) + } + } + for _, code := range [][]byte{ + rawBF16Dot(0, 8, 1, 2, 0, false, false, false), + rawBF16Dot(1, 1, 8, 2, 0, false, false, false), + rawBF16Dot(2, 1, 2, 8, 0, false, false, false), + } { + if _, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 BF16 extended register", map[string]bool{}); err == nil { + t.Fatalf("accepted 386 extended register %x", code) + } + } +} + +func TestTranslateRawBF16DotLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT rawBF16Dot(SB),4,$0-0\n") + for length := byte(0); length < 3; length++ { + for _, memory := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && !memory { + continue + } + code := rawBF16Dot(length, 1, 2, 3, 1, memory, broadcast, true) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + } + } + source.WriteString("RET\n") + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawBF16Dot": {Name: "rawBF16Dot", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"@llvm.x86.avx512bf16.dpbf16ps.128", "@llvm.x86.avx512bf16.dpbf16ps.256", + "@llvm.x86.avx512bf16.dpbf16ps.512", "@llvm.masked.load.v4i32", "phi i32", "+avx512bf16"} { + if !strings.Contains(ir, want) { + t.Fatalf("missing %s", want) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-bf16-dot.ll", "raw-bf16-dot.o", ir) + cmd := exec.Command(llc, "-O0", "-mtriple="+target.triple, "-filetype=asm", "-o", "-", "-") + cmd.Stdin = strings.NewReader(ir) + assembly, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("LLVM 22 assembly: %v\n%s", err, assembly) + } + if !strings.Contains(string(assembly), "vdpbf16ps") { + t.Fatalf("BF16 intrinsic did not lower to the native instruction:\n%s", assembly) + } + }) + } +} + +func TestRawBF16DotRejectsNamedGoForms(t *testing.T) { + const source = "TEXT namedBF16Dot(SB),4,$0-0\nVDPBF16PS X0, X1, X2\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedBF16Dot": {Name: "namedBF16Dot", Ret: Void}}, + }); err == nil { + t.Fatal("accepted VDPBF16PS absent from Go's named assembler table") + } +} + +func TestDecodeRawBF16RIPLiteral(t *testing.T) { + for length := byte(0); length < 3; length++ { + for _, broadcast := range []bool{false, true} { + for _, dot := range []bool{false, true} { + code := rawBF16Dot(length, 1, 0, 2, 1, false, broadcast, true) + if !dot { + code = encodeX86RawBF16Convert(int(length), 0, 2, 1, true, broadcast, false) + } + code[5] = 0x15 + code = append(code, 1, 0, 0, 0, 0xc3) + width := 16 << length + if broadcast { + width = 4 + } + for index := 0; index < width; index++ { + code = append(code, byte(index+1)) + } + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "BF16 RIP literal", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Args[0].Kind != OpSym || !got[0].x86RIPLiteral || got[1].Op != OpRET { + t.Fatalf("decoded %x as %+v, want source-local literal and RET", code, got) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated BF16 pool", map[string]bool{}); err == nil { + t.Fatal("accepted truncated source-local BF16 pool") + } + } + } + } +} diff --git a/amd64_raw_binary_float_literal_test.go b/amd64_raw_binary_float_literal_test.go new file mode 100644 index 000000000..e0452b14c --- /dev/null +++ b/amd64_raw_binary_float_literal_test.go @@ -0,0 +1,179 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawBinaryFloatLiteral(opcode, pp byte, evex bool, width int, broadcast bool) []byte { + var code []byte + if evex { + code = encodeX86EVEXBinaryFloat(opcode, pp, byte(map[int]int{16: 0, 32: 1, 64: 2}[width]), broadcast, true, 7, 0, 0, 0) + } else { + code = encodeX86VEXBinaryFloat(opcode, pp, true, width == 32, false, 0, 0, 0) + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if pp >= 2 || broadcast { + dataWidth = 4 + if pp == 1 || pp == 3 { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*11+17)) + } + return code +} + +func TestDecodeX86RawBinaryFloatRIPDataCompleteFamily(t *testing.T) { + for _, opcode := range []byte{0x51, 0x58, 0x59, 0x5c, 0x5d, 0x5e, 0x5f} { + for pp := byte(0); pp < 4; pp++ { + if opcode == 0x51 && pp < 2 { + continue + } + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + if pp >= 2 { + widths = []int{16} + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && pp < 2 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + stem := map[byte]string{0x51: "VSQRT", 0x58: "VADD", 0x59: "VMUL", 0x5c: "VSUB", 0x5d: "VMIN", 0x5e: "VDIV", 0x5f: "VMAX"}[opcode] + name := fmt.Sprintf("%s%s/evex%t/%d/bcst%t", stem, []string{"PS", "PD", "SS", "SD"}[pp], evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawBinaryFloatLiteral(opcode, pp, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op(stem + []string{"PS", "PD", "SS", "SD"}[pp]) + if broadcast { + want += ".BCST" + } + if evex { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } + } +} + +func TestDecodeX86RawBinaryFloatRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawBinaryFloatLiteral(0x58, 0, true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawBinaryFloatRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + index := 0 + for _, opcode := range []byte{0x51, 0x58, 0x59, 0x5c, 0x5d, 0x5e, 0x5f} { + for pp := byte(0); pp < 4; pp++ { + if opcode == 0x51 && pp < 2 { + continue + } + evex := index%2 == 0 + width := 32 + if evex { + width = 64 + } + if pp >= 2 { + width = 16 + } + broadcast := evex && pp < 2 + code := x86RawBinaryFloatLiteral(opcode, pp, evex, width, broadcast) + name := fmt.Sprintf("binaryFloatLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if pp >= 2 || broadcast { + dataWidth = 4 + if pp == 1 || pp == 3 { + dataWidth = 8 + } + } + pools = append(pools, code[len(code)-dataWidth:]) + index++ + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Retain the entire original raw RET and constant pools as Go + // objects and public Context controls. This format-only source + // has no vector or mask producers; initialized public objects + // are independently owned by the complete 124-case directory. + x86UnboundVoidFormatSources(t, source.String(), "amd64", triple) + ir := x86ScalarFormatObjectUnit(t, normalized, sigs, "amd64", triple, 0) + compileLLVMToObject(t, llc, triple, "binary-float-literal.ll", "binary-float-literal.o", ir) + }) + } +} diff --git a/amd64_raw_binary_vector.go b/amd64_raw_binary_vector.go new file mode 100644 index 000000000..c77d6ec7f --- /dev/null +++ b/amd64_raw_binary_vector.go @@ -0,0 +1,84 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Decode the shared two-source vector grammar after the family has selected +// its operation and validated generation/W. The memory source is first in +// Plan 9 order; EVEX masks and broadcast retain their typed semantics. +func decodedX86BinaryVectorOperands(code []byte, p x86RawVectorEncoding, mode int, op Op, laneBits int) (Instr, int, bool, error) { + return decodedX86BinaryVectorOperandsWithMemory(code, p, mode, op, laneBits, x86RawMemoryDecoder{}) +} + +func decodedX86BinaryVectorOperandsWithMemory(code []byte, p x86RawVectorEncoding, mode int, op Op, laneBits int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("%s: %s", op, message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.pp != 1 || p.vectorLength > 2 || p.evex && !p.fixed { + return fail("invalid prefix, fixed bit or vector length") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires K1-K7") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.broadcast && (laneBits < 32 || code[p.modRM]>>6 == 3) { + return fail("broadcast requires D/Q memory source") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + + width := []string{"X", "Y", "Z"}[p.vectorLength] + scale := 16 << p.vectorLength + if p.broadcast { + scale = laneBits / 8 + op += ".BCST" + } + if p.zero { + op += ".Z" + } + + var source Operand + var size int + var err error + if p.evex { + source, size, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, width, scale) + } else { + source, size, err = memory.vex(code[p.modRM:], mode, p.b, p.x, p.segment, width) + } + if err != nil { + return Instr{}, 0, true, err + } + + args := []Operand{ + source, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))}, + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + destination := int(code[p.modRM]>>3&7) + p.r*8 + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destination))}) + + parts := make([]string, len(args)) + for i, arg := range args { + parts[i] = arg.String() + } + instruction := Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(parts, ", ")), + x86Encoded: true, + } + return instruction, p.modRM + size, true, nil +} diff --git a/amd64_raw_bitcount_test.go b/amd64_raw_bitcount_test.go new file mode 100644 index 000000000..617860662 --- /dev/null +++ b/amd64_raw_bitcount_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86RawBitCountCompleteGo127EncodingFamily(t *testing.T) { + type form struct { + name string + op Op + code []byte + mem bool + } + var forms []form + for _, family := range []struct { + name string + opcode byte + }{ + {"LZCNT", 0xbd}, + {"TZCNT", 0xbc}, + } { + for _, width := range []struct { + suffix string + prefix []byte + }{ + {"W", []byte{0x66, 0xf3}}, + {"L", []byte{0xf3}}, + {"Q", []byte{0xf3, 0x48}}, + } { + for _, source := range []struct { + name string + modRM byte + mem bool + }{ + {"register", 0xc1, false}, + {"memory", 0x08, true}, + } { + code := append(append([]byte{}, width.prefix...), 0x0f, family.opcode, source.modRM) + forms = append(forms, form{ + name: family.name + width.suffix + "/" + source.name, + op: Op(family.name + width.suffix), code: code, mem: source.mem, + }) + } + } + } + for _, goarch := range []string{"386", "amd64"} { + for _, form := range forms { + if goarch == "386" && strings.Contains(form.name, "Q/") { + continue + } + t.Run(goarch+"/"+form.name, func(t *testing.T) { + fn := Func{} + for _, value := range form.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != form.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want one %s", form.code, got.Instrs, form.op) + } + ins := got.Instrs[0] + wantDestination := AX + if form.mem { + wantDestination = CX + } + if ins.Args[1].Kind != OpReg || ins.Args[1].Reg != wantDestination { + t.Fatalf("decoded destination %#v, want %s", ins.Args[1], wantDestination) + } + if form.mem { + if ins.Args[0].Kind != OpMem || ins.Args[0].Mem.Base != AX { + t.Fatalf("decoded memory source %#v, want (AX)", ins.Args[0]) + } + } else if ins.Args[0].Kind != OpReg || ins.Args[0].Reg != CX { + t.Fatalf("decoded register source %#v, want CX", ins.Args[0]) + } + }) + } + } +} + +func TestX86RawBitCountExtendedRegisters(t *testing.T) { + for _, family := range []struct { + opcode byte + op Op + }{ + {0xbc, "TZCNTQ"}, + {0xbd, "LZCNTQ"}, + } { + for _, test := range []struct { + name string + code []byte + source Operand + dest Reg + }{ + { + name: "extended register", + code: []byte{0xf3, 0x4d, 0x0f, family.opcode, 0xc1}, + source: Operand{Kind: OpReg, Reg: "R9"}, dest: "R8", + }, + { + name: "extended memory base", + code: []byte{0xf3, 0x49, 0x0f, family.opcode, 0x08}, + source: Operand{Kind: OpMem, Mem: MemRef{Base: "R8"}}, dest: CX, + }, + } { + t.Run(string(family.op)+"/"+test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != family.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", test.code, got.Instrs, family.op) + } + ins := got.Instrs[0] + if ins.Args[0].Kind != test.source.Kind || ins.Args[0].Reg != test.source.Reg || ins.Args[0].Mem != test.source.Mem || ins.Args[1].Reg != test.dest { + t.Fatalf("decoded %#x operands as %#v, want %#v, %s", test.code, ins.Args, test.source, test.dest) + } + }) + } + } +} + +func TestX86RawBitCountCompilesAllForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawbitcount(SB),NOSPLIT,$0-0\n") + for _, opcode := range []byte{0xbc, 0xbd} { + for _, prefix := range [][]byte{{0x66, 0xf3}, {0xf3}, {0xf3, 0x48}} { + if target.goarch == "386" && len(prefix) == 2 && prefix[1] == 0x48 { + continue + } + for _, modRM := range []byte{0xc1, 0x08} { + for _, value := range append(append([]byte{}, prefix...), 0x0f, opcode, modRM) { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + } + } + source.WriteString("\tRET\n") + // The original complete sequence has no CX producer, and its + // partial AX result cannot establish the later memory address. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + + word, move, typ := 8, "MOVQ", "uint64" + if target.goarch == "386" { + word, move, typ = 4, "MOVL", "uint32" + } + var initialized, declarations strings.Builder + forms := 0 + for _, opcode := range []byte{0xbc, 0xbd} { + for _, prefix := range [][]byte{{0x66, 0xf3}, {0xf3}, {0xf3, 0x48}} { + if target.goarch == "386" && len(prefix) == 2 && prefix[1] == 0x48 { + continue + } + for _, modRM := range []byte{0xc1, 0x08} { + name := fmt.Sprintf("RawCount%d", forms) + inputType, input, result := typ, "CX", "AX" + if modRM == 0x08 { + inputType, input, result = "*"+typ, "AX", "CX" + } + fmt.Fprintf(&declarations, "func %s(x %s) %s\n", name, inputType, typ) + fmt.Fprintf(&initialized, "TEXT ·%s(SB),4,$0-%d\n%s $0,%s\n%s x+0(FP),%s\n", + name, 2*word, move, result, move, input) + initialized.WriteString(x86RawDisplacementDirectives(append(append([]byte{}, prefix...), 0x0f, opcode, modRM))) + fmt.Fprintf(&initialized, "%s %s,ret+%d(FP)\nRET\n", move, result, word) + forms++ + } + } + } + want := 12 + if target.goarch == "386" { + want = 8 + } + if forms != want { + t.Fatal("initialized raw count form inventory changed", forms, want) + } + x86LEAActualTargetGoObject(t, target.goarch, target.triple, initialized.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, initialized.String(), declarations.String(), target.goarch, target.triple, api) + if err != nil { + t.Fatal("independently initialized original raw count form", api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + } + }) + } +} + +func TestX86RawBitCountRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution requires amd64; required amd64 CI covers this host-inapplicable test") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86RawCountOriginalRuntimeSource + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{ + "rawbitcountsemantics": { + Name: "rawbitcountsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void rawbitcountsemantics(uint8_t *); +int main(void) { + uint8_t out[72] = {0}; + rawbitcountsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0x12340010)) return 10; + if (out[8] != 1 || out[9] != 0) return 11; + if (*(uint64_t *)(out+16) != 0) return 12; + if (out[24] != 0 || out[25] != 1) return 13; + if (*(uint64_t *)(out+32) != 64) return 14; + if (out[40] != 1 || out[41] != 0) return 15; + if (*(uint64_t *)(out+56) != 55) return 16; + if (out[64] != 0 || out[65] != 0) return 17; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_bitcount_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_raw_bitcount_vector.go b/amd64_raw_bitcount_vector.go new file mode 100644 index 000000000..0f9214b45 --- /dev/null +++ b/amd64_raw_bitcount_vector.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawVectorBitCountForm struct { + op Op + opcode int + w bool +} + +// The six Go 1.27 _yvexpandpd bit-count rows share the same X/Y/Z and +// mask grammar. Opcode and W choose the lane operation; only D/Q rows +// enable scalar-memory broadcast. +var x86RawVectorBitCountForms = [...]x86RawVectorBitCountForm{ + {op: "VPLZCNTD", opcode: 0x44}, + {op: "VPLZCNTQ", opcode: 0x44, w: true}, + {op: "VPOPCNTB", opcode: 0x54}, + {op: "VPOPCNTW", opcode: 0x54, w: true}, + {op: "VPOPCNTD", opcode: 0x55}, + {op: "VPOPCNTQ", opcode: 0x55, w: true}, +} + +func decodedX86RawVectorBitCountInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || !p.evex || p.mapNumber != 2 || p.pp != 1 { + return Instr{}, 0, false, nil + } + var form x86RawVectorBitCountForm + for _, candidate := range x86RawVectorBitCountForms { + if candidate.opcode == p.opcode && candidate.w == p.w { + form = candidate + break + } + } + if form.op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("vector bit count: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.upper != 0 || p.vectorLength > 2 { + return fail("invalid reserved EVEX encoding bit") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + spec := amd64PackedBitCountSpecs[form.op] + modRM := code[p.modRM] + registerSource := modRM>>6 == 3 + if p.broadcast && (registerSource || !spec.broadcast) { + return fail("broadcast requires a supported scalar-memory source") + } + if mode == 32 && !registerSource && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + + vectorPrefix := [...]string{"X", "Y", "Z"}[p.vectorLength] + vectorBytes := 16 << p.vectorLength + registerNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && p.vectorLength == 2 && registerNumber >= 8 { + return fail("386 Z destination register exceeds Go's encoder class") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, registerNumber))} + accessBytes := vectorBytes + if p.broadcast { + accessBytes = spec.laneBits / 8 + } + source, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, vectorPrefix, accessBytes) + if err != nil { + return Instr{}, 0, true, err + } + op := form.op + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_bitcount_vector_test.go b/amd64_raw_bitcount_vector_test.go new file mode 100644 index 000000000..d5fc3f82f --- /dev/null +++ b/amd64_raw_bitcount_vector_test.go @@ -0,0 +1,205 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVectorBitCount(form x86RawVectorBitCountForm, vectorBits, destination, source, mask int, zeroing, broadcast, memory bool) []byte { + if memory { + source = 0 + } + p0 := byte((1-destination/8&1)<<7 | (1-source/16)<<6 | + (1-source/8&1)<<5 | (1-destination/16)<<4 | 2) + p1 := byte(0x7d) + if form.w { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + if memory { + modRM = byte(0x40 | (destination&7)<<3) + } + code := []byte{0x62, p0, p1, p2, byte(form.opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestTranslateRawBitCountGorseRegression(t *testing.T) { + const source = `TEXT rawVectorPopcount(SB), $0-0 + LONG $0x48fdf262 + WORD $0xdb55 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawVectorPopcount": {Name: "rawVectorPopcount", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "llvm.ctpop.v8i64") { + t.Fatalf("raw VPOPCNTQ omitted eight-lane population count:\n%s", ir) + } +} + +func TestDecodedX86RawVectorBitCountCompleteGoRows(t *testing.T) { + if len(x86RawVectorBitCountForms) != len(amd64PackedBitCountSpecs) { + t.Fatalf("raw bit-count grammar has %d rows, named lowerer has %d", len(x86RawVectorBitCountForms), len(amd64PackedBitCountSpecs)) + } + seen := make(map[Op]bool) + for _, form := range x86RawVectorBitCountForms { + spec, ok := amd64PackedBitCountSpecs[form.op] + if !ok || seen[form.op] { + t.Fatalf("missing or duplicated bit-count spec for %s", form.op) + } + seen[form.op] = true + for vectorBits, prefix := range [...]string{"X", "Y", "Z"} { + vectorBytes := 16 << vectorBits + for _, memory := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + for _, zeroing := range []bool{false, true} { + if zeroing && !masked { + continue + } + for _, broadcast := range []bool{false, true} { + if broadcast && (!memory || !spec.broadcast) { + continue + } + code := encodeX86RawVectorBitCount(form, vectorBits, 21, 20, mask, zeroing, broadcast, memory) + got, length, recognized, err := decodedX86RawVectorBitCountInstruction(code, 64) + if err != nil || !recognized || length != len(code) { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + wantOp := form.op + if broadcast { + wantOp += ".BCST" + } + if zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s21", prefix) { + t.Fatalf("decode %x = %+v, want %s and %s21", code, got, wantOp, prefix) + } + if masked && (len(got.Args) != 3 || got.Args[1].String() != "K3") || !masked && len(got.Args) != 2 { + t.Fatalf("decode %x = %+v, lost mask", code, got) + } + source := got.Args[0] + if memory { + wantBytes := vectorBytes + if broadcast { + wantBytes = spec.laneBits / 8 + } + if source.Kind != OpMem || source.Mem.Base != AX || source.Mem.Off != int64(wantBytes) { + t.Fatalf("decode %x = %+v, want compressed displacement %d", code, got, wantBytes) + } + } else if source.String() != fmt.Sprintf("%s20", prefix) { + t.Fatalf("decode %x = %+v, want %s20", code, got, prefix) + } + } + } + } + } + } + } +} + +func TestDecodedX86RawVectorBitCountRejectsReservedBits(t *testing.T) { + form := x86RawVectorBitCountForms[5] + base := encodeX86RawVectorBitCount(form, 2, 2, 1, 3, true, false, false) + for _, test := range []struct { + name string + code []byte + }{ + {"fixed bit", []byte{base[0], base[1], base[2] &^ 4, base[3], base[4], base[5]}}, + {"vvvv", []byte{base[0], base[1], base[2] &^ 8, base[3], base[4], base[5]}}, + {"V prime", []byte{base[0], base[1], base[2], base[3] &^ 8, base[4], base[5]}}, + {"reserved length", []byte{base[0], base[1], base[2], base[3] | 0x20, base[4], base[5]}}, + {"zero without mask", []byte{base[0], base[1], base[2], base[3] &^ 7, base[4], base[5]}}, + {"register broadcast", []byte{base[0], base[1], base[2], base[3] | 0x10, base[4], base[5]}}, + {"missing ModRM", base[:5]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawVectorBitCountInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted reserved encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + broadcastWord := encodeX86RawVectorBitCount(x86RawVectorBitCountForms[3], 2, 2, 0, 0, false, true, true) + if _, _, recognized, err := decodedX86RawVectorBitCountInstruction(broadcastWord, 64); !recognized || err == nil { + t.Fatalf("accepted VPOPCNTW broadcast %x", broadcastWord) + } +} + +func TestTranslateRawVectorBitCountLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawBitCountFamily(SB), $0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for _, form := range x86RawVectorBitCountForms { + for vectorBits := 0; vectorBits < 3; vectorBits++ { + appendBytes(encodeX86RawVectorBitCount(form, vectorBits, 2, 1, 0, false, false, false)) + appendBytes(encodeX86RawVectorBitCount(form, vectorBits, 2, 0, 3, true, false, true)) + if amd64PackedBitCountSpecs[form.op].broadcast { + appendBytes(encodeX86RawVectorBitCount(form, vectorBits, 2, 0, 4, false, true, true)) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawBitCountFamily": {Name: "rawBitCountFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "llvm.ctpop") || !strings.Contains(ir, "llvm.ctlz") { + t.Fatalf("raw vector bit-count family omitted operations:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-vector-bit-count.ll", "raw-vector-bit-count.o", ir) + }) + } +} diff --git a/amd64_raw_blend_imm_literal_test.go b/amd64_raw_blend_imm_literal_test.go new file mode 100644 index 000000000..fde9821ce --- /dev/null +++ b/amd64_raw_blend_imm_literal_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawImmediatePackedBlendLiteral(spec rawImmediatePackedBlendSpec, vex bool, width int) []byte { + var code []byte + if vex { + code = encodeX86RawVEXImmediatePackedBlend(spec, byte((width-16)/16), 0, 1, 2, 0xa5) + } else { + code = encodeX86RawLegacyImmediatePackedBlend(spec, 0, 2, 0xa5) + } + modRMIndex := len(code) - 2 + code[modRMIndex] = 0x15 // RIP+disp32; destination remains X/Y2. + code = append(code[:modRMIndex+1], append([]byte{1, 0, 0, 0}, code[modRMIndex+1:]...)...) + code = append(code, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*7+3)) + } + return code +} + +func TestDecodeX86RawImmediatePackedBlendRIPDataCompleteFamily(t *testing.T) { + for _, spec := range rawImmediatePackedBlendSpecs { + for _, vex := range []bool{false, true} { + if !vex && spec.legacy == "" { + continue + } + widths := []int{16} + if vex { + widths = append(widths, 32) + } + for _, width := range widths { + name := fmt.Sprintf("%s/vex%t/%d", spec.vector, vex, width) + t.Run(name, func(t *testing.T) { + code := x86RawImmediatePackedBlendLiteral(spec, vex, width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := spec.legacy + if vex { + want = spec.vector + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpImm || decoded[0].Args[0].Imm != 0xa5 || + decoded[0].Args[1].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + if !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-width:]) { + t.Fatalf("source-local data = %x, want %x", decoded[0].x86RIPLiteralData, code[len(code)-width:]) + } + }) + } + } + } +} + +func TestDecodeX86RawImmediatePackedBlendRIPDataRejectsUnsafeSources(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { + code[5] = 127 + return code + }, + "overlapping instruction": func(code []byte) []byte { + code[5] = 0 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawImmediatePackedBlendLiteral(rawImmediatePackedBlendSpecs[2], true, 32)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe packed-blend RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawImmediatePackedBlendRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for _, spec := range rawImmediatePackedBlendSpecs { + for _, vex := range []bool{false, true} { + if !vex && spec.legacy == "" { + continue + } + name := string(spec.legacy) + width := 16 + if vex { + name = string(spec.vector) + width = 32 + } + fmt.Fprintf(&source, "TEXT literal%s(SB),$0-0\n", name) + for _, value := range x86RawImmediatePackedBlendLiteral(spec, vex, width) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs["literal"+name] = FuncSig{Name: "literal" + name, Ret: Void} + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(sigs) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(sigs)) + } + for _, fn := range normalized.Funcs { + found := false + for _, instruction := range fn.Instrs { + if !instruction.x86RIPLiteral { + continue + } + if len(instruction.Args) < 2 || !strings.Contains(instruction.Raw, instruction.Args[1].Sym) { + t.Fatalf("%s raw text does not name its materialized source: %+v", fn.Sym, instruction) + } + found = true + } + if !found { + t.Fatalf("%s has no decoded packed-blend literal", fn.Sym) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Keep the complete original raw RET and literal bytes as actual + // Go objects plus three public Context outcomes. These seven bare + // functions have no old-destination or second-source producers. + // Initialized full register/native-memory programs have the separate + // 22-form five-target object and whole-producer replay owners. + x86UnboundVoidFormatSources(t, source.String(), "amd64", triple) + ir := x86InstructionFormatObjectUnit(t, normalized, sigs, "amd64", triple, 0, + func(c *amd64Ctx, instruction Instr) (bool, bool, error) { + return c.lowerImmediatePackedBlend(instruction.Op, instruction) + }) + for _, input := range []string{ir, x86DwordProductIROptimize(t, ir)} { + compileLLVMToObject(t, llc, triple, "raw-blend-imm-literal.ll", "raw-blend-imm-literal.o", input) + } + }) + } +} diff --git a/amd64_raw_blend_imm_test.go b/amd64_raw_blend_imm_test.go new file mode 100644 index 000000000..431e0359e --- /dev/null +++ b/amd64_raw_blend_imm_test.go @@ -0,0 +1,201 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +type rawImmediatePackedBlendSpec struct { + opcode byte + legacy Op + vector Op +} + +var rawImmediatePackedBlendSpecs = []rawImmediatePackedBlendSpec{ + {opcode: 0x0e, legacy: "PBLENDW", vector: "VPBLENDW"}, + {opcode: 0x0c, legacy: "BLENDPS", vector: "VBLENDPS"}, + {opcode: 0x0d, legacy: "BLENDPD", vector: "VBLENDPD"}, + {opcode: 0x02, vector: "VPBLENDD"}, +} + +func encodeX86RawLegacyImmediatePackedBlend(spec rawImmediatePackedBlendSpec, source, destination int, immediate byte) []byte { + rex := byte(0x40 | source>>3&1 | destination>>3&1<<2) + code := []byte{0x66} + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0x3a, spec.opcode, byte(0xc0|destination&7<<3|source&7), immediate) +} + +func encodeX86RawVEXImmediatePackedBlend(spec rawImmediatePackedBlendSpec, vectorBits byte, first, second, destination int, immediate byte) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^first>>3)&1)<<5 | 3 + p1 := byte(^second&15)<<3 | vectorBits<<2 | 1 + return []byte{0xc4, p0, p1, spec.opcode, byte(0xc0 | destination&7<<3 | first&7), immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVBLENDPD(t *testing.T) { + // vblendpd $12, %ymm1, %ymm0, %ymm0; + // vunpcklps %xmm3, %xmm5, %xmm0 + code := []byte{0xc4, 0xe3, 0x7d, 0x0d, 0xc1, 0x0c, 0xc5, 0xd0, 0x14, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VBLENDPD sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VBLENDPD $12, Y1, Y0, Y0 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawImmediatePackedBlendCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawImmediatePackedBlendSpecs { + if spec.legacy != "" { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawLegacyImmediatePackedBlend(spec, source, destination, 0xa5) + got, length, ok, err := decodedX86ImmediatePackedBlendInstruction(code, 64) + want := fmt.Sprintf("%s $165, X%d, X%d", spec.legacy, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + for vectorBits := byte(0); vectorBits < 2; vectorBits++ { + vectorName := [...]string{"X", "Y"}[vectorBits] + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXImmediatePackedBlend(spec, vectorBits, first, second, destination, 0x5a) + got, length, ok, err := decodedX86ImmediatePackedBlendInstruction(code, 64) + want := fmt.Sprintf("%s $90, %s%d, %s%d, %s%d", spec.vector, vectorName, first, vectorName, second, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + if count != 33536 { + t.Fatalf("covered %d immediate packed-blend register encodings, want 33536", count) + } +} + +func TestDecodedX86RawImmediatePackedBlendMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0x66, 0x47, 0x0f, 0x3a, 0x0e, 0x7c, 0x8b, 0x01, 0x7f} + got, length, ok, err := decodedX86ImmediatePackedBlendInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "PBLENDW" || got.Args[0].Imm != 127 || got.Args[2].String() != "X15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x03, 0x75, 0x0d, 0x7c, 0x8b, 0x02, 0xfe} + got, length, ok, err = decodedX86ImmediatePackedBlendInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VBLENDPD" || got.Args[0].Imm != 254 || got.Args[2].String() != "Y1" || got.Args[3].String() != "Y15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + + validLegacy := encodeX86RawLegacyImmediatePackedBlend(rawImmediatePackedBlendSpecs[0], 7, 7, 1) + validVEX := encodeX86RawVEXImmediatePackedBlend(rawImmediatePackedBlendSpecs[0], 0, 7, 7, 7, 1) + reservedW := encodeX86RawVEXImmediatePackedBlend(rawImmediatePackedBlendSpecs[3], 0, 7, 7, 7, 1) + reservedW[2] |= 0x80 // VPBLENDD alone is W0; the other three VEX members are WIG. + for name, invalid := range map[string][]byte{ + "legacy missing immediate": validLegacy[:len(validLegacy)-1], + "vex missing immediate": validVEX[:len(validVEX)-1], + "reserved VPBLENDD W": reservedW, + "vex prefix": {validVEX[0], validVEX[1], validVEX[2] &^ 0x01, validVEX[3], validVEX[4], validVEX[5]}, + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86ImmediatePackedBlendInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + for _, mode := range []int{32, 64} { + for _, w := range []byte{0, 0x80} { + physical := append([]byte(nil), validVEX...) + physical[2] |= w + x86NumericalNativeRequireMC(t, physical, mode, 1, "+avx2") + instruction, length, matched, decodeErr := decodedX86ImmediatePackedBlendInstruction(physical, mode) + if !matched || decodeErr != nil || length != len(physical) || instruction.Op != "VPBLENDW" || !instruction.x86Encoded { + t.Fatalf("physical mode%d WIG VEX encoding %x decoded as %+v, ok=%v, err=%v", mode, physical, instruction, matched, decodeErr) + } + } + } +} + +func TestX86ImmediatePackedBlendGo127ArchitectureOracle(t *testing.T) { + const accepted386 = `TEXT ok(SB),$0-0 + PBLENDW $7, X6, X7 + BLENDPS $7, X6, X7 + BLENDPD $7, X6, X7 + RET +` + requireX86GoAssemblerResult(t, "386", accepted386, true) + for _, instruction := range []string{ + "VPBLENDW $7, X0, X1, X2", + "VPBLENDD $7, Y0, Y1, Y2", + "VBLENDPS $7, X0, X1, X2", + "VBLENDPD $7, Y0, Y1, Y2", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } +} + +func TestTranslateX86RawImmediatePackedBlendTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + registerMax int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", registerMax: 15}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registerMax: 15}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registerMax: 15}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", registerMax: 7}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", registerMax: 7}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_immediate_packed_blend(SB),NOSPLIT,$0-0\n") + for _, spec := range rawImmediatePackedBlendSpecs { + var encodings [][]byte + if spec.legacy != "" { + encodings = append(encodings, encodeX86RawLegacyImmediatePackedBlend(spec, target.registerMax, target.registerMax, 0xa5)) + } + if target.goarch == "amd64" { + encodings = append(encodings, encodeX86RawVEXImmediatePackedBlend(spec, 1, target.registerMax, target.registerMax, target.registerMax, 0x5a)) + } + for _, encoding := range encodings { + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "raw_immediate_packed_blend": {Name: "raw_immediate_packed_blend", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-immediate-packed-blend-"+target.name+".ll", "raw-immediate-packed-blend-"+target.name+".o", ll) + }) + } +} diff --git a/amd64_raw_blend_variable.go b/amd64_raw_blend_variable.go new file mode 100644 index 000000000..54f1d976f --- /dev/null +++ b/amd64_raw_blend_variable.go @@ -0,0 +1,74 @@ +package plan9asm + +import "fmt" + +// VEX variable blends encode the mask register in imm8[7:4], not an ordinary +// immediate. Intel's AVX/AVX2 tables specify 66.0F3A.W0, X/Y widths and no +// EVEX forms. The low nibble is ignored; in 32-bit mode bit 7 is also ignored. +func decodedX86VariableBlendInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 3 { + return Instr{}, 0, false, nil + } + var op Op + for candidate, spec := range amd64VariableBlendSpecs { + if spec.vector && spec.opcode == p.opcode { + op = candidate + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("variable blend: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.w || p.pp != 1 { + return fail("requires VEX.128/256.66.0F3A.W0") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 { + if p.r != 0 || p.x != 0 { + return fail("legacy LES encoding is not a 32-bit VEX instruction") + } + // Three-byte VEX ignores these high register selectors outside + // 64-bit mode, independently of the imm8 mask-register selector. + p.b = 0 + p.upper &= 7 + } + prefix := "X" + if p.vectorLength != 0 { + prefix = "Y" + } + source, size, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, prefix) + if err != nil { + return Instr{}, 0, true, err + } + immediate := p.modRM + size + if len(code) <= immediate { + return fail("missing mask-register selector byte") + } + mask := int(code[immediate] >> 4) + if mode == 32 { + mask &= 7 + } + reg := func(index int) Operand { + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, index))} + } + args := []Operand{reg(mask), source, reg(p.upper), reg(int(code[p.modRM]>>3&7) + p.r*8)} + return Instr{ + // This exact physical four-operand instruction is valid in mode32, + // unlike Go's named frontend spelling. No other raw family acquires + // that syntax role; the complete source replay seals this flag too. + Op: op, Args: args, x86Encoded: true, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, args[0].String(), source.String(), args[2].String(), args[3].String()), + }, immediate + 1, true, nil +} diff --git a/amd64_raw_blend_variable_test.go b/amd64_raw_blend_variable_test.go new file mode 100644 index 000000000..cbbd3df37 --- /dev/null +++ b/amd64_raw_blend_variable_test.go @@ -0,0 +1,337 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86VariableBlendTestFamilies = []struct { + legacy, vector string + bits int + legacyCode byte + vectorCode byte +}{ + {"BLENDVPS", "VBLENDVPS", 32, 0x14, 0x4a}, + {"BLENDVPD", "VBLENDVPD", 64, 0x15, 0x4b}, + {"PBLENDVB", "VPBLENDVB", 8, 0x10, 0x4c}, +} + +func TestX86VariableBlendGrammarMatchesGoEncoder(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + vector, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, grammar := range []struct { + text, table string + operands []string + }{ + {string(legacy), "yblendvpd", []string{"Yxr0, Yxm, Yxr"}}, + {string(vector), "_yvblendvpd", []string{"Yxr, Yxm, Yxr, Yxr", "Yyr, Yym, Yyr, Yyr"}}, + } { + pattern := `(?s)var ` + grammar.table + ` = \[\]ytab\{(.*?)\n\}` + rows := regexp.MustCompile(pattern).FindStringSubmatch(grammar.text) + if len(rows) != 2 || strings.Count(rows[1], "argList{") != len(grammar.operands) { + t.Fatalf("Go %s operand grammar changed", grammar.table) + } + for _, operands := range grammar.operands { + if !strings.Contains(rows[1], "argList{"+operands+"}") { + t.Fatalf("Go %s is missing %s", grammar.table, operands) + } + } + } + for _, family := range x86VariableBlendTestFamilies { + for _, vex := range []bool{false, true} { + op, opcode := family.legacy, family.legacyCode + if vex { + op, opcode = family.vector, family.vectorCode + } + spec, ok := amd64VariableBlendSpecs[Op(op)] + if !ok || spec.laneBits != family.bits || spec.vector != vex || spec.opcode != int(opcode) { + t.Fatalf("incorrect typed spec for %s: %+v", op, spec) + } + if !vex { + row := fmt.Sprintf("{A%s, yblendvpd, Pq4, opBytes{0x%02x}}", op, opcode) + if !strings.Contains(string(legacy), row) { + t.Fatalf("Go %s opcode changed", op) + } + continue + } + pattern := `(?s)\{as: A` + op + `, ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}` + rows := regexp.MustCompile(pattern).FindAllStringSubmatch(string(vector), -1) + if len(rows) != 1 || rows[0][1] != "_yvblendvpd" { + t.Fatalf("Go %s opcode grammar changed", op) + } + entries := strings.Split(strings.TrimSpace(rows[0][2]), "\n") + if len(entries) != 2 { + t.Fatalf("Go %s must have exactly X/Y forms", op) + } + for i, width := range []int{128, 256} { + want := fmt.Sprintf("avxEscape | vex%d | vex66 | vex0F3A | vexW0, 0x%02X,", width, opcode) + if strings.TrimSpace(entries[i]) != want { + t.Fatalf("Go %s encoding = %s, want %s", op, entries[i], want) + } + } + } + } + if len(amd64VariableBlendSpecs) != 6 { + t.Fatal("variable-blend specs must cover all six Go mnemonics") + } +} + +func TestX86RawVariableBlendEncodingFields(t *testing.T) { + for _, family := range x86VariableBlendTestFamilies { + for _, length := range []byte{0, 1} { + for selector := 0; selector < 256; selector++ { + code := []byte{0xc4, 0xe3, 0x69 | length<<2, family.vectorCode, 0xd9, byte(selector)} + for _, mode := range []int{32, 64} { + got, size, matched, err := decodedX86VariableBlendInstruction(code, mode) + mask := selector >> 4 + if mode == 32 { + mask &= 7 + } + prefix := "X" + if length != 0 { + prefix = "Y" + } + want := []Operand{ + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, mask))}, + {Kind: OpReg, Reg: Reg(prefix + "1")}, + {Kind: OpReg, Reg: Reg(prefix + "2")}, + {Kind: OpReg, Reg: Reg(prefix + "3")}, + } + if err != nil || !matched || size != len(code) || got.Op != Op(family.vector) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x mode=%d: %+v %v, want %+v", code, mode, got, err, want) + } + } + } + } + } +} + +func TestX86RawVariableBlend386IgnoredRegisterBits(t *testing.T) { + // Intel XED's XMM/YMM_N_32 and XMM/YMM_B_32 tables ignore the high + // vvvv and B selectors. R/X must still distinguish VEX from legacy LES. + for _, family := range x86VariableBlendTestFamilies { + for _, length := range []byte{0, 1} { + for _, modRM := range []byte{0xd9, 0x1b} { + canonical := []byte{0xc4, 0xe3, 0x69 | length<<2, family.vectorCode, modRM, 0x70} + want, _, matched, err := decodedX86VariableBlendInstruction(canonical, 32) + if !matched || err != nil { + t.Fatalf("canonical %x: %v", canonical, err) + } + for bits := 0; bits < 4; bits++ { + code := append([]byte(nil), canonical...) + if bits&1 != 0 { + code[1] &^= 0x20 + } + if bits&2 != 0 { + code[2] &^= 0x40 + } + code[5] = 0xff + got, size, matched, err := decodedX86VariableBlendInstruction(code, 32) + if !matched || err != nil || size != len(code) || !reflect.DeepEqual(got.Args, want.Args) { + t.Fatalf("386 ignored fields %x: %+v %v, want %+v", code, got, err, want) + } + } + } + } + } +} + +func TestX86RawVariableBlendInvalidAndMemoryForms(t *testing.T) { + for _, family := range x86VariableBlendTestFamilies { + code := []byte{0xc4, 0xe3, 0x69, family.vectorCode, 0xd9, 0x00} + for _, mutate := range []func([]byte) []byte{ + func(b []byte) []byte { b[2] |= 0x80; return b }, // W1 is reserved. + func(b []byte) []byte { b[2] &^= 3; return b }, // No 66 prefix. + func(b []byte) []byte { return b[:4] }, // Missing ModRM. + func(b []byte) []byte { return b[:5] }, // Missing selector. + func(b []byte) []byte { b[4] = 0x04; return b[:5] }, + func(b []byte) []byte { b[4] = 0x80; return b[:5] }, + func(b []byte) []byte { return append([]byte{0x67}, b...) }, + func(b []byte) []byte { return []byte{0x62, 0xf3, 0x6d, 0x08, b[3], b[4], b[5]} }, + } { + invalid := mutate(append([]byte(nil), code...)) + if _, _, matched, err := decodedX86VariableBlendInstruction(invalid, 64); !matched || err == nil { + t.Fatalf("accepted reserved/truncated encoding %x", invalid) + } + } + if _, _, matched, err := decodedX86VariableBlendInstruction(code, 16); !matched || err == nil { + t.Fatal("accepted unsupported mode") + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + memory := []byte{segment.prefix, 0xc4, 0x03, 0x69, family.vectorCode, 0x64, 0x88, 0xf9, 0xf7} + got, size, matched, err := decodedX86VariableBlendInstruction(memory, 64) + want := []Operand{ + {Kind: OpReg, Reg: "X15"}, + {Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: -7}}, + {Kind: OpReg, Reg: "X2"}, + {Kind: OpReg, Reg: "X12"}, + } + if err != nil || !matched || size != len(memory) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x: %+v %v, want %+v", memory, got, err, want) + } + if _, _, matched, err := decodedX86VariableBlendInstruction(memory, 32); !matched || err == nil { + t.Fatal("accepted legacy LES as a 32-bit VEX instruction") + } + } + rip := []byte{0xc4, 0xe3, 0x69, family.vectorCode, 0x05, 0, 0, 0, 0, 0} + if _, _, matched, err := decodedX86VariableBlendInstruction(rip, 64); !matched || err == nil { + t.Fatal("accepted source-layout-unsafe RIP addressing") + } + if _, size, matched, err := decodedX86VariableBlendInstruction(rip, 32); !matched || err != nil || size != len(rip) { + t.Fatalf("absolute 32-bit addressing: matched=%t size=%d err=%v", matched, size, err) + } + } + for _, code := range [][]byte{nil, {0xc4}, {0xc4, 0xe2, 0x69, 0x4b, 0xc0, 0}, {0xc4, 0xe3, 0x69, 0x4d, 0xc0, 0}} { + if _, _, matched, _ := decodedX86VariableBlendInstruction(code, 64); matched { + t.Fatalf("matched unrelated instruction %x", code) + } + } +} + +func TestX86RawVariableBlendSourceLocalRIPData(t *testing.T) { + for _, family := range x86VariableBlendTestFamilies { + for vectorLength, width := range []int{16, 32} { + code := []byte{ + 0xc4, 0xe3, byte(0x69 | vectorLength<<2), family.vectorCode, + 0x05, 0x01, 0, 0, 0, 0x70, 0xc3, + } + for value := 0; value < width; value++ { + code = append(code, byte(value)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "variable blend RIP", map[string]bool{}) + if err != nil { + t.Fatalf("%s width %d: %v", family.vector, width, err) + } + if len(decoded) != 2 || decoded[0].Op != Op(family.vector) || + len(decoded[0].x86RIPLiteralData) != width || + decoded[0].Args[1].Kind != OpSym { + t.Fatalf("%s width %d: %#v", family.vector, width, decoded) + } + } + } +} + +func TestX86VariableBlendGoRejectedForms(t *testing.T) { + for _, family := range x86VariableBlendTestFamilies { + for _, form := range []struct { + op string + args []string + }{ + {family.legacy, []string{" X0,X1", " X1,X2,X3", " X0,(AX),(BX)", " X0,Y1,X2", " X0,X1,X16", ".Z X0,X1,X2"}}, + {family.vector, []string{" X0,X1,X2", " (AX),X1,X2,X3", " X0,X1,(AX),X3", " X0,X1,X2,(AX)", + " X0,Y1,X2,X3", " X0,X1,Y2,X3", " X0,X1,X2,Y3", " Z0,Z1,Z2,Z3", " X16,X1,X2,X3", + " X0,X16,X2,X3", " X0,X1,X16,X3", " X0,X1,X2,X16", ".Z X0,X1,X2,X3", ".BCST X0,(AX),X2,X3", ".SAE X0,X1,X2,X3"}}, + } { + for _, args := range form.args { + line := form.op + args + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),4,$0-0\n"+line+"\nRET\n", false) + assertX86VariableBlendRejected(t, "amd64", "x86_64-unknown-linux-gnu", line) + } + } + } +} + +func TestX86RawVariableBlendGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + for _, family := range x86VariableBlendTestFamilies { + for _, width := range []int{0, 16, 32} { + t.Run(fmt.Sprintf("%s/%s/%d", arch, family.vector, width), func(t *testing.T) { + limit := 16 + if arch == "386" { + limit = 8 + } + prefix, op := "X", family.legacy + if width != 0 { + op = family.vector + } + if width == 32 { + prefix = "Y" + } + memories := []string{"(AX)", "-7(BP)(CX*4)", "8192(SP)"} + if arch == "amd64" { + memories = append(memories, "(R8)", "-7(R13)(R15*8)") + } + var source strings.Builder + source.WriteString("TEXT blend(SB),4,$0-0\n") + for reg := 0; reg < limit; reg++ { + if width == 0 { + fmt.Fprintf(&source, "%s X0,X%d,X2\n%s X0,X1,X%d\n", op, reg, op, reg) + } else { + // Vary the four register fields independently, not as a + // redundant Cartesian product of the same grammar. + for field := 0; field < 4; field++ { + args := []string{prefix + "0", prefix + "1", prefix + "2", prefix + "3"} + args[field] = fmt.Sprintf("%s%d", prefix, reg) + fmt.Fprintf(&source, "%s %s\n", op, strings.Join(args, ",")) + } + } + for _, memory := range memories { + if width == 0 { + fmt.Fprintf(&source, "%s X0,%s,X%d\n", op, memory, reg) + } else { + fmt.Fprintf(&source, "%s %s%d,%s,%s2,%s%d\n", op, prefix, reg, memory, prefix, prefix, reg) + } + } + } + source.WriteString("RET\n") + encoderArch := arch + if arch == "386" && width != 0 { + // Go's 386 parser rejects four operands before the shared + // VEX encoder. The same low-register/address encodings + // are valid on i386; decode and compile them there too. + requireX86GoAssemblerResult(t, arch, "TEXT bad(SB),4,$0-0\n"+op+" X0,X1,X2,X3\nRET\n", false) + encoderArch = "amd64" + } + code := assembleX87ControlBytes(t, encoderArch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !x86RawOperandsMatchGoSource(got.Args, want[i].Args) { + t.Fatalf("instruction %d %s operands=%#v, want source %#v", i, got.Op, got.Args, want[i].Args) + } + } + file := x86RawStackObservingFormProbeFile(t, arch, "blend", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir := x86UnboundBlendMinposProbeObject(t, file, arch, triple) + compileLLVMToObject(t, llc, triple, "raw-blend.ll", "raw-blend.o", ir) + } + t.Logf("checked %d actual Go encodings", len(want)-1) + }) + } + } + } +} diff --git a/amd64_raw_bmi2_shift.go b/amd64_raw_bmi2_shift.go new file mode 100644 index 000000000..155dd0cbb --- /dev/null +++ b/amd64_raw_bmi2_shift.go @@ -0,0 +1,64 @@ +package plan9asm + +import "fmt" + +// All six Go _ybextrl forms use VEX.L0. Intel XED's hsw-bmi-vex-isa +// table leaves W unconstrained in not64 mode, so Q bytes decode as L on 386. +func decodedX86BMI2ShiftInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86BMI2ShiftInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86BMI2ShiftInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || p.opcode != 0xf7 || p.pp == 0 { + return Instr{}, 0, false, nil + } + + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("BMI2 variable shift: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.vectorLength != 0 { + return fail("requires VEX.L0 encoding") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + + bits := 32 + if p.w && mode == 64 { + bits = 64 + } + var op Op + for name, spec := range amd64BMI2ShiftSpecs { + if !spec.immediate && spec.bits == bits && spec.prefix == p.pp { + op = name + break + } + } + + source, size, err := memory.scalar(code[p.modRM:], mode, p.b, p.x, p.segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + count, _ := decodedX86GeneralRegister(p.upper) + destination, _ := decodedX86GeneralRegister(int(code[p.modRM]>>3&7) + p.r*8) + instruction := Instr{ + Op: op, + Args: []Operand{ + {Kind: OpReg, Reg: count}, + source, + {Kind: OpReg, Reg: destination}, + }, + Raw: fmt.Sprintf("%s %s, %s, %s", op, count, source.String(), destination), + } + return instruction, p.modRM + size, true, nil +} diff --git a/amd64_raw_bmi2_shift_test.go b/amd64_raw_bmi2_shift_test.go new file mode 100644 index 000000000..951342ddb --- /dev/null +++ b/amd64_raw_bmi2_shift_test.go @@ -0,0 +1,251 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86BMI2ShiftTestOps = []string{ + "SARXL", "SARXQ", "SHLXL", "SHLXQ", "SHRXL", "SHRXQ", +} + +func TestX86BMI2ShiftGrammarMatchesGoEncoder(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + pattern := `(?s)\{as: A((?:SARX|SHLX|SHRX)[LQ]), ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}` + rows := regexp.MustCompile(pattern).FindAllStringSubmatch(string(data), -1) + if len(rows) != 6 || len(amd64BMI2ShiftSpecs) != 8 { + t.Fatalf("Go family=%d, typed family=%d", len(rows), len(amd64BMI2ShiftSpecs)) + } + for _, row := range rows { + spec, ok := amd64BMI2ShiftSpecs[Op(row[1])] + prefix := map[int]string{1: "vex66", 2: "vexF3", 3: "vexF2"}[spec.prefix] + wide := strings.HasSuffix(row[1], "Q") + if !ok || row[2] != "_ybextrl" || (spec.bits == 64) != wide { + t.Fatalf("incorrect grammar for %s: %+v", row[1], spec) + } + for _, token := range []string{"vex128", "vex0F38", "0xF7", prefix} { + if !strings.Contains(row[3], token) { + t.Fatalf("Go %s missing %s", row[1], token) + } + } + if strings.Contains(row[3], "vexW1") != wide { + t.Fatalf("incorrect W for %s", row[1]) + } + } + pattern = `(?s)var _ybextrl = \[\]ytab\{(.*?)\n\}` + table := regexp.MustCompile(pattern).FindStringSubmatch(string(data)) + if len(table) != 2 || strings.Count(table[1], "argList{") != 1 || !strings.Contains(table[1], "argList{Yrl, Yml, Yrl}") { + t.Fatal("Go BMI2 shift operand grammar changed") + } +} + +func TestX86RawBMI2ShiftEncodingFields(t *testing.T) { + cases := 0 + for pp, stem := range map[int]string{1: "SHLX", 2: "SARX", 3: "SHRX"} { + for _, wide := range []bool{false, true} { + for source := 0; source < 16; source++ { + for count := 0; count < 16; count++ { + for dst := 0; dst < 16; dst++ { + code := encodeRawScalarMove(false, pp, 0xf7, source, count, dst, 0, false) + code[1] = code[1]&0xe0 | 2 + if wide { + code[2] |= 0x80 + } + for _, mode := range []int{32, 64} { + got, n, matched, err := decodedX86BMI2ShiftInstruction(code, mode) + if mode == 32 && (source >= 8 || count >= 8 || dst >= 8) { + if !matched || err == nil { + t.Fatalf("386 accepted %x", code) + } + continue + } + width := "L" + if wide && mode == 64 { + width = "Q" + } + sr, _ := decodedX86GeneralRegister(source) + cr, _ := decodedX86GeneralRegister(count) + dr, _ := decodedX86GeneralRegister(dst) + want := []Operand{ + {Kind: OpReg, Reg: cr}, + {Kind: OpReg, Reg: sr}, + {Kind: OpReg, Reg: dr}, + } + if err != nil || !matched || n != len(code) || got.Op != Op(stem+width) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x mode=%d: %+v %v, want %s %+v", code, mode, got, err, stem+width, want) + } + } + cases++ + } + } + } + } + } + if cases != 24576 { + t.Fatalf("checked %d encodings, want 24576", cases) + } +} + +func TestX86RawBMI2ShiftInvalidAndSegmentForms(t *testing.T) { + invalid := [][]byte{ + {0xc4, 0xe2, 0x7d, 0xf7, 0xc0}, // VEX.L1. + {0x62, 0xf2, 0x7d, 0x08, 0xf7, 0xc0}, // EVEX. + {0xc4, 0xe2, 0x79, 0xf7}, // Missing ModRM. + {0xc4, 0xe2, 0x79, 0xf7, 0x04}, // Missing SIB. + {0xc4, 0xe2, 0x79, 0xf7, 0x80}, // Missing displacement. + {0xc4, 0xe2, 0x79, 0xf7, 0x05, 0, 0, 0, 0}, // RIP-relative. + {0x67, 0xc4, 0xe2, 0x79, 0xf7, 0x00}, // Address override. + } + for _, code := range invalid { + if _, _, matched, err := decodedX86BMI2ShiftInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x", code) + } + } + if _, _, matched, err := decodedX86BMI2ShiftInstruction([]byte{0xc4, 0xe2, 0x79, 0xf7, 0xc0}, 16); !matched || err == nil { + t.Fatal("accepted invalid mode") + } + for _, code := range [][]byte{nil, {0xc4}, {0xc4, 0xe2, 0x78, 0xf7, 0xc0}, {0xc4, 0xe1, 0x79, 0xf7, 0xc0}} { + if _, _, matched, _ := decodedX86BMI2ShiftInstruction(code, 64); matched { + t.Fatalf("matched unrelated encoding %x", code) + } + } + + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + code := []byte{segment.prefix, 0xc4, 0x02, 0xb1, 0xf7, 0x64, 0x88, 0xf9} + got, n, matched, err := decodedX86BMI2ShiftInstruction(code, 64) + want := []Operand{ + {Kind: OpReg, Reg: "R9"}, + {Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: -7}}, + {Kind: OpReg, Reg: "R12"}, + } + if err != nil || !matched || n != len(code) || got.Op != "SHLXQ" || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v %v, want %+v", code, got, err, want) + } + } +} + +func TestX86BMI2ShiftGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86BMI2ShiftTestOps { + args := []string{ + " AX,BX", " X0,AX,BX", " AX,X0,BX", " AX,BX,X0", + " (AX),BX,CX", " AX,BX,(CX)", " $1,BX,CX", + ".Z AX,BX,CX", ".BCST AX,(BX),CX", + } + if arch == "386" { + args = append(args, " R8,BX,CX", " AX,R8,CX", " AX,BX,R8") + } + for _, operands := range args { + source := "TEXT bad(SB),4,$0-0\n" + op + operands + "\nRET\n" + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: arch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("translator accepted %s%s on %s", op, operands, arch) + } + } + } + } +} + +func TestX86RawBMI2ShiftGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + regs := []string{"AX", "SP", "DI"} + if arch == "amd64" { + regs = append(regs, "R8", "R15") + } + + var source strings.Builder + source.WriteString("TEXT bmi2shift(SB),4,$0-0\n") + for _, op := range x86BMI2ShiftTestOps { + for _, reg := range regs { + for _, input := range []string{reg, "(BX)", "-7(BP)(CX*4)", "8192(SI)"} { + fmt.Fprintf(&source, "%s %s,%s,%s\n", op, reg, input, reg) + } + } + } + source.WriteString("RET\n") + + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if arch == "386" && strings.HasSuffix(string(want[i].Op), "Q") { + want[i].Op = Op(strings.TrimSuffix(string(want[i].Op), "Q") + "L") + } + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + + file := x86RawStackObservingFormProbeFile(t, arch, "bmi2shift", code) + triples := []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir := x86UnboundVoidRawProbeObject(t, file, arch, triple) + compileLLVMToObject(t, llc, triple, "raw-bmi2-shift.ll", "raw-bmi2-shift.o", ir) + } + t.Logf("checked %d Go encodings", len(want)-1) + }) + } +} + +func TestX86BMI2Shift386IgnoresW(t *testing.T) { + for _, op := range []string{"SARXQ", "SHLXQ", "SHRXQ"} { + t.Run(op, func(t *testing.T) { + // Retain the original Go-accepted source and its missing-input + // Context controls. Initialized whole programs have separate owners. + source := "TEXT shift(SB),4,$0-0\n" + op + " AX,(BX),CX\nRET\n" + ir := x86UnboundVoidScalarFormatObject(t, source, "shift", "386", "i386-unknown-linux-gnu") + if !strings.Contains(ir, "load volatile i32, ptr") || !strings.Contains(ir, ", 31\n") || + strings.Contains(ir, "ashr i64") || strings.Contains(ir, "shl i64") || strings.Contains(ir, "lshr i64") { + t.Fatalf("386 %s must read exactly four bytes:\n%s", op, ir) + } + }) + } +} diff --git a/amd64_raw_broadcast_block.go b/amd64_raw_broadcast_block.go new file mode 100644 index 000000000..f479430ef --- /dev/null +++ b/amd64_raw_broadcast_block.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawBlockBroadcastForm struct { + op Op + sourceBytes int + minLength int + allowX bool +} + +// Go 1.27's complete EVEX packed-block broadcast rows. The corresponding +// VEX F128/I128 memory forms have a separate two-form decoder. +var x86RawBlockBroadcastForms = map[[2]int]x86RawBlockBroadcastForm{ + {0x19, 0}: {op: "VBROADCASTF32X2", sourceBytes: 8, minLength: 1, allowX: true}, + {0x59, 0}: {op: "VBROADCASTI32X2", sourceBytes: 8, minLength: 0, allowX: true}, + {0x1a, 0}: {op: "VBROADCASTF32X4", sourceBytes: 16, minLength: 1}, + {0x1a, 1}: {op: "VBROADCASTF64X2", sourceBytes: 16, minLength: 1}, + {0x5a, 0}: {op: "VBROADCASTI32X4", sourceBytes: 16, minLength: 1}, + {0x5a, 1}: {op: "VBROADCASTI64X2", sourceBytes: 16, minLength: 1}, + {0x1b, 0}: {op: "VBROADCASTF32X8", sourceBytes: 32, minLength: 2}, + {0x1b, 1}: {op: "VBROADCASTF64X4", sourceBytes: 32, minLength: 2}, + {0x5b, 0}: {op: "VBROADCASTI32X8", sourceBytes: 32, minLength: 2}, + {0x5b, 1}: {op: "VBROADCASTI64X4", sourceBytes: 32, minLength: 2}, +} + +// decodedX86RawBlockBroadcastInstruction decodes every EVEX block-broadcast +// shape, including register-backed X2 sources and K merge/zero masking. +func decodedX86RawBlockBroadcastInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86RawBlockBroadcastInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86RawBlockBroadcastInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 { + return Instr{}, 0, false, nil + } + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawBlockBroadcastForms[[2]int{p.opcode, w}] + if !recognized { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX block broadcast: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.addressOverride || p.broadcast || p.upper != 0 || + p.vectorLength < form.minLength || p.vectorLength > 2 { + return fail("invalid EVEX fixed, address, broadcast, vvvv, or vector-length bits") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + modRM := code[p.modRM] + if modRM>>6 == 3 && !form.allowX { + return fail("this block width requires a memory source") + } + destinationNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && p.vectorLength == 2 && destinationNumber >= 8 { + return fail("386 Z-register exceeds the Go assembler frontend's register class") + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + source, consumed, err := memory.evex( + code[p.modRM:], mode, p.b, p.x, p.segment, "X", form.sourceBytes, + ) + if err != nil { + return Instr{}, 0, true, err + } + op := form.op + if p.zero { + op += ".Z" + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, destinationNumber))}) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_broadcast_block_test.go b/amd64_raw_broadcast_block_test.go new file mode 100644 index 000000000..0816cac40 --- /dev/null +++ b/amd64_raw_broadcast_block_test.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawBlockBroadcast(opcode, widthBit, vectorBits, mask int, zeroing, registerSource bool) []byte { + p1 := byte(0x7d) + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0x40) // disp8(AX), destination 0. + if registerSource { + modRM = 0xc1 // X1, destination 0. + } + code := []byte{0x62, 0xf2, p1, p2, byte(opcode), modRM} + if !registerSource { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawBlockBroadcastSeBiShogunRegression(t *testing.T) { + // Go disassembles the instruction as VBROADCASTI32X4 RIP+1, Z1. + code := []byte{0x62, 0xf2, 0x7d, 0x48, 0x5a, 0x0d, 1, 0, 0, 0, 0xc3} + for index := 0; index < 16; index++ { + code = append(code, byte(index+1)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "onesCountInt32 AVX512 block broadcast", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VBROADCASTI32X4" || + decoded[0].Args[1].Reg != "Z1" || !decoded[0].x86RIPLiteral { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawBlockBroadcastCompleteGoFamily(t *testing.T) { + if len(x86RawBlockBroadcastForms) != 10 { + t.Fatalf("modeled %d EVEX block-broadcast rows, want ten", len(x86RawBlockBroadcastForms)) + } + for encoding, form := range x86RawBlockBroadcastForms { + for vectorBits := form.minLength; vectorBits <= 2; vectorBits++ { + for _, masking := range []struct { + mask, zero int + }{ + {}, {mask: 3}, {mask: 7, zero: 1}, + } { + for _, registerSource := range []bool{false, true} { + if registerSource && !form.allowX { + continue + } + name := fmt.Sprintf("%s/length%d/mask%d/zero%d/reg%t", form.op, vectorBits, masking.mask, masking.zero, registerSource) + t.Run(name, func(t *testing.T) { + code := encodeX86RawBlockBroadcast(encoding[0], encoding[1], vectorBits, masking.mask, masking.zero != 0, registerSource) + got, length, ok, err := decodedX86RawBlockBroadcastInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if masking.zero != 0 { + wantOp += ".Z" + } + wantDestination := fmt.Sprintf("%s0", [...]string{"X", "Y", "Z"}[vectorBits]) + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != wantDestination || + registerSource && got.Args[0].String() != "X1" || + len(got.Args) != 2+map[bool]int{true: 1}[masking.mask != 0] { + t.Fatalf("decode %x = %+v, want %s ... %s", code, got, wantOp, wantDestination) + } + }) + } + } + } + } +} + +func TestDecodeX86RawBlockBroadcastRIPLiteralCompleteFamily(t *testing.T) { + for encoding, form := range x86RawBlockBroadcastForms { + for vectorBits := form.minLength; vectorBits <= 2; vectorBits++ { + code := encodeX86RawBlockBroadcast(encoding[0], encoding[1], vectorBits, 1, false, false) + code = code[:6] + code[5] = 0x05 // RIP-relative source, destination 0. + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < form.sourceBytes; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/length%d", form.op, vectorBits) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != form.sourceBytes || got[0].Op != form.op { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestDecodeX86RawBlockBroadcastRejectsInvalidForms(t *testing.T) { + base := encodeX86RawBlockBroadcast(0x5a, 0, 2, 0, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x10 }, // no EVEX.b broadcast + func(code []byte) { code[3] |= 0x80 }, // zeroing needs a mask + func(code []byte) { code[3] |= 0x20 }, // vector length 3 reserved + func(code []byte) { code[2] &^= 0x08 }, // reserved vvvv + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86RawBlockBroadcastInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + registerSource := encodeX86RawBlockBroadcast(0x5a, 0, 2, 0, false, true) + if _, _, ok, err := decodedX86RawBlockBroadcastInstruction(registerSource, 64); !ok || err == nil { + t.Fatalf("accepted m128-only register source %x: ok=%v err=%v", registerSource, ok, err) + } +} + +func TestTranslateX86RawBlockBroadcastLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawBlockBroadcast(SB),$0-0\n") + for encoding, form := range x86RawBlockBroadcastForms { + for vectorBits := form.minLength; vectorBits <= 2; vectorBits++ { + for _, masking := range []struct { + mask int + zero bool + }{ + {}, {mask: 3}, {mask: 7, zero: true}, + } { + code := encodeX86RawBlockBroadcast(encoding[0], encoding[1], vectorBits, masking.mask, masking.zero, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-block-broadcast.ll", "raw-block-broadcast.o", ir) + }) + } +} diff --git a/amd64_raw_broadcast_literal.go b/amd64_raw_broadcast_literal.go new file mode 100644 index 000000000..8f8cc52fc --- /dev/null +++ b/amd64_raw_broadcast_literal.go @@ -0,0 +1,1433 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "strings" + + "golang.org/x/arch/x86/x86asm" +) + +type x86RawLiteralRange struct { + first int + last int + source int +} + +// decodeX86RawPackedBroadcastLiteral resolves a RIP-relative packed scalar +// broadcast only when its source bytes live in the same raw directive group. +// The caller must still prove that those bytes are not reachable instructions. +// A displacement into another TEXT, a missing constant pool, and segment- or +// address-size-relative loads stay on the rejecting decoder path. +func decodeX86RawPackedBroadcastLiteral(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + + // A segment or address-size prefix changes the effective source. + switch code[offset] { + case 0x64, 0x65, 0x67: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + + modRMIndex := -1 + laneBytes := 0 + if len(code) >= offset+5 && code[offset] == 0xc4 && code[offset+1]&0x1f == 2 { + modRMIndex = offset + 4 + switch code[offset+3] { + case 0x78: + laneBytes = 1 + case 0x79: + laneBytes = 2 + case 0x58: + laneBytes = 4 + case 0x59: + laneBytes = 8 + } + } else if len(code) >= offset+6 && code[offset] == 0x62 && code[offset+1]&0x0f == 2 { + modRMIndex = offset + 5 + switch code[offset+4] { + case 0x78: + laneBytes = 1 + case 0x79: + laneBytes = 2 + case 0x58: + laneBytes = 4 + case 0x59: + laneBytes = 8 + } + } + if laneBytes == 0 || modRMIndex < 0 || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + patched, value, literal, err := x86RawRIPLiteral(code, offset, modRMIndex, laneBytes) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + length := len(patched) + instruction, consumed, ok, err := decodedX86VEXPackedBroadcastInstruction(patched, mode) + if err != nil || !ok || consumed != length { + if err == nil { + err = fmt.Errorf("RIP-relative packed broadcast did not match its VEX/EVEX grammar") + } + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + + setX86RawRIPLiteral(&instruction, value) + return instruction, length, literal, true, nil +} + +// decodeX86RawVMOVIntegerLiteral covers all memory-to-X VMOVW/VMOVD/VMOVQ VEX and +// EVEX encodings, including VMOVQ's alternate F3 7E encoding. The ordinary +// decoder validates the full grammar after the fixed-length ModRM rewrite. +func decodeX86RawVMOVIntegerLiteral(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + + switch code[offset] { + case 0x64, 0x65, 0x67: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := -1 + width := 0 + if len(code) >= offset+4 && code[offset] == 0xc5 { + modRMIndex = offset + 3 + width = x86RawVMOVLoadWidth(code[offset+2], code[offset+1]&3, false, false) + } else if len(code) >= offset+5 && code[offset] == 0xc4 && code[offset+1]&0x1f == 1 { + modRMIndex = offset + 4 + width = x86RawVMOVLoadWidth(code[offset+3], code[offset+2]&3, code[offset+2]&0x80 != 0, false) + } else if len(code) >= offset+6 && code[offset] == 0x62 { + modRMIndex = offset + 5 + if code[offset+1]&0x0f == 1 { + width = x86RawVMOVLoadWidth(code[offset+4], code[offset+2]&3, code[offset+2]&0x80 != 0, true) + } else if code[offset+1]&0x0f == 5 && code[offset+4] == 0x6e && code[offset+2]&3 == 1 { + width = 2 + } + } + if width == 0 || modRMIndex < 0 || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + + patched, value, literal, err := x86RawRIPLiteral(code, offset, modRMIndex, width) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + instruction, consumed, ok, err := decodedX86VMOVQInstruction(patched, mode) + if err != nil || !ok || consumed != len(patched) || + len(instruction.Args) != 2 || instruction.Args[0].Kind != OpMem { + if err == nil { + err = fmt.Errorf("RIP-relative VMOVW/VMOVD/VMOVQ load did not match its VEX/EVEX grammar") + } + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + setX86RawRIPLiteral(&instruction, value) + return instruction, len(patched), literal, true, nil +} + +// decodeX86RawScalarFloatBroadcastLiteral resolves VBROADCASTSS/SD loads +// through the same source-local, unreachable constant-pool proof. +func decodeX86RawScalarFloatBroadcastLiteral(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 2 || p.pp != 1 || + (p.opcode != 0x18 && p.opcode != 0x19) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 4 + if p.opcode == 0x19 { + width = 8 + } + patched, value, literal, err := x86RawRIPLiteral(code, offset, modRMIndex, width) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + instruction, consumed, ok, err := decodedX86ScalarBroadcastInstruction(patched, mode) + if err != nil || !ok || consumed != len(patched) || + len(instruction.Args) < 2 || instruction.Args[0].Kind != OpMem { + if err == nil { + err = fmt.Errorf("RIP-relative scalar float broadcast did not match its VEX/EVEX grammar") + } + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + setX86RawRIPLiteral(&instruction, value) + return instruction, len(patched), literal, true, nil +} + +func x86RawVMOVLoadWidth(opcode, pp byte, width64, evex bool) int { + if opcode == 0x6e && pp == 1 { + if width64 { + return 8 + } + return 4 + } + if opcode == 0x7e && pp == 2 && (width64 == evex) { + return 8 + } + return 0 +} + +func x86RawRIPLiteral(code []byte, offset, modRMIndex, width int) ([]byte, int64, x86RawLiteralRange, error) { + patched, data, literal, err := x86RawRIPBytes(code, offset, modRMIndex, width) + if err != nil { + return nil, 0, x86RawLiteralRange{}, err + } + if width > 8 { + return nil, 0, x86RawLiteralRange{}, fmt.Errorf("%d-byte RIP literal does not fit an immediate", width) + } + var value uint64 + for index, b := range data { + value |= uint64(b) << (8 * index) + } + return patched, int64(value), literal, nil +} + +func x86RawRIPBytes(code []byte, offset, modRMIndex, width int) ([]byte, []byte, x86RawLiteralRange, error) { + return x86RawRIPBytesWithSuffix(code, offset, modRMIndex, width, 0) +} + +func x86RawRIPBytesWithSuffix(code []byte, offset, modRMIndex, width, trailingBytes int) ([]byte, []byte, x86RawLiteralRange, error) { + if trailingBytes < 0 || len(code) < modRMIndex+5+trailingBytes { + return nil, nil, x86RawLiteralRange{}, fmt.Errorf("truncated RIP-relative literal") + } + length := modRMIndex + 5 + trailingBytes - offset + displacement := int32(binary.LittleEndian.Uint32(code[modRMIndex+1 : modRMIndex+5])) + first := offset + length + int(displacement) + last := first + width + if width <= 0 || first < 0 || last > len(code) || last < first { + return nil, nil, x86RawLiteralRange{}, fmt.Errorf("RIP-relative literal [%d,%d) is outside directive group", first, last) + } + + // A base-register disp32 has the same byte length as RIP+disp32, allowing + // the existing VEX/EVEX grammar to validate every other encoding field. + patched := append([]byte(nil), code[offset:offset+length]...) + patched[modRMIndex-offset] = patched[modRMIndex-offset]&0x38 | 0x80 + data := append([]byte(nil), code[first:last]...) + return patched, data, x86RawLiteralRange{first: first, last: last}, nil +} + +// decodeX86RawPackedMoveRIPData handles source-local X/Y/Z memory loads for +// the complete packed float and VEX integer move decoder families. +func decodeX86RawPackedMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + packedFloat := (p.opcode == 0x10 || p.opcode == 0x28) && p.pp <= 1 + packedInteger := p.opcode == 0x6f && !p.evex && (p.pp == 1 || p.pp == 2) + if !matched || p.mapNumber != 1 || (!packedFloat && !packedInteger) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedMoveInstruction, "packed move", + ) +} + +// decodeX86RawEVEXPackedIntegerMoveRIPData covers the six EVEX VMOVDQA/DQU +// load opcodes across X/Y/Z widths and optional K masks. +func decodeX86RawEVEXPackedIntegerMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 1 || p.opcode != 0x6f || + (p.pp != 1 && p.pp != 2 && p.pp != 3) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86EVEXPackedIntegerMoveInstruction, "EVEX packed integer move", + ) +} + +// decodeX86RawVBROADCAST128RIPData resolves both VEX.256 F128/I128 +// m128-to-Y forms through their shared source-memory grammar. +func decodeX86RawVBROADCAST128RIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.evex || p.mapNumber != 2 || p.pp != 1 || + p.vectorLength != 1 || (p.opcode != 0x1a && p.opcode != 0x5a) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, 16, + decodedX86VBROADCAST128Instruction, "VBROADCASTF128/VBROADCASTI128", + ) +} + +// decodeX86RawPackedLogicalRIPData covers VEX VPAND/ANDN/OR/XOR and their +// EVEX D/Q counterparts. EVEX.b reads one scalar lane; other forms read the +// full X/Y/Z-width vector from the same unreachable source-local pool. +func decodeX86RawPackedLogicalRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || p.pp != 1 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + switch p.opcode { + case 0xdb, 0xdf, 0xeb, 0xef: + default: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + decoder := x86RawInstructionDecoder(decodedX86VEXPackedIntegerLogicalInstruction) + family := "VEX packed integer logical" + if p.evex { + decoder = decodedX86EVEXPackedLogicalInstruction + family = "EVEX packed logical" + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, decoder, family, + ) +} + +// decodeX86RawFloatLogicalRIPData covers VEX/EVEX VAND/ANDN/OR/XOR PS/PD +// reads from an unreachable source-local X/Y/Z-width constant pool. +func decodeX86RawFloatLogicalRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || p.pp > 1 || + p.opcode < 0x54 || p.opcode > 0x57 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + decoder := x86RawInstructionDecoder(decodedX86VEXPackedFloatLogicalInstruction) + if p.evex { + decoder = decodedX86RawEVEXFloatLogicalInstruction + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, decoder, "packed floating logical", + ) +} + +// decodeX86RawHorizontalFloatRIPData covers VHADD/VHSUB/VADDSUB PS/PD +// VEX X/Y reads from unreachable source-local vector constants. +func decodeX86RawHorizontalFloatRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.evex || p.mapNumber != 1 || + (p.pp != 1 && p.pp != 3) || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + switch p.opcode { + case 0x7c, 0x7d, 0xd0: + default: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, 16<= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := -1 + width := 16 + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if matched && !p.evex && p.mapNumber == 3 && p.pp == 1 && + p.segment == "" && !p.addressOverride { + switch p.opcode { + case 0x02, 0x0c, 0x0d, 0x0e: + modRMIndex = offset + p.modRM + width <<= p.vectorLength + } + } + if modRMIndex < 0 && code[offset] == 0x66 { + i := offset + 1 + if i < len(code) && code[i]&0xf0 == 0x40 { + i++ // Optional REX extension for the legacy X registers. + } + if len(code) >= i+4 && code[i] == 0x0f && code[i+1] == 0x3a { + switch code[i+2] { + case 0x0c, 0x0d, 0x0e: + modRMIndex = i + 3 + } + } + } + if modRMIndex < 0 || len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86ImmediatePackedBlendInstruction, "immediate packed blend", + ) +} + +// decodeX86RawCLMULRIPData covers legacy PCLMULQDQ and all VEX/EVEX +// VPCLMULQDQ widths. The imm8 follows the source displacement. +func decodeX86RawCLMULRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := -1 + width := 16 + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if matched && p.mapNumber == 3 && p.pp == 1 && p.opcode == 0x44 && + p.segment == "" && !p.addressOverride && p.vectorLength <= 2 { + modRMIndex = offset + p.modRM + width <<= p.vectorLength + } + if modRMIndex < 0 && code[offset] == 0x66 { + i := offset + 1 + if i < len(code) && code[i]&0xf0 == 0x40 { + i++ + } + if len(code) >= i+4 && code[i] == 0x0f && code[i+1] == 0x3a && code[i+2] == 0x44 { + modRMIndex = i + 3 + } + } + if modRMIndex < 0 || len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86RawCLMULInstruction, "carryless multiply", + ) +} + +// decodeX86RawMaskBlendRIPData folds source-local vector constants, or one +// D/Q broadcast lane, for all six EVEX mask-blend operations. +func decodeX86RawMaskBlendRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || + p.opcode < 0x64 || p.opcode > 0x66 || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86RawMaskBlendInstruction, "EVEX mask blend", + ) +} + +// decodeX86RawScalarMoveRIPData covers memory-to-X VMOVSS/VMOVSD VEX and +// EVEX encodings plus raw EVEX VMOVSH. The typed scalar decoder enforces +// reserved fields and masks. +func decodeX86RawScalarMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + half := p.evex && p.mapNumber == 5 && p.pp == 2 + if !matched || (p.mapNumber != 1 && !half) || p.opcode != 0x10 || + (p.pp != 2 && p.pp != 3) || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 4 + if half { + width = 2 + } else if p.pp == 3 { + width = 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86ScalarMoveInstruction, "scalar move", + ) +} + +// decodeX86RawMinMaxRIPData covers the complete VEX/EVEX packed integer +// min/max family. EVEX.b reads one D/Q lane; other forms read a full vector. +func decodeX86RawMinMaxRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + for _, spec := range amd64PackedIntegerMinMaxSpecs { + if p.mapNumber != spec.mapNumber || p.opcode != spec.opcode { + continue + } + if !p.evex && spec.laneBits == 64 { + continue + } + if p.evex && spec.laneBits >= 32 && p.w != (spec.laneBits == 64) { + continue + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = spec.laneBits / 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedIntegerMinMaxInstruction, "packed integer min/max", + ) + } + return Instr{}, 0, x86RawLiteralRange{}, false, nil +} + +// decodeX86RawPackedCompareRIPData covers all VPCMPEQ/GT B/W/D/Q source +// memory forms. The ordinary decoder checks the EVEX mask-result grammar. +func decodeX86RawPackedCompareRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + for _, spec := range amd64PackedIntegerCompareSpecs { + if p.mapNumber != spec.mapNumber || p.opcode != spec.opcode { + continue + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = spec.laneBits / 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedIntegerCompareInstruction, "packed integer compare", + ) + } + return Instr{}, 0, x86RawLiteralRange{}, false, nil +} + +// decodeX86RawVariableShiftRIPData covers the VEX and EVEX per-lane variable +// shift family, including scalar D/Q memory broadcast in EVEX encodings. +func decodeX86RawVariableShiftRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 2 || p.pp != 1 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + for _, spec := range amd64PerLaneVariableShiftSpecs { + if p.opcode != int(spec.opcode) || p.w != (spec.laneBits != 32) || + !p.evex && !spec.vex { + continue + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = spec.laneBits / 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PerLaneVariableShiftInstruction, "per-lane variable shift", + ) + } + return Instr{}, 0, x86RawLiteralRange{}, false, nil +} + +// decodeX86RawVPSHUFBRIPData covers every VEX and EVEX vector-width load +// accepted by Go's VPSHUFB table. The instruction has no scalar broadcast. +func decodeX86RawVPSHUFBRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 2 || p.pp != 1 || p.opcode != 0 || + p.w || p.broadcast || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86VPSHUFBInstruction, "VPSHUFB", + ) +} + +// decodeX86RawPackedArithmeticRIPData covers the complete VEX/EVEX VPADD +// and VPSUB families, including their D/Q scalar memory broadcasts. +func decodeX86RawPackedArithmeticRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || p.pp != 1 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + properties, recognized := decodedX86PackedIntegerArithmeticOps[byte(p.opcode)] + if !recognized { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = properties.laneBytes + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedIntegerArithmeticInstruction, "packed integer arithmetic", + ) +} + +// decodeX86RawFMA3RIPData handles all packed and scalar FMA3 encodings. +// Scalar operations read one lane; packed EVEX.b memory forms broadcast it. +func decodeX86RawFMA3RIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + half := p.evex && p.mapNumber == 6 + if !matched || p.mapNumber != 2 && !half || p.pp != 1 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + spec, recognized := decodedX86VEXFMA3Ops[byte(p.opcode)] + if !recognized { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if spec.scalar || p.evex && p.broadcast { + width = 4 + if p.w { + width = 8 + } + if half { + width = 2 + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86VEXFMA3Instruction, "FMA3", + ) +} + +// decodeX86RawBinaryFloatRIPData covers VADD/MUL/SUB/MIN/DIV/MAX packed and +// scalar forms, including raw AVX-512 FP16, plus scalar VSQRT and packed +// memory broadcast. +func decodeX86RawBinaryFloatRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + half := p.evex && p.mapNumber == 5 && (p.pp == 0 || p.pp == 2) + if !matched || (p.mapNumber != 1 && !half) || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + switch p.opcode { + case 0x58, 0x59, 0x5c, 0x5d, 0x5e, 0x5f: + case 0x51: + if p.pp < 2 && !half { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + default: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if half { + if p.pp == 2 || p.broadcast { + width = 2 + } + } else if p.pp >= 2 || p.evex && p.broadcast { + width = 4 + if p.pp == 1 || p.pp == 3 { + width = 8 + } + } + decoder := decodedX86VEXBinaryFloatInstruction + if half && p.opcode == 0x51 && p.pp == 0 { + decoder = decodedX86SameWidthConversionInstruction + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decoder, "binary floating", + ) +} + +// decodeX86RawScalarFlagCompareRIPData covers VCOMIS{S,D} and +// VUCOMIS{S,D} VEX/EVEX memory sources. EVEX.b remains register-only SAE. +func decodeX86RawScalarFlagCompareRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || (p.opcode != 0x2e && p.opcode != 0x2f) || + (p.pp != 0 && p.pp != 1) || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 4 + if p.pp == 1 { + width = 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86ScalarFlagCompareInstruction, "scalar flag compare", + ) +} + +// decodeX86RawVectorFloatCompareRIPData accounts for VCMP's trailing imm8 +// when resolving the RIP displacement and its second Plan 9 operand. +func decodeX86RawVectorFloatCompareRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || p.opcode != 0xc2 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.pp >= 2 || p.evex && p.broadcast { + width = 4 + if p.pp == 1 || p.pp == 3 { + width = 8 + } + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86VectorFloatCompareInstruction, "vector floating compare", + ) +} + +// decodeX86RawPackedMultiplyRIPData covers the five packed word/D/Q multiply +// opcode rows, including EVEX D/Q memory broadcast and masked widths. +func decodeX86RawPackedMultiplyRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.pp != 1 || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + form, recognized := x86PackedMultiplyFormFor(byte(p.mapNumber), byte(p.opcode)) + if !recognized || p.evex && p.broadcast && form.broadcastByte == 0 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = form.broadcastByte + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedMultiplyInstruction, "packed multiply", + ) +} + +// decodeX86RawPackedAbsRIPData resolves source-local X/Y/Z VPABS loads, +// including the D/Q EVEX memory-broadcast lanes. +func decodeX86RawPackedAbsRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 2 || p.pp != 1 || p.opcode < 0x1c || p.opcode > 0x1f || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = [...]int{1, 2, 4, 8}[p.opcode-0x1c] + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86RawPackedAbsInstruction, "VPABS", + ) +} + +// decodeX86RawVEXRoundRIPData accounts for VROUND's trailing imm8 while +// resolving packed or scalar source-local constant loads. +func decodeX86RawVEXRoundRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 3 || p.pp != 1 || p.opcode < 0x08 || p.opcode > 0x0b || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.opcode == 0x0a { + width = 4 + } else if p.opcode == 0x0b { + width = 8 + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86RawVEXRoundInstruction, "VROUND", + ) +} + +// decodeX86RawDuplicateMoveRIPData covers legacy, VEX and EVEX MOVDDUP, +// MOVSLDUP and MOVSHDUP. The X-width DDUP form reads only one 64-bit lane; +// all other forms read the complete source vector. +func decodeX86RawDuplicateMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := -1 + width := 0 + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if matched && p.mapNumber == 1 && p.segment == "" && !p.addressOverride { + double := p.pp == 3 && p.opcode == 0x12 + single := p.pp == 2 && (p.opcode == 0x12 || p.opcode == 0x16) + if double || single { + modRMIndex = offset + p.modRM + width = 16 << p.vectorLength + if double && p.vectorLength == 0 { + width = 8 + } + } + } else { + i := offset + prefix := code[i] + if prefix == 0xf2 || prefix == 0xf3 { + i++ + if i < len(code) && code[i]&0xf0 == 0x40 { + i++ + } + if len(code) >= i+3 && code[i] == 0x0f && + (prefix == 0xf2 && code[i+1] == 0x12 || + prefix == 0xf3 && (code[i+1] == 0x12 || code[i+1] == 0x16)) { + modRMIndex = i + 2 + width = 16 + if prefix == 0xf2 { + width = 8 + } + } + } + } + if modRMIndex < 0 || len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86DuplicateMoveInstruction, "duplicate move", + ) +} + +// decodeX86RawVectorByteShiftRIPData resolves EVEX X/Y/Z memory forms of +// VPSLLDQ and VPSRLDQ, accounting for the trailing shift-count imm8. +func decodeX86RawVectorByteShiftRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 || p.opcode != 0x73 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + group := int(code[modRMIndex]>>3) & 7 + if group != 3 && group != 7 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, 16<= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.pp != 1 || p.segment != "" || p.addressOverride || + p.evex && p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + if !(p.mapNumber == 1 && p.opcode == 0xf5 || + p.mapNumber == 2 && p.opcode == 0x04) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + decode := x86RawInstructionDecoder(decodedX86VEXPackedMADDInstruction) + if p.evex { + decode = decodedX86EVEXPackedMADDInstruction + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, 16<= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.evex || p.mapNumber != 3 || p.pp != 1 || p.w || + p.segment != "" || p.addressOverride || p.vectorLength > 1 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + if p.opcode != 0x4a && p.opcode != 0x4b && p.opcode != 0x4c { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, 16<= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 || + p.opcode != int(opcode) || p.segment != "" || p.addressOverride || + p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, decode, family, + ) +} + +func decodeX86RawPackedWidenConversionRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + ops := x86RawPackedWidenConversionByOpcode + if p.w { + ops = x86RawPackedDoubleConversionByOpcode + } + if _, recognized := ops[p.opcode]; !recognized { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 8 << p.vectorLength + if p.w { + width *= 2 + } + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 0, 0, + decodedX86PackedWidenConversionInstruction, "packed widening conversion", + ) +} + +// decodeX86RawLegacySIMDMoveRIPData covers the six 128-bit and two scalar +// legacy MOV loads in Go's yxmov table. The other direction is a store and +// does not read the RIP-relative source bytes. +func decodeX86RawLegacySIMDMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + i := offset + prefix := byte(0) + unsafeAddress := false + for i < len(code) { + switch code[i] { + case 0x64, 0x65, 0x67: + unsafeAddress = true + case 0x66, 0xf2, 0xf3: + if prefix != 0 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + prefix = code[i] + default: + if code[i] >= 0x40 && code[i] <= 0x4f { + i++ + } + goto opcode + } + i++ + } + +opcode: + if len(code) < i+3 || code[i] != 0x0f { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + opcode := code[i+1] + valid := opcode == 0x10 || + opcode == 0x28 && (prefix == 0 || prefix == 0x66) || + opcode == 0x6f && (prefix == 0x66 || prefix == 0xf3) + if !valid || code[i+2]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + if unsafeAddress { + return Instr{}, 0, x86RawLiteralRange{}, true, fmt.Errorf("segment/address override is not source-layout safe") + } + width := 16 + if opcode == 0x10 && prefix == 0xf3 { + width = 4 + } else if opcode == 0x10 && prefix == 0xf2 { + width = 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, i+2, width, + decodedX86LegacySIMDMoveInstruction, "legacy SIMD move", + ) +} + +func decodedX86LegacySIMDMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + inst, err := x86asm.Decode(code, mode) + if err != nil || inst.Len <= 0 { + return Instr{}, 0, true, err + } + syntax, err := decodedX86GoSyntax(inst, code[:inst.Len]) + if err != nil { + return Instr{}, 0, true, err + } + instrs, err := parseDecodedX86Instruction(syntax) + if err != nil || len(instrs) != 1 { + if err == nil { + err = fmt.Errorf("legacy SIMD move decoded as %d instructions", len(instrs)) + } + return Instr{}, 0, true, err + } + return instrs[0], inst.Len, true, nil +} + +type x86RawInstructionDecoder func([]byte, int) (Instr, int, bool, error) + +// decodeX86RawIndexedPermuteRIPData preserves local vector/scalar constants +// for the complete VPERMI2/VPERMT2 family. +func decodeX86RawIndexedPermuteRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || + (p.opcode < 0x75 || p.opcode > 0x77) && (p.opcode < 0x7d || p.opcode > 0x7f) || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + if p.opcode == 0x76 || p.opcode == 0x77 || p.opcode == 0x7e || p.opcode == 0x7f { + if p.w { + width = 8 + } + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86IndexedPermuteInstruction, "EVEX indexed permute", + ) +} + +// decodeX86RawEVEXVariableDwordPermuteRIPData preserves local Y/Z data or +// scalar broadcast constants for VPERMD and VPERMPS. +func decodeX86RawEVEXVariableDwordPermuteRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || p.w || + (p.opcode != 0x36 && p.opcode != 0x16) || + p.segment != "" || p.addressOverride || p.vectorLength < 1 || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86EVEXVariableDwordPermuteInstruction, "EVEX variable dword permute", + ) +} + +// decodeX86RawMaskCompareRIPData resolves local full-vector or D/Q broadcast +// constants and accounts for the comparison's trailing imm8. +func decodeX86RawMaskCompareRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawMaskCompareForms[[2]int{p.opcode, w}] + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = form.laneBytes + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86RawMaskCompareInstruction, "EVEX mask compare", + ) +} + +// decodeX86RawScaledRoundRIPData accounts for the trailing imm8 while +// resolving packed vectors, packed broadcasts, and scalar local constants. +func decodeX86RawScaledRoundRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawScaledRoundForms[[2]int{p.opcode, w}] + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if form.scalar || p.broadcast { + width = form.laneBytes + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86RawScaledRoundInstruction, "EVEX scaled round", + ) +} + +// decodeX86RawEVEXUnpackRIPData preserves source-local full-vector and +// broadcast-lane constants for all eight EVEX integer interleave rows. +func decodeX86RawEVEXUnpackRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + form, recognized := x86RawEVEXUnpackForms[p.opcode] + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = form.broadcastBytes + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86EVEXPackedUnpackInstruction, "EVEX packed unpack", + ) +} + +// decodeX86RawBlockBroadcastRIPData resolves source-local m64/m128/m256 +// blocks for all Go EVEX packed-block broadcasts. +func decodeX86RawBlockBroadcastRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawBlockBroadcastForms[[2]int{p.opcode, w}] + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, form.sourceBytes, + decodedX86RawBlockBroadcastInstruction, "EVEX block broadcast", + ) +} + +// decodeX86RawVariableRotateRIPData preserves local vector or scalar +// constant pools for all four EVEX per-lane variable-rotate forms. +func decodeX86RawVariableRotateRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || + (p.opcode != 0x14 && p.opcode != 0x15) || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86RawVariableRotateInstruction, "EVEX variable rotate", + ) +} + +// decodeX86RawQQToFloatRIPData resolves source-local literal pools for all +// QWORD-to-PS/PD encodings before the raw-byte decoder treats them as memory. +func decodeX86RawQQToFloatRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + _, _, recognized := x86RawQQToFloatForm(p) + if !matched || !p.evex || !p.w || p.mapNumber != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86RawQQToFloatInstruction, "QWORD-to-float conversion", + ) +} + +func x86RawRIPDataThroughDecoder( + code []byte, offset, mode, modRMIndex, width int, + decode x86RawInstructionDecoder, family string, +) (Instr, int, x86RawLiteralRange, bool, error) { + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 0, 0, decode, family, + ) +} + +func x86RawRIPDataThroughDecoderOperand( + code []byte, offset, mode, modRMIndex, width, trailingBytes, sourceIndex int, + decode x86RawInstructionDecoder, family string, +) (Instr, int, x86RawLiteralRange, bool, error) { + patched, data, literal, err := x86RawRIPBytesWithSuffix(code, offset, modRMIndex, width, trailingBytes) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + instruction, consumed, ok, err := decode(patched, mode) + if err != nil || !ok || consumed != len(patched) || + len(instruction.Args) <= sourceIndex || instruction.Args[sourceIndex].Kind != OpMem { + if err == nil { + err = fmt.Errorf("RIP-relative %s did not match its grammar", family) + } + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + setX86RawRIPDataOperand(&instruction, sourceIndex, data) + return instruction, len(patched), literal, true, nil +} + +func setX86RawRIPDataOperand(instruction *Instr, sourceIndex int, data []byte) { + instruction.Args[sourceIndex] = Operand{Kind: OpSym, Sym: "·__plan9asm_raw_literal_pending(SB)"} + instruction.x86Encoded = true + instruction.x86RIPLiteral = true + instruction.x86RIPLiteralData = data + args := append([]Operand(nil), instruction.Args...) + for index := range args { + args[index].RegList = append([]Reg(nil), args[index].RegList...) + } + instruction.x86RIPMemoryRead = x86RawStaticRead{ + op: instruction.Op, source: sourceIndex, width: int64(len(data)), + symbol: instruction.Args[sourceIndex].Sym, + object: append([]byte(nil), data...), + args: args, + } + rawArgs := make([]string, len(instruction.Args)) + for index, operand := range instruction.Args { + rawArgs[index] = operand.String() + } + instruction.Raw = fmt.Sprintf("%s %s", instruction.Op, strings.Join(rawArgs, ", ")) +} + +func setX86RawRIPLiteral(instruction *Instr, value int64) { + instruction.Args[0] = Operand{Kind: OpImm, Imm: value} + instruction.x86Encoded = true + instruction.x86RIPLiteral = true + rawArgs := make([]string, len(instruction.Args)) + for index, operand := range instruction.Args { + rawArgs[index] = operand.String() + } + instruction.Raw = fmt.Sprintf("%s %s", instruction.Op, strings.Join(rawArgs, ", ")) +} diff --git a/amd64_raw_broadcast_literal_test.go b/amd64_raw_broadcast_literal_test.go new file mode 100644 index 000000000..177854abd --- /dev/null +++ b/amd64_raw_broadcast_literal_test.go @@ -0,0 +1,497 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86RawBroadcastLiteral(op Op, encoding string, vectorBits, mask int, zeroing bool) ([]byte, int64) { + opcode := map[Op]byte{ + "VPBROADCASTB": 0x78, + "VPBROADCASTW": 0x79, + "VPBROADCASTD": 0x58, + "VPBROADCASTQ": 0x59, + }[op] + width := map[Op]int{ + "VPBROADCASTB": 1, + "VPBROADCASTW": 2, + "VPBROADCASTD": 4, + "VPBROADCASTQ": 8, + }[op] + value := int64(0) + data := make([]byte, width) + for i := range data { + data[i] = byte(0x11 + i*0x22) + value |= int64(data[i]) << (8 * i) + } + + var code []byte + if encoding == "VEX" { + p2 := byte(0x79) + if vectorBits == 256 { + p2 |= 0x04 + } + code = []byte{0xc4, 0xe2, p2, opcode, 0x05, 1, 0, 0, 0} + } else { + p1 := byte(0x7d) + if op == "VPBROADCASTQ" { + p1 |= 0x80 + } + lengthBits := map[int]byte{128: 0, 256: 1, 512: 2}[vectorBits] + p2 := lengthBits<<5 | 0x08 | byte(mask) + if zeroing { + p2 |= 0x80 + } + code = []byte{0x62, 0xf2, p1, p2, opcode, 0x05, 1, 0, 0, 0} + } + code = append(code, 0xc3) + code = append(code, data...) + return code, value +} + +func TestDecodeX86RawPackedBroadcastRIPLiteralCompleteFamily(t *testing.T) { + for _, op := range []Op{"VPBROADCASTB", "VPBROADCASTW", "VPBROADCASTD", "VPBROADCASTQ"} { + for _, encoding := range []string{"VEX", "EVEX"} { + for _, vectorBits := range []int{128, 256, 512} { + if encoding == "VEX" && vectorBits == 512 { + continue + } + for _, masking := range []struct { + mask int + zeroing bool + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true}, + } { + if encoding == "VEX" && masking.mask != 0 { + continue + } + name := fmt.Sprintf("%s/%s/%d/mask%d/zero%t", op, encoding, vectorBits, masking.mask, masking.zeroing) + t.Run(name, func(t *testing.T) { + code, value := x86RawBroadcastLiteral(op, encoding, vectorBits, masking.mask, masking.zeroing) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != op && decoded[0].Op != op+".Z" || + len(decoded[0].Args) < 2 || decoded[0].Args[0].Kind != OpImm || + decoded[0].Args[0].Imm != value || !decoded[0].x86Encoded || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want a literal %s and RET", code, decoded, op) + } + }) + } + } + } + } +} + +func TestTranslateX86RawPackedBroadcastRIPLiteralObjects(t *testing.T) { + var source strings.Builder + for index, op := range []Op{"VPBROADCASTB", "VPBROADCASTW", "VPBROADCASTD", "VPBROADCASTQ"} { + name := fmt.Sprintf("literalBroadcast%d", index) + code, _ := x86RawBroadcastLiteral(op, "EVEX", 512, 3, false) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Literal data does not supply the original K3 or merge input. + // Initialized complete-family programs have their own public objects. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-broadcast-literal.ll", "raw-broadcast-literal.o", ir) + }) + } +} + +func TestDecodeX86RawPackedBroadcastRIPLiteralRejectsExecutableAndMissingData(t *testing.T) { + for _, displacement := range []byte{0xf7, 0xf6, 0x20} { + code, _ := x86RawBroadcastLiteral("VPBROADCASTD", "VEX", 256, 0, false) + code[5] = displacement + if displacement != 0x20 { + copy(code[6:9], []byte{0xff, 0xff, 0xff}) + } + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid RIP literal", map[string]bool{}); err == nil { + t.Fatalf("accepted RIP literal displacement %#x", displacement) + } + } + code, _ := x86RawBroadcastLiteral("VPBROADCASTQ", "EVEX", 512, 0, false) + code[6] = 5 // Only the last four bytes remain; a QWORD read is truncated. + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "truncated RIP literal", map[string]bool{}); err == nil { + t.Fatal("accepted truncated RIP-relative QWORD literal") + } + + for _, prefix := range []byte{0x64, 0x65, 0x67} { + code, _ := x86RawBroadcastLiteral("VPBROADCASTD", "VEX", 256, 0, false) + code = append([]byte{prefix}, code...) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "prefixed RIP literal", map[string]bool{}); err == nil { + t.Fatalf("folded a source with prefix %#x", prefix) + } + } +} + +func TestDecodeX86RawPackedBroadcastRIPLiteralDoesNotChange386AbsoluteMemory(t *testing.T) { + code, _ := x86RawBroadcastLiteral("VPBROADCASTD", "VEX", 256, 0, false) + decoded, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 absolute source", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) < 1 || decoded[0].Args[0].Kind != OpMem || decoded[0].x86RIPLiteral { + t.Fatalf("changed a 386 absolute memory operand into a literal: %#v", decoded) + } +} + +func TestTranslateX86RawPackedBroadcastRIPLiteralRejectsAddressObservedText(t *testing.T) { + code, _ := x86RawBroadcastLiteral("VPBROADCASTD", "VEX", 256, 0, false) + var source strings.Builder + source.WriteString("TEXT address(SB),$0-0\n\tLEAQ raw+1(SB), AX\n\tRET\n") + source.WriteString("TEXT raw(SB),$0-0\n\tMOVQ AX, AX\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{ + "address": {Name: "address", Ret: Void}, + "raw": {Name: "raw", Ret: Void}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "address-observed RIP-relative raw literal") { + t.Fatalf("address-observed raw TEXT was silently folded: %v", err) + } +} + +func x86RawVMOVIntegerLiteral(encoding string, width int, alternateQ bool) ([]byte, int64) { + var code []byte + switch encoding { + case "VEX2": + if alternateQ { + code = []byte{0xc5, 0xfa, 0x7e, 0x05} + } else { + code = []byte{0xc5, 0xf9, 0x6e, 0x05} + } + case "VEX3": + p1 := byte(0x79) + if width == 8 { + p1 |= 0x80 + } + opcode := byte(0x6e) + if alternateQ { + p1 = 0x7a + opcode = 0x7e + } + code = []byte{0xc4, 0xe1, p1, opcode, 0x05} + case "EVEX": + p1 := byte(0x7d) + if width == 8 { + p1 |= 0x80 + } + opcode := byte(0x6e) + if alternateQ { + p1 = 0xfe + opcode = 0x7e + } + code = []byte{0x62, 0xf1, p1, 0x08, opcode, 0x05} + } + code = append(code, 1, 0, 0, 0, 0xc3) + var value int64 + for index := 0; index < width; index++ { + b := byte(0x13 + index*0x11) + code = append(code, b) + value |= int64(b) << (8 * index) + } + return code, value +} + +func TestDecodeX86RawVMOVIntegerRIPLiteralCompleteLoadForms(t *testing.T) { + for _, test := range []struct { + encoding string + width int + alternateQ bool + }{ + {"VEX2", 4, false}, + {"VEX3", 4, false}, + {"EVEX", 4, false}, + {"VEX2", 8, true}, + {"VEX3", 8, false}, + {"VEX3", 8, true}, + {"EVEX", 8, false}, + {"EVEX", 8, true}, + } { + name := fmt.Sprintf("%s/%d/alternate%t", test.encoding, test.width, test.alternateQ) + t.Run(name, func(t *testing.T) { + code, value := x86RawVMOVIntegerLiteral(test.encoding, test.width, test.alternateQ) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + op := Op("VMOVD") + if test.width == 8 { + op = "VMOVQ" + } + if len(decoded) != 2 || decoded[0].Op != op || len(decoded[0].Args) != 2 || + decoded[0].Args[0].Kind != OpImm || decoded[0].Args[0].Imm != value || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s literal load and RET", code, decoded, op) + } + }) + } +} + +func TestTranslateX86RawVMOVIntegerRIPLiteralObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, test := range []struct { + encoding string + width int + alternateQ bool + }{ + {"VEX2", 4, false}, {"VEX3", 4, false}, {"EVEX", 4, false}, + {"VEX2", 8, true}, {"VEX3", 8, false}, {"VEX3", 8, true}, + {"EVEX", 8, false}, {"EVEX", 8, true}, + } { + name := fmt.Sprintf("literalVMOV%d", index) + code, _ := x86RawVMOVIntegerLiteral(test.encoding, test.width, test.alternateQ) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-vmov-literal.ll", "raw-vmov-literal.o", ir) + }) + } +} + +func TestDecodeX86RawVMOVIntegerRIPLiteralRejectsUnsafeSources(t *testing.T) { + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "inside_instruction", edit: func(code []byte) []byte { + copy(code[5:9], []byte{0xfa, 0xff, 0xff, 0xff}) // Target the opcode. + return code + }}, + {name: "missing_pool", edit: func(code []byte) []byte { + code[5] = 0x20 + return code + }}, + {name: "truncated_pool", edit: func(code []byte) []byte { + code[5] = 5 // Only the last four bytes remain. + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x64}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code, _ := x86RawVMOVIntegerLiteral("VEX3", 8, false) + code = test.edit(code) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("folded unsafe raw VMOVQ source: %x", code) + } + }) + } +} + +func TestTranslateX86RawVMOVIntegerRIPLiteralRejectsAddressObservedText(t *testing.T) { + code, _ := x86RawVMOVIntegerLiteral("VEX3", 8, false) + var source strings.Builder + source.WriteString("TEXT address(SB),$0-0\n\tLEAQ raw+1(SB), AX\n\tRET\n") + source.WriteString("TEXT raw(SB),$0-0\n\tMOVQ AX, AX\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{ + "address": {Name: "address", Ret: Void}, + "raw": {Name: "raw", Ret: Void}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "address-observed RIP-relative raw literal") { + t.Fatalf("address-observed raw TEXT was silently folded: %v", err) + } +} + +func x86RawScalarFloatBroadcastLiteral(double, evex bool, vectorBits, mask int, zeroing bool) ([]byte, int64) { + opcode := byte(0x18) + width := 4 + if double { + opcode = 0x19 + width = 8 + } + var code []byte + if evex { + p1 := byte(0x7d) + if double { + p1 |= 0x80 + } + p2 := byte(map[int]int{128: 0, 256: 1, 512: 2}[vectorBits]<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + code = []byte{0x62, 0xf2, p1, p2, opcode, 0x05} + } else { + p1 := byte(0x79) + if vectorBits == 256 { + p1 |= 0x04 + } + code = []byte{0xc4, 0xe2, p1, opcode, 0x05} + } + code = append(code, 1, 0, 0, 0, 0xc3) + var value int64 + for index := 0; index < width; index++ { + b := byte(0x21 + index*0x13) + code = append(code, b) + value |= int64(b) << (8 * index) + } + return code, value +} + +func TestDecodeX86RawScalarFloatBroadcastRIPLiteral(t *testing.T) { + for _, test := range []struct { + double, evex bool + vectorBits int + mask int + zeroing bool + }{ + {vectorBits: 128}, {vectorBits: 256}, {double: true, vectorBits: 256}, + {evex: true, vectorBits: 128}, {evex: true, vectorBits: 256, mask: 3}, + {evex: true, vectorBits: 512, mask: 7, zeroing: true}, + {double: true, evex: true, vectorBits: 256}, + {double: true, evex: true, vectorBits: 512, mask: 7, zeroing: true}, + } { + name := fmt.Sprintf("double%t/evex%t/%d/mask%d/zero%t", test.double, test.evex, test.vectorBits, test.mask, test.zeroing) + t.Run(name, func(t *testing.T) { + code, value := x86RawScalarFloatBroadcastLiteral(test.double, test.evex, test.vectorBits, test.mask, test.zeroing) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + op := Op("VBROADCASTSS") + if test.double { + op = "VBROADCASTSD" + } + if test.zeroing { + op += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != op || + decoded[0].Args[0].Kind != OpImm || decoded[0].Args[0].Imm != value || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want scalar broadcast of literal", code, decoded) + } + }) + } +} + +func TestTranslateX86RawScalarFloatBroadcastRIPLiteralObjects(t *testing.T) { + var source strings.Builder + for index, test := range []struct { + double, evex bool + vectorBits int + mask int + zeroing bool + }{ + {vectorBits: 128}, {vectorBits: 256}, {double: true, vectorBits: 256}, + {evex: true, vectorBits: 128}, {evex: true, vectorBits: 256, mask: 3}, + {evex: true, vectorBits: 512, mask: 7, zeroing: true}, + {double: true, evex: true, vectorBits: 256}, + {double: true, evex: true, vectorBits: 512, mask: 7, zeroing: true}, + } { + name := fmt.Sprintf("scalarFloatLiteral%d", index) + code, _ := x86RawScalarFloatBroadcastLiteral(test.double, test.evex, test.vectorBits, test.mask, test.zeroing) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Preserve all original masked/unmasked formats without inventing + // missing K or merge bytes in the complete source. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "scalar-float-literal.ll", "scalar-float-literal.o", ir) + }) + } +} + +func TestDecodeX86RawScalarFloatBroadcastRIPLiteralRejectsUnsafeSources(t *testing.T) { + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "instruction_overlap", edit: func(code []byte) []byte { + copy(code[6:10], []byte{0xfa, 0xff, 0xff, 0xff}) + return code + }}, + {name: "outside_group", edit: func(code []byte) []byte { + code[6] = 0x20 + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x65}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code, _ := x86RawScalarFloatBroadcastLiteral(true, true, 512, 0, false) + code = test.edit(code) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("folded unsafe scalar broadcast source: %x", code) + } + }) + } +} diff --git a/amd64_raw_byte_shift.go b/amd64_raw_byte_shift.go new file mode 100644 index 000000000..31bd4db48 --- /dev/null +++ b/amd64_raw_byte_shift.go @@ -0,0 +1,74 @@ +package plan9asm + +import "fmt" + +// decodedX86RawVectorByteShiftInstruction covers Go's VPSLLDQ/VPSRLDQ +// _yvpslldq table: VEX X/Y register forms and EVEX X/Y/Z register or memory +// forms, all with an imm8. ModRM.reg is the opcode extension (/7 or /3), +// while VEX.vvvv or EVEX.vvvvv names the destination. +func decodedX86RawVectorByteShiftInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86RawVectorByteShiftInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86RawVectorByteShiftInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || p.pp != 1 || p.opcode != 0x73 { + return Instr{}, 0, false, nil + } + if len(code) <= p.modRM { + return Instr{}, 0, true, fmt.Errorf("vector byte shift: missing ModRM byte") + } + group := int(code[p.modRM]>>3) & 7 + if group != 3 && group != 7 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("vector byte shift: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride || p.r != 0 || p.vectorLength > 2 || + p.evex && !p.fixed || p.mask != 0 || p.zero || p.broadcast { + return fail("invalid prefix, mask or vector length") + } + registerSource := code[p.modRM]>>6 == 3 + if !p.evex && !registerSource { + return fail("VEX byte shift requires a register source") + } + if mode == 32 && (p.upper >= 8 || registerSource && (p.b != 0 || p.x != 0)) { + return fail("extended vector register in 32-bit mode") + } + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, width, 16<= 8) { + return fail("extended register in 32-bit mode") + } + op := Op(stem + "L") + if p.w && mode == 64 { + op = Op(stem + "Q") + } + source, size, err := memory.scalar(code[p.modRM:], mode, p.b, p.x, p.segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + index, _ := decodedX86GeneralRegister(p.upper) + destination, _ := decodedX86GeneralRegister(int(code[p.modRM]>>3&7) + p.r*8) + return Instr{ + Op: op, + Args: []Operand{ + {Kind: OpReg, Reg: index}, source, {Kind: OpReg, Reg: destination}, + }, + Raw: fmt.Sprintf("%s %s, %s, %s", op, index, source.String(), destination), + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_bzhi_test.go b/amd64_raw_bzhi_test.go new file mode 100644 index 000000000..87c9eeb86 --- /dev/null +++ b/amd64_raw_bzhi_test.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "reflect" + "testing" +) + +func encodeX86RawBZHI(wide bool, source, index, destination int) []byte { + code := encodeRawScalarMove(false, 0, 0xf5, source, index, destination, 0, false) + code[1] = code[1]&0xe0 | 2 + if wide { + code[2] |= 0x80 + } + return code +} + +func TestDecodedX86RawBZHICompleteRegisterFamily(t *testing.T) { + count := 0 + for _, wide := range []bool{false, true} { + for source := 0; source < 16; source++ { + for index := 0; index < 16; index++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawBZHI(wide, source, index, destination) + got, length, ok, err := decodedX86RawBZHIInstruction(code, 64) + wantOp := Op("BZHIL") + if wide { + wantOp = "BZHIQ" + } + sr, _ := decodedX86GeneralRegister(source) + ir, _ := decodedX86GeneralRegister(index) + dr, _ := decodedX86GeneralRegister(destination) + want := []Operand{ + {Kind: OpReg, Reg: ir}, + {Kind: OpReg, Reg: sr}, + {Kind: OpReg, Reg: dr}, + } + if err != nil || !ok || length != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x = %+v length=%d ok=%t err=%v, want %s %+v", code, got, length, ok, err, wantOp, want) + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d BZHI register forms, want 8192", count) + } +} + +func TestDecodedX86RawBZHIRealMemoryAndInvalidForms(t *testing.T) { + code := []byte{0xc4, 0x62, 0xf0, 0xf5, 0x04, 0xc6} + got, length, ok, err := decodedX86RawBZHIInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != "BZHIQ" || + len(got.Args) != 3 || got.Args[0].String() != "CX" || + got.Args[1].String() != "0(SI)(AX*8)" || got.Args[2].String() != "R8" { + t.Fatalf("decode real simd BZHI %x = %+v length=%d ok=%t err=%v", code, got, length, ok, err) + } + for name, edit := range map[string]func([]byte) []byte{ + "L1": func(b []byte) []byte { b[2] |= 4; return b }, + "address override": func(b []byte) []byte { return append([]byte{0x67}, b...) }, + "truncated ModRM": func(b []byte) []byte { return b[:4] }, + } { + t.Run(name, func(t *testing.T) { + bad := edit(append([]byte(nil), code...)) + if _, _, matched, err := decodedX86RawBZHIInstruction(bad, 64); !matched || err == nil { + t.Fatalf("accepted invalid BZHI %x, matched=%t err=%v", bad, matched, err) + } + }) + } + badPP := append([]byte(nil), code...) + badPP[2] |= 1 + if _, _, matched, _ := decodedX86RawBZHIInstruction(badPP, 64); matched { + t.Fatalf("accepted non-BZHI pp encoding %x", badPP) + } +} + +func TestTranslateX86RawBZHIRealMemoryObjects(t *testing.T) { + code := []byte{0xc4, 0x62, 0xf0, 0xf5, 0x04, 0xc6, 0xc3} + file := x86RawUnprovedReturnFormProbeFile(t, "amd64", "rawBZHI", code) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // The original physical memory/index inputs are not initialized. + // Keep Go objects and public Context separately from emitter objects. + ir := x86UnboundVoidRawProbeObject(t, file, "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-bzhi.ll", "raw-bzhi.o", ir) + }) + } +} diff --git a/amd64_raw_clmul.go b/amd64_raw_clmul.go new file mode 100644 index 000000000..b2c9e15a2 --- /dev/null +++ b/amd64_raw_clmul.go @@ -0,0 +1,138 @@ +package plan9asm + +import "fmt" + +// decodedX86RawCLMULInstruction covers Go's complete PCLMULQDQ and +// VPCLMULQDQ tables: legacy X, VEX X/Y, and EVEX X/Y/Z, each with imm8. +// The vector ISA has no mask or broadcast, but W is ignored. Go selects W0 +// and limits its named four-operand frontend to amd64; neither restriction +// invalidates original low-bank WIG bytes in physical 32-bit mode. +func decodedX86RawCLMULInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if matched && p.mapNumber == 3 && p.pp == 1 && p.opcode == 0x44 { + return decodedX86RawVectorCLMULInstruction(code, mode, p) + } + return decodedX86RawLegacyCLMULInstruction(code, mode) +} + +func decodedX86RawVectorCLMULInstruction(code []byte, mode int, p x86RawVectorEncoding) (Instr, int, bool, error) { + return decodedX86RawVectorCLMULInstructionWithMemory(code, mode, p, x86RawMemoryDecoder{}) +} + +func decodedX86RawVectorCLMULInstructionWithMemory(code []byte, mode int, p x86RawVectorEncoding, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VPCLMULQDQ: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride || p.vectorLength > 2 || + p.evex && (!p.fixed || p.mask != 0 || p.zero || p.broadcast) { + return fail("invalid prefix, mask, broadcast or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + if mode == 32 && (p.r != 0 || p.upper >= 8 || p.b != 0 || p.x != 0) { + return fail("extended register is unavailable in physical 32-bit mode") + } + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = memory.evex( + code[p.modRM:], mode, p.b, p.x, p.segment, width, 16<>3&7)+p.r*8))} + imm := Operand{Kind: OpImm, Imm: int64(code[immIndex])} + return Instr{ + Op: "VPCLMULQDQ", + Args: []Operand{imm, source, second, destination}, + Raw: fmt.Sprintf("VPCLMULQDQ %s, %s, %s, %s", imm.String(), source.String(), second.String(), destination.String()), + x86Encoded: true, + }, immIndex + 1, true, nil +} + +func decodedX86RawLegacyCLMULInstruction(code []byte, mode int) (Instr, int, bool, error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + case 0x65: + segment = GS + case 0x67: + addressOverride = true + default: + goto encoding + } + i++ + } + +encoding: + if len(code) <= i || code[i] != 0x66 { + return Instr{}, 0, false, nil + } + i++ + rex := byte(0) + if i < len(code) && code[i]&0xf0 == 0x40 { + rex = code[i] + i++ + } + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0x3a || code[i+2] != 0x44 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("PCLMULQDQ: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if addressOverride || mode == 32 && rex != 0 { + return fail("address override or REX prefix is invalid") + } + modRMIndex := i + 3 + if len(code) <= modRMIndex { + return fail("missing ModRM byte") + } + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + int(rex>>2&1)*8 + if mode == 32 && destinationNumber >= 8 { + return fail("high X register is unavailable in 386 mode") + } + source, consumed, err := decodedX86VEXVectorRMOperand( + code[modRMIndex:], mode, int(rex&1), int(rex>>1&1), segment, "X", + ) + if err != nil { + return Instr{}, 0, true, err + } + immIndex := modRMIndex + consumed + if len(code) <= immIndex { + return fail("missing imm8") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + imm := Operand{Kind: OpImm, Imm: int64(code[immIndex])} + return Instr{ + Op: "PCLMULQDQ", + Args: []Operand{imm, source, destination}, + Raw: fmt.Sprintf("PCLMULQDQ %s, %s, %s", imm.String(), source.String(), destination.String()), + x86Encoded: true, + }, immIndex + 1, true, nil +} diff --git a/amd64_raw_clmul_test.go b/amd64_raw_clmul_test.go new file mode 100644 index 000000000..345bc95e2 --- /dev/null +++ b/amd64_raw_clmul_test.go @@ -0,0 +1,259 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawCLMULCode(encoding string, width int, source, second, destination int, immediate byte) []byte { + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + switch encoding { + case "legacy": + code := []byte{0x66} + rex := byte(0x40 | (destination>>3&1)<<2 | source>>3&1) + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0x3a, 0x44, modRM, immediate) + case "vex": + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-source/8)<<5 | 3) + p1 := byte(((^second)&15)<<3 | ((width-16)/16)<<2 | 1) + return []byte{0xc4, p0, p1, 0x44, modRM, immediate} + default: + p0 := byte((1-destination/8&1)<<7 | (1-source/16&1)<<6 | + (1-source/8&1)<<5 | (1-destination/16&1)<<4 | 3) + p1 := byte(((^second)&15)<<3 | 0x05) + p2 := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]<<5 | (1-second/16)<<3) + return []byte{0x62, p0, p1, p2, 0x44, modRM, immediate} + } +} + +func x86RawCLMULLiteral(encoding string, width int) []byte { + code := x86RawCLMULCode(encoding, width, 0, 1, 2, 0x11) + modRMIndex := len(code) - 2 + code[modRMIndex] = 0x15 // RIP+disp32, destination X/Y/Z2. + code = append(code[:modRMIndex+1], append([]byte{1, 0, 0, 0}, code[modRMIndex+1:]...)...) + code = append(code, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*9+7)) + } + return code +} + +func TestDecodeX86RawCLMULCompleteRegisterFamily(t *testing.T) { + for _, encoding := range []string{"legacy", "vex", "evex"} { + widths := []int{16} + if encoding != "legacy" { + widths = append(widths, 32) + } + if encoding == "evex" { + widths = append(widths, 64) + } + for _, width := range widths { + limit := 16 + if encoding == "evex" { + limit = 32 + } + for _, source := range []int{0, 7, 8, limit - 1} { + for _, second := range []int{0, 7, limit - 1} { + if encoding == "legacy" && second != 0 { + continue + } + for _, destination := range []int{0, 7, limit - 1} { + name := fmt.Sprintf("%s/%d/%d/%d/%d", encoding, width, source, second, destination) + t.Run(name, func(t *testing.T) { + code := x86RawCLMULCode(encoding, width, source, second, destination, 0x11) + got, length, ok, err := decodedX86RawCLMULInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + prefix := "X" + if width == 32 { + prefix = "Y" + } else if width == 64 { + prefix = "Z" + } + want := fmt.Sprintf("VPCLMULQDQ $17, %s%d, %s%d, %s%d", prefix, source, prefix, second, prefix, destination) + if encoding == "legacy" { + want = fmt.Sprintf("PCLMULQDQ $17, X%d, X%d", source, destination) + } + if got.Raw != want { + t.Fatalf("decode %x = %q, want %q", code, got.Raw, want) + } + }) + } + } + } + } + } +} + +func TestDecodeX86RawCLMULChecksumAVX2Regression(t *testing.T) { + code := []byte{0xc4, 0xe3, 0x61, 0x44, 0xf4, 0x00, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "checksum crc32c AVX2", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Raw != "VPCLMULQDQ $0, X4, X3, X6 /* decoded from checksum crc32c AVX2 */" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawCLMULMemoryForms(t *testing.T) { + for _, encoding := range []string{"legacy", "vex", "evex"} { + code := x86RawCLMULCode(encoding, 16, 0, 1, 2, 0x11) + modRMIndex := len(code) - 2 + code[modRMIndex] = 0x54 // disp8(SIB), destination X2. + code = append(code[:modRMIndex+1], append([]byte{0x88, 0x20}, code[modRMIndex+1:]...)...) + got, length, ok, err := decodedX86RawCLMULInstruction(code, 64) + wantOffset := int64(32) + if encoding == "evex" { + wantOffset = 512 // EVEX disp8 scales by the 128-bit memory operand. + } + if err != nil || !ok || length != len(code) || got.Args[1].Kind != OpMem || + got.Args[1].Mem.Base != AX || got.Args[1].Mem.Index != CX || + got.Args[1].Mem.Scale != 4 || got.Args[1].Mem.Off != wantOffset { + t.Fatalf("%s memory source %x decoded as %+v, length=%d, ok=%v, err=%v", encoding, code, got, length, ok, err) + } + } +} + +func TestDecodeX86RawCLMULRIPDataCompleteFamily(t *testing.T) { + for _, encoding := range []string{"legacy", "vex", "evex"} { + widths := []int{16} + if encoding != "legacy" { + widths = append(widths, 32) + } + if encoding == "evex" { + widths = append(widths, 64) + } + for _, width := range widths { + name := fmt.Sprintf("%s/%d", encoding, width) + t.Run(name, func(t *testing.T) { + code := x86RawCLMULLiteral(encoding, width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Args[0].Imm != 0x11 || + decoded[0].Args[1].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET || + !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-width:]) { + t.Fatalf("decoded CLMUL literal %x as %#v", code, decoded) + } + }) + } + } +} + +func TestDecodeX86RawCLMULRIPDataRejectsUnsafeSources(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { code[6] = 127; return code }, + "overlapping instruction": func(code []byte) []byte { code[6] = 0; return code }, + "segment override": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawCLMULLiteral("evex", 64)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe CLMUL literal %x", code) + } + }) + } +} + +func TestDecodeX86RawCLMULRejectsReservedForms(t *testing.T) { + valid := x86RawCLMULCode("evex", 64, 0, 1, 2, 0x11) + for name, mutate := range map[string]func([]byte) []byte{ + "mask": func(code []byte) []byte { code[3] |= 1; return code }, + "zeroing": func(code []byte) []byte { code[3] |= 0x80; return code }, + "broadcast": func(code []byte) []byte { code[3] |= 0x10; return code }, + "width": func(code []byte) []byte { code[3] |= 0x60; return code }, + "fixed bit": func(code []byte) []byte { code[2] &^= 4; return code }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + "missing immediate": func(code []byte) []byte { return code[:len(code)-1] }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, _, ok, err := decodedX86RawCLMULInstruction(code, 64); !ok || err == nil { + t.Fatalf("accepted reserved CLMUL encoding %x: ok=%v err=%v", code, ok, err) + } + }) + } +} + +func TestTranslateX86RawCLMULObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, encoding := range []string{"legacy", "vex", "evex"} { + width := []int{16, 32, 64}[index] + name := fmt.Sprintf("rawCLMUL%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawCLMULCode(encoding, width, 0, 1, 2, 0x11) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + source.WriteString("\tRET\n") + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-clmul.ll", "raw-clmul.o", ir) + }) + } +} + +func TestTranslateX86RawCLMULRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, encoding := range []string{"legacy", "vex", "evex"} { + width := []int{16, 32, 64}[index] + name := fmt.Sprintf("literalCLMUL%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawCLMULLiteral(encoding, width) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-clmul-literal.ll", "raw-clmul-literal.o", ir) + }) + } +} diff --git a/amd64_raw_compare_float_test.go b/amd64_raw_compare_float_test.go new file mode 100644 index 000000000..18ffcf829 --- /dev/null +++ b/amd64_raw_compare_float_test.go @@ -0,0 +1,373 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Go's yxcmpi table uses X/m, X, signed-imm8 for all four legacy +// floating compares. Intel decoding instead returns destination, source, +// unsigned predicate; a generic operand reversal is not the Go grammar. +func TestX86RawLegacyFloatingCompareCompleteForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var named, raw strings.Builder + named.WriteString("TEXT compareforms(SB),4,$0-0\n") + raw.WriteString("TEXT compareforms(SB),4,$0-0\n") + var expected []byte + for _, family := range []struct { + op string + prefix []byte + }{{"CMPPS", nil}, {"CMPPD", []byte{0x66}}, {"CMPSS", []byte{0xf3}}, {"CMPSD", []byte{0xf2}}} { + forms := []struct { + source, destination string + rex byte + address []byte + }{{"X1", "X0", 0, []byte{0xc1}}, {"-7(BX)(CX*4)", "X7", 0, []byte{0x7c, 0x8b, 0xf9}}} + if arch == "amd64" { + forms = append(forms, struct { + source, destination string + rex byte + address []byte + }{"X15", "X14", 0x45, []byte{0xf7}}) + } + for _, form := range forms { + for predicate := 0; predicate < 256; predicate++ { + code := append([]byte(nil), family.prefix...) + if form.rex != 0 { + code = append(code, form.rex) + } + code = append(code, 0x0f, 0xc2) + code = append(code, form.address...) + code = append(code, byte(predicate)) + instruction := fmt.Sprintf("%s %s, %s, $%d", family.op, form.source, form.destination, int8(predicate)) + fmt.Fprintln(&named, instruction) + expected = append(expected, code...) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + want, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\n") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != want.Funcs[0].Instrs[1].Op || !reflect.DeepEqual(decoded.Instrs[0].Args, want.Funcs[0].Instrs[1].Args) { + t.Fatalf("decoded %x as %+v, want %s", code, decoded.Instrs, instruction) + } + // All encodings are checked against Go above; compiling the + // low predicates and signed boundary cases exercises every + // semantic selector without bloating every target's IR. + if predicate < 8 || predicate == 127 || predicate == 128 || predicate == 255 { + for _, b := range code { + fmt.Fprintf(&raw, "BYTE $%#02x\n", b) + } + } + } + } + } + named.WriteString("RET\n") + raw.WriteString("RET\n") + expected = append(expected, 0xc3) + if got := assembleX87ControlBytes(t, arch, named.String()); !bytes.HasPrefix(got, expected) { + t.Fatalf("Go %s compare encodings disagree: got %d bytes, want prefix of %d bytes", arch, len(got), len(expected)) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + ir := x86UnboundVoidRawScalarFormatObject(t, raw.String(), arch, triple) + compileLLVMToObject(t, llc, triple, "raw-compares.ll", "raw-compares.o", ir) + } + }) + } +} + +func TestX86RawLegacyFloatingCompareRejectsMalformedPredicate(t *testing.T) { + for _, predicate := range []x86asm.Arg{nil, x86asm.X0, x86asm.Imm(-1), x86asm.Imm(256)} { + inst := x86asm.Inst{Op: x86asm.CMPPS, Mode: 64, Args: x86asm.Args{x86asm.X0, x86asm.X1, predicate}} + if _, err := decodedX86GoSyntax(inst, nil); err == nil { + t.Fatalf("accepted malformed decoded predicate %v", predicate) + } + } +} + +func TestCrossLinuxRuntimeMatrixX86RawLegacyFloatingCompareRuntime(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, functions, originals, originalFunctions, goOpcodes strings.Builder + goOpcodes.WriteString("TEXT originalopcodes(SB),4,$0-0\n") + var expectedOpcodes []byte + sigs := make(map[string]FuncSig) + for family, prefix := range [][]byte{nil, {0x66}, {0xf3}, {0xf2}} { + for memory := 0; memory < 2; memory++ { + for predicate := 0; predicate < 256; predicate++ { + name := fmt.Sprintf("rawcmp_%d_%d_%d", family, memory, predicate) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),DI\nMOVQ left+8(FP),AX\nMOVQ right+16(FP),BX\nMOVOU (AX),X0\nMOVOU (BX),X1\n", name) + code := append([]byte(nil), prefix...) + modrm := byte(0xc1) + if memory == 1 { + modrm = 0x03 // X0, (BX) + } + code = append(code, 0x0f, 0xc2, modrm, byte(predicate)) + expectedOpcodes = append(expectedOpcodes, code...) + sourceOperand := "X1" + if memory == 1 { + sourceOperand = "(BX)" + } + opcode := [...]string{"CMPPS", "CMPPD", "CMPSS", "CMPSD"}[family] + fmt.Fprintf(&goOpcodes, "%s %s,X0,$%d\n", opcode, sourceOperand, int8(predicate)) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + source.WriteString("MOVOU X0,(DI)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void *,const void *,const void *);\n", name) + fmt.Fprintf(&functions, "%s,\n", name) + originals.WriteString(x86RawLegacyFloatingCompareOriginalKernel(name, code)) + fmt.Fprintf(&originalFunctions, "original_%s,\n", name) + } + } + } + goOpcodes.WriteString("RET\n") + expectedOpcodes = append(expectedOpcodes, 0xc3) + if got := assembleX87ControlBytes(t, "amd64", goOpcodes.String()); !bytes.HasPrefix(got, expectedOpcodes) { + t.Fatalf("actual Go full imm8 register/(BX) opcode bytes differ: got=%d expected prefix=%d", len(got), len(expectedOpcodes)) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +#include +#ifndef _WIN32 +#include +#include +#include +#include +#endif +` + declarations.String() + ` +typedef void (*comparison)(void *,const void *,const void *); +static comparison functions[]={ +` + functions.String() + `}; +` + originals.String() + ` +static comparison originals[]={ +` + originalFunctions.String() + `}; + +// Only predicates 0..7 have the legacy IEEE truth table below. Bits 3..7 +// are reserved: an observed machine result is not a mathematical definition. +static int predicate(double a,double b,int p) { + int unordered=isnan(a)||isnan(b); + switch(p) { + case 0: return !unordered&&a==b; + case 1: return !unordered&&a=b; + case 6: return unordered||a>b; + case 7: return !unordered; + default: return -1; + } +} + +static unsigned observe(comparison fn,void *out,const void *left,const void *right) { + unsigned control=0x1f80,observed; + __asm__ volatile("ldmxcsr %0"::"m"(control):"memory"); + fn(out,left,right); + __asm__ volatile("stmxcsr %0":"=m"(observed)::"memory"); + return observed; +} + +#ifndef _WIN32 +// Preserve the original packed memory opcode's unaligned behavior. The +// outcome is an observation of this execution environment, not a new policy. +static int same_unaligned_outcome(comparison actual,comparison original) { + int statuses[2]; + for(int which=0;which<2;which++) { + pid_t child=fork(); + if(child<0) return 1; + if(child==0) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + _Alignas(16) unsigned char left[32]={0},right[32]={0},out[16]; + observe(which?original:actual,out,left,right+1); + _exit(0); + } + if(waitpid(child,&statuses[which],0)!=child) return 1; + if(WIFEXITED(statuses[which])) { + if(WEXITSTATUS(statuses[which])!=0) return 1; + } else if(!WIFSIGNALED(statuses[which]) || + (WTERMSIG(statuses[which])!=SIGSEGV && WTERMSIG(statuses[which])!=SIGBUS)) { + return 1; + } + } + if(WIFSIGNALED(statuses[0])!=WIFSIGNALED(statuses[1]) || + (WIFSIGNALED(statuses[1]) && WTERMSIG(statuses[0])!=WTERMSIG(statuses[1]))) { + fprintf(stderr,"unaligned original opcode outcome differs actual=%x original=%x\n",statuses[0],statuses[1]); + return 1; + } + return 0; +} +#endif + +int main(void) { + uint32_t f[]={0,0x80000000u,0x3f800000u,0xbf800000u,0x7f800000u,0xff800000u,0x7fc12345u}; + uint64_t d[]={0,0x8000000000000000ull,0x3ff0000000000000ull,0xbff0000000000000ull,0x7ff0000000000000ull,0xfff0000000000000ull,0x7ff8123456789abcull}; + for(int family=0;family<4;family++) { + for(int mem=0;mem<2;mem++) { + for(int p=0;p<256;p++) { + for(int a=0;a<7;a++) { + for(int b=0;b<7;b++) { + _Alignas(16) unsigned char left_storage[64],right_storage[64]; + unsigned char actual_storage[48],original_storage[48],expected[16]; + unsigned char left_before[64],right_before[64]; + memset(left_storage,0xa7,sizeof(left_storage)); + memset(right_storage,0xb9,sizeof(right_storage)); + memset(actual_storage,0xc3,sizeof(actual_storage)); + memset(original_storage,0xd5,sizeof(original_storage)); + unsigned char *left=left_storage+16; + // Packed memory remains aligned; scalar memory additionally + // exercises a real unaligned address without inventing a shadow. + unsigned char *right=right_storage+16+(family>=2 && mem?3:0); + unsigned char *got=actual_storage+16,*observed=original_storage+16; + int bytes=family&1?8:4,lanes=16/bytes; + for(int lane=0;lane=2?1:lanes);lane++) { + double x,y; + if(bytes==8) { + memcpy(&x,left+lane*bytes,8); + memcpy(&y,right+lane*bytes,8); + } else { + float xf,yf; + memcpy(&xf,left+lane*bytes,4); + memcpy(&yf,right+lane*bytes,4); + x=xf; + y=yf; + } + memset(expected+lane*bytes,predicate(x,y,p)?255:0,bytes); + } + if(memcmp(got,expected,16)) { + fprintf(stderr,"independent IEEE mismatch family=%d mem=%d predicate=%d a=%d b=%d\n",family,mem,p,a,b); + return 4; + } + } + } + } + } +#ifndef _WIN32 + if(mem && family<2) { + const int predicates[]={0,8,255}; + for(int axis=0;axis<3;axis++) { + int p=predicates[axis]; + int index=(family*2+mem)*256+p; + if(same_unaligned_outcome(functions[index],originals[index])) return 5; + } + } +#endif + } + } + return 0; +} +` + execution.compileAndRunLLVM(t, "raw_compare_float", ir, mainC, "-O2") + t.Logf("2048 exact original imm8 opcodes ×49 inputs: full X bytes, canaries and MXCSR agree; only 0..7 have independent IEEE truth; transport=%v", execution.prefix) +} + +func x86RawLegacyFloatingCompareOriginalKernel(name string, code []byte) string { + var directive strings.Builder + for index, b := range code { + if index != 0 { + directive.WriteByte(',') + } + fmt.Fprintf(&directive, "0x%02x", b) + } + return fmt.Sprintf(` +static __attribute__((noinline)) void original_%s(void *out,const void *left,const void *right) { + __asm__ volatile( + "movdqu (%%1),%%%%xmm0\n\t" + "movdqu (%%%%rbx),%%%%xmm1\n\t" + ".byte %s\n\t" + "movdqu %%%%xmm0,(%%0)" + ::"r"(out),"r"(left),"b"(right) + :"xmm0","xmm1","memory"); +} +`, name, directive.String()) +} + +func TestX86RawLegacyFloatingCompareOriginalKernelRetainsReservedBytes(t *testing.T) { + for _, prefix := range [][]byte{nil, {0x66}, {0xf3}, {0xf2}} { + for _, modrm := range []byte{0xc1, 0x03} { + for predicate := 0; predicate < 256; predicate++ { + code := append([]byte(nil), prefix...) + code = append(code, 0x0f, 0xc2, modrm, byte(predicate)) + kernel := x86RawLegacyFloatingCompareOriginalKernel("byte_identity", code) + var exact strings.Builder + for index, b := range code { + if index != 0 { + exact.WriteByte(',') + } + fmt.Fprintf(&exact, "0x%02x", b) + } + if strings.Count(kernel, ".byte ") != 1 || !strings.Contains(kernel, ".byte "+exact.String()+`\n\t`) { + t.Fatalf("original kernel altered source bytes %x:\n%s", code, kernel) + } + for _, required := range []string{`movdqu (%1),%%xmm0`, `movdqu (%%rbx),%%xmm1`, `movdqu %%xmm0,(%0)`, `"b"(right)`, `"memory"`} { + if !strings.Contains(kernel, required) { + t.Fatalf("original register/memory/output binding lost %q: %x", required, code) + } + } + if predicate >= 8 { + folded := append([]byte(nil), code...) + folded[len(folded)-1] &= 7 + if x86RawLegacyFloatingCompareOriginalKernel("byte_identity", folded) == kernel { + t.Fatalf("reserved imm8 %d folded to a mathematical predicate", predicate) + } + } + } + } + } +} diff --git a/amd64_raw_compare_float_transport_test.go b/amd64_raw_compare_float_transport_test.go new file mode 100644 index 000000000..8adba7148 --- /dev/null +++ b/amd64_raw_compare_float_transport_test.go @@ -0,0 +1,49 @@ +package plan9asm + +import ( + "go/ast" + "go/parser" + "go/token" + "testing" +) + +func TestX86RawLegacyFloatingCompareRuntimeRequiresOwnedGuestTransport(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "amd64_raw_compare_float_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + const required = "TestCrossLinuxRuntimeMatrixX86RawLegacyFloatingCompareRuntime" + var body *ast.BlockStmt + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == required { + body = function.Body + } + } + if body == nil { + t.Fatal("raw compare runtime is not registered in the required cross-runtime matrix") + } + + tools, consumer := 0, 0 + ast.Inspect(body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if function, ok := call.Fun.(*ast.Ident); ok && function.Name == "x86FloatCompareExecutionTools" { + tools++ + } + if method, ok := call.Fun.(*ast.SelectorExpr); ok { + if method.Sel.Name == "Skip" || method.Sel.Name == "Skipf" || method.Sel.Name == "SkipNow" { + t.Fatal("required raw compare execution cannot substitute a host skip for its guest counterpart") + } + if receiver, ok := method.X.(*ast.Ident); ok && receiver.Name == "execution" && method.Sel.Name == "compileAndRunLLVM" { + consumer++ + } + } + return true + }) + if tools != 1 || consumer != 1 { + t.Fatalf("required raw comparison must consume its authenticated tools and execution route exactly once: tools=%d consumer=%d", tools, consumer) + } +} diff --git a/amd64_raw_compress_test.go b/amd64_raw_compress_test.go new file mode 100644 index 000000000..6c36aee59 --- /dev/null +++ b/amd64_raw_compress_test.go @@ -0,0 +1,157 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawCompressForms = []struct { + opcode int + widthBit int + op Op + laneBytes int +}{ + {0x63, 0, "VPCOMPRESSB", 1}, + {0x63, 1, "VPCOMPRESSW", 2}, + {0x8a, 0, "VCOMPRESSPS", 4}, + {0x8a, 1, "VCOMPRESSPD", 8}, + {0x8b, 0, "VPCOMPRESSD", 4}, + {0x8b, 1, "VPCOMPRESSQ", 8}, +} + +func encodeX86RawCompress(opcode, widthBit, vectorBits, mask int, zeroing, memory bool) []byte { + p1 := byte(0x7d) // EVEX.vvvv is unused, pp = 66. + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xd3) // source register 2, destination register 3. + if memory { + modRM = 0x50 // source register 2, disp8(AX) destination. + } + code := []byte{0x62, 0xf2, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawCompressSeBiShogunRegression(t *testing.T) { + // Go disassembles the bytes as VCOMPRESSPS Z0, K1, 0(SI)(R9*4). + code := []byte{0x62, 0xb2, 0x7d, 0x49, 0x8a, 0x04, 0x8e, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "compressBitsFloat32 AVX512 VCOMPRESSPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VCOMPRESSPS" || + decoded[0].Args[0].Reg != "Z0" || decoded[0].Args[1].Reg != "K1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawCompressCompleteGoForms(t *testing.T) { + if len(x86RawCompressForms) != len(amd64PackedCompressLaneBits) { + t.Fatalf("raw/lower compress grammar rows = %d/%d", len(x86RawCompressForms), len(amd64PackedCompressLaneBits)) + } + for _, form := range x86RawCompressForms { + for vectorBits, vector := range []string{"X", "Y", "Z"} { + for _, variant := range []struct { + name string + mask int + zeroing bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "zeroing register", mask: 7, zeroing: true}, + {name: "memory", memory: true}, + {name: "masked memory", mask: 1, memory: true}, + } { + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawCompress(form.opcode, form.widthBit, vectorBits, variant.mask, variant.zeroing, variant.memory) + got, length, ok, err := decodedX86PackedCompressInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[0].String() != vector+"2" || + !variant.memory && got.Args[len(got.Args)-1].String() != vector+"3" || + variant.memory && got.Args[len(got.Args)-1].String() != fmt.Sprintf("%d(AX)", form.laneBytes) || + len(got.Args) != 2+map[bool]int{true: 1}[variant.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawCompressRejectsInvalidForms(t *testing.T) { + base := encodeX86RawCompress(0x8a, 0, 2, 0, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x80 }, // zeroing without mask + func(code []byte) { code[3] |= 0x60 }, // reserved vector length + func(code []byte) { code[3] |= 0x10 }, // broadcast unavailable + func(code []byte) { code[2] &^= 0x08 }, // EVEX.vvvv is reserved + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86PackedCompressInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + badMode32 := encodeX86RawCompress(0x8a, 0, 2, 1, false, false) + badMode32[1] &^= 0x80 // The source field has no physical mode32 register. + if _, _, ok, err := decodedX86PackedCompressInstruction(badMode32, 32); !ok || err == nil { + t.Fatalf("accepted unbound mode32 source bank: ok=%v err=%v", ok, err) + } +} + +func TestTranslateX86RawCompressLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawCompress(SB),$0-0\n") + for _, form := range x86RawCompressForms { + for _, memory := range []bool{false, true} { + mask := 1 + if target.goarch == "386" { + mask = 0 + } + code := encodeX86RawCompress(form.opcode, form.widthBit, 0, mask, false, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + // Keep the complete original encoding program, not fictitious + // source/mask/address inputs. Initialized raw counterparts have + // separate complete X/Y/Z public-object matrices. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-compress.ll", "raw-compress.o", ir) + }) + } +} diff --git a/amd64_raw_convert_half.go b/amd64_raw_convert_half.go new file mode 100644 index 000000000..0baabf3c2 --- /dev/null +++ b/amd64_raw_convert_half.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawHalfConversionForm struct { + op Op + mapNumber int + opcode int + immediate bool + rawOnly bool +} + +// The complete F16C/AVX-512 half/single conversion rows. The FP16 map5/map6 +// forms are raw-only in Go 1.27. Only legacy VCVTPS2PH reverses ModRM roles +// and appends imm8; FP16 narrowing instead loads and supports broadcasts. +var x86RawHalfConversionForms = [...]x86RawHalfConversionForm{ + {op: "VCVTPH2PS", mapNumber: 2, opcode: 0x13}, + {op: "VCVTPS2PH", mapNumber: 3, opcode: 0x1d, immediate: true}, + {op: "VCVTPH2PSX", mapNumber: 6, opcode: 0x13, rawOnly: true}, + {op: "VCVTPS2PHX", mapNumber: 5, opcode: 0x1d, rawOnly: true}, +} + +func decodedX86PackedHalfConversionInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || p.pp != 1 { + return Instr{}, 0, false, nil + } + var form x86RawHalfConversionForm + for _, candidate := range x86RawHalfConversionForms { + if p.mapNumber == candidate.mapNumber && p.opcode == candidate.opcode { + form = candidate + break + } + } + if form.op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed half conversion: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if form.rawOnly && !p.evex { + return fail("FP16 conversion requires EVEX") + } + if p.upper != 0 || p.w || p.evex && !p.fixed { + return fail("invalid reserved vvvv/V', W or EVEX fixed bit") + } + if p.evex && p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + registerSource := code[p.modRM]>>6 == 3 + vectorLength := p.vectorLength + op := form.op + if p.broadcast && form.rawOnly { + if registerSource { + vectorLength = 2 + if form.op == "VCVTPS2PHX" { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } else { + // LLVM emits LL=00 for SAE-only; LL does not encode VL here. + op += ".SAE" + } + } else { + op += ".BCST" + } + } + if vectorLength > 2 { + return fail("reserved vector length") + } + if form.rawOnly && mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register or address in 32-bit mode") + } + if !p.evex && mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended VEX register in 32-bit mode") + } + if p.evex && mode == 32 && code[p.modRM]>>6 != 3 && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + if p.broadcast && !form.rawOnly { + if !p.evex || p.vectorLength != 2 || form.op == "VCVTPH2PS" && code[p.modRM]>>6 != 3 { + return fail("SAE requires the EVEX Z-width register form") + } + op += ".SAE" + } + + vector := [...]string{"X", "Y", "Z"}[vectorLength] + shorter := "X" + if vectorLength == 2 { + shorter = "Y" + } + if p.zero { + op += ".Z" + } + modRM := code[p.modRM] + regNumber := int(modRM>>3&7) + p.r*8 + register := func(prefix string) Operand { + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, regNumber))} + } + decodeRM := func(prefix string) (Operand, int, error) { + if p.evex { + width := 8 << vectorLength + if form.op == "VCVTPS2PHX" { + width *= 2 + } + if form.rawOnly && p.broadcast && !registerSource { + width = amd64HalfConversionSpecs[string(form.op)].inputBits / 8 + } + return decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, prefix, width) + } + return decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, prefix) + } + + args := make([]Operand, 0, 4) + var consumed int + if !form.immediate { + sourcePrefix, destinationPrefix := shorter, vector + if form.op == "VCVTPS2PHX" { + sourcePrefix, destinationPrefix = vector, shorter + } + source, n, err := decodeRM(sourcePrefix) + if err != nil { + return Instr{}, 0, true, err + } + consumed = n + args = append(args, source) + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, register(destinationPrefix)) + } else { + destination, n, err := decodeRM(shorter) + if err != nil { + return Instr{}, 0, true, err + } + consumed = n + if len(code) <= p.modRM+consumed { + return fail("missing imm8 rounding control") + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[p.modRM+consumed])}) + args = append(args, register(vector)) + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + consumed++ + } + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + encoding := x86RawNamedHalfAbsent + if !form.rawOnly { + encoding = x86RawNamedHalfVEX + if p.evex { + encoding = x86RawNamedHalfEVEX + } + } + return Instr{ + Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86Encoded: form.rawOnly, x86VectorBytes: 16 << vectorLength, x86NamedHalfEncoding: encoding, + }, p.modRM + consumed, true, nil +} + +func decodeX86RawFP16ConversionRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, ok := decodeX86RawVectorEncoding(code[offset:]) + if !ok || !p.evex || p.pp != 1 || p.segment != "" || p.addressOverride || + !(p.mapNumber == 6 && p.opcode == 0x13 || p.mapNumber == 5 && p.opcode == 0x1d) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRM := offset + p.modRM + if len(code) <= modRM || code[modRM]&0xc7 != 5 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 8 << p.vectorLength + if p.broadcast { + width = 2 + } + if p.mapNumber == 5 { + width *= 2 + } + return x86RawRIPDataThroughDecoder(code, offset, mode, modRM, width, decodedX86PackedHalfConversionInstruction, "FP16 conversion") +} diff --git a/amd64_raw_convert_half_test.go b/amd64_raw_convert_half_test.go new file mode 100644 index 000000000..9a331ea84 --- /dev/null +++ b/amd64_raw_convert_half_test.go @@ -0,0 +1,241 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVEXHalfConversion(form x86RawHalfConversionForm, width256 bool, reg, rm int, memory bool, immediate byte) []byte { + if memory { + rm = 0 + } + l := byte(0) + if width256 { + l = 1 + } + p0 := byte((1-reg/8)<<7 | 1<<6 | (1-rm/8)<<5 | form.mapNumber) + p1 := byte(0x79 | l<<2) + modRM := byte(0xc0 | (reg&7)<<3 | rm&7) + if memory { + modRM = byte(0x40 | (reg&7)<<3) + } + code := []byte{0xc4, p0, p1, byte(form.opcode), modRM} + if memory { + code = append(code, 7) + } + if form.immediate { + code = append(code, immediate) + } + return code +} + +func encodeX86RawEVEXHalfConversion(form x86RawHalfConversionForm, vectorBits, mask int, zeroing, sae bool, reg, rm int, memory bool, immediate byte) []byte { + if memory { + rm = 0 + } + p0 := byte((1-reg/8&1)<<7 | (1-rm/16)<<6 | (1-rm/8&1)<<5 | (1-reg/16)<<4 | form.mapNumber) + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if sae { + p2 |= 0x10 + } + modRM := byte(0xc0 | (reg&7)<<3 | rm&7) + if memory { + modRM = byte(0x40 | (reg&7)<<3) + } + code := []byte{0x62, p0, 0x7d, p2, byte(form.opcode), modRM} + if memory { + code = append(code, 7) + } + if form.immediate { + code = append(code, immediate) + } + return code +} + +func TestTranslateRawHalfConversionGorseRegression(t *testing.T) { + const source = `TEXT rawHalfToSingle(SB), $0-0 + LONG $0x137da2c4 + WORD $0x474c + BYTE $0xf0 + RET +` + ir := x86NamedHalfOriginalFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("original Gorse half format requires LLVM22") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "gorse-half-format.ll", "gorse-half-format.o", ir) +} + +func TestDecodedX86PackedHalfConversionGorseEVEXMemory(t *testing.T) { + cases := []struct { + name string + code []byte + want MemRef + }{ + { + name: "SIB", + code: []byte{0x62, 0xf2, 0x7d, 0x48, 0x13, 0x0c, 0x4f}, + want: MemRef{Base: DI, Index: CX, Scale: 2}, + }, + { + name: "SIB compressed displacement", + code: []byte{0x62, 0xf2, 0x7d, 0x48, 0x13, 0x4c, 0x4f, 0x01}, + want: MemRef{Base: DI, Index: CX, Scale: 2, Off: 32}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, length, matched, err := decodedX86PackedHalfConversionInstruction(tc.code, 64) + if err != nil || !matched || length != len(tc.code) || got.Op != "VCVTPH2PS" { + t.Fatalf("decode %x = %+v length=%d matched=%v err=%v", tc.code, got, length, matched, err) + } + if len(got.Args) != 2 || got.Args[0].Kind != OpMem || got.Args[0].Mem != tc.want || got.Args[1].Reg != "Z1" { + t.Fatalf("decode %x = %+v, want memory %+v and Z1", tc.code, got, tc.want) + } + }) + } +} + +func TestDecodedX86PackedHalfConversionGoRowsAndWidths(t *testing.T) { + if len(x86RawHalfConversionForms) != 4 || + x86RawHalfConversionForms[0] != (x86RawHalfConversionForm{op: "VCVTPH2PS", mapNumber: 2, opcode: 0x13}) || + x86RawHalfConversionForms[1] != (x86RawHalfConversionForm{op: "VCVTPS2PH", mapNumber: 3, opcode: 0x1d, immediate: true}) { + t.Fatalf("packed half conversion grammar does not match the two Go 1.27 opcode rows") + } + for _, form := range x86RawHalfConversionForms[:2] { + for _, width256 := range []bool{false, true} { + vector := "X" + if width256 { + vector = "Y" + } + for _, memory := range []bool{false, true} { + code := encodeX86RawVEXHalfConversion(form, width256, 2, 1, memory, 7) + got, length, matched, err := decodedX86PackedHalfConversionInstruction(code, 64) + if err != nil || !matched || length != len(code) || got.Op != form.op { + t.Fatalf("VEX %x = %+v length=%d matched=%v err=%v", code, got, length, matched, err) + } + if form.immediate { + if len(got.Args) != 3 || got.Args[0].String() != "$7" || got.Args[1].String() != vector+"2" { + t.Fatalf("VEX single-to-half %x = %+v", code, got) + } + } else if len(got.Args) != 2 || got.Args[1].String() != vector+"2" { + t.Fatalf("VEX half-to-single %x = %+v", code, got) + } + } + } + for vectorBits, vector := range []string{"X", "Y", "Z"} { + for _, memory := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + code := encodeX86RawEVEXHalfConversion(form, vectorBits, mask, masked && !memory, false, 21, 20, memory, 9) + got, length, matched, err := decodedX86PackedHalfConversionInstruction(code, 64) + if err != nil || !matched || length != len(code) { + t.Fatalf("EVEX %x = %+v length=%d matched=%v err=%v", code, got, length, matched, err) + } + wantOp := form.op + if masked && !memory { + wantOp += ".Z" + } + if got.Op != wantOp { + t.Fatalf("EVEX %x op=%s, want %s", code, got.Op, wantOp) + } + registerIndex := 1 + if !form.immediate { + registerIndex = len(got.Args) - 1 + } + if got.Args[registerIndex].String() != fmt.Sprintf("%s21", vector) { + t.Fatalf("EVEX %x = %+v, lost high register", code, got) + } + } + } + } + } +} + +func TestDecodedX86PackedHalfConversionSAEAndInvalidForms(t *testing.T) { + for _, form := range x86RawHalfConversionForms[:2] { + code := encodeX86RawEVEXHalfConversion(form, 2, 2, true, true, 1, 2, false, 7) + got, length, matched, err := decodedX86PackedHalfConversionInstruction(code, 64) + if err != nil || !matched || length != len(code) || got.Op != form.op+".SAE.Z" { + t.Fatalf("SAE %x = %+v length=%d matched=%v err=%v", code, got, length, matched, err) + } + invalid := append([]byte(nil), code...) + invalid[2] |= 0x80 + if _, _, matched, err := decodedX86PackedHalfConversionInstruction(invalid, 64); !matched || err == nil { + t.Fatalf("W=1 encoding accepted: %x", invalid) + } + invalid = append([]byte(nil), code...) + invalid[3] &^= 7 + if _, _, matched, err := decodedX86PackedHalfConversionInstruction(invalid, 64); !matched || err == nil { + t.Fatalf("zeroing without mask accepted: %x", invalid) + } + } + toHalf := x86RawHalfConversionForms[1] + missingImmediate := encodeX86RawVEXHalfConversion(toHalf, false, 1, 2, false, 7) + missingImmediate = missingImmediate[:len(missingImmediate)-1] + if _, _, matched, err := decodedX86PackedHalfConversionInstruction(missingImmediate, 64); !matched || err == nil { + t.Fatalf("missing imm8 accepted: %x", missingImmediate) + } + fromHalf := x86RawHalfConversionForms[0] + invalidMemorySAE := encodeX86RawEVEXHalfConversion(fromHalf, 2, 0, false, true, 1, 0, true, 0) + if _, _, matched, err := decodedX86PackedHalfConversionInstruction(invalidMemorySAE, 64); !matched || err == nil { + t.Fatalf("half-to-single memory SAE accepted: %x", invalidMemorySAE) + } +} + +func TestTranslateRawPackedHalfConversionLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawPackedHalfConversions(SB), $0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for _, form := range x86RawHalfConversionForms[:2] { + for _, width256 := range []bool{false, true} { + appendBytes(encodeX86RawVEXHalfConversion(form, width256, 2, 1, false, 7)) + appendBytes(encodeX86RawVEXHalfConversion(form, width256, 2, 0, true, 7)) + } + for vectorBits := 0; vectorBits < 3; vectorBits++ { + mask := 3 + if target.goarch == "386" && form.immediate { + mask = 0 + } + reg, rm := 21, 20 + if target.goarch == "386" { + reg, rm = 2, 1 + } + appendBytes(encodeX86RawEVEXHalfConversion(form, vectorBits, mask, mask != 0, vectorBits == 2, reg, rm, false, 9)) + appendBytes(encodeX86RawEVEXHalfConversion(form, vectorBits, mask, false, false, 1, 0, true, 9)) + } + } + source.WriteString("\tRET\n") + ir := x86NamedHalfOriginalFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-half-conversions.ll", "raw-half-conversions.o", ir) + }) + } +} diff --git a/amd64_raw_decode_guard_test.go b/amd64_raw_decode_guard_test.go new file mode 100644 index 000000000..a9d117865 --- /dev/null +++ b/amd64_raw_decode_guard_test.go @@ -0,0 +1,57 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawDecoderRejectsUnrecognizedVEXInsteadOfLegacyInstructions(t *testing.T) { + // x/arch v0.14 can return SUB AL,imm instead of an error for these + // invalid scalar conversions. Such a successful decode is not semantic support. + for _, code := range [][]byte{{0xc5, 0xf8, 0x2c, 0xc1}, {0xc5, 0xf8, 0x2d, 0xc1}, {0xc4, 0xe1, 0xf8, 0x2c, 0xc1}} { + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + for _, b := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}}); err == nil { + t.Errorf("%s raw %x must fail closed instead of executing legacy arithmetic: %v", arch, code, err) + } + } + } +} + +func TestX86RawDecoderRecognizesSupportedScalarConversionsBeforeLegacyFallback(t *testing.T) { + for _, tc := range []struct { + code []byte + op Op + }{ + {[]byte{0xc5, 0xfb, 0x2c, 0xc1}, "VCVTTSD2SI"}, + {[]byte{0xc5, 0xfa, 0x2c, 0xc1}, "VCVTTSS2SI"}, + {[]byte{0xc4, 0xe1, 0xfb, 0x2c, 0xc1}, "VCVTTSD2SIQ"}, + } { + decoded, err := decodeX86RawDirectives(rawX86Function(tc.code), "amd64") + if err != nil || len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != tc.op { + t.Errorf("supported VEX %x: %+v, %v; want %s", tc.code, decoded.Instrs, err, tc.op) + } + } +} + +func TestX86RawDecoderRetainsRecognizedVEXFallback(t *testing.T) { + for _, code := range [][]byte{ + {0xc5, 0xf8, 0x77}, {0xc5, 0xf9, 0x6f, 0xc1}, {0xc5, 0xfa, 0x6f, 0xc1}, + {0xc5, 0xf9, 0xe7, 0x00}, {0xc4, 0xe2, 0x79, 0x2a, 0x00}, + } { + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil || len(decoded.Instrs) != 1 { + t.Fatalf("recognized VEX %x: %+v, %v", code, decoded.Instrs, err) + } + } +} diff --git a/amd64_raw_decode_test.go b/amd64_raw_decode_test.go new file mode 100644 index 000000000..92d11841e --- /dev/null +++ b/amd64_raw_decode_test.go @@ -0,0 +1,395 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestAMD64RawDirectiveSequenceUsesSemanticLowering(t *testing.T) { + source := `TEXT rawLoad(SB), NOSPLIT, $256-0 + MOVQ $0, R11 + BYTE $0x4e + BYTE $0x0f + BYTE $0xb7 + BYTE $0x7c + BYTE $0x5c + BYTE $0x78 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatalf("Parse() error = %v", err) + } + triple := "x86_64-unknown-linux-gnu" + x86LEAActualTargetGoObject(t, "amd64", triple, "#include \"textflag.h\"\n"+source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func rawLoad()", "amd64", triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original indexed physical SP bytes acquired a source/frame role", api, err) + } + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + ir := x86ScalarFormatObjectUnit(t, normalized, map[string]FuncSig{"rawLoad": {Name: "rawLoad", Ret: Void}}, "amd64", triple, 256) + if strings.Contains(ir, "module asm") { + t.Fatalf("raw instructions bypassed semantic lowering:\n%s", ir) + } + if !strings.Contains(ir, "load i16") || !strings.Contains(ir, "zext i16") { + t.Fatalf("decoded MOVWQZX semantics missing:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + // This independent source initializes the exact original physical tuple + // and attempts a real Go frame capture. The current frame producer has + // no raw-extension contract; retain that boundary as Context explicitly. + initialized := strings.Replace(source, "TEXT rawLoad(SB), NOSPLIT, $256-0", + "TEXT ·RawLoad(SB),4,$256-16\nMOVQ x+0(FP),AX\nMOVW AX,120(SP)", 1) + initialized = strings.Replace(initialized, "\tRET", "\tMOVQ R15,ret+8(FP)\n\tRET", 1) + x86LEAActualTargetGoObject(t, "amd64", triple, initialized) + pkg, request, _, options := fixedLocalTransportFixture(t, initialized, "func RawLoad(x uint64) uint64", "linux/amd64", triple) + goAMD64FramePrepareOwnedToolCache(t) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + observation, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("initialized raw-SP capture failed outside its source boundary", err) + } + t.Log("original initialized raw-SP Go object retains source-capture Context; initialized named extension objects have separate owners") + return + } + defer func() { + if err := observation.Close(); err != nil { + t.Error(err) + } + }() + expanded, err := observation.preprocess(request.AssemblyFile, []byte(initialized)) + if err != nil { + t.Fatal(err) + } + initializedFile, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + if _, err := observation.BindFixedLocal(ctx, pkg, initializedFile, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("captured frame silently acquired a raw physical-extension consumer", err) + } + t.Log("original raw-SP formats and captured initialized raw-SP Context remain separate from initialized named-extension objects") +} + +func TestX86RawOnlyFunctionUsesSemanticLowering(t *testing.T) { + for _, test := range []struct { + name string + goarch string + triple string + source string + }{ + { + name: "amd64", + goarch: "amd64", + triple: "x86_64-unknown-linux-gnu", + source: "TEXT raw(SB), NOSPLIT, $0-0\nBYTE $0x90\nBYTE $0xc3\n", + }, + { + name: "386", + goarch: "386", + triple: "i386-unknown-linux-gnu", + source: "TEXT raw(SB), NOSPLIT, $0-0\nBYTE $0x90\nBYTE $0xc3\n", + }, + } { + t.Run(test.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, test.source) + if err != nil { + t.Fatalf("Parse() error = %v", err) + } + ir, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: test.triple, + Sigs: map[string]FuncSig{ + "raw": {Name: "raw", Ret: Void}, + }, + }) + if err != nil { + t.Fatalf("Translate() error = %v", err) + } + if strings.Contains(ir, "module asm") { + t.Fatalf("raw-only function bypassed instruction validation:\n%s", ir) + } + if !strings.Contains(ir, "define void @raw()") || !strings.Contains(ir, "ret void") { + t.Fatalf("semantic function body missing:\n%s", ir) + } + }) + } +} + +func TestX86RawPCRelativeInstructionOutsideDirectiveGroupFailsClosed(t *testing.T) { + source := `TEXT rawBranch(SB), NOSPLIT, $0-0 + MOVQ $0, AX + BYTE $0xeb + BYTE $0x7f + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatalf("Parse() error = %v", err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{ + "rawBranch": {Name: "rawBranch", Ret: Void}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "PC-relative") { + t.Fatalf("Translate() error = %v, want explicit PC-relative raw-instruction rejection", err) + } +} + +func TestX86RawInternalConditionalBranchCompleteFamily(t *testing.T) { + targets := []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } + forms := []struct { + name string + bytes func(condition byte) []byte + }{ + {name: "short_forward", bytes: func(condition byte) []byte { + return []byte{0x70 + condition, 0x01, 0x90, 0xc3} + }}, + {name: "short_backward", bytes: func(condition byte) []byte { + return []byte{0x90, 0x70 + condition, 0xfd, 0xc3} + }}, + {name: "near_forward", bytes: func(condition byte) []byte { + return []byte{0x0f, 0x80 + condition, 0x01, 0, 0, 0, 0x90, 0xc3} + }}, + {name: "near_backward", bytes: func(condition byte) []byte { + return []byte{0x90, 0x0f, 0x80 + condition, 0xf9, 0xff, 0xff, 0xff, 0xc3} + }}, + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for condition := byte(0); condition < 16; condition++ { + for _, form := range forms { + name := fmt.Sprintf("rawJcc_%02x_%s", condition, form.name) + fmt.Fprintf(&source, "TEXT %s(SB), NOSPLIT, $0-0\n", name) + for _, value := range form.bytes(condition) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, amd64LLVMBlockName("raw_jcc")) { + t.Fatalf("synthetic raw branch labels missing:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-jcc-"+target.goarch+".ll", "raw-jcc-"+target.goarch+".o", ir) + }) + } +} + +func TestX86RawMULXReportedEncoding(t *testing.T) { + const source = `TEXT rawMULX(SB), NOSPLIT, $0-0 + BYTE $0xc4 + BYTE $0x42 + BYTE $0xe3 + BYTE $0xf6 + BYTE $0xf6 + RET +` + // These original bytes have no R14/DX producers. Keep the complete Go + // encoding and private LLVM format object, not a fictitious public entry. + ir := x86MULXOriginalFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu", + []string{"MULXQ R14,BX,R14"}) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-mulx.ll", "raw-mulx.o", ir) +} + +func TestX86RawMULXCompleteGoAssemblerForms(t *testing.T) { + // Go's x86 table defines both register and Yml memory sources for MULXL + // and (on amd64) MULXQ. Exercise both widths plus direct, SIB, disp8 and + // base-less disp32 ModRM forms, including extended registers. + targets := []struct { + goarch string + triple string + forms [][]byte + rows []string + }{ + { + goarch: "amd64", + triple: "x86_64-unknown-linux-gnu", + forms: [][]byte{ + {0xc4, 0x42, 0xe3, 0xf6, 0xf6}, // MULXQ R14, BX, R14 + {0xc4, 0x42, 0x3b, 0xf6, 0xfe}, // MULXL R14, R8, R15 + {0xc4, 0x22, 0xab, 0xf6, 0x5c, 0x88, 0x10}, // MULXQ 16(AX)(R9*4), R10, R11 + {0xc4, 0xe2, 0x6b, 0xf6, 0x5c, 0x48, 0x08}, // MULXL 8(AX)(CX*2), DX, BX + {0xc4, 0xe2, 0xfb, 0xf6, 0x0c, 0x25, 0x78, 0x56, 0x34, 0x12}, // MULXQ 0x12345678, AX, CX + }, + rows: []string{ + "MULXQ R14,BX,R14", "MULXL R14,R8,R15", + "MULXQ 16(AX)(R9*4),R10,R11", "MULXL 8(AX)(CX*2),DX,BX", + "MULXQ 0x12345678,AX,CX", + }, + }, + { + goarch: "386", + triple: "i386-unknown-linux-gnu", + forms: [][]byte{ + {0xc4, 0xe2, 0x73, 0xf6, 0xd0}, // MULXL AX, CX, DX + {0xc4, 0xe2, 0x6b, 0xf6, 0x5c, 0x48, 0x08}, // MULXL 8(AX)(CX*2), DX, BX + {0xc4, 0xe2, 0x7b, 0xf6, 0x0d, 0x78, 0x56, 0x34, 0x12}, // MULXL 0x12345678, AX, CX + }, + rows: []string{"MULXL AX,CX,DX", "MULXL 8(AX)(CX*2),DX,BX", "MULXL 0x12345678,AX,CX"}, + }, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + for i, encoding := range target.forms { + name := fmt.Sprintf("rawMULX_%d", i) + fmt.Fprintf(&source, "TEXT %s(SB), NOSPLIT, $0-0\n", name) + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + source.WriteString("\tRET\n") + } + // The absolute-address rows remain unbound formats. They cannot + // borrow a numerical pointer, DX or a raw-entry permission. + ir := x86MULXOriginalFormatObject(t, source.String(), target.goarch, target.triple, target.rows) + compileLLVMToObject(t, llc, target.triple, "raw-mulx-forms.ll", "raw-mulx-forms.o", ir) + }) + } +} + +func TestX86RawMULXRejectsLayoutDependentAddressForms(t *testing.T) { + tests := []struct { + name string + encoding []byte + }{ + {name: "address_size_override", encoding: []byte{0x67, 0xc4, 0xe2, 0x6b, 0xf6, 0x5c, 0x48, 0x08}}, + {name: "rip_relative", encoding: []byte{0xc4, 0xe2, 0xfb, 0xf6, 0x0d, 0, 0, 0, 0}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawMULX(SB), NOSPLIT, $0-0\n") + for _, value := range test.encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawMULX": {Name: "rawMULX", Ret: Void}}}) + if err == nil || !strings.Contains(err.Error(), "source-layout safe") { + t.Fatalf("Translate() error = %v, want source-layout safety rejection", err) + } + }) + } +} + +func TestX86RawForwardJumpSkipsEmbeddedDataButStillLowersReachableCode(t *testing.T) { + const source = `TEXT rawSignature(SB), NOSPLIT, $0-0 + BYTE $0xeb + BYTE $0x04 + LONG $0xe77da1c4 + BYTE $0xc3 +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawSignature": {Name: "rawSignature", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(ir, "module asm") || !strings.Contains(ir, "ret void") { + t.Fatalf("reachable raw control flow was not lowered semantically:\n%s", ir) + } +} + +func encodeX86RawRetpoline(register int) []byte { + return []byte{0xe8, 0x04, 0, 0, 0, 0xf3, 0x90, 0xeb, 0xfc, byte(0x48 | (register&8)>>1), 0x89, byte(0x04 | (register&7)<<3), 0x24, 0xc3} +} + +func TestX86RawRetpolineFamilyLowersToIndirectTailCalls(t *testing.T) { + names := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + var source strings.Builder + sigs := map[string]FuncSig{} + for register, name := range names { + if register == 4 { + continue + } + function := "retpoline" + name + fmt.Fprintf(&source, "TEXT %s(SB),NOSPLIT,$0-0\n", function) + for _, value := range encodeX86RawRetpoline(register) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[function] = FuncSig{Name: function, Ret: Void} + if got, ok := x86RawRetpolineTarget(encodeX86RawRetpoline(register)); !ok || got != Reg(name) { + t.Fatalf("decoded retpoline register %d as %s, ok=%v", register, got, ok) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(ir, "module asm") || !strings.Contains(ir, "call i64 %") || !strings.Contains(ir, "ret void") { + t.Fatalf("raw retpolines were not lowered as semantic indirect tail calls:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-retpoline.ll", "amd64-raw-retpoline.o", ir) +} + +func TestX86RawRetpolineDecoderRejectsNearMisses(t *testing.T) { + for _, offset := range []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13} { + nearMiss := append([]byte(nil), encodeX86RawRetpoline(0)...) + nearMiss[offset] ^= 1 + if _, ok := x86RawRetpolineTarget(nearMiss); ok { + t.Fatalf("retpoline decoder accepted byte %d near miss %#v", offset, nearMiss) + } + } + if _, ok := x86RawRetpolineTarget(encodeX86RawRetpoline(4)); ok { + t.Fatal("retpoline decoder accepted impossible SP target") + } +} diff --git a/amd64_raw_descriptor_alias_test.go b/amd64_raw_descriptor_alias_test.go new file mode 100644 index 000000000..7a5938e4d --- /dev/null +++ b/amd64_raw_descriptor_alias_test.go @@ -0,0 +1,98 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawDescriptorStoreAliases(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + want string + }{ + {name: "sldt-memory-amd64", code: []byte{0x0f, 0x00, 0x07}, mode: 64, want: "SLDTW"}, + {name: "sldt-memory-386", code: []byte{0x0f, 0x00, 0x07}, mode: 32, want: "SLDTW"}, + {name: "sldt-register-word", code: []byte{0x66, 0x0f, 0x00, 0xc0}, mode: 64, want: "SLDTW"}, + {name: "sldt-register-long", code: []byte{0x0f, 0x00, 0xc0}, mode: 64, want: "SLDTL"}, + {name: "sldt-register-quad", code: []byte{0x48, 0x0f, 0x00, 0xc0}, mode: 64, want: "SLDTQ"}, + {name: "str-memory-amd64", code: []byte{0x0f, 0x00, 0x0f}, mode: 64, want: "STRW"}, + {name: "str-memory-386", code: []byte{0x0f, 0x00, 0x0f}, mode: 32, want: "STRW"}, + {name: "str-register-word", code: []byte{0x66, 0x0f, 0x00, 0xc8}, mode: 64, want: "STRW"}, + {name: "str-register-long", code: []byte{0x0f, 0x00, 0xc8}, mode: 64, want: "STRL"}, + {name: "str-register-quad", code: []byte{0x48, 0x0f, 0x00, 0xc8}, mode: 64, want: "STRQ"}, + {name: "smsw-memory-amd64", code: []byte{0x0f, 0x01, 0x27}, mode: 64, want: "SMSWW"}, + {name: "smsw-memory-386", code: []byte{0x0f, 0x01, 0x27}, mode: 32, want: "SMSWW"}, + {name: "smsw-register-word", code: []byte{0x66, 0x0f, 0x01, 0xe0}, mode: 64, want: "SMSWW"}, + {name: "smsw-register-long", code: []byte{0x0f, 0x01, 0xe0}, mode: 64, want: "SMSWL"}, + {name: "smsw-register-quad", code: []byte{0x48, 0x0f, 0x01, 0xe0}, mode: 64, want: "SMSWQ"}, + } { + t.Run(tc.name, func(t *testing.T) { + inst, err := x86asm.Decode(tc.code, tc.mode) + if err != nil { + t.Fatal(err) + } + syntax, err := decodedX86GoSyntax(inst, tc.code) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(syntax, tc.want+" ") { + t.Fatalf("raw %x decoded as %q, want %s", tc.code, syntax, tc.want) + } + }) + } +} + +func TestX86RawDescriptorStoreCompiles(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawDescriptorStore(SB),$0-0\n") + codes := [][]byte{ + {0x0f, 0x00, 0x07}, {0x0f, 0x00, 0x0f}, {0x0f, 0x01, 0x27}, + {0x66, 0x0f, 0x00, 0xc0}, {0x0f, 0x00, 0xc0}, + {0x66, 0x0f, 0x00, 0xc8}, {0x0f, 0x00, 0xc8}, + {0x66, 0x0f, 0x01, 0xe0}, {0x0f, 0x01, 0xe0}, + } + if target.goarch == "amd64" { + codes = append(codes, + []byte{0x48, 0x0f, 0x00, 0xc0}, + []byte{0x48, 0x0f, 0x00, 0xc8}, + []byte{0x48, 0x0f, 0x01, 0xe0}, + ) + } + for _, code := range codes { + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawDescriptorStore": {Name: "rawDescriptorStore", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-descriptor-store.ll", "raw-descriptor-store.o", ir) + }) + } +} diff --git a/amd64_raw_displacement_test.go b/amd64_raw_displacement_test.go new file mode 100644 index 000000000..d3e18ebe8 --- /dev/null +++ b/amd64_raw_displacement_test.go @@ -0,0 +1,180 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func x86RawLEADisplacementEncoding(mode, width, addressBits, shape, scale int, displacement int32) []byte { + var encoding []byte + if width == 16 { + encoding = append(encoding, 0x66) + } + if addressBits != mode { + encoding = append(encoding, 0x67) + } + if width == 64 { + encoding = append(encoding, 0x48) + } + encoding = append(encoding, 0x8d) + if addressBits == 16 { + // mod=10, reg=CX, r/m=BX+SI; the displacement is physical disp16. + encoding = append(encoding, 0x88) + return binary.LittleEndian.AppendUint16(encoding, uint16(displacement)) + } + switch shape { + case 0: // AX + disp32 + encoding = append(encoding, 0x88) + case 1: // AX + BX*scale + disp32 + encoding = append(encoding, 0x8c, byte(scale<<6|3<<3)) + case 2: // AX*scale + disp32; no base is encoded. + encoding = append(encoding, 0x0c, byte(scale<<6|5)) + case 3: // disp32; SIB has neither a base nor an index. + encoding = append(encoding, 0x0c, 0x25) + default: + panic("unknown raw LEA displacement fixture shape") + } + return binary.LittleEndian.AppendUint32(encoding, uint32(displacement)) +} + +func x86RawDisplacementDirectives(encoding []byte) string { + var source strings.Builder + for _, value := range encoding { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + return source.String() +} + +func TestX86RawDisplacementRetainsPhysicalSignedField(t *testing.T) { + // Intel SDM Vol. 2, section 2.2.1: ModR/M/SIB disp32 is sign-extended + // in a 64-bit effective address. x/arch v0.14 exposes it as uint32. + for _, mode := range []int{32, 64} { + for _, shape := range []int{0, 1, 2, 3} { + for _, displacement := range []int32{-1 << 31, -129, -128, -64, -1, 0, 127, 128, 1<<31 - 1} { + t.Run(fmt.Sprintf("mode%d/shape%d/%d", mode, shape, displacement), func(t *testing.T) { + encoding := x86RawLEADisplacementEncoding(mode, mode, mode, shape, 2, displacement) + inst, err := x86asm.Decode(encoding, mode) + if err != nil || inst.Len != len(encoding) || inst.Op != x86asm.LEA { + t.Fatalf("physical LEA decode: %+v, %v", inst, err) + } + syntax, err := decodedX86GoSyntax(inst, encoding) + if err != nil { + t.Fatal(err) + } + instruction := x86RawDisplacementDecodedLEA(t, mode, encoding) + if got := instruction.Args[0].Mem.Off; got != int64(displacement) { + t.Fatalf("physical signed disp32 became %d, want %d: %q", got, displacement, syntax) + } + }) + } + } + } + // The independently discovered go-cv-simd bytes use REX.R/X and no base. + encoding := []byte{0x4e, 0x8d, 0x04, 0x9d, 0xc0, 0xff, 0xff, 0xff} + inst, err := x86asm.Decode(encoding, 64) + if err != nil { + t.Fatal(err) + } + if syntax, err := decodedX86GoSyntax(inst, encoding); err != nil || syntax != "LEAQ -0x40(R11*4), R8" { + t.Fatalf("original no-base extended-index LEA: %q, %v", syntax, err) + } +} + +func x86RawDisplacementDecodedLEA(t *testing.T, mode int, encoding []byte) Instr { + t.Helper() + arch := "amd64" + if mode == 32 { + arch = "386" + } + file, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+x86RawDisplacementDirectives(encoding)+"RET\n") + if err != nil { + t.Fatal(err) + } + function, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal(err) + } + for _, instruction := range function.Instrs { + if strings.HasPrefix(string(instruction.Op), "LEA") { + if len(instruction.Args) != 2 || instruction.Args[0].Kind != OpMem { + t.Fatalf("lost physical LEA memory operand: %+v", instruction) + } + return instruction + } + } + t.Fatal("decoded raw source lost its LEA instruction") + return Instr{} +} + +func TestX86RawDisplacementDoesNotSignExtendMOVAbsoluteOffset(t *testing.T) { + for _, opcode := range []byte{0xa0, 0xa1, 0xa2, 0xa3} { + for _, offset := range []uint64{0x80000000, 0xffffffc0, 0x1122334480000000} { + t.Run(fmt.Sprintf("%x/%x", opcode, offset), func(t *testing.T) { + encoding := binary.LittleEndian.AppendUint64([]byte{0x48, opcode}, offset) + inst, err := x86asm.Decode(encoding, 64) + if err != nil || inst.Len != len(encoding) { + t.Fatalf("physical MOV moffs decode: %+v, %v", inst, err) + } + syntax, err := decodedX86GoSyntax(inst, encoding) + if err != nil { + t.Fatal(err) + } + if syntax != x86asm.GoSyntax(inst, 0, nil) { + t.Fatalf("absolute MOV spelling changed: %q", syntax) + } + found := false + normalized := normalizeDecodedX86MemoryDisplacements(inst) + for _, operand := range normalized.Args { + if memory, ok := operand.(x86asm.Mem); ok { + found = true + if uint64(memory.Disp) != offset { + t.Fatalf("absolute native offset narrowed: %q", syntax) + } + } + } + if !found { + t.Fatalf("missing absolute memory operand: %q", syntax) + } + }) + } + } +} + +func TestX86RawDisplacementLEAAllWidthsAndFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + mode := 64 + if target.arch == "386" { + mode = 32 + } + for _, width := range []int{16, 32, 64} { + if width > mode { + continue + } + for _, shape := range []int{0, 1, 2, 3} { + t.Run(fmt.Sprintf("%s/width%d/shape%d", target.triple, width, shape), func(t *testing.T) { + encoding := x86RawLEADisplacementEncoding(mode, width, mode, shape, 2, -64) + body := "@MOV x+0(FP),AX\n@MOV $3,BX\n@MOV $0,CX\n" + x86RawDisplacementDirectives(encoding) + "@MOV CX,ret+@WORD(FP)\nRET" + source, sig, _ := x86SourceStackNumericalFixture(target.arch, body) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-disp.ll", "raw-disp.o", ir) + }) + } + } + } +} diff --git a/amd64_raw_double_dword.go b/amd64_raw_double_dword.go new file mode 100644 index 000000000..7860568f9 --- /dev/null +++ b/amd64_raw_double_dword.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Opcode E6 has three Go 1.27 conversion grammars. VEX.W is ignored; EVEX.W +// distinguishes the dword/double family from qword/double instructions. LL +// selects the vector width except when EVEX.b enables rounding or SAE on a +// register-source 512-bit double-to-dword form. +func decodedX86PackedDoubleDwordInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedDoubleDwordInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedDoubleDwordInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || p.opcode != 0xe6 || (p.pp != 1 && p.pp != 2 && p.pp != 3) { + return Instr{}, 0, false, nil + } + if p.pp != 2 && (!p.evex || p.w) { + return decodedX86PackedNumericConversionInstructionWithMemory(code, mode, memory) + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed double/dword conversion: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 { + return fail("reserved vvvv/V' field") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.evex { + if p.w != (p.pp != 2) { + // EVEX.W selects another conversion family, such as VCVTQQ2PD. + return Instr{}, 0, false, nil + } + if !p.fixed || p.zero && p.mask == 0 { + return fail("invalid EVEX fixed, mask or zeroing field") + } + } else if p.vectorLength > 1 { + return fail("invalid VEX vector length") + } + registerSource := code[p.modRM]>>6 == 3 + if mode == 32 { + if p.pp == 2 && (p.r != 0 || registerSource && (p.b != 0 || p.x != 0)) { + return fail("signed dword-to-double has no extended physical vector register in 32-bit mode") + } + if !p.evex && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended VEX register in 32-bit mode") + } + if p.evex && !registerSource && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + } + rounding := p.evex && p.broadcast && registerSource + broadcast := p.evex && p.broadcast && !registerSource + if p.pp == 2 && rounding { + return fail("dword-to-double has no SAE or rounding form") + } + if p.evex && !rounding && p.vectorLength > 2 { + return fail("reserved EVEX vector length") + } + if p.pp != 2 && rounding && !p.evex { + return fail("rounding or SAE requires EVEX") + } + + vectorLength := p.vectorLength + if rounding { + vectorLength = 2 // Register-source SAE/rounding implies 512 bits. + } + var op Op + var sourcePrefix, destinationPrefix string + var memoryScale int + if p.pp == 2 { + op = "VCVTDQ2PD" + destinationPrefix = [...]string{"X", "Y", "Z"}[vectorLength] + sourcePrefix = "X" + if vectorLength == 2 { + sourcePrefix = "Y" + } + memoryScale = 8 << vectorLength + } else { + stem := "VCVTPD2DQ" + if p.pp == 1 { + stem = "VCVTTPD2DQ" + } + sourcePrefix = [...]string{"X", "Y", "Z"}[vectorLength] + destinationPrefix = "X" + if vectorLength == 2 { + destinationPrefix = "Y" + } + op = Op(stem) + if vectorLength < 2 { + op = Op(stem + [...]string{"X", "Y"}[vectorLength]) + } + memoryScale = 16 << vectorLength + } + if mode == 32 { + if sourcePrefix == "Z" && registerSource && (p.b != 0 || p.x != 0) { + return fail("Z8-Z31 are not Go assembler source registers in 32-bit mode") + } + if destinationPrefix == "Z" && p.r != 0 { + return fail("Z8-Z31 are not Go assembler destination registers in 32-bit mode") + } + } + if _, exists := amd64PackedDoubleDwordOps[string(op)]; !exists { + return fail("encoding has no Go packed conversion form") + } + if rounding { + if p.pp == 1 { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } else if broadcast { + op += ".BCST" + if p.pp == 2 { + memoryScale = 4 + } else { + memoryScale = 8 + } + } + if p.zero { + op += ".Z" + } + + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix, memoryScale) + } else { + source, consumed, err = memory.vex(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix) + } + if err != nil { + return Instr{}, 0, true, err + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationPrefix, int(code[p.modRM]>>3&7)+p.r*8))} + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for i, arg := range args { + printed[i] = arg.String() + } + return Instr{ + Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86PackedNumericEVEX: p.evex, + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_double_dword_test.go b/amd64_raw_double_dword_test.go new file mode 100644 index 000000000..cca7db94d --- /dev/null +++ b/amd64_raw_double_dword_test.go @@ -0,0 +1,224 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawPackedDoubleDwordBaseForms = []struct { + op Op + pp byte + length byte + evex bool + src Reg + dst Reg +}{ + {"VCVTDQ2PD", 2, 0, false, "X1", "X0"}, + {"VCVTDQ2PD", 2, 1, false, "X1", "Y0"}, + {"VCVTDQ2PD", 2, 0, true, "X1", "X0"}, + {"VCVTDQ2PD", 2, 1, true, "X1", "Y0"}, + {"VCVTDQ2PD", 2, 2, true, "Y1", "Z0"}, + {"VCVTPD2DQX", 3, 0, false, "X1", "X0"}, + {"VCVTPD2DQY", 3, 1, false, "Y1", "X0"}, + {"VCVTPD2DQX", 3, 0, true, "X1", "X0"}, + {"VCVTPD2DQY", 3, 1, true, "Y1", "X0"}, + {"VCVTPD2DQ", 3, 2, true, "Z1", "Y0"}, + {"VCVTTPD2DQX", 1, 0, false, "X1", "X0"}, + {"VCVTTPD2DQY", 1, 1, false, "Y1", "X0"}, + {"VCVTTPD2DQX", 1, 0, true, "X1", "X0"}, + {"VCVTTPD2DQY", 1, 1, true, "Y1", "X0"}, + {"VCVTTPD2DQ", 1, 2, true, "Z1", "Y0"}, +} + +func TestX86RawPackedDoubleDwordCompleteGo127BaseForms(t *testing.T) { + if len(amd64PackedDoubleDwordOps) != 9 { + t.Fatalf("packed double/dword grammar has %d entries, want nine", len(amd64PackedDoubleDwordOps)) + } + for _, tc := range x86RawPackedDoubleDwordBaseForms { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("%s/LL=%d/EVEX=%t/memory=%t", tc.op, tc.length, tc.evex, memory) + t.Run(name, func(t *testing.T) { + modRM := byte(0xc1) + if memory { + modRM = 0x08 + } + var code []byte + if tc.evex { + w := byte(0) + if tc.pp != 2 { + w = 0x80 + } + code = []byte{0x62, 0xf1, 0x7c | tc.pp | w, 0x08 | tc.length<<5, 0xe6, modRM} + } else { + code = []byte{0xc5, 0xf8 | tc.pp | tc.length<<2, 0xe6, modRM} + } + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != tc.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", code, got.Instrs, tc.op) + } + args := got.Instrs[0].Args + if memory { + if args[0].Kind != OpMem || args[0].Mem.Base != AX { + t.Fatalf("memory source %#x = %+v", code, args[0]) + } + } else if args[0].Kind != OpReg || args[0].Reg != tc.src { + t.Fatalf("register source %#x = %+v, want %s", code, args[0], tc.src) + } + wantDestination := tc.dst + if memory { + wantDestination = Reg(string(tc.dst[:1]) + "1") + } + if args[1].Reg != wantDestination { + t.Fatalf("destination %#x = %s, want %s", code, args[1].Reg, wantDestination) + } + }) + } + } +} + +func TestX86RawPackedDoubleDwordEncodingAxes(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + op Op + args []string + }{ + {"VEX-W-ignored", []byte{0xc4, 0xe1, 0xfe, 0xe6, 0xc1}, "VCVTDQ2PD", []string{"X1", "Y0"}}, + {"masked", []byte{0x62, 0xf1, 0x7e, 0x09, 0xe6, 0xc1}, "VCVTDQ2PD", []string{"X1", "K1", "X0"}}, + {"masked-broadcast-zero", []byte{0x62, 0xf1, 0x7e, 0x99, 0xe6, 0x00}, "VCVTDQ2PD.BCST.Z", []string{"0(AX)", "K1", "X0"}}, + {"round-down-zero", []byte{0x62, 0xf1, 0xff, 0xb9, 0xe6, 0xd8}, "VCVTPD2DQ.RD_SAE.Z", []string{"Z0", "K1", "Y3"}}, + {"truncate-SAE", []byte{0x62, 0xf1, 0xfd, 0x18, 0xe6, 0xc1}, "VCVTTPD2DQ.SAE", []string{"Z1", "Y0"}}, + {"extended-source-and-destination", []byte{0x62, 0xa1, 0x7e, 0x08, 0xe6, 0xc1}, "VCVTDQ2PD", []string{"X17", "X16"}}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86PackedDoubleDwordInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) || got.Op != tc.op { + t.Fatalf("decode %#x = %+v, %d, %v, %v; want %s", tc.code, got, length, recognized, err, tc.op) + } + if len(got.Args) != len(tc.args) { + t.Fatalf("decode %#x operands = %+v, want %v", tc.code, got.Args, tc.args) + } + for index, arg := range got.Args { + if arg.String() != tc.args[index] { + t.Fatalf("decode %#x operand %d = %s, want %s", tc.code, index, arg.String(), tc.args[index]) + } + } + }) + } + for _, tc := range []struct { + name string + code []byte + off int64 + }{ + {"dword-Z-disp8", []byte{0x62, 0xf1, 0x7e, 0x48, 0xe6, 0x40, 0x02}, 64}, + {"double-Y-disp8", []byte{0x62, 0xf1, 0xff, 0x28, 0xe6, 0x40, 0x02}, 64}, + {"double-broadcast-disp8", []byte{0x62, 0xf1, 0xff, 0x38, 0xe6, 0x40, 0x02}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + got, _, recognized, err := decodedX86PackedDoubleDwordInstruction(tc.code, 64) + if !recognized || err != nil || got.Args[0].Kind != OpMem || got.Args[0].Mem.Off != tc.off { + t.Fatalf("decode %#x = %+v, %v, %v; want offset %d", tc.code, got, recognized, err, tc.off) + } + }) + } + for _, code := range [][]byte{ + {0x62, 0xa1, 0x7e, 0x08, 0xe6, 0xc1}, // X17 -> X16. + {0x62, 0xa1, 0xff, 0x28, 0xe6, 0xc1}, // Y17 -> X16. + } { + got, _, recognized, err := decodedX86PackedDoubleDwordInstruction(code, 32) + // Both directions now preserve native state. Go accepts these + // spellings, but neither can invent physical high mode32 banks. + if !recognized || err == nil { + t.Fatal("mode32 native conversion fabricated a high physical bank", got, err) + } + } + requireX86GoAssemblerResult(t, "386", "TEXT high(SB),4,$0-0\n\tVCVTDQ2PD X17, K1, X16\n\tVCVTPD2DQY Y17, K1, X16\n\tRET\n", true) +} + +func TestX86RawPackedDoubleDwordRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"VEX-vvvv", []byte{0xc5, 0xee, 0xe6, 0xc1}, 64}, + {"EVEX-vvvv", []byte{0x62, 0xf1, 0x6e, 0x08, 0xe6, 0xc1}, 64}, + {"EVEX-zero-without-mask", []byte{0x62, 0xf1, 0x7e, 0x88, 0xe6, 0xc1}, 64}, + {"EVEX-LL3-without-rounding", []byte{0x62, 0xf1, 0x7e, 0x68, 0xe6, 0xc1}, 64}, + {"dword-register-broadcast", []byte{0x62, 0xf1, 0x7e, 0x18, 0xe6, 0xc1}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xfe, 0xe6, 0xc1}, 64}, + {"missing-ModRM", []byte{0xc5, 0xfe, 0xe6}, 64}, + {"extended-386", []byte{0xc4, 0x61, 0x7e, 0xe6, 0xc1}, 32}, + {"Z-destination-386", []byte{0x62, 0xe1, 0x7e, 0x48, 0xe6, 0xc1}, 32}, + {"Z-source-386", []byte{0x62, 0xb1, 0xff, 0x48, 0xe6, 0xc1}, 32}, + {"extended-memory-386", []byte{0x62, 0xb1, 0x7e, 0x08, 0xe6, 0x01}, 32}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86PackedDoubleDwordInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } +} + +func TestX86RawPackedDoubleDwordCompilesEveryBaseForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawdoubleconversion(SB),4,$0-0\n") + encodings := make([][]byte, 0, len(x86RawPackedDoubleDwordBaseForms)+4) + for _, form := range x86RawPackedDoubleDwordBaseForms { + modRM := byte(0xc1) + var code []byte + if form.evex { + w := byte(0) + if form.pp != 2 { + w = 0x80 + } + code = []byte{0x62, 0xf1, 0x7c | form.pp | w, 0x08 | form.length<<5, 0xe6, modRM} + } else { + code = []byte{0xc5, 0xf8 | form.pp | form.length<<2, 0xe6, modRM} + } + encodings = append(encodings, code) + } + encodings = append(encodings, + []byte{0x62, 0xf1, 0x7e, 0x99, 0xe6, 0x00}, // Dword broadcast, mask, zero. + []byte{0x62, 0xf1, 0xff, 0xb9, 0xe6, 0xd8}, // Double embedded rounding. + []byte{0x62, 0xf1, 0xff, 0x38, 0xe6, 0x40, 0x02}, // Double broadcast disp8. + ) + if target.goarch == "amd64" { + encodings = append(encodings, []byte{0x62, 0xa1, 0x7e, 0x08, 0xe6, 0xc1}) // Physical high X. + } + for _, code := range encodings { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-double-conversion.ll", "raw-double-conversion.o", ir) + }) + } +} diff --git a/amd64_raw_duplicate_literal_test.go b/amd64_raw_duplicate_literal_test.go new file mode 100644 index 000000000..e88e88f73 --- /dev/null +++ b/amd64_raw_duplicate_literal_test.go @@ -0,0 +1,124 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86RawDuplicateMoveConstant(spec rawDuplicateMoveSpec, encoding string, width int, mask int, zeroing bool) []byte { + var code []byte + switch encoding { + case "legacy": + code = encodeX86RawLegacyDuplicateMove(spec, 0, 0) + case "vex": + code = encodeX86RawVEXDuplicateMove(spec, byte(map[int]int{16: 0, 32: 1}[width]), 0, 0) + case "evex": + code = encodeX86RawEVEXDuplicateMove(spec, byte(map[int]int{16: 0, 32: 1, 64: 2}[width]), mask, zeroing, 0, 0) + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if spec.double && width == 16 { + dataWidth = 8 + } + for i := 0; i < dataWidth; i++ { + code = append(code, byte(i*37+3)) + } + return code +} + +func TestDecodeX86RawDuplicateMoveConstantCompleteFamily(t *testing.T) { + for _, spec := range rawDuplicateMoveSpecs { + for _, encoding := range []string{"legacy", "vex", "evex"} { + widths := []int{16} + if encoding != "legacy" { + widths = append(widths, 32) + } + if encoding == "evex" { + widths = append(widths, 64) + } + for _, width := range widths { + mask := 0 + zeroing := false + if encoding == "evex" && width == 64 { + mask, zeroing = 3, true + } + name := fmt.Sprintf("%s/%s/%d", spec.vector, encoding, width) + t.Run(name, func(t *testing.T) { + code := x86RawDuplicateMoveConstant(spec, encoding, width, mask, zeroing) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := spec.vector + if encoding == "legacy" { + want = spec.legacy + } + if zeroing { + want += ".Z" + } + if len(got) != 2 || got[0].Op != want || + got[0].Args[0].Kind != OpSym || !got[0].x86RIPLiteral || + got[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, got, want) + } + }) + } + } + } +} + +func TestDecodeX86RawDuplicateMoveConstantRejectsUnsafeSources(t *testing.T) { + valid := x86RawDuplicateMoveConstant(rawDuplicateMoveSpecs[0], "vex", 32, 0, false) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[5] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[5] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe duplicate-move RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawDuplicateMoveConstantObjects(t *testing.T) { + var source strings.Builder + for index, spec := range rawDuplicateMoveSpecs { + code := x86RawDuplicateMoveConstant(spec, "evex", 64, 3, true) + name := fmt.Sprintf("rawDuplicateConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + original, signatures := x86UnboundVoidFormatSources(t, source.String(), "amd64", triple) + normalized, err := normalizeX86RawFile(original, "amd64") + if err != nil { + t.Fatal(err) + } + ir := x86DuplicateMoveFormatObjectUnit(t, normalized, signatures, "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-duplicate-literal.ll", "raw-duplicate-literal.o", ir) + }) + } +} diff --git a/amd64_raw_duplicate_test.go b/amd64_raw_duplicate_test.go new file mode 100644 index 000000000..a7708b256 --- /dev/null +++ b/amd64_raw_duplicate_test.go @@ -0,0 +1,334 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" +) + +type rawDuplicateMoveSpec struct { + legacyPrefix byte + opcode byte + legacy Op + vector Op + double bool +} + +var rawDuplicateMoveSpecs = []rawDuplicateMoveSpec{ + {legacyPrefix: 0xf2, opcode: 0x12, legacy: "MOVDDUP", vector: "VMOVDDUP", double: true}, + {legacyPrefix: 0xf3, opcode: 0x16, legacy: "MOVSHDUP", vector: "VMOVSHDUP"}, + {legacyPrefix: 0xf3, opcode: 0x12, legacy: "MOVSLDUP", vector: "VMOVSLDUP"}, +} + +func encodeX86RawLegacyDuplicateMove(spec rawDuplicateMoveSpec, source, destination int) []byte { + rex := byte(0x40 | source>>3&1 | destination>>3&1<<2) + code := []byte{spec.legacyPrefix} + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, spec.opcode, byte(0xc0|destination&7<<3|source&7)) +} + +func encodeX86RawVEXDuplicateMove(spec rawDuplicateMoveSpec, vectorBits byte, source, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^source>>3)&1)<<5 | 1 + pp := byte(2) + if spec.double { + pp = 3 + } + p1 := byte(0x78 | vectorBits<<2 | pp) + return []byte{0xc4, p0, p1, spec.opcode, byte(0xc0 | destination&7<<3 | source&7)} +} + +func encodeX86RawEVEXDuplicateMove(spec rawDuplicateMoveSpec, vectorBits byte, mask int, zeroing bool, source, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^source>>4)&1)<<6 | + byte((^source>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 1 + pp := byte(2) + if spec.double { + pp = 3 + } + p1 := byte(0x7c | pp) + if spec.double { + p1 |= 0x80 + } + p2 := vectorBits<<5 | 0x08 | byte(mask) + if zeroing { + p2 |= 0x80 + } + return []byte{0x62, p0, p1, p2, spec.opcode, byte(0xc0 | destination&7<<3 | source&7)} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVMOVSHDUP(t *testing.T) { + // vmovshdup %xmm0, %xmm1; + // vpermilps $208, %xmm7, %xmm0 + code := []byte{0xc5, 0xfa, 0x16, 0xc8, 0xc4, 0xe3, 0x79, 0x04, 0xc7, 0xd0} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VMOVSHDUP sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VMOVSHDUP X0, X1 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawDuplicateMoveCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawDuplicateMoveSpecs { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawLegacyDuplicateMove(spec, source, destination) + got, length, ok, err := decodedX86DuplicateMoveInstruction(code, 64) + want := fmt.Sprintf("%s X%d, X%d", spec.legacy, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for vectorBits := byte(0); vectorBits < 2; vectorBits++ { + vectorName := [...]string{"X", "Y"}[vectorBits] + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXDuplicateMove(spec, vectorBits, source, destination) + got, length, ok, err := decodedX86DuplicateMoveInstruction(code, 64) + want := fmt.Sprintf("%s %s%d, %s%d", spec.vector, vectorName, source, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{{}, {mask: 3}, {mask: 7, zeroing: true, suffix: ".Z"}} { + for source := 0; source < 32; source++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXDuplicateMove(spec, vectorBits, masking.mask, masking.zeroing, source, destination) + got, length, ok, err := decodedX86DuplicateMoveInstruction(code, 64) + args := []string{fmt.Sprintf("%s%d", vectorName, source)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(spec.vector) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + if count != 29952 { + t.Fatalf("covered %d duplicate-move register encodings, want 29952", count) + } +} + +func TestDecodedX86RawDuplicateMoveMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0xf3, 0x47, 0x0f, 0x16, 0x7c, 0x8b, 0x01} + got, length, ok, err := decodedX86DuplicateMoveInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "MOVSHDUP" || got.Args[1].String() != "X15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x01, 0x7e, 0x16, 0x7c, 0x8b, 0x02} + got, length, ok, err = decodedX86DuplicateMoveInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VMOVSHDUP" || got.Args[1].String() != "Y15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexMemory := []byte{0x62, 0xf1, 0xff, 0x8b, 0x12, 0x50, 0x02} + got, length, ok, err = decodedX86DuplicateMoveInstruction(evexMemory, 64) + wantMemory = MemRef{Base: "AX", Off: 16} + if err != nil || !ok || length != len(evexMemory) || got.Op != "VMOVDDUP.Z" || got.Args[1].String() != "K3" || got.Args[2].String() != "X2" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + validVEX := encodeX86RawVEXDuplicateMove(rawDuplicateMoveSpecs[1], 0, 7, 7) + validEVEX := encodeX86RawEVEXDuplicateMove(rawDuplicateMoveSpecs[1], 2, 0, false, 7, 5) + for name, invalid := range map[string][]byte{ + "vex vvvv": {validVEX[0], validVEX[1], validVEX[2] &^ 0x08, validVEX[3], validVEX[4]}, + "evex fixed bit": {validEVEX[0], validEVEX[1], validEVEX[2] &^ 0x04, validEVEX[3], validEVEX[4], validEVEX[5]}, + "evex wrong W": {validEVEX[0], validEVEX[1], validEVEX[2] | 0x80, validEVEX[3], validEVEX[4], validEVEX[5]}, + "evex vvvvv": {validEVEX[0], validEVEX[1], validEVEX[2] &^ 0x08, validEVEX[3] &^ 0x08, validEVEX[4], validEVEX[5]}, + "evex broadcast": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x10, validEVEX[4], validEVEX[5]}, + "evex reserved length": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x60, validEVEX[4], validEVEX[5]}, + "zero without mask": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x80, validEVEX[4], validEVEX[5]}, + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86DuplicateMoveInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + highZ386 := encodeX86RawEVEXDuplicateMove(rawDuplicateMoveSpecs[1], 2, 0, false, 8, 2) + if instruction, _, matched, decodeErr := decodedX86DuplicateMoveInstruction(highZ386, 32); !matched || decodeErr == nil { + t.Fatalf("386 high-Z encoding %x decoded as %+v, ok=%v, err=%v", highZ386, instruction, matched, decodeErr) + } +} + +func TestX86DuplicateMoveRawGo127ArchitectureOracle(t *testing.T) { + const accepted386 = `TEXT ok(SB),$0-0 + MOVDDUP X6, X7 + MOVSHDUP X6, X7 + MOVSLDUP X6, X7 + VMOVDDUP Y6, Y7 + VMOVSHDUP X20, K1, X21 + VMOVSLDUP.Z Y20, K2, Y21 + VMOVDDUP Z6, K3, Z7 + RET +` + requireX86GoAssemblerResult(t, "386", accepted386, true) + for _, instruction := range []string{ + "MOVSHDUP X8, X0", + "VMOVDDUP Z8, Z0", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } +} + +func TestTranslateX86RawDuplicateMoveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_duplicate_move(SB),NOSPLIT,$0-0\n") + for _, spec := range rawDuplicateMoveSpecs { + var encodings [][]byte + if target.goarch == "amd64" { + encodings = append(encodings, + encodeX86RawLegacyDuplicateMove(spec, 15, 14), + encodeX86RawVEXDuplicateMove(spec, 1, 15, 14), + encodeX86RawEVEXDuplicateMove(spec, 2, 7, true, 31, 30), + ) + } else { + encodings = append(encodings, + encodeX86RawLegacyDuplicateMove(spec, 7, 6), + encodeX86RawVEXDuplicateMove(spec, 1, 7, 6), + encodeX86RawEVEXDuplicateMove(spec, 0, 1, false, 20, 21), + encodeX86RawEVEXDuplicateMove(spec, 2, 2, true, 6, 7), + ) + } + for _, encoding := range encodings { + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, signatures := x86UnboundVoidFormatSources(t, source.String(), target.goarch, target.triple) + normalized, err := normalizeX86RawFile(file, target.goarch) + if err != nil { + t.Fatal(err) + } + ll := x86DuplicateMoveFormatObjectUnit(t, normalized, signatures, target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-duplicate-move-"+target.name+".ll", "raw-duplicate-move-"+target.name+".o", ll) + }) + } +} + +func TestAMD64DuplicateMoveRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT duplicateMoveSemantics(SB),NOSPLIT,$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + KMOVQ mask+24(FP), K1 + VMOVUPS (BX), Y0 + VMOVSHDUP Y0, Y1 + VMOVUPS Y1, 0(AX) + VMOVSLDUP Y0, Y2 + VMOVUPS Y2, 32(AX) + VMOVDDUP (BX), X3 + VMOVUPS X3, 64(AX) + VMOVUPS (CX), Y4 + VMOVSHDUP Y0, K1, Y4 + VMOVUPS Y4, 80(AX) + VMOVUPS (CX), Y4 + VMOVSLDUP.Z Y0, K1, Y4 + VMOVUPS Y4, 112(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "duplicateMoveSemantics": {Name: "duplicateMoveSemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void duplicateMoveSemantics(uint32_t *, const uint32_t *, const uint32_t *, uint64_t); +int main(void) { + uint32_t source[8] = {10, 11, 20, 21, 30, 31, 40, 41}; + uint32_t old[8] = {100, 101, 102, 103, 104, 105, 106, 107}; + uint32_t out[36] = {0}; + uint64_t mask = 0x55; + duplicateMoveSemantics(out, source, old, mask); + uint32_t sh[8] = {11, 11, 21, 21, 31, 31, 41, 41}; + uint32_t sl[8] = {10, 10, 20, 20, 30, 30, 40, 40}; + uint32_t dd[4] = {10, 11, 10, 11}; + for (int i = 0; i < 8; i++) if (out[i] != sh[i]) return 1 + i; + for (int i = 0; i < 8; i++) if (out[8+i] != sl[i]) return 20 + i; + for (int i = 0; i < 4; i++) if (out[16+i] != dd[i]) return 40 + i; + for (int i = 0; i < 8; i++) { + uint32_t want = (mask >> i) & 1 ? sh[i] : old[i]; + if (out[20+i] != want) return 50 + i; + } + for (int i = 0; i < 8; i++) { + uint32_t want = (mask >> i) & 1 ? sl[i] : 0; + if (out[28+i] != want) return 70 + i; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "duplicate_move", triple, ll, mainC, runPrefix) +} diff --git a/amd64_raw_endbranch.go b/amd64_raw_endbranch.go new file mode 100644 index 000000000..bad9f5a3f --- /dev/null +++ b/amd64_raw_endbranch.go @@ -0,0 +1,43 @@ +package plan9asm + +// ENDBR changes CET tracking, not GP/SIMD, FLAGS or storage. Numerical +// observers may preserve existing facts only for the exact named/raw form. +// The lowerer still emits the native opcode; this supplies no entry authority. +func x86EndBranchPreservesSourceState(ins Instr) bool { + if ins.Op != "ENDBR64" && ins.Op != "ENDBR32" || len(ins.Args) != 0 || + ins.x86RawNearReturn != nil || ins.x86AddressBits != 0 || + ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 { + return false + } + origin := ins.x86SourceRaw + if origin == nil { + return ins.Op == "ENDBR64" && !ins.x86Encoded + } + return ins.x86Encoded && origin.op == ins.Op && len(origin.args) == 0 +} + +// These instructions have different native effects. Only their preservation +// of numerical source facts is shared, never their lowering or CET semantics. +func x86PreservesNumericalSourceState(ins Instr) bool { + return x86NOPPreservesSourceState(ins) || x86EndBranchPreservesSourceState(ins) +} + +// ENDBR32/ENDBR64 have fixed, operand-free encodings (Intel SDM, volume 2). +// x86asm does not decode them yet. Preserve the actual instruction rather than +// replacing the CET state transition with a NOP. Go names only ENDBR64, so +// x86Encoded also keeps ENDBR32 confined to validated raw input. +func decodedX86EndBranchInstruction(code []byte) (Instr, int, bool) { + if len(code) < 4 || code[0] != 0xf3 || code[1] != 0x0f || code[2] != 0x1e { + return Instr{}, 0, false + } + var op Op + switch code[3] { + case 0xfa: + op = "ENDBR64" + case 0xfb: + op = "ENDBR32" + default: + return Instr{}, 0, false + } + return Instr{Op: op, Raw: string(op), x86Encoded: true}, 4, true +} diff --git a/amd64_raw_endbranch_test.go b/amd64_raw_endbranch_test.go new file mode 100644 index 000000000..c99c7a70f --- /dev/null +++ b/amd64_raw_endbranch_test.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +// Intel SDM lists the operand-free F3 0F 1E FA/FB ENDBR64/ENDBR32 +// encodings. Go 1.27 names only ENDBR64; either encoding is valid BYTE data. +func TestX86RawEndBranchFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + for _, form := range []struct { + op Op + last byte + }{ + {"ENDBR64", 0xfa}, + {"ENDBR32", 0xfb}, + } { + t.Run(string(form.op)+"/"+target.triple, func(t *testing.T) { + source := fmt.Sprintf(`TEXT endbranch(SB),$0-0 + MOVL $17, AX + CMPL AX, AX + BYTE $0xf3; BYTE $0x0f; BYTE $0x1e; BYTE $0x%02x + SETEQ BL + ADDL $3, AX + RET +`, form.last) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 7 || decoded.Instrs[3].Op != form.op { + t.Fatalf("wrong instruction boundaries: %+v", decoded.Instrs) + } + ir, err := Translate(file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"endbranch": {Name: "endbranch", Ret: I32}}, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("frame-less manual result acquired an ABI return slot", err) + } + want := fmt.Sprintf(".byte 0xf3, 0x0f, 0x1e, 0x%02x", form.last) + for _, result := range []bool{false, true} { + program, declaration := source, "func endbranch()" + if result { + program = strings.Replace(program, "$0-0", "$0-4", 1) + program = strings.Replace(program, "\tRET", "\tMOVL AX,ret+0(FP)\n\tRET", 1) + declaration = "func endbranch() uint32" + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, program) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, program, declaration, target.arch, target.triple, api) + if err != nil || !strings.Contains(ir, want) { + t.Fatal("source-bound ENDBR program lost its encoding", api, result, err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + } + } + }) + } + } +} + +func TestX86RawEndBranchPreservesBoundaries(t *testing.T) { + // JE targets the second ENDBR after a first four-byte ENDBR and NOP. + code := []byte{0x74, 5, 0xf3, 0x0f, 0x1e, 0xfa, 0x90, 0xf3, 0x0f, 0x1e, 0xfb, 0xc3} + for _, mode := range []int{32, 64} { + instructions, err := decodeX86RawDirectiveGroup(code, mode, 0, "endbranch fixture", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + var ops []Op + for _, instruction := range instructions { + if instruction.Op == "ENDBR32" || instruction.Op == "ENDBR64" { + ops = append(ops, instruction.Op) + } + } + if len(ops) != 2 || ops[0] != "ENDBR64" || ops[1] != "ENDBR32" { + t.Fatalf("mode %d: ENDBR instructions = %v", mode, ops) + } + for _, incomplete := range [][]byte{{0xf3}, {0xf3, 0x0f}, {0xf3, 0x0f, 0x1e}} { + if _, err := decodeX86RawDirectiveGroup(incomplete, mode, 0, "truncated ENDBR", map[string]bool{}); err == nil { + t.Fatalf("accepted truncated encoding %x", incomplete) + } + } + } +} + +func TestX86EndBranch32RemainsRawOnly(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, instruction := range []string{"ENDBR32", "ENDBR32 AX", "ENDBR32.Z"} { + source := "TEXT invalid(SB),$0-0\n" + instruction + "\nRET\n" + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if _, err := Translate(file, Options{ + Goarch: arch, + Sigs: map[string]FuncSig{"invalid": {Name: "invalid", Ret: Void}}, + }); err == nil { + t.Fatalf("accepted Go-undefined text form %q on %s", instruction, arch) + } + } + } +} + +const x86EndBranchRuntimeSource = `TEXT endbranchstate(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVL $17, AX + CMPL AX, AX + BYTE $0xf3; BYTE $0x0f; BYTE $0x1e; BYTE $0x%02x + SETEQ 0(DI) + SETCS 1(DI) + MOVL AX, 4(DI) + RET +` + +func TestX86RawEndBranchRuntimePreservesState(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution requires amd64; required amd64 CI covers this host-inapplicable test") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + // This checks GP/flags preservation, not operating-system CET enforcement. + // Both instructions remain executable bytes, even on pre-CET CPUs. + for _, last := range []byte{0xfa, 0xfb} { + t.Run(fmt.Sprintf("%02x", last), func(t *testing.T) { + source := fmt.Sprintf(x86EndBranchRuntimeSource, last) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{ + "endbranchstate": { + Name: "endbranchstate", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +struct result { uint8_t zf, cf, padding[2]; uint32_t value; }; +extern void endbranchstate(struct result *); +int main(void) { + struct result out = {0}; + endbranchstate(&out); + return out.zf != 1 || out.cf != 0 || out.value != 17; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_endbranch_state", triple, ir, mainC, runPrefix) + }) + } +} diff --git a/amd64_raw_enqueue.go b/amd64_raw_enqueue.go new file mode 100644 index 000000000..c2f3336c9 --- /dev/null +++ b/amd64_raw_enqueue.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// Decode the shared ModRM/SIB grammar using an equivalent LEA understood by +// x/arch. ENQCMD's register width follows address size, not REX.W or 0x66. +func decodedX86EnqueueInstruction(code []byte, mode int) (Instr, int, bool, error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + var mandatory, segment, rex byte + addressOverride, locked := false, false + for i < len(code) { + switch code[i] { + case 0xf2, 0xf3: + mandatory, rex = code[i], 0 + case 0x64, 0x65: + segment, rex = code[i], 0 + case 0x26, 0x2e, 0x36, 0x3e: + segment, rex = 0, 0 + case 0x66: + rex = 0 + case 0x67: + addressOverride, rex = true, 0 + case 0xf0: + locked, rex = true, 0 + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + continue + } + goto opcode + } + i++ + } + +opcode: + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0x38 || code[i+2] != 0xf8 { + return Instr{}, 0, false, nil + } + var op Op + for name, prefix := range x86EnqueueOps { + if prefix == mandatory { + op = Op(name) + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(reason string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("%s: %s", op, reason) + } + if locked { + return fail("LOCK is not permitted") + } + if len(code) <= i+3 { + return fail("missing ModRM byte") + } + if code[i+3]>>6 == 3 { + return fail("the command source must be memory") + } + var lea []byte + if segment != 0 { + lea = append(lea, segment) + } + if addressOverride { + lea = append(lea, 0x67) + } + if mode == 64 { + lea = append(lea, rex|0x48) + } + lea = append(lea, 0x8d) + leaHeader := len(lea) + lea = append(lea, code[i+3:]...) + decoded, err := x86asm.Decode(lea, mode) + if err != nil || decoded.Op != x86asm.LEA { + return fail("truncated or invalid memory address") + } + memory, ok := decoded.Args[1].(x86asm.Mem) + if !ok { + return fail("decoded source is not memory") + } + if memory.Base == x86asm.RIP || memory.Base == x86asm.EIP { + return fail("PC-relative command source requires source-layout context") + } + // x/arch prints segment overrides as FS:..., while the Plan 9 parser + // represents them as (...)(FS). Carry the segment structurally instead. + memory.Segment = 0 + decoded.Args[1] = memory + parsed, err := parseDecodedX86Instruction(x86asm.GoSyntax(decoded, 0, nil)) + if err != nil || len(parsed) != 1 || len(parsed[0].Args) != 2 { + return fail("cannot represent decoded memory address") + } + length := i + 3 + decoded.Len - leaHeader + if length > 15 { + return fail("encoding exceeds the 15-byte instruction limit") + } + ins := parsed[0] + if memory.Base == 0 && memory.Index == 0 { + ins.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Off: memory.Disp}} + } + if ins.Args[0].Kind != OpMem { + return fail("cannot represent decoded source as memory") + } + switch segment { + case 0x64: + ins.Args[0].Mem.Segment = FS + case 0x65: + ins.Args[0].Mem.Segment = GS + } + ins.Op = op + ins.x86AddressBits = decoded.AddrSize + ins.Raw = fmt.Sprintf("%s %s, %s", op, ins.Args[0], ins.Args[1]) + return ins, length, true, nil +} diff --git a/amd64_raw_enqueue_test.go b/amd64_raw_enqueue_test.go new file mode 100644 index 000000000..285bc2bdd --- /dev/null +++ b/amd64_raw_enqueue_test.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os/exec" + "strings" + "testing" +) + +// Intel's ENQCMD family uses F2/F3 0F 38 F8 /r, a 64-byte memory +// source, a GP destination address, and ZF to report retry. Neither +// spelling is present in Go 1.27's x86 opcode tables; ixl-go uses BYTE. +// https://www.intel.com/content/dam/develop/external/us/en/documents/architecture-instruction-set-extensions-programming-reference-737410.pdf +func TestX86RawEnqueueFamilyObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, family := range []struct { + name string + prefix byte + }{ + {"ENQCMD", 0xf2}, + {"ENQCMDS", 0xf3}, + } { + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(family.name+"/"+target.triple, func(t *testing.T) { + source := fmt.Sprintf(`TEXT enqueue(SB),$0-0 + MOVL $0, AX + MOVL $0, BX + BYTE $0x%02x; BYTE $0x0f; BYTE $0x38; BYTE $0xf8; BYTE $0x03 + SETEQ AX + RET +`, family.prefix) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + instruction := decoded.Instrs[len(decoded.Instrs)-3] + if instruction.Op != Op(family.name) || len(instruction.Args) != 2 || + instruction.Args[0].Kind != OpMem || instruction.Args[0].Mem.Base != BX || + instruction.Args[1].Kind != OpReg || instruction.Args[1].Reg != AX { + t.Fatalf("wrong command source/destination: %+v", instruction) + } + // The original source has no ABI0 result field and uses null + // device/source addresses. It is a format witness, not a closed + // public entry or an executable hardware/PASID contract. + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func enqueue() uint32", target.arch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("device format invented an initialized ABI0 return", api, err) + } + } + original := *file + original.Funcs = []Func{decoded} + ir := x86ScalarFormatObjectUnit(t, &original, + map[string]FuncSig{"enqueue": {Name: "enqueue", Ret: I32}}, target.arch, target.triple, 0) + if !strings.Contains(ir, strings.ToLower(family.name)) { + t.Fatalf("lost device command %s", family.name) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + }) + } + } +} + +func TestX86RawEnqueueAddressGrammar(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, test := range []struct { + name string + mode int + code []byte + address string + destination Reg + bits int + }{ + {"base64", 64, []byte{0xf2, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)", AX, 64}, + {"base32", 32, []byte{0xf3, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)", AX, 32}, + {"sib_disp8", 64, []byte{0xf2, 0x44, 0x0f, 0x38, 0xf8, 0x54, 0x8b, 0x80}, "-128(BX)(CX*4)", "R10", 64}, + {"extended_base_index", 64, []byte{0xf2, 0x47, 0x0f, 0x38, 0xf8, 0x14, 0x8b}, "0(R11)(R9*4)", "R10", 64}, + {"disp32", 32, []byte{0xf2, 0x0f, 0x38, 0xf8, 0xb0, 0x78, 0x56, 0x34, 0x12}, "305419896(AX)", SI, 32}, + {"absolute32", 32, []byte{0xf2, 0x0f, 0x38, 0xf8, 0x05, 0x00, 0x10, 0x00, 0x00}, "4096()", AX, 32}, + {"override32", 64, []byte{0x67, 0xf2, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)", AX, 32}, + {"override16", 32, []byte{0x67, 0xf2, 0x0f, 0x38, 0xf8, 0x00}, "0(BX)(SI*1)", AX, 16}, + {"fs", 64, []byte{0x64, 0xf2, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)(FS)", AX, 64}, + {"gs", 32, []byte{0x65, 0xf3, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)(GS)", AX, 32}, + {"rex_reset", 64, []byte{0x44, 0xf2, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)", AX, 64}, + } { + t.Run(test.name, func(t *testing.T) { + ins, length, ok, err := decodedX86EnqueueInstruction(test.code, test.mode) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode = %+v, %d, %v, %v", ins, length, ok, err) + } + if ins.Args[0].String() != test.address || ins.Args[1].Reg != test.destination || ins.x86AddressBits != test.bits { + t.Fatalf("decoded %+v address=%s, want %s, %s, %d bits", ins, ins.Args[0], test.address, test.destination, test.bits) + } + arch, triple := "amd64", "x86_64-unknown-linux-gnu" + if test.mode == 32 { + arch, triple = "386", "i386-unknown-linux-gnu" + } + // Retain every original physical address form and its actual Go + // bytes. This format has no initialized source/device addresses. + source := "TEXT enqueue(SB),$0-0\n" + x86RawDisplacementDirectives(test.code) + "RET\n" + ir := x86UnboundVoidRawScalarFormatObject(t, source, arch, triple) + for _, flag := range []string{"cf", "of", "slt", "pf"} { + if strings.Count(ir, "store i1 false, ptr %flags_"+flag) < 2 { + t.Fatalf("missing cleared %s status flag", flag) + } + } + if !strings.Contains(ir, "elementtype([64 x i8])") || !strings.Contains(ir, "; sete $0") { + t.Fatal("lost command width or device retry status") + } + compileLLVMToObject(t, llc, triple, "address.ll", "address.o", ir) + if segment := ins.Args[0].Mem.Segment; segment != "" { + cmd := exec.Command(llc, "-O0", "-mtriple="+triple, "-filetype=asm", "-o", "-", "-") + cmd.Stdin = strings.NewReader(ir) + assembly, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("emit assembly: %v\n%s", err, assembly) + } + if !strings.Contains(string(assembly), "%"+strings.ToLower(string(segment))+":") { + t.Fatalf("source segment override was lost:\n%s", assembly) + } + } + }) + } +} + +func TestX86RawEnqueueRejectsInvalidEncodings(t *testing.T) { + for _, code := range [][]byte{ + {0xf2, 0x0f, 0x38, 0xf8}, + {0xf2, 0x0f, 0x38, 0xf8, 0xc3}, + {0xf3, 0x0f, 0x38, 0xf8, 0x04}, + {0xf2, 0x0f, 0x38, 0xf8, 0x43}, + {0xf2, 0x0f, 0x38, 0xf8, 0x83, 0x01}, + {0xf0, 0xf2, 0x0f, 0x38, 0xf8, 0x03}, + } { + for _, mode := range []int{32, 64} { + if _, _, ok, err := decodedX86EnqueueInstruction(code, mode); !ok || err == nil { + t.Fatalf("accepted malformed %x in mode %d: recognized=%v err=%v", code, mode, ok, err) + } + } + } + for _, code := range [][]byte{ + {0x0f, 0x38, 0xf8, 0x03}, + {0xf2, 0x0f, 0x38, 0xf9, 0x03}, + {0xb8, 0xf2, 0x0f, 0x38, 0xf8}, + } { + if _, _, ok, _ := decodedX86EnqueueInstruction(code, 64); ok { + t.Fatalf("claimed another instruction: %x", code) + } + } +} + +func TestX86EnqueueRejectsInvalidOperands(t *testing.T) { + for _, op := range []string{"ENQCMD", "ENQCMDS"} { + for _, tail := range []string{" AX, BX", " (AX), X0", " (AX), (BX)", " (AX)", ".Z (AX), BX"} { + file, err := Parse(ArchAMD64, "TEXT invalid(SB),$0-0\n"+op+tail+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{ + "invalid": {Name: "invalid", Ret: Void}, + }}); err == nil { + t.Fatalf("accepted %s%s", op, tail) + } + } + } +} diff --git a/amd64_raw_evex_float_logical.go b/amd64_raw_evex_float_logical.go new file mode 100644 index 000000000..d702be118 --- /dev/null +++ b/amd64_raw_evex_float_logical.go @@ -0,0 +1,83 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86RawEVEXFloatLogicalInstruction covers Go 1.27's EVEX +// VAND/ANDN/OR/XOR PS/PD rows across X/Y/Z, masks, and scalar broadcast. +func decodedX86RawEVEXFloatLogicalInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86RawEVEXFloatLogicalInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86RawEVEXFloatLogicalInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 1 || p.opcode < 0x54 || p.opcode > 0x57 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX floating logical: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.addressOverride || p.vectorLength > 2 || p.pp > 1 || p.w != (p.pp == 1) { + return fail("invalid prefix or vector length") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires K1-K7") + } + if mode == 32 && p.mask != 0 { + return fail("386 masked form exceeds Go's three-operand frontend") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.broadcast && code[p.modRM]>>6 == 3 { + return fail("broadcast requires a memory source") + } + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + scale := 16 << p.vectorLength + if p.broadcast { + scale = 4 + if p.w { + scale = 8 + } + } + source, consumed, err := memory.evex( + code[p.modRM:], mode, p.b, p.x, p.segment, width, scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + stem := [...]string{"VAND", "VANDN", "VOR", "VXOR"}[p.opcode-0x54] + suffix := "PS" + if p.w { + suffix = "PD" + } + op := Op(stem + suffix) + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + destNumber := int(code[p.modRM]>>3&7) + p.r*8 + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destNumber))} + args := []Operand{source, second} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{ + Op: op, Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86Encoded: true, + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_evex_float_logical_test.go b/amd64_raw_evex_float_logical_test.go new file mode 100644 index 000000000..7ddbe7a90 --- /dev/null +++ b/amd64_raw_evex_float_logical_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawEVEXFloatLogicalEncoding(opcode, pp byte, width, mask int, zeroing, broadcast, rip bool) []byte { + p1 := byte(0x7c | pp) // EVEX.vvvv names Z0, fixed bit set. + if pp == 1 { + p1 |= 0x80 // PD requires EVEX.W1. + } + p2 := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // Z3 first source, Z2 destination. + if broadcast { + modRM = 0x13 // 0(BX) first source. + } + if rip { + modRM = 0x15 // RIP+disp32 first source. + } + code := []byte{0x62, 0xf1, p1, p2, opcode, modRM} + if rip { + code = append(code, 1, 0, 0, 0) + } + return code +} + +func x86RawEVEXFloatLogicalLiteral(opcode, pp byte, width, mask int, zeroing, broadcast bool) []byte { + code := x86RawEVEXFloatLogicalEncoding(opcode, pp, width, mask, zeroing, broadcast, true) + code = append(code, 0xc3) + dataWidth := width + if broadcast { + dataWidth = 4 + if pp == 1 { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*13+3)) + } + return code +} + +func TestDecodeX86RawEVEXFloatLogicalArithAVX512Regression(t *testing.T) { + // Go's assembler decodes these bytes as VANDPS 0(SI)(R8*4), Z0, Z1. + code := []byte{0x62, 0xb1, 0x7c, 0x48, 0x54, 0x0c, 0x86, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "arith AVX512 VANDPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VANDPS" || + decoded[0].Args[0].Kind != OpMem || + decoded[0].Args[0].Mem.Base != SI || + decoded[0].Args[0].Mem.Index != "R8" || + decoded[0].Args[0].Mem.Scale != 4 || + decoded[0].Args[1].Reg != "Z0" || + decoded[0].Args[2].Reg != "Z1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawEVEXFloatLogicalCompleteGo127Family(t *testing.T) { + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + for _, width := range []int{16, 32, 64} { + for _, masking := range []struct { + mask int + zero bool + }{ + {}, {mask: 3}, {mask: 7, zero: true}, + } { + for _, broadcast := range []bool{false, true} { + code := x86RawEVEXFloatLogicalEncoding(opcode, pp, width, masking.mask, masking.zero, broadcast, false) + got, length, ok, err := decodedX86RawEVEXFloatLogicalInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + want := Op(stem + suffix) + if broadcast { + want += ".BCST" + } + if masking.zero { + want += ".Z" + } + if got.Op != want || len(got.Args) != 3+map[bool]int{false: 0, true: 1}[masking.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, want) + } + if broadcast && got.Args[0].Kind != OpMem || !broadcast && got.Args[0].Reg == "" { + t.Fatalf("decode %x has wrong first source: %+v", code, got) + } + } + } + } + } + } + high := []byte{0x62, 0x01, 0x04, 0x40, 0x54, 0xff} + got, length, ok, err := decodedX86RawEVEXFloatLogicalInstruction(high, 64) + if err != nil || !ok || length != len(high) || got.Raw != "VANDPS Z31, Z31, Z31" { + t.Fatalf("high EVEX registers %x decoded as %+v, length=%d, ok=%v, err=%v", high, got, length, ok, err) + } +} + +func TestDecodeX86RawEVEXFloatLogicalRIPDataCompleteFamily(t *testing.T) { + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + name := fmt.Sprintf("%s%s/%d/bcst%t", stem, suffix, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawEVEXFloatLogicalLiteral(opcode, pp, width, 7, true, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + dataWidth := width + if broadcast { + dataWidth = 4 + if pp == 1 { + dataWidth = 8 + } + } + if len(decoded) != 2 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET || + !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-dataWidth:]) { + t.Fatalf("decoded floating logical literal %x as %#v", code, decoded) + } + }) + } + } + } + } +} + +func TestDecodeX86RawEVEXFloatLogicalRejectsUnsafeForms(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated literal": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { code[6] = 127; return code }, + "overlapping instruction": func(code []byte) []byte { code[6] = 0; return code }, + "segment override": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawEVEXFloatLogicalLiteral(0x54, 0, 64, 7, true, true)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe floating logical literal %x", code) + } + }) + } + for _, code := range [][]byte{ + x86RawEVEXFloatLogicalEncoding(0x54, 0, 16, 0, true, false, false), + {0x62, 0xf1, 0x7c, 0x19, 0x54, 0xd3}, // EVEX.b with a register source. + } { + if _, _, ok, err := decodedX86RawEVEXFloatLogicalInstruction(code, 64); !ok || err == nil { + t.Fatalf("accepted reserved floating logical form %x: ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawEVEXFloatLogicalRIPDataObjects(t *testing.T) { + var source strings.Builder + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + name := "literal" + stem + suffix + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawEVEXFloatLogicalLiteral(opcode, pp, 64, 7, true, true) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // The original complete bytes have no K7 producer. Preserve their + // objects and public Context, not a fabricated initialized mask. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-evex-float-logical-literal.ll", "raw-evex-float-logical-literal.o", ir) + }) + } +} diff --git a/amd64_raw_evex_insert_test.go b/amd64_raw_evex_insert_test.go new file mode 100644 index 000000000..8f9b80ced --- /dev/null +++ b/amd64_raw_evex_insert_test.go @@ -0,0 +1,216 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawEVEXPackedInsert(op Op, vectorBits byte, mask int, zeroing bool, first, second, destination int, immediate byte) []byte { + opcode := map[Op]byte{ + "VINSERTF32X4": 0x18, + "VINSERTF64X2": 0x18, + "VINSERTF32X8": 0x1a, + "VINSERTF64X4": 0x1a, + "VINSERTI32X4": 0x38, + "VINSERTI64X2": 0x38, + "VINSERTI32X8": 0x3a, + "VINSERTI64X4": 0x3a, + }[op] + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 3 + p1 := byte(^second&15)<<3 | 5 + if strings.Contains(string(op), "64") { + p1 |= 0x80 + } + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0x62, p0, p1, p2, opcode, modRM, immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedHasteEVEXPackedInsert(t *testing.T) { + code := []byte{0x62, 0xf3, 0xfd, 0x48, 0x38, 0x47, 0x01, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "haste VINSERTI64X2 sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VINSERTI64X2 $1, 16(DI), Z0, Z0" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawEVEXPackedInsertCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, family := range []struct { + ops []Op + vectorBits []byte + }{ + { + ops: []Op{"VINSERTF32X4", "VINSERTF64X2", "VINSERTI32X4", "VINSERTI64X2"}, + vectorBits: []byte{1, 2}, + }, + { + ops: []Op{"VINSERTF32X8", "VINSERTF64X4", "VINSERTI32X8", "VINSERTI64X4"}, + vectorBits: []byte{2}, + }, + } { + for _, op := range family.ops { + for _, vectorBits := range family.vectorBits { + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + firstName := "X" + if strings.HasSuffix(string(op), "X8") || strings.HasSuffix(string(op), "X4") && strings.Contains(string(op), "64X4") { + firstName = "Y" + } + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + immediate := byte(first*8 + destination) + code := encodeX86RawEVEXPackedInsert(op, vectorBits, masking.mask, masking.zeroing, first, second, destination, immediate) + got, length, ok, err := decodedX86EVEXPackedInsertInstruction(code, 64) + args := []string{ + fmt.Sprintf("$%d", immediate), + fmt.Sprintf("%s%d", firstName, first), + fmt.Sprintf("%s%d", vectorName, second), + } + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(op) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + } + if count != 1179648 { + t.Fatalf("covered %d EVEX packed-insert register encodings, want 1179648", count) + } +} + +func TestDecodedX86RawEVEXPackedInsertMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x62, 0x03, 0x15, 0xc3, 0x3a, 0x64, 0x8b, 0x02, 0xff} + got, length, ok, err := decodedX86EVEXPackedInsertInstruction(code, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 64} + if err != nil || !ok || length != len(code) || got.Op != "VINSERTI32X8.Z" || len(got.Args) != 5 || got.Args[0].Imm != 255 || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) || got.Args[2].String() != "Z29" || got.Args[3].String() != "K3" || got.Args[4].String() != "Z28" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86RawEVEXPackedInsert("VINSERTI64X2", 2, 0, false, 0, 1, 2, 3) + for name, mutate := range map[string]func([]byte){ + "missing EVEX fixed bit": func(code []byte) { code[2] &^= 0x04 }, + "EVEX.b": func(code []byte) { code[3] |= 0x10 }, + "reserved length": func(code []byte) { code[3] |= 0x60 }, + "zero without mask": func(code []byte) { code[3] |= 0x80 }, + "x4 with X output": func(code []byte) { code[3] &^= 0x60 }, + "x8 with Y output": func(code []byte) { + code[3] = code[3]&^0x60 | 0x20 + code[4] = 0x3a + }, + } { + invalid := append([]byte(nil), valid...) + mutate(invalid) + if instruction, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + for _, invalid := range [][]byte{ + {0x62}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5], valid[6]}, + {valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4], valid[5], valid[6]}, + {valid[0], valid[1], valid[2], valid[3], 0x19, valid[5], valid[6]}, + } { + if instruction, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("unrelated encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + truncatedImmediate := valid[:len(valid)-1] + if _, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(truncatedImmediate, 64); !matched || decodeErr == nil { + t.Fatalf("truncated immediate returned ok=%v err=%v", matched, decodeErr) + } + ripRelative := []byte{0x62, 0xf3, 0xfd, 0x48, 0x38, 0x05, 0, 0, 0, 0, 1} + if _, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(ripRelative, 64); !matched || decodeErr == nil { + t.Fatalf("RIP-relative encoding returned ok=%v err=%v", matched, decodeErr) + } + addressOverride := append([]byte{0x67}, valid...) + if _, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(addressOverride, 64); !matched || decodeErr == nil { + t.Fatalf("address-override encoding returned ok=%v err=%v", matched, decodeErr) + } + for _, op := range []Op{"VINSERTF32X4", "VINSERTF64X2", "VINSERTF32X8", "VINSERTF64X4", "VINSERTI32X4", "VINSERTI64X2", "VINSERTI32X8", "VINSERTI64X4"} { + vectorBits := byte(1) + if strings.HasSuffix(string(op), "X8") || strings.Contains(string(op), "64X4") { + vectorBits = 2 + } + invalid := encodeX86RawEVEXPackedInsert(op, vectorBits, 0, false, 0, 0, 1, 0) + if _, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(invalid, 32); !matched || decodeErr == nil { + t.Fatalf("386 %s encoding %x returned ok=%v err=%v", op, invalid, matched, decodeErr) + } + } +} + +func TestTranslateX86RawEVEXPackedInsertTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + code := [][]byte{ + encodeX86RawEVEXPackedInsert("VINSERTI64X2", 1, 3, true, 20, 20, 21, 1), + } + code = append(code, + []byte{0x62, 0xf3, 0xfd, 0x48, 0x38, 0x47, 0x01, 0x01}, + []byte{0x62, 0x03, 0x15, 0xc3, 0x3a, 0x64, 0x8b, 0x02, 0xff}, + ) + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_insert(SB),NOSPLIT,$0-0\n") + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_evex_packed_insert": {Name: "raw_evex_packed_insert", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-evex-packed-insert-"+target.name+".ll", "raw-evex-packed-insert-"+target.name+".o", ir) + }) + } +} diff --git a/amd64_raw_evex_madd.go b/amd64_raw_evex_madd.go new file mode 100644 index 000000000..ca5bd7395 --- /dev/null +++ b/amd64_raw_evex_madd.go @@ -0,0 +1,38 @@ +package plan9asm + +import ( + "fmt" +) + +// decodedX86EVEXPackedMADDInstruction covers the EVEX rows of Go's +// _yvandnpd optab shared by VPMADDUBSW and VPMADDWD. Neither row permits +// EVEX.b broadcasting, but both permit K masks and zeroing at X/Y/Z widths. +func decodedX86EVEXPackedMADDInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86EVEXPackedMADDInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXPackedMADDInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.pp != 1 { + return Instr{}, 0, false, nil + } + op := map[[2]int]Op{ + {1, 0xf5}: "VPMADDWD", + {2, 0x04}: "VPMADDUBSW", + }[[2]int{p.mapNumber, p.opcode}] + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX packed multiply-add: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.broadcast || p.addressOverride || p.vectorLength > 2 { + return fail("invalid EVEX fixed, width, broadcast, address, or vector-length bits") + } + // Both opcode-specific ISA rows are WIG. Full validated physical bytes, + // including mode32 masks, use the common private decoder producer. + return decodedX86BinaryVectorOperandsWithMemory(code, p, mode, op, 16, memory) +} diff --git a/amd64_raw_evex_madd_test.go b/amd64_raw_evex_madd_test.go new file mode 100644 index 000000000..d183a6462 --- /dev/null +++ b/amd64_raw_evex_madd_test.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawEVEXPackedMADD(op Op, width, mask, destination, first, second int, zeroing, memory bool) []byte { + opcode, opcodeMap := byte(0xf5), byte(1) + if op == "VPMADDUBSW" { + opcode, opcodeMap = 0x04, 2 + } + p0 := byte((1-(destination>>3)&1)<<7 | (1-(first>>4)&1)<<6 | + (1-(first>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | int(opcodeMap)) + p1 := byte(((^second)&15)<<3 | 0x05) + p2 := byte(width<<5 | (1-((second>>4)&1))<<3 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | first&7) + if memory { + modRM = byte(0x40 | (destination&7)<<3) + p0 = byte((1-(destination>>3)&1)<<7 | 0x60 | (1-(destination>>4)&1)<<4 | int(opcodeMap)) + } + code := []byte{0x62, p0, p1, p2, opcode, modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawEVEXMADDSeBiShogunRegression(t *testing.T) { + // Go's assembler disassembles these bytes as VPMADDUBSW Z1, Z5, Z1. + code := []byte{0x62, 0xf2, 0x55, 0x48, 0x04, 0xc9, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "mulInt8 AVX512 VPMADDUBSW", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPMADDUBSW" || + decoded[0].Args[0].Reg != "Z1" || decoded[0].Args[1].Reg != "Z5" || + decoded[0].Args[2].Reg != "Z1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawEVEXPackedMADDCompleteGoForms(t *testing.T) { + for _, op := range []Op{"VPMADDWD", "VPMADDUBSW"} { + for width := 0; width < 3; width++ { + vector := [...]string{"X", "Y", "Z"}[width] + for _, form := range []struct { + name string + mask int + zeroing bool + memory bool + destination int + first int + second int + }{ + {name: "register", destination: 2, first: 3, second: 4}, + {name: "high registers", destination: 18, first: 19, second: 20}, + {name: "masked", mask: 3, destination: 2, first: 3, second: 4}, + {name: "zeroing", mask: 7, zeroing: true, destination: 2, first: 3, second: 4}, + {name: "memory", memory: true, destination: 2, second: 4}, + {name: "zeroing memory", memory: true, mask: 5, zeroing: true, destination: 2, second: 4}, + } { + name := fmt.Sprintf("%s/%s/%s", op, vector, form.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawEVEXPackedMADD( + op, width, form.mask, form.destination, form.first, form.second, form.zeroing, form.memory, + ) + got, length, ok, err := decodedX86EVEXPackedMADDInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := op + if form.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[1].String() != fmt.Sprintf("%s%d", vector, form.second) || + got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s%d", vector, form.destination) { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + if !form.memory && got.Args[0].String() != fmt.Sprintf("%s%d", vector, form.first) { + t.Fatalf("decode %x first source = %s", code, got.Args[0]) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXPackedMADDRIPLiteral(t *testing.T) { + for _, op := range []Op{"VPMADDWD", "VPMADDUBSW"} { + for width := 0; width < 3; width++ { + code := encodeX86RawEVEXPackedMADD(op, width, 1, 2, 0, 4, false, false) + code[5] = 0x15 // RIP-relative first source, destination 2. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << width + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/width%d", op, width) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || len(got[0].x86RIPLiteralData) != literalWidth || got[0].Op != op { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestDecodeX86RawEVEXPackedMADDRejectsInvalidForms(t *testing.T) { + base := encodeX86RawEVEXPackedMADD("VPMADDUBSW", 2, 0, 2, 3, 4, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x10 }, // unsupported broadcast + func(code []byte) { code[3] |= 0x80 }, // zeroing without mask + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86EVEXPackedMADDInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawEVEXPackedMADDLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawEVEXPackedMADD(SB),$0-0\n") + for _, op := range []Op{"VPMADDWD", "VPMADDUBSW"} { + for width := 0; width < 3; width++ { + for _, memory := range []bool{false, true} { + code := encodeX86RawEVEXPackedMADD(op, width, 0, 2, 3, 4, false, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawEVEXPackedMADD": {Name: "rawEVEXPackedMADD", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-evex-madd.ll", "raw-evex-madd.o", ir) + }) + } +} diff --git a/amd64_raw_evex_packed_broadcast_test.go b/amd64_raw_evex_packed_broadcast_test.go new file mode 100644 index 000000000..9473f0b00 --- /dev/null +++ b/amd64_raw_evex_packed_broadcast_test.go @@ -0,0 +1,230 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawEVEXPackedBroadcast(op Op, vectorBits byte, mask int, zeroing, vectorSource bool, source, destination int) []byte { + opcode := byte(0) + width64 := false + if vectorSource { + switch op { + case "VPBROADCASTB": + opcode = 0x78 + case "VPBROADCASTW": + opcode = 0x79 + case "VPBROADCASTD": + opcode = 0x58 + case "VPBROADCASTQ": + opcode = 0x59 + width64 = true + } + } else { + switch op { + case "VPBROADCASTB": + opcode = 0x7a + case "VPBROADCASTW": + opcode = 0x7b + case "VPBROADCASTD": + opcode = 0x7c + case "VPBROADCASTQ": + opcode = 0x7c + width64 = true + } + } + p0 := byte((^destination>>3)&1)<<7 | + byte((^source>>4)&1)<<6 | + byte((^source>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 2 + p1 := byte(0x7d) + if width64 { + p1 |= 0x80 + } + p2 := vectorBits<<5 | 0x08 | byte(mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | source&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestDecodeX86RawDirectiveGroupReportedHasteEVEXBroadcast(t *testing.T) { + code := []byte{0x62, 0x52, 0xfd, 0x48, 0x7c, 0xf2} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "haste VPBROADCASTQ sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VPBROADCASTQ R10, Z14" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawEVEXPackedBroadcastCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, op := range []Op{"VPBROADCASTB", "VPBROADCASTW", "VPBROADCASTD", "VPBROADCASTQ"} { + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for _, sourceKind := range []struct { + vector bool + count int + prefix string + }{ + {count: 16, prefix: "R"}, + {vector: true, count: 32, prefix: "X"}, + } { + for source := 0; source < sourceKind.count; source++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXPackedBroadcast(op, byte(vectorBits), masking.mask, masking.zeroing, sourceKind.vector, source, destination) + got, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + args := []string{fmt.Sprintf("%s%d", sourceKind.prefix, source)} + if !sourceKind.vector { + args[0] = decodedX86RegisterNameForTest(source) + } + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(op) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 55296 { + t.Fatalf("covered %d EVEX packed-broadcast register encodings, want 55296", count) + } +} + +func TestDecodedX86RawEVEXPackedBroadcastMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x65, 0x62, 0x12, 0x7d, 0x2b, 0x58, 0x64, 0x8b, 0x20} + got, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + wantMemory := MemRef{Segment: GS, Base: "R11", Index: "R9", Scale: 4, Off: 128} + if err != nil || !ok || length != len(code) || got.Op != "VPBROADCASTD" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) || got.Args[1].String() != "K3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, false, 0, 0) + for name, mutate := range map[string]func([]byte){ + "W on byte": func(code []byte) { code[2] |= 0x80; code[4] = 0x7a }, + "used vvvv": func(code []byte) { code[2] &^= 0x08 }, + "used high vvvv": func(code []byte) { code[3] &^= 0x08 }, + "EVEX.b": func(code []byte) { code[3] |= 0x10 }, + "reserved length": func(code []byte) { code[3] |= 0x60 }, + "zero without mask": func(code []byte) { code[3] |= 0x80 }, + "GP high extension": func(code []byte) { code[1] &^= 0x40 }, + "GP opcode on memory": func(code []byte) { code[5] &^= 0xc0 }, + } { + invalid := append([]byte(nil), valid...) + mutate(invalid) + if instruction, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + for _, invalid := range [][]byte{ + {0x62}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2], valid[3], 0x77, valid[5]}, + } { + if instruction, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("unrelated encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + + ripRelative := []byte{0x62, 0xf2, 0x7d, 0x08, 0x58, 0x05, 0, 0, 0, 0} + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(ripRelative, 64); !matched || decodeErr == nil { + t.Fatalf("RIP-relative encoding returned ok=%v err=%v", matched, decodeErr) + } + addressOverride := append([]byte{0x67}, valid...) + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(addressOverride, 64); !matched || decodeErr == nil { + t.Fatalf("address-override encoding returned ok=%v err=%v", matched, decodeErr) + } + for name, invalid := range map[string][]byte{ + "extended destination on 386": encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, false, 0, 8), + "extended source on 386": encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, true, 8, 0), + } { + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(invalid, 32); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, invalid, matched, decodeErr) + } + } + valid386 := encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, false, 7, 7) + got, length, ok, err = decodedX86VEXPackedBroadcastInstruction(valid386, 32) + if err != nil || !ok || length != len(valid386) || got.Raw != "VPBROADCASTD DI, X7" { + t.Fatalf("386 base-register form decoded as %+v, length=%d ok=%v err=%v", got, length, ok, err) + } +} + +func TestTranslateX86RawEVEXPackedBroadcastTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + code := [][]byte{ + encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, false, 0, 1), + encodeX86RawEVEXPackedBroadcast("VPBROADCASTB", 1, 1, true, true, 1, 2), + } + if target.goarch == "amd64" { + code = append(code, []byte{0x62, 0x52, 0xfd, 0x48, 0x7c, 0xf2}) + } + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_broadcast(SB),NOSPLIT,$0-0\n") + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-evex-packed-broadcast-"+target.name+".ll", "raw-evex-packed-broadcast-"+target.name+".o", ir) + + setup := "MOVL $19, AX\nMOVL $37, CX\nMOVL CX, X1\nKMOVW CX, K1\n" + if target.goarch == "amd64" { + setup += "MOVL $79, R10\n" + } + initialized := strings.Replace(source.String(), "\n", "\n"+setup, 1) + x86LEAActualTargetGoObject(t, target.goarch, target.triple, "#include \"textflag.h\"\n"+initialized) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, initialized, "func raw_evex_packed_broadcast()", target.goarch, target.triple, api) + if err != nil { + t.Fatal("initialized original raw broadcast sequence", err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + }) + } + }) + } +} diff --git a/amd64_raw_evex_packed_float_move_test.go b/amd64_raw_evex_packed_float_move_test.go new file mode 100644 index 000000000..9ee8ff8bd --- /dev/null +++ b/amd64_raw_evex_packed_float_move_test.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawEVEXPackedFloatMoveCompleteGo127Family(t *testing.T) { + // Go 1.27's _yvmovapd table gives VMOVAPS/APD/UPS/UPD the same + // bidirectional X/Y/Z and K-masked EVEX forms in addition to the four + // legacy VEX rows. These LLVM-verified encodings cover every EVEX opcode, + // direction, vector width, mask mode, high-register field, and compressed + // disp8 addressing shape used by that table. + tests := []struct { + name string + code []byte + want string + }{ + {name: "aps x load masked", code: []byte{0x62, 0xf1, 0x7c, 0x09, 0x28, 0xc1}, want: "VMOVAPS X1, K1, X0"}, + {name: "aps x store compressed disp8", code: []byte{0x62, 0xf1, 0x7c, 0x0a, 0x29, 0x50, 0x01}, want: "VMOVAPS X2, K2, 16(AX)"}, + {name: "apd y high load masked", code: []byte{0x62, 0xa1, 0xfd, 0x29, 0x28, 0xe5}, want: "VMOVAPD Y21, K1, Y20"}, + {name: "apd y store masked sib", code: []byte{0x62, 0x91, 0xfd, 0x2a, 0x29, 0x64, 0x88, 0x03}, want: "VMOVAPD Y4, K2, 96(R8)(R9*4)"}, + {name: "apd z high load zeroing", code: []byte{0x62, 0xa1, 0xfd, 0xcb, 0x28, 0xe5}, want: "VMOVAPD.Z Z21, K3, Z20"}, + {name: "ups z high load zeroing", code: []byte{0x62, 0x01, 0x7c, 0xcf, 0x10, 0xfe}, want: "VMOVUPS.Z Z30, K7, Z31"}, + {name: "ups z store compressed disp8", code: []byte{0x62, 0xf1, 0x7c, 0x4e, 0x11, 0x7f, 0x07}, want: "VMOVUPS Z7, K6, 448(DI)"}, + {name: "upd x load masked", code: []byte{0x62, 0xf1, 0xfd, 0x09, 0x10, 0xd3}, want: "VMOVUPD X3, K1, X2"}, + {name: "upd x store compressed disp8", code: []byte{0x62, 0xf1, 0xfd, 0x0a, 0x11, 0x60, 0x02}, want: "VMOVUPD X4, K2, 32(AX)"}, + {name: "ups y memory load zeroing sib", code: []byte{0x62, 0x91, 0x7c, 0xaa, 0x10, 0x64, 0x88, 0x03}, want: "VMOVUPS.Z 96(R8)(R9*4), K2, Y4"}, + {name: "ups y high store masked", code: []byte{0x62, 0x41, 0x7c, 0x2b, 0x11, 0x4c, 0x24, 0x02}, want: "VMOVUPS Y25, K3, 64(R12)"}, + {name: "upd z high store masked sib", code: []byte{0x62, 0x01, 0xfd, 0x4b, 0x11, 0x4c, 0x6c, 0x02}, want: "VMOVUPD Z25, K3, 128(R12)(R13*2)"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := decodeX86RawDirectives(rawX86Function(test.code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || !strings.HasPrefix(got.Instrs[0].Raw, test.want+" ") { + t.Fatalf("decoded %#x as %#v, want %q", test.code, got.Instrs, test.want) + } + }) + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_float_move(SB),NOSPLIT,$0-0\n") + for _, test := range tests { + for _, value := range test.code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidVectorMoveFormatObject(t, source.String(), "amd64", "x86_64-unknown-linux-gnu") + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-evex-packed-float-move.ll", "raw-evex-packed-float-move.o", ir) +} + +func TestX86RawEVEXPackedFloatMoveReportedMinioInstruction(t *testing.T) { + code := []byte{0x62, 0xc1, 0x7c, 0x48, 0x10, 0x04, 0x09} + got, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || !strings.HasPrefix(got.Instrs[0].Raw, "VMOVUPS 0(R9)(CX*1), Z16 ") { + t.Fatalf("decoded reported minio bytes as %#v", got.Instrs) + } +} diff --git a/amd64_raw_evex_packed_integer_move_test.go b/amd64_raw_evex_packed_integer_move_test.go new file mode 100644 index 000000000..7aaaa7bf1 --- /dev/null +++ b/amd64_raw_evex_packed_integer_move_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawEVEXPackedIntegerMoveCompleteGo127Family(t *testing.T) { + // Go 1.27 gives VMOVDQA32/64 and VMOVDQU8/16/32/64 the same + // 12-row _yvmovdqa32 table: X/Y/Z, load/store, with and without a + // nonzero K mask. These LLVM-verified encodings exercise every opcode, + // both directions, every vector width, high registers, SIB addressing, + // zeroing, and EVEX compressed disp8 scaling. + tests := []struct { + name string + code []byte + want string + }{ + {name: "dqa32 x load", code: []byte{0x62, 0xf1, 0x7d, 0x08, 0x6f, 0xc1}, want: "VMOVDQA32 X1, X0"}, + {name: "dqa32 x store compressed disp8", code: []byte{0x62, 0xf1, 0x7d, 0x08, 0x7f, 0x50, 0x01}, want: "VMOVDQA32 X2, 16(AX)"}, + {name: "dqa64 y high load masked", code: []byte{0x62, 0xa1, 0xfd, 0x29, 0x6f, 0xe5}, want: "VMOVDQA64 Y21, K1, Y20"}, + {name: "dqa64 y store masked sib", code: []byte{0x62, 0x91, 0xfd, 0x2a, 0x7f, 0x64, 0x88, 0x03}, want: "VMOVDQA64 Y4, K2, 96(R8)(R9*4)"}, + {name: "dqu8 z high load zeroing", code: []byte{0x62, 0x01, 0x7f, 0xcf, 0x6f, 0xfe}, want: "VMOVDQU8.Z Z30, K7, Z31"}, + {name: "dqu8 z store compressed disp8", code: []byte{0x62, 0xf1, 0x7f, 0x4e, 0x7f, 0x7f, 0x07}, want: "VMOVDQU8 Z7, K6, 448(DI)"}, + {name: "dqu16 x load masked", code: []byte{0x62, 0xf1, 0xff, 0x09, 0x6f, 0xd3}, want: "VMOVDQU16 X3, K1, X2"}, + {name: "dqu16 x store compressed disp8", code: []byte{0x62, 0xf1, 0xff, 0x0a, 0x7f, 0x60, 0x02}, want: "VMOVDQU16 X4, K2, 32(AX)"}, + {name: "dqu32 y memory load zeroing sib", code: []byte{0x62, 0x91, 0x7e, 0xaa, 0x6f, 0x64, 0x88, 0x03}, want: "VMOVDQU32.Z 96(R8)(R9*4), K2, Y4"}, + {name: "dqu32 y high store masked", code: []byte{0x62, 0x41, 0x7e, 0x2b, 0x7f, 0x4c, 0x24, 0x02}, want: "VMOVDQU32 Y25, K3, 64(R12)"}, + {name: "dqu64 z high load masked", code: []byte{0x62, 0x01, 0xfe, 0x4f, 0x6f, 0xf7}, want: "VMOVDQU64 Z31, K7, Z30"}, + {name: "dqu64 z high store masked sib", code: []byte{0x62, 0x01, 0xfe, 0x4b, 0x7f, 0x4c, 0x6c, 0x02}, want: "VMOVDQU64 Z25, K3, 128(R12)(R13*2)"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := decodeX86RawDirectives(rawX86Function(test.code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || !strings.HasPrefix(got.Instrs[0].Raw, test.want+" ") { + t.Fatalf("decoded %#x as %#v, want %q", test.code, got.Instrs, test.want) + } + }) + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_integer_move(SB),NOSPLIT,$0-0\n") + for _, test := range tests { + for _, value := range test.code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidVectorMoveFormatObject(t, source.String(), "amd64", "x86_64-unknown-linux-gnu") + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-evex-packed-integer-move.ll", "raw-evex-packed-integer-move.o", ir) +} + +func TestX86RawEVEXPackedIntegerMoveReportedMinioSequence(t *testing.T) { + var code []byte + for register := byte(0); register < 8; register++ { + modRM := byte(0x07 | register<<3) + code = append(code, 0x62, 0xf1, 0x7e, 0x48, 0x6f, modRM) + if register != 0 { + code[len(code)-1] |= 0x40 + code = append(code, register) + } + } + got, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 8 { + t.Fatalf("decoded minio sequence as %d instructions: %#v", len(got.Instrs), got.Instrs) + } + for index, instruction := range got.Instrs { + want := fmt.Sprintf("VMOVDQU32 %d(DI), Z%d", index*64, index) + if !strings.HasPrefix(instruction.Raw, want+" ") { + t.Fatalf("instruction %d = %q, want %q", index, instruction.Raw, want) + } + } +} + +func TestX86RawEVEXPackedIntegerMoveZeroingRegisterStoreEncoding(t *testing.T) { + for _, spec := range []struct { + op Op + pp int + w byte + }{{"VMOVDQA32", 1, 0}, {"VMOVDQA64", 1, 0x80}, {"VMOVDQU8", 3, 0}, {"VMOVDQU16", 3, 0x80}, {"VMOVDQU32", 2, 0}, {"VMOVDQU64", 2, 0x80}} { + for width, prefix := range []string{"X", "Y", "Z"} { + for src := 0; src < 32; src++ { + for dst := 0; dst < 32; dst++ { + code := encodeRawScalarMove(true, spec.pp, 0x11, src, 0, dst, 7, true) + code[2] = code[2]&0x7f | spec.w + code[3] |= byte(width) << 5 + code[4] = 0x7f + got, size, matched, err := decodedX86EVEXPackedIntegerMoveInstruction(code, 64) + want := fmt.Sprintf("%s.Z %s%d, K7, %s%d", spec.op, prefix, src, prefix, dst) + if err != nil || !matched || size != len(code) || got.Raw != want { + t.Fatalf("decode %x=%q size=%d matched=%v err=%v, want %q", code, got.Raw, size, matched, err, want) + } + } + } + } + } +} + +func TestX86RawEVEXPackedIntegerMoveRejectsInvalidEVEXForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + }{ + {name: "broadcast bit", code: []byte{0x62, 0xf1, 0x7e, 0x58, 0x6f, 0xc1}}, + {name: "zeroing without mask", code: []byte{0x62, 0xf1, 0x7e, 0xc8, 0x6f, 0xc1}}, + {name: "zeroing store", code: []byte{0x62, 0xf1, 0x7e, 0x89, 0x7f, 0x08}}, + {name: "reserved vector length", code: []byte{0x62, 0xf1, 0x7e, 0x68, 0x6f, 0xc1}}, + {name: "address override", code: []byte{0x67, 0x62, 0xf1, 0x7e, 0x48, 0x6f, 0x07}}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, ok, err := decodedX86EVEXPackedIntegerMoveInstruction(test.code, 64); !ok || err == nil { + t.Fatalf("decoded invalid encoding %#x with ok=%v err=%v", test.code, ok, err) + } + }) + } +} + +func rawX86Function(code []byte) Func { + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + return fn +} diff --git a/amd64_raw_evex_packed_logical_test.go b/amd64_raw_evex_packed_logical_test.go new file mode 100644 index 000000000..5d07749b1 --- /dev/null +++ b/amd64_raw_evex_packed_logical_test.go @@ -0,0 +1,247 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawEVEXPackedLogical(op Op, vectorBits byte, mask int, zeroing, broadcast bool, first, second, destination int) []byte { + opcode := map[Op]byte{ + "VPANDD": 0xdb, + "VPANDQ": 0xdb, + "VPANDND": 0xdf, + "VPANDNQ": 0xdf, + "VPORD": 0xeb, + "VPORQ": 0xeb, + "VPXORD": 0xef, + "VPXORQ": 0xef, + }[op] + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 1 + p1 := byte(^second&15)<<3 | 5 + if strings.HasSuffix(string(op), "Q") { + p1 |= 0x80 + } + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestDecodeX86RawDirectiveGroupReportedHasteEVEXPackedLogical(t *testing.T) { + code := []byte{0x62, 0xf1, 0xf5, 0x48, 0xef, 0xc9} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "haste VPXORQ sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VPXORQ Z1, Z1, Z1" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawEVEXPackedLogicalCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, op := range []Op{"VPANDD", "VPANDQ", "VPANDND", "VPANDNQ", "VPORD", "VPORQ", "VPXORD", "VPXORQ"} { + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXPackedLogical(op, byte(vectorBits), masking.mask, masking.zeroing, false, first, second, destination) + got, length, ok, err := decodedX86EVEXPackedLogicalInstruction(code, 64) + args := []string{ + fmt.Sprintf("%s%d", vectorName, first), + fmt.Sprintf("%s%d", vectorName, second), + } + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(op) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 2359296 { + t.Fatalf("covered %d EVEX packed-logical register encodings, want 2359296", count) + } +} + +func TestDecodedX86RawEVEXPackedLogicalMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x62, 0x01, 0x95, 0xd3, 0xdb, 0x64, 0x8b, 0x20} + got, length, ok, err := decodedX86EVEXPackedLogicalInstruction(code, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 256} + if err != nil || !ok || length != len(code) || got.Op != "VPANDQ.BCST.Z" || len(got.Args) != 4 || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) || got.Args[1].String() != "Z29" || got.Args[2].String() != "K3" || got.Args[3].String() != "Z28" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86RawEVEXPackedLogical("VPXORD", 0, 0, false, false, 0, 1, 2) + for name, mutate := range map[string]func([]byte){ + "missing EVEX fixed bit": func(code []byte) { code[2] &^= 0x04 }, + "reserved length": func(code []byte) { code[3] |= 0x60 }, + "zero without mask": func(code []byte) { code[3] |= 0x80 }, + "broadcast register": func(code []byte) { code[3] |= 0x10 }, + } { + invalid := append([]byte(nil), valid...) + mutate(invalid) + if instruction, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + for _, invalid := range [][]byte{ + {0x62}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2], valid[3], 0xee, valid[5]}, + } { + if instruction, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("unrelated encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + ripRelative := []byte{0x62, 0xf1, 0x75, 0x08, 0xef, 0x15, 0, 0, 0, 0} + if _, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(ripRelative, 64); !matched || decodeErr == nil { + t.Fatalf("RIP-relative encoding returned ok=%v err=%v", matched, decodeErr) + } + addressOverride := append([]byte{0x67}, valid...) + if _, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(addressOverride, 64); !matched || decodeErr == nil { + t.Fatalf("address-override encoding returned ok=%v err=%v", matched, decodeErr) + } + for name, invalid := range map[string][]byte{ + "masked on 386": encodeX86RawEVEXPackedLogical("VPXORD", 0, 1, false, false, 0, 1, 2), + "high Z first on 386": encodeX86RawEVEXPackedLogical("VPXORD", 2, 0, false, false, 8, 1, 2), + "high Z second on 386": encodeX86RawEVEXPackedLogical("VPXORD", 2, 0, false, false, 0, 8, 2), + "high Z destination 386": encodeX86RawEVEXPackedLogical("VPXORD", 2, 0, false, false, 0, 1, 8), + } { + if _, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(invalid, 32); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, invalid, matched, decodeErr) + } + } + valid386 := encodeX86RawEVEXPackedLogical("VPXORD", 1, 0, false, false, 29, 30, 31) + got, length, ok, err = decodedX86EVEXPackedLogicalInstruction(valid386, 32) + if err != nil || !ok || length != len(valid386) || got.Raw != "VPXORD Y29, Y30, Y31" { + t.Fatalf("386 high X/Y register form decoded as %+v, length=%d ok=%v err=%v", got, length, ok, err) + } +} + +func TestTranslateX86RawEVEXPackedLogicalTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + first, second, destination := 29, 30, 31 + if target.goarch == "386" { + // Only the physical low bank belongs to this object fixture. + // The original high-bank bytes remain a separate Context case. + first, second, destination = 5, 6, 7 + } + code := [][]byte{ + encodeX86RawEVEXPackedLogical("VPXORD", 1, 0, false, false, first, second, destination), + } + if target.goarch == "amd64" { + code = append(code, + []byte{0x62, 0xf1, 0xf5, 0x48, 0xef, 0xc9}, + []byte{0x62, 0x01, 0x95, 0xd3, 0xdb, 0x64, 0x8b, 0x20}, + ) + } + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_logical(SB),NOSPLIT,$0-0\n") + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + if target.goarch == "amd64" { + // This original includes a memory access with no address or + // mask input. Keep complete format objects and public Context. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-evex-packed-logical-"+target.name+".ll", "raw-evex-packed-logical-"+target.name+".o", ir) + return + } + // The original mode32 body only discards a pure register XOR; + // it neither accesses native memory nor promises result bytes. + x86LEAActualTargetGoObject(t, target.goarch, target.triple, "#include \"textflag.h\"\n"+source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func raw_evex_packed_logical()", target.goarch, target.triple, api) + if err != nil { + t.Fatal("original pure dead XOR acquired an unrelated input requirement", api, err) + } + compileLLVMToObject(t, llc, target.triple, "raw-evex-packed-logical-"+target.name+".ll", "raw-evex-packed-logical-"+target.name+".o", ir) + } + }) + } +} + +func TestX86RawEVEXPackedLogicalMode32HighRegistersNeedContext(t *testing.T) { + checked := 0 + for _, op := range []Op{"VPANDD", "VPANDQ", "VPANDND", "VPANDNQ", "VPORD", "VPORQ", "VPXORD", "VPXORQ"} { + for _, vectorBits := range []byte{0, 1} { + for _, position := range []int{0, 1, 2} { + for _, high := range []int{8, 15, 16, 31} { + registers := [3]int{0, 1, 2} + registers[position] = high + code := encodeX86RawEVEXPackedLogical(op, vectorBits, 0, false, false, registers[0], registers[1], registers[2]) + var source strings.Builder + source.WriteString("TEXT raw_mode32_high(SB),4,$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"raw_mode32_high": {Name: "raw_mode32_high", Ret: Void}}, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("raw mode32 high-bank bytes %x acquired an unproved physical binding: %v", code, err) + } + checked++ + } + } + } + } + if checked != 192 { + t.Fatalf("checked %d raw mode32 high-bank cases, want 192", checked) + } +} diff --git a/amd64_raw_evex_qword.go b/amd64_raw_evex_qword.go new file mode 100644 index 000000000..79aff5aca --- /dev/null +++ b/amd64_raw_evex_qword.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// These complete EVEX-only qword families share physical field decoding, +// not operand authority or their independent numerical and memory effects. +func decodedX86EVEXQwordBinaryInstruction(code []byte, mode int, lookup bool) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 || code[i+1]&0x0f != 2 || code[i+2]&0x07 != 5 { + return Instr{}, 0, false, nil + } + op := Op("") + switch code[i+4] { + case 0xb4: + if lookup { + return Instr{}, 0, false, nil + } + op = "VPMADD52LUQ" + case 0xb5: + if lookup { + return Instr{}, 0, false, nil + } + op = "VPMADD52HUQ" + case 0x83, 0x8f: + if !lookup { + return Instr{}, 0, false, nil + } + op = "VPMULTISHIFTQB" + if code[i+4] == 0x8f { + op = "VPSHUFBITQMB" + } + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if (p1&0x80 != 0) != (op != "VPSHUFBITQMB") { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match %s", op) + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if op == "VPSHUFBITQMB" && (zeroing || p2&0x10 != 0) { + return Instr{}, 0, true, fmt.Errorf("mask-result lookup has no zeroing or broadcast encoding") + } + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if op == "VPSHUFBITQMB" && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("%w: unbound extended K destination field", ErrProbeNeedsContext) + } + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = 8 + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destinationName := vectorName + if op == "VPSHUFBITQMB" { + destinationName = "K" + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationName, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} diff --git a/amd64_raw_evex_unpack.go b/amd64_raw_evex_unpack.go new file mode 100644 index 000000000..d3444054f --- /dev/null +++ b/amd64_raw_evex_unpack.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawEVEXUnpackForm struct { + op Op + w bool + broadcastBytes int +} + +// Go 1.27's eight EVEX integer interleave rows. Dword and QDQ sources may +// use scalar-memory broadcast; byte and word rows may not. +var x86RawEVEXUnpackForms = map[int]x86RawEVEXUnpackForm{ + 0x60: {op: "VPUNPCKLBW"}, + 0x61: {op: "VPUNPCKLWD"}, + 0x62: {op: "VPUNPCKLDQ", broadcastBytes: 4}, + 0x6c: {op: "VPUNPCKLQDQ", w: true, broadcastBytes: 8}, + 0x68: {op: "VPUNPCKHBW"}, + 0x69: {op: "VPUNPCKHWD"}, + 0x6a: {op: "VPUNPCKHDQ", broadcastBytes: 4}, + 0x6d: {op: "VPUNPCKHQDQ", w: true, broadcastBytes: 8}, +} + +func decodedX86EVEXPackedUnpackInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 { + return Instr{}, 0, false, nil + } + form, recognized := x86RawEVEXUnpackForms[p.opcode] + if !recognized { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX packed unpack: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.w != form.w || p.addressOverride || p.vectorLength > 2 { + return fail("invalid EVEX fixed, element width, address, or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if mode == 32 && p.mask != 0 { + return fail("386 mask form exceeds the Go assembler frontend's operand limit") + } + modRM := code[p.modRM] + if p.broadcast && (form.broadcastBytes == 0 || modRM>>6 == 3) { + return fail("broadcast requires an enabled scalar-memory row") + } + destinationNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && + (destinationNumber >= 8 || p.upper >= 8 || + modRM>>6 == 3 && int(modRM&7)+p.b*8+p.x*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("%w: EVEX unpack high register has no proved physical mode32 binding", ErrProbeNeedsContext) + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + disp8Scale := 16 << p.vectorLength + if p.broadcast { + disp8Scale = form.broadcastBytes + } + first, consumed, err := decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, vector, disp8Scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + op := form.op + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, destinationNumber))} + args := []Operand{first, second} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_evex_unpack_test.go b/amd64_raw_evex_unpack_test.go new file mode 100644 index 000000000..7c09d5ae9 --- /dev/null +++ b/amd64_raw_evex_unpack_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawEVEXUnpack(opcode, width, mask int, zeroing, broadcast, memory bool) []byte { + form := x86RawEVEXUnpackForms[opcode] + p1 := byte(0x5d) // EVEX.vvvv = source register 4, pp = 66. + if form.w { + p1 |= 0x80 + } + p2 := byte(width<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // first register 3, destination register 2. + if memory { + modRM = 0x50 // disp8(AX), destination register 2. + } + code := []byte{0x62, 0xf1, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawEVEXUnpackSeBiShogunRegression(t *testing.T) { + // Go's assembler disassembles these bytes as VPUNPCKHDQ Z2, Z3, Z7. + code := []byte{0x62, 0xf1, 0x65, 0x48, 0x6a, 0xfa, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "onesCountInt32 AVX512 VPUNPCKHDQ", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPUNPCKHDQ" || + decoded[0].Args[0].Reg != "Z2" || decoded[0].Args[1].Reg != "Z3" || + decoded[0].Args[2].Reg != "Z7" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawEVEXUnpackCompleteGoForms(t *testing.T) { + if len(x86RawEVEXUnpackForms) != 8 { + t.Fatalf("modeled %d EVEX unpack rows, want eight", len(x86RawEVEXUnpackForms)) + } + for opcode, form := range x86RawEVEXUnpackForms { + for width := 0; width < 3; width++ { + vector := [...]string{"X", "Y", "Z"}[width] + for _, variant := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "memory", memory: true}, + {name: "zeroing memory", mask: 7, zeroing: true, memory: true}, + {name: "broadcast memory", broadcast: true, memory: true}, + {name: "broadcast zero memory", broadcast: true, mask: 7, zeroing: true, memory: true}, + } { + if variant.broadcast && form.broadcastBytes == 0 { + continue + } + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawEVEXUnpack(opcode, width, variant.mask, variant.zeroing, variant.broadcast, variant.memory) + got, length, ok, err := decodedX86EVEXPackedUnpackInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.broadcast { + wantOp += ".BCST" + } + if variant.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[1].String() != vector+"4" || + got.Args[len(got.Args)-1].String() != vector+"2" || + !variant.memory && got.Args[0].String() != vector+"3" { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXUnpackRIPLiteral(t *testing.T) { + for opcode, form := range x86RawEVEXUnpackForms { + for width := 0; width < 3; width++ { + for _, broadcast := range []bool{false, true} { + if broadcast && form.broadcastBytes == 0 { + continue + } + code := encodeX86RawEVEXUnpack(opcode, width, 1, false, broadcast, false) + code[5] = 0x15 // RIP-relative first source, destination 2. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << width + if broadcast { + literalWidth = form.broadcastBytes + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/width%d/broadcast%t", form.op, width, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + wantOp := form.op + if broadcast { + wantOp += ".BCST" + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != literalWidth || got[0].Op != wantOp { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXUnpackRejectsInvalidForms(t *testing.T) { + base := encodeX86RawEVEXUnpack(0x6a, 2, 0, false, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[2] |= 0x80 }, // HDQ requires EVEX.W0 + func(code []byte) { code[3] |= 0x80 }, // zeroing needs a mask + func(code []byte) { code[3] |= 0x10 }, // broadcast requires memory + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86EVEXPackedUnpackInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + byteBroadcast := encodeX86RawEVEXUnpack(0x60, 2, 0, false, true, true) + if _, _, ok, err := decodedX86EVEXPackedUnpackInstruction(byteBroadcast, 64); !ok || err == nil { + t.Fatalf("accepted VPUNPCKLBW broadcast %x: ok=%v err=%v", byteBroadcast, ok, err) + } +} + +func TestTranslateX86RawEVEXUnpackLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawEVEXUnpack(SB),$0-0\n") + for opcode, form := range x86RawEVEXUnpackForms { + for width := 0; width < 3; width++ { + for _, variant := range []struct { + memory, broadcast bool + }{ + {}, {memory: true}, {memory: true, broadcast: true}, + } { + if variant.broadcast && form.broadcastBytes == 0 { + continue + } + code := encodeX86RawEVEXUnpack(opcode, width, 0, false, variant.broadcast, variant.memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + // Operand formats do not initialize the original vector/address + // inputs. Keep public rejection separate from emitter objects. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-evex-unpack.ll", "raw-evex-unpack.o", ir) + }) + } +} diff --git a/amd64_raw_expand_test.go b/amd64_raw_expand_test.go new file mode 100644 index 000000000..dd7dce1be --- /dev/null +++ b/amd64_raw_expand_test.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawExpandForms = []struct { + opcode int + widthBit int + op Op + laneBytes int +}{ + {0x62, 0, "VPEXPANDB", 1}, + {0x62, 1, "VPEXPANDW", 2}, + {0x88, 0, "VEXPANDPS", 4}, + {0x88, 1, "VEXPANDPD", 8}, + {0x89, 0, "VPEXPANDD", 4}, + {0x89, 1, "VPEXPANDQ", 8}, +} + +func encodeX86RawExpand(opcode, widthBit, vectorBits, mask int, zeroing, memory bool) []byte { + p1 := byte(0x7d) // EVEX.vvvv is unused, pp = 66. + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xd3) // source register 3, destination register 2. + if memory { + modRM = 0x50 // disp8(AX) source, destination register 2. + } + code := []byte{0x62, 0xf2, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawExpandSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VEXPANDPS.Z Z2, K1, Z3. + code := []byte{0x62, 0xf2, 0x7d, 0xc9, 0x88, 0xda, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "fastCumSumFloat32 VEXPANDPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VEXPANDPS.Z" || + decoded[0].Args[0].Reg != "Z2" || decoded[0].Args[1].Reg != "K1" || + decoded[0].Args[2].Reg != "Z3" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawExpandCompleteGoForms(t *testing.T) { + if len(x86RawExpandForms) != len(amd64PackedExpandSpecs) { + t.Fatalf("raw/lower expand grammar rows = %d/%d", len(x86RawExpandForms), len(amd64PackedExpandSpecs)) + } + for _, form := range x86RawExpandForms { + for vectorBits, vector := range []string{"X", "Y", "Z"} { + for _, variant := range []struct { + name string + mask int + zeroing bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "zeroing register", mask: 7, zeroing: true}, + {name: "memory", memory: true}, + {name: "masked memory", mask: 1, memory: true}, + {name: "zeroing memory", mask: 4, zeroing: true, memory: true}, + } { + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawExpand(form.opcode, form.widthBit, vectorBits, variant.mask, variant.zeroing, variant.memory) + got, length, ok, err := decodedX86PackedExpandInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != vector+"2" || + !variant.memory && got.Args[0].String() != vector+"3" || + variant.memory && got.Args[0].String() != fmt.Sprintf("%d(AX)", form.laneBytes) || + len(got.Args) != 2+map[bool]int{true: 1}[variant.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawExpandRejectsInvalidForms(t *testing.T) { + base := encodeX86RawExpand(0x88, 0, 2, 0, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x80 }, // zeroing without mask + func(code []byte) { code[3] |= 0x60 }, // reserved vector length + func(code []byte) { code[3] |= 0x10 }, // broadcast unavailable + func(code []byte) { code[2] &^= 0x08 }, // EVEX.vvvv is reserved + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86PackedExpandInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + badMode32 := encodeX86RawExpand(0x88, 0, 2, 1, false, false) + badMode32[1] &^= 0x80 // The destination field has no physical mode32 register. + if _, _, ok, err := decodedX86PackedExpandInstruction(badMode32, 32); !ok || err == nil { + t.Fatalf("accepted unbound mode32 destination bank: ok=%v err=%v", ok, err) + } +} + +func TestTranslateX86RawExpandLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawExpand(SB),$0-0\n") + for _, form := range x86RawExpandForms { + for _, memory := range []bool{false, true} { + mask := 1 + if target.goarch == "386" { + mask = 0 + } + code := encodeX86RawExpand(form.opcode, form.widthBit, 0, mask, false, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + // These original bytes do not establish numerical/native inputs. + // Preserve Go, public Context and emitter objects independently + // from the complete initialized raw public-object counterparts. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-expand.ll", "raw-expand.o", ir) + }) + } +} diff --git a/amd64_raw_explicit_mask_move.go b/amd64_raw_explicit_mask_move.go new file mode 100644 index 000000000..ff3ee6daa --- /dev/null +++ b/amd64_raw_explicit_mask_move.go @@ -0,0 +1,67 @@ +package plan9asm + +import ( + "fmt" +) + +// Go 1.27's _yvmaskmovpd table describes the four VEX X/Y masked load/store +// families. Opcode bit 1 selects store; W selects the D/Q integer lane. +var x86RawExplicitMaskMoveOps = map[int]Op{ + 0x2c: "VMASKMOVPS", + 0x2d: "VMASKMOVPD", + 0x8c: "VPMASKMOVD", +} + +func decodedX86ExplicitMaskMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.evex || p.mapNumber != 2 { + return Instr{}, 0, false, nil + } + // The floating store opcodes are 2E/2F; integer stores use 8E. + base := p.opcode &^ 2 + op, known := x86RawExplicitMaskMoveOps[base] + if !known { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("explicit mask move: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.pp != 1 || p.vectorLength > 1 || p.w && base != 0x8c { + return fail("invalid VEX prefix or width") + } + if base == 0x8c && p.w { + op = "VPMASKMOVQ" + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if code[p.modRM]>>6 == 3 { + return fail("explicit mask move requires a memory operand") + } + width := "X" + if p.vectorLength == 1 { + width = "Y" + } + memory, consumed, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]>>3&7)+p.r*8))} + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + args := []Operand{memory, mask, data} + if p.opcode&2 != 0 { + args = []Operand{data, mask, memory} + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s, %s", op, args[0].String(), args[1].String(), args[2].String()), + x86Encoded: true, + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_explicit_mask_move_test.go b/amd64_raw_explicit_mask_move_test.go new file mode 100644 index 000000000..a063fbd19 --- /dev/null +++ b/amd64_raw_explicit_mask_move_test.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawExplicitMaskMoveGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawmaskmoveforms(SB),4,$0-0\n") + for _, op := range []string{"VMASKMOVPS", "VMASKMOVPD", "VPMASKMOVD", "VPMASKMOVQ"} { + for _, width := range []string{"X", "Y"} { + for _, reg := range []int{1, 7, 15} { + fmt.Fprintf(&source, "%s 8(AX), %s2, %s%d\n", op, width, width, reg) + fmt.Fprintf(&source, "%s %s%d, %s2, 16(BX)\n", op, width, reg, width) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d = %s %v, want %s %v", index, got.Op, got.Args, want[index].Op, want[index].Args) + } + } + if len(code) == 0 || code[len(code)-1] != 0xc3 { + t.Fatal("Go assembler did not end the fixture with RET") + } + var raw strings.Builder + raw.WriteString("TEXT rawmaskmoveforms(SB),4,$0-0\n") + for _, value := range code[:len(code)-1] { + fmt.Fprintf(&raw, "BYTE $%#02x\n", value) + } + raw.WriteString("RET\n") + file, err := Parse(ArchAMD64, raw.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + signatures := map[string]FuncSig{"rawmaskmoveforms": {Name: "rawmaskmoveforms", Ret: Void}} + ir, err := Translate(file, Options{ + Goarch: arch, TargetTriple: triple, + Sigs: signatures, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original raw formats borrowed missing native address/data/mask inputs", err) + } + if arch == "386" { + // The decoder's exact original high-bank inventory stays above; + // physical mode32 success is owned by separate initialized + // low-bank named/raw objects, not by rewriting these bytes. + continue + } + normalized, err := normalizeX86RawFile(file, arch) + if err != nil { + t.Fatal(err) + } + ir = x86ScalarFormatObjectUnit(t, normalized, signatures, arch, triple, 0) + compileLLVMToObject(t, llc, triple, "raw-explicit-mask-move.ll", "raw-explicit-mask-move.o", ir) + } + }) + } +} + +func TestX86RawExplicitMaskMoveRejectsInvalidEncodings(t *testing.T) { + for _, test := range []struct { + name string + code []byte + }{ + {name: "missing ModRM", code: []byte{0xc4, 0xa2, 0x69, 0x8e}}, + {name: "register destination", code: []byte{0xc4, 0xe2, 0x69, 0x8e, 0xc1}}, + {name: "wrong pp", code: []byte{0xc4, 0xa2, 0x68, 0x8e, 0x0c, 0x82}}, + {name: "floating W1", code: []byte{0xc4, 0xe2, 0xe9, 0x2c, 0x01}}, + {name: "address override", code: []byte{0x67, 0xc4, 0xa2, 0x69, 0x8e, 0x0c, 0x82}}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, matched, err := decodedX86ExplicitMaskMoveInstruction(test.code, 64); !matched || err == nil { + t.Fatalf("invalid %x matched=%v error=%v", test.code, matched, err) + } + }) + } + for _, code := range [][]byte{ + {0xc4, 0xe3, 0x69, 0x8e, 0xc1}, + {0x62, 0xe2, 0x69, 0x08, 0x8e, 0xc1}, + } { + if _, _, matched, err := decodedX86ExplicitMaskMoveInstruction(code, 64); matched || err != nil { + t.Fatalf("other opcode %x matched=%v error=%v", code, matched, err) + } + } +} diff --git a/amd64_raw_far_return.go b/amd64_raw_far_return.go new file mode 100644 index 000000000..1d07064c4 --- /dev/null +++ b/amd64_raw_far_return.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// decodedX86FarReturn preserves the actual operand-size attribute before +// x86asm.GoSyntax erases LRET's width or emits Intel's IRET/IRETD aliases. +// In both 32- and 64-bit modes a bare far return has a 32-bit operand size; +// only an effective REX.W promotes it to 64 bits. See Intel SDM volume 2, +// RET and IRET/IRETD/IRETQ, and Go's shared x86 ynone/yret encoder rows. +func decodedX86FarReturn(inst x86asm.Inst) (syntax string, recognized bool, err error) { + kind := amd64InterruptReturn + switch inst.Op { + case x86asm.LRET: + kind = amd64FarReturn + case x86asm.IRET: + if inst.DataSize != 16 { + return "", true, fmt.Errorf("raw IRET has inconsistent operand size %d", inst.DataSize) + } + case x86asm.IRETD: + if inst.DataSize != 32 { + return "", true, fmt.Errorf("raw IRETD has inconsistent operand size %d", inst.DataSize) + } + case x86asm.IRETQ: + if inst.DataSize != 64 { + return "", true, fmt.Errorf("raw IRETQ has inconsistent operand size %d", inst.DataSize) + } + default: + return "", false, nil + } + if (inst.Mode != 32 && inst.Mode != 64) || (inst.Mode == 32 && inst.DataSize == 64) { + return "", true, fmt.Errorf("raw %s operand size %d is not supported in mode %d", inst.Op, inst.DataSize, inst.Mode) + } + for _, prefix := range inst.Prefix { + if prefix == 0 { + continue + } + // Register only the size prefixes represented by the typed family. + // Never silently drop LOCK, REP, segment or address-size prefixes. + // Decoder-rejected prefix orderings remain explicit decode failures. + if prefix&x86asm.PrefixInvalid != 0 || (prefix&0xff != x86asm.PrefixDataSize && !prefix.IsREX()) { + return "", true, fmt.Errorf("raw %s has an invalid or unregistered prefix %s", inst.Op, prefix) + } + if prefix.IsREX() && inst.Mode != 64 { + return "", true, fmt.Errorf("raw %s has a REX prefix outside 64-bit mode", inst.Op) + } + } + for _, extra := range inst.Args[1:] { + if extra != nil { + return "", true, fmt.Errorf("raw %s has extra operands", inst.Op) + } + } + argument := "" + if inst.Args[0] != nil { + immediate, ok := inst.Args[0].(x86asm.Imm) + if kind != amd64FarReturn || !ok || immediate < 0 || immediate > 65535 { + return "", true, fmt.Errorf("raw %s requires no operand or an encoded unsigned imm16", inst.Op) + } + argument = fmt.Sprintf(" $%d", immediate) + } + for op, spec := range amd64FarReturnSpecs { + if spec.kind == kind && spec.bits == inst.DataSize { + return string(op) + argument, true, nil + } + } + return "", true, fmt.Errorf("raw %s has unregistered operand size %d", inst.Op, inst.DataSize) +} diff --git a/amd64_raw_far_return_test.go b/amd64_raw_far_return_test.go new file mode 100644 index 000000000..0b3e61679 --- /dev/null +++ b/amd64_raw_far_return_test.go @@ -0,0 +1,408 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86FarReturnGrammarMatchesActualGoRows(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86", "asm6.go") + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + t.Fatal(err) + } + want := make(map[Op]amd64FarReturnSpec) + ast.Inspect(file, func(node ast.Node) bool { + row, ok := node.(*ast.CompositeLit) + if !ok || len(row.Elts) == 0 { + return true + } + name, ok := row.Elts[0].(*ast.Ident) + if !ok || (!strings.HasPrefix(name.Name, "AIRET") && !strings.HasPrefix(name.Name, "ARETF")) { + return true + } + if len(row.Elts) != 4 { + t.Fatalf("unrecognized actual Go far-return opcode row %s", name.Name) + } + operands, ok := row.Elts[1].(*ast.Ident) + if !ok || (operands.Name != "ynone" && operands.Name != "yret") { + t.Fatalf("unrecognized actual Go far-return operand row: %#v", row.Elts[1]) + } + prefix, ok := row.Elts[2].(*ast.Ident) + if !ok { + t.Fatalf("unrecognized actual Go far-return prefix: %#v", row.Elts[2]) + } + bits := map[string]int{"Pe": 16, "Px": 32, "Pw": 64}[prefix.Name] + if bits == 0 { + t.Fatalf("unrecognized actual Go far-return width prefix %s", prefix.Name) + } + kind := amd64InterruptReturn + if operands.Name == "yret" { + kind = amd64FarReturn + } + want[Op(strings.TrimPrefix(name.Name, "A"))] = amd64FarReturnSpec{ + kind: kind, bits: bits, allowImmediate: operands.Name == "yret", + } + return false + }) + if len(want) != len(amd64FarReturnSpecs) { + t.Fatalf("actual Go rows %v do not match typed inventory %v", want, amd64FarReturnSpecs) + } + for op, expected := range want { + spec, ok := amd64FarReturnSpecs[op] + if !ok || spec != expected { + t.Fatalf("actual Go %s spec=%+v is missing or differs", op, expected) + } + } + // The row names alone are not operand completeness: audit both entire + // referenced ytabs, including the operand-free RETF and imm16 truncation. + seen := make(map[string]bool) + ast.Inspect(file, func(node ast.Node) bool { + value, ok := node.(*ast.ValueSpec) + if !ok || len(value.Names) != 1 || (value.Names[0].Name != "ynone" && value.Names[0].Name != "yret") { + return true + } + name := value.Names[0].Name + if len(value.Values) != 1 { + t.Fatalf("unrecognized %s initializer", name) + } + table, ok := value.Values[0].(*ast.CompositeLit) + if !ok { + t.Fatalf("unrecognized %s table", name) + } + classes := make(map[string]bool) + for _, element := range table.Elts { + row, ok := element.(*ast.CompositeLit) + if !ok || len(row.Elts) != 3 { + t.Fatalf("unrecognized %s operand row", name) + } + operands, ok := row.Elts[2].(*ast.CompositeLit) + if !ok || len(operands.Elts) > 1 { + t.Fatalf("unrecognized %s argument list", name) + } + class := "none" + if len(operands.Elts) == 1 { + identifier, ok := operands.Elts[0].(*ast.Ident) + if !ok { + t.Fatalf("unrecognized %s argument class", name) + } + class = identifier.Name + } + if classes[class] { + t.Fatalf("duplicate %s argument class %s", name, class) + } + classes[class] = true + } + wantCount := 1 + if name == "yret" { + wantCount = 2 + } + if len(classes) != wantCount || !classes["none"] || (name == "yret" && !classes["Yi32"]) { + t.Fatalf("actual Go %s forms changed: %v", name, classes) + } + seen[name] = true + return false + }) + if len(seen) != 2 { + t.Fatal("actual Go far-return operand tables were not both observed") + } +} + +func TestX86RawFarReturnPrefixAndModeAxes(t *testing.T) { + for _, test := range []struct { + name string + mode int + code []byte + want string + }{ + {name: "bare-386", mode: 32, code: []byte{0xcb}, want: "RETFL"}, + {name: "bare-amd64", mode: 64, code: []byte{0xcb}, want: "RETFL"}, + {name: "operand-386", mode: 32, code: []byte{0x66, 0xcb}, want: "RETFW"}, + {name: "operand-amd64", mode: 64, code: []byte{0x66, 0xcb}, want: "RETFW"}, + {name: "rex-without-w", mode: 64, code: []byte{0x47, 0xcb}, want: "RETFL"}, + {name: "rex-w", mode: 64, code: []byte{0x4f, 0xcb}, want: "RETFQ"}, + {name: "rex-w-overrides-66", mode: 64, code: []byte{0x66, 0x48, 0xcb}, want: "RETFQ"}, + {name: "repeated-operand", mode: 64, code: []byte{0x66, 0x66, 0xcf}, want: "IRETW"}, + } { + t.Run(test.name, func(t *testing.T) { + inst, err := x86asm.Decode(test.code, test.mode) + if err != nil || inst.Len != len(test.code) { + t.Fatalf("actual decoder: %+v %v", inst, err) + } + got, err := decodedX86GoSyntax(inst, test.code) + if err != nil || got != test.want { + t.Fatalf("actual prefixes %+v DataSize%d: %q %v, want %q", inst.Prefix, inst.DataSize, got, err, test.want) + } + }) + } + // 48 is an actual DEC opcode in 32-bit mode. Never treat the same + // byte string as an unsupported 64-bit far return in that mode. + decoded, err := decodeX86RawDirectiveGroup([]byte{0x48, 0xcb}, 32, 0, "32-bit DEC then far return", nil) + if err != nil || len(decoded) != 2 || decoded[0].Op != "DECL" || decoded[1].Op != "RETFL" { + t.Fatalf("32-bit 48 CB changed instruction boundaries: %+v %v", decoded, err) + } + decoded, err = decodeX86RawDirectiveGroup([]byte{0x48, 0x66, 0xcb}, 32, 0, "32-bit DEC then 16-bit far return", nil) + if err != nil || len(decoded) != 2 || decoded[0].Op != "DECL" || decoded[1].Op != "RETFW" { + t.Fatalf("32-bit 48 66 CB changed instruction boundaries: %+v %v", decoded, err) + } +} + +func TestX86RawFarReturnRejectsUnprovedForms(t *testing.T) { + for _, code := range [][]byte{ + {0xf0, 0xcb}, {0xf0, 0xcf}, {0xf2, 0xcb}, {0xf3, 0xcf}, + {0x67, 0xcb}, {0x64, 0xcf}, // Not a registered width-only prefix. + {0xca}, {0xca, 0x01}, {0x66}, {0x48}, + {0x48, 0x66, 0xcb}, // Pinned decoder rejects nonadjacent REX. + } { + for _, mode := range []int{32, 64} { + if mode == 32 && code[0] == 0x48 { + continue // Valid DEC, not a REX prefix or prefix ordering. + } + _, err := decodeX86RawDirectiveGroup(code, mode, 0, "invalid or unregistered raw far return", nil) + if err == nil { + t.Errorf("accepted unproved %x mode%d", code, mode) + } + } + } + for _, inst := range []x86asm.Inst{ + {Op: x86asm.LRET, Mode: 16, DataSize: 16}, + {Op: x86asm.LRET, Mode: 32, DataSize: 64}, + {Op: x86asm.LRET, Mode: 64, DataSize: 8}, + {Op: x86asm.IRETQ, Mode: 64, DataSize: 32}, + {Op: x86asm.IRETD, Mode: 64, DataSize: 16}, + {Op: x86asm.LRET, Mode: 64, DataSize: 32, Args: x86asm.Args{x86asm.Imm(-1)}}, + {Op: x86asm.LRET, Mode: 64, DataSize: 32, Args: x86asm.Args{x86asm.Imm(65536)}}, + {Op: x86asm.LRET, Mode: 64, DataSize: 32, Args: x86asm.Args{x86asm.EAX}}, + {Op: x86asm.LRET, Mode: 64, DataSize: 32, Args: x86asm.Args{nil, x86asm.Imm(1)}}, + {Op: x86asm.IRETD, Mode: 64, DataSize: 32, Args: x86asm.Args{x86asm.Imm(1)}}, + } { + if got, err := decodedX86GoSyntax(inst, nil); err == nil { + t.Errorf("accepted inconsistent raw form %+v as %q", inst, got) + } + } +} + +func TestX86FarReturnRejectsFloatingAndWidthlessOperands(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + instructions := []string{"LRET", "LRET $4", "RETFL $unknown"} + for op := range amd64FarReturnSpecs { + for _, argument := range []string{"$0.0", "$-0.0", "$1.5", "$-1.5"} { + instructions = append(instructions, string(op)+" "+argument) + } + } + for _, instruction := range instructions { + source := "TEXT bad(SB),516,$0-0\n" + instruction + "\n" + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if _, err := Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Errorf("accepted %s %q outside actual Go integer/width grammar", arch, instruction) + } + } + } +} + +func TestX86RawFarReturnRetainsDecodedWidthAndImmediate(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, width := range []struct { + bits int + prefix []byte + suffix string + }{ + {bits: 16, prefix: []byte{0x66}, suffix: "W"}, + {bits: 32, suffix: "L"}, + {bits: 64, prefix: []byte{0x48}, suffix: "Q"}, + } { + if mode == 32 && width.bits == 64 { + continue // 48 is DEC in 32-bit mode, not a width prefix. + } + for _, form := range []struct { + name string + code []byte + args string + }{ + {name: "IRET", code: []byte{0xcf}}, + {name: "RETF", code: []byte{0xcb}}, + {name: "RETF", code: []byte{0xca, 0x00, 0x00}, args: " $0"}, + {name: "RETF", code: []byte{0xca, 0x34, 0x12}, args: " $4660"}, + {name: "RETF", code: []byte{0xca, 0xff, 0xff}, args: " $65535"}, + } { + code := append(append([]byte(nil), width.prefix...), form.code...) + inst, err := x86asm.Decode(code, mode) + if err != nil || inst.Len != len(code) || inst.DataSize != width.bits { + t.Fatalf("actual decoder %x mode%d: %+v %v", code, mode, inst, err) + } + want := form.name + width.suffix + form.args + got, err := decodedX86GoSyntax(inst, code) + if err != nil || got != want { + t.Errorf("%x mode%d DataSize%d: got %q %v, want %q", code, mode, inst.DataSize, got, err, want) + } + decoded, err := decodeX86RawDirectiveGroup(code, mode, 0, "raw far-return bytes", nil) + if err != nil || len(decoded) != 1 || string(decoded[0].Op) != form.name+width.suffix { + t.Errorf("whole raw group %x mode%d: %+v %v", code, mode, decoded, err) + } + } + } + } +} + +func TestX86RawFarReturnCompleteFiveTargetObjectsMatchGo(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 is required") + } + for _, target := range []struct{ os, arch, triple string }{ + {os: "darwin", arch: "amd64", triple: "x86_64-apple-darwin"}, + {os: "linux", arch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {os: "windows", arch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {os: "linux", arch: "386", triple: "i386-unknown-linux-gnu"}, + {os: "windows", arch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var rawSource strings.Builder + sigs := make(map[string]FuncSig) + wantBytes := make(map[string][]byte) + for op, spec := range amd64FarReturnSpecs { + if target.arch == "386" && spec.bits == 64 { + continue + } + forms := []amd64FarReturnForm{{}} + if spec.allowImmediate { + for _, value := range []uint16{0, 1, 0x1234, 0xffff} { + forms = append(forms, amd64FarReturnForm{hasImmediate: true, immediate: value}) + } + } + for _, form := range forms { + name := fmt.Sprintf("far_%s_%t_%d", op, form.hasImmediate, form.immediate) + prefix := []byte(nil) + if spec.bits == 16 { + prefix = []byte{0x66} + } else if spec.bits == 64 { + prefix = []byte{0x48} + } + code := append(prefix, byte(0xcb)) + if spec.kind == amd64InterruptReturn { + code[len(code)-1] = 0xcf + } else if form.hasImmediate { + code[len(code)-1] = 0xca + code = append(code, byte(form.immediate), byte(form.immediate>>8)) + } + raw := fmt.Sprintf("TEXT %s(SB),516,$0-0\n", name) + for _, value := range code { + raw += fmt.Sprintf("BYTE $0x%02x\n", value) + } + named := fmt.Sprintf("TEXT %s(SB),516,$0-0\n%s", name, op) + if form.hasImmediate { + named += fmt.Sprintf(" $%d", form.immediate) + } + named += "\n" + // Independently authenticate the raw and canonical named + // encodings with Go, not merely the LLVM assembly spelling. + for _, source := range []string{raw, named} { + dir := t.TempDir() + path := filepath.Join(dir, "far.s") + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + command := exec.Command("go", "tool", "asm", "-S", "-o", filepath.Join(dir, "far.o"), path) + command.Env = append(os.Environ(), "GOOS="+target.os, "GOARCH="+target.arch, "GOTOOLCHAIN=local") + output, err := command.CombinedOutput() + parts := make([]string, len(code)) + for i, value := range code { + parts[i] = fmt.Sprintf("%02x", value) + } + want := regexp.MustCompile(`(?m)^\s*0x0000 ` + strings.Join(parts, " ") + `(?:\s|$)`) + if err != nil || !want.Match(output) { + t.Fatalf("original Go bytes %x: %v\n%s\n%s", code, err, source, output) + } + } + rawSource.WriteString(raw) + sigs[name] = FuncSig{Name: name, Ret: Void} + wantBytes[name] = code + } + } + file, err := Parse(ArchAMD64, rawSource.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for op, spec := range amd64FarReturnSpecs { + if target.arch == "386" && spec.bits == 64 { + continue + } + if !strings.Contains(ir, spec.encoding(amd64FarReturnForm{})) { + t.Errorf("raw %s lost typed byte lowering", op) + } + } + if strings.Count(ir, "unreachable") != len(sigs) { + t.Fatal("raw far return lost terminal control semantics") + } + requireX86RawFarReturnObjectBytes(t, llc, target.triple, ir, wantBytes) + }) + } +} + +func requireX86RawFarReturnObjectBytes(t *testing.T, llc, triple, ir string, want map[string][]byte) { + t.Helper() + objdump := findLLVM22Tool("llvm-objdump") + if objdump == "" { + t.Fatal("LLVM22 llvm-objdump is required") + } + dir := t.TempDir() + llPath := filepath.Join(dir, "far.ll") + object := filepath.Join(dir, "far.o") + if err := os.WriteFile(llPath, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + output, err := exec.Command(llc, "-O0", "-mtriple="+triple, "-filetype=obj", llPath, "-o", object).CombinedOutput() + if err != nil { + t.Fatalf("LLVM22 object: %v\n%s", err, output) + } + output, err = exec.Command(objdump, "-d", object).CombinedOutput() + if err != nil { + t.Fatalf("LLVM22 disassembly: %v\n%s", err, output) + } + // Match each terminal instruction's complete bytes inside its own symbol, + // not merely a coincidental sequence in another function or an IR string. + sections := regexp.MustCompile(`(?m)^[0-9a-f]+ <_?(far_[^>]+)>:\n`).FindAllSubmatchIndex(output, -1) + seen := make(map[string]bool) + for index, section := range sections { + name := string(output[section[2]:section[3]]) + code, ok := want[name] + if !ok { + t.Fatalf("unexpected emitted far-return symbol %s", name) + } + end := len(output) + if index+1 < len(sections) { + end = sections[index+1][0] + } + parts := make([]string, len(code)) + for i, value := range code { + parts[i] = fmt.Sprintf("%02x", value) + } + instruction := regexp.MustCompile(`(?m)^\s*[0-9a-f]+:\s+` + strings.Join(parts, " ") + `\s+(?:iret|lret)`) + if !instruction.Match(output[section[1]:end]) { + t.Fatalf("LLVM22 symbol %s lacks exact terminal bytes %x:\n%s", name, code, output[section[1]:end]) + } + seen[name] = true + } + if len(seen) != len(want) { + t.Fatalf("LLVM22 object contains %d/%d authenticated far-return symbols", len(seen), len(want)) + } +} diff --git a/amd64_raw_float_logical_literal_test.go b/amd64_raw_float_logical_literal_test.go new file mode 100644 index 000000000..6e966888d --- /dev/null +++ b/amd64_raw_float_logical_literal_test.go @@ -0,0 +1,109 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawFloatLogicalLiteral(opcode, pp byte, vex3 bool, width int) []byte { + code := encodeX86VEXPackedFloatLogical(opcode, pp, vex3, width == 32, false, 1, 2, 0) + code[len(code)-1] = 0x0d // RIP+disp32 rather than X0; destination stays X/Y1. + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*13+5)) + } + return code +} + +func TestDecodeX86RawFloatLogicalRIPDataCompleteVEXFamily(t *testing.T) { + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + for _, vex3 := range []bool{false, true} { + for _, width := range []int{16, 32} { + name := fmt.Sprintf("%s%s/vex3%t/%d", stem, suffix, vex3, width) + t.Run(name, func(t *testing.T) { + code := x86RawFloatLogicalLiteral(opcode, pp, vex3, width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != Op(stem+suffix) || + len(decoded[0].Args) != 3 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s%s source-local data", code, decoded, stem, suffix) + } + if !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-width:]) { + t.Fatalf("source-local data = %x, want %x", decoded[0].x86RIPLiteralData, code[len(code)-width:]) + } + }) + } + } + } + } +} + +func TestDecodeX86RawFloatLogicalRIPDataRejectsUnsafeSources(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { + code[5] = 127 + return code + }, + "overlapping instruction": func(code []byte) []byte { + code[5] = 0 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawFloatLogicalLiteral(0x57, 1, true, 32)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe floating logical RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawFloatLogicalRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + name := fmt.Sprintf("literal%s%s", stem, suffix) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawFloatLogicalLiteral(opcode, pp, true, 32) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-float-logical-literal.ll", "raw-float-logical-literal.o", ir) + }) + } +} diff --git a/amd64_raw_fma3_literal_test.go b/amd64_raw_fma3_literal_test.go new file mode 100644 index 000000000..5713e0539 --- /dev/null +++ b/amd64_raw_fma3_literal_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "sort" + "strings" + "testing" +) + +func x86FMA3Opcodes() []byte { + opcodes := make([]byte, 0, len(decodedX86VEXFMA3Ops)) + for opcode := range decodedX86VEXFMA3Ops { + opcodes = append(opcodes, opcode) + } + sort.Slice(opcodes, func(i, j int) bool { return opcodes[i] < opcodes[j] }) + return opcodes +} + +func x86RawFMA3Literal(opcode byte, width64, evex bool, width int, broadcast bool) []byte { + spec := decodedX86VEXFMA3Ops[opcode] + var code []byte + if evex { + code = encodeX86EVEXFMA3(opcode, width64, byte(map[int]int{16: 0, 32: 1, 64: 2}[width]), broadcast, true, 7, 0, 0, 0) + } else { + code = encodeX86VEXFMA3(opcode, width64, width == 32, 0, 0, 0) + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if spec.scalar || broadcast { + dataWidth = 4 + if width64 { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*31+13)) + } + return code +} + +func TestDecodeX86RawFMA3RIPDataCompleteFamily(t *testing.T) { + for _, opcode := range x86FMA3Opcodes() { + spec := decodedX86VEXFMA3Ops[opcode] + for _, width64 := range []bool{false, true} { + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + if spec.scalar { + widths = []int{16} + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && !spec.scalar { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", spec.op(width64), evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawFMA3Literal(opcode, width64, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := spec.op(width64) + if broadcast { + want += ".BCST" + } + if evex { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } + } +} + +func TestDecodeX86RawFMA3RIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawFMA3Literal(0xb8, false, true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawFMA3RIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, opcode := range x86FMA3Opcodes() { + spec := decodedX86VEXFMA3Ops[opcode] + width64 := index%2 == 0 + evex := index%3 == 0 + width := 32 + if spec.scalar { + width = 16 + } else if evex { + width = 64 + } + broadcast := evex && !spec.scalar + code := x86RawFMA3Literal(opcode, width64, evex, width, broadcast) + name := fmt.Sprintf("fma3Literal%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if spec.scalar || broadcast { + dataWidth = 4 + if width64 { + dataWidth = 8 + } + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Preserve the entire original raw RET and DATA directory as actual + // Go objects plus three-API Context: it has no vector/mask inputs. + // TestX86RawFMA3InitializedRIPWholeSourceAndRawReturn separately + // owns all 60 genuinely initialized public RIP counterparts. + x86UnboundVoidFormatSources(t, source.String(), "amd64", triple) + ir := x86ScalarFormatObjectUnit(t, normalized, sigs, "amd64", triple, 0) + for _, function := range normalized.Funcs { + for _, instruction := range function.Instrs { + form, known, err := amd64FMA3Properties(instruction, "amd64") + if !known { + continue + } + if err != nil { + t.Fatal("original RIP format lost its physical operand grammar", err) + } + mnemonic, _, _ := strings.Cut(strings.ToLower(string(instruction.Op)), ".") + x86RawFMA3RequireOriginalNativeFormat(t, ir, mnemonic, form.byteWidth, form.memoryBytes, + x86FMA3NativeEVEX(instruction, form)) + } + } + compileLLVMToObject(t, llc, triple, "fma3-literal.ll", "fma3-literal.o", ir) + }) + } +} diff --git a/amd64_raw_fp16_binary_test.go b/amd64_raw_fp16_binary_test.go new file mode 100644 index 000000000..af05ca28e --- /dev/null +++ b/amd64_raw_fp16_binary_test.go @@ -0,0 +1,240 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var rawFP16BinaryOpcodes = map[byte]Op{ + 0x58: "VADDPH", + 0x59: "VMULPH", + 0x5c: "VSUBPH", + 0x5d: "VMINPH", + 0x5e: "VDIVPH", + 0x5f: "VMAXPH", +} + +func encodeRawFP16Binary(opcode, width byte, source1, source2, destination, mask int, memory, broadcast, zero bool) []byte { + code := encodeX86EVEXBinaryFloat(opcode, 0, width, broadcast, zero, mask, destination, source1, source2) + code[1] = code[1]&^byte(0x0f) | 5 + if memory { + code[5] = byte(0x40 | (destination&7)<<3) + code = append(code, 1) + } + return code +} + +func TestDecodeRawFP16BinaryGoATReproduction(t *testing.T) { + // VADDPH 64(SI)(R8*2), Z1, Z1 from go-highway's GoAT output. + code := []byte{0x62, 0xb5, 0x74, 0x48, 0x58, 0x4c, 0x46, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "GoAT VADDPH", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VADDPH" { + t.Fatalf("decoded %#v, want one VADDPH", decoded) + } +} + +func TestDecodeRawFP16BinaryCompleteForms(t *testing.T) { + for opcode, baseOp := range rawFP16BinaryOpcodes { + for width := byte(0); width < 3; width++ { + for _, memory := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && !memory { + continue + } + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + for _, zero := range []bool{false, true} { + if zero && !masked { + continue + } + code := encodeRawFP16Binary(opcode, width, 20, 21, 22, mask, memory, broadcast, zero) + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("%x: decoded %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := baseOp + if broadcast { + wantOp += ".BCST" + } + if zero { + wantOp += ".Z" + } + prefix := [...]string{"X", "Y", "Z"}[width] + if got.Op != wantOp || !got.x86Encoded || + got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s22", prefix) || + got.Args[1].String() != fmt.Sprintf("%s20", prefix) { + t.Fatalf("%x: decoded %+v, want %s on %s", code, got, wantOp, prefix) + } + if !memory && got.Args[0].String() != fmt.Sprintf("%s21", prefix) { + t.Fatalf("%x: wrong register source %+v", code, got) + } + if masked && (len(got.Args) != 4 || got.Args[2].String() != "K3") { + t.Fatalf("%x: missing mask %+v", code, got) + } + } + } + } + } + } + } +} + +func TestTranslateRawFP16BinaryLLVM22Targets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16Binary(SB),4,$0-0\n") + for opcode := range rawFP16BinaryOpcodes { + for width := byte(0); width < 3; width++ { + for _, memory := range []bool{false, true} { + code := encodeRawFP16Binary(opcode, width, 1, 2, 3, 1, memory, memory, true) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Keep the entire original format source. Its absent numerical + // inputs require public Context; initialized counterparts below + // own complete public objects, independently of this emitter unit. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + if strings.Count(ir, "asm sideeffect") != 36 { + t.Fatal("original packed FP16 format inventory changed") + } + for _, op := range rawFP16BinaryOpcodes { + if !strings.Contains(ir, strings.ToLower(string(op))+" ") { + t.Fatal("missing original native FP16 operation", op) + } + } + for _, control := range []string{"{1to8}", "{1to16}", "{1to32}", "kmovd "} { + if !strings.Contains(ir, control) { + t.Fatal("missing original broadcast or full mask", control) + } + } + compileLLVMToObject(t, llc, triple, "raw-fp16-binary.ll", "raw-fp16-binary.o", ir) + }) + } +} + +func TestDecodeRawFP16BinaryEmbeddedControlAndRejections(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16Control(SB),4,$0-0\n") + for opcode, baseOp := range rawFP16BinaryOpcodes { + for control := byte(0); control < 4; control++ { + code := encodeRawFP16Binary(opcode, control, 1, 2, 3, 1, false, true, true) + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("embedded control %x: %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + want := baseOp + ".SAE.Z" + if opcode != 0x5d && opcode != 0x5f { + want = baseOp + "." + [...]Op{"RN_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", "RZ_SAE.Z"}[control] + } + if got.Op != want || got.Args[0].String() != "Z2" || + got.Args[1].String() != "Z1" || got.Args[2].String() != "K1" || + got.Args[3].String() != "Z3" { + t.Fatalf("embedded control %x: %+v, want %s Z2,Z1,K1,Z3", code, got, want) + } + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", "x86_64-unknown-linux-gnu") + if strings.Count(ir, "asm sideeffect") != 24 { + t.Fatal("original embedded-control format inventory changed") + } + for _, control := range []string{"{rn-sae}", "{rd-sae}", "{ru-sae}", "{rz-sae}", "{sae}"} { + if !strings.Contains(ir, control) { + t.Fatal("native FP16 operation lost an original embedded control", control) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-fp16-control.ll", "raw-fp16-control.o", ir) + + valid := encodeRawFP16Binary(0x58, 0, 1, 2, 3, 1, false, false, false) + for _, mutation := range []struct { + name string + index int + mask byte + }{ + {"EVEX.W", 2, 0x80}, + {"zero without mask", 3, 0x81}, + {"reserved width", 3, 0x60}, + } { + bad := append([]byte(nil), valid...) + bad[mutation.index] ^= mutation.mask + if _, _, matched, err := decodedX86VEXBinaryFloatInstruction(bad, 64); !matched || err == nil { + t.Errorf("%s %x: matched=%v, err=%v", mutation.name, bad, matched, err) + } + } + if _, _, matched, err := decodedX86VEXBinaryFloatInstruction(valid[:5], 64); matched && err == nil { + t.Fatal("accepted truncated FP16 binary encoding") + } +} + +func TestTranslateRawFP16Binary386Mask(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16386(SB),4,$0-0\n") + for _, code := range [][]byte{ + encodeRawFP16Binary(0x58, 0, 1, 2, 3, 1, false, false, true), + encodeRawFP16Binary(0x5c, 1, 1, 0, 3, 1, true, true, true), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "386", triple) + if strings.Count(ir, "asm sideeffect") != 2 || + !strings.Contains(ir, "vaddph ") || !strings.Contains(ir, "vsubph ") { + t.Fatal("original mode32 FP16 format inventory changed") + } + compileLLVMToObject(t, llc, triple, "raw-fp16-386.ll", "raw-fp16-386.o", ir) + }) + } +} + +func TestRawFP16BinaryRejectsNamedGoForm(t *testing.T) { + const source = "TEXT invalidNamedHalf(SB),4,$0-0\n\tVADDPH X0, X1, X2\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"invalidNamedHalf": {Name: "invalidNamedHalf", Ret: Void}}, + }); err == nil { + t.Fatal("accepted FP16 mnemonic absent from Go's named assembler table") + } +} diff --git a/amd64_raw_fp16_convert_runtime_test.go b/amd64_raw_fp16_convert_runtime_test.go new file mode 100644 index 000000000..73444858d --- /dev/null +++ b/amd64_raw_fp16_convert_runtime_test.go @@ -0,0 +1,253 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestRawFP16ConversionPortableRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, calls strings.Builder + sigs := make(map[string]FuncSig) + for _, narrow := range []bool{false, true} { + for length := 0; length < 3; length++ { + for _, broadcast := range []bool{false, true} { + for _, zero := range []bool{false, true} { + name := fmt.Sprintf("halfConversion_%t_%d_%t_%t", narrow, length, broadcast, zero) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ input+0(FP), AX\nMOVQ out+8(FP), SI\nMOVQ old+16(FP), DI\nMOVQ mask+24(FP), CX\nKMOVQ CX, K1\nVMOVDQU64 (DI), Z0\n") + code := rawFP16Conversion(narrow, length, 1, 0, 0, true, broadcast, zero) + code[len(code)-1] = 0 + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("VMOVDQU64 Z0, (SI)\nRET\n") + sigs[name] = FuncSig{ + Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Attrs: "nounwind", + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void *, void *, const void *, uint64_t);\n", name) + input, output, bytes, total, inactive := "halves", "singles", 2, 16, "UINT32_C(0xa5a5a5a5)" + if narrow { + input, output, bytes, total, inactive = "singles", "halves", 4, 32, "0xa5a5" + } + if zero { + inactive = "0" + } + index := "lane" + if broadcast { + index = "0" + } + fmt.Fprintf(&calls, ` + { + const void *pointer = %s; + if ((mask & ((UINT64_C(1) << %d) - 1)) == 0) pointer = NULL; + if (mask == 1) { + memcpy(guard + page - %d, %s, %d); + pointer = guard + page - %d; + } + %s(pointer, &output, old, mask); + for (int lane = 0; lane < %d; lane++) { + uint32_t expected = lane < %d ? ((mask & (UINT64_C(1) << lane)) ? %s[%s] : %s) : 0; + uint32_t actual = output.%s[lane]; + if (actual != expected) { + fprintf(stderr, "%s mask %%llx lane %%d: %%08x != %%08x\n", + (unsigned long long)mask, lane, actual, expected); + return 2; + } + } + } +`, input, 4<> 10; + double value = bits & 1023; + int power = -24; + if (exponent) { + value += 1024; + power = (int)exponent - 25; + } + while (power < 0) { value *= 0.5; power++; } + while (power > 0) { value *= 2; power--; } + return value; +} + +// Independent numeric oracle: bracket the float between adjacent positive +// half values by binary search, then choose a bound according to sign/mode. +static uint16_t reference(uint32_t bits, int mode) { + unsigned sign = bits >> 31; + bits &= 0x7fffffff; + if (bits >= 0x7f800000) { + unsigned payload = bits == 0x7f800000 ? 0 : ((bits >> 13) & 1023) | 512; + return (uint16_t)((sign << 15) | 0x7c00 | payload); + } + float source; + memcpy(&source, &bits, 4); + double value = source; + unsigned low = 0, high = 0x7bff; + while (low < high) { + unsigned mid = (low + high + 1) / 2; + if (half_value(mid) <= value) low = mid; + else high = mid - 1; + } + unsigned result = low; + double bottom = half_value(low); + if (value != bottom) { + if (mode == 0) { + double top = low == 0x7bff ? 65536.0 : half_value(low + 1); + double midpoint = (bottom + top) * 0.5; + if (value > midpoint || (value == midpoint && (low & 1))) result++; + } else if ((mode == 1 && sign) || (mode == 2 && !sign)) { + result++; + } + } + return (uint16_t)((sign << 15) | result); +} + +int main(void) { + // Exact halfway values, signed overflow, and half-subnormal ties. + const uint32_t values[8] = {0x3f801000, 0x3f803000, 0xbf801000, 0xbf803000, + 0x47800000, 0xc7800000, 0x33000000, 0xb3000000}; + const uint16_t expected[4][8] = { + {0x3c00, 0x3c02, 0xbc00, 0xbc02, 0x7c00, 0xfc00, 0, 0x8000}, + {0x3c00, 0x3c01, 0xbc01, 0xbc02, 0x7bff, 0xfc00, 0, 0x8001}, + {0x3c01, 0x3c02, 0xbc00, 0xbc01, 0x7c00, 0xfbff, 1, 0x8000}, + {0x3c00, 0x3c01, 0xbc00, 0xbc01, 0x7bff, 0xfbff, 0, 0x8000} + }; + uint32_t input[16]; + uint16_t output[32]; + for (int lane = 0; lane < 16; lane++) input[lane] = values[lane % 8]; +` + calls.String() + ` + void (*functions[])(const void *, void *) = {halfRound0, halfRound1, halfRound2, halfRound3}; + uint32_t random = 0x13579bdf; + for (int batch = 0; batch < 4096; batch++) { + for (int lane = 0; lane < 16; lane++) { + random ^= random << 13; + random ^= random >> 17; + random ^= random << 5; + input[lane] = random; + } + for (int mode = 0; mode < 4; mode++) { + functions[mode](input, output); + for (int lane = 0; lane < 16; lane++) { + uint16_t want = reference(input[lane], mode); + if (output[lane] != want) { + fprintf(stderr, "mode %d input %08x: %04x != %04x\n", mode, input[lane], output[lane], want); + return 2; + } + } + } + } + return 0; +} +` + if legacy { + x86NamedHalfCheckOriginalRoundingC(t, mainC) + } + compileAndRunRuntimeTest(t, llc, clang, "fp16_conversion_rounding", ir, mainC) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + const triple = "x86_64-apple-macosx" + x86IR := x86FP16NumericalModelIR(t, file, sigs, triple) + compileAndRunRuntimeTestForTarget(t, llc, clang, "fp16_conversion_rounding_x86", triple, x86IR, mainC, []string{"/usr/bin/arch", "-x86_64"}) + } +} diff --git a/amd64_raw_fp16_convert_test.go b/amd64_raw_fp16_convert_test.go new file mode 100644 index 000000000..339a5a65a --- /dev/null +++ b/amd64_raw_fp16_convert_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func rawFP16Conversion(narrow bool, length, mask, destination, source int, memory, control, zero bool) []byte { + form := x86RawHalfConversionForm{mapNumber: 6, opcode: 0x13} + if narrow { + form.mapNumber, form.opcode = 5, 0x1d + } + code := encodeX86RawEVEXHalfConversion(form, length, mask, zero, control, destination, source, memory, 0) + if memory { + code[len(code)-1] = 1 + } + return code +} + +func TestRawFP16ConversionCompleteFormats(t *testing.T) { + for _, narrow := range []bool{false, true} { + for length := 0; length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if length == 3 && (memory || !control) { + continue + } + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := rawFP16Conversion(narrow, length, mask, 20, 21, memory, control, zero) + got, size, ok, err := decodedX86PackedHalfConversionInstruction(code, 64) + if err != nil || !ok || size != len(code) || !got.x86Encoded { + t.Fatalf("%x: %+v, size=%d, ok=%v, err=%v", code, got, size, ok, err) + } + vl := length + if control && !memory { + vl = 2 + } + wide, short := [...]string{"X", "Y", "Z"}[vl], [...]string{"X", "X", "Y"}[vl] + src, dst, width := short+"21", wide+"20", 8< %s", code, got, want, src, dst) + } + } + } + } + } + } + } +} + +func TestRawFP16ConversionInvalidFormats(t *testing.T) { + for _, narrow := range []bool{false, true} { + for _, change := range []struct { + index int + bits byte + }{ + {2, 0x80}, {2, 0x04}, {2, 0x08}, {3, 0x08}, {3, 0x80}, {3, 0x60}, + } { + code := rawFP16Conversion(narrow, 0, 0, 0, 1, false, false, false) + code[change.index] ^= change.bits + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid FP16 conversion", map[string]bool{}); err == nil { + t.Fatalf("accepted reserved encoding %x", code) + } + } + for _, code := range [][]byte{ + rawFP16Conversion(narrow, 0, 0, 8, 1, false, false, false), + rawFP16Conversion(narrow, 0, 0, 1, 8, false, false, false), + } { + if _, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 high FP16 conversion", map[string]bool{}); err == nil { + t.Fatalf("accepted extended register %x", code) + } + } + } +} + +func TestRawFP16ConversionLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT rawFP16Conversion(SB),4,$0-0\n") + for _, narrow := range []bool{false, true} { + for length := 0; length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if length == 3 && (memory || !control) { + continue + } + for _, mask := range []int{0, 1} { + for _, b := range rawFP16Conversion(narrow, length, mask, 1, 2, memory, control, mask != 0) { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + } + } + } + } + source.WriteString("RET\n") + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.arch, target.triple) + for _, want := range []string{"vcvtph2psx ", "vcvtps2phx ", "vcvtps2phxx ", "vcvtps2phxy ", + "{rn-sae}", "{rd-sae}", "{ru-sae}", "{rz-sae}", "{sae}", "asm sideeffect", "+avx512fp16", "+avx512vl"} { + if !strings.Contains(ir, want) { + t.Fatalf("missing %s", want) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-fp16-conversion.ll", "raw-fp16-conversion.o", ir) + }) + } +} + +func TestRawFP16ConversionRIPAndNamedForms(t *testing.T) { + for _, narrow := range []bool{false, true} { + for length := 0; length < 3; length++ { + for _, broadcast := range []bool{false, true} { + code := rawFP16Conversion(narrow, length, 1, 0, 0, false, broadcast, true) + code[5] = 5 + code = append(code, 1, 0, 0, 0, 0xc3) + width := 8 << length + if broadcast { + width = 2 + } + if narrow { + width *= 2 + } + code = append(code, make([]byte, width)...) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "FP16 conversion literal", map[string]bool{}) + if err != nil || len(got) != 2 || !got[0].x86RIPLiteral || got[0].Args[0].Kind != OpSym { + t.Fatalf("%x: %+v, err=%v", code, got, err) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated FP16 pool", map[string]bool{}); err == nil { + t.Fatal("accepted truncated constant pool") + } + } + } + } + for _, op := range []string{"VCVTPH2PSX", "VCVTPS2PHX"} { + source := "TEXT namedFP16Conversion(SB),4,$0-0\n" + op + " X0, X1\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedFP16Conversion": {Name: "namedFP16Conversion", Ret: Void}}, + }); err == nil { + t.Fatalf("accepted named %s", op) + } + } +} diff --git a/amd64_raw_fp16_fma_runtime_test.go b/amd64_raw_fp16_fma_runtime_test.go new file mode 100644 index 000000000..10709f65c --- /dev/null +++ b/amd64_raw_fp16_fma_runtime_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Execute the portable IR on the host, without requiring an AVX512-FP16 +// CPU. Separate object tests check the actual x86 features and every target. +func TestRawFP16FMAPortableRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, cases strings.Builder + sigs := make(map[string]FuncSig) + ops := rawFP16FMAOps() + for _, opcode := range x86FMA3Opcodes() { + op := ops[opcode] + scalar := strings.HasSuffix(string(op), "SH") + lengths := []byte{1} + if scalar { + lengths = []byte{0} + } else if opcode == 0x98 { + lengths = []byte{0, 1, 2} + } + for _, length := range lengths { + for _, zero := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && (scalar || opcode != 0x98) { + continue + } + name := fmt.Sprintf("halfFMA_%x_%d_%t_%t", opcode, length, zero, broadcast) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ input+0(FP), DI\nMOVQ memory+8(FP), AX\nMOVQ out+16(FP), SI\nMOVQ mask+24(FP), CX\n") + source.WriteString("KMOVQ CX, K1\nVMOVDQU64 (DI), Z0\nVMOVDQU64 64(DI), Z1\n") + code := encodeRawFP16FMA(opcode, length, 1, 0, 0, 1, true, broadcast, zero) + code[len(code)-1] = 0 + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + source.WriteString("VMOVDQU64 Z0, (SI)\nRET\n") + sigs[name] = FuncSig{ + Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Attrs: "nounwind", // Suppress x86-only inference for this portable oracle. + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + order := []int{132, 213, 231}[(opcode>>4)-9] + mode := int(opcode&15) - 8 + if mode < 0 { + mode += 6 // 6/7 are alternating subtract-even/subtract-odd. + } else { + mode /= 2 + } + lanes := 8 << length + if scalar { + lanes = 1 + } + flag := func(value bool) int { + if value { + return 1 + } + return 0 + } + fmt.Fprintf(&declarations, "extern void %s(const void *, const void *, void *, uint64_t);\n", name) + fmt.Fprintf(&cases, " {%s, %d, %d, %d, %d, %d, %d},\n", name, order, mode, lanes, flag(scalar), flag(zero), flag(broadcast)) + } + } + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir := x86FP16NumericalModelIR(t, file, sigs, testTargetTriple(runtime.GOOS, runtime.GOARCH)) + mainC := "#include \n#include \n#include \n#include \n" + declarations.String() + x86TestGuardPagesC + ` +typedef void (*operation)(const void *, const void *, void *, uint64_t); +struct test_case { operation run; int order, mode, lanes, scalar, zero, broadcast; }; +static struct test_case cases[] = { +` + cases.String() + `}; + +// These operands and their exact products/sums fit in binary64. Round only +// once to FP16, independently of the translator's half-precision FMA. +static double from_half(uint16_t bits) { + _Float16 value; + memcpy(&value, &bits, 2); + return value; +} + +static uint16_t to_half(double value) { + _Float16 result = (_Float16)value; + uint16_t bits; + memcpy(&bits, &result, 2); + return bits; +} + +int main(void) { + if (setup_guard()) return 1; + uint16_t input[64], memory[32], output[32]; + for (int lane = 0; lane < 32; lane++) { + input[lane] = 0x3c01; // 1 + 2^-10 + input[32 + lane] = 0xbc00; // -1 + memory[lane] = 0x3bfe; // 1 - 2^-10 + } + const uint64_t masks[] = {0, 1, 0xaaaaaaaa, 0xffffffff, UINT64_C(1) << 60}; + for (size_t index = 0; index < sizeof(cases) / sizeof(cases[0]); index++) { + struct test_case test = cases[index]; + for (size_t mi = 0; mi < sizeof(masks) / sizeof(masks[0]); mi++) { + uint64_t mask = masks[mi]; + const void *pointer = memory; + if ((mask & ((UINT64_C(1) << test.lanes) - 1)) == 0) pointer = NULL; + if (mask == 1) { + memcpy(guard + page - 3, memory, 2); + pointer = guard + page - 3; + } + memset(output, 0xcc, sizeof(output)); + test.run(input, pointer, output, mask); + for (int lane = 0; lane < 32; lane++) { + uint16_t expected = 0; + if (lane < test.lanes && (mask & (UINT64_C(1) << lane))) { + double d = from_half(input[lane]); + double s = from_half(input[32 + lane]); + double m = from_half(memory[test.broadcast ? 0 : lane]); + double a = d, b = m, c = s; + if (test.order == 213) { a = s; b = d; c = m; } + if (test.order == 231) { a = s; b = m; c = d; } + if (test.mode == 2 || test.mode == 3) a = -a; + if (test.mode == 1 || test.mode == 3 || + (test.mode == 4 && lane % 2 == 0) || + (test.mode == 5 && lane % 2 != 0)) c = -c; + expected = to_half(a * b + c); + } else if (lane < test.lanes && !test.zero) { + expected = input[lane]; + } else if (test.scalar && lane < 8 && lane != 0) { + expected = input[lane]; + } + if (output[lane] != expected) { + fprintf(stderr, "case %zu order %d mode %d mask %llx lane %d: %04x != %04x\n", + index, test.order, test.mode, (unsigned long long)mask, lane, output[lane], expected); + return 2; + } + } + } + } + return free_guard(); +} +` + compileAndRunRuntimeTest(t, llc, clang, "fp16_fma_portable", ir, mainC) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + const triple = "x86_64-apple-macosx" + x86IR, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "fp16_fma_portable_x86", triple, x86IR, mainC, []string{"/usr/bin/arch", "-x86_64"}) + } +} diff --git a/amd64_raw_fp16_fma_test.go b/amd64_raw_fp16_fma_test.go new file mode 100644 index 000000000..b5bc6ad9c --- /dev/null +++ b/amd64_raw_fp16_fma_test.go @@ -0,0 +1,266 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Intel AVX512-FP16 specification, sections 5.44-5.49: MAP6.66.W0, +// packed X/Y/Z and scalar LLIG, with the same opcode/order axes as FMA3. +// Go 1.27 has no named PH/SH FMA forms, so these are raw encodings only. +func rawFP16FMAOps() map[byte]Op { + ops := make(map[byte]Op) + for orderIndex, order := range []int{132, 213, 231} { + for familyIndex, family := range []string{"VFMADD", "VFMSUB", "VFNMADD", "VFNMSUB"} { + for scalarIndex, suffix := range []string{"PH", "SH"} { + opcode := byte(0x98 + orderIndex*16 + familyIndex*2 + scalarIndex) + ops[opcode] = Op(fmt.Sprintf("%s%d%s", family, order, suffix)) + } + } + ops[byte(0x96+orderIndex*16)] = Op(fmt.Sprintf("VFMADDSUB%dPH", order)) + ops[byte(0x97+orderIndex*16)] = Op(fmt.Sprintf("VFMSUBADD%dPH", order)) + } + return ops +} + +func encodeRawFP16FMA(opcode, length byte, source1, source2, destination, mask int, memory, control, zero bool) []byte { + code := encodeX86EVEXFMA3(opcode, false, length, control, zero, mask, destination, source1, source2) + code[1] = code[1]&^byte(15) | 6 + if memory { + // Keep register extension tests independent of memory base extensions. + code[1] |= 0x60 + code[5] = byte(0x40 | (destination&7)<<3) + code = append(code, 1) + } + return code +} + +func TestDecodeRawFP16FMAGoATReproduction(t *testing.T) { + for _, tc := range []struct { + code []byte + op Op + }{ + {[]byte{0x62, 0xb6, 0x7d, 0x48, 0xa8, 0x24, 0x4a}, "VFMADD213PH"}, + {[]byte{0x62, 0xf6, 0x7d, 0x48, 0x98, 0x0e}, "VFMADD132PH"}, + {[]byte{0x62, 0xb6, 0x7d, 0x08, 0x99, 0x0c, 0x76}, "VFMADD132SH"}, + } { + got, err := decodeX86RawDirectiveGroup(tc.code, 64, 0, "GoAT FP16 FMA", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Op != tc.op || !got[0].x86Encoded { + t.Fatalf("%x decoded as %+v, want raw %s", tc.code, got, tc.op) + } + } +} + +func TestDecodeRawFP16FMACompleteFormats(t *testing.T) { + for opcode, base := range rawFP16FMAOps() { + scalar := strings.HasSuffix(string(base), "SH") + for length := byte(0); length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if scalar && memory && control || !scalar && length == 3 && (!control || memory) { + continue + } + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := encodeRawFP16FMA(opcode, length, 20, 21, 22, mask, memory, control, zero) + got, size, ok, err := decodedX86VEXFMA3Instruction(code, 64) + if err != nil || !ok || size != len(code) { + t.Fatalf("%x: got %+v, size=%d, ok=%v, err=%v", code, got, size, ok, err) + } + want := base + prefix, width := "X", 16 + if !scalar { + vl := length + if control && !memory { + vl = 2 + } + prefix, width = [...]string{"X", "Y", "Z"}[vl], 16<>6 != 3 { + return fail("scalar conversion does not support memory broadcast") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register or address in 32-bit mode") + } + spec := amd64HalfConversionSpecs[string(form.op)] + source, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", spec.inputBits/8) + if err != nil { + return Instr{}, 0, true, err + } + op := form.op + if p.broadcast { + if spec.inputBits == 16 { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } + if p.zero { + op += ".Z" + } + args := []Operand{source, {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + destination := int(code[p.modRM]>>3&7) + p.r*8 + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destination))}) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), x86Encoded: true}, p.modRM + consumed, true, nil +} + +func decodeX86RawScalarHalfConversionRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, recognized := decodeX86RawVectorEncoding(code[offset:]) + form, matched := x86RawScalarHalfConversionFormFor(p) + if !recognized || !matched || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRM := offset + p.modRM + if len(code) <= modRM || code[modRM]&0xc7 != 5 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := amd64HalfConversionSpecs[string(form.op)].inputBits / 8 + return x86RawRIPDataThroughDecoder(code, offset, mode, modRM, width, decodedX86ScalarHalfConversionInstruction, "scalar half conversion") +} diff --git a/amd64_raw_fp16_scalar_convert_runtime_test.go b/amd64_raw_fp16_scalar_convert_runtime_test.go new file mode 100644 index 000000000..3e85e0579 --- /dev/null +++ b/amd64_raw_fp16_scalar_convert_runtime_test.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestRawFP16ScalarConversionPortableRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, calls strings.Builder + sigs := make(map[string]FuncSig) + for index, tc := range fp16ScalarConversionCases { + for _, zero := range []bool{false, true} { + name := fmt.Sprintf("scalarHalf_%d_%t", index, zero) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ input+0(FP), AX\nMOVQ vectors+8(FP), DI\nMOVQ out+16(FP), SI\nMOVQ mask+24(FP), CX\nKMOVQ CX, K1\nVMOVDQU64 (DI), Z0\nVMOVDQU64 64(DI), Z1\n") + code := rawFP16ScalarConversion(index, 3, 1, 0, 0, 1, true, false, zero) + code[len(code)-1] = 0 + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("VMOVDQU64 Z0, (SI)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Attrs: "nounwind", + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + bits := map[int]uint64{16: 0x4000, 32: 0x40000000, 64: 0x4000000000000000} + fmt.Fprintf(&declarations, "extern void %s(const void *, const void *, void *, uint64_t);\n", name) + inactive := "memcpy(expected, vectors, output_bytes);" + if zero { + inactive = "memset(expected, 0, output_bytes);" + } + fmt.Fprintf(&calls, ` + { + const int input_bytes = %d, output_bytes = %d; + uint64_t input = UINT64_C(%d), result = UINT64_C(%d); + const void *pointer = NULL; + if (mask & 1) { + memcpy(guard + page - input_bytes - 1, &input, input_bytes); + pointer = guard + page - input_bytes - 1; + } + memset(expected, 0, sizeof(expected)); + if (mask & 1) memcpy(expected, &result, output_bytes); + else { %s } + memcpy(expected + output_bytes, vectors + 64 + output_bytes, 16 - output_bytes); + %s(pointer, vectors, output, mask); + if (memcmp(expected, output, 64)) { + fprintf(stderr, "%s mask %%llx: incorrect low, preserved or upper lanes\n", (unsigned long long)mask); + return 2; + } + } +`, tc.inputBits/8, tc.outBits/8, bits[tc.inputBits], bits[tc.outBits], inactive, name, name) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir := x86FP16NumericalModelIR(t, file, sigs, testTargetTriple(runtime.GOOS, runtime.GOARCH)) + mainC := "#include \n#include \n#include \n#include \n" + declarations.String() + x86TestGuardPagesC + ` +int main(void) { + if (setup_guard()) return 1; + unsigned char vectors[128], output[64], expected[64]; + for (int i = 0; i < 128; i++) vectors[i] = (unsigned char)(0x20 + i); + const uint64_t masks[] = {0, 1, 2, UINT64_C(1) << 60, UINT64_MAX}; + for (int mi = 0; mi < 5; mi++) { + uint64_t mask = masks[mi]; +` + calls.String() + ` + } + return free_guard(); +} +` + compileAndRunRuntimeTest(t, llc, clang, "fp16_scalar_conversion", ir, mainC) +} + +func TestRawFP16ScalarConversionRoundingRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, calls strings.Builder + sigs := make(map[string]FuncSig) + for _, index := range []int{1, 3} { + for rounding := 0; rounding < 4; rounding++ { + name := fmt.Sprintf("scalarRound_%d_%d", index, rounding) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-16\nMOVQ input+0(FP), AX\nMOVQ out+8(FP), SI\nVMOVDQU64 (AX), Z2\n", name) + source.WriteString("VPXORD Z1,Z1,Z1\n") // The complete output includes this scalar passthrough. + for _, b := range rawFP16ScalarConversion(index, rounding, 1, 2, 0, 0, false, true, false) { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("VMOVDQU64 Z0, (SI)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, Attrs: "nounwind", + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void *, void *);\n", name) + input := "single" + if index == 3 { + input = "wide" + } + fmt.Fprintf(&calls, ` + %s(%s, output); + if (output[0] != expected[%d][sample]) { + fprintf(stderr, "%s sample %%d: %%04x != %%04x\n", sample, output[0], expected[%d][sample]); + return 1; + } +`, name, input, rounding, name, rounding) + if index == 3 { + fmt.Fprintf(&calls, ` + // These exactly representable doubles straddle a half tie but round + // to that same tie in float; a double->float->half chain is wrong. + if (sample == 0) { + wide[0] = 1.0 + 0x1p-11 + 0x1p-40; + %s(wide, output); + const uint16_t above[] = {0x3c01, 0x3c00, 0x3c01, 0x3c00}; + if (output[0] != above[%d]) return 2; + wide[0] = 1.0 + 0x1p-11 - 0x1p-40; + %s(wide, output); + const uint16_t below[] = {0x3c00, 0x3c00, 0x3c01, 0x3c00}; + if (output[0] != below[%d]) return 3; + wide[0] = values[sample]; + } +`, name, rounding, name, rounding) + } + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + original := strings.ReplaceAll(source.String(), "VPXORD Z1,Z1,Z1\n", "") + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", original) + originalFile, err := Parse(ArchAMD64, original) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(originalFile, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original full-vector store borrowed missing scalar passthrough bytes", err) + } + ir := x86FP16NumericalModelIR(t, file, sigs, testTargetTriple(runtime.GOOS, runtime.GOARCH)) + mainC := "#include \n#include \n" + declarations.String() + ` +int main(void) { + const double values[] = {1 + 0x1p-11, 1 + 3 * 0x1p-11, -1 - 0x1p-11, -1 - 3 * 0x1p-11, + 65536, -65536, 0x1p-25, -0x1p-25}; + const uint16_t expected[4][8] = { + {0x3c00, 0x3c02, 0xbc00, 0xbc02, 0x7c00, 0xfc00, 0, 0x8000}, + {0x3c00, 0x3c01, 0xbc01, 0xbc02, 0x7bff, 0xfc00, 0, 0x8001}, + {0x3c01, 0x3c02, 0xbc00, 0xbc01, 0x7c00, 0xfbff, 1, 0x8000}, + {0x3c00, 0x3c01, 0xbc00, 0xbc01, 0x7bff, 0xfbff, 0, 0x8000} + }; + uint16_t output[32]; + for (int sample = 0; sample < 8; sample++) { + double wide[8] = {values[sample]}; + float single[16] = {(float)values[sample]}; +` + calls.String() + ` + } + return 0; +} +` + compileAndRunRuntimeTest(t, llc, clang, "fp16_scalar_rounding", ir, mainC) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + const triple = "x86_64-apple-macosx" + x86IR := x86FP16NumericalModelIR(t, file, sigs, triple) + compileAndRunRuntimeTestForTarget(t, llc, clang, "fp16_scalar_rounding_x86", triple, x86IR, mainC, []string{"/usr/bin/arch", "-x86_64"}) + } +} diff --git a/amd64_raw_fp16_scalar_convert_test.go b/amd64_raw_fp16_scalar_convert_test.go new file mode 100644 index 000000000..d0622d51a --- /dev/null +++ b/amd64_raw_fp16_scalar_convert_test.go @@ -0,0 +1,163 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Independent Intel AVX512-FP16 sections 5.18, 5.19, 5.21 and 5.24 rows. +var fp16ScalarConversionCases = []struct { + op Op + mapNumber, pp, code byte + inputBits, outBits int + w bool +}{ + {"VCVTSH2SS", 6, 0, 0x13, 16, 32, false}, + {"VCVTSS2SH", 5, 0, 0x1d, 32, 16, false}, + {"VCVTSH2SD", 5, 2, 0x5a, 16, 64, false}, + {"VCVTSD2SH", 5, 3, 0x5a, 64, 16, true}, +} + +func rawFP16ScalarConversion(index, length, source1, source2, destination, mask int, memory, control, zero bool) []byte { + tc := fp16ScalarConversionCases[index] + code := encodeX86EVEXFMA3(tc.code, tc.w, byte(length), control, zero, mask, destination, source1, source2) + code[1] = code[1]&0xf0 | tc.mapNumber + code[2] = code[2]&^3 | tc.pp + if memory { + code[1] |= 0x60 + code[5] = 0x40 | byte(destination&7)<<3 + code = append(code, 1) + } + return code +} + +func TestRawFP16ScalarConversionFormats(t *testing.T) { + for index, tc := range fp16ScalarConversionCases { + for length := 0; length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if control && memory { + continue + } + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := rawFP16ScalarConversion(index, length, 20, 21, 22, mask, memory, control, zero) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "FP16 scalar conversion", map[string]bool{}) + if err != nil || len(got) != 1 || !got[0].x86Encoded { + t.Fatalf("%x: %+v, err=%v", code, got, err) + } + want := tc.op + if control { + if tc.inputBits == 16 { + want += ".SAE" + } else { + want += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[length] + } + } + if zero { + want += ".Z" + } + source := "X21" + if memory { + source = fmt.Sprintf("%d(AX)", tc.inputBits/8) + } + ins := got[0] + if ins.Op != want || ins.Args[0].String() != source || ins.Args[1].String() != "X20" || ins.Args[len(ins.Args)-1].String() != "X22" { + t.Fatalf("%x: %+v, want %s %s, X20, [mask], X22", code, ins, want, source) + } + } + } + } + } + } + } +} + +func TestRawFP16ScalarConversionInvalidAndNamedForms(t *testing.T) { + for index, tc := range fp16ScalarConversionCases { + for _, change := range []struct { + index int + bits byte + }{{2, 4}, {2, 0x80}, {3, 0x80}} { + code := rawFP16ScalarConversion(index, 0, 1, 2, 3, 0, false, false, false) + code[change.index] ^= change.bits + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid scalar conversion", map[string]bool{}); err == nil { + t.Fatalf("accepted reserved %x", code) + } + } + code := rawFP16ScalarConversion(index, 0, 1, 2, 3, 0, true, true, false) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid scalar broadcast", map[string]bool{}); err == nil { + t.Fatalf("accepted scalar broadcast %x", code) + } + for operand := 0; operand < 3; operand++ { + registers := [3]int{1, 2, 3} + registers[operand] = 8 + code := rawFP16ScalarConversion(index, 0, registers[0], registers[1], registers[2], 1, false, false, false) + if _, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 high scalar conversion", map[string]bool{}); err == nil { + t.Fatalf("accepted 386 extended register %x", code) + } + } + source := "TEXT namedScalarHalf(SB),4,$0-0\n" + string(tc.op) + " X0, X1, X2\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedScalarHalf": {Name: "namedScalarHalf", Ret: Void}}, + }); err == nil { + t.Fatalf("accepted named %s", tc.op) + } + } +} + +func TestRawFP16ScalarConversionLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT rawScalarHalf(SB),4,$0-0\n") + for index := range fp16ScalarConversionCases { + for length := 0; length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, mask := range []int{0, 1} { + for _, b := range rawFP16ScalarConversion(index, length, 1, 2, 3, mask, memory, !memory, mask != 0) { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + } + } + } + source.WriteString("RET\n") + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.arch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-scalar-half.ll", "raw-scalar-half.o", ir) + }) + } +} + +func TestRawFP16ScalarConversionRIPLiteral(t *testing.T) { + for index, tc := range fp16ScalarConversionCases { + code := rawFP16ScalarConversion(index, 3, 1, 0, 0, 1, false, false, true) + code[5] = 5 + code = append(code, 1, 0, 0, 0, 0xc3) + code = append(code, make([]byte, tc.inputBits/8)...) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "scalar FP16 literal", map[string]bool{}) + if err != nil || len(got) != 2 || !got[0].x86RIPLiteral || got[0].Args[0].Kind != OpSym { + t.Fatalf("%x: %+v, err=%v", code, got, err) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated scalar FP16 literal", map[string]bool{}); err == nil { + t.Fatal("accepted truncated constant pool") + } + } +} diff --git a/amd64_raw_fp16_scalar_move_test.go b/amd64_raw_fp16_scalar_move_test.go new file mode 100644 index 000000000..cf5e2bcb9 --- /dev/null +++ b/amd64_raw_fp16_scalar_move_test.go @@ -0,0 +1,211 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeRawFP16ScalarMove(opcode, low, upper, dst, mask int, zero bool, length byte) []byte { + code := encodeRawScalarMove(true, 2, opcode, low, upper, dst, mask, zero) + code[1] = code[1]&^byte(15) | 5 + code[3] = code[3]&^byte(0x60) | length<<5 + return code +} + +func TestDecodeRawFP16ScalarMoveGoATReproduction(t *testing.T) { + for _, code := range [][]byte{ + // VMOVSH (DI)(CX*2), X0 and VMOVSH X0, (DX)(CX*2) + // from go-highway's GoAT output. + {0x62, 0xf5, 0x7e, 0x08, 0x10, 0x04, 0x4f}, + {0x62, 0xf5, 0x7e, 0x08, 0x11, 0x04, 0x4a}, + // A complete add/store group, not just a standalone move. + {0x62, 0xf5, 0x7e, 0x08, 0x58, 0x04, 0x4e, + 0x62, 0xf5, 0x7e, 0x08, 0x11, 0x04, 0x4a}, + } { + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "GoAT VMOVSH", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + move := decoded[len(decoded)-1] + if move.Op != "VMOVSH" || !move.x86Encoded { + t.Fatalf("decoded %#v, want final raw VMOVSH", decoded) + } + } +} + +func TestDecodeRawFP16ScalarMoveCompleteForms(t *testing.T) { + for _, opcode := range []int{0x10, 0x11} { + for length := byte(0); length < 4; length++ { + for _, registers := range [][3]int{{0, 1, 2}, {7, 8, 15}, {16, 23, 31}} { + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if mask == 0 && zero { + continue + } + low, upper, dst := registers[0], registers[1], registers[2] + code := encodeRawFP16ScalarMove(opcode, low, upper, dst, mask, zero, length) + got, size, ok, err := decodedX86ScalarMoveInstruction(code, 64) + wantOp := Op("VMOVSH") + if zero { + wantOp += ".Z" + } + want := []Operand{ + {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", low))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", upper))}, + } + if mask != 0 { + want = append(want, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + want = append(want, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", dst))}) + if err != nil || !ok || size != len(code) || got.Op != wantOp || + !got.x86Encoded || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("%x: got %+v, size=%d, ok=%v, err=%v; want %s %+v", code, got, size, ok, err, wantOp, want) + } + } + } + } + } + } + for _, opcode := range []byte{0x10, 0x11} { + for length := byte(0); length < 4; length++ { + for _, mask := range []byte{0, 1, 7} { + for _, zero := range []bool{false, true} { + if opcode == 0x11 && zero || mask == 0 && zero { + continue + } + code := []byte{0x62, 0xf5, 0x7e, 0x08 | length<<5 | mask, opcode, 0x04, 0x4f} + if zero { + code[3] |= 0x80 + } + got, size, ok, err := decodedX86ScalarMoveInstruction(code, 64) + if err != nil || !ok || size != len(code) || got.Op != "VMOVSH" && got.Op != "VMOVSH.Z" { + t.Fatalf("%x: got %+v, size=%d, ok=%v, err=%v", code, got, size, ok, err) + } + if opcode == 0x10 && (got.Args[0].Kind != OpMem || got.Args[len(got.Args)-1].Reg != "X0") { + t.Fatalf("%x: wrong memory load %+v", code, got) + } + if opcode == 0x11 && (got.Args[0].Reg != "X0" || got.Args[len(got.Args)-1].Kind != OpMem) { + t.Fatalf("%x: wrong memory store %+v", code, got) + } + } + } + } + } +} + +func TestDecodeRawFP16ScalarMoveInvalidForms(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"missing-modrm", []byte{0x62, 0xf5, 0x7e, 0x08, 0x10}, 64}, + {"address-override", []byte{0x67, 0x62, 0xf5, 0x7e, 0x08, 0x10, 0xc0}, 64}, + {"wrong-width", []byte{0x62, 0xf5, 0xfe, 0x08, 0x10, 0xc0}, 64}, + {"missing-fixed", []byte{0x62, 0xf5, 0x7a, 0x08, 0x10, 0xc0}, 64}, + {"broadcast", []byte{0x62, 0xf5, 0x7e, 0x18, 0x10, 0xc0}, 64}, + {"zero-k0", []byte{0x62, 0xf5, 0x7e, 0x88, 0x10, 0xc0}, 64}, + {"zero-store", []byte{0x62, 0xf5, 0x7e, 0x89, 0x11, 0x00}, 64}, + {"memory-vvvv", []byte{0x62, 0xf5, 0x76, 0x08, 0x10, 0x00}, 64}, + {"memory-vprime", []byte{0x62, 0xf5, 0x7e, 0x00, 0x10, 0x00}, 64}, + {"386-high-register", []byte{0x62, 0x75, 0x7e, 0x08, 0x10, 0xc0}, 32}, + {"wrong-mode", []byte{0x62, 0xf5, 0x7e, 0x08, 0x10, 0xc0}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, ok, err := decodedX86ScalarMoveInstruction(tc.code, tc.mode); !ok || err == nil { + t.Fatalf("invalid %x: matched=%v, err=%v", tc.code, ok, err) + } + }) + } +} + +func TestDecodeRawFP16ScalarMoveRIPLiteral(t *testing.T) { + code := []byte{0x62, 0xf5, 0x7e, 0x08, 0x10, 0x05, 1, 0, 0, 0, 0xc3, 0x00, 0x3c} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VMOVSH RIP literal", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VMOVSH" || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want source-local FP16 literal", code, decoded) + } +} + +func TestTranslateRawFP16ScalarMoveLLVM22Targets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16ScalarMove(SB),4,$0-0\n") + for _, code := range [][]byte{ + {0x62, 0xf5, 0x7e, 0x08, 0x10, 0x04, 0x4f}, + {0x62, 0xf5, 0x7e, 0x08, 0x11, 0x04, 0x4f}, + encodeRawFP16ScalarMove(0x10, 1, 2, 3, 7, true, 3), + encodeRawFP16ScalarMove(0x11, 4, 5, 6, 1, false, 1), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawFP16ScalarMove": {Name: "rawFP16ScalarMove", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "load i16") || !strings.Contains(ir, "store i16") { + t.Fatalf("missing scalar FP16 memory operation:\n%s", ir) + } + compileLLVMToObject(t, llc, triple, "raw-fp16-scalar-move.ll", "raw-fp16-scalar-move.o", ir) + }) + } +} + +func TestTranslateRawFP16ScalarMove386LLVM22Targets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16ScalarMove386(SB),4,$0-0\n") + for _, code := range [][]byte{ + {0x62, 0xf5, 0x7e, 0x08, 0x10, 0x04, 0x4f}, + {0x62, 0xf5, 0x7e, 0x09, 0x11, 0x04, 0x4f}, + encodeRawFP16ScalarMove(0x10, 1, 2, 3, 1, true, 2), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "386", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawFP16ScalarMove386": {Name: "rawFP16ScalarMove386", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-fp16-scalar-move386.ll", "raw-fp16-scalar-move386.o", ir) + }) + } +} diff --git a/amd64_raw_fp16_sqrt_test.go b/amd64_raw_fp16_sqrt_test.go new file mode 100644 index 000000000..2852bda18 --- /dev/null +++ b/amd64_raw_fp16_sqrt_test.go @@ -0,0 +1,252 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Intel AVX512-FP16 sections 5.74/5.75. Packed SQRT reserves vvvv; +// scalar SQRT instead uses it as the source of preserved upper X lanes. +func rawFP16PackedSqrt(length byte, source, destination, mask int, memory, control, zero bool) []byte { + code := encodeRawFP16Binary(0x51, length, 0, source, destination, mask, false, control, zero) + if memory { + code[1] |= 0x60 + code[5] = byte(0x40 | (destination&7)<<3) + code = append(code, 1) + } + return code +} + +func TestRawFP16PackedSqrtRejectsNamedForms(t *testing.T) { + const source = "TEXT namedHalfSqrt(SB),4,$0-0\nVSQRTPH X0, X1\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedHalfSqrt": {Name: "namedHalfSqrt", Ret: Void}}, + }); err == nil { + t.Fatal("accepted VSQRTPH absent from Go's named assembler table") + } +} + +func TestRawFP16SqrtPortableRuntime(t *testing.T) { + runRawFP16SqrtPortable(t, 0) +} + +func TestRawFP16SqrtWideMaskPortableRuntime(t *testing.T) { + for _, length := range []byte{1, 2} { + t.Run(fmt.Sprintf("lanes%d", 8<>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | operation.mapValue + p1 := operation.w<<7 | byte(^second&15)<<3 | 5 + p2 := byte(vectorBits)<<5 | byte((^second>>4)&1)<<3 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0x62, p0, p1, p2, operation.opcode, modRM} + if !operation.variable { + encoding = append(encoding, 0xa5) + } + instruction, length, ok, err := decodedX86FunnelShiftInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("op=%s width=%s first=%d second=%d destination=%d: ok=%v length=%d err=%v", operation.op, vectorName, first, second, destination, ok, length, err) + } + immediate := "" + if !operation.variable { + immediate = "$165, " + } + want := fmt.Sprintf("%s %s%s%d, %s%d, %s%d", operation.op, immediate, vectorName, first, vectorName, second, vectorName, destination) + if instruction.Raw != want { + t.Fatalf("encoding %x decoded as %q, want %q", encoding, instruction.Raw, want) + } + } + } + } + } + } +} + +func TestDecodedX86RawFunnelShiftRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "386 immediate operand limit", mode: 32, encoding: []byte{0x62, 0xf3, 0xf5, 0x08, 0x73, 0xc2, 1}, wantOK: true}, + {name: "386 variable masked operand limit", mode: 32, encoding: []byte{0x62, 0xf2, 0xf5, 0x09, 0x73, 0xc2}, wantOK: true}, + {name: "386 variable extended register", mode: 32, encoding: []byte{0x62, 0x72, 0xf5, 0x08, 0x73, 0xc2}, wantOK: true}, + {name: "reserved vector length", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x68, 0x73, 0xc2}, wantOK: true}, + {name: "word broadcast", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x18, 0x72, 0x10}, wantOK: true}, + {name: "broadcast register", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x18, 0x73, 0xc2}, wantOK: true}, + {name: "zero without mask", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x88, 0x73, 0xc2}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0x62, 0xf2, 0xf5, 0x08, 0x73, 0xc2}, wantOK: true}, + {name: "wrong map", mode: 64, encoding: []byte{0x62, 0xf1, 0xf5, 0x08, 0x73, 0xc2}, wantOK: false}, + {name: "wrong prefix", mode: 64, encoding: []byte{0x62, 0xf2, 0xf4, 0x08, 0x73, 0xc2}, wantOK: false}, + {name: "word wrong w", mode: 64, encoding: []byte{0x62, 0xf2, 0x75, 0x08, 0x72, 0xc2}, wantOK: true}, + {name: "truncated immediate", mode: 64, encoding: []byte{0x62, 0xf3, 0xf5, 0x08, 0x73, 0xc2}, wantOK: false}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86FunnelShiftInstruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x: ok=%v, want %v (err=%v)", test.encoding, ok, test.wantOK, err) + } + if test.wantOK && err == nil { + t.Fatalf("encoding %x was recognized without rejection", test.encoding) + } + }) + } +} diff --git a/amd64_raw_gather.go b/amd64_raw_gather.go new file mode 100644 index 000000000..f1edd36cc --- /dev/null +++ b/amd64_raw_gather.go @@ -0,0 +1,179 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// The eight data-gather spellings share the same opcode axes in Go's +// _yvgatherdps, _yvgatherdpd and _yvgatherqps tables. The ModRM/SIB operand +// is VSIB memory: its index is a vector register, never a GP register. +var x86GatherOps = [4][2]Op{ + {"VPGATHERDD", "VPGATHERDQ"}, + {"VPGATHERQD", "VPGATHERQQ"}, + {"VGATHERDPS", "VGATHERDPD"}, + {"VGATHERQPS", "VGATHERQPD"}, +} + +// The four packed and four floating-point scatter spellings mirror the EVEX +// gather VSIB axes, but encode the data vector in ModRM.reg and store to VSIB. +var x86ScatterOps = [4][2]Op{ + {"VPSCATTERDD", "VPSCATTERDQ"}, + {"VPSCATTERQD", "VPSCATTERQQ"}, + {"VSCATTERDPS", "VSCATTERDPD"}, + {"VSCATTERQPS", "VSCATTERQPD"}, +} + +func decodedX86RawScatterInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || !p.evex || p.mapNumber != 2 || p.pp != 1 || + p.opcode < 0xa0 || p.opcode > 0xa3 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("raw scatter: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.vectorLength > 2 || p.mask == 0 || p.zero || + p.broadcast || p.upper&15 != 0 { + return fail("invalid EVEX fixed, length, mask, zeroing, broadcast or vvvv bits") + } + if len(code) < p.modRM+2 { + return fail("missing ModRM or VSIB byte") + } + modRM := code[p.modRM] + if modRM>>6 == 3 || modRM&7 != 4 { + return fail("scatter requires VSIB memory") + } + w := 0 + if p.w { + w = 1 + } + op := x86ScatterOps[p.opcode-0xa0][w] + spec := amd64ScatterSpecs[string(op)] + widths := [...]string{"X", "Y", "Z"} + dataWidth := widths[p.vectorLength] + indexWidth := dataWidth + switch spec.table { + case amd64GatherDPD: + if p.vectorLength > 0 { + indexWidth = widths[p.vectorLength-1] + } + case amd64GatherQPS: + if p.vectorLength > 0 { + dataWidth = widths[p.vectorLength-1] + } + } + dataNumber := int(modRM>>3&7) + p.r*8 + indexNumber := int(code[p.modRM+1]>>3&7) + p.x*8 + p.upper&16 + if mode == 32 && (indexNumber >= 8 || p.b != 0 || + dataWidth == "Z" && dataNumber >= 8) { + return fail("extended register unavailable in 32-bit mode") + } + memory, consumed, err := decodedX86RMSource( + code[p.modRM:], mode, p.b, p.x, p.segment, spec.elemBits/8, + ) + if err != nil { + return Instr{}, 0, true, err + } + memory.Mem.Index = Reg(fmt.Sprintf("%s%d", indexWidth, indexNumber)) + memory.Mem.Scale = int64(1 << (code[p.modRM+1] >> 6)) + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", dataWidth, dataNumber))} + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))} + args := []Operand{data, mask, memory} + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s, %s", op, data.String(), mask.String(), memory.String()), + }, p.modRM + consumed, true, nil +} + +func decodedX86RawGatherInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || p.mapNumber != 2 || p.pp != 1 || p.opcode < 0x90 || p.opcode > 0x93 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("raw gather: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.evex && (!p.fixed || p.vectorLength > 2 || p.mask == 0 || p.zero || p.broadcast || p.upper&15 != 0) { + return fail("invalid EVEX fixed, length, mask, zeroing, broadcast or vvvv bits") + } + if len(code) < p.modRM+2 { + return fail("missing ModRM or VSIB byte") + } + modRM := code[p.modRM] + if modRM>>6 == 3 || modRM&7 != 4 { + return fail("gather requires VSIB memory") + } + + w := 0 + if p.w { + w = 1 + } + op := x86GatherOps[p.opcode-0x90][w] + spec := amd64GatherSpecs[op] + widths := [...]string{"X", "Y", "Z"} + destinationWidth := widths[p.vectorLength] + indexWidth := destinationWidth + switch spec.table { + case amd64GatherDPD: + if p.vectorLength > 0 { + indexWidth = widths[p.vectorLength-1] + } + case amd64GatherQPS: + if p.vectorLength > 0 { + destinationWidth = widths[p.vectorLength-1] + } + } + destinationNumber := int(modRM>>3&7) + p.r*8 + indexNumber := int(code[p.modRM+1]>>3&7) + p.x*8 + if p.evex { + indexNumber += p.upper & 16 + } + if mode == 32 && (indexNumber >= 8 || p.b != 0 || + !p.evex && (destinationNumber >= 8 || p.upper >= 8) || + p.evex && destinationWidth == "Z" && destinationNumber >= 8) { + return fail("extended register unavailable in 32-bit mode") + } + + disp8Scale := 1 + if p.evex { + disp8Scale = spec.elemBits / 8 + } + memory, consumed, err := decodedX86RMSource(code[p.modRM:], mode, p.b, p.x, p.segment, disp8Scale) + if err != nil { + return Instr{}, 0, true, err + } + memory.Mem.Index = Reg(fmt.Sprintf("%s%d", indexWidth, indexNumber)) + memory.Mem.Scale = int64(1 << (code[p.modRM+1] >> 6)) + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationWidth, destinationNumber))} + var args []Operand + if p.evex { + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))} + args = []Operand{memory, mask, destination} + } else { + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationWidth, p.upper))} + args = []Operand{mask, memory, destination} + } + printed := make([]string, len(args)) + for i, arg := range args { + printed[i] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_gather_test.go b/amd64_raw_gather_test.go new file mode 100644 index 000000000..c709fb11b --- /dev/null +++ b/amd64_raw_gather_test.go @@ -0,0 +1,215 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawVEXGatherCorpusEncoding(t *testing.T) { + const source = "TEXT rawgather(SB),4,$0-0\n\tVPGATHERDD Y2, (R8)(Y0*2), Y1\n\tRET\n" + code := assembleX87ControlBytes(t, "amd64", source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } +} + +func TestX86RawVEXGatherGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT gatherforms(SB),4,$0-0\n") + for _, variants := range x86GatherOps { + for _, op := range variants { + for width := 0; width < 2; width++ { + destinationWidth, indexWidth := "X", "X" + if width == 1 { + switch amd64GatherSpecs[op].table { + case amd64GatherDPS: + destinationWidth, indexWidth = "Y", "Y" + case amd64GatherDPD: + destinationWidth = "Y" + case amd64GatherQPS: + indexWidth = "Y" + } + } + fmt.Fprintf(&source, "%s %s1, 4(BX)(%s2*2), %s3\n", op, destinationWidth, indexWidth, destinationWidth) + fmt.Fprintf(&source, "%s %s1, 256(%s4*8), %s3\n", op, destinationWidth, indexWidth, destinationWidth) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s14, -32(R8)(%s13*4), %s15\n", op, destinationWidth, indexWidth, destinationWidth) + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + }) + } +} + +func TestX86RawEVEXGatherGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT evexgatherforms(SB),4,$0-0\n") + for _, variants := range x86GatherOps { + for _, op := range variants { + for width := 0; width < 3; width++ { + widths := [...]string{"X", "Y", "Z"} + destinationWidth := widths[width] + indexWidth := destinationWidth + switch amd64GatherSpecs[op].table { + case amd64GatherDPD: + if width > 0 { + indexWidth = widths[width-1] + } + case amd64GatherQPS: + if width > 0 { + destinationWidth = widths[width-1] + } + } + fmt.Fprintf(&source, "%s 8(BX)(%s2*4), K1, %s3\n", op, indexWidth, destinationWidth) + if arch == "amd64" { + fmt.Fprintf(&source, "%s -32(R8)(%s18*2), K7, %s21\n", op, indexWidth, destinationWidth) + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + }) + } +} + +func TestX86RawVEXGatherRejectsMalformedForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {"missing SIB", []byte{0xc4, 0xc2, 0x6d, 0x90, 0x0c}, 64}, + {"register source", []byte{0xc4, 0xc2, 0x6d, 0x90, 0xc0, 0x40}, 64}, + {"GP index", []byte{0xc4, 0xc2, 0x6d, 0x90, 0x08, 0x40}, 64}, + {"missing displacement", []byte{0xc4, 0xc2, 0x6d, 0x90, 0x4c, 0x40}, 64}, + {"address override", []byte{0x67, 0xc4, 0xc2, 0x6d, 0x90, 0x0c, 0x40}, 64}, + {"high VSIB index on 386", []byte{0xc4, 0x82, 0x6d, 0x90, 0x0c, 0x40}, 32}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, matched, err := decodedX86RawGatherInstruction(test.code, test.mode); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", test.code, matched, err) + } + }) + } +} + +func TestX86RawEVEXGatherRejectsReservedForms(t *testing.T) { + const source = "TEXT evexgather(SB),4,$0-0\n\tVPGATHERDD 8(BX)(X2*4), K1, X3\n\tRET\n" + assembled := assembleX87ControlBytes(t, "amd64", source) + if len(assembled) < 8 || assembled[0] != 0x62 { + t.Fatalf("Go did not emit an EVEX gather: %x", assembled) + } + valid := assembled[:len(assembled)-1] + for _, test := range []struct { + name string + mutate func([]byte) []byte + }{ + {"missing mask", func(code []byte) []byte { code[3] &^= 7; return code }}, + {"zeroing", func(code []byte) []byte { code[3] |= 0x80; return code }}, + {"broadcast", func(code []byte) []byte { code[3] |= 0x10; return code }}, + {"reserved length", func(code []byte) []byte { code[3] |= 0x60; return code }}, + {"reserved vvvv", func(code []byte) []byte { code[2] &^= 0x08; return code }}, + {"missing fixed bit", func(code []byte) []byte { code[2] &^= 0x04; return code }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.mutate(append([]byte(nil), valid...)) + if _, _, matched, err := decodedX86RawGatherInstruction(code, 64); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", code, matched, err) + } + }) + } +} + +func TestTranslateX86RawGatherLLVM22AllTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, form := range []struct { + name string + source string + }{ + {"VEX", "TEXT rawgather(SB),4,$0-0\n\tVPGATHERDD X2, (BX)(X0*2), X1\n\tRET\n"}, + {"EVEX", "TEXT rawgather(SB),4,$0-0\n\tVPGATHERDD (BX)(X0*2), K1, X2\n\tRET\n"}, + } { + t.Run(form.name, func(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + code := assembleX87ControlBytes(t, target.goarch, form.source) + file := x86RawUnprovedReturnFormProbeFile(t, target.goarch, "rawgather", code) + // A terminal byte cannot supply the preceding gather's + // index, mask, merge input or native-memory authority. + ir := x86UnboundVoidRawProbeObject(t, file, target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-gather.ll", "raw-gather.o", ir) + }) + } + }) + } +} diff --git a/amd64_raw_gfni.go b/amd64_raw_gfni.go new file mode 100644 index 000000000..aac52af6b --- /dev/null +++ b/amd64_raw_gfni.go @@ -0,0 +1,58 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// The Go 1.27 table has one GFNI family: multiply (0F38 CF, W0) and the +// affine/inverse pair (0F3A CE/CF, W1). Its VEX rows are X/Y; EVEX adds Z, +// masks, zeroing and, only for the affine matrix source, qword broadcast. +func decodedX86GFNIInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok { + return Instr{}, 0, false, nil + } + + var op Op + var immediate bool + switch { + case p.mapNumber == 2 && p.opcode == 0xcf: + op = "VGF2P8MULB" + case p.mapNumber == 3 && p.opcode == 0xce: + op, immediate = "VGF2P8AFFINEQB", true + case p.mapNumber == 3 && p.opcode == 0xcf: + op, immediate = "VGF2P8AFFINEINVQB", true + default: + return Instr{}, 0, false, nil + } + if p.w != immediate { + return Instr{}, 0, true, fmt.Errorf("%s has invalid W bit", op) + } + if p.broadcast && !immediate { + return Instr{}, 0, true, fmt.Errorf("%s has no broadcast form", op) + } + + laneBits := 8 + if immediate { + laneBits = 64 + } + instruction, length, _, err := decodedX86BinaryVectorOperands(code, p, mode, op, laneBits) + if err != nil { + return Instr{}, 0, true, err + } + if !immediate { + return instruction, length, true, nil + } + if len(code) <= length { + return Instr{}, 0, true, fmt.Errorf("%s is missing its imm8", op) + } + instruction.Args = append([]Operand{{Kind: OpImm, Imm: int64(code[length])}}, instruction.Args...) + length++ + parts := make([]string, len(instruction.Args)) + for index, arg := range instruction.Args { + parts[index] = arg.String() + } + instruction.Raw = fmt.Sprintf("%s %s", instruction.Op, strings.Join(parts, ", ")) + return instruction, length, true, nil +} diff --git a/amd64_raw_gfni_test.go b/amd64_raw_gfni_test.go new file mode 100644 index 000000000..045651907 --- /dev/null +++ b/amd64_raw_gfni_test.go @@ -0,0 +1,173 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawGFNIGoFECRegression(t *testing.T) { + // GoFEC v1.4.2 emits this EVEX-512 affine instruction as BYTE directives. + code := []byte{0x62, 0xf3, 0x85, 0x48, 0xce, 0xc8, 0x00} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "GoFEC GFNI", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, "VGF2P8AFFINEQB $0, Z0, Z15, Z1 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawGFNICompleteGoFamily(t *testing.T) { + for _, family := range []struct { + name string + mapID byte + opcode byte + w byte + imm bool + }{ + {name: "VGF2P8MULB", mapID: 2, opcode: 0xcf}, + {name: "VGF2P8AFFINEQB", mapID: 3, opcode: 0xce, w: 1, imm: true}, + {name: "VGF2P8AFFINEINVQB", mapID: 3, opcode: 0xcf, w: 1, imm: true}, + } { + for _, encoding := range []struct { + name string + width int + evex bool + masked bool + zeroing bool + }{ + {name: "vex-x", width: 0}, + {name: "vex-y", width: 1}, + {name: "evex-x", width: 0, evex: true}, + {name: "evex-y-mask", width: 1, evex: true, masked: true}, + {name: "evex-z-zero", width: 2, evex: true, masked: true, zeroing: true}, + } { + t.Run(family.name+"/"+encoding.name, func(t *testing.T) { + width := []string{"X", "Y", "Z"}[encoding.width] + var code []byte + if encoding.evex { + last := byte(encoding.width<<5) | 0x08 + if encoding.masked { + last |= 1 + } + if encoding.zeroing { + last |= 0x80 + } + code = []byte{0x62, 0xf0 | family.mapID, family.w<<7 | 0x75, last, family.opcode, 0xd0} + } else { + code = []byte{0xc4, 0xe0 | family.mapID, family.w<<7 | 0x71 | byte(encoding.width<<2), family.opcode, 0xd0} + } + if family.imm { + code = append(code, 0x63) + } + instruction, length, ok, err := decodedX86GFNIInstruction(code, 64) + if !ok || err != nil || length != len(code) { + t.Fatalf("decode %x: ok=%v length=%d err=%v", code, ok, length, err) + } + args := fmt.Sprintf("%s0, %s1", width, width) + if family.imm { + args = "$99, " + args + } + if encoding.masked { + args += ", K1" + } + wantOp := family.name + if encoding.zeroing { + wantOp += ".Z" + } + want := fmt.Sprintf("%s %s, %s2", wantOp, args, width) + if instruction.Raw != want { + t.Fatalf("decode %x as %q, want %q", code, instruction.Raw, want) + } + }) + } + } +} + +func TestDecodeX86RawGFNIRejectsReservedForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + }{ + {name: "affine wrong W", code: []byte{0x62, 0xf3, 0x75, 0x48, 0xce, 0xd0, 0x63}}, + {name: "multiply wrong W", code: []byte{0x62, 0xf2, 0xf5, 0x48, 0xcf, 0xd0}}, + {name: "multiply broadcast", code: []byte{0x62, 0xf2, 0x75, 0x58, 0xcf, 0x50, 0x01}}, + {name: "affine broadcast register", code: []byte{0x62, 0xf3, 0xf5, 0x58, 0xce, 0xd0, 0x63}}, + {name: "zero without mask", code: []byte{0x62, 0xf3, 0xf5, 0xc8, 0xce, 0xd0, 0x63}}, + {name: "truncated immediate", code: []byte{0x62, 0xf3, 0xf5, 0x48, 0xce, 0xd0}}, + {name: "reserved vector length", code: []byte{0x62, 0xf3, 0xf5, 0x68, 0xce, 0xd0, 0x63}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86GFNIInstruction(test.code, 64) + if !ok || err == nil { + t.Fatalf("decode %x: ok=%v err=%v, want recognized rejection", test.code, ok, err) + } + }) + } +} + +func TestDecodeX86RawGFNIMemoryAndBroadcast(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + {name: "vex multiply memory", code: []byte{0xc4, 0xe2, 0x71, 0xcf, 0x50, 0x08}, want: "VGF2P8MULB 8(AX), X1, X2"}, + {name: "evex affine broadcast", code: []byte{0x62, 0xf3, 0xf5, 0x59, 0xce, 0x50, 0x01, 0x63}, want: "VGF2P8AFFINEQB.BCST $99, 8(AX), Z1, K1, Z2"}, + {name: "evex inverse broadcast zero", code: []byte{0x62, 0xf3, 0xf5, 0xd9, 0xcf, 0x50, 0x01, 0xa5}, want: "VGF2P8AFFINEINVQB.BCST.Z $165, 8(AX), Z1, K1, Z2"}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86GFNIInstruction(test.code, 64) + if !ok || err != nil || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode %x = %q, length=%d, ok=%v, err=%v; want %q", test.code, instruction.Raw, length, ok, err, test.want) + } + }) + } +} + +func TestTranslateX86RawGFNIAllObjectTargets(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawgfni(SB),$0-0\n") + for _, instruction := range [][]byte{ + {0xc4, 0xe2, 0x71, 0xcf, 0xd0}, + {0xc4, 0xe3, 0xf1, 0xce, 0xd0, 0x63}, + {0x62, 0xf3, 0xf5, 0xc9, 0xcf, 0xd0, 0xa5}, + } { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawgfni": {Name: "rawgfni", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "raw-gfni.ll", "raw-gfni.o", ir) + }) + } +} diff --git a/amd64_raw_horizontal_float_literal_test.go b/amd64_raw_horizontal_float_literal_test.go new file mode 100644 index 000000000..fc3bb3160 --- /dev/null +++ b/amd64_raw_horizontal_float_literal_test.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawHorizontalFloatLiteral(opcode, pp byte, vex3 bool, width int) []byte { + code := encodeX86VEXHorizontalFloat(opcode, pp, vex3, width == 32, false, 1, 2, 0) + code[len(code)-1] = 0x0d // RIP+disp32, destination X/Y1. + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*11+7)) + } + return code +} + +func TestDecodeX86RawHorizontalFloatRIPDataCompleteFamily(t *testing.T) { + for opcode, stem := range map[byte]string{0x7c: "VHADD", 0x7d: "VHSUB", 0xd0: "VADDSUB"} { + for pp, suffix := range map[byte]string{1: "PD", 3: "PS"} { + for _, vex3 := range []bool{false, true} { + for _, width := range []int{16, 32} { + name := fmt.Sprintf("%s%s/vex3%t/%d", stem, suffix, vex3, width) + t.Run(name, func(t *testing.T) { + code := x86RawHorizontalFloatLiteral(opcode, pp, vex3, width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != Op(stem+suffix) || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET || + !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-width:]) { + t.Fatalf("decoded %x as %#v", code, decoded) + } + }) + } + } + } + } +} + +func TestDecodeX86RawHorizontalFloatRIPDataRejectsUnsafeSources(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { code[5] = 127; return code }, + "overlapping instruction": func(code []byte) []byte { code[5] = 0; return code }, + "segment override": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawHorizontalFloatLiteral(0xd0, 1, true, 32)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe horizontal/alternating RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawHorizontalFloatRIPDataObjects(t *testing.T) { + var source strings.Builder + for opcode, stem := range map[byte]string{0x7c: "VHADD", 0x7d: "VHSUB", 0xd0: "VADDSUB"} { + for pp, suffix := range map[byte]string{1: "PD", 3: "PS"} { + name := "literal" + stem + suffix + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawHorizontalFloatLiteral(opcode, pp, true, 32) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-horizontal-float-literal.ll", "raw-horizontal-float-literal.o", ir) + }) + } +} diff --git a/amd64_raw_horizontal_integer.go b/amd64_raw_horizontal_integer.go new file mode 100644 index 000000000..b619228f4 --- /dev/null +++ b/amd64_raw_horizontal_integer.go @@ -0,0 +1,52 @@ +package plan9asm + +import "fmt" + +// Go's VPHADD/PHSUB family has two _yvaddsubpd rows, X and Y, without +// EVEX/masks. W is ignored (Go emits W0); Intel XED leaves W unconstrained: +// https://github.com/intelxed/xed/blob/main/datafiles/avx/avx-isa.txt +func decodedX86VEXHorizontalIntegerInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 { + return Instr{}, 0, false, nil + } + var op Op + for name, spec := range amd64HorizontalIntegerSpecs { + if spec.vector && int(spec.opcode) == p.opcode { + op = name + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("horizontal integer: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.pp != 1 || p.vectorLength > 1 { + return fail("requires VEX.66.0F38 X/Y encoding") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.x != 0 || p.b != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + width := []string{"X", "Y"}[p.vectorLength] + source2, size, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + source1 := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]>>3&7)+p.r*8))} + return Instr{ + Op: op, Args: []Operand{source2, source1, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, source2.String(), source1.String(), destination.String()), + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_horizontal_integer_test.go b/amd64_raw_horizontal_integer_test.go new file mode 100644 index 000000000..fe8272af7 --- /dev/null +++ b/amd64_raw_horizontal_integer_test.go @@ -0,0 +1,338 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86RawHorizontalIntegerEncodingAxes(t *testing.T) { + count := 0 + for _, form := range []struct { + op Op + opcode byte + }{ + {"VPHADDW", 0x01}, {"VPHADDD", 0x02}, {"VPHADDSW", 0x03}, + {"VPHSUBW", 0x05}, {"VPHSUBD", 0x06}, {"VPHSUBSW", 0x07}, + } { + for width, prefix := range []string{"X", "Y"} { + for _, w := range []bool{false, true} { + for dst := 0; dst < 16; dst++ { + for src1 := 0; src1 < 16; src1++ { + for src2 := 0; src2 < 16; src2++ { + code := encodeX86VEXHorizontalFloat(form.opcode, 1, true, width == 1, w, dst, src1, src2) + code[1] = code[1]&0xe0 | 2 + got, size, matched, err := decodedX86VEXHorizontalIntegerInstruction(code, 64) + want := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src2))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src1))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, dst))}} + if err != nil || !matched || size != len(code) || got.Op != form.op || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v size=%d matched=%v err=%v; want %s %+v", code, got, size, matched, err, form.op, want) + } + _, _, matched, err = decodedX86VEXHorizontalIntegerInstruction(code, 32) + if !matched || (err != nil) != (dst >= 8 || src1 >= 8 || src2 >= 8) { + t.Fatalf("386 decode %x matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + if count != 98304 { + t.Fatalf("covered %d encodings, want 98304", count) + } +} + +func TestX86RawHorizontalIntegerInvalidAndSegmentForms(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe2, 0x79, 0x02}, // Missing ModRM. + {0xc4, 0xe2, 0x79, 0x02, 0x04}, // Missing SIB. + {0xc4, 0xe2, 0x79, 0x02, 0x45}, // Missing disp8. + {0xc4, 0xe2, 0x78, 0x02, 0xc0}, // pp=none. + {0xc4, 0xe2, 0x7a, 0x02, 0xc0}, // pp=F3. + {0xc4, 0xe2, 0x7b, 0x02, 0xc0}, // pp=F2. + {0x62, 0xf2, 0x7d, 0x08, 0x02, 0xc0}, // No EVEX variant. + {0x67, 0xc4, 0xe2, 0x79, 0x02, 0x00}, + } { + if _, _, matched, err := decodedX86VEXHorizontalIntegerInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x matched=%v err=%v", code, matched, err) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + code := []byte{segment.prefix, 0xc4, 0x02, 0xe5, 0x06, 0x64, 0x88, 0x20} + got, size, matched, err := decodedX86VEXHorizontalIntegerInstruction(code, 64) + want := []Operand{{Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: 32}}, {Kind: OpReg, Reg: "Y3"}, {Kind: OpReg, Reg: "Y12"}} + if err != nil || !matched || size != len(code) || got.Op != "VPHSUBD" || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v %v", code, got, err) + } + } +} + +func TestX86HorizontalIntegerGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []string{"VPHADDW", "VPHADDD", "VPHADDSW", "VPHSUBW", "VPHSUBD", "VPHSUBSW"} { + lines := []string{op + " X0,Y1,Y2", op + " Z0,Z1,Z2", op + " X0,X1,K1,X2", op + " X0,X1,(AX)", op + ".Z X0,X1,X2", op + ".SAE X0,X1,X2", op + " X16,X0,X1"} + for _, line := range lines { + dir := t.TempDir() + path := filepath.Join(dir, "invalid.s") + if err := os.WriteFile(path, []byte("TEXT invalid(SB),4,$0-0\n"+line+"\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), path) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + if output, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("Go %s accepted %s; repair oracle: %s", arch, line, output) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + assertX86HorizontalIntegerRejected(t, arch, triple, line) + } + } + } +} + +func TestTranslateX86HorizontalIntegerGo386RegisterCompatibility(t *testing.T) { + // Go's VEX Yxr/Yyr classes accept X/Y8-15 even for its 386 frontend. + // This named compatibility grammar is distinct from architectural raw + // 32-bit decoding, which still rejects extended register fields. + var source strings.Builder + source.WriteString("TEXT compat(SB),4,$0-0\n") + for _, op := range []string{"VPHADDW", "VPHADDD", "VPHADDSW", "VPHSUBW", "VPHSUBD", "VPHSUBSW"} { + for _, width := range []string{"X", "Y"} { + for _, regs := range [][3]int{{8, 0, 1}, {0, 8, 1}, {0, 1, 8}, {15, 14, 13}} { + fmt.Fprintf(&source, "%s %s%d,%s%d,%s%d\n", op, width, regs[0], width, regs[1], width, regs[2]) + } + } + } + source.WriteString("RET\n") + assembleX87ControlBytes(t, "386", source.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"compat": {Name: "compat", Ret: Void}}} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go frontend acceptance invented high physical mode32 registers: %v", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("direct module invented mode32 high registers", err) + } + translated, err := translateGoModuleInContext(ctx, x86SADGoPackage(t, "386", "()"), []byte(strings.Replace(source.String(), "compat(SB)", "·sad(SB)", 1)), GoModuleOptions{GOARCH: "386", TargetTriple: triple}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("typed module invented mode32 high registers", err) + } + }) + } +} + +// All six VEX instructions share Go's two _yvaddsubpd rows (X/Y). +// The legacy siblings use yxm_q4, except PHADDD's MMX-capable ymmxmm0f38. +func TestX86RawHorizontalIntegerGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + last := 7 + if arch == "amd64" { + last = 15 + } + var source strings.Builder + source.WriteString("TEXT horizontal(SB),4,$0-0\nPHADDD M0,M7\nPHADDD -7(BX)(CX*4),M0\n") + for _, op := range []string{"PHADDD", "PHADDSW", "PHADDW", "PHSUBD", "PHSUBSW", "PHSUBW"} { + for _, reg := range []int{0, last} { + for _, input := range []string{fmt.Sprintf("X%d", last-reg), "(AX)", "-7(BX)(CX*4)", "8192(SI)"} { + fmt.Fprintf(&source, "%s %s,X%d\n", op, input, reg) + } + for _, width := range []string{"X", "Y"} { + for _, input := range []string{fmt.Sprintf("%s%d", width, last-reg), "(AX)", "-7(BX)(CX*4)", "8192(SI)"} { + fmt.Fprintf(&source, "V%s %s,%s%d,%s%d\n", op, input, width, last-reg, width, reg) + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(want) != 147 || len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions; expected %d, including RET", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + file := x86RawUnprovedReturnFormProbeFile(t, arch, "horizontal", code) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + // A terminal UD2 removes only the return obligation. It does + // not initialize the preceding MMX/vector/address inputs. + // Keep all original formats as private objects, independently + // of the three public APIs' required Context observations. + ir := x86UnboundVoidRawProbeObject(t, file, arch, triple) + compileLLVMToObject(t, llc, triple, "raw-horizontal.ll", "raw-horizontal.o", ir) + } + }) + } +} + +func TestX86HorizontalIntegerInitializedMMXRegionKeepsRawBoundary(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + const declaration = "func horizontalMMX(a,b uint64) uint64\n" + const source = "TEXT ·horizontalMMX(SB),4,$0-24\nMOVQ a+0(FP),M0\nMOVQ b+8(FP),M7\nPHADDD M0,M7\nMOVQ M7,ret+16(FP)\nEMMS\nRET\n" + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + x86LEAActualTargetGoObject(t, arch, triple, source) + code := assembleX87ControlBytes(t, arch, "TEXT row(SB),4,$0-0\nPHADDD M0,M7\nRET\n") + if !reflect.DeepEqual(code, []byte{0x0f, 0x38, 0x02, 0xf8, 0xc3}) { + t.Fatal("initialized counterpart lost the original MMX row identity", code) + } + var bytes strings.Builder + for _, value := range code[:len(code)-1] { + fmt.Fprintf(&bytes, "BYTE $0x%02x\n", value) + } + raw := strings.Replace(source, "PHADDD M0,M7\n", bytes.String(), 1) + x86LEAActualTargetGoObject(t, arch, triple, raw) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, arch, triple, api) + if err != nil { + t.Fatal("complete named MMX region lost its bounded inputs/EMMS", api, err) + } + if !strings.Contains(ir, "phaddd") || !strings.Contains(ir, "emms") { + t.Fatal("initialized counterpart lost the physical MMX operation/tag closure", api) + } + compileLLVMToObject(t, llc, triple, "initialized-horizontal-mmx-"+api+".ll", "initialized-horizontal-mmx-"+api+".o", ir) + if ir, err := x86ScalarFPBitViewAPI(t, raw, declaration, arch, triple, api); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("typed values and EMMS invented an authenticated raw MMX region", api, err) + } + } + }) + } +} + +func TestX86HorizontalIntegerViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; separate five-target object tests are required") + } + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, spec := range []struct { + op string + bits, subtract, saturate int + }{{"VPHADDW", 16, 0, 0}, {"VPHADDD", 32, 0, 0}, {"VPHADDSW", 16, 0, 1}, {"VPHSUBW", 16, 1, 0}, {"VPHSUBD", 32, 1, 0}, {"VPHSUBSW", 16, 1, 1}} { + for width, prefix := range []string{"X", "Y"} { + for _, memory := range []bool{false, true} { + for dst := 0; dst < 3; dst++ { + name := fmt.Sprintf("horizontal%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),DI\nMOVQ old+24(FP),SI\nVMOVUPS (AX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (SI),Z2\n", name) + input := prefix + "1" + if memory { + input = "(BX)" + } + fmt.Fprintf(&source, "%s %s,%s0,%s%d\nVMOVUPS Z%d,(DI)\nVMOVUPS Y%d,64(DI)\nVMOVUPS X%d,96(DI)\nRET\n", spec.op, input, prefix, prefix, dst, dst, dst, dst) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: Ptr, Index: 3, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *,const void *);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d,%d,%d,%d)) return %d;\n", name, 16< +#include +#include +` + declarations.String() + ` +static int64_t lane(const uint8_t *p,int bytes) { + if(bytes==2) {int16_t v;memcpy(&v,p,2);return v;} + int32_t v;memcpy(&v,p,4);return v; +} +static int check(void (*fn)(const void *,const void *,void *,const void *),int bytes,int laneBytes,int sub,int sat) { + for(int seed=0;seed<128;seed++) { + uint8_t a[64],b[64],old[64],out[112],want[64]={0}; + for(int i=0;i<64;i++) {a[i]=(uint8_t)(seed*17+i*31);b[i]=(uint8_t)(seed*43+i*7);old[i]=(uint8_t)(193-i);} + for(int i=0;i32767)result=32767;} + uint64_t bits=(uint64_t)result; + for(int j=0;j>(j*8)); + } + fn(a,b,out,old); + for(int view=0;view<3;view++) { + int offset=view==0?0:view==1?64:96; + for(int i=0;i<(64>>view);i++) if(out[offset+i]!=want[i]) { + fprintf(stderr,"horizontal mismatch bytes=%d lane=%d sub=%d sat=%d seed=%d view=%d byte=%d\n",bytes,laneBytes,sub,sat,seed,view,i);return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "horizontal_integer", triple, ir, mainC, commandPrefix) +} diff --git a/amd64_raw_hypercall_test.go b/amd64_raw_hypercall_test.go new file mode 100644 index 000000000..4545be9bd --- /dev/null +++ b/amd64_raw_hypercall_test.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// KVM defines both vendor encodings of the same zero-operand hypercall ABI: +// https://docs.kernel.org/virt/kvm/x86/hypercalls.html +func TestX86RawKVMHypercallFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, op := range []struct { + name string + last byte + }{ + {name: "VMCALL", last: 0xc1}, + {name: "VMMCALL", last: 0xd9}, + } { + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(op.name+"/"+target.triple, func(t *testing.T) { + source := fmt.Sprintf(`TEXT hypercall(SB),$0-0 + MOVL $9, AX + MOVL $17, BX + MOVL $23, CX + MOVL $29, DX + MOVL $31, SI + BYTE $0x0f; BYTE $0x01; BYTE $0x%02x + RET +`, op.last) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + if decoded.Instrs[len(decoded.Instrs)-2].Op != Op(op.name) { + t.Fatalf("wrong hypercall decoding: %+v", decoded.Instrs) + } + word := I64 + if target.arch == "386" { + word = I32 + } + ir, err := Translate(file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"hypercall": {Name: "hypercall", Ret: word}}, + }) + if target.arch == "386" { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("logical mode32 scalar result acquired a raw/native entry contract", err) + } + normalized, err := normalizeX86RawFile(file, target.arch) + if err != nil { + t.Fatal(err) + } + // Retain the actual hypercall opcode/constraints and AX + // return as an object, not an executable logical ABI proof. + ir = x86ScalarFormatObjectUnit(t, normalized, + map[string]FuncSig{"hypercall": {Name: "hypercall", Ret: word}}, target.arch, target.triple, 0) + } else if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `asm sideeffect "` + strings.ToLower(op.name) + `"`, + `={ax},0,{bx},{cx},{dx},{si},`, `~{memory}`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("missing hypercall ABI %q", want) + } + } + compileLLVMToObject(t, llc, target.triple, "hypercall.ll", "hypercall.o", ir) + }) + } + } +} + +func TestX86KVMHypercallRejectsExplicitOperandsAndSuffixes(t *testing.T) { + for _, instruction := range []string{"VMCALL AX", "VMMCALL AX", "VMCALL.Z", "VMMCALL.Z"} { + file, err := Parse(ArchAMD64, "TEXT invalid(SB),$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"invalid": {Name: "invalid", Ret: Void}}, + }) + if err == nil { + t.Fatalf("accepted invalid hypercall %q", instruction) + } + } +} diff --git a/amd64_raw_ifma_test.go b/amd64_raw_ifma_test.go new file mode 100644 index 000000000..44839f777 --- /dev/null +++ b/amd64_raw_ifma_test.go @@ -0,0 +1,98 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawDirectiveGroupIFMACompleteFamily(t *testing.T) { + // Bytes come from Go 1.27's avx512_ifma assembler test table and cover + // both opcodes, all widths, high registers, memory, and masking. + for _, test := range []struct { + name string + code []byte + want string + }{ + {name: "high x high registers", code: []byte{0x62, 0xe2, 0xa5, 0x09, 0xb5, 0xd7}, want: "VPMADD52HUQ X7, X11, K1, X18"}, + {name: "high y high registers", code: []byte{0x62, 0x82, 0x85, 0x27, 0xb5, 0xcc}, want: "VPMADD52HUQ Y28, Y31, K7, Y17"}, + {name: "low y memory", code: []byte{0x62, 0xf2, 0xb5, 0x29, 0xb4, 0x94, 0x2c, 0x11, 0, 0, 0}, want: "VPMADD52LUQ 17(SP)(BP*1), Y9, K1, Y2"}, + {name: "low z high registers", code: []byte{0x62, 0x22, 0xd5, 0x47, 0xb4, 0xe0}, want: "VPMADD52LUQ Z16, Z21, K7, Z28"}, + {name: "low z broadcast zeroing", code: []byte{0x62, 0xf2, 0xf5, 0xdb, 0xb4, 0x50, 0x7f}, want: "VPMADD52LUQ.BCST.Z 1016(AX), Z1, K3, Z2"}, + } { + t.Run(test.name, func(t *testing.T) { + decoded, err := decodeX86RawDirectiveGroup(test.code, 64, 0, test.name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, test.want+" ") { + t.Fatalf("decoded %x as %#v, want %q", test.code, decoded, test.want) + } + }) + } +} + +func TestDecodedX86RawIFMAAllRegisterFields(t *testing.T) { + for _, operation := range []struct { + opcode byte + op Op + }{ + {opcode: 0xb4, op: "VPMADD52LUQ"}, + {opcode: 0xb5, op: "VPMADD52HUQ"}, + } { + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 2 + p1 := byte(1)<<7 | byte(^second&15)<<3 | 5 + p2 := byte(vectorBits)<<5 | byte((^second>>4)&1)<<3 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0x62, p0, p1, p2, operation.opcode, modRM} + instruction, length, ok, err := decodedX86IFMAInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("op=%s width=%s first=%d second=%d destination=%d: ok=%v length=%d err=%v", operation.op, vectorName, first, second, destination, ok, length, err) + } + want := fmt.Sprintf("%s %s%d, %s%d, %s%d", operation.op, vectorName, first, vectorName, second, vectorName, destination) + if instruction.Raw != want { + t.Fatalf("encoding %x decoded as %q, want %q", encoding, instruction.Raw, want) + } + } + } + } + } + } +} + +func TestDecodedX86RawIFMARejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "386 masked extended source", mode: 32, encoding: []byte{0x62, 0xf2, 0xb5, 0x09, 0xb4, 0xc2}, wantOK: true}, + {name: "386 extended register", mode: 32, encoding: []byte{0x62, 0x72, 0xf5, 0x08, 0xb4, 0xc2}, wantOK: true}, + {name: "reserved vector length", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x68, 0xb4, 0xc2}, wantOK: true}, + {name: "broadcast register", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x18, 0xb4, 0xc2}, wantOK: true}, + {name: "zero without mask", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x88, 0xb5, 0xc2}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0x62, 0xf2, 0xf5, 0x08, 0xb4, 0xc2}, wantOK: true}, + {name: "wrong map", mode: 64, encoding: []byte{0x62, 0xf3, 0xf5, 0x08, 0xb4, 0xc2}, wantOK: false}, + {name: "wrong prefix", mode: 64, encoding: []byte{0x62, 0xf2, 0xf4, 0x08, 0xb4, 0xc2}, wantOK: false}, + {name: "wrong w", mode: 64, encoding: []byte{0x62, 0xf2, 0x75, 0x08, 0xb4, 0xc2}, wantOK: true}, + {name: "truncated", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x08, 0xb4}, wantOK: false}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86IFMAInstruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x: ok=%v, want %v (err=%v)", test.encoding, ok, test.wantOK, err) + } + if test.wantOK && err == nil { + t.Fatalf("encoding %x was recognized without rejection", test.encoding) + } + }) + } +} diff --git a/amd64_raw_indexed_permute_test.go b/amd64_raw_indexed_permute_test.go new file mode 100644 index 000000000..d8c815486 --- /dev/null +++ b/amd64_raw_indexed_permute_test.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawIndexedPermuteForms = []struct { + opcode int + widthBit int + op Op + laneBytes int +}{ + {0x75, 0, "VPERMI2B", 1}, {0x75, 1, "VPERMI2W", 2}, + {0x76, 0, "VPERMI2D", 4}, {0x76, 1, "VPERMI2Q", 8}, + {0x77, 0, "VPERMI2PS", 4}, {0x77, 1, "VPERMI2PD", 8}, + {0x7d, 0, "VPERMT2B", 1}, {0x7d, 1, "VPERMT2W", 2}, + {0x7e, 0, "VPERMT2D", 4}, {0x7e, 1, "VPERMT2Q", 8}, + {0x7f, 0, "VPERMT2PS", 4}, {0x7f, 1, "VPERMT2PD", 8}, +} + +func TestDecodeX86RawIndexedPermuteSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VPERMT2W Y18, Y19, Y21. + code := []byte{0x62, 0xa2, 0xe5, 0x20, 0x7d, 0xea, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "validUTF8 AVX512 VPERMT2W", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPERMT2W" || + decoded[0].Args[0].Reg != "Y18" || decoded[0].Args[1].Reg != "Y19" || + decoded[0].Args[2].Reg != "Y21" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func encodeX86RawIndexedPermute(opcode, widthBit, vectorBits, mask int, broadcast, memory bool) []byte { + p1 := byte(0x75) // EVEX.vvvv = vector register 1, pp = 66. + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // first source register 3, destination register 2. + if memory { + modRM = 0x50 // disp8(AX), destination register 2. + } + code := []byte{0x62, 0xf2, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawIndexedPermuteCompleteGoForms(t *testing.T) { + if len(x86RawIndexedPermuteForms) != 12 || len(amd64IndexedPermuteSpecs) != 14 { + t.Fatalf("raw/indexed-permute grammar has %d/%d rows, want 12/14", len(x86RawIndexedPermuteForms), len(amd64IndexedPermuteSpecs)) + } + for _, form := range x86RawIndexedPermuteForms { + for vectorBits, vector := range []string{"X", "Y", "Z"} { + for _, variant := range []struct { + name string + mask int + broadcast bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "memory", memory: true}, + {name: "masked memory", mask: 7, memory: true}, + {name: "broadcast memory", broadcast: true, memory: true}, + } { + if variant.broadcast && form.laneBytes < 4 { + continue + } + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawIndexedPermute(form.opcode, form.widthBit, vectorBits, variant.mask, variant.broadcast, variant.memory) + got, length, ok, err := decodedX86IndexedPermuteInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.broadcast { + wantOp += ".BCST" + } + if got.Op != wantOp || got.Args[1].String() != vector+"1" || + got.Args[len(got.Args)-1].String() != vector+"2" || + len(got.Args) != 3+map[bool]int{true: 1}[variant.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawIndexedPermuteRIPLiteral(t *testing.T) { + for _, form := range x86RawIndexedPermuteForms { + for _, broadcast := range []bool{false, true} { + if broadcast && form.laneBytes < 4 { + continue + } + code := encodeX86RawIndexedPermute(form.opcode, form.widthBit, 2, 1, broadcast, false) + code[5] = 0x15 // RIP-relative first source. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 64 + if broadcast { + literalWidth = form.laneBytes + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/broadcast%t", form.op, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + wantOp := form.op + if broadcast { + wantOp += ".BCST" + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != literalWidth || got[0].Op != wantOp { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestTranslateX86RawIndexedPermuteLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawIndexedPermute(SB),$0-0\n") + for _, opcode := range []int{0x75, 0x76, 0x77, 0x7d, 0x7e, 0x7f} { + for _, widthBit := range []int{0, 1} { + code := encodeX86RawIndexedPermute(opcode, widthBit, 0, 0, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + if opcode == 0x75 || opcode == 0x7d { + continue + } + mask := 3 + if target.goarch == "386" { + mask = 0 + } + code = encodeX86RawIndexedPermute(opcode, widthBit, 0, mask, true, true) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawIndexedPermute": {Name: "rawIndexedPermute", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-indexed-permute.ll", "raw-indexed-permute.o", ir) + }) + } +} diff --git a/amd64_raw_insertps_mode32_test.go b/amd64_raw_insertps_mode32_test.go new file mode 100644 index 000000000..c6057a89c --- /dev/null +++ b/amd64_raw_insertps_mode32_test.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +func x86RawINSERTPSSource(name string, code []byte) string { + var source strings.Builder + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + source.WriteString("RET\n") + return source.String() +} + +func TestX86RawINSERTPSMode32CompleteRegisterGrammar(t *testing.T) { + // Go's 386 frontend rejects four operands. BYTE acceptance is a distinct + // source role, not permission to accept the equivalent named instruction. + for _, row := range []string{ + "VINSERTPS $28,X7,X7,X7", + "VINSERTPS $28,1(BX),X7,X7", + } { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + ins.Raw += " /* decoded from BYTE $0xc4 */" + if _, known, err := x86InsertExtractProperties(ins, "386"); !known || err == nil { + t.Fatalf("a textual annotation granted physical mode32 identity: %+v", ins) + } + } + goAssemblerOracleOutput(t, "386", x86RawINSERTPSSource("f", encodeX86RawVEXINSERTPS(7, 7, 7, 28)), "", true) + + count := 0 + for source := 0; source < 8; source++ { + for base := 0; base < 8; base++ { + for destination := 0; destination < 8; destination++ { + immediate := byte(source*32 + base*4 + destination) + code := encodeX86RawVEXINSERTPS(source, base, destination, immediate) + ins, length, known, err := decodedX86InsertPSInstruction(code, 32) + if err != nil || !known || length != len(code) || !ins.x86Encoded { + t.Fatalf("physical mode32 decode %x: %+v, length=%d, known=%t, err=%v", code, ins, length, known, err) + } + form, known, err := x86InsertExtractProperties(ins, "386") + if err != nil || !known || form.immediate != immediate || form.width != 16 || form.output != 16 || form.memory != -1 { + t.Fatalf("physical mode32 grammar %x: %+v, known=%t, err=%v", code, form, known, err) + } + count++ + } + } + } + if count != 512 { + t.Fatalf("mode32 register axis has %d cases, want 512", count) + } +} + +func TestX86RawINSERTPSVEXIgnoredWLLVMOracle(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM 22 llvm-mc not found") + } + // Intel XED's VINSERTPS patterns do not constrain REXW, unlike adjacent + // VINSERTF128 patterns: datafiles/avx/avx-isa.txt. Both VEX.W values decode + // to the same instruction; Go's named encoder always chooses W=0. + for _, mode := range []int{32, 64} { + var input strings.Builder + for _, w := range []byte{0, 0x80} { + code := encodeX86RawVEXINSERTPS(7, 7, 7, 28) + code[2] |= w + for _, value := range code { + fmt.Fprintf(&input, "%#02x ", value) + } + input.WriteByte('\n') + ins, length, known, err := decodedX86InsertPSInstruction(code, mode) + if err != nil || !known || length != len(code) || ins.Raw != "VINSERTPS $28, X7, X7, X7" { + t.Fatalf("VEX.WIG mode%d decode %x: %+v, length=%d, known=%t, err=%v", mode, code, ins, length, known, err) + } + } + triple := "i386-unknown-linux-gnu" + if mode == 64 { + triple = "x86_64-unknown-linux-gnu" + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple="+triple, "--mattr=+avx") + command.Stdin = strings.NewReader(input.String()) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || strings.Count(string(output), "vinsertps\t$28, %xmm7, %xmm7, %xmm7") != 2 { + t.Fatalf("LLVM 22 independent VEX.WIG mode%d: %v\n%s", mode, err, output) + } + } +} + +func TestX86RawINSERTPSMode32MemoryAndFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Independent ModRM/SIB axes: direct, signed disp8/disp32 and every scale. + memory := [][]byte{ + {0x3b}, {0x7b, 0x80}, {0x7b, 0x7f}, + {0xbb, 0, 0, 0, 0x80}, {0xbb, 0xff, 0xff, 0xff, 0x7f}, + {0x7c, 0x33, 1}, {0x7c, 0x73, 1}, {0x7c, 0xb3, 1}, {0x7c, 0xf3, 1}, + } + var source strings.Builder + sigs := map[string]FuncSig{} + for index, modRM := range memory { + for _, w := range []byte{0, 0x80} { + code := append([]byte{0xc4, 0xe3, 0x41 | w, 0x21}, modRM...) + code = append(code, 0xff) + ins, length, known, err := decodedX86InsertPSInstruction(code, 32) + if err != nil || !known || length != len(code) { + t.Fatalf("mode32 memory decode %x: %+v, length=%d, known=%t, err=%v", code, ins, length, known, err) + } + form, known, err := x86InsertExtractProperties(ins, "386") + if err != nil || !known || form.memory != 1 || form.spec.chunk != 4 || form.immediate != 255 { + t.Fatalf("mode32 memory grammar %x: %+v, known=%t, err=%v", code, form, known, err) + } + name := fmt.Sprintf("memory%d_w%d", index, w>>7) + source.WriteString(x86RawINSERTPSSource(name, code)) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + for _, arch := range []string{"386", "amd64"} { + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triples := []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + if arch == "amd64" { + triples = []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-insertps-memory.ll", "raw-insertps-memory.o", ir) + } + } +} diff --git a/amd64_raw_insertps_test.go b/amd64_raw_insertps_test.go new file mode 100644 index 000000000..c09308c6b --- /dev/null +++ b/amd64_raw_insertps_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawLegacyINSERTPS(source, destination int, immediate byte) []byte { + rex := byte(0x40 | source>>3&1 | destination>>3&1<<2) + code := []byte{0x66} + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0x3a, 0x21, byte(0xc0|destination&7<<3|source&7), immediate) +} + +func encodeX86RawVEXINSERTPS(source, base, destination int, immediate byte) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^source>>3)&1)<<5 | 3 + p1 := byte(^base&15)<<3 | 1 + return []byte{0xc4, p0, p1, 0x21, byte(0xc0 | destination&7<<3 | source&7), immediate} +} + +func encodeX86RawEVEXINSERTPS(source, base, destination int, immediate byte) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^source>>4)&1)<<6 | + byte((^source>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 3 + p1 := byte(^base&15)<<3 | 5 + p2 := byte((^base>>4)&1) << 3 + return []byte{0x62, p0, p1, p2, 0x21, byte(0xc0 | destination&7<<3 | source&7), immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVINSERTPS(t *testing.T) { + // vinsertps $0x1c, %xmm10, %xmm8, %xmm1; + // vshufps $0x24, %xmm0, %xmm1, %xmm0 + code := []byte{0xc4, 0xc3, 0x39, 0x21, 0xca, 0x1c, 0xc5, 0xf0, 0xc6, 0xc0, 0x24} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VINSERTPS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VINSERTPS $28, X10, X8, X1 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawINSERTPSCompleteRegisterFamily(t *testing.T) { + count := 0 + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawLegacyINSERTPS(source, destination, immediate) + got, length, ok, err := decodedX86InsertPSInstruction(code, 64) + want := fmt.Sprintf("INSERTPS $%d, X%d, X%d", immediate, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for source := 0; source < 16; source++ { + for base := 0; base < 16; base++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawVEXINSERTPS(source, base, destination, immediate) + got, length, ok, err := decodedX86InsertPSInstruction(code, 64) + want := fmt.Sprintf("VINSERTPS $%d, X%d, X%d, X%d", immediate, source, base, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + for source := 0; source < 32; source++ { + for base := 0; base < 32; base++ { + for destination := 0; destination < 32; destination++ { + immediate := byte(source*8 + destination) + code := encodeX86RawEVEXINSERTPS(source, base, destination, immediate) + got, length, ok, err := decodedX86InsertPSInstruction(code, 64) + want := fmt.Sprintf("VINSERTPS $%d, X%d, X%d, X%d", immediate, source, base, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + if count != 37120 { + t.Fatalf("covered %d INSERTPS/VINSERTPS register encodings, want 37120", count) + } +} + +func TestDecodedX86RawINSERTPSMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0x66, 0x47, 0x0f, 0x3a, 0x21, 0x7c, 0x8b, 0x01, 0xff} + got, length, ok, err := decodedX86InsertPSInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "INSERTPS" || got.Args[2].String() != "X15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x03, 0x01, 0x21, 0x7c, 0x8b, 0x02, 0x80} + got, length, ok, err = decodedX86InsertPSInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VINSERTPS" || got.Args[2].String() != "X15" || got.Args[3].String() != "X15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexMemory := []byte{0x62, 0x03, 0x05, 0x00, 0x21, 0x7c, 0x8b, 0x02, 0x40} + got, length, ok, err = decodedX86InsertPSInstruction(evexMemory, 64) + wantMemory.Off = 8 + if err != nil || !ok || length != len(evexMemory) || got.Op != "VINSERTPS" || got.Args[2].String() != "X31" || got.Args[3].String() != "X31" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + validVEX := encodeX86RawVEXINSERTPS(7, 7, 7, 1) + validEVEX := encodeX86RawEVEXINSERTPS(31, 31, 31, 1) + for name, invalid := range map[string][]byte{ + "vex vector length": {validVEX[0], validVEX[1], validVEX[2] | 0x04, validVEX[3], validVEX[4], validVEX[5]}, + "evex W": {validEVEX[0], validEVEX[1], validEVEX[2] | 0x80, validEVEX[3], validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex mask": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 1, validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex zero": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x80, validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex broadcast": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x10, validEVEX[4], validEVEX[5], validEVEX[6]}, + "truncated vex immediate": validVEX[:len(validVEX)-1], + "truncated evex immediate": validEVEX[:len(validEVEX)-1], + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86InsertPSInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + if instruction, _, matched, decodeErr := decodedX86InsertPSInstruction(validEVEX, 32); !matched || decodeErr == nil { + t.Fatalf("386 EVEX encoding %x decoded as %+v, ok=%v, err=%v", validEVEX, instruction, matched, decodeErr) + } + extended386 := encodeX86RawVEXINSERTPS(8, 0, 0, 1) + if instruction, _, matched, decodeErr := decodedX86InsertPSInstruction(extended386, 32); !matched || decodeErr == nil { + t.Fatalf("386 extended encoding %x decoded as %+v, ok=%v, err=%v", extended386, instruction, matched, decodeErr) + } +} + +func TestTranslateX86RawINSERTPSTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + registerMax int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", registerMax: 15}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registerMax: 15}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registerMax: 15}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", registerMax: 7}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", registerMax: 7}, + } { + t.Run(target.name, func(t *testing.T) { + code := [][]byte{ + encodeX86RawLegacyINSERTPS(target.registerMax, target.registerMax, 0xff), + encodeX86RawVEXINSERTPS(target.registerMax, target.registerMax, target.registerMax, 0x1c), + } + if target.goarch == "amd64" { + code = append(code, encodeX86RawEVEXINSERTPS(31, 30, 29, 0x80)) + } + var source strings.Builder + source.WriteString("TEXT raw_insertps(SB),NOSPLIT,$0-0\n") + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_insertps": {Name: "raw_insertps", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-insertps-"+target.name+".ll", "raw-insertps-"+target.name+".o", ir) + }) + } +} diff --git a/amd64_raw_integer_dot.go b/amd64_raw_integer_dot.go new file mode 100644 index 000000000..f76102e29 --- /dev/null +++ b/amd64_raw_integer_dot.go @@ -0,0 +1,36 @@ +package plan9asm + +import "fmt" + +func decodedX86IntegerPackedDotInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || p.evex && p.pp == 2 { + return Instr{}, 0, false, nil // BF16 owns EVEX.F3, not integer VNNI. + } + var op Op + for name, spec := range amd64PackedDotProductSpecs { + if !spec.bfloat && int(spec.opcode) == p.opcode { + op = name + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + if p.w { + return Instr{}, 0, true, fmt.Errorf("integer dot-product encoding requires W0") + } + ins, size, known, err := decodedX86BinaryVectorOperands(code, p, mode, op, 32) + if err != nil || !known { + return ins, size, known, err + } + if size > 15 { + return Instr{}, 0, true, fmt.Errorf("integer dot-product instruction exceeds 15 bytes") + } + ins.x86VectorBytes = 16 << p.vectorLength + ins.x86PackedDotEncoding = x86RawPackedDotVEX + if p.evex { + ins.x86PackedDotEncoding = x86RawPackedDotEVEX + } + return ins, size, true, nil +} diff --git a/amd64_raw_invalid_prefix_observation_test.go b/amd64_raw_invalid_prefix_observation_test.go new file mode 100644 index 000000000..230963ab2 --- /dev/null +++ b/amd64_raw_invalid_prefix_observation_test.go @@ -0,0 +1,201 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// BYTE acceptance establishes original bytes only. These instructions are not +// executed: Intel specifies #UD for their LOCK prefix. The observed decoder +// marks that prefix invalid; a textual conversion must not erase that fact. +func TestX86RawInvalidLockPrefixesMustNotDisappear(t *testing.T) { + rows := []struct { + op string + machine []byte + }{ + {"RDTSC", []byte{0x0f, 0x31}}, + {"RDTSCP", []byte{0x0f, 0x01, 0xf9}}, + {"LFENCE", []byte{0x0f, 0xae, 0xe8}}, + {"MFENCE", []byte{0x0f, 0xae, 0xf0}}, + {"SFENCE", []byte{0x0f, 0xae, 0xf8}}, + {"PAUSE", []byte{0xf3, 0x90}}, + } + for _, mode := range []int{32, 64} { + arch := "386" + if mode == 64 { + arch = "amd64" + } + for _, row := range rows { + for _, prefix := range [][]byte{{0xf0}, {0xf0, 0x66}} { + t.Run(fmt.Sprintf("%d/%s/%x", mode, row.op, prefix), func(t *testing.T) { + code := append(append([]byte(nil), prefix...), row.machine...) + original := "TEXT invalid(SB),4,$0-0\n" + for _, value := range code { + original += fmt.Sprintf("BYTE $%d\n", value) + } + goAssemblerOracleOutput(t, arch, original+"RET\n", "", true) + physical, err := x86asm.Decode(code, mode) + if err != nil { + t.Fatal("actual pinned decoder observation", err) + } + invalid := false + for _, value := range physical.Prefix { + invalid = invalid || value&x86asm.PrefixInvalid != 0 + } + if !invalid { + t.Fatalf("decoder did not identify specified LOCK invalidity: %#v", physical) + } + decoded, conversion := decodeX86RawDirectiveGroup(code, mode, 0, original, nil) + t.Logf("actual bytes=%x prefix=%#v GoSyntax=%q whole=%#v error=%v", + code, physical.Prefix, x86asm.GoSyntax(physical, 0, nil), decoded, conversion) + if conversion == nil { + t.Fatal("whole raw decoder erased a machine-invalid prefix") + } + }) + } + } + } +} + +func x86PrefixOriginalBytes(t *testing.T, mode int, code []byte) { + t.Helper() + arch := "386" + if mode == 64 { + arch = "amd64" + } + source := "TEXT prefix(SB),4,$0-0\n" + for _, value := range code { + source += fmt.Sprintf("BYTE $%d\n", value) + } + actual := x86OrderingHintGoBytes(t, arch, source+"RET\n") + want := append(append([]byte(nil), code...), 0xc3) + if !bytes.Equal(actual, want) { + t.Fatalf("original BYTE object identity %x, want %x", actual, want) + } +} + +// The rule is decoder metadata, not an opcode whitelist. Include width, +// arithmetic, control, x87 and mnemonic-alias paths through GoSyntax. +func TestX86RawInvalidPrefixEntireDecodedClass(t *testing.T) { + rows := []struct { + name string + code []byte + }{ + {"MOV", []byte{0x8b, 0xc1}}, + {"ADD register", []byte{0x01, 0xc8}}, + {"CMP memory", []byte{0x39, 0x08}}, + {"TEST memory", []byte{0x85, 0x08}}, + {"MUL", []byte{0xf7, 0xe1}}, + {"DIV", []byte{0xf7, 0xf1}}, + {"LEA", []byte{0x8d, 0x00}}, + {"NOP", []byte{0x90}}, + {"INT3", []byte{0xcc}}, + {"UD2 fault", []byte{0x0f, 0x0b}}, + {"RET", []byte{0xc3}}, + {"PACK alias", []byte{0x66, 0x0f, 0x6b, 0xca}}, + {"FINIT alias", []byte{0xdb, 0xe3}}, + } + for _, mode := range []int{32, 64} { + for _, row := range rows { + for _, prefix := range [][]byte{{0xf0}, {0xf0, 0x66}, {0x66, 0xf0}, {0xf0, 0xf0}} { + t.Run(fmt.Sprintf("%d/%s/%x", mode, row.name, prefix), func(t *testing.T) { + code := append(append([]byte(nil), prefix...), row.code...) + x86PrefixOriginalBytes(t, mode, code) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("complete original decode %+v: %v", physical, err) + } + invalid := false + for _, value := range physical.Prefix { + invalid = invalid || value&x86asm.PrefixInvalid != 0 + } + if !invalid { + t.Fatal("actual original bytes lack the expected invalid LOCK metadata") + } + if syntax, err := decodedX86GoSyntax(physical, code); err == nil { + t.Fatalf("invalid metadata became legal syntax %q", syntax) + } + if got, err := decodeX86RawDirectiveGroup(code, mode, 0, row.name, nil); err == nil { + t.Fatalf("invalid metadata escaped whole decoding: %+v", got) + } + }) + } + } + } +} + +func TestX86RawIgnoredPrefixesRemainDistinctFromInvalid(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, row := range []struct { + name string + code []byte + want Op + }{ + {"duplicate operand override", []byte{0x66, 0x66, 0x89, 0xc8}, "MOVW"}, + {"duplicate address override", []byte{0x67, 0x67, 0x0f, 0x31}, "RDTSC"}, + {"ignored operand override", []byte{0x66, 0x66, 0x0f, 0x31}, "RDTSC"}, + } { + t.Run(fmt.Sprintf("%d/%s", mode, row.name), func(t *testing.T) { + x86PrefixOriginalBytes(t, mode, row.code) + physical, err := x86asm.Decode(row.code, mode) + if err != nil || physical.Len != len(row.code) { + t.Fatalf("complete original decode %+v: %v", physical, err) + } + ignored := false + for _, prefix := range physical.Prefix { + if prefix&x86asm.PrefixInvalid != 0 { + t.Fatal("legal control has invalid metadata") + } + ignored = ignored || prefix&x86asm.PrefixIgnored != 0 + } + if !ignored { + t.Fatal("control does not observe PrefixIgnored") + } + got, err := decodeX86RawDirectiveGroup(row.code, mode, 0, row.name, nil) + if err != nil || len(got) != 1 || got[0].Op != row.want { + t.Fatalf("legal ignored prefix changed: %+v, %v", got, err) + } + }) + } + } +} + +func TestX86RawValidLockRequiresItsWholeAtomicEffect(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, row := range []struct { + name string + code []byte + }{ + {"ADD", []byte{0x01, 0x08}}, + {"AND", []byte{0x21, 0x08}}, + {"XADD", []byte{0x0f, 0xc1, 0x08}}, + {"XCHG", []byte{0x87, 0x08}}, + } { + for _, prefix := range [][]byte{{0xf0}, {0xf0, 0xf0}} { + t.Run(fmt.Sprintf("%d/%s/%x", mode, row.name, prefix), func(t *testing.T) { + code := append(append([]byte(nil), prefix...), row.code...) + x86PrefixOriginalBytes(t, mode, code) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("complete original decode %+v: %v", physical, err) + } + for _, value := range physical.Prefix { + if value&x86asm.PrefixInvalid != 0 { + t.Fatal("actual legitimate LOCK classified as machine invalid") + } + } + got, err := decodeX86RawDirectiveGroup(code, mode, 0, row.name, nil) + t.Logf("bytes=%x syntax=%q parsed=%+v error=%v", code, + x86asm.GoSyntax(physical, 0, nil), got, err) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("raw atomic-prefix effect is not bound; may not become a standalone LOCK no-op: %+v/%v", got, err) + } + }) + } + } + } +} diff --git a/amd64_raw_klogical.go b/amd64_raw_klogical.go new file mode 100644 index 000000000..c22cef5ce --- /dev/null +++ b/amd64_raw_klogical.go @@ -0,0 +1,110 @@ +package plan9asm + +import "fmt" + +// decodedX86RawKLogicalInstruction models Go 1.27's _ykaddb and _yknotb +// mask-register tables as opcode, width prefix, and K-operand arity. It +// includes KADD, KAND/ANDN, KOR, KXNOR/XOR, KUNPCK, KNOT, KTEST and KORTEST. +func decodedX86RawKLogicalInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.evex || p.mapNumber != 1 { + return Instr{}, 0, false, nil + } + stem, arity := x86RawKLogicalOpcode(p.opcode) + if stem == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("K-register operation: %s", message) + } + if p.pp > 1 { + return fail("reserved VEX.pp") + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride || p.segment != "" || p.r != 0 || p.b != 0 || p.x != 0 { + return fail("invalid address or register extension prefix") + } + length := 0 + if arity == 3 { + length = 1 + } + if p.vectorLength != length || p.upper >= 8 || arity == 2 && p.upper != 0 { + return fail("invalid VEX length or vvvv K register") + } + op, ok := x86RawKLogicalOperation(stem, p.pp, p.w) + if !ok { + return fail("reserved width or opcode combination") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + if modRM>>6 != 3 { + return fail("K-register operation requires register operands") + } + source1 := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM&7))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM>>3&7))} + args := []Operand{source1} + if arity == 3 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.upper))}) + } + args = append(args, destination) + raw := fmt.Sprintf("%s %s, %s", op, source1.String(), destination.String()) + if arity == 3 { + raw = fmt.Sprintf("%s %s, %s, %s", op, source1.String(), args[1].String(), destination.String()) + } + return Instr{Op: op, Args: args, Raw: raw}, p.modRM + 1, true, nil +} + +func x86RawKLogicalOpcode(opcode int) (stem string, arity int) { + switch opcode { + case 0x4a: + return "KADD", 3 + case 0x41: + return "KAND", 3 + case 0x42: + return "KANDN", 3 + case 0x45: + return "KOR", 3 + case 0x46: + return "KXNOR", 3 + case 0x47: + return "KXOR", 3 + case 0x4b: + return "KUNPCK", 3 + case 0x44: + return "KNOT", 2 + case 0x98: + return "KORTEST", 2 + case 0x99: + return "KTEST", 2 + } + return "", 0 +} + +func x86RawKLogicalOperation(stem string, pp int, w bool) (Op, bool) { + if stem == "KUNPCK" { + switch { + case pp == 1 && !w: + return "KUNPCKBW", true + case pp == 0 && !w: + return "KUNPCKWD", true + case pp == 0 && w: + return "KUNPCKDQ", true + } + return "", false + } + suffix := "W" + if pp == 1 { + suffix = "B" + } + if w { + suffix = "Q" + if pp == 1 { + suffix = "D" + } + } + return Op(stem + suffix), true +} diff --git a/amd64_raw_klogical_test.go b/amd64_raw_klogical_test.go new file mode 100644 index 000000000..bb0897145 --- /dev/null +++ b/amd64_raw_klogical_test.go @@ -0,0 +1,174 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86RawKLogicalRow struct { + opcode byte + pp byte + w bool + name Op + ary int +} + +func x86RawKLogicalRows() []x86RawKLogicalRow { + var rows []x86RawKLogicalRow + for opcode, stem := range map[byte]string{ + 0x4a: "KADD", 0x41: "KAND", 0x42: "KANDN", 0x45: "KOR", + 0x46: "KXNOR", 0x47: "KXOR", 0x44: "KNOT", + 0x98: "KORTEST", 0x99: "KTEST", + } { + arity := 3 + if opcode == 0x44 || opcode == 0x98 || opcode == 0x99 { + arity = 2 + } + for _, width := range []struct { + pp byte + w bool + suffix string + }{ + {pp: 1, suffix: "B"}, {pp: 0, suffix: "W"}, + {pp: 1, w: true, suffix: "D"}, {pp: 0, w: true, suffix: "Q"}, + } { + rows = append(rows, x86RawKLogicalRow{opcode, width.pp, width.w, Op(stem + width.suffix), arity}) + } + } + rows = append(rows, + x86RawKLogicalRow{0x4b, 1, false, "KUNPCKBW", 3}, + x86RawKLogicalRow{0x4b, 0, false, "KUNPCKWD", 3}, + x86RawKLogicalRow{0x4b, 0, true, "KUNPCKDQ", 3}, + ) + return rows +} + +func encodeX86RawKLogical(row x86RawKLogicalRow, source1, source2, destination int) []byte { + length := byte(0) + if row.ary == 3 { + length = 4 + } + vvvv := 0 + if row.ary == 3 { + vvvv = source2 + } + p1 := byte((^vvvv&15)<<3) | length | row.pp + modRM := byte(0xc0 | destination<<3 | source1) + if row.w { + return []byte{0xc4, 0xe1, p1 | 0x80, row.opcode, modRM} + } + return []byte{0xc5, p1 | 0x80, row.opcode, modRM} +} + +func TestDecodedX86RawKLogicalCompleteGo127Family(t *testing.T) { + rows := x86RawKLogicalRows() + if len(rows) != 39 { + t.Fatalf("modeled %d Go 1.27 K logical/arithmetic/unpack forms, want 39", len(rows)) + } + for _, row := range rows { + if _, ok := amd64MaskLogicalSpecs[row.name]; !ok { + if _, ok := amd64MaskArithmeticSpecs[row.name]; !ok { + if _, ok := amd64MaskUnpackBits[row.name]; !ok { + t.Fatalf("%s has no lowering grammar", row.name) + } + } + } + for source1 := 0; source1 < 8; source1++ { + for source2 := 0; source2 < 8; source2++ { + if row.ary == 2 && source2 != 0 { + continue + } + for destination := 0; destination < 8; destination++ { + code := encodeX86RawKLogical(row, source1, source2, destination) + got, length, ok, err := decodedX86RawKLogicalInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != row.name || len(got.Args) != row.ary { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %s", code, got, length, ok, err, row.name) + } + want := fmt.Sprintf("%s K%d, K%d", row.name, source1, destination) + if row.ary == 3 { + want = fmt.Sprintf("%s K%d, K%d, K%d", row.name, source1, source2, destination) + } + if got.Raw != want { + t.Fatalf("decode %x = %q, want %q", code, got.Raw, want) + } + if _, _, ok, err := decodedX86RawKLogicalInstruction(code, 32); !ok || err != nil { + t.Fatalf("386 decode %x: ok=%v err=%v", code, ok, err) + } + } + } + } + } +} + +func TestTranslateRawKLogicalArgReduceAVX512Regression(t *testing.T) { + const source = "TEXT rawKORW(SB),$0-0\n\tLONG $0xc945fcc5\n\tRET\n" + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", triple) + if !strings.Contains(ir, "or i64") { + t.Fatalf("KORW did not lower to mask OR:\n%s", ir) + } + compileLLVMToObject(t, llc, triple, "raw-korw.ll", "raw-korw.o", ir) + }) + } +} + +func TestTranslateRawKLogicalCompleteGo127FamilyObjects(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawKLogicalFamily(SB),$0-0\n") + for _, row := range x86RawKLogicalRows() { + for _, value := range encodeX86RawKLogical(row, 1, 2, 3) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + // Keep every original byte and its Go/public-Context/emitter + // checks; initialized raw logical programs have separate owners. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-klogical-family.ll", "raw-klogical-family.o", ir) + }) + } +} + +func TestDecodedX86RawKLogicalRejectsReservedForms(t *testing.T) { + row := x86RawKLogicalRow{0x45, 0, false, "KORW", 3} + base := encodeX86RawKLogical(row, 1, 2, 3) + for name, code := range map[string][]byte{ + "length": {base[0], base[1] &^ 4, base[2], base[3]}, + "prefix": {base[0], base[1] | 2, base[2], base[3]}, + "high vvvv": {base[0], base[1] &^ 0x40, base[2], base[3]}, + "memory source": {base[0], base[1], base[2], base[3] &^ 0xc0}, + "address override": append([]byte{0x67}, base...), + "missing ModRM": base[:3], + } { + t.Run(name, func(t *testing.T) { + if _, _, ok, err := decodedX86RawKLogicalInstruction(code, 64); !ok || err == nil { + t.Fatalf("accepted reserved K encoding %x: ok=%v err=%v", code, ok, err) + } + }) + } +} diff --git a/amd64_raw_kshift.go b/amd64_raw_kshift.go new file mode 100644 index 000000000..ffa78ae57 --- /dev/null +++ b/amd64_raw_kshift.go @@ -0,0 +1,59 @@ +package plan9asm + +import "fmt" + +// Go 1.27 _ykshiftlb uses opcode 30/31 for right and 32/33 for left; +// VEX.W distinguishes B/W or D/Q at each opcode. +var x86RawMaskShiftOps = [4][2]Op{ + {"KSHIFTRB", "KSHIFTRW"}, + {"KSHIFTRD", "KSHIFTRQ"}, + {"KSHIFTLB", "KSHIFTLW"}, + {"KSHIFTLD", "KSHIFTLQ"}, +} + +func decodedX86RawMaskShiftInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || p.evex || p.mapNumber != 3 || p.pp != 1 || p.opcode < 0x30 || p.opcode > 0x33 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("mask shift: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.vectorLength != 0 || p.upper != 0 || p.r != 0 || p.b != 0 || p.x != 0 { + return fail("invalid reserved VEX bit") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + if modRM>>6 != 3 { + return fail("KSHIFT requires a K-register source") + } + if len(code) <= p.modRM+1 { + return fail("missing imm8 shift count") + } + w := 0 + if p.w { + w = 1 + } + op := x86RawMaskShiftOps[p.opcode-0x30][w] + count := code[p.modRM+1] + source := modRM & 7 + destination := modRM >> 3 & 7 + args := []Operand{ + {Kind: OpImm, Imm: int64(count)}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", source))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", destination))}, + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s $%d, K%d, K%d", op, count, source, destination), + }, p.modRM + 2, true, nil +} diff --git a/amd64_raw_kshift_test.go b/amd64_raw_kshift_test.go new file mode 100644 index 000000000..4c843ea68 --- /dev/null +++ b/amd64_raw_kshift_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawMaskShift(opcode byte, w bool, source, destination int, count byte) []byte { + p1 := byte(0x79) + if w { + p1 |= 0x80 + } + return []byte{0xc4, 0xe3, p1, opcode, byte(0xc0 | destination<<3 | source), count} +} + +func TestTranslateRawMaskShiftGorseRegression(t *testing.T) { + const source = `TEXT rawMaskShift(SB), $0-0 + LONG $0x3179e3c4 + WORD $0x10d1 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawMaskShift": {Name: "rawMaskShift", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "lshr i64") { + t.Fatalf("raw KSHIFTRD omitted mask shift:\n%s", ir) + } +} + +func TestDecodedX86RawMaskShiftCompleteGoRows(t *testing.T) { + if len(x86RawMaskShiftOps) != 4 { + t.Fatal("Go 1.27 KSHIFT opcode family should have four direction/width rows") + } + seen := make(map[Op]bool) + for row, widths := range x86RawMaskShiftOps { + for width, op := range widths { + spec, ok := amd64MaskArithmeticSpecs[op] + if !ok || seen[op] || spec.shift == "" { + t.Fatalf("KSHIFT row %d/%d = %s, spec %+v", row, width, op, spec) + } + seen[op] = true + for source := 0; source < 8; source++ { + for destination := 0; destination < 8; destination++ { + for _, count := range []byte{0, 7, 8, 16, 32, 64, 255} { + code := encodeX86RawMaskShift(byte(0x30+row), width != 0, source, destination, count) + got, length, recognized, err := decodedX86RawMaskShiftInstruction(code, 64) + if err != nil || !recognized || length != len(code) || got.Op != op { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + if len(got.Args) != 3 || got.Args[0].Imm != int64(count) || + got.Args[1].String() != fmt.Sprintf("K%d", source) || + got.Args[2].String() != fmt.Sprintf("K%d", destination) { + t.Fatalf("decode %x = %+v, want $%d, K%d, K%d", code, got, count, source, destination) + } + if _, _, recognized, err := decodedX86RawMaskShiftInstruction(code, 32); !recognized || err != nil { + t.Fatalf("386 decode %x: recognized=%v err=%v", code, recognized, err) + } + } + } + } + } + } + if len(seen) != 8 { + t.Fatalf("covered %d KSHIFT opcodes, want eight", len(seen)) + } +} + +func TestDecodedX86RawMaskShiftRejectsInvalidForms(t *testing.T) { + base := encodeX86RawMaskShift(0x31, false, 1, 2, 16) + for _, test := range []struct { + name string + code []byte + }{ + {"VEX length", []byte{base[0], base[1], base[2] | 4, base[3], base[4], base[5]}}, + {"vvvv", []byte{base[0], base[1], base[2] &^ 8, base[3], base[4], base[5]}}, + {"extended register", []byte{base[0], base[1] &^ 0x20, base[2], base[3], base[4], base[5]}}, + {"memory source", []byte{base[0], base[1], base[2], base[3], base[4] &^ 0xc0, base[5]}}, + {"missing immediate", base[:5]}, + {"missing ModRM", base[:4]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawMaskShiftInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } +} + +func TestTranslateRawMaskShiftLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawMaskShiftFamily(SB), $0-0\n") + for row, widths := range x86RawMaskShiftOps { + for width := range widths { + for _, count := range []byte{0, 16, 255} { + code := encodeX86RawMaskShift(byte(0x30+row), width != 0, 1, 2, count) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawMaskShiftFamily": {Name: "rawMaskShiftFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-mask-shift.ll", "raw-mask-shift.o", ir) + }) + } +} diff --git a/amd64_raw_lea_address_test.go b/amd64_raw_lea_address_test.go new file mode 100644 index 000000000..1938f3006 --- /dev/null +++ b/amd64_raw_lea_address_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawDisplacementLEAAddressSizeIdentity(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, addressBits := range []int{mode, mode / 2} { + for _, width := range []int{16, 32, 64} { + if width > mode { + continue + } + t.Run(fmt.Sprintf("mode%d/address%d/width%d", mode, addressBits, width), func(t *testing.T) { + encoding := x86RawLEADisplacementEncoding(mode, width, addressBits, 0, 0, -64) + instruction := x86RawDisplacementDecodedLEA(t, mode, encoding) + want := 0 + if addressBits != mode { + want = addressBits + } + if instruction.x86AddressBits != want { + t.Fatalf("raw LEA lost physical address size: got %d, want %d", instruction.x86AddressBits, want) + } + }) + } + } + } +} + +func TestX86RawDisplacementLEANarrowAddressFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + mode := 64 + if target.arch == "386" { + mode = 32 + } + for _, width := range []int{16, 32, 64} { + if width > mode { + continue + } + t.Run(fmt.Sprintf("%s/width%d", target.triple, width), func(t *testing.T) { + encoding := x86RawLEADisplacementEncoding(mode, width, mode/2, 0, 0, -64) + body := "@MOV x+0(FP),AX\n@MOV AX,BX\n@MOV $3,SI\n@MOV $0,CX\n" + + x86RawDisplacementDirectives(encoding) + "@MOV CX,ret+@WORD(FP)\nRET" + source, sig, _ := x86SourceStackNumericalFixture(target.arch, body) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err != nil { + t.Fatal(err) + } + want := "zext i32" + if mode == 32 { + want = "zext i16" + } + if !strings.Contains(ir, want) { + t.Fatalf("narrow effective offset was not zero-extended before destination: %s", want) + } + compileLLVMToObject(t, llc, target.triple, "raw-lea-address.ll", "raw-lea-address.o", ir) + }) + } + } +} + +func TestX86RawDisplacementLEARejectsUnboundAddressMarkers(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, bits := range []int{1, 8, 16, 32, 64, 128} { + if target.arch == "386" && bits == 16 || target.arch == "amd64" && bits == 32 { + continue + } + t.Run(fmt.Sprintf("%s/%d", target.triple, bits), func(t *testing.T) { + instruction := Instr{Op: "LEAL", x86AddressBits: bits, Args: []Operand{ + {Kind: OpMem, Mem: MemRef{Base: AX}}, {Kind: OpReg, Reg: CX}, + }} + if _, err := x86LEAOperandWidth(instruction, target.arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("forged physical address size gained a grammar: %v", err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + ctx.goarch = target.arch + before := output.String() + if ok, terminated, err := ctx.lowerArith("LEAL", instruction); !ok || terminated || !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("invalid raw address marker emitted IR: (%v, %v, %v)", ok, terminated, err) + } + }) + } + } +} + +func TestX86RawDisplacementLEARelativeAndTruncatedRemainClosed(t *testing.T) { + for _, encoding := range [][]byte{ + {0x48, 0x8d, 0x0d, 0xc0, 0xff, 0xff, 0xff}, + {0x67, 0x48, 0x8d, 0x0d, 0xc0, 0xff, 0xff, 0xff}, + {0x48, 0x8d}, {0x48, 0x8d, 0x0c}, + {0x48, 0x8d, 0x0c, 0x25, 0xc0, 0xff, 0xff}, + } { + t.Run(fmt.Sprintf("%x", encoding), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+x86RawDisplacementDirectives(encoding)+"RET\n") + if err != nil { + t.Fatal(err) + } + if _, err := decodeX86RawDirectives(file.Funcs[0], "amd64"); err == nil { + t.Fatal("unbound relative target or truncated LEA became a numerical address") + } + }) + } +} + +func TestX86RawDisplacementLEAIgnoresSegmentBase(t *testing.T) { + for _, prefix := range []byte{0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65} { + t.Run(fmt.Sprintf("prefix%x", prefix), func(t *testing.T) { + encoding := append([]byte{prefix}, x86RawLEADisplacementEncoding(64, 64, 64, 0, 0, -64)...) + instruction := x86RawDisplacementDecodedLEA(t, 64, encoding) + memory := instruction.Args[0].Mem + if memory.Segment != "" || memory.Base != AX || memory.Off != -64 { + t.Fatalf("LEA used a segment base instead of its effective offset: %+v", memory) + } + }) + } +} + +func TestX86RawDisplacementEIPCannotBorrowRIPRelocation(t *testing.T) { + for _, opcode := range []byte{0x8d, 0x8b, 0x03} { + t.Run(fmt.Sprintf("opcode%x", opcode), func(t *testing.T) { + encoding := []byte{0x67, 0x48, opcode, 0x0d, 0xc0, 0xff, 0xff, 0xff} + inst, err := x86asm.Decode(encoding, 64) + if err != nil || inst.Len != len(encoding) || inst.AddrSize != 32 || inst.PCRel != 0 { + t.Fatalf("EIP decoder witness changed: %+v, %v", inst, err) + } + if syntax, err := decodedX86GoSyntax(inst, encoding); !errors.Is(err, ErrProbeNeedsContext) || syntax != "" { + t.Fatalf("unbound EIP-relative address acquired a source relocation: %q, %v", syntax, err) + } + }) + } +} + +func TestX86RawDisplacementNonLEASegmentOverridesStayPhysical(t *testing.T) { + for _, prefix := range []byte{0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65} { + t.Run(fmt.Sprintf("prefix%x", prefix), func(t *testing.T) { + encoding := []byte{prefix, 0x48, 0x8b, 0x80, 0xc0, 0xff, 0xff, 0xff} + inst, err := x86asm.Decode(encoding, 64) + if err != nil || inst.Len != len(encoding) || inst.Op != x86asm.MOV { + t.Fatalf("physical segmented MOV witness: %+v, %v", inst, err) + } + if got := normalizeDecodedX86LEASegments(inst); !reflect.DeepEqual(got, inst) { + t.Fatal("LEA normalization stripped an actual load's segment") + } + original, ok := inst.Args[1].(x86asm.Mem) + // In 64-bit mode x/arch already ignores ES/CS/SS/DS. FS and + // GS still carry a physical segment base and must not be erased. + if !ok || (prefix == 0x64 || prefix == 0x65) && original.Segment == 0 { + t.Fatalf("physical MOV lost its segment: %+v", inst) + } + normalized := normalizeDecodedX86MemoryDisplacements(inst) + memory := normalized.Args[1].(x86asm.Mem) + if memory.Segment != original.Segment || memory.Base != original.Base || memory.Disp != -64 { + t.Fatalf("signed displacement changed actual memory identity: %+v", memory) + } + }) + } +} + +func TestX86RawDisplacementNarrowLEADoesNotGrantCallerPointerOrStack(t *testing.T) { + for _, modRM := range []byte{0x87, 0x85} { // DI caller pointer, BP private stack. + t.Run(fmt.Sprintf("base%x", modRM), func(t *testing.T) { + encoding := []byte{0x67, 0x48, 0x8d, modRM, 0xc0, 0xff, 0xff, 0xff} + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", + x86RawDisplacementDirectives(encoding)+"MOVL AX,(DX)", 1) + requireLEANativeGoObject(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("narrow physical address borrowed native caller/stack identity: %v", err) + } + }) + } +} diff --git a/amd64_raw_lea_runtime_test.go b/amd64_raw_lea_runtime_test.go new file mode 100644 index 000000000..c94989d94 --- /dev/null +++ b/amd64_raw_lea_runtime_test.go @@ -0,0 +1,244 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +type x86RawLEAAddressRuntimeForm struct { + name string + addressBits, width, shape int + scale int + alias bool +} + +func x86RawLEAAddressRuntimeForms(arch string) []x86RawLEAAddressRuntimeForm { + mode := 64 + if arch == "386" { + mode = 32 + } + var forms []x86RawLEAAddressRuntimeForm + for _, addressBits := range []int{mode, mode / 2} { + for _, width := range []int{16, 32, 64} { + if width > mode { + continue + } + for _, shape := range []int{0, 1, 2, 3} { + if addressBits == 16 && shape != 0 { + continue // The physical fixture here is BX+SI, not a SIB. + } + for scale := 0; scale < 4; scale++ { + if (shape == 0 || shape == 3) && scale != 0 { + continue + } + for _, alias := range []bool{false, true} { + name := fmt.Sprintf("leaA%dW%dS%dC%d", addressBits, width, shape, scale) + if alias { + name += "Alias" + } + forms = append(forms, x86RawLEAAddressRuntimeForm{ + name: name, addressBits: addressBits, width: width, + shape: shape, scale: scale, alias: alias, + }) + } + } + } + } + } + return forms +} + +func x86RawLEAAddressRuntimeSources(arch string) (string, string, string, string, map[string]FuncSig) { + mode, word, move, typ, ctype, cmax := 64, 8, "MOVQ", "uint64", "uint64_t", "UINT64_MAX" + seed := uint64(0x1122334455667788) + if arch == "386" { + mode, word, move, typ, ctype, cmax = 32, 4, "MOVL", "uint32", "uint32_t", "UINT32_MAX" + seed = 0x55667788 + } + var declarations, assembly, goTest, c strings.Builder + declarations.WriteString("package rawlea\n") + fmt.Fprintf(&goTest, "package rawlea\nimport \"testing\"\nfunc TestPhysicalAddressMathematics(t *testing.T) {\n"+ + " values := []%s{0,1,63,64,65,65535,65536,1<<31-1,1<<31,1<<32-1,^%s(0)}\n", typ, typ) + if mode == 64 { + goTest.WriteString(" values = append(values,1<<32,1<<63,0x1122334455667788)\n") + } + c.WriteString("#include \n#if defined(__APPLE__)\n#define LABEL(name) \"_\" name\n#else\n#define LABEL(name) name\n#endif\n") + sigs := make(map[string]FuncSig) + for _, form := range x86RawLEAAddressRuntimeForms(arch) { + fmt.Fprintf(&declarations, "func %s(x %s) %s\n", form.name, typ, typ) + encoding := x86RawLEADisplacementEncoding(mode, form.width, form.addressBits, form.shape, form.scale, -64) + destination, initial := "CX", fmt.Sprintf("0x%x", seed) + if form.alias { + destination, initial = "AX", "x" + register := byte(0) + if form.addressBits == 16 { + destination, register = "BX", 3 + } + opcode := bytes.IndexByte(encoding, 0x8d) + encoding[opcode+1] = encoding[opcode+1]&^0x38 | register<<3 + } + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-%d\n%s x+0(FP),AX\n%s $3,BX\n%s $0x%x,CX\n", + form.name, 2*word, move, move, move, seed) + if form.addressBits == 16 { + fmt.Fprintf(&assembly, "%s AX,BX\n%s $3,SI\n", move, move) + } + assembly.WriteString(x86RawDisplacementDirectives(encoding)) + fmt.Fprintf(&assembly, "%s %s,ret+%d(FP)\nRET\n", move, destination, word) + sig := crossUnarySig(arch, "rawlea."+form.name) + sigs[sig.Name] = sig + + // These two independent scalar oracles model effective-address width + // first and destination width second, never decoded IR or GoSyntax. + effective := "x - 64" + if form.addressBits == 16 { + effective = "x + 3 - 64" + } else if form.shape == 1 { + effective = fmt.Sprintf("x + 3*%d - 64", 1<= 8 { + code = append(code, 0x44) + } + code = append(code, 0x0f, opcode, 0x05, 1, 0, 0, 0, 0xc3) + for i := 0; i < width; i++ { + code = append(code, byte(i*31+7)) + } + return code +} + +func TestDecodeX86RawLegacyScalarMoveConstantCompleteFamily(t *testing.T) { + for _, form := range []struct { + op Op + prefix byte + width int + }{ + {"MOVSS", 0xf3, 4}, + {"MOVSD", 0xf2, 8}, + } { + for _, register := range []int{0, 8} { + t.Run(fmt.Sprintf("%s/X%d", form.op, register), func(t *testing.T) { + code := x86RawLegacySIMDMoveConstant(form.prefix, 0x10, register, form.width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, string(form.op), map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != form.op || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[0].Args[1].String() != fmt.Sprintf("X%d", register) || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, form.op) + } + }) + } + } +} + +func TestTranslateX86RawLegacyScalarMoveConstantObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, form := range []struct { + prefix byte + width int + }{ + {0xf3, 4}, {0xf2, 8}, + } { + code := x86RawLegacySIMDMoveConstant(form.prefix, 0x10, 8, form.width) + name := fmt.Sprintf("legacyScalarMoveConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != 2 || len(normalized.Data[0].Payload) != 4 || len(normalized.Data[1].Payload) != 8 { + t.Fatalf("materialized wrong scalar constants: %#v", normalized.Data) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "legacy-scalar-move-literal.ll", "legacy-scalar-move-literal.o", ir) + }) + } +} + +func TestDecodeX86RawLegacyPackedMoveConstantCompleteFamily(t *testing.T) { + for _, form := range x86RawLegacyPackedMoveForms { + for _, register := range []int{0, 8} { + t.Run(fmt.Sprintf("%s/X%d", form.name, register), func(t *testing.T) { + code := x86RawLegacyPackedMoveConstantRegister(form.prefix, form.opcode, register) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, string(form.name), map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != form.name || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[0].Args[1].String() != fmt.Sprintf("X%d", register) || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, form.name) + } + }) + } + } +} + +func TestDecodeX86RawLegacyPackedMoveConstantRejectsUnsafeSources(t *testing.T) { + valid := x86RawLegacyPackedMoveConstant(0, 0x10) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[3] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[3] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawLegacyPackedMoveConstantObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, form := range x86RawLegacyPackedMoveForms { + code := x86RawLegacyPackedMoveConstantRegister(form.prefix, form.opcode, 8) + name := fmt.Sprintf("legacyPackedMoveConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + pools = append(pools, code[len(code)-16:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "legacy-packed-move-literal.ll", "legacy-packed-move-literal.o", ir) + }) + } +} diff --git a/amd64_raw_legacy_scalar_rip.go b/amd64_raw_legacy_scalar_rip.go new file mode 100644 index 000000000..b6ae2b74d --- /dev/null +++ b/amd64_raw_legacy_scalar_rip.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + "strings" + + "golang.org/x/arch/x86/x86asm" +) + +// These Go legacy SSE scalar families read a fixed-width RIP-relative source +// and update only a vector register or flags. A source-local constant can be +// materialized without losing an in-place memory write or relocation. +var x86RawLegacyScalarReadOps = map[Op]bool{ + "MOVL": true, "MOVQ": true, "MOVLQSX": true, + "UCOMISD": true, "UCOMISS": true, "COMISD": true, "COMISS": true, + "CMPSS": true, "CMPSD": true, + "ADDSS": true, "ADDSD": true, "SUBSS": true, "SUBSD": true, + "MULSS": true, "MULSD": true, "DIVSS": true, "DIVSD": true, + "MINSS": true, "MINSD": true, "MAXSS": true, "MAXSD": true, + "SQRTSS": true, "SQRTSD": true, +} + +func decodeX86RawLegacyScalarRIPData( + code []byte, offset int, inst x86asm.Inst, syntax string, +) (Instr, x86RawLiteralRange, bool, error) { + if inst.Mode != 64 || inst.AddrSize != 64 || inst.PCRel != 4 || + inst.MemBytes != 4 && inst.MemBytes != 8 { + return Instr{}, x86RawLiteralRange{}, false, nil + } + op := Op(strings.Fields(syntax)[0]) + if !x86RawLegacyScalarReadOps[op] { + return Instr{}, x86RawLiteralRange{}, false, nil + } + if _, ok := inst.Args[0].(x86asm.Reg); !ok { + return Instr{}, x86RawLiteralRange{}, false, nil + } + mem, ok := inst.Args[1].(x86asm.Mem) + if !ok || mem.Base != x86asm.RIP { + return Instr{}, x86RawLiteralRange{}, false, nil + } + if len(code) <= offset || code[offset] == 0x64 || code[offset] == 0x65 || code[offset] == 0x67 { + return Instr{}, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + inst.PCRelOff - 1 + if modRMIndex < offset || modRMIndex >= len(code) || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, x86RawLiteralRange{}, false, nil + } + trailing := inst.Len - inst.PCRelOff - inst.PCRel + if trailing != 0 && (op != "CMPSS" && op != "CMPSD" || trailing != 1) { + return Instr{}, x86RawLiteralRange{}, false, nil + } + patched, data, literal, err := x86RawRIPBytesWithSuffix(code, offset, modRMIndex, inst.MemBytes, trailing) + if err != nil { + return Instr{}, x86RawLiteralRange{}, true, err + } + physical, err := x86asm.Decode(patched, 64) + if err != nil || physical.Len != len(patched) { + return Instr{}, x86RawLiteralRange{}, true, fmt.Errorf("patched legacy scalar encoding is invalid: %v", err) + } + patchedSyntax, err := decodedX86GoSyntax(physical, patched) + if err != nil { + return Instr{}, x86RawLiteralRange{}, true, err + } + instrs, err := parseDecodedX86Instruction(patchedSyntax) + if err != nil || len(instrs) != 1 || len(instrs[0].Args) < 2 || instrs[0].Args[0].Kind != OpMem { + return Instr{}, x86RawLiteralRange{}, true, fmt.Errorf("patched legacy scalar instruction %q has no memory source: %v", patchedSyntax, err) + } + setX86RawRIPDataOperand(&instrs[0], 0, data) + return instrs[0], literal, true, nil +} diff --git a/amd64_raw_legacy_scalar_rip_test.go b/amd64_raw_legacy_scalar_rip_test.go new file mode 100644 index 000000000..a24daa552 --- /dev/null +++ b/amd64_raw_legacy_scalar_rip_test.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawLegacyScalarSourceLocalRIPData(t *testing.T) { + for _, op := range []string{ + "MOVD", "UCOMISD", "UCOMISS", "COMISD", "COMISS", + "CMPSS", "CMPSD", + "ADDSS", "ADDSD", "SUBSS", "SUBSD", "MULSS", "MULSD", + "DIVSS", "DIVSD", "MINSS", "MINSD", "MAXSS", "MAXSD", + "SQRTSS", "SQRTSD", + } { + for _, destination := range []string{"X0", "X10"} { + t.Run(op+"/"+destination, func(t *testing.T) { + instruction := op + " pool(SB), " + destination + if op == "CMPSS" || op == "CMPSD" { + instruction += ", $1" + } + code := assembleX87ControlBytes(t, "amd64", + "TEXT scalarRIP(SB),4,$0-0\n\t"+instruction+"\n\tRET\n") + inst, err := x86asm.Decode(code, 64) + if err != nil || inst.PCRel != 4 || inst.MemBytes <= 0 || inst.Len >= len(code) { + t.Fatalf("Go assembler %s %s: %x, decode %+v, %v", op, destination, code, inst, err) + } + binary.LittleEndian.PutUint32(code[inst.PCRelOff:], 1) + for value := 0; value < inst.MemBytes; value++ { + code = append(code, byte(value)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "scalar RIP", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || len(decoded[0].x86RIPLiteralData) != inst.MemBytes { + t.Fatalf("%s %s decoded as %#v", op, destination, decoded) + } + }) + } + } +} + +func TestX86RawMOVD32PreservesPhysicalWidth(t *testing.T) { + const source = "TEXT movd32(SB),4,$0-0\n\tMOVL pool(SB), X0\n\tRET\n" + code := assembleX87ControlBytes(t, "amd64", source) + physical, err := x86asm.Decode(code, 64) + if err != nil || physical.Op != x86asm.MOVD || physical.MemBytes != 4 { + t.Fatalf("Go MOVL to X0 encoding: %x, %+v, %v", code, physical, err) + } + binary.LittleEndian.PutUint32(code[physical.PCRelOff:], 1) + code = append(code, 1, 2, 3, 4) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "MOVD32 literal", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "MOVL" || + len(decoded[0].x86RIPLiteralData) != 4 { + t.Fatalf("MOVD32 decoded as %#v", decoded) + } + var raw strings.Builder + raw.WriteString("TEXT ·rawMovd32(SB),4,$0-0\n") + for _, value := range code { + fmt.Fprintf(&raw, "\tBYTE $0x%02x\n", value) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, raw.String(), "func rawMovd32()\n", "amd64", triple, api) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "load volatile i32, ptr") || + strings.Contains(ir, "load volatile i64, ptr") { + t.Fatal("physical MOVD32 must retain exactly one fault-observable 32-bit load") + } + compileLLVMToObject(t, llc, triple, "raw-movd32-"+api+".ll", "raw-movd32-"+api+".o", ir) + }) + } + }) + } +} diff --git a/amd64_raw_local_address_test.go b/amd64_raw_local_address_test.go new file mode 100644 index 000000000..054b61ff8 --- /dev/null +++ b/amd64_raw_local_address_test.go @@ -0,0 +1,150 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawLocalAddressPreservesPointerOffsets(t *testing.T) { + // Two RIP-relative LEAs address different bytes of one unreachable table. + code := []byte{ + 0x48, 0x8d, 0x05, 0x08, 0, 0, 0, // LEAQ table+0, AX + 0x48, 0x8d, 0x1d, 0x04, 0, 0, 0, // LEAQ table+3, BX + 0xc3, + 1, 2, 3, 4, 5, 6, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "local address table", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 3 || decoded[0].Op != "LEAQ" || decoded[1].Op != "LEAQ" || + decoded[0].Args[1].Reg != AX || decoded[1].Args[1].Reg != BX || + decoded[0].x86RIPAddressOff != 0 || decoded[1].x86RIPAddressOff != 3 { + t.Fatalf("decoded %x as %#v", code, decoded) + } + var source strings.Builder + source.WriteString("TEXT localAddress(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != 1 || len(normalized.Funcs[0].Instrs) != 4 { + t.Fatalf("shared local address layout: %d globals, %d instructions", len(normalized.Data), len(normalized.Funcs[0].Instrs)) + } + firstAddress := normalized.Funcs[0].Instrs[1].Args[0].Sym + secondAddress := normalized.Funcs[0].Instrs[2].Args[0].Sym + if firstAddress != normalized.Data[0].Sym+"(SB)" || + secondAddress != normalized.Data[0].Sym+"+3(SB)" { + t.Fatalf("shared address offsets = %q, %q", firstAddress, secondAddress) + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"localAddress": {Name: "localAddress", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "raw_address") { + t.Fatal("shared local-address data global missing") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "raw-local-address.ll", "raw-local-address.o", ir) + }) + } +} + +func TestDecodeX86RawLocalAddressSharesLiteralPool(t *testing.T) { + code := []byte{ + 0x48, 0x8d, 0x05, 0x09, 0, 0, 0, // LEAQ table, AX + 0x66, 0x0f, 0x6f, 0x05, 0x01, 0, 0, 0, // MOVDQU table, X0 + 0xc3, + } + for value := byte(0); value < 16; value++ { + code = append(code, value) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "shared literal pool", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 3 || len(decoded[0].x86RIPAddressData) != 16 || + len(decoded[1].x86RIPAddressData) != 16 || + decoded[0].x86RIPAddressOff != 0 || decoded[1].x86RIPAddressOff != 0 { + t.Fatalf("shared pool decoded as %#v", decoded) + } + var source strings.Builder + source.WriteString("TEXT localAddressAndLiteral(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != 1 || + len(normalized.Funcs[0].Instrs) != 4 || + normalized.Funcs[0].Instrs[1].Args[0].Sym != normalized.Data[0].Sym+"(SB)" || + normalized.Funcs[0].Instrs[2].Args[0].Sym != normalized.Data[0].Sym+"(SB)" { + t.Fatalf("local address and literal did not share data: %d globals, %d instructions", len(normalized.Data), len(normalized.Funcs[0].Instrs)) + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"localAddressAndLiteral": {Name: "localAddressAndLiteral", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "raw-shared-pool.ll", "raw-shared-pool.o", ir) + }) + } +} + +func TestDecodeX86RawLocalAddressRejectsReachableCode(t *testing.T) { + code := []byte{ + 0x48, 0x8d, 0x05, 0x00, 0, 0, 0, // LEAQ next instruction, AX + 0xc3, + } + _, err := decodeX86RawDirectiveGroup(code, 64, 0, "code address", map[string]bool{}) + if err == nil || !strings.Contains(err.Error(), "no preceding RET") { + t.Fatalf("address of reachable code: %v", err) + } +} + +func TestDecodeX86RawLocalAddressSignExtendsNegativeDisplacement(t *testing.T) { + // LEAQ -7(RIP), AX points to its own first byte, not byte 2^32. + code := []byte{0x48, 0x8d, 0x05, 0xf9, 0xff, 0xff, 0xff, 0xc3} + _, err := decodeX86RawDirectiveGroup(code, 64, 0, "self address", map[string]bool{}) + if err == nil || !strings.Contains(err.Error(), "no preceding RET") { + t.Fatalf("self-addressed native TEXT must fail with layout reason, got %v", err) + } +} diff --git a/amd64_raw_logical_literal_test.go b/amd64_raw_logical_literal_test.go new file mode 100644 index 000000000..e4578d5e6 --- /dev/null +++ b/amd64_raw_logical_literal_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawPackedLogicalLiteral(op Op, evex bool, width, mask int, zeroing, broadcast bool) []byte { + var code []byte + if evex { + length := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) + code = encodeX86RawEVEXPackedLogical(op, length, mask, zeroing, broadcast, 0, 0, 0) + } else { + opcode := map[Op]byte{"VPAND": 0xdb, "VPANDN": 0xdf, "VPOR": 0xeb, "VPXOR": 0xef}[op] + code = encodeX86VEXPackedIntegerLogical(opcode, true, width == 32, false, 0, 0, 0) + } + code[len(code)-1] = 0x05 // RIP+disp32 rather than X0. + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = 4 + if strings.HasSuffix(string(op), "Q") { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*9+7)) + } + return code +} + +func TestTranslateX86RawPackedLogicalRIPDataObjects(t *testing.T) { + var source strings.Builder + var pools [][]byte + add := func(name string, code []byte, width int) { + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + pools = append(pools, code[len(code)-width:]) + } + for index, op := range []Op{"VPAND", "VPANDN", "VPOR", "VPXOR"} { + add(fmt.Sprintf("vexLogical%d", index), x86RawPackedLogicalLiteral(op, false, 32, 0, false, false), 32) + } + for index, op := range []Op{ + "VPANDD", "VPANDQ", "VPANDND", "VPANDNQ", + "VPORD", "VPORQ", "VPXORD", "VPXORQ", + } { + width := 4 + if strings.HasSuffix(string(op), "Q") { + width = 8 + } + add(fmt.Sprintf("evexLogical%d", index), x86RawPackedLogicalLiteral(op, true, 64, 7, true, true), width) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Retain every original VEX and EVEX function. The EVEX functions + // lack K7; separate complete initialized programs own public objects. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "logical-literal.ll", "logical-literal.o", ir) + }) + } +} + +func TestDecodeX86RawPackedLogicalRIPDataRejectsUnsafeSources(t *testing.T) { + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "outside_pool", edit: func(code []byte) []byte { + code[6] = 0x7f + return code + }}, + {name: "overlapping_instruction", edit: func(code []byte) []byte { + copy(code[6:10], []byte{0xfa, 0xff, 0xff, 0xff}) + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x64}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.edit(x86RawPackedLogicalLiteral("VPXORD", true, 64, 0, false, false)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe packed logical RIP source %x", code) + } + }) + } +} + +func TestDecodeX86RawPackedLogicalRIPDataCompleteFamily(t *testing.T) { + for _, op := range []Op{"VPAND", "VPANDN", "VPOR", "VPXOR"} { + for _, width := range []int{16, 32} { + name := fmt.Sprintf("%s/%d", op, width) + t.Run(name, func(t *testing.T) { + code := x86RawPackedLogicalLiteral(op, false, width, 0, false, false) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != op || + len(decoded[0].Args) != 3 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s literal source", code, decoded, op) + } + }) + } + } + for _, op := range []Op{ + "VPANDD", "VPANDQ", "VPANDND", "VPANDNQ", + "VPORD", "VPORQ", "VPXORD", "VPXORQ", + } { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + name := fmt.Sprintf("%s/%d/bcst%t", op, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawPackedLogicalLiteral(op, true, width, 7, true, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := op + if broadcast { + want += ".BCST" + } + want += ".Z" + if len(decoded) != 2 || decoded[0].Op != want || + len(decoded[0].Args) != 4 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s literal source", code, decoded, want) + } + }) + } + } + } +} diff --git a/amd64_raw_mask_blend.go b/amd64_raw_mask_blend.go new file mode 100644 index 000000000..6268203b0 --- /dev/null +++ b/amd64_raw_mask_blend.go @@ -0,0 +1,41 @@ +package plan9asm + +import "fmt" + +// decodedX86RawMaskBlendInstruction covers all six Go 1.27 _yvblendmpd +// operations across EVEX X/Y/Z, optional K mask and zeroing, and permitted +// D/Q scalar broadcasts. The shared typed operand decoder handles ModRM/SIB. +func decodedX86RawMaskBlendInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 { + return Instr{}, 0, false, nil + } + var op Op + var laneBits int + switch p.opcode { + case 0x65: + if p.w { + op, laneBits = "VBLENDMPD", 64 + } else { + op, laneBits = "VBLENDMPS", 32 + } + case 0x64: + if p.w { + op, laneBits = "VPBLENDMQ", 64 + } else { + op, laneBits = "VPBLENDMD", 32 + } + case 0x66: + if p.w { + op, laneBits = "VPBLENDMW", 16 + } else { + op, laneBits = "VPBLENDMB", 8 + } + default: + return Instr{}, 0, false, nil + } + if mode == 32 && p.mask != 0 { + return Instr{}, 0, true, fmt.Errorf("386 masked %s exceeds Go's three-operand frontend", op) + } + return decodedX86BinaryVectorOperands(code, p, mode, op, laneBits) +} diff --git a/amd64_raw_mask_blend_test.go b/amd64_raw_mask_blend_test.go new file mode 100644 index 000000000..2474e6626 --- /dev/null +++ b/amd64_raw_mask_blend_test.go @@ -0,0 +1,203 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawMaskBlendEncoding(op Op, width, mask int, zeroing, broadcast, rip bool) []byte { + properties := map[Op]struct { + opcode byte + w bool + }{ + "VBLENDMPS": {0x65, false}, "VBLENDMPD": {0x65, true}, + "VPBLENDMD": {0x64, false}, "VPBLENDMQ": {0x64, true}, + "VPBLENDMB": {0x66, false}, "VPBLENDMW": {0x66, true}, + }[op] + p1 := byte(0x75) // source 2 is register 1, fixed bit and pp=66. + if properties.w { + p1 |= 0x80 + } + p2 := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // register 3, destination register 2. + if broadcast { + modRM = 0x13 // 0(BX), destination register 2. + } + if rip { + modRM = 0x15 // RIP+disp32, destination register 2. + } + code := []byte{0x62, 0xf2, p1, p2, properties.opcode, modRM} + if rip { + code = append(code, 1, 0, 0, 0) + } + return code +} + +func TestDecodeX86RawMaskBlend386FrontendBoundary(t *testing.T) { + plain := x86RawMaskBlendEncoding("VBLENDMPS", 16, 0, false, false, false) + if _, _, ok, err := decodedX86RawMaskBlendInstruction(plain, 32); !ok || err != nil { + t.Fatalf("386 unmasked EVEX form: ok=%v err=%v", ok, err) + } + masked := x86RawMaskBlendEncoding("VBLENDMPS", 16, 1, false, false, false) + if _, _, ok, err := decodedX86RawMaskBlendInstruction(masked, 32); !ok || err == nil { + t.Fatalf("386 masked EVEX form accepted: ok=%v err=%v", ok, err) + } +} + +func x86RawMaskBlendLiteral(op Op, width, mask int, zeroing, broadcast bool) []byte { + code := x86RawMaskBlendEncoding(op, width, mask, zeroing, broadcast, true) + code = append(code, 0xc3) + dataWidth := width + if broadcast { + dataWidth = amd64MaskBlendSpecs[op].laneBits / 8 + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*7+5)) + } + return code +} + +func TestDecodeX86RawMaskBlendCompleteGo127Family(t *testing.T) { + if len(amd64MaskBlendSpecs) != 6 { + t.Fatalf("modeled %d mask-blend operations, want six", len(amd64MaskBlendSpecs)) + } + for op, spec := range amd64MaskBlendSpecs { + for _, width := range []int{16, 32, 64} { + for _, masking := range []struct { + mask int + zero bool + }{ + {}, {mask: 5}, {mask: 7, zero: true}, + } { + for _, broadcast := range []bool{false, true} { + if broadcast && !spec.broadcast { + continue + } + code := x86RawMaskBlendEncoding(op, width, masking.mask, masking.zero, broadcast, false) + got, length, ok, err := decodedX86RawMaskBlendInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %s %x = %+v, length=%d, ok=%v, err=%v", op, code, got, length, ok, err) + } + want := op + if broadcast { + want += ".BCST" + } + if masking.zero { + want += ".Z" + } + if got.Op != want || len(got.Args) != 3+map[bool]int{false: 0, true: 1}[masking.mask != 0] || + got.Args[1].Reg != "Z1" && width == 64 { + t.Fatalf("decode %s %x = %+v, want %s", op, code, got, want) + } + } + } + } + } +} + +func TestDecodeX86RawMaskBlendArgReduceAVX512Regression(t *testing.T) { + code := []byte{0x62, 0xf2, 0x75, 0x4d, 0x65, 0xdb, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "argreduce AVX512 mask blend", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VBLENDMPS" || + decoded[0].Raw != "VBLENDMPS Z3, Z1, K5, Z3 /* decoded from argreduce AVX512 mask blend */" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawMaskBlendRIPDataCompleteFamily(t *testing.T) { + for op, spec := range amd64MaskBlendSpecs { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + if broadcast && !spec.broadcast { + continue + } + name := fmt.Sprintf("%s/%d/bcst%t", op, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawMaskBlendLiteral(op, width, 7, true, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + if len(decoded) != 2 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET || + !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-dataWidth:]) { + t.Fatalf("decoded mask-blend literal %x as %#v", code, decoded) + } + }) + } + } + } +} + +func TestDecodeX86RawMaskBlendRejectsUnsafeForms(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated literal": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { code[6] = 127; return code }, + "overlapping instruction": func(code []byte) []byte { code[6] = 0; return code }, + "segment override": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawMaskBlendLiteral("VBLENDMPS", 64, 7, true, true)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe mask-blend literal %x", code) + } + }) + } + for _, code := range [][]byte{ + x86RawMaskBlendEncoding("VPBLENDMB", 16, 3, false, true, false), + x86RawMaskBlendEncoding("VBLENDMPS", 16, 0, true, false, false), + } { + if _, _, ok, err := decodedX86RawMaskBlendInstruction(code, 64); !ok || err == nil { + t.Fatalf("accepted reserved mask blend %x: ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawMaskBlendRIPDataObjects(t *testing.T) { + var source strings.Builder + index := 0 + for op, spec := range amd64MaskBlendSpecs { + for _, broadcast := range []bool{false, true} { + if broadcast && !spec.broadcast { + continue + } + name := fmt.Sprintf("maskBlendLiteral%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawMaskBlendLiteral(op, 64, 7, true, broadcast) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-mask-blend-literal.ll", "raw-mask-blend-literal.o", ir) + }) + } +} diff --git a/amd64_raw_mask_broadcast_test.go b/amd64_raw_mask_broadcast_test.go new file mode 100644 index 000000000..6b60ba647 --- /dev/null +++ b/amd64_raw_mask_broadcast_test.go @@ -0,0 +1,246 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" +) + +// This physical directory is independent of the decoder and is also checked +// against actual cmd/asm output below. K is a data operand, not a writemask. +func x86MaskBroadcastPhysicalBytes(op string, width, source, destination int) []byte { + p0 := byte(0xf2 ^ ((destination >> 3 & 1) << 7) ^ ((destination >> 4 & 1) << 4)) + p1, opcode := byte(0x7e), byte(0x3a) + if op == "VPBROADCASTMB2Q" { + p1, opcode = 0xfe, 0x2a + } + return []byte{0x62, p0, p1, byte(width<<5 | 8), opcode, byte(0xc0 | destination&7<<3 | source)} +} + +func TestX86BroadcastRawMaskDataOriginalGoBytes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + var expected []byte + for _, op := range []string{"VPBROADCASTMB2Q", "VPBROADCASTMW2D"} { + for width, bank := range []string{"X", "Y", "Z"} { + for _, input := range []int{0, 7} { + destinations := []int{0, 7} + if arch == "amd64" { + destinations = append(destinations, 31) + } + for _, destination := range destinations { + fmt.Fprintf(&source, "%s K%d,%s%d\n", op, input, bank, destination) + expected = append(expected, x86MaskBroadcastPhysicalBytes(op, width, input, destination)...) + } + } + } + } + source.WriteString("RET\n") + expected = append(expected, 0xc3) + actual := assembleX87ControlBytes(t, arch, source.String()) + if !bytes.Equal(actual, expected) { + t.Fatalf("%s actual Go mask broadcast bytes %x, expected %x", arch, actual, expected) + } + for offset := 0; offset < len(actual)-1; offset += 6 { + mode := 64 + if arch == "386" { + mode = 32 + } + ins, length, known, err := decodedX86VEXPackedBroadcastInstruction(actual[offset:offset+6], mode) + if err != nil || !known || length != 6 { + t.Fatalf("%s actual Go bytes %x: known%t length%d err%v", arch, actual[offset:offset+6], known, length, err) + } + form, recognized, err := x86BroadcastProperties(ins.Op, ins, arch) + if err != nil || !recognized || form.kind != x86BroadcastMask { + t.Fatalf("%s physical mask grammar %+v: known%t err%v", arch, form, recognized, err) + } + // This private decoder returns a plain instruction, not a raw-source + // entry contract. Its numerical effect requires only its actual K + // data input; neither another initialized mask nor the decoder can + // manufacture that input or grant native-memory/entry permission. + for _, consumer := range x86ResultGPSIMDConsumers { + input, _ := amd64ParseKReg(ins.Args[0].Reg) + destination, _ := amd64VectorRegisterIndex(ins.Args[1].Reg, form.byteWidth) + for _, initialized := range []bool{false, true} { + state := x86ResultState{flags: x86ByteFlagsAll} + state.masks[(input+1)%len(state.masks)] = true + state.masks[input] = initialized + before := state.clone() + if accepted := consumer.step(ins, &state, nil, 0, arch); accepted != initialized { + t.Fatalf("%s %s K%d initialized%t: accepted%t", arch, consumer.name, input, initialized, accepted) + } + expected := before.clone() + if initialized { + expected.writeVector(destination, form.byteWidth, ^uint64(0), false) + } + if !reflect.DeepEqual(state, expected) { + t.Fatalf("%s %s changed unrelated state or invented missing input: %+v", arch, consumer.name, state) + } + } + } + } + } +} + +func TestX86BroadcastMaskDataUnboundPublicSourceNeedsContext(t *testing.T) { + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + for _, row := range []string{"VPBROADCASTMB2Q K0,X2", "VPBROADCASTMW2D K7,Z2"} { + for _, raw := range []bool{false, true} { + t.Run(target.triple+"/"+row+fmt.Sprint("/raw", raw), func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + emitX86GoEncodedTestInstructionForArch(t, &source, row, raw, target.arch) + source.WriteString("RET\n") + // Raw normalization and a Void result cannot supply the + // missing K data. Preserve the original Go object as well + // as Context with empty IR through all three public APIs. + x86UnboundVoidFormatSource(t, source.String(), "f", target.arch, target.triple) + }) + } + } + } +} + +func TestX86BroadcastRawMaskDataCompletePhysicalBanks(t *testing.T) { + count := 0 + for _, op := range []string{"VPBROADCASTMB2Q", "VPBROADCASTMW2D"} { + for width, bank := range []string{"X", "Y", "Z"} { + for input := 0; input < 8; input++ { + for destination := 0; destination < 32; destination++ { + code := x86MaskBroadcastPhysicalBytes(op, width, input, destination) + ins, length, known, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + want := fmt.Sprintf("%s K%d, %s%d", op, input, bank, destination) + if err != nil || !known || length != 6 || ins.Raw != want { + t.Fatalf("decode %x: %+v length%d known%t err%v; want %q", code, ins, length, known, err, want) + } + _, _, known32, err32 := decodedX86VEXPackedBroadcastInstruction(code, 32) + if !known32 || (err32 != nil) != (destination >= 8) { + t.Fatalf("mode32 destination%d: known%t err%v", destination, known32, err32) + } + count++ + } + } + } + } + if count != 1536 { + t.Fatalf("physical directory covered %d encodings", count) + } +} + +func TestX86BroadcastRawMaskDataReservedAndUnprovedForms(t *testing.T) { + for _, op := range []string{"VPBROADCASTMB2Q", "VPBROADCASTMW2D"} { + valid := x86MaskBroadcastPhysicalBytes(op, 0, 0, 0) + for name, mutate := range map[string]func([]byte){ + "wrong W": func(code []byte) { code[2] ^= 0x80 }, + "missing fixed bit": func(code []byte) { code[2] &^= 4 }, + "used vvvv": func(code []byte) { code[2] &^= 8 }, + "used high vvvv": func(code []byte) { code[3] &^= 8 }, + "source extension B": func(code []byte) { code[1] &^= 0x20 }, + "source extension X": func(code []byte) { code[1] &^= 0x40 }, + "writemask": func(code []byte) { code[3] |= 1 }, + "zeroing": func(code []byte) { code[3] |= 0x80 }, + "broadcast bit": func(code []byte) { code[3] |= 0x10 }, + "reserved length": func(code []byte) { code[3] |= 0x60 }, + "memory input": func(code []byte) { code[5] &^= 0xc0 }, + } { + code := append([]byte(nil), valid...) + mutate(code) + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(code, 64); !known || err == nil { + t.Fatalf("%s %s %x: known%t err%v", op, name, code, known, err) + } + } + for _, prefix := range []byte{0x64, 0x65, 0x67} { + code := append([]byte{prefix}, valid...) + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(code, 64); !known || err == nil { + t.Fatalf("%s forbidden prefix%x: known%t err%v", op, prefix, known, err) + } + } + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(valid[:5], 64); !known || err == nil { + t.Fatalf("%s missing ModRM: known%t err%v", op, known, err) + } + for length := 0; length < 5; length++ { + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(valid[:length], 64); known || err != nil { + t.Fatalf("%s incomplete prefix length%d: known%t err%v", op, length, known, err) + } + } + for name, mutate := range map[string]func([]byte){ + "other map": func(code []byte) { code[1] ^= 1 }, + "other prefix": func(code []byte) { code[2] ^= 1 }, + "other opcode": func(code []byte) { code[4] ^= 1 }, + } { + code := append([]byte(nil), valid...) + mutate(code) + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(code, 64); known || err != nil { + t.Fatalf("%s %s outside grammar: known%t err%v", op, name, known, err) + } + } + } +} + +func TestX86BroadcastRawMaskDataMandatoryPrefixCollision(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + mode := 64 + if arch == "386" { + mode = 32 + } + for _, row := range []string{"VPBROADCASTMB2Q K0,X0", "VPBROADCASTMW2D K7,Z3", "VPMINUW Z1,Z2,Z3"} { + code := assembleX87ControlBytes(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if len(code) != 7 || code[0] != 0x62 || code[1]&15 != 2 { + t.Fatalf("%s original Go opcode-map witness %q: %x", arch, row, code) + } + want := Op(strings.Fields(row)[0]) + prefix := byte(2) + if want == "VPMINUW" { + prefix = 1 + } + if code[2]&3 != prefix || want != "VPBROADCASTMB2Q" && code[4] != 0x3a { + t.Fatalf("%s original mandatory-prefix/opcode witness %q: %x", arch, row, code) + } + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + for _, value := range code[:6] { + fmt.Fprintf(&source, "BYTE $%#x\n", value) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatalf("%s actual Go %q directive route: %v", arch, row, err) + } + ins := normalized.Instrs[1] + if ins.Op != want || ins.x86SourceRaw == nil || ins.x86SourceRaw.op != want { + t.Fatalf("%s %q lost original decoded source identity: %+v", arch, row, ins) + } + if want == "VPMINUW" { + if len(ins.Args) != 3 || ins.Args[0].String() != "Z1" || ins.Args[1].String() != "Z2" || ins.Args[2].String() != "Z3" { + t.Fatalf("legal min/max sibling changed: %+v", ins) + } + continue + } + for name, mutate := range map[string]func([]byte){ + "reserved mandatory prefix": func(code []byte) { code[2] ^= 1 }, + "reserved fixed bit": func(code []byte) { code[2] &^= 4 }, + "reserved W": func(code []byte) { code[2] ^= 0x80 }, + "unproved memory": func(code []byte) { code[5] &^= 0xc0 }, + } { + invalid := append([]byte(nil), code[:6]...) + mutate(invalid) + if _, err := decodeX86RawDirectiveGroup(invalid, mode, 0, name, map[string]bool{}); err == nil { + t.Fatalf("%s %q %s accepted through another decoder: %x", arch, row, name, invalid) + } + } + } + } +} diff --git a/amd64_raw_mask_compare.go b/amd64_raw_mask_compare.go new file mode 100644 index 000000000..b92e4ba91 --- /dev/null +++ b/amd64_raw_mask_compare.go @@ -0,0 +1,91 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawMaskCompareForm struct { + op Op + laneBytes int +} + +// The eight signed and unsigned Go 1.27 _yvpcmpb opcode rows are selected +// by the opcode and EVEX.W, not by a separate decoder for each mnemonic. +var x86RawMaskCompareForms = map[[2]int]x86RawMaskCompareForm{ + {0x3f, 0}: {op: "VPCMPB", laneBytes: 1}, + {0x3f, 1}: {op: "VPCMPW", laneBytes: 2}, + {0x1f, 0}: {op: "VPCMPD", laneBytes: 4}, + {0x1f, 1}: {op: "VPCMPQ", laneBytes: 8}, + {0x3e, 0}: {op: "VPCMPUB", laneBytes: 1}, + {0x3e, 1}: {op: "VPCMPUW", laneBytes: 2}, + {0x1e, 0}: {op: "VPCMPUD", laneBytes: 4}, + {0x1e, 1}: {op: "VPCMPUQ", laneBytes: 8}, +} + +func decodedX86RawMaskCompareInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 { + return Instr{}, 0, false, nil + } + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawMaskCompareForms[[2]int{p.opcode, w}] + if !recognized { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX mask compare: %s", message) + } + if mode != 64 && mode != 32 { + return fail("unknown physical x86 mode") + } + if !p.fixed || p.addressOverride || p.vectorLength > 2 || p.zero || p.r != 0 { + return fail("invalid EVEX fixed, address, vector length, zeroing, or K destination extension") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + if mode == 32 && (p.upper >= 8 || modRM>>6 == 3 && int(modRM&7)+8*p.b+16*p.x >= 8) { + return fail("extended source vector has no physical mode32 binding") + } + if p.broadcast && (form.laneBytes < 4 || modRM>>6 == 3) { + return fail("broadcast requires a D/Q scalar-memory first source") + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + disp8Scale := 16 << p.vectorLength + if p.broadcast { + disp8Scale = form.laneBytes + } + first, consumed, err := decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, vector, disp8Scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + immIndex := p.modRM + consumed + if len(code) <= immIndex { + return fail("missing imm8") + } + op := form.op + if p.broadcast { + op += ".BCST" + } + args := []Operand{ + {Kind: OpImm, Imm: int64(code[immIndex])}, + first, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, p.upper))}, + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM>>3&7))}) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, immIndex + 1, true, nil +} diff --git a/amd64_raw_mask_compare_test.go b/amd64_raw_mask_compare_test.go new file mode 100644 index 000000000..7bd3f9614 --- /dev/null +++ b/amd64_raw_mask_compare_test.go @@ -0,0 +1,194 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func encodeX86RawMaskCompare(opcode, widthBit, vectorBits, writeMask int, broadcast, memory bool) []byte { + p1 := byte(0x5d) // EVEX.vvvv = vector register 4, pp = 66. + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | writeMask) + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // first register 3, K destination 2. + if memory { + modRM = 0x50 // disp8(AX), K destination 2. + } + code := []byte{0x62, 0xf3, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return append(code, 6) +} + +func TestDecodeX86RawMaskCompareSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VPCMPUB $6, 0(SI)(R8*1), Z0, K0. + code := []byte{0x62, 0xb3, 0x7d, 0x48, 0x3e, 0x04, 0x06, 0x06, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "maskBitsLess AVX512 VPCMPUB", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPCMPUB" || + decoded[0].Args[0].Imm != 6 || decoded[0].Args[2].Reg != "Z0" || + decoded[0].Args[3].Reg != "K0" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawMaskCompareCompleteGoForms(t *testing.T) { + if len(x86RawMaskCompareForms) != 8 { + t.Fatalf("modeled %d EVEX mask-compare rows, want eight", len(x86RawMaskCompareForms)) + } + for encoding, form := range x86RawMaskCompareForms { + for vectorBits := 0; vectorBits < 3; vectorBits++ { + vector := [...]string{"X", "Y", "Z"}[vectorBits] + for _, variant := range []struct { + name string + writeMask int + broadcast bool + memory bool + }{ + {name: "register"}, + {name: "masked register", writeMask: 3}, + {name: "memory", memory: true}, + {name: "masked memory", writeMask: 7, memory: true}, + {name: "broadcast memory", broadcast: true, memory: true}, + } { + if variant.broadcast && form.laneBytes < 4 { + continue + } + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawMaskCompare(encoding[0], encoding[1], vectorBits, variant.writeMask, variant.broadcast, variant.memory) + got, length, ok, err := decodedX86RawMaskCompareInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.broadcast { + wantOp += ".BCST" + } + if got.Op != wantOp || got.Args[0].Imm != 6 || got.Args[2].String() != vector+"4" || + got.Args[len(got.Args)-1].String() != "K2" || + !variant.memory && got.Args[1].String() != vector+"3" || + len(got.Args) != 4+map[bool]int{true: 1}[variant.writeMask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawMaskCompareRIPLiteral(t *testing.T) { + for encoding, form := range x86RawMaskCompareForms { + for _, broadcast := range []bool{false, true} { + if broadcast && form.laneBytes < 4 { + continue + } + code := encodeX86RawMaskCompare(encoding[0], encoding[1], 2, 1, broadcast, false) + code = code[:6] + code[5] = 0x15 // RIP-relative first source, K destination 2. + code = append(code, 1, 0, 0, 0, 6, 0xc3) + literalWidth := 64 + if broadcast { + literalWidth = form.laneBytes + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/broadcast%t", form.op, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + wantOp := form.op + if broadcast { + wantOp += ".BCST" + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != literalWidth || got[0].Op != wantOp { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestDecodeX86RawMaskCompareRejectsInvalidForms(t *testing.T) { + base := encodeX86RawMaskCompare(0x3e, 0, 2, 0, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x80 }, // no EVEX zeroing + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + func(code []byte) { code[3] |= 0x10 }, // byte broadcast unavailable + func(code []byte) { code[1] &^= 0x80 }, // K destination extension + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86RawMaskCompareInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + extended := append([]byte(nil), base...) + extended[2] &^= 0x40 // Physical source X12, not the valid low-register mode32 row. + if _, _, ok, err := decodedX86RawMaskCompareInstruction(extended, 32); !ok || err == nil { + t.Fatalf("accepted unbound mode32 source vector: ok=%v err=%v", ok, err) + } +} + +func TestTranslateX86RawMaskCompareLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawMaskCompare(SB),$0-0\n") + for encoding, form := range x86RawMaskCompareForms { + code := encodeX86RawMaskCompare(encoding[0], encoding[1], 0, 0, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + if form.laneBytes >= 4 { + code = encodeX86RawMaskCompare(encoding[0], encoding[1], 0, 3, true, true) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawMaskCompare": {Name: "rawMaskCompare", Ret: Void}}, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original raw compare formats invented native/mask inputs", err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal("original raw compare format bytes no longer decode", err) + } + ir = x86InstructionFormatObjectUnit(t, normalized, + map[string]FuncSig{"rawMaskCompare": {Name: "rawMaskCompare", Ret: Void}}, "amd64", triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerImmediatePackedCompare(ins.Op, ins) }) + compileLLVMToObject(t, llc, triple, "raw-mask-compare.ll", "raw-mask-compare.o", ir) + }) + } +} diff --git a/amd64_raw_mask_move_test.go b/amd64_raw_mask_move_test.go new file mode 100644 index 000000000..8e06463bb --- /dev/null +++ b/amd64_raw_mask_move_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86RawMaskMoveCompleteGo127Family(t *testing.T) { + // Go 1.27's shared _ykmovb table gives KMOVB/W/D/Q the same four + // encoding rows: K->memory, K->GP, K-or-memory->K, and GP->K. Cover + // every row at every width, plus the extended GP form reported by + // github.com/minio/sha256-simd. + tests := []struct { + name string + code []byte + want string + }{ + {name: "b k to k", code: []byte{0xc5, 0xf9, 0x90, 0xcd}, want: "KMOVB K5, K1"}, + {name: "b memory to k", code: []byte{0xc5, 0xf9, 0x90, 0x08}, want: "KMOVB 0(AX), K1"}, + {name: "b k to memory", code: []byte{0xc5, 0xf9, 0x91, 0x00}, want: "KMOVB K0, 0(AX)"}, + {name: "b k to gp", code: []byte{0xc5, 0xf9, 0x93, 0xc5}, want: "KMOVB K5, AX"}, + {name: "b gp to k", code: []byte{0xc5, 0xf9, 0x92, 0xe8}, want: "KMOVB AX, K5"}, + {name: "w k to k", code: []byte{0xc5, 0xf8, 0x90, 0xcd}, want: "KMOVW K5, K1"}, + {name: "w memory to k", code: []byte{0xc5, 0xf8, 0x90, 0x08}, want: "KMOVW 0(AX), K1"}, + {name: "w k to memory", code: []byte{0xc5, 0xf8, 0x91, 0x00}, want: "KMOVW K0, 0(AX)"}, + {name: "w k to gp", code: []byte{0xc5, 0xf8, 0x93, 0xc5}, want: "KMOVW K5, AX"}, + {name: "w gp to k", code: []byte{0xc5, 0xf8, 0x92, 0xe8}, want: "KMOVW AX, K5"}, + {name: "d k to k", code: []byte{0xc4, 0xe1, 0xf9, 0x90, 0xcd}, want: "KMOVD K5, K1"}, + {name: "d memory to k", code: []byte{0xc4, 0xe1, 0xf9, 0x90, 0x08}, want: "KMOVD 0(AX), K1"}, + {name: "d k to memory", code: []byte{0xc4, 0xe1, 0xf9, 0x91, 0x00}, want: "KMOVD K0, 0(AX)"}, + {name: "d k to gp", code: []byte{0xc5, 0xfb, 0x93, 0xc5}, want: "KMOVD K5, AX"}, + {name: "d gp to k", code: []byte{0xc5, 0xfb, 0x92, 0xe8}, want: "KMOVD AX, K5"}, + {name: "q k to k", code: []byte{0xc4, 0xe1, 0xf8, 0x90, 0xcd}, want: "KMOVQ K5, K1"}, + {name: "q memory to k", code: []byte{0xc4, 0xe1, 0xf8, 0x90, 0x08}, want: "KMOVQ 0(AX), K1"}, + {name: "q k to memory", code: []byte{0xc4, 0xe1, 0xf8, 0x91, 0x00}, want: "KMOVQ K0, 0(AX)"}, + {name: "q k to gp", code: []byte{0xc4, 0xe1, 0xfb, 0x93, 0xc5}, want: "KMOVQ K5, AX"}, + {name: "q gp to k", code: []byte{0xc4, 0xe1, 0xfb, 0x92, 0xe8}, want: "KMOVQ AX, K5"}, + {name: "q reported extended gp to k", code: []byte{0xc4, 0xc1, 0xfb, 0x92, 0xce}, want: "KMOVQ R14, K1"}, + {name: "q extended k to gp", code: []byte{0xc4, 0x61, 0xfb, 0x93, 0xd7}, want: "KMOVQ K7, R10"}, + {name: "q extended sib memory to k", code: []byte{0xc4, 0x81, 0xf8, 0x90, 0x4c, 0x88, 0x20}, want: "KMOVQ 32(R8)(R9*4), K1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || !strings.HasPrefix(got.Instrs[0].Raw, test.want+" ") { + t.Fatalf("decoded %#x as %#v, want %q", test.code, got.Instrs, test.want) + } + }) + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch+"/lower", func(t *testing.T) { + var unbound strings.Builder + unbound.WriteString("TEXT raw_mask_move(SB),NOSPLIT,$0-0\n") + for _, test := range tests[:20] { + unbound.WriteString(x86RawDisplacementDirectives(test.code)) + } + unbound.WriteString("RET\n") + // Preserve the original complete source as a Go-format object and + // public Context control. Unknown AX/K inputs are not an ABI. + x86UnboundVoidFormatSources(t, unbound.String(), target.goarch, target.triple) + for index, test := range tests[:20] { + t.Run(test.name, func(t *testing.T) { + name := fmt.Sprintf("rawMask%d", index) + word, move := 8, "MOVQ" + if target.goarch == "386" { + word, move = 4, "MOVL" + } + arguments, extent := "", 0 + setup := "MOVL $21,AX\n" + if strings.Contains(test.want, "(AX)") { + arguments, extent = "p *uint64", word + setup = move + " p+0(FP),AX\n" + } + // Each original opcode gets an independent function: a K->AX + // result must never become the next row's memory address. + source := fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\nMOVL $91,CX\nKMOVD CX,K0\nKMOVD CX,K5\n%s%sRET\n", + name, extent, setup, x86RawDisplacementDirectives(test.code)) + declaration := fmt.Sprintf("func %s(%s)", name, arguments) + x86LEAActualTargetGoObject(t, target.goarch, target.triple, source) + // GP transfers ignore encoded W in 32-bit mode: these + // original KMOVQ bytes transfer 32 bits. K/memory Q + // transfers retain their independent 64-bit width. + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.goarch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("independently initialized original raw mask-move rejected", err) + } + for _, body := range []string{ir, x86DwordProductIROptimize(t, ir)} { + if arguments != "" { + bits := []int{8, 16, 32, 64}[index/5] + operation := "load" + if strings.Contains(test.name, "k to memory") { + operation = "store" + } + if !strings.Contains(body, fmt.Sprintf("%s volatile i%d", operation, bits)) { + t.Fatal("dead raw mask access lost its exact volatile width after optimization", operation, bits) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-mask-move.ll", "raw-mask-move.o", body) + } + }) + } + }) + } + }) + } +} + +func TestX86RawMaskMoveWholeProducerIdentity(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + for _, store := range []bool{false, true} { + code := []byte{0xc4, 0xe1, 0xf8, 0x90, 0x08} // KMOVQ 0(AX),K1. + if store { + code[3], code[4] = 0x91, 0x00 // KMOVQ K0,0(AX). + } + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\nMOVL $91,CX\nKMOVD CX,K0\n%s p+0(FP),AX\n%sRET\n", + word, move, x86RawDisplacementDirectives(code)) + check := x86LegacyFloatReplayCheck(t, arch, source, "func Value(p *uint64)") + file, err := normalizeX86RawFile(mustParseX86RawReturn(t, source), arch) + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0] + ordinal := len(original.Instrs) - 2 + for _, mutation := range []string{ + "valid", "origin", "missing-replay", "producer", "opcode", "operand", "original-bytes", "snapshot", + } { + t.Run(fmt.Sprintf("%s/store=%t/%s", arch, store, mutation), func(t *testing.T) { + candidate := original + candidate.Instrs = cloneX86ReplayInstrs(original.Instrs) + ins := &candidate.Instrs[ordinal] + switch mutation { + case "origin": + ins.x86SourceRaw = nil + case "missing-replay": + candidate.x86RawReplay = nil + case "producer": + candidate.Instrs[1].Raw += " changed" + case "opcode": + ins.Op = "KMOVW" + case "operand": + index := 0 + if store { + index = 1 + } + ins.Args[index].Mem.Off = 1 + case "original-bytes", "snapshot": + replay := *original.x86RawReplay + if mutation == "original-bytes" { + replay.source = strings.Replace(replay.source, "BYTE $", "BYTE $1+", 1) + } else { + replay.normalized = cloneX86ReplayInstrs(replay.normalized) + replay.normalized[ordinal].Raw += " changed" + } + candidate.x86RawReplay = &replay + } + err := check(candidate) + if mutation == "valid" { + if err != nil { + t.Fatal("initialized original mask source lost its native continuation", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("edited original raw mask source acquired native access", err) + } + if err := original.x86RawReplay.validate(original, arch); err != nil { + t.Fatal("mutation damaged the unchanged original producer", err) + } + }) + } + } + } +} diff --git a/amd64_raw_mask_vector_move.go b/amd64_raw_mask_vector_move.go new file mode 100644 index 000000000..b9d7fdbc7 --- /dev/null +++ b/amd64_raw_mask_vector_move.go @@ -0,0 +1,113 @@ +package plan9asm + +import "fmt" + +type decodedX86MaskVectorMoveProperties struct { + op Op + maskToVector bool +} + +func decodedX86MaskVectorMoveOpcode(opcode byte, width64 bool) (decodedX86MaskVectorMoveProperties, bool) { + width := byte(0) + if width64 { + width = 1 + } + properties := map[[2]byte]decodedX86MaskVectorMoveProperties{ + {0x28, 0}: {op: "VPMOVM2B", maskToVector: true}, + {0x28, 1}: {op: "VPMOVM2W", maskToVector: true}, + {0x38, 0}: {op: "VPMOVM2D", maskToVector: true}, + {0x38, 1}: {op: "VPMOVM2Q", maskToVector: true}, + {0x29, 0}: {op: "VPMOVB2M"}, + {0x29, 1}: {op: "VPMOVW2M"}, + {0x39, 0}: {op: "VPMOVD2M"}, + {0x39, 1}: {op: "VPMOVQ2M"}, + }[[2]byte{opcode, width}] + return properties, properties.op != "" +} + +// decodedX86MaskVectorMoveInstruction recognizes Go 1.27's complete +// VPMOVM2{B,W,D,Q} and VPMOV{B,W,D,Q}2M family. These EVEX-only instructions +// have register-only K/X/Y/Z forms and do not permit writemasking, zeroing, or +// EVEX.b. Recover all vector registers and widths before the generic decoder. +func decodedX86MaskVectorMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64, 0x65: + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 || len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 2 || p1&3 != 2 { + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86MaskVectorMoveOpcode(opcode, p1&0x80 != 0) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if p2&0x97 != 0 { + return Instr{}, 0, true, fmt.Errorf("mask/vector moves do not support masking, zeroing, or EVEX.b") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + if modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("mask/vector moves require register operands") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + vectorNumber := 0 + maskNumber := 0 + if properties.maskToVector { + if xExt != 0 || bExt != 0 { + return Instr{}, 0, true, fmt.Errorf("K source register cannot be extended") + } + vectorNumber = int(modRM>>3&7) + rExt*8 + rHighExt*16 + maskNumber = int(modRM & 7) + } else { + if rExt != 0 || rHighExt != 0 { + return Instr{}, 0, true, fmt.Errorf("K destination register cannot be extended") + } + vectorNumber = int(modRM&7) + bExt*8 + xExt*16 + maskNumber = int(modRM >> 3 & 7) + } + if mode == 32 && vectorNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + vector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, vectorNumber))} + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))} + args := []Operand{vector, mask} + if properties.maskToVector { + args[0], args[1] = args[1], args[0] + } + return Instr{Op: properties.op, Args: args, Raw: fmt.Sprintf("%s %s, %s", properties.op, args[0].String(), args[1].String())}, modRMIndex + 1, true, nil +} diff --git a/amd64_raw_mask_vector_move_test.go b/amd64_raw_mask_vector_move_test.go new file mode 100644 index 000000000..3ef86ff7f --- /dev/null +++ b/amd64_raw_mask_vector_move_test.go @@ -0,0 +1,157 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type rawMaskVectorMoveSpec struct { + opcode byte + width64 bool + op Op + laneBits int + maskToVector bool +} + +var rawMaskVectorMoveSpecs = []rawMaskVectorMoveSpec{ + {opcode: 0x28, op: "VPMOVM2B", laneBits: 8, maskToVector: true}, + {opcode: 0x28, width64: true, op: "VPMOVM2W", laneBits: 16, maskToVector: true}, + {opcode: 0x38, op: "VPMOVM2D", laneBits: 32, maskToVector: true}, + {opcode: 0x38, width64: true, op: "VPMOVM2Q", laneBits: 64, maskToVector: true}, + {opcode: 0x29, op: "VPMOVB2M", laneBits: 8}, + {opcode: 0x29, width64: true, op: "VPMOVW2M", laneBits: 16}, + {opcode: 0x39, op: "VPMOVD2M", laneBits: 32}, + {opcode: 0x39, width64: true, op: "VPMOVQ2M", laneBits: 64}, +} + +func encodeX86EVEXMaskVectorMove(spec rawMaskVectorMoveSpec, vectorBits byte, vector, mask int) []byte { + p0 := byte(2) + regField, rmField := vector, mask + if spec.maskToVector { + p0 |= byte((1-(vector>>3)&1)<<7 | 1<<6 | 1<<5 | (1-(vector>>4)&1)<<4) + } else { + regField, rmField = mask, vector + p0 |= byte(1<<7 | (1-(vector>>4)&1)<<6 | (1-(vector>>3)&1)<<5 | 1<<4) + } + p1 := byte(15<<3 | 1<<2 | 2) + if spec.width64 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 1<<3) + modRM := byte(0xc0 | (regField&7)<<3 | rmField&7) + return []byte{0x62, p0, p1, p2, spec.opcode, modRM} +} + +func TestDecodedX86EVEXMaskVectorMoveCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawMaskVectorMoveSpecs { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + for vector := 0; vector < 32; vector++ { + for mask := 0; mask < 8; mask++ { + code := encodeX86EVEXMaskVectorMove(spec, vectorBits, vector, mask) + got, length, ok, err := decodedX86MaskVectorMoveInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, vector), fmt.Sprintf("K%d", mask)} + if spec.maskToVector { + wantArgs[0], wantArgs[1] = wantArgs[1], wantArgs[0] + } + if got.Op != spec.op || len(got.Args) != 2 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 6144 { + t.Fatalf("covered %d EVEX mask/vector move encodings, want 6144", count) + } +} + +func TestDecodedX86EVEXMaskVectorMoveGoAndWeaviateEncodings(t *testing.T) { + tests := []struct { + code []byte + op Op + args []string + }{ + {code: []byte{0x62, 0x72, 0x7e, 0x48, 0x38, 0xc0}, op: "VPMOVM2D", args: []string{"K0", "Z8"}}, + {code: []byte{0x62, 0x62, 0x7e, 0x08, 0x28, 0xd4}, op: "VPMOVM2B", args: []string{"K4", "X26"}}, + {code: []byte{0x62, 0xd2, 0x7e, 0x48, 0x29, 0xc9}, op: "VPMOVB2M", args: []string{"Z9", "K1"}}, + {code: []byte{0x62, 0x92, 0xfe, 0x28, 0x29, 0xe3}, op: "VPMOVW2M", args: []string{"Y27", "K4"}}, + } + for _, test := range tests { + got, length, ok, err := decodedX86MaskVectorMoveInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != 2 || got.Args[0].String() != test.args[0] || got.Args[1].String() != test.args[1] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } +} + +func TestDecodedX86EVEXMaskVectorMoveInvalidForms(t *testing.T) { + spec := rawMaskVectorMoveSpecs[0] + base := encodeX86EVEXMaskVectorMove(spec, 0, 0, 0) + invalid := make([][]byte, 0, 9) + reservedVVVV := append([]byte(nil), base...) + reservedVVVV[2] &^= 0x08 + reservedHighV := append([]byte(nil), base...) + reservedHighV[3] &^= 0x08 + reservedLength := append([]byte(nil), base...) + reservedLength[3] |= 0x60 + masking := append([]byte(nil), base...) + masking[3] |= 0x01 + zeroing := append([]byte(nil), base...) + zeroing[3] |= 0x80 + broadcast := append([]byte(nil), base...) + broadcast[3] |= 0x10 + memory := append([]byte(nil), base...) + memory[5] &^= 0xc0 + extendedKSource := append([]byte(nil), base...) + extendedKSource[1] &^= 0x20 + vectorToMask := encodeX86EVEXMaskVectorMove(rawMaskVectorMoveSpecs[4], 0, 0, 0) + extendedKDestination := append([]byte(nil), vectorToMask...) + extendedKDestination[1] &^= 0x80 + invalid = append(invalid, reservedVVVV, reservedHighV, reservedLength, masking, zeroing, broadcast, memory, extendedKSource, extendedKDestination) + for _, code := range invalid { + if _, _, ok, err := decodedX86MaskVectorMoveInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateRawEVEXMaskToVectorWeaviateBlockRegression(t *testing.T) { + const source = `TEXT rawMaskToVectorBlock(SB), $0-0 + LONG $0x487e7262; WORD $0xc038 + LONG $0x4875d162; WORD $0xc8fa + LONG $0x487e7262; WORD $0xc738 + LONG $0x4875d162; WORD $0xc8fa + LONG $0x487e7262; WORD $0xc138 + LONG $0x486dd162; WORD $0xd0fa + LONG $0x487e7262; WORD $0xc238 + LONG $0x4865d162; WORD $0xd8fa + LONG $0x487e7262; WORD $0xc338 + LONG $0x485dd162; WORD $0xe0fa + LONG $0x487e7262; WORD $0xc438 + LONG $0x4855d162; WORD $0xe8fa + LONG $0x487e7262; WORD $0xc538 + LONG $0x484dd162; WORD $0xf0fa + LONG $0x487e7262; WORD $0xc638 + LONG $0x4845d162; WORD $0xf8fa + RET +` + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + if strings.Count(ir, "select i1") < 8*16 { + t.Fatalf("raw VPMOVM2D block omitted mask expansion:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-mask-to-vector.ll", "amd64-raw-mask-to-vector.o", ir) +} diff --git a/amd64_raw_memory_decoder.go b/amd64_raw_memory_decoder.go new file mode 100644 index 000000000..b10327232 --- /dev/null +++ b/amd64_raw_memory_decoder.go @@ -0,0 +1,34 @@ +package plan9asm + +// The zero value retains the raw source decoder's layout restrictions. Only +// original native byte observers select physicalPC: they observe whole native +// instruction boundaries and discards these operands. A physical RIP+disp32 +// is not a source symbol, bounded literal, numerical value, or read permission. +type x86RawMemoryDecoder struct { + physicalPC bool +} + +func (d x86RawMemoryDecoder) vexTransfer(code []byte, mode, bExt, xExt int, segment Reg, vector bool) (Operand, int, error) { + if len(code) == 0 || code[0]>>6 == 3 || !d.physicalPC { + return decodedX86VEXRMOperand(code, mode, bExt, xExt, segment, vector) + } + return decodedX86RMSourceWithPC(code, mode, bExt, xExt, segment, 1, true) +} + +func (d x86RawMemoryDecoder) scalar(code []byte, mode, bExt, xExt int, segment Reg, width int) (Operand, int, error) { + return decodedX86RMSourceWithPC(code, mode, bExt, xExt, segment, width, d.physicalPC) +} + +func (d x86RawMemoryDecoder) evex(code []byte, mode, bExt, xExt int, segment Reg, bank string, scale int) (Operand, int, error) { + if len(code) == 0 || code[0]>>6 == 3 { + return decodedX86EVEXRMOperand(code, mode, bExt, xExt, segment, bank, scale) + } + return decodedX86RMSourceWithPC(code, mode, bExt, xExt, segment, scale, d.physicalPC) +} + +func (d x86RawMemoryDecoder) vex(code []byte, mode, bExt, xExt int, segment Reg, bank string) (Operand, int, error) { + if len(code) == 0 || code[0]>>6 == 3 { + return decodedX86VEXVectorRMOperand(code, mode, bExt, xExt, segment, bank) + } + return decodedX86RMSourceWithPC(code, mode, bExt, xExt, segment, 1, d.physicalPC) +} diff --git a/amd64_raw_minmax.go b/amd64_raw_minmax.go new file mode 100644 index 000000000..b542ec229 --- /dev/null +++ b/amd64_raw_minmax.go @@ -0,0 +1,30 @@ +package plan9asm + +// VEX B/W/D ignores W. EVEX B/W also ignores W; D/Q shares an opcode with +// fixed W0/W1, and Q has no VEX encoding. The remaining operand grammar is +// shared with per-lane shifts, not inferred from the legacy disassembler. +func decodedX86PackedIntegerMinMaxInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedIntegerMinMaxInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedIntegerMinMaxInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched { + return Instr{}, 0, false, nil + } + + for name, spec := range amd64PackedIntegerMinMaxSpecs { + if p.mapNumber != spec.mapNumber || p.opcode != spec.opcode { + continue + } + if !p.evex && spec.laneBits == 64 { + continue + } + if p.evex && spec.laneBits >= 32 && p.w != (spec.laneBits == 64) { + continue + } + + return decodedX86BinaryVectorOperandsWithMemory(code, p, mode, Op(name), spec.laneBits, memory) + } + return Instr{}, 0, false, nil +} diff --git a/amd64_raw_minmax_literal_test.go b/amd64_raw_minmax_literal_test.go new file mode 100644 index 000000000..ee8e40587 --- /dev/null +++ b/amd64_raw_minmax_literal_test.go @@ -0,0 +1,169 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawMinMaxLiteral(op string, evex bool, width, mask int, zeroing, broadcast bool) []byte { + spec := amd64PackedIntegerMinMaxSpecs[op] + code := encodeRawScalarMove(evex, 1, spec.opcode, 0, 0, 0, mask, zeroing) + code[1] = code[1]&0xf0 | byte(spec.mapNumber) + code[2] &^= 0x80 + if spec.laneBits == 64 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + if broadcast { + code[3] |= 0x10 + } + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*13+9)) + } + return code +} + +func TestDecodeX86RawMinMaxRIPDataCompleteFamily(t *testing.T) { + for _, op := range x86MinMaxTestOps { + spec := amd64PackedIntegerMinMaxSpecs[op] + for _, evex := range []bool{false, true} { + if !evex && spec.laneBits == 64 { + continue + } + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && spec.broadcast { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + mask, zeroing := 0, false + if evex { + mask, zeroing = 7, true + } + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", op, evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawMinMaxLiteral(op, evex, width, mask, zeroing, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op(op) + if broadcast { + want += ".BCST" + } + if zeroing { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawMinMaxRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawMinMaxLiteral("VPMINSD", true, 64, 7, true, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawMinMaxRIPDataObjects(t *testing.T) { + var source strings.Builder + var pools [][]byte + for index, op := range x86MinMaxTestOps { + spec := amd64PackedIntegerMinMaxSpecs[op] + evex := spec.laneBits == 64 + width := 32 + if evex { + width = 64 + } + broadcast := evex && spec.broadcast + mask, zeroing := 0, false + if evex { + mask, zeroing = 7, true + } + code := x86RawMinMaxLiteral(op, evex, width, mask, zeroing, broadcast) + name := fmt.Sprintf("minMaxLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "minmax-literal.ll", "minmax-literal.o", ir) + }) + } +} diff --git a/amd64_raw_minmax_test.go b/amd64_raw_minmax_test.go new file mode 100644 index 000000000..aa49e0ef2 --- /dev/null +++ b/amd64_raw_minmax_test.go @@ -0,0 +1,435 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86MinMaxTestOps = []string{ + "VPMINSB", "VPMINUB", "VPMAXSB", "VPMAXUB", + "VPMINSW", "VPMINUW", "VPMAXSW", "VPMAXUW", + "VPMINSD", "VPMINUD", "VPMAXSD", "VPMAXUD", + "VPMINSQ", "VPMINUQ", "VPMAXSQ", "VPMAXUQ", +} + +func TestX86MinMaxGrammarMatchesGoEncoder(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + pattern := `(?s)\{as: A(VPM(?:IN|AX)[SU][BWDQ]), ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}` + rows := regexp.MustCompile(pattern).FindAllStringSubmatch(string(data), -1) + if len(rows) != 16 || len(amd64PackedIntegerMinMaxSpecs) != len(rows) { + t.Fatalf("Go family=%d, typed family=%d", len(rows), len(amd64PackedIntegerMinMaxSpecs)) + } + + for _, row := range rows { + spec, ok := amd64PackedIntegerMinMaxSpecs[row[1]] + if !ok { + t.Fatalf("missing %s", row[1]) + } + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[row[1][len(row[1])-1]] + if spec.laneBits != bits || spec.signed != (row[1][5] == 'S') || spec.minimum != strings.HasPrefix(row[1], "VPMIN") { + t.Fatalf("incorrect typed semantics for %s: %+v", row[1], spec) + } + if (row[2] == "_yvandnpd") != (bits != 64) || strings.Contains(row[3], "| vex128") != (bits != 64) { + t.Fatalf("incorrect encoding generation for %s", row[1]) + } + if strings.Contains(row[3], "0F38") != (spec.mapNumber == 2) || strings.Contains(row[3], "evexBcst") != spec.broadcast { + t.Fatalf("incorrect map/broadcast for %s", row[1]) + } + opcodes := regexp.MustCompile(`0x[0-9A-F]+`).FindAllString(row[3], -1) + wantRows := 5 + if bits == 64 { + wantRows = 3 + } + if len(opcodes) != wantRows { + t.Fatalf("%s has %d encodings, want %d", row[1], len(opcodes), wantRows) + } + for _, opcode := range opcodes { + if opcode != fmt.Sprintf("0x%X", spec.opcode) { + t.Fatalf("%s opcode=%s, typed=%x", row[1], opcode, spec.opcode) + } + } + } + + operandTables := map[string][]string{ + "_yvandnpd": { + "Yxm, Yxr, Yxr", "Yym, Yyr, Yyr", + "YxmEvex, YxrEvex, YxrEvex", "YxmEvex, YxrEvex, Yknot0, YxrEvex", + "YymEvex, YyrEvex, YyrEvex", "YymEvex, YyrEvex, Yknot0, YyrEvex", + "Yzm, Yzr, Yzr", "Yzm, Yzr, Yknot0, Yzr", + }, + "_yvblendmpd": { + "YxmEvex, YxrEvex, YxrEvex", "YxmEvex, YxrEvex, Yknot0, YxrEvex", + "YymEvex, YyrEvex, YyrEvex", "YymEvex, YyrEvex, Yknot0, YyrEvex", + "Yzm, Yzr, Yzr", "Yzm, Yzr, Yknot0, Yzr", + }, + } + for name, want := range operandTables { + pattern := `(?s)var ` + name + ` = \[\]ytab\{(.*?)\n\}` + table := regexp.MustCompile(pattern).FindStringSubmatch(string(data)) + if len(table) != 2 { + t.Fatal("missing Go operand table", name) + } + args := regexp.MustCompile(`argList\{([^}]+)\}`).FindAllStringSubmatch(table[1], -1) + var got []string + for _, arg := range args { + got = append(got, arg[1]) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("Go %s rows=%v, want %v", name, got, want) + } + } +} + +func TestX86MinMaxGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86MinMaxTestOps { + lines := []string{ + op + " X0,Y1,Y2", + op + " X0,X1,(AX)", + op + ".Z X0,X1,X2", + op + ".SAE X0,X1,X2", + op + " X0,X1,K0,X2", + op + ".BCST X0,X1,X2", + op + ".Z.BCST (AX),X1,K1,X2", + op + " $1,X1,X2", + } + if strings.HasSuffix(op, "B") || strings.HasSuffix(op, "W") { + lines = append(lines, op+".BCST (AX),X1,X2") + } + if arch == "386" { + lines = append(lines, op+" Z8,Z0,Z1", op+" X0,X1,K1,X2") + } + + for _, line := range lines { + dir := t.TempDir() + path := filepath.Join(dir, "invalid.s") + source := "TEXT bad(SB),4,$0-0\n" + line + "\nRET\n" + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), path) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + out, err := cmd.CombinedOutput() + historicalZero := strings.HasPrefix(runtime.Version(), "go1.20") && strings.Contains(line, ".Z X0") + if historicalZero && err != nil || !historicalZero && err == nil { + t.Fatalf("Go %s unexpected result for %s: %v %s", arch, line, err, out) + } + + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + assertX86PackedIntegerMinMaxRejected(t, arch, triple, line) + } + } + } +} + +func TestX86MinMaxZeroMaskDoesNotAccessMemory(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("portable values require the native 64-bit host counterpart") + } + + const original = `TEXT zero_mask(SB),4,$0-24 +MOVQ source+0(FP), AX +MOVQ out+8(FP), DI +KMOVQ mask+16(FP), K1 +VPMINUD (AX), Y0, K1, Y1 +VMOVUPS Y1, (DI) +RET +` + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", original) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, original, + "func zero_mask(source,out *byte,mask uint64)", "amd64", triple, api) + x86VectorScratchCodegenControl(t, original, "func zero_mask(source,out *byte,mask uint64)", + "amd64", triple, ir, err) + } + // Keep the original source above. This separate initialized program owns + // the actual masked-off NULL read, not an invented incoming Y0/Y1 value. + initialized := strings.Replace(original, "zero_mask(SB)", "zero_mask_initialized(SB)", 1) + initialized = strings.Replace(initialized, "KMOVQ mask", "VPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\nKMOVQ mask", 1) + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", initialized) + ir, err := x86ScalarFPBitViewAPI(t, initialized, + "func zero_mask_initialized(source,out *byte,mask uint64)", "amd64", triple, "text") + if err != nil { + t.Fatal(err) + } + + mainC := ` +#include +#include + +extern void zero_mask(const void *source, void *out, uint64_t mask); + +int main(void) { + uint8_t out[32]; + zero_mask(NULL, out, 0); + return 0; +} +` + mainC = strings.ReplaceAll(mainC, "zero_mask", "zero_mask_initialized") + x86ConflictFunnelPortableRuntime(t, llc, clang, "minmax_zero_mask_initialized", triple, ir, mainC) +} + +func TestX86RawMinMaxEncodingFields(t *testing.T) { + // The shared operand decoder already has the exhaustive shift-register + // product test. Here each min/max member checks every register field and + // W interpretation independently, without repeating that product 16 times. + opcodes := map[string]byte{ + "VPMINSB": 0x38, "VPMINUB": 0xda, "VPMAXSB": 0x3c, "VPMAXUB": 0xde, + "VPMINSW": 0xea, "VPMINUW": 0x3a, "VPMAXSW": 0xee, "VPMAXUW": 0x3e, + "VPMINSD": 0x39, "VPMINUD": 0x3b, "VPMAXSD": 0x3d, "VPMAXUD": 0x3f, + "VPMINSQ": 0x39, "VPMINUQ": 0x3b, "VPMAXSQ": 0x3d, "VPMAXUQ": 0x3f, + } + count := 0 + for _, op := range x86MinMaxTestOps { + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + mapNumber := byte(2) + if op == "VPMINUB" || op == "VPMAXUB" || op == "VPMINSW" || op == "VPMAXSW" { + mapNumber = 1 + } + for _, evex := range []bool{false, true} { + if !evex && bits == 64 { + continue + } + limit, widths := 16, 2 + if evex { + limit, widths = 32, 3 + } + for width := 0; width < widths; width++ { + for axis := 0; axis < 3; axis++ { + for register := 0; register < limit; register++ { + regs := [3]int{0, 1, 2} // source2, source1, destination. + regs[axis] = register + variants := 1 + if !evex || bits < 32 { + variants = 2 + } + for variant := 0; variant < variants; variant++ { + mask, zero := 0, false + if evex { + mask = register % 8 + zero = mask != 0 && register&1 != 0 + } + code := encodeRawScalarMove(evex, 1, 0, regs[0], regs[1], regs[2], mask, zero) + code[len(code)-2] = opcodes[op] + code[1] = code[1]&0xf0 | mapNumber + code[2] &^= 0x80 + if variant != 0 || bits == 64 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(width) << 5 + } else { + code[2] |= byte(width) << 2 + } + + prefix := []string{"X", "Y", "Z"}[width] + args := []Operand{ + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, regs[0]))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, regs[1]))}, + } + if mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, regs[2]))}) + wantOp := Op(op) + if zero { + wantOp += ".Z" + } + + got, size, matched, err := decodedX86PackedIntegerMinMaxInstruction(code, 64) + if err != nil || !matched || size != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, args) { + t.Fatalf("decode %x: %+v %d %v %v; want %s %+v", code, got, size, matched, err, wantOp, args) + } + _, _, matched, err = decodedX86PackedIntegerMinMaxInstruction(code, 32) + if !matched || (err != nil) != (register >= 8) { + t.Fatalf("386 decode %x: matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + if count != 9216 { + t.Fatalf("covered %d field cases, want 9216", count) + } +} + +func TestX86RawMinMax386MaskedObjects(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT masked_minmax(SB),4,$0-0\n") + for _, op := range x86MinMaxTestOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, input := range []string{width + "0", "(AX)"} { + for _, suffix := range []string{"", ".Z"} { + line := fmt.Sprintf("%s%s %s,%s1,K7,%s2", op, suffix, input, width, width) + emitX86GoEncodedTestInstruction(t, &source, line, true) + } + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + line := fmt.Sprintf("%s.BCST.Z (AX),%s1,K7,%s2", op, width, width) + emitX86GoEncodedTestInstruction(t, &source, line, true) + } + } + } + source.WriteString("RET\n") + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "386", triple) + compileLLVMToObject(t, llc, triple, "masked-minmax-386.ll", "masked-minmax-386.o", ir) + } +} + +func TestX86RawMinMaxGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + last := 7 + if arch == "amd64" { + last = 31 + } + + var source strings.Builder + source.WriteString("TEXT minmax(SB),4,$0-0\n") + for _, op := range x86MinMaxTestOps { + broadcast := strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") + for _, width := range []string{"X", "Y", "Z"} { + for _, reg := range []int{0, last} { + inputs := []string{ + fmt.Sprintf("%s%d", width, last-reg), + "(AX)", "-7(BX)(CX*4)", "64(SI)", "8192(DI)", + } + for _, input := range inputs { + fmt.Fprintf(&source, "%s %s,%s%d,%s%d\n", op, input, width, reg, width, last-reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s,%s%d,K1,%s%d\n", op, input, width, reg, width, last-reg) + fmt.Fprintf(&source, "%s.Z %s,%s%d,K7,%s%d\n", op, input, width, reg, width, last-reg) + } + } + if broadcast { + fmt.Fprintf(&source, "%s.BCST -8(BX)(CX*4),%s%d,%s%d\n", op, width, reg, width, last-reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s.BCST.Z 64(SI),%s%d,K7,%s%d\n", op, width, reg, width, last-reg) + } + } + } + if width != "Z" { + fmt.Fprintf(&source, "%s %s0,%s1,%s2\n", op, width, width, width) + fmt.Fprintf(&source, "%s (AX),%s1,%s2\n", op, width, width) + } + } + } + source.WriteString("RET\n") + + code := assembleX87ControlBytes(t, arch, source.String()) + got, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(got.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(got.Instrs), len(want)) + } + for i, instruction := range got.Instrs { + if instruction.Op != want[i].Op || !reflect.DeepEqual(instruction.Args, want[i].Args) { + t.Fatalf("instruction %d: %+v, want %+v", i, instruction, want[i]) + } + } + + file := x86RawUnprovedReturnFormProbeFile(t, arch, "minmax", code) + targets := []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + "i386-unknown-linux-gnu", "i686-pc-windows-msvc", + } + for _, triple := range targets { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + ir := x86UnboundVoidRawProbeObject(t, file, arch, triple) + compileLLVMToObject(t, llc, triple, "minmax.ll", "minmax.o", ir) + } + t.Logf("%s: %d actual Go encodings", arch, len(want)-1) + }) + } +} + +func TestX86RawMinMaxInvalidAndMemoryForms(t *testing.T) { + invalid := [][]byte{ + {0x62, 0xf2, 0x71, 0x0b, 0x39, 0xe0}, // EVEX fixed bit clear. + {0x62, 0xf2, 0x75, 0x6b, 0x39, 0xe0}, // Reserved vector length. + {0x62, 0xf2, 0x75, 0x88, 0x39, 0xe0}, // Zeroing without mask. + {0x62, 0xf2, 0x75, 0x1b, 0x39, 0xe0}, // Register broadcast. + {0x62, 0xf2, 0x75, 0x1b, 0x38, 0x00}, // Byte broadcast. + {0x62, 0xf2, 0x74, 0x0b, 0x39, 0xe0}, // Wrong mandatory prefix. + {0x62, 0xf2, 0x75, 0x0b, 0x39}, // Missing ModRM. + {0x62, 0xf2, 0x75, 0x0b, 0x39, 0x04}, // Missing SIB. + {0x62, 0xf2, 0x75, 0x0b, 0x39, 0x40}, // Missing disp8. + {0x67, 0xc5, 0xf1, 0xda, 0xc0}, // Address-size override. + {0xc5, 0xf1, 0xda, 0x05, 0, 0, 0, 0}, // Unrelocated RIP source. + } + for _, code := range invalid { + _, _, matched, err := decodedX86PackedIntegerMinMaxInstruction(code, 64) + if !matched || err == nil { + t.Fatalf("accepted invalid %x", code) + } + } + if _, _, matched, err := decodedX86PackedIntegerMinMaxInstruction([]byte{0xc5, 0xf1, 0xda, 0xc0}, 16); !matched || err == nil { + t.Fatal("accepted 16-bit mode") + } + for _, code := range [][]byte{nil, {0x62}, {0x90}, {0xc4, 0xe3, 0x79, 0x39, 0xc0}, {0xc4, 0xe2, 0x79, 0x40, 0xc0}} { + if _, _, matched, err := decodedX86PackedIntegerMinMaxInstruction(code, 64); matched || err != nil { + t.Fatalf("claimed unrelated %x", code) + } + } + + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + for _, broadcast := range []bool{false, true} { + code := []byte{segment.prefix, 0x62, 0x92, 0xf5, 0xcb, 0x39, 0x64, 0x88, 0xfe} + offset, op := int64(-128), Op("VPMINSQ.Z") + if broadcast { + code[4] |= 0x10 + offset, op = -16, "VPMINSQ.BCST.Z" + } + got, size, matched, err := decodedX86PackedIntegerMinMaxInstruction(code, 64) + want := MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: offset} + if err != nil || !matched || size != len(code) || got.Op != op || !reflect.DeepEqual(got.Args[0].Mem, want) { + t.Fatalf("decode %x: %+v, err=%v", code, got, err) + } + } + } +} diff --git a/amd64_raw_minpos.go b/amd64_raw_minpos.go new file mode 100644 index 000000000..32455ed50 --- /dev/null +++ b/amd64_raw_minpos.go @@ -0,0 +1,41 @@ +package plan9asm + +import "fmt" + +// Intel XED's AVX table specifies VL128, V66, V0F38 and NOVSR; W is ignored. +// There are no Y/Z, EVEX, mask, immediate or three-operand forms. +func decodedX86MinimumPositionInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + spec := amd64MinimumPositionSpecs["VPHMINPOSUW"] + if !matched || p.mapNumber != spec.mapNumber || p.opcode != spec.opcode { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("minimum position: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.vectorLength != 0 || p.pp != 1 || p.upper != 0 { + return fail("requires VEX.128.66 with reserved vvvv") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.x != 0 || p.b != 0) { + return fail("extended register in 32-bit mode") + } + source, size, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + if err != nil { + return Instr{}, 0, true, err + } + dst := Reg(fmt.Sprintf("X%d", int(code[p.modRM]>>3&7)+p.r*8)) + return Instr{ + Op: "VPHMINPOSUW", + Args: []Operand{source, {Kind: OpReg, Reg: dst}}, + Raw: fmt.Sprintf("VPHMINPOSUW %s, %s", source.String(), dst), + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_minpos_address_test.go b/amd64_raw_minpos_address_test.go new file mode 100644 index 000000000..009fcfdf1 --- /dev/null +++ b/amd64_raw_minpos_address_test.go @@ -0,0 +1,178 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "strconv" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func x86MinimumPositionOriginalGoBytes(t *testing.T, arch, source string) []byte { + t.Helper() + listing, err := x86ScalarLiteralGoListing(t, arch, source) + if err != nil { + t.Fatal(err) + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + text := strings.TrimLeft(line, "\t ") + fields := strings.Fields(text) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") || len(fields[1]) != 2 { + continue + } + offset, err := strconv.ParseUint(fields[0][2:], 16, 32) + start := strings.IndexByte(text, ' ') + if err != nil || int(offset) != len(code) || start < 0 || len(text) < start+48 { + t.Fatalf("incomplete/noncontiguous stock Go hex row: %s", line) + } + gap := false + for column := 0; column < 16; column++ { + field := text[start+3*column : start+3*(column+1)] + if field == " " { + gap = true + continue + } + value, err := strconv.ParseUint(field[1:], 16, 8) + if gap || field[0] != ' ' || err != nil { + t.Fatalf("invalid stock Go hex column: %q", field) + } + code = append(code, byte(value)) + } + } + if len(code) == 0 { + t.Fatal("stock Go omitted the original instruction bytes") + } + return code +} + +func TestX86RawMinimumPositionInactiveSIBScalePhysicalWitness(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + mode := 64 + if arch == "386" { + mode = 32 + } + for shift := byte(0); shift < 4; shift++ { + // Original legacy PHMINPOSUW 8192(SP),X0. SIB.index=4 + // suppresses the index; all four scale bit patterns are ignored. + code := []byte{0x66, 0x0f, 0x38, 0x41, 0x84, 0x24 | shift<<6, 0, 0x20, 0, 0} + original := append([]byte(nil), code...) + x86HandoffOriginalBytes(t, arch, code) + physical, err := x86asm.Decode(code, mode) + memory, ok := physical.Args[1].(x86asm.Mem) + base := x86asm.RSP + if mode == 32 { + base = x86asm.ESP + } + if err != nil || physical.Len != len(code) || physical.Op != x86asm.PHMINPOSUW || + !ok || memory.Base != base || memory.Index != 0 || memory.Scale != 1<4 GiB synthetic allocator: llc O0 rejects its +// frame offsets. This is neither public success nor native stack permission. +func TestX86RawMinimumPositionHugeSPAllocatorBoundaryObservation(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Keep the complete mode32 address inventory in both source forms. The + // original encoded RET remains a negative stack/continuation control; + // the raw compile-only probe replaces only that RET with encoded UD2. + source, _, count, _ := x86MinimumPositionAddressSources("386", "giant", "UD2") + if count != 240 { + t.Fatalf("complete mode32 address denominator changed: %d", count) + } + namedCode := x86MinimumPositionOriginalGoBytes(t, "386", source) + namedFile, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + rawSource, _, _, _ := x86MinimumPositionAddressSources("386", "giant", "RET") + rawCode := x86MinimumPositionOriginalGoBytes(t, "386", rawSource) + x86UnboundVoidFormatSources(t, "TEXT giant(SB),4,$0-0\n"+x86DwordProductDirectives(rawCode), + "386", "i386-unknown-linux-gnu") + rawFile := x86RawStackObservingFormProbeFile(t, "386", "giant", rawCode) + for _, fixture := range []struct { + name string + file *File + source string + code []byte + }{ + {"named", namedFile, source, namedCode}, + {"original-byte-probe", rawFile, rawSource, rawCode}, + } { + t.Run(fixture.name, func(t *testing.T) { + file := fixture.file + before := fmt.Sprintf("%#v", file.Funcs) + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + // The same complete 240-row typed emitter remains an explicit + // private allocation diagnostic, with no cached input proof. + privateIR := x86UnboundBlendMinposProbeObject(t, file, "386", triple) + if !strings.Contains(privateIR, "%local_stack = alloca [4294967439 x i8]") { + t.Fatal("private allocator lost the complete original SP envelope") + } + for _, api := range []string{"text", "module", "context"} { + t.Run(triple+"/"+api, func(t *testing.T) { + options := Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"giant": {Name: "giant", Ret: Void}}} + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + module, callErr := TranslateModule(file, options) + err = callErr + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "context": + ctx := llvm.NewContext() + defer ctx.Dispose() + module, callErr := TranslateModuleInContext(ctx, file, options) + err = callErr + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + } + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" || before != fmt.Sprintf("%#v", file.Funcs) { + t.Fatalf("unbound source acquired a public AUTO/stack role: %v; complete source unchanged=%t", err, before == fmt.Sprintf("%#v", file.Funcs)) + } + // The exact private source envelope is [-2^31, 2^31-1+16), + // plus the original two 64-byte synthetic guards. Reusing + // this private IR is not an API translation certificate. + dir := t.TempDir() + input, object := filepath.Join(dir, "giant.ll"), filepath.Join(dir, "giant.o") + if err := os.WriteFile(input, []byte(privateIR), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, llc, "-O0", "-filetype=obj", "-mtriple="+triple, "-o", object, input) + output, err := runtimeOracleCombinedOutput(ctx, command) + if ctx.Err() != nil { + t.Fatalf("bounded compiler observation expired: %v: %s", err, output) + } + exit, ordinaryExit := err.(*exec.ExitError) + if !ordinaryExit || exit.ExitCode() != 1 { + t.Fatalf("expected plain LLVM32 O0 offset failure, got %v: %s", err, output) + } + diagnostic := "error: :0:0: 64-bit offset calculated but target is 32-bit" + warning := regexp.MustCompile(`^warning: :0:0: stack frame size \(([0-9]+)\) exceeds limit \(4294967295\) in function 'InstructionFormat0'$`) + errors, warnings := 0, 0 + text := strings.ReplaceAll(string(output), "\r\n", "\n") + if strings.Contains(text, "\r") { + t.Fatalf("malformed diagnostic line endings: %q", output) + } + for _, line := range strings.Split(strings.TrimSpace(text), "\n") { + if line == diagnostic { + errors++ + continue + } + match := warning.FindStringSubmatch(line) + if len(match) != 2 { + t.Fatalf("unknown diagnostic is not the observed allocator boundary: %s", output) + } + frameBytes, err := strconv.ParseUint(match[1], 10, 64) + if err != nil || frameBytes <= 4294967295 { + t.Fatalf("compiler warning did not observe an oversized frame: %s", line) + } + warnings++ + } + if warnings != 1 || errors == 0 { + t.Fatalf("missing/changed exact allocator diagnostics: errors=%d warnings=%d", errors, warnings) + } + t.Logf("LLVM_ALLOCATOR_BOUNDARY_OBSERVATION: source SHA=%x; original Go code SHA=%x; %s public %s=Context; private InstructionFormat0 local_stack=4294967439 bytes; compiler O0 offset errors=%d, frame warnings=%d; no object/stack correctness/runtime or source-N/A credit", sha256.Sum256([]byte(fixture.source)), sha256.Sum256(fixture.code), fixture.name, api, errors, warnings) + }) + } + } + }) + } +} + +func TestX86MinimumPositionInitializedBoundedAUTOPublicObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + const initialized = `TEXT Bounded(SB),4,$32-4 +MOVL $0x12345678,AX +MOVL AX,tmp-16(SP) +MOVL AX,tmp-12(SP) +MOVL AX,tmp-8(SP) +MOVL AX,tmp-4(SP) +VMOVUPS tmp-16(SP),X1 +PHMINPOSUW X1,X2 +VPHMINPOSUW tmp-16(SP),X3 +MOVL X2,AX +MOVL X3,BX +XORL BX,AX +MOVL AX,ret+0(FP) +RET +` + for _, target := range x86RawReturnTargets { + for _, missing := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/missing=%t", target.triple, missing), func(t *testing.T) { + source := initialized + if missing { + source = strings.Replace(source, "MOVL AX,tmp-12(SP)\n", "", 1) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Bounded() uint32", target.arch, target.triple, api) + if missing { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("bounded AUTO invented the missing original initialized word", err) + } + return + } + if err != nil || ir == "" { + t.Fatal("complete small-frame numerical source rejected", err) + } + for _, input := range []string{ir, x86DwordProductIROptimize(t, ir)} { + x86PackedStringCompileObject(t, llc, target.triple, input, "-O0") + } + }) + } + }) + } + } +} diff --git a/amd64_raw_minpos_test.go b/amd64_raw_minpos_test.go new file mode 100644 index 000000000..3153317b5 --- /dev/null +++ b/amd64_raw_minpos_test.go @@ -0,0 +1,225 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86MinimumPositionGrammarMatchesGoEncoder(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + for _, form := range []struct { + file, op, table, operands string + vector bool + }{ + {"asm6.go", "PHMINPOSUW", "yxm_q4", "Yxm, Yxr", false}, + {"avx_optabs.go", "VPHMINPOSUW", "_yvaesimc", "Yxm, Yxr", true}, + } { + data, err := os.ReadFile(filepath.Join(root, form.file)) + if err != nil { + t.Fatal(err) + } + text := string(data) + spec, ok := amd64MinimumPositionSpecs[Op(form.op)] + if !ok || spec.vector != form.vector || spec.mapNumber != 2 || spec.opcode != 0x41 { + t.Fatalf("incorrect typed spec for %s: %+v", form.op, spec) + } + pattern := `(?s)var ` + form.table + ` = \[\]ytab\{(.*?)\n\}` + table := regexp.MustCompile(pattern).FindStringSubmatch(text) + if len(table) != 2 || strings.Count(table[1], "argList{") != 1 || + !strings.Contains(table[1], "argList{"+form.operands+"}") { + t.Fatalf("Go %s operand grammar changed", form.op) + } + if form.vector { + pattern = `(?s)\{as: AVPHMINPOSUW, ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}` + rows := regexp.MustCompile(pattern).FindAllStringSubmatch(text, -1) + if len(rows) != 1 || rows[0][1] != form.table || strings.Count(rows[0][2], "0x41") != 1 { + t.Fatal("Go VPHMINPOSUW opcode forms changed") + } + for _, token := range []string{"vex128", "vex66", "vex0F38", "vexW0"} { + if !strings.Contains(rows[0][2], token) { + t.Fatalf("Go VPHMINPOSUW missing %s", token) + } + } + } else if !strings.Contains(text, "{APHMINPOSUW, yxm_q4, Pq4, opBytes{0x41}}") { + t.Fatal("Go PHMINPOSUW encoding changed") + } + } + if len(amd64MinimumPositionSpecs) != 2 { + t.Fatal("minimum-position family must match both Go opcodes") + } +} + +func TestX86RawMinimumPositionEncodingFields(t *testing.T) { + for _, wide := range []bool{false, true} { + for src := 0; src < 16; src++ { + for dst := 0; dst < 16; dst++ { + code := encodeRawScalarMove(false, 1, 0x41, src, 0, dst, 0, false) + code[1] = code[1]&0xe0 | 2 + if wide { + code[2] |= 0x80 + } + for _, mode := range []int{32, 64} { + got, size, matched, err := decodedX86MinimumPositionInstruction(code, mode) + if mode == 32 && (src >= 8 || dst >= 8) { + if !matched || err == nil { + t.Fatalf("386 accepted %x", code) + } + continue + } + want := []Operand{ + {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", src))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", dst))}, + } + if err != nil || !matched || size != len(code) || got.Op != "VPHMINPOSUW" || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x mode=%d: %+v %v, want %+v", code, mode, got, err, want) + } + } + } + } + } +} + +func TestX86RawMinimumPositionInvalidAndSegmentForms(t *testing.T) { + invalid := [][]byte{ + {0xc4, 0xe2, 0x7d, 0x41, 0xc0}, // VEX.L1. + {0xc4, 0xe2, 0x71, 0x41, 0xc0}, // vvvv is not reserved. + {0xc4, 0xe2, 0x78, 0x41, 0xc0}, // No 66 prefix. + {0x62, 0xf2, 0x7d, 0x08, 0x41, 0xc0}, // No EVEX form. + {0xc4, 0xe2, 0x79, 0x41}, // Missing ModRM. + {0xc4, 0xe2, 0x79, 0x41, 0x04}, // Missing SIB. + {0xc4, 0xe2, 0x79, 0x41, 0x80}, // Missing displacement. + {0xc4, 0xe2, 0x79, 0x41, 0x05, 0, 0, 0, 0}, // RIP relative. + {0x67, 0xc4, 0xe2, 0x79, 0x41, 0x00}, // Address override. + } + for _, code := range invalid { + if _, _, matched, err := decodedX86MinimumPositionInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x", code) + } + } + if _, _, matched, err := decodedX86MinimumPositionInstruction([]byte{0xc4, 0xe2, 0x79, 0x41, 0xc0}, 16); !matched || err == nil { + t.Fatal("accepted invalid x86 mode") + } + for _, code := range [][]byte{nil, {0xc4}, {0xc4, 0xe1, 0x79, 0x41, 0xc0}, {0xc4, 0xe2, 0x79, 0x40, 0xc0}} { + if _, _, matched, _ := decodedX86MinimumPositionInstruction(code, 64); matched { + t.Fatalf("matched unrelated encoding %x", code) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + code := []byte{segment.prefix, 0xc4, 0x02, 0x79, 0x41, 0x64, 0x88, 0xf9} + got, size, matched, err := decodedX86MinimumPositionInstruction(code, 64) + want := []Operand{ + {Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: -7}}, + {Kind: OpReg, Reg: "X12"}, + } + if err != nil || !matched || size != len(code) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x: %+v %v, want %+v", code, got, err, want) + } + } +} + +func TestX86MinimumPositionGoRejectedForms(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + for _, op := range []string{"PHMINPOSUW", "VPHMINPOSUW"} { + forms := []string{ + " X0", " X0,X1,X2", " X0,K1,X1", " X0,(AX)", + " M0,M1", " Y0,Y1", " Z0,Z1", " X16,X0", " X0,X16", + ".Z X0,X1", ".BCST (AX),X1", ".SAE X0,X1", + } + if arch == "386" && op == "PHMINPOSUW" { + forms = append(forms, " X8,X0", " X0,X8") + } + for _, operands := range forms { + line := op + operands + requireX86GoAssemblerResult(t, arch, "TEXT bad(SB),4,$0-0\n"+line+"\nRET\n", false) + assertX86PackedUnsignedWordMinimumPositionRejected(t, arch, triple, line) + } + } + } +} + +func TestX86RawMinimumPositionGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + limit := 16 + if arch == "386" { + limit = 8 + } + memories := []string{"(AX)", "-7(BP)(CX*4)", "8192(SP)"} + if arch == "amd64" { + memories = append(memories, "(R8)", "-7(R13)(R15*8)") + } + var source strings.Builder + source.WriteString("TEXT minpos(SB),4,$0-0\n") + for _, op := range []string{"PHMINPOSUW", "VPHMINPOSUW"} { + for dst := 0; dst < limit; dst++ { + for src := 0; src < limit; src++ { + fmt.Fprintf(&source, "%s X%d,X%d\n", op, src, dst) + } + for _, memory := range memories { + fmt.Fprintf(&source, "%s %s,X%d\n", op, memory, dst) + } + } + } + source.WriteString("RET\n") + code := x86MinimumPositionOriginalGoBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !x86RawOperandsMatchGoSource(got.Args, want[i].Args) { + t.Fatalf("instruction %d %s operands=%#v, want source %#v", i, got.Op, got.Args, want[i].Args) + } + } + file := x86RawStackObservingFormProbeFile(t, arch, "minpos", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir := x86UnboundBlendMinposProbeObject(t, file, arch, triple) + compileLLVMToObject(t, llc, triple, "raw-minpos.ll", "raw-minpos.o", ir) + } + t.Logf("checked %d actual Go encodings", len(want)-1) + }) + } +} + +func TestX86MinimumPositionGoAccepted386HighNamedRegisters(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT minpos(SB),4,$0-0\n") + for reg := 8; reg < 16; reg++ { + fmt.Fprintf(&source, "VPHMINPOSUW X%d,X0\nVPHMINPOSUW X0,X%d\nVPHMINPOSUW (AX),X%d\n", reg, reg, reg) + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + x86UnboundVoidFormatSources(t, source.String(), "386", triple) + } + t.Log("24 Go-accepted high named rows retain original Go objects and public Context, not physical mode32 format objects; complete initialized low-bank objects have a separate five-target owner") +} diff --git a/amd64_raw_mixed_layout.go b/amd64_raw_mixed_layout.go new file mode 100644 index 000000000..8c13c150f --- /dev/null +++ b/amd64_raw_mixed_layout.go @@ -0,0 +1,51 @@ +package plan9asm + +// These legacy SSE/SSE2 symbol loads occur between generated raw instruction +// spans. Their exact Go-assembler lengths are needed only to locate raw +// PC-relative targets; the named instructions still lower semantically. +var x86RawMixedSSESymbolReadOps = map[Op]bool{ + "ADDPD": true, "ADDPS": true, "MULPD": true, "MULPS": true, + "ANDPD": true, "ANDPS": true, "ANDNPD": true, "ANDNPS": true, + "XORPD": true, "XORPS": true, "CMPPD": true, "CMPPS": true, + "PAND": true, "PANDN": true, "POR": true, "PXOR": true, + "PADDB": true, "PADDW": true, "PADDL": true, "PADDQ": true, + "PSUBL": true, "PCMPGTL": true, "PCMPEQL": true, + "PMULHUW": true, "PMADDWL": true, "PMINUB": true, "PMAXUB": true, + "MOVQ": true, +} + +func x86RawMixedSSESymbolReadSize(ins Instr) (int, bool) { + if !x86RawMixedSSESymbolReadOps[ins.Op] || len(ins.Args) < 2 || + ins.Args[0].Kind != OpSym || ins.Args[1].Kind != OpReg { + return 0, false + } + if _, _, ok := parseSBRef(ins.Args[0].Sym); !ok { + return 0, false + } + register, ok := amd64ParseXReg(ins.Args[1].Reg) + if !ok || register >= 16 { + return 0, false + } + compare := ins.Op == "CMPPD" || ins.Op == "CMPPS" + if compare { + if len(ins.Args) != 3 || ins.Args[2].Kind != OpImm || ins.Args[2].Imm < 0 || ins.Args[2].Imm > 255 { + return 0, false + } + } else if len(ins.Args) != 2 { + return 0, false + } + // 0F opcode, ModRM and disp32 = 7 bytes. Packed-double/integer and + // MOVQ forms have one mandatory prefix; X8..X15 add one REX byte. + length := 7 + if ins.Op == "MOVQ" || len(ins.Op) > 0 && ins.Op[0] == 'P' || + len(ins.Op) >= 2 && ins.Op[len(ins.Op)-2:] == "PD" { + length++ + } + if register >= 8 { + length++ + } + if compare { + length++ + } + return length, true +} diff --git a/amd64_raw_mixed_layout_test.go b/amd64_raw_mixed_layout_test.go new file mode 100644 index 000000000..68754640d --- /dev/null +++ b/amd64_raw_mixed_layout_test.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestDecodeX86RawByteIncrementWidth(t *testing.T) { + code := []byte{0xfe, 0xc1, 0xc3} + inst, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + if inst.Op != x86asm.INC || x86asm.GoSyntax(inst, 0, nil) != "INCL CL" { + t.Fatalf("x/arch INC baseline changed: %+v", inst) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "byte increment", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "INCB" || decoded[0].Args[0].Reg != CL { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawIncrementDecrementCompleteGoForms(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + var source strings.Builder + source.WriteString("TEXT rawIncDec(SB),4,$0-0\n") + for _, op := range []string{"INC", "DEC"} { + for _, form := range []struct { + suffix, register string + }{ + {"B", "CL"}, {"W", "CX"}, {"L", "CX"}, {"Q", "CX"}, + } { + if goarch == "386" && form.suffix == "Q" { + continue + } + fmt.Fprintf(&source, "\t%s%s %s\n", op, form.suffix, form.register) + fmt.Fprintf(&source, "\t%s%s 8(AX)\n", op, form.suffix) + } + } + source.WriteString("\tRET\n") + code := assembleX87ControlBytes(t, goarch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), goarch) + if err != nil { + t.Fatalf("%s: %v", goarch, err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("%s decoded %d instructions, want %d", goarch, len(decoded.Instrs), len(want)) + } + for index, ins := range decoded.Instrs { + if ins.Op != want[index].Op || !reflect.DeepEqual(ins.Args, want[index].Args) { + t.Fatalf("%s instruction %d=%+v, want %+v", goarch, index, ins, want[index]) + } + } + } +} + +func TestX86MixedSSESymbolReadLengthsMatchGoAssembler(t *testing.T) { + for op := range x86RawMixedSSESymbolReadOps { + for _, register := range []string{"X0", "X10"} { + t.Run(string(op)+"/"+register, func(t *testing.T) { + instruction := fmt.Sprintf("%s pool(SB), %s", op, register) + if op == "CMPPS" || op == "CMPPD" { + instruction += ", $5" + } + source := "TEXT width(SB),4,$0-0\n\t" + instruction + "\n\tRET\n" + code := assembleX87ControlBytes(t, "amd64", source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + length, ok := x86RawMixedSSESymbolReadSize(file.Funcs[0].Instrs[1]) + if !ok || length != len(code)-1 { + t.Fatalf("%s length=%d, recognized=%t, Go bytes=%x", instruction, length, ok, code) + } + }) + } + } +} + +func TestTranslateX86MixedRawJumpAcrossNamedSSERead(t *testing.T) { + const source = `TEXT mixedRawJump(SB),4,$0-0 + BYTE $0xe9 + LONG $8 + POR pool(SB), X0 + BYTE $0xc3 +` + code := assembleX87ControlBytes(t, "amd64", source) + if len(code) != 14 || code[0] != 0xe9 || code[13] != 0xc3 { + t.Fatalf("unexpected Go assembler layout: %x", code) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"mixedRawJump": {Name: "mixedRawJump", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, amd64LLVMBlockName("raw_jcc_")) { + t.Fatal("cross-segment branch label missing") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "mixed-raw-jump.ll", "mixed-raw-jump.o", ir) + }) + } +} diff --git a/amd64_raw_movbe_width_test.go b/amd64_raw_movbe_width_test.go new file mode 100644 index 000000000..1b2a47211 --- /dev/null +++ b/amd64_raw_movbe_width_test.go @@ -0,0 +1,75 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +func x86RawMOVBE(width int, store bool, register int) []byte { + var code []byte + if width == 16 { + code = append(code, 0x66) + } + rex := byte(0x40 | (register>>3&1)<<2) + if width == 64 { + rex |= 0x08 + } + if rex != 0x40 { + code = append(code, rex) + } + opcode := byte(0xf0) + if store { + opcode = 0xf1 + } + code = append(code, 0x0f, 0x38, opcode, byte((register&7)<<3|0x02), 0xc3) + return code +} + +func TestDecodeX86RawMOVBECompleteWidthsAndDirections(t *testing.T) { + for _, width := range []int{16, 32, 64} { + for _, store := range []bool{false, true} { + for _, register := range []int{0, 7, 8, 15} { + name := fmt.Sprintf("%d/store%t/reg%d", width, store, register) + t.Run(name, func(t *testing.T) { + code := x86RawMOVBE(width, store, register) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := map[int]Op{16: "MOVBEW", 32: "MOVBEL", 64: "MOVBEQ"}[width] + if len(got) != 2 || got[0].Op != want || got[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s", code, got, want) + } + }) + } + } + } +} + +func TestTranslateX86RawMOVBEObjects(t *testing.T) { + file := &File{Arch: ArchAMD64} + index := 0 + for _, width := range []int{16, 32, 64} { + for _, store := range []bool{false, true} { + name := fmt.Sprintf("rawMOVBE%d", index) + probe := x86RawUnprovedReturnFormProbeFile(t, "amd64", name, x86RawMOVBE(width, store, 8)) + file.Funcs = append(file.Funcs, probe.Funcs...) + index++ + } + } + if index != 6 { + t.Fatal("complete raw MOVBE width/direction directory changed", index) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawProbeObject(t, file, "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-movbe.ll", "raw-movbe.o", ir) + }) + } +} diff --git a/amd64_raw_movemask.go b/amd64_raw_movemask.go new file mode 100644 index 000000000..38a5d3dfe --- /dev/null +++ b/amd64_raw_movemask.go @@ -0,0 +1,60 @@ +package plan9asm + +import "fmt" + +// Decode Go's VEX _yvmovmskpd family before x/arch can mistake opcode 50/D7 +// for a legacy PUSH/XLATB. The source is always a vector register; ModRM.reg +// names the GP destination and vvvv is reserved. +func decodedX86VEXMoveMaskInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.evex || p.mapNumber != 1 { + return Instr{}, 0, false, nil + } + var op Op + for name, spec := range amd64MoveMaskSpecs { + if spec.vector && spec.opcode == p.opcode && spec.prefix == p.pp { + op = name + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VEX move mask: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 || p.vectorLength > 1 { + return fail("reserved vvvv or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if code[p.modRM]>>6 != 3 { + return fail("source must be a vector register") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + width := "X" + if p.vectorLength == 1 { + width = "Y" + } + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]&7)+p.b*8))} + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + destinationReg, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return fail("invalid GP destination register") + } + destination := Operand{Kind: OpReg, Reg: destinationReg} + return Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + }, p.modRM + 1, true, nil +} diff --git a/amd64_raw_movemask_test.go b/amd64_raw_movemask_test.go new file mode 100644 index 000000000..10ade7c72 --- /dev/null +++ b/amd64_raw_movemask_test.go @@ -0,0 +1,185 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +var x86RawMoveMaskForms = []struct { + name string + code []byte + op Op + src Reg + dst Reg +}{ + {"PMOVMSKB/MMX", []byte{0x0f, 0xd7, 0xc1}, "PMOVMSKB", "M1", AX}, + {"PMOVMSKB/X", []byte{0x66, 0x0f, 0xd7, 0xc1}, "PMOVMSKB", "X1", AX}, + {"MOVMSKPS/X", []byte{0x0f, 0x50, 0xc1}, "MOVMSKPS", "X1", AX}, + {"MOVMSKPD/X", []byte{0x66, 0x0f, 0x50, 0xc1}, "MOVMSKPD", "X1", AX}, + {"VPMOVMSKB/X", []byte{0xc5, 0xf9, 0xd7, 0xc1}, "VPMOVMSKB", "X1", AX}, + {"VPMOVMSKB/Y", []byte{0xc5, 0xfd, 0xd7, 0xc1}, "VPMOVMSKB", "Y1", AX}, + {"VMOVMSKPS/X", []byte{0xc5, 0xf8, 0x50, 0xc1}, "VMOVMSKPS", "X1", AX}, + {"VMOVMSKPS/Y", []byte{0xc5, 0xfc, 0x50, 0xc1}, "VMOVMSKPS", "Y1", AX}, + {"VMOVMSKPD/X", []byte{0xc5, 0xf9, 0x50, 0xc1}, "VMOVMSKPD", "X1", AX}, + {"VMOVMSKPD/Y", []byte{0xc5, 0xfd, 0x50, 0xc1}, "VMOVMSKPD", "Y1", AX}, +} + +func TestX86RawMoveMaskCompleteGo127Family(t *testing.T) { + if len(amd64MoveMaskSpecs) != 6 { + t.Fatalf("move-mask grammar has %d operations, want six", len(amd64MoveMaskSpecs)) + } + for _, goarch := range []string{"386", "amd64"} { + for _, form := range x86RawMoveMaskForms { + t.Run(goarch+"/"+form.name, func(t *testing.T) { + fn := Func{} + for _, value := range form.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != form.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", form.code, got.Instrs, form.op) + } + args := got.Instrs[0].Args + if args[0].Kind != OpReg || args[0].Reg != form.src || args[1].Kind != OpReg || args[1].Reg != form.dst { + t.Fatalf("decoded %#x as %#v, want %s, %s", form.code, args, form.src, form.dst) + } + }) + } + } +} + +func TestX86RawMoveMaskRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"reserved-vvvv", []byte{0xc5, 0xe9, 0xd7, 0xc1}, 64}, + {"memory-source", []byte{0xc5, 0xfd, 0xd7, 0x01}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xfd, 0xd7, 0xc1}, 64}, + {"missing-modrm", []byte{0xc5, 0xfd, 0xd7}, 64}, + {"extended-386", []byte{0xc4, 0x61, 0x7d, 0xd7, 0xc1}, 32}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86VEXMoveMaskInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } + for _, tc := range []struct { + name string + code []byte + src Reg + dst Reg + }{ + {"W-ignored", []byte{0xc4, 0xe1, 0xfd, 0xd7, 0xc1}, "Y1", AX}, + {"extended-vector-and-GP", []byte{0xc4, 0x41, 0x7d, 0xd7, 0xc8}, "Y8", "R9"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86VEXMoveMaskInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) { + t.Fatalf("encoding %#x: instruction=%+v length=%d recognized=%v error=%v", tc.code, got, length, recognized, err) + } + if got.Args[0].Reg != tc.src || got.Args[1].Reg != tc.dst { + t.Fatalf("encoding %#x operands = %+v, want %s, %s", tc.code, got.Args, tc.src, tc.dst) + } + }) + } +} + +func TestX86RawMoveMaskCompilesEveryGoForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawmovemaskforms(SB),4,$0-0\n") + for _, form := range x86RawMoveMaskForms { + for _, value := range form.code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawmovemaskforms": {Name: "rawmovemaskforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-movemask.ll", "raw-movemask.o", ir) + }) + } +} + +func TestX86RawMoveMaskRuntime(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := x86RawMoveMaskOriginalRuntimeSource + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawmovemasksemantics": { + Name: "rawmovemasksemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void rawmovemasksemantics(const uint8_t *, uint32_t *); +int main(void) { + uint8_t input[32] = {0}; + uint32_t out[3] = {0}; + input[3] = 0x80; input[7] = 0x80; input[15] = 0x80; input[31] = 0x80; + rawmovemasksemantics(input, out); + if (out[0] != ((1u << 3) | (1u << 7) | (1u << 15) | (1u << 31))) return 1; + if (out[1] != ((1u << 0) | (1u << 1) | (1u << 3))) return 2; + if (out[2] != ((1u << 0) | (1u << 1) | (1u << 3))) return 3; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_move_mask", triple, ir, mainC, runPrefix) +} diff --git a/amd64_raw_movhlps.go b/amd64_raw_movhlps.go new file mode 100644 index 000000000..d5332dbde --- /dev/null +++ b/amd64_raw_movhlps.go @@ -0,0 +1,65 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var x86RawVectorHighLowMoveOps = map[int]Op{ + 0x12: "VMOVHLPS", + 0x16: "VMOVLHPS", +} + +// The Go 1.27 _yvmovhlps table has exactly two register-only 128-bit rows. +// ModRM memory encodings belong to the separate VMOVLPS/VMOVHPS family. +func decodedX86RawVectorHighLowMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || p.mapNumber != 1 || p.pp != 0 { + return Instr{}, 0, false, nil + } + op, owned := x86RawVectorHighLowMoveOps[p.opcode] + if !owned { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("vector high/low move: %s", message) + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if code[p.modRM]>>6 != 3 { + return Instr{}, 0, false, nil + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.vectorLength != 0 || p.evex && (!p.fixed || p.w || p.mask != 0 || p.zero || p.broadcast) { + return fail("invalid 128-bit VEX/EVEX register form") + } + if !p.evex && mode == 32 && (p.r != 0 || p.upper >= 8) { + return fail("extended VEX register in 32-bit mode") + } + var first Operand + var consumed int + var err error + if p.evex { + first, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", 16) + } else { + first, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))} + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + args := []Operand{first, second, destination} + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_movhlps_test.go b/amd64_raw_movhlps_test.go new file mode 100644 index 000000000..a3a50f8e3 --- /dev/null +++ b/amd64_raw_movhlps_test.go @@ -0,0 +1,176 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func encodeX86RawVectorHighLowMove(opcode byte, encoding string, first, second, destination int, wig bool) []byte { + modRM := byte(0xc0 | (destination&7)<<3 | first&7) + switch encoding { + case "VEX2": + p1 := byte((1-destination/8)<<7 | (^second&15)<<3) + return []byte{0xc5, p1, opcode, modRM} + case "VEX3": + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-first/8)<<5 | 1) + p1 := byte((^second & 15) << 3) + if wig { + p1 |= 0x80 + } + return []byte{0xc4, p0, p1, opcode, modRM} + case "EVEX": + p0 := byte((1-destination/8&1)<<7 | (1-first/16)<<6 | + (1-first/8&1)<<5 | (1-destination/16)<<4 | 1) + p1 := byte(4 | (^second&15)<<3) + p2 := byte((1 - second/16) << 3) + return []byte{0x62, p0, p1, p2, opcode, modRM} + default: + panic("unknown high/low move encoding") + } +} + +func TestTranslateRawVectorHighLowMoveGorseRegression(t *testing.T) { + const source = `TEXT rawMoveLowToHigh(SB), $0-0 + LONG $0xc016f0c5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawMoveLowToHigh": {Name: "rawMoveLowToHigh", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "shufflevector <2 x i64>") { + t.Fatalf("raw VMOVLHPS omitted half-lane move:\n%s", ir) + } +} + +func TestDecodedX86RawVectorHighLowMoveGoRowsAndEncodings(t *testing.T) { + if len(x86RawVectorHighLowMoveOps) != 2 || + x86RawVectorHighLowMoveOps[0x12] != "VMOVHLPS" || + x86RawVectorHighLowMoveOps[0x16] != "VMOVLHPS" { + t.Fatal("Go 1.27 _yvmovhlps raw opcode grammar changed") + } + for opcode, op := range x86RawVectorHighLowMoveOps { + for _, test := range []struct { + encoding string + first, second, dest int + wig bool + }{ + {"VEX2", 1, 2, 3, false}, + {"VEX3", 12, 13, 14, false}, + {"VEX3", 12, 13, 14, true}, // VEX.W is ignored by the hardware. + {"EVEX", 20, 21, 22, false}, + } { + code := encodeX86RawVectorHighLowMove(byte(opcode), test.encoding, test.first, test.second, test.dest, test.wig) + got, length, recognized, err := decodedX86RawVectorHighLowMoveInstruction(code, 64) + if err != nil || !recognized || length != len(code) || got.Op != op { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + want := []string{ + fmt.Sprintf("X%d", test.first), + fmt.Sprintf("X%d", test.second), + fmt.Sprintf("X%d", test.dest), + } + if len(got.Args) != len(want) { + t.Fatalf("decode %x = %+v, want %v", code, got, want) + } + for index, operand := range got.Args { + if operand.String() != want[index] { + t.Fatalf("decode %x = %+v, want %v", code, got, want) + } + } + } + } +} + +func TestDecodedX86RawVectorHighLowMoveRejectsNonGoForms(t *testing.T) { + base := encodeX86RawVectorHighLowMove(0x16, "EVEX", 1, 2, 3, false) + for _, test := range []struct { + name string + code []byte + }{ + {"fixed bit", []byte{base[0], base[1], base[2] &^ 4, base[3], base[4], base[5]}}, + {"EVEX W", []byte{base[0], base[1], base[2] | 0x80, base[3], base[4], base[5]}}, + {"vector length", []byte{base[0], base[1], base[2], base[3] | 0x20, base[4], base[5]}}, + {"mask", []byte{base[0], base[1], base[2], base[3] | 1, base[4], base[5]}}, + {"zero", []byte{base[0], base[1], base[2], base[3] | 0x80, base[4], base[5]}}, + {"broadcast", []byte{base[0], base[1], base[2], base[3] | 0x10, base[4], base[5]}}, + {"missing ModRM", base[:5]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawVectorHighLowMoveInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + memory := append([]byte(nil), base...) + memory[5] = 0x43 + if _, _, recognized, err := decodedX86RawVectorHighLowMoveInstruction(memory, 64); recognized || err != nil { + t.Fatalf("claimed VMOVHPS memory encoding %x: recognized=%v err=%v", memory, recognized, err) + } +} + +func TestTranslateRawVectorHighLowMoveLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawHighLowMoveFamily(SB), $0-0\n") + for opcode := range x86RawVectorHighLowMoveOps { + for _, code := range [][]byte{ + encodeX86RawVectorHighLowMove(byte(opcode), "VEX2", 1, 2, 3, false), + encodeX86RawVectorHighLowMove(byte(opcode), "VEX3", 4, 5, 6, true), + encodeX86RawVectorHighLowMove(byte(opcode), "EVEX", 20, 21, 22, false), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + x86LEAActualTargetGoObject(t, target.goarch, target.triple, source.String()) + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawHighLowMoveFamily": {Name: "rawHighLowMoveFamily", Ret: Void}}, + }) + if target.goarch == "386" { + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("original high-bank whole source acquired mode32 numerical authority", err) + } + return + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-high-low-move.ll", "raw-high-low-move.o", ir) + }) + } +} diff --git a/amd64_raw_movsxd.go b/amd64_raw_movsxd.go new file mode 100644 index 000000000..7cfede571 --- /dev/null +++ b/amd64_raw_movsxd.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// Opcode 63 is ARPL outside 64-bit mode. In 64-bit mode its default +// operation is a 32-bit copy, 66 selects a 16-bit copy, and effective +// REX.W selects a 32-to-64-bit sign extension. See Intel SDM volume 2B, +// MOVSX/MOVSXD. A 16-bit memory operand remains Context: Intel's r/m16 +// and AMD/x86asm's r/m32 descriptions do not establish one read extent. +func decodedX86MOVSXD(inst x86asm.Inst, encoding []byte) (string, error) { + fail := func(reason string) (string, error) { + return "", fmt.Errorf("%w: raw MOVSXD %s", ErrProbeNeedsContext, reason) + } + if inst.Op != x86asm.MOVSXD || inst.Mode != 64 || inst.Len != len(encoding) { + return fail("requires its complete 64-bit-mode physical encoding") + } + physical, decodeErr := x86asm.Decode(encoding, 64) + if decodeErr != nil || physical != inst { + return fail("does not match the independently decoded original bytes") + } + // Bind the effective size and register extension to actual bytes, not a + // public DataSize or a widthless mnemonic. Deliberately leave unregistered + // ignored/repeated/segment/address prefixes outside this closed grammar. + index, bits := 0, 32 + if index < len(encoding) && encoding[index] == 0x66 { + bits = 16 + index++ + } + if index < len(encoding) && encoding[index]&0xf0 == 0x40 { + if encoding[index]&8 != 0 { + bits = 64 + } + index++ + } + if index+1 >= len(encoding) || encoding[index] != 0x63 || inst.DataSize != bits || inst.AddrSize != 64 { + return fail("has an unregistered prefix, opcode or effective size") + } + destination, register := inst.Args[0].(x86asm.Reg) + if !register || decodedX86RegisterBits(destination) != bits || inst.Args[2] != nil || inst.Args[3] != nil { + return fail("has inconsistent destination or extra operands") + } + switch source := inst.Args[1].(type) { + case x86asm.Reg: + // The pinned decoder describes every 63 source as r/m32, including + // 66 register forms. The latter consume only its low sixteen bits. + if decodedX86RegisterBits(source) != 32 || encoding[index+1]&0xc0 != 0xc0 { + return fail("has inconsistent register source or ModRM") + } + if bits == 16 { + inst.Args[1] = x86asm.AX + (source - x86asm.EAX) + } + case x86asm.Mem: + if encoding[index+1]&0xc0 == 0xc0 || inst.MemBytes != 4 || source.Segment != 0 { + return fail("has inconsistent memory source or native read extent") + } + if bits == 16 { + return fail("16-bit memory needs a source-bound vendor read-width contract") + } + default: + return fail("requires a register or memory source") + } + if bits == 64 { + // Go has a width-specific name for the genuine sign extension. + syntax := x86asm.GoSyntax(inst, 0, nil) + return "MOVLQSX" + syntax[len("MOVSXD"):], nil + } + inst.Op = x86asm.MOV + if bits == 16 { + inst.MemBytes = 2 + } + return x86asm.GoSyntax(inst, 0, nil), nil +} + +func x86RawDwordLiteralInput(ins Instr) bool { + // This is only the source-stack transfer's input-availability check. The + // final consumer must still bind the exact bytes to the allocated DATA + // object through x86RawReturnDecodedReadBound; no GP input is inferred. + read := ins.x86RIPMemoryRead + if ins.Op != "MOVL" && ins.Op != "MOVLQSX" || read.source != 0 || read.width != 4 || len(ins.Args) != 2 || + ins.Args[1].Kind != OpReg || !x86RawReturnDecodedReadIdentity(ins) { + return false + } + result, valid := x86CallerGP(ins.Args[1].Reg) + return valid && result != SP && result != BP && result == ins.Args[1].Reg +} diff --git a/amd64_raw_movsxd_runtime_test.go b/amd64_raw_movsxd_runtime_test.go new file mode 100644 index 000000000..f4cf66e9f --- /dev/null +++ b/amd64_raw_movsxd_runtime_test.go @@ -0,0 +1,325 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86MOVSXDNumericalForm struct { + name string + encoding []byte + bits int + offset int + setup string + result string + inPlace bool + flagSeed int +} + +func x86MOVSXDNumericalForms() []x86MOVSXDNumericalForm { + forms := []x86MOVSXDNumericalForm{ + {name: "wordBX", encoding: []byte{0x66, 0x63, 0xc3}, bits: 16}, + {name: "wordSI", encoding: []byte{0x66, 0x63, 0xc6}, bits: 16, setup: "MOVQ BX,SI\n"}, + {name: "wordR9", encoding: []byte{0x66, 0x41, 0x63, 0xc1}, bits: 16, setup: "MOVQ BX,R9\n"}, + {name: "wordR8", encoding: []byte{0x66, 0x44, 0x63, 0xc3}, bits: 16, result: "MOVQ R8,AX\n"}, + {name: "wordAlias", encoding: []byte{0x66, 0x63, 0xc0}, bits: 16, setup: "MOVQ BX,AX\n", inPlace: true}, + {name: "longBX", encoding: []byte{0x63, 0xc3}, bits: 32}, + {name: "longR9", encoding: []byte{0x41, 0x63, 0xc1}, bits: 32, setup: "MOVQ BX,R9\n"}, + {name: "longR8", encoding: []byte{0x44, 0x63, 0xc3}, bits: 32, result: "MOVQ R8,AX\n"}, + {name: "longAlias", encoding: []byte{0x63, 0xc0}, bits: 32, setup: "MOVQ BX,AX\n", inPlace: true}, + {name: "longMemory", encoding: []byte{0x63, 0x07}, bits: 32}, + {name: "longUnaligned", encoding: []byte{0x63, 0x47, 0x01}, bits: 32, offset: 1}, + {name: "longOffset4", encoding: []byte{0x63, 0x47, 0x04}, bits: 32, offset: 4}, + {name: "longIndexed", encoding: []byte{0x63, 0x04, 0x8f}, bits: 32, setup: "MOVQ $0,CX\n"}, + {name: "quadBX", encoding: []byte{0x48, 0x63, 0xc3}, bits: 64}, + {name: "quadR9", encoding: []byte{0x49, 0x63, 0xc1}, bits: 64, setup: "MOVQ BX,R9\n"}, + {name: "quadR8", encoding: []byte{0x4c, 0x63, 0xc3}, bits: 64, result: "MOVQ R8,AX\n"}, + {name: "quadAlias", encoding: []byte{0x48, 0x63, 0xc0}, bits: 64, setup: "MOVQ BX,AX\n", inPlace: true}, + {name: "quadMemory", encoding: []byte{0x48, 0x63, 0x07}, bits: 64}, + {name: "quadUnaligned", encoding: []byte{0x48, 0x63, 0x47, 0x01}, bits: 64, offset: 1}, + {name: "quadOffset4", encoding: []byte{0x48, 0x63, 0x47, 0x04}, bits: 64, offset: 4}, + {name: "quadIndexed", encoding: []byte{0x48, 0x63, 0x04, 0x8f}, bits: 64, setup: "MOVQ $0,CX\n"}, + {name: "quadOperandOverride", encoding: []byte{0x66, 0x48, 0x63, 0x07}, bits: 64}, + } + var seeded []x86MOVSXDNumericalForm + for _, form := range forms { + for seed, suffix := range []string{"", "Carry", "Overflow"} { + variant := form + variant.name += suffix + variant.flagSeed = seed + seeded = append(seeded, variant) + } + } + return seeded +} + +func x86MOVSXDFlagSeed(seed int) string { + switch seed { + case 1: + // 0-1: CF=1, OF=0, SF=1, PF=1, ZF=0. + return "MOVQ $0,R15\nCMPQ R15,$1\n" + case 2: + // INT64_MIN-1: CF=0, OF=1, SF=0, PF=1, ZF=0. + return "MOVQ $0x8000000000000000,R15\nCMPQ R15,$1\n" + default: + return "CMPQ BX,$0\n" + } +} + +func x86MOVSXDNumericalSource() (string, string) { + var declarations, assembly strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + for _, form := range x86MOVSXDNumericalForms() { + declaration := strings.ReplaceAll(x86CallerFixtureSource, "float32", "uint64") + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", form.name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+form.name+"(", 1) + declarations.WriteString(declaration) + body := "MOVQ (DI),BX\nMOVQ $0x5a6b7c8d9eaf1021,AX\nMOVQ AX,R8\n" + form.setup + body += x86MOVSXDFlagSeed(form.flagSeed) + x86MOVSXDBytes(form.encoding) + form.result + // Neutral MOVs capture each modeled flag before any arithmetic. The + // branch joins preserve definite numeric bytes, not guessed values. + body += `MOVQ $0,R9 +MOVQ $0,R10 +MOVQ $0,R11 +MOVQ $0,R12 +MOVQ $0,R13 +JCC noCarry +MOVQ $1,R9 +noCarry: +JOC noOverflow +MOVQ $2,R10 +noOverflow: +JPL noSign +MOVQ $4,R11 +noSign: +JPC noParity +MOVQ $8,R12 +noParity: +JNE noZero +MOVQ $16,R13 +noZero: +JE done +ADDQ $7,AX +done: +ORQ R9,R10 +ORQ R11,R10 +ORQ R12,R10 +ORQ R13,R10 +SHLQ $48,R10 +XORQ R10,AX +MOVQ AX,(DX)` + machine := strings.ReplaceAll(x86CallerFixtureAssembly, "·kernel", "·"+form.name) + assembly.WriteString(strings.Replace(machine, "MOVL $7,AX\nMOVL AX,(DX)", body, 1)) + } + return declarations.String(), assembly.String() +} + +func x86MOVSXDGoOracle() string { + var source strings.Builder + source.WriteString(`package caller +import "testing" +func reference(input uint64, bits, offset uint, inPlace bool, flagSeed uint) uint64 { + value := uint64(uint32(input >> (offset*8))) + if bits == 16 { + seed := uint64(0x5a6b7c8d9eaf1021) + if inPlace { seed = input } + value = seed & ^uint64(65535) | input & 65535 + } else if bits == 64 { + value = uint64(int64(int32(value))) + } + if input != 0 || flagSeed != 0 { value += 7 } + flags := uint64(0) + if input>>63 != 0 { flags |= 4 } + var ones uint + for byteValue := uint8(input); byteValue != 0; byteValue >>= 1 { ones += uint(byteValue&1) } + if ones%2 == 0 { flags |= 8 } + if input == 0 { flags |= 16 } + if flagSeed == 1 { flags = 1 | 4 | 8 } + if flagSeed == 2 { flags = 2 | 8 } + return value ^ (flags<<48) +} +func TestOriginalMOVSXDNumerical(t *testing.T) { +`) + for _, form := range x86MOVSXDNumericalForms() { + fmt.Fprintf(&source, ` + for iteration := 0; iteration < 16; iteration++ { + for _, input := range []uint64{0, 1, 127, 128, 65535, 1<<31, 1<<63, ^uint64(0)} { + x := []uint64{input, 0x1122334455667788} + y := []uint64{0x8877665544332211, 23} + canary := [4]uint64{11, 23, 37, 41} + got := Call%[1]s(x, y) + want := reference(input, %[2]d, %[3]d, %[4]t, %[5]d) + if got != want || x[0] != input || x[1] != 0x1122334455667788 || + y[0] != 0x8877665544332211 || y[1] != 23 || canary != [4]uint64{11,23,37,41} { + t.Fatalf("%[1]s got=%%#x want=%%#x input=%%#x canary=%%v", got, want, input, canary) + } + } + } +`, form.name, form.bits, form.offset, form.inPlace, form.flagSeed) + } + source.WriteString("}\n") + return source.String() +} + +func x86MOVSXDActualGo(t *testing.T, execute bool) { + t.Helper() + declaration, assembly := x86MOVSXDNumericalSource() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module caller\n\ngo 1.20\n", "value.go": declaration, + "value_amd64.s": assembly, "value_test.go": x86MOVSXDGoOracle(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + binary := filepath.Join(dir, "original.test") + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-c", "-o", binary, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go original raw MOVSXD build: %v\n%s", err, output) + } + if !execute { + return // Object/Go entry transport evidence is not runtime success. + } + command = exec.CommandContext(ctx, binary, "-test.v", "-test.timeout=2m") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("original Linux/amd64 MOVSXD mathematical/canary oracle: %v\n%s", err, output) + } +} + +func x86MOVSXDCOracle() string { + var declarations, checks strings.Builder + for index, form := range x86MOVSXDNumericalForms() { + fmt.Fprintf(&declarations, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", form.name, form.name) + fmt.Fprintf(&checks, " if (check(%s, %d, %d, %t, %d)) return %d;\n", + form.name, form.bits, form.offset, form.inPlace, form.flagSeed, index+1) + } + return `#include +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +` + declarations.String() + ` +static uint64_t reference(uint64_t input, unsigned bits, unsigned offset, bool in_place, unsigned flag_seed) { + uint64_t value = (uint32_t)(input >> (offset*8)); + if (bits == 16) { + uint64_t seed = in_place ? input : UINT64_C(0x5a6b7c8d9eaf1021); + value = (seed & ~UINT64_C(65535)) | (input & UINT64_C(65535)); + } else if (bits == 64) { + value = (uint64_t)(int64_t)(int32_t)value; + } + value += input != 0 || flag_seed != 0 ? 7 : 0; + uint64_t flags = input >> 63 ? 4 : 0; + unsigned ones = 0; + for (unsigned byte_value = (unsigned char)input; byte_value; byte_value >>= 1) ones += byte_value & 1; + if (!(ones & 1)) flags |= 8; + if (!input) flags |= 16; + if (flag_seed == 1) flags = 1 | 4 | 8; + if (flag_seed == 2) flags = 2 | 8; + return value ^ (flags << 48); +} +static int check(void (*fn)(void*,void*,void*,void*), unsigned bits, unsigned offset, bool in_place, unsigned flag_seed) { + const uint64_t inputs[] = {0, 1, 127, 128, 65535, UINT64_C(1)<<31, UINT64_C(1)<<63, UINT64_MAX}; + for (unsigned iteration = 0; iteration < 16; iteration++) { + for (unsigned index = 0; index < sizeof(inputs)/sizeof(inputs[0]); index++) { + uint64_t x[] = {inputs[index], UINT64_C(0x1122334455667788)}; + uint64_t y[] = {UINT64_C(0x8877665544332211), 23}, length = 2; + unsigned char output[10]; + memset(output, 0xa5, sizeof(output)); + fn(x, y, output+1, &length); + uint64_t got; + memcpy(&got, output+1, sizeof(got)); + if (got != reference(inputs[index], bits, offset, in_place, flag_seed) || + output[0] != 0xa5 || output[9] != 0xa5 || length != 2 || + x[0] != inputs[index] || x[1] != UINT64_C(0x1122334455667788) || + y[0] != UINT64_C(0x8877665544332211) || y[1] != 23) return 1; + } + } + return 0; +} +int main(void) { +` + checks.String() + " return 0;\n}\n" +} + +func x86MOVSXDLLVMNumerical(t *testing.T, runner []string) { + t.Helper() + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang required") + } + declaration, assembly := x86MOVSXDNumericalSource() + pkg := x86CallerRuntimePackage(t, declaration, assembly) + file := pkg.SourceSelection.Assembly["value_amd64.s"] + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, "amd64") + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "movsxd-text", triple, ir, x86MOVSXDCOracle(), runner) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "movsxd-module", triple, module.String(), x86MOVSXDCOracle(), runner) + ctx = llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, ResolveSym: resolve, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "movsxd-go-module", triple, translation.Module.String(), x86MOVSXDCOracle(), runner) +} + +func TestX86RawMOVSXDOriginalGoBuild(t *testing.T) { + x86MOVSXDActualGo(t, false) +} + +func TestX86RawMOVSXDLLVMHostNumerical(t *testing.T) { + var runner []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + runner = []string{"/usr/bin/arch", "-x86_64"} + t.Log("LLVM numerical diagnostic uses Rosetta; the original no-REX 63 environment failure is retained separately") + } else if runtime.GOARCH != "amd64" { + t.Skip("requires the mandatory Linux/amd64 original-Go plus LLVM counterpart") + } + x86MOVSXDLLVMNumerical(t, runner) +} + +func TestCrossLinuxRuntimeMatrixAMD64MOVSXDNativeWidths(t *testing.T) { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Skip("required actual Linux/amd64 original-Go and LLVM counterpart; Darwin/Rosetta is not equivalent") + } + x86MOVSXDActualGo(t, true) + x86MOVSXDLLVMNumerical(t, nil) +} diff --git a/amd64_raw_movsxd_test.go b/amd64_raw_movsxd_test.go new file mode 100644 index 000000000..f4e159f69 --- /dev/null +++ b/amd64_raw_movsxd_test.go @@ -0,0 +1,464 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +func x86MOVSXDBytes(encoding []byte) string { + var source strings.Builder + for _, value := range encoding { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + return source.String() +} + +func TestX86RawMOVSXDNativeWidths(t *testing.T) { + for _, test := range []struct { + name string + encoding []byte + want string + }{ + {"long register", []byte{0x63, 0xc3}, "MOVL BX, AX"}, + {"long high source", []byte{0x41, 0x63, 0xc1}, "MOVL R9, AX"}, + {"long high result", []byte{0x44, 0x63, 0xc3}, "MOVL BX, R8"}, + {"long alias", []byte{0x63, 0xc0}, "MOVL AX, AX"}, + {"long memory", []byte{0x63, 0x07}, "MOVL 0(DI), AX"}, + {"long indexed", []byte{0x63, 0x44, 0x8f, 0x03}, "MOVL 0x3(DI)(CX*4), AX"}, + {"word register", []byte{0x66, 0x63, 0xc3}, "MOVW BX, AX"}, + {"word physical SI", []byte{0x66, 0x63, 0xc6}, "MOVW SI, AX"}, + {"word high source", []byte{0x66, 0x41, 0x63, 0xc1}, "MOVW R9, AX"}, + {"word high result", []byte{0x66, 0x44, 0x63, 0xc3}, "MOVW BX, R8"}, + {"quad register", []byte{0x48, 0x63, 0xc3}, "MOVLQSX BX, AX"}, + {"quad memory", []byte{0x48, 0x63, 0x07}, "MOVLQSX 0(DI), AX"}, + } { + t.Run(test.name, func(t *testing.T) { + inst, err := x86asm.Decode(test.encoding, 64) + if err != nil || inst.Op != x86asm.MOVSXD || inst.Len != len(test.encoding) { + t.Fatalf("physical decode: %+v %v", inst, err) + } + got, err := decodedX86GoSyntax(inst, test.encoding) + if err != nil || got != test.want { + t.Fatalf("physical width: got %q, want %q: %v", got, test.want, err) + } + }) + } +} + +func TestX86RawMOVSXDLongCallerOriginalBytes(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX\nMOVL AX,(DX)", + "MOVQ $0x80000000,BX\nWORD $0xc363\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } +} + +func TestX86RawMOVSXDCompleteRegisterModRM(t *testing.T) { + registers := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + for _, bits := range []int{16, 32, 64} { + for result := range registers { + for source := range registers { + var encoding []byte + if bits == 16 { + encoding = append(encoding, 0x66) + } + rex := byte(0x40) + if bits == 64 { + rex |= 8 + } + if result >= 8 { + rex |= 4 + } + if source >= 8 { + rex |= 1 + } + if rex != 0x40 { + encoding = append(encoding, rex) + } + encoding = append(encoding, 0x63, byte(0xc0+(result%8)*8+source%8)) + inst, err := x86asm.Decode(encoding, 64) + if err != nil { + t.Fatal(err) + } + before := inst + got, err := decodedX86GoSyntax(inst, encoding) + op := map[int]string{16: "MOVW", 32: "MOVL", 64: "MOVLQSX"}[bits] + want := fmt.Sprintf("%s %s, %s", op, registers[source], registers[result]) + if err != nil || got != want || !reflect.DeepEqual(inst, before) { + t.Fatalf("bits=%d bytes=%x: got %q want %q changed=%t: %v", bits, encoding, got, want, inst != before, err) + } + } + } + } +} + +func x86MOVSXDDenied(t *testing.T, source string) { + t.Helper() + requireX86GoAssemblerResult(t, "amd64", source, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, source) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved raw MOVSXD Translate: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved raw MOVSXD module: %v", err) + } + if err := x86CallerCoreTranslate(t, pkg, source, triple); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved raw MOVSXD Go module: %v", err) + } + } +} + +func TestX86RawMOVSXDInputAndVendorBoundaries(t *testing.T) { + for _, test := range []struct{ name, body string }{ + {"undefined long source", "WORD $0xc363"}, + {"partial long source", "MOVW $7,BX\nWORD $0xc363"}, + {"undefined word source", "BYTE $0x66\nWORD $0xc363"}, + {"partial word source", "MOVB $7,BX\nBYTE $0x66\nWORD $0xc363"}, + {"caller pointer long source", "WORD $0xc763"}, + {"caller pointer word source", "BYTE $0x66\nWORD $0xc763"}, + {"word overwrites surviving pointer bits", "MOVQ DI,AX\nBYTE $0x66\nWORD $0xc363"}, + {"native SP source", "WORD $0xc463"}, + {"native BP source", "BYTE $0x66\nWORD $0xc563"}, + {"native SP result", "WORD $0xe363"}, + {"native BP result", "BYTE $0x66\nWORD $0xeb63"}, + {"g source", "BYTE $0x41\nWORD $0xc663"}, + {"g result", "BYTE $0x44\nWORD $0xf363"}, + {"unknown memory", "WORD $0x0363"}, + {"partial FP pointer slot", "WORD $0x4463\nWORD $0x0824"}, + {"return PC memory", "WORD $0x0463\nBYTE $0x24"}, + {"word memory vendor identity absent", "BYTE $0x66\nWORD $0x0763"}, + {"word indexed vendor identity absent", "BYTE $0x66\nLONG $0x038f4463"}, + {"address-size memory", "BYTE $0x67\nWORD $0x0763"}, + {"different initialized bytes join", "CMPQ AX,$0\nJE other\nMOVL $7,BX\nother:\nWORD $0xc363"}, + } { + t.Run(test.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", test.body+"\nMOVL AX,(DX)", 1) + x86MOVSXDDenied(t, assembly) + }) + } +} + +func TestX86RawMOVSXDRequiresExactEncodingIdentity(t *testing.T) { + encoding := []byte{0x63, 0xc3} + physical, err := x86asm.Decode(encoding, 64) + if err != nil { + t.Fatal(err) + } + for _, change := range []string{"source", "result", "width", "mode", "extra operand", "bytes", "truncated", "extended"} { + t.Run(change, func(t *testing.T) { + inst, bytes := physical, append([]byte(nil), encoding...) + switch change { + case "source": + inst.Args[1] = x86asm.ECX + case "result": + inst.Args[0] = x86asm.ECX + case "width": + inst.DataSize = 64 + case "mode": + inst.Mode = 32 + case "extra operand": + inst.Args[2] = x86asm.EAX + case "bytes": + bytes[1] = 0xc1 + case "truncated": + bytes = bytes[:1] + case "extended": + bytes = append(bytes, 0x90) + } + if _, err := decodedX86GoSyntax(inst, bytes); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mismatched native MOVSXD width acquired permission: %v", err) + } + }) + } +} + +func TestX86RawMOVSXDNoncallerDoesNotAcquireInputProof(t *testing.T) { + for _, prefix := range []string{"", "MOVW $7,BX\n", "MOVQ x+0(FP),BX\n"} { + for _, raw := range []string{"WORD $0xc363", "BYTE $0x66\nWORD $0xc363"} { + source, sig, pkg := x86SourceStackNumericalFixture("amd64", prefix+raw+"\nMOVQ AX,ret+8(FP)\nBYTE $0xc3") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig) + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller raw scalar width acquired input proof: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller raw scalar module acquired input proof: %v", err) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + }) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller raw scalar Go module acquired input proof: %v", err) + } + } + } +} + +func TestX86RawMOVSXDLegacyModeRemainsARPL(t *testing.T) { + for _, encoding := range [][]byte{{0x63, 0xc3}, {0x66, 0x63, 0xc3}} { + inst, err := x86asm.Decode(encoding, 32) + if err != nil || inst.Op != x86asm.ARPL { + t.Fatalf("legacy mode opcode identity: %+v %v", inst, err) + } + syntax, err := decodedX86GoSyntax(inst, encoding) + if err != nil || !strings.HasPrefix(syntax, "ARPL ") { + t.Fatalf("legacy ARPL must not become a numerical copy: %q %v", syntax, err) + } + } +} + +func TestX86RawMOVSXDUnregisteredPrefixesStayContext(t *testing.T) { + for _, encoding := range [][]byte{ + {0x66, 0x66, 0x63, 0xc3}, {0x40, 0x41, 0x63, 0xc3}, + {0xf2, 0x63, 0xc3}, {0xf3, 0x63, 0xc3}, + {0x67, 0x63, 0x07}, {0x64, 0x63, 0x07}, {0x65, 0x48, 0x63, 0x07}, + } { + inst, err := x86asm.Decode(encoding, 64) + if err != nil || inst.Op != x86asm.MOVSXD { + continue // Decoder rejection is also closed; it is not object/runtime P. + } + if _, err := decodedX86GoSyntax(inst, encoding); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unregistered physical prefix %x was silently discarded: %v", encoding, err) + } + } + if inst, err := x86asm.Decode([]byte{0xf0, 0x63, 0xc3}, 64); err == nil && inst.Op == x86asm.MOVSXD { + if _, err := decodedX86GoSyntax(inst, []byte{0xf0, 0x63, 0xc3}); err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("machine-invalid LOCK MOVSXD was accepted or misclassified as missing context", err) + } + } +} + +func x86MOVSXDModuleObject(t *testing.T, file *File, options Options, triple, name string) { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, name+".ll", name+".o", module.String()) +} + +func x86MOVSXDGoModuleObject(t *testing.T, pkg GoPackage, source, triple, name string) { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: triple, + }) + if translation != nil { + defer translation.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, name+".ll", name+".o", translation.Module.String()) +} + +func TestX86RawMOVSXDSafeFormatsThreeAPIObjects(t *testing.T) { + for _, body := range []string{ + "MOVL $0x80000000,BX\nWORD $0xc363", + "MOVL $0x80000000,R9\nBYTE $0x41\nWORD $0xc163", + "MOVL $0x80000000,BX\nBYTE $0x44\nWORD $0xc363\nMOVL R8,AX", + "MOVL $0x80000000,AX\nWORD $0xc063", + "WORD $0x0763", "WORD $0x4763\nBYTE $1", "MOVL $0,CX\nLONG $0x038f4463", + "MOVL $0,CX\nLONG $0x008f4463", + "MOVQ $0x1122334455667788,AX\nMOVW $0x8000,BX\nBYTE $0x66\nWORD $0xc363", + "MOVQ $0x1122334455667788,AX\nMOVQ $0x8000,SI\nBYTE $0x66\nWORD $0xc663", + "MOVQ $0x1122334455667788,AX\nMOVW $0x8000,R9\nWORD $0x4166\nWORD $0xc163", + "MOVL $0x80000000,BX\nBYTE $0x48\nWORD $0xc363", + "BYTE $0x48\nWORD $0x0763", "WORD $0x4866\nWORD $0x0763", + } { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body+"\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatalf("Translate %s: %v\n%s", triple, err, body) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + compileLLVMToObject(t, llc, triple, "movsxd-text.ll", "movsxd-text.o", ir) + x86MOVSXDModuleObject(t, file, options, triple, "movsxd-module") + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + } +} + +func TestX86RawMOVSXDLiteralExactReadExtent(t *testing.T) { + for _, prefix := range []string{"", "BYTE $0x48\n"} { + // The literal lies after the native C3, twelve/thirteen bytes from the + // entry. Displacement six spans the encoded full result-slot store. + body := prefix + "WORD $0x0563\nLONG $6\nWORD $0x8948\nWORD $0x2444\nBYTE $16\nBYTE $0xc3\nLONG $0x80000000" + source, sig, pkg := x86SourceStackNumericalFixture("amd64", body) + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := x86SourceStackOptions("amd64", triple, sig) + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "movsxd-literal.ll", "movsxd-literal.o", ir) + x86MOVSXDModuleObject(t, file, options, triple, "movsxd-literal-module") + x86MOVSXDGoModuleObject(t, pkg, source, triple, "movsxd-literal-go") + } + // Preserve the real opcode and displacement; only truncate the object. + truncated := strings.TrimSuffix(source, "LONG $0x80000000\n") + "WORD $0x8000\n" + truncatedFile, err := Parse(ArchAMD64, truncated) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(truncatedFile, x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig)); err == nil { + t.Fatal("four-byte MOVSXD literal read borrowed a truncated object") + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, truncatedFile, x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig)) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if err == nil { + t.Fatal("module borrowed a truncated MOVSXD literal") + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(truncated), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + }) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if err == nil { + t.Fatal("Go module borrowed a truncated MOVSXD literal") + } + } +} + +func TestX86RawMOVSXDOriginalGoByteWitness(t *testing.T) { + for _, encoding := range [][]byte{ + {0x63, 0xc3}, {0x66, 0x63, 0xc3}, {0x48, 0x63, 0xc3}, + {0x41, 0x63, 0xc1}, {0x44, 0x63, 0xc3}, + {0x66, 0x41, 0x63, 0xc1}, {0x66, 0x44, 0x63, 0xc3}, + {0x63, 0x07}, {0x63, 0x47, 0x01}, {0x63, 0x44, 0x8f, 0x03}, + {0x48, 0x63, 0x07}, {0x66, 0x48, 0x63, 0x07}, + {0x66, 0x63, 0x07}, // Go accepts the raw bytes; native extent remains Context. + } { + listing, err := x86ExtensionGoListing(t, "amd64", x86MOVSXDBytes(encoding)) + var bytes strings.Builder + for _, value := range append(append([]byte(nil), encoding...), 0xc3) { + fmt.Fprintf(&bytes, "%02x ", value) + } + if err != nil || !strings.Contains(listing, strings.TrimSpace(bytes.String())) { + t.Fatalf("actual Go %x byte witness: %v\n%s", encoding, err, listing) + } + } +} + +func TestX86RawMOVSXDLiteralCannotBorrowStaleCertificate(t *testing.T) { + body := "BYTE $0x48\nWORD $0x0563\nLONG $6\nWORD $0x8948\nWORD $0x2444\nBYTE $16\nBYTE $0xc3\nLONG $0x80000000" + source, sig, _ := x86SourceStackNumericalFixture("amd64", body) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig) + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + for _, change := range []string{"source width", "opcode", "result", "source", "payload", "missing object", "g result"} { + t.Run(change, func(t *testing.T) { + mutated := *normalized + mutated.Data = append([]DataStmt(nil), normalized.Data...) + mutated.Funcs = append([]Func(nil), normalized.Funcs...) + mutated.Funcs[0].Instrs = append([]Instr(nil), normalized.Funcs[0].Instrs...) + instruction := &mutated.Funcs[0].Instrs[1] + instruction.Args = append([]Operand(nil), instruction.Args...) + switch change { + case "source width": + instruction.x86RIPMemoryRead.width = 8 + case "opcode": + instruction.Op = "MOVQ" + case "result": + instruction.Args[1].Reg = BX + case "source": + instruction.Args[0] = Operand{Kind: OpReg, Reg: BX} + case "g result": + instruction.Args[1].Reg = "R14" + case "payload": + mutated.Data[0].Payload = []byte{1, 2, 3, 4} + case "missing object": + mutated.Data = nil + } + if _, err := Translate(&mutated, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed literal Translate: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, &mutated, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed literal module: %v", err) + } + // Go-module input is original bytes, not a File carrying a private + // certificate. Its independently regenerated, truncated source is + // tested above; it cannot accept this mutated certificate as input. + }) + } +} diff --git a/amd64_raw_mul_dword_qword.go b/amd64_raw_mul_dword_qword.go new file mode 100644 index 000000000..bc071c1bb --- /dev/null +++ b/amd64_raw_mul_dword_qword.go @@ -0,0 +1,253 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86DwordToQwordMultiplyInstruction recognizes every VEX and EVEX +// encoding of VPMULDQ and VPMULUDQ in Go 1.27's _yvandnpd table. x/arch +// v0.14 does not decode these encodings, so raw BYTE/LONG directive users +// need the complete X/Y/Z, memory, mask, zeroing, and broadcast family here. +func decodedX86DwordToQwordMultiplyInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86DwordToQwordMultiplyInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86DwordToQwordMultiplyInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if i >= len(code) { + return Instr{}, 0, false, nil + } + + var ( + opcodeIndex int + modRMIndex int + mapSelect int + width int + rExt int + rHigh int + xExt int + bExt int + vRegister int + vHigh int + rmHigh int + masked bool + zeroing bool + broadcast bool + mask int + evex bool + ) + + switch code[i] { + case 0xc5: + if len(code) < i+3 { + return Instr{}, 0, false, nil + } + vexByte := code[i+1] + opcodeIndex, modRMIndex = i+2, i+3 + mapSelect = 1 + if vexByte&3 != 1 || code[opcodeIndex] != 0xf4 { + return Instr{}, 0, false, nil + } + rExt = int(^vexByte>>7) & 1 + vRegister = int(^vexByte>>3) & 15 + width = 16 + if vexByte&4 != 0 { + width = 32 + } + case 0xc4: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + vexExt, vexByte := code[i+1], code[i+2] + opcodeIndex, modRMIndex = i+3, i+4 + mapSelect = int(vexExt & 0x1f) + if vexByte&3 != 1 || !x86DwordToQwordMultiplyOpcode(mapSelect, code[opcodeIndex]) { + return Instr{}, 0, false, nil + } + ok = true + if vexByte&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + rExt = int(^vexExt>>7) & 1 + xExt = int(^vexExt>>6) & 1 + bExt = int(^vexExt>>5) & 1 + vRegister = int(^vexByte>>3) & 15 + width = 16 + if vexByte&4 != 0 { + width = 32 + } + case 0x62: + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcodeIndex, modRMIndex = i+4, i+5 + mapSelect = int(p0 & 3) + if p0&0x0c != 0 || p1&3 != 1 || !x86DwordToQwordMultiplyOpcode(mapSelect, code[opcodeIndex]) { + return Instr{}, 0, false, nil + } + ok, evex = true, true + if p1&0x84 != 0x84 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be one and reserved bit must be set") + } + rExt = int(^p0>>7) & 1 + rHigh = int(^p0>>4) & 1 + xExt = int(^p0>>6) & 1 + bExt = int(^p0>>5) & 1 + vRegister = int(^p1>>3) & 15 + vHigh = int(^p2>>3) & 1 + rmHigh = xExt + zeroing = p2&0x80 != 0 + broadcast = p2&0x10 != 0 + mask = int(p2 & 7) + masked = mask != 0 + switch p2 >> 5 & 3 { + case 0: + width = 16 + case 1: + width = 32 + case 2: + width = 64 + default: + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + default: + return Instr{}, 0, false, nil + } + + if !ok { + ok = true + } + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRM := code[modRMIndex] + if evex && broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + if zeroing && !masked { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + destinationNumber := int(modRM>>3&7) + rExt*8 + rHigh*16 + secondNumber := vRegister + vHigh*16 + if mode == 32 && (destinationNumber >= 8 || secondNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector or address register in 32-bit mode") + } + destination, valid := decodedX86VectorRegister(destinationNumber, width) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid destination vector register %d", destinationNumber) + } + second, valid := decodedX86VectorRegister(secondNumber, width) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid second-source vector register %d", secondNumber) + } + disp8Scale := 1 + if evex { + disp8Scale = width + if broadcast { + disp8Scale = 8 + } + } + var first Operand + var consumed int + if memory.physicalPC && modRM>>6 != 3 { + first, consumed, err = memory.scalar(code[modRMIndex:], mode, bExt, xExt, segment, disp8Scale) + } else { + first, consumed, err = decodedX86VectorRMSource(code[modRMIndex:], mode, bExt, xExt, rmHigh, width, segment, disp8Scale) + } + if err != nil { + return Instr{}, 0, true, err + } + + op := "VPMULUDQ" + if mapSelect == 2 { + op = "VPMULDQ" + } + var suffixes []string + if broadcast { + suffixes = append(suffixes, "BCST") + } + if zeroing { + suffixes = append(suffixes, "Z") + } + if len(suffixes) != 0 { + op += "." + strings.Join(suffixes, ".") + } + args := []Operand{first, {Kind: OpReg, Reg: second}} + if masked { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: destination}) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: Op(op), Args: args, Raw: op + " " + strings.Join(rawArgs, ", "), x86Encoded: true}, modRMIndex + consumed, true, nil +} + +func x86DwordToQwordMultiplyOpcode(mapSelect int, opcode byte) bool { + return mapSelect == 1 && opcode == 0xf4 || mapSelect == 2 && opcode == 0x28 +} + +func decodedX86VectorRMSource(code []byte, mode, bExt, xExt, highExt, width int, segment Reg, disp8Scale int) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if code[0]>>6 != 3 { + return decodedX86RMSource(code, mode, bExt, xExt, segment, disp8Scale) + } + number := int(code[0]&7) + bExt*8 + highExt*16 + if mode == 32 && number >= 8 { + return Operand{}, 0, fmt.Errorf("extended vector register in 32-bit mode") + } + register, ok := decodedX86VectorRegister(number, width) + if !ok { + return Operand{}, 0, fmt.Errorf("invalid first-source vector register %d", number) + } + return Operand{Kind: OpReg, Reg: register}, 1, nil +} + +func decodedX86VectorRegister(number, width int) (Reg, bool) { + prefix := "" + switch width { + case 16: + prefix = "X" + case 32: + prefix = "Y" + case 64: + prefix = "Z" + default: + return "", false + } + if number < 0 || number >= 32 { + return "", false + } + return Reg(fmt.Sprintf("%s%d", prefix, number)), true +} diff --git a/amd64_raw_mul_dword_qword_test.go b/amd64_raw_mul_dword_qword_test.go new file mode 100644 index 000000000..fdcaccd53 --- /dev/null +++ b/amd64_raw_mul_dword_qword_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestX86RawDwordToQwordMultiplyReportedEncoding(t *testing.T) { + const source = ` +TEXT rawVPMULUDQ(SB),$0-0 + BYTE $0xc5 + BYTE $0xfd + BYTE $0xf4 + BYTE $0xc2 + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i686-unknown-linux-gnu"}, + {goarch: "386", triple: "i686-pc-windows-msvc"}, + {goarch: "amd64", triple: "x86_64-apple-darwin"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + // The original reported bytes have unknown Y inputs. Preserve the + // real Go object, public Context and complete emitter object; + // initialized product-family objects own the numerical contract. + ir := x86UnboundVoidRawScalarFormatObject(t, source, target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-vpmuludq.ll", "raw-vpmuludq.o", ir) + }) + } +} + +func TestX86RawDwordToQwordMultiplyCompleteVEXRegisterFields(t *testing.T) { + for _, family := range []struct { + name string + mapID byte + opcode byte + }{ + {name: "VPMULUDQ", mapID: 1, opcode: 0xf4}, + {name: "VPMULDQ", mapID: 2, opcode: 0x28}, + } { + for _, width := range []struct { + prefix string + l byte + }{ + {prefix: "X", l: 0}, + {prefix: "Y", l: 1}, + } { + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + p0 := byte((^destination>>3)&1)<<7 | 1<<6 | byte((^first>>3)&1)<<5 | family.mapID + p1 := byte((^second)&15)<<3 | width.l<<2 | 1 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0xc4, p0, p1, family.opcode, modRM} + instruction, length, ok, err := decodedX86DwordToQwordMultiplyInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("%s %s registers %d,%d,%d: ok=%v length=%d err=%v", family.name, width.prefix, first, second, destination, ok, length, err) + } + want := []Reg{ + Reg(fmt.Sprintf("%s%d", width.prefix, first)), + Reg(fmt.Sprintf("%s%d", width.prefix, second)), + Reg(fmt.Sprintf("%s%d", width.prefix, destination)), + } + if instruction.Op != Op(family.name) || len(instruction.Args) != len(want) { + t.Fatalf("encoding %x decoded as %#v", encoding, instruction) + } + for index := range want { + if instruction.Args[index].Kind != OpReg || instruction.Args[index].Reg != want[index] { + t.Fatalf("encoding %x arg %d = %#v, want %s", encoding, index, instruction.Args[index], want[index]) + } + } + } + } + } + } + } +} + +func TestX86RawDwordToQwordMultiplyCompleteEVEXRegisterFields(t *testing.T) { + for _, family := range []struct { + name string + mapID byte + opcode byte + }{ + {name: "VPMULUDQ", mapID: 1, opcode: 0xf4}, + {name: "VPMULDQ", mapID: 2, opcode: 0x28}, + } { + for _, width := range []struct { + prefix string + ll byte + }{ + {prefix: "X", ll: 0}, + {prefix: "Y", ll: 1}, + {prefix: "Z", ll: 2}, + } { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + p0 := byte((^destination>>3)&1)<<7 | byte((^first>>4)&1)<<6 | byte((^first>>3)&1)<<5 | byte((^destination>>4)&1)<<4 | family.mapID + p1 := byte(0x85 | ((^second)&15)<<3) + p2 := width.ll<<5 | byte((^second>>4)&1)<<3 | 1 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0x62, p0, p1, p2, family.opcode, modRM} + instruction, length, ok, err := decodedX86DwordToQwordMultiplyInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("%s %s registers %d,%d,%d: ok=%v length=%d err=%v", family.name, width.prefix, first, second, destination, ok, length, err) + } + want := []Reg{ + Reg(fmt.Sprintf("%s%d", width.prefix, first)), + Reg(fmt.Sprintf("%s%d", width.prefix, second)), + "K1", + Reg(fmt.Sprintf("%s%d", width.prefix, destination)), + } + if instruction.Op != Op(family.name) || len(instruction.Args) != len(want) { + t.Fatalf("encoding %x decoded as %#v", encoding, instruction) + } + for index := range want { + if instruction.Args[index].Kind != OpReg || instruction.Args[index].Reg != want[index] { + t.Fatalf("encoding %x arg %d = %#v, want %s", encoding, index, instruction.Args[index], want[index]) + } + } + } + } + } + } + } +} + +func TestX86RawDwordToQwordMultiplyMemoryMaskAndBroadcastForms(t *testing.T) { + for _, test := range []struct { + name string + encoding []byte + wantOp Op + wantArgs int + wantMem bool + }{ + {name: "vex signed base", encoding: []byte{0xc4, 0xe2, 0x31, 0x28, 0x13}, wantOp: "VPMULDQ", wantArgs: 3, wantMem: true}, + {name: "vex unsigned extended base", encoding: []byte{0xc4, 0xc1, 0x31, 0xf4, 0x13}, wantOp: "VPMULUDQ", wantArgs: 3, wantMem: true}, + {name: "evex masked memory", encoding: []byte{0x62, 0x71, 0xfd, 0x0e, 0xf4, 0xbc, 0x75, 0xef, 0xff, 0xff, 0xff}, wantOp: "VPMULUDQ", wantArgs: 4, wantMem: true}, + {name: "evex broadcast", encoding: []byte{0x62, 0x71, 0xfd, 0x1e, 0xf4, 0xbc, 0x75, 0xef, 0xff, 0xff, 0xff}, wantOp: "VPMULUDQ.BCST", wantArgs: 4, wantMem: true}, + {name: "evex broadcast zero", encoding: []byte{0x62, 0x71, 0xfd, 0x9e, 0xf4, 0xbc, 0x75, 0xef, 0xff, 0xff, 0xff}, wantOp: "VPMULUDQ.BCST.Z", wantArgs: 4, wantMem: true}, + {name: "evex unmasked", encoding: []byte{0x62, 0x31, 0xfd, 0x08, 0xf4, 0xf8}, wantOp: "VPMULUDQ", wantArgs: 3}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86DwordToQwordMultiplyInstruction(test.encoding, 64) + if !ok || err != nil || length != len(test.encoding) { + t.Fatalf("encoding %x: ok=%v length=%d err=%v", test.encoding, ok, length, err) + } + if instruction.Op != test.wantOp || len(instruction.Args) != test.wantArgs || (instruction.Args[0].Kind == OpMem) != test.wantMem { + t.Fatalf("encoding %x decoded as %#v", test.encoding, instruction) + } + }) + } +} + +func TestX86RawDwordToQwordMultiplyRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + }{ + {name: "vex w", mode: 64, encoding: []byte{0xc4, 0xe1, 0xfd, 0xf4, 0xc2}}, + {name: "evex w", mode: 64, encoding: []byte{0x62, 0x31, 0x7d, 0x0e, 0xf4, 0xf8}}, + {name: "evex reserved length", mode: 64, encoding: []byte{0x62, 0x31, 0xfd, 0x6e, 0xf4, 0xf8}}, + {name: "zero without mask", mode: 64, encoding: []byte{0x62, 0x31, 0xfd, 0x88, 0xf4, 0xf8}}, + {name: "broadcast register", mode: 64, encoding: []byte{0x62, 0x31, 0xfd, 0x1e, 0xf4, 0xf8}}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc5, 0xfd, 0xf4, 0xc2}}, + {name: "rip relative", mode: 64, encoding: []byte{0xc5, 0xfd, 0xf4, 0x05, 0, 0, 0, 0}}, + {name: "extended 386", mode: 32, encoding: []byte{0xc4, 0x61, 0x79, 0xf4, 0xc0}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86DwordToQwordMultiplyInstruction(test.encoding, test.mode) + if !ok || err == nil { + t.Fatalf("encoding %x: ok=%v err=%v, want recognized rejection", test.encoding, ok, err) + } + }) + } +} diff --git a/amd64_raw_mul_low.go b/amd64_raw_mul_low.go new file mode 100644 index 000000000..afbc3f1d3 --- /dev/null +++ b/amd64_raw_mul_low.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27's packed-multiply rows share _yvandnpd's operand grammar. The +// opcode/map selects the lane operation; EVEX.W selects Q for opcode 40/0F38. +// VEX.W is ignored, including VPMULLD; it cannot select a VEX qword form. +// The word operations have no broadcast row, whereas D/Q permit it. +type x86PackedMultiplyForm struct { + mapID byte + opcode byte + vexOp Op + evexW0Op Op + evexW1Op Op + broadcastByte int +} + +var x86PackedMultiplyForms = [...]x86PackedMultiplyForm{ + {mapID: 1, opcode: 0xd5, vexOp: "VPMULLW", evexW0Op: "VPMULLW", evexW1Op: "VPMULLW"}, + {mapID: 1, opcode: 0xe5, vexOp: "VPMULHW", evexW0Op: "VPMULHW", evexW1Op: "VPMULHW"}, + {mapID: 1, opcode: 0xe4, vexOp: "VPMULHUW", evexW0Op: "VPMULHUW", evexW1Op: "VPMULHUW"}, + {mapID: 2, opcode: 0x0b, vexOp: "VPMULHRSW", evexW0Op: "VPMULHRSW", evexW1Op: "VPMULHRSW"}, + {mapID: 2, opcode: 0x40, vexOp: "VPMULLD", evexW0Op: "VPMULLD", evexW1Op: "VPMULLQ", broadcastByte: 4}, +} + +func x86PackedMultiplyFormFor(mapID, opcode byte) (x86PackedMultiplyForm, bool) { + for _, form := range x86PackedMultiplyForms { + if form.mapID == mapID && form.opcode == opcode { + return form, true + } + } + return x86PackedMultiplyForm{}, false +} + +// Keep the direct decoder used by the exhaustive VPMULLD/Q tests separate +// from the dispatcher, while sharing its parser with the word-multiply rows. +func decodedX86PackedMultiplyLowInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + instruction, length, ok, err = decodedX86PackedMultiplyInstruction(code, mode) + if !ok || err != nil || instruction.Op == "VPMULLD" || strings.HasPrefix(string(instruction.Op), "VPMULLD.") || + instruction.Op == "VPMULLQ" || strings.HasPrefix(string(instruction.Op), "VPMULLQ.") { + return instruction, length, ok, err + } + return Instr{}, 0, false, nil +} + +func decodedX86PackedMultiplyInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86PackedMultiplyInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedMultiplyInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized { + return Instr{}, 0, false, nil + } + if p.evex { + return decodedX86EVEXPackedMultiplyInstructionWithMemory( + code, p.modRM-5, mode, p.segment, p.addressOverride, memory, + ) + } + if p.pp != 1 { + return Instr{}, 0, false, nil + } + form, recognized := x86PackedMultiplyFormFor(byte(p.mapNumber), byte(p.opcode)) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p.addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRMIndex := p.modRM + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorName := "X" + if p.vectorLength != 0 { + vectorName = "Y" + } + secondSourceNumber := p.upper + destinationNumber := int(code[modRMIndex]>>3&7) + p.r*8 + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8 || + code[modRMIndex]>>6 == 3 && int(code[modRMIndex]&7)+p.b*8 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + firstSource, consumed, decodeErr := memory.vex( + code[modRMIndex:], mode, p.b, p.x, p.segment, vectorName, + ) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + instruction = Instr{ + Op: form.vexOp, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", form.vexOp, firstSource.String(), secondSource.String(), destination.String()), + } + instruction.x86Encoded = true + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXPackedMultiplyInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedMultiplyInstructionWithMemory(code, i, mode, segment, addressOverride, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXPackedMultiplyInstructionWithMemory(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p1&0x07 != 5 { + return Instr{}, 0, false, nil + } + form, recognized := x86PackedMultiplyFormFor(p0&0x0f, code[i+4]) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + width64 := p1&0x80 != 0 + op := form.evexW0Op + if width64 { + op = form.evexW1Op + } + laneBytes := form.broadcastByte + if width64 && laneBytes != 0 { + laneBytes = 8 + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && laneBytes == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is not available for %s", op) + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + firstSourceNumber := int(modRM&7) + bExt*8 + xExt*16 + if mode == 32 && vectorName == "Z" && (secondSourceNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && firstSourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("high Z register is unavailable in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + x86Encoded: true}, modRMIndex + consumed, true, nil +} diff --git a/amd64_raw_mul_low_test.go b/amd64_raw_mul_low_test.go new file mode 100644 index 000000000..2a7a1df15 --- /dev/null +++ b/amd64_raw_mul_low_test.go @@ -0,0 +1,168 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXPackedMultiplyLow(width256 bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 2) + p1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0xc4, p0, p1, 0x40, modRM} +} + +func encodeX86EVEXPackedMultiplyLow(width64 bool, vectorBits byte, mask int, zeroing bool, destination, source1, source2 int) []byte { + p0 := byte((1-destination/8&1)<<7 | (1-source2/16)<<6 | (1-source2/8&1)<<5 | (1-destination/16)<<4 | 2) + p1 := byte(((^source1)&15)<<3 | 5) + if width64 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | byte((^source1>>4)&1)<<3 | byte(mask)) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0x62, p0, p1, p2, 0x40, modRM} +} + +func TestTranslateRawEVEXVPMULLDWeaviateRegression(t *testing.T) { + const source = `TEXT rawVPMULLD(SB), $0-0 + LONG $0x483df262 + WORD $0xe440 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawVPMULLD": {Name: "rawVPMULLD", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "mul <16 x i32>") { + t.Fatalf("raw VPMULLD lowering omitted packed multiply:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-evex-vpmulld.ll", "amd64-raw-evex-vpmulld.o", ir) +} + +func TestDecodedX86PackedMultiplyLowCompleteVEXRegisterFamily(t *testing.T) { + count := 0 + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < 16; source2++ { + code := encodeX86VEXPackedMultiplyLow(width256, destination, source1, source2) + got, length, ok, err := decodedX86PackedMultiplyLowInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != "VPMULLD" || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want VPMULLD %s", code, got, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d VEX packed multiply-low register encodings, want 8192", count) + } +} + +func TestDecodedX86PackedMultiplyLowCompleteEVEXRegisterFamily(t *testing.T) { + count := 0 + for _, width := range []struct { + width64 bool + op Op + }{ + {op: "VPMULLD"}, + {width64: true, op: "VPMULLQ"}, + } { + for vectorBits, prefix := range []string{"X", "Y", "Z"} { + for destination := 0; destination < 32; destination++ { + for source1 := 0; source1 < 32; source1++ { + for source2 := 0; source2 < 32; source2++ { + code := encodeX86EVEXPackedMultiplyLow(width.width64, byte(vectorBits), 1, false, destination, source1, source2) + got, length, ok, err := decodedX86PackedMultiplyLowInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), "K1", fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != width.op || len(got.Args) != 4 { + t.Fatalf("decode %x = %+v, want %s %s", code, got, width.op, strings.Join(wantArgs, ", ")) + } + for index := range wantArgs { + if got.Args[index].String() != wantArgs[index] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, width.op, strings.Join(wantArgs, ", ")) + } + } + count++ + } + } + } + } + } + if count != 196608 { + t.Fatalf("covered %d EVEX packed multiply-low register encodings, want 196608", count) + } +} + +func TestDecodedX86PackedMultiplyLowMemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + width64 bool + vectorBits byte + op Op + laneBytes int + }{ + {name: "dword", vectorBits: 2, op: "VPMULLD.BCST.Z", laneBytes: 4}, + {name: "qword", width64: true, vectorBits: 2, op: "VPMULLQ.BCST.Z", laneBytes: 8}, + } { + t.Run(test.name, func(t *testing.T) { + code := encodeX86EVEXPackedMultiplyLow(test.width64, test.vectorBits, 3, true, 21, 20, 0) + code[3] |= 0x10 + code[5] = 0x68 + code = append(code, 0x7f) + got, length, ok, err := decodedX86PackedMultiplyLowInstruction(code, 64) + wantSource := fmt.Sprintf("%d(AX)", test.laneBytes*127) + if err != nil || !ok || length != len(code) || got.Op != test.op || len(got.Args) != 4 || got.Args[0].String() != wantSource || got.Args[1].String() != "Z20" || got.Args[2].String() != "K3" || got.Args[3].String() != "Z21" { + t.Fatalf("decode %x = %+v, length=%d ok=%v err=%v; want %s %s, Z20, K3, Z21", code, got, length, ok, err, test.op, wantSource) + } + }) + } + + for name, code := range map[string][]byte{ + "truncated": {0xc4, 0xe2}, + "wrong map": {0xc4, 0xe1, 0x79, 0x40, 0xc0}, + "wrong pp": {0xc4, 0xe2, 0x78, 0x40, 0xc0}, + "wrong opcode": {0xc4, 0xe2, 0x79, 0x41, 0xc0}, + } { + if instruction, _, matched, decodeErr := decodedX86PackedMultiplyLowInstruction(code, 64); matched || decodeErr != nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, code, instruction, matched, decodeErr) + } + } + invalidBroadcast := encodeX86EVEXPackedMultiplyLow(false, 2, 1, false, 0, 0, 0) + invalidBroadcast[3] |= 0x10 + if _, _, matched, decodeErr := decodedX86PackedMultiplyLowInstruction(invalidBroadcast, 64); !matched || decodeErr == nil { + t.Fatalf("register broadcast returned ok=%v err=%v", matched, decodeErr) + } + zeroWithoutMask := encodeX86EVEXPackedMultiplyLow(false, 2, 0, true, 0, 0, 0) + if _, _, matched, decodeErr := decodedX86PackedMultiplyLowInstruction(zeroWithoutMask, 64); !matched || decodeErr == nil { + t.Fatalf("zero without mask returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_muldiv_width_test.go b/amd64_raw_muldiv_width_test.go new file mode 100644 index 000000000..518555666 --- /dev/null +++ b/amd64_raw_muldiv_width_test.go @@ -0,0 +1,96 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawImplicitMulDivWidths(t *testing.T) { + for _, form := range []struct { + name string + regBit byte + }{ + {"MUL", 4}, {"IMUL", 5}, {"DIV", 6}, {"IDIV", 7}, + } { + for _, width := range []struct { + name string + prefix byte + opcode byte + }{ + {"B", 0, 0xf6}, + {"W", 0x66, 0xf7}, + {"L", 0, 0xf7}, + {"Q", 0x48, 0xf7}, + } { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("%s%s/mem%t", form.name, width.name, memory) + t.Run(name, func(t *testing.T) { + var code []byte + if width.prefix != 0 { + code = append(code, width.prefix) + } + modRM := byte(0xc2 | form.regBit<<3) + if memory { + modRM = 0x02 | form.regBit<<3 + } + code = append(code, width.opcode, modRM, 0xc3) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || string(got[0].Op) != form.name+width.name || got[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s", code, got, name) + } + }) + } + } + } +} + +func TestTranslateX86RawImplicitMulDivWidthsObjects(t *testing.T) { + var source strings.Builder + for index, code := range [][]byte{ + {0xf6, 0xe2, 0xc3}, + {0x66, 0xf7, 0xea, 0xc3}, + {0xf7, 0xf2, 0xc3}, + {0x48, 0xf7, 0xfa, 0xc3}, + } { + name := fmt.Sprintf("rawMulDivWidth%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + original, signatures := x86UnboundVoidFormatSources(t, source.String(), "amd64", triple) + normalized, err := normalizeX86RawFile(original, "amd64") + if err != nil { + t.Fatal(err) + } + // Original raw bytes and all public Context results stay bound + // above. These private units measure native encoding syntax only; + // they supply no implicit inputs or raw/native entry contract. + ir := x86InstructionFormatObjectUnit(t, normalized, signatures, "amd64", triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + if form, known, err := x86ImplicitDivideProperties(ins.Op, ins, "amd64"); known { + if err != nil { + return true, false, err + } + return true, false, c.emitNativeImplicitDivide(form) + } + terminated, err := c.lowerInstr(0, 0, ins, nil, nil) + return true, terminated, err + }) + compileLLVMToObject(t, llc, triple, "raw-muldiv-width.ll", "raw-muldiv-width.o", ir) + }) + } +} diff --git a/amd64_raw_mulword_test.go b/amd64_raw_mulword_test.go new file mode 100644 index 000000000..f60615409 --- /dev/null +++ b/amd64_raw_mulword_test.go @@ -0,0 +1,273 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86PackedWordMultiplyGoRows = []struct { + mapID byte + opcode byte + op Op +}{ + {1, 0xd5, "VPMULLW"}, + {1, 0xe5, "VPMULHW"}, + {1, 0xe4, "VPMULHUW"}, + {2, 0x0b, "VPMULHRSW"}, +} + +func encodeX86VEXPackedWordMultiply(mapID, opcode byte, width256, w bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | int(mapID)) + p1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + if w { + p1 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0xc4, p0, p1, opcode, modRM} +} + +func encodeX86VEX2PackedWordMultiply(opcode byte, width256 bool, destination, source1, source2 int) []byte { + vectorBit := 0 + if width256 { + vectorBit = 1 + } + prefix := byte((1-destination/8)<<7 | (^source1&15)<<3 | vectorBit<<2 | 1) + modRM := byte(0xc0 | destination&7<<3 | source2&7) + return []byte{0xc5, prefix, opcode, modRM} +} + +func encodeX86EVEXPackedWordMultiply(mapID, opcode byte, vectorBits, mask int, w, zeroing bool, destination, source1, source2 int) []byte { + p0 := byte((1-destination/8&1)<<7 | (1-source2/16)<<6 | (1-source2/8&1)<<5 | (1-destination/16)<<4 | int(mapID)) + p1 := byte(((^source1)&15)<<3 | 5) + if w { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | ((^source1>>4)&1)<<3 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestTranslateRawVPMULHWOldCorpusRegression(t *testing.T) { + const source = `TEXT rawVPMULHW(SB), $0-0 + LONG $0xe57161c4 + BYTE $0xc6 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawVPMULHW": {Name: "rawVPMULHW", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "mul <8 x i32>") || !strings.Contains(ir, "ashr <8 x i32>") { + t.Fatalf("raw VPMULHW omitted packed word multiply:\n%s", ir) + } +} + +func TestTranslateRawVEX2PackedWordMultiplyGoccRegression(t *testing.T) { + const source = `TEXT rawVEX2WordMultiply(SB), $0-0 + LONG $0xdbd5cdc5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawVEX2WordMultiply": {Name: "rawVEX2WordMultiply", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "mul <16 x i16>") { + t.Fatalf("raw VEX2 VPMULLW omitted packed multiplication:\n%s", ir) + } +} + +func TestDecodedX86PackedWordMultiplyCompleteGoRows(t *testing.T) { + if len(x86PackedMultiplyForms) != len(x86PackedWordMultiplyGoRows)+1 { + t.Fatalf("packed multiply grammar has %d rows, want four word rows plus D/Q", len(x86PackedMultiplyForms)) + } + for _, row := range x86PackedWordMultiplyGoRows { + form, ok := x86PackedMultiplyFormFor(row.mapID, row.opcode) + if !ok || form.vexOp != row.op || form.evexW0Op != row.op || form.evexW1Op != row.op || form.broadcastByte != 0 { + t.Fatalf("Go word-multiply row %s/%02x is not modeled completely: %+v", row.op, row.opcode, form) + } + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, w := range []bool{false, true} { + code := encodeX86VEXPackedWordMultiply(row.mapID, row.opcode, width256, w, 8, 1, 6) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + want := fmt.Sprintf("%s %s6, %s1, %s8", row.op, prefix, prefix, prefix) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("VEX %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + if row.mapID == 1 { + code := encodeX86VEX2PackedWordMultiply(row.opcode, width256, 8, 1, 6) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + want := fmt.Sprintf("%s %s6, %s1, %s8", row.op, prefix, prefix, prefix) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("VEX2 %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + } + for vectorBits, prefix := range []string{"X", "Y", "Z"} { + for _, w := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 2 + } + code := encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, vectorBits, mask, w, masked, 21, 20, 22) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + wantOp := string(row.op) + wantArgs := fmt.Sprintf("%s22, %s20, %s21", prefix, prefix, prefix) + if masked { + wantOp += ".Z" + wantArgs = fmt.Sprintf("%s22, %s20, K2, %s21", prefix, prefix, prefix) + } + want := wantOp + " " + wantArgs + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("EVEX %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + } + } + } +} + +func TestDecodedX86PackedWordMultiplyMemoryAndRejectedForms(t *testing.T) { + for _, row := range x86PackedWordMultiplyGoRows { + if row.mapID == 1 { + code := encodeX86VEX2PackedWordMultiply(row.opcode, true, 2, 1, 0) + code[3] = 0x50 + code = append(code, 0x20) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + want := fmt.Sprintf("%s 32(AX), Y1, Y2", row.op) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("VEX2 memory %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + code := encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, 2, 0, false, false, 21, 20, 0) + code[5] = 0x68 + code = append(code, 0x7f) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + want := fmt.Sprintf("%s 8128(AX), Z20, Z21", row.op) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("memory %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + broadcast := append([]byte(nil), code...) + broadcast[3] |= 0x10 + if _, _, matched, err := decodedX86PackedMultiplyInstruction(broadcast, 64); !matched || err == nil { + t.Fatalf("word multiply incorrectly accepted EVEX broadcast %x", broadcast) + } + for vectorBits, prefix := range []string{"X", "Y"} { + code := encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, vectorBits, 0, false, false, 21, 20, 22) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 32) + want := fmt.Sprintf("%s %s22, %s20, %s21", row.op, prefix, prefix, prefix) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("386 high-register %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + } + invalidZeroing := encodeX86EVEXPackedWordMultiply(1, 0xe5, 0, 0, false, true, 0, 1, 2) + if _, _, matched, err := decodedX86PackedMultiplyInstruction(invalidZeroing, 64); !matched || err == nil { + t.Fatalf("zero without mask accepted: %x", invalidZeroing) + } + invalidLength := encodeX86EVEXPackedWordMultiply(1, 0xe5, 3, 0, false, false, 0, 1, 2) + if _, _, matched, err := decodedX86PackedMultiplyInstruction(invalidLength, 64); !matched || err == nil { + t.Fatalf("reserved vector length accepted: %x", invalidLength) + } + for _, code := range [][]byte{ + {0xc4, 0xe3, 0x79, 0xe5, 0xc0}, + {0xc4, 0xe1, 0x78, 0xe5, 0xc0}, + {0xc4, 0xe1, 0x79, 0xe6, 0xc0}, + } { + if got, _, matched, err := decodedX86PackedMultiplyInstruction(code, 64); matched || err != nil { + t.Fatalf("invalid opcode form %x decoded as %+v, matched=%v err=%v", code, got, matched, err) + } + } +} + +func TestTranslateRawPackedWordMultiplyLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawPackedWordMultiply(SB), $0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for _, row := range x86PackedWordMultiplyGoRows { + for _, width256 := range []bool{false, true} { + appendBytes(encodeX86VEXPackedWordMultiply(row.mapID, row.opcode, width256, false, 2, 1, 0)) + if row.mapID == 1 { + appendBytes(encodeX86VEX2PackedWordMultiply(row.opcode, width256, 2, 1, 0)) + } + } + for vectorBits := 0; vectorBits < 3; vectorBits++ { + mask := 3 + zeroing := true + if target.goarch == "386" { + mask = 0 + zeroing = false + } + destination, second, first := 21, 20, 22 + if target.goarch == "386" { + destination, second, first = 2, 1, 0 + } + appendBytes(encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, vectorBits, mask, false, zeroing, destination, second, first)) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawPackedWordMultiply": {Name: "rawPackedWordMultiply", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-packed-word-multiply.ll", "raw-packed-word-multiply.o", ir) + }) + } +} diff --git a/amd64_raw_narrow_single_source.go b/amd64_raw_narrow_single_source.go new file mode 100644 index 000000000..e13caebc0 --- /dev/null +++ b/amd64_raw_narrow_single_source.go @@ -0,0 +1,81 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27's _yvpmovdb/_yvpmovdw rows form a three-by-six grammar: +// unsigned-saturating, signed-saturating and truncating conversions, each +// narrowing W/D/Q into B/W/D. The low opcode nibble selects the width pair. +var x86RawSingleSourceNarrowOps = [3][6]Op{ + {"VPMOVUSWB", "VPMOVUSDB", "VPMOVUSQB", "VPMOVUSDW", "VPMOVUSQW", "VPMOVUSQD"}, + {"VPMOVSWB", "VPMOVSDB", "VPMOVSQB", "VPMOVSDW", "VPMOVSQW", "VPMOVSQD"}, + {"VPMOVWB", "VPMOVDB", "VPMOVQB", "VPMOVDW", "VPMOVQW", "VPMOVQD"}, +} + +func decodedX86SingleSourceNarrowInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || !p.evex || p.mapNumber != 2 || p.pp != 2 { + return Instr{}, 0, false, nil + } + group := p.opcode>>4 - 1 + width := p.opcode & 15 + if group < 0 || group >= len(x86RawSingleSourceNarrowOps) || width >= 6 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("single-source narrowing: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.w || p.upper != 0 || p.broadcast || p.vectorLength > 2 { + return fail("invalid reserved EVEX encoding bit") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + + baseOp := x86RawSingleSourceNarrowOps[group][width] + spec := amd64SingleSourceNarrowSpecs[string(baseOp)] + inputBytes := 16 << p.vectorLength + outputBytes := inputBytes * spec.outputBits / spec.inputBits + destinationPrefix := "X" + if outputBytes > 16 { + destinationPrefix = "Y" + } + modRM := code[p.modRM] + registerNumber := int(modRM>>3&7) + p.r*8 + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", [...]string{"X", "Y", "Z"}[p.vectorLength], registerNumber))} + if mode == 32 && p.vectorLength == 2 && registerNumber >= 8 { + return fail("386 Z source register exceeds Go's encoder class") + } + if mode == 32 && modRM>>6 != 3 && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + destination, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, destinationPrefix, outputBytes) + if err != nil { + return Instr{}, 0, true, err + } + op := baseOp + if p.zero { + op += ".Z" + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_narrow_single_source_test.go b/amd64_raw_narrow_single_source_test.go new file mode 100644 index 000000000..19ed71037 --- /dev/null +++ b/amd64_raw_narrow_single_source_test.go @@ -0,0 +1,214 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func encodeX86RawSingleSourceNarrow(group, width, vectorBits, source, destination, mask int, zeroing, memory bool) []byte { + if memory { + destination = 0 + } + p0 := byte((1-source/8&1)<<7 | (1-destination/16)<<6 | + (1-destination/8&1)<<5 | (1-source/16)<<4 | 2) + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (source&7)<<3 | destination&7) + if memory { + modRM = byte(0x40 | (source&7)<<3) + } + code := []byte{0x62, p0, 0x7e, p2, byte((group+1)<<4 | width), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestTranslateRawNarrowSingleSourceGorseRegression(t *testing.T) { + const source = `TEXT rawNarrowSingleSource(SB), $0-0 + LONG $0x487ef262 + WORD $0x2c11 + BYTE $0x07 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawNarrowSingleSource": {Name: "rawNarrowSingleSource", Ret: Void}}, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original raw regression invented native input registers/address", err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + file.Funcs[0] = decoded + units, signatures := x86SingleSourceNarrowingFormatUnits(file) + if len(units.Funcs) != 1 { + t.Fatal("original physical regression lost its single conversion", len(units.Funcs)) + } + ir = x86SingleInstructionFormatObjectUnits(t, units, signatures, "amd64", "x86_64-unknown-linux-gnu", + func(c *amd64Ctx, ins Instr) (bool, bool, error) { return c.lowerSingleSourceIntegerNarrow(ins.Op, ins) }) + if !strings.Contains(ir, "icmp ugt") { + t.Fatal("raw VPMOVUSDB format omitted unsigned input saturation") + } +} + +func TestDecodedX86RawSingleSourceNarrowCompleteGoRows(t *testing.T) { + if len(amd64SingleSourceNarrowSpecs) != 18 { + t.Fatalf("Go 1.27 single-source narrowing grammar has %d members, want 18", len(amd64SingleSourceNarrowSpecs)) + } + inputBits := [...]int{16, 32, 64, 32, 64, 64} + outputBits := [...]int{8, 8, 8, 16, 16, 32} + seen := make(map[Op]bool) + for group, row := range x86RawSingleSourceNarrowOps { + for width, op := range row { + spec, ok := amd64SingleSourceNarrowSpecs[string(op)] + if !ok || seen[op] || spec.inputBits != inputBits[width] || spec.outputBits != outputBits[width] || + spec.domain != [...]x86SingleSourceNarrowDomain{x86NarrowUnsigned, x86NarrowSigned, x86NarrowTruncate}[group] { + t.Fatalf("raw opcode row %d/%d = %s, spec %+v", group, width, op, spec) + } + seen[op] = true + for vectorBits, prefix := range [...]string{"X", "Y", "Z"} { + outputBytes := (16 << vectorBits) * spec.outputBits / spec.inputBits + outputPrefix := "X" + if outputBytes > 16 { + outputPrefix = "Y" + } + for _, memory := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + for _, zeroing := range []bool{false, true} { + if zeroing && !masked { + continue + } + code := encodeX86RawSingleSourceNarrow(group, width, vectorBits, 21, 20, mask, zeroing, memory) + got, length, recognized, err := decodedX86SingleSourceNarrowInstruction(code, 64) + if err != nil || !recognized || length != len(code) { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + wantOp := op + if zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[0].String() != fmt.Sprintf("%s21", prefix) { + t.Fatalf("decode %x = %+v, want %s and %s21", code, got, wantOp, prefix) + } + if masked && (len(got.Args) != 3 || got.Args[1].String() != "K3") || !masked && len(got.Args) != 2 { + t.Fatalf("decode %x = %+v, lost mask", code, got) + } + destination := got.Args[len(got.Args)-1] + if memory { + if destination.Kind != OpMem || destination.Mem.Base != AX || destination.Mem.Off != int64(outputBytes) { + t.Fatalf("decode %x = %+v, want compressed displacement %d", code, got, outputBytes) + } + } else if destination.String() != fmt.Sprintf("%s20", outputPrefix) { + t.Fatalf("decode %x = %+v, want %s20", code, got, outputPrefix) + } + } + } + } + } + } + } + if len(seen) != len(amd64SingleSourceNarrowSpecs) { + t.Fatalf("raw grammar has %d distinct ops, named lowerer has %d", len(seen), len(amd64SingleSourceNarrowSpecs)) + } +} + +func TestDecodedX86RawSingleSourceNarrowRejectsReservedBits(t *testing.T) { + base := encodeX86RawSingleSourceNarrow(0, 1, 2, 2, 1, 3, true, false) + for _, test := range []struct { + name string + code []byte + }{ + {"W bit", []byte{base[0], base[1], base[2] | 0x80, base[3], base[4], base[5]}}, + {"fixed bit", []byte{base[0], base[1], base[2] &^ 4, base[3], base[4], base[5]}}, + {"vvvv", []byte{base[0], base[1], base[2] &^ 8, base[3], base[4], base[5]}}, + {"V prime", []byte{base[0], base[1], base[2], base[3] &^ 8, base[4], base[5]}}, + {"broadcast", []byte{base[0], base[1], base[2], base[3] | 0x10, base[4], base[5]}}, + {"reserved length", []byte{base[0], base[1], base[2], base[3] | 0x20, base[4], base[5]}}, + {"zero without mask", []byte{base[0], base[1], base[2], base[3] &^ 7, base[4], base[5]}}, + {"missing ModRM", base[:5]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86SingleSourceNarrowInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted reserved encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + if _, _, recognized, err := decodedX86PerLaneVariableShiftInstruction(base, 64); recognized || err != nil { + t.Fatalf("per-lane shift incorrectly owns F3 narrow opcode: recognized=%v err=%v", recognized, err) + } +} + +func TestTranslateRawSingleSourceNarrowLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawNarrowFamily(SB), $0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for group, row := range x86RawSingleSourceNarrowOps { + for width := range row { + appendBytes(encodeX86RawSingleSourceNarrow(group, width, 0, 2, 1, 0, false, false)) + } + appendBytes(encodeX86RawSingleSourceNarrow(group, 5, 2, 2, 1, 2, true, false)) + appendBytes(encodeX86RawSingleSourceNarrow(group, 1, 1, 2, 0, 3, false, true)) + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawNarrowFamily": {Name: "rawNarrowFamily", Ret: Void}}, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("bare raw formats invented initialized native entry values", err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.goarch) + if err != nil { + t.Fatal(err) + } + file.Funcs[0] = decoded + units, signatures := x86SingleSourceNarrowingFormatUnits(file) + if len(units.Funcs) != 24 { + t.Fatal("original physical family format disappeared", len(units.Funcs)) + } + x86SingleSourceNarrowingFormatObjects(t, llc, units, signatures, target.goarch, target.triple) + }) + } +} diff --git a/amd64_raw_near_return.go b/amd64_raw_near_return.go new file mode 100644 index 000000000..4e3680f50 --- /dev/null +++ b/amd64_raw_near_return.go @@ -0,0 +1,449 @@ +package plan9asm + +import ( + "bytes" + "encoding/binary" + "fmt" + "reflect" + "strings" +) + +type x86RawNearReturnForm struct { + nativeWidth bool + cleanup uint16 +} + +// x86RawStaticRead is produced only by a source-local RIP decoder after its +// complete encoding grammar and access width have been validated. It is not a +// mnemonic whitelist, a general pointer contract, or permission to store. +type x86RawStaticRead struct { + op Op + source int + width int64 + symbol string + offset int64 + object []byte + args []Operand +} + +// Only the normalizer's symbol relocation may change a proved operand. Keep +// an independent snapshot so a caller's edited File cannot authorize a wider +// vector access, a different mask, or an out-of-range proof index. +func rebindX86RawStaticRead(ins *Instr, source int, symbol string) error { + if source < 0 || source >= len(ins.Args) { + return fmt.Errorf("%w: invalid raw static read operand", ErrProbeNeedsContext) + } + read := ins.x86RIPMemoryRead + if read.width != 0 { + if read.source != source || read.op != ins.Op || !reflect.DeepEqual(read.args, ins.Args) { + return fmt.Errorf("%w: changed raw static read cannot inherit decoder proof", ErrProbeNeedsContext) + } + read.args = append([]Operand(nil), read.args...) + read.args[source].Sym = symbol + read.symbol = symbol + ins.x86RIPMemoryRead = read + } + ins.Args[source].Sym = symbol + return nil +} + +// C3 and C2 imm16 use the target's native return-PC width. Operand-size +// overrides and other prefixes require their own physical entry contract; +// successful decoding alone does not make them ordinary Go/C returns. +func x86RawNearReturnEncoding(code []byte) x86RawNearReturnForm { + if len(code) == 1 && code[0] == 0xc3 { + return x86RawNearReturnForm{nativeWidth: true} + } + if len(code) == 3 && code[0] == 0xc2 { + return x86RawNearReturnForm{nativeWidth: true, cleanup: binary.LittleEndian.Uint16(code[1:])} + } + return x86RawNearReturnForm{} +} + +func validateX86RawNearReturns(fn Func, file *File, opt Options, bindFrame bool) error { + hasReturn := false + for _, ins := range fn.Instrs { + form := ins.x86RawNearReturn + if form == nil { + continue + } + hasReturn = true + if !form.nativeWidth || form.cleanup != 0 { + return fmt.Errorf("%w: raw near RET needs a physical return-width and caller-stack cleanup contract: %q", ErrProbeNeedsContext, ins.Raw) + } + } + if !hasReturn { + return nil + } + name := fn.Sym + if opt.ResolveSym != nil { + name = opt.ResolveSym(name) + } + if bindFrame && x86SourceStackBoolCandidate(fn, opt.Sigs[name]) && + (fn.x86SourceStack == nil || fn.x86SourceStack.scalarResult == nil) { + return fmt.Errorf("%w: raw ABI0 bool RET requires its complete typed canonical source-stack proof; %s", ErrProbeNeedsContext, fn.x86SourceStackFailure) + } + // The raw instruction bypasses obj6's RET epilogue. A zero-sized, + // stack-unobserving leaf has the same return-PC location as its caller. + // All other cases need a native frame/continuation contract, not a guessed + // epilogue or an inline RET inside LLVM's independently allocated frame. + if fn.FrameSize != 0 || (!x86RawReturnLeafStackUnobserved(fn) && fn.x86SourceStack == nil) { + diagnostic := "" + if fn.x86SourceStackFailure != "" { + diagnostic = "; " + fn.x86SourceStackFailure + } + return fmt.Errorf("%w: raw near RET requires a proved source stack/continuation; Go's TEXT frame epilogue is not encoded in raw bytes%s", ErrProbeNeedsContext, diagnostic) + } + if !x86RawReturnMemoryBound(fn, file, opt, bindFrame) { + return fmt.Errorf("%w: raw near RET needs bounded typed FP/static memory; ordinary pointers have no native return-PC nonalias contract", ErrProbeNeedsContext) + } + return nil +} + +// This bounded contract does not infer noalias from a Go pointer type or a +// register value. It proves only scalar MOV accesses to bound FP slots or a +// same-file static object, and shared raw decoders' exact source-local reads. +// Other memory forms need their own complete effects and access-range proof. +// In particular, this is not a general store whitelist. +func x86RawReturnMemoryBound(fn Func, file *File, opt Options, bindFrame bool) bool { + name := fn.Sym + if opt.ResolveSym != nil { + name = opt.ResolveSym(name) + } + sig := opt.Sigs[name] + caller := bindFrame && x86CallerNormalizedSourceMatches(fn, sig) + if fn.x86SourceStack != nil && fn.x86SourceStack.caller != nil && !caller { + return false + } + for _, ins := range fn.Instrs { + if ins.x86RawMOVSXDNeedsCaller && !caller { + return false // A native width is not an initialized GP-input proof. + } + read := ins.x86RIPMemoryRead + if read.width != 0 && (opt.Goarch == "386" || !x86RawReturnDecodedReadBound(ins, file)) { + return false + } + op := Op(normalizeInstructionOpcode(ins.Op)) + if _, extension := amd64ScalarExtensionMoveSpecs[op]; extension && !caller && !ins.x86Encoded { + // A named extension gains new source-frame effects only through + // actual typed caller input availability. Keep the old separately + // bound physical/RIP decoder contract distinct from that producer. + return false + } + if _, _, _, memory := x86StringProperties(op); memory { + return false + } + if _, _, memory := x86PortStringProperties(op); memory { + return false + } + if _, memory := amd64ImplicitMaskMoveSpecs[op]; memory || op == "XLAT" { + return false + } + for index, arg := range ins.Args { + switch arg.Kind { + case OpMem: + if caller { + if !x86CallerMemoryCertified(ins, index) && op != "LEAW" && op != "LEAL" && op != "LEAQ" { + return false + } + // Caller pointers require their own exact access certificate, + // including physical-SP imports. The old numerical FP proof + // is not an alternative permission after that certificate is lost. + continue + } + if offset, proved := ins.x86SourceStackFP[index]; bindFrame && fn.x86SourceStack != nil && proved && + x86RawReturnFPBound(offset, x86ScalarMoveMemoryBytes(ins, index, opt.Goarch), sig, opt.Goarch) { + continue // Instruction-local, freshly recomputed physical-SP slot proof. + } + if !bindFrame && fn.x86SourceStack != nil && x86RawReturnStackMemory(arg.Mem) && x86ScalarMoveMemoryBytes(ins, index, opt.Goarch) != 0 { + continue // Pure preparation; final normalization binds the exact slot for lowering. + } + // Even a static symbol plus an index is not a bounded object. + // Address-only register expressions have no dereference, but + // their later uses still have to satisfy this same contract. + if op != "LEAW" && op != "LEAL" && op != "LEAQ" { + return false + } + case OpFP: + if caller { + if !x86CallerMemoryCertified(ins, index) { + return false + } + continue + } + if !bindFrame { + continue // Pure prepartition normalization retains metadata. + } + if fn.x86SourceStack == nil && fn.x86FixedLocal == nil && + x86RawReturnMaskParameterReadBound(ins, index, sig, opt.Goarch) { + continue // Exact logical numeric input bytes, not native frame/pointer permission. + } + width := x86ScalarMoveMemoryBytes(ins, index, opt.Goarch) + if width == 0 || !x86RawReturnFPBound(arg.FPOffset, width, sig, opt.Goarch) { + return false + } + case OpSym: + if read.width != 0 && read.source == index { + continue // Exact source and allocated bytes were checked above. + } + symbol := strings.TrimSpace(arg.Sym) + if strings.HasPrefix(symbol, "$") || ins.Op == OpTEXT || ins.Op == OpLABEL { + continue + } + if op == "LEAW" || op == "LEAL" || op == "LEAQ" { + continue // Address formation, like OpMem LEA; not a read. + } + base, offset, static := parseSBRef(symbol) + if !static { + continue // Non-memory control operands retain their own checks. + } + if !x86RawReturnStaticBound(base, offset, x86ScalarMoveMemoryBytes(ins, index, opt.Goarch), file) { + return false + } + } + } + } + return true +} + +func x86RawReturnDecodedReadIdentity(ins Instr) bool { + read := ins.x86RIPMemoryRead + if !ins.x86Encoded || !ins.x86RIPLiteral || + read.op != ins.Op || read.source < 0 || read.source >= len(ins.Args) || + !reflect.DeepEqual(read.args, ins.Args) || + read.width <= 0 || read.offset < 0 || + read.width > int64(len(read.object)) || read.offset > int64(len(read.object))-read.width { + return false + } + operand := ins.Args[read.source] + if operand.Kind != OpSym || operand.Sym != read.symbol { + return false + } + if len(ins.x86RIPLiteralData) != 0 && + !bytes.Equal(ins.x86RIPLiteralData, read.object[read.offset:read.offset+read.width]) { + return false + } + return true +} + +func x86RawReturnDecodedReadBound(ins Instr, file *File) bool { + if file == nil || !x86RawReturnDecodedReadIdentity(ins) { + return false + } + read := ins.x86RIPMemoryRead + symbol, offset, ok := parseSBRef(ins.Args[read.source].Sym) + if !ok || offset != read.offset || !x86RawReturnStaticBound(symbol, offset, read.width, file) { + return false + } + // Re-normalization and bounded module partitioning retain the data. Do + // not use stale constant-specialization bytes after a caller changes it. + matched := false + for _, datum := range file.Data { + if datum.Sym != symbol { + continue + } + if matched || datum.Off != 0 || datum.Width != int64(len(read.object)) || datum.Addr != "" || + !bytes.Equal(datum.Payload, read.object) { + return false + } + matched = true + } + return matched +} + +func x86RawReturnScalarMoveBytes(op Op) int64 { + switch op { + case "MOVB": + return 1 + case "MOVW": + return 2 + case "MOVL": + return 4 + case "MOVQ": + return 8 + default: + return 0 + } +} + +func x86RawReturnFPBound(offset, width int64, sig FuncSig, goarch string) bool { + if offset < 0 || width <= 0 { + return false + } + const maxOffset = int64(1<<63 - 1) + var checked []FrameSlot + matched := false + for group, slots := range [][]FrameSlot{sig.Frame.Params, sig.Frame.Results} { + for _, slot := range slots { + size := x86RawReturnSlotBytes(slot.Type, goarch) + if !x86RawReturnSlotMatchesSignature(slot, group, sig) || size == 0 || + slot.Offset < 0 || slot.Offset > maxOffset-size { + return false + } + for _, previous := range checked { + previousSize := x86RawReturnSlotBytes(previous.Type, goarch) + if slot.Offset < previous.Offset+previousSize && previous.Offset < slot.Offset+size { + return false + } + } + checked = append(checked, slot) + // Exact whole-slot moves avoid relying on generic partial FP writes + // preserving the untouched high bytes. This does not repair those + // writes elsewhere in the translator. + matched = matched || slot.Offset == offset && width == size + } + } + return matched +} + +func x86RawReturnSlotMatchesSignature(slot FrameSlot, group int, sig FuncSig) bool { + if slot.Index < 0 { + return false + } + path := frameSlotFields(slot) + if group == 0 { + if slot.Index >= len(sig.Args) { + return false + } + typ := sig.Args[slot.Index] + if len(path) != 0 { + // Reuse the existing flat literal-aggregate grammar. Nested paths + // and arrays need a richer validated binding, not a new guessed + // layout parser inside the native-return proof. + fields, aggregate := parseLiteralStructFields(typ) + if !aggregate || len(path) != 1 || path[0] < 0 || path[0] >= len(fields) { + return false + } + typ = fields[path[0]] + } + return slot.Type == typ + } + // Classic result slots index the already flattened return tuple. Field + // paths are parameter extraction metadata, not a result-slot binding. + if sig.Ret == Void || sig.Ret == "" || len(path) != 0 { + return false + } + if fields, aggregate := parseLiteralStructFields(sig.Ret); aggregate { + return slot.Index < len(fields) && slot.Type == fields[slot.Index] + } + return slot.Index == 0 && slot.Type == sig.Ret +} + +func x86RawReturnSlotBytes(typ LLVMType, goarch string) int64 { + return x86FrameScalarBytes(typ, goarch) +} + +func x86RawReturnStaticBound(symbol string, offset, width int64, file *File) bool { + if file == nil || offset < 0 || width <= 0 { + return false + } + var size int64 + for _, object := range file.Globl { + if object.Sym != symbol { + continue + } + if object.SizeRaw != "" || object.Size < 0 || object.Size > maxDataGlobalSize { + return false + } + if object.Size > size { + size = object.Size + } + } + // Match the actual DATA allocation consumer. DATA without a GLOBL still + // creates a same-file byte object; holes are initialized to zero. Reuse + // its existing checked extent rather than guessing a fixed object size. + for _, datum := range file.Data { + if datum.Sym != symbol { + continue + } + end, err := dataStmtEnd(datum) + if err != nil { + return false + } + if end > size { + size = end + } + } + return size >= width && offset <= size-width +} + +func x86RawReturnLeafStackUnobserved(fn Func) bool { + hasText := false + for _, ins := range fn.Instrs { + if ins.Op == OpTEXT { + hasText = true + _, rest := splitOpcode(ins.Raw) + parts := strings.Split(rest, ",") + if len(parts) == 3 { + expression := globlFlagName.ReplaceAllStringFunc(parts[1], func(name string) string { + if value, ok := goTextFlagValues[name]; ok { + return value + } + return name + }) + flags, known := parseImmExpr(strings.TrimSpace(expression)) + if !known || flags & ^uint64(2|4|512) != 0 { + return false + } + } else if len(parts) != 2 { + return false + } + continue + } + if x86SourceGOperand(ins) { + // Stack-unobserving does not mean native-context-unobserving. + // Incoming g is opaque; numerical use or clobber needs the same + // complete caller and exact saved-identity proof as a framed leaf. + return false + } + op := strings.ToUpper(string(ins.Op)) + if strings.HasPrefix(op, "PUSH") || strings.HasPrefix(op, "POP") || + strings.HasPrefix(op, "IRET") || strings.HasPrefix(op, "RETF") { + return false + } + switch Op(op) { + case OpCALL, "ADJSP", "ENTER", "ENTERL", "ENTERQ", "ENTERW", "LEAVEL", "LEAVEQ", "LEAVEW": + return false + case OpRET: + if len(ins.Args) != 0 { + return false + } + case OpJMP: + if len(ins.Args) != 1 || (ins.Args[0].Kind != OpIdent && ins.Args[0].Kind != OpLabel) { + return false + } + } + for _, operand := range ins.Args { + if operand.Kind == OpFPAddr || (operand.Kind == OpFP && operand.FPOffset < 0) { + return false + } + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + return false + } + if operand.Kind == OpMem && x86RawReturnStackMemory(operand.Mem) { + return false + } + // Go's effective-address immediate is OpSym, not OpMem. Use the + // same parser and typed register roots as its actual lowering. + if operand.Kind == OpSym { + symbol := strings.TrimSpace(operand.Sym) + if strings.HasPrefix(symbol, "$") { + address, ok := parseMem(strings.TrimSpace(strings.TrimPrefix(symbol, "$"))) + if ok && x86RawReturnStackMemory(address) { + return false + } + } + } + } + } + return hasText +} + +func x86RawReturnStackRegister(reg Reg) bool { + root, known := amd64FullRegBase(reg) + return known && (root == SP || root == BP) +} + +func x86RawReturnStackMemory(memory MemRef) bool { + return x86RawReturnStackRegister(memory.Base) || x86RawReturnStackRegister(memory.Index) +} diff --git a/amd64_raw_near_return_runtime_test.go b/amd64_raw_near_return_runtime_test.go new file mode 100644 index 000000000..0676e27cb --- /dev/null +++ b/amd64_raw_near_return_runtime_test.go @@ -0,0 +1,146 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// Observe C2's real caller-SP effect with a source-native continuation. The +// trampoline restores the original SP before returning to the Go runtime. +// It is an independent machine-code witness, not a translated C tail call. +func x86RawNearReturnGoStackOracle(t *testing.T, arch string, runPrefix []string) { + t.Helper() + move, sub, add, address := "MOVQ", "SUBQ", "ADDQ", "LEAQ" + wordBytes := 8 + if arch == "386" { + move, sub, add, address = "MOVL", "SUBL", "ADDL", "LEAL" + wordBytes = 4 + } + var assembly strings.Builder + for _, cleanup := range []int{0, 4} { + fmt.Fprintf(&assembly, `TEXT ·Measure%d(SB),516,$0-%d +%s SP,BX +%s $16,SP +%s resume%d<>(SB),AX +%s AX,0(SP) +JMP native%d<>(SB) +TEXT native%d<>(SB),516,$0-0 +BYTE $0xc2 +WORD $%d +TEXT resume%d<>(SB),516,$0-0 +%s SP,AX +%s BX,AX +%s $%d,AX +%s BX,SP +%s AX,ret+0(FP) +RET +`, cleanup, wordBytes, move, sub, address, cleanup, move, cleanup, cleanup, + cleanup, cleanup, move, sub, add, 16-wordBytes, move, move) + } + for _, encoding := range []struct{ name, code string }{ + {"C3", "BYTE $0xc3"}, {"C2", "BYTE $0xc2\nWORD $0"}, + } { + fmt.Fprintf(&assembly, `TEXT ·Result%s(SB),4,$0-4 +CALL value%s<>(SB) +MOVL AX,ret+0(FP) +RET +TEXT value%s<>(SB),4,$0-0 +MOVL $17,AX +%s +RET +`, encoding.name, encoding.name, encoding.name, encoding.code) + } + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module example.com/rawreturn\n\ngo 1.20\n", + "oracle_" + arch + ".s": assembly.String(), + "oracle_test.go": `package rawreturn +import "testing" +func Measure0() int +func Measure4() int +func ResultC2() int32 +func ResultC3() int32 +func TestActualMachineStack(t *testing.T) { + if delta := Measure0(); delta != 0 { t.Fatalf("C2 zero cleanup: SP delta %d", delta) } + if delta := Measure4(); delta != 4 { t.Fatalf("C2 imm16=4: SP delta %d", delta) } + if ResultC2() != 17 || ResultC3() != 17 { t.Fatal("zero-cleanup raw return lost AX") } +} +`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + args := []string{"test", "-vet=off", "-count=1"} + if len(runPrefix) != 0 { + args = append(args, "-exec", strings.Join(runPrefix, " ")) + } + args = append(args, ".") + cmd := exec.Command("go", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH="+arch, "CGO_ENABLED=0", "GODEBUG=asyncpreemptoff=1") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("actual Go %s raw-return stack oracle: %v\n%s", arch, err, out) + } +} + +func TestCrossLinuxRuntimeMatrixX86RawNearReturn(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("x86 source-stack oracle runs in the required Linux/amd64 cross-runtime gate") + } + x86RawNearReturnGoStackOracle(t, "amd64", nil) + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var runPrefix []string + if crossRosetta { + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + var source strings.Builder + for _, encoding := range []struct{ name, code string }{ + {"rawC3", "BYTE $0xc3"}, {"rawC2", "BYTE $0xc2\nWORD $0"}, + } { + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\nMOVL $17,AX\n%s\nRET\n", encoding.name, encoding.code) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawC3": {Name: "rawC3", Ret: I32}, "rawC2": {Name: "rawC2", Ret: I32}}}) + if err != nil { + t.Fatal(err) + } + const main = `#include +extern int32_t rawC2(void), rawC3(void); +int main(void) { + for (unsigned i=0; i<1024; i++) { + volatile uint64_t canary = UINT64_C(0x123456789abcdef0); + if (rawC2()!=17 || rawC3()!=17 || canary!=UINT64_C(0x123456789abcdef0)) return 1; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_near_return", triple, ir, main, runPrefix) + if os.Getenv("PLAN9ASM_CROSS_EXEC") == "1" { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("PLAN9ASM_CROSS_EXEC raw-return matrix requires Linux/amd64") + } + x86RawNearReturnGoStackOracle(t, "386", []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"}) + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawC3": {Name: "rawC3", Ret: I32}, "rawC2": {Name: "rawC2", Ret: I32}}}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestWithCompiler(t, llc, []string{"i686-linux-gnu-gcc", "-no-pie"}, + "raw_near_return386", "i386-unknown-linux-gnu", ir, main, []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"}) + } +} diff --git a/amd64_raw_near_return_test.go b/amd64_raw_near_return_test.go new file mode 100644 index 000000000..a15fdc490 --- /dev/null +++ b/amd64_raw_near_return_test.go @@ -0,0 +1,218 @@ +package plan9asm + +import ( + "encoding/binary" + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86RawNearReturnCompleteImmediateEncoding(t *testing.T) { + if form := x86RawNearReturnEncoding([]byte{0xc3}); !form.nativeWidth || form.cleanup != 0 { + t.Fatalf("C3 has no caller cleanup: %+v", form) + } + code := []byte{0xc2, 0, 0} + for cleanup := 0; cleanup <= 65535; cleanup++ { + binary.LittleEndian.PutUint16(code[1:], uint16(cleanup)) + form := x86RawNearReturnEncoding(code) + if !form.nativeWidth || form.cleanup != uint16(cleanup) { + t.Fatalf("C2 imm16=%d was changed: %+v", cleanup, form) + } + } + for _, code := range [][]byte{nil, {0xc2}, {0xc2, 0}, {0xcb}, {0xca, 0, 0}, {0x66, 0xc3}, {0xf2, 0xc3}, {0xf3, 0xc3}, {0x48, 0xc3}} { + if x86RawNearReturnEncoding(code).nativeWidth { + t.Errorf("unproved/truncated near return %x became an ordinary return", code) + } + } +} + +var x86RawReturnTargets = []struct { + arch string + triple string +}{ + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, +} + +func TestX86RawNearReturnNeedsNativeStackContract(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, cleanup := range []uint16{1, 4, 255, 256, 65535} { + t.Run(fmt.Sprintf("%s/pop%d", target.triple, cleanup), func(t *testing.T) { + // Unlike textual RET $n, these are valid Go BYTE/WORD inputs. + // C2 pops the return PC and then adds imm16 to the caller SP. + source := fmt.Sprintf("TEXT rawret(SB),4,$0-0\nBYTE $0xc2\nWORD $%d\nRET\n", cleanup) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawret": {Name: "rawret", Ret: Void}}} + if _, err := Translate(file, opt); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("native caller cleanup=%d must remain Context, got %v", cleanup, err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module API lost native cleanup=%d contract, got %v", cleanup, err) + } + if err := ProbeInstructionSequence(ArchAMD64, target.arch, file.Funcs[0].Instrs[1:3]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("isolated raw cleanup became an ordinary return: %v", err) + } + }) + } + } +} + +func TestX86RawNearReturnCannotInventGoEpilogue(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, push, pop := "MOVQ", "PUSHQ", "POPQ" + if target.arch == "386" { + move, push, pop = "MOVL", "PUSHL", "POPL" + } + for _, source := range []string{ + "TEXT rawret(SB),4,$8-0\nBYTE $0xc3\nRET\n", + "TEXT rawret(SB),4,$0-0\n" + move + " $0,0(SP)\nBYTE $0xc3\nRET\n", + "TEXT rawret(SB),4,$0-0\n" + push + " AX\nBYTE $0xc3\n" + pop + " AX\nRET\n", + "TEXT rawret(SB),4,$0-0\nBYTE $0x66\nBYTE $0xc3\nRET\n", + "TEXT rawret(SB),4,$0-0\nBYTE $0xf2\nBYTE $0xc3\nRET\n", + "TEXT rawret(SB),4,$0-0\nBYTE $0xf3\nBYTE $0xc3\nRET\n", + } { + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawret": {Name: "rawret", Ret: Void}}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s raw machine return cannot acquire a Go epilogue or normal stack width; got %v\n%s", target.triple, err, source) + } + } + } +} + +func x86RawStackObservingFormProbeFile(t *testing.T, arch, symbol string, code []byte) *File { + t.Helper() + return x86RawUnprovedReturnFormProbeFile(t, arch, symbol, code) +} + +// Retain the original Go encoder object as a negative return-contract test. +// A compile-only operand probe ends in UD2: it cannot promise a caller return +// after observing a physical stack or dereferencing unbound native pointers. +// Every preceding instruction byte is unchanged. These objects prove operand +// lowering, not execution of the original machine-entry function. +func x86RawUnprovedReturnFormProbeFile(t *testing.T, arch, symbol string, code []byte) *File { + t.Helper() + if len(code) == 0 || code[len(code)-1] != 0xc3 { + t.Fatalf("Go form fixture must end in its ordinary encoded RET, got %x", code) + } + makeFile := func(bytes []byte) *File { + var source strings.Builder + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", symbol) + for _, value := range bytes { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + return file + } + original := makeFile(code) + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + _, err := Translate(original, Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{symbol: {Name: symbol, Ret: Void}}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("original format fixture must retain its native return-contract failure, got %v", err) + } + if !strings.Contains(err.Error(), "source stack/continuation") && + !strings.Contains(err.Error(), "bounded typed FP/static memory") { + t.Fatalf("unrelated Context error is not a native return-contract witness: %v", err) + } + probe := append(append([]byte(nil), code[:len(code)-1]...), 0x0f, 0x0b) + return makeFile(probe) +} + +func TestX86RawUnprovedReturnFormProbePreservesInstructions(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + for _, instruction := range []string{"MOVL (AX),CX", "MOVL CX,(AX)"} { + t.Run(target.triple+"/"+instruction, func(t *testing.T) { + code := assembleX87ControlBytes(t, target.arch, + "TEXT form(SB),4,$0-0\n"+instruction+"\nRET\n") + original := append([]byte(nil), code...) + file := x86RawUnprovedReturnFormProbeFile(t, target.arch, "form", code) + if string(code) != string(original) { + t.Fatal("form probe modified its independent Go encoder input") + } + body := file.Funcs[0].Instrs[1:] + if len(body) != len(code)+1 { + t.Fatalf("probe directives=%d, want %d", len(body), len(code)+1) + } + want := append(append([]byte(nil), code[:len(code)-1]...), 0x0f, 0x0b) + for index, ins := range body { + if ins.Op != OpBYTE || len(ins.Args) != 1 || + ins.Args[0].Kind != OpImm || ins.Args[0].Imm != int64(want[index]) { + t.Fatalf("changed operand-form byte %d: %+v, want %#x", index, ins, want[index]) + } + } + // Replacing the unproved return with UD2 does not initialize + // AX/CX or authorize the preceding native load/store. Retain + // the public Context controls and private operand-format object. + ir := x86UnboundVoidRawProbeObject(t, file, target.arch, target.triple) + if !strings.Contains(ir, `asm sideeffect "ud2"`) || !strings.Contains(ir, "unreachable") { + t.Fatal("compile-only form probe invented an ordinary caller return") + } + compileLLVMToObject(t, llc, target.triple, "memory-form.ll", "memory-form.o", ir) + }) + } + } +} + +func TestX86RawNearReturnZeroCleanupLeafObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + for _, encoding := range []string{"BYTE $0xc3", "BYTE $0xc2\nWORD $0"} { + t.Run(target.triple+"/"+strings.ReplaceAll(encoding, "\n", ";"), func(t *testing.T) { + source := "TEXT rawret(SB),4,$0-0\nMOVL $17,AX\n" + encoding + "\nRET\n" + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawret": {Name: "rawret", Ret: I32}}, + }) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, target.triple, "raw-near-ret.ll", "raw-near-ret.o", module.String()) + }) + } + } +} diff --git a/amd64_raw_nop.go b/amd64_raw_nop.go new file mode 100644 index 000000000..21d194a8d --- /dev/null +++ b/amd64_raw_nop.go @@ -0,0 +1,50 @@ +package plan9asm + +import "golang.org/x/arch/x86/x86asm" + +// The decoder discards NOP's inert ModRM/SIB fields only after validating the +// complete encoding. Preserve existing source facts through that exact handoff, +// not through a different physical instruction or an unbound address contract. +// This observation creates no numerical, pointer, memory or raw-entry fact. +func x86NOPPreservesSourceState(ins Instr) bool { + if ins.Op != "NOP" || len(ins.Args) != 0 || ins.x86Encoded || ins.x86RawNearReturn != nil || + ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 { + return false + } + origin := ins.x86SourceRaw + return origin == nil || origin.op == "NOP" && len(origin.args) == 0 +} + +// The pinned decoder lacks the REX.W form of 0F 1F /0 (LLVM 22 NOOPQ). +// Only its operand width is inert. Keep every other prefix and use the decoder +// for the complete ModRM/SIB/displacement length and prefix validity. This +// narrowly scoped adapter never changes 90's XCHG/PAUSE aliases or another +// opcode, and its result is consumed only by the operand-free NOP handoff. +func decodedX86REXWNOP(code []byte) (x86asm.Inst, bool) { + if len(code) > 15 { + code = code[:15] + } + i := 0 + for i < len(code) { + switch code[i] { + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65, 0x66, 0x67, 0xf0, 0xf2, 0xf3: + i++ + default: + goto opcode + } + } + +opcode: + if i+4 > len(code) || code[i]&0xf8 != 0x48 || + code[i+1] != 0x0f || code[i+2] != 0x1f || code[i+3]&0x38 != 0 { + return x86asm.Inst{}, false + } + var adjusted [15]byte + copy(adjusted[:], code) + adjusted[i] &^= 8 + inst, err := x86asm.Decode(adjusted[:len(code)], 64) + if err != nil || inst.Op != x86asm.NOP || inst.Len <= i+3 { + return x86asm.Inst{}, false + } + return inst, true +} diff --git a/amd64_raw_nop_handoff_test.go b/amd64_raw_nop_handoff_test.go new file mode 100644 index 000000000..fe5531a35 --- /dev/null +++ b/amd64_raw_nop_handoff_test.go @@ -0,0 +1,207 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Intel SDM NOP (90 and 0F 1F /0) has no register or memory effect. +// Independently enumerate ModRM/SIB and prefix axes, including addresses +// which would need a caller, segment, stack or PC contract if dereferenced. +func x86RawNOPWitnesses(mode int) [][]byte { + forms := [][]byte{{0x90}, {0x66, 0x90}} + for mod := byte(0); mod < 4; mod++ { + for rm := byte(0); rm < 8; rm++ { + code := []byte{0x0f, 0x1f, mod<<6 | rm} + if mod != 3 && rm == 4 { + code = append(code, 0x24) + } + switch { + case mod == 1: + code = append(code, 0xff) + case mod == 2 || mod == 0 && rm == 5: + code = append(code, 0xff, 0xff, 0xff, 0x7f) + } + forms = append(forms, code) + } + } + for sib := 0; sib < 256; sib++ { + code := []byte{0x0f, 0x1f, 0x04, byte(sib)} + if sib&7 == 5 { + code = append(code, 0xff, 0xff, 0xff, 0x7f) + } + forms = append(forms, code) + } + for _, prefix := range []byte{0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65, 0x66} { + forms = append(forms, []byte{prefix, 0x0f, 0x1f, 0x00}) + } + forms = append(forms, []byte{0x66, 0x0f, 0x1f, 0x84, 0x00, 0, 0, 0, 0}) + if mode == 32 { + // The 16-bit address decoder must not manufacture BX/SI inputs. + forms = append(forms, []byte{0x67, 0x0f, 0x1f, 0x00}) + } else { + // Inert EIP-relative address and actual extended REX banks. + forms = append(forms, []byte{0x67, 0x0f, 0x1f, 0x05, 0xff, 0xff, 0xff, 0x7f}) + for rex := byte(0x40); rex < 0x50; rex++ { + forms = append(forms, []byte{rex, 0x0f, 0x1f, 0x04, 0x00}) + } + } + return forms +} + +func TestX86RawNOPPhysicalFieldsAreInert(t *testing.T) { + for _, mode := range []int{32, 64} { + arch := "amd64" + if mode == 32 { + arch = "386" + } + forms := x86RawNOPWitnesses(mode) + var complete []byte + for _, code := range forms { + complete = append(complete, code...) + } + x86HandoffOriginalBytes(t, arch, complete) + for _, code := range forms { + t.Run(fmt.Sprintf("mode%d/%x", mode, code), func(t *testing.T) { + physical, err := x86asm.Decode(code, mode) + // The independent literal object and runtime oracle cover the + // pinned decoder's missing REX.W NOP row; do not use the + // production adapter as its own acceptance oracle. + missingWidth := mode == 64 && code[0]&0xf8 == 0x48 + if !missingWidth && (err != nil || physical.Op != x86asm.NOP || physical.Len != len(code)) { + t.Fatalf("independent NOP encoding: %+v %v", physical, err) + } + instructions, err := decodeX86RawDirectiveGroup(code, mode, 0, "NOP witness", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(instructions) != 1 || instructions[0].Op != "NOP" || len(instructions[0].Args) != 0 || + instructions[0].x86UnboundAddressOverride || instructions[0].x86AddressBits != 0 { + t.Fatalf("inert address acquired an effect: %+v", instructions) + } + }) + } + } +} + +func TestX86RawNOPGoEncoderWitnesses(t *testing.T) { + table, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, registration := range []string{ + "var ydivl = []ytab{\n\t{Zm_o, 2, argList{Yml}},\n}", + "{ANOPW, ydivl, Pe, opBytes{0x0f, 0x1f, 00}}", + "{ANOPL, ydivl, Px, opBytes{0x0f, 0x1f, 00}}", + } { + if !strings.Contains(string(table), registration) { + t.Fatal("Go NOP encoder changed:", registration) + } + } + for _, arch := range []string{"386", "amd64"} { + for _, spelling := range []string{"NOPL AX", "NOPW AX", "NOPL (AX)", "NOPW -1(AX)(CX*8)"} { + code := x86OrderingHintGoBytes(t, arch, "TEXT probe(SB),4,$0-0\n"+spelling+"\n") + mode := 64 + if arch == "386" { + mode = 32 + } + decoded, err := decodeX86RawDirectiveGroup(code, mode, 0, spelling, map[string]bool{}) + if err != nil || len(decoded) != 1 || decoded[0].Op != "NOP" { + t.Fatalf("actual %s %s bytes %x: %+v %v", arch, spelling, code, decoded, err) + } + } + } +} + +func TestX86RawNOPDoesNotEraseOtherEffectsOrBoundaries(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, code := range [][]byte{ + {0xf0, 0x90}, {0xf0, 0x0f, 0x1f, 0x00}, + {0xf0, 0x48, 0x0f, 0x1f, 0x00}, + {0x0f}, {0x0f, 0x1f}, {0x0f, 0x1f, 0x84, 0x00}, + {0x48, 0x0f, 0x1f, 0x84, 0x00}, + } { + if _, err := decodeX86RawDirectiveGroup(code, mode, 0, "invalid NOP", map[string]bool{}); err == nil { + t.Fatalf("mode %d accepted invalid/truncated bytes %x", mode, code) + } + } + for extension := byte(1); extension < 8; extension++ { + code := []byte{0x0f, 0x1f, extension << 3} + if _, err := decodeX86RawDirectiveGroup(code, mode, 0, "reserved NOP extension", map[string]bool{}); err == nil { + t.Fatalf("reserved /%d NOP accepted", extension) + } + if _, ok := decodedX86REXWNOP(append([]byte{0x48}, code...)); ok { + t.Fatalf("REX.W adapter accepted reserved /%d", extension) + } + } + for _, code := range [][]byte{{0xf3, 0x90}, {0x87, 0xc0}, {0x8b, 0x00}} { + instructions, err := decodeX86RawDirectiveGroup(code, mode, 0, "not NOP", map[string]bool{}) + if err != nil || len(instructions) != 1 || instructions[0].Op == "NOP" { + t.Fatalf("mode %d lost non-NOP effect %x: %+v %v", mode, code, instructions, err) + } + } + // A branch into a NOP's ModRM byte is still a mid-instruction edge. + interior := []byte{0x74, 1, 0x0f, 0x1f, 0x00, 0xc3} + if _, err := decodeX86RawDirectiveGroup(interior, mode, 0, "interior", map[string]bool{}); err == nil { + t.Fatal("branch into multi-byte NOP accepted") + } + boundary := []byte{0x74, 3, 0x0f, 0x1f, 0x00, 0x90, 0xc3} + if _, err := decodeX86RawDirectiveGroup(boundary, mode, 0, "boundary", map[string]bool{}); err != nil { + t.Fatal("valid branch/NOP boundaries:", err) + } + // The generic physical handoff remains strict for real memory. + physical, err := x86asm.Decode([]byte{0x8b, 0x00}, mode) + if err != nil { + t.Fatal(err) + } + if _, err := parseDecodedX86PhysicalInstruction(physical, "NOP"); err == nil { + t.Fatal("formatter erased a real memory read") + } + } +} + +func TestX86RawNOPThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86OrderingHintTargets { + mode := 64 + if target.arch == "386" { + mode = 32 + } + var source strings.Builder + source.WriteString("TEXT ·probe(SB),4,$0-4\n") + for _, code := range x86RawNOPWitnesses(mode) { + source.WriteString(x86RawDisplacementDirectives(code)) + } + source.WriteString("MOVL $73,ret+0(FP)\nRET\n") + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), "package ordering\nfunc probe() uint32\n", target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + if bytes.Contains([]byte(ir), []byte("inttoptr")) { + t.Fatal("inert NOP address became a native pointer") + } + for _, level := range []string{"-O0", "-O2"} { + dir := t.TempDir() + input, object := filepath.Join(dir, "nop.ll"), filepath.Join(dir, "nop.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+target.triple, "-filetype=obj", "-o", object, input)) + } + }) + } + } +} diff --git a/amd64_raw_nop_result_go127_test.go b/amd64_raw_nop_result_go127_test.go new file mode 100644 index 000000000..dfef90aa0 --- /dev/null +++ b/amd64_raw_nop_result_go127_test.go @@ -0,0 +1,12 @@ +//go:build go1.27 + +package plan9asm + +import "testing" + +// The discovering current-Go source combines PCALIGN with literal NOP padding. +// Go 1.20 rejects PCALIGN; the untagged root still checks every literal NOP and +// paired native-read contract on both the historical and current SDKs. +func TestX86RawNOPResultReadWithCurrentGoAlignment(t *testing.T) { + x86RawNOPResultReadComposition(t, true) +} diff --git a/amd64_raw_nop_result_runtime_unix_test.go b/amd64_raw_nop_result_runtime_unix_test.go new file mode 100644 index 000000000..27b93ae29 --- /dev/null +++ b/amd64_raw_nop_result_runtime_unix_test.go @@ -0,0 +1,72 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86RawNOPPairedNativeRead(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + literal := x86RawDisplacementDirectives([]byte{0x66, 0x0f, 0x1f, 0x84, 0, 0, 0, 0, 0}) + var source, declarations, prototypes, originals, rows strings.Builder + count := 0 + for _, vector := range []bool{false, true} { + for _, dead := range []bool{false, true} { + name, original := fmt.Sprintf("F%d", count), fmt.Sprintf("Original%d", count) + read, width, inputByte, outputByte := "MOVQ (DX),AX\n", 8, 0, 0 + if vector { + read, width, inputByte, outputByte = "MOVOU (DX),X1\nMOVQ X1,AX\n", 16, 1, 1 + } + body := literal + read + literal + discarded := 0 + if dead { + body += "MOVQ $0x87654321fedcba98,AX\n" + discarded = 1 + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),DX\n%sMOVQ AX,ret+8(FP)\nRET\n", name, body) + fmt.Fprintf(&declarations, "func %s(p *[64]byte) uint64\n", name) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+strings.ReplaceAll(body, "(DX)", "(DI)")+"RET\n") + var encoded []string + for _, b := range code { + encoded = append(encoded, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(encoded, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\nextern uint64_t %s(const uint8_t*);\n", name, original) + fmt.Fprintf(&rows, "{%s,%s,%d,%d,%d,%d},\n", name, original, width, discarded, inputByte, outputByte) + count++ + } + } + if count != 4 { + t.Fatal("incomplete scalar/vector live/dead matrix", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf(x86NativeScalarReadRuntimeC, prototypes.String(), originals.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text, level := ir, "-O0" + if optimized { + text, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + output := execution.compileAndRunLLVM(t, "nop-native-read", text, driver, level) + if string(output) != "4 native scalar kernels: numerical 512, faults 48, tripwires 2; failures 0\n" { + t.Fatal("original NOP/read composition differs from LLVM", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } +} diff --git a/amd64_raw_nop_result_test.go b/amd64_raw_nop_result_test.go new file mode 100644 index 000000000..7fa090869 --- /dev/null +++ b/amd64_raw_nop_result_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86RawNOPResultReadComposition(t *testing.T) { + x86RawNOPResultReadComposition(t, false) +} + +func x86RawNOPResultReadComposition(t *testing.T, align bool) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + mode, word, move := 64, 8, "MOVQ" + if target.arch == "386" { + mode, word, move = 32, 4, "MOVL" + } + var literals strings.Builder + if align { + literals.WriteString("PCALIGN $32\n") + } + for _, code := range x86RawNOPWitnesses(mode) { + literals.WriteString(x86RawDisplacementDirectives(code)) + } + body := fmt.Sprintf("%s p+0(FP),DX\n%sVMOVDQU (DX),Y1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+%d(FP)\nRET", + move, literals.String(), word) + _, file, sig, source := x86ResultInitializationFixture(t, target.arch, "func Value(p *[64]byte) uint32", body) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + normalized, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + if err := CheckX86ABI0ResultInitialization(normalized, sig, target.arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("inert raw bytes granted a standalone native read", err) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[64]byte) uint32", target.arch, target.triple, api) + if err != nil { + t.Fatal("inert NOPs blocked the independently paired native read", api, err) + } + compileLLVMToObject(t, llc, target.triple, "nop-result.ll", "nop-result.o", ir) + compileLLVMToObject(t, llc, target.triple, "nop-result-opt.ll", "nop-result-opt.o", x86MulDivRoundOptimize(t, ir)) + } + }) + } +} + +func TestX86RawNOPResultReadRetainsRawBoundaries(t *testing.T) { + literal := x86RawDisplacementDirectives([]byte{0x66, 0x0f, 0x1f, 0x84, 0, 0, 0, 0, 0}) + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[64]byte) uint32", + "MOVQ p+0(FP),DX\n"+literal+"VMOVDQU (DX),Y1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+8(FP)\nRET") + normalized, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + nop := -1 + for i, ins := range normalized.Instrs { + if ins.x86SourceRaw != nil { + if nop != -1 || ins.Op != "NOP" { + t.Fatal("fixture did not decode one inert raw instruction") + } + nop = i + } + } + if nop == -1 { + t.Fatal("missing actual raw NOP handoff") + } + for _, mutation := range []string{"original opcode", "original operands", "normalized opcode", "normalized operands", "encoded", "address size", "raw return", "raw TEXT", "source stack", "fixed local"} { + t.Run(mutation, func(t *testing.T) { + fn := normalized + fn.Instrs = append([]Instr(nil), fn.Instrs...) + ins := &fn.Instrs[nop] + origin := *ins.x86SourceRaw + ins.x86SourceRaw = &origin + switch mutation { + case "original opcode": + origin.op = "PAUSE" + case "original operands": + origin.args = []Operand{{Kind: OpReg, Reg: AX}} + case "normalized opcode": + ins.Op = "PAUSE" + case "normalized operands": + ins.Args = []Operand{{Kind: OpReg, Reg: AX}} + case "encoded": + ins.x86Encoded = true + case "address size": + ins.x86AddressBits = 32 + case "raw return": + ins.x86RawNearReturn = new(x86RawNearReturnForm) + case "raw TEXT": + fn.X86RawText = []byte{0x90} + case "source stack": + fn.x86SourceStack = new(x86SourceStackProof) + case "fixed local": + fn.x86FixedLocal = new(x86FixedLocalProof) + } + ctx := &amd64Ctx{fpScalarSource: fn, sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("inert padding granted another raw or entry contract", err) + } + }) + } +} diff --git a/amd64_raw_nop_runtime_test.go b/amd64_raw_nop_runtime_test.go new file mode 100644 index 000000000..d91a00488 --- /dev/null +++ b/amd64_raw_nop_runtime_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +// Reuse only the independent GP/FLAGS/MXCSR/canary driver. Replace every tested +// instruction with a disjoint part of the complete literal-NOP witness set. +// Noncanonical/null effective addresses must not fault or change native state. +func x86RawNOPRuntimeFixture(bits int) (declarations, assembly, driver, reference string) { + declarations, assembly, driver, reference = x86OrderingHintRuntimeFixture(bits) + var batches [4]strings.Builder + for i, code := range x86RawNOPWitnesses(bits) { + batches[i%len(batches)].WriteString(x86RawDisplacementDirectives(code)) + } + for i, op := range []string{"LFENCE", "MFENCE", "SFENCE", "PAUSE"} { + assembly = strings.ReplaceAll(assembly, "\n"+op+"\n", "\n"+batches[i].String()) + } + return +} + +// Portable LLVM execution is distinct from the required original-Go native +// roots. Rosetta rejects some valid register NOPs (for example 0F 1F C2), so +// it cannot certify this family's original machine execution. +func TestX86RawNOPLLVMRuntime(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, assembly, _, reference := x86RawNOPRuntimeFixture(64) + ir := x86NarrowOracleIR(t, assembly, declarations, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "nop", ir, reference, level) + if string(output) != "ordering observations 256\n" { + t.Fatalf("LLVM NOP GP/FLAGS/MXCSR/canary oracle: %q", output) + } + }) + } +} + +func TestX86RawNOPOriginal386StaticELFCompileOnly(t *testing.T) { + declarations, assembly, driver, _ := x86RawNOPRuntimeFixture(32) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go IA32 NOP ELF bytes=%d SHA256=%x; compile only", sealed.size, sealed.hash) +} diff --git a/amd64_raw_operand_handoff.go b/amd64_raw_operand_handoff.go new file mode 100644 index 000000000..9d019d6d8 --- /dev/null +++ b/amd64_raw_operand_handoff.go @@ -0,0 +1,205 @@ +package plan9asm + +import ( + "fmt" + "strings" + + "golang.org/x/arch/x86/x86asm" +) + +// parseDecodedX86PhysicalInstruction is the private decoder handoff, not a Go +// source grammar extension. GoSyntax orders operands and selects the audited +// mnemonic; the actual decoder Mem supplies the physical address. In +// particular, FS:..., default DS/ES, and RIP are not source register groups. +// Prefix, PC-relative layout, and address-size admission remain the caller's +// independent checks. No memory/caller/frame certificate is manufactured here. +func parseDecodedX86PhysicalInstruction(inst x86asm.Inst, syntax string) ([]Instr, error) { + physical := normalizeDecodedX86LEASegments(normalizeDecodedX86MemoryDisplacements(inst)) + const marker = "__plan9asm_decoded_memory_" + if strings.Contains(syntax, marker) { + return nil, fmt.Errorf("decoded syntax collides with private memory placeholder") + } + type memoryBinding struct { + name string + operand Operand + } + bindings := make(map[string]memoryBinding) + byName := make(map[string]Operand) + memoryCount := 0 + for _, arg := range physical.Args { + memory, ok := arg.(x86asm.Mem) + if !ok { + continue + } + operand, err := decodedX86PhysicalMemory(memory) + if err != nil { + return nil, err + } + name := fmt.Sprintf("%s%d", marker, memoryCount) + memoryCount++ + byName[name] = operand + // Use the same pinned formatter only to locate this complete operand + // token. Never interpret its text as a public memory/address role. + probe := physical + probe.Op = x86asm.NOP + probe.Prefix = x86asm.Prefixes{} + probe.Args = x86asm.Args{memory} + text := x86asm.GoSyntax(probe, 0, nil) + space := strings.IndexByte(text, ' ') + if space < 0 { + return nil, fmt.Errorf("decoder formatter omitted its physical memory operand") + } + text = text[space+1:] + spellings := []string{text, strings.TrimPrefix(normalizeDecodedX86SegmentMemorySyntax("NOP "+text), "NOP ")} + for _, spelling := range spellings { + if prior, exists := bindings[spelling]; exists && prior.operand.Mem != operand.Mem { + return nil, fmt.Errorf("ambiguous decoded physical memory %q", spelling) + } + bindings[spelling] = memoryBinding{name: name, operand: operand} + } + } + for _, prefix := range physical.Prefix { + if memoryCount == 0 || prefix&(x86asm.PrefixIgnored|x86asm.PrefixImplicit) != 0 { + continue + } + switch prefix & 0xff { + case x86asm.PrefixES, x86asm.PrefixCS, x86asm.PrefixSS, x86asm.PrefixDS, x86asm.PrefixFS, x86asm.PrefixGS: + bound := false + for _, argument := range physical.Args { + if memory, ok := argument.(x86asm.Mem); ok && memory.Segment.String() == prefix.String() { + bound = true + } + } + if !bound { + // A later operand-size prefix can hide this unused segment + // in GoSyntax. Absence from text is not an ignored witness. + return nil, fmt.Errorf("%w: raw %s has an unbound physical %s segment prefix", ErrProbeNeedsContext, inst.Op, prefix) + } + } + } + + parts := strings.Split(syntax, ";") + consumed := 0 + for i, part := range parts { + part = strings.TrimSpace(part) + // GoSyntax prints an architecturally ignored segment before the + // opcode (for example FS SCASB). Drop only that actual metadata-tagged + // prefix, not a used segment, an invalid prefix, or a textual guess. + for _, prefix := range physical.Prefix { + switch prefix & 0xff { + case x86asm.PrefixES, x86asm.PrefixCS, x86asm.PrefixSS, x86asm.PrefixDS, x86asm.PrefixFS, x86asm.PrefixGS: + if !strings.HasPrefix(part, prefix.String()+" ") { + continue + } + if prefix&x86asm.PrefixIgnored == 0 { + return nil, fmt.Errorf("%w: raw %s has an unbound physical %s segment prefix", ErrProbeNeedsContext, inst.Op, prefix) + } + part = strings.TrimPrefix(part, prefix.String()+" ") + } + } + space := strings.IndexByte(part, ' ') + if space >= 0 { + args := splitTopLevelCSV(part[space+1:]) + for j, arg := range args { + if binding, ok := bindings[strings.TrimSpace(arg)]; ok { + args[j] = binding.name + consumed++ + } + } + part = part[:space+1] + strings.Join(args, ", ") + } + parts[i] = part + } + if consumed != memoryCount { + return nil, fmt.Errorf("decoded physical memory handoff consumed %d operands, want %d", consumed, memoryCount) + } + instructions, err := parseDecodedX86Instruction(strings.Join(parts, "; ")) + if err != nil { + return nil, err + } + if len(instructions) != len(parts) { + return nil, fmt.Errorf("decoded instruction/prefix handoff changed instruction count") + } + restored := 0 + for i := range instructions { + for j, arg := range instructions[i].Args { + if arg.Kind != OpIdent { + continue + } + if operand, ok := byName[arg.Ident]; ok { + instructions[i].Args[j] = operand + restored++ + } + } + instructions[i].Raw = strings.TrimSpace(strings.Split(syntax, ";")[i]) + } + if restored != memoryCount { + return nil, fmt.Errorf("decoded physical memory handoff restored %d operands, want %d", restored, memoryCount) + } + return instructions, nil +} + +// The pinned decoder appends L to these environment-load mnemonics, whereas +// Go's complete ysvrs_mo rows use FLDENV/FRSTOR for the 32-bit layout. Bind the +// real opcode, full prefix metadata, and complete encoding before dropping +// that formatter suffix. The 14/94-byte operand-size override is not this +// 28/108-byte Go layout and remains unsupported. +func decodedX86X87EnvironmentLoad(inst x86asm.Inst, encoding []byte) (Op, bool, error) { + if inst.Op != x86asm.FLDENV && inst.Op != x86asm.FRSTOR { + return "", false, nil + } + witness, err := x86asm.Decode(encoding, inst.Mode) + witness = normalizeDecodedX86LEASegments(normalizeDecodedX86MemoryDisplacements(witness)) + memory, hasMemory := inst.Args[0].(x86asm.Mem) + witnessMemory, hasWitnessMemory := witness.Args[0].(x86asm.Mem) + if err != nil || witness.Len != len(encoding) || witness.Len != inst.Len || + witness.Op != inst.Op || witness.Opcode != inst.Opcode || witness.Prefix != inst.Prefix || + witness.DataSize != inst.DataSize || witness.AddrSize != inst.AddrSize || witness.MemBytes != inst.MemBytes || + !hasMemory || !hasWitnessMemory || memory != witnessMemory || + inst.Args[1] != nil || inst.Args[2] != nil || inst.Args[3] != nil { + return "", true, fmt.Errorf("%w: raw x87 environment load lacks its complete decoder encoding", ErrProbeNeedsContext) + } + if inst.DataSize != 32 { + return "", true, fmt.Errorf("raw %s uses a %d-bit environment layout not represented by Go %s", inst.Op, inst.DataSize, inst.Op) + } + return Op(inst.Op.String()), true, nil +} + +func decodedX86PhysicalMemory(memory x86asm.Mem) (Operand, error) { + register := func(reg x86asm.Reg) (Reg, error) { + if reg == 0 { + return "", nil + } + if reg == x86asm.RIP { + return Reg("IP"), nil // Consumed only by the separate RIP layout proof. + } + text := x86asm.GoSyntax(x86asm.Inst{Op: x86asm.NOP, Args: x86asm.Args{reg}}, 0, nil) + space := strings.IndexByte(text, ' ') + if space < 0 { + return "", fmt.Errorf("decoder formatter omitted its physical address register") + } + text = text[space+1:] + value, ok := parseReg(text) + if !ok { + return "", fmt.Errorf("unsupported decoded physical address register %s", reg) + } + return value, nil + } + base, err := register(memory.Base) + if err != nil { + return Operand{}, err + } + index, err := register(memory.Index) + if err != nil { + return Operand{}, err + } + segment := Reg("") + switch memory.Segment { + case 0: + case x86asm.ES, x86asm.CS, x86asm.SS, x86asm.DS, x86asm.FS, x86asm.GS: + segment = Reg(memory.Segment.String()) + default: + return Operand{}, fmt.Errorf("unsupported decoded physical segment %s", memory.Segment) + } + return Operand{Kind: OpMem, Mem: MemRef{Base: base, Index: index, Scale: int64(memory.Scale), Off: memory.Disp, Segment: segment}}, nil +} diff --git a/amd64_raw_operand_handoff_test.go b/amd64_raw_operand_handoff_test.go new file mode 100644 index 000000000..89fcbfc38 --- /dev/null +++ b/amd64_raw_operand_handoff_test.go @@ -0,0 +1,557 @@ +package plan9asm + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +// This oracle assembles literal BYTE directives, not GoSyntax's synthetic +// memory spelling. Its complete contiguous text bytes independently anchor +// the private decoder input on both supported SDK test lanes. +func x86HandoffOriginalBytes(t *testing.T, arch string, code []byte) { + t.Helper() + source := "TEXT handoff(SB),4,$0-0\n" + x86RawDisplacementDirectives(code) + listing, err := x86ScalarLiteralGoListing(t, arch, source) + if err != nil { + t.Fatal(err) + } + var actual []byte + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") || len(fields[1]) != 2 { + continue + } + offset, err := strconv.ParseUint(fields[0][2:], 16, 32) + if err != nil || int(offset) != len(actual) { + t.Fatalf("noncontiguous stock Go bytes: %s", line) + } + // Both audited objfile.go printers use sixteen fixed three-byte + // hex columns, then two spaces and ASCII. Never parse the ASCII + // tail as another byte, even when it happens to be "ab" or "f.". + text := strings.TrimLeft(line, "\t ") + start := strings.IndexByte(text, ' ') + if start < 0 || len(text) < start+48 { + t.Fatalf("missing complete stock Go hex columns: %s", line) + } + gap := false + for i := 0; i < 16; i++ { + field := text[start+3*i : start+3*(i+1)] + if field == " " { + gap = true + continue + } + value, err := strconv.ParseUint(field[1:], 16, 8) + if gap || field[0] != ' ' || err != nil { + t.Fatalf("invalid stock Go hex column: %q", field) + } + actual = append(actual, byte(value)) + } + } + if !bytes.Equal(actual, code) { + t.Fatalf("stock Go raw text = %x, want complete %x", actual, code) + } +} + +func TestX86PrivateDecoderX87PhysicalMemoryHandoff(t *testing.T) { + forms := []struct { + op Op + code []byte + }{ + {"FSAVE", []byte{0xdd, 0x74, 0x8b, 0xf9}}, + {"FSTCW", []byte{0xd9, 0x7c, 0x8b, 0xf9}}, + {"FSTENV", []byte{0xd9, 0x74, 0x8b, 0xf9}}, + {"FSTSW", []byte{0xdd, 0x7c, 0x8b, 0xf9}}, + {"FLDCW", []byte{0xd9, 0x6c, 0x8b, 0xf9}}, + {"FLDENV", []byte{0xd9, 0x64, 0x8b, 0xf9}}, + {"FRSTOR", []byte{0xdd, 0x64, 0x8b, 0xf9}}, + } + segments := []struct { + prefix byte + reg Reg + }{{0x26, ES}, {0x2e, CS}, {0x36, SS}, {0x3e, DS}, {0x64, FS}, {0x65, GS}} + for _, arch := range []string{"386", "amd64"} { + var complete []byte + for _, form := range forms { + for _, segment := range segments { + complete = append(complete, segment.prefix) + complete = append(complete, form.code...) + } + } + complete = append(complete, 0xc3) + x86HandoffOriginalBytes(t, arch, complete) + for _, form := range forms { + for _, segment := range segments { + t.Run(arch+"/"+string(form.op)+"/"+string(segment.reg), func(t *testing.T) { + code := append([]byte{segment.prefix}, form.code...) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + want := Operand{Kind: OpMem, Mem: MemRef{Base: BX, Index: CX, Scale: 4, Off: -7, Segment: segment.reg}} + if arch == "amd64" && segment.reg != FS && segment.reg != GS { + // In mode64 the actual decoder ignores the legacy flat + // segment prefix; it supplies no physical override. + want.Mem.Segment = "" + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != form.op || len(decoded.Instrs[0].Args) != 1 || !reflect.DeepEqual(decoded.Instrs[0].Args[0], want) { + t.Fatalf("physical memory changed: %+v, want %s %+v", decoded.Instrs, form.op, want) + } + }) + } + } + } +} + +func TestX86PrivateDecoderImplicitPhysicalSegments(t *testing.T) { + forms := []struct { + name string + opcode byte + source bool + dest bool + }{ + {"MOVS", 0xa4, true, true}, {"CMPS", 0xa6, true, true}, + {"STOS", 0xaa, false, true}, {"LODS", 0xac, true, false}, + {"SCAS", 0xae, false, true}, {"INS", 0x6c, false, true}, + {"OUTS", 0x6e, true, false}, + } + for _, mode := range []int{32, 64} { + arch := "amd64" + if mode == 32 { + arch = "386" + } + var complete []byte + for _, form := range forms { + for _, segment := range []byte{0, 0x64, 0x65} { + for _, repeat := range []byte{0, 0xf2, 0xf3} { + code := []byte{0x67} + if repeat != 0 { + code = append(code, repeat) + } + if segment != 0 { + code = append(code, segment) + } + code = append(code, form.opcode) + complete = append(complete, code...) + } + } + } + complete = append(complete, 0xc3) + x86HandoffOriginalBytes(t, arch, complete) + for _, form := range forms { + for _, segment := range []byte{0, 0x64, 0x65} { + for _, repeat := range []byte{0, 0xf2, 0xf3} { + t.Run(fmt.Sprintf("mode%d/%s/seg%02x/rep%02x", mode, form.name, segment, repeat), func(t *testing.T) { + code := []byte{0x67} + if repeat != 0 { + code = append(code, repeat) + } + if segment != 0 { + code = append(code, segment) + } + code = append(code, form.opcode) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("complete physical decode: %+v %v", physical, err) + } + instructions, err := x86AddressOverrideDecode(mode, code) + if !form.source && segment != 0 { + // x/arch does not mark this unused FS/GS prefix as + // ignored. Do not guess or strip it: precise Context + // replaces the old accidental source-parser failure. + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "unbound physical") { + t.Fatalf("unbound physical prefix: %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + var memories []MemRef + for _, ins := range instructions { + if ins.Op == OpTEXT || ins.Op == OpRET { + continue + } + if !ins.x86UnboundAddressOverride || ins.x86AddressBits != mode/2 { + t.Fatalf("lost physical address restriction: %+v", ins) + } + for _, arg := range ins.Args { + if arg.Kind == OpMem { + memories = append(memories, arg.Mem) + } + } + ctx, out := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := out.String() + if terminated, err := ctx.lowerInstr(0, 0, ins, nil, nil); terminated || !errors.Is(err, ErrProbeNeedsContext) || out.String() != before { + t.Fatalf("unbound physical memory emitted IR: %t %v", terminated, err) + } + } + wantSource := DS + if segment == 0x64 { + wantSource = FS + } else if segment == 0x65 { + wantSource = GS + } + seenSource, seenDest := false, false + for _, memory := range memories { + if memory.Base == SI && memory.Segment == wantSource { + seenSource = true + } else if memory.Base == DI && memory.Segment == ES { + seenDest = true + } else { + t.Fatalf("unexpected implicit physical memory: %+v", memory) + } + } + if seenSource != form.source || seenDest != form.dest { + t.Fatalf("lost source/destination segments: %+v", memories) + } + }) + } + } + } + } +} + +func TestX86PrivateDecoderRIPPoolAllPublicAPIs(t *testing.T) { + code := []byte{0x48, 0x8d, 0x05, 1, 0, 0, 0, 0xc3, 0x41} + x86HandoffOriginalBytes(t, "amd64", code) + file := mustParseX86RawReturn(t, x86StaticReturnSource("Bound", code)) + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue // RIP is a mode64-only machine addressing form. + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + for _, api := range []string{"text", "module", "owned-module"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, opt) + case "module": + module, callErr := TranslateModule(file, opt) + err = callErr + if err == nil { + ir = module.String() + module.Dispose() + } + default: + ctx := llvm.NewContext() + module, callErr := TranslateModuleInContext(ctx, file, opt) + err = callErr + if err == nil { + ir = module.String() + module.Dispose() + } + ctx.Dispose() + } + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + compileLLVMToObject(t, llc, target.triple, "handoff.ll", "handoff.o", ir) + }) + } + } +} + +func TestX86PrivateDecoderX87FiveTargetObjects(t *testing.T) { + // Object-only controls for the already-supported FS/GS address spaces. + // They do not execute an unknown native pointer or prove caller bounds. + var code []byte + for _, opcode := range [][]byte{ + {0xdd, 0x30}, {0xd9, 0x38}, {0xd9, 0x30}, {0xdd, 0x38}, + {0xd9, 0x28}, {0xd9, 0x20}, {0xdd, 0x20}, + {0xdf, 0x20}, {0xdf, 0x30}, // packed BCD load/store + } { + for _, segment := range []byte{0x64, 0x65} { + code = append(code, segment) + code = append(code, opcode...) + } + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + // This is the same named RET domain as the old x87 positive. + // A raw RET plus unknown native memory remains Context. + source := x86StaticReturnSource("Bound", code) + + "FMOVX 0(AX)(FS),F0\nFMOVXP F0,0(AX)(FS)\n" + + "FMOVX 0(AX)(GS),F0\nFMOVXP F0,0(AX)(GS)\nRET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + file := mustParseX86RawReturn(t, source) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, X87Mode: X87Software, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + for _, api := range []string{"text", "module", "owned-module"} { + t.Run(api, func(t *testing.T) { + ir, err := x86X87UnboundOriginalAPI(t, file, opt, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" || !strings.Contains(err.Error(), "FSAVE FS:0(AX)") { + t.Fatal("complete original raw source acquired native AX/segment storage or an x87 image", err) + } + }) + } + t.Run("lowerer-unit-object", func(t *testing.T) { + ir := x86X87LowererObjectUnit(t, file, target.arch, target.triple) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + compileLLVMToObject(t, llc, target.triple, "physical-x87-unit.ll", "physical-x87-unit.o", ir) + }) + }) + } +} + +func TestX86PrivateDecoderHandoffDoesNotWidenSourceOrLayout(t *testing.T) { + for _, source := range []string{"FSAVE FS:0(AX)", "MOVSB DS:0(SI),ES:0(DI)", "LEAQ 1(IP),AX"} { + t.Run("public/"+source, func(t *testing.T) { + if _, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+source+"\nRET\n"); err == nil { + t.Fatal("private physical memory spelling became public source syntax") + } + }) + } + for _, opcode := range [][]byte{{0xdd, 0x30}, {0xd9, 0x30}, {0xd9, 0x20}, {0xdd, 0x20}} { + for _, mode := range []int{32, 64} { + t.Run(fmt.Sprintf("legacy-layout/%d/%x", mode, opcode), func(t *testing.T) { + code := append([]byte{0x66}, opcode...) + arch := "amd64" + if mode == 32 { + arch = "386" + } + x86HandoffOriginalBytes(t, arch, append(append([]byte(nil), code...), 0xc3)) + _, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err == nil || !strings.Contains(err.Error(), "16-bit environment layout") { + t.Fatalf("14/94-byte layout acquired the 28/108-byte alias: %v", err) + } + }) + } + } + code := []byte{0xd9, 0x20} + inst, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + for name, mutate := range map[string]func(*x86asm.Inst){ + "prefix": func(i *x86asm.Inst) { i.Prefix[0] = x86asm.PrefixFS }, + "size": func(i *x86asm.Inst) { i.DataSize = 16 }, + "length": func(i *x86asm.Inst) { i.Len++ }, + "memory": func(i *x86asm.Inst) { i.Args[0] = x86asm.Mem{Base: x86asm.RBX} }, + "extra": func(i *x86asm.Inst) { i.Args[1] = x86asm.RAX }, + } { + t.Run("alias-witness/"+name, func(t *testing.T) { + changed := inst + mutate(&changed) + if _, recognized, err := decodedX86X87EnvironmentLoad(changed, code); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed decoded observation borrowed the environment alias: %v", err) + } + }) + } + for _, changed := range [][]byte{{0xd9}, {0xd9, 0x28}, {0xd9, 0x20, 0x90}} { + t.Run(fmt.Sprintf("alias-bytes/%x", changed), func(t *testing.T) { + if _, recognized, err := decodedX86X87EnvironmentLoad(inst, changed); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed encoding borrowed the environment alias: %v", err) + } + }) + } + for _, code := range [][]byte{{0x67, 0x48, 0x8d, 0x05, 0, 0, 0, 0}, {0xf0, 0xd9, 0x20}} { + t.Run(fmt.Sprintf("physical-boundary/%x", code), func(t *testing.T) { + _, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err == nil { + t.Fatal("unbound EIP or invalid prefix acquired private memory admission") + } + }) + } +} + +func TestX86PrivateDecoderNamedX87MemoryConsumers(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT Bound(SB),4,$0-0\n") + var forms []string + for _, segment := range []string{"FS", "GS"} { + // Named FLDENV is the separately observed stock assembler defect + // below, not an accepted object control. Its complete raw opcode + // remains in the physical-memory and five-target object matrices. + for _, op := range []string{"FSAVE", "FSTCW", "FSTENV", "FSTSW", "FLDCW", "FRSTOR"} { + forms = append(forms, fmt.Sprintf("%s 0(AX)(%s)", op, segment)) + } + for _, suffix := range []string{"B", "X", "F", "D", "W", "L", "V"} { + forms = append(forms, fmt.Sprintf("FMOV%s 0(AX)(%s),F0", suffix, segment), + fmt.Sprintf("FMOV%sP F0,0(AX)(%s)", suffix, segment)) + } + } + source.WriteString(strings.Join(forms, "\n") + "\n") + source.WriteString("RET\n") + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + for _, form := range forms { + t.Run("original-Go/"+form, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, "TEXT Bound(SB),4,$0-0\n"+form+"\nRET\n") + }) + } + if t.Failed() { + return // Do not claim LLVM equivalence for a rejected Go input. + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + file := mustParseX86RawReturn(t, source.String()) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, X87Mode: X87Software, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + for _, api := range []string{"text", "module", "owned-module"} { + t.Run(api, func(t *testing.T) { + ir, err := x86X87UnboundOriginalAPI(t, file, opt, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" || !strings.Contains(err.Error(), "FSAVE 0(AX)(FS)") { + t.Fatal("complete original named source acquired native AX/segment storage or an x87 image", err) + } + }) + } + t.Run("lowerer-unit-object", func(t *testing.T) { + ir := x86X87LowererObjectUnit(t, file, target.arch, target.triple) + for _, addressSpace := range []string{"ptr addrspace(256)", "ptr addrspace(257)"} { + if !strings.Contains(ir, "getelementptr i8, "+addressSpace) || !strings.Contains(ir, "load i8, "+addressSpace) { + t.Fatalf("environment/BCD emitter unit lost %s", addressSpace) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + compileLLVMToObject(t, llc, target.triple, "named-x87-unit.ll", "named-x87-unit.o", ir) + }) + }) + } +} + +func TestX86PrivateDecoderNamedFLDENVStockGoDefect(t *testing.T) { + // Do not turn a positionless backend defect into a generic source-rejection + // certificate. Bind the entire one-instruction fixture and exact process + // outcome separately; raw D9 /4 is independently accepted above. + for _, target := range x86RawReturnTargets { + for _, segment := range []string{"FS", "GS"} { + t.Run(target.triple+"/"+segment, func(t *testing.T) { + original := "TEXT Bound(SB),4,$0-0\nFLDENV 0(AX)(" + segment + ")\nRET\n" + dir := t.TempDir() + input, object := filepath.Join(dir, "fldenv.s"), filepath.Join(dir, "fldenv.o") + if err := os.WriteFile(input, []byte(original), 0600); err != nil { + t.Fatal(err) + } + stage, cancelStage := runtimeOracleStageContext(t) + defer cancelStage() + ctx, cancel := context.WithTimeout(stage, 30*time.Second) + defer cancel() + driver, driverHash := x86LEAActualSDKDriver(t) + goos := "linux" + if strings.Contains(target.triple, "apple") { + goos = "darwin" + } else if strings.Contains(target.triple, "windows") { + goos = "windows" + } + command := exec.CommandContext(ctx, driver, "tool", "asm", "-p", "sourceframe", "-o", object, input) + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", + "GOOS="+goos, "GOARCH="+target.arch, "CGO_ENABLED=0") + output, err := runtimeOracleCombinedOutput(ctx, command) + const diagnostic = "asm: asmand: bad address \nasm: assembly failed\n" + if failure := x86UnnamedAsmidxOutcome(output, err, ctx.Err(), diagnostic); failure != nil { + t.Fatalf("unknown named FLDENV backend observation: %v\n%s", failure, output) + } + if goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), false, target.arch) == nil { + t.Fatal("positionless backend defect became a source-rejection certificate") + } + for _, changed := range []string{"", diagnostic + diagnostic, diagnostic + "unexpected\n", "asm: assembly failed\n"} { + if x86UnnamedAsmidxOutcome([]byte(changed), err, nil, diagnostic) == nil { + t.Fatal("changed/missing/repeated backend diagnostic accepted") + } + } + for _, changed := range []error{nil, errors.New("tool launch"), errors.Join(err, errors.New("cleanup"))} { + if x86UnnamedAsmidxOutcome(output, changed, nil, diagnostic) == nil { + t.Fatal("missing/unknown/cleanup process failure accepted") + } + } + if x86UnnamedAsmidxOutcome(output, err, context.DeadlineExceeded, diagnostic) == nil { + t.Fatal("expired backend observation accepted") + } + if data, readErr := os.ReadFile(input); readErr != nil || string(data) != original { + t.Fatal("original named FLDENV source changed", readErr) + } + t.Logf("GO_BACKEND_DEFECT not-source-exclusion target=%s source=%q sourceSHA=%x driverSHA=%s diagnosticSHA=%x", + target.triple, original, sha256.Sum256([]byte(original)), driverHash, sha256.Sum256(output)) + }) + } + } +} + +func TestX86PrivateDecoderUnboundSegmentCannotEmit(t *testing.T) { + // These are exactly the eighteen old parser-error controls: mode32 + // STOSB/STOSL/INSB/INSL and mode64 plus STOSQ, each with FS/GS. They + // remain unbound machine Context, not new public syntax or object passes. + for _, target := range x86RawReturnTargets { + forms := [][]byte{{0xaa}, {0xab}, {0x6c}, {0x6d}} + mode := 32 + if target.arch == "amd64" { + mode = 64 + forms = append(forms, []byte{0x48, 0xab}) + } + for _, opcode := range forms { + for _, segment := range []byte{0x64, 0x65} { + code := append([]byte{0x67, segment}, opcode...) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatal("complete segment observation", physical, err) + } + memory, ok := physical.Args[0].(x86asm.Mem) + if !ok || memory.Segment != x86asm.ES || physical.Prefix[1]&x86asm.PrefixIgnored != 0 { + t.Fatal("unbound prefix borrowed an ignored/default-segment identity", physical) + } + for _, api := range []string{"text", "module", "owned-module"} { + t.Run(fmt.Sprintf("%s/%x/%s", target.triple, code, api), func(t *testing.T) { + file := mustParseX86RawReturn(t, x86StaticReturnSource("Bound", code)+"RET\n") + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + var callErr error + switch api { + case "text": + ir, err := Translate(file, opt) + callErr = err + if ir != "" { + t.Fatal("unbound segment emitted text") + } + case "module": + module, err := TranslateModule(file, opt) + callErr = err + if !module.IsNil() { + module.Dispose() + t.Fatal("unbound segment emitted a module") + } + default: + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + callErr = err + if !module.IsNil() { + module.Dispose() + ctx.Dispose() + t.Fatal("unbound segment emitted an owned module") + } + ctx.Dispose() + } + if !errors.Is(callErr, ErrProbeNeedsContext) || !strings.Contains(callErr.Error(), "unbound physical") { + t.Fatalf("unbound segment lost Context: %v", callErr) + } + }) + } + } + } + } +} diff --git a/amd64_raw_pack.go b/amd64_raw_pack.go new file mode 100644 index 000000000..70ed38492 --- /dev/null +++ b/amd64_raw_pack.go @@ -0,0 +1,38 @@ +package plan9asm + +import "fmt" + +// Go 1.27's four VPACK rows share the same three-operand grammar. The two +// dword-to-word operations also permit EVEX 32-bit memory broadcast. +var x86RawPackedNarrowOps = map[[2]int]struct { + op Op + broadcast bool + evexW0 bool +}{ + {1, 0x6b}: {op: "VPACKSSDW", broadcast: true, evexW0: true}, + {1, 0x63}: {op: "VPACKSSWB"}, + {2, 0x2b}: {op: "VPACKUSDW", broadcast: true, evexW0: true}, + {1, 0x67}: {op: "VPACKUSWB"}, +} + +func decodedX86VectorPackedSaturatingNarrowInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched { + return Instr{}, 0, false, nil + } + spec, known := x86RawPackedNarrowOps[[2]int{p.mapNumber, p.opcode}] + if !known { + return Instr{}, 0, false, nil + } + // Intel's VEX pack rows ignore W. EVEX dword inputs require W0, + // while EVEX word inputs also ignore W. Go happens to emit W0 for all; + // that chosen encoding is not the complete legal physical grammar. + if p.evex && spec.evexW0 && p.w { + return Instr{}, 0, true, fmt.Errorf("packed saturating narrow: unsupported W bit") + } + bits := 16 + if spec.broadcast { + bits = 32 + } + return decodedX86BinaryVectorOperands(code, p, mode, spec.op, bits) +} diff --git a/amd64_raw_pack_test.go b/amd64_raw_pack_test.go new file mode 100644 index 000000000..ee9fd150a --- /dev/null +++ b/amd64_raw_pack_test.go @@ -0,0 +1,162 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawPackedSaturatingNarrowGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawpackedforms(SB),4,$0-0\n") + for _, op := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB"} { + if arch == "amd64" && op != "PACKUSDW" { + fmt.Fprintf(&source, "%s M1, M2\n%s (AX), M2\n", op, op) + } + fmt.Fprintf(&source, "%s X1, X2\n%s (AX), X2\n", op, op) + } + if arch == "amd64" { + for _, op := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "%s %s1, %s2, %s3\n", op, width, width, width) + fmt.Fprintf(&source, "%s 32(AX), %s2, %s3\n", op, width, width) + fmt.Fprintf(&source, "%s %s20, %s21, K1, %s22\n", op, width, width, width) + } + fmt.Fprintf(&source, "%s Z20, Z21, K7, Z22\n", op) + fmt.Fprintf(&source, "%s.Z (AX), Z21, K7, Z22\n", op) + if op == "VPACKSSDW" || op == "VPACKUSDW" { + fmt.Fprintf(&source, "%s.BCST.Z 64(AX), Z21, K7, Z22\n", op) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d = %s %v, want %s %v", index, got.Op, got.Args, want[index].Op, want[index].Args) + } + } + if len(code) == 0 || code[len(code)-1] != 0xc3 { + t.Fatal("Go assembler did not end the fixture with RET") + } + var raw strings.Builder + raw.WriteString("TEXT rawpackedforms(SB),4,$0-0\n") + for _, value := range code[:len(code)-1] { + fmt.Fprintf(&raw, "BYTE $%#02x\n", value) + } + raw.WriteString("RET\n") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + // Preserve MMX/X/Y/Z and all mask/broadcast forms as original + // objects; neither physical MMX nor unknown inputs gain an entry. + ir := x86UnboundVoidRawScalarFormatObject(t, raw.String(), arch, triple) + compileLLVMToObject(t, llc, triple, "raw-packed-saturating-narrow.ll", "raw-packed-saturating-narrow.o", ir) + } + }) + } +} + +func TestX86RawPackedSaturatingNarrowRegisterAxes(t *testing.T) { + for key, spec := range x86RawPackedNarrowOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, regs := range [][3]int{{0, 1, 2}, {7, 15, 8}, {16, 20, 31}, {31, 30, 29}} { + for _, evex := range []bool{false, true} { + if !evex && (width == "Z" || regs[0] >= 16 || regs[1] >= 16 || regs[2] >= 16) { + continue + } + var code []byte + if evex { + vectorBits := byte(strings.Index("XYZ", width)) + code = encodeX86EVEXPackedMultiplyLow(false, vectorBits, 7, true, regs[0], regs[1], regs[2]) + code[1] = code[1]&^0x0f | byte(key[0]) + code[4] = byte(key[1]) + } else { + code = encodeX86VEXPackedMultiplyLow(width == "Y", regs[0], regs[1], regs[2]) + code[1] = code[1]&^0x1f | byte(key[0]) + code[3] = byte(key[1]) + } + got, length, matched, err := decodedX86VectorPackedSaturatingNarrowInstruction(code, 64) + if !matched || err != nil || length != len(code) { + t.Fatalf("decode %x = %+v length=%d matched=%v err=%v", code, got, length, matched, err) + } + wantOp := spec.op + wantArgs := []string{ + fmt.Sprintf("%s%d", width, regs[2]), + fmt.Sprintf("%s%d", width, regs[1]), + } + if evex { + wantOp += ".Z" + wantArgs = append(wantArgs, "K7") + } + wantArgs = append(wantArgs, fmt.Sprintf("%s%d", width, regs[0])) + if got.Op != wantOp || len(got.Args) != len(wantArgs) { + t.Fatalf("decode %x = %+v, want %s %v", code, got, wantOp, wantArgs) + } + for index, arg := range got.Args { + if arg.String() != wantArgs[index] { + t.Fatalf("decode %x arg %d = %s, want %s", code, index, arg.String(), wantArgs[index]) + } + } + } + } + } + } +} + +func TestX86RawPackedSaturatingNarrowRejectsReservedForms(t *testing.T) { + base := encodeX86EVEXPackedMultiplyLow(false, 2, 1, false, 2, 3, 4) + base[1] = base[1]&^0x0f | 2 + base[4] = 0x2b + bytePackBroadcast := append([]byte(nil), base...) + bytePackBroadcast[1] = bytePackBroadcast[1]&^0x0f | 1 + bytePackBroadcast[3] |= 0x10 + bytePackBroadcast[4] = 0x63 + for _, test := range []struct { + name string + code []byte + }{ + {name: "missing ModRM", code: base[:5]}, + {name: "address override", code: append([]byte{0x67}, base...)}, + {name: "wrong pp", code: replaceX86Byte(base, 2, base[2]&^3)}, + {name: "wrong W", code: replaceX86Byte(base, 2, base[2]|0x80)}, + {name: "missing fixed bit", code: replaceX86Byte(base, 2, base[2]&^4)}, + {name: "reserved vector length", code: replaceX86Byte(base, 3, base[3]&^0x60|0x60)}, + {name: "zero without mask", code: replaceX86Byte(base, 3, base[3]&^7|0x80)}, + {name: "broadcast register", code: replaceX86Byte(base, 3, base[3]|0x10)}, + {name: "byte pack broadcast", code: bytePackBroadcast}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, matched, err := decodedX86VectorPackedSaturatingNarrowInstruction(test.code, 64); !matched || err == nil { + t.Fatalf("invalid %x matched=%v error=%v", test.code, matched, err) + } + }) + } +} + +func replaceX86Byte(source []byte, index int, value byte) []byte { + copyOfSource := append([]byte(nil), source...) + copyOfSource[index] = value + return copyOfSource +} diff --git a/amd64_raw_packed_arithmetic_literal_test.go b/amd64_raw_packed_arithmetic_literal_test.go new file mode 100644 index 000000000..df94a575c --- /dev/null +++ b/amd64_raw_packed_arithmetic_literal_test.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "sort" + "strings" + "testing" +) + +func x86RawPackedArithmeticLiteral(opcode byte, evex bool, width int, broadcast bool) []byte { + properties := decodedX86PackedIntegerArithmeticOps[opcode] + mask, zeroing := 0, false + if evex { + mask, zeroing = 7, true + } + code := encodeRawScalarMove(evex, 1, int(opcode), 0, 0, 0, mask, zeroing) + code[1] = code[1]&0xf0 | 1 + code[2] &^= 0x80 + if evex && properties.laneBytes == 8 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + if broadcast { + code[3] |= 0x10 + } + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = properties.laneBytes + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*29+11)) + } + return code +} + +func x86PackedArithmeticOpcodes() []byte { + opcodes := make([]byte, 0, len(decodedX86PackedIntegerArithmeticOps)) + for opcode := range decodedX86PackedIntegerArithmeticOps { + opcodes = append(opcodes, opcode) + } + sort.Slice(opcodes, func(i, j int) bool { return opcodes[i] < opcodes[j] }) + return opcodes +} + +func TestDecodeX86RawPackedArithmeticRIPDataCompleteFamily(t *testing.T) { + for _, opcode := range x86PackedArithmeticOpcodes() { + properties := decodedX86PackedIntegerArithmeticOps[opcode] + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && properties.laneBytes >= 4 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", properties.op, evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawPackedArithmeticLiteral(opcode, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := properties.op + if broadcast { + want += ".BCST" + } + if evex { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawPackedArithmeticRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawPackedArithmeticLiteral(0xd4, true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawPackedArithmeticRIPDataObjects(t *testing.T) { + var source strings.Builder + var pools [][]byte + for index, opcode := range x86PackedArithmeticOpcodes() { + properties := decodedX86PackedIntegerArithmeticOps[opcode] + evex := index%2 == 0 + width := 32 + if evex { + width = 64 + } + broadcast := evex && properties.laneBytes >= 4 + code := x86RawPackedArithmeticLiteral(opcode, evex, width, broadcast) + name := fmt.Sprintf("packedArithmeticLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + dataWidth := width + if broadcast { + dataWidth = properties.laneBytes + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // The complete original source includes EVEX forms with missing K7. + // Keep its Go/emitter objects and public Context independently of + // the initialized full-family public object programs. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "packed-arithmetic-literal.ll", "packed-arithmetic-literal.o", ir) + }) + } +} diff --git a/amd64_raw_packed_compare.go b/amd64_raw_packed_compare.go new file mode 100644 index 000000000..7cc99c769 --- /dev/null +++ b/amd64_raw_packed_compare.go @@ -0,0 +1,86 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func decodedX86PackedIntegerCompareInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedIntegerCompareInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedIntegerCompareInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched { + return Instr{}, 0, false, nil + } + var op Op + var spec amd64PackedIntegerCompareSpec + for name, candidate := range amd64PackedIntegerCompareSpecs { + if p.mapNumber == candidate.mapNumber && p.opcode == candidate.opcode { + op, spec = name, candidate + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed integer compare: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.pp != 1 || p.vectorLength > 2 { + return fail("invalid prefix or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + // Intel XED leaves VEX.W and EVEX byte/word W unconstrained. D/Q + // EVEX requires W0/W1. MASK_R reserves both destination extension bits. + if p.evex && (!p.fixed || p.zero || p.r != 0 || spec.laneBits >= 32 && p.w != (spec.laneBits == 64)) { + return fail("invalid EVEX fixed, zeroing, destination or width field") + } + if p.broadcast && (spec.laneBits < 32 || code[p.modRM]>>6 == 3) { + return fail("broadcast requires D/Q memory source") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + width := []string{"X", "Y", "Z"}[p.vectorLength] + byteWidth := 16 << p.vectorLength + scale := byteWidth + if p.broadcast { + scale = spec.laneBits / 8 + op += ".BCST" + } + var source Operand + var size int + var err error + if p.evex { + source, size, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, width, scale) + } else { + source, size, err = memory.vex(code[p.modRM:], mode, p.b, p.x, p.segment, width) + } + if err != nil { + return Instr{}, 0, true, err + } + dst := Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]>>3&7)+p.r*8)) + args := []Operand{source, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))}} + if p.evex { + dst = Reg(fmt.Sprintf("K%d", code[p.modRM]>>3&7)) + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + } + args = append(args, Operand{Kind: OpReg, Reg: dst}) + parts := make([]string, len(args)) + for i, arg := range args { + parts[i] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(parts, ", ")), x86Encoded: true}, p.modRM + size, true, nil +} diff --git a/amd64_raw_packed_compare_literal_test.go b/amd64_raw_packed_compare_literal_test.go new file mode 100644 index 000000000..e71aebcd2 --- /dev/null +++ b/amd64_raw_packed_compare_literal_test.go @@ -0,0 +1,157 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawPackedCompareLiteral(op string, evex bool, width int, broadcast bool) []byte { + spec := amd64PackedIntegerCompareSpecs[Op(op)] + code := encodeRawScalarMove(evex, 1, spec.opcode, 0, 0, 0, 0, false) + code[1] = code[1]&0xf0 | byte(spec.mapNumber) + code[2] &^= 0x80 + if evex && spec.laneBits == 64 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + if broadcast { + code[3] |= 0x10 + } + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*17+5)) + } + return code +} + +func TestDecodeX86RawPackedCompareRIPDataCompleteFamily(t *testing.T) { + for _, op := range x86PackedCompareTestOps { + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && amd64PackedIntegerCompareSpecs[Op(op)].laneBits >= 32 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", op, evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawPackedCompareLiteral(op, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op(op) + if broadcast { + want += ".BCST" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawPackedCompareRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawPackedCompareLiteral("VPCMPEQD", true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x65}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawPackedCompareRIPDataObjects(t *testing.T) { + var source strings.Builder + var pools [][]byte + for index, op := range x86PackedCompareTestOps { + spec := amd64PackedIntegerCompareSpecs[Op(op)] + evex := index%2 == 0 + width := 32 + if evex { + width = 64 + } + broadcast := evex && spec.laneBits >= 32 + code := x86RawPackedCompareLiteral(op, evex, width, broadcast) + name := fmt.Sprintf("packedCompareLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Original encodings leave the second vector input undefined. + // Preserve their full Go/emitter objects and public Context; + // independent initialized literal programs own public objects. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "packed-compare-literal.ll", "packed-compare-literal.o", ir) + }) + } +} diff --git a/amd64_raw_packed_compare_test.go b/amd64_raw_packed_compare_test.go new file mode 100644 index 000000000..79c002f08 --- /dev/null +++ b/amd64_raw_packed_compare_test.go @@ -0,0 +1,518 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86PackedCompareTestOps = []string{"VPCMPEQB", "VPCMPEQW", "VPCMPEQD", "VPCMPEQQ", "VPCMPGTB", "VPCMPGTW", "VPCMPGTD", "VPCMPGTQ"} + +func TestX86PackedCompareGrammarMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`(?s)\{as: A(\w+), ytab: _yvpcmpeqb, prefix: Pavx, op: opBytes\{(.*?)\}\}`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 8 || len(rows) != len(amd64PackedIntegerCompareSpecs) { + t.Fatalf("Go has %d family entries, grammar has %d", len(rows), len(amd64PackedIntegerCompareSpecs)) + } + for _, row := range rows { + spec, ok := amd64PackedIntegerCompareSpecs[Op(row[1])] + if !ok { + t.Fatalf("missing Go family member %s", row[1]) + } + width := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[row[1][len(row[1])-1]] + if spec.laneBits != width || spec.greater != strings.Contains(row[1], "GT") { + t.Fatalf("incorrect semantics for %s: %+v", row[1], spec) + } + opcodes := regexp.MustCompile(`0x[0-9A-F]+`).FindAllString(row[2], -1) + if len(opcodes) != 5 { + t.Fatalf("Go %s has %d encodings", row[1], len(opcodes)) + } + for _, opcode := range opcodes { + if opcode != fmt.Sprintf("0x%X", spec.opcode) { + t.Fatalf("Go opcode %s != spec %+v", opcode, spec) + } + } + if strings.Contains(row[2], "0F38") != (spec.mapNumber == 2) || strings.Contains(row[2], "evexBcst") != (width >= 32) { + t.Fatalf("Go map/broadcast does not match %+v: %s", spec, row[2]) + } + } + table := regexp.MustCompile(`(?s)var _yvpcmpeqb = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(string(data)) + if len(table) != 2 { + t.Fatal("missing Go operand table") + } + args := regexp.MustCompile(`argList\{([^}]+)\}`).FindAllStringSubmatch(table[1], -1) + want := []string{"Yxm, Yxr, Yxr", "Yym, Yyr, Yyr", "YxmEvex, YxrEvex, Yk", "YxmEvex, YxrEvex, Yknot0, Yk", "YymEvex, YyrEvex, Yk", "YymEvex, YyrEvex, Yknot0, Yk", "Yzm, Yzr, Yk", "Yzm, Yzr, Yknot0, Yk"} + if len(args) != len(want) { + t.Fatalf("Go operand rows %d != %d", len(args), len(want)) + } + for i, arg := range args { + if arg[1] != want[i] { + t.Fatalf("Go row %d=%s, want %s", i, arg[1], want[i]) + } + } +} + +func TestX86RawPackedCompareEncodingAxes(t *testing.T) { + count := 0 + for _, op := range x86PackedCompareTestOps { + // The test's encoding map is independent of the implementation spec. + opcode := map[string]byte{"VPCMPEQB": 0x74, "VPCMPEQW": 0x75, "VPCMPEQD": 0x76, "VPCMPEQQ": 0x29, "VPCMPGTB": 0x64, "VPCMPGTW": 0x65, "VPCMPGTD": 0x66, "VPCMPGTQ": 0x37}[op] + mapNumber := byte(1) + if strings.HasSuffix(op, "Q") { + mapNumber = 2 + } + for _, evex := range []bool{false, true} { + limit, destLimit, widths := 16, 16, 2 + if evex { + limit, destLimit, widths = 32, 8, 3 + } + for width := 0; width < widths; width++ { + for src1 := 0; src1 < limit; src1++ { + for src2 := 0; src2 < limit; src2++ { + for dst := 0; dst < destLimit; dst++ { + code := encodeRawScalarMove(evex, 1, int(opcode), src2, src1, dst, 0, false) + code[1] = code[1]&0xe0 | mapNumber + if evex { + // Preserve R' (K destination extension) when replacing map. + code[1] |= 0x10 + code[2] &^= 0x80 + if mapNumber == 2 { + code[2] |= 0x80 + } + code[3] |= byte(width)<<5 | byte(dst) + } else { + code[2] |= byte(width) << 2 + } + variants := 1 + if !evex { + variants = 2 + } + for variant := 0; variant < variants; variant++ { + if variant != 0 { + code[2] |= 0x80 + } + got, size, matched, err := decodedX86PackedIntegerCompareInstruction(code, 64) + prefix := []string{"X", "Y", "Z"}[width] + args := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src2))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src1))}} + dest := Reg(fmt.Sprintf("%s%d", prefix, dst)) + if evex { + dest = Reg(fmt.Sprintf("K%d", dst)) + if dst != 0 { + args = append(args, Operand{Kind: OpReg, Reg: dest}) + } + } + args = append(args, Operand{Kind: OpReg, Reg: dest}) + if err != nil || !matched || size != len(code) || got.Op != Op(op) || !reflect.DeepEqual(got.Args, args) { + t.Fatalf("decode %x: %+v %d %v %v; want %s %+v", code, got, size, matched, err, op, args) + } + _, _, matched, err = decodedX86PackedIntegerCompareInstruction(code, 32) + if !matched || (err != nil) != (src1 >= 8 || src2 >= 8 || !evex && dst >= 8) { + t.Fatalf("386 decode %x: matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + } + if count != 327680 { + t.Fatalf("covered %d encodings, want 327680", count) + } +} + +func TestX86RawPackedCompareInvalidAndMemoryForms(t *testing.T) { + valid := []byte{0x62, 0xf1, 0x75, 0x0b, 0x76, 0xe0} // VPCMPEQD X0,X1,K3,K4. + for name, mutate := range map[string]func([]byte){ + "fixed": func(b []byte) { b[2] &^= 4 }, "length": func(b []byte) { b[3] |= 0x60 }, + "zero": func(b []byte) { b[3] |= 0x80 }, "broadcast-register": func(b []byte) { b[3] |= 0x10 }, + "width": func(b []byte) { b[2] |= 0x80 }, "mask-dest-r": func(b []byte) { b[1] &^= 0x80 }, + "mask-dest-rprime": func(b []byte) { b[1] &^= 0x10 }, "prefix": func(b []byte) { b[2] &^= 3 }, + } { + b := append([]byte(nil), valid...) + mutate(b) + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64); !matched || err == nil { + t.Fatalf("accepted %s %x", name, b) + } + } + for _, b := range [][]byte{valid[:5], {0x62, 0xf1, 0x75, 0x0b, 0x76, 0x04}, {0x62, 0xf1, 0x75, 0x0b, 0x76, 0x40}, append([]byte{0x67}, valid...), {0x62, 0xf1, 0x75, 0x1b, 0x74, 0x00}, {0xc5, 0xf9, 0x74}, {0xc4, 0xe1, 0x79, 0x76, 0x05, 0, 0, 0, 0}} { + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64); !matched || err == nil { + t.Fatalf("accepted invalid %x", b) + } + } + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(valid, 16); !matched || err == nil { + t.Fatal("accepted mode16") + } + for _, b := range [][]byte{nil, {0x62}, {0x90}, {0xc5, 0xf9, 0x73, 0xc0}} { + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64); matched || err != nil { + t.Fatalf("claimed unrelated %x", b) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + for _, broadcast := range []bool{false, true} { + b := []byte{segment.prefix, 0x62, 0x91, 0x75, 0x4b, 0x76, 0x64, 0x88, 0xfe} + offset := int64(-128) + op := Op("VPCMPEQD") + if broadcast { + b[4] |= 0x10 + offset = -8 + op += ".BCST" + } + got, size, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64) + want := MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: offset} + if err != nil || !matched || size != len(b) || got.Op != op || !reflect.DeepEqual(got.Args[0].Mem, want) { + t.Fatalf("decode %x=%+v: %v", b, got, err) + } + } + } + // Byte/word EVEX.W is ignored, whereas dword and qword W is fixed. + for _, opcode := range []byte{0x74, 0x75, 0x64, 0x65} { + b := append([]byte(nil), valid...) + b[4] = opcode + b[2] |= 0x80 + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64); !matched || err != nil { + t.Fatalf("ignored W: %x %v", b, err) + } + } +} + +func TestX86RawPackedCompareGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + last := 7 + if arch == "amd64" { + last = 31 + } + var source strings.Builder + source.WriteString("TEXT compare(SB),4,$0-0\n") + for _, op := range x86PackedCompareTestOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, reg := range []int{0, last} { + inputs := []string{fmt.Sprintf("%s%d", width, last-reg), "(AX)", "-7(BX)(CX*4)", "64(SI)", "8192(DI)"} + for _, input := range inputs { + fmt.Fprintf(&source, "%s %s,%s%d,K0\n", op, input, width, reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s,%s%d,K7,K7\n", op, input, width, reg) + } + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + for _, input := range []string{"(AX)", "-8(BX)(CX*4)", "64(SI)", "8192(DI)"} { + fmt.Fprintf(&source, "%s.BCST %s,%s%d,K7\n", op, input, width, reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s.BCST %s,%s%d,K1,K0\n", op, input, width, reg) + } + } + } + } + if width != "Z" { + for _, input := range []string{width + "0", "(AX)", "-7(BX)(CX*4)", "8192(DI)"} { + fmt.Fprintf(&source, "%s %s,%s6,%s7\n", op, input, width, width) + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + got, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(got.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(got.Instrs), len(want)) + } + for i, instruction := range got.Instrs { + if instruction.Op != want[i].Op || !reflect.DeepEqual(instruction.Args, want[i].Args) { + t.Fatalf("instruction %d: %+v, want %+v", i, instruction, want[i]) + } + } + file := x86RawUnprovedReturnFormProbeFile(t, arch, "compare", code) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + ir := x86UnboundVoidRawProbeObject(t, file, arch, triple) + compileLLVMToObject(t, llc, triple, "compare.ll", "compare.o", ir) + } + t.Logf("%s: %d actual Go encodings", arch, len(want)-1) + }) + } +} + +func TestX86PackedCompareGo386HighRegisters(t *testing.T) { + for _, op := range x86PackedCompareTestOps { + for _, width := range []string{"X", "Y"} { + for _, regs := range [][3]int{{8, 0, 1}, {0, 8, 1}, {0, 1, 8}, {15, 14, 13}} { + operands := fmt.Sprintf("%s%d,%s%d,%s%d", width, regs[0], width, regs[1], width, regs[2]) + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(op+"/"+operands+"/"+triple, func(t *testing.T) { + source := "TEXT ·compare(SB),4,$0-0\n" + op + " " + operands + "\nRET\n" + // Preserve actual Go frontend acceptance, without treating + // those names as physical mode32 register identities. + x86LEAActualTargetGoObject(t, "386", triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86PackedIntegerCompareAPI(t, source, "func compare()\n", "386", triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("frontend high bank borrowed physical permission: IR=%d err=%v", len(ir), err) + } + }) + } + }) + } + } + } + } +} + +func TestX86PackedCompareGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86PackedCompareTestOps { + lines := []string{op + " X0,Y1,Y2", op + " Z0,Z1,Z2", op + " X16,X1,X2", op + " X0,X1,(AX)", op + ".Z X0,X1,K1", op + ".SAE X0,X1,K1", op + " X0,X1,K0,K2", op + ".BCST X0,X1,K2"} + if strings.HasSuffix(op, "B") || strings.HasSuffix(op, "W") { + lines = append(lines, op+".BCST (AX),X1,K2") + } + if arch == "386" { + lines = append(lines, op+" Z8,Z0,K1", op+" X0,X1,K1,K2") + } + for _, line := range lines { + dir := t.TempDir() + path := filepath.Join(dir, "invalid.s") + if err := os.WriteFile(path, []byte("TEXT bad(SB),4,$0-0\n"+line+"\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), path) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + if out, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("Go %s accepted %s: %s", arch, line, out) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + assertX86PackedSignedGreaterThanRejected(t, arch, triple, line) + } + } + } +} + +func TestX86PackedCompareMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + cross := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !cross { + t.Skip("execution requires amd64 or Rosetta; five-target object tests remain required") + } + var source, decl strings.Builder + checks := map[string]*strings.Builder{"views": {}, "memory": {}} + sigs := map[string]FuncSig{} + index := 0 + for _, op := range x86PackedCompareTestOps { + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + greater := 0 + if strings.Contains(op, "GT") { + greater = 1 + } + for w, width := range []string{"X", "Y", "Z"} { + for mode := 0; mode < 5; mode++ { // VEX reg/mem; EVEX reg/mem/broadcast. + if mode < 2 && w == 2 || mode == 4 && bits < 32 { + continue + } + for alias := 0; alias < 2; alias++ { + for _, raw := range []bool{false, true} { + name := fmt.Sprintf("compare%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),DI\nVMOVUPS (AX),Z0\n", name) + input := "(BX)" + if mode == 0 || mode == 2 { + source.WriteString("VMOVUPS (BX),Z1\n") + input = width + "1" + } + dst := width + "0" + if alias == 1 { + dst = width + "1" + } + if mode < 2 { + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s %s,%s0,%s", op, input, width, dst), raw) + fmt.Fprintf(&source, "VMOVUPS Z%d,(DI)\nVMOVUPS Y%d,64(DI)\nVMOVUPS X%d,96(DI)\n", alias, alias, alias) + } else { + dst = "K7" + if alias == 1 { + dst = "K1" + } + suffix := "" + if mode == 4 { + suffix = ".BCST" + } + source.WriteString("KMOVQ mask+24(FP),K1\n") + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s%s %s,%s0,K1,%s", op, suffix, input, width, dst), raw) + fmt.Fprintf(&source, "KMOVQ %s,(DI)\nVMOVUPS Z0,16(DI)\n", dst) + } + source.WriteString("RET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + fmt.Fprintf(&decl, "extern void %s(const void*,const void*,void*,uint64_t);\n", name) + kind := "views" + if mode >= 2 { + kind = "memory" + } + fmt.Fprintf(checks[kind], "if(check(%s,%d,%d,%d,%d)) {fprintf(stderr,\"%s failed\\n\");return 1;}\n", name, 16< +#include +#include +` + x86TestGuardPagesC + decl.String() + ` +static uint64_t lane(const uint8_t *p,int bytes) {uint64_t v=0;memcpy(&v,p,bytes);return v;} +static int cmp(const uint8_t *a,const uint8_t *b,int bytes,int gt) { + uint64_t x=lane(a,bytes),y=lane(b,bytes),sign=UINT64_C(1)<<(bytes*8-1); + return gt ? (x^sign)>(y^sign) : x==y; +} +static int check(void(*fn)(const void*,const void*,void*,uint64_t),int width,int bytes,int gt,int mode) { + for(int n=0;n<64;n++) { + uint8_t ab[65],bb[65],out[128],want[112]={0};uint8_t *a=ab+1,*b=bb+1; + for(int i=0;i<64;i++) {a[i]=(uint8_t)(i*37+n*19);b[i]=(n%3==0)?a[i]:(uint8_t)(i*13-n*7);} + uint64_t mask=(n==0)?0:(n==1)?UINT64_C(1)<<63:(n==2)?UINT64_MAX:UINT64_C(0x5a5a01239876cdef)>>(n%32),expected=0; + int lanes=width/bytes; + if(mode<2) { + for(int i=0;i>i)&1) expected|=(uint64_t)cmp(a+i*bytes,b+(mode==4?0:i*bytes),bytes,gt)<=3 && !active)?NULL:b,out,mask); + if(lane(out,8)!=expected || memcmp(out+16,a,64)) {fprintf(stderr,"mask compare mismatch mode=%d width=%d bytes=%d n=%d\n",mode,width,bytes,n);return 1;} + } + } + if(mode>=3) { + // Only the enabled source lanes are accessible. This detects an eager + // full-width read even when the mask is nonzero, not just NULL/zero mask. + uint8_t a[64],out[128];memset(a,0,64); + int lanes=width/bytes; + for(int enabled=1;enabled<=lanes;enabled++) { + int readable=(mode==4)?bytes:enabled*bytes; + uint8_t *b=guard+page-readable;memset(b,0,(size_t)readable); + uint64_t mask=enabled==64?UINT64_MAX:(UINT64_C(1)<= i+5 && code[i] == 0x62 && code[i+1]&7 == 3 && + (code[i+4] == 0x40 || code[i+4] == 0x41) { + return Instr{}, 0, true, fmt.Errorf("packed dot products have no EVEX encoding") + } + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 3 || + (code[i+3] != 0x40 && code[i+3] != 0x41) { + return Instr{}, 0, false, nil + } + fail := func(reason string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VEX packed dot: %s", reason) + } + if invalidPrefix { + return fail("machine-invalid legacy/REX prefix before VEX") + } + if physicalSegment || addressOverride { + return Instr{}, 0, true, fmt.Errorf("%w: VEX packed dot needs its original physical segment/address layout", ErrProbeNeedsContext) + } + p0, p1, opcode := code[i+1], code[i+2], code[i+3] + if p1&3 != 1 { + return fail("VEX.pp must select 66") + } + if opcode == 0x41 && p1&4 != 0 { + return fail("VDPPD requires VEX.128") + } + modRMIndex := i + 4 + if len(code) <= modRMIndex { + return fail("missing ModRM byte") + } + rExt, xExt, bExt := int(^p0>>7)&1, int(^p0>>6)&1, int(^p0>>5)&1 + secondNumber := int(^p1>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondNumber >= 8 || destinationNumber >= 8) { + return fail("extended register in physical mode32") + } + bank, width := "X", 16 + if p1&4 != 0 { + bank, width = "Y", 32 + } + first, consumed, err := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, bank) + if err != nil { + return Instr{}, 0, true, err + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return fail("missing immediate byte") + } + length := immediateIndex + 1 + if length > 15 { + return fail("instruction exceeds 15 bytes") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", bank, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", bank, destinationNumber))} + op := Op("VDPPS") + if opcode == 0x41 { + op = "VDPPD" + } + return Instr{ + Op: op, Args: []Operand{immediate, first, second, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate, first, second, destination), + x86Encoded: true, x86VectorBytes: width, + }, length, true, nil +} diff --git a/amd64_raw_packed_dot_test.go b/amd64_raw_packed_dot_test.go new file mode 100644 index 000000000..e71e645c4 --- /dev/null +++ b/amd64_raw_packed_dot_test.go @@ -0,0 +1,234 @@ +package plan9asm + +import ( + "encoding/binary" + "errors" + "fmt" + "reflect" + "testing" +) + +type x86RawDotOracleShape struct { + op Op + code byte + bank string + width int +} + +var x86RawDotOracleShapes = []x86RawDotOracleShape{ + {"VDPPD", 0x41, "X", 16}, + {"VDPPS", 0x40, "X", 16}, + {"VDPPS", 0x40, "Y", 32}, +} + +func x86RawDotOracleHeader(shape x86RawDotOracleShape, destination, second, bExt, xExt int, widthIgnored bool) []byte { + p0 := byte((1-destination/8)<<7 | (1-xExt)<<6 | (1-bExt)<<5 | 3) + p1 := byte((^second&15)<<3 | 1) + if shape.width == 32 { + p1 |= 4 + } + if widthIgnored { + p1 |= 0x80 + } + return []byte{0xc4, p0, p1, shape.code} +} + +func x86RawDotOracleInstruction(shape x86RawDotOracleShape, first Operand, second, destination int, immediate byte) Instr { + return Instr{Op: shape.op, Args: []Operand{ + {Kind: OpImm, Imm: int64(immediate)}, first, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", shape.bank, second))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", shape.bank, destination))}, + }} +} + +func x86RawDotRequireDecode(t *testing.T, code []byte, mode int, shape x86RawDotOracleShape, want Instr) { + t.Helper() + got, length, known, err := decodedX86VEXPackedDotInstruction(code, mode) + if err != nil || !known || length != len(code) || got.Op != want.Op || + !reflect.DeepEqual(got.Args, want.Args) || !got.x86Encoded || got.x86VectorBytes != shape.width { + t.Fatalf("complete original packed dot %x mode%d: got=%+v length=%d known=%v err=%v want=%+v", code, mode, got, length, known, err, want) + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + if _, known, err := x86PackedReductionProperties(got, arch); !known || err != nil { + t.Fatalf("exact physical dot did not share complete typed grammar: %+v: %v", got, err) + } +} + +func TestDecodedX86VEXPackedDotCompleteRegistersAndSelectors(t *testing.T) { + count := 0 + for _, mode := range []int{32, 64} { + limit := 16 + if mode == 32 { + limit = 8 + } + for _, shape := range x86RawDotOracleShapes { + t.Run(fmt.Sprintf("mode%d/%s/%s", mode, shape.op, shape.bank), func(t *testing.T) { + for _, widthIgnored := range []bool{false, true} { + for destination := 0; destination < limit; destination++ { + for second := 0; second < limit; second++ { + for first := 0; first < limit; first++ { + code := x86RawDotOracleHeader(shape, destination, second, first/8, 0, widthIgnored) + code = append(code, byte(0xc0|(destination&7)<<3|first&7), 0xa5) + input := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", shape.bank, first))} + x86RawDotRequireDecode(t, code, mode, shape, x86RawDotOracleInstruction(shape, input, second, destination, 0xa5)) + count++ + } + } + } + // Every imm8 is independent of the register-bank axis. Keep + // all ignored and meaningful selector bits for both sources. + for immediate := 0; immediate < 256; immediate++ { + for _, memory := range []bool{false, true} { + code := x86RawDotOracleHeader(shape, 2, 1, 0, 0, widthIgnored) + input := Operand{Kind: OpReg, Reg: Reg(shape.bank + "0")} + modRM := byte(0xd0) + if memory { + input = Operand{Kind: OpMem, Mem: MemRef{Base: BX}} + modRM = 0x13 + } + code = append(code, modRM, byte(immediate)) + x86RawDotRequireDecode(t, code, mode, shape, x86RawDotOracleInstruction(shape, input, 1, 2, byte(immediate))) + } + } + } + }) + } + } + if count != 27648 { + t.Fatal("complete two-mode physical register matrix lost rows", count) + } +} + +func TestDecodedX86VEXPackedDotMemoryPrefixesAndInvalidBoundaries(t *testing.T) { + registers := []Reg{AX, CX, DX, BX, SP, BP, SI, DI} + for _, mode := range []int{32, 64} { + for _, shape := range x86RawDotOracleShapes { + t.Run(fmt.Sprintf("mode%d/%s/%s", mode, shape.op, shape.bank), func(t *testing.T) { + for mod := 0; mod < 3; mod++ { + for rm := 0; rm < 8; rm++ { + sibLimit := 1 + if rm == 4 { + sibLimit = 256 + } + for sib := 0; sib < sibLimit; sib++ { + if mod == 0 && rm == 5 && mode == 64 { + continue // Original RIP layout has its separate negative below. + } + code := x86RawDotOracleHeader(shape, 2, 1, 0, 0, false) + code = append(code, byte(mod<<6|2<<3|rm)) + mem := MemRef{Base: registers[rm]} + absolute := mod == 0 && rm == 5 + if rm == 4 { + code = append(code, byte(sib)) + mem.Base = registers[sib&7] + if sib>>3&7 != 4 { + mem.Index, mem.Scale = registers[sib>>3&7], int64(1<>6)) + } + absolute = mod == 0 && sib&7 == 5 + } + if absolute { + mem.Base = "" + } + switch { + case mod == 1: + code = append(code, 0xf9) + mem.Off = -7 + case mod == 2 || absolute: + var displacement [4]byte + binary.LittleEndian.PutUint32(displacement[:], 0x89abcdef) + code = append(code, displacement[:]...) + mem.Off = -0x76543211 + } + code = append(code, 0x5a) + x86RawDotRequireDecode(t, code, mode, shape, + x86RawDotOracleInstruction(shape, Operand{Kind: OpMem, Mem: mem}, 1, 2, 0x5a)) + } + } + } + }) + } + } + shape := x86RawDotOracleShapes[2] + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + code := []byte{segment.prefix, 0xc4, 0x03, 0xe5, 0x40, 0x64, 0x88, 0x20, 0xa5} + input := Operand{Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: 32}} + x86RawDotRequireDecode(t, code, 64, shape, x86RawDotOracleInstruction(shape, input, 3, 12, 0xa5)) + if _, _, known, err := decodedX86VEXPackedDotInstruction(code, 32); !known || err == nil { + t.Fatal("physical mode32 borrowed extended GP/vector banks") + } + } + x86RawDotInvalidBoundaryOracle(t) +} + +func x86RawDotInvalidBoundaryOracle(t *testing.T) { + t.Helper() + valid := []byte{0xc4, 0xe3, 0x71, 0x41, 0xd0, 0xff} + for name, code := range map[string][]byte{ + "missing ModRM": valid[:4], + "missing imm8": valid[:5], + "VDPPD Y": {0xc4, 0xe3, 0x75, 0x41, 0xd0, 0xff}, + "wrong pp": {0xc4, 0xe3, 0x70, 0x41, 0xd0, 0xff}, + "missing SIB": {0xc4, 0xe3, 0x71, 0x41, 0x14}, + "partial disp8": {0xc4, 0xe3, 0x71, 0x41, 0x53}, + "partial disp32": {0xc4, 0xe3, 0x71, 0x41, 0x93, 1, 2, 3}, + "illegal EVEX": {0x62, 0xf3, 0x75, 0x08, 0x41, 0xd0, 0xff}, + } { + t.Run(name, func(t *testing.T) { + if got, _, known, err := decodedX86VEXPackedDotInstruction(code, 64); !known || err == nil { + t.Fatalf("invalid/truncated original bytes %x decoded as %+v known=%v err=%v", code, got, known, err) + } + }) + } + for _, mode := range []int{32, 64} { + for _, prefix := range []byte{0x66, 0xf2, 0xf3, 0xf0} { + code := append([]byte{prefix}, valid...) + if _, _, known, err := decodedX86VEXPackedDotInstruction(code, mode); !known || err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("invalid prefix %x mode%d was erased or called Context: known=%v err=%v", code, mode, known, err) + } + } + for _, prefix := range []byte{0x26, 0x2e, 0x36, 0x3e, 0x67} { + code := append([]byte{prefix}, valid...) + if _, _, known, err := decodedX86VEXPackedDotInstruction(code, mode); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unbound original physical prefix %x mode%d gained source authority: known=%v err=%v", code, mode, known, err) + } + } + } + for _, code := range [][]byte{ + {0xc4, 0x63, 0x71, 0x41, 0xd0, 0xff}, // R extension. + {0xc4, 0xa3, 0x71, 0x41, 0x14, 0x0b, 0xff}, // X extension. + {0xc4, 0xc3, 0x71, 0x41, 0xd0, 0xff}, // B extension. + {0xc4, 0xe3, 0x39, 0x41, 0xd0, 0xff}, // vvvv source8. + } { + if _, _, known, err := decodedX86VEXPackedDotInstruction(code, 32); !known || err == nil { + t.Fatalf("mode32 high-bank original bytes %x gained authority", code) + } + } + for rex := byte(0x40); rex <= 0x4f; rex++ { + if _, _, known, err := decodedX86VEXPackedDotInstruction(append([]byte{rex}, valid...), 64); !known || err == nil { + t.Fatal("machine-invalid REX+VEX prefix was discarded", rex) + } + } + tooLong := append([]byte{0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64}, valid...) + if _, _, known, err := decodedX86VEXPackedDotInstruction(tooLong, 64); !known || err == nil { + t.Fatal("overlong original instruction accepted") + } + rip := []byte{0xc4, 0xe3, 0x71, 0x41, 0x15, 1, 2, 3, 4, 0xff} + if _, _, known, err := decodedX86VEXPackedDotInstruction(rip, 64); !known || err == nil { + t.Fatal("unbound RIP input gained source-local storage") + } + physical, length, known, err := decodedX86VEXPackedDotInstructionWithMemory(rip, 64, x86RawMemoryDecoder{physicalPC: true}) + if !known || err != nil || length != len(rip) || physical.Args[1].Mem.Base != "RIP" { + t.Fatal("physical native boundary observer lost the complete RIP instruction", physical, length, known, err) + } + for _, code := range [][]byte{{0xc4}, {0xc4, 0xe2, 0x71, 0x41, 0xd0, 0xff}, {0xc4, 0xe3, 0x71, 0x42, 0xd0, 0xff}} { + if _, _, known, err := decodedX86VEXPackedDotInstruction(code, 64); known || err != nil { + t.Fatalf("unrelated opcode/map %x was claimed by dot family", code) + } + } +} diff --git a/amd64_raw_packed_extend_test.go b/amd64_raw_packed_extend_test.go new file mode 100644 index 000000000..2753703cd --- /dev/null +++ b/amd64_raw_packed_extend_test.go @@ -0,0 +1,234 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type rawPackedExtendTestSpec struct { + opcode byte + op Op + inputBits int + outputBits int +} + +var rawPackedExtendTestSpecs = []rawPackedExtendTestSpec{ + {opcode: 0x20, op: "VPMOVSXBW", inputBits: 8, outputBits: 16}, + {opcode: 0x21, op: "VPMOVSXBD", inputBits: 8, outputBits: 32}, + {opcode: 0x22, op: "VPMOVSXBQ", inputBits: 8, outputBits: 64}, + {opcode: 0x23, op: "VPMOVSXWD", inputBits: 16, outputBits: 32}, + {opcode: 0x24, op: "VPMOVSXWQ", inputBits: 16, outputBits: 64}, + {opcode: 0x25, op: "VPMOVSXDQ", inputBits: 32, outputBits: 64}, + {opcode: 0x30, op: "VPMOVZXBW", inputBits: 8, outputBits: 16}, + {opcode: 0x31, op: "VPMOVZXBD", inputBits: 8, outputBits: 32}, + {opcode: 0x32, op: "VPMOVZXBQ", inputBits: 8, outputBits: 64}, + {opcode: 0x33, op: "VPMOVZXWD", inputBits: 16, outputBits: 32}, + {opcode: 0x34, op: "VPMOVZXWQ", inputBits: 16, outputBits: 64}, + {opcode: 0x35, op: "VPMOVZXDQ", inputBits: 32, outputBits: 64}, +} + +func encodeX86VEXPackedExtend(opcode byte, width256 bool, destination, source int) []byte { + l := byte(0) + if width256 { + l = 1 + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source/8)<<5 | 2) + vex1 := byte(15<<3 | int(l)<<2 | 1) + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func encodeX86EVEXPackedExtend(opcode byte, vectorBits byte, mask int, zeroing bool, destination, source int) []byte { + p0 := byte((1-destination/8&1)<<7 | (1-source/16)<<6 | (1-source/8&1)<<5 | (1-destination/16)<<4 | 2) + p1 := byte(0x7d) + p2 := byte(vectorBits<<5 | 1<<3 | byte(mask)) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestTranslateRawEVEXPackedExtendWeaviateRegression(t *testing.T) { + const source = `TEXT rawPackedExtend(SB), $0-0 + LONG $0x487df262 + WORD $0x2432 + BYTE $0x07 + RET +` + // Original bytes lack DI/AX producers. Keep their Go object and + // three-API Context; the independent initialized block owns public success. + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + if !strings.Contains(ir, "zext i8") || !strings.Contains(ir, "to i64") { + t.Fatalf("raw VPMOVZXBQ lowering omitted byte-to-qword zero extension:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-evex-packed-extend.ll", "amd64-raw-evex-packed-extend.o", ir) +} + +func TestTranslateRawVEXPackedExtendWeaviateBlockRegression(t *testing.T) { + lastExtend := []byte{0xc4, 0xe2, 0x7d, 0x31, 0x7e, 0x18} + if got, length, ok, err := decodedX86PackedExtendInstruction(lastExtend, 64); err != nil || !ok || length != len(lastExtend) || got.Op != "VPMOVZXBD" { + t.Fatalf("direct decode %x = %+v, length=%d, ok=%v, err=%v", lastExtend, got, length, ok, err) + } + const source = `TEXT rawPackedExtendBlock(SB), $0-0 + LONG $0x317de2c4; BYTE $0x26 + LONG $0x317de2c4; WORD $0x086e + LONG $0xe45bfcc5 + LONG $0xed5bfcc5 + LONG $0x317de2c4; WORD $0x1076 + LONG $0xf65bfcc5 + LONG $0x317de2c4; WORD $0x187e + LONG $0xff5bfcc5 + LONG $0xb85de2c4; BYTE $0x1f + LONG $0xb855e2c4; WORD $0x2057 + LONG $0xb84de2c4; WORD $0x404f + LONG $0xb845e2c4; WORD $0x6047 + RET +` + // This complete unchanged block lacks SI/DI and old accumulator inputs. + // Its format object is not initialized public translation or execution. + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + if strings.Count(ir, "zext i8") < 4 || strings.Count(ir, "asm sideeffect \"vcvtdq2ps") != 4 { + t.Fatalf("raw VPMOVZXBD/VCVTDQ2PS block omitted conversions:\n%s", ir) + } + ir = x86SignedIntegerFloatOptimize(t, ir) + if strings.Count(ir, "asm sideeffect \"vcvtdq2ps") != 4 { + t.Fatal("O2 dropped raw VCVTDQ2PS side effects") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-packed-extend-block.ll", "amd64-raw-packed-extend-block.o", ir) +} + +func TestDecodedX86PackedExtendCompleteVEXRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawPackedExtendTestSpecs { + for _, width256 := range []bool{false, true} { + destinationPrefix := "X" + if width256 { + destinationPrefix = "Y" + } + for destination := 0; destination < 16; destination++ { + for source := 0; source < 16; source++ { + code := encodeX86VEXPackedExtend(spec.opcode, width256, destination, source) + got, length, ok, err := decodedX86PackedExtendInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantSource := fmt.Sprintf("X%d", source) + wantDestination := fmt.Sprintf("%s%d", destinationPrefix, destination) + if got.Op != spec.op || len(got.Args) != 2 || got.Args[0].String() != wantSource || got.Args[1].String() != wantDestination { + t.Fatalf("decode %x = %+v, want %s %s, %s", code, got, spec.op, wantSource, wantDestination) + } + count++ + } + } + } + } + if count != 6144 { + t.Fatalf("covered %d VEX packed-extend register encodings, want 6144", count) + } +} + +func TestDecodedX86PackedExtendCompleteEVEXRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawPackedExtendTestSpecs { + for vectorBits, destinationPrefix := range []string{"X", "Y", "Z"} { + destinationBytes := 16 << vectorBits + sourcePrefix := "X" + if destinationBytes*spec.inputBits/spec.outputBits > 16 { + sourcePrefix = "Y" + } + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for destination := 0; destination < 32; destination++ { + for source := 0; source < 32; source++ { + code := encodeX86EVEXPackedExtend(spec.opcode, byte(vectorBits), masking.mask, masking.zeroing, destination, source) + got, length, ok, err := decodedX86PackedExtendInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", sourcePrefix, source)} + if masking.mask != 0 { + wantArgs = append(wantArgs, fmt.Sprintf("K%d", masking.mask)) + } + wantArgs = append(wantArgs, fmt.Sprintf("%s%d", destinationPrefix, destination)) + if got.Op != spec.op+Op(masking.suffix) || len(got.Args) != len(wantArgs) { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op+Op(masking.suffix), strings.Join(wantArgs, ", ")) + } + for index := range wantArgs { + if got.Args[index].String() != wantArgs[index] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op+Op(masking.suffix), strings.Join(wantArgs, ", ")) + } + } + count++ + } + } + } + } + } + if count != 110592 { + t.Fatalf("covered %d EVEX packed-extend register encodings, want 110592", count) + } +} + +func TestDecodedX86PackedExtendCompressedMemoryAndInvalidForms(t *testing.T) { + for _, spec := range rawPackedExtendTestSpecs { + for vectorBits, destinationPrefix := range []string{"X", "Y", "Z"} { + destinationBytes := 16 << vectorBits + sourceBytes := destinationBytes * spec.inputBits / spec.outputBits + code := encodeX86EVEXPackedExtend(spec.opcode, byte(vectorBits), 2, true, 21, 0) + code[5] = 0x68 + code = append(code, 0x7f) + got, length, ok, err := decodedX86PackedExtendInstruction(code, 64) + wantSource := fmt.Sprintf("%d(AX)", sourceBytes*127) + wantDestination := fmt.Sprintf("%s21", destinationPrefix) + if err != nil || !ok || length != len(code) || got.Op != spec.op+".Z" || len(got.Args) != 3 || got.Args[0].String() != wantSource || got.Args[1].String() != "K2" || got.Args[2].String() != wantDestination { + t.Fatalf("decode %x = %+v, length=%d ok=%v err=%v; want %s.Z %s, K2, %s", code, got, length, ok, err, spec.op, wantSource, wantDestination) + } + } + } + + valid := encodeX86VEXPackedExtend(0x32, false, 0, 0) + invalidVVVV := append([]byte(nil), valid...) + invalidVVVV[2] &^= 0x08 + for name, code := range map[string][]byte{ + "truncated": {0xc4, 0xe2}, + "wrong map": {0xc4, 0xe1, 0x79, 0x32, 0xc0}, + "wrong pp": {0xc4, 0xe2, 0x78, 0x32, 0xc0}, + "unrelated opcode": {0xc4, 0xe2, 0x79, 0x2f, 0xc0}, + } { + if instruction, _, matched, decodeErr := decodedX86PackedExtendInstruction(code, 64); matched || decodeErr != nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, code, instruction, matched, decodeErr) + } + } + // This family's VEX.W is ignored. It must not be confused with EVEX + // dword-to-qword W0, or rejected merely because Go emits canonical W0. + if ins, _, matched, err := decodedX86PackedExtendInstruction([]byte{0xc4, 0xe2, 0xf9, 0x32, 0xc0}, 64); !matched || err != nil || ins.Op != "VPMOVZXBQ" { + t.Fatalf("legal VEX.WIG form rejected: %+v/%v", ins, err) + } + if _, _, matched, decodeErr := decodedX86PackedExtendInstruction(invalidVVVV, 64); !matched || decodeErr == nil { + t.Fatalf("non-reserved VEX.vvvv encoding %x returned ok=%v err=%v", invalidVVVV, matched, decodeErr) + } + invalidBroadcast := encodeX86EVEXPackedExtend(0x32, 2, 1, false, 0, 0) + invalidBroadcast[3] |= 0x10 + invalidBroadcast[5] = 0x40 + invalidBroadcast = append(invalidBroadcast, 1) + if _, _, matched, decodeErr := decodedX86PackedExtendInstruction(invalidBroadcast, 64); !matched || decodeErr == nil { + t.Fatalf("EVEX broadcast returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_packed_float_move.go b/amd64_raw_packed_float_move.go new file mode 100644 index 000000000..505887ec1 --- /dev/null +++ b/amd64_raw_packed_float_move.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's packed move grammars share width, direction and memory axes. The pinned +// generic decoder can misread VEX moves as legacy instructions or lose their +// vector width, so decode the complete float and VEX integer families here. +func decodedX86PackedMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedMoveInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedMoveInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || p.mapNumber != 1 { + return Instr{}, 0, false, nil + } + var op Op + store := false + switch p.opcode { + case 0x28, 0x29: + if p.pp > 1 { + return Instr{}, 0, false, nil + } + op = Op("VMOVA" + []string{"PS", "PD"}[p.pp]) + store = p.opcode == 0x29 + case 0x10, 0x11: + if p.pp > 1 { + return Instr{}, 0, false, nil + } + op = Op("VMOVU" + []string{"PS", "PD"}[p.pp]) + store = p.opcode == 0x11 + case 0x6f, 0x7f: + if p.evex || (p.pp != 1 && p.pp != 2) { + return Instr{}, 0, false, nil + } + op = Op([]string{"", "VMOVDQA", "VMOVDQU"}[p.pp]) + store = p.opcode == 0x7f + default: + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed float move: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 { + return fail("vvvv and V' must be reserved") + } + if p.evex && (!p.fixed || p.w != (p.pp == 1)) { + return fail("invalid EVEX fixed or width bit") + } + if p.vectorLength > 2 || p.broadcast { + return fail("reserved vector length or broadcast bit") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + registerForm := code[p.modRM]>>6 == 3 + if p.zero && (p.mask == 0 || store && !registerForm) { + return fail("zeroing requires a nonzero mask and register destination") + } + width := []string{"X", "Y", "Z"}[p.vectorLength] + regNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && ((!p.evex || width == "Z") && regNumber >= 8) { + return fail("extended register unavailable in 32-bit mode") + } + reg := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, regNumber))} + var rm Operand + var consumed int + var err error + if p.evex { + rm, consumed, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, width, 16<= 0x78 && p.opcode <= 0x7b { + names := [...]Op{"VCVTTPS2UQQ", "VCVTPS2UQQ", "VCVTTPS2QQ", "VCVTPS2QQ"} + name := names[p.opcode-0x78] + if p.w { + name = Op(strings.Replace(string(name), "PS", "PD", 1)) + } + return name + } + switch { + case p.opcode == 0x5b && p.pp == 1 && !p.w: + return "VCVTPS2DQ" + case p.opcode == 0x5b && p.pp == 2 && !p.w: + return "VCVTTPS2DQ" + case p.opcode == 0xe6 && p.pp == 3 && p.w: + return "VCVTPD2DQ" + case p.opcode == 0xe6 && p.pp == 1 && p.w: + return "VCVTTPD2DQ" + case (p.opcode == 0x78 || p.opcode == 0x79) && p.pp == 0: + name := "VCVT" + if p.opcode == 0x78 { + name += "T" + } + if p.w { + return Op(name + "PD2UDQ") + } + return Op(name + "PS2UDQ") + case p.opcode == 0x7a && p.pp == 2 && !p.w: + return "VCVTUDQ2PD" + case p.opcode == 0x7a && p.pp == 3 && !p.w: + return "VCVTUDQ2PS" + case p.opcode == 0xe6 && p.pp == 2 && p.w: + return "VCVTQQ2PD" + case p.opcode == 0x7a && p.pp == 2 && p.w: + return "VCVTUQQ2PD" + case p.opcode == 0x5b && p.pp == 0 && p.w: + return "VCVTQQ2PS" + case p.opcode == 0x7a && p.pp == 3 && p.w: + return "VCVTUQQ2PS" + } + return "" +} + +func decodedX86PackedNumericConversionInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedNumericConversionInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedNumericConversionInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched { + return Instr{}, 0, false, nil + } + name := x86RawPackedNumericName(p) + if name == "" { + return Instr{}, 0, false, nil + } + fail := func(reason string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed numeric conversion: %s", reason) + } + if mode != 32 && mode != 64 || p.addressOverride || p.upper != 0 || + p.evex && (!p.fixed || p.zero && p.mask == 0) || len(code) <= p.modRM { + return fail("invalid physical mode, prefix, reserved field, or truncated ModRM") + } + register := code[p.modRM]>>6 == 3 + embedded := p.evex && p.broadcast && register + vl := p.vectorLength + if embedded { + vl = 2 + } + if vl > 2 || !p.evex && vl > 1 { + return fail("reserved vector length") + } + spec, _ := x86PackedConversionSpecification(name) + if spec.sourceBits > spec.destinationBits && vl < 2 { + name += [...]Op{"X", "Y"}[vl] + spec, _ = x86PackedConversionSpecification(name) + } + shape := spec.shapes[0] + if len(spec.shapes) > 1 { + shape = spec.shapes[vl] + } + if embedded && spec.explicit == "" { + return fail("embedded rounding/SAE absent from this conversion") + } + width := shape.lanes * spec.sourceBits / 8 + if p.broadcast && !register { + width = spec.sourceBits / 8 + } + prefix := map[int]string{16: "X", 32: "Y", 64: "Z"} + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, prefix[shape.sourceBytes], width) + } else { + source, consumed, err = memory.vex(code[p.modRM:], mode, p.b, p.x, p.segment, prefix[shape.sourceBytes]) + } + if err != nil { + return Instr{}, 0, true, err + } + if embedded { + if spec.explicit == "sae" { + name += ".SAE" // LL is ignored for SAE-only register encodings. + } else { + name += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } else if p.broadcast { + name += ".BCST" + } + if p.zero { + name += ".Z" + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix[shape.destinationBytes], int(code[p.modRM]>>3&7)+p.r*8))}) + ins := Instr{Op: name, Args: args, x86Encoded: true, x86PackedNumericEVEX: p.evex} + arch := "amd64" + if mode == 32 { + arch = "386" + } + shapeInput := ins + if memory.physicalPC && source.Kind == OpMem && source.Mem.Base == "RIP" { + // The physical decoder has already checked the complete disp32. + // Check the conversion's memory class, not a Go source RIP spelling. + // Return the original physical operand; only a boundary observer may + // request this mode and it discards all decoded operands. + shapeInput.Args = append([]Operand(nil), ins.Args...) + shapeInput.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Base: AX}} + } + _, _, err = x86PackedConversionProperties(shapeInput, arch) + if err != nil { + return Instr{}, 0, true, err + } + printed := make([]string, len(args)) + for index, operand := range args { + printed[index] = operand.String() + } + ins.Raw = string(name) + " " + strings.Join(printed, ", ") + return ins, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_packed_precision_convert.go b/amd64_raw_packed_precision_convert.go new file mode 100644 index 000000000..ea2675bc1 --- /dev/null +++ b/amd64_raw_packed_precision_convert.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Opcode 5A with no prefix or 66 converts packed singles to doubles or +// doubles to singles. Go 1.27 exposes the VEX.128/.256 and EVEX.128/.256/.512 +// forms under VCVTPS2PD and VCVTPD2PSX/Y (or unsuffixed Z). Scalar F2/F3 5A +// belongs to the separate scalar-precision grammar. +func decodedX86PackedPrecisionConvertInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || p.opcode != 0x5a || (p.pp != 0 && p.pp != 1) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed precision conversion: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 { + return fail("reserved vvvv/V' field") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.evex { + if !p.fixed || p.w != (p.pp == 1) || p.zero && p.mask == 0 { + return fail("invalid EVEX fixed, width, mask or zeroing field") + } + } + + registerSource := code[p.modRM]>>6 == 3 + if mode == 32 { + if !p.evex && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended VEX register in 32-bit mode") + } + if p.evex && !registerSource && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + } + embeddedControl := p.evex && p.broadcast && registerSource + broadcast := p.evex && p.broadcast && !registerSource + if p.evex && !embeddedControl && p.vectorLength > 2 { + return fail("reserved EVEX vector length") + } + + vectorLength := p.vectorLength + if embeddedControl { + vectorLength = 2 // SAE or rounding implies the 512-bit register form. + } + var op Op + var sourcePrefix, destinationPrefix string + var memoryScale int + if p.pp == 0 { + op = "VCVTPS2PD" + destinationPrefix = [...]string{"X", "Y", "Z"}[vectorLength] + sourcePrefix = "X" + if vectorLength == 2 { + sourcePrefix = "Y" + } + memoryScale = 8 << vectorLength + } else { + op = "VCVTPD2PS" + if vectorLength < 2 { + op += [...]Op{"X", "Y"}[vectorLength] + } + sourcePrefix = [...]string{"X", "Y", "Z"}[vectorLength] + destinationPrefix = "X" + if vectorLength == 2 { + destinationPrefix = "Y" + } + memoryScale = 16 << vectorLength + } + if mode == 32 { + if sourcePrefix == "Z" && registerSource && (p.b != 0 || p.x != 0) { + return fail("Z8-Z31 are not Go assembler source registers in 32-bit mode") + } + if destinationPrefix == "Z" && p.r != 0 { + return fail("Z8-Z31 are not Go assembler destination registers in 32-bit mode") + } + } + if embeddedControl { + if p.pp == 0 { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } else if broadcast { + op += ".BCST" + if p.pp == 0 { + memoryScale = 4 + } else { + memoryScale = 8 + } + } + if p.zero { + op += ".Z" + } + + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix, memoryScale) + } else { + source, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix) + } + if err != nil { + return Instr{}, 0, true, err + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationPrefix, int(code[p.modRM]>>3&7)+p.r*8))} + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for i, arg := range args { + printed[i] = arg.String() + } + return Instr{Op: op, Args: args, x86Encoded: true, x86PackedPrecisionEVEX: p.evex, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_packed_precision_convert_test.go b/amd64_raw_packed_precision_convert_test.go new file mode 100644 index 000000000..134d9282c --- /dev/null +++ b/amd64_raw_packed_precision_convert_test.go @@ -0,0 +1,309 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +var x86RawPackedPrecisionForms = []struct { + op Op + pp byte + length byte + evex bool + src Reg + dst Reg +}{ + {"VCVTPS2PD", 0, 0, false, "X1", "X0"}, + {"VCVTPS2PD", 0, 1, false, "X1", "Y0"}, + {"VCVTPS2PD", 0, 0, true, "X1", "X0"}, + {"VCVTPS2PD", 0, 1, true, "X1", "Y0"}, + {"VCVTPS2PD", 0, 2, true, "Y1", "Z0"}, + {"VCVTPD2PSX", 1, 0, false, "X1", "X0"}, + {"VCVTPD2PSY", 1, 1, false, "Y1", "X0"}, + {"VCVTPD2PSX", 1, 0, true, "X1", "X0"}, + {"VCVTPD2PSY", 1, 1, true, "Y1", "X0"}, + {"VCVTPD2PS", 1, 2, true, "Z1", "Y0"}, +} + +func rawPackedPrecisionBytes(pp, length byte, evex, memory bool) []byte { + modRM := byte(0xc1) + if memory { + modRM = 0x08 + } + if evex { + w := byte(0) + if pp == 1 { + w = 0x80 + } + return []byte{0x62, 0xf1, 0x7c | pp | w, 0x08 | length<<5, 0x5a, modRM} + } + return []byte{0xc5, 0xf8 | pp | length<<2, 0x5a, modRM} +} + +func TestX86RawPackedPrecisionConvertArrowRegression(t *testing.T) { + // Arrow emits this VEX.256 VCVTPS2PD as raw bytes. x/arch v0.14 + // incorrectly starts with a legacy POP rather than decoding the vector op. + code := []byte{0xc5, 0xfc, 0x5a, 0x04, 0xba} + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != "VCVTPS2PD" { + t.Fatalf("raw %#x decoded as %+v", code, decoded.Instrs) + } +} + +func TestX86RawPackedPrecisionConvertCompleteGo127BaseForms(t *testing.T) { + if len(x86RawPackedPrecisionForms) != 10 { + t.Fatal("Go 1.27 packed 5A table requires ten VEX/EVEX width forms") + } + for _, form := range x86RawPackedPrecisionForms { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("%s/LL=%d/EVEX=%t/memory=%t", form.op, form.length, form.evex, memory) + t.Run(name, func(t *testing.T) { + code := rawPackedPrecisionBytes(form.pp, form.length, form.evex, memory) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != form.op { + t.Fatalf("raw %#x decoded as %+v; want %s", code, decoded.Instrs, form.op) + } + args := decoded.Instrs[0].Args + if len(args) != 2 { + t.Fatalf("raw %#x operands = %+v", code, args) + } + if memory { + if args[0].Kind != OpMem || args[0].Mem.Base != AX { + t.Fatalf("raw %#x memory source = %+v", code, args[0]) + } + } else if args[0].Kind != OpReg || args[0].Reg != form.src { + t.Fatalf("raw %#x source = %+v; want %s", code, args[0], form.src) + } + wantDestination := form.dst + if memory { + wantDestination = Reg(string(form.dst[:1]) + "1") + } + if args[1].Reg != wantDestination { + t.Fatalf("raw %#x destination = %s; want %s", code, args[1].Reg, wantDestination) + } + }) + } + } +} + +func TestX86RawPackedPrecisionConvertEncodingAxes(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + op Op + args []string + }{ + {"VEX-W-ignored", []byte{0xc4, 0xe1, 0xfc, 0x5a, 0xc1}, "VCVTPS2PD", []string{"X1", "Y0"}}, + {"masked", []byte{0x62, 0xf1, 0x7c, 0x09, 0x5a, 0xc1}, "VCVTPS2PD", []string{"X1", "K1", "X0"}}, + {"broadcast-zero", []byte{0x62, 0xf1, 0x7c, 0x99, 0x5a, 0x00}, "VCVTPS2PD.BCST.Z", []string{"0(AX)", "K1", "X0"}}, + {"single-to-double-SAE", []byte{0x62, 0xf1, 0x7c, 0x18, 0x5a, 0xc1}, "VCVTPS2PD.SAE", []string{"Y1", "Z0"}}, + {"double-to-single-round-down", []byte{0x62, 0xf1, 0xfd, 0x39, 0x5a, 0xd8}, "VCVTPD2PS.RD_SAE", []string{"Z0", "K1", "Y3"}}, + {"extended-registers", []byte{0x62, 0xa1, 0x7c, 0x08, 0x5a, 0xc1}, "VCVTPS2PD", []string{"X17", "X16"}}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86PackedPrecisionConvertInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) || got.Op != tc.op { + t.Fatalf("decode %#x = %+v, %d, %v, %v; want %s", tc.code, got, length, recognized, err, tc.op) + } + if len(got.Args) != len(tc.args) { + t.Fatalf("decode %#x operands = %+v; want %v", tc.code, got.Args, tc.args) + } + for index, arg := range got.Args { + if arg.String() != tc.args[index] { + t.Fatalf("decode %#x operand %d = %s; want %s", tc.code, index, arg.String(), tc.args[index]) + } + } + }) + } + for _, tc := range []struct { + name string + code []byte + off int64 + }{ + {"single-Z-disp8", []byte{0x62, 0xf1, 0x7c, 0x48, 0x5a, 0x40, 0x02}, 64}, + {"double-Y-disp8", []byte{0x62, 0xf1, 0xfd, 0x28, 0x5a, 0x40, 0x02}, 64}, + {"double-broadcast-disp8", []byte{0x62, 0xf1, 0xfd, 0x38, 0x5a, 0x40, 0x02}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + got, _, recognized, err := decodedX86PackedPrecisionConvertInstruction(tc.code, 64) + if !recognized || err != nil || got.Args[0].Kind != OpMem || got.Args[0].Mem.Off != tc.off { + t.Fatalf("decode %#x = %+v, %v, %v; want offset %d", tc.code, got, recognized, err, tc.off) + } + }) + } + for _, code := range [][]byte{ + {0x62, 0xa1, 0x7c, 0x08, 0x5a, 0xc1}, // X17 -> X16. + {0x62, 0xa1, 0xfd, 0x28, 0x5a, 0xc1}, // Y17 -> X16. + } { + if got, _, recognized, err := decodedX86PackedPrecisionConvertInstruction(code, 32); !recognized || err != nil || len(got.Args) != 2 { + t.Fatalf("386 EVEX high X/Y register %#x = %+v, %v, %v", code, got, recognized, err) + } + } + requireX86GoAssemblerResult(t, "386", "TEXT high(SB),4,$0-0\n\tVCVTPS2PD X17, K1, X16\n\tVCVTPD2PSY Y17, K1, X16\n\tRET\n", true) +} + +func TestX86RawPackedPrecisionConvertRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"VEX-vvvv", []byte{0xc5, 0xe8, 0x5a, 0xc1}, 64}, + {"EVEX-vvvv", []byte{0x62, 0xf1, 0x6c, 0x08, 0x5a, 0xc1}, 64}, + {"EVEX-zero-without-mask", []byte{0x62, 0xf1, 0x7c, 0x88, 0x5a, 0xc1}, 64}, + {"EVEX-LL3-without-control", []byte{0x62, 0xf1, 0x7c, 0x68, 0x5a, 0xc1}, 64}, + {"EVEX-W-mismatch", []byte{0x62, 0xf1, 0xfc, 0x08, 0x5a, 0xc1}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xf8, 0x5a, 0xc1}, 64}, + {"missing-ModRM", []byte{0xc5, 0xf8, 0x5a}, 64}, + {"extended-386", []byte{0xc4, 0x61, 0x78, 0x5a, 0xc1}, 32}, + {"Z-destination-386", []byte{0x62, 0xe1, 0x7c, 0x48, 0x5a, 0xc1}, 32}, + {"Z-source-386", []byte{0x62, 0xb1, 0xfd, 0x48, 0x5a, 0xc1}, 32}, + {"extended-memory-386", []byte{0x62, 0xb1, 0x7c, 0x08, 0x5a, 0x01}, 32}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86PackedPrecisionConvertInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } +} + +func TestX86RawPackedPrecisionConvertCompilesEveryBaseForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawprecisionconversion(SB),4,$0-0\n") + encodings := make([][]byte, 0, len(x86RawPackedPrecisionForms)+5) + for _, form := range x86RawPackedPrecisionForms { + encodings = append(encodings, rawPackedPrecisionBytes(form.pp, form.length, form.evex, false)) + } + encodings = append(encodings, + []byte{0x62, 0xf1, 0x7c, 0x99, 0x5a, 0x00}, // PS broadcast, mask, zero. + []byte{0x62, 0xf1, 0x7c, 0x18, 0x5a, 0xc1}, // PS suppress exceptions. + []byte{0x62, 0xf1, 0xfd, 0x39, 0x5a, 0xd8}, // PD embedded rounding. + []byte{0x62, 0xf1, 0xfd, 0x38, 0x5a, 0x40, 0x02}, // PD broadcast disp8. + []byte{0x62, 0xa1, 0x7c, 0x08, 0x5a, 0xc1}, // EVEX high X in 386/amd64. + ) + for _, code := range encodings { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + source.WriteString("\tRET\n") + // Every original byte/NOP/RET and its actual Go object stay intact. + // The complete bare source lacks inputs; only the private emitter + // checks bound formats, while the initialized raw family is separate. + ir := x86PackedPrecisionOriginalFormatObject(t, source.String(), "rawprecisionconversion", + target.goarch, target.triple, true, 15, 1) + compileLLVMToObject(t, llc, target.triple, "raw-precision-conversion.ll", "raw-precision-conversion.o", ir) + }) + } +} + +func TestX86RawPackedPrecisionConvertRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const namedSource = ` +TEXT packedprecision(SB),4,$0-24 + MOVQ out+0(FP), AX + MOVQ singles+8(FP), BX + MOVQ doubles+16(FP), CX + VCVTPS2PD (BX), Y1 + VMOVUPD Y1, (AX) + VCVTPD2PSY (CX), X2 + VMOVUPS X2, 32(AX) + VZEROUPPER + RET +` + rawSource := strings.NewReplacer( + "VCVTPS2PD (BX), Y1", "BYTE $0xc5; BYTE $0xfc; BYTE $0x5a; BYTE $0x0b", + "VCVTPD2PSY (CX), X2", "BYTE $0xc5; BYTE $0xfd; BYTE $0x5a; BYTE $0x11", + ).Replace(namedSource) + requireX86GoAssemblerResult(t, "amd64", namedSource, true) + requireX86GoAssemblerResult(t, "amd64", rawSource, true) + + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + options := Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedprecision": { + Name: "packedprecision", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + }, + } + const mainC = ` +#include +#include +extern void packedprecision(uint8_t *, const float *, const double *); +int main(void) { + const float singles[4] = {1.5f, -2.25f, 3.0f, 4.0f}; + const double doubles[4] = {1.5, -2.25, 3.0, 4.0}; + uint8_t out[48]; + double widened[4]; + float narrowed[4]; + memset(out, 0xcc, sizeof(out)); + packedprecision(out, singles, doubles); + memcpy(widened, out, sizeof(widened)); + memcpy(narrowed, out + 32, sizeof(narrowed)); + for (int i = 0; i < 4; i++) { + if (widened[i] != (double)singles[i]) return 10 + i; + if (narrowed[i] != (float)doubles[i]) return 20 + i; + } + return 0; +} +` + for _, tc := range []struct{ name, source string }{ + {"named", namedSource}, + {"raw", rawSource}, + } { + t.Run(tc.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, tc.source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_precision_"+tc.name, triple, ir, mainC, runPrefix) + }) + } +} diff --git a/amd64_raw_packed_sign.go b/amd64_raw_packed_sign.go new file mode 100644 index 000000000..c280020d5 --- /dev/null +++ b/amd64_raw_packed_sign.go @@ -0,0 +1,44 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27's _yvaddsubpd table gives all three VPSIGN instructions exactly +// the same VEX.128/VEX.256 register-and-memory operand grammar. +var x86RawPackedSignOps = [...]Op{"VPSIGNB", "VPSIGNW", "VPSIGND"} + +func decodedX86RawPackedSignInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || p.opcode < 0x08 || p.opcode > 0x0a { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed sign: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.pp != 1 || p.addressOverride { + return fail("packed sign requires VEX.66.0F38.WIG without address override") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + width := [...]string{"X", "Y"}[p.vectorLength] + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (destinationNumber >= 8 || p.upper >= 8) { + return fail("extended vector register unavailable in 32-bit mode") + } + source, consumed, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))} + op := x86RawPackedSignOps[p.opcode-0x08] + args := []Operand{source, data, destination} + printed := []string{source.String(), data.String(), destination.String()} + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_packed_sign_test.go b/amd64_raw_packed_sign_test.go new file mode 100644 index 000000000..05eaf783b --- /dev/null +++ b/amd64_raw_packed_sign_test.go @@ -0,0 +1,173 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "os/exec" + "reflect" + "strings" + "testing" +) + +func TestX86RawPackedSignGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawpackedsign(SB),4,$0-0\n") + for _, op := range []string{"VPSIGNB", "VPSIGNW", "VPSIGND"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "%s %s2, %s3, %s4\n", op, width, width, width) + fmt.Fprintf(&source, "%s 24(BX), %s3, %s4\n", op, width, width) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s12, %s13, %s14\n", op, width, width, width) + fmt.Fprintf(&source, "%s -32(R8), %s13, %s14\n", op, width, width) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d=%+v, want %+v", index, got, want[index]) + } + } + }) + } +} + +func TestX86RawPackedSignRejectsFormsOutsideGoTable(t *testing.T) { + const source = "TEXT rawpackedsign(SB),4,$0-0\nVPSIGNB Y2, Y3, Y4\nRET\n" + assembled := assembleX87ControlBytes(t, "amd64", source) + if len(assembled) < 6 || assembled[0] != 0xc4 { + t.Fatalf("Go did not emit a VEX packed sign: %x", assembled) + } + valid := assembled[:len(assembled)-1] + for _, test := range []struct { + name string + mutate func([]byte) []byte + }{ + {"wrong prefix", func(code []byte) []byte { code[2] &^= 3; return code }}, + {"EVEX outside SIGN", func([]byte) []byte { + return encodeX86EVEXPackedWordMultiply(2, 0x08, 0, 0, false, false, 4, 3, 2) + }}, + {"address override", func(code []byte) []byte { return append([]byte{0x67}, code...) }}, + {"missing ModRM", func(code []byte) []byte { return code[:4] }}, + {"high register on 386", func(code []byte) []byte { code[1] &^= 0x80; return code }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.mutate(append([]byte(nil), valid...)) + mode := 64 + if test.name == "high register on 386" { + mode = 32 + } + if _, _, recognized, err := decodedX86RawPackedSignInstruction(code, mode); !recognized || err == nil { + t.Fatalf("invalid %x: recognized=%v err=%v", code, recognized, err) + } + }) + } +} + +func TestTranslateX86RawPackedSignLLVM22AllTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, form := range []string{ + "VPSIGNB Y2, Y3, Y4", + "VPSIGNW X2, X3, X4", + "VPSIGND 16(BX), Y3, Y4", + } { + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(form+"/"+target.triple, func(t *testing.T) { + source := "TEXT rawpackedsign(SB),4,$0-0\n" + form + "\nRET\n" + code := assembleX87ControlBytes(t, target.goarch, source) + x86RawPackedSignIndependentPhysicalPrior(t, form, code, target.triple) + var raw strings.Builder + raw.WriteString("TEXT rawpackedsign(SB),4,$0-0\n") + for _, value := range code { + fmt.Fprintf(&raw, "BYTE $%#02x\n", value) + } + // All original bytes and RET remain unchanged. A bare raw + // format has no vector/address inputs: its three public APIs + // must report Context, while a private emitter proves objects. + // The independent 114-row initialized family is exercised by + // TestX86PackedAbsSignInitializedObjectsThreeAPIsFiveTargets. + file, signatures := x86UnboundVoidFormatSources(t, raw.String(), target.goarch, target.triple) + if len(file.Funcs) != 1 || len(file.Data) != 0 || len(file.Globl) != 0 { + t.Fatal("original packed-sign format lost its single byte-only body") + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.goarch) + if err != nil || len(decoded.Instrs) != 3 || decoded.Instrs[1].Op != Op(strings.Fields(form)[0]) || + decoded.Instrs[2].Op != OpRET || decoded.Instrs[2].x86RawNearReturn == nil { + t.Fatal("original packed-sign instruction and encoded RET were not retained", err) + } + // Private format emission is not the public normalizer's + // physical return/nonalias contract. In particular the memory + // row cannot borrow that contract merely to compile its format. + unit := *file + unit.Funcs = []Func{decoded} + ir := x86ScalarFormatObjectUnit(t, &unit, signatures, target.goarch, target.triple, 0) + for _, operation := range []string{"icmp slt <", "icmp sgt <", "sub <", "select <"} { + if !strings.Contains(ir, operation) { + t.Fatal("original raw sign format lost its complete integer operation", operation) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-packed-sign.ll", "raw-packed-sign.o", ir) + }) + } + } +} + +func x86RawPackedSignIndependentPhysicalPrior(t *testing.T, form string, code []byte, triple string) { + t.Helper() + // Independent original Go bytes, including the ordinary encoded RET. + // Neither a decoder nor a private LLVM format object authenticates these. + want := map[string][]byte{ + "VPSIGNB Y2, Y3, Y4": {0xc4, 0xe2, 0x65, 0x08, 0xe2, 0xc3}, + "VPSIGNW X2, X3, X4": {0xc4, 0xe2, 0x61, 0x09, 0xe2, 0xc3}, + "VPSIGND 16(BX), Y3, Y4": {0xc4, 0xe2, 0x65, 0x0a, 0x63, 0x10, 0xc3}, + } + if !bytes.Equal(code, want[form]) { + t.Fatalf("original complete packed-sign Go bytes changed: %s got=%x want=%x", form, code, want[form]) + } + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("independent packed-sign physical prior requires LLVM22 llvm-mc") + } + var input strings.Builder + for _, value := range code { + fmt.Fprintf(&input, "0x%02x ", value) + } + input.WriteByte('\n') + stage, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(stage, mc, "--disassemble", "--triple="+triple, "--mattr=+avx2") + command.Stdin = strings.NewReader(input.String()) + output, err := runtimeOracleCombinedOutput(stage, command) + mnemonic := strings.ToLower(strings.Fields(form)[0]) + if err != nil || strings.Contains(string(output), "warning") || strings.Contains(string(output), "invalid") || + strings.Count(string(output), mnemonic) != 1 { + t.Fatalf("independent LLVM22 original packed-sign decoding failed: %v\n%s", err, output) + } +} diff --git a/amd64_raw_packed_testmask.go b/amd64_raw_packed_testmask.go new file mode 100644 index 000000000..e665384c1 --- /dev/null +++ b/amd64_raw_packed_testmask.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27's _yvpshufbitqmb forms are the product of test polarity (66/F3), +// B/W versus D/Q opcode, and EVEX.W. Each row has X/Y/Z widths and an +// optional K write-mask; only D/Q permits scalar-memory broadcast. +var x86RawPackedTestMaskOps = [2][2][2]Op{ + { + {"VPTESTMB", "VPTESTMW"}, + {"VPTESTMD", "VPTESTMQ"}, + }, + { + {"VPTESTNMB", "VPTESTNMW"}, + {"VPTESTNMD", "VPTESTNMQ"}, + }, +} + +func decodedX86RawPackedTestMaskInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || !p.evex || p.mapNumber != 2 || + (p.pp != 1 && p.pp != 2) || (p.opcode != 0x26 && p.opcode != 0x27) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed test mask: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.vectorLength > 2 || p.r != 0 || p.zero { + return fail("invalid reserved EVEX encoding bit") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + registerSource := modRM>>6 == 3 + if p.broadcast && (registerSource || p.opcode != 0x27) { + return fail("broadcast requires a D/Q scalar-memory source") + } + if mode == 32 && !registerSource && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + + polarity := p.pp - 1 + widthGroup := p.opcode - 0x26 + w := 0 + if p.w { + w = 1 + } + op := x86RawPackedTestMaskOps[polarity][widthGroup][w] + laneBits := amd64PackedTestMaskSpecs[string(op)].laneBits + vectorPrefix := [...]string{"X", "Y", "Z"}[p.vectorLength] + accessBytes := 16 << p.vectorLength + if p.broadcast { + accessBytes = laneBits / 8 + op += ".BCST" + } + first, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, vectorPrefix, accessBytes) + if err != nil { + return Instr{}, 0, true, err + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, p.upper))} + args := []Operand{first, second} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + destination := modRM >> 3 & 7 + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", destination))}) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86Encoded: true, + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_packed_testmask_test.go b/amd64_raw_packed_testmask_test.go new file mode 100644 index 000000000..1e9633c3b --- /dev/null +++ b/amd64_raw_packed_testmask_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func encodeX86RawPackedTestMask(polarity, widthGroup, w, vectorBits, first, second, destination, writeMask int, memory, broadcast bool) []byte { + if memory { + first = 0 + } + p0 := byte(1<<7 | (1-first/16)<<6 | (1-first/8&1)<<5 | 1<<4 | 2) + p1 := byte(4 | (polarity + 1) | (^second&15)<<3) + if w != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | (1-second/16)<<3 | writeMask) + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | destination<<3 | first&7) + if memory { + modRM = byte(0x40 | destination<<3) + } + code := []byte{0x62, p0, p1, p2, byte(0x26 + widthGroup), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestTranslateRawPackedTestMaskGorseRegression(t *testing.T) { + const source = `TEXT rawPackedTestMask(SB), $0-0 + LONG $0x4875f262 + WORD $0xc026 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawPackedTestMask": {Name: "rawPackedTestMask", Ret: Void}}, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original raw test-to-mask invented native entry inputs", err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + file.Funcs[0] = decoded + ir = x86InstructionFormatObjectUnit(t, file, map[string]FuncSig{ + "rawPackedTestMask": {Name: "rawPackedTestMask", Ret: Void}, + }, "amd64", "x86_64-unknown-linux-gnu", 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerPackedTestMask(ins.Op, ins) + }) + if !strings.Contains(ir, "icmp ne") { + t.Fatalf("raw VPTESTMB omitted per-lane mask test:\n%s", ir) + } +} + +func TestDecodedX86RawPackedTestMaskCompleteGoRows(t *testing.T) { + if len(amd64PackedTestMaskSpecs) != 8 { + t.Fatalf("Go 1.27 packed test-mask family has %d members, want eight", len(amd64PackedTestMaskSpecs)) + } + seen := make(map[Op]bool) + for polarity, groups := range x86RawPackedTestMaskOps { + for widthGroup, widths := range groups { + for w, op := range widths { + spec, ok := amd64PackedTestMaskSpecs[string(op)] + if !ok || seen[op] || spec.testZero != (polarity != 0) || + spec.laneBits != (8<<(widthGroup*2+w)) { + t.Fatalf("packed test-mask row %d/%d/%d = %s, spec %+v", polarity, widthGroup, w, op, spec) + } + seen[op] = true + for vectorBits, prefix := range [...]string{"X", "Y", "Z"} { + for _, memory := range []bool{false, true} { + for _, writeMask := range []int{0, 3} { + for _, broadcast := range []bool{false, true} { + if broadcast && (!memory || widthGroup == 0) { + continue + } + code := encodeX86RawPackedTestMask(polarity, widthGroup, w, vectorBits, 20, 21, 2, writeMask, memory, broadcast) + got, length, recognized, err := decodedX86RawPackedTestMaskInstruction(code, 64) + if err != nil || !recognized || length != len(code) { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + wantOp := op + if broadcast { + wantOp += ".BCST" + } + if got.Op != wantOp || got.Args[1].String() != fmt.Sprintf("%s21", prefix) || + got.Args[len(got.Args)-1].String() != "K2" { + t.Fatalf("decode %x = %+v, want %s %s21, K2", code, got, wantOp, prefix) + } + if writeMask != 0 && (len(got.Args) != 4 || got.Args[2].String() != "K3") || + writeMask == 0 && len(got.Args) != 3 { + t.Fatalf("decode %x = %+v, lost write mask", code, got) + } + first := got.Args[0] + if memory { + wantBytes := 16 << vectorBits + if broadcast { + wantBytes = spec.laneBits / 8 + } + if first.Kind != OpMem || first.Mem.Base != AX || first.Mem.Off != int64(wantBytes) { + t.Fatalf("decode %x = %+v, want compressed displacement %d", code, got, wantBytes) + } + } else if first.String() != fmt.Sprintf("%s20", prefix) { + t.Fatalf("decode %x = %+v, want %s20", code, got, prefix) + } + } + } + } + } + } + } + } + if len(seen) != len(amd64PackedTestMaskSpecs) { + t.Fatalf("decoded %d distinct packed test-mask ops, want %d", len(seen), len(amd64PackedTestMaskSpecs)) + } +} + +func TestDecodedX86RawPackedTestMaskRejectsReservedBits(t *testing.T) { + base := encodeX86RawPackedTestMask(0, 0, 0, 2, 1, 2, 3, 0, false, false) + for _, test := range []struct { + name string + code []byte + }{ + {"fixed bit", []byte{base[0], base[1], base[2] &^ 4, base[3], base[4], base[5]}}, + {"K register extension", []byte{base[0], base[1] &^ 0x80, base[2], base[3], base[4], base[5]}}, + {"reserved length", []byte{base[0], base[1], base[2], base[3] | 0x20, base[4], base[5]}}, + {"zeroing", []byte{base[0], base[1], base[2], base[3] | 0x80, base[4], base[5]}}, + {"register broadcast", []byte{base[0], base[1], base[2], base[3] | 0x10, base[4], base[5]}}, + {"missing ModRM", base[:5]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawPackedTestMaskInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + byteBroadcast := encodeX86RawPackedTestMask(0, 0, 0, 2, 0, 1, 2, 0, true, true) + if _, _, recognized, err := decodedX86RawPackedTestMaskInstruction(byteBroadcast, 64); !recognized || err == nil { + t.Fatalf("accepted byte broadcast %x", byteBroadcast) + } +} + +func TestTranslateRawPackedTestMaskLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawPackedTestMaskFamily(SB), $0-0\n") + for polarity, groups := range x86RawPackedTestMaskOps { + for widthGroup, widths := range groups { + for w := range widths { + for vectorBits := 0; vectorBits < 3; vectorBits++ { + for _, code := range [][]byte{ + encodeX86RawPackedTestMask(polarity, widthGroup, w, vectorBits, 1, 2, 3, 0, false, false), + encodeX86RawPackedTestMask(polarity, widthGroup, w, vectorBits, 0, 2, 3, 4, true, widthGroup != 0), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawPackedTestMaskFamily": {Name: "rawPackedTestMaskFamily", Ret: Void}}, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("bare physical test-to-mask formats invented source/mask values", err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.goarch) + if err != nil { + t.Fatal(err) + } + file.Funcs[0] = decoded + if len(decoded.Instrs) != 50 { + t.Fatal("original 48 raw forms lost TEXT/RET or an instruction", len(decoded.Instrs)) + } + ir = x86InstructionFormatObjectUnit(t, file, map[string]FuncSig{ + "rawPackedTestMaskFamily": {Name: "rawPackedTestMaskFamily", Ret: Void}, + }, target.goarch, target.triple, 0, func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerPackedTestMask(ins.Op, ins) + }) + compileLLVMToObject(t, llc, target.triple, "raw-packed-test-mask.ll", "raw-packed-test-mask.o", ir) + }) + } +} diff --git a/amd64_raw_packed_widen_conversion.go b/amd64_raw_packed_widen_conversion.go new file mode 100644 index 000000000..4ae731b95 --- /dev/null +++ b/amd64_raw_packed_widen_conversion.go @@ -0,0 +1,27 @@ +package plan9asm + +// The eight packed float-to-(u)int64 conversions share Go 1.27's +// _yvcvtps2qq and _yvcvtpd2qq grammars. Their source width is determined +// by the float lane size and destination width, with common mask, zeroing, +// broadcast, and embedded rounding/SAE rules. +var x86RawPackedWidenConversionByOpcode = map[int]Op{ + 0x7b: "VCVTPS2QQ", + 0x79: "VCVTPS2UQQ", + 0x7a: "VCVTTPS2QQ", + 0x78: "VCVTTPS2UQQ", +} + +var x86RawPackedDoubleConversionByOpcode = map[int]Op{ + 0x7b: "VCVTPD2QQ", + 0x79: "VCVTPD2UQQ", + 0x7a: "VCVTTPD2QQ", + 0x78: "VCVTTPD2UQQ", +} + +func decodedX86PackedWidenConversionInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 || p.opcode < 0x78 || p.opcode > 0x7b { + return Instr{}, 0, false, nil + } + return decodedX86PackedNumericConversionInstruction(code, mode) +} diff --git a/amd64_raw_packed_widen_conversion_test.go b/amd64_raw_packed_widen_conversion_test.go new file mode 100644 index 000000000..c55f8767c --- /dev/null +++ b/amd64_raw_packed_widen_conversion_test.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +var x86RawPackedWidenConversionOps = []Op{ + "VCVTPS2QQ", "VCVTPS2UQQ", "VCVTTPS2QQ", "VCVTTPS2UQQ", + "VCVTPD2QQ", "VCVTPD2UQQ", "VCVTTPD2QQ", "VCVTTPD2UQQ", +} + +func x86RawPackedWidenConversionGoForms() string { + var source strings.Builder + source.WriteString("TEXT rawPackedWidenConversion(SB),4,$0-0\n") + for _, op := range x86RawPackedWidenConversionOps { + registers := []struct{ source, destination string }{ + {"X0", "X1"}, {"X0", "Y1"}, {"Y0", "Z1"}, + } + if strings.Contains(string(op), "PD2") { + registers = []struct{ source, destination string }{ + {"X0", "X1"}, {"Y0", "Y1"}, {"Z0", "Z1"}, + } + } + for _, registers := range registers { + fmt.Fprintf(&source, "\t%s %s, %s\n", op, registers.source, registers.destination) + fmt.Fprintf(&source, "\t%s 0(AX), %s\n", op, registers.destination) + fmt.Fprintf(&source, "\t%s %s, K1, %s\n", op, registers.source, registers.destination) + fmt.Fprintf(&source, "\t%s.Z 0(AX), K1, %s\n", op, registers.destination) + fmt.Fprintf(&source, "\t%s.BCST 0(AX), %s\n", op, registers.destination) + } + roundingSource := "Y0" + if strings.Contains(string(op), "PD2") { + roundingSource = "Z0" + } + if strings.HasPrefix(string(op), "VCVTT") { + fmt.Fprintf(&source, "\t%s.SAE %s, Z1\n", op, roundingSource) + } else { + fmt.Fprintf(&source, "\t%s.RN_SAE %s, Z1\n", op, roundingSource) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestDecodeX86RawPackedWidenConversionCompleteGoForms(t *testing.T) { + source := x86RawPackedWidenConversionGoForms() + code := assembleX87ControlBytes(t, "amd64", source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d=%+v, want %+v", index, got, want[index]) + } + } +} + +func TestDecodeX86RawPackedWidenConversionSourceLocalRIPData(t *testing.T) { + for _, double := range []bool{false, true} { + ops := x86RawPackedWidenConversionByOpcode + if double { + ops = x86RawPackedDoubleConversionByOpcode + } + for opcode, op := range ops { + for vectorLength := 0; vectorLength < 3; vectorLength++ { + for _, broadcast := range []bool{false, true} { + p2 := byte(0x08 | vectorLength<<5) + width := 8 << vectorLength + p1 := byte(0x7d) + if double { + p1 |= 0x80 + width *= 2 + } + if broadcast { + p2 |= 0x10 + width = 4 + if double { + width = 8 + } + } + code := []byte{0x62, 0xf1, p1, p2, byte(opcode), 0x05, 1, 0, 0, 0, 0xc3} + for value := 0; value < width; value++ { + code = append(code, byte(value)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "packed widening conversion RIP", map[string]bool{}) + if err != nil { + t.Fatalf("%s width %d broadcast=%t: %v", op, width, broadcast, err) + } + if len(decoded) != 2 || len(decoded[0].x86RIPLiteralData) != width || + decoded[0].Args[0].Kind != OpSym { + t.Fatalf("%s width %d broadcast=%t: %#v", op, width, broadcast, decoded) + } + } + } + } + } +} + +func TestTranslateX86RawPackedWidenConversionLLVM22Objects(t *testing.T) { + source := x86RawPackedWidenConversionGoForms() + code := assembleX87ControlBytes(t, "amd64", source) + file := x86RawUnprovedReturnFormProbeFile(t, "amd64", "rawPackedWidenConversion", code) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawProbeObject(t, file, "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-packed-widen.ll", "raw-packed-widen.o", ir) + }) + } +} diff --git a/amd64_raw_parallel_bits.go b/amd64_raw_parallel_bits.go new file mode 100644 index 000000000..1ec0d6d07 --- /dev/null +++ b/amd64_raw_parallel_bits.go @@ -0,0 +1,53 @@ +package plan9asm + +import "fmt" + +// PDEP/PEXT share Go's _yandnl register/memory-mask grammar. Intel XED's +// datafiles/hswbmi/hsw-bmi-vex-isa.xed.txt confirms W is ignored in not64 +// mode, while L must be zero. This is BMI2, not an arbitrary AVX fallback. +func decodedX86ParallelBitsInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86ParallelBitsInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86ParallelBitsInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || p.opcode != 0xf5 || (p.pp != 2 && p.pp != 3) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("parallel bit operation: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.vectorLength != 0 { + return fail("requires VEX.L0 encoding") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + bits := 32 + if p.w && mode == 64 { + bits = 64 + } + var op Op + for name, spec := range x86TernaryGPSpecs { + if spec.operation != x86TernaryANDN && spec.bits == bits && spec.prefix == p.pp { + op = name + break + } + } + mask, size, err := memory.scalar(code[p.modRM:], mode, p.b, p.x, p.segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + src, _ := decodedX86GeneralRegister(p.upper) + dst, _ := decodedX86GeneralRegister(int(code[p.modRM]>>3&7) + p.r*8) + return Instr{Op: op, Args: []Operand{mask, {Kind: OpReg, Reg: src}, {Kind: OpReg, Reg: dst}}, Raw: fmt.Sprintf("%s %s, %s, %s", op, mask.String(), src, dst)}, p.modRM + size, true, nil +} diff --git a/amd64_raw_parallel_bits_test.go b/amd64_raw_parallel_bits_test.go new file mode 100644 index 000000000..43d6cf6b7 --- /dev/null +++ b/amd64_raw_parallel_bits_test.go @@ -0,0 +1,209 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86RawParallelBitsGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + regs := []string{"AX", "SP", "DI"} + if arch == "amd64" { + regs = append(regs, "R8", "R15") + } + var source strings.Builder + source.WriteString("TEXT parallelbits(SB),4,$0-0\n") + // Go _yandnl: one register/memory mask, source GP, destination GP + // row, independently encoded by each of PDEPL/Q and PEXTL/Q. + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + for _, dst := range regs { + for _, mask := range []string{dst, "(BX)", "-7(BP)(CX*4)", "8192(SI)"} { + fmt.Fprintf(&source, "%s %s,%s,%s\n", op, mask, dst, dst) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if arch == "386" && strings.HasSuffix(string(want[i].Op), "Q") { + want[i].Op = Op(strings.TrimSuffix(string(want[i].Op), "Q") + "L") + } + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + file := x86RawStackObservingFormProbeFile(t, arch, "parallelbits", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir := x86UnboundVoidRawProbeObject(t, file, arch, triple) + compileLLVMToObject(t, llc, triple, "raw-parallel-bits.ll", "raw-parallel-bits.o", ir) + } + t.Logf("checked %d Go encodings", len(want)-1) + }) + } +} + +func TestX86RawParallelBitsEncodingAxes(t *testing.T) { + count := 0 + for _, deposit := range []bool{false, true} { + for _, wide := range []bool{false, true} { + pp, op := 2, "PEXT" + if deposit { + pp, op = 3, "PDEP" + } + for mask := 0; mask < 16; mask++ { + for src := 0; src < 16; src++ { + for dst := 0; dst < 16; dst++ { + code := encodeRawScalarMove(false, pp, 0xf5, mask, src, dst, 0, false) + code[1] = code[1]&0xe0 | 2 + if wide { + code[2] |= 0x80 + } + for _, mode := range []int{32, 64} { + got, n, matched, err := decodedX86ParallelBitsInstruction(code, mode) + if mode == 32 && (mask >= 8 || src >= 8 || dst >= 8) { + if !matched || err == nil { + t.Fatalf("386 accepted %x", code) + } + continue + } + width := "L" + if wide && mode == 64 { + width = "Q" + } + mr, _ := decodedX86GeneralRegister(mask) + sr, _ := decodedX86GeneralRegister(src) + dr, _ := decodedX86GeneralRegister(dst) + want := []Operand{{Kind: OpReg, Reg: mr}, {Kind: OpReg, Reg: sr}, {Kind: OpReg, Reg: dr}} + if err != nil || !matched || n != len(code) || got.Op != Op(op+width) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x mode=%d: %+v %v, want %s %+v", code, mode, got, err, op+width, want) + } + } + count++ + } + } + } + } + } + if count != 16384 { + t.Fatalf("covered %d encodings", count) + } +} + +func TestX86RawParallelBitsInvalidAndSegmentForms(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe2, 0x7a, 0xf5}, {0xc4, 0xe2, 0x7a, 0xf5, 0x04}, {0xc4, 0xe2, 0x7a, 0xf5, 0x45}, + {0xc4, 0xe2, 0x7e, 0xf5, 0xc0}, // Reserved L=1. + {0x62, 0xf2, 0x7e, 0x08, 0xf5, 0xc0}, // Not a Go EVEX family. + {0x67, 0xc4, 0xe2, 0x7a, 0xf5, 0x00}, + } { + if _, _, matched, err := decodedX86ParallelBitsInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x", code) + } + } + if _, _, matched, err := decodedX86ParallelBitsInstruction([]byte{0xc4, 0xe2, 0x7a, 0xf5, 0xc0}, 16); !matched || err == nil { + t.Fatal("accepted invalid mode") + } + for _, code := range [][]byte{nil, {0xc4}, {0xc4, 0xe2, 0x78, 0xf5, 0xc0}, {0xc4, 0xe1, 0x7a, 0xf5, 0xc0}} { + if _, _, matched, _ := decodedX86ParallelBitsInstruction(code, 64); matched { + t.Fatalf("matched unrelated encoding %x", code) + } + } + for _, seg := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + code := []byte{seg.prefix, 0xc4, 0x02, 0xb2, 0xf5, 0x64, 0x88, 0xf9} + got, n, matched, err := decodedX86ParallelBitsInstruction(code, 64) + want := []Operand{{Kind: OpMem, Mem: MemRef{Segment: seg.reg, Base: "R8", Index: "R9", Scale: 4, Off: -7}}, {Kind: OpReg, Reg: "R9"}, {Kind: OpReg, Reg: "R12"}} + if err != nil || !matched || n != len(code) || got.Op != "PEXTQ" || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v %v, want %+v", code, got, err, want) + } + } +} + +func TestX86ParallelBitsGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + for _, args := range []string{" AX,BX", " X0,AX,BX", " AX,X0,BX", " AX,BX,X0", " AX,(BX),CX", " AX,BX,(CX)", " $1,BX,CX", ".Z AX,BX,CX", ".BCST (AX),BX,CX"} { + source := "TEXT invalid(SB),4,$0-0\n" + op + args + "\nRET\n" + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"invalid": {Name: "invalid", Ret: Void}}}); err == nil { + t.Fatalf("translator accepted %s%s on %s", op, args, arch) + } + } + } + } +} + +func TestX86ParallelBitsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; separate five-target object tests are required") + } + source, decl, checks, sigs := x86ParallelBitsOriginalRuntimeFixture(t) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := x86ParallelBitsOriginalRuntimeDriver(decl.String(), checks.String()) + compileAndRunRuntimeTestForTarget(t, llc, clang, "parallel_bits", triple, ir, mainC, prefix) +} + +func TestX86ParallelBits386IgnoresW(t *testing.T) { + // Intel XED hsw-bmi-vex-isa.xed.txt has unconstrained W in not64 + // mode: both Go-accepted L and Q spellings access only a 32-bit mask. + for _, op := range []string{"PDEPQ", "PEXTQ"} { + source := "TEXT width386(SB),4,$0-0\n" + op + " (AX),BX,CX\nRET\n" + assembleX87ControlBytes(t, "386", source) + ir := x86UnboundVoidScalarFormatObject(t, source, "width386", "386", "i386-unknown-linux-gnu") + if strings.Contains(ir, "lshr i64") || !strings.Contains(ir, "lshr i32") || !strings.Contains(ir, "load volatile i32, ptr") { + t.Fatalf("386 %s failed to normalize width", op) + } + } +} diff --git a/amd64_raw_permute128_test.go b/amd64_raw_permute128_test.go new file mode 100644 index 000000000..b0057ceeb --- /dev/null +++ b/amd64_raw_permute128_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVPERM2X128(opcode, immediate byte, first, second, destination int) []byte { + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-first/8)<<5 | 3) + p1 := byte((^second&15)<<3 | 5) + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0xc4, p0, p1, opcode, modRM, immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedPsiphonVPERM2I128(t *testing.T) { + code := []byte{0xc4, 0xe3, 0x6d, 0x46, 0xc3, 0x20} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "Psiphon VPERM2I128 sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VPERM2I128 $32, Y3, Y2, Y0" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawVPERM2X128CompleteRegisterFamily(t *testing.T) { + count := 0 + for _, operation := range []struct { + opcode byte + name string + }{ + {opcode: 0x06, name: "VPERM2F128"}, + {opcode: 0x46, name: "VPERM2I128"}, + } { + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(first*16 + destination) + code := encodeX86RawVPERM2X128(operation.opcode, immediate, first, second, destination) + got, length, ok, err := decodedX86Permute128Instruction(code, 64) + want := fmt.Sprintf("%s $%d, Y%d, Y%d, Y%d", operation.name, immediate, first, second, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d VPERM2F128/VPERM2I128 register encodings, want 8192", count) + } +} + +func TestDecodedX86RawVPERM2X128MemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + { + name: "VPERM2F128 extended SIB", + code: []byte{0x65, 0xc4, 0x03, 0x15, 0x06, 0x64, 0x8b, 0x20, 0xff}, + want: "VPERM2F128 $255, 32(R11)(R9*4)(GS), Y13, Y12", + }, + { + name: "VPERM2I128 extended SIB", + code: []byte{0x64, 0xc4, 0x03, 0x15, 0x46, 0x64, 0x8b, 0x20, 0xfe}, + want: "VPERM2I128 $254, 32(R11)(R9*4)(FS), Y13, Y12", + }, + } { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86Permute128Instruction(test.code, 64) + if err != nil || !ok || length != len(test.code) || got.Raw != test.want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", test.code, got, length, ok, err, test.want) + } + }) + } + + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "386 unavailable vvvv register", mode: 32, encoding: encodeX86RawVPERM2X128(0x46, 1, 0, 8, 2), wantOK: true}, + {name: "wrong pp", mode: 64, encoding: []byte{0xc4, 0xe3, 0x6c, 0x46, 0xc3, 4}, wantOK: true}, + {name: "W", mode: 64, encoding: []byte{0xc4, 0xe3, 0xed, 0x46, 0xc3, 4}, wantOK: true}, + {name: "128-bit vector length", mode: 64, encoding: []byte{0xc4, 0xe3, 0x69, 0x46, 0xc3, 4}, wantOK: true}, + {name: "truncated immediate", mode: 64, encoding: []byte{0xc4, 0xe3, 0x6d, 0x46, 0xc3}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe3, 0x6d, 0x46, 0xc3, 4}, wantOK: true}, + {name: "legacy prefix", mode: 64, encoding: []byte{0x66, 0xc4, 0xe3, 0x6d, 0x46, 0xc3, 4}}, + {name: "wrong opcode", mode: 64, encoding: []byte{0xc4, 0xe3, 0x6d, 0x47, 0xc3, 4}}, + {name: "wrong map", mode: 64, encoding: []byte{0xc4, 0xe2, 0x6d, 0x46, 0xc3, 4}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86Permute128Instruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x returned ok=%v err=%v, want ok=%v", test.encoding, ok, err, test.wantOK) + } + if test.wantOK && err == nil { + t.Fatalf("invalid encoding %x was accepted", test.encoding) + } + if !test.wantOK && err != nil { + t.Fatalf("unrelated encoding %x returned err=%v", test.encoding, err) + } + }) + } +} diff --git a/amd64_raw_permute_inlane_test.go b/amd64_raw_permute_inlane_test.go new file mode 100644 index 000000000..c34cfc3e6 --- /dev/null +++ b/amd64_raw_permute_inlane_test.go @@ -0,0 +1,283 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawVEXInLaneFloatingPermute(double, immediate bool, vectorBits byte, first, second, destination int, imm byte) []byte { + mapNumber := byte(2) + opcode := byte(0x0c) + if immediate { + mapNumber = 3 + opcode = 0x04 + } + if double { + opcode++ + } + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^first>>3)&1)<<5 | mapNumber + p1 := byte(^second&15)<<3 | vectorBits<<2 | 1 + if immediate { + p1 = 0x78 | vectorBits<<2 | 1 + } + code := []byte{0xc4, p0, p1, opcode, byte(0xc0 | destination&7<<3 | first&7)} + if immediate { + code = append(code, imm) + } + return code +} + +func encodeX86RawEVEXInLaneFloatingPermute(double, immediate bool, vectorBits byte, mask int, zeroing, broadcast bool, first, second, destination int, imm byte) []byte { + mapNumber := byte(2) + opcode := byte(0x0c) + if immediate { + mapNumber = 3 + opcode = 0x04 + } + if double { + opcode++ + } + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | mapNumber + p1 := byte(^second&15)<<3 | 5 + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if immediate { + p1 = 0x7d + p2 |= 0x08 + } + if double { + p1 |= 0x80 + } + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + code := []byte{0x62, p0, p1, p2, opcode, byte(0xc0 | destination&7<<3 | first&7)} + if immediate { + code = append(code, imm) + } + return code +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVPERMILPS(t *testing.T) { + // vpermilps $180, %ymm2, %ymm3; + // vblendps $72, %ymm3, %ymm1, %ymm3 + code := []byte{0xc4, 0xe3, 0x7d, 0x04, 0xda, 0xb4, 0xc4, 0xe3, 0x75, 0x0c, 0xdb, 0x48} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VPERMILPS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VPERMILPS $180, Y2, Y3 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawInLaneFloatingPermuteCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, double := range []bool{false, true} { + op := "VPERMILPS" + if double { + op = "VPERMILPD" + } + for vectorBits := byte(0); vectorBits < 2; vectorBits++ { + vectorName := [...]string{"X", "Y"}[vectorBits] + for first := 0; first < 16; first++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXInLaneFloatingPermute(double, true, vectorBits, first, 0, destination, 0xa5) + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code, 64) + want := fmt.Sprintf("%s $165, %s%d, %s%d", op, vectorName, first, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXInLaneFloatingPermute(double, false, vectorBits, first, second, destination, 0) + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code, 64) + want := fmt.Sprintf("%s %s%d, %s%d, %s%d", op, vectorName, first, vectorName, second, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{{}, {mask: 3}, {mask: 7, zeroing: true, suffix: ".Z"}} { + for first := 0; first < 32; first++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXInLaneFloatingPermute(double, true, vectorBits, masking.mask, masking.zeroing, false, first, 0, destination, 0x5a) + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code, 64) + args := []string{"$90", fmt.Sprintf("%s%d", vectorName, first)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := op + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXInLaneFloatingPermute(double, false, vectorBits, masking.mask, masking.zeroing, false, first, second, destination, 0) + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code, 64) + args := []string{fmt.Sprintf("%s%d", vectorName, first), fmt.Sprintf("%s%d", vectorName, second)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := op + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 625664 { + t.Fatalf("covered %d in-lane floating-permute register encodings, want 625664", count) + } +} + +func TestDecodedX86RawInLaneFloatingPermuteMemoryAndInvalidForms(t *testing.T) { + vexMemory := []byte{0xc4, 0x03, 0x7d, 0x05, 0x7c, 0x8b, 0x02, 0xfe} + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(vexMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 2} + if err != nil || !ok || length != len(vexMemory) || got.Op != "VPERMILPD" || got.Args[0].Imm != 254 || got.Args[2].String() != "Y15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexImmediateMemory := []byte{0x62, 0xf3, 0xfd, 0xdb, 0x05, 0x50, 0x02, 0x07} + got, length, ok, err = decodedX86InLaneFloatingPermuteInstruction(evexImmediateMemory, 64) + wantMemory = MemRef{Base: "AX", Off: 16} + if err != nil || !ok || length != len(evexImmediateMemory) || got.Op != "VPERMILPD.BCST.Z" || got.Args[0].Imm != 7 || got.Args[2].String() != "K3" || got.Args[3].String() != "Z2" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexImmediateMemory, got, length, ok, err) + } + evexVariableMemory := []byte{0x62, 0xf2, 0x75, 0x38, 0x0c, 0x50, 0x02} + got, length, ok, err = decodedX86InLaneFloatingPermuteInstruction(evexVariableMemory, 64) + wantMemory = MemRef{Base: "AX", Off: 8} + if err != nil || !ok || length != len(evexVariableMemory) || got.Op != "VPERMILPS.BCST" || got.Args[1].String() != "Y1" || got.Args[2].String() != "Y2" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexVariableMemory, got, length, ok, err) + } + + validVEXImmediate := encodeX86RawVEXInLaneFloatingPermute(false, true, 0, 7, 0, 7, 1) + validEVEXVariable := encodeX86RawEVEXInLaneFloatingPermute(false, false, 2, 0, false, false, 7, 6, 5, 0) + validEVEXImmediate := encodeX86RawEVEXInLaneFloatingPermute(false, true, 2, 0, false, false, 7, 0, 5, 1) + for name, invalid := range map[string][]byte{ + "vex missing immediate": validVEXImmediate[:len(validVEXImmediate)-1], + "vex W": {validVEXImmediate[0], validVEXImmediate[1], validVEXImmediate[2] | 0x80, validVEXImmediate[3], validVEXImmediate[4], validVEXImmediate[5]}, + "vex prefix": {validVEXImmediate[0], validVEXImmediate[1], validVEXImmediate[2] &^ 0x01, validVEXImmediate[3], validVEXImmediate[4], validVEXImmediate[5]}, + "vex immediate vvvv": {validVEXImmediate[0], validVEXImmediate[1], validVEXImmediate[2] &^ 0x08, validVEXImmediate[3], validVEXImmediate[4], validVEXImmediate[5]}, + "evex fixed bit": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2] &^ 0x04, validEVEXVariable[3], validEVEXVariable[4], validEVEXVariable[5]}, + "evex wrong W": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2] | 0x80, validEVEXVariable[3], validEVEXVariable[4], validEVEXVariable[5]}, + "evex immediate vvvvv": {validEVEXImmediate[0], validEVEXImmediate[1], validEVEXImmediate[2] &^ 0x08, validEVEXImmediate[3] &^ 0x08, validEVEXImmediate[4], validEVEXImmediate[5], validEVEXImmediate[6]}, + "evex reserved length": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2], validEVEXVariable[3] | 0x60, validEVEXVariable[4], validEVEXVariable[5]}, + "zero without mask": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2], validEVEXVariable[3] | 0x80, validEVEXVariable[4], validEVEXVariable[5]}, + "register broadcast": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2], validEVEXVariable[3] | 0x10, validEVEXVariable[4], validEVEXVariable[5]}, + "address override": append([]byte{0x67}, validVEXImmediate...), + } { + if instruction, _, matched, decodeErr := decodedX86InLaneFloatingPermuteInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + masked386 := encodeX86RawEVEXInLaneFloatingPermute(false, false, 0, 1, false, false, 0, 1, 2, 0) + if instruction, _, matched, decodeErr := decodedX86InLaneFloatingPermuteInstruction(masked386, 32); !matched || decodeErr == nil { + t.Fatalf("386 masked encoding %x decoded as %+v, ok=%v, err=%v", masked386, instruction, matched, decodeErr) + } + highZ386 := encodeX86RawEVEXInLaneFloatingPermute(false, false, 2, 0, false, false, 8, 1, 2, 0) + if instruction, _, matched, decodeErr := decodedX86InLaneFloatingPermuteInstruction(highZ386, 32); !matched || decodeErr == nil { + t.Fatalf("386 high-Z encoding %x decoded as %+v, ok=%v, err=%v", highZ386, instruction, matched, decodeErr) + } +} + +func TestX86InLaneFloatingPermuteRawGo127ArchitectureOracle(t *testing.T) { + const accepted386 = `TEXT ok(SB),$0-0 + VPERMILPS $7, X0, X1 + VPERMILPD X0, X1, X2 + VPERMILPS $7, X20, X21 + VPERMILPD Z0, Z6, Z7 + RET +` + requireX86GoAssemblerResult(t, "386", accepted386, true) + for _, instruction := range []string{ + "VPERMILPS $7, Z8, Z0", + "VPERMILPD Z0, Z1, Z8", + "VPERMILPS X0, X1, K1, X2", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } +} + +func TestTranslateX86RawInLaneFloatingPermuteTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_in_lane_floating_permute(SB),NOSPLIT,$0-0\n") + for _, double := range []bool{false, true} { + var encodings [][]byte + if target.goarch == "amd64" { + encodings = append(encodings, + encodeX86RawVEXInLaneFloatingPermute(double, true, 1, 15, 0, 14, 0xa5), + encodeX86RawVEXInLaneFloatingPermute(double, false, 1, 15, 14, 13, 0), + encodeX86RawEVEXInLaneFloatingPermute(double, true, 2, 7, true, false, 31, 0, 30, 0x5a), + encodeX86RawEVEXInLaneFloatingPermute(double, false, 2, 3, false, false, 31, 30, 29, 0), + ) + } else { + encodings = append(encodings, + encodeX86RawVEXInLaneFloatingPermute(double, true, 1, 7, 0, 6, 0xa5), + encodeX86RawVEXInLaneFloatingPermute(double, false, 1, 7, 6, 5, 0), + encodeX86RawEVEXInLaneFloatingPermute(double, true, 0, 0, false, false, 20, 0, 21, 0x5a), + encodeX86RawEVEXInLaneFloatingPermute(double, false, 2, 0, false, false, 5, 6, 7, 0), + ) + } + for _, encoding := range encodings { + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + physical, contextual := 8, 0 + if target.goarch == "386" { + physical, contextual = 6, 2 + } + ll := x86InLanePermuteUnboundFormatObject(t, source.String(), target.goarch, target.triple, physical, contextual) + compileLLVMToObject(t, llc, target.triple, "raw-in-lane-floating-permute-"+target.name+".ll", "raw-in-lane-floating-permute-"+target.name+".o", ll) + }) + } +} diff --git a/amd64_raw_portio.go b/amd64_raw_portio.go new file mode 100644 index 000000000..2428c7984 --- /dev/null +++ b/amd64_raw_portio.go @@ -0,0 +1,54 @@ +package plan9asm + +import "fmt" + +// decodedX86ScalarPortIOInstruction covers Go's complete yin table: input +// and output, byte/word/long, with either an immediate port or implicit DX. +// x/arch's GoSyntax prints the architectural accumulator and DX operands, +// whereas Go's assembler source syntax omits both. Its old decoder also calls +// the byte encodings INL/OUTL. Decode these small fixed forms from the opcode. +func decodedX86ScalarPortIOInstruction(code []byte) (Instr, int, bool, error) { + if len(code) == 0 { + return Instr{}, 0, false, nil + } + index := 0 + word := false + if code[index] == 0x66 { + word = true + index++ + if len(code) == index { + return Instr{}, 0, false, nil + } + } + opcode := code[index] + if opcode < 0xe4 || (opcode > 0xe7 && opcode < 0xec) || opcode > 0xef { + return Instr{}, 0, false, nil + } + input := opcode == 0xe4 || opcode == 0xe5 || opcode == 0xec || opcode == 0xed + byteWidth := opcode&1 == 0 + name := "OUT" + if input { + name = "IN" + } + width := "L" + if byteWidth { + width = "B" + } else if word { + width = "W" + } + index++ + var args []Operand + if opcode >= 0xe4 && opcode <= 0xe7 { + if len(code) == index { + return Instr{}, 0, true, fmt.Errorf("missing immediate port byte") + } + args = []Operand{{Kind: OpImm, Imm: int64(code[index])}} + index++ + } + op := Op(name + width) + raw := string(op) + if len(args) != 0 { + raw = fmt.Sprintf("%s $%d", op, args[0].Imm) + } + return Instr{Op: op, Args: args, Raw: raw}, index, true, nil +} diff --git a/amd64_raw_portio_test.go b/amd64_raw_portio_test.go new file mode 100644 index 000000000..a35a5d359 --- /dev/null +++ b/amd64_raw_portio_test.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawScalarPortIOCompleteEncodingFamily(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + port int64 + }{ + {"in byte immediate", []byte{0xe4, 0x7f}, "INB", 0x7f}, + {"in word immediate", []byte{0x66, 0xe5, 0x7f}, "INW", 0x7f}, + {"in long immediate", []byte{0xe5, 0x7f}, "INL", 0x7f}, + {"in byte DX", []byte{0xec}, "INB", -1}, + {"in word DX", []byte{0x66, 0xed}, "INW", -1}, + {"in long DX", []byte{0xed}, "INL", -1}, + {"out byte immediate", []byte{0xe6, 0x7f}, "OUTB", 0x7f}, + {"out word immediate", []byte{0x66, 0xe7, 0x7f}, "OUTW", 0x7f}, + {"out long immediate", []byte{0xe7, 0x7f}, "OUTL", 0x7f}, + {"out byte DX", []byte{0xee}, "OUTB", -1}, + {"out word DX", []byte{0x66, 0xef}, "OUTW", -1}, + {"out long DX", []byte{0xef}, "OUTL", -1}, + } + for _, goarch := range []string{"386", "amd64"} { + for _, test := range tests { + t.Run(goarch+"/"+test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 { + t.Fatalf("decoded %#x as %#v, want one instruction", test.code, got.Instrs) + } + ins := got.Instrs[0] + wantArgs := 0 + if test.port >= 0 { + wantArgs = 1 + } + if ins.Op != test.op || len(ins.Args) != wantArgs { + t.Fatalf("decoded %#x as %#v, want %s with port %d", test.code, ins, test.op, test.port) + } + if test.port >= 0 && (ins.Args[0].Kind != OpImm || ins.Args[0].Imm != test.port) { + t.Fatalf("decoded %#x port as %#v, want $%d", test.code, ins.Args[0], test.port) + } + }) + } + } +} + +func TestX86RawScalarPortIOImmediateByteRange(t *testing.T) { + for _, opcode := range []byte{0xe4, 0xe5, 0xe6, 0xe7} { + for port := 0; port < 256; port++ { + instruction, length, ok, err := decodedX86ScalarPortIOInstruction([]byte{opcode, byte(port)}) + if !ok || err != nil || length != 2 || len(instruction.Args) != 1 || instruction.Args[0].Imm != int64(port) { + t.Fatalf("opcode %#x port %d: instruction=%#v length=%d recognized=%t error=%v", opcode, port, instruction, length, ok, err) + } + } + _, _, ok, err := decodedX86ScalarPortIOInstruction([]byte{opcode}) + if !ok || err == nil { + t.Fatalf("opcode %#x with missing port: recognized=%t error=%v", opcode, ok, err) + } + } +} + +func TestX86RawScalarPortIOCompilesEveryForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawportio(SB),NOSPLIT,$0-0\n") + for _, code := range [][]byte{ + {0xe4, 0x7f}, {0x66, 0xe5, 0x7f}, {0xe5, 0x7f}, + {0xec}, {0x66, 0xed}, {0xed}, + {0xe6, 0x7f}, {0x66, 0xe7, 0x7f}, {0xe7, 0x7f}, + {0xee}, {0x66, 0xef}, {0xef}, + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawportio": {Name: "rawportio", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-portio.ll", "raw-portio.o", ir) + }) + } +} diff --git a/amd64_raw_prefetch_hint_test.go b/amd64_raw_prefetch_hint_test.go new file mode 100644 index 000000000..275642faf --- /dev/null +++ b/amd64_raw_prefetch_hint_test.go @@ -0,0 +1,79 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawExtendedPrefetchHints(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawExtendedPrefetch(SB),$0-0\n") + codes := [][]byte{ + {0x0f, 0x18, 0x20}, // /4, base register + {0x0f, 0x18, 0x30}, // /6, base register + {0x0f, 0x18, 0x38}, // /7, base register + {0x65, 0x0f, 0x18, 0x74, 0x8b, 0xf9}, // /6, GS and SIB with disp8 + } + if target.goarch == "amd64" { + codes = append(codes, []byte{0x4d, 0x0f, 0x18, 0x7c, 0x8c, 0x10}) // /7, extended base/index + } + for _, code := range codes { + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawExtendedPrefetch": {Name: "rawExtendedPrefetch", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-extended-prefetch.ll", "raw-extended-prefetch.o", ir) + }) + } +} + +func TestX86RawExtendedPrefetchHintsRejectInvalidForms(t *testing.T) { + for _, code := range [][]byte{ + {0x0f, 0x18, 0x28}, // /5 is reserved. + {0x0f, 0x18, 0xf0}, // /6 requires memory. + {0x67, 0x0f, 0x18, 0x30}, // Address-size override is not modeled. + {0x0f, 0x18, 0x35, 0, 0, 0, 0}, // RIP-relative address has no proven source layout. + } { + var source strings.Builder + source.WriteString("TEXT badRawPrefetch(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"badRawPrefetch": {Name: "badRawPrefetch", Ret: Void}}, + }); err == nil { + t.Fatalf("accepted unsupported raw prefetch form %x", code) + } + } +} diff --git a/amd64_raw_prefix.go b/amd64_raw_prefix.go new file mode 100644 index 000000000..69f4c743c --- /dev/null +++ b/amd64_raw_prefix.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// Intel specifies that the r/m field is ignored for each unprefixed register +// fence encoding. The pinned decoder recognizes only the canonical r/m=0, +// and the cache-line recognizer otherwise mistakes SFENCE /7 for CLFLUSH. +// Do not strip or reinterpret any prefix here: those remain independently +// decoded and validated, including LOCK and the distinct 66 memory forms. +func decodedX86OrderingFenceInstruction(code []byte, mode int) (Instr, int, bool) { + if mode != 32 && mode != 64 || len(code) < 3 || code[0] != 0x0f || code[1] != 0xae || code[2]>>6 != 3 { + return Instr{}, 0, false + } + op := Op("") + switch code[2] >> 3 & 7 { + case 5: + op = "LFENCE" + case 6: + op = "MFENCE" + case 7: + op = "SFENCE" + default: + return Instr{}, 0, false + } + return Instr{Op: op}, 3, true +} + +// x/arch keeps invalidity in metadata that GoSyntax can erase or print as a +// standalone prefix with the real instruction in its operands. Reject before +// any width/alias rewrite. PrefixIgnored alone is not machine invalidity. +func decodedX86PrefixError(inst x86asm.Inst) error { + for _, prefix := range inst.Prefix { + if prefix&x86asm.PrefixInvalid != 0 { + return fmt.Errorf("raw %s has a machine-invalid prefix 0x%02x", inst.Op, byte(prefix)) + } + } + for _, prefix := range inst.Prefix { + if prefix&0xff == x86asm.PrefixLOCK && prefix&x86asm.PrefixIgnored == 0 { + // This is valid native syntax, not invalid source. The existing + // standalone LOCK lowerer cannot bind a printed whole RMW or its + // atomic effect. Do not silently discard the following opcode. + return fmt.Errorf("%w: raw LOCK %s requires its complete atomic prefix/operand effect", ErrProbeNeedsContext, inst.Op) + } + if prefix&(x86asm.PrefixIgnored|x86asm.PrefixImplicit) != 0 { + continue + } + switch prefix { + case x86asm.PrefixXACQUIRE, x86asm.PrefixXRELEASE, x86asm.PrefixBND: + // GoSyntax uses only the low prefix byte for REP/REPNE. That + // textual repeater is not the original transactional or bounds + // effect and can leak onto a later string operation. + return fmt.Errorf("%w: raw %s %s requires its complete native prefix effect", ErrProbeNeedsContext, prefix, inst.Op) + } + } + return nil +} diff --git a/amd64_raw_prefix_class_test.go b/amd64_raw_prefix_class_test.go new file mode 100644 index 000000000..a0824a60f --- /dev/null +++ b/amd64_raw_prefix_class_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86OrderingHintsPhysicalFenceRMIsIgnored(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, row := range []struct { + op Op + base byte + }{ + {"LFENCE", 0xe8}, + {"MFENCE", 0xf0}, + {"SFENCE", 0xf8}, + } { + for rm := byte(0); rm < 8; rm++ { + t.Run(fmt.Sprintf("%d/%s/rm=%d", mode, row.op, rm), func(t *testing.T) { + code := []byte{0x0f, 0xae, row.base | rm} + x86PrefixOriginalBytes(t, mode, code) + got, err := decodeX86RawDirectiveGroup(code, mode, 0, "physical fence", nil) + if err != nil || len(got) != 1 || got[0].Op != row.op || len(got[0].Args) != 0 { + t.Fatalf("physical ignored-r/m fence changed: %x: %+v/%v", code, got, err) + } + }) + } + } + } +} + +// The complete Intel LOCK list is independent of the translator. The original +// named Go object binds width, destination and opcode; no RMW is executed here. +// Valid raw LOCK remains Context until the complete atomic effect is bound. +func TestX86RawValidLockCompleteRMWClass(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + mode := 32 + widths := "BWL" + if arch == "amd64" { + mode = 64 + widths += "Q" + } + var forms []string + for _, stem := range []string{"ADD", "ADC", "AND", "OR", "SBB", "SUB", "XOR", "CMPXCHG", "XADD", "XCHG"} { + for _, width := range widths { + reg := "CX" + if width == 'B' { + reg = "CL" + } + forms = append(forms, fmt.Sprintf("%s%c %s,(AX)", stem, width, reg)) + } + } + for _, stem := range []string{"INC", "DEC", "NEG", "NOT"} { + for _, width := range widths { + forms = append(forms, fmt.Sprintf("%s%c (AX)", stem, width)) + } + } + for _, stem := range []string{"BTC", "BTR", "BTS"} { + for _, width := range strings.TrimPrefix(widths, "B") { + forms = append(forms, fmt.Sprintf("%s%c CX,(AX)", stem, width)) + } + } + forms = append(forms, "CMPXCHG8B (AX)") + if arch == "amd64" { + forms = append(forms, "CMPXCHG16B (AX)") + } + want := 49 + if arch == "amd64" { + want = 67 + } + if len(forms) != want { + t.Fatal("complete native-width LOCK inventory changed", arch, len(forms)) + } + for _, form := range forms { + t.Run(arch+"/"+form, func(t *testing.T) { + source := "TEXT locked(SB),4,$0-0\nLOCK\n" + form + "\nRET\n" + actual := x86OrderingHintGoBytes(t, arch, source) + if len(actual) < 3 || actual[0] != 0xf0 || actual[len(actual)-1] != 0xc3 { + t.Fatalf("original named LOCK object lacks complete prefix/body/RET: %x", actual) + } + code := actual[:len(actual)-1] + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("actual complete LOCK instruction %+v: %v", physical, err) + } + for _, prefix := range physical.Prefix { + if prefix&x86asm.PrefixInvalid != 0 { + t.Fatal("accepted memory RMW has invalid machine LOCK", physical) + } + } + got, err := decodeX86RawDirectiveGroup(code, mode, 0, source, nil) + if !errors.Is(err, ErrProbeNeedsContext) || len(got) != 0 { + t.Fatalf("complete valid LOCK silently lost atomic/body effects: %+v/%v", got, err) + } + // The simple named (AX) form ends in its ModRM byte. Change only + // mod to register addressing, retaining the original opcode/width. + // BYTE acceptance binds invalid machine bytes, not ISA validity. + bad := append([]byte(nil), code...) + bad[len(bad)-1] |= 0xc0 + x86PrefixOriginalBytes(t, mode, bad) + if got, err := decodeX86RawDirectiveGroup(bad, mode, 0, "invalid register-only LOCK "+form, nil); err == nil { + t.Fatalf("complete LOCK class admitted register-only RMW: %x/%+v", bad, got) + } + }) + } + } +} + +func TestX86RawVisibleEffectPrefixesNeedWholeContract(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, code := range [][]byte{ + {0xf3, 0x89, 0x08}, // XRELEASE MOVL CX,(AX), no explicit LOCK. + {0xf3, 0x87, 0x08}, // XRELEASE on implicitly locked XCHG. + {0xf2, 0x87, 0x08}, // XACQUIRE on implicitly locked XCHG. + {0xf2, 0xc3}, // BND RET, not an ordinary near return. + } { + t.Run(fmt.Sprintf("%d/%x", mode, code), func(t *testing.T) { + x86PrefixOriginalBytes(t, mode, code) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatal("complete original prefixed object", physical, err) + } + got, err := decodeX86RawDirectiveGroup(code, mode, 0, "whole effect prefix", nil) + t.Logf("bytes=%x actualPrefix=%#v GoSyntax=%q parsed=%+v error=%v", code, + physical.Prefix, x86asm.GoSyntax(physical, 0, nil), got, err) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("visible native effect became an ordinary opcode/string repeat without a contract", err) + } + }) + } + } +} + +// These BYTE objects observe conversion, not a new string/caller contract. +// Visible prefixes must either retain the complete body or fail closed. +func TestX86RawVisibleStringPrefixesRetainBody(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, row := range []struct { + name string + code []byte + body Op + }{ + {"REP MOVSB", []byte{0xf3, 0xa4}, "MOVSB"}, + {"REPE CMPSB", []byte{0xf3, 0xa6}, "CMPSB"}, + {"REPNE CMPSB", []byte{0xf2, 0xa6}, "CMPSB"}, + {"duplicate REP MOVSB", []byte{0xf3, 0xf3, 0xa4}, "MOVSB"}, + {"XRELEASE MOV", []byte{0xf3, 0x89, 0x08}, "MOVL"}, + {"XACQUIRE LOCK ADD", []byte{0xf2, 0xf0, 0x01, 0x08}, "ADDL"}, + {"XRELEASE LOCK ADD", []byte{0xf3, 0xf0, 0x01, 0x08}, "ADDL"}, + } { + t.Run(fmt.Sprintf("%d/%s", mode, row.name), func(t *testing.T) { + x86PrefixOriginalBytes(t, mode, row.code) + physical, err := x86asm.Decode(row.code, mode) + if err != nil || physical.Len != len(row.code) { + t.Fatal("complete physical visible prefix", physical, err) + } + got, err := decodeX86RawDirectiveGroup(row.code, mode, 0, row.name, nil) + t.Logf("actual bytes=%x syntax=%q whole=%+v error=%v", row.code, + x86asm.GoSyntax(physical, 0, nil), got, err) + if err != nil { + return + } + body := false + for _, instruction := range got { + body = body || instruction.Op == row.body + } + if !body { + t.Fatal("visible prefix conversion discarded its actual instruction body", got) + } + }) + } + } +} diff --git a/amd64_raw_qq_to_float.go b/amd64_raw_qq_to_float.go new file mode 100644 index 000000000..38b3869f2 --- /dev/null +++ b/amd64_raw_qq_to_float.go @@ -0,0 +1,32 @@ +package plan9asm + +// x86RawQQToFloatForm maps opcode and mandatory prefix to the Go 1.27 +// QWORD-to-PS/PD optabs. PS narrows the Y/Z destination by one vector width. +func x86RawQQToFloatForm(p x86RawVectorEncoding) (base Op, narrow bool, ok bool) { + switch { + case p.opcode == 0x5b && p.pp == 0: + return "VCVTQQ2PS", true, true + case p.opcode == 0x7a && p.pp == 3: + return "VCVTUQQ2PS", true, true + case p.opcode == 0xe6 && p.pp == 2: + return "VCVTQQ2PD", false, true + case p.opcode == 0x7a && p.pp == 2: + return "VCVTUQQ2PD", false, true + default: + return "", false, false + } +} + +// decodedX86RawQQToFloatInstruction handles all four EVEX QWORD-to-PS/PD +// conversion families, including every X/Y/Z, K-mask, broadcast, and +// maximum-width embedded-rounding form. +func decodedX86RawQQToFloatInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || !p.w || p.mapNumber != 1 { + return Instr{}, 0, false, nil + } + if _, _, known := x86RawQQToFloatForm(p); !known { + return Instr{}, 0, false, nil + } + return decodedX86PackedNumericConversionInstruction(code, mode) +} diff --git a/amd64_raw_qq_to_float_test.go b/amd64_raw_qq_to_float_test.go new file mode 100644 index 000000000..4f8c77066 --- /dev/null +++ b/amd64_raw_qq_to_float_test.go @@ -0,0 +1,284 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86RawQQToPSEncoding(unsigned bool, vectorBits, mask int, zeroing, broadcast, memory bool) []byte { + p1, opcode := byte(0xfc), byte(0x5b) + if unsigned { + p1, opcode = 0xff, 0x7a + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc1) // source register 1, destination register 0. + if memory { + modRM = 0x40 // disp8(AX), destination register 0. + } + code := []byte{0x62, 0xf1, p1, p2, opcode, modRM} + if memory { + code = append(code, 1) + } + return code +} + +func x86RawQQToPDEncoding(unsigned bool, vectorBits, mask int, zeroing, broadcast, memory bool) []byte { + code := x86RawQQToPSEncoding(unsigned, vectorBits, mask, zeroing, broadcast, memory) + code[2] = 0xfe // EVEX.W1, F3 mandatory prefix. + if !unsigned { + code[4] = 0xe6 + } + return code +} + +func TestDecodeX86RawQQToPSRampAVX512Regression(t *testing.T) { + // Go's assembler decodes these bytes as VCVTQQ2PS Z5, Y16. + code := []byte{0x62, 0xe1, 0xfc, 0x48, 0x5b, 0xc5, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "ramp AVX512 QQ to PS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VCVTQQ2PS" || + decoded[0].Args[0].Reg != "Z5" || decoded[0].Args[1].Reg != "Y16" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawQQToPDRampAVX512Regression(t *testing.T) { + // Go's assembler decodes these bytes as VCVTQQ2PD Z4, Z12. + code := []byte{0x62, 0x71, 0xfe, 0x48, 0xe6, 0xe4, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "ramp AVX512 QQ to PD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VCVTQQ2PD" || + decoded[0].Args[0].Reg != "Z4" || decoded[0].Args[1].Reg != "Z12" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawQQToPSCompleteGoForms(t *testing.T) { + for _, unsigned := range []bool{false, true} { + base := "VCVTQQ2PS" + if unsigned { + base = "VCVTUQQ2PS" + } + for width := 0; width < 3; width++ { + baseOp := base + [...]string{"X", "Y", ""}[width] + source := [...]string{"X1", "Y1", "Z1"}[width] + destination := [...]string{"X0", "X0", "Y0"}[width] + for _, form := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + op string + }{ + {name: "register", op: baseOp}, + {name: "masked register", mask: 3, op: baseOp}, + {name: "zero register", mask: 7, zeroing: true, op: baseOp + ".Z"}, + {name: "memory", memory: true, op: baseOp}, + {name: "broadcast memory", memory: true, broadcast: true, op: baseOp + ".BCST"}, + {name: "broadcast zero memory", mask: 7, zeroing: true, memory: true, broadcast: true, op: baseOp + ".BCST.Z"}, + } { + name := fmt.Sprintf("%s/width%d/%s", base, width, form.name) + t.Run(name, func(t *testing.T) { + code := x86RawQQToPSEncoding(unsigned, width, form.mask, form.zeroing, form.broadcast, form.memory) + got, length, ok, err := decodedX86RawQQToFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if got.Op != Op(form.op) || got.Args[len(got.Args)-1].String() != destination { + t.Fatalf("decode %x = %+v, want %s ... %s", code, got, form.op, destination) + } + if !form.memory && got.Args[0].String() != source { + t.Fatalf("decode %x source = %s, want %s", code, got.Args[0], source) + } + if len(got.Args) != 2+map[bool]int{true: 1}[form.mask != 0] { + t.Fatalf("decode %x operands = %+v", code, got.Args) + } + }) + } + } + for rounding := 0; rounding < 4; rounding++ { + code := x86RawQQToPSEncoding(unsigned, rounding, 4, true, true, false) + got, length, ok, err := decodedX86RawQQToFloatInstruction(code, 64) + want := base + [...]string{".RN_SAE.Z", ".RD_SAE.Z", ".RU_SAE.Z", ".RZ_SAE.Z"}[rounding] + if err != nil || !ok || length != len(code) || string(got.Op) != want || + got.Args[0].String() != "Z1" || got.Args[2].String() != "Y0" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %s Z1, K4, Y0", code, got, length, ok, err, want) + } + } + } +} + +func TestDecodeX86RawQQToPDCompleteGoForms(t *testing.T) { + for _, unsigned := range []bool{false, true} { + base := "VCVTQQ2PD" + if unsigned { + base = "VCVTUQQ2PD" + } + for width := 0; width < 3; width++ { + vector := [...]string{"X", "Y", "Z"}[width] + for _, form := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + suffix string + }{ + {name: "register"}, + {name: "masked zero register", mask: 5, zeroing: true, suffix: ".Z"}, + {name: "memory", memory: true}, + {name: "broadcast memory", memory: true, broadcast: true, suffix: ".BCST"}, + {name: "broadcast zero memory", memory: true, broadcast: true, mask: 7, zeroing: true, suffix: ".BCST.Z"}, + } { + name := fmt.Sprintf("%s/width%d/%s", base, width, form.name) + t.Run(name, func(t *testing.T) { + code := x86RawQQToPDEncoding(unsigned, width, form.mask, form.zeroing, form.broadcast, form.memory) + got, length, ok, err := decodedX86RawQQToFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != Op(base+form.suffix) || + got.Args[len(got.Args)-1].String() != vector+"0" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if !form.memory && got.Args[0].String() != vector+"1" { + t.Fatalf("decode %x source = %s, want %s1", code, got.Args[0], vector) + } + }) + } + } + for rounding := 0; rounding < 4; rounding++ { + code := x86RawQQToPDEncoding(unsigned, rounding, 4, true, true, false) + got, length, ok, err := decodedX86RawQQToFloatInstruction(code, 64) + want := base + [...]string{".RN_SAE.Z", ".RD_SAE.Z", ".RU_SAE.Z", ".RZ_SAE.Z"}[rounding] + if err != nil || !ok || length != len(code) || string(got.Op) != want || + got.Args[0].String() != "Z1" || got.Args[2].String() != "Z0" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %s Z1, K4, Z0", code, got, length, ok, err, want) + } + } + } +} + +func TestDecodeX86RawQQToFloatRIPLiteral(t *testing.T) { + for _, kind := range []struct { + name string + encode func(bool, int, int, bool, bool, bool) []byte + }{ + {name: "PS", encode: x86RawQQToPSEncoding}, + {name: "PD", encode: x86RawQQToPDEncoding}, + } { + for _, unsigned := range []bool{false, true} { + for width := 0; width < 3; width++ { + for _, broadcast := range []bool{false, true} { + code := kind.encode(unsigned, width, 1, false, broadcast, false) + code[5] = 0x05 // RIP-relative source, destination 0. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << width + if broadcast { + literalWidth = 8 + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/unsigned%t/width%d/broadcast%t", kind.name, unsigned, width, broadcast) + t.Run(name, func(t *testing.T) { + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !decoded[0].x86RIPLiteral || + len(decoded[0].x86RIPLiteralData) != literalWidth || + broadcast != strings.Contains(string(decoded[0].Op), ".BCST") { + t.Fatalf("decoded %x as %+v", code, decoded) + } + }) + } + } + } + } +} + +// Preserve this original complete byte-only fixture for both its direct-format +// objects and its separate public missing-input contract. +func x86RawQQToFloatOriginalSource(arch string) string { + var source strings.Builder + source.WriteString("TEXT rawQQToFloat(SB),$0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for _, encode := range []func(bool, int, int, bool, bool, bool) []byte{ + x86RawQQToPSEncoding, + x86RawQQToPDEncoding, + } { + for _, unsigned := range []bool{false, true} { + for width := 0; width < 3; width++ { + appendBytes(encode(unsigned, width, 0, false, false, false)) + appendBytes(encode(unsigned, width, 0, false, false, true)) + appendBytes(encode(unsigned, width, 0, false, true, true)) + if arch == "amd64" { + appendBytes(encode(unsigned, width, 7, true, false, false)) + appendBytes(encode(unsigned, width, 7, true, true, true)) + } + } + if arch == "amd64" { + for rounding := 0; rounding < 4; rounding++ { + appendBytes(encode(unsigned, rounding, 4, true, true, false)) + } + } + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86RawQQToFloatLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86RawQQToFloatOriginalSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + // These unchanged byte-only formats have no input producers. + // Keep their actual Go objects and all three public Context + // controls, then compile every row through the format-only owner. + // Initialized public objects have their independent counterpart. + ir := x86UnboundVoidRawScalarFormatObject(t, source, target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-qq-to-float.ll", "raw-qq-to-float.o", ir) + }) + } +} + +func TestDecodeX86RawQQToPSRejectsInvalidForms(t *testing.T) { + zeroWithoutMask := x86RawQQToPSEncoding(false, 0, 0, true, false, false) + reservedLength := x86RawQQToPSEncoding(false, 3, 0, false, false, false) + reservedVVVV := x86RawQQToPSEncoding(false, 0, 0, false, false, false) + reservedVVVV[2] &^= 0x08 + for _, code := range [][]byte{zeroWithoutMask, reservedLength, reservedVVVV} { + if _, _, ok, err := decodedX86RawQQToFloatInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid %x returned ok=%v, err=%v", code, ok, err) + } + } +} diff --git a/amd64_raw_qword_permute.go b/amd64_raw_qword_permute.go new file mode 100644 index 000000000..fa0bf1291 --- /dev/null +++ b/amd64_raw_qword_permute.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's _yvpermq table gives VPERMQ and VPERMPD the same operand grammar. +// The 0F3A encodings use imm8 control; the EVEX-only 0F38 encodings use a +// vector control operand. The table below keeps the opcode split explicit. +var x86RawQwordPermutes = map[[2]int]Op{ + {3, 0x00}: "VPERMQ", + {3, 0x01}: "VPERMPD", + {2, 0x36}: "VPERMQ", + {2, 0x16}: "VPERMPD", +} + +func decodedX86RawQwordPermuteInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.pp != 1 || !p.w { + return Instr{}, 0, false, nil + } + op, known := x86RawQwordPermutes[[2]int{p.mapNumber, p.opcode}] + if !known { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("qword permute: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.vectorLength < 1 || p.vectorLength > 2 || !p.evex && p.vectorLength != 1 { + return fail("only Y and Z vector lengths are in Go's qword permute table") + } + if p.evex && !p.fixed { + return fail("invalid EVEX fixed bit") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + immediate := p.mapNumber == 3 + if !immediate && !p.evex { + return fail("variable-control form requires EVEX") + } + if immediate && p.upper != 0 { + return fail("immediate form reserves vvvv and V'") + } + registerSource := code[p.modRM]>>6 == 3 + if p.broadcast && (registerSource || !p.evex) { + return fail("broadcast requires EVEX memory source") + } + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (!p.evex && destinationNumber >= 8 || + width == "Z" && (destinationNumber >= 8 || !immediate && p.upper >= 8)) { + return fail("destination or control register unavailable in 32-bit mode") + } + + scale := 16 << p.vectorLength + if p.broadcast { + scale = 8 + } + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width, scale) + } else { + source, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + } + if err != nil { + return Instr{}, 0, true, err + } + if mode == 32 && width == "Z" && source.Kind == OpReg { + index, _ := amd64VectorRegisterIndex(source.Reg, 64) + if index >= 8 { + return fail("Z source register unavailable in 32-bit mode") + } + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))} + args := make([]Operand, 0, 4) + if immediate { + if len(code) <= p.modRM+consumed { + return fail("missing immediate byte") + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[p.modRM+consumed])}) + } else { + args = append(args, source) + source = Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + } + args = append(args, source) + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + length := p.modRM + consumed + if immediate { + length++ + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, length, true, nil +} diff --git a/amd64_raw_qword_permute_test.go b/amd64_raw_qword_permute_test.go new file mode 100644 index 000000000..a442b90f2 --- /dev/null +++ b/amd64_raw_qword_permute_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawQwordPermuteGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawqwordpermute(SB),4,$0-0\n") + for _, op := range []string{"VPERMQ", "VPERMPD"} { + fmt.Fprintf(&source, "%s $216, Y2, Y2\n", op) + fmt.Fprintf(&source, "%s $1, 64(BX), Y3\n", op) + fmt.Fprintf(&source, "%s $2, Y20, Y21\n", op) + if arch == "amd64" { + fmt.Fprintf(&source, "%s $3, Z20, Z21\n", op) + } else { + fmt.Fprintf(&source, "%s $3, Z0, Z1\n", op) + } + fmt.Fprintf(&source, "%s.BCST $4, 8(BX), Y21\n", op) + fmt.Fprintf(&source, "%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&source, "%s (BX), Y1, Y2\n", op) + fmt.Fprintf(&source, "%s Y20, Y21, Y22\n", op) + if arch == "amd64" { + fmt.Fprintf(&source, "%s.BCST 16(BX), Z21, Z22\n", op) + } else { + fmt.Fprintf(&source, "%s.BCST 16(BX), Z1, Z2\n", op) + } + if arch == "amd64" { + fmt.Fprintf(&source, "%s.Z $5, -32(R8), K2, Z21\n", op) + fmt.Fprintf(&source, "%s.Z Z20, Z21, K3, Z22\n", op) + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d=%+v, want %+v", index, got, want[index]) + } + } + }) + } +} + +func TestX86RawQwordPermuteRejectsMalformedForms(t *testing.T) { + const source = "TEXT rawqwordpermute(SB),4,$0-0\nVPERMQ $1, Y2, Y3\nRET\n" + assembled := assembleX87ControlBytes(t, "amd64", source) + if len(assembled) < 7 || assembled[0] != 0xc4 { + t.Fatalf("Go did not emit a VEX permute: %x", assembled) + } + valid := assembled[:len(assembled)-1] + for _, test := range []struct { + name string + mutate func([]byte) []byte + }{ + {"missing immediate", func(code []byte) []byte { return code[:len(code)-1] }}, + {"reserved vvvv", func(code []byte) []byte { code[2] &^= 0x08; return code }}, + {"wrong vector length", func(code []byte) []byte { code[2] &^= 0x04; return code }}, + {"address override", func(code []byte) []byte { return append([]byte{0x67}, code...) }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.mutate(append([]byte(nil), valid...)) + if _, _, recognized, err := decodedX86RawQwordPermuteInstruction(code, 64); !recognized || err == nil { + t.Fatalf("invalid %x: recognized=%v err=%v", code, recognized, err) + } + }) + } + const evexSource = "TEXT rawqwordpermute(SB),4,$0-0\nVPERMPD.Z Z1, Z2, K1, Z3\nRET\n" + evex := assembleX87ControlBytes(t, "amd64", evexSource) + if len(evex) < 7 || evex[0] != 0x62 { + t.Fatalf("Go did not emit an EVEX permute: %x", evex) + } + for _, test := range []struct { + name string + mutate func([]byte) []byte + }{ + {"missing mask", func(code []byte) []byte { code[3] &^= 7; return code }}, + {"invalid vector length", func(code []byte) []byte { code[3] &^= 0x60; return code }}, + {"missing fixed bit", func(code []byte) []byte { code[2] &^= 4; return code }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.mutate(append([]byte(nil), evex[:len(evex)-1]...)) + if _, _, recognized, err := decodedX86RawQwordPermuteInstruction(code, 64); !recognized || err == nil { + t.Fatalf("invalid %x: recognized=%v err=%v", code, recognized, err) + } + }) + } +} + +func TestTranslateX86RawQwordPermuteLLVM22AllTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, form := range []string{ + "VPERMQ $216, Y2, Y3", + "VPERMPD Z1, Z2, Z3", + } { + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(form+"/"+target.triple, func(t *testing.T) { + source := "TEXT rawqwordpermute(SB),4,$0-0\n" + form + "\nRET\n" + code := assembleX87ControlBytes(t, target.goarch, source) + var raw strings.Builder + raw.WriteString("TEXT rawqwordpermute(SB),4,$0-0\n") + for _, value := range code { + fmt.Fprintf(&raw, "BYTE $%#02x\n", value) + } + if actual := assembleX87ControlBytes(t, target.goarch, raw.String()); !bytes.Equal(actual, code) { + t.Fatal("complete original Go BYTE source changed its instruction or RET bytes", actual, code) + } + // Preserve complete BYTE/RET Go objects and all public Context + // controls before independently normalizing the private emitter + // unit. Neither raw identity nor an unbound RET supplies NUM. + ir := x86ElementPermuteUnboundFormatObject(t, raw.String(), target.goarch, target.triple, 1, 0) + x86ElementPermuteCompileFormatObjects(t, llc, target.triple, ir) + }) + } + } +} diff --git a/amd64_raw_rdpid_test.go b/amd64_raw_rdpid_test.go new file mode 100644 index 000000000..dafd8d26f --- /dev/null +++ b/amd64_raw_rdpid_test.go @@ -0,0 +1,61 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodeX86RawRDPIDCompleteRegisterFamily(t *testing.T) { + registers := [...]string{ + "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15", + } + for _, mode := range []int{32, 64} { + count := 8 + if mode == 64 { + count = len(registers) + } + for register := 0; register < count; register++ { + code := []byte{0xf3} + if register >= 8 { + code = append(code, 0x41) + } + code = append(code, 0x0f, 0xc7, byte(0xf8|register&7)) + decoded, err := decodeX86RawDirectiveGroup(code, mode, 0, "RDPID raw bytes", nil) + if err != nil { + t.Fatalf("mode %d, %x: %v", mode, code, err) + } + if len(decoded) != 1 || decoded[0].Op != "RDPID" || len(decoded[0].Args) != 1 || + decoded[0].Args[0].Reg != Reg(registers[register]) { + t.Fatalf("mode %d, %x decoded as %+v", mode, code, decoded) + } + } + } +} + +func TestTranslateX86RawRDPIDReportedByteArena(t *testing.T) { + const source = `TEXT rawRDPID(SB),$0-0 + BYTE $0xF3; BYTE $0x0F; BYTE $0xC7; BYTE $0xF9 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawRDPID": {Name: "rawRDPID", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "@llvm.x86.rdpid()") { + t.Fatalf("raw RDPID did not call the RDPID intrinsic:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-rdpid.ll", "raw-rdpid.o", ir) +} diff --git a/amd64_raw_return_fp_width_contract_test.go b/amd64_raw_return_fp_width_contract_test.go new file mode 100644 index 000000000..1ade2f726 --- /dev/null +++ b/amd64_raw_return_fp_width_contract_test.go @@ -0,0 +1,228 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86RawReturnScalarFrame(word int, typ LLVMType) FuncSig { + return FuncSig{Name: "Raw", Args: []LLVMType{typ}, Ret: typ, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: int64(word), Type: typ, Index: 0, Field: -1}}, + }} +} + +// A byte/word write to a wider FP slot must preserve the untouched bytes. +// The bounded raw-return contract does not establish that partial-width +// lowering, even when the access is inside the declared scalar's bounds. +// These original returning sources are never executed or rewritten as passes. +func TestX86RawReturnPartialFPAccessNeedsContext(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + partialMoves := []string{"MOVB", "MOVW"} + if word == 8 { + partialMoves = append(partialMoves, "MOVL") + } + for _, partial := range partialMoves { + for _, access := range []string{ + partial + " $17,x+0(FP)\n" + move + " x+0(FP),AX", + partial + " x+0(FP),AX", + fmt.Sprintf("%s $17,ret+%d(FP)", partial, word), + fmt.Sprintf("%s ret+%d(FP),AX", partial, word), + } { + for _, encoding := range []string{"BYTE $0xc3", "BYTE $0xc2\nWORD $0"} { + t.Run(target.triple+"/"+access+"/"+encoding, func(t *testing.T) { + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s\n%s\nRET\n", 2*word, access, encoding) + requireX86GoAssemblerResult(t, target.arch, source, true) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": x86RawReturnScalarFrame(word, typ)}} + x86RawReturnPointerAPIs(t, source, opt, true) + }) + } + } + } + } +} + +func TestX86RawReturnActualGoDeclarationRejectsPartialFPStore(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, declaration := "MOVQ", 8, "func Raw(x uint64) uint64" + if target.arch == "386" { + move, word, declaration = "MOVL", 4, "func Raw(x uint32) uint32" + } + source := fmt.Sprintf("TEXT ·Raw(SB),4,$0-%d\nMOVB $17,x+0(FP)\n%s x+0(FP),AX\n%s AX,ret+%d(FP)\nBYTE $0xc3\nRET\n", 2*word, move, move, word) + requireX86GoAssemblerResult(t, target.arch, source, true) + pkg := mustGoPackage(t, "test/rawret", "package rawret\n"+declaration+"\n") + options := GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, + ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }} + ctx := llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), options) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s actual declaration authorized an unproved partial FP store: %v", target.triple, err) + } + } +} + +func TestX86RawReturnFrameSlotMustMatchSignature(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, test := range []struct { + name string + mutate func(*FuncSig) + }{ + {"overlap-parameter", func(sig *FuncSig) { + sig.Args = append(sig.Args, I32) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(word / 2), Type: I32, Index: 1, Field: -1}) + }}, + {"overlap-result", func(sig *FuncSig) { sig.Frame.Results[0].Offset = int64(word / 2) }}, + {"parameter-type", func(sig *FuncSig) { sig.Args[0] = I8 }}, + {"parameter-scalar-field", func(sig *FuncSig) { sig.Frame.Params[0].Field = 0 }}, + {"parameter-scalar-path", func(sig *FuncSig) { sig.Frame.Params[0].Fields = []int{0} }}, + {"parameter-field-type", func(sig *FuncSig) { + sig.Args[0] = LLVMType("{ i8, ptr }") + sig.Frame.Params[0].Field = 0 + }}, + {"parameter-field-index", func(sig *FuncSig) { + sig.Args[0] = LLVMType("{ ptr, " + string(typ) + " }") + sig.Frame.Params[0].Field = 2 + }}, + {"parameter-negative-path", func(sig *FuncSig) { + sig.Args[0] = LLVMType("{ ptr, " + string(typ) + " }") + sig.Frame.Params[0].Fields = []int{-1} + }}, + {"parameter-nested-path", func(sig *FuncSig) { + sig.Args[0] = LLVMType("{ ptr, " + string(typ) + " }") + sig.Frame.Params[0].Fields = []int{1, 0} + }}, + {"result-type", func(sig *FuncSig) { sig.Ret = I8 }}, + {"result-scalar-field", func(sig *FuncSig) { sig.Frame.Results[0].Field = 0 }}, + {"result-scalar-path", func(sig *FuncSig) { sig.Frame.Results[0].Fields = []int{0} }}, + {"result-tuple-type", func(sig *FuncSig) { sig.Ret = "{ i8, ptr }" }}, + {"result-tuple-field", func(sig *FuncSig) { + sig.Ret = LLVMType("{ " + string(typ) + ", ptr }") + sig.Frame.Results[0].Field = 1 + }}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+0(FP),AX\n%s AX,ret+%d(FP)\nBYTE $0xc3\nRET\n", 2*word, move, move, word) + requireX86GoAssemblerResult(t, target.arch, source, true) + sig := x86RawReturnScalarFrame(word, typ) + test.mutate(&sig) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": sig}}, true) + }) + } + } +} + +func TestX86RawReturnBoundAggregateScalarSlotObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc not found") + } + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, explicitPath := range []bool{false, true} { + for _, tupleResult := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/path%t/tuple%t", target.triple, explicitPath, tupleResult), func(t *testing.T) { + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+%d(FP),AX\n%s AX,ret+%d(FP)\nBYTE $0xc3\nRET\n", 3*word, move, word, move, 2*word) + requireX86GoAssemblerResult(t, target.arch, source, true) + sig := FuncSig{Name: "Raw", Args: []LLVMType{LLVMType("{ ptr, " + string(typ) + " }")}, Ret: typ, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: int64(word), Type: typ, Index: 0, Field: 1}}, + Results: []FrameSlot{{Offset: int64(2 * word), Type: typ, Index: 0, Field: -1}}, + }} + if explicitPath { + sig.Frame.Params[0].Field = -1 + sig.Frame.Params[0].Fields = []int{1} + } + if tupleResult { + sig.Ret = LLVMType("{ ptr, " + string(typ) + " }") + sig.Frame.Results = []FrameSlot{ + {Offset: int64(2 * word), Type: Ptr, Index: 0, Field: -1}, + {Offset: int64(3 * word), Type: typ, Index: 1, Field: -1}, + } + source = fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+%d(FP),AX\n%s $0,ret0+%d(FP)\n%s AX,ret1+%d(FP)\nBYTE $0xc3\nRET\n", 4*word, move, word, move, 2*word, move, 3*word) + requireX86GoAssemblerResult(t, target.arch, source, true) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Raw": sig}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(mustParseX86RawReturn(t, source), opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "bound-field.ll", "bound-field.o", ir) + }) + } + } + } +} + +// Object evidence only: this independent UD2 harness cannot return to a caller +// and is never executed. It exposes existing generic partial FP lowering; a +// compiled object does not certify preservation of bytes outside the write. +func TestX86PartialFPStoreCompileOnlyEvidence(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + word, typ := 8, I64 + if target.arch == "386" { + word, typ = 4, I32 + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\nMOVB $17,x+0(FP)\nUD2\nRET\n", 2*word) + dir := t.TempDir() + asm, object := filepath.Join(dir, "partial.s"), filepath.Join(dir, "partial.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-p", "example.com/partial", "-o", object, asm) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+target.arch) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("Go assembler failed: %v\n%s", err, out) + } + out, err := exec.Command("go", "tool", "objdump", "-s", "Raw", object).CombinedOutput() + if err != nil || !strings.Contains(string(out), "MOVB $0x11") { + t.Fatalf("Go object did not retain the one-byte write: %v\n%s", err, out) + } + t.Logf("actual Go object, not executed:\n%s", out) + ir, err := Translate(mustParseX86RawReturn(t, source), Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": x86RawReturnScalarFrame(word, typ)}, + }) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(ir, "\n") { + if strings.Contains(line, "17") || strings.Contains(line, "fp_arg") || strings.Contains(line, "zext i8") || + strings.Contains(line, "store "+string(typ)+" %") || strings.Contains(line, "classic_frame") { + t.Log("generic lowering observation (not a semantic pass):", line) + } + } + compileLLVMToObject(t, llc, target.triple, "partial-compile-only.ll", "partial-compile-only.o", ir) + }) + } +} diff --git a/amd64_raw_return_pointer_contract_test.go b/amd64_raw_return_pointer_contract_test.go new file mode 100644 index 000000000..ffe4b0785 --- /dev/null +++ b/amd64_raw_return_pointer_contract_test.go @@ -0,0 +1,300 @@ +package plan9asm + +import ( + "errors" + "fmt" + "sort" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// These are compile-only contract counterexamples, not programs that mutate a +// live return address. Go acceptance proves source grammar, never alias safety. +func x86RawReturnPointerAPIs(t *testing.T, source string, opt Options, wantContext bool) { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "owned-module", "annotated"} { + var callErr error + switch api { + case "text": + _, callErr = Translate(file, opt) + case "module": + module, err := TranslateModule(file, opt) + if err == nil { + module.Dispose() + } + callErr = err + default: + ctx := llvm.NewContext() + options := opt + options.AnnotateSource = api == "annotated" + module, err := TranslateModuleInContext(ctx, file, options) + if err == nil { + module.Dispose() + } + ctx.Dispose() + callErr = err + } + if wantContext && !errors.Is(callErr, ErrProbeNeedsContext) { + t.Errorf("%s did not reject an unproved native continuation: %v", api, callErr) + } + if !wantContext && callErr != nil { + t.Errorf("%s rejected a bounded FP/static leaf: %v", api, callErr) + } + } +} + +func TestX86RawReturnPointerMemoryNeedsContinuationContract(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + instructions := []string{ + "MOVB $0,(AX)", "MOVW $0,1(AX)", "MOVL $0,4(AX)", + "MOVB (AX),AL", "ADDL $1,(AX)", "XCHGL AX,(AX)", + "MOVOU X0,(AX)", "SETEQ (AX)", "BTSL $1,(AX)", + "VMASKMOVPS X0,X1,(AX)", "VMASKMOVPD X0,X1,(AX)", + "VPMASKMOVD X0,X1,(AX)", "VPMASKMOVQ X0,X1,(AX)", + "VMOVDQU32 X0,K1,(AX)", + move + " AX,BX\nMOVB $0,(BX)", + move + " $saved<>(SB),AX\nMOVB $0,(AX)", + "MOVB $0,saved<>(SB)(AX*1)", + "MOVB $0,0(AX)(CX*2)", + } + if target.arch == "amd64" { + instructions = append(instructions, "MOVQ $0,8(AX)") + } + for _, instruction := range instructions { + t.Run(target.triple+"/"+instruction, func(t *testing.T) { + source := fmt.Sprintf("GLOBL saved<>(SB),8,$32\nTEXT Raw(SB),4,$0-%d\n%s p+0(FP),AX\n%s\nBYTE $0xc3\nRET\n", word, move, instruction) + // Go permits indexed static-base addressing in 386, but rejects + // this spelling in amd64. Keep the latter as a defensive parsed + // source negative, not a positive Go format witness. + goAccepted := target.arch == "386" || instruction != "MOVB $0,saved<>(SB)(AX*1)" + requireX86GoAssemblerResult(t, target.arch, source, goAccepted) + sig := FuncSig{Name: "Raw", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": sig}}, true) + }) + } + } +} + +func TestX86RawReturnImplicitMemoryNeedsContinuationContract(t *testing.T) { + instructions := []string{"XLAT", "MASKMOVQ M0,M1", "MASKMOVOU X0,X1", "MASKMOVDQU X0,X1", "VMASKMOVDQU X0,X1"} + for op := range x86StringSpecs { + instructions = append(instructions, string(op)) + } + for _, op := range []string{"INSB", "INSW", "INSL", "OUTSB", "OUTSW", "OUTSL"} { + instructions = append(instructions, op) + } + sort.Strings(instructions) + for _, target := range x86RawReturnTargets { + for _, instruction := range instructions { + if target.arch == "386" && strings.HasSuffix(instruction, "Q") && !strings.HasPrefix(instruction, "MASK") { + continue // Go's 32-bit string table has no Q-width forms. + } + t.Run(target.triple+"/"+instruction, func(t *testing.T) { + source := "TEXT Raw(SB),4,$0-0\n" + instruction + "\nBYTE $0xc3\nRET\n" + requireX86GoAssemblerResult(t, target.arch, source, true) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Ret: Void}}}, true) + }) + } + } +} + +func TestX86RawReturnFPRequiresBoundTypedSlot(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, frame := range []FrameLayout{ + {}, {Params: []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}}}, + } { + offset := 0 + if len(frame.Params) != 0 { + offset = 100 + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+%d(FP),AX\nBYTE $0xc3\nRET\n", word, move, offset) + requireX86GoAssemblerResult(t, target.arch, source, true) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Args: []LLVMType{typ}, Ret: typ, Frame: frame}}}, true) + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s AX,ret+%d(FP)\nBYTE $0xc3\nRET\n", 2*word, move, word) + requireX86GoAssemblerResult(t, target.arch, source, true) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Args: []LLVMType{typ}, Ret: typ, + Frame: FrameLayout{Results: []FrameSlot{{Offset: int64(word), Type: typ, Index: 1, Field: -1}}}}}}, true) + } +} + +func TestX86RawReturnMemoryRangeAndUnknownShapeNeedContext(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, test := range []struct { + name, instruction string + slot FrameSlot + }{ + {"fp-too-narrow", move + " x+0(FP),AX", FrameSlot{Type: I8, Index: 0, Field: -1}}, + {"fp-unrecognized-type", move + " x+0(FP),AX", FrameSlot{Type: "i128", Index: 0, Field: -1}}, + {"fp-unbound-index", move + " x+0(FP),AX", FrameSlot{Type: typ, Index: 1, Field: -1}}, + {"static-before-region", "MOVL AX,saved<>-1(SB)", FrameSlot{Type: typ, Index: 0, Field: -1}}, + {"static-after-region", "MOVL AX,saved<>+32(SB)", FrameSlot{Type: typ, Index: 0, Field: -1}}, + {"static-cross-end", "MOVL AX,saved<>+30(SB)", FrameSlot{Type: typ, Index: 0, Field: -1}}, + {"static-no-object-contract", "MOVL AX,unknown(SB)", FrameSlot{Type: typ, Index: 0, Field: -1}}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source := fmt.Sprintf("GLOBL saved<>(SB),8,$32\nTEXT Raw(SB),4,$0-%d\n%s\nBYTE $0xc3\nRET\n", word, test.instruction) + requireX86GoAssemblerResult(t, target.arch, source, true) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Args: []LLVMType{typ}, Ret: typ, + Frame: FrameLayout{Params: []FrameSlot{test.slot}}}}}, true) + }) + } + } +} + +func TestX86RawReturnBoundFPAndStaticObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc not found") + } + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, encoding := range []string{"BYTE $0xc3", "BYTE $0xc2\nWORD $0"} { + for _, pointer := range []bool{false, true} { + valueType := typ + if pointer { + valueType = Ptr + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+0(FP),AX\n%s AX,ret+%d(FP)\n%s\nRET\n", 2*word, move, move, word, encoding) + requireX86GoAssemblerResult(t, target.arch, source, true) + sig := FuncSig{Name: "Raw", Args: []LLVMType{valueType}, Ret: valueType, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: valueType, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: int64(word), Type: valueType, Index: 0, Field: -1}}}} + opt := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Raw": sig}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(mustParseX86RawReturn(t, source), opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "fp-return.ll", "fp-return.o", ir) + } + source := "GLOBL saved<>(SB),8,$32\nTEXT Raw(SB),4,$0-0\nMOVL $17,AX\nMOVL AX,saved<>+4(SB)\nMOVL saved<>+4(SB),AX\n" + encoding + "\nRET\n" + requireX86GoAssemblerResult(t, target.arch, source, true) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Ret: I32}}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(mustParseX86RawReturn(t, source), opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "static-return.ll", "static-return.o", ir) + } + } +} + +func TestX86RawReturnPrepartitionRetainsFinalFrameValidation(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+0(FP),AX\nBYTE $0xc3\nRET\n", word, move) + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), target.arch) + if err != nil { + t.Fatal("pure preparation must not discard the future typed frame:", err) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Args: []LLVMType{typ}, Ret: typ}}} + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("prepartition preparation erased the final missing-frame contract:", err) + } + sig := opt.Sigs["Raw"] + sig.Frame.Params = []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}} + opt.Sigs["Raw"] = sig + ctx = llvm.NewContext() + module, err = TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if err != nil { + t.Fatal("existing consumer's bound signature was not consumed:", err) + } + } +} + +func TestX86RawReturnActualGoDeclarationContract(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc not found") + } + for _, target := range x86RawReturnTargets { + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + for _, test := range []struct { + name, declaration, body string + argSize int + context bool + }{ + {"pointer-store", "func Raw(*byte)", move + " p+0(FP),AX\nMOVB $0,(AX)", word, true}, + {"unknown-fp", "func Raw(uintptr) uintptr", move + " p+100(FP),AX", 2 * word, true}, + {"typed-pointer-transport", "func Raw(*byte) *byte", fmt.Sprintf("%s p+0(FP),AX\n%s AX,ret+%d(FP)", move, move, word), 2 * word, false}, + {"typed-static-global", "func Raw() uint32", "MOVL $17,AX\nMOVL AX,saved<>+4(SB)\nMOVL saved<>+4(SB),AX\nMOVL AX,ret+0(FP)", 4, false}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source := fmt.Sprintf("GLOBL saved<>(SB),8,$32\nTEXT ·Raw(SB),4,$0-%d\n%s\nBYTE $0xc3\nRET\n", test.argSize, test.body) + requireX86GoAssemblerResult(t, target.arch, source, true) + pkg := mustGoPackage(t, "test/rawret", "package rawret\n"+test.declaration+"\n") + options := GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, + ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }} + tr, err := TranslateGoModule(pkg, []byte(source), options) + if tr != nil { + defer tr.Module.Dispose() + } + if test.context { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a Go pointer/declaration invented a return-PC contract:", err) + } + return + } + if err != nil { + t.Fatal("actual declaration's bounded typed frame rejected:", err) + } + compileLLVMToObject(t, llc, target.triple, "declared-return.ll", "declared-return.o", tr.Module.String()) + }) + } + } +} + +func mustParseX86RawReturn(t *testing.T, source string) *File { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + return file +} diff --git a/amd64_raw_return_source_address_runtime_test.go b/amd64_raw_return_source_address_runtime_test.go new file mode 100644 index 000000000..a2e73c1ce --- /dev/null +++ b/amd64_raw_return_source_address_runtime_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// The original native reference executes Go's actual LEA and raw C3 bytes, +// not an encoding manufactured by the translator. Register arguments here +// are an explicit numerical custom ABI, never a native-pointer permission. +func TestCrossLinuxRuntimeMatrixX86RawReturnSourceAddressNumerical(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + var assembly, oracle, checks strings.Builder + oracle.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + sigs := make(map[string]FuncSig) + for _, width := range []string{"L", "Q"} { + for _, scale := range []int{1, 2, 4, 8} { + name := fmt.Sprintf("address%s%d", width, scale) + source := fmt.Sprintf("TEXT %s(SB),4,$0-0\nMOV%s $-7(AX)(CX*%d),AX\nBYTE $0xc3\nRET\n", name, width, scale) + assembly.WriteString(source) + code := x86PackedShiftActualGoBytes(t, "amd64", source) + instruction, err := x86asm.Decode(code, 64) + if err != nil || instruction.Op != x86asm.LEA || instruction.Len >= len(code) || code[instruction.Len] != 0xc3 { + t.Fatalf("actual Go address rewrite/raw return lost: %x / %v", code, err) + } + var bytes strings.Builder + for _, value := range code[:instruction.Len+1] { + if bytes.Len() != 0 { + bytes.WriteByte(',') + } + fmt.Fprintf(&bytes, "0x%02x", value) + } + oracle.WriteString(x86OriginalSysVDeclaration("uint64_t", "original"+name, "uint64_t,uint64_t")) + fmt.Fprintf(&oracle, ` +extern uint64_t %[1]s(uint64_t,uint64_t); +__asm__(".text\n.globl " LABEL("original%[1]s") "\n" + LABEL("original%[1]s") ":\n" + "movq %%rdi,%%rax\nmovq %%rsi,%%rcx\n.byte %[2]s\n"); +`, name, bytes.String()) + want := fmt.Sprintf("a+%d*b-7", scale) + if width == "L" { + want = "(uint64_t)(uint32_t)(" + want + ")" + } + fmt.Fprintf(&checks, ` + { uint64_t want=%[1]s; + if (%[2]s(a,b)!=want || original%[2]s(a,b)!=want) { + fprintf(stderr,"raw RET address %[2]s seed %%u\n",seed); return 1; + } + } +`, want, name) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{I64, I64}, ArgRegs: []Reg{AX, CX}, Ret: I64} + } + } + oracle.WriteString(`int main(void) { + for (unsigned seed=0; seed<256; seed++) { + uint64_t a=UINT64_C(0x1313939382827272)*seed,b=~a; +` + checks.String() + ` + } + puts("RAW_RET_NUMERICAL_ADDRESS_2048_OK"); + return 0; +} +`) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + output := execution.compileAndRunLLVM(t, "raw-return-address", ir, oracle.String(), level) + if string(output) != "RAW_RET_NUMERICAL_ADDRESS_2048_OK\n" { + t.Fatalf("missing complete numerical original/LLVM oracle: %q", output) + } + } +} diff --git a/amd64_raw_return_source_address_test.go b/amd64_raw_return_source_address_test.go new file mode 100644 index 000000000..8ff3cecae --- /dev/null +++ b/amd64_raw_return_source_address_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Go progedit rewrites the whole MOVL/Q TYPE_ADDR family to LEAL/Q before +// classification. A raw near RET must not prohibit a purely numerical, +// stack-unobserving address calculation or inherit unrelated source proofs. +func TestX86RawReturnSourceAddressLeafThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + widths := []string{"L"} + if target.arch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + for _, address := range []string{"8(AX)", "-8(AX)(CX*1)", "16(AX)(CX*2)", "32(AX)(CX*4)", "64(AX)(CX*8)", "8(CX*4)", "(1<<4)(AX)"} { + t.Run(target.triple+"/"+width+"/"+address, func(t *testing.T) { + source := fmt.Sprintf("TEXT ·addr(SB),4,$0-0\nMOVL $3,AX\nMOVL $2,CX\nMOV%s $%s,BX\nBYTE $0xc3\nRET\n", width, address) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + before := fmt.Sprintf("%#v", file.Funcs) + pkg := x86AddSubLiteralPackage(t, target.arch, "func addr()\n") + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "literal.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, target.arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + context := llvm.NewContext() + defer context.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var result *GoModuleTranslation + result, err = translateGoModuleInContext(context, pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + ir = result.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-return-address.ll", "raw-return-address.o", ir) + if before != fmt.Sprintf("%#v", file.Funcs) { + t.Fatal("derived address changed the original source") + } + }) + } + }) + } + } + } +} + +func TestX86RawReturnSourceAddressCannotInheritProof(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + file, err := Parse(ArchAMD64, "TEXT addr(SB),4,$0-0\nMOVL $8(AX)(CX*4),BX\nBYTE $0xc3\nRET\n") + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, Sigs: map[string]FuncSig{"addr": {Name: "addr", Ret: Void}}} + normalized, err := normalizeX86RawFile(file, arch, options) + if err != nil { + t.Fatal(err) + } + for _, field := range []string{"sourceStack", "fixedLocal", "rawText", "LOCK", "frame", "scalarConstant", "g", "memory", "encodedSibling"} { + t.Run(arch+"/"+field, func(t *testing.T) { + copy := *normalized + copy.Funcs = append([]Func(nil), normalized.Funcs...) + fn := ©.Funcs[0] + fn.Instrs = append([]Instr(nil), fn.Instrs...) + fn.Instrs[1].Args = append([]Operand(nil), fn.Instrs[1].Args...) + switch field { + case "sourceStack": + fn.x86SourceStack = &x86SourceStackProof{} + case "fixedLocal": + fn.x86FixedLocal = &x86FixedLocalProof{} + case "rawText": + fn.X86RawText = []byte{0xc3} + case "LOCK": + fn.Instrs = append(fn.Instrs, Instr{Op: "LOCK"}) + case "frame": + fn.FrameSize = 8 + case "scalarConstant": + fn.Instrs[1].Op = "MOVW" + case "g": + fn.Instrs = append(fn.Instrs, Instr{Op: "MOVQ", Args: []Operand{{Kind: OpReg, Reg: "R14"}, {Kind: OpReg, Reg: AX}}}) + case "memory": + fn.Instrs = append(fn.Instrs, Instr{Op: "MOVL", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX}}, {Kind: OpReg, Reg: DX}}}) + case "encodedSibling": + fn.Instrs = append(fn.Instrs, Instr{Op: "MOVL", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}, x86Encoded: true}) + } + before := fmt.Sprintf("%#v", fn) + if _, err := normalizeX86SourceIntegerAddresses(©, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("adapter inherited %s proof or native effect: %v", field, err) + } + if before != fmt.Sprintf("%#v", fn) || !reflect.DeepEqual(file.Funcs[0].Instrs[1].Args, normalized.Funcs[0].Instrs[1].Args) { + t.Fatal("rejection modified original instructions") + } + }) + } + } +} diff --git a/amd64_raw_return_source_stack.go b/amd64_raw_return_source_stack.go new file mode 100644 index 000000000..b5aced6ee --- /dev/null +++ b/amd64_raw_return_source_stack.go @@ -0,0 +1,971 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" +) + +// This is a source ABI0 stack, not LLVM's physical stack or an incoming native +// pointer. It is enabled only after all returns and all stack uses are proved. +// Ordinary pointers need a separately consumed complete Go caller proof. +// Nonzero TEXT frames, calls and private-stack address escape cannot use it. +type x86SourceStackProof struct { + minimum int64 + word int64 + caller *x86GoCallerRegionProof + scalarResult *x86SourceStackScalarProof + // Exact public normalized source bytes, excluding private certificates. + normalized [32]byte + // Diagnostic-only finite replay counts, never used as permission inputs. + countedIterations [4]int +} + +// Bind raw stack producer identity before consulting Go's named PUSH/POP +// deltasp rules. Edited normalized instructions cannot borrow that origin. +type x86SourceRawInstruction struct { + op Op + args []Operand + vectorBytes int + addressBits int // Exact decoded mode; not an address-size or native-entry permission. + // Conversion-only physical identity, captured explicitly by the decoder. + // A zero value is absent, not an SSE/VEX or non-MMX assertion. + packedConversion x86RawPackedConversionIdentity + // A low-bank unmasked packed move still retains its physical generation. + packedMove x86RawPackedMoveIdentity + packedReduction x86RawPackedReductionIdentity + ternaryLogic x86RawTernaryLogicIdentity + // An absent mode snapshot cannot claim a legacy compare continuation. + legacyFloatCompare x86RawLegacyFloatCompareIdentity + comi x86RawCOMIIdentity + floatMinMax x86RawFloatMinMaxIdentity + // Neither an absent snapshot nor a low bank implies a VEX conversion. + scalarPrecision x86RawScalarPrecisionIdentity + scalarFloatInteger x86RawScalarFloatIntegerIdentity + packedPrecision x86RawPackedPrecisionIdentity + namedHalf x86RawNamedHalfIdentity + fma3 x86RawFMA3Identity + explicitMaskMove x86RawExplicitMaskMoveIdentity + packedShift x86RawPackedShiftIdentity + packedAbs x86RawPackedAbsIdentity + packedDot x86RawPackedDotIdentity + permute128 x86RawPermute128Identity + zeroExtend x86RawZeroExtendIdentity + scalarSegmentLoad x86RawScalarSegmentLoadIdentity +} + +type x86SourceStackAddress struct { + // original distinguishes the caller's opaque BP from an entry-SP offset. + original bool + offset int64 +} + +type x86SourceStackCell struct { + offset int64 + width int64 + bp *x86SourceStackAddress + flags bool + numericFlags bool + partialNumericFlags uint8 + caller *x86CallerPointerValue + known uint8 + canonical uint8 + scalarFlags uint8 + gOriginal bool +} + +type x86SourceStackState struct { + sp int64 + bp x86SourceStackAddress + cells []x86SourceStackCell + numericFlags bool + partialNumericFlags uint8 + callers map[Reg]x86CallerPointerValue + known map[Reg]uint8 + vectors map[Reg]uint64 + masks map[Reg]bool + gOriginal bool + scalarResult *x86SourceStackScalarState +} + +const x86SourceStackLimit = int64(1 << 20) + +// Recompute the proof on every final normalization. No caller-supplied signature, +// changed source, or prepartition result inherits a cached admission flag. +func normalizeX86RawReturnSourceStack(fn Func, opt Options, bindFrame bool, original ...Func) Func { + input := fn + if len(original) != 0 { + input = original[0] + } + fn.x86SourceStack = nil + fn.x86SourceStackFailure = "" + reject := func(index int, reason string) Func { + fn.x86SourceStackFailure = fmt.Sprintf("normalized instruction %d: %s", index, reason) + return fn + } + if fn.FrameSize != 0 || strings.Contains(fn.Sym, "") { + return fn + } + hasRawReturn := false + for _, ins := range fn.Instrs { + if form := ins.x86RawNearReturn; form != nil { + if !form.nativeWidth || form.cleanup != 0 { + return fn + } + hasRawReturn = true + } + } + if !hasRawReturn { + return fn + } + word, nativeMove, nativeSuffix := int64(8), Op("MOVQ"), "Q" + if opt.Goarch == "386" { + word, nativeMove, nativeSuffix = 4, "MOVL", "L" + } + name := fn.Sym + if opt.ResolveSym != nil { + name = opt.ResolveSym(name) + } + sig, hasSig := opt.Sigs[name] + physicalBool := bindFrame && hasSig && x86SourceStackBoolCandidate(fn, sig) + if physicalBool { + // This argument is a request-local Options value. Match the public + // ArchAMD64 default for the complete grammar; do not rewrite the + // caller's bound Options or the raw decoder's original mode spelling. + opt.Goarch = x86SourceStackScalarArch(opt.Goarch) + } + caller := bindFrame && hasSig && x86CallerRegionSourceMatches(sig, input, opt.Goarch) + if !caller { + for index, instruction := range fn.Instrs { + if _, known, _ := x86ExplicitIMULProperties(instruction.Op, instruction, opt.Goarch); known { + return reject(index, "explicit multiply needs a complete typed caller input proof") + } + if _, known, _ := x86ConditionalMoveProperties(instruction.Op, instruction, opt.Goarch); known { + return reject(index, "conditional move needs a complete typed caller input and flags proof") + } + if _, conditionalSet, _ := x86ConditionalSetProperties(instruction.Op, instruction, opt.Goarch); conditionalSet { + return reject(index, "conditional byte output needs a complete typed caller input proof") + } + if _, extension := amd64ScalarExtensionMoveSpecs[instruction.Op]; extension || instruction.x86RawMOVSXDNeedsCaller { + if x86RawDwordLiteralInput(instruction) { + continue + } + // The old scalar slot contract proves frame bounds, not GP + // input availability. It cannot authorize a new extension read + // of unknown or partially initialized native entry registers. + return reject(index, "scalar extension needs a complete typed caller input proof") + } + } + } + if !caller && !physicalBool && x86RawReturnLeafStackUnobserved(fn) { + return fn + } + if bindFrame && hasSig && !caller { + opt.Goarch = x86SourceStackScalarArch(opt.Goarch) + } + if bindFrame && (!hasSig || !caller && !x86SourceStackScalarABI0(sig, fn.ArgSize, opt.Goarch)) { + return fn + } + + // Validate the complete source, including dead instructions. A dead CALL + // still changes Go's source prologue; unknown stack effects stay Context. + labels := map[string]int{} + var instructions []int + goDeltas := map[int]int64{} + var goDelta int64 + hasText := false + for index, ins := range fn.Instrs { + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpTEXT { + if hasText || !x86RawReturnLeafStackUnobserved(Func{Instrs: []Instr{ins}}) { + return fn + } + hasText = true + } + if op == OpLABEL { + if len(ins.Args) != 1 || ins.Args[0].Kind != OpLabel { + return fn + } + if _, duplicate := labels[ins.Args[0].Sym]; duplicate { + return fn + } + labels[ins.Args[0].Sym] = len(instructions) + continue + } + vectorMove := caller && x86CallerVectorMoveKnown(ins) + if !x86SourceStackInstructionKnown(ins, nativeMove, nativeSuffix, opt.Goarch) && !vectorMove { + return reject(index, "unregistered source-stack effect: "+string(ins.Op)) + } + if x86LEAWidth(ins.Op, opt.Goarch) != 0 && ins.Op != Op("LEA"+nativeSuffix) && !caller { + // The older scalar stack proof establishes a continuation, not + // numerical GP initialization. Only the actual caller's complete + // must-defined transfer can authorize these additional widths. + return reject(index, "non-native-width LEA requires actual numerical caller source") + } + if opt.Goarch == "amd64" && !caller && x86SourceGOperand(ins) { + if _, stack := amd64StackWidthSpecs[op]; !stack { + return reject(index, "g numerical/address use requires actual caller source") + } + } + if caller && !vectorMove && !x86CallerScalarOperands(ins) { + return reject(index, "unregistered caller-region operand bank: "+string(ins.Op)) + } + if _, stack := amd64StackWidthSpecs[op]; stack && ins.x86SourceRaw != nil && + (ins.x86SourceRaw.op != op || !reflect.DeepEqual(ins.x86SourceRaw.args, ins.Args)) { + return fn + } + for _, operand := range ins.Args { + if operand.Kind == OpFPAddr || (operand.Kind == OpFP && operand.FPOffset < 0) { + return fn + } + if operand.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(operand.Sym), "$") { + // Address-of is not an ordinary numeric immediate, regardless of + // the register spelling or whether its later value is overwritten. + return fn + } + } + instructions = append(instructions, index) + goDeltas[index] = goDelta + // cmd/internal/obj/x86 preprocess adjusts named FP operands using + // its linear deltasp. BYTE/WORD pushes are opaque to that pass. + if spec, stack := amd64StackWidthSpecs[op]; stack && ins.x86SourceRaw == nil { + width := int64(spec.bits / 8) + if spec.mode == amd64StackPushValue || spec.mode == amd64StackPushFlags { + goDelta += width + } else { + goDelta -= width + } + if goDelta < -x86SourceStackLimit || goDelta > x86SourceStackLimit { + return fn + } + } + } + if !hasText || len(instructions) == 0 { + return fn + } + + copyFn := fn + copyFn.Instrs = append([]Instr(nil), fn.Instrs...) + for index := range copyFn.Instrs { + copyFn.Instrs[index].Args = append([]Operand(nil), fn.Instrs[index].Args...) + copyFn.Instrs[index].x86SourceStackFP = nil + copyFn.Instrs[index].x86CallerMemory = nil + } + states := map[int]x86SourceStackState{} + work := []int{0} + states[0] = x86SourceStackState{bp: x86SourceStackAddress{original: true}, gOriginal: true} + proof := &x86SourceStackProof{word: word} + if bindFrame && hasSig && (!caller || sig.Ret == I1) { + states[0] = x86SourceStackState{bp: x86SourceStackAddress{original: true}, gOriginal: true, + scalarResult: x86SourceStackScalarInitial(fn, sig, opt.Goarch)} + if states[0].scalarResult == nil || caller { + return reject(0, "physical scalar result requires its independent scalar declaration") + } + } + if caller { + proof.caller = sig.x86CallerRegions + states[0] = x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, + callers: make(map[Reg]x86CallerPointerValue), + known: make(map[Reg]uint8), + vectors: make(map[Reg]uint64), + masks: make(map[Reg]bool), + gOriginal: true, + } + } + transfer := func(pc int, ins *Instr, state *x86SourceStackState) (bool, string) { + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpRET { + if word == 8 && !state.gOriginal { + return false, "unrestored original g identity at source return" + } + if len(ins.Args) != 0 || state.sp != 0 || !state.bp.original || goDeltas[instructions[pc]] != 0 { + return false, "unproved source-stack or ABI0 continuation effect" + } + if value := state.scalarResult; value != nil && !value.complete() { + return false, "physical scalar result lacks initialized/canonical bytes on every return" + } + return true, "" + } + for _, operand := range ins.Args { + if operand.Kind == OpFP && state.sp+goDeltas[instructions[pc]] != 0 { + return false, "unproved source-stack or ABI0 continuation effect" + } + } + + if spec, stack := amd64StackWidthSpecs[op]; stack { + if !x86SourceStackStep(spec, ins, state, word) { + return false, "unproved source-stack or ABI0 continuation effect" + } + } else if !x86SourceStackAddressStep(ins, state, nativeMove, nativeSuffix, word) { + return false, "unproved source-stack or ABI0 continuation effect" + } + if state.sp < -x86SourceStackLimit || state.sp > 0 { + return false, "unproved source-stack or ABI0 continuation effect" + } + if state.sp < proof.minimum { + proof.minimum = state.sp + } + for operandIndex, operand := range ins.Args { + if operand.Kind != OpMem || !x86RawReturnStackMemory(operand.Mem) { + continue + } + if x86LEAWidth(ins.Op, opt.Goarch) != 0 { + continue // The address step proved its destination and range. + } + width := x86ScalarMoveMemoryBytes(*ins, operandIndex, opt.Goarch) + memory := operand.Mem + if width == 0 || memory.Sym != "" || memory.Index != "" || memory.Segment != "" || + ins.x86AddressBits != 0 { + return false, "unproved source-stack or ABI0 continuation effect" + } + base := state.sp + if memory.Base == BP { + if state.bp.original { + return false, "unproved source-stack or ABI0 continuation effect" + } + base = state.bp.offset + } else if memory.Base != SP { + return false, "unproved source-stack or ABI0 continuation effect" + } + // Source entry SP[0:word] is the return PC. Only declaration-backed + // data at entry SP+word can be mapped to FP shadow storage. + if memory.Off < -x86SourceStackLimit || memory.Off > x86SourceStackLimit { + return false, "unproved source-stack or ABI0 continuation effect" + } + offset := base + memory.Off - word + if offset < 0 || offset > fn.ArgSize || width > fn.ArgSize-offset { + return false, "unproved source-stack or ABI0 continuation effect" + } + if bindFrame { + if !x86RawReturnFPBound(offset, width, sig, opt.Goarch) { + return false, "unproved source-stack or ABI0 continuation effect" + } + if ins.x86SourceStackFP == nil { + ins.x86SourceStackFP = map[int]int64{} + } + ins.x86SourceStackFP[operandIndex] = offset + } + } + pointerFlags := false + if caller { + var valid bool + if x86CallerVectorMoveKnown(*ins) { + valid = x86CallerVectorMoveStep(ins, state, sig.x86CallerRegions) + } else { + pointerFlags, valid = x86CallerScalarStep(ins, state, sig.x86CallerRegions) + } + if !valid { + return false, "unproved caller-region effect: " + string(ins.Op) + } + x86CallerGDefinition(*ins, state) + } + if !x86SourceStackFlagsStepForArch(*ins, state, opt.Goarch) { + return false, "unproved source-stack or ABI0 continuation effect" + } + if pointerFlags { + x86SourceStackSetNumericFlags(state, 0) + } + if state.scalarResult != nil && !state.scalarResult.step(*ins, opt.Goarch) { + return false, "physical scalar result has an undefined numerical input or effect" + } + + return false, "" + } + returned := false + acyclic := (caller || states[0].scalarResult != nil) && x86CallerReachableAcyclic(copyFn, instructions, labels) + if caller && !acyclic { + if plan := deriveX86CallerCountedPlan(input, copyFn, instructions, labels); plan != nil { + first, reason := replayX86CallerCountedPlan(plan, ©Fn, instructions, states[0], proof.caller, transfer) + if reason != "" { + return reject(0, reason) + } + minimum := proof.minimum + proof.minimum = 0 + for index := range copyFn.Instrs { + copyFn.Instrs[index].x86CallerMemory = nil + copyFn.Instrs[index].x86SourceStackFP = nil + } + finalPlan := deriveX86CallerCountedPlan(input, copyFn, instructions, labels) + if !reflect.DeepEqual(plan, finalPlan) { + return reject(0, "counted-loop plan changed during independent replay") + } + second, reason := replayX86CallerCountedPlan(finalPlan, ©Fn, instructions, states[0], proof.caller, transfer) + if reason != "" || minimum != proof.minimum || !reflect.DeepEqual(first, second) { + return reject(0, "counted-loop final access/state replay changed: "+reason) + } + for _, visit := range second { + if _, header := finalPlan.headers[visit.pc]; header { + proof.countedIterations[visit.residue]++ + } + } + var err error + proof.normalized, err = x86CallerSourceFingerprint(copyFn) + if err != nil { + return fn + } + copyFn.x86SourceStack = proof + return copyFn + } + } + final := false + scalarByteWork := 0 + for budget := 0; ; budget++ { + if len(work) == 0 { + if !acyclic || final { + break + } + // Discard every provisional certificate. Independently replay all + // final reachable inputs, once, after the finite merge fixed point. + final = true + budget = 0 + for pc, index := range instructions { + copyFn.Instrs[index].x86CallerMemory = nil + copyFn.Instrs[index].x86SourceStackFP = nil + if _, reachable := states[pc]; reachable { + work = append(work, pc) + } + } + } + if budget >= 8192 { + return fn + } + pc := work[0] + work = work[1:] + state := states[pc] + if value := state.scalarResult; value != nil { + scalarByteWork += 8 * (2*len(value.bytes.fp) + 33 + 2*len(state.cells)) + if scalarByteWork > 64<<20 { + return reject(instructions[pc], "physical scalar source-CFG byte budget exhausted") + } + } + if caller { + state = x86CloneCallerState(state) + } else { + state.cells = append([]x86SourceStackCell(nil), state.cells...) + state.scalarResult = state.scalarResult.clone() + } + ins := ©Fn.Instrs[instructions[pc]] + ins.x86CallerMemory = nil + ins.x86SourceStackFP = nil + op := Op(normalizeInstructionOpcode(ins.Op)) + isReturn, reason := transfer(pc, ins, &state) + if reason != "" { + return reject(instructions[pc], reason) + } + if isReturn { + returned = true + continue + } + successors := []int{pc + 1} + if op == OpJMP || isAMD64ConditionalBranch(op) { + if len(ins.Args) != 1 { + return fn + } + operand := ins.Args[0] + label := operand.Sym + if operand.Kind == OpIdent { + label = operand.Ident + } + target, known := labels[label] + if operand.Kind == OpMem && operand.Mem.Base == PC && operand.Mem.Index == "" && + operand.Mem.Sym == "" && operand.Mem.Off >= -int64(len(instructions)) && operand.Mem.Off <= int64(len(instructions)) { + target, known = pc+int(operand.Mem.Off), true + } else if operand.Kind != OpLabel && operand.Kind != OpIdent { + return fn + } + if !known { + return fn + } + if op == OpJMP { + successors = []int{target} + } else { + successors = append(successors, target) + } + } + for _, next := range successors { + if next < 0 || next >= len(instructions) { + return fn // Fallthrough cannot invent a source continuation. + } + if old, visited := states[next]; visited { + if !reflect.DeepEqual(old, state) { + if !acyclic { + return reject(instructions[next], "different source-stack or caller-region states at join") + } + var merged x86SourceStackState + var valid bool + if caller { + merged, valid = x86CallerAcyclicMerge(old, state, sig.x86CallerRegions) + } else { + merged, valid = x86SourceStackScalarMerge(old, state) + } + if !valid { + return reject(instructions[next], "different source-stack or caller-region states at join") + } + if reflect.DeepEqual(old, merged) { + continue + } + if final { + return reject(instructions[next], "caller-region changed during final certificate replay") + } + states[next] = merged + work = append(work, next) + } + continue + } + if final { + return reject(instructions[next], "new caller-region state during final certificate replay") + } + states[next] = state + work = append(work, next) + } + } + if !returned { + return fn + } + if states[0].scalarResult != nil { + proof.scalarResult = sealX86SourceStackScalar(copyFn, input, sig, proof) + if proof.scalarResult == nil { + return reject(0, "physical scalar result source seal failed") + } + } + if caller { + var err error + proof.normalized, err = x86CallerSourceFingerprint(copyFn) + if err != nil { + return fn + } + } + copyFn.x86SourceStack = proof + return copyFn +} + +func x86SourceStackScalarABI0(sig FuncSig, argSize int64, arch string) bool { + if len(sig.ArgRegs) != 0 || argSize <= 0 || argSize > x86SourceStackLimit || len(sig.Frame.Results) != 1 || + len(sig.Frame.Params) != len(sig.Args) || len(sig.Args) == 0 { + return false + } + seen := map[int]bool{} + for _, slot := range sig.Frame.Params { + width := x86FrameScalarBytes(slot.Type, arch) + if !x86SourceStackScalarType(slot.Type) || len(frameSlotFields(slot)) != 0 || + slot.Index < 0 || slot.Index >= len(sig.Args) || slot.Type != sig.Args[slot.Index] || + width == 0 || !x86RawReturnFPBound(slot.Offset, width, sig, arch) || + slot.Offset > argSize || width > argSize-slot.Offset || seen[slot.Index] { + return false + } + seen[slot.Index] = true + } + slot := sig.Frame.Results[0] + width := x86FrameScalarBytes(slot.Type, arch) + return x86SourceStackScalarType(slot.Type) && slot.Type == sig.Ret && width != 0 && + x86RawReturnFPBound(slot.Offset, width, sig, arch) && slot.Offset <= argSize && width <= argSize-slot.Offset +} + +func x86SourceStackScalarType(typ LLVMType) bool { + switch typ { + case I1, I8, I16, I32, I64: + return true + default: + return false + } +} + +func x86SourceStackInstructionKnown(ins Instr, nativeMove Op, suffix, arch string) bool { + if _, known, err := x86CounterControlProperties(ins.Op, ins, arch); known { + return err == nil // Original raw/address metadata cannot borrow named count semantics. + } + if _, known, err := x86ExplicitIMULProperties(ins.Op, ins, arch); known { + return err == nil + } + if _, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch); known { + return err == nil + } + if _, known, err := x86OrderingHintProperties(ins.Op, ins, arch); known { + return err == nil + } + op := Op(normalizeInstructionOpcode(x86SourceMoveInstruction(ins, arch).Op)) + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); recognized { + return err == nil + } + if op == OpTEXT || op == OpRET || op == OpJMP || isAMD64ConditionalBranch(op) { + return true + } + if spec, stack := amd64StackWidthSpecs[op]; stack { + return string(ins.Op) == string(op) && (spec.bits == 16 || (spec.bits == 32 && arch == "386") || (spec.bits == 64 && arch == "amd64")) + } + if op == nativeMove || x86LEAWidth(ins.Op, arch) != 0 || x86RawReturnScalarMoveBytes(op) != 0 { + return true + } + if _, recognized, err := x86ScalarExtensionProperties(op, ins, arch); recognized { + return err == nil + } + // This bounded numerical contract does not infer effects for opaque raw, + // native/system calls, aggregate/vector memory, or instructions with hidden + // stack inputs. These remain Context, not unsupported decoder forms. + for _, base := range []string{"ADD", "SUB", "ADC", "SBB", "AND", "OR", "XOR", "CMP", "TEST", "INC", "DEC", "NEG", "NOT", "SHL", "SHR", "SAR"} { + for _, width := range []string{"B", "W", "L", "Q"} { + if op == Op(base+width) && (width != "Q" || arch == "amd64") { + return true + } + } + } + return false +} + +func x86SourceStackStep(spec amd64StackWidthSpec, ins *Instr, state *x86SourceStackState, word int64) bool { + width := int64(spec.bits / 8) + push := spec.mode == amd64StackPushValue || spec.mode == amd64StackPushFlags + flags := spec.mode == amd64StackPushFlags || spec.mode == amd64StackPopFlags + if flags && len(ins.Args) != 0 || !flags && len(ins.Args) != 1 { + return false + } + cell := x86SourceStackCell{width: width, flags: flags, numericFlags: state.numericFlags, partialNumericFlags: state.partialNumericFlags} + if state.scalarResult != nil && flags { + cell.scalarFlags = state.scalarResult.bytes.flags + } + if !flags { + operand := ins.Args[0] + if operand.Kind != OpReg && (!push || operand.Kind != OpImm) { + return false // Memory/segment stack transfers need a separate contract. + } + if operand.Kind == OpReg { + if x86SegmentRegisterOperand(operand) || operand.Reg == SP { + return false + } + if x86RawReturnStackRegister(operand.Reg) { + if operand.Reg != BP || width != word { + return false + } + value := state.bp + cell.bp = &value + } + if x86SourceGRegister(operand.Reg) && state.gOriginal && push { + if word != 8 || width != word || operand.Reg != "R14" { + return false // A partial token cannot preserve the opaque g identity. + } + cell.gOriginal = true + } + if state.callers != nil && operand.Reg != BP { + root, valid := x86CallerGP(operand.Reg) + if !valid { + return false + } + if pointer, present := state.callers[root]; present { + if width != word || operand.Reg != root { + return false + } + value := pointer + cell.caller = &value + } + cell.known = state.known[root] + } + if state.scalarResult != nil && operand.Reg != BP { + arch := "amd64" + if word == 4 { + arch = "386" + } + index, mask, valid := x86ResultGP(operand.Reg, width, arch) + if !valid { + return false + } + cell.known = state.scalarResult.bytes.gp[index] & mask + cell.canonical = state.scalarResult.bytes.canonicalGP[index] & mask + } + } + if state.scalarResult != nil && operand.Kind == OpImm { + if operand.ImmRaw != "" || operand.ImmIsFloat { + return false + } + cell.known = uint8((uint16(1) << uint(width)) - 1) + cell.canonical = fixedLocalConstantBooleans(operand.Imm, width) + } + } + if push { + state.sp -= width + cell.offset = state.sp + // A new store replaces the covered old cell, never reuses stale data. + for len(state.cells) != 0 && state.cells[len(state.cells)-1].offset <= state.sp { + state.cells = state.cells[:len(state.cells)-1] + } + state.cells = append(state.cells, cell) + return true + } + if len(state.cells) == 0 { + return false + } + // Restoring a saved SP may discard lower private cells. The next POP must + // still read a complete initialized source PUSH of the same physical width. + for len(state.cells) != 0 && state.cells[len(state.cells)-1].offset < state.sp { + state.cells = state.cells[:len(state.cells)-1] + } + if len(state.cells) == 0 { + return false + } + saved := state.cells[len(state.cells)-1] + if saved.offset != state.sp || saved.width != width { + return false + } + if flags != saved.flags { + return false // Scalar bytes and complete hardware FLAGS are not interchangeable. + } + if flags { + state.numericFlags = saved.numericFlags + state.partialNumericFlags = saved.partialNumericFlags + if state.scalarResult != nil { + state.scalarResult.bytes.flags = saved.scalarFlags + } + } + if state.scalarResult != nil && !flags && ins.Args[0].Reg != BP { + arch := "amd64" + if word == 4 { + arch = "386" + } + index, mask, valid := x86ResultGP(ins.Args[0].Reg, width, arch) + if !valid { + return false + } + value := &state.scalarResult.bytes + value.gp[index] = value.gp[index]&^mask | saved.known&mask + value.canonicalGP[index] = value.canonicalGP[index]&^mask | saved.canonical&mask + if width == 4 && word == 8 { + value.gp[index] |= 240 + value.canonicalGP[index] |= 240 + } + } + if saved.bp != nil { + if flags || ins.Args[0].Reg != BP { + return false // A saved frame pointer cannot become numeric output. + } + state.bp = *saved.bp + } else if !flags && ins.Args[0].Reg == BP { + return false + } + if saved.gOriginal { + if flags || word != 8 || width != word || ins.Args[0].Reg != "R14" { + return false // The saved opaque g never becomes an ordinary GP value. + } + state.gOriginal = true + delete(state.callers, "R14") + delete(state.known, "R14") + } else if !flags && x86SourceGRegister(ins.Args[0].Reg) { + state.gOriginal = false + } + if state.callers != nil && !flags && ins.Args[0].Reg != BP { + root, valid := x86CallerGP(ins.Args[0].Reg) + if !valid { + return false + } + if _, pointer := state.callers[root]; pointer && width != word { + return false + } + delete(state.callers, root) + if saved.caller != nil { + if width != word || ins.Args[0].Reg != root { + return false + } + state.callers[root] = *saved.caller + } + if saved.gOriginal { + delete(state.known, root) + } else if width == word { + state.known[root] = saved.known + } else { + mask := x86CallerByteMask(ins.Args[0].Reg, width) + state.known[root] = state.known[root]&^mask | saved.known&mask + } + } + state.cells = state.cells[:len(state.cells)-1] + state.sp += width + return true +} + +// Source SP arithmetic operates on a Go stack address, not our LLVM private +// allocation's address. Its flags cannot be assumed equivalent. This bounded +// proof permits numerical redefinitions and matching FLAGS save/restore, but +// never exposes incomplete FLAGS as scalar data or invents incoming flags. +func x86SourceStackFlagsStep(ins Instr, state *x86SourceStackState) bool { + return x86SourceStackFlagsStepForArch(ins, state, "amd64") +} + +func x86SourceStackFlagsStepForArch(ins Instr, state *x86SourceStackState, arch string) bool { + if form, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch); known { + return err == nil && x86SourceStackNumericFlagsRead(state, x86ByteConditionFlags(form.condition)) + } + op := Op(normalizeInstructionOpcode(ins.Op)) + if condition, conditionalSet := x86ConditionalSetConditions[op]; conditionalSet { + return x86SourceStackNumericFlagsRead(state, x86ByteConditionFlags(condition)) // SETcc preserves flags. + } + if effect, known, err := x86TypedCounterControlByteEffect(ins.Op, ins, arch); known { + if err != nil || state.callers == nil && state.scalarResult == nil { + return false // A physical stack identity alone proves no numerical counter. + } + // The existing caller or scalar-result transfer owns every NUM read/write. + // Zero FLAGS reads require no producer; no status bit is defined here. + return x86SourceStackNumericFlagMask(state)&effect.flagsRead == effect.flagsRead + } + if isAMD64ConditionalBranch(op) { + return isAMD64CounterZeroBranch(op) || x86SourceStackNumericFlagsRead(state, x86ByteBranchFlags(op)) + } + if state.callers != nil { + if effect, known, err := x86TypedGPByteEffect(ins.Op, ins, arch); known { + mask := x86SourceStackNumericFlagMask(state) + if err != nil || mask&effect.flagsRead != effect.flagsRead { + return false + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + x86SourceStackSetNumericFlags(state, 0) + return true // Source address arithmetic never defines numerical flags. + } + } + x86SourceStackSetNumericFlags(state, mask&^effect.flagsKilled|effect.flagsDefined) + return true + } + } + name := string(op) + if len(name) < 2 { + return true + } + base := name[:len(name)-1] + switch base { + case "ADC", "SBB": + return state.numericFlags + case "ADD", "SUB", "AND", "OR", "XOR", "CMP", "TEST", "NEG": + x86SourceStackSetNumericFlags(state, x86ByteFlagsAll) + for _, operand := range ins.Args { + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + x86SourceStackSetNumericFlags(state, 0) + } + } + case "SHL", "SHR", "SAR": + // Masked zero counts preserve prior flags; OF is undefined for larger + // counts. They need a per-bit contract before serving as a new witness. + x86SourceStackSetNumericFlags(state, 0) + } + return true +} + +func x86SourceStackAddressStep(ins *Instr, state *x86SourceStackState, nativeMove Op, suffix string, word int64) bool { + arch := "amd64" + if word == 4 { + arch = "386" + } + op := Op(normalizeInstructionOpcode(x86SourceMoveInstruction(*ins, arch).Op)) + lea := x86LEAWidth(ins.Op, arch) != 0 + if lea { + if _, err := x86LEAOperandWidth(*ins, arch); err != nil { + return false + } + } + if form, recognized, err := x86ScalarExtensionProperties(op, *ins, arch); recognized { + if err != nil || x86RawReturnStackRegister(form.result) || + form.source.Kind == OpReg && x86RawReturnStackRegister(form.source.Reg) { + return false + } + } + if len(ins.Args) == 2 && op == nativeMove && ins.Args[0].Kind == OpReg && ins.Args[1].Kind == OpReg && + x86RawReturnStackRegister(ins.Args[0].Reg) && x86RawReturnStackRegister(ins.Args[1].Reg) { + source, destination := ins.Args[0].Reg, ins.Args[1].Reg + if source != SP && source != BP || destination != SP && destination != BP { + return false + } + value := state.bp + if source == SP { + value = x86SourceStackAddress{offset: state.sp} + } + if destination == BP { + state.bp = value + return true + } + if value.original { + return false + } + state.sp = value.offset + return true + } + if len(ins.Args) == 2 && ins.Args[1].Kind == OpReg && ins.Args[1].Reg == SP { + if ins.Args[0].Kind == OpImm && ins.Args[0].ImmRaw == "" { + value := int64(ins.Args[0].Imm) + if word == 4 { + value = int64(int32(value)) // Raw 386 immediates retain unsigned bits. + } + if value < -x86SourceStackLimit || value > x86SourceStackLimit { + return false + } + switch op { + case Op("ADD" + suffix): + state.sp += value + return true + case Op("SUB" + suffix): + state.sp -= value + return true + case Op("AND" + suffix): + alignment := -value + if alignment < 1 || alignment > word || alignment&(alignment-1) != 0 { + return false // The Go entry contract proves native-word alignment only. + } + state.sp &= value + return true + } + } + if op == Op("LEA"+suffix) && ins.Args[0].Kind == OpMem { + memory := ins.Args[0].Mem + if memory.Index != "" || memory.Sym != "" || memory.Segment != "" || + memory.Off < -x86SourceStackLimit || memory.Off > x86SourceStackLimit || ins.x86AddressBits != 0 { + return false + } + base := state.sp + if memory.Base == BP && !state.bp.original { + base = state.bp.offset + } else if memory.Base != SP { + return false + } + state.sp = base + memory.Off + return true + } + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + return false // No implicit narrowing, arithmetic, or address escape. + } + if lea && operand.Kind == OpMem && x86RawReturnStackMemory(operand.Mem) { + return false // An owned stack address may not escape into an ordinary GP. + } + } + return true +} + +// Source pushes and pops must use the same byte-addressable SP as MOV/LEA/ +// alignment. The legacy amd64 virtual value stack is intentionally unchanged +// for sources without this complete source-stack proof. +func (c *amd64Ctx) pushSourceStackValue(typ LLVMType, width int64, value string) error { + sp, err := c.loadReg(SP) + if err != nil { + return err + } + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i64 %s, %d\n", next, sp, width) + if err := c.storeRegUnchecked(SP, "%"+next); err != nil { + return err + } + ptr := c.ptrFromAddrI64("%" + next) + fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", typ, value, ptr) + return nil +} + +func (c *amd64Ctx) popSourceStackValue(typ LLVMType, width int64) (string, error) { + sp, err := c.loadReg(SP) + if err != nil { + return "", err + } + ptr := c.ptrFromAddrI64(sp) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", value, typ, ptr) + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %d\n", next, sp, width) + if err := c.storeRegUnchecked(SP, "%"+next); err != nil { + return "", err + } + return "%" + value, nil +} diff --git a/amd64_raw_return_source_stack_runtime_test.go b/amd64_raw_return_source_stack_runtime_test.go new file mode 100644 index 000000000..6a17adf5a --- /dev/null +++ b/amd64_raw_return_source_stack_runtime_test.go @@ -0,0 +1,287 @@ +package plan9asm + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +type x86SourceStackRuntimeCase struct { + name string + width int + body string + flags bool +} + +func x86SourceStackRuntimeCases(arch string) []x86SourceStackRuntimeCase { + word, suffix, rawMove := 8, "Q", "WORD $0x8948\nBYTE $0xe5" + rawRestore := "WORD $0x8948\nBYTE $0xec" + rawAlign := "LONG $0xf8e48348" + if arch == "386" { + word, suffix, rawMove = 4, "L", "WORD $0xe589" + rawRestore, rawAlign = "WORD $0xec89", "WORD $0xe483\nBYTE $0xfc" + } + var cases []x86SourceStackRuntimeCase + for _, width := range []int{8, 16, 32, 64} { + if arch == "386" && width == 64 { + continue + } + move := map[int]string{8: "MOVB", 16: "MOVW", 32: "MOVL", 64: "MOVQ"}[width] + cases = append(cases, x86SourceStackRuntimeCase{ + name: fmt.Sprintf("Physical%d", width), width: width, + body: fmt.Sprintf("%s %d(SP),AX\nADD%s $7,AX\n%s AX,%d(SP)\nBYTE $0xc3", move, word, suffix, move, 2*word), + }) + } + finish := fmt.Sprintf("ADD%s $7,AX\nMOV%s AX,ret+%d(FP)\nBYTE $0xc3", suffix, suffix, word) + start := "MOV" + suffix + " x+0(FP),AX\n" + cases = append(cases, + x86SourceStackRuntimeCase{name: "BalancedRaw", width: word * 8, + body: start + "BYTE $0x55\n" + rawMove + "\n" + rawAlign + "\n" + rawRestore + "\nBYTE $0x5d\n" + finish}, + x86SourceStackRuntimeCase{name: "BalancedNamed", width: word * 8, + body: start + "PUSH" + suffix + " BP\nMOV" + suffix + " SP,BP\nPUSH" + suffix + " AX\nMOV" + suffix + " $0,AX\nPOP" + suffix + " AX\nMOV" + suffix + " BP,SP\nPOP" + suffix + " BP\n" + finish}, + x86SourceStackRuntimeCase{name: "WordData", width: word * 8, + body: start + "PUSHW AX\nMOVW $0,AX\nPOPW AX\n" + finish}, + x86SourceStackRuntimeCase{name: "WordImmediate", width: word * 8, + body: start + "PUSHW $65530\nPOPW AX\n" + finish}, + x86SourceStackRuntimeCase{name: "Branch", width: word * 8, + body: start + "CMP" + suffix + " AX,$1\nJE done\nADD" + suffix + " $0,BX\ndone:\n" + finish}, + x86SourceStackRuntimeCase{name: "FlagsStack", width: word * 8, flags: true, + body: start + "CMP" + suffix + " AX,$1\nPUSHF" + suffix + "\n" + + fmt.Sprintf("SUB%s $%d,SP\nADD%s $%d,SP\n", suffix, word, suffix, word) + + "POPF" + suffix + "\nADC" + suffix + " $0,AX\n" + finish}, + ) + for _, flagSuffix := range []string{"W", suffix} { + cases = append(cases, x86SourceStackRuntimeCase{ + name: "Flags" + flagSuffix, width: word * 8, flags: true, + body: start + "CMP" + suffix + " AX,$1\nPUSHF" + flagSuffix + "\nADD" + suffix + " $0,BX\nPOPF" + flagSuffix + "\nADC" + suffix + " $0,AX\n" + finish, + }) + } + return cases +} + +func x86SourceStackRuntimeSource(arch string) (string, map[string]FuncSig) { + var source strings.Builder + sigs := map[string]FuncSig{} + for _, test := range x86SourceStackRuntimeCases(arch) { + word, move := int64(8), "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + ty := map[int]LLVMType{8: I8, 16: I16, 32: I32, 64: I64}[test.width] + text, sig, _ := x86SourceStackTypedFixture(arch, fmt.Sprintf("uint%d", test.width), ty, word, move, test.body) + source.WriteString(strings.ReplaceAll(text, "·Value", "·"+test.name)) + sig.Name = test.name + sigs[test.name] = sig + } + word := int64(8) + if arch == "386" { + word = 4 + } + text, sig, _ := x86SourceStackTypedFixture(arch, "bool", I1, word, "MOVB", + "MOVB @WORD(SP),AX\nMOVB AX,@RESULT(SP)\nBYTE $0xc3") + source.WriteString(strings.ReplaceAll(text, "·Value", "·PhysicalBool")) + sig.Name = "PhysicalBool" + sigs[sig.Name] = sig + // Preserve the original identity kernel above byte-for-byte. This sibling + // observes a physical result byte before its final logical bool conversion: + // prematurely truncating the intermediate 2 to i1 would return false. + text, sig, _ = x86SourceStackTypedFixture(arch, "bool", I1, word, "MOVB", + "MOVB $2,@RESULT(SP)\nMOVB @RESULT(SP),AX\nSHRB $1,AX\nANDB $1,AX\nMOVB AX,@RESULT(SP)\nBYTE $0xc3") + source.WriteString(strings.ReplaceAll(text, "·Value", "·PhysicalBoolIntermediate")) + sig.Name = "PhysicalBoolIntermediate" + sigs[sig.Name] = sig + text, sig, _ = x86SourceStackTypedFixture(arch, "bool", I1, word, "MOVB", + "MOVB $2,ret+@WORD(FP)\nMOVB ret+@WORD(FP),AX\nSHRB $1,AX\nANDB $1,AX\nMOVB AX,ret+@WORD(FP)\nBYTE $0xc3") + source.WriteString(strings.ReplaceAll(text, "·Value", "·PhysicalBoolNamedFP")) + sig.Name = "PhysicalBoolNamedFP" + sigs[sig.Name] = sig + return source.String(), sigs +} + +func x86SourceStackGoOracle(arch string) string { + var goSource strings.Builder + goSource.WriteString("package sourceframe\nimport \"testing\"\n") + for _, test := range x86SourceStackRuntimeCases(arch) { + fmt.Fprintf(&goSource, "func %s(x uint%d) uint%d\n", test.name, test.width, test.width) + } + goSource.WriteString("func PhysicalBool(x bool) bool\n") + goSource.WriteString("func PhysicalBoolIntermediate(x bool) bool\n") + goSource.WriteString("func PhysicalBoolNamedFP(x bool) bool\n") + goSource.WriteString("func TestNumerical(t *testing.T) {\n") + for _, test := range x86SourceStackRuntimeCases(arch) { + fmt.Fprintf(&goSource, ` + for iteration := 0; iteration < 64; iteration++ { + for _, input := range []uint%d{0, 1, 17, 253, ^uint%d(0)-3} { + want := input + 7 +`, test.width, test.width) + if test.flags { + goSource.WriteString("\t\t\tif input == 0 { want++ }\n") + } + if test.name == "WordImmediate" { + fmt.Fprintf(&goSource, "\t\t\twant = ((input & ^uint%d(65535)) | 65530) + 7\n", test.width) + } + fmt.Fprintf(&goSource, ` + canary := [4]uint64{11, 23, 37, 41} + if got := %s(input); got != want || canary != [4]uint64{11, 23, 37, 41} { + t.Fatalf("%s(%%#x)=%%#x, want %%#x; canary=%%v", input, got, want, canary) + } + } + } +`, test.name, test.name) + } + goSource.WriteString("\tif PhysicalBool(false) || !PhysicalBool(true) { t.Fatal(\"bool byte slot\") }\n") + goSource.WriteString("\tcanary := [4]uint64{11,23,37,41}\n\tif !PhysicalBoolIntermediate(false) || !PhysicalBoolIntermediate(true) || !PhysicalBoolNamedFP(false) || !PhysicalBoolNamedFP(true) || canary != [4]uint64{11,23,37,41} { t.Fatal(\"physical/named intermediate bool byte/canary\") }\n}\n") + return goSource.String() +} + +func x86SourceStackActualGoNumerical(t *testing.T, arch string, runner []string) { + t.Helper() + dir := t.TempDir() + source, _ := x86SourceStackRuntimeSource(arch) + for name, text := range map[string]string{ + "go.mod": "module example.invalid/sourceframeoracle\n\ngo 1.20\n", + "value_" + arch + ".s": source, + "value_test.go": x86SourceStackGoOracle(arch), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(text), 0600); err != nil { + t.Fatal(err) + } + } + args := []string{"test", "-p=2", "-vet=off", "-count=1"} + if len(runner) != 0 { + args = append(args, "-exec", strings.Join(runner, " ")) + } + args = append(args, ".") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + env := append(os.Environ(), "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local", "GOARCH="+arch, "CGO_ENABLED=0", "GOMAXPROCS=2") + command := exec.CommandContext(ctx, "go", args...) + command.Dir, command.Env, command.WaitDelay = dir, env, 2*time.Second + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("actual Go %s original source numerical oracle: %v\n%s", arch, err, output) + } +} + +func x86SourceStackCOracle(arch string) string { + var source strings.Builder + source.WriteString("#include \n") + source.WriteString("#include \nextern bool PhysicalBool(bool);\nextern bool PhysicalBoolIntermediate(bool);\n") + source.WriteString("extern bool PhysicalBoolNamedFP(bool);\n") + for _, test := range x86SourceStackRuntimeCases(arch) { + fmt.Fprintf(&source, "extern uint%d_t %s(uint%d_t);\n", test.width, test.name, test.width) + } + source.WriteString("int main(void) {\n") + for _, test := range x86SourceStackRuntimeCases(arch) { + fmt.Fprintf(&source, ` + for (unsigned iteration = 0; iteration < 64; iteration++) { + const uint%d_t inputs[] = {0, 1, 17, 253, (uint%d_t)-4}; + for (unsigned index = 0; index < sizeof(inputs)/sizeof(inputs[0]); index++) { + const uint%d_t input = inputs[index]; + uint%d_t want = input + 7; +`, test.width, test.width, test.width, test.width) + if test.flags { + source.WriteString("\t\t\tif (input == 0) want++;\n") + } + if test.name == "WordImmediate" { + fmt.Fprintf(&source, "\t\t\twant = (input & ~(uint%d_t)65535) | 65530;\n\t\t\twant += 7;\n", test.width) + } + fmt.Fprintf(&source, ` + volatile uint64_t canary = UINT64_C(0x123456789abcdef0); + if (%s(input) != want || canary != UINT64_C(0x123456789abcdef0)) return 1; + } + } +`, test.name) + } + source.WriteString("\tif (PhysicalBool(false) || !PhysicalBool(true)) return 2;\n") + source.WriteString("\tvolatile uint64_t boolCanary = UINT64_C(0x123456789abcdef0);\n\tif (!PhysicalBoolIntermediate(false) || !PhysicalBoolIntermediate(true) || !PhysicalBoolNamedFP(false) || !PhysicalBoolNamedFP(true) || boolCanary != UINT64_C(0x123456789abcdef0)) return 3;\n\treturn 0;\n}\n") + return source.String() +} + +func TestX86RawReturnBoundSourceStackRuntimeForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + source, sigs := x86SourceStackRuntimeSource(target.arch) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "source-stack-forms.ll", "source-stack-forms.o", ir) + } +} + +func TestCrossLinuxRuntimeMatrixX86RawSourceStackNumerical(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("x86 source-stack numerical oracle runs in the required Linux/amd64 cross-runtime gate") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + t.Run("amd64", func(t *testing.T) { + x86SourceStackActualGoNumerical(t, "amd64", nil) + source, sigs := x86SourceStackRuntimeSource("amd64") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "source_stack64", triple, ir, x86SourceStackCOracle("amd64"), runner) + }) + // This required native owner must execute the original physical 386 bool + // bytes too, even when the broader ARM/QEMU matrix is not selected. Explicit + // cross execution retains its old dynamic guest transport independently. + if os.Getenv("PLAN9ASM_CROSS_EXEC") == "1" || runtime.GOOS == "linux" && runtime.GOARCH == "amd64" { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("PLAN9ASM_CROSS_EXEC source-stack matrix requires Linux/amd64") + } + t.Run("386", func(t *testing.T) { + runner = nil + compiler := []string{"i686-linux-gnu-gcc", "-static", "-no-pie"} + if os.Getenv("PLAN9ASM_CROSS_EXEC") == "1" { + runner = []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"} + compiler = []string{"i686-linux-gnu-gcc", "-no-pie"} + } + x86SourceStackActualGoNumerical(t, "386", runner) + source, sigs := x86SourceStackRuntimeSource("386") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, + "source_stack32", "i386-unknown-linux-gnu", ir, x86SourceStackCOracle("386"), runner) + }) + } +} diff --git a/amd64_raw_return_source_stack_test.go b/amd64_raw_return_source_stack_test.go new file mode 100644 index 000000000..ee4a777ec --- /dev/null +++ b/amd64_raw_return_source_stack_test.go @@ -0,0 +1,389 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86SourceStackNumericalFixture(arch, body string) (string, FuncSig, GoPackage) { + word, move, typ, llvmType := int64(8), "MOVQ", "uint64", I64 + if arch == "386" { + word, move, typ, llvmType = 4, "MOVL", "uint32", I32 + } + return x86SourceStackTypedFixture(arch, typ, llvmType, word, move, body) +} + +func x86SourceStackTypedFixture(arch, typ string, llvmType LLVMType, word int64, move, body string) (string, FuncSig, GoPackage) { + body = strings.NewReplacer("@MOV", move, "@WORD", fmt.Sprint(word), "@RESULT", fmt.Sprint(2*word)).Replace(body) + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\n%s\n", 2*word, body) + sig := FuncSig{Name: "sourceframe.Value", Args: []LLVMType{llvmType}, Ret: llvmType, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: llvmType, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: word, Type: llvmType, Index: 0, Field: -1}}, + }} + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "value.go", "package sourceframe\nfunc Value(x "+typ+") "+typ+"\nfunc callee()\n", 0) + if err != nil { + panic(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("sourceframe", fset, []*ast.File{file}, nil) + if err != nil { + panic(err) + } + return source, sig, GoPackage{Path: "sourceframe", Types: pkg, Syntax: []*ast.File{file}} +} + +func x86SourceStackOptions(arch, triple string, sig FuncSig) Options { + return Options{Goarch: arch, TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "sourceframe.", 1) }, + Sigs: map[string]FuncSig{sig.Name: sig}, + } +} + +func TestX86RawReturnBoundSourceStackNumerical(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + push, pop, arithmetic, address, align := "PUSHQ", "POPQ", "ADDQ", "LEAQ", 8 + if target.arch == "386" { + push, pop, arithmetic, address, align = "PUSHL", "POPL", "ADDL", "LEAL", 4 + } + for _, fixture := range []struct{ name, body string }{ + {"physical", "@MOV @WORD(SP),AX\n" + arithmetic + " $7,AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3"}, + {"balanced", "@MOV x+0(FP),AX\n" + push + " BP\n@MOV SP,BP\nAND" + strings.TrimPrefix(arithmetic, "ADD") + fmt.Sprintf(" $-%d,SP\n", align) + arithmetic + " $7,AX\n@MOV BP,SP\n" + pop + " BP\n@MOV AX,ret+@WORD(FP)\nBYTE $0xc3"}, + {"nested", "@MOV x+0(FP),AX\n" + push + " BP\n@MOV SP,BP\n" + push + " BX\n" + push + " AX\n" + address + " -2*@WORD(BP),SP\n" + pop + " AX\n" + pop + " BX\n@MOV BP,SP\n" + pop + " BP\n" + arithmetic + " $7,AX\n@MOV AX,ret+@WORD(FP)\nBYTE $0xc3"}, + } { + t.Run(target.triple+"/"+fixture.name, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, fixture.body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + ir, err := Translate(file, opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "source-frame.ll", "source-frame.o", ir) + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + module.Dispose() + ctx.Dispose() + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + translation.Module.Dispose() + ctx.Dispose() + }) + } + } +} + +func TestX86RawReturnBoundSourceStackRetainsContext(t *testing.T) { + for _, target := range x86RawReturnTargets { + push, pop, sub, align := "PUSHQ", "POPQ", "SUBQ", 16 + if target.arch == "386" { + push, pop, sub, align = "PUSHL", "POPL", "SUBL", 8 + } + for _, body := range []string{ + "@MOV @WORD(SP),AX\n@MOV AX,0(SP)\nBYTE $0xc3", + "@MOV @WORD(SP),AX\n@MOV AX,@WORD-1(SP)\nBYTE $0xc3", + push + " AX\nBYTE $0xc3", + push + " BP\n@MOV SP,BP\n" + pop + " AX\nBYTE $0xc3", + push + " BP\n@MOV SP,BP\nAND" + strings.TrimPrefix(sub, "SUB") + fmt.Sprintf(" $-%d,SP\n", align) + "@MOV BP,SP\n" + pop + " BP\nBYTE $0xc3", + push + " BP\n@MOV SP,BP\n@MOV BP,AX\n@MOV BP,SP\n" + pop + " BP\nBYTE $0xc3", + "LEA" + strings.TrimPrefix(sub, "SUB") + " (SP),AX\nBYTE $0xc3", + "@MOV @WORD(SP),AX\n@MOV AX,(BX)\nBYTE $0xc3", + "CALL ·callee(SB)\nBYTE $0xc3", + "@MOV $0,SP\nBYTE $0xc3", + "CMP" + strings.TrimPrefix(sub, "SUB") + " $0,AX\nJE alternate\n" + push + " AX\nalternate:\nBYTE $0xc3", + } { + t.Run(target.triple+"/"+body, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, body) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + if _, err := Translate(file, opt); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source continuation became callable: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module accepted unproved source continuation: %v", err) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if err == nil { + translation.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("declaration API accepted unproved source continuation: %v", err) + } + }) + } + } +} + +func TestX86RawReturnBoundSourceStackGoFPProducerOffsets(t *testing.T) { + for _, target := range x86RawReturnTargets { + word, push, pop, move, rawPush, rawPop := int64(8), "PUSHQ", "POPQ", "MOVQ", "BYTE $0x50", "BYTE $0x59" + if target.arch == "386" { + word, push, pop, move = 4, "PUSHL", "POPL", "MOVL" + } + for _, test := range []struct { + name, body string + accepted bool + }{ + {"named FP after named push", push + " CX\n" + move + " x+0(FP),AX\n" + pop + " CX\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", true}, + {"raw push physical shifted argument", rawPush + "\n" + move + " @RESULT(SP),AX\n" + rawPop + "\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", true}, + {"raw push cannot shift Go FP", rawPush + "\n" + move + " x+0(FP),AX\n" + rawPop + "\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", false}, + {"partial scalar access", fmt.Sprintf("MOVB %d(SP),AX\n%s AX,ret+@WORD(FP)\nBYTE $0xc3", word, move), false}, + {"stack index", move + " @WORD(SP)(CX*1),AX\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", false}, + {"original BP is not argument frame", move + " @WORD(BP),AX\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", false}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, test.body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + _, err = Translate(file, opt) + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("source ABI0 transport accepted=%t: %v", test.accepted, err) + } + ctx := llvm.NewContext() + module, moduleErr := TranslateModuleInContext(ctx, file, opt) + if moduleErr == nil { + module.Dispose() + } + ctx.Dispose() + if test.accepted && moduleErr != nil || !test.accepted && !errors.Is(moduleErr, ErrProbeNeedsContext) { + t.Fatalf("module ABI0 transport accepted=%t: %v", test.accepted, moduleErr) + } + ctx = llvm.NewContext() + translation, bindingErr := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if bindingErr == nil { + translation.Module.Dispose() + } + ctx.Dispose() + if test.accepted && bindingErr != nil || !test.accepted && !errors.Is(bindingErr, ErrProbeNeedsContext) { + t.Fatalf("declaration ABI0 transport accepted=%t: %v", test.accepted, bindingErr) + } + }) + } + } +} + +func TestX86RawReturnBoundSourceStackScalarMoveWidths(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + word, add := int64(8), "ADDQ" + if target.arch == "386" { + word, add = 4, "ADDL" + } + for _, typ := range []struct { + goType string + llvm LLVMType + move string + }{ + {"bool", I1, "MOVB"}, + {"uint8", I8, "MOVB"}, {"uint16", I16, "MOVW"}, {"uint32", I32, "MOVL"}, {"uint64", I64, "MOVQ"}, + } { + if target.arch == "386" && typ.llvm == I64 { + continue // Go's 386 GP ABI transports this value as two slots. + } + t.Run(target.triple+"/"+typ.goType, func(t *testing.T) { + body := "@MOV @WORD(SP),AX\n" + add + " $7,AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3" + if typ.llvm == I1 { + body = "@MOV @WORD(SP),AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3" + } + source, sig, pkg := x86SourceStackTypedFixture(target.arch, typ.goType, typ.llvm, word, typ.move, body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "scalar-sp.ll", "scalar-sp.o", ir) + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + module.Dispose() + ctx.Dispose() + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + translation.Module.Dispose() + ctx.Dispose() + }) + } + } +} + +func TestX86RawReturnBoundSourceStackDoesNotTrustChangedSignature(t *testing.T) { + for _, target := range x86RawReturnTargets { + source, sig, _ := x86SourceStackNumericalFixture(target.arch, "@MOV @WORD(SP),AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + prepared, err := NormalizeRawFileForTranslation(file, target.arch) + if err != nil { + t.Fatal(err) + } + for _, change := range []string{"no frame", "wrong type", "wrong offset", "wrong index", "custom registers", "pointer", "wrong size"} { + t.Run(target.triple+"/"+change, func(t *testing.T) { + changed := sig + changed.Frame.Params = append([]FrameSlot(nil), sig.Frame.Params...) + changed.Frame.Results = append([]FrameSlot(nil), sig.Frame.Results...) + changed.Args = append([]LLVMType(nil), sig.Args...) + switch change { + case "no frame": + changed.Frame = FrameLayout{} + case "wrong type": + changed.Frame.Params[0].Type = I8 + case "wrong offset": + changed.Frame.Params[0].Offset = 1 + case "wrong index": + changed.Frame.Params[0].Index = 1 + case "custom registers": + changed.ArgRegs = []Reg{AX} + case "pointer": + changed.Args[0], changed.Frame.Params[0].Type = Ptr, Ptr + case "wrong size": + changed.Frame.Results[0].Offset = 1 << 20 + } + if _, err := Translate(prepared, x86SourceStackOptions(target.arch, target.triple, changed)); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("prepartition metadata trusted a changed contract: %v", err) + } + }) + } + } +} + +func TestX86RawReturnBoundSourceStackFlagsNeedNumericalProvenance(t *testing.T) { + for _, target := range x86RawReturnTargets { + suffix, word := "Q", 8 + if target.arch == "386" { + suffix, word = "L", 4 + } + start := "@MOV @WORD(SP),AX\n" + finish := "@MOV AX,@RESULT(SP)\nBYTE $0xc3" + movement := fmt.Sprintf("SUB%s $%d,SP\nADD%s $%d,SP\n", suffix, word, suffix, word) + for _, test := range []struct { + name, body string + accepted bool + }{ + {"entry carry", start + "ADC" + suffix + " $0,AX\n" + finish, false}, + {"SP-derived flags", start + movement + "JS done\ndone:\n" + finish, false}, + {"flags exposed as scalar", start + "CMP" + suffix + " AX,$1\nPUSHF" + suffix + "\nPOP" + suffix + " AX\n" + finish, false}, + {"scalar replaces flags", start + "PUSH" + suffix + " AX\nPOPF" + suffix + "\nADC" + suffix + " $0,AX\n" + finish, false}, + {"different flag origins join", start + "CMP" + suffix + " AX,$1\nJE done\n" + movement + "done:\nADC" + suffix + " $0,AX\n" + finish, false}, + {"saved numeric flags restored", start + "CMP" + suffix + " AX,$1\nPUSHF" + suffix + "\n" + movement + "POPF" + suffix + "\nADC" + suffix + " $0,AX\n" + finish, true}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, test.body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, x86SourceStackOptions(target.arch, target.triple, sig)) + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("numerical flags contract accepted=%t: %v", test.accepted, err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module flags contract accepted=%t: %v", test.accepted, err) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if err == nil { + translation.Module.Dispose() + } + ctx.Dispose() + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("declaration flags contract accepted=%t: %v", test.accepted, err) + } + }) + } + } +} + +func TestX86RawReturnBoundSourceStackDoesNotTrustEditedRawProducer(t *testing.T) { + for _, target := range x86RawReturnTargets { + source, sig, _ := x86SourceStackNumericalFixture(target.arch, + "BYTE $0x50\n@MOV @RESULT(SP),AX\nBYTE $0x59\n@MOV AX,ret+@WORD(FP)\nBYTE $0xc3") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + prepared, err := NormalizeRawFileForTranslation(file, target.arch) + if err != nil { + t.Fatal(err) + } + for index := range prepared.Funcs[0].Instrs { + ins := &prepared.Funcs[0].Instrs[index] + if spec, stack := amd64StackWidthSpecs[ins.Op]; stack && spec.mode == amd64StackPushValue { + ins.Args = []Operand{{Kind: OpReg, Reg: BX}} + break + } + } + if _, err := Translate(prepared, x86SourceStackOptions(target.arch, target.triple, sig)); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s edited raw producer kept an original-Go prologue certificate: %v", target.triple, err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, prepared, x86SourceStackOptions(target.arch, target.triple, sig)) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s module trusted edited raw producer: %v", target.triple, err) + } + } +} diff --git a/amd64_raw_rorx.go b/amd64_raw_rorx.go new file mode 100644 index 000000000..5d86b79d1 --- /dev/null +++ b/amd64_raw_rorx.go @@ -0,0 +1,86 @@ +package plan9asm + +import "fmt" + +// decodedX86RORXInstruction recognizes the complete BMI2 RORXL/RORXQ Yml, +// Yrl, imm8 family, including register and every source-layout-safe ModRM/SIB +// memory form. x/arch v0.14 reports the VEX.0F3A encoding as an unknown AVX +// opcode, so it must be recovered before generic decoding. +func decodedX86RORXInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86RORXInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86RORXInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 3 || code[i+3] != 0xf0 { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + vexExt, vexByte := code[i+1], code[i+2] + if vexByte&3 != 3 || vexByte&4 != 0 || vexByte&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("reserved RORX VEX prefix") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if len(code) <= i+4 { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + rExt := int(^vexExt>>7) & 1 + xExt := int(^vexExt>>6) & 1 + bExt := int(^vexExt>>5) & 1 + width64 := vexByte&0x80 != 0 + modRM := code[i+4] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + destination, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid destination register %d", destinationNumber) + } + source, consumed, err := memory.scalar(code[i+4:], mode, bExt, xExt, segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + if len(code) <= i+4+consumed { + return Instr{}, 0, true, fmt.Errorf("missing imm8") + } + immediate := int64(code[i+4+consumed]) + op := Op("RORXL") + if width64 && mode == 64 { + op = "RORXQ" + } + instruction = Instr{ + Op: op, + Args: []Operand{ + {Kind: OpImm, Imm: immediate}, + source, + {Kind: OpReg, Reg: destination}, + }, + Raw: fmt.Sprintf("%s $%d, %s, %s", op, immediate, source.String(), destination), + } + return instruction, i + 5 + consumed, true, nil +} diff --git a/amd64_raw_rotate_variable.go b/amd64_raw_rotate_variable.go new file mode 100644 index 000000000..0f6564d27 --- /dev/null +++ b/amd64_raw_rotate_variable.go @@ -0,0 +1,85 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86RawVariableRotateInstruction covers the four EVEX per-lane +// variable rotates in Go's _yvblendmpd table: left/right, D/Q, X/Y/Z, +// optional mask, zeroing, and scalar-memory broadcast. +func decodedX86RawVariableRotateInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || + (p.opcode != 0x14 && p.opcode != 0x15) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX variable rotate: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.addressOverride || p.vectorLength > 2 { + return fail("invalid EVEX fixed, address, or vector-length bits") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if mode == 32 && p.mask != 0 { + return fail("386 mask form exceeds the Go assembler frontend's operand limit") + } + modRM := code[p.modRM] + if p.broadcast && modRM>>6 == 3 { + return fail("EVEX.b broadcast requires memory") + } + destinationNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && (p.upper >= 8 || destinationNumber >= 8 || + modRM>>6 == 3 && int(modRM&7)+p.b*8+p.x*16 >= 8) { + return fail("extended vector register in 32-bit mode") + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + width := 16 << p.vectorLength + laneBytes := 4 + if p.w { + laneBytes = 8 + } + disp8Scale := width + if p.broadcast { + disp8Scale = laneBytes + } + first, consumed, err := decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, vector, disp8Scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + op := Op("VPRORVD") + if p.opcode == 0x15 { + op = "VPROLVD" + } + if p.w { + op = Op(strings.TrimSuffix(string(op), "D") + "Q") + } + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, destinationNumber))} + args := []Operand{first, second} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_rotate_variable_test.go b/amd64_raw_rotate_variable_test.go new file mode 100644 index 000000000..4aae5cdde --- /dev/null +++ b/amd64_raw_rotate_variable_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVariableRotate(op Op, width, mask, destination, first, second int, zeroing, broadcast, memory bool) []byte { + opcode := byte(0x14) + if strings.HasPrefix(string(op), "VPROL") { + opcode = 0x15 + } + p0 := byte((1-(destination>>3)&1)<<7 | (1-(first>>4)&1)<<6 | + (1-(first>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 2) + p1 := byte(((^second)&15)<<3 | 0x05) + if strings.HasSuffix(string(op), "Q") { + p1 |= 0x80 + } + p2 := byte(width<<5 | (1-((second>>4)&1))<<3 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | (destination&7)<<3 | first&7) + if memory { + modRM = byte(0x40 | (destination&7)<<3) + p0 = byte((1-(destination>>3)&1)<<7 | 0x60 | (1-(destination>>4)&1)<<4 | 2) + } + code := []byte{0x62, p0, p1, p2, opcode, modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawVariableRotateSeBiShogunRegression(t *testing.T) { + // Go's assembler disassembles these bytes as VPROLVD Z0, Z1, Z1. + code := []byte{0x62, 0xf2, 0x75, 0x48, 0x15, 0xc8, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "rotlInt32 AVX512 VPROLVD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPROLVD" || + decoded[0].Args[0].Reg != "Z0" || decoded[0].Args[1].Reg != "Z1" || + decoded[0].Args[2].Reg != "Z1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawVariableRotateCompleteGoForms(t *testing.T) { + for _, op := range []Op{"VPROLVD", "VPROLVQ", "VPRORVD", "VPRORVQ"} { + for width := 0; width < 3; width++ { + vector := [...]string{"X", "Y", "Z"}[width] + for _, form := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + destination int + first int + second int + }{ + {name: "register", destination: 2, first: 3, second: 4}, + {name: "high registers", destination: 18, first: 19, second: 20}, + {name: "masked zeroing", mask: 7, zeroing: true, destination: 2, first: 3, second: 4}, + {name: "memory", memory: true, destination: 2, second: 4}, + {name: "broadcast memory", broadcast: true, memory: true, destination: 2, second: 4}, + {name: "broadcast zeroing memory", broadcast: true, memory: true, mask: 5, zeroing: true, destination: 2, second: 4}, + } { + name := fmt.Sprintf("%s/%s/%s", op, vector, form.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawVariableRotate( + op, width, form.mask, form.destination, form.first, form.second, + form.zeroing, form.broadcast, form.memory, + ) + got, length, ok, err := decodedX86RawVariableRotateInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := op + if form.broadcast { + wantOp += ".BCST" + } + if form.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[1].String() != fmt.Sprintf("%s%d", vector, form.second) || + got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s%d", vector, form.destination) { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + if !form.memory && got.Args[0].String() != fmt.Sprintf("%s%d", vector, form.first) { + t.Fatalf("decode %x first source = %s", code, got.Args[0]) + } + }) + } + } + } +} + +func TestDecodeX86RawVariableRotateRIPLiteral(t *testing.T) { + for _, op := range []Op{"VPROLVD", "VPROLVQ", "VPRORVD", "VPRORVQ"} { + for width := 0; width < 3; width++ { + for _, broadcast := range []bool{false, true} { + code := encodeX86RawVariableRotate(op, width, 1, 2, 0, 4, false, broadcast, false) + code[5] = 0x15 // RIP-relative first source, destination 2. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << width + if broadcast { + literalWidth = 4 + if strings.HasSuffix(string(op), "Q") { + literalWidth = 8 + } + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/width%d/broadcast%t", op, width, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || len(got[0].x86RIPLiteralData) != literalWidth { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } + } +} + +func TestDecodeX86RawVariableRotateRejectsInvalidForms(t *testing.T) { + base := encodeX86RawVariableRotate("VPROLVD", 2, 0, 2, 3, 4, false, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x10 }, // broadcast requires memory + func(code []byte) { code[3] |= 0x80 }, // zeroing requires a mask + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86RawVariableRotateInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawVariableRotateLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawVariableRotate(SB),$0-0\n") + for _, op := range []Op{"VPROLVD", "VPROLVQ", "VPRORVD", "VPRORVQ"} { + for width := 0; width < 3; width++ { + for _, form := range []struct { + memory, broadcast bool + }{ + {}, {memory: true}, {memory: true, broadcast: true}, + } { + code := encodeX86RawVariableRotate(op, width, 0, 2, 3, 4, false, form.broadcast, form.memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + ir := x86PackedRotateUnboundFormatObject(t, source.String(), target.goarch, target.triple, 36, 0) + compileLLVMToObject(t, llc, target.triple, "raw-variable-rotate.ll", "raw-variable-rotate.o", ir) + }) + } +} diff --git a/amd64_raw_round.go b/amd64_raw_round.go new file mode 100644 index 000000000..7810be8e6 --- /dev/null +++ b/amd64_raw_round.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86RawVEXRoundInstruction implements Go's four VROUND PS/PD/SS/SD +// opcode rows. The packed forms reserve vvvv; scalar forms use it as their +// upper-lane source. Every form has a trailing imm8. +func decodedX86RawVEXRoundInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 3 || p.pp != 1 || p.opcode < 0x08 || p.opcode > 0x0b { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VROUND: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.w || p.addressOverride { + return fail("requires VEX.W0 without address override") + } + scalar := p.opcode >= 0x0a + if scalar && p.vectorLength != 0 || !scalar && p.upper != 0 { + return fail("invalid scalar vector length or packed reserved vvvv") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (destinationNumber >= 8 || p.upper >= 8 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + width := "X" + if p.vectorLength != 0 { + width = "Y" + } + source, consumed, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + immIndex := p.modRM + consumed + if len(code) <= immIndex { + return fail("missing imm8") + } + args := []Operand{{Kind: OpImm, Imm: int64(code[immIndex])}, source} + if scalar { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))}) + op := [...]Op{"VROUNDPS", "VROUNDPD", "VROUNDSS", "VROUNDSD"}[p.opcode-0x08] + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, immIndex + 1, true, nil +} diff --git a/amd64_raw_round_test.go b/amd64_raw_round_test.go new file mode 100644 index 000000000..570fde9b1 --- /dev/null +++ b/amd64_raw_round_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVEXRound(opcode byte, width256 bool, destination, upper, source int, imm byte) []byte { + code := encodeX86VEXPackedWordMultiply(3, opcode, width256, false, destination, upper, source) + return append(code, imm) +} + +func TestDecodedX86RawVEXRoundCompleteFamily(t *testing.T) { + for _, form := range []struct { + opcode byte + op Op + scalar bool + }{ + {0x08, "VROUNDPS", false}, + {0x09, "VROUNDPD", false}, + {0x0a, "VROUNDSS", true}, + {0x0b, "VROUNDSD", true}, + } { + for _, width256 := range []bool{false, true} { + if form.scalar && width256 { + continue + } + for _, source := range []int{0, 7, 8, 15} { + for _, imm := range []byte{0, 9, 255} { + name := fmt.Sprintf("%s/y%t/src%d/imm%d", form.op, width256, source, imm) + t.Run(name, func(t *testing.T) { + code := encodeX86RawVEXRound(form.opcode, width256, 11, 3, source, imm) + if !form.scalar { + code = encodeX86RawVEXRound(form.opcode, width256, 11, 0, source, imm) + } + got, length, matched, err := decodedX86RawVEXRoundInstruction(code, 64) + if err != nil || !matched || length != len(code) || got.Op != form.op || + got.Args[0].Kind != OpImm || got.Args[0].Imm != int64(imm) || + got.Args[1].String() != fmt.Sprintf("%s%d", map[bool]string{false: "X", true: "Y"}[width256], source) { + t.Fatalf("decode %x = %+v length=%d matched=%t err=%v", code, got, length, matched, err) + } + if form.scalar && (len(got.Args) != 4 || got.Args[2].String() != "X3") { + t.Fatalf("scalar upper-lane operand missing: %+v", got) + } + }) + } + } + } + } + // Actual bytes in simd@v1.21.1 arith_avx2_amd64.s. + real := []byte{0xc4, 0xa3, 0x7d, 0x08, 0x04, 0x86, 0x09} + got, length, matched, err := decodedX86RawVEXRoundInstruction(real, 64) + if err != nil || !matched || length != len(real) || got.Op != "VROUNDPS" || + got.Args[1].String() != "0(SI)(R8*4)" || got.Args[2].String() != "Y0" { + t.Fatalf("decode real VROUNDPS %x = %+v length=%d matched=%t err=%v", real, got, length, matched, err) + } +} + +func TestDecodedX86RawVEXRoundRejectsInvalidForms(t *testing.T) { + packed := encodeX86RawVEXRound(0x08, true, 0, 0, 0, 9) + scalar := encodeX86RawVEXRound(0x0a, false, 0, 1, 0, 9) + for name, bad := range map[string][]byte{ + "packed reserved vvvv": func() []byte { + b := append([]byte(nil), packed...) + b[2] &^= 0x08 + return b + }(), + "scalar L1": func() []byte { + b := append([]byte(nil), scalar...) + b[2] |= 0x04 + return b + }(), + "W1": func() []byte { + b := append([]byte(nil), packed...) + b[2] |= 0x80 + return b + }(), + "truncated imm8": packed[:len(packed)-1], + "address override": append([]byte{0x67}, packed...), + } { + t.Run(name, func(t *testing.T) { + if _, _, matched, err := decodedX86RawVEXRoundInstruction(bad, 64); !matched || err == nil { + t.Fatalf("accepted invalid VROUND %x: matched=%t err=%v", bad, matched, err) + } + }) + } +} + +func x86RawVEXRoundConstant(opcode byte, width256 bool) []byte { + code := encodeX86RawVEXRound(opcode, width256, 0, 0, 0, 9) + code[len(code)-2] = 0x05 + imm := code[len(code)-1] + code = append(code[:len(code)-1], 1, 0, 0, 0, imm, 0xc3) + width := 16 + if width256 { + width = 32 + } + if opcode == 0x0a { + width = 4 + } else if opcode == 0x0b { + width = 8 + } + for i := 0; i < width; i++ { + code = append(code, byte(i*17+11)) + } + return code +} + +func TestDecodeX86RawVEXRoundConstantCompleteFamily(t *testing.T) { + for _, opcode := range []byte{0x08, 0x09, 0x0a, 0x0b} { + for _, wide := range []bool{false, true} { + if opcode >= 0x0a && wide { + continue + } + name := fmt.Sprintf("%02x/y%t", opcode, wide) + t.Run(name, func(t *testing.T) { + code := x86RawVEXRoundConstant(opcode, wide) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Args[1].Kind != OpSym || !got[0].x86RIPLiteral || got[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want source-local round constant", code, got) + } + }) + } + } +} + +func TestDecodeX86RawVEXRoundConstantRejectsUnsafeSources(t *testing.T) { + valid := x86RawVEXRoundConstant(0x08, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[5] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[5] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe VROUND RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVEXRoundObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, opcode := range []byte{0x08, 0x09, 0x0a, 0x0b} { + code := x86RawVEXRoundConstant(opcode, opcode < 0x0a) + name := fmt.Sprintf("rawVEXRound%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86RawRoundOriginalModuleObjects(t, file, source.String(), triple, sigs) + compileLLVMToObject(t, llc, triple, "raw-vex-round.ll", "raw-vex-round.o", ir) + }) + } +} diff --git a/amd64_raw_sad.go b/amd64_raw_sad.go new file mode 100644 index 000000000..d8c5e6bba --- /dev/null +++ b/amd64_raw_sad.go @@ -0,0 +1,103 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27 has three vector SAD encodings: VPSADBW is 66/0F/F6 in +// VEX X/Y and EVEX X/Y/Z, VMPSADBW is VEX 66/0F3A/42 with imm8 in X/Y, +// and VDBPSADBW is the corresponding EVEX X/Y/Z immediate form with an +// optional writemask. Legacy PSADBW and MPSADBW use the x86asm path. +func decodedX86RawPackedSADInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || p.pp != 1 { + return Instr{}, 0, false, nil + } + var op Op + withImmediate := false + switch { + case p.mapNumber == 1 && p.opcode == 0xf6: + op = "VPSADBW" + case p.mapNumber == 3 && p.opcode == 0x42 && !p.evex: + op, withImmediate = "VMPSADBW", true + case p.mapNumber == 3 && p.opcode == 0x42 && p.evex: + op, withImmediate = "VDBPSADBW", true + default: + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("%s: %s", op, message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.w && op == "VDBPSADBW" || p.vectorLength > 2 || p.evex && !p.fixed { + return fail("invalid width or EVEX fixed bit") + } + if !p.evex && p.vectorLength == 2 { + return fail("VEX does not encode Z registers") + } + if p.broadcast || p.mask != 0 && op != "VDBPSADBW" || + p.zero && (op != "VDBPSADBW" || p.mask == 0) { + return fail("invalid broadcast, writemask or zeroing form") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended vector register in 32-bit mode") + } + + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + var first Operand + var consumed int + var err error + if p.evex { + first, consumed, err = decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, width, 16<>3&7) + p.r*8 + if mode == 32 && destinationNumber >= 8 { + return fail("extended destination register in 32-bit mode") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))} + args := []Operand{first, second} + if withImmediate { + args = append([]Operand{{Kind: OpImm, Imm: int64(code[length])}}, args...) + length++ + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + if p.zero { + op += ".Z" + } + parts := make([]string, len(args)) + for index, arg := range args { + parts[index] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(parts, ", ")), + x86Encoded: true, + }, length, true, nil +} diff --git a/amd64_raw_sad_test.go b/amd64_raw_sad_test.go new file mode 100644 index 000000000..1dfcbad35 --- /dev/null +++ b/amd64_raw_sad_test.go @@ -0,0 +1,205 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawPackedSAD(op Op, vectorBits, first, second, destination, mask int, zero, memory bool) []byte { + if memory { + first = 0 + } + mapNumber, opcode := byte(1), byte(0xf6) + if op != "VPSADBW" && op != "VPSADBW.EVEX" { + mapNumber, opcode = 3, 0x42 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + if memory { + modRM = byte(0x40 | destination&7<<3) + } + var code []byte + if op == "VDBPSADBW" || op == "VPSADBW.EVEX" { + p0 := byte((1-destination/8&1)<<7 | (1-first/16)<<6 | + (1-first/8&1)<<5 | (1-destination/16)<<4 | int(mapNumber)) + p1 := byte((^second&15)<<3 | 4 | 1) + p2 := byte(vectorBits<<5 | (1-second/16)<<3 | mask) + if zero { + p2 |= 0x80 + } + code = []byte{0x62, p0, p1, p2, opcode, modRM} + } else { + p0 := byte((1-destination/8&1)<<7 | 1<<6 | + (1-first/8&1)<<5 | int(mapNumber)) + p1 := byte((^second&15)<<3 | vectorBits<<2 | 1) + code = []byte{0xc4, p0, p1, opcode, modRM} + } + if memory { + code = append(code, 1) + } + if op != "VPSADBW" && op != "VPSADBW.EVEX" { + code = append(code, 0x39) + } + return code +} + +func TestTranslateRawPackedSADGorseRegression(t *testing.T) { + const source = `TEXT rawPackedSAD(SB), $0-0 + LONG $0xd3f6edc5 + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + ir := x86UnboundVoidRawScalarFormatObject(t, source, target.arch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-gorse-sad.ll", "raw-gorse-sad.o", ir) + } +} + +func TestTranslateRawPackedSADLegacyForms(t *testing.T) { + const source = `TEXT rawLegacySAD(SB), $0-0 + BYTE $0x66; BYTE $0x0f; BYTE $0xf6; BYTE $0xc1 + BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x42; BYTE $0xc1; BYTE $0x01 + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + ir := x86UnboundVoidRawScalarFormatObject(t, source, target.arch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-legacy-sad.ll", "raw-legacy-sad.o", ir) + } +} + +func TestDecodedX86RawPackedSADGoRows(t *testing.T) { + for _, test := range []struct { + encoded Op + want Op + widths int + }{ + {"VPSADBW", "VPSADBW", 2}, + {"VPSADBW.EVEX", "VPSADBW", 3}, + {"VMPSADBW", "VMPSADBW", 2}, + {"VDBPSADBW", "VDBPSADBW", 3}, + } { + if _, ok := amd64PackedSADSpecs[test.want]; !ok { + t.Fatalf("missing lowerer for Go row %s", test.want) + } + for vectorBits := 0; vectorBits < test.widths; vectorBits++ { + for _, memory := range []bool{false, true} { + for _, mask := range []int{0, 3} { + if mask != 0 && test.encoded != "VDBPSADBW" { + continue + } + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := encodeX86RawPackedSAD(test.encoded, vectorBits, 4, 5, 2, mask, zero, memory) + got, length, recognized, err := decodedX86RawPackedSADInstruction(code, 64) + if err != nil || !recognized || length != len(code) { + t.Fatalf("decode %x: %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + wantOp := test.want + if zero { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s2", "XYZ"[vectorBits:vectorBits+1]) { + t.Fatalf("decode %x: %+v, want %s", code, got, wantOp) + } + if memory { + firstIndex := 0 + if test.want != "VPSADBW" { + firstIndex = 1 + } + if got.Args[firstIndex].Kind != OpMem { + t.Fatalf("decode %x did not retain memory: %+v", code, got) + } + } + } + } + } + } + } +} + +func TestDecodedX86RawPackedSADRejectsInvalidForms(t *testing.T) { + base := encodeX86RawPackedSAD("VPSADBW", 1, 1, 2, 3, 0, false, false) + for _, test := range []struct { + name string + code []byte + }{ + // WIG W0/W1 pairs have their own independent LLVM decoder oracle. + {"VDB W1", func() []byte { + code := encodeX86RawPackedSAD("VDBPSADBW", 0, 1, 2, 3, 0, false, false) + code[2] |= 0x80 + return code + }()}, + {"missing ModRM", base[:4]}, + {"address override", append([]byte{0x67}, base...)}, + {"missing immediate", encodeX86RawPackedSAD("VMPSADBW", 0, 1, 2, 3, 0, false, false)[:5]}, + {"VDB broadcast", func() []byte { + code := encodeX86RawPackedSAD("VDBPSADBW", 0, 0, 2, 3, 0, false, true) + code[3] |= 0x10 + return code + }()}, + {"VPSADBW writemask", func() []byte { + code := encodeX86RawPackedSAD("VPSADBW.EVEX", 0, 1, 2, 3, 0, false, false) + code[3] |= 3 + return code + }()}, + {"VDB zero without mask", encodeX86RawPackedSAD("VDBPSADBW", 0, 1, 2, 3, 0, true, false)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawPackedSADInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } +} + +func TestTranslateRawPackedSADLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawPackedSADFamily(SB), $0-0\n") + for _, test := range []struct { + op Op + width int + mask int + zero bool + memory bool + }{ + {op: "VPSADBW", width: 1}, + {op: "VPSADBW.EVEX", width: 2, memory: true}, + {op: "VMPSADBW", width: 1, memory: true}, + {op: "VDBPSADBW", width: 2, mask: 3, zero: true, memory: true}, + } { + code := encodeX86RawPackedSAD( + test.op, test.width, 1, 2, 3, test.mask, test.zero, test.memory, + ) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-packed-sad.ll", "raw-packed-sad.o", ir) + }) + } +} diff --git a/amd64_raw_same_width_conversion.go b/amd64_raw_same_width_conversion.go new file mode 100644 index 000000000..9da6d1820 --- /dev/null +++ b/amd64_raw_same_width_conversion.go @@ -0,0 +1,236 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type decodedX86SameWidthConversionProperties struct { + op Op + laneBytes int + sae bool +} + +func decodedX86SameWidthConversionOpcode(opcode, pp byte) (decodedX86SameWidthConversionProperties, bool) { + properties := map[[2]byte]decodedX86SameWidthConversionProperties{ + {0x5b, 0}: {op: "VCVTDQ2PS", laneBytes: 4}, + {0x5b, 1}: {op: "VCVTPS2DQ", laneBytes: 4}, + {0x5b, 2}: {op: "VCVTTPS2DQ", laneBytes: 4, sae: true}, + {0x51, 0}: {op: "VSQRTPS", laneBytes: 4}, + {0x51, 1}: {op: "VSQRTPD", laneBytes: 8}, + }[[2]byte{opcode, pp}] + return properties, properties.op != "" +} + +// decodedX86SameWidthConversionInstruction recognizes every VEX and EVEX +// encoding behind Go 1.27's _yvcvtdq2ps table: VCVTDQ2PS, VCVTPS2DQ, +// VCVTTPS2DQ, VSQRTPD, and VSQRTPS. It covers X/Y/Z registers, ModRM/SIB +// memory, masking, zeroing, broadcasts, embedded rounding, SAE, and compressed +// displacements before the generic decoder can consume an incorrect length. +func decodedX86SameWidthConversionInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86SameWidthConversionInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86SameWidthConversionInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if p, valid := decodeX86RawVectorEncoding(code); valid && p.mapNumber == 1 && p.opcode == 0x5b && + (p.pp == 1 || p.pp == 2) && (!p.evex || !p.w) { + return decodedX86PackedNumericConversionInstructionWithMemory(code, mode, memory) + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vectorPrefix + } + } + +vectorPrefix: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXSameWidthConversionInstructionWithMemory(code, i, mode, segment, addressOverride, memory) + } + + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+4 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+5 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86SameWidthConversionOpcode(opcode, vexBits&3) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + // CVTDQ2PS and packed SQRT are VEX.WIG in both machine modes. + // This does not relax their independently validated EVEX.W fields. + ignoredWidth := properties.op == "VCVTDQ2PS" || properties.op == "VSQRTPS" || properties.op == "VSQRTPD" + if vexLength == 3 && vexBits&0x80 != 0 && !ignoredWidth { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: properties.op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", properties.op, source.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXSameWidthConversionInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXSameWidthConversionInstructionWithMemory(code, i, mode, segment, addressOverride, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXSameWidthConversionInstructionWithMemory(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+6 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + half := p0&0x0f == 5 && opcode == 0x51 && p1&3 == 0 + if p0&0x0f != 1 && !half { + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86SameWidthConversionOpcode(opcode, p1&3) + if half { + properties = decodedX86SameWidthConversionProperties{op: "VSQRTPH", laneBytes: 2} + } + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + width64 := p1&0x80 != 0 + wantWidth64 := properties.laneBytes == 8 + if width64 != wantWidth64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match conversion element type") + } + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + evexB := p2&0x10 != 0 + registerSource := modRM>>6 == 3 + embeddedControl := evexB && registerSource + broadcast := evexB && !registerSource + if !embeddedControl && vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + vectorPrefix := "Z" + vectorWidth := 64 + if !embeddedControl { + vectorPrefix = [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth = [...]int{16, 32, 64}[vectorBits] + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = properties.laneBytes + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + sqrt := opcode == 0x51 && !half + if mode == 32 && (maskNumber != 0 && !half && !sqrt && properties.op != "VCVTDQ2PS" || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register or mask in 32-bit mode") + } + source, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + + op := properties.op + if broadcast { + op += ".BCST" + } + if embeddedControl { + if properties.sae { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[vectorBits] + } + } + if zeroing { + op += ".Z" + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{ + Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + x86Encoded: half || sqrt, x86SqrtEVEX: sqrt, + x86PackedNumericEVEX: properties.op == "VCVTDQ2PS", + }, modRMIndex + consumed, true, nil +} diff --git a/amd64_raw_same_width_conversion_test.go b/amd64_raw_same_width_conversion_test.go new file mode 100644 index 000000000..48d98a08b --- /dev/null +++ b/amd64_raw_same_width_conversion_test.go @@ -0,0 +1,171 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type rawSameWidthConversionSpec struct { + opcode byte + pp byte + evexW bool + op Op + laneBytes int + sae bool +} + +var rawSameWidthConversionSpecs = []rawSameWidthConversionSpec{ + {opcode: 0x5b, pp: 0, op: "VCVTDQ2PS", laneBytes: 4}, + {opcode: 0x5b, pp: 1, op: "VCVTPS2DQ", laneBytes: 4}, + {opcode: 0x5b, pp: 2, op: "VCVTTPS2DQ", laneBytes: 4, sae: true}, + {opcode: 0x51, pp: 1, evexW: true, op: "VSQRTPD", laneBytes: 8}, + {opcode: 0x51, pp: 0, op: "VSQRTPS", laneBytes: 4}, +} + +func encodeX86VEXSameWidthConversion(spec rawSameWidthConversionSpec, vex3, width256 bool, destination, source int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + if !vex3 { + vex1 := byte((1-(destination>>3)&1)<<7 | 15<<3 | int(l)<<2 | int(spec.pp)) + return []byte{0xc5, vex1, spec.opcode, modRM} + } + vex0 := byte((1-(destination>>3)&1)<<7 | 1<<6 | (1-(source>>3)&1)<<5 | 1) + vex1 := byte(15<<3 | int(l)<<2 | int(spec.pp)) + return []byte{0xc4, vex0, vex1, spec.opcode, modRM} +} + +func encodeX86EVEXSameWidthConversion(spec rawSameWidthConversionSpec, vectorBits byte, broadcastOrControl, zeroing bool, mask, destination, source int) []byte { + p0 := byte((1-(destination>>3)&1)<<7 | (1-(source>>4)&1)<<6 | (1-(source>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 1) + p1 := byte(15<<3 | 1<<2 | int(spec.pp)) + if spec.evexW { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | 1<<3 | mask) + if broadcastOrControl { + p2 |= 0x10 + } + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + return []byte{0x62, p0, p1, p2, spec.opcode, modRM} +} + +func TestDecodedX86VEXSameWidthConversionCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawSameWidthConversionSpecs { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + sourceLimit := 16 + if !vex3 { + sourceLimit = 8 + } + for destination := 0; destination < 16; destination++ { + for source := 0; source < sourceLimit; source++ { + code := encodeX86VEXSameWidthConversion(spec, vex3, width256, destination, source) + got, length, ok, err := decodedX86SameWidthConversionInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != spec.op || len(got.Args) != 2 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + if count != 3840 { + t.Fatalf("covered %d VEX same-width conversion encodings, want 3840", count) + } +} + +func TestDecodedX86EVEXSameWidthConversionCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawSameWidthConversionSpecs { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + for destination := 0; destination < 32; destination++ { + for source := 0; source < 32; source++ { + code := encodeX86EVEXSameWidthConversion(spec, vectorBits, false, false, 1, destination, source) + got, length, ok, err := decodedX86SameWidthConversionInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source), "K1", fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != spec.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 15360 { + t.Fatalf("covered %d EVEX same-width conversion encodings, want 15360", count) + } +} + +func TestDecodedX86EVEXSameWidthConversionModifiersMemoryAndInvalid(t *testing.T) { + find := func(op Op) rawSameWidthConversionSpec { + t.Helper() + for _, spec := range rawSameWidthConversionSpecs { + if spec.op == op { + return spec + } + } + t.Fatalf("missing raw same-width spec %s", op) + return rawSameWidthConversionSpec{} + } + convert := find("VCVTDQ2PS") + truncate := find("VCVTTPS2DQ") + sqrt := find("VSQRTPD") + tests := []struct { + name string + code []byte + op Op + args []string + }{ + {name: "vex weaviate", code: []byte{0xc5, 0xfc, 0x5b, 0xe4}, op: "VCVTDQ2PS", args: []string{"Y4", "Y4"}}, + {name: "broadcast zeroing", code: []byte{0x62, 0xf1, 0x7c, 0xdb, 0x5b, 0x50, 0x7f}, op: "VCVTDQ2PS.BCST.Z", args: []string{"508(AX)", "K3", "Z2"}}, + {name: "round up", code: encodeX86EVEXSameWidthConversion(convert, 2, true, false, 2, 4, 5), op: "VCVTDQ2PS.RU_SAE", args: []string{"Z5", "K2", "Z4"}}, + {name: "truncate sae zeroing", code: encodeX86EVEXSameWidthConversion(truncate, 0, true, true, 3, 6, 7), op: "VCVTTPS2DQ.SAE.Z", args: []string{"Z7", "K3", "Z6"}}, + {name: "sqrt round zero", code: encodeX86EVEXSameWidthConversion(sqrt, 3, true, false, 0, 8, 9), op: "VSQRTPD.RZ_SAE", args: []string{"Z9", "Z8"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86SameWidthConversionInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != len(test.args) { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + for index := range test.args { + if got.Args[index].String() != test.args[index] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } + }) + } + + zeroWithoutMask := encodeX86EVEXSameWidthConversion(convert, 0, false, true, 0, 0, 0) + reservedLength := encodeX86EVEXSameWidthConversion(convert, 3, false, false, 0, 0, 0) + wrongW := encodeX86EVEXSameWidthConversion(convert, 0, false, false, 0, 0, 0) + wrongW[2] |= 0x80 + for _, code := range [][]byte{zeroWithoutMask, reservedLength, wrongW} { + if _, _, ok, err := decodedX86SameWidthConversionInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} diff --git a/amd64_raw_scalar_broadcast.go b/amd64_raw_scalar_broadcast.go new file mode 100644 index 000000000..7d492e2c5 --- /dev/null +++ b/amd64_raw_scalar_broadcast.go @@ -0,0 +1,77 @@ +package plan9asm + +import "fmt" + +// Go's _yvbroadcastss/_yvbroadcastsd tables share the typed VEX/EVEX +// prefix grammar. The scalar input width, output width and mask are separate +// axes: SD has no 128-bit destination, and disp8 scales by scalar input size. +func decodedX86ScalarBroadcastInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86ScalarBroadcastInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86ScalarBroadcastInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || (p.opcode != 0x18 && p.opcode != 0x19) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("scalar broadcast: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.pp != 1 || p.upper != 0 || p.broadcast { + return fail("reserved prefix, vvvv/V' or broadcast field") + } + double := p.opcode == 0x19 + if p.vectorLength > 2 || double && p.vectorLength == 0 { + return fail("invalid vector width") + } + if p.evex { + if !p.fixed || p.w != double { + return fail("invalid EVEX fixed or width bit") + } + } else if p.w { + return fail("VEX.W must be zero") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + op, bytes := Op("VBROADCASTSS"), 4 + if double { + op, bytes = "VBROADCASTSD", 8 + } + var source Operand + var size int + var err error + if p.evex { + source, size, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, "X", bytes) + } else { + source, size, err = memory.vex(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + reg := int(code[p.modRM]>>3&7) + p.r*8 + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", []string{"X", "Y", "Z"}[p.vectorLength], reg))} + args := []Operand{source, destination} + printed := source.String() + ", " + destination.String() + if p.mask != 0 { + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))} + args = []Operand{source, mask, destination} + printed = source.String() + ", " + mask.String() + ", " + destination.String() + } + if p.zero { + op += ".Z" + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, printed)}, p.modRM + size, true, nil +} diff --git a/amd64_raw_scalar_broadcast_test.go b/amd64_raw_scalar_broadcast_test.go new file mode 100644 index 000000000..681aec420 --- /dev/null +++ b/amd64_raw_scalar_broadcast_test.go @@ -0,0 +1,293 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86RawScalarBroadcastGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Go 1.27 _yvbroadcastss has 2 VEX + 6 EVEX rows; + // _yvbroadcastsd has 1 VEX + 4 EVEX rows (no X destination). + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT broadcasts(SB),4,$0-0\n") + regs := []int{0, 7} + if arch == "amd64" { + regs = append(regs, 15, 16, 31) + } + for _, op := range []string{"VBROADCASTSS", "VBROADCASTSD"} { + for _, width := range []string{"X", "Y", "Z"} { + if op == "VBROADCASTSD" && width == "X" { + continue + } + for _, reg := range regs { + for _, input := range []string{fmt.Sprintf("X%d", reg), "(AX)", "16(BX)(CX*4)", "-64(SI)"} { + for _, form := range []struct{ suffix, mask string }{{"", ""}, {"", "K1,"}, {".Z", "K7,"}} { + fmt.Fprintf(&source, "%s%s %s,%s%s%d\n", op, form.suffix, input, form.mask, width, reg) + } + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + file := x86RawUnprovedReturnFormProbeFile(t, arch, "broadcasts", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir := x86UnboundVoidRawProbeObject(t, file, arch, triple) + compileLLVMToObject(t, llc, triple, "raw-scalar-broadcast.ll", "raw-scalar-broadcast.o", ir) + } + }) + } +} + +func TestX86RawScalarBroadcastEncodingAxes(t *testing.T) { + count := 0 + for _, evex := range []bool{false, true} { + limit, widths := 16, 2 + if evex { + limit, widths = 32, 3 + } + for _, double := range []bool{false, true} { + for width := 0; width < widths; width++ { + if double && width == 0 { + continue + } + for src := 0; src < limit; src++ { + for dst := 0; dst < limit; dst++ { + mask, zero := 0, false + if evex { + mask = (src + dst) & 7 + zero = mask != 0 && dst&1 != 0 + } + code := encodeRawScalarMove(evex, 1, 0x10, src, 0, dst, mask, zero) + op, opcode := Op("VBROADCASTSS"), byte(0x18) + if double { + op, opcode = "VBROADCASTSD", 0x19 + } + if evex { + code[1] = code[1]&0xf0 | 2 + code[3] |= byte(width) << 5 + code[4] = opcode + if double { + code[2] |= 0x80 + } + } else { + code[1] = code[1]&0xe0 | 2 + code[2] |= byte(width) << 2 + code[3] = opcode + } + got, size, ok, err := decodedX86ScalarBroadcastInstruction(code, 64) + if zero { + op += ".Z" + } + args := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", src))}} + if mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", []string{"X", "Y", "Z"}[width], dst))}) + if err != nil || !ok || size != len(code) || got.Op != op || !reflect.DeepEqual(got.Args, args) { + t.Fatalf("decode %x=%+v size=%d ok=%v err=%v; want %s %+v", code, got, size, ok, err, op, args) + } + count++ + } + } + } + } + } + if count != 5888 { + t.Fatalf("tested %d encodings, want 5888", count) + } +} + +func TestX86RawScalarBroadcastRejectsReservedEncodings(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe2, 0x79, 0x18}, // Missing ModRM. + {0xc4, 0xe2, 0x79, 0x18, 0x04}, // Missing SIB. + {0xc4, 0xe2, 0x79, 0x18, 0x45}, // Missing displacement. + {0xc4, 0xe2, 0x79, 0x19, 0xc0}, // SD has no 128-bit output. + {0xc4, 0xe2, 0xf9, 0x18, 0xc0}, // VEX.W must be zero. + {0xc4, 0xe2, 0x71, 0x18, 0xc0}, // Reserved vvvv. + {0x67, 0xc4, 0xe2, 0x79, 0x18, 0x00}, + {0x62, 0xf2, 0x79, 0x08, 0x18, 0xc0}, // Missing EVEX fixed bit. + {0x62, 0xf2, 0xfd, 0x08, 0x18, 0xc0}, // SS W1. + {0x62, 0xf2, 0x7d, 0x28, 0x19, 0xc0}, // SD W0. + {0x62, 0xf2, 0x7d, 0x00, 0x18, 0xc0}, // Reserved V'. + {0x62, 0xf2, 0x7d, 0x68, 0x18, 0xc0}, // Reserved LL. + {0x62, 0xf2, 0x7d, 0x18, 0x18, 0xc0}, // Reserved EVEX.b. + {0x62, 0xf2, 0x7d, 0x88, 0x18, 0xc0}, // Zeroing with K0. + } { + if _, _, matched, err := decodedX86ScalarBroadcastInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x matched=%v err=%v", code, matched, err) + } + } +} + +func TestX86ScalarBroadcastGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []string{"VBROADCASTSS", "VBROADCASTSD"} { + instructions := []string{op + " AX,Y0", op + " Y0,Y1", op + " X0,AX", op + " X0,K0,Y0", op + ".Z X0,Y0", op + ".SAE X0,Y0", op + " X0,K1,(AX)"} + if op == "VBROADCASTSD" { + instructions = append(instructions, op+" X0,X1") + } + if arch == "386" { + instructions = append(instructions, op+" X0,Z8") + } + for _, instruction := range instructions { + source := "TEXT invalidbroadcast(SB),4,$0-0\n" + instruction + "\nRET\n" + dir := t.TempDir() + name := filepath.Join(dir, "invalid.s") + if err := os.WriteFile(name, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), name) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + if output, err := cmd.CombinedOutput(); err == nil { + if instruction != op+".Z X0,Y0" || goToolchainAtLeast(runtime.Version(), 1, 27) { + t.Fatalf("Go %s accepted %s; repair the oracle expectation: %s", arch, instruction, output) + } + // Old Go accepts maskless .Z but emits a reserved EVEX + // zeroing/K0 encoding. Verify both that exact historical + // mismatch and the raw decoder's rejection, then require + // the named translator's rejection below too. + code := assembleX87ControlBytes(t, arch, source) + mode := 64 + if arch == "386" { + mode = 32 + } + p, prefixMatched := decodeX86RawVectorEncoding(code) + _, _, matched, err := decodedX86ScalarBroadcastInstruction(code, mode) + if !prefixMatched || !p.evex || !p.zero || p.mask != 0 || !matched || err == nil || !strings.Contains(err.Error(), "zeroing requires a nonzero mask") { + t.Fatalf("historical maskless .Z did not emit rejected EVEX zeroing/K0: %x %v", code, err) + } + t.Logf("%s/%s emits reserved zeroing/K0; both translators reject it", runtime.Version(), arch) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + _, err = Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"invalidbroadcast": {Name: "invalidbroadcast", Ret: Void}}}) + if err == nil { + t.Errorf("%s accepted Go-rejected %s", arch, instruction) + } + } + } + } +} + +func TestX86ScalarBroadcastMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; separate five-target object tests are required") + } + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, spec := range []struct { + op string + bits int + }{{"VBROADCASTSS", 32}, {"VBROADCASTSD", 64}} { + for width, reg := range []string{"X0", "Y0", "Z0"} { + if spec.bits == 64 && width == 0 { + continue + } + for form := 0; form < 6; form++ { + name := fmt.Sprintf("broadcast%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ old+0(FP),BX\nMOVQ mem+8(FP),AX\nMOVQ out+16(FP),DI\nMOVQ mask+24(FP),CX\nKMOVQ CX,K7\nVMOVUPS (BX),Z0\n", name) + input := "(AX)" + if form >= 3 { + source.WriteString("VMOVUPS (AX),X1\n") + input = "X1" + } + op, mask := spec.op, "" + if form%3 != 0 { + mask = "K7," + } + if form%3 == 2 { + op += ".Z" + } + fmt.Fprintf(&source, "%s %s,%s%s\nVMOVUPS Z0,(DI)\nVMOVUPS Y0,64(DI)\nVMOVUPS X0,96(DI)\nRET\n", op, input, mask, reg) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *,uint64_t);\n", name) + faultcheck := 0 + if form < 3 && form%3 != 0 { + faultcheck = 1 + } + fmt.Fprintf(&checks, "if(check(%s,%d,%d,%d,%d)) return %d;\n", name, 16< +#include +` + declarations.String() + ` +static int check(void (*fn)(const void *,const void *,void *,uint64_t),int bytes,int lane,int form,int faultcheck) { + uint64_t masks[]={0,1,2,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX,UINT64_C(1)<<63}; + for(int m=0;m<6;m++) { + _Alignas(64) uint8_t old[64],mem[16],out[112]; + for(int i=0;i<64;i++) old[i]=(uint8_t)(197-i*3); + for(int i=0;i<16;i++) mem[i]=(uint8_t)(17+i*7); + uint64_t active=masks[m]&((UINT64_C(1)<<(bytes/lane))-1); + fn(old,faultcheck && !active ? 0 : mem,out,masks[m]); + for(int view=0;view<3;view++) for(int i=0;i<(64>>view);i++) { + int enabled=form==0 || ((masks[m]>>(i/lane))&1); + uint8_t want=i>=bytes ? 0 : enabled ? mem[i%lane] : form==2 ? 0 : old[i]; + int offset=view==0 ? 0 : view==1 ? 64 : 96; + if(out[offset+i]!=want) {fprintf(stderr,"broadcast mismatch bytes=%d lane=%d form=%d mask=%d view=%d byte=%d\n",bytes,lane,form,m,view,i);return 1;} + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_broadcast", triple, ir, mainC, prefix) +} diff --git a/amd64_raw_scalar_extract_test.go b/amd64_raw_scalar_extract_test.go new file mode 100644 index 000000000..fe211e74e --- /dev/null +++ b/amd64_raw_scalar_extract_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +type rawPackedScalarExtractSpec struct { + op Op + mapNumber byte + opcode byte + width64 bool +} + +var rawPackedScalarExtractSpecs = []rawPackedScalarExtractSpec{ + {op: "VPEXTRB", mapNumber: 3, opcode: 0x14}, + {op: "VPEXTRW", mapNumber: 3, opcode: 0x15}, + {op: "VPEXTRW", mapNumber: 1, opcode: 0xc5}, + {op: "VPEXTRD", mapNumber: 3, opcode: 0x16}, + {op: "VPEXTRQ", mapNumber: 3, opcode: 0x16, width64: true}, + {op: "VEXTRACTPS", mapNumber: 3, opcode: 0x17}, +} + +func encodeX86RawVEXPackedScalarExtract(spec rawPackedScalarExtractSpec, source, destination int, immediate byte) []byte { + p0 := byte((^source>>3)&1)<<7 | 0x40 | byte((^destination>>3)&1)<<5 | spec.mapNumber + p1 := byte(0x79) + if spec.width64 { + p1 |= 0x80 + } + modRM := byte(0xc0 | source&7<<3 | destination&7) + return []byte{0xc4, p0, p1, spec.opcode, modRM, immediate} +} + +func encodeX86RawEVEXPackedScalarExtract(spec rawPackedScalarExtractSpec, source, destination int, immediate byte) []byte { + p0 := byte((^source>>3)&1)<<7 | + 0x40 | + byte((^destination>>3)&1)<<5 | + byte((^source>>4)&1)<<4 | + spec.mapNumber + p1 := byte(0x7d) + if spec.width64 { + p1 |= 0x80 + } + modRM := byte(0xc0 | source&7<<3 | destination&7) + return []byte{0x62, p0, p1, 0x08, spec.opcode, modRM, immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVEXTRACTPS(t *testing.T) { + // vmovss %xmm0, (%rdi); vextractps $1, %xmm0, 4(%rdi) + code := []byte{0xc5, 0xfa, 0x11, 0x07, 0xc4, 0xe3, 0x79, 0x17, 0x47, 0x04, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VEXTRACTPS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VMOVSS X0, 0(DI) ") || !strings.HasPrefix(decoded[1].Raw, "VEXTRACTPS $1, X0, 4(DI) ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawPackedScalarExtractCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawPackedScalarExtractSpecs { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawVEXPackedScalarExtract(spec, source, destination, immediate) + got, length, ok, err := decodedX86PackedScalarExtractInstruction(code, 64) + want := fmt.Sprintf("%s $%d, X%d, %s", spec.op, immediate, source, mustDecodedX86GeneralRegister(t, destination)) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for source := 0; source < 32; source++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawEVEXPackedScalarExtract(spec, source, destination, immediate) + got, length, ok, err := decodedX86PackedScalarExtractInstruction(code, 64) + want := fmt.Sprintf("%s $%d, X%d, %s", spec.op, immediate, source, mustDecodedX86GeneralRegister(t, destination)) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + if count != 4608 { + t.Fatalf("covered %d packed scalar-extract register encodings, want 4608", count) + } +} + +func TestDecodedX86RawPackedScalarExtractMemoryAndInvalidForms(t *testing.T) { + vexMemory := []byte{0xc4, 0x03, 0x79, 0x17, 0x7c, 0x8b, 0x01, 0xff} + got, length, ok, err := decodedX86PackedScalarExtractInstruction(vexMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(vexMemory) || got.Op != "VEXTRACTPS" || len(got.Args) != 3 || got.Args[1].String() != "X15" || got.Args[2].Kind != OpMem || !reflect.DeepEqual(got.Args[2].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + + evexMemory := []byte{0x62, 0x03, 0xfd, 0x08, 0x16, 0x7c, 0x8b, 0x02, 0x01} + got, length, ok, err = decodedX86PackedScalarExtractInstruction(evexMemory, 64) + wantMemory.Off = 16 + if err != nil || !ok || length != len(evexMemory) || got.Op != "VPEXTRQ" || len(got.Args) != 3 || got.Args[1].String() != "X31" || got.Args[2].Kind != OpMem || !reflect.DeepEqual(got.Args[2].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + twoByteVEX := []byte{0xc5, 0xf9, 0xc5, 0xc1, 0x01} + got, length, ok, err = decodedX86PackedScalarExtractInstruction(twoByteVEX, 64) + if err != nil || !ok || length != len(twoByteVEX) || got.Raw != "VPEXTRW $1, X0, CX" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", twoByteVEX, got, length, ok, err) + } + + validVEX := encodeX86RawVEXPackedScalarExtract(rawPackedScalarExtractSpecs[0], 7, 7, 1) + validEVEX := encodeX86RawEVEXPackedScalarExtract(rawPackedScalarExtractSpecs[0], 31, 15, 1) + for name, invalid := range map[string][]byte{ + "vex reserved vvvv": {validVEX[0], validVEX[1], validVEX[2] &^ 0x08, validVEX[3], validVEX[4], validVEX[5]}, + "vex vector length": {validVEX[0], validVEX[1], validVEX[2] | 0x04, validVEX[3], validVEX[4], validVEX[5]}, + "evex mask": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 1, validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex vector length": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x20, validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex missing fixed bit": {validEVEX[0], validEVEX[1], validEVEX[2] &^ 0x04, validEVEX[3], validEVEX[4], validEVEX[5], validEVEX[6]}, + "truncated vex immediate": validVEX[:len(validVEX)-1], + "truncated evex immediate": validEVEX[:len(validEVEX)-1], + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86PackedScalarExtractInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + if instruction, _, matched, decodeErr := decodedX86PackedScalarExtractInstruction(validEVEX, 32); !matched || decodeErr == nil { + t.Fatalf("386 EVEX encoding %x decoded as %+v, ok=%v, err=%v", validEVEX, instruction, matched, decodeErr) + } + extended386 := encodeX86RawVEXPackedScalarExtract(rawPackedScalarExtractSpecs[0], 8, 0, 1) + if instruction, _, matched, decodeErr := decodedX86PackedScalarExtractInstruction(extended386, 32); !matched || decodeErr == nil { + t.Fatalf("386 extended encoding %x decoded as %+v, ok=%v, err=%v", extended386, instruction, matched, decodeErr) + } +} + +func TestTranslateX86RawPackedScalarExtractTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + registerMax int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", registerMax: 15}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registerMax: 15}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registerMax: 15}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", registerMax: 7}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", registerMax: 7}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_packed_scalar_extract(SB),NOSPLIT,$0-0\n") + for _, spec := range rawPackedScalarExtractSpecs { + code := encodeX86RawVEXPackedScalarExtract(spec, target.registerMax, target.registerMax, 1) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + if target.goarch == "amd64" { + code = encodeX86RawEVEXPackedScalarExtract(spec, 31, 15, 1) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_packed_scalar_extract": {Name: "raw_packed_scalar_extract", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-packed-scalar-extract-"+target.name+".ll", "raw-packed-scalar-extract-"+target.name+".o", ir) + }) + } +} + +func mustDecodedX86GeneralRegister(t *testing.T, number int) Reg { + t.Helper() + reg, ok := decodedX86GeneralRegister(number) + if !ok { + t.Fatalf("missing general register %d", number) + } + return reg +} diff --git a/amd64_raw_scalar_flag_compare.go b/amd64_raw_scalar_flag_compare.go new file mode 100644 index 000000000..1f4c245a7 --- /dev/null +++ b/amd64_raw_scalar_flag_compare.go @@ -0,0 +1,71 @@ +package plan9asm + +import "fmt" + +// Go's four _yvcomisd entries share X/m,X and EVEX register-only SAE. +// Prefix fields stay typed: EVEX.b means SAE here, never memory broadcast; +// disp8 scales by the scalar width. VEX W and EVEX LL are ignored. Intel +// SDM COMISS/UCOMISS explicitly warns that VEX.L=1 can behave unpredictably +// across processor generations; decoder acceptance is not a portable opcode +// contract and cannot authorize changing those original bytes to VEX.L=0. +func decodedX86ScalarFlagCompareInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || (p.opcode != 0x2e && p.opcode != 0x2f) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("scalar flag compare: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if (p.pp != 0 && p.pp != 1) || p.upper != 0 || p.mask != 0 || p.zero { + return fail("reserved prefix, vvvv/V', mask or zeroing field") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + double := p.pp == 1 + if p.evex && (!p.fixed || p.w != double) { + return fail("invalid EVEX fixed or scalar width bit") + } + if p.broadcast && code[p.modRM]>>6 != 3 { + return fail("SAE requires a register source") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + op := Op("VCOMISS") + if p.opcode == 0x2e { + op = "VUCOMISS" + } + bytes := 4 + if double { + op = op[:len(op)-1] + "D" + bytes = 8 + } + if p.broadcast { + op += ".SAE" + } + var source Operand + var size int + var err error + if p.evex { + source, size, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", bytes) + } else { + source, size, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + if !p.evex && p.vectorLength != 0 { + return Instr{}, 0, true, fmt.Errorf("%w: scalar flag comparison VEX.L=1 lacks a portable physical opcode contract", ErrProbeNeedsContext) + } + dst := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", int(code[p.modRM]>>3&7)+p.r*8))} + return Instr{Op: op, Args: []Operand{source, dst}, Raw: fmt.Sprintf("%s %s, %s", op, source.String(), dst.String()), + x86ScalarFlagCompareEVEX: p.evex, + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_scalar_flag_compare_literal_test.go b/amd64_raw_scalar_flag_compare_literal_test.go new file mode 100644 index 000000000..ce4362100 --- /dev/null +++ b/amd64_raw_scalar_flag_compare_literal_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawScalarFlagCompareLiteral(op string, encoding string) []byte { + pp, opcode := 0, 0x2f + if strings.HasPrefix(op, "VU") { + opcode = 0x2e + } + width := 4 + if strings.HasSuffix(op, "SD") { + pp, width = 1, 8 + } + var code []byte + if encoding == "vex2" { + code = []byte{0xc5, byte(0xf8 | pp), byte(opcode), 0x05} + } else { + evex := encoding == "evex" + code = encodeRawScalarMove(evex, pp, opcode, 0, 0, 0, 0, false) + if evex && width == 8 { + code[2] |= 0x80 + } + code[len(code)-1] = 0x05 + } + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*37+21)) + } + return code +} + +func TestDecodeX86RawScalarFlagCompareRIPDataCompleteFamily(t *testing.T) { + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, encoding := range []string{"vex2", "vex3", "evex"} { + name := op + "/" + encoding + t.Run(name, func(t *testing.T) { + code := x86RawScalarFlagCompareLiteral(op, encoding) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != Op(op) || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local scalar", code, decoded, op) + } + }) + } + } +} + +func TestDecodeX86RawScalarFlagCompareRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawScalarFlagCompareLiteral("VUCOMISD", "evex") + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x65}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + "EVEX SAE memory": func(code []byte) []byte { + code[3] |= 0x10 + return code + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawScalarFlagCompareRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + index := 0 + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, encoding := range []string{"vex2", "vex3", "evex"} { + code := x86RawScalarFlagCompareLiteral(op, encoding) + name := fmt.Sprintf("flagCompareLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + width := 4 + if strings.HasSuffix(op, "SD") { + width = 8 + } + pools = append(pools, code[len(code)-width:]) + index++ + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + if len(sigs) != len(file.Funcs) || len(sigs) != 12 { + t.Fatal("complete original COMI RIP TEXT/declaration inventory changed") + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Supply only the missing X0 input in a separately named companion. + // Inserting before the complete original BYTE block preserves every raw + // byte, its RIP displacement, its terminal RET and all source-local data. + const header = "(SB),$0-0\n" + const initializedHeader = "(SB),$0-0\nPXOR X0,X0\n" + initialized := strings.ReplaceAll(source.String(), header, initializedHeader) + if strings.Count(source.String(), header) != 12 || strings.Count(initialized, "PXOR X0,X0\n") != 12 || + strings.ReplaceAll(initialized, initializedHeader, header) != source.String() { + t.Fatal("initialized RIP companion changed more than each missing X0 producer") + } + var declarations strings.Builder + for _, function := range file.Funcs { + fmt.Fprintf(&declarations, "func %s()\n", function.Sym) + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86COMIOriginalFormatObject(t, source.String(), "amd64", triple) + x86COMICompileOriginalFormatObjects(t, llc, triple, "flag-compare-literal", ir) + x86COMIOriginalProducerObjects(t, initialized, declarations.String(), "amd64", triple) + }) + } +} diff --git a/amd64_raw_scalar_flag_compare_test.go b/amd64_raw_scalar_flag_compare_test.go new file mode 100644 index 000000000..68343beb1 --- /dev/null +++ b/amd64_raw_scalar_flag_compare_test.go @@ -0,0 +1,314 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawScalarFlagCompareGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Go 1.27 _yvcomisd has one VEX and one EVEX X/m,X row for + // each of the four opcodes. Only EVEX register sources permit SAE. + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + regs := []int{0, 7} + if arch == "amd64" { + regs = append(regs, 8, 15, 16, 31) + } + var source strings.Builder + source.WriteString("TEXT flagcompare(SB),4,$0-0\n") + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, dst := range regs { + for _, src := range regs { + fmt.Fprintf(&source, "%s X%d,X%d\n%s.SAE X%d,X%d\n", op, src, dst, op, src, dst) + } + for _, mem := range []string{"(AX)", "12(BX)(CX*4)", "-32(SI)", "8192(DI)"} { + fmt.Fprintf(&source, "%s %s,X%d\n", op, mem, dst) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + file := x86RawUnprovedReturnFormProbeFile(t, arch, "flagcompare", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + // Retain the original named and byte-exact RET programs as + // uninitialized public Context, independently of the UD2 probe. + x86UnboundVoidFormatSource(t, source.String(), "flagcompare", arch, triple) + originalRaw := "TEXT flagcompare(SB),4,$0-0\n" + x86DwordProductDirectives(code) + x86UnboundVoidFormatSource(t, originalRaw, "flagcompare", arch, triple) + ir := x86COMIOriginalProbeFormatObject(t, file, arch, triple) + x86COMICompileOriginalFormatObjects(t, llc, triple, "raw-flag-compare", ir) + } + t.Logf("checked %d Go encodings", len(want)-1) + }) + } +} + +func TestX86RawScalarFlagCompareEncodingAxes(t *testing.T) { + count, contexts := 0, 0 + for _, evex := range []bool{false, true} { + limit, variants := 16, 4 // WIG; L1 is an explicit source-context boundary. + if evex { + limit, variants = 32, 8 // EVEX.LL ignored, SAE on/off. + } + for _, double := range []bool{false, true} { + for _, unordered := range []bool{false, true} { + pp, opcode, op := 0, 0x2f, Op("VCOMISS") + if unordered { + opcode, op = 0x2e, "VUCOMISS" + } + if double { + pp, op = 1, op[:len(op)-1]+"D" + } + for src := 0; src < limit; src++ { + for dst := 0; dst < limit; dst++ { + for variant := 0; variant < variants; variant++ { + code := encodeRawScalarMove(evex, pp, opcode, src, 0, dst, 0, false) + wantOp := op + if evex { + code[2] &^= 0x80 + if double { + code[2] |= 0x80 + } + code[3] |= byte(variant&3) << 5 + if variant&4 != 0 { + code[3] |= 0x10 + wantOp += ".SAE" + } + } else { + code[2] |= byte(variant&1)<<2 | byte(variant>>1)<<7 + } + got, size, matched, err := decodedX86ScalarFlagCompareInstruction(code, 64) + if !evex && variant&1 != 0 { + if !matched || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("VEX.L1 lost physical opcode boundary: %x %v", code, err) + } + count++ + contexts++ + continue + } + want := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", src))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", dst))}} + if err != nil || !matched || size != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v size=%d matched=%v err=%v; want %s %+v", code, got, size, matched, err, wantOp, want) + } + _, _, matched, err = decodedX86ScalarFlagCompareInstruction(code, 32) + if !matched || (err != nil) != (src >= 8 || dst >= 8) { + t.Fatalf("386 decode %x matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + if count != 36864 || contexts != 2048 { + t.Fatalf("tested %d encodings with %d VEX.L1 contexts, want 36864/2048", count, contexts) + } +} + +func TestX86RawScalarFlagCompareInvalidAndMemoryForms(t *testing.T) { + for _, code := range [][]byte{nil, {0xc5}, {0xc4, 0xe2, 0x78, 0x2e, 0xc0}, {0xc5, 0xf8, 0x58, 0xc0}} { + if _, _, matched, _ := decodedX86ScalarFlagCompareInstruction(code, 64); matched { + t.Fatalf("matched unrelated encoding %x", code) + } + } + if _, _, matched, err := decodedX86ScalarFlagCompareInstruction([]byte{0xc5, 0xf8, 0x2e, 0xc0}, 16); !matched || err == nil { + t.Fatalf("accepted unsupported mode: matched=%v err=%v", matched, err) + } + for _, code := range [][]byte{ + {0xc5, 0xf8, 0x2e}, + {0xc5, 0xf8, 0x2e, 0x04}, // Missing SIB. + {0xc5, 0xf8, 0x2e, 0x45}, // Missing displacement. + {0xc5, 0xfa, 0x2e, 0xc0}, // Wrong pp. + {0xc5, 0xfb, 0x2e, 0xc0}, + {0xc5, 0xf0, 0x2e, 0xc0}, // Reserved vvvv. + {0x67, 0xc5, 0xf8, 0x2e, 0x00}, + {0x62, 0xf1, 0x78, 0x08, 0x2e, 0xc0}, // Missing fixed bit. + {0x62, 0xf1, 0xfc, 0x08, 0x2e, 0xc0}, // SS W1. + {0x62, 0xf1, 0x7d, 0x08, 0x2e, 0xc0}, // SD W0. + {0x62, 0xf1, 0x7c, 0x00, 0x2e, 0xc0}, // Reserved V'. + {0x62, 0xf1, 0x7c, 0x88, 0x2e, 0xc0}, // Zeroing. + {0x62, 0xf1, 0x7c, 0x09, 0x2e, 0xc0}, // Masking. + {0x62, 0xf1, 0x7c, 0x18, 0x2e, 0x00}, // SAE with memory. + } { + if _, _, matched, err := decodedX86ScalarFlagCompareInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x matched=%v err=%v", code, matched, err) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + for _, double := range []bool{false, true} { + for _, evex := range []bool{false, true} { + code := []byte{segment.prefix, 0xc4, 0x01, 0x78, 0x2e, 0x64, 0x88, 0xf9} + wantOff, op := int64(-7), Op("VUCOMISS") + if double { + code[3] |= 1 + op = "VUCOMISD" + } + if evex { + code = []byte{segment.prefix, 0x62, 0x01, 0x7c, 0x08, 0x2e, 0x64, 0x88, 0xf9} + wantOff *= 4 + if double { + code[3] |= 0x81 + wantOff *= 2 + } + } + dst := Reg("X12") + if evex { + dst = "X28" + } + want := []Operand{{Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: wantOff}}, {Kind: OpReg, Reg: dst}} + got, size, matched, err := decodedX86ScalarFlagCompareInstruction(code, 64) + if err != nil || !matched || size != len(code) || got.Op != op || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v %v; want %s %+v", code, got, err, op, want) + } + } + } + } +} + +func TestX86ScalarFlagCompareGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, args := range []string{" Y0,X1", " X0,Y1", " Z0,X1", " (AX),(BX)", " X0,K1,X1", ".Z X0,X1", ".BCST (AX),X1", ".SAE (AX),X1", ".RN_SAE X0,X1"} { + line := op + args + source := "TEXT rejected(SB),4,$0-0\n" + line + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"rejected": {Name: "rejected", Ret: Void}}}); err == nil { + t.Fatalf("translator accepted %s for %s", line, arch) + } + } + } + } +} + +func x86COMIRawNativeCompare(t *testing.T) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + source, sigs, mainC, _ := x86COMIRawNativeFixture(t) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%v", optimized), func(t *testing.T) { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, input) + } + for _, backend := range []string{"-O0", "-O2"} { + t.Run("backend"+backend, func(t *testing.T) { + execution.compileAndRunLLVM(t, "raw-comi", input, mainC, backend) + }) + } + }) + } +} + +func x86COMIRawNativeFixture(t *testing.T) (string, map[string]FuncSig, string, int) { + t.Helper() + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, reg := range []int{0, 16} { + for _, form := range []string{"register", "memory", "sae"} { + name := fmt.Sprintf("flagcompare%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),DI\nVMOVUPS (AX),Z%d\nVMOVUPS (BX),Z%d\nMOVL $2147483647,CX\nADDL $1,CX\n", name, reg, reg+1) + input, suffix := fmt.Sprintf("X%d", reg+1), "" + if form == "memory" { + input = "(BX)" + } + if form == "sae" { + suffix = ".SAE" + } + code := assembleX87ControlBytes(t, "amd64", fmt.Sprintf("TEXT probe(SB),4,$0-0\n%s%s %s,X%d\nRET\n", op, suffix, input, reg)) + for _, b := range code[:len(code)-1] { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + source.WriteString("SETEQ (DI)\nSETCS 1(DI)\nSETPS 2(DI)\nSETMI 3(DI)\nSETOS 4(DI)\n") + fmt.Fprintf(&source, "VMOVUPS Z%d,8(DI)\nVMOVUPS Z%d,72(DI)\nRET\n", reg, reg+1) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *);\n", name) + double := 0 + if strings.HasSuffix(op, "SD") { + double = 1 + } + fmt.Fprintf(&checks, "if(check(%s,%d)) return %d;\n", name, double, index) + } + } + } + // Compare EFLAGS with native UCOMIS{S,D}. This oracle checks comparison, + // NaN, signed-zero and register-preservation semantics, not MXCSR exception + // delivery or SAE suppression, which require a separate floating-state audit. + mainC := `#include +#include +#include +` + declarations.String() + ` +static const uint64_t doubles[]={0,0x8000000000000000ULL,0x3ff0000000000000ULL,0xbff0000000000000ULL,0x4004000000000000ULL,0xc004000000000000ULL,0x7ff0000000000000ULL,0xfff0000000000000ULL,0x7ff8000000000001ULL,0x7ff0000000000001ULL,1,0x7fefffffffffffffULL}; +static const uint32_t singles[]={0,0x80000000,0x3f800000,0xbf800000,0x40200000,0xc0200000,0x7f800000,0xff800000,0x7fc00001,0x7f800001,1,0x7f7fffff}; +static int check(void (*fn)(const void *,const void *,void *),int isDouble) { + for(int i=0;i<12;i++) for(int j=0;j<12;j++) { + uint8_t a[64],b[64],out[136]={0},want[5]; + for(int k=0;k<64;k++) {a[k]=(uint8_t)(k*7+91);b[k]=(uint8_t)(k*31+5);} + if(isDouble) { + double x,y;memcpy(a,&doubles[i],8);memcpy(b,&doubles[j],8);memcpy(&x,a,8);memcpy(&y,b,8); + __asm__ volatile("ucomisd %6,%5; sete %0; setb %1; setp %2; sets %3; seto %4":"=qm"(want[0]),"=qm"(want[1]),"=qm"(want[2]),"=qm"(want[3]),"=qm"(want[4]):"x"(x),"x"(y):"cc"); + } else { + float x,y;memcpy(a,&singles[i],4);memcpy(b,&singles[j],4);memcpy(&x,a,4);memcpy(&y,b,4); + __asm__ volatile("ucomiss %6,%5; sete %0; setb %1; setp %2; sets %3; seto %4":"=qm"(want[0]),"=qm"(want[1]),"=qm"(want[2]),"=qm"(want[3]),"=qm"(want[4]):"x"(x),"x"(y):"cc"); + } + fn(a,b,out); + if(memcmp(out,want,5)||memcmp(out+8,a,64)||memcmp(out+72,b,64)) { + fprintf(stderr,"flag compare mismatch double=%d i=%d j=%d\n",isDouble,i,j);return 1; + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + return source.String(), sigs, mainC, index +} diff --git a/amd64_raw_scalar_float_to_int.go b/amd64_raw_scalar_float_to_int.go new file mode 100644 index 000000000..60a7c82bd --- /dev/null +++ b/amd64_raw_scalar_float_to_int.go @@ -0,0 +1,108 @@ +package plan9asm + +import "fmt" + +// Decode the complete Go 1.27 scalar floating-point-to-GP family. Signed +// forms have VEX and EVEX encodings; unsigned forms are EVEX-only. The named +// lowerer and this decoder share amd64ScalarFloatToIntSpecs so W, F2/F3, +// truncation, and signedness cannot drift between the two entry points. +func decodedX86ScalarFloatToIntegerInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || (p.pp != 2 && p.pp != 3) { + return Instr{}, 0, false, nil + } + if p.opcode != 0x2c && p.opcode != 0x2d && p.opcode != 0x78 && p.opcode != 0x79 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("scalar float-to-integer: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + unsigned := p.opcode == 0x78 || p.opcode == 0x79 + truncating := p.opcode == 0x2c || p.opcode == 0x78 + if unsigned && !p.evex { + return fail("unsigned conversion requires EVEX") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 { + return fail("reserved vvvv/V' field") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register encoding in 32-bit mode") + } + if p.evex { + if !p.fixed || p.mask != 0 || p.zero || p.r >= 2 { + return fail("invalid EVEX fixed, mask, zeroing or GP extension field") + } + if p.broadcast && code[p.modRM]>>6 != 3 { + return fail("SAE or embedded rounding requires register source") + } + // EVEX.L'L is ignored without embedded rounding (and for CVTT SAE). + // Unlike VEX.L=1, Intel defines these as LLIG encodings. + } else if p.vectorLength != 0 { + return fail("scalar conversion requires VEX.128") + } + + inputBits := 32 + if p.pp == 3 { + inputBits = 64 + } + outputBits := 32 + if p.w { + outputBits = 64 + } + var op Op + for name, spec := range amd64ScalarFloatToIntSpecs { + if spec.vector && spec.inputBits == inputBits && spec.outputBits == outputBits && + spec.truncating == truncating && spec.unsigned == unsigned { + op = name + break + } + } + if op == "" { + return fail("encoding has no Go scalar conversion form") + } + if p.evex && p.broadcast { + if truncating { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", inputBits/8) + } else { + source, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + destinationNumber := int(code[p.modRM]>>3&7) + (p.r&1)*8 + destinationReg, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return fail("invalid GP destination") + } + destination := Operand{Kind: OpReg, Reg: destinationReg} + instruction := Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + x86ScalarFloatIntegerEVEX: p.evex, + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + _, _, err = x86ScalarFloatIntegerProperties(instruction, arch) + return instruction, p.modRM + consumed, true, err +} diff --git a/amd64_raw_scalar_float_to_int_test.go b/amd64_raw_scalar_float_to_int_test.go new file mode 100644 index 000000000..53126f7f1 --- /dev/null +++ b/amd64_raw_scalar_float_to_int_test.go @@ -0,0 +1,255 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func rawScalarFloatToIntBytes(spec amd64ScalarFloatToIntSpec, memory, evex bool) []byte { + opcode := byte(0x2d) + if spec.truncating { + opcode = 0x2c + } + if spec.unsigned { + opcode = 0x79 + if spec.truncating { + opcode = 0x78 + } + } + pp := byte(2) // F3: single precision. + if spec.inputBits == 64 { + pp = 3 // F2: double precision. + } + modRM := byte(0xc1) // X1 -> AX. + if memory { + modRM = 0x08 // (AX) -> CX. + } + if evex { + p1 := byte(0x7c) | pp + if spec.outputBits == 64 { + p1 |= 0x80 + } + return []byte{0x62, 0xf1, p1, 0x08, opcode, modRM} + } + if spec.outputBits == 64 { + return []byte{0xc4, 0xe1, 0xf8 | pp, opcode, modRM} + } + return []byte{0xc5, 0xf8 | pp, opcode, modRM} +} + +func TestX86RawScalarFloatToIntegerCompleteGo127Family(t *testing.T) { + count := 0 + for op, spec := range amd64ScalarFloatToIntSpecs { + if !spec.vector { + continue + } + for _, evex := range []bool{false, true} { + if spec.unsigned && !evex { + continue // Go has no unsigned VEX encoding. + } + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("%s/evex=%t/memory=%t", op, evex, memory) + t.Run(name, func(t *testing.T) { + code := rawScalarFloatToIntBytes(spec, memory, evex) + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", code, got.Instrs, op) + } + wantSource := Operand{Kind: OpReg, Reg: "X1"} + wantDestination := AX + if memory { + wantSource = Operand{Kind: OpMem, Mem: MemRef{Base: AX}} + wantDestination = CX + } + args := got.Instrs[0].Args + if args[0].Kind != wantSource.Kind || args[0].Reg != wantSource.Reg || args[0].Mem != wantSource.Mem || args[1].Reg != wantDestination { + t.Fatalf("decoded %#x operands as %#v, want %+v, %s", code, args, wantSource, wantDestination) + } + }) + count++ + } + } + } + if count != 48 { + t.Fatalf("tested %d raw VEX/EVEX forms, want 48", count) + } +} + +func TestX86RawScalarFloatToIntegerRoundingAndEncodingAxes(t *testing.T) { + base := []byte{0x62, 0xf1, 0x7f, 0x08, 0x2d, 0xc1} + for _, tc := range []struct { + name string + p2 byte + want Op + }{ + {"round-nearest", 0x18, "VCVTSD2SI.RN_SAE"}, + {"round-down", 0x38, "VCVTSD2SI.RD_SAE"}, + {"round-up", 0x58, "VCVTSD2SI.RU_SAE"}, + {"round-zero", 0x78, "VCVTSD2SI.RZ_SAE"}, + } { + t.Run(tc.name, func(t *testing.T) { + code := append([]byte(nil), base...) + code[3] = tc.p2 + got, length, recognized, err := decodedX86ScalarFloatToIntegerInstruction(code, 64) + if !recognized || err != nil || length != len(code) || got.Op != tc.want { + t.Fatalf("decode %#x = %+v, %d, %v, %v; want %s", code, got, length, recognized, err, tc.want) + } + }) + } + for _, tc := range []struct { + name string + code []byte + want Op + src Reg + dst Reg + }{ + {"signed-SAE", []byte{0x62, 0xf1, 0x7f, 0x18, 0x2c, 0xc1}, "VCVTTSD2SI.SAE", "X1", AX}, + {"unsigned-SAE", []byte{0x62, 0xf1, 0x7f, 0x18, 0x78, 0xc1}, "VCVTTSD2USIL.SAE", "X1", AX}, + {"extended-vector", []byte{0x62, 0xb1, 0x7f, 0x08, 0x78, 0xc1}, "VCVTTSD2USIL", "X17", AX}, + {"extended-GP", []byte{0x62, 0x71, 0x7f, 0x08, 0x78, 0xc1}, "VCVTTSD2USIL", "X1", "R8"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86ScalarFloatToIntegerInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) || got.Op != tc.want || + got.Args[0].Reg != tc.src || got.Args[1].Reg != tc.dst { + t.Fatalf("decode %#x = %+v, %d, %v, %v; want %s %s, %s", tc.code, got, length, recognized, err, tc.want, tc.src, tc.dst) + } + }) + } + for _, tc := range []struct { + name string + code []byte + off int64 + }{ + {"SS-disp8-scale", []byte{0x62, 0xf1, 0x7e, 0x08, 0x79, 0x48, 0x02}, 8}, + {"SD-disp8-scale", []byte{0x62, 0xf1, 0x7f, 0x08, 0x79, 0x48, 0x02}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + got, _, recognized, err := decodedX86ScalarFloatToIntegerInstruction(tc.code, 64) + if !recognized || err != nil || got.Args[0].Kind != OpMem || got.Args[0].Mem.Off != tc.off { + t.Fatalf("decode %#x = %+v, %v, %v; want displacement %d", tc.code, got, recognized, err, tc.off) + } + }) + } +} + +func TestX86RawScalarFloatToIntegerRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"VEX-vector-length", []byte{0xc5, 0xff, 0x2c, 0xc1}, 64}, + {"VEX-vvvv", []byte{0xc5, 0xeb, 0x2c, 0xc1}, 64}, + {"EVEX-mask", []byte{0x62, 0xf1, 0x7f, 0x09, 0x2c, 0xc1}, 64}, + {"EVEX-zero", []byte{0x62, 0xf1, 0x7f, 0x88, 0x2c, 0xc1}, 64}, + {"EVEX-vvvv", []byte{0x62, 0xf1, 0x6f, 0x08, 0x2c, 0xc1}, 64}, + {"EVEX-SAE-memory", []byte{0x62, 0xf1, 0x7f, 0x18, 0x2c, 0x00}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xfb, 0x2c, 0xc1}, 64}, + {"missing-ModRM", []byte{0xc5, 0xfb, 0x2c}, 64}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86ScalarFloatToIntegerInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } +} + +func TestX86RawScalarFloatToIntegerCompilesEveryGoForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawscalarconvert(SB),4,$0-0\n") + for _, spec := range amd64ScalarFloatToIntSpecs { + if !spec.vector { + continue + } + for _, evex := range []bool{false, true} { + if spec.unsigned && !evex { + continue + } + for _, memory := range []bool{false, true} { + for _, value := range rawScalarFloatToIntBytes(spec, memory, evex) { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + } + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-scalar-convert.ll", "raw-scalar-convert.o", ir) + }) + } +} + +func x86RawScalarFloatToIntegerNativeRuntime(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + const source = `TEXT rawfloattointsemantics(SB),4,$0-16 + MOVQ input+0(FP), BX + MOVQ out+8(FP), DI + BYTE $0xc5; BYTE $0xfb; BYTE $0x2c; BYTE $0x03 + MOVL AX, 0(DI) + BYTE $0xc5; BYTE $0xfb; BYTE $0x2d; BYTE $0x03 + MOVL AX, 4(DI) + BYTE $0x62; BYTE $0xf1; BYTE $0x7f; BYTE $0x08; BYTE $0x78; BYTE $0x03 + MOVL AX, 8(DI) + BYTE $0x62; BYTE $0xf1; BYTE $0x7f; BYTE $0x08; BYTE $0x79; BYTE $0x03 + MOVL AX, 12(DI) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: execution.triple, + Sigs: map[string]FuncSig{"rawfloattointsemantics": { + Name: "rawfloattointsemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void rawfloattointsemantics(const double *, uint32_t *); +int main(void) { + double input = 2.75; + uint32_t out[4] = {0}; + rawfloattointsemantics(&input, out); + if (out[0] != 2) return 1; + if (out[1] != 3) return 2; + if (out[2] != 2) return 3; + if (out[3] != 3) return 4; + return 0; +} +` + execution.compileAndRunLLVM(t, "raw-float-to-integer", ir, mainC, "-O2") +} diff --git a/amd64_raw_scalar_integer_float_test.go b/amd64_raw_scalar_integer_float_test.go new file mode 100644 index 000000000..0a5ddc014 --- /dev/null +++ b/amd64_raw_scalar_integer_float_test.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXScalarIntegerFloat(pp byte, width64, vex3 bool, destination, passthrough, source int) []byte { + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + if !vex3 { + vex1 := byte((1-(destination>>3)&1)<<7 | (^passthrough&15)<<3 | int(pp)) + return []byte{0xc5, vex1, 0x2a, modRM} + } + vex0 := byte((1-(destination>>3)&1)<<7 | 1<<6 | (1-(source>>3)&1)<<5 | 1) + vex1 := byte((^passthrough&15)<<3 | int(pp)) + if width64 { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, 0x2a, modRM} +} + +func encodeX86EVEXScalarIntegerFloat(unsigned bool, pp byte, width64 bool, vectorBits byte, rounding bool, destination, passthrough, source int) []byte { + opcode := byte(0x2a) + if unsigned { + opcode = 0x7b + } + p0 := byte((1-(destination>>3)&1)<<7 | 1<<6 | (1-(source>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 1) + p1 := byte((^passthrough&15)<<3 | 1<<2 | int(pp)) + if width64 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^passthrough>>4&1)<<3) + if rounding { + p2 |= 0x10 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func scalarIntegerFloatRawOp(unsigned bool, pp byte, width64 bool) Op { + prefix := "VCVTSI2" + if unsigned { + prefix = "VCVTUSI2" + } + result := "SS" + if pp == 3 { + result = "SD" + } + source := "L" + if width64 { + source = "Q" + } + return Op(prefix + result + source) +} + +func TestTranslateRawVEXScalarIntegerFloatWeaviateRegression(t *testing.T) { + const source = `TEXT rawScalarIntegerFloat(SB), $0-0 + LONG $0xc82aeac5 + LONG $0xc02a9ac5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawScalarIntegerFloat": {Name: "rawScalarIntegerFloat", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if strings.Count(ir, "call float @llvm.experimental.constrained.sitofp.f32.i32") != 2 { + t.Fatalf("raw VCVTSI2SSL lowering omitted signed conversions:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-scalar-integer-float.ll", "amd64-raw-scalar-integer-float.o", ir) +} + +func TestDecodedX86VEXScalarIntegerFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, pp := range []byte{2, 3} { + for destination := 0; destination < 16; destination++ { + for passthrough := 0; passthrough < 16; passthrough++ { + for source := 0; source < 8; source++ { + code := encodeX86VEXScalarIntegerFloat(pp, false, false, destination, passthrough, source) + assertDecodedX86ScalarIntegerFloat(t, code, scalarIntegerFloatRawOp(false, pp, false), destination, passthrough, source, "") + count++ + } + } + } + for _, width64 := range []bool{false, true} { + for destination := 0; destination < 16; destination++ { + for passthrough := 0; passthrough < 16; passthrough++ { + for source := 0; source < 16; source++ { + code := encodeX86VEXScalarIntegerFloat(pp, width64, true, destination, passthrough, source) + assertDecodedX86ScalarIntegerFloat(t, code, scalarIntegerFloatRawOp(false, pp, width64), destination, passthrough, source, "") + count++ + } + } + } + } + } + if count != 20480 { + t.Fatalf("covered %d VEX scalar integer-to-float encodings, want 20480", count) + } +} + +func TestDecodedX86EVEXScalarIntegerFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, unsigned := range []bool{false, true} { + for _, pp := range []byte{2, 3} { + for _, width64 := range []bool{false, true} { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + for destination := 0; destination < 32; destination++ { + for passthrough := 0; passthrough < 32; passthrough++ { + for source := 0; source < 16; source++ { + code := encodeX86EVEXScalarIntegerFloat(unsigned, pp, width64, vectorBits, false, destination, passthrough, source) + assertDecodedX86ScalarIntegerFloat(t, code, scalarIntegerFloatRawOp(unsigned, pp, width64), destination, passthrough, source, "") + count++ + } + } + } + } + } + } + } + if count != 393216 { + t.Fatalf("covered %d EVEX scalar integer-to-float encodings, want 393216", count) + } +} + +func TestDecodedX86EVEXScalarIntegerFloatRoundingMemoryAndInvalid(t *testing.T) { + for vectorBits, suffix := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + code := encodeX86EVEXScalarIntegerFloat(false, 2, true, byte(vectorBits), true, 17, 18, 9) + assertDecodedX86ScalarIntegerFloat(t, code, "VCVTSI2SSQ."+Op(suffix), 17, 18, 9, "") + } + memory := []byte{0x62, 0xe1, 0xff, 0x00, 0x2a, 0x60, 0x02} + got, length, ok, err := decodedX86ScalarIntegerFloatInstruction(memory, 64) + if err != nil || !ok || length != len(memory) || got.Op != "VCVTSI2SDQ" || len(got.Args) != 3 || got.Args[0].String() != "16(AX)" || got.Args[1].String() != "X16" || got.Args[2].String() != "X20" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", memory, got, length, ok, err) + } + + roundingMemory := encodeX86EVEXScalarIntegerFloat(false, 2, true, 0, true, 0, 0, 0) + roundingMemory[len(roundingMemory)-1] = 0 + nonRounding := encodeX86EVEXScalarIntegerFloat(false, 3, false, 0, true, 0, 0, 0) + invalid := [][]byte{ + encodeX86VEXScalarIntegerFloat(1, false, false, 0, 0, 0), + encodeX86EVEXScalarIntegerFloat(false, 2, false, 3, false, 0, 0, 0), + roundingMemory, + nonRounding, + } + for _, code := range invalid { + if _, _, ok, err := decodedX86ScalarIntegerFloatInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func assertDecodedX86ScalarIntegerFloat(t *testing.T, code []byte, op Op, destination, passthrough, source int, sourceText string) { + t.Helper() + got, length, ok, err := decodedX86ScalarIntegerFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if sourceText == "" { + sourceText = decodedX86ScalarIntegerFloatTestRegister(source) + } + wantArgs := []string{sourceText, fmt.Sprintf("X%d", passthrough), fmt.Sprintf("X%d", destination)} + if got.Op != op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, op, strings.Join(wantArgs, ", ")) + } +} + +func decodedX86ScalarIntegerFloatTestRegister(number int) string { + register, ok := decodedX86GeneralRegister(number) + if !ok { + return fmt.Sprintf("invalid%d", number) + } + return string(register) +} diff --git a/amd64_raw_scalar_move.go b/amd64_raw_scalar_move.go new file mode 100644 index 000000000..944e5d004 --- /dev/null +++ b/amd64_raw_scalar_move.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// x86RawVectorEncoding separates the shared VEX/EVEX prefix axes from a +// family's operand grammar. Register numbers are decoded (not complemented). +type x86RawVectorEncoding struct { + modRM, mapNumber, pp, opcode int + r, x, b, upper int + vectorLength, mask int + evex, w, zero, broadcast bool + fixed, addressOverride bool + segment Reg +} + +func decodeX86RawVectorEncoding(code []byte) (p x86RawVectorEncoding, ok bool) { + i := 0 + for i < len(code) { + switch code[i] { + case 0x64: + p.segment = FS + case 0x65: + p.segment = GS + case 0x67: + p.addressOverride = true + default: + goto vector + } + i++ + } + return p, false +vector: + if i >= len(code) { + return p, false + } + var bits byte + switch code[i] { + case 0xc5: + if len(code) < i+3 { + return p, false + } + bits = code[i+1] + p.r, p.mapNumber, p.modRM = int(^bits>>7)&1, 1, i+3 + case 0xc4: + if len(code) < i+4 { + return p, false + } + first := code[i+1] + bits = code[i+2] + p.r, p.x, p.b = int(^first>>7)&1, int(^first>>6)&1, int(^first>>5)&1 + p.mapNumber, p.modRM, p.w = int(first&31), i+4, bits&0x80 != 0 + case 0x62: + if len(code) < i+5 { + return p, false + } + first, last := code[i+1], code[i+3] + bits = code[i+2] + p.evex, p.w, p.fixed = true, bits&0x80 != 0, bits&4 != 0 + p.r, p.x, p.b = (int(^first>>7)&1)|(int(^first>>4)&1)<<1, int(^first>>6)&1, int(^first>>5)&1 + p.mapNumber, p.modRM = int(first&15), i+5 + p.vectorLength, p.mask = int(last>>5)&3, int(last&7) + p.zero, p.broadcast = last&0x80 != 0, last&0x10 != 0 + p.upper = (int(^last>>3) & 1) << 4 + default: + return p, false + } + p.opcode, p.pp = int(code[p.modRM-1]), int(bits&3) + p.upper |= int(^bits>>3) & 15 + if !p.evex { + p.vectorLength = int(bits>>2) & 1 + } + return p, true +} + +// Go's _yvmovsd grammar is shared by VMOVSS and VMOVSD. Opcodes 10 and 11 +// both encode three-register merges; memory forms reserve vvvv and EVEX V'. +func decodedX86ScalarMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || (p.opcode != 0x10 && p.opcode != 0x11) { + return Instr{}, 0, false, nil + } + fp16 := p.evex && p.mapNumber == 5 && p.pp == 2 + if !fp16 && (p.mapNumber != 1 || (p.pp != 2 && p.pp != 3)) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("scalar move: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + // EVEX VMOVSH is LLIG; VMOVSS/SD reserve every nonzero length. + if (!fp16 && p.vectorLength != 0) || p.broadcast { + return fail("scalar moves require 128-bit encoding without broadcast/rounding") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + spec := map[int]struct { + op Op + bytes int + }{2: {"VMOVSS", 4}, 3: {"VMOVSD", 8}}[p.pp] + if fp16 { + spec.op, spec.bytes = "VMOVSH", 2 + } + if p.evex && (!p.fixed || p.w != (spec.bytes == 8)) { + return fail("invalid EVEX fixed or width bit") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + registerForm := code[p.modRM]>>6 == 3 + if !registerForm && p.upper != 0 { + return fail("memory forms reserve vvvv and V'") + } + store := p.opcode == 0x11 + if store && !registerForm && p.zero { + return fail("memory stores cannot zero-mask") + } + regNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (regNumber >= 8 || p.upper >= 8 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + var rm Operand + var consumed int + var err error + if p.evex { + rm, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", spec.bytes) + } else { + rm, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + reg := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", regNumber))} + source, destination := rm, reg + if store { + source, destination = reg, rm + } + args := []Operand{source} + if registerForm { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}) + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + op := spec.op + if p.zero { + op += ".Z" + } + printed := make([]string, len(args)) + for i := range args { + printed[i] = args[i].String() + } + return Instr{ + Op: op, Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86Encoded: fp16, + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_scalar_move_literal_test.go b/amd64_raw_scalar_move_literal_test.go new file mode 100644 index 000000000..6bfb2910e --- /dev/null +++ b/amd64_raw_scalar_move_literal_test.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawScalarMoveLiteral(double bool, encoding string, mask int, zeroing bool) []byte { + pp := byte(2) + width := 4 + if double { + pp = 3 + width = 8 + } + var code []byte + switch encoding { + case "VEX2": + code = []byte{0xc5, 0xf8 | pp, 0x10, 0x05} + case "VEX3": + code = []byte{0xc4, 0xe1, 0x78 | pp, 0x10, 0x05} + case "EVEX": + p1 := byte(0x7c) | pp + if double { + p1 |= 0x80 + } + p2 := byte(0x08 | mask) + if zeroing { + p2 |= 0x80 + } + code = []byte{0x62, 0xf1, p1, p2, 0x10, 0x05} + } + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*17+5)) + } + return code +} + +func TestDecodeX86RawScalarMoveRIPDataCompleteLoadForms(t *testing.T) { + type masking struct { + mask int + zeroing bool + } + for _, double := range []bool{false, true} { + for _, encoding := range []string{"VEX2", "VEX3", "EVEX"} { + maskings := []masking{{}} + if encoding == "EVEX" { + maskings = append(maskings, masking{mask: 3}, masking{mask: 7, zeroing: true}) + } + for _, masking := range maskings { + name := fmt.Sprintf("double%t/%s/mask%d/zero%t", double, encoding, masking.mask, masking.zeroing) + t.Run(name, func(t *testing.T) { + code := x86RawScalarMoveLiteral(double, encoding, masking.mask, masking.zeroing) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op("VMOVSS") + if double { + want = "VMOVSD" + } + if masking.zeroing { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } +} + +func TestTranslateX86RawScalarMoveRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for _, double := range []bool{false, true} { + for _, encoding := range []string{"VEX2", "VEX3", "EVEX"} { + name := fmt.Sprintf("scalarMoveLiteral%d", len(sigs)) + mask, zeroing := 0, false + if encoding == "EVEX" { + mask, zeroing = 7, true + } + code := x86RawScalarMoveLiteral(double, encoding, mask, zeroing) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + width := 4 + if double { + width = 8 + } + pools = append(pools, code[len(code)-width:]) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + var initializedSource, initializedDeclarations strings.Builder + for index, function := range file.Funcs { + if index%3 == 2 { + continue // The original EVEX source has no K7 producer. + } + text, valid := x86RawReplaySourceText(function) + if !valid { + t.Fatal("original initialized VEX constant source lost its bytes") + } + initializedSource.WriteString(text) + fmt.Fprintf(&initializedDeclarations, "func %s()\n", function.Sym) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Keep all six original functions and pools in format coverage. + // Only the two EVEX functions lack numerical mask inputs. + ir := x86UnboundVoidVectorMoveFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "scalar-move-literal.ll", "scalar-move-literal.o", ir) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, initializedSource.String(), initializedDeclarations.String(), "amd64", triple, api) + if err != nil { + t.Fatal("valid original unmasked constants regressed", api, err) + } + compileLLVMToObject(t, llc, triple, "initialized-scalar-literal.ll", "initialized-scalar-literal.o", ir) + } + }) + } +} + +func TestDecodeX86RawScalarMoveRIPDataRejectsUnsafeSources(t *testing.T) { + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "missing_pool", edit: func(code []byte) []byte { + code[6] = 0x40 + return code + }}, + {name: "instruction_overlap", edit: func(code []byte) []byte { + copy(code[6:10], []byte{0xfa, 0xff, 0xff, 0xff}) + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x65}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.edit(x86RawScalarMoveLiteral(true, "EVEX", 0, false)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe scalar-move source %x", code) + } + }) + } +} diff --git a/amd64_raw_scalar_move_test.go b/amd64_raw_scalar_move_test.go new file mode 100644 index 000000000..7ef9d4172 --- /dev/null +++ b/amd64_raw_scalar_move_test.go @@ -0,0 +1,332 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestDecodeX86RawScalarMoveCompleteGoForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT scalarmoves(SB),4,$0-0\n") + var lines []string + for _, op := range []string{"VMOVSS", "VMOVSD"} { + for _, registers := range [][]int{{0, 1, 7}, {8, 9, 15}, {16, 23, 31}} { + for _, mask := range []string{"", "K1, ", "K7, "} { + for _, suffix := range []string{"", ".Z"} { + if mask == "" && suffix != "" { + continue + } + for _, memory := range []string{"(AX)", "-8(R11)(R9*4)", "1024(R11)"} { + lines = append(lines, fmt.Sprintf("%s%s %s, %sX%d", op, suffix, memory, mask, registers[2])) + if suffix == "" { + lines = append(lines, fmt.Sprintf("%s X%d, %s%s", op, registers[0], mask, memory)) + } + } + lines = append(lines, fmt.Sprintf("%s%s X%d, X%d, %sX%d", op, suffix, registers[0], registers[1], mask, registers[2])) + } + } + } + } + source.WriteString(strings.Join(lines, "\n") + "\nRET\n") + code := assembleX87ControlBytes(t, "amd64", source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := file.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, ins := range decoded.Instrs { + if ins.Op != want[i].Op || !reflect.DeepEqual(ins.Args, want[i].Args) { + t.Fatalf("instruction %d decoded %+v, want %+v", i, ins, want[i]) + } + } + rawFile := x86RawUnprovedReturnFormProbeFile(t, "amd64", "scalarmoves", code) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + ir := x86UnboundVoidRawVectorMoveProbeObject(t, rawFile, "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-scalar-moves.ll", "raw-scalar-moves.o", ir) + } +} + +func TestTranslate386VectorScalarMoveCompleteGoForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT moves386(SB),4,$0-0\n") + for _, op := range []string{"VMOVSS", "VMOVSD"} { + for _, x := range []string{"X1", "X15", "X31"} { + fmt.Fprintf(&source, "%s (AX),%s\n%s %s,(BX)\n%s %s,X2,X3\n%s (AX),K7,%s\n%s.Z (AX),K7,%s\n%s %s,K7,(BX)\n", op, x, op, x, op, x, op, x, op, x, op, x) + } + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir := x86UnboundVoidVectorMoveFormatObject(t, source.String(), "386", triple) + compileLLVMToObject(t, llc, triple, "scalar-moves386.ll", "scalar-moves386.o", ir) + } + // Raw 32-bit encodings use the architectural X0-X7 range; named Go + // compatibility rows above also expose high EVEX register classes. + rawSource := strings.NewReplacer("X15", "X1", "X31", "X1").Replace(source.String()) + code := assembleX87ControlBytes(t, "386", rawSource) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "386") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, rawSource) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("386 raw instruction count=%d, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("386 instruction %d=%+v, want %+v", i, got, want[i]) + } + } +} + +// Unlike Go's canonical encoder, exercise both equivalent register directions +// and every independent R/R'/B/X/vvvv/V' bit, including EVEX masking. +func encodeRawScalarMove(evex bool, pp, opcode, low, upper, dst, mask int, zero bool) []byte { + reg, rm := dst, low + if opcode == 0x11 { + reg, rm = low, dst + } + modrm := byte(0xc0 | (reg&7)<<3 | rm&7) + if !evex { + return []byte{0xc4, byte((^reg>>3&1)<<7 | 0x40 | (^rm>>3&1)<<5 | 1), byte((^upper&15)<<3 | pp), byte(opcode), modrm} + } + p0 := byte((^reg>>3&1)<<7 | (^rm>>4&1)<<6 | (^rm>>3&1)<<5 | (^reg>>4&1)<<4 | 1) + p1 := byte((^upper&15)<<3 | 4 | pp) + if pp == 3 { + p1 |= 0x80 + } + p2 := byte((^upper>>4&1)<<3 | mask) + if zero { + p2 |= 0x80 + } + return []byte{0x62, p0, p1, p2, byte(opcode), modrm} +} + +func TestDecodedX86RawScalarMoveRegisterEncodings(t *testing.T) { + count := 0 + for _, evex := range []bool{false, true} { + limit := 16 + if evex { + limit = 32 + } + for pp, op := range map[int]Op{2: "VMOVSS", 3: "VMOVSD"} { + for _, opcode := range []int{0x10, 0x11} { + for low := 0; low < limit; low++ { + for upper := 0; upper < limit; upper++ { + for dst := 0; dst < limit; dst++ { + mask, zero := 0, false + if evex { + mask = (low + upper + dst) & 7 + zero = mask != 0 && dst&1 != 0 + } + code := encodeRawScalarMove(evex, pp, opcode, low, upper, dst, mask, zero) + got, size, ok, err := decodedX86ScalarMoveInstruction(code, 64) + wantOp := op + if zero { + wantOp += ".Z" + } + want := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", low))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", upper))}} + if mask != 0 { + want = append(want, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + want = append(want, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", dst))}) + if err != nil || !ok || size != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x = %+v, size=%d, ok=%v, err=%v; want %s %+v", code, got, size, ok, err, wantOp, want) + } + count++ + } + } + } + } + } + } + if count != 147456 { + t.Fatalf("covered %d encodings, want 147456", count) + } +} + +func TestDecodedX86RawScalarMoveMemoryAndInvalidForms(t *testing.T) { + for pp, op := range map[int]Op{2: "VMOVSS", 3: "VMOVSD"} { + for _, segment := range []byte{0x64, 0x65} { + code := []byte{segment, 0x62, 0x01, byte(0x7c | pp), 0x08, 0x10, 0x7c, 0x8b, 0xff} + scale := int64(4) + if pp == 3 { + code[3] |= 0x80 + scale = 8 + } + got, size, ok, err := decodedX86ScalarMoveInstruction(code, 64) + seg := FS + if segment == 0x65 { + seg = GS + } + wantMem := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: -scale, Segment: seg} + if err != nil || !ok || size != len(code) || got.Op != op || len(got.Args) != 2 || !reflect.DeepEqual(got.Args[0].Mem, wantMem) || got.Args[1].Reg != "X31" { + t.Fatalf("decode %x = %+v, size=%d, ok=%v, err=%v; want memory %+v", code, got, size, ok, err, wantMem) + } + } + } + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"missing-modrm", []byte{0xc5, 0xfa, 0x10}, 64}, + {"missing-sib", []byte{0xc5, 0xfa, 0x10, 0x04}, 64}, + {"missing-displacement", []byte{0xc5, 0xfa, 0x10, 0x45}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xfa, 0x10, 0xc0}, 64}, + {"vex-length", []byte{0xc5, 0xfe, 0x10, 0xc0}, 64}, + {"vex-memory-vvvv", []byte{0xc5, 0xf2, 0x10, 0x00}, 64}, + {"evex-fixed", []byte{0x62, 0xf1, 0x7a, 0x08, 0x10, 0xc0}, 64}, + {"evex-ss-width", []byte{0x62, 0xf1, 0xfe, 0x08, 0x10, 0xc0}, 64}, + {"evex-sd-width", []byte{0x62, 0xf1, 0x7f, 0x08, 0x10, 0xc0}, 64}, + {"evex-length", []byte{0x62, 0xf1, 0x7e, 0x28, 0x10, 0xc0}, 64}, + {"evex-broadcast", []byte{0x62, 0xf1, 0x7e, 0x18, 0x10, 0xc0}, 64}, + {"zero-k0", []byte{0x62, 0xf1, 0x7e, 0x88, 0x10, 0xc0}, 64}, + {"memory-vprime", []byte{0x62, 0xf1, 0x7e, 0x00, 0x10, 0x00}, 64}, + {"zero-store", []byte{0x62, 0xf1, 0x7e, 0x89, 0x11, 0x00}, 64}, + {"386-high-register", []byte{0xc5, 0x7a, 0x10, 0xc0}, 32}, + {"wrong-mode", []byte{0xc5, 0xfa, 0x10, 0xc0}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, ok, err := decodedX86ScalarMoveInstruction(tc.code, tc.mode); !ok || err == nil { + t.Fatalf("invalid %x: matched=%v, err=%v", tc.code, ok, err) + } + }) + } + for _, code := range [][]byte{nil, {0x64}, {0xc5}, {0xc4, 0xe1}, {0x62, 0xf1, 0x7e}, {0xc5, 0xf9, 0x10, 0xc0}, {0xc5, 0xfa, 0x5a, 0xc0}} { + if _, _, ok, err := decodedX86ScalarMoveInstruction(code, 64); ok || err != nil { + t.Fatalf("unrelated/truncated prefix %x: matched=%v, err=%v", code, ok, err) + } + } +} + +func TestX86RawScalarMoveRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; three-target object coverage is required separately") + } + var source, declarations, calls strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, pp := range []int{2, 3} { + width := 4 + if pp == 3 { + width = 8 + } + for _, kind := range []string{"merge10", "merge11", "load", "store"} { + for _, encoding := range []string{"vex", "evex", "masked", "zero"} { + if kind == "store" && encoding == "zero" { + continue + } + mask, zero := 0, encoding == "zero" + if encoding == "masked" || zero { + mask = 1 + } + opcode := 0x10 + if kind == "merge11" || kind == "store" { + opcode = 0x11 + } + code := encodeRawScalarMove(encoding != "vex", pp, opcode, 0, 1, 2, mask, zero) + if kind == "load" || kind == "store" { + // Memory is (AX), destination/source X2. vvvv/V' must be reserved. + code = encodeRawScalarMove(encoding != "vex", pp, opcode, 2, 0, 2, mask, zero) + code[len(code)-1] = 0x10 + } + name := fmt.Sprintf("rawscalarmove%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),DI\nMOVQ in+8(FP),SI\nMOVQ mem+16(FP),AX\nMOVQ mask+24(FP),CX\nKMOVQ CX,K1\nVMOVUPS 32(SI),Y2\nVMOVUPS 32(SI),Z2\nMOVOU (SI),X0\nMOVOU 16(SI),X1\nMOVOU 32(SI),X2\n", name) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + source.WriteString("MOVOU X2,(DI)\nVMOVUPS Y2,16(DI)\nVMOVUPS Z2,48(DI)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void *,const void *,void *,uint64_t);\n", name) + zeroFlag := 0 + if zero { + zeroFlag = 1 + } + fmt.Fprintf(&calls, "if (check(%s,%q,%d,%d,%d)) return %d;\n", name, kind, width, mask, zeroFlag, index) + } + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple, runPrefix = "x86_64-apple-macosx", []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +` + declarations.String() + ` +static int check(void (*fn)(void *,const void *,void *,uint64_t), const char *kind, int width, int masked, int zero) { + for (int m=0;m<4;m++) { + uint8_t in[96], out[112], want[16], mem[32], old[32]; + for (int i=0;i<96;i++) in[i]=(uint8_t)(17+i*3); + for (int i=0;i<32;i++) mem[i]=old[i]=(uint8_t)(255-i*7); + int active=!masked || (m&1), store=strcmp(kind,"store")==0, load=strcmp(kind,"load")==0; + memcpy(want,in+32,16); + if (!store) { + if (load) memset(want,0,16); else memcpy(want,in+16,16); + if (active) memcpy(want,load ? mem+1 : in,width); + else if (zero) memset(want,0,width); else memcpy(want,in+32,width); + } + fn(out,in,mem+1,(uint64_t)m); + if (memcmp(out,want,16)) { fprintf(stderr,"%s width=%d mask=%d zero=%d register mismatch\n",kind,width,m,zero); return 1; } + for (int v=0;v<2;v++) { + int start=v ? 48 : 16, bytes=v ? 64 : 32; + for (int b=0;b>3&1 | destination>>3&1<<2) + code := []byte{spec.legacyPrefix} + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0x5a, byte(0xc0|destination&7<<3|source&7)) +} + +func encodeX86RawVEXScalarPrecisionConvert(spec rawScalarPrecisionConvertSpec, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^first>>3)&1)<<5 | 1 + pp := byte(3) + if spec.inputBytes == 4 { + pp = 2 + } + p1 := byte(^second&15)<<3 | pp + return []byte{0xc4, p0, p1, 0x5a, byte(0xc0 | destination&7<<3 | first&7)} +} + +func encodeX86RawEVEXScalarPrecisionConvert(spec rawScalarPrecisionConvertSpec, control int, mask int, zeroing bool, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 1 + pp := byte(3) + if spec.inputBytes == 4 { + pp = 2 + } + p1 := byte(^second&15)<<3 | 4 | pp + if spec.inputBytes == 8 { + p1 |= 0x80 + } + p2 := byte((^second>>4)&1)<<3 | byte(mask) + if control >= 0 { + p2 |= 0x10 | byte(control)<<5 + } + if zeroing { + p2 |= 0x80 + } + return []byte{0x62, p0, p1, p2, 0x5a, byte(0xc0 | destination&7<<3 | first&7)} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVCVTSD2SS(t *testing.T) { + // vcvtsd2ss %xmm0, %xmm0, %xmm0; + // vmovsd (%rcx), %xmm1 + code := []byte{0xc5, 0xfb, 0x5a, 0xc0, 0xc5, 0xfb, 0x10, 0x09} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VCVTSD2SS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VCVTSD2SS X0, X0, X0 ") || !strings.HasPrefix(decoded[1].Raw, "VMOVSD 0(CX), X1 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawScalarPrecisionConvertCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawScalarPrecisionConvertSpecs { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawLegacyScalarPrecisionConvert(spec, source, destination) + got, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(code, 64) + want := fmt.Sprintf("%s X%d, X%d", spec.legacy, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXScalarPrecisionConvert(spec, first, second, destination) + got, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(code, 64) + want := fmt.Sprintf("%s X%d, X%d, X%d", spec.vector, first, second, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + controls := []int{-1, 0} + if spec.inputBytes == 8 { + controls = []int{-1, 0, 1, 2, 3} + } + for _, control := range controls { + for _, masking := range []struct { + mask int + zeroing bool + }{{}, {mask: 3}, {mask: 7, zeroing: true}} { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXScalarPrecisionConvert(spec, control, masking.mask, masking.zeroing, first, second, destination) + got, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(code, 64) + suffix := "" + if control >= 0 { + if spec.inputBytes == 4 { + suffix = ".SAE" + } else { + suffix = "." + [...]string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"}[control] + } + } + if masking.zeroing { + suffix += ".Z" + } + args := []string{fmt.Sprintf("X%d", first), fmt.Sprintf("X%d", second)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("X%d", destination)) + want := string(spec.vector) + suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 696832 { + t.Fatalf("covered %d scalar precision-conversion register encodings, want 696832", count) + } +} + +func TestDecodedX86RawScalarPrecisionConvertMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0xf3, 0x47, 0x0f, 0x5a, 0x7c, 0x8b, 0x01} + got, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "CVTSS2SD" || got.Args[1].String() != "X15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x01, 0x73, 0x5a, 0x7c, 0x8b, 0x02} + got, length, ok, err = decodedX86ScalarPrecisionConvertInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VCVTSD2SS" || got.Args[1].String() != "X1" || got.Args[2].String() != "X15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexMemory := []byte{0x62, 0xf1, 0x56, 0x83, 0x5a, 0x50, 0x02} + got, length, ok, err = decodedX86ScalarPrecisionConvertInstruction(evexMemory, 64) + wantMemory = MemRef{Base: "AX", Off: 8} + if err != nil || !ok || length != len(evexMemory) || got.Op != "VCVTSS2SD.Z" || got.Args[1].String() != "X21" || got.Args[2].String() != "K3" || got.Args[3].String() != "X2" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + validVEX := encodeX86RawVEXScalarPrecisionConvert(rawScalarPrecisionConvertSpecs[0], 7, 6, 5) + validEVEXSS := encodeX86RawEVEXScalarPrecisionConvert(rawScalarPrecisionConvertSpecs[0], -1, 0, false, 7, 6, 5) + validEVEXSD := encodeX86RawEVEXScalarPrecisionConvert(rawScalarPrecisionConvertSpecs[1], -1, 0, false, 7, 6, 5) + for name, invalid := range map[string][]byte{ + "vex W": {validVEX[0], validVEX[1], validVEX[2] | 0x80, validVEX[3], validVEX[4]}, + "vex L": {validVEX[0], validVEX[1], validVEX[2] | 0x04, validVEX[3], validVEX[4]}, + "evex fixed bit": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2] &^ 0x04, validEVEXSS[3], validEVEXSS[4], validEVEXSS[5]}, + "evex SS wrong W": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2] | 0x80, validEVEXSS[3], validEVEXSS[4], validEVEXSS[5]}, + "evex SD wrong W": {validEVEXSD[0], validEVEXSD[1], validEVEXSD[2] &^ 0x80, validEVEXSD[3], validEVEXSD[4], validEVEXSD[5]}, + "evex length without SAE": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2], validEVEXSS[3] | 0x20, validEVEXSS[4], validEVEXSS[5]}, + "zero without mask": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2], validEVEXSS[3] | 0x80, validEVEXSS[4], validEVEXSS[5]}, + "SS explicit rounding": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2], validEVEXSS[3] | 0x30, validEVEXSS[4], validEVEXSS[5]}, + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86ScalarPrecisionConvertInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + roundingMemory := []byte{0x62, 0xf1, 0xff, 0x10, 0x5a, 0x00} + if instruction, _, matched, decodeErr := decodedX86ScalarPrecisionConvertInstruction(roundingMemory, 64); !matched || decodeErr == nil { + t.Fatalf("rounding memory encoding %x decoded as %+v, ok=%v, err=%v", roundingMemory, instruction, matched, decodeErr) + } + masked386 := encodeX86RawEVEXScalarPrecisionConvert(rawScalarPrecisionConvertSpecs[0], -1, 1, false, 0, 1, 2) + if instruction, _, matched, decodeErr := decodedX86ScalarPrecisionConvertInstruction(masked386, 32); !matched || decodeErr == nil { + t.Fatalf("386 masked encoding %x decoded as %+v, ok=%v, err=%v", masked386, instruction, matched, decodeErr) + } +} + +func TestTranslateX86RawScalarPrecisionConvertTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_scalar_precision_convert(SB),NOSPLIT,$0-0\n") + for _, spec := range rawScalarPrecisionConvertSpecs { + var encodings [][]byte + if target.goarch == "amd64" { + control := 0 + if spec.inputBytes == 8 { + control = 3 + } + encodings = append(encodings, + encodeX86RawLegacyScalarPrecisionConvert(spec, 15, 14), + encodeX86RawVEXScalarPrecisionConvert(spec, 15, 14, 13), + encodeX86RawEVEXScalarPrecisionConvert(spec, -1, 3, true, 31, 30, 29), + encodeX86RawEVEXScalarPrecisionConvert(spec, control, 7, true, 31, 30, 29), + ) + } else { + control := 0 + if spec.inputBytes == 8 { + control = 2 + } + encodings = append(encodings, + encodeX86RawLegacyScalarPrecisionConvert(spec, 7, 6), + encodeX86RawVEXScalarPrecisionConvert(spec, 7, 6, 5), + encodeX86RawEVEXScalarPrecisionConvert(spec, -1, 0, false, 20, 21, 22), + encodeX86RawEVEXScalarPrecisionConvert(spec, control, 0, false, 20, 21, 22), + ) + } + for _, encoding := range encodings { + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + ll := x86ScalarPrecisionOriginalFormatObject(t, source.String(), "raw_scalar_precision_convert", target.goarch, target.triple, true) + compileLLVMToObject(t, llc, target.triple, "raw-scalar-precision-convert-"+target.name+".ll", "raw-scalar-precision-convert-"+target.name+".o", ll) + }) + } +} diff --git a/amd64_raw_scalar_reciprocal.go b/amd64_raw_scalar_reciprocal.go new file mode 100644 index 000000000..32baf491a --- /dev/null +++ b/amd64_raw_scalar_reciprocal.go @@ -0,0 +1,80 @@ +package plan9asm + +import "fmt" + +// Intel RCPSS/RSQRTSS share legacy F3.0F and VEX.LIG.F3.0F.WIG +// register/m32 rows. Ignored W/L bits do not widen the selected scalar or +// change the upper source. EVEX belongs to other estimate families. +func decodedX86RawScalarReciprocalInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, vector := decodeX86RawVectorEncoding(code) + if vector { + if p.mapNumber != 1 || p.pp != 2 || p.opcode != 0x52 && p.opcode != 0x53 { + return Instr{}, 0, false, nil + } + } else { + i := 0 + mandatory := false + for i < len(code) { + switch code[i] { + case 0x64: + p.segment = FS + case 0x65: + p.segment = GS + case 0x67: + p.addressOverride = true + case 0xf3: + mandatory = true + default: + goto legacy + } + i++ + } + legacy: + rex := byte(0) + if i < len(code) && code[i]&0xf0 == 0x40 { + rex = code[i] + i++ + } + if !mandatory || len(code) < i+2 || code[i] != 0x0f || code[i+1] != 0x52 && code[i+1] != 0x53 { + return Instr{}, 0, false, nil + } + p.opcode, p.modRM = int(code[i+1]), i+2 + p.r, p.x, p.b = int(rex>>2&1), int(rex>>1&1), int(rex&1) + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar reciprocal: REX unavailable in 32-bit mode") + } + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("scalar reciprocal: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.addressOverride { + return fail("EVEX or address-size override has no scalar estimate source contract") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.x != 0 || p.b != 0 || p.upper >= 8) { + return fail("extended physical register in 32-bit mode") + } + source, consumed, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + if err != nil { + return Instr{}, 0, true, err + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", int(code[p.modRM]>>3&7)+p.r*8))} + op := Op("RCPSS") + if p.opcode == 0x52 { + op = "RSQRTSS" + } + args := []Operand{source, destination} + printed := source.String() + ", " + destination.String() + if vector { + op = "V" + op + base := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))} + args = []Operand{source, base, destination} + printed = source.String() + ", " + base.String() + ", " + destination.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, printed)}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_scaled_round.go b/amd64_raw_scaled_round.go new file mode 100644 index 000000000..fb0fb3029 --- /dev/null +++ b/amd64_raw_scaled_round.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawScaledRoundForm struct { + op Op + laneBytes int + scalar bool +} + +// Go 1.27's eight EVEX RNDSCALE/REDUCE opcode rows. Both operations share +// the same packed/scalar immediate and modifier grammar. +var x86RawScaledRoundForms = map[[2]int]x86RawScaledRoundForm{ + {0x08, 0}: {op: "VRNDSCALEPS", laneBytes: 4}, + {0x09, 1}: {op: "VRNDSCALEPD", laneBytes: 8}, + {0x0a, 0}: {op: "VRNDSCALESS", laneBytes: 4, scalar: true}, + {0x0b, 1}: {op: "VRNDSCALESD", laneBytes: 8, scalar: true}, + {0x56, 0}: {op: "VREDUCEPS", laneBytes: 4}, + {0x56, 1}: {op: "VREDUCEPD", laneBytes: 8}, + {0x57, 0}: {op: "VREDUCESS", laneBytes: 4, scalar: true}, + {0x57, 1}: {op: "VREDUCESD", laneBytes: 8, scalar: true}, +} + +func decodedX86RawScaledRoundInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 { + return Instr{}, 0, false, nil + } + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawScaledRoundForms[[2]int{p.opcode, w}] + if !recognized { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX scaled round: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.addressOverride || p.vectorLength > 2 || + form.scalar && (mode == 32 || p.vectorLength != 0) || + !form.scalar && p.upper != 0 { + return fail("invalid EVEX fixed, address, vector length, or vvvv") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if mode == 32 && p.mask != 0 { + return fail("386 mask form exceeds the Go assembler frontend's operand limit") + } + modRM := code[p.modRM] + registerSource := modRM>>6 == 3 + sae := p.broadcast && registerSource + broadcast := p.broadcast && !registerSource + if form.scalar && broadcast || !form.scalar && sae && p.vectorLength != 2 { + return fail("SAE or broadcast is unavailable for this operand width") + } + destinationNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && p.vectorLength == 2 && + (destinationNumber >= 8 || registerSource && int(modRM&7)+p.b*8+p.x*16 >= 8) { + return fail("386 Z-register exceeds the Go assembler frontend's register class") + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + disp8Scale := 16 << p.vectorLength + if form.scalar || broadcast { + disp8Scale = form.laneBytes + } + source, consumed, err := decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, vector, disp8Scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + immIndex := p.modRM + consumed + if len(code) <= immIndex { + return fail("missing imm8") + } + op := form.op + if broadcast { + op += ".BCST" + } + if sae { + op += ".SAE" + } + if p.zero { + op += ".Z" + } + args := []Operand{{Kind: OpImm, Imm: int64(code[immIndex])}, source} + if form.scalar { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}) + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, destinationNumber))}) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, immIndex + 1, true, nil +} diff --git a/amd64_raw_scaled_round_test.go b/amd64_raw_scaled_round_test.go new file mode 100644 index 000000000..96c11d6dc --- /dev/null +++ b/amd64_raw_scaled_round_test.go @@ -0,0 +1,206 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawScaledRound(op Op, vectorBits, mask int, zeroing, evexB, memory bool) []byte { + var opcode byte + var w, scalar bool + for encoding, form := range x86RawScaledRoundForms { + if form.op == op { + opcode = byte(encoding[0]) + w = encoding[1] != 0 + scalar = form.scalar + break + } + } + p1 := byte(0x7d) + if scalar { + p1 = 0x6d // EVEX.vvvv = X2 passthrough. + } + if w { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if evexB { + p2 |= 0x10 + } + modRM := byte(0xd1) // first register 1, destination register 2. + if scalar { + modRM = 0xd9 // first register 1, destination register 3. + } + if memory { + modRM = 0x50 // disp8(AX), destination register 2. + if scalar { + modRM = 0x58 // disp8(AX), destination register 3. + } + } + code := []byte{0x62, 0xf3, p1, p2, opcode, modRM} + if memory { + code = append(code, 1) + } + return append(code, 9) +} + +func TestDecodeX86RawScaledRoundSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VRNDSCALEPS $9, 0(SI)(R8*4), Z0. + code := []byte{0x62, 0xb3, 0x7d, 0x48, 0x08, 0x04, 0x86, 0x09, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "floorFloat32 AVX512 VRNDSCALEPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VRNDSCALEPS" || + decoded[0].Args[0].Imm != 9 || decoded[0].Args[2].Reg != "Z0" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawScaledRoundCompleteGoForms(t *testing.T) { + if len(x86RawScaledRoundForms) != 8 { + t.Fatalf("modeled %d scaled-round rows, want eight", len(x86RawScaledRoundForms)) + } + for _, form := range x86RawScaledRoundForms { + maxLength := 2 + if form.scalar { + maxLength = 0 + } + for vectorBits := 0; vectorBits <= maxLength; vectorBits++ { + for _, variant := range []struct { + name string + mask int + zeroing bool + evexB bool + memory bool + suffix string + }{ + {name: "register"}, + {name: "masked zero register", mask: 7, zeroing: true, suffix: ".Z"}, + {name: "memory", memory: true}, + {name: "broadcast memory", evexB: true, memory: true, suffix: ".BCST"}, + {name: "SAE register", evexB: true, suffix: ".SAE"}, + } { + if form.scalar && variant.suffix == ".BCST" || + !form.scalar && variant.suffix == ".SAE" && vectorBits != 2 { + continue + } + name := fmt.Sprintf("%s/length%d/%s", form.op, vectorBits, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawScaledRound(form.op, vectorBits, variant.mask, variant.zeroing, variant.evexB, variant.memory) + got, length, ok, err := decodedX86RawScaledRoundInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != form.op+Op(variant.suffix) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantDestination := fmt.Sprintf("%s2", [...]string{"X", "Y", "Z"}[vectorBits]) + if form.scalar { + wantDestination = "X3" + } + if got.Args[0].Imm != 9 || got.Args[len(got.Args)-1].String() != wantDestination { + t.Fatalf("decode %x = %+v, want $9 ... %s", code, got, wantDestination) + } + }) + } + } + } +} + +func TestDecodeX86RawScaledRoundRIPLiteral(t *testing.T) { + for _, form := range x86RawScaledRoundForms { + for _, broadcast := range []bool{false, true} { + if broadcast && form.scalar { + continue + } + vectorBits := 2 + if form.scalar { + vectorBits = 0 + } + code := encodeX86RawScaledRound(form.op, vectorBits, 1, false, broadcast, false) + code = code[:6] + code[5] = 0x15 // RIP-relative first source, destination 2. + if form.scalar { + code[5] = 0x1d // destination 3. + } + code = append(code, 1, 0, 0, 0, 9, 0xc3) + literalWidth := 16 << vectorBits + if form.scalar || broadcast { + literalWidth = form.laneBytes + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/broadcast%t", form.op, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || len(got[0].x86RIPLiteralData) != literalWidth { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestDecodeX86RawScaledRoundRejectsInvalidForms(t *testing.T) { + base := encodeX86RawScaledRound("VRNDSCALEPS", 2, 0, false, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x80 }, // zeroing needs a mask + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + func(code []byte) { code[2] &^= 0x08 }, // packed reserved vvvv + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86RawScaledRoundInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + scalarBroadcast := encodeX86RawScaledRound("VRNDSCALESS", 0, 0, false, true, true) + if _, _, ok, err := decodedX86RawScaledRoundInstruction(scalarBroadcast, 64); !ok || err == nil { + t.Fatalf("accepted scalar broadcast %x: ok=%v err=%v", scalarBroadcast, ok, err) + } +} + +func TestTranslateX86RawScaledRoundLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawScaledRound(SB),$0-0\n") + for _, op := range []Op{"VRNDSCALEPS", "VRNDSCALEPD", "VREDUCEPS", "VREDUCEPD"} { + code := encodeX86RawScaledRound(op, 0, 0, false, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + if target.goarch == "amd64" { + for _, op := range []Op{"VRNDSCALESS", "VRNDSCALESD", "VREDUCESS", "VREDUCESD"} { + code := encodeX86RawScaledRound(op, 0, 0, false, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-scaled-round.ll", "raw-scaled-round.o", ir) + }) + } +} diff --git a/amd64_raw_scatter_test.go b/amd64_raw_scatter_test.go new file mode 100644 index 000000000..21d5a4a19 --- /dev/null +++ b/amd64_raw_scatter_test.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestDecodeX86RawScatterSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VSCATTERQPS Y2, K1, 0(DI)(Z1*4). + code := []byte{0x62, 0xf2, 0x7d, 0x49, 0xa3, 0x14, 0x8f, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "convolveFloat32 AVX512 VSCATTERQPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VSCATTERQPS" || + decoded[0].Args[0].Reg != "Y2" || decoded[0].Args[1].Reg != "K1" || + decoded[0].Args[2].Mem.Index != "Z1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func x86RawScatterGoForms(goarch string) string { + var source strings.Builder + source.WriteString("TEXT rawScatterForms(SB),4,$0-0\n") + widths := [...]string{"X", "Y", "Z"} + for _, family := range x86ScatterOps { + for _, op := range family { + spec := amd64ScatterSpecs[string(op)] + for width := 0; width < 3; width++ { + dataWidth := widths[width] + indexWidth := dataWidth + switch spec.table { + case amd64GatherDPD: + if width > 0 { + indexWidth = widths[width-1] + } + case amd64GatherQPS: + if width > 0 { + dataWidth = widths[width-1] + } + } + fmt.Fprintf(&source, "\t%s %s1, K1, 8(BX)(%s2*4)\n", op, dataWidth, indexWidth) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s21, K7, -32(R8)(%s18*2)\n", op, dataWidth, indexWidth) + } + } + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestDecodeX86RawScatterCompleteGoForms(t *testing.T) { + expected := map[Op]bool{ + "VPSCATTERDD": true, "VPSCATTERDQ": true, + "VPSCATTERQD": true, "VPSCATTERQQ": true, + "VSCATTERDPS": true, "VSCATTERDPD": true, + "VSCATTERQPS": true, "VSCATTERQPD": true, + } + for _, family := range x86ScatterOps { + for _, op := range family { + if !expected[op] { + t.Fatalf("unexpected or repeated scatter opcode %s", op) + } + delete(expected, op) + } + } + if len(expected) != 0 { + t.Fatalf("missing Go scatter opcodes: %v", expected) + } + for _, goarch := range []string{"amd64", "386"} { + t.Run(goarch, func(t *testing.T) { + source := x86RawScatterGoForms(goarch) + code := assembleX87ControlBytes(t, goarch, source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), goarch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + }) + } +} + +func TestDecodeX86RawScatterRejectsMalformedForms(t *testing.T) { + valid := []byte{0x62, 0xf2, 0x7d, 0x49, 0xa3, 0x14, 0x8f} + for _, variant := range []struct { + name string + mutate func([]byte) []byte + }{ + {"missing mask", func(code []byte) []byte { code[3] &^= 7; return code }}, + {"zeroing", func(code []byte) []byte { code[3] |= 0x80; return code }}, + {"broadcast", func(code []byte) []byte { code[3] |= 0x10; return code }}, + {"reserved length", func(code []byte) []byte { code[3] |= 0x60; return code }}, + {"register destination", func(code []byte) []byte { code[5] |= 0xc0; return code }}, + {"missing SIB", func(code []byte) []byte { return code[:6] }}, + {"reserved vvvv", func(code []byte) []byte { code[2] &^= 0x08; return code }}, + } { + t.Run(variant.name, func(t *testing.T) { + code := variant.mutate(append([]byte(nil), valid...)) + if _, _, matched, err := decodedX86RawScatterInstruction(code, 64); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", code, matched, err) + } + }) + } +} + +func TestTranslateX86RawScatterLLVM22AllTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + source := x86RawScatterGoForms(target.goarch) + code := assembleX87ControlBytes(t, target.goarch, source) + file := x86RawUnprovedReturnFormProbeFile(t, target.goarch, "rawScatterForms", code) + // Bare encoder rows have no index, data, mask or address producers. + // Retain their three public Context controls and actual Go bytes, + // then compile every unchanged row through the private format owner. + ir := x86UnboundVoidRawProbeObject(t, file, target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-scatter.ll", "raw-scatter.o", ir) + }) + } +} diff --git a/amd64_raw_segment_absolute_test.go b/amd64_raw_segment_absolute_test.go new file mode 100644 index 000000000..a13dfad57 --- /dev/null +++ b/amd64_raw_segment_absolute_test.go @@ -0,0 +1,65 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawSegmentAbsoluteMemory(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, tc := range []struct { + name string + code []byte + want string + base bool + }{ + {name: "gs-absolute", code: []byte{0x65, 0x48, 0x8b, 0x04, 0x25, 0x30, 0, 0, 0}, want: "48(GS)"}, + {name: "fs-absolute", code: []byte{0x64, 0x48, 0x8b, 0x04, 0x25, 0x30, 0, 0, 0}, want: "48(FS)"}, + {name: "gs-base", code: []byte{0x65, 0x48, 0x8b, 0x40, 0x30}, want: "48(AX)(GS)", base: true}, + {name: "fs-base", code: []byte{0x64, 0x48, 0x8b, 0x40, 0x30}, want: "48(AX)(FS)", base: true}, + } { + t.Run(tc.name, func(t *testing.T) { + inst, err := x86asm.Decode(tc.code, 64) + if err != nil { + t.Fatal(err) + } + syntax, err := decodedX86GoSyntax(inst, tc.code) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(syntax, tc.want) { + t.Fatalf("raw %x decoded as %q, want memory %s", tc.code, syntax, tc.want) + } + + var source strings.Builder + source.WriteString("TEXT rawSegmentMemory(SB),$0-0\n") + for _, value := range tc.code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + source.WriteString("RET\n") + assembly := source.String() + triple := "x86_64-unknown-linux-gnu" + if tc.base { + ir := x86UnboundVoidRawScalarFormatObject(t, assembly, "amd64", triple) + x86PackedStringCompileObject(t, llc, triple, ir, "-O0") + assembly = strings.Replace(assembly, "\nBYTE", "\nMOVQ $0,AX\nBYTE", 1) + } + x86LEAActualTargetGoObject(t, "amd64", triple, assembly) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, assembly, "func rawSegmentMemory()", "amd64", triple, api) + if err != nil { + t.Fatal("raw hardware segment with all explicit numerical offsets", api, err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, ir, level) + } + } + }) + } +} diff --git a/amd64_raw_segment_base_test.go b/amd64_raw_segment_base_test.go new file mode 100644 index 000000000..5a36ecaaa --- /dev/null +++ b/amd64_raw_segment_base_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Intel's F3 0F AE /0..3 register encodings correspond to all eight rows in +// Go's asm6.go: RDFSBASE, RDGSBASE, WRFSBASE and WRGSBASE, each with L/Q width. +var rawSegmentBaseTestOps = [...]string{"RDFSBASE", "RDGSBASE", "WRFSBASE", "WRGSBASE"} + +func rawSegmentBaseTestEncoding(operation, bits, register int) []byte { + code := []byte{0xf3} + rex := byte(0x40) + if bits == 64 { + rex |= 8 + } + if register >= 8 { + rex |= 1 + } + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0xae, byte(0xc0|operation<<3|register&7)) +} + +func TestX86RawSegmentBaseCompleteRegisterFamily(t *testing.T) { + if len(amd64SegmentBaseSpecs) != 2*len(rawSegmentBaseTestOps) { + t.Fatal("raw encoding tests must cover the complete named segment-base grammar") + } + registers := [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + for operation, name := range rawSegmentBaseTestOps { + for _, width := range []struct { + bits int + suffix string + }{{32, "L"}, {64, "Q"}} { + for register, wantRegister := range registers { + wantOp := Op(name + width.suffix) + t.Run(string(wantOp)+"/"+string(wantRegister), func(t *testing.T) { + code := rawSegmentBaseTestEncoding(operation, width.bits, register) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segment-base bytes", nil) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != wantOp || len(decoded[0].Args) != 1 || + decoded[0].Args[0].Kind != OpReg || decoded[0].Args[0].Reg != wantRegister { + t.Fatalf("%x decoded as %+v, want %s %s", code, decoded, wantOp, wantRegister) + } + if _, ok := amd64SegmentBaseSpecs[wantOp]; !ok { + t.Fatalf("raw opcode %s has no typed named grammar", wantOp) + } + }) + } + } + } +} + +func TestX86RawSegmentBaseIgnoredPrefixesPreserveRegisterWidth(t *testing.T) { + for operation, name := range rawSegmentBaseTestOps { + for _, bits := range []int{32, 64} { + for _, prefix := range []byte{0x66, 0x67, 0x64, 0x65} { + for _, afterREP := range []bool{false, true} { + code := rawSegmentBaseTestEncoding(operation, bits, 11) + if afterREP { + code = append([]byte{0xf3, prefix}, code[1:]...) + } else { + code = append([]byte{prefix}, code...) + } + inst, err := x86asm.Decode(code, 64) + if err != nil || inst.Len != len(code) { + t.Fatalf("decode %x: %+v, %v", code, inst, err) + } + got, err := decodedX86GoSyntax(inst, code) + width := "L" + if bits == 64 { + width = "Q" + } + want := name + width + " R11" + if err != nil || got != want { + t.Errorf("%x (DataSize=%d): got %q, %v; want %q", code, inst.DataSize, got, err, want) + } + } + } + } + } +} + +func TestX86RawSegmentBaseRejectsInvalidDecodedForms(t *testing.T) { + for _, op := range []x86asm.Op{x86asm.RDFSBASE, x86asm.RDGSBASE, x86asm.WRFSBASE, x86asm.WRGSBASE} { + for _, args := range []x86asm.Args{ + {}, {x86asm.Imm(1)}, {x86asm.Mem{Base: x86asm.RAX}}, + {x86asm.AL}, {x86asm.AX}, {x86asm.X0}, + {x86asm.RAX, x86asm.RBX}, {x86asm.RAX, nil, nil, x86asm.RBX}, + } { + inst := x86asm.Inst{Op: op, Mode: 64, DataSize: 64, Args: args} + if got, err := decodedX86GoSyntax(inst, nil); err == nil { + t.Errorf("accepted invalid %s operands %v as %q", op, args, got) + } + } + } + for operation := range rawSegmentBaseTestOps { + code := rawSegmentBaseTestEncoding(operation, 64, 3) + for _, invalid := range []struct { + name string + code []byte + mode int + }{ + {"lock", append([]byte{0xf0}, code...), 64}, + {"memory", []byte{0xf3, 0x48, 0x0f, 0xae, byte(operation << 3)}, 64}, + {"truncated", code[:len(code)-1], 64}, + {"32-bit-mode", rawSegmentBaseTestEncoding(operation, 32, 3), 32}, + } { + t.Run(rawSegmentBaseTestOps[operation]+"/"+invalid.name, func(t *testing.T) { + if got, err := decodeX86RawDirectiveGroup(invalid.code, invalid.mode, 0, "invalid segment-base bytes", nil); err == nil { + t.Fatalf("accepted invalid %x as %+v", invalid.code, got) + } + }) + } + } +} + +func TestTranslateX86RawSegmentBaseMatchesNamedFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + var raw, named strings.Builder + for _, source := range []*strings.Builder{&raw, &named} { + source.WriteString("TEXT segmentbase(SB),$0-0\nMOVQ $-1, BX\n") + } + // Keep the whole reported gVisor sequence, including SWAPGS before + // RDGSBASEQ, as well as every sibling operation and operand width. + for _, value := range []byte{0x0f, 0x01, 0xf8, 0xf3, 0x48, 0x0f, 0xae, 0xcb} { + fmt.Fprintf(&raw, "BYTE $%#x\n", value) + } + named.WriteString("SWAPGS\nRDGSBASEQ BX\n") + for operation, name := range rawSegmentBaseTestOps { + for _, bits := range []int{32, 64} { + for _, value := range rawSegmentBaseTestEncoding(operation, bits, 3) { + fmt.Fprintf(&raw, "BYTE $%#x\n", value) + } + width := "L" + if bits == 64 { + width = "Q" + } + fmt.Fprintf(&named, "%s%s BX\n", name, width) + } + } + var irs []string + for _, source := range []*strings.Builder{&raw, &named} { + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"segmentbase": {Name: "segmentbase", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + // LLVM assigns a different temporary filename on each parse. + // Compare every semantic line, excluding only that metadata. + lines := strings.Split(ir, "\n") + for index, line := range lines { + if strings.HasPrefix(line, "; ModuleID = ") || strings.HasPrefix(line, "source_filename = ") { + lines[index] = "" + } + } + irs = append(irs, strings.Join(lines, "\n")) + } + if irs[0] != irs[1] { + t.Fatalf("raw and named segment-base semantics differ:\nraw:\n%s\nnamed:\n%s", irs[0], irs[1]) + } + compileLLVMToObject(t, llc, triple, "raw-segment-base.ll", "raw-segment-base.o", irs[0]) + }) + } +} diff --git a/amd64_raw_segment_load_test.go b/amd64_raw_segment_load_test.go new file mode 100644 index 000000000..2b4c6ca66 --- /dev/null +++ b/amd64_raw_segment_load_test.go @@ -0,0 +1,205 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawSegmentInitializedLoadForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + forms := 0 + for _, segment := range []struct { + prefix byte + }{{0x64}, {0x65}} { + for _, width := range []int{1, 2, 4, 8} { + if target.arch == "386" && width == 8 { + continue + } + for _, address := range []string{"absolute", "base", "indexed"} { + name := fmt.Sprintf("Segment%d", forms) + fmt.Fprintf(&declarations, "func %s()\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\nMOVL $0,AX\nMOVL $1,CX\n", name) + code := []byte{segment.prefix} + if width == 2 { + code = append(code, 0x66) + } else if width == 8 { + code = append(code, 0x48) + } + opcode := byte(0x8b) + if width == 1 { + opcode = 0x8a + } + code = append(code, opcode) + switch address { + case "absolute": + if target.arch == "386" { + code = append(code, 0x15) + } else { + code = append(code, 0x14, 0x25) + } + code = append(code, 48, 0, 0, 0) + case "base": + code = append(code, 0x50, 48) + case "indexed": + code = append(code, 0x54, 0x48, 48) + } + source.WriteString(x86RawDisplacementDirectives(code)) + source.WriteString("RET\n") + forms++ + } + } + } + want := 24 + if target.arch == "386" { + want = 18 + } + if forms != want { + t.Fatal("complete raw segment load inventory changed", forms, want) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("initialized physical segment load lost its paired native read", api, err) + } + for _, output := range []string{ir, x86DwordProductIROptimize(t, ir)} { + if strings.Count(output, "load volatile ") != want { + t.Fatal("dead-result physical segment read was lost or duplicated", api, output) + } + for _, space := range []int{256, 257} { + for _, width := range []int{1, 2, 4, 8} { + if target.arch == "386" && width == 8 { + continue + } + read := fmt.Sprintf("load volatile i%d, ptr addrspace(%d)", width*8, space) + if strings.Count(output, read) != 3 { + t.Fatal("physical segment read width or address space changed", api, read, output) + } + } + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, output, level) + } + } + for _, foreign := range []string{"aarch64-unknown-linux-gnu", "wasm32-unknown-unknown"} { + if ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, foreign, api); err == nil || ir != "" { + t.Fatal("physical segment source acquired a foreign backend", api, foreign, err) + } + } + } + }) + } +} + +func TestX86RawSegmentLoadRetainsInputsAndSourceIdentity(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + mode, word := 32, int64(4) + if arch == "amd64" { + mode, word = 64, 8 + } + for _, prefix := range []byte{0x64, 0x65} { + instructions := x86AddressOverrideDecoded(t, mode, []byte{prefix, 0x8b, 0x54, 0x48, 48}) + if len(instructions) != 3 || instructions[0].Op != OpTEXT || instructions[2].Op != OpRET { + t.Fatal("single physical segment load lost identity", instructions) + } + original := instructions[1] + if !x86RawScalarSegmentLoadSource(original, arch) { + t.Fatal("exact physical segment load lacks decoder-bound identity", arch, original) + } + for _, register := range []Reg{AX, CX} { + index, _, _ := x86ResultGP(register, word, arch) + for missing := 0; missing <= int(word); missing++ { + state := x86ResultState{flags: x86ByteFlagsAll} + state.gp[0], state.gp[1] = 0xff, 0xff + if missing < int(word) { + state.gp[index] &^= 1 << uint(missing) + } else { + state.pointerGP[index] = true + } + before := state.clone() + if accepted, known := x86LoweredScalarNativeLoad(original, &state, nil, 0, arch); !known || accepted || !reflect.DeepEqual(state, before) { + t.Fatal("physical segment offset acquired missing NUM or pointer bits", arch, register, missing) + } + } + } + for _, mutation := range []struct { + name string + change func(*Instr) + }{ + {"origin", func(ins *Instr) { ins.x86SourceRaw = nil }}, + {"encoding", func(ins *Instr) { ins.x86Encoded = false }}, + {"absent-mode", func(ins *Instr) { ins.x86SourceRaw.scalarSegmentLoad.arch = "" }}, + {"foreign-mode", func(ins *Instr) { ins.x86SourceRaw.scalarSegmentLoad.arch = "other" }}, + {"physical-mode", func(ins *Instr) { + ins.x86SourceRaw.scalarSegmentLoad.arch = "386" + if arch == "386" { + ins.x86SourceRaw.scalarSegmentLoad.arch = "amd64" + } + }}, + {"width", func(ins *Instr) { ins.x86SourceRaw.scalarSegmentLoad.bytes++ }}, + {"prefix", func(ins *Instr) { ins.x86SegmentPrefix = 0x64 }}, + {"origin-prefix", func(ins *Instr) { ins.x86SourceRaw.scalarSegmentLoad.segmentPrefix = 0x65 }}, + {"address", func(ins *Instr) { ins.x86AddressBits = 16 }}, + {"override", func(ins *Instr) { ins.x86UnboundAddressOverride = true }}, + {"opcode", func(ins *Instr) { ins.Op = "MOVW" }}, + {"source", func(ins *Instr) { ins.Args[0].Mem.Off++ }}, + {"destination", func(ins *Instr) { ins.Args[1].Reg = BX }}, + {"segment", func(ins *Instr) { ins.Args[0].Mem.Segment = GS }}, + } { + if mutation.name == "segment" && prefix == 0x65 { + continue // The FS case independently changes the actual segment. + } + changed := cloneX86ReplayInstrs([]Instr{original})[0] + mutation.change(&changed) + state := x86ResultState{flags: x86ByteFlagsAll} + state.gp[0], state.gp[1] = 0xff, 0xff + before := state.clone() + if x86RawScalarSegmentLoadSource(changed, arch) { + t.Fatal("edited raw segment source reused its decoder identity", arch, mutation.name) + } + if accepted, _ := x86LoweredScalarNativeLoad(changed, &state, nil, 0, arch); accepted || !reflect.DeepEqual(state, before) { + t.Fatal("edited raw segment source obtained a numerical continuation", arch, mutation.name) + } + } + } + } + assembly := "TEXT ·Segment(SB),4,$0-0\nMOVL $0,AX\n" + x86RawDisplacementDirectives([]byte{0x64, 0x8b, 0x40, 48}) + "RET\n" + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + file, err = normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + for _, mutation := range []string{"operand", "receipt", "replay"} { + changed := *file + changed.Funcs = append([]Func(nil), file.Funcs...) + changed.Funcs[0].Instrs = cloneX86ReplayInstrs(file.Funcs[0].Instrs) + switch mutation { + case "operand": + changed.Funcs[0].Instrs[2].Args[0].Mem.Off++ + case "receipt": + changed.Funcs[0].Instrs[2].x86SourceRaw = nil + case "replay": + replay := *changed.Funcs[0].x86RawReplay + replay.source = strings.Replace(replay.source, "MOVL $0,AX", "MOVL $1,AX", 1) + changed.Funcs[0].x86RawReplay = &replay + } + ir, err := Translate(&changed, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"Segment": {Name: "Segment", Ret: Void}}, + ResolveSym: func(name string) string { return strings.TrimPrefix(name, "·") }}) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("edited complete raw source bypassed its sealed replay", mutation, err) + } + } +} diff --git a/amd64_raw_segment_move_test.go b/amd64_raw_segment_move_test.go new file mode 100644 index 000000000..63309a2bc --- /dev/null +++ b/amd64_raw_segment_move_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestDecodedX86RawSegmentMoveCompleteRegisterFamily(t *testing.T) { + segments := []string{"ES", "CS", "SS", "DS", "FS", "GS"} + count := 0 + for _, mode := range []int{32, 64} { + registers := 8 + if mode == 64 { + registers = 16 + } + for segmentNumber, segment := range segments { + for general := 0; general < registers; general++ { + rex := byte(0) + if mode == 64 && general >= 8 { + rex = 0x41 + } + for _, direction := range []struct { + opcode byte + want string + }{ + {opcode: 0x8e, want: fmt.Sprintf("MOVW %s, %s", decodedX86RegisterNameForTest(general), segment)}, + {opcode: 0x8c, want: fmt.Sprintf("MOVW %s, %s", segment, decodedX86RegisterNameForTest(general))}, + } { + code := make([]byte, 0, 3) + if rex != 0 { + code = append(code, rex) + } + code = append(code, direction.opcode, byte(0xc0|segmentNumber<<3|general&7)) + got, length, ok, err := decodedX86SegmentMoveInstruction(code, mode) + if err != nil || !ok || length != len(code) || got.Raw != direction.want { + t.Fatalf("decode mode=%d %x = %+v, length=%d, ok=%v, err=%v; want %q", mode, code, got, length, ok, err, direction.want) + } + count++ + } + } + } + } + if count != 288 { + t.Fatalf("covered %d raw segment-register moves, want 288", count) + } +} + +func decodedX86RegisterNameForTest(number int) string { + reg, ok := decodedX86GeneralRegister(number) + if !ok { + panic(number) + } + return string(reg) +} + +func TestDecodedX86RawSegmentMoveMemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + code []byte + want string + }{ + { + name: "load FS from extended SIB with FS override", + mode: 64, + code: []byte{0x64, 0x43, 0x8e, 0x64, 0x8b, 0x20}, + want: "MOVW 32(R11)(R9*4)(FS), FS", + }, + { + name: "store GS to extended SIB with GS override", + mode: 64, + code: []byte{0x65, 0x43, 0x8c, 0x6c, 0x8b, 0x20}, + want: "MOVW GS, 32(R11)(R9*4)(GS)", + }, + { + name: "386 absolute memory", + mode: 32, + code: []byte{0x66, 0x8e, 0x1d, 0x78, 0x56, 0x34, 0x12}, + want: "MOVW 305419896(), DS", + }, + } { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86SegmentMoveInstruction(test.code, test.mode) + if err != nil || !ok || length != len(test.code) || got.Raw != test.want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", test.code, got, length, ok, err, test.want) + } + }) + } + + for _, test := range []struct { + name string + encoding []byte + wantOK bool + }{ + {name: "reserved segment six", encoding: []byte{0x8e, 0xf0}, wantOK: true}, + {name: "reserved segment seven", encoding: []byte{0x8c, 0xf8}, wantOK: true}, + {name: "REX.R reserved segment", encoding: []byte{0x44, 0x8e, 0xc0}, wantOK: true}, + {name: "RIP relative", encoding: []byte{0x8e, 0x1d, 0, 0, 0, 0}, wantOK: true}, + {name: "address override", encoding: []byte{0x67, 0x8e, 0xd8}, wantOK: true}, + {name: "truncated", encoding: []byte{0x8e}, wantOK: false}, + {name: "unrelated opcode", encoding: []byte{0x8f, 0xd8}, wantOK: false}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86SegmentMoveInstruction(test.encoding, 64) + if ok != test.wantOK { + t.Fatalf("encoding %x returned ok=%v err=%v, want ok=%v", test.encoding, ok, err, test.wantOK) + } + if test.wantOK && err == nil { + t.Fatalf("invalid encoding %x was accepted", test.encoding) + } + if !test.wantOK && err != nil { + t.Fatalf("unrelated encoding %x returned err=%v", test.encoding, err) + } + }) + } +} diff --git a/amd64_raw_segment_prefixes.go b/amd64_raw_segment_prefixes.go new file mode 100644 index 000000000..818627ee5 --- /dev/null +++ b/amd64_raw_segment_prefixes.go @@ -0,0 +1,20 @@ +package plan9asm + +// x86RawSegmentPrefixes retains physical prefix observations without deciding +// whether a particular instruction or source mode accepts them. +func x86RawSegmentPrefixes(code []byte) (length int, segment Reg, addressOverride bool) { + for length < len(code) { + switch code[length] { + case 0x64: + segment = FS + case 0x65: + segment = GS + case 0x67: + addressOverride = true + default: + return + } + length++ + } + return +} diff --git a/amd64_raw_segment_prefixes_bench_test.go b/amd64_raw_segment_prefixes_bench_test.go new file mode 100644 index 000000000..e92b13c50 --- /dev/null +++ b/amd64_raw_segment_prefixes_bench_test.go @@ -0,0 +1,55 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +// The bytes retain independent VEX float, EVEX funnel, SHA and scalar +// precision fixtures from their existing raw-family tests. +type rawPrefixQualificationFixture struct { + name string + code []byte + decode func([]byte, int) (Instr, int, bool, error) +} + +func rawPrefixQualificationFixtures() []rawPrefixQualificationFixture { + return []rawPrefixQualificationFixture{ + {"vex-float-memory", []byte{0xc4, 0x01, 0xe5, 0x58, 0x64, 0x88, 0x20}, decodedX86VEXBinaryFloatInstruction}, + {"evex-funnel-memory", []byte{0x62, 0xf3, 0xf5, 0xdb, 0x73, 0x50, 0x7f, 0x09}, decodedX86FunnelShiftInstruction}, + {"legacy-sha-register", []byte{0x0f, 0x38, 0xc8, 0xca}, decodedX86SHAInstruction}, + {"scalar-precision-memory", []byte{0x62, 0xf1, 0x56, 0x83, 0x5a, 0x50, 0x02}, decodedX86ScalarPrecisionConvertInstruction}, + } +} + +var rawPrefixQualificationSink Instr + +func BenchmarkX86RawPrefixDecoders(b *testing.B) { + for _, fixture := range rawPrefixQualificationFixtures() { + for _, mode := range []int{32, 64} { + for _, variant := range []struct { + name string + prefix []byte + cut int + }{ + {"original", nil, len(fixture.code)}, + {"last-segment", []byte{0x65, 0x64}, len(fixture.code)}, + {"address-override", []byte{0x64, 0x67, 0x65}, len(fixture.code)}, + {"nonmatch", []byte{0x90}, len(fixture.code)}, + {"truncated", []byte{0x64}, 1}, + } { + b.Run(fmt.Sprintf("%s/mode%d/%s", fixture.name, mode, variant.name), func(b *testing.B) { + code := append(append([]byte(nil), variant.prefix...), fixture.code[:variant.cut]...) + // Compare acceptance and exact errors outside timing. Cases + // deliberately include matched, nonmatching and invalid input. + fixture.decode(code, mode) + b.ReportAllocs() + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + rawPrefixQualificationSink, _, _, _ = fixture.decode(code, mode) + } + }) + } + } + } +} diff --git a/amd64_raw_segment_prefixes_test.go b/amd64_raw_segment_prefixes_test.go new file mode 100644 index 000000000..7dc2bb3cc --- /dev/null +++ b/amd64_raw_segment_prefixes_test.go @@ -0,0 +1,57 @@ +package plan9asm + +import "testing" + +// Candidate-only owner. Do not install/run this against baseline, where the +// helper does not exist; a missing symbol would not establish a product RED. +func TestX86RawSegmentPrefixTuples(t *testing.T) { + for _, test := range []struct { + code []byte + length int + segment Reg + address bool + }{ + {nil, 0, "", false}, + {[]byte{0x64}, 1, FS, false}, + {[]byte{0x65}, 1, GS, false}, + {[]byte{0x67, 0x67}, 2, "", true}, + {[]byte{0x64, 0x65}, 2, GS, false}, + {[]byte{0x65, 0x64}, 2, FS, false}, + {[]byte{0x65, 0x67, 0x64, 0x67}, 4, FS, true}, + {[]byte{0x64, 0x90, 0x65, 0x67}, 1, FS, false}, + } { + length, segment, address := x86RawSegmentPrefixes(test.code) + if length != test.length || segment != test.segment || address != test.address { + t.Fatalf("%x: got (%d,%s,%v), want (%d,%s,%v)", test.code, length, segment, address, test.length, test.segment, test.address) + } + } + // All 256 next bytes, preceded by each physical segment/address state. + for _, prefix := range []struct { + code []byte + segment Reg + address bool + }{ + {nil, "", false}, {[]byte{0x64}, FS, false}, {[]byte{0x65}, GS, false}, + {[]byte{0x67}, "", true}, {[]byte{0x64, 0x67}, FS, true}, {[]byte{0x65, 0x67}, GS, true}, + } { + for next := 0; next < 256; next++ { + code := append(append([]byte(nil), prefix.code...), byte(next), 0x90, 0x65, 0x67) + wantLength, wantSegment, wantAddress := len(prefix.code), prefix.segment, prefix.address + switch byte(next) { + case 0x64: + wantLength++ + wantSegment = FS + case 0x65: + wantLength++ + wantSegment = GS + case 0x67: + wantLength++ + wantAddress = true + } + length, segment, address := x86RawSegmentPrefixes(code) + if length != wantLength || segment != wantSegment || address != wantAddress { + t.Fatalf("%x: got (%d,%s,%v), want (%d,%s,%v)", code, length, segment, address, wantLength, wantSegment, wantAddress) + } + } + } +} diff --git a/amd64_raw_sha_test.go b/amd64_raw_sha_test.go new file mode 100644 index 000000000..7875b2b32 --- /dev/null +++ b/amd64_raw_sha_test.go @@ -0,0 +1,50 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86SHACompleteGo127Family(t *testing.T) { + // Go 1.27 exposes every Intel SHA extension opcode. Cover register and + // memory sources, REX-extended X registers, the SHA1 immediate, and the + // implicit X0 operand used by SHA256RNDS2. + tests := []struct { + name string + code []byte + want string + }{ + {name: "sha1 next e", code: []byte{0x0f, 0x38, 0xc8, 0xca}, want: "SHA1NEXTE X2, X1"}, + {name: "sha1 message 1 high destination", code: []byte{0x44, 0x0f, 0x38, 0xc9, 0xf4}, want: "SHA1MSG1 X4, X14"}, + {name: "sha1 message 2 memory", code: []byte{0x41, 0x0f, 0x38, 0xca, 0x54, 0x8c, 0x20}, want: "SHA1MSG2 32(R12)(CX*4), X2"}, + {name: "sha256 message 1", code: []byte{0x0f, 0x38, 0xcc, 0xe5}, want: "SHA256MSG1 X5, X4"}, + {name: "sha256 message 2", code: []byte{0x0f, 0x38, 0xcd, 0xf5}, want: "SHA256MSG2 X5, X6"}, + {name: "sha1 rounds 4", code: []byte{0x0f, 0x3a, 0xcc, 0xd1, 0x03}, want: "SHA1RNDS4 $3, X1, X2"}, + {name: "sha256 rounds 2 implicit x0", code: []byte{0x0f, 0x38, 0xcb, 0xda}, want: "SHA256RNDS2 X0, X2, X3"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86SHAInstruction(test.code, 64) + if !ok { + t.Fatal("SHA encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioSHA256RNDS2(t *testing.T) { + code := []byte{0x0f, 0x38, 0xcb, 0xda} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio SHA256RNDS2", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "SHA256RNDS2" || !strings.HasPrefix(decoded[0].Raw, "SHA256RNDS2 X0, X2, X3 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_shiftrotate_test.go b/amd64_raw_shiftrotate_test.go new file mode 100644 index 000000000..3669004fd --- /dev/null +++ b/amd64_raw_shiftrotate_test.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawShiftRotateCompleteGoForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var lines []string + widths := []string{"B", "W", "L"} + mode := 32 + if arch == "amd64" { + mode = 64 + widths = append(widths, "Q") + } + for _, stem := range []string{"ROL", "ROR", "RCL", "RCR", "SHL", "SHR", "SAR"} { + for _, width := range widths { + destinations := []string{"AX", "-7(BX)(CX*4)"} + if width == "B" { + destinations = []string{"AL", "AH", "-7(BX)(CX*4)"} + } + if arch == "amd64" { + destinations = append(destinations, "R11") + } + for _, dst := range destinations { + for _, count := range []string{"$1", "$255", "CL"} { + lines = append(lines, fmt.Sprintf("%s%s %s, %s", stem, width, count, dst)) + } + } + } + } + for _, stem := range []string{"SHL", "SHR"} { + for _, width := range widths[1:] { + for _, dst := range []string{"BX", "-7(BX)(CX*4)"} { + for _, count := range []string{"$-128", "$127", "CL"} { + lines = append(lines, fmt.Sprintf("%s%s %s, AX, %s", stem, width, count, dst)) + } + } + } + } + named := "TEXT shifts(SB),4,$0-0\n" + strings.Join(lines, "\n") + "\nRET\n" + code := assembleX87ControlBytes(t, arch, named) + offset := 0 + var raw strings.Builder + raw.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, line := range lines { + inst, err := x86asm.Decode(code[offset:], mode) + if err != nil { + t.Fatalf("Go encoded %s: %v", line, err) + } + encoded := code[offset : offset+inst.Len] + decoded, err := decodeX86RawDirectives(rawX86Function(encoded), arch) + if err != nil { + t.Fatal(err) + } + want, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+line+"\n") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != want.Funcs[0].Instrs[1].Op || !reflect.DeepEqual(decoded.Instrs[0].Args, want.Funcs[0].Instrs[1].Args) { + t.Fatalf("decoded %x as %+v, want %s", encoded, decoded.Instrs, line) + } + for _, b := range encoded { + fmt.Fprintf(&raw, "BYTE $%#02x\n", b) + } + offset += inst.Len + } + raw.WriteString("RET\n") + file, err := Parse(ArchAMD64, raw.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + // Counts, carry, data and physical addresses remain unbound. + // Replay the complete original source before direct format emission. + ir := x86UnboundVoidRawProbeObject(t, file, arch, triple) + compileLLVMToObject(t, llc, triple, "raw-shifts.ll", "raw-shifts.o", ir) + } + }) + } +} diff --git a/amd64_raw_static_return_runtime_test.go b/amd64_raw_static_return_runtime_test.go new file mode 100644 index 000000000..9174e8aef --- /dev/null +++ b/amd64_raw_static_return_runtime_test.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +// The Go oracle uses an ordinary declared result slot. It does not pretend +// that Go ABI0 returns a scalar in AX. The translated raw-RIP counterpart has +// the same fixed-width read and data, transported through its typed C result. +func runX86StaticReadNumericOracle(t *testing.T, arch, triple string, compiler, runner []string) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + values := []uint64{0, 1, 0x80000001, 0xffffffff, 0x89abcdef01234567, 0xfedcba9876543210} + var raw, native, declarations, goMain, cMain strings.Builder + goMain.WriteString("package main\nfunc main(){\n") + cMain.WriteString("typedef unsigned int u32; typedef unsigned long long u64;\n") + var cChecks strings.Builder + cChecks.WriteString("int main(void){\n") + sigs := make(map[string]FuncSig) + count := 0 + for _, width := range []int{4, 8} { + if arch == "386" && width == 8 { + continue // The 386 scalar MOVQ spelling is not an integer GP read. + } + for index, value := range values { + name := fmt.Sprintf("Read%d_%d", width, index) + typ, goType, cType, move := I32, "uint32", "u32", "MOVL" + if width == 8 { + typ, goType, cType, move = I64, "uint64", "u64", "MOVQ" + } else { + value = uint64(uint32(value)) + } + fmt.Fprintf(&declarations, "func %s()(r %s)\n", name, goType) + fmt.Fprintf(&goMain, "if got:=%s(); got!=%s(0x%x){panic(\"%s\")}\n", name, goType, value, name) + fmt.Fprintf(&cMain, "extern %s %s(void);\n", cType, name) + fmt.Fprintf(&cChecks, "if(%s()!=0x%xULL)return %d;\n", name, value, count+1) + fmt.Fprintf(&native, "DATA pool%d_%d<>+0(SB)/%d,$0x%x\nGLOBL pool%d_%d<>(SB),8,$%d\nTEXT ·%s(SB),4,$0-%d\n%s pool%d_%d<>(SB),AX\n%s AX,r+0(FP)\nBYTE $0xc3\n", + width, index, width, value, width, index, width, name, width, move, width, index, move) + if arch == "amd64" { + code := []byte{0x8b, 0x05, 1, 0, 0, 0, 0xc3} + if width == 8 { + code = append([]byte{0x48}, code...) + } + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, value) + raw.WriteString(x86StaticReturnSource(name, append(code, data[:width]...))) + } else { + // Same-file DATA allocations are architecture-neutral. In 386 + // mode these are static absolute references, never RIP reads. + fmt.Fprintf(&raw, "DATA pool%d_%d<>+0(SB)/%d,$0x%x\nTEXT %s(SB),4,$0-0\n%s pool%d_%d<>(SB),AX\nBYTE $0xc3\n", + width, index, width, value, name, move, width, index) + } + sigs[name] = FuncSig{Name: name, Ret: typ} + count++ + } + } + if arch == "amd64" { + // A source-local LEA makes the pool mutable through its returned + // address. The shared read proof must not resurrect specialization. + declarations.WriteString("func Mutable()(p *uint64,v uint64)\n") + native.WriteString("DATA mutable<>+0(SB)/8,$0x123456789abcdef\nGLOBL mutable<>(SB),16,$8\nTEXT ·Mutable(SB),4,$0-16\nLEAQ mutable<>(SB),AX\nMOVQ AX,p+0(FP)\nMOVQ mutable<>(SB),BX\nMOVQ BX,v+8(FP)\nBYTE $0xc3\n") + goMain.WriteString("p,v:=Mutable();if p==nil||v!=0x123456789abcdef{panic(\"initial mutable\")}\n*p=0xfedcba9876543210\nq,v:=Mutable();if p!=q||v!=0xfedcba9876543210{panic(\"changed mutable\")}\n") + code := []byte{ + 0x48, 0x8d, 0x1d, 0x08, 0, 0, 0, // LEAQ pool,BX. + 0x48, 0x8b, 0x05, 0x01, 0, 0, 0, // MOVQ pool,AX. + 0xc3, + } + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, 0x123456789abcdef) + raw.WriteString(x86StaticReturnSource("Mutable", append(code, data...))) + sigs["Mutable"] = FuncSig{Name: "Mutable", Ret: I64} + cMain.WriteString("extern u64 Mutable(void); extern u64 MutablePool[];\n") + cChecks.WriteString("if(Mutable()!=0x123456789abcdefULL)return 98;\nMutablePool[0]=0xfedcba9876543210ULL;\nif(Mutable()!=0xfedcba9876543210ULL)return 99;\n") + } + goMain.WriteString("}\n") + cChecks.WriteString("return 0;}\n") + cMain.WriteString(cChecks.String()) + goSource := strings.Replace(goMain.String(), "func main", declarations.String()+"func main", 1) + project := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module example.com/static-read-oracle\ngo 1.20\n", + "main.go": goSource, "read_" + arch + ".s": native.String(), + } { + if err := os.WriteFile(filepath.Join(project, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(project, gotoolchain.HostExecutable("oracle")) + cmd := exec.Command("go", "build", "-o", executable, ".") + cmd.Dir = project + goos := runtime.GOOS + if len(runner) != 0 { + goos = "linux" + } + cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+arch, "CGO_ENABLED=0", "GOWORK=off", "GOTOOLCHAIN=local") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("actual Go static-read oracle build: %v\n%s", err, output) + } + run := append(append([]string(nil), runner...), executable) + if output, err := exec.Command(run[0], run[1:]...).CombinedOutput(); err != nil { + t.Fatalf("actual Go static-read oracle run: %v\n%s", err, output) + } + t.Logf("actual Go %s: %d ordinary static scalar result cases", arch, count) + file := mustParseX86RawReturn(t, raw.String()) + opt := Options{Goarch: arch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { + if strings.HasPrefix(symbol, "·__plan9asm_raw_address_") { + return "MutablePool" + } + return symbol + }} + for _, api := range []string{"text", "module", "owned-module"} { + var ir string + switch api { + case "text": + var err error + ir, err = Translate(file, opt) + if err != nil { + t.Fatal(err) + } + case "module": + module, err := TranslateModule(file, opt) + if err != nil { + t.Fatal(err) + } + ir = module.String() + module.Dispose() + default: + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + ir = module.String() + module.Dispose() + ctx.Dispose() + } + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, "static-read-"+api, triple, ir, cMain.String(), runner) + t.Logf("LLVM 22 %s/%s: %d scalar cases", arch, api, count) + } +} + +func TestX86RawReturnStaticReadNumericOracle(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("host-inapplicable: x86 scalar oracle runs in the required Linux/amd64 cross-runtime gate") + } + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("LLVM 22 clang not found") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + compiler := []string{clang, "-target", triple} + if runtime.GOOS == "darwin" { + output, err := exec.Command("xcrun", "--show-sdk-path").CombinedOutput() + if err != nil { + t.Fatalf("required Darwin SDK: %v\n%s", err, output) + } + compiler = append(compiler, "-isysroot", strings.TrimSpace(string(output))) + } + runX86StaticReadNumericOracle(t, "amd64", triple, compiler, nil) +} + +func TestCrossLinuxRuntimeMatrixX86RawStaticRead(t *testing.T) { + if os.Getenv("PLAN9ASM_CROSS_EXEC") != "1" { + return // The explicit cross job runs this required counterpart. + } + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("cross static-read oracle requires Linux/amd64 runner") + } + qemu, err := exec.LookPath("qemu-i386") + if err != nil { + t.Fatal("required pinned qemu-i386:", err) + } + output, err := exec.Command(qemu, "--version").CombinedOutput() + if err != nil || !strings.Contains(string(output), "version 10.2.3") { + t.Fatalf("required QEMU 10.2.3: %v\n%s", err, output) + } + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if err != nil { + t.Fatal("required 386 cross compiler:", err) + } + runX86StaticReadNumericOracle(t, "386", "i386-unknown-linux-gnu", []string{compiler, "-no-pie"}, []string{qemu, "-L", "/usr/i686-linux-gnu"}) +} diff --git a/amd64_raw_static_return_test.go b/amd64_raw_static_return_test.go new file mode 100644 index 000000000..d0d4061db --- /dev/null +++ b/amd64_raw_static_return_test.go @@ -0,0 +1,249 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86StaticReturnSource(name string, code []byte) string { + var source strings.Builder + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + return source.String() +} + +func TestX86RawReturnSharedRIPReadContract(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + cases := []struct { + name string + code []byte + width int + }{ + {"legacy-i32", []byte{0x8b, 0x05, 1, 0, 0, 0, 0xc3, 1, 2, 3, 4}, 4}, + {"legacy-i64", []byte{0x48, 0x8b, 0x05, 1, 0, 0, 0, 0xc3, 1, 2, 3, 4, 5, 6, 7, 8}, 8}, + {"legacy-vector", x86RawLegacyPackedMoveConstant(0xf3, 0x6f), 16}, + {"vex-vector", x86RawPackedMoveLiteral(false, 32), 32}, + {"evex-vector", x86RawPackedMoveLiteral(true, 64), 64}, + {"broadcast-f32", x86RawBinaryFloatLiteral(0x58, 0, true, 64, true), 4}, + {"broadcast-f64", x86RawBinaryFloatLiteral(0x58, 1, true, 64, true), 8}, + {"scalar-f32", x86RawBinaryFloatLiteral(0x58, 2, false, 16, false), 4}, + {"scalar-f64", x86RawBinaryFloatLiteral(0x58, 3, false, 16, false), 8}, + {"nonfirst-source", x86RawCLMULLiteral("legacy", 16), 16}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + source := x86StaticReturnSource("Bound", test.code) + requireX86GoAssemblerResult(t, "amd64", source, true) + prepared, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), "amd64") + if err != nil { + t.Fatal("shared RIP read rejected before materialization:", err) + } + if len(prepared.Data) != 1 || prepared.Data[0].Width != int64(test.width) { + t.Fatalf("materialized data: %+v, want exact width %d", prepared.Data, test.width) + } + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue // RIP-relative addressing exists only in 64-bit mode. + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(prepared, opt) + if err != nil { + t.Fatal("prepartition/final validation disagreed:", err) + } + compileLLVMToObject(t, llc, target.triple, "bounded-rip.ll", "bounded-rip.o", ir) + } + }) + } +} + +func TestX86RawReturnStaticDATAExtents(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + for _, offset := range []int{0, 4, 5} { + source := fmt.Sprintf("DATA pool<>+4(SB)/4,$0x76543210\nTEXT Bound(SB),4,$0-0\nMOVL pool<>+%d(SB),AX\nBYTE $0xc3\n", offset) + requireX86GoAssemblerResult(t, target.arch, source, true) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: I32}}} + x86RawReturnPointerAPIs(t, source, opt, offset == 5) + if offset != 5 { + ir, err := Translate(mustParseX86RawReturn(t, source), opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "bounded-data.ll", "bounded-data.o", ir) + } + } + } +} + +func TestX86RawReturnMaterializedReadCannotAuthorizeChangedSource(t *testing.T) { + source := x86StaticReturnSource("Bound", x86RawPackedMoveLiteral(false, 32)) + mutations := map[string]func(*File){ + "opcode store": func(file *File) { + file.Funcs[0].Instrs[1].Op = "VMOVDQU" + args := file.Funcs[0].Instrs[1].Args + args[0], args[1] = args[1], args[0] + }, + "unrecognized opcode": func(file *File) { + file.Funcs[0].Instrs[1].Op = "UNRECOGNIZED" + }, + "pointer operand": func(file *File) { + file.Funcs[0].Instrs[1].Args[0] = Operand{Kind: OpMem, Mem: MemRef{Base: AX}} + }, + "shifted source": func(file *File) { + file.Funcs[0].Instrs[1].Args[0].Sym = file.Data[0].Sym + "+1(SB)" + }, + "missing data": func(file *File) { + file.Data = nil + }, + "short object": func(file *File) { + file.Data[0].Width-- + file.Data[0].Payload = file.Data[0].Payload[:len(file.Data[0].Payload)-1] + }, + "changed payload": func(file *File) { + payload := append([]byte(nil), file.Data[0].Payload...) + payload[0] ^= 0xff + file.Data[0].Payload = payload + }, + "extra DATA write": func(file *File) { + file.Data = append(file.Data, DataStmt{Sym: file.Data[0].Sym, Width: 1, Value: 0xff}) + }, + "wrong pending index": func(file *File) { + file.Funcs[0].Instrs[1].Args[1] = Operand{Kind: OpSym, Sym: "·__plan9asm_raw_literal_pending(SB)"} + }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), "amd64") + if err != nil { + t.Fatal(err) + } + mutate(file) + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + // Test every final API on this exact prepared/changed file. + if x86RawReturnMemoryBound(file.Funcs[0], file, opt, true) { + t.Fatal("a decoded read authorized changed source or missing backing") + } + for _, api := range []string{"text", "module", "owned-module"} { + var callErr error + switch api { + case "text": + _, callErr = Translate(file, opt) + case "module": + module, err := TranslateModule(file, opt) + if err == nil { + module.Dispose() + } + callErr = err + default: + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + ctx.Dispose() + callErr = err + } + if !errors.Is(callErr, ErrProbeNeedsContext) { + t.Errorf("%s changed prepared source bypassed the read contract: %v", api, callErr) + } + } + } + }) + } +} + +func TestX86RawReturnRIPReadRequires64BitDecodeContext(t *testing.T) { + source := x86StaticReturnSource("Bound", x86RawPackedMoveLiteral(false, 32)) + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), "amd64") + if err != nil { + t.Fatal(err) + } + opt := Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + _, err = Translate(file, opt) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("64-bit RIP proof was reused as a 32-bit absolute-address proof: %v", err) + } +} + +func TestX86RawReturnSharedAddressPoolRetainsExactReadRange(t *testing.T) { + code := []byte{ + 0x48, 0x8d, 0x05, 0x09, 0, 0, 0, // LEAQ pool,AX. + 0xf3, 0x0f, 0x6f, 0x05, 0x09, 0, 0, 0, // MOVOU pool+8,X0. + 0xc3, + } + code = append(code, make([]byte, 32)...) + source := x86StaticReturnSource("Bound", code) + requireX86GoAssemblerResult(t, "amd64", source, true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue + } + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), target.arch) + if err != nil { + t.Fatal(err) + } + read := file.Funcs[0].Instrs[2] + if len(read.x86RIPLiteralData) != 0 || read.x86RIPMemoryRead.offset != 8 || + read.x86RIPMemoryRead.width != 16 || len(read.x86RIPMemoryRead.object) != 32 { + t.Fatalf("shared runtime read lost range or acquired specialization: %+v", read) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(file, opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "shared-pool.ll", "shared-pool.o", ir) + file.Funcs[0].Instrs[2].Args[0].Sym = file.Data[0].Sym + "+17(SB)" + if x86RawReturnMemoryBound(file.Funcs[0], file, opt, true) { + t.Fatal("changed read extending outside the shared object retained decoder proof") + } + } +} + +func TestX86RawReturnReadProofBindsCompleteOperandShape(t *testing.T) { + source := x86StaticReturnSource("Bound", x86RawPackedMoveLiteral(true, 32)) + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), "amd64") + if err != nil { + t.Fatal(err) + } + // The same valid EVEX opcode now requests twice the memory width. A + // private proof tied only to Op/source-index would incorrectly survive. + read := &file.Funcs[0].Instrs[1] + read.Args[len(read.Args)-1].Reg = "Z0" + opt := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + if x86RawReturnMemoryBound(file.Funcs[0], file, opt, true) { + t.Error("wider destination inherited a 32-byte decoded read proof") + } + _, err = Translate(file, opt) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("changed vector width reached final translation without Context: %v", err) + } +} diff --git a/amd64_raw_svm.go b/amd64_raw_svm.go new file mode 100644 index 000000000..8517e834b --- /dev/null +++ b/amd64_raw_svm.go @@ -0,0 +1,30 @@ +package plan9asm + +// decodedX86SystemManagementInstruction covers AMD's fixed 0F 01 D8-DF +// system-management family and Intel's matching KVM hypercall, VMCALL. +// golang.org/x/arch/x86asm does not decode +// these privileged instructions, so packages conventionally spell them as +// Plan 9 BYTE directives even though LLVM 22 can assemble their mnemonics. +func decodedX86SystemManagementInstruction(code []byte) (Instr, int, bool) { + if len(code) < 3 || code[0] != 0x0f || code[1] != 0x01 { + return Instr{}, 0, false + } + if code[2] == 0xc1 { + return Instr{Op: "VMCALL", Raw: "VMCALL"}, 3, true + } + if code[2] < 0xd8 || code[2] > 0xdf { + return Instr{}, 0, false + } + ops := [...]Op{ + "VMRUN", + "VMMCALL", + "VMLOAD", + "VMSAVE", + "STGI", + "CLGI", + "SKINIT", + "INVLPGA", + } + op := ops[code[2]-0xd8] + return Instr{Op: op, Raw: string(op)}, 3, true +} diff --git a/amd64_raw_svm_test.go b/amd64_raw_svm_test.go new file mode 100644 index 000000000..c8faba1a1 --- /dev/null +++ b/amd64_raw_svm_test.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86RawDecoderCompleteAMDSystemManagementFamily(t *testing.T) { + family := []struct { + opcode byte + op Op + }{ + {opcode: 0xd8, op: "VMRUN"}, + {opcode: 0xd9, op: "VMMCALL"}, + {opcode: 0xda, op: "VMLOAD"}, + {opcode: 0xdb, op: "VMSAVE"}, + {opcode: 0xdc, op: "STGI"}, + {opcode: 0xdd, op: "CLGI"}, + {opcode: 0xde, op: "SKINIT"}, + {opcode: 0xdf, op: "INVLPGA"}, + } + for _, goarch := range []string{"386", "amd64"} { + for _, test := range family { + t.Run(goarch+"/"+string(test.op), func(t *testing.T) { + fn := Func{Instrs: []Instr{ + {Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: 0x0f}}}, + {Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: 0x01}}}, + {Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(test.opcode)}}}, + }} + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.op || len(got.Instrs[0].Args) != 0 { + t.Fatalf("decoded 0f 01 %02x as %#v, want one operand-free %s", test.opcode, got.Instrs, test.op) + } + }) + } + } +} + +func TestTranslateX86RawAMDSystemManagementFamilyAllTargets(t *testing.T) { + const source = `TEXT svmfamily(SB),$0-0 + BYTE $0x0f; BYTE $0x01; BYTE $0xd8 // VMRUN + BYTE $0x0f; BYTE $0x01; BYTE $0xd9 // VMMCALL + BYTE $0x0f; BYTE $0x01; BYTE $0xda // VMLOAD + BYTE $0x0f; BYTE $0x01; BYTE $0xdb // VMSAVE + BYTE $0x0f; BYTE $0x01; BYTE $0xdc // STGI + BYTE $0x0f; BYTE $0x01; BYTE $0xdd // CLGI + BYTE $0x0f; BYTE $0x01; BYTE $0xde // SKINIT + BYTE $0x0f; BYTE $0x01; BYTE $0xdf // INVLPGA + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"svmfamily": {Name: "svmfamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, op := range []string{"vmrun", "vmmcall", "vmload", "vmsave", "stgi", "clgi", "skinit", "invlpga"} { + if !strings.Contains(ll, `asm sideeffect "`+op+`"`) { + t.Fatalf("translated IR does not retain %s:\n%s", op, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-svm-family-"+target.name+".ll", "raw-svm-family-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86AMDSystemManagementFamilyRejectsOperands(t *testing.T) { + for _, op := range []string{"VMRUN", "VMMCALL", "VMLOAD", "VMSAVE", "STGI", "CLGI", "SKINIT", "INVLPGA"} { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+op+" AX\n") + if err != nil { + t.Fatalf("Parse(%s AX) error = %v", op, err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted operand form for fixed no-operand %s", op) + } + } +} diff --git a/amd64_raw_text_blob_test.go b/amd64_raw_text_blob_test.go new file mode 100644 index 000000000..e9c9311ec --- /dev/null +++ b/amd64_raw_text_blob_test.go @@ -0,0 +1,260 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +func TestX86AddressSensitiveRawTextBlobReportedURootFarJump(t *testing.T) { + const source = x86URootFarJumpOriginalSource + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + file, signatures := x86UnboundVoidFormatSources(t, source, target.goarch, target.triple) + normalized := x86URootFarJumpNormalizedFile(t, file, target.goarch) + + // The complete original setup has public Context above. This + // private object owns only the selected raw TEXT's physical bytes; + // it supplies no executable setup or writable-code contract. + var format strings.Builder + fmt.Fprintf(&format, "target triple = %q\n", target.triple) + if err := emitX86AddressSensitiveRawText(&format, normalized.Funcs[1], signatures["farjump64"]); err != nil { + t.Fatal(err) + } + ir := format.String() + if !strings.Contains(ir, "define void @farjump64()") || + !strings.Contains(ir, "naked noinline") || + !strings.Contains(ir, `.byte 255, 45, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0`) { + t.Fatalf("byte-exact naked raw TEXT body missing:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "raw-text-"+target.name+".ll", "raw-text-"+target.name+".o", ir) + assertX86RawTextAssemblyBytes(t, llc, target.triple, ir, []byte{0xff, 0x2d, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0}) + }) + } +} + +func TestX86AddressSensitiveRawTextLeadingPCALIGNFormats(t *testing.T) { + for _, alignment := range []int{8, 16, 32, 64, 128, 256, 512, 1024, 2048} { + t.Run(strconv.Itoa(alignment), func(t *testing.T) { + source := fmt.Sprintf(`TEXT refer(SB),$0-0 + LEAQ data(SB), AX + RET +TEXT data(SB),$0 + PCALIGN $%d + LONG $0xdeadbeef +`, alignment) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for pass := 0; pass < 2; pass++ { + file, err = normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if got := file.Funcs[1].X86RawAlign; got != int64(alignment) { + t.Fatalf("alignment after normalization = %d, want %d", got, alignment) + } + } + }) + } +} + +func assertX86RawTextAssemblyBytes(t *testing.T, llc, triple, ir string, want []byte) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "raw-text.ll") + if err := os.WriteFile(path, []byte(ir), 0o600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(llc, "-mtriple="+triple, "-filetype=asm", "-o", "-", path) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("LLVM 22 assembly emission failed: %v\n%s", err, out) + } + var got []byte + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) != 2 || fields[0] != ".byte" { + continue + } + value, err := strconv.ParseUint(fields[1], 0, 8) + if err != nil { + t.Fatalf("parse emitted byte line %q: %v", line, err) + } + got = append(got, byte(value)) + } + if string(got) != string(want) { + t.Fatalf("LLVM 22 emitted raw TEXT bytes %v, want %v\n%s", got, want, out) + } +} + +func TestX86AddressSensitiveRawTextBlobSelection(t *testing.T) { + const source = `TEXT refs(SB),NOSPLIT,$0-0 + MOVL AX, farjump32+1(SB) + LEAQ gdt(SB), CX + MOVL info(SB), BX + RET +TEXT farjump32(SB),NOSPLIT,$0-0 + BYTE $0xea; LONG $0; WORD $0x18 +TEXT gdt(SB),NOSPLIT,$0-0 + QUAD $0; QUAD $0x00cf9a000000ffff +TEXT info(SB),NOSPLIT,$0-0 + LONG $0 +TEXT semantic(SB),NOSPLIT,$0-0 + BYTE $0x90; BYTE $0xc3 +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + wants := map[string][]byte{ + "farjump32": {0xea, 0, 0, 0, 0, 0x18, 0}, + "gdt": {0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0, 0, 0x9a, 0xcf, 0}, + "info": {0, 0, 0, 0}, + } + for _, fn := range normalized.Funcs { + want, selected := wants[fn.Sym] + if selected && string(fn.X86RawText) != string(want) { + t.Fatalf("%s raw TEXT bytes %v, want %v", fn.Sym, fn.X86RawText, want) + } + if fn.Sym == "semantic" && fn.X86RawText != nil { + t.Fatal("unaddressed raw instruction body bypassed semantic decoding") + } + } +} + +// Older x86 assemblers recognize PCALIGN in the parser but have no backend +// implementation. Observe that exact tool defect separately; it is neither a +// successful Go object nor a source-format rejection. LLVM objects below are +// still required on every supported library toolchain. +func x86AlignedRawTextGoObject(t *testing.T, source string) { + t.Helper() + directory := t.TempDir() + assembly, object := filepath.Join(directory, "aligned.s"), filepath.Join(directory, "aligned.o") + if err := os.WriteFile(assembly, []byte(source), 0600); err != nil { + t.Fatal(err) + } + stage, cancelStage := runtimeOracleStageContext(t) + defer cancelStage() + ctx, cancel := context.WithTimeout(stage, time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-p", "example.com/forms", "-o", object, assembly) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + // Joined pipe or cleanup failures are infrastructure errors, even when + // they contain the assembler's expected exit status and diagnostic. + exit, directExit := err.(*exec.ExitError) + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + if !goToolchainAtLeast(runtime.Version(), 1, 22) && ctx.Err() == nil && directExit && exit.ExitCode() == 1 && + len(lines) == 2 && strings.HasPrefix(lines[0], "asm: asmins: missing op ") && + strings.Contains(lines[0], "()\tPCALIGN\t$32") && lines[1] == "asm: assembly failed" { + t.Logf("GO_PCAlign_BACKEND_DEFECT_OBSERVED compiler=%s target=linux/amd64; no successful Go object claimed: %s", runtime.Version(), output) + return + } + if failure := goAssemblerOracleOutcome(assembly, object, output, err, ctx.Err(), true, "amd64", "example.com/forms"); failure != nil { + t.Fatalf("actual aligned raw TEXT compiler observation: %v\n%s", failure, output) + } +} + +func TestX86AddressSensitiveAlignedRawTextBlobReportedURoot(t *testing.T) { + const source = `TEXT trampoline_start(SB),$0-0 + LEAQ stack_top(SB), AX + MOVQ (AX), AX + RET +TEXT stack_top(SB),$0 + PCALIGN $32 + LONG $0xdeadbeef +` + x86AlignedRawTextGoObject(t, source) + + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + var blob *Func + for index := range normalized.Funcs { + if normalized.Funcs[index].Sym == "stack_top" { + blob = &normalized.Funcs[index] + } + } + if blob == nil || string(blob.X86RawText) != string([]byte{0xef, 0xbe, 0xad, 0xde}) || blob.X86RawAlign != 32 { + t.Fatalf("aligned raw TEXT was not retained: %+v", blob) + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + addressOnly := strings.Replace(source, "\tMOVQ (AX), AX\n", "", 1) + x86AlignedRawTextGoObject(t, addressOnly) + for _, target := range []struct { + name string + triple string + }{ + {name: "linux", triple: "x86_64-unknown-linux-gnu"}, + {name: "darwin", triple: "x86_64-apple-darwin"}, + {name: "windows", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + options := Options{ + Goarch: "amd64", + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "trampoline_start": {Name: "trampoline_start", Ret: Void}, + "stack_top": {Name: "stack_top", Ret: Void}, + }, + } + ir, err := Translate(file, options) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("addressing four raw TEXT bytes authorized an eight-byte read", err) + } + // The original source and blob checks above remain unchanged. + // This separate address-only program checks alignment and emitted + // bytes without pretending that the original load had an owner. + addressFile, err := Parse(ArchAMD64, addressOnly) + if err != nil { + t.Fatal(err) + } + ir, err = Translate(addressFile, options) + if err != nil || ir == "" { + t.Fatal("separate address-only raw TEXT program lost its object path", err) + } + if !strings.Contains(ir, "define void @stack_top()") || !strings.Contains(ir, " align 32 {") { + t.Fatal("aligned raw TEXT declaration missing") + } + compileLLVMToObject(t, llc, target.triple, "aligned-raw-text.ll", "aligned-raw-text.o", ir) + assertX86RawTextAssemblyBytes(t, llc, target.triple, ir, []byte{0xef, 0xbe, 0xad, 0xde}) + }) + } +} diff --git a/amd64_raw_text_contract_test.go b/amd64_raw_text_contract_test.go new file mode 100644 index 000000000..21c463f20 --- /dev/null +++ b/amd64_raw_text_contract_test.go @@ -0,0 +1,92 @@ +package plan9asm + +import ( + "bytes" + "errors" + "reflect" + "strings" + "testing" +) + +const x86URootFarJumpOriginalSource = `TEXT setup(SB),NOSPLIT,$0-0 + MOVL AX, farjump64+6(SB) + JMP farjump64(SB) +TEXT farjump64(SB),NOSPLIT,$0-0 + BYTE $0xff; BYTE $0x2d; LONG $0 + LONG $0 + LONG $8 +` + +func x86URootFarJumpNormalizedFile(t *testing.T, file *File, arch string) *File { + t.Helper() + normalized, err := normalizeX86RawFile(file, arch) + if err != nil { + t.Fatal("complete original far-jump source could not be normalized", err) + } + if len(normalized.Funcs) != 2 || normalized.Funcs[0].Sym != "setup" || normalized.Funcs[1].Sym != "farjump64" || + len(normalized.Data) != 0 || len(normalized.Globl) != 0 { + t.Fatal("original setup/raw-TEXT inventory changed") + } + if len(normalized.Funcs[0].Instrs) != 3 || normalized.Funcs[0].X86RawText != nil || + !reflect.DeepEqual(normalized.Funcs[0].Instrs, file.Funcs[0].Instrs) { + t.Fatal("raw TEXT selection omitted or rewrote the original setup instructions") + } + want := []byte{0xff, 0x2d, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0} + if len(normalized.Funcs[1].Instrs) != 6 || !bytes.Equal(normalized.Funcs[1].X86RawText, want) { + t.Fatalf("complete original 14-byte far-jump inventory changed: %x", normalized.Funcs[1].X86RawText) + } + return normalized +} + +func TestX86AddressSensitiveRawTextFarJumpOriginalContextAndInventory(t *testing.T) { + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + // Stock Go accepts the complete original source. Its public LLVM + // translation still needs initialized AX and writable TEXT backing. + file, _ := x86UnboundVoidFormatSources(t, x86URootFarJumpOriginalSource, target.arch, target.triple) + x86URootFarJumpNormalizedFile(t, file, target.arch) + }) + } +} + +func TestX86AddressSensitiveRawTextStoreNeedsSourceAndWritableData(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, row := range []struct { + name string + initialize string + text bool + context bool + }{ + {name: "owned-data-initialized", initialize: "MOVL $7, AX\n"}, + {name: "owned-data-uninitialized", context: true}, + {name: "owned-data-partial-input", initialize: "MOVW $7, AX\n", context: true}, + {name: "text-initialized", initialize: "MOVL $7, AX\n", text: true, context: true}, + {name: "text-uninitialized", text: true, context: true}, + } { + t.Run(target.triple+"/"+row.name, func(t *testing.T) { + storage := "GLOBL cell(SB),NOPTR,$14\n" + var tail string + declarations := "func patch()\n" + if row.text { + storage = "" + tail = "TEXT cell(SB),NOSPLIT,$0-0\nBYTE $0xff; BYTE $0x2d; LONG $0\nLONG $0\nLONG $8\n" + declarations += "func cell()\n" + } + source := storage + "TEXT patch(SB),NOSPLIT,$0-0\n" + row.initialize + "MOVL AX, cell+6(SB)\nRET\n" + tail + x86LEAActualTargetGoObject(t, target.arch, target.triple, "#include \"textflag.h\"\n"+source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, target.arch, target.triple, api) + if row.context { + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "MOVL AX, cell+6(SB)") { + t.Fatal("static store invented numerical input or writable TEXT backing", err) + } + } else if err != nil || ir == "" { + t.Fatal("initialized store to actual module-owned writable DATA failed", err) + } + }) + } + }) + } + } +} diff --git a/amd64_raw_text_frame.go b/amd64_raw_text_frame.go new file mode 100644 index 000000000..164368807 --- /dev/null +++ b/amd64_raw_text_frame.go @@ -0,0 +1,51 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Naked emission preserves the raw directive bytes, not obj6's additional +// Go frame, wrapper or stack-split prologue. Keep this source boundary explicit +// even when the body's bytes are data rather than decodable instructions. +func validateX86AddressSensitiveRawText(fn Func, goarch string) error { + if fn.FrameSize != 0 || fn.ArgSize != 0 { + return fmt.Errorf("%w: address-sensitive raw TEXT needs its source Go frame/argument transport contract", ErrProbeNeedsContext) + } + if len(fn.Instrs) == 0 || fn.Instrs[0].Op != OpTEXT { + return fmt.Errorf("%w: address-sensitive raw TEXT needs an actual source TEXT header", ErrProbeNeedsContext) + } + _, rest := splitOpcode(fn.Instrs[0].Raw) + parts := strings.Split(rest, ",") + if len(parts) != 2 && len(parts) != 3 { + return fmt.Errorf("%w: address-sensitive raw TEXT needs a resolved source header", ErrProbeNeedsContext) + } + if !strings.HasPrefix(strings.TrimSpace(parts[len(parts)-1]), "$") { + return fmt.Errorf("%w: address-sensitive raw TEXT needs a known source frame", ErrProbeNeedsContext) + } + frame, args, err := parseTEXTFrame(parts) + if err != nil || frame != fn.FrameSize || args != fn.ArgSize { + return fmt.Errorf("%w: address-sensitive raw TEXT source frame differs from its retained metadata", ErrProbeNeedsContext) + } + var flags uint64 + if len(parts) == 3 { + expression := globlFlagName.ReplaceAllStringFunc(parts[1], func(name string) string { + if value, known := goTextFlagValues[name]; known { + return value + } + return name + }) + var known bool + flags, known = parseImmExpr(strings.TrimSpace(expression)) + if !known { + return fmt.Errorf("%w: address-sensitive raw TEXT flags need actual Go source context", ErrProbeNeedsContext) + } + } + if flags & ^uint64(2|4|512) != 0 { + return fmt.Errorf("%w: address-sensitive raw TEXT requires unmodeled Go prologue/entry flags", ErrProbeNeedsContext) + } + if goarch == "386" && flags&4 == 0 { + return fmt.Errorf("%w: address-sensitive 386 raw TEXT cannot omit Go's stack-split prologue", ErrProbeNeedsContext) + } + return nil +} diff --git a/amd64_raw_text_frame_test.go b/amd64_raw_text_frame_test.go new file mode 100644 index 000000000..5ebdbda45 --- /dev/null +++ b/amd64_raw_text_frame_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "errors" + "fmt" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86AddressSensitiveFrameSource(arch, text string) string { + address := "LEAQ" + if arch == "386" { + address = "LEAL" + } + return fmt.Sprintf("TEXT refer(SB),4,$0-0\n%s blob(SB),AX\nRET\nTEXT blob(SB),%s\nBYTE $0xc3\n", address, text) +} + +func requireX86RawTextContextAPIs(t *testing.T, file *File, opt Options) { + t.Helper() + for _, api := range []string{"text", "module", "owned-module", "annotated"} { + var callErr error + switch api { + case "text": + _, callErr = Translate(file, opt) + case "module": + module, err := TranslateModule(file, opt) + if err == nil { + module.Dispose() + } + callErr = err + default: + ctx := llvm.NewContext() + options := opt + options.AnnotateSource = api == "annotated" + module, err := TranslateModuleInContext(ctx, file, options) + if err == nil { + module.Dispose() + } + ctx.Dispose() + callErr = err + } + if !errors.Is(callErr, ErrProbeNeedsContext) { + t.Errorf("%s omitted raw TEXT's source prologue/Go-frame contract: %v", api, callErr) + } + } +} + +func TestX86AddressSensitiveRawTextNeedsSourceFrameContract(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, text := range []string{"4,$8-0", "4,$16-0", "4,$0-8", "32,$0-0"} { + t.Run(target.triple+"/"+text, func(t *testing.T) { + source := x86AddressSensitiveFrameSource(target.arch, text) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + requireX86RawTextContextAPIs(t, file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"refer": {Name: "refer", Ret: Void}, "blob": {Name: "blob", Ret: Void}}, + }) + }) + } + } +} + +func TestX86AddressSensitiveRawTextCannotBypassGoFrameTransport(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, sig := range []FuncSig{ + {Name: "blob", Ret: Void, Args: []LLVMType{I32}, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I32, Field: -1}}}}, + {Name: "blob", Ret: I32, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I32, Field: -1}}}}, + } { + t.Run(target.triple+"/"+string(sig.Ret), func(t *testing.T) { + source := x86AddressSensitiveFrameSource(target.arch, "4,$0-0") + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + requireX86RawTextContextAPIs(t, file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"refer": {Name: "refer", Ret: Void}, "blob": sig}, + }) + }) + } + } +} + +func TestX86AddressSensitiveRawText386CannotOmitStackSplit(t *testing.T) { + // obj6's automatic NOSPLIT leaf promotion is amd64-only. An unflagged + // 386 TEXT has real TLS/stack-check bytes before the raw source body. + plain := assembleX87ControlBytes(t, "386", "TEXT blob(SB),$0-0\nBYTE $0xc3\n") + nosplit := assembleX87ControlBytes(t, "386", "TEXT blob(SB),4,$0-0\nBYTE $0xc3\n") + if len(nosplit) != 1 || nosplit[0] != 0xc3 || len(plain) <= len(nosplit) { + t.Fatalf("Go source prologue witness is not independent: plain=%x NOSPLIT=%x", plain, nosplit) + } + t.Logf("actual Go 386 source TEXT bytes: unflagged=%x NOSPLIT=%x", plain, nosplit) + file, err := Parse(ArchAMD64, x86AddressSensitiveFrameSource("386", "$0-0")) + if err != nil { + t.Fatal(err) + } + requireX86RawTextContextAPIs(t, file, Options{ + Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"refer": {Name: "refer", Ret: Void}, "blob": {Name: "blob", Ret: Void}}, + }) +} + +func TestX86AddressSensitiveRawTextFramelessBytesRemainExact(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + flags := []string{"4,$0-0", "6,$0-0", "516,$0-0"} + if target.arch == "amd64" { + // obj6 promotes this raw-only, zero-frame amd64 leaf to NOSPLIT. + flags = append(flags, "$0-0") + } + for _, text := range flags { + t.Run(target.triple+"/"+text, func(t *testing.T) { + original := assembleX87ControlBytes(t, target.arch, "TEXT blob(SB),"+text+"\nBYTE $0xc3\n") + if len(original) != 1 || original[0] != 0xc3 { + t.Fatalf("source Go TEXT unexpectedly adds a native prologue: %x", original) + } + file, err := Parse(ArchAMD64, x86AddressSensitiveFrameSource(target.arch, text)) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"refer": {Name: "refer", Ret: Void}, "blob": {Name: "blob", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, target.triple, "frameless-raw-text.ll", "frameless-raw-text.o", module.String()) + assertX86RawTextAssemblyBytes(t, llc, target.triple, module.String(), original) + }) + } + } +} diff --git a/amd64_raw_unaligned_load.go b/amd64_raw_unaligned_load.go new file mode 100644 index 000000000..0f9dd45e1 --- /dev/null +++ b/amd64_raw_unaligned_load.go @@ -0,0 +1,34 @@ +package plan9asm + +import "fmt" + +func decodedX86VEXUnalignedLoadInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || p.opcode != 0xf0 || p.pp != 3 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VLDDQU: %s", message) + } + if mode != 32 && mode != 64 || p.evex || p.vectorLength > 1 || p.upper != 0 || p.addressOverride { + return fail("invalid mode, VEX generation, length, reserved vvvv or address size") + } + if len(code) <= p.modRM || code[p.modRM]>>6 == 3 { + return fail("missing or non-memory ModRM") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register unavailable in physical mode32") + } + bank := []string{"X", "Y"}[p.vectorLength] + source, size, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, bank) + if err != nil { + return Instr{}, 0, true, err + } + if p.modRM+size > 15 { + return fail("instruction exceeds 15 bytes") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", bank, int(code[p.modRM]>>3&7)+p.r*8))} + ins := Instr{Op: "VLDDQU", Args: []Operand{source, destination}, x86Encoded: true} + ins.Raw = fmt.Sprintf("VLDDQU %s,%s", source.String(), destination.String()) + return ins, p.modRM + size, true, nil +} diff --git a/amd64_raw_unaligned_load_test.go b/amd64_raw_unaligned_load_test.go new file mode 100644 index 000000000..fa2d5e66d --- /dev/null +++ b/amd64_raw_unaligned_load_test.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "testing" +) + +func TestDecodedX86VEXUnalignedLoadCompleteMemoryForms(t *testing.T) { + for _, mode := range []int{32, 64} { + limit := 8 + if mode == 64 { + limit = 16 + } + for _, memory := range []struct { + encoded []byte + source string + }{ + {[]byte{0x00}, "(AX)"}, + {[]byte{0x40, 0x81}, "-127(AX)"}, + {[]byte{0x80, 0x78, 0x56, 0x34, 0x12}, "305419896(AX)"}, + {[]byte{0x44, 0x8b, 0x10}, "16(BX)(CX*4)"}, + } { + for _, segment := range []struct { + prefix []byte + name Reg + }{{nil, ""}, {[]byte{0x64}, FS}, {[]byte{0x65}, GS}} { + for width, bank := range []string{"X", "Y"} { + for destination := 0; destination < limit; destination++ { + for _, header := range [][]byte{ + {0xc5, byte(0xfb ^ destination/8<<7 | width<<2), 0xf0}, + {0xc4, byte(0xe1 ^ destination/8<<7), byte(0x7b | width<<2), 0xf0}, + {0xc4, byte(0xe1 ^ destination/8<<7), byte(0xfb | width<<2), 0xf0}, + } { + code := append(append([]byte(nil), segment.prefix...), header...) + address := append([]byte(nil), memory.encoded...) + address[0] |= byte(destination&7) << 3 + code = append(code, address...) + want := x86ScalarLaneInstruction(t, fmt.Sprintf("VLDDQU %s,%s%d", memory.source, bank, destination)) + want.Args[0].Mem.Segment = segment.name + got, length, known, err := decodedX86VEXUnalignedLoadInstruction(append(code, 0xcc), mode) + if err != nil || !known || length != len(code) || got.Op != want.Op || + !reflect.DeepEqual(got.Args, want.Args) || !got.x86Encoded { + t.Fatalf("complete VLDDQU %x mode%d: got=%+v length=%d known=%v err=%v want=%+v", code, mode, got, length, known, err, want) + } + } + } + } + } + } + } +} + +func TestDecodedX86VEXUnalignedLoadInvalidAndWholeBoundaries(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, test := range []struct { + name string + code []byte + }{ + {"reserved-vvvv", []byte{0xc4, 0xe1, 0x73, 0xf0, 0x00}}, + {"register-source", []byte{0xc4, 0xe1, 0x7b, 0xf0, 0xc0}}, + {"missing-modrm", []byte{0xc4, 0xe1, 0x7b, 0xf0}}, + {"missing-sib", []byte{0xc4, 0xe1, 0x7b, 0xf0, 0x04}}, + {"missing-disp8", []byte{0xc4, 0xe1, 0x7b, 0xf0, 0x40}}, + {"short-disp32", []byte{0xc4, 0xe1, 0x7b, 0xf0, 0x80, 0, 0, 0}}, + {"no-evex-form", []byte{0x62, 0xf1, 0x7f, 0x08, 0xf0, 0x00}}, + {"address-layout", []byte{0x67, 0xc4, 0xe1, 0x7b, 0xf0, 0x00}}, + } { + t.Run(fmt.Sprintf("mode%d/%s", mode, test.name), func(t *testing.T) { + _, _, known, err := decodedX86VEXUnalignedLoadInstruction(test.code, mode) + if !known || err == nil { + t.Fatal("invalid or incomplete physical VLDDQU accepted", known, err) + } + }) + } + t.Run(fmt.Sprintf("mode%d/maximum-instruction-length", mode), func(t *testing.T) { + base := []byte{0xc4, 0xe1, 0x7b, 0xf0, 0x40, 1} + valid := append(bytes.Repeat([]byte{0x64}, 9), base...) + if _, length, known, err := decodedX86VEXUnalignedLoadInstruction(valid, mode); !known || err != nil || length != 15 { + t.Fatal("complete legal 15-byte boundary lost", length, known, err) + } + invalid := append(bytes.Repeat([]byte{0x64}, 10), base...) + if _, _, known, err := decodedX86VEXUnalignedLoadInstruction(invalid, mode); !known || err == nil { + t.Fatal("machine-invalid instruction longer than 15 bytes accepted", known, err) + } + }) + } +} diff --git a/amd64_raw_unpack_float_test.go b/amd64_raw_unpack_float_test.go new file mode 100644 index 000000000..4f6378d74 --- /dev/null +++ b/amd64_raw_unpack_float_test.go @@ -0,0 +1,284 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +type rawFloatingUnpackSpec struct { + opcode byte + double bool + high bool +} + +var rawFloatingUnpackSpecs = []rawFloatingUnpackSpec{ + {opcode: 0x14}, + {opcode: 0x15, high: true}, + {opcode: 0x14, double: true}, + {opcode: 0x15, double: true, high: true}, +} + +func (spec rawFloatingUnpackSpec) op(vector bool) Op { + name := "UNPCKL" + if spec.high { + name = "UNPCKH" + } + if vector { + name = "V" + name + } + if spec.double { + name += "PD" + } else { + name += "PS" + } + return Op(name) +} + +func encodeX86RawLegacyFloatingUnpack(spec rawFloatingUnpackSpec, source, destination int) []byte { + code := []byte{} + if spec.double { + code = append(code, 0x66) + } + rex := byte(0x40 | source>>3&1 | destination>>3&1<<2) + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, spec.opcode, byte(0xc0|destination&7<<3|source&7)) +} + +func encodeX86RawVEXFloatingUnpack(spec rawFloatingUnpackSpec, vectorBits byte, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^first>>3)&1)<<5 | 1 + p1 := byte(^second&15)<<3 | vectorBits<<2 + if spec.double { + p1 |= 1 + } + return []byte{0xc4, p0, p1, spec.opcode, byte(0xc0 | destination&7<<3 | first&7)} +} + +func encodeX86RawEVEXFloatingUnpack(spec rawFloatingUnpackSpec, vectorBits byte, mask int, zeroing, broadcast bool, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 1 + p1 := byte(^second&15)<<3 | 4 + if spec.double { + p1 |= 0x81 + } + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + return []byte{0x62, p0, p1, p2, spec.opcode, byte(0xc0 | destination&7<<3 | first&7)} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVUNPCKLPS(t *testing.T) { + // vunpcklps %xmm3, %xmm5, %xmm0; + // vinsertf128 $1, %xmm0, %ymm0, %ymm0 + code := []byte{0xc5, 0xd0, 0x14, 0xc3, 0xc4, 0xe3, 0x7d, 0x18, 0xc0, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VUNPCKLPS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VUNPCKLPS X3, X5, X0 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawFloatingUnpackCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawFloatingUnpackSpecs { + for first := 0; first < 16; first++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawLegacyFloatingUnpack(spec, first, destination) + got, length, ok, err := decodedX86FloatingUnpackInstruction(code, 64) + want := fmt.Sprintf("%s X%d, X%d", spec.op(false), first, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for vectorBits := byte(0); vectorBits < 2; vectorBits++ { + vectorName := [...]string{"X", "Y"}[vectorBits] + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXFloatingUnpack(spec, vectorBits, first, second, destination) + got, length, ok, err := decodedX86FloatingUnpackInstruction(code, 64) + want := fmt.Sprintf("%s %s%d, %s%d, %s%d", spec.op(true), vectorName, first, vectorName, second, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{{}, {mask: 3}, {mask: 7, zeroing: true, suffix: ".Z"}} { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXFloatingUnpack(spec, vectorBits, masking.mask, masking.zeroing, false, first, second, destination) + got, length, ok, err := decodedX86FloatingUnpackInstruction(code, 64) + args := []string{fmt.Sprintf("%s%d", vectorName, first), fmt.Sprintf("%s%d", vectorName, second)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(spec.op(true)) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 1213440 { + t.Fatalf("covered %d floating-unpack register encodings, want 1213440", count) + } +} + +func TestDecodedX86RawFloatingUnpackMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0x66, 0x47, 0x0f, 0x15, 0x7c, 0x8b, 0x01} + got, length, ok, err := decodedX86FloatingUnpackInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "UNPCKHPD" || got.Args[1].String() != "X15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x01, 0x75, 0x14, 0x7c, 0x8b, 0x02} + got, length, ok, err = decodedX86FloatingUnpackInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VUNPCKLPD" || got.Args[1].String() != "Y1" || got.Args[2].String() != "Y15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexMemory := []byte{0x62, 0x01, 0x85, 0xdf, 0x15, 0x7c, 0x8b, 0x02} + got, length, ok, err = decodedX86FloatingUnpackInstruction(evexMemory, 64) + wantMemory.Off = 16 + if err != nil || !ok || length != len(evexMemory) || got.Op != "VUNPCKHPD.BCST.Z" || got.Args[1].String() != "Z15" || got.Args[2].String() != "K7" || got.Args[3].String() != "Z31" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + validVEX := encodeX86RawVEXFloatingUnpack(rawFloatingUnpackSpecs[0], 0, 7, 7, 7) + validEVEX := encodeX86RawEVEXFloatingUnpack(rawFloatingUnpackSpecs[0], 2, 0, false, false, 7, 7, 7) + for name, invalid := range map[string][]byte{ + "vex W": {validVEX[0], validVEX[1], validVEX[2] | 0x80, validVEX[3], validVEX[4]}, + "vex prefix": {validVEX[0], validVEX[1], validVEX[2] | 0x02, validVEX[3], validVEX[4]}, + "evex missing fixed bit": {validEVEX[0], validEVEX[1], validEVEX[2] &^ 0x04, validEVEX[3], validEVEX[4], validEVEX[5]}, + "evex prefix": {validEVEX[0], validEVEX[1], validEVEX[2] | 0x02, validEVEX[3], validEVEX[4], validEVEX[5]}, + "evex reserved length": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x60, validEVEX[4], validEVEX[5]}, + "zero without mask": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x80, validEVEX[4], validEVEX[5]}, + "register broadcast": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x10, validEVEX[4], validEVEX[5]}, + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86FloatingUnpackInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + masked386 := encodeX86RawEVEXFloatingUnpack(rawFloatingUnpackSpecs[0], 2, 1, false, false, 0, 1, 2) + if instruction, _, matched, decodeErr := decodedX86FloatingUnpackInstruction(masked386, 32); !matched || decodeErr == nil { + t.Fatalf("386 masked encoding %x decoded as %+v, ok=%v, err=%v", masked386, instruction, matched, decodeErr) + } + highZ386 := encodeX86RawEVEXFloatingUnpack(rawFloatingUnpackSpecs[0], 2, 0, false, false, 8, 1, 2) + if instruction, _, matched, decodeErr := decodedX86FloatingUnpackInstruction(highZ386, 32); !matched || decodeErr == nil { + t.Fatalf("386 high-Z encoding %x decoded as %+v, ok=%v, err=%v", highZ386, instruction, matched, decodeErr) + } +} + +func TestX86FloatingUnpackGo127ArchitectureOracle(t *testing.T) { + const accepted386 = `TEXT ok(SB),$0-0 + VUNPCKLPS X20, X21, X22 + VUNPCKLPD Y20, Y21, Y22 + VUNPCKHPS Z5, Z6, Z7 + VUNPCKHPD.BCST 8(AX), X20, X21 + RET +` + requireX86GoAssemblerResult(t, "386", accepted386, true) + for _, instruction := range []string{ + "VUNPCKLPS Z8, Z1, Z2", + "VUNPCKHPS Z0, Z1, K1, Z2", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } +} + +func TestTranslateX86RawFloatingUnpackTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + registerMax int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", registerMax: 15}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registerMax: 15}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registerMax: 15}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", registerMax: 7}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", registerMax: 7}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_floating_unpack(SB),NOSPLIT,$0-0\n") + for _, spec := range rawFloatingUnpackSpecs { + code := [][]byte{ + encodeX86RawLegacyFloatingUnpack(spec, target.registerMax, target.registerMax), + encodeX86RawVEXFloatingUnpack(spec, 1, target.registerMax, target.registerMax, target.registerMax), + } + if target.goarch == "amd64" { + code = append(code, encodeX86RawEVEXFloatingUnpack(spec, 2, 7, true, false, 31, 30, 29)) + } else { + code = append(code, encodeX86RawEVEXFloatingUnpack(spec, 2, 0, false, false, 7, 6, 5)) + } + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + if target.goarch == "386" { + // These unmasked, register-only rows may propagate unknown + // bytes when every output is dead. They do not invent NUM. + x86LEAActualTargetGoObject(t, target.goarch, target.triple, "#include \"textflag.h\"\n"+source.String()) + exposed := strings.Replace(source.String(), "$0-0", "$0-4", 1) + exposed = strings.Replace(exposed, "\tRET\n", "MOVL X5,AX\nMOVL AX,ret0+0(FP)\nRET\n", 1) + x86LEAActualTargetGoObject(t, target.goarch, target.triple, "#include \"textflag.h\"\n"+exposed) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func raw_floating_unpack()", target.goarch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("dead register-only unpack lost its complete source check", api, err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, text, level) + } + } + ir, err = x86ScalarFPBitViewAPI(t, exposed, "func raw_floating_unpack() uint32", target.goarch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("dead unpack scratch acquired an observable initialized result", api, err) + } + } + return + } + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-floating-unpack-"+target.name+".ll", "raw-floating-unpack-"+target.name+".o", ir) + }) + } +} diff --git a/amd64_raw_unreachable_tail_test.go b/amd64_raw_unreachable_tail_test.go new file mode 100644 index 000000000..1096e2d11 --- /dev/null +++ b/amd64_raw_unreachable_tail_test.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86UnreachableRawTailAfterReturn(t *testing.T) { + const source = ` +TEXT rawtail(SB),$0-0 + RET +GLOBL structtbl(SB),$8 + BYTE $0x7b; BYTE $0x7d; BYTE $0x5b; BYTE $0x5d + BYTE $0x22; BYTE $0x3a; BYTE $0x2c; BYTE $0x27 +GLOBL wordtbl(SB),$4 + BYTE $0x74; BYTE $0x68; BYTE $0x6e; BYTE $0x6b +` + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "rawtail": {Name: "rawtail", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "structtbl") || !strings.Contains(ll, "wordtbl") { + t.Fatal("GLOBL declarations disappeared from IR") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "x86-raw-unreachable-tail.ll", "x86-raw-unreachable-tail.o", ll) + }) + } +} + +func TestTranslateX86LabeledRawTailRemainsReachable(t *testing.T) { + const source = ` +TEXT rawtail(SB),$0-0 + JMP tail + RET +tail: + BYTE $0x7b; BYTE $0x7d + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "rawtail": {Name: "rawtail", Ret: Void}, + }, + }); err == nil || !strings.Contains(err.Error(), "outside directive group") { + t.Fatalf("reachable malformed raw tail was not rejected: %v", err) + } +} + +func TestTranslateX86AddressObservedRawTailFailsClosed(t *testing.T) { + const source = ` +TEXT rawtail(SB),$0-0 + RET + BYTE $0x7b; BYTE $0x7d +TEXT address(SB),$0-0 + MOVQ $rawtail(SB), AX + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "rawtail": {Name: "rawtail", Ret: Void}, + "address": {Name: "address", Ret: Void}, + }, + }); err == nil || !strings.Contains(err.Error(), "address-observed raw tail") { + t.Fatalf("accepted layout-sensitive raw tail: %v", err) + } +} diff --git a/amd64_raw_valign_test.go b/amd64_raw_valign_test.go new file mode 100644 index 000000000..c86661631 --- /dev/null +++ b/amd64_raw_valign_test.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawVectorAlignCompleteGo127Family(t *testing.T) { + tests := []struct { + name string + code []byte + want string + }{ + {name: "dword x high masked", code: []byte{0x62, 0xa3, 0x55, 0x01, 0x03, 0xf4, 0xca}, want: "VALIGND $202, X20, X21, K1, X22"}, + {name: "qword y memory zeroing", code: []byte{0x62, 0xd3, 0xf5, 0xaa, 0x03, 0x50, 0x03, 0x96}, want: "VALIGNQ.Z $150, 96(R8), Y1, K2, Y2"}, + {name: "dword z register", code: []byte{0x62, 0x73, 0x55, 0x48, 0x03, 0xce, 0xca}, want: "VALIGND $202, Z6, Z5, Z9"}, + {name: "dword z broadcast high masked", code: []byte{0x62, 0x53, 0x7d, 0x54, 0x03, 0x4c, 0x24, 0x7f, 0x44}, want: "VALIGND.BCST $68, 508(R12), Z16, K4, Z9"}, + {name: "qword z broadcast zeroing", code: []byte{0x62, 0xf3, 0xf5, 0xdb, 0x03, 0x50, 0x7f, 0x09}, want: "VALIGNQ.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + decoded, err := decodeX86RawDirectiveGroup(test.code, 64, 0, "VALIGN", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, test.want+" ") { + t.Fatalf("decoded %x as %#v, want %q", test.code, decoded, test.want) + } + }) + } +} + +func TestDecodedX86RawVectorAlignAllRegisterFields(t *testing.T) { + for _, lane := range []struct { + w byte + op Op + }{ + {w: 0, op: "VALIGND"}, + {w: 1, op: "VALIGNQ"}, + } { + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 3 + p1 := lane.w<<7 | byte(^second&15)<<3 | 5 + p2 := byte(vectorBits)<<5 | byte((^second>>4)&1)<<3 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0x62, p0, p1, p2, 0x03, modRM, 0xa5} + instruction, length, ok, err := decodedX86VectorAlignInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("op=%s width=%s first=%d second=%d destination=%d: ok=%v length=%d err=%v", lane.op, vectorName, first, second, destination, ok, length, err) + } + want := fmt.Sprintf("%s $165, %s%d, %s%d, %s%d", lane.op, vectorName, first, vectorName, second, vectorName, destination) + if instruction.Raw != want { + t.Fatalf("encoding %x decoded as %q, want %q", encoding, instruction.Raw, want) + } + } + } + } + } + } +} + +func TestDecodedX86RawVectorAlignRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "386 operand limit", mode: 32, encoding: []byte{0x62, 0xf3, 0x75, 0x08, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "reserved vector length", mode: 64, encoding: []byte{0x62, 0xf3, 0x75, 0x68, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "broadcast register", mode: 64, encoding: []byte{0x62, 0xf3, 0x75, 0x18, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "zero without mask", mode: 64, encoding: []byte{0x62, 0xf3, 0x75, 0x88, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0x62, 0xf3, 0x75, 0x08, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "missing immediate", mode: 64, encoding: []byte{0x62, 0xf3, 0x75, 0x08, 0x03, 0xc2}, wantOK: false}, + {name: "wrong map", mode: 64, encoding: []byte{0x62, 0xf2, 0x75, 0x08, 0x03, 0xc2, 0x01}, wantOK: false}, + {name: "wrong prefix", mode: 64, encoding: []byte{0x62, 0xf3, 0x74, 0x08, 0x03, 0xc2, 0x01}, wantOK: false}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VectorAlignInstruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x: ok=%v, want %v (err=%v)", test.encoding, ok, test.wantOK, err) + } + if test.wantOK && err == nil { + t.Fatalf("encoding %x was recognized without rejection", test.encoding) + } + }) + } +} diff --git a/amd64_raw_variable_shift.go b/amd64_raw_variable_shift.go new file mode 100644 index 000000000..51ad2eb09 --- /dev/null +++ b/amd64_raw_variable_shift.go @@ -0,0 +1,54 @@ +package plan9asm + +import "fmt" + +// Go has five VEX+EVEX D/Q members and four EVEX-only W/Q members. +// All use per-element counts, not the scalar count of VPSLL/PSRL/PSRA. +func decodedX86PerLaneVariableShiftInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PerLaneVariableShiftInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PerLaneVariableShiftInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + // F3.0F38 opcodes 10-12 are single-source narrowing instructions, + // not a malformed 66.0F38 per-lane shift. Other wrong prefixes remain + // owned here so callers get an explicit invalid-form diagnostic. + if !matched || p.mapNumber != 2 || p.pp == 2 { + return Instr{}, 0, false, nil + } + var op Op + var spec amd64PerLaneVariableShiftSpec + owned := false + for name, candidate := range amd64PerLaneVariableShiftSpecs { + if int(candidate.opcode) != p.opcode { + continue + } + owned = true + if p.w == (candidate.laneBits != 32) { + op, spec = Op(name), candidate + break + } + } + if !owned { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("per-lane variable shift: %s", message) + } + if op == "" { + return fail("invalid element width bit") + } + if !p.evex && !spec.vex { + return fail("instruction requires EVEX encoding") + } + + ins, consumed, known, err := decodedX86BinaryVectorOperandsWithMemory(code, p, mode, op, spec.laneBits, memory) + if known && err == nil { + ins.x86PackedShiftEncoding = x86RawPackedShiftVEX + if p.evex { + ins.x86PackedShiftEncoding = x86RawPackedShiftEVEX + } + ins.x86PackedShiftW = p.w + } + return ins, consumed, known, err +} diff --git a/amd64_raw_variable_shift_literal_test.go b/amd64_raw_variable_shift_literal_test.go new file mode 100644 index 000000000..a8562e2f4 --- /dev/null +++ b/amd64_raw_variable_shift_literal_test.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawVariableShiftLiteral(op string, evex bool, width int, broadcast bool) []byte { + spec := amd64PerLaneVariableShiftSpecs[op] + mask, zeroing := 0, false + if evex { + mask, zeroing = 7, true + } + code := encodeRawScalarMove(evex, 1, int(spec.opcode), 0, 0, 0, mask, zeroing) + code[1] = code[1]&0xf0 | 2 + code[2] &^= 0x80 + if spec.laneBits != 32 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + if broadcast { + code[3] |= 0x10 + } + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*19+3)) + } + return code +} + +func TestDecodeX86RawVariableShiftRIPDataCompleteFamily(t *testing.T) { + for _, op := range x86VariableShiftTestOps { + spec := amd64PerLaneVariableShiftSpecs[op] + for _, evex := range []bool{false, true} { + if !evex && !spec.vex { + continue + } + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && spec.laneBits >= 32 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", op, evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawVariableShiftLiteral(op, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op(op) + if broadcast { + want += ".BCST" + } + if evex { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawVariableShiftRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawVariableShiftLiteral("VPSLLVQ", true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVariableShiftRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, op := range x86VariableShiftTestOps { + spec := amd64PerLaneVariableShiftSpecs[op] + evex := !spec.vex || index%2 == 0 + width := 32 + if evex { + width = 64 + } + broadcast := evex && spec.laneBits >= 32 + code := x86RawVariableShiftLiteral(op, evex, width, broadcast) + name := fmt.Sprintf("variableShiftLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86PackedShiftRawFormatObject(t, file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + compileLLVMToObject(t, llc, triple, "variable-shift-literal.ll", "variable-shift-literal.o", ir) + }) + } +} diff --git a/amd64_raw_variable_shift_test.go b/amd64_raw_variable_shift_test.go new file mode 100644 index 000000000..bf98eebe8 --- /dev/null +++ b/amd64_raw_variable_shift_test.go @@ -0,0 +1,469 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86VariableShiftTestOps = []string{"VPSLLVW", "VPSLLVD", "VPSLLVQ", "VPSRLVW", "VPSRLVD", "VPSRLVQ", "VPSRAVW", "VPSRAVD", "VPSRAVQ"} + +func TestX86VariableShiftGrammarMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`(?s)\{as: A(VPS(?:LL|RL|RA)V[WDQ]), ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 9 || len(rows) != len(amd64PerLaneVariableShiftSpecs) { + t.Fatalf("Go entries=%d, grammar=%d", len(rows), len(amd64PerLaneVariableShiftSpecs)) + } + for _, row := range rows { + spec, ok := amd64PerLaneVariableShiftSpecs[row[1]] + if !ok { + t.Fatalf("missing %s", row[1]) + } + bits := map[byte]int{'W': 16, 'D': 32, 'Q': 64}[row[1][len(row[1])-1]] + operation := "shl" + if strings.HasPrefix(row[1], "VPSRL") { + operation = "lshr" + } + if strings.HasPrefix(row[1], "VPSRA") { + operation = "ashr" + } + if spec.laneBits != bits || spec.operation != operation || spec.arithmetic != (operation == "ashr") { + t.Fatalf("Go semantics differ for %s: %+v", row[1], spec) + } + vex := bits != 16 && row[1] != "VPSRAVQ" + if spec.vex != vex || (row[2] == "_yvandnpd") != vex || strings.Contains(row[3], "| vex128") != vex || strings.Contains(row[3], "evexBcst") != (bits != 16) { + t.Fatalf("Go encoding surface changed for %s: %s", row[1], row[3]) + } + for _, opcode := range regexp.MustCompile(`0x[0-9A-F]+`).FindAllString(row[3], -1) { + if opcode != fmt.Sprintf("0x%X", spec.opcode) { + t.Fatalf("Go opcode %s != spec %+v", opcode, spec) + } + } + } +} + +func TestX86RawVariableShiftGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + last := 7 + if arch == "amd64" { + last = 31 + } + var source strings.Builder + source.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, op := range x86VariableShiftTestOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, reg := range []int{0, last} { + for _, input := range []string{fmt.Sprintf("%s%d", width, last-reg), "(AX)", "-7(BX)(CX*4)", "64(SI)", "8192(DI)"} { + fmt.Fprintf(&source, "%s %s,%s%d,%s%d\n", op, input, width, reg, width, last-reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s,%s%d,K1,%s%d\n%s.Z %s,%s%d,K7,%s%d\n", op, input, width, reg, width, last-reg, op, input, width, reg, width, last-reg) + } + } + if !strings.HasSuffix(op, "W") { + for _, input := range []string{"-8(BX)(CX*4)", "64(SI)"} { + fmt.Fprintf(&source, "%s.BCST %s,%s%d,%s%d\n", op, input, width, reg, width, last-reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s.BCST %s,%s%d,K1,%s%d\n%s.BCST.Z %s,%s%d,K7,%s%d\n", op, input, width, reg, width, last-reg, op, input, width, reg, width, last-reg) + } + } + } + } + // Low registers force the VEX row where available, even when the + // high-register rows above necessarily select EVEX on amd64. + if width != "Z" { + fmt.Fprintf(&source, "%s %s0,%s1,%s2\n%s (AX),%s1,%s2\n", op, width, width, width, op, width, width) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + got, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(got.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(got.Instrs), len(want)) + } + for i, instruction := range got.Instrs { + if instruction.Op != want[i].Op || !reflect.DeepEqual(instruction.Args, want[i].Args) { + t.Fatalf("instruction %d: %+v, want %+v", i, instruction, want[i]) + } + } + var raw strings.Builder + raw.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, value := range code { + fmt.Fprintf(&raw, "BYTE $%#02x\n", value) + } + file, err := Parse(ArchAMD64, raw.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + options := Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"shifts": {Name: "shifts", Ret: Void}}} + if ir, err := Translate(file, options); ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original raw near RET acquired a native return/memory/input contract", err) + } + // Whole-file normalization correctly rejects this unbound + // native RET. Compile the independently byte-checked decoder + // output as an instruction-only unit, preserving its RET; + // do not replace RET by UD2 to manufacture public success. + units := *file + units.Funcs = []Func{got} + units.Funcs[0].Sym = "shifts" + units.Funcs[0].Instrs = append([]Instr{file.Funcs[0].Instrs[0]}, got.Instrs...) + ir := x86InstructionFormatObjectUnit(t, &units, options.Sigs, arch, triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + return c.lowerX86PackedShift(ins.Op, ins) + }) + compileLLVMToObject(t, llc, triple, "shifts.ll", "shifts.o", ir) + } + t.Logf("%s: %d actual Go encodings", arch, len(want)-1) + }) + } +} + +func TestX86VariableShiftGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86VariableShiftTestOps { + lines := []string{op + " X0,Y1,Y2", op + " X0,X1,(AX)", op + ".Z X0,X1,X2", op + ".SAE X0,X1,X2", op + " X0,X1,K0,X2", op + ".BCST X0,X1,X2", op + ".Z.BCST (AX),X1,K1,X2", op + " $1,X1,X2"} + if strings.HasSuffix(op, "W") { + lines = append(lines, op+".BCST (AX),X1,X2") + } + if arch == "386" { + lines = append(lines, op+" Z8,Z0,Z1", op+" X0,X1,K1,X2") + } + for _, line := range lines { + // Go 1.20 emitted reserved zeroing/K0 for maskless .Z. Preserve + // the strict named/raw rejection even for that historical encoder. + { + dir := t.TempDir() + path := filepath.Join(dir, "invalid.s") + if err := os.WriteFile(path, []byte("TEXT bad(SB),4,$0-0\n"+line+"\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), path) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + out, err := cmd.CombinedOutput() + historicalZero := strings.HasPrefix(runtime.Version(), "go1.20") && strings.Contains(line, ".Z X0") + if historicalZero && err != nil || !historicalZero && err == nil { + t.Fatalf("Go %s unexpected acceptance for %s (historical zeroing=%v): %v %s", arch, line, historicalZero, err, out) + } + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + assertX86PerLaneVariableShiftRejected(t, arch, triple, line) + } + } + } +} + +func TestX86RawVariableShiftEncodingAxes(t *testing.T) { + count := 0 + for _, op := range x86VariableShiftTestOps { + bits := map[byte]int{'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + opcode := map[string]int{"VPSLLVW": 0x12, "VPSLLVD": 0x47, "VPSLLVQ": 0x47, "VPSRLVW": 0x10, "VPSRLVD": 0x45, "VPSRLVQ": 0x45, "VPSRAVW": 0x11, "VPSRAVD": 0x46, "VPSRAVQ": 0x46}[op] + for _, evex := range []bool{false, true} { + if !evex && (bits == 16 || op == "VPSRAVQ") { + continue + } + limit, widths := 16, 2 + if evex { + limit, widths = 32, 3 + } + for width := 0; width < widths; width++ { + for src1 := 0; src1 < limit; src1++ { + for src2 := 0; src2 < limit; src2++ { + for dst := 0; dst < limit; dst++ { + mask := 0 + zero := false + if evex { + mask = (dst + src1 + src2) % 8 + zero = mask != 0 && src2&1 != 0 + } + code := encodeRawScalarMove(evex, 1, 0, src2, src1, dst, mask, zero) + code[len(code)-2] = byte(opcode) + code[1] = code[1]&0xf0 | 2 + code[2] &^= 0x80 + if bits != 32 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(width) << 5 + } else { + code[2] |= byte(width) << 2 + } + got, size, matched, err := decodedX86PerLaneVariableShiftInstruction(code, 64) + prefix := []string{"X", "Y", "Z"}[width] + wantOp := Op(op) + if zero { + wantOp += ".Z" + } + args := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src2))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src1))}} + if mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, dst))}) + if err != nil || !matched || size != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, args) { + t.Fatalf("decode %x: %+v %d %v %v; want %s %+v", code, got, size, matched, err, wantOp, args) + } + _, _, matched, err = decodedX86PerLaneVariableShiftInstruction(code, 32) + if !matched || (err != nil) != (src1 >= 8 || src2 >= 8 || dst >= 8) { + t.Fatalf("386 decode %x matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + if count != 925696 { + t.Fatalf("covered %d encodings, want 925696", count) + } +} + +func TestX86RawVariableShiftInvalidAndMemoryForms(t *testing.T) { + valid := []byte{0x62, 0xf2, 0xf5, 0x0b, 0x47, 0xe0} + for name, mutate := range map[string]func([]byte){"fixed": func(b []byte) { b[2] &^= 4 }, "length": func(b []byte) { b[3] |= 0x60 }, "zero-mask": func(b []byte) { b[3] = 0x88 }, "broadcast-register": func(b []byte) { b[3] |= 0x10 }, "prefix": func(b []byte) { b[2] &^= 3 }} { + b := append([]byte(nil), valid...) + mutate(b) + if _, _, matched, err := decodedX86PerLaneVariableShiftInstruction(b, 64); !matched || err == nil { + t.Fatalf("accepted %s %x", name, b) + } + } + for _, b := range [][]byte{valid[:5], {0x62, 0xf2, 0xf5, 0x0b, 0x47, 0x04}, {0x62, 0xf2, 0xf5, 0x0b, 0x47, 0x40}, append([]byte{0x67}, valid...), {0x62, 0xf2, 0xf5, 0x1b, 0x12, 0x00}, {0x62, 0xf2, 0x75, 0x0b, 0x12, 0xc0}, {0xc4, 0xe2, 0xf9, 0x46, 0xc0}, {0xc4, 0xe2, 0xf9, 0x12, 0xc0}, {0xc4, 0xe2, 0xf9, 0x47, 0x05, 0, 0, 0, 0}} { + if _, _, matched, err := decodedX86PerLaneVariableShiftInstruction(b, 64); !matched || err == nil { + t.Fatalf("accepted invalid %x", b) + } + } + if _, _, matched, err := decodedX86PerLaneVariableShiftInstruction(valid, 16); !matched || err == nil { + t.Fatal("accepted mode16") + } + for _, b := range [][]byte{nil, {0x62}, {0x90}, {0xc4, 0xe1, 0xf9, 0x47, 0xc0}, {0xc4, 0xe2, 0xf9, 0x48, 0xc0}} { + if _, _, matched, err := decodedX86PerLaneVariableShiftInstruction(b, 64); matched || err != nil { + t.Fatalf("claimed unrelated %x", b) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + for _, broadcast := range []bool{false, true} { + b := []byte{segment.prefix, 0x62, 0x92, 0xf5, 0xcb, 0x47, 0x64, 0x88, 0xfe} + offset := int64(-128) + op := Op("VPSLLVQ.Z") + if broadcast { + b[4] |= 0x10 + offset = -16 + op = "VPSLLVQ.BCST.Z" + } + got, size, matched, err := decodedX86PerLaneVariableShiftInstruction(b, 64) + want := MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: offset} + if err != nil || !matched || size != len(b) || got.Op != op || !reflect.DeepEqual(got.Args[0].Mem, want) { + t.Fatalf("decode %x=%+v: %v", b, got, err) + } + } + } +} + +func TestX86RawVariableShift386MaskedObjects(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, op := range x86VariableShiftTestOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, input := range []string{width + "0", "(AX)"} { + for _, suffix := range []string{"", ".Z"} { + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s%s %s,%s1,K7,%s2", op, suffix, input, width, width), true) + } + } + if !strings.HasSuffix(op, "W") { + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s.BCST.Z (AX),%s1,K7,%s2", op, width, width), true) + } + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir := x86PackedShiftRawFormatObject(t, file, Options{Goarch: "386", TargetTriple: triple, + Sigs: map[string]FuncSig{"shifts": {Name: "shifts", Ret: Void}}}) + compileLLVMToObject(t, llc, triple, "masked-shifts-386.ll", "masked-shifts-386.o", ir) + } +} + +func TestX86VariableShiftMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + cross := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !cross { + t.Skip("execution requires amd64 or Rosetta; five-target object tests remain required") + } + var source, decl strings.Builder + checks := map[string]*strings.Builder{"views": {}, "memory": {}} + sigs := map[string]FuncSig{} + index := 0 + for _, op := range x86VariableShiftTestOps { + bits := map[byte]int{'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + operation := 0 + if strings.HasPrefix(op, "VPSRL") { + operation = 1 + } + if strings.HasPrefix(op, "VPSRA") { + operation = 2 + } + for w, width := range []string{"X", "Y", "Z"} { + for mode := 0; mode < 5; mode++ { // reg/mem unmasked, reg/mem masked, broadcast masked. + if mode == 4 && bits == 16 { + continue + } + for _, zero := range []bool{false, true} { + if zero && mode < 2 { + continue + } + for _, raw := range []bool{false, true} { + name := fmt.Sprintf("shift%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ data+0(FP),AX\nMOVQ counts+8(FP),BX\nMOVQ out+16(FP),DI\nVMOVUPS (AX),Z0\n", name) + input := "(BX)" + if mode == 0 || mode == 2 { + source.WriteString("VMOVUPS (BX),Z1\n") + input = width + "1" + } else if raw { + source.WriteString("VMOVUPS (AX),Z1\n") + } + suffix, mask := "", "" + if mode == 4 { + suffix = ".BCST" + } + if zero { + suffix += ".Z" + } + if mode >= 2 { + source.WriteString("KMOVQ mask+24(FP),K1\n") + mask = "K1," + } + dst := 0 + if raw { + dst = 1 + } + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s%s %s,%s0,%s%s%d", op, suffix, input, width, mask, width, dst), raw) + fmt.Fprintf(&source, "VMOVUPS Z%d,(DI)\nVMOVUPS Y%d,64(DI)\nVMOVUPS X%d,96(DI)\nRET\n", dst, dst, dst) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + fmt.Fprintf(&decl, "extern void %s(const void*,const void*,void*,uint64_t);\n", name) + kind := "views" + if mode >= 2 { + kind = "memory" + } + z := 0 + if zero { + z = 1 + } + oldCounts := 0 + if raw && (mode == 0 || mode == 2) { + oldCounts = 1 + } + fmt.Fprintf(checks[kind], "if(check(%s,%d,%d,%d,%d,%d,%d)){fprintf(stderr,\"%s failed\\n\");return 1;}\n", name, 16< +#include +#include +` + x86TestGuardPagesC + decl.String() + ` +static uint64_t lane(const uint8_t *p,int bytes){uint64_t v=0;memcpy(&v,p,bytes);return v;} +static uint64_t shifted(uint64_t value,uint64_t count,int bits,int op){ + uint64_t limit=bits==64?UINT64_MAX:(UINT64_C(1)<=(uint64_t)bits)return op==2 && (value&sign)?limit:0; + if(op==0)return (value<>count; + if(op==2 && (value&sign) && count)result|=limit^(limit>>count); + return result; +} +static int check(void(*fn)(const void*,const void*,void*,uint64_t),int width,int bytes,int op,int mode,int zero,int oldCounts){ + for(int n=0;n<96;n++){ + uint8_t ab[65],bb[65],out[112],want[112]={0};uint8_t *a=ab+1,*b=bb+1; + uint64_t counts[]={0,1,(uint64_t)bytes*8-1,(uint64_t)bytes*8,(uint64_t)bytes*8+1,UINT64_MAX,UINT64_C(1)<<(bytes*8-1)}; + for(int i=0;i<64;i++)a[i]=(uint8_t)(i*31+n*17); + for(int i=0;i<64/bytes;i++){uint64_t count=counts[(n+i)%7];memcpy(b+i*bytes,&count,bytes);} + int lanes=width/bytes;uint64_t mask=n==0?0:n==1?UINT64_C(1)<<63:n==2?UINT64_MAX:UINT64_C(0x5a5a01239876cdef)>>(n%32); + uint64_t active=mask&((UINT64_C(1)<>i)&1)?shifted(value,count,bytes*8,op):zero?0:oldCounts?lane(b+i*bytes,bytes):value; + memcpy(want+i*bytes,&result,bytes); + } + memcpy(want+64,want,32);memcpy(want+96,want,16); + fn(a,mode>=3 && !active?NULL:b,out,mask); + if(memcmp(out,want,112)){fprintf(stderr,"width=%d bytes=%d op=%d mode=%d zero=%d n=%d\n",width,bytes,op,mode,zero,n);return 1;} + } + if(mode>=3){ + int lanes=width/bytes; + for(int enabled=1;enabled<=lanes;enabled++){ + uint8_t a[64],out[112],want[112]={0};memset(a,0xff,64); + int readable=mode==4?bytes:enabled*bytes;uint8_t *b=guard+page-readable; + memset(b,0,(size_t)readable); + uint64_t mask=(UINT64_C(1)< 1 { + return fail("invalid 66 prefix, reserved vvvv or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.x != 0 || p.b != 0) { + return fail("extended register in 32-bit mode") + } + width := "X" + if p.vectorLength == 1 { + width = "Y" + } + source, size, err := memory.vex(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]>>3&7)+p.r*8))} + return Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_vecflags_test.go b/amd64_raw_vecflags_test.go new file mode 100644 index 000000000..9ebd4f85d --- /dev/null +++ b/amd64_raw_vecflags_test.go @@ -0,0 +1,202 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawVectorTestCompleteGo127EncodingFamily(t *testing.T) { + if len(x86RawVectorTestOpcodes) != len(amd64VectorTestSpecs)-1 { + t.Fatalf("raw VEX vector-test opcode table has %d entries, named grammar has %d VEX entries", len(x86RawVectorTestOpcodes), len(amd64VectorTestSpecs)-1) + } + for _, op := range x86RawVectorTestOpcodes { + if spec, ok := amd64VectorTestSpecs[op]; !ok || !spec.vector { + t.Fatalf("raw opcode %s has no matching named VEX grammar", op) + } + } + + type form struct { + name string + op Op + code []byte + src Operand + dst Reg + } + var forms []form + for _, spec := range []struct { + name Op + opcode byte + legacy bool + }{ + {"PTEST", 0x17, true}, + {"VPTEST", 0x17, false}, + {"VTESTPS", 0x0e, false}, + {"VTESTPD", 0x0f, false}, + } { + for _, width := range []struct { + name string + vex3 byte + }{ + {"X", 0x79}, + {"Y", 0x7d}, + } { + if spec.legacy && width.name == "Y" { + continue + } + for _, source := range []struct { + name string + modRM byte + value Operand + }{ + {"register", 0xc1, Operand{Kind: OpReg, Reg: Reg(width.name + "1")}}, + {"memory", 0x08, Operand{Kind: OpMem, Mem: MemRef{Base: AX}}}, + } { + code := []byte{0xc4, 0xe2, width.vex3, spec.opcode, source.modRM} + if spec.legacy { + code = []byte{0x66, 0x0f, 0x38, spec.opcode, source.modRM} + } + dest := Reg(width.name + "0") + if source.name == "memory" { + dest = Reg(width.name + "1") + } + forms = append(forms, form{ + name: string(spec.name) + "/" + width.name + "/" + source.name, + op: spec.name, code: code, src: source.value, dst: dest, + }) + } + } + } + for _, goarch := range []string{"386", "amd64"} { + for _, form := range forms { + t.Run(goarch+"/"+form.name, func(t *testing.T) { + fn := Func{} + for _, value := range form.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != form.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", form.code, got.Instrs, form.op) + } + ins := got.Instrs[0] + if ins.Args[0].Kind != form.src.Kind || ins.Args[0].Reg != form.src.Reg || ins.Args[0].Mem != form.src.Mem || ins.Args[1].Reg != form.dst { + t.Fatalf("decoded %#x operands as %#v, want %#v, %s", form.code, ins.Args, form.src, form.dst) + } + }) + } + } +} + +func TestX86RawVectorTestRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"missing-66", []byte{0xc4, 0xe2, 0x78, 0x17, 0xc1}, 64}, + {"nonzero-vvvv", []byte{0xc4, 0xe2, 0x71, 0x17, 0xc1}, 64}, + {"address-override", []byte{0x67, 0xc4, 0xe2, 0x79, 0x17, 0x08}, 64}, + {"missing-modrm", []byte{0xc4, 0xe2, 0x79, 0x17}, 64}, + {"extended-386", []byte{0xc4, 0x62, 0x79, 0x17, 0xc1}, 32}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86VEXVectorTestInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } + for _, tc := range []struct { + name string + code []byte + want Operand + }{ + {"W-ignored", []byte{0xc4, 0xe2, 0xf9, 0x17, 0xc1}, Operand{Kind: OpReg, Reg: "X1"}}, + {"SIB-memory", []byte{0xc4, 0xe2, 0x7d, 0x0f, 0x44, 0x88, 0x20}, Operand{Kind: OpMem, Mem: MemRef{Base: AX, Index: CX, Scale: 4, Off: 32}}}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86VEXVectorTestInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) { + t.Fatalf("encoding %#x: instruction=%+v length=%d recognized=%v error=%v", tc.code, got, length, recognized, err) + } + if !reflect.DeepEqual(got.Args[0], tc.want) { + t.Fatalf("source = %+v, want %+v", got.Args[0], tc.want) + } + }) + } +} + +func TestX86RawVectorTestCompilesEveryForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawvectortest(SB),NOSPLIT,$0-0\n") + for _, spec := range []struct { + opcode byte + legacy bool + }{ + {0x17, true}, {0x17, false}, {0x0e, false}, {0x0f, false}, + } { + for _, width := range []byte{0x79, 0x7d} { + if spec.legacy && width == 0x7d { + continue + } + for _, modRM := range []byte{0xc1, 0x08} { + code := []byte{0xc4, 0xe2, width, spec.opcode, modRM} + if spec.legacy { + code = []byte{0x66, 0x0f, 0x38, spec.opcode, modRM} + } + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawvectortest": {Name: "rawvectortest", Ret: Void}}, + }) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original raw vector-test formats invented native or register inputs", err) + } + x86LEAActualTargetGoObject(t, target.goarch, target.triple, "#include \"textflag.h\"\n"+source.String()) + file, err = normalizeX86RawFile(file, target.goarch) + if err != nil { + t.Fatal(err) + } + ir = x86InstructionFormatObjectUnit(t, file, + map[string]FuncSig{"rawvectortest": {Name: "rawvectortest", Ret: Void}}, target.goarch, target.triple, 0, + func(c *amd64Ctx, ins Instr) (bool, bool, error) { + if ins.Op == "NOP" { + return true, false, nil + } + return c.lowerVectorTest(ins.Op, ins) + }) + compileLLVMToObject(t, llc, target.triple, "raw-vector-test.ll", "raw-vector-test.o", ir) + }) + } +} diff --git a/amd64_raw_vector_float_compare_literal_test.go b/amd64_raw_vector_float_compare_literal_test.go new file mode 100644 index 000000000..00b51fc32 --- /dev/null +++ b/amd64_raw_vector_float_compare_literal_test.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawVectorFloatCompareLiteral(pp byte, encoding string, width int, broadcast bool) []byte { + var code []byte + switch encoding { + case "vex2": + code = encodeX86VEX2VectorFloatCompare(pp, width == 32, 0, 0, 0, 12) + case "vex3": + code = encodeX86VEXVectorFloatCompare(pp, width == 32, 0, 0, 0, 12) + case "evex": + code = encodeX86EVEXVectorFloatCompare( + pp, byte(map[int]int{16: 0, 32: 1, 64: 2}[width]), + broadcast, 1, 1, 0, 0, 12, + ) + } + modRMIndex := len(code) - 2 + code[modRMIndex] = 0x05 + immediate := code[len(code)-1] + code = append(code[:modRMIndex+1], 1, 0, 0, 0, immediate, 0xc3) + dataWidth := width + if pp >= 2 || broadcast { + dataWidth = 4 + if pp == 1 || pp == 3 { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*41+15)) + } + return code +} + +func TestDecodeX86RawVectorFloatCompareRIPDataCompleteFamily(t *testing.T) { + for pp := byte(0); pp < 4; pp++ { + for _, encoding := range []string{"vex2", "vex3", "evex"} { + widths := []int{16, 32} + if encoding == "evex" { + widths = append(widths, 64) + } + if pp >= 2 { + widths = []int{16} + } + for _, width := range widths { + broadcasts := []bool{false} + if encoding == "evex" && pp < 2 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/%s/%d/bcst%t", vectorFloatCompareRawOp(pp), encoding, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawVectorFloatCompareLiteral(pp, encoding, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := vectorFloatCompareRawOp(pp) + if broadcast { + want += ".BCST" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpImm || decoded[0].Args[0].Imm != 12 || + decoded[0].Args[1].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawVectorFloatCompareRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawVectorFloatCompareLiteral(1, "evex", 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x65}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + "EVEX zeroing": func(code []byte) []byte { + code[3] |= 0x80 + return code + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVectorFloatCompareRIPDataObjects(t *testing.T) { + var source strings.Builder + var pools [][]byte + index := 0 + for pp := byte(0); pp < 4; pp++ { + for _, encoding := range []string{"vex2", "vex3", "evex"} { + width := 16 + if pp < 2 { + width = 32 + if encoding == "evex" { + width = 64 + } + } + broadcast := encoding == "evex" && pp < 2 + code := x86RawVectorFloatCompareLiteral(pp, encoding, width, broadcast) + name := fmt.Sprintf("vectorCompareLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + dataWidth := width + if pp >= 2 || broadcast { + dataWidth = 4 + if pp == 1 || pp == 3 { + dataWidth = 8 + } + } + pools = append(pools, code[len(code)-dataWidth:]) + index++ + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // Original encodings lack vector and mask inputs. Preserve their + // entire Go/emitter object and public Context contracts; independent + // initialized literal programs own the public object matrix. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "vector-compare-literal.ll", "vector-compare-literal.o", ir) + }) + } +} diff --git a/amd64_raw_vector_float_compare_test.go b/amd64_raw_vector_float_compare_test.go new file mode 100644 index 000000000..eded5c4f3 --- /dev/null +++ b/amd64_raw_vector_float_compare_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXVectorFloatCompare(pp byte, width256 bool, destination, secondSource, firstSource int, immediate byte) []byte { + l := byte(0) + if width256 { + l = 1 + } + vex0 := byte((1-(destination>>3)&1)<<7 | 1<<6 | (1-(firstSource>>3)&1)<<5 | 1) + vex1 := byte((^secondSource&15)<<3 | int(l)<<2 | int(pp)) + modRM := byte(0xc0 | (destination&7)<<3 | firstSource&7) + return []byte{0xc4, vex0, vex1, 0xc2, modRM, immediate} +} + +func encodeX86VEX2VectorFloatCompare(pp byte, width256 bool, destination, secondSource, firstSource int, immediate byte) []byte { + l := byte(0) + if width256 { + l = 1 + } + vex1 := byte((1-(destination>>3)&1)<<7 | (^secondSource&15)<<3 | int(l)<<2 | int(pp)) + modRM := byte(0xc0 | (destination&7)<<3 | firstSource&7) + return []byte{0xc5, vex1, 0xc2, modRM, immediate} +} + +func encodeX86EVEXVectorFloatCompare(pp, vectorBits byte, broadcastOrSAE bool, writeMask, destinationMask, secondSource, firstSource int, immediate byte) []byte { + p0 := byte(1<<7 | (1-(firstSource>>4)&1)<<6 | (1-(firstSource>>3)&1)<<5 | 1<<4 | 1) + p1 := byte((^secondSource&15)<<3 | 1<<2 | int(pp)) + if pp == 1 || pp == 3 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^secondSource>>4&1)<<3 | writeMask) + if broadcastOrSAE { + p2 |= 0x10 + } + modRM := byte(0xc0 | (destinationMask&7)<<3 | firstSource&7) + return []byte{0x62, p0, p1, p2, 0xc2, modRM, immediate} +} + +func vectorFloatCompareRawOp(pp byte) Op { + return [...]Op{"VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"}[pp] +} + +func TestTranslateRawEVEXVectorFloatCompareWeaviateRegression(t *testing.T) { + const source = `TEXT rawVectorFloatCompare(SB), $0-0 + LONG $0x483cf162; WORD $0x06c2; BYTE $0x0c + RET +` + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + if !strings.Contains(ir, `asm sideeffect "vcmpps $$12,`) { + t.Fatalf("raw EVEX VCMPPS lowering omitted floating compare:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vector-float-compare.ll", "amd64-raw-vector-float-compare.o", ir) +} + +func TestDecodedX86VEXVectorFloatCompareCompleteRegisterFamily(t *testing.T) { + count := 0 + for pp := byte(0); pp < 4; pp++ { + widths := []bool{false, true} + if pp >= 2 { + widths = []bool{false} + } + for _, width256 := range widths { + prefix := "X" + if width256 { + prefix = "Y" + } + for destination := 0; destination < 16; destination++ { + for secondSource := 0; secondSource < 16; secondSource++ { + for firstSource := 0; firstSource < 16; firstSource++ { + code := encodeX86VEXVectorFloatCompare(pp, width256, destination, secondSource, firstSource, 0xa5) + got, length, ok, err := decodedX86VectorFloatCompareInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{"$165", fmt.Sprintf("%s%d", prefix, firstSource), fmt.Sprintf("%s%d", prefix, secondSource), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != vectorFloatCompareRawOp(pp) || len(got.Args) != 4 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, vectorFloatCompareRawOp(pp), strings.Join(wantArgs, ", ")) + } + count++ + } + for firstSource := 0; firstSource < 8; firstSource++ { + code := encodeX86VEX2VectorFloatCompare(pp, width256, destination, secondSource, firstSource, 0xa5) + got, length, ok, err := decodedX86VectorFloatCompareInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{"$165", fmt.Sprintf("%s%d", prefix, firstSource), fmt.Sprintf("%s%d", prefix, secondSource), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != vectorFloatCompareRawOp(pp) || len(got.Args) != 4 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, vectorFloatCompareRawOp(pp), strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + if count != 36864 { + t.Fatalf("covered %d VEX vector floating compare encodings, want 36864", count) + } +} + +func TestDecodedX86EVEXVectorFloatCompareCompleteRegisterFamily(t *testing.T) { + count := 0 + for pp := byte(0); pp < 4; pp++ { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + if pp >= 2 { + prefix = "X" + } + for destinationMask := 0; destinationMask < 8; destinationMask++ { + for secondSource := 0; secondSource < 32; secondSource++ { + for firstSource := 0; firstSource < 32; firstSource++ { + code := encodeX86EVEXVectorFloatCompare(pp, vectorBits, false, 1, destinationMask, secondSource, firstSource, 0x5a) + got, length, ok, err := decodedX86VectorFloatCompareInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{"$90", fmt.Sprintf("%s%d", prefix, firstSource), fmt.Sprintf("%s%d", prefix, secondSource), "K1", fmt.Sprintf("K%d", destinationMask)} + if got.Op != vectorFloatCompareRawOp(pp) || len(got.Args) != 5 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] || got.Args[4].String() != wantArgs[4] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, vectorFloatCompareRawOp(pp), strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + if count != 98304 { + t.Fatalf("covered %d EVEX vector floating compare encodings, want 98304", count) + } +} + +func TestDecodedX86EVEXVectorFloatCompareModifiersMemoryAndInvalid(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + args []string + }{ + {name: "weaviate memory", code: []byte{0x62, 0xf1, 0x3c, 0x48, 0xc2, 0x06, 0x0c}, op: "VCMPPS", args: []string{"$12", "0(SI)", "Z8", "K0"}}, + {name: "packed broadcast", code: []byte{0x62, 0xf1, 0xfd, 0x59, 0xc2, 0x48, 0x02, 0x7f}, op: "VCMPPD.BCST", args: []string{"$127", "16(AX)", "Z0", "K1", "K1"}}, + {name: "packed sae", code: encodeX86EVEXVectorFloatCompare(0, 2, true, 2, 7, 20, 21, 31), op: "VCMPPS.SAE", args: []string{"$31", "Z21", "Z20", "K2", "K7"}}, + {name: "scalar lig", code: encodeX86EVEXVectorFloatCompare(3, 1, false, 0, 4, 3, 21, 27), op: "VCMPSD", args: []string{"$27", "X21", "X3", "K4"}}, + {name: "scalar sae", code: encodeX86EVEXVectorFloatCompare(2, 3, true, 3, 5, 1, 11, 47), op: "VCMPSS.SAE", args: []string{"$47", "X11", "X1", "K3", "K5"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86VectorFloatCompareInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != len(test.args) { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + for index := range test.args { + if got.Args[index].String() != test.args[index] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } + }) + } + + scalarBroadcast := encodeX86EVEXVectorFloatCompare(2, 0, true, 0, 0, 0, 0, 0) + scalarBroadcast[len(scalarBroadcast)-2] = 0 + invalid := [][]byte{ + encodeX86VEXVectorFloatCompare(2, true, 0, 0, 0, 0), + encodeX86EVEXVectorFloatCompare(0, 3, false, 0, 0, 0, 0, 0), + scalarBroadcast, + } + for _, code := range invalid { + if _, _, ok, err := decodedX86VectorFloatCompareInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} diff --git a/amd64_raw_vector_literal_test.go b/amd64_raw_vector_literal_test.go new file mode 100644 index 000000000..364feac21 --- /dev/null +++ b/amd64_raw_vector_literal_test.go @@ -0,0 +1,341 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawPackedMoveLiteral(evex bool, width int) []byte { + var code []byte + if evex { + length := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) + code = []byte{0x62, 0xf1, 0x7c, length<<5 | 0x08, 0x10, 0x05} + } else { + p1 := byte(0x7e) + if width == 16 { + p1 = 0x7a + } + code = []byte{0xc5, p1, 0x6f, 0x05} + } + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*7+3)) + } + return code +} + +func TestDecodeX86RawPackedMoveRIPDataCompleteWidths(t *testing.T) { + for _, test := range []struct { + evex bool + width int + }{ + {false, 16}, {false, 32}, + {true, 16}, {true, 32}, {true, 64}, + } { + name := fmt.Sprintf("evex%t/width%d", test.evex, test.width) + t.Run(name, func(t *testing.T) { + code := x86RawPackedMoveLiteral(test.evex, test.width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !decoded[0].x86RIPLiteral || + len(decoded[0].Args) < 2 || decoded[0].Args[0].Kind != OpSym || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want source-local vector constant", code, decoded) + } + }) + } +} + +func TestTranslateX86RawPackedMoveRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, test := range []struct { + evex bool + width int + }{ + {false, 16}, {false, 32}, + {true, 16}, {true, 32}, {true, 64}, + } { + name := fmt.Sprintf("vectorLiteral%d", index) + code := x86RawPackedMoveLiteral(test.evex, test.width) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(sigs) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(sigs)) + } + for index, datum := range normalized.Data { + width := []int{16, 32, 16, 32, 64}[index] + code := x86RawPackedMoveLiteral(index >= 2, width) + want := code[len(code)-width:] + if datum.Width != int64(width) || !bytes.Equal(datum.Payload, want) { + t.Fatalf("constant %d = %x width %d, want %x", index, datum.Payload, datum.Width, want) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "vector-literal.ll", "vector-literal.o", ir) + }) + } +} + +func TestDecodeX86RawPackedMoveRIPDataRejectsUnsafeSources(t *testing.T) { + // The same opcode byte is used by scalar VMOVSS/SD; this packed-move + // decoder must leave those encodings to the scalar grammar. + for _, code := range [][]byte{ + {0xc5, 0xfa, 0x10, 0x05, 1, 0, 0, 0}, + {0xc5, 0xfb, 0x10, 0x05, 1, 0, 0, 0}, + } { + if _, _, _, matched, _ := decodeX86RawPackedMoveRIPData(code, 0, 64); matched { + t.Fatalf("packed move claimed scalar encoding %x", code) + } + } + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "instruction_overlap", edit: func(code []byte) []byte { + copy(code[6:10], []byte{0xfa, 0xff, 0xff, 0xff}) + return code + }}, + {name: "missing_pool", edit: func(code []byte) []byte { + code[6] = 0x7f + return code + }}, + {name: "truncated_pool", edit: func(code []byte) []byte { + code[6] = 2 + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x64}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.edit(x86RawPackedMoveLiteral(true, 64)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP vector source: %x", code) + } + }) + } +} + +func TestTranslateX86RawPackedMoveRIPDataRejectsAddressObservedText(t *testing.T) { + code := x86RawPackedMoveLiteral(true, 64) + var source strings.Builder + source.WriteString("TEXT address(SB),$0-0\n\tLEAQ raw+1(SB), AX\n\tRET\n") + source.WriteString("TEXT raw(SB),$0-0\n\tMOVQ AX, AX\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{ + "address": {Name: "address", Ret: Void}, + "raw": {Name: "raw", Ret: Void}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "address-observed RIP-relative raw literal") { + t.Fatalf("address-observed raw TEXT was silently folded: %v", err) + } +} + +func x86RawEVEXPackedIntegerMoveLiteral(pp byte, wide bool, width, mask int, zeroing bool) []byte { + p1 := byte(0x7c) | pp + if wide { + p1 |= 0x80 + } + p2 := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + code := []byte{0x62, 0xf1, p1, p2, 0x6f, 0x05, 1, 0, 0, 0, 0xc3} + for index := 0; index < width; index++ { + code = append(code, byte(index*5+1)) + } + return code +} + +func TestDecodeX86RawEVEXPackedIntegerMoveRIPDataCompleteFamily(t *testing.T) { + for _, form := range []struct { + pp byte + wide bool + op Op + }{ + {1, false, "VMOVDQA32"}, {1, true, "VMOVDQA64"}, + {2, false, "VMOVDQU32"}, {2, true, "VMOVDQU64"}, + {3, false, "VMOVDQU8"}, {3, true, "VMOVDQU16"}, + } { + for _, width := range []int{16, 32, 64} { + for _, mask := range []int{0, 3, 7} { + zeroing := mask == 7 + name := fmt.Sprintf("%s/%d/mask%d", form.op, width, mask) + t.Run(name, func(t *testing.T) { + code := x86RawEVEXPackedIntegerMoveLiteral(form.pp, form.wide, width, mask, zeroing) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + op := form.op + if zeroing { + op += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != op || + len(decoded[0].Args) < 2 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data load", code, decoded, op) + } + }) + } + } + } +} + +func TestTranslateX86RawEVEXPackedIntegerMoveRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + forms := []struct { + pp byte + wide bool + }{ + {1, false}, {1, true}, {2, false}, + {2, true}, {3, false}, {3, true}, + } + for index, form := range forms { + name := fmt.Sprintf("evexIntegerLiteral%d", index) + code := x86RawEVEXPackedIntegerMoveLiteral(form.pp, form.wide, 64, 7, true) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + // These original six sources have no K7 producer. Preserve their + // complete byte/format coverage, not invented initialized masks. + ir := x86UnboundVoidVectorMoveFormatObject(t, source.String(), "amd64", triple) + compileLLVMToObject(t, llc, triple, "evex-integer-literal.ll", "evex-integer-literal.o", ir) + }) + } +} + +func TestDecodeX86RawEVEXPackedIntegerMoveRIPDataRejectsUnsafeSource(t *testing.T) { + for _, displacement := range []byte{0x7f, 2} { + code := x86RawEVEXPackedIntegerMoveLiteral(2, false, 64, 0, false) + code[6] = displacement + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "unsafe EVEX integer source", map[string]bool{}); err == nil { + t.Fatalf("accepted out-of-group EVEX integer source %x", code) + } + } +} + +func x86RawVBROADCASTI128Literal() []byte { + code := []byte{0xc4, 0xe2, 0x7d, 0x5a, 0x05, 1, 0, 0, 0, 0xc3} + for index := 0; index < 16; index++ { + code = append(code, byte(index*11+2)) + } + return code +} + +func TestDecodeX86RawVBROADCASTI128RIPData(t *testing.T) { + code := x86RawVBROADCASTI128Literal() + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VBROADCASTI128 literal", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VBROADCASTI128" || + len(decoded[0].Args) != 2 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want VBROADCASTI128 source-local data", code, decoded) + } +} + +func TestTranslateX86RawVBROADCASTI128RIPDataObjects(t *testing.T) { + code := x86RawVBROADCASTI128Literal() + var source strings.Builder + source.WriteString("TEXT blockLiteral(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != 1 || !bytes.Equal(normalized.Data[0].Payload, code[len(code)-16:]) { + t.Fatalf("VBROADCASTI128 constant = %#v, want the exact source bytes", normalized.Data) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"blockLiteral": {Name: "blockLiteral", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "block-literal.ll", "block-literal.o", ir) + }) + } +} + +func TestDecodeX86RawVBROADCASTI128RIPDataRejectsUnsafeSource(t *testing.T) { + for _, displacement := range []byte{3, 0x7f} { + code := x86RawVBROADCASTI128Literal() + code[5] = displacement + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "unsafe block broadcast", map[string]bool{}); err == nil { + t.Fatalf("accepted out-of-group VBROADCASTI128 source %x", code) + } + } +} diff --git a/amd64_raw_vector_zero.go b/amd64_raw_vector_zero.go new file mode 100644 index 000000000..25cc82dcb --- /dev/null +++ b/amd64_raw_vector_zero.go @@ -0,0 +1,48 @@ +package plan9asm + +import "fmt" + +// The pinned x/arch decoder calls both VEX.L variants VZEROUPPER and accepts +// reserved pp/vvvv values. Decode this complete zero-operand family before +// generic fallback, binding the actual bytes rather than its printed opcode. +func decodedX86VectorZeroInstruction(code []byte, mode int) (Instr, int, bool, error) { + i, forbidden := 0, false + for i < len(code) { + prefix := code[i] + switch { + case prefix == 0x26 || prefix == 0x2e || prefix == 0x36 || prefix == 0x3e || prefix == 0x64 || prefix == 0x65 || prefix == 0x67: + case prefix == 0x66 || prefix == 0xf0 || prefix == 0xf2 || prefix == 0xf3 || mode == 64 && prefix >= 0x40 && prefix <= 0x4f: + forbidden = true + default: + goto prefixEnd + } + i++ + } +prefixEnd: + if i >= len(code) || code[i] != 0xc4 && code[i] != 0xc5 { + return Instr{}, 0, false, nil + } + payload, opcode, length := i+1, i+2, i+3 + if code[i] == 0xc4 { + payload, opcode, length = i+2, i+3, i+4 + } + if opcode >= len(code) { + return Instr{}, 0, false, nil + } + if mode == 32 && code[i+1]&0xc0 != 0xc0 { + // These bytes are legacy LES/LDS, not a mode32 VEX prefix. + return Instr{}, 0, false, nil + } + if code[opcode] != 0x77 || code[i] == 0xc4 && code[i+1]&31 != 1 { + return Instr{}, 0, false, nil + } + if mode != 32 && mode != 64 || forbidden || length > 15 || code[payload]&3 != 0 || code[payload]&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("reserved raw VEX vector-zero encoding %x in mode%d", code[:length], mode) + } + for op, spec := range x86VectorZeroSpecs { + if spec.vexLength == code[payload]>>2&1 { + return Instr{Op: op, Raw: string(op)}, length, true, nil + } + } + return Instr{}, 0, true, fmt.Errorf("unregistered raw VEX vector-zero length") +} diff --git a/amd64_raw_vex_binary_float_test.go b/amd64_raw_vex_binary_float_test.go new file mode 100644 index 000000000..a0fe58ab1 --- /dev/null +++ b/amd64_raw_vex_binary_float_test.go @@ -0,0 +1,346 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86VEXBinaryFloatTestOpcodes = map[byte]string{ + 0x58: "VADD", + 0x59: "VMUL", + 0x5c: "VSUB", + 0x5d: "VMIN", + 0x5e: "VDIV", + 0x5f: "VMAX", +} + +func encodeX86VEXBinaryFloat(opcode, pp byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | int(pp)) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | int(pp)) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func encodeX86EVEXBinaryFloat(opcode, pp, vectorBits byte, broadcastOrRounding, zeroing bool, mask, destination, source1, source2 int) []byte { + p0 := byte((1-(destination>>3)&1)<<7 | (1-(source2>>4)&1)<<6 | (1-(source2>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 1) + p1 := byte((^source1&15)<<3 | 1<<2 | int(pp)) + if pp == 1 || pp == 3 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^source1>>4&1)<<3 | mask) + if broadcastOrRounding { + p2 |= 0x10 + } + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestTranslateRawVEXBinaryFloatWeaviateRegression(t *testing.T) { + const source = `TEXT rawBinary(SB), $0-0 + LONG $0x0e5cf2c5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawBinary": {Name: "rawBinary", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "@llvm.experimental.constrained.fsub.f32(") || !strings.Contains(ir, "@__plan9asm_add_sub_special_f32(") { + t.Fatalf("raw VSUBSS lowering omitted scalar subtraction:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-binary-float.ll", "amd64-raw-vex-binary-float.o", ir) +} + +func TestTranslateRawScalarSqrtKelindarRegression(t *testing.T) { + const source = `TEXT rawScalarSqrt(SB), $0-0 + LONG $0xdc51dac5 + RET +` + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + if !strings.Contains(ir, `asm sideeffect "vsqrtss`) { + t.Fatalf("raw VSQRTSS omitted square root:\n%s", ir) + } +} + +func TestDecodedX86RawScalarSqrtCompleteGoRows(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + {"VEX single register", []byte{0xc5, 0xea, 0x51, 0xd9}, "VSQRTSS X1, X2, X3"}, + {"VEX double register", []byte{0xc5, 0xeb, 0x51, 0xd9}, "VSQRTSD X1, X2, X3"}, + {"VEX single memory", []byte{0xc5, 0xea, 0x51, 0x18}, "VSQRTSS 0(AX), X2, X3"}, + {"VEX double memory", []byte{0xc5, 0xeb, 0x51, 0x58, 0x10}, "VSQRTSD 16(AX), X2, X3"}, + {"EVEX single mask zero", []byte{0x62, 0xa1, 0x56, 0x81, 0x51, 0xf4}, "VSQRTSS.Z X20, X21, K1, X22"}, + {"EVEX double mask zero", []byte{0x62, 0xa1, 0xd7, 0x81, 0x51, 0xf4}, "VSQRTSD.Z X20, X21, K1, X22"}, + {"EVEX single rounding", []byte{0x62, 0xf1, 0x6e, 0x18, 0x51, 0xd9}, "VSQRTSS.RN_SAE X1, X2, X3"}, + {"EVEX double rounding", []byte{0x62, 0xf1, 0xef, 0x38, 0x51, 0xd9}, "VSQRTSD.RD_SAE X1, X2, X3"}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, recognized, err := decodedX86VEXBinaryFloatInstruction(test.code, 64) + if err != nil || !recognized || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode %x: raw=%q length=%d recognized=%v err=%v; want %q", test.code, instruction.Raw, length, recognized, err, test.want) + } + }) + } +} + +func TestTranslateRawScalarSqrtLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + encodings := [][]byte{ + {0xc5, 0xea, 0x51, 0xd9}, + {0xc5, 0xeb, 0x51, 0x58, 0x10}, + {0x62, 0xf1, 0x6e, 0x18, 0x51, 0xd9}, + {0x62, 0xf1, 0xef, 0x38, 0x51, 0xd9}, + } + if target.goarch == "amd64" { + encodings = append(encodings, + []byte{0x62, 0xa1, 0x56, 0x81, 0x51, 0xf4}, + []byte{0x62, 0xa1, 0xd7, 0x81, 0x51, 0xf4}, + ) + } + var source strings.Builder + source.WriteString("TEXT rawScalarSqrtFamily(SB), $0-0\n") + for _, code := range encodings { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-scalar-sqrt.ll", "raw-scalar-sqrt.o", ir) + }) + } +} + +func TestTranslateRawEVEXBinaryFloatWeaviateRegression(t *testing.T) { + const source = `TEXT rawBinaryEVEX(SB), $0-0 + LONG $0x48347162 + WORD $0x0e5c + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawBinaryEVEX": {Name: "rawBinaryEVEX", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "@llvm.experimental.constrained.fsub.v16f32(") || !strings.Contains(ir, "@__plan9asm_add_sub_special_f32(") { + t.Fatalf("raw EVEX VSUBPS lowering omitted packed subtraction:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-evex-binary-float.ll", "amd64-raw-evex-binary-float.o", ir) +} + +func TestDecodedX86VEXBinaryFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + suffixes := map[byte]string{0: "PS", 1: "PD", 2: "SS", 3: "SD"} + for opcode, stem := range x86VEXBinaryFloatTestOpcodes { + for pp, suffix := range suffixes { + scalar := pp >= 2 + for _, vex3 := range []bool{false, true} { + widths := []bool{false, true} + if scalar { + widths = []bool{false} + } + for _, width256 := range widths { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXBinaryFloat(opcode, pp, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := Op(stem + suffix) + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + } + if count != 368640 { + t.Fatalf("covered %d VEX binary floating register encodings, want 368640", count) + } +} + +func TestDecodedX86EVEXBinaryFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + suffixes := map[byte]string{0: "PS", 1: "PD", 2: "SS", 3: "SD"} + for opcode, stem := range x86VEXBinaryFloatTestOpcodes { + for pp, suffix := range suffixes { + vectorLengths := []byte{0, 1, 2} + for _, vectorBits := range vectorLengths { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + if pp >= 2 { + prefix = "X" + } + for destination := 0; destination < 32; destination++ { + for source1 := 0; source1 < 32; source1++ { + for source2 := 0; source2 < 32; source2++ { + code := encodeX86EVEXBinaryFloat(opcode, pp, vectorBits, false, false, 1, destination, source1, source2) + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := Op(stem + suffix) + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), "K1", fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 4 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + if count != 2359296 { + t.Fatalf("covered %d EVEX binary floating register encodings, want 2359296", count) + } +} + +func TestDecodedX86EVEXBinaryFloatModifiersAndMemory(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + args []string + }{ + {name: "packed zeroing", code: encodeX86EVEXBinaryFloat(0x58, 0, 1, false, true, 3, 29, 28, 21), op: "VADDPS.Z", args: []string{"Y21", "Y28", "K3", "Y29"}}, + {name: "packed broadcast zeroing", code: []byte{0x62, 0xf1, 0xed, 0xd9, 0x58, 0x08}, op: "VADDPD.BCST.Z", args: []string{"0(AX)", "Z2", "K1", "Z1"}}, + {name: "packed round nearest", code: []byte{0x62, 0xf1, 0xed, 0x19, 0x58, 0xcb}, op: "VADDPD.RN_SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed round down", code: []byte{0x62, 0xf1, 0xed, 0x39, 0x58, 0xcb}, op: "VADDPD.RD_SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed round up", code: []byte{0x62, 0xf1, 0xed, 0x59, 0x58, 0xcb}, op: "VADDPD.RU_SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed round zero zeroing", code: []byte{0x62, 0xf1, 0xed, 0xf9, 0x58, 0xcb}, op: "VADDPD.RZ_SAE.Z", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed sae", code: []byte{0x62, 0xf1, 0xed, 0x59, 0x5f, 0xcb}, op: "VMAXPD.SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "scalar round up", code: encodeX86EVEXBinaryFloat(0x5c, 2, 2, true, false, 2, 5, 6, 7), op: "VSUBSS.RU_SAE", args: []string{"X7", "X6", "K2", "X5"}}, + {name: "scalar sae zeroing", code: encodeX86EVEXBinaryFloat(0x5d, 3, 0, true, true, 4, 8, 9, 10), op: "VMINSD.SAE.Z", args: []string{"X10", "X9", "K4", "X8"}}, + {name: "compressed memory", code: []byte{0x62, 0x71, 0x34, 0x48, 0x5c, 0x4e, 0x02}, op: "VSUBPS", args: []string{"128(SI)", "Z9", "Z9"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != len(test.args) { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + for index := range test.args { + if got.Args[index].String() != test.args[index] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } + }) + } + + scalarBroadcast := encodeX86EVEXBinaryFloat(0x58, 2, 0, true, false, 0, 0, 0, 0) + scalarBroadcast[len(scalarBroadcast)-1] = 0 + wrongW := encodeX86EVEXBinaryFloat(0x58, 0, 0, false, false, 0, 0, 0, 0) + wrongW[2] ^= 0x80 + invalid := [][]byte{ + encodeX86EVEXBinaryFloat(0x58, 0, 0, false, true, 0, 0, 0, 0), + encodeX86EVEXBinaryFloat(0x58, 0, 3, false, false, 0, 0, 0, 0), + scalarBroadcast, + wrongW, + } + for _, code := range invalid { + if _, _, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid EVEX encoding %x returned ok=%v err=%v", code, ok, err) + } + } + + mode32 := encodeX86EVEXBinaryFloat(0x58, 0, 0, false, false, 1, 0, 0, 0) + if _, _, ok, err := decodedX86VEXBinaryFloatInstruction(mode32, 32); !ok || err == nil { + t.Fatalf("masked EVEX form in 32-bit mode returned ok=%v err=%v", ok, err) + } +} + +func TestDecodedX86VEXBinaryFloatMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x01, 0xe5, 0x58, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VADDPD" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXBinaryFloat(0x58, 0, true, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc5}, + {0xc4, valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXBinaryFloat(0x5a, 0, true, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXBinaryFloatInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + scalarY := encodeX86VEXBinaryFloat(0x58, 2, true, true, false, 0, 0, 0) + if _, _, matched, decodeErr := decodedX86VEXBinaryFloatInstruction(scalarY, 64); !matched || decodeErr == nil { + t.Fatalf("scalar VEX.256 encoding %x returned ok=%v err=%v", scalarY, matched, decodeErr) + } + extended := encodeX86VEXBinaryFloat(0x58, 0, true, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXBinaryFloatInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_fma3_test.go b/amd64_raw_vex_fma3_test.go new file mode 100644 index 000000000..e99d8b19f --- /dev/null +++ b/amd64_raw_vex_fma3_test.go @@ -0,0 +1,304 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXFMA3(opcode byte, width64, width256 bool, destination, source1, source2 int) []byte { + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 2) + vex1 := byte(((^source1)&15)<<3 | 1) + if width64 { + vex1 |= 0x80 + } + if width256 { + vex1 |= 0x04 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func encodeX86EVEXFMA3(opcode byte, width64 bool, vectorBits byte, broadcastOrRounding, zeroing bool, mask, destination, source1, source2 int) []byte { + p0 := byte((1-(destination>>3)&1)<<7 | (1-(source2>>4)&1)<<6 | (1-(source2>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 2) + p1 := byte((^source1&15)<<3 | 1<<2 | 1) + if width64 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^source1>>4&1)<<3 | mask) + if broadcastOrRounding { + p2 |= 0x10 + } + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestTranslateRawVEXFMA3WeaviateRegression(t *testing.T) { + const source = `TEXT rawFMA(SB), $0-0 + LONG $0x985de2c4; BYTE $0x1e + RET +` + // The complete original lacks its SI address and Y3/Y4 producers. Keep + // its actual Go object and all three public Context results. This private + // emitter object is not initialized public translation or native execution; + // TestX86RawFMA3OriginalWeaviateInputsAndInitializedCounterparts owns that + // independent, genuinely initialized public object matrix. + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + x86RawFMA3RequireOriginalNativeFormat(t, ir, "vfmadd132ps", 32, 32, false) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-fma3.ll", "amd64-raw-vex-fma3.o", ir) +} + +func TestTranslateRawEVEXFMA3WeaviateRegression(t *testing.T) { + const source = `TEXT rawFMAEVEX(SB), $0-0 + LONG $0x4835d262; WORD $0xc9b8 + RET +` + // Unchanged original bytes have no Z1/Z9 producers. The public Context, + // private format object and initialized public counterpart are distinct. + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + x86RawFMA3RequireOriginalNativeFormat(t, ir, "vfmadd231ps", 64, 0, true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-evex-fma3.ll", "amd64-raw-evex-fma3.o", ir) +} + +func x86RawFMA3RequireOriginalNativeFormat(t *testing.T, ir, mnemonic string, width, memoryBytes int, evex bool) { + t.Helper() + var native string + for _, line := range strings.Split(ir, "\n") { + if strings.Contains(line, "asm sideeffect") && strings.Contains(line, mnemonic+" ") { + if native != "" { + t.Fatal("original format duplicated its native FMA opcode", mnemonic) + } + native = line + } + if strings.Contains(line, "call ") && strings.Contains(line, "@llvm.fma.") { + t.Fatal("original native FMA format was replaced by pure arithmetic", mnemonic, line) + } + } + constraint := "=x,0," + if evex { + constraint = "=v,0," + } + if native == "" || !strings.Contains(native, fmt.Sprintf("<%d x float> asm sideeffect", width/4)) || + !strings.Contains(native, "\""+constraint) || !strings.Contains(native, "~{fpsr}") || + !strings.Contains(native, "~{memory}") || strings.Contains(native, "~{flags}") { + t.Fatal("original native FMA lost physical carrier, tied old destination, FP/access effects or FLAGS preservation", mnemonic, native) + } + if memoryBytes != 0 { + if !strings.Contains(native, fmt.Sprintf("elementtype([%d x i8])", memoryBytes)) || !strings.Contains(native, ",*m") { + t.Fatal("original native FMA lost its exact memory tuple", mnemonic, native) + } + } else if strings.Contains(native, "elementtype(") || strings.Contains(native, ",*m") { + t.Fatal("original register FMA invented a memory operand", mnemonic, native) + } +} + +func TestDecodedX86VEXFMA3CompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, spec := range decodedX86VEXFMA3Ops { + for _, width64 := range []bool{false, true} { + widths := []bool{false, true} + if spec.scalar { + widths = []bool{false} + } + for _, width256 := range widths { + prefix := "X" + if width256 { + prefix = "Y" + } + var registers [16]Reg + for index := range registers { + registers[index] = Reg(fmt.Sprintf("%s%d", prefix, index)) + } + wantOp := spec.op(width64) + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < 16; source2++ { + code := encodeX86VEXFMA3(opcode, width64, width256, destination, source1, source2) + got, length, ok, err := decodedX86VEXFMA3Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if got.Op != wantOp || len(got.Args) != 3 || + got.Args[0].Kind != OpReg || got.Args[0].Reg != registers[source2] || + got.Args[1].Kind != OpReg || got.Args[1].Reg != registers[source1] || + got.Args[2].Kind != OpReg || got.Args[2].Reg != registers[destination] { + t.Fatalf("decode %x = %+v, want %s %s,%s,%s", code, got, wantOp, registers[source2], registers[source1], registers[destination]) + } + count++ + } + } + } + } + } + } + if count != 393216 { + t.Fatalf("covered %d VEX FMA3 register encodings, want 393216", count) + } + t.Logf("exhaustively checked %d VEX FMA3 register encodings", count) +} + +func TestDecodedX86EVEXFMA3CompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, spec := range decodedX86VEXFMA3Ops { + for _, width64 := range []bool{false, true} { + vectorLengths := []byte{0, 1, 2} + for _, vectorBits := range vectorLengths { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + if spec.scalar { + prefix = "X" + } + var registers [32]Reg + for index := range registers { + registers[index] = Reg(fmt.Sprintf("%s%d", prefix, index)) + } + wantOp := spec.op(width64) + for destination := 0; destination < 32; destination++ { + for source1 := 0; source1 < 32; source1++ { + for source2 := 0; source2 < 32; source2++ { + code := encodeX86EVEXFMA3(opcode, width64, vectorBits, false, false, 1, destination, source1, source2) + got, length, ok, err := decodedX86VEXFMA3Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if got.Op != wantOp || len(got.Args) != 4 || + got.Args[0].Kind != OpReg || got.Args[0].Reg != registers[source2] || + got.Args[1].Kind != OpReg || got.Args[1].Reg != registers[source1] || + got.Args[2].Kind != OpReg || got.Args[2].Reg != "K1" || + got.Args[3].Kind != OpReg || got.Args[3].Reg != registers[destination] { + t.Fatalf("decode %x = %+v, want %s %s,%s,K1,%s", code, got, wantOp, registers[source2], registers[source1], registers[destination]) + } + count++ + } + } + } + } + } + } + if count != 5898240 { + t.Fatalf("covered %d EVEX FMA3 register encodings, want 5898240", count) + } + t.Logf("exhaustively checked %d EVEX FMA3 register encodings", count) +} + +func TestDecodedX86EVEXFMA3ModifiersAndMemory(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + args []string + }{ + {name: "packed zeroing", code: encodeX86EVEXFMA3(0xb8, false, 1, false, true, 3, 29, 28, 21), op: "VFMADD231PS.Z", args: []string{"Y21", "Y28", "K3", "Y29"}}, + {name: "packed broadcast zeroing", code: []byte{0x62, 0xf2, 0xed, 0xd9, 0x98, 0x08}, op: "VFMADD132PD.BCST.Z", args: []string{"0(AX)", "Z2", "K1", "Z1"}}, + {name: "packed round nearest", code: encodeX86EVEXFMA3(0xa8, true, 0, true, false, 1, 1, 2, 3), op: "VFMADD213PD.RN_SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed round down", code: encodeX86EVEXFMA3(0xba, false, 1, true, false, 2, 4, 5, 6), op: "VFMSUB231PS.RD_SAE", args: []string{"Z6", "Z5", "K2", "Z4"}}, + {name: "packed round up", code: encodeX86EVEXFMA3(0x9c, true, 2, true, false, 3, 7, 8, 9), op: "VFNMADD132PD.RU_SAE", args: []string{"Z9", "Z8", "K3", "Z7"}}, + {name: "packed round zero zeroing", code: encodeX86EVEXFMA3(0xae, false, 3, true, true, 4, 10, 11, 12), op: "VFNMSUB213PS.RZ_SAE.Z", args: []string{"Z12", "Z11", "K4", "Z10"}}, + {name: "scalar round up", code: encodeX86EVEXFMA3(0xb9, false, 2, true, false, 5, 13, 14, 15), op: "VFMADD231SS.RU_SAE", args: []string{"X15", "X14", "K5", "X13"}}, + {name: "compressed memory", code: []byte{0x62, 0xd2, 0x35, 0x48, 0xb8, 0x49, 0x02}, op: "VFMADD231PS", args: []string{"128(R9)", "Z9", "Z1"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86VEXFMA3Instruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != len(test.args) { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + for index := range test.args { + if got.Args[index].String() != test.args[index] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } + }) + } + + scalarBroadcast := encodeX86EVEXFMA3(0x99, false, 0, true, false, 0, 0, 0, 0) + scalarBroadcast[len(scalarBroadcast)-1] = 0 + wrongPP := encodeX86EVEXFMA3(0x98, false, 0, false, false, 0, 0, 0, 0) + wrongPP[2] &^= 1 + invalid := [][]byte{ + encodeX86EVEXFMA3(0x98, false, 0, false, true, 0, 0, 0, 0), + encodeX86EVEXFMA3(0x98, false, 3, false, false, 0, 0, 0, 0), + scalarBroadcast, + wrongPP, + } + for _, code := range invalid { + if _, _, ok, err := decodedX86VEXFMA3Instruction(code, 64); !ok || err == nil { + t.Fatalf("invalid EVEX encoding %x returned ok=%v err=%v", code, ok, err) + } + } + + mode32 := encodeX86EVEXFMA3(0x98, false, 0, false, false, 1, 0, 0, 0) + got, length, ok, err := decodedX86VEXFMA3Instruction(mode32, 32) + if !ok || err != nil || length != len(mode32) || got.Op != "VFMADD132PS" || len(got.Args) != 4 || + got.x86FMA3Encoding != x86RawFMA3EVEX || got.x86Encoded { + t.Fatalf("valid physical mode32 masked EVEX lost its complete decode/generation: %+v length=%d ok=%v err=%v", got, length, ok, err) + } + for index, want := range []string{"X0", "X0", "K1", "X0"} { + if got.Args[index].Kind != OpReg || got.Args[index].String() != want { + t.Fatal("physical mode32 masked EVEX changed an original register/mask operand", index, got.Args) + } + } + // Physical low-bank EVEX validity does not authorize unaddressable mode32 + // banks. Exercise each source/destination extension independently. + for _, registers := range [][3]int{{8, 0, 0}, {0, 8, 0}, {0, 0, 8}, {16, 0, 0}, {0, 16, 0}, {0, 0, 16}} { + extended := encodeX86EVEXFMA3(0x98, false, 0, false, false, 1, + registers[0], registers[1], registers[2]) + if _, _, matched, decodeErr := decodedX86VEXFMA3Instruction(extended, 32); !matched || decodeErr == nil || + !strings.Contains(decodeErr.Error(), "extended register in 32-bit mode") { + t.Fatalf("physical mode32 extended-bank form %v returned ok=%v err=%v", registers, matched, decodeErr) + } + } + // Go's named four-operand mode32 grammar remains a distinct restriction. + // Require an actual normal assembler rejection as well as the typed guard; + // raw physical acceptance must not silently widen that named grammar. + const named = "TEXT named386(SB),4,$0-0\nVFMADD132PS X0,X0,K1,X0\nRET\n" + goAssemblerOracleOutput(t, "386", named, "", false) + if _, known, err := amd64FMA3Properties(x86ScalarLaneInstruction(t, "VFMADD132PS X0,X0,K1,X0"), "386"); !known || err == nil { + t.Fatal("raw physical acceptance widened Go's named mode32 mask grammar", known, err) + } +} + +func TestDecodedX86VEXFMA3MemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x02, 0xe5, 0x98, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXFMA3Instruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VFMADD132PD" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXFMA3(0x98, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc4}, + {valid[0], valid[1] ^ 3, valid[2], valid[3], valid[4]}, + encodeX86VEXFMA3(0x95, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXFMA3Instruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + invalidPP := []byte{valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4]} + if _, _, matched, decodeErr := decodedX86VEXFMA3Instruction(invalidPP, 64); !matched || decodeErr == nil { + t.Fatalf("invalid pp encoding %x returned ok=%v err=%v", invalidPP, matched, decodeErr) + } + scalarY := encodeX86VEXFMA3(0x99, false, true, 0, 0, 0) + if _, _, matched, decodeErr := decodedX86VEXFMA3Instruction(scalarY, 64); !matched || decodeErr == nil { + t.Fatalf("scalar VEX.256 encoding %x returned ok=%v err=%v", scalarY, matched, decodeErr) + } + extended := encodeX86VEXFMA3(0x98, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXFMA3Instruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_horizontal_float_test.go b/amd64_raw_vex_horizontal_float_test.go new file mode 100644 index 000000000..99ea1daa4 --- /dev/null +++ b/amd64_raw_vex_horizontal_float_test.go @@ -0,0 +1,171 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func encodeX86VEXHorizontalFloat(opcode, pp byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | int(pp)) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | int(pp)) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestTranslateRawVEXHorizontalFloatWeaviateRegression(t *testing.T) { + const source = `TEXT rawHorizontal(SB), $0-0 + LONG $0xc07cffc5 + RET +` + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", "x86_64-unknown-linux-gnu") + if !strings.Contains(ir, "vhaddps ") || !strings.Contains(ir, "~{fpsr}") { + t.Fatalf("raw VHADDPS lowering omitted horizontal addition:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-horizontal-float.ll", "amd64-raw-vex-horizontal-float.o", ir) +} + +func TestDecodedX86VEXHorizontalFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, stem := range map[byte]string{0x7c: "VHADD", 0x7d: "VHSUB", 0xd0: "VADDSUB"} { + for pp, suffix := range map[byte]string{1: "PD", 3: "PS"} { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXHorizontalFloat(opcode, pp, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXHorizontalFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := Op(stem + suffix) + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + } + if count != 122880 { + t.Fatalf("covered %d VEX horizontal and alternating floating register encodings, want 122880", count) + } +} + +func TestTranslateRawVEXAddSubComplexAVX2Regression(t *testing.T) { + const source = "TEXT rawAddSub(SB),$0-0\n\tLONG $0xc1d0f9c5\n\tRET\n" + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", triple) + if !strings.Contains(ir, "fsub double") || !strings.Contains(ir, "fadd double") { + t.Fatalf("VADDSUBPD omitted alternating arithmetic:\n%s", ir) + } + compileLLVMToObject(t, llc, triple, "raw-vaddsub.ll", "raw-vaddsub.o", ir) + }) + } +} + +func TestX86RawAddSubInitializedPublicInputs(t *testing.T) { + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + // The original VADDSUBPD bytes are unchanged. Independent typed + // inputs provide both vectors, and the complete result is observed. + loadSecond := "MOVUPD b+16(FP),X1\n" + source := "TEXT Value(SB),4,$0-48\nMOVUPD a+0(FP),X0\n" + loadSecond + + "LONG $0xc1d0f9c5\nMOVUPD X0,ret+32(FP)\nRET\n" + const declaration = "func Value(a,b [16]byte) [16]byte" + requireX86GoAssemblerResult(t, target.arch, source, true) + named := strings.Replace(source, "LONG $0xc1d0f9c5", "VADDSUBPD X1,X0,X0", 1) + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, named, declaration, target.arch, target.triple, api); err != nil || ir == "" { + t.Fatal("initialized named counterpart must precede raw comparison", api, err) + } + } + x86RawNumericalFamilyObjects(t, source, declaration, target.arch, target.triple) + for _, api := range []string{"text", "module", "Go"} { + missing := strings.Replace(source, loadSecond, "", 1) + ir, err := x86ScalarFPBitViewAPI(t, missing, declaration, target.arch, target.triple, api) + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("raw arithmetic borrowed its missing second vector", api, err) + } + } + }) + } +} + +func TestDecodedX86VEXHorizontalFloatMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x01, 0xe5, 0x7d, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXHorizontalFloatInstruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VHSUBPD" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + code[4] = 0xd0 + got, length, ok, err = decodedX86VEXHorizontalFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != "VADDSUBPD" || + got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || + got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("VADDSUBPD memory source %x decoded as %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXHorizontalFloat(0x7c, 1, true, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc5}, + {0xc4, valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXHorizontalFloat(0x7e, 1, true, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXHorizontalFloatInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + invalidPP := encodeX86VEXHorizontalFloat(0x7c, 0, true, false, false, 0, 0, 0) + if _, _, matched, decodeErr := decodedX86VEXHorizontalFloatInstruction(invalidPP, 64); !matched || decodeErr == nil { + t.Fatalf("invalid pp encoding %x returned ok=%v err=%v", invalidPP, matched, decodeErr) + } + extended := encodeX86VEXHorizontalFloat(0x7c, 1, true, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXHorizontalFloatInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_integration_test.go b/amd64_raw_vex_integration_test.go new file mode 100644 index 000000000..9ab0d6fb8 --- /dev/null +++ b/amd64_raw_vex_integration_test.go @@ -0,0 +1,35 @@ +package plan9asm + +import "testing" + +func TestTranslateX86RawVEXVNNIAndVariableDwordPermuteAllTargets(t *testing.T) { + const source = `TEXT rawvexfamilies(SB),$0-0 + // {vex} VPDPBUSD X2, X1, X0. + BYTE $0xc4; BYTE $0xe2; BYTE $0x71; BYTE $0x50; BYTE $0xc2 + // {vex} VPERMD Y5, Y3, Y4. + BYTE $0xc4; BYTE $0xe2; BYTE $0x65; BYTE $0x36; BYTE $0xe5 + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + // Keep both original raw opcodes and their whole Go object, but + // do not mistake this input-free format unit for a numerical ABI. + ll := x86UnboundVoidRawScalarFormatObject(t, source, target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-vex-families-"+target.name+".ll", "raw-vex-families-"+target.name+".o", ll) + }) + } +} diff --git a/amd64_raw_vex_packed_float_logical_test.go b/amd64_raw_vex_packed_float_logical_test.go new file mode 100644 index 000000000..9f2665c69 --- /dev/null +++ b/amd64_raw_vex_packed_float_logical_test.go @@ -0,0 +1,170 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86VEXPackedFloatLogicalTestOpcodes = map[byte]string{ + 0x54: "VAND", + 0x55: "VANDN", + 0x56: "VOR", + 0x57: "VXOR", +} + +func encodeX86VEXPackedFloatLogical(opcode, pp byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | int(pp)) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | int(pp)) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestTranslateRawVEXPackedFloatLogicalWeaviateRegression(t *testing.T) { + const source = `TEXT rawLogical(SB), $0-0 + WORD $0x8944; BYTE $0xc8 + WORD $0xe083; BYTE $0x03 + LONG $0xc057f8c5 + WORD $0xc931 + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + // Keep every original byte, including the read of unknown R9. + // Its format object is not an initialized public source program. + ir := x86UnboundVoidRawScalarFormatObject(t, source, "amd64", triple) + if !strings.Contains(ir, "xor <4 x i32>") { + t.Fatalf("raw VXORPS lowering omitted vector xor:\n%s", ir) + } + compileLLVMToObject(t, llc, triple, "amd64-raw-vex-logical.ll", "amd64-raw-vex-logical.o", ir) + + initialized := strings.Replace(source, "TEXT rawLogical(SB), $0-0\n", + "TEXT rawLogical(SB), $0-4\nMOVL value+0(FP), R9\n", 1) + x86LEAActualTargetGoObject(t, "amd64", triple, initialized) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, initialized, "func rawLogical(value uint32)", "amd64", triple, api) + if err != nil { + t.Fatal("initialized original raw logical sequence", err) + } + for _, text := range []string{ir, x86DwordProductIROptimize(t, ir)} { + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, triple, text, level) + } + } + }) + } + }) + } +} + +func TestDecodedX86VEXPackedFloatLogicalCompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedFloatLogical(opcode, pp, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXPackedFloatLogicalInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := Op(stem + suffix) + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + } + if count != 163840 { + t.Fatalf("covered %d VEX packed floating logical register encodings, want 163840", count) + } +} + +func TestDecodedX86VEXPackedFloatLogicalMemoryForms(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + source MemRef + second Reg + dest Reg + }{ + {name: "base", code: []byte{0xc4, 0xe1, 0x78, 0x57, 0x00}, op: "VXORPS", source: MemRef{Base: AX}, second: "X0", dest: "X0"}, + {name: "extended SIB displacement and FS", code: []byte{0x64, 0xc4, 0x01, 0x65, 0x54, 0x64, 0x88, 0x20}, op: "VANDPD", source: MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32}, second: "Y3", dest: "Y12"}, + {name: "base displacement and GS", code: []byte{0x65, 0xc5, 0xec, 0x56, 0x4d, 0x7f}, op: "VORPS", source: MemRef{Segment: GS, Base: BP, Off: 0x7f}, second: "Y2", dest: "Y1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86VEXPackedFloatLogicalInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != test.source || got.Args[1].Kind != OpReg || got.Args[1].Reg != test.second || got.Args[2].Kind != OpReg || got.Args[2].Reg != test.dest { + t.Fatalf("decode %x = %+v, want %s %+v, %s, %s", test.code, got, test.op, test.source, test.second, test.dest) + } + }) + } +} + +func TestDecodedX86VEXPackedFloatLogicalRejectsReservedForms(t *testing.T) { + valid := encodeX86VEXPackedFloatLogical(0x57, 0, true, false, false, 0, 0, 0) + for _, code := range [][]byte{ + {0xc5}, + {0xc4, valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXPackedFloatLogical(0x58, 0, true, false, false, 0, 0, 0), + } { + if got, _, ok, err := decodedX86VEXPackedFloatLogicalInstruction(code, 64); ok || err != nil { + t.Fatalf("reserved encoding %x decoded as %+v, ok=%v, err=%v", code, got, ok, err) + } + } + reservedPP := encodeX86VEXPackedFloatLogical(0x57, 2, true, false, false, 0, 0, 0) + if _, _, ok, err := decodedX86VEXPackedFloatLogicalInstruction(reservedPP, 64); !ok || err == nil { + t.Fatalf("reserved pp encoding %x returned ok=%v err=%v", reservedPP, ok, err) + } + ripRelative := []byte{0xc5, 0xec, 0x56, 0x0d, 0x78, 0x56, 0x34, 0x12} + if _, _, ok, err := decodedX86VEXPackedFloatLogicalInstruction(ripRelative, 64); !ok || err == nil { + t.Fatalf("source-layout-unsafe RIP-relative encoding %x returned ok=%v err=%v", ripRelative, ok, err) + } + extended := encodeX86VEXPackedFloatLogical(0x57, 0, true, false, false, 8, 8, 8) + if _, _, ok, err := decodedX86VEXPackedFloatLogicalInstruction(extended, 32); !ok || err == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", ok, err) + } +} diff --git a/amd64_raw_vex_packed_integer_logical_test.go b/amd64_raw_vex_packed_integer_logical_test.go new file mode 100644 index 000000000..576222fc5 --- /dev/null +++ b/amd64_raw_vex_packed_integer_logical_test.go @@ -0,0 +1,125 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86VEXPackedIntegerLogicalTestOpcodes = map[byte]Op{ + 0xdb: "VPAND", + 0xdf: "VPANDN", + 0xeb: "VPOR", + 0xef: "VPXOR", +} + +func encodeX86VEXPackedIntegerLogical(opcode byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | 1) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestTranslateRawVEXPackedIntegerLogicalWeaviateRegression(t *testing.T) { + const source = `TEXT rawLogical(SB), $0-0 + LONG $0xc0eff9c5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawLogical": {Name: "rawLogical", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "xor <16 x i8>") { + t.Fatalf("raw VPXOR lowering omitted vector xor:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-integer-logical.ll", "amd64-raw-vex-integer-logical.o", ir) +} + +func TestDecodedX86VEXPackedIntegerLogicalCompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, wantOp := range x86VEXPackedIntegerLogicalTestOpcodes { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedIntegerLogical(opcode, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXPackedIntegerLogicalInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + if count != 81920 { + t.Fatalf("covered %d VEX packed integer logical register encodings, want 81920", count) + } +} + +func TestDecodedX86VEXPackedIntegerLogicalMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x01, 0xe5, 0xdb, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXPackedIntegerLogicalInstruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VPAND" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXPackedIntegerLogical(0xef, true, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc5}, + {0xc4, valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXPackedIntegerLogical(0xee, true, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXPackedIntegerLogicalInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + invalidPP := []byte{valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4]} + if _, _, matched, decodeErr := decodedX86VEXPackedIntegerLogicalInstruction(invalidPP, 64); !matched || decodeErr == nil { + t.Fatalf("invalid pp encoding %x returned ok=%v err=%v", invalidPP, matched, decodeErr) + } + extended := encodeX86VEXPackedIntegerLogical(0xef, true, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXPackedIntegerLogicalInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_packed_integer_move_test.go b/amd64_raw_vex_packed_integer_move_test.go new file mode 100644 index 000000000..17ef6b597 --- /dev/null +++ b/amd64_raw_vex_packed_integer_move_test.go @@ -0,0 +1,62 @@ +package plan9asm + +import ( + "reflect" + "testing" +) + +func TestX86RawVEXPackedIntegerMoveCorpusEncoding(t *testing.T) { + const source = "TEXT corpusmove(SB),4,$0-0\n\tVMOVDQU -32(SI)(CX*4), Y14\n\tRET\n" + code := assembleX87ControlBytes(t, "amd64", source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } +} + +func TestX86RawVEXPackedIntegerMoveRejectsReservedForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {"missing ModRM", []byte{0xc5, 0xfe, 0x6f}, 64}, + {"missing SIB", []byte{0xc5, 0xfe, 0x6f, 0x04}, 64}, + {"missing displacement", []byte{0xc5, 0xfe, 0x6f, 0x44, 0x24}, 64}, + {"reserved vvvv", []byte{0xc5, 0xf6, 0x6f, 0xc0}, 64}, + {"extended register in 386", []byte{0xc5, 0x7e, 0x6f, 0xc0}, 32}, + {"address override", []byte{0x67, 0xc5, 0xfe, 0x6f, 0xc0}, 64}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, matched, err := decodedX86PackedMoveInstruction(test.code, test.mode); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", test.code, matched, err) + } + }) + } +} + +func TestX86RawVEXPackedIntegerMoveIgnoresWBit(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe1, 0x7e, 0x6f, 0xc0}, + {0xc4, 0xe1, 0xfe, 0x6f, 0xc0}, + } { + got, size, matched, err := decodedX86PackedMoveInstruction(code, 64) + if err != nil || !matched || size != len(code) || got.Op != "VMOVDQU" || + !reflect.DeepEqual(got.Args, []Operand{{Kind: OpReg, Reg: "Y0"}, {Kind: OpReg, Reg: "Y0"}}) { + t.Fatalf("decode %x=%+v size=%d matched=%v err=%v", code, got, size, matched, err) + } + } +} diff --git a/amd64_raw_vex_packed_madd_literal_test.go b/amd64_raw_vex_packed_madd_literal_test.go new file mode 100644 index 000000000..84df43a1a --- /dev/null +++ b/amd64_raw_vex_packed_madd_literal_test.go @@ -0,0 +1,137 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawVEXPackedMADDConstant(opcode byte, opcodeMap int, vex3, width256, w bool) []byte { + code := encodeX86VEXPackedMADD(opcode, opcodeMap, vex3, width256, w, 0, 0, 0) + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + width := 16 + if width256 { + width = 32 + } + for i := 0; i < width; i++ { + code = append(code, byte(i*29+13)) + } + return code +} + +func TestDecodeX86RawVEXPackedMADDLiteralCompleteFamily(t *testing.T) { + for _, form := range []struct { + opcode byte + mapID int + op Op + vex2 bool + }{ + {0xf5, 1, "VPMADDWD", true}, + {0x04, 2, "VPMADDUBSW", false}, + } { + for _, vex3 := range []bool{false, true} { + if !vex3 && !form.vex2 { + continue + } + for _, width256 := range []bool{false, true} { + for _, w := range []bool{false, true} { + if !vex3 && w { + continue + } + name := fmt.Sprintf("%s/vex3=%t/y=%t/w=%t", form.op, vex3, width256, w) + t.Run(name, func(t *testing.T) { + code := x86RawVEXPackedMADDConstant(form.opcode, form.mapID, vex3, width256, w) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != form.op || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, form.op) + } + }) + } + } + } + } +} + +func TestDecodeX86RawVEXPackedMADDLiteralRejectsUnsafeSources(t *testing.T) { + valid := x86RawVEXPackedMADDConstant(0xf5, 1, false, false, false) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[4] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[4] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVEXPackedMADDLiteralObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, form := range []struct { + opcode byte + mapID int + }{ + {0xf5, 1}, {0x04, 2}, + } { + code := x86RawVEXPackedMADDConstant(form.opcode, form.mapID, true, true, false) + name := fmt.Sprintf("packedMADDConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + pools = append(pools, code[len(code)-32:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "packed-madd-literal.ll", "packed-madd-literal.o", ir) + }) + } +} diff --git a/amd64_raw_vex_packed_madd_test.go b/amd64_raw_vex_packed_madd_test.go new file mode 100644 index 000000000..052c64c0e --- /dev/null +++ b/amd64_raw_vex_packed_madd_test.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXPackedMADD(opcode byte, opcodeMap int, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | 1) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | opcodeMap) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestTranslateRawVEXPackedMADDWeaviateRegression(t *testing.T) { + const source = `TEXT rawMADD(SB), $0-0 + LONG $0xd4f5edc5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawMADD": {Name: "rawMADD", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "mul <16 x i32>") { + t.Fatalf("raw VPMADDWD lowering omitted packed multiplication:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-packed-madd.ll", "amd64-raw-vex-packed-madd.o", ir) +} + +func TestDecodedX86VEXPackedMADDCompleteRegisterFamily(t *testing.T) { + tests := []struct { + opcode byte + opcodeMap int + op Op + vex2 bool + }{ + {opcode: 0xf5, opcodeMap: 1, op: "VPMADDWD", vex2: true}, + {opcode: 0x04, opcodeMap: 2, op: "VPMADDUBSW"}, + } + count := 0 + for _, test := range tests { + for _, vex3 := range []bool{false, true} { + if !vex3 && !test.vex2 { + continue + } + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedMADD(test.opcode, test.opcodeMap, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXPackedMADDInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != test.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, test.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + if count != 36864 { + t.Fatalf("covered %d VEX packed multiply-add register encodings, want 36864", count) + } +} + +func TestDecodedX86VEXPackedMADDMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x02, 0xe5, 0x04, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXPackedMADDInstruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VPMADDUBSW" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXPackedMADD(0xf5, 1, true, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc5}, + encodeX86VEXPackedMADD(0xf4, 1, true, false, false, 0, 0, 0), + encodeX86VEXPackedMADD(0x04, 1, true, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXPackedMADDInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + invalidPP := []byte{valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4]} + if _, _, matched, decodeErr := decodedX86VEXPackedMADDInstruction(invalidPP, 64); !matched || decodeErr == nil { + t.Fatalf("invalid pp encoding %x returned ok=%v err=%v", invalidPP, matched, decodeErr) + } + extended := encodeX86VEXPackedMADD(0xf5, 1, true, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXPackedMADDInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_packed_shuffle_test.go b/amd64_raw_vex_packed_shuffle_test.go new file mode 100644 index 000000000..eaef494b4 --- /dev/null +++ b/amd64_raw_vex_packed_shuffle_test.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXPackedShuffle(pp byte, vex3, width256, widthIgnored bool, destination, source int, immediate byte) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | 15<<3 | int(l)<<2 | int(pp)) + return []byte{0xc5, vex1, 0x70, modRM, immediate} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source/8)<<5 | 1) + vex1 := byte(15<<3 | int(l)<<2 | int(pp)) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, 0x70, modRM, immediate} +} + +func TestTranslateRawVEXPackedShuffleWeaviateRegression(t *testing.T) { + const source = `TEXT rawShuffle(SB), $0-0 + LONG $0xc870f9c5 + BYTE $0x1b + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawShuffle": {Name: "rawShuffle", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "shufflevector <16 x i8>") || + !strings.Contains(ir, "<16 x i32> ") { + t.Fatalf("raw VPSHUFD lowering omitted packed shuffle:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-packed-shuffle.ll", "amd64-raw-vex-packed-shuffle.o", ir) +} + +func TestDecodedX86PackedShuffleCompleteVEXRegisterFamily(t *testing.T) { + tests := []struct { + pp byte + op Op + }{ + {pp: 1, op: "VPSHUFD"}, + {pp: 2, op: "VPSHUFHW"}, + {pp: 3, op: "VPSHUFLW"}, + } + count := 0 + for _, test := range tests { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + sourceLimit := 16 + if !vex3 { + sourceLimit = 8 + } + for destination := 0; destination < 16; destination++ { + for source := 0; source < sourceLimit; source++ { + immediate := byte(destination<<4 | source) + code := encodeX86VEXPackedShuffle(test.pp, vex3, width256, widthIgnored, destination, source, immediate) + got, length, ok, err := decodedX86PackedShuffleInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("$%d", immediate), fmt.Sprintf("%s%d", prefix, source), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != test.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, test.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + if count != 3840 { + t.Fatalf("covered %d VEX packed shuffle register encodings, want 3840", count) + } +} + +func TestDecodedX86PackedShuffleCompleteGo127Family(t *testing.T) { + // Go 1.27 assigns these three instructions the shared _yvpshufd table. + // The cases cover VEX/EVEX X/Y/Z, WIG, high registers, memory, masks, + // zeroing, the VPSHUFD-only broadcast, and compressed disp8. + tests := []struct { + name string + code []byte + want string + }{ + {name: "vex d reported weaviate", code: []byte{0xc5, 0xf9, 0x70, 0xc8, 0x1b}, want: "VPSHUFD $27, X0, X1"}, + {name: "vex hw y WIG memory", code: []byte{0x64, 0xc4, 0x01, 0xfe, 0x70, 0x64, 0x88, 0x20, 0x5a}, want: "VPSHUFHW $90, 32(R8)(R9*4)(FS), Y12"}, + {name: "evex d x high masked", code: []byte{0x62, 0x71, 0x7d, 0x0c, 0x70, 0xca, 0x7e}, want: "VPSHUFD $126, X2, K4, X9"}, + {name: "evex hw x high masked WIG", code: []byte{0x62, 0x41, 0xfe, 0x0a, 0x70, 0xfb, 0x0d}, want: "VPSHUFHW $13, X11, K2, X31"}, + {name: "evex lw y high masked", code: []byte{0x62, 0xe1, 0x7f, 0x2a, 0x70, 0xdf, 0x00}, want: "VPSHUFLW $0, Y7, K2, Y19"}, + {name: "evex d z broadcast zeroing", code: []byte{0x62, 0xf1, 0x7d, 0xdb, 0x70, 0x50, 0x7f, 0x09}, want: "VPSHUFD.BCST.Z $9, 508(AX), K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86PackedShuffleInstruction(test.code, 64) + if !ok { + t.Fatal("packed shuffle encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodedX86PackedShuffleRejectsInvalidForms(t *testing.T) { + valid := encodeX86VEXPackedShuffle(1, true, false, false, 0, 0, 1) + invalidVVVV := append([]byte(nil), valid...) + invalidVVVV[2] &^= 0x08 + for name, code := range map[string][]byte{ + "truncated": {0xc5}, + "wrong opcode": {0xc5, 0xf9, 0x71, 0xc0, 1}, + "wrong map": {0xc4, 0xe2, 0x79, 0x70, 0xc0, 1}, + "unrelated pp": {0xc5, 0xf8, 0x70, 0xc0, 1}, + "EVEX D with W1": {0x62, 0xf1, 0xfd, 0x08, 0x70, 0xc0, 1}, + } { + if instruction, _, matched, decodeErr := decodedX86PackedShuffleInstruction(code, 64); matched || decodeErr != nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, code, instruction, matched, decodeErr) + } + } + if _, _, matched, decodeErr := decodedX86PackedShuffleInstruction(invalidVVVV, 64); !matched || decodeErr == nil { + t.Fatalf("non-reserved VEX.vvvv encoding %x returned ok=%v err=%v", invalidVVVV, matched, decodeErr) + } + extended := encodeX86VEXPackedShuffle(1, true, false, false, 8, 8, 1) + if _, _, matched, decodeErr := decodedX86PackedShuffleInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } + invalidBroadcast := []byte{0x62, 0xf1, 0x7e, 0x5b, 0x70, 0x50, 0x7f, 1} + if _, _, matched, decodeErr := decodedX86PackedShuffleInstruction(invalidBroadcast, 64); !matched || decodeErr == nil { + t.Fatalf("VPSHUFHW broadcast returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vextract128_test.go b/amd64_raw_vextract128_test.go new file mode 100644 index 000000000..3cde8176d --- /dev/null +++ b/amd64_raw_vextract128_test.go @@ -0,0 +1,248 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXTRACT128(opcode, immediate byte, source, destination int) []byte { + vex0 := byte((1-source/8)<<7 | 1<<6 | (1-destination/8)<<5 | 3) + modRM := byte(0xc0 | (source&7)<<3 | destination&7) + return []byte{0xc4, vex0, 0x7d, opcode, modRM, immediate} +} + +func encodeX86EVEXTRACT(opcode, immediate byte, width64 bool, vectorBits byte, mask int, zeroing bool, source, destination int) []byte { + p0 := byte((1-source/8&1)<<7 | (1-destination/16)<<6 | (1-destination/8&1)<<5 | (1-source/16)<<4 | 3) + p1 := byte(0x7d) + if width64 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 1<<3 | byte(mask)) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (source&7)<<3 | destination&7) + return []byte{0x62, p0, p1, p2, opcode, modRM, immediate} +} + +func TestTranslateRawVEXTRACT128WeaviateRegression(t *testing.T) { + const source = `TEXT rawExtract(SB), $0-0 + LONG $0x197de3c4; WORD $0x01c1 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawExtract": {Name: "rawExtract", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "shufflevector <32 x i8>") { + t.Fatalf("raw VEXTRACTF128 lowering omitted vector extract:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vextract128.ll", "amd64-raw-vextract128.o", ir) +} + +func TestDecodedX86VEXTRACT128CompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, wantOp := range map[byte]Op{0x19: "VEXTRACTF128", 0x39: "VEXTRACTI128"} { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + for immediate := 0; immediate < 256; immediate++ { + code := encodeX86VEXTRACT128(opcode, byte(immediate), source, destination) + got, length, ok, err := decodedX86VEXTRACT128Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].Imm != int64(immediate) || got.Args[1].String() != fmt.Sprintf("Y%d", source) || got.Args[2].String() != fmt.Sprintf("X%d", destination) { + t.Fatalf("decode %x = %+v, want %s $%d, Y%d, X%d", code, got, wantOp, immediate, source, destination) + } + count++ + } + } + } + } + if count != 131072 { + t.Fatalf("covered %d VEXTRACT128 register encodings, want 131072", count) + } +} + +func TestDecodedX86VEXTRACT128MemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x03, 0x7d, 0x39, 0x64, 0x88, 0x20, 0xff} + got, length, ok, err := decodedX86VEXTRACT128Instruction(code, 64) + wantDestination := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VEXTRACTI128" || len(got.Args) != 3 || got.Args[0].Imm != 0xff || got.Args[1].String() != "Y12" || got.Args[2].Kind != OpMem || got.Args[2].Mem != wantDestination { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXTRACT128(0x19, 1, 0, 0) + for _, invalid := range [][]byte{ + {0xc4}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5]}, + encodeX86VEXTRACT128(0x18, 1, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + for _, invalid := range [][]byte{ + {valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2] &^ 4, valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2] | 0x80, valid[3], valid[4], valid[5]}, + } { + if _, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("reserved encoding %x returned ok=%v err=%v", invalid, matched, decodeErr) + } + } + extended := encodeX86VEXTRACT128(0x19, 1, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} + +func TestTranslateRawEVEXTRACTWeaviateRegression(t *testing.T) { + const source = `TEXT rawEVEXTRACT(SB), $0-0 + LONG $0x48fdf362 + WORD $0xc13b + BYTE $0x01 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawEVEXTRACT": {Name: "rawEVEXTRACT", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "shufflevector <64 x i8>") { + t.Fatalf("raw VEXTRACTI64X4 lowering omitted 256-bit block extract:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-evextract.ll", "amd64-raw-evextract.o", ir) +} + +func TestDecodedX86VEXTRACTCompleteEVEXRegisterFamily(t *testing.T) { + tests := []struct { + opcode byte + width64 bool + op Op + outputBytes int + vectorBits []byte + }{ + {opcode: 0x19, op: "VEXTRACTF32X4", outputBytes: 16, vectorBits: []byte{1, 2}}, + {opcode: 0x19, width64: true, op: "VEXTRACTF64X2", outputBytes: 16, vectorBits: []byte{1, 2}}, + {opcode: 0x1b, op: "VEXTRACTF32X8", outputBytes: 32, vectorBits: []byte{2}}, + {opcode: 0x1b, width64: true, op: "VEXTRACTF64X4", outputBytes: 32, vectorBits: []byte{2}}, + {opcode: 0x39, op: "VEXTRACTI32X4", outputBytes: 16, vectorBits: []byte{1, 2}}, + {opcode: 0x39, width64: true, op: "VEXTRACTI64X2", outputBytes: 16, vectorBits: []byte{1, 2}}, + {opcode: 0x3b, op: "VEXTRACTI32X8", outputBytes: 32, vectorBits: []byte{2}}, + {opcode: 0x3b, width64: true, op: "VEXTRACTI64X4", outputBytes: 32, vectorBits: []byte{2}}, + } + count := 0 + for _, test := range tests { + for _, vectorBits := range test.vectorBits { + sourcePrefix := [...]string{"", "Y", "Z"}[vectorBits] + destinationPrefix := "X" + if test.outputBytes == 32 { + destinationPrefix = "Y" + } + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for source := 0; source < 32; source++ { + for destination := 0; destination < 32; destination++ { + immediate := byte(source<<3 | destination&7) + code := encodeX86EVEXTRACT(test.opcode, immediate, test.width64, vectorBits, masking.mask, masking.zeroing, source, destination) + got, length, ok, err := decodedX86VEXTRACT128Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("$%d", immediate), fmt.Sprintf("%s%d", sourcePrefix, source)} + if masking.mask != 0 { + wantArgs = append(wantArgs, fmt.Sprintf("K%d", masking.mask)) + } + wantArgs = append(wantArgs, fmt.Sprintf("%s%d", destinationPrefix, destination)) + wantOp := test.op + Op(masking.suffix) + if got.Op != wantOp || len(got.Args) != len(wantArgs) { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + for index := range wantArgs { + if got.Args[index].String() != wantArgs[index] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + } + count++ + } + } + } + } + } + if count != 36864 { + t.Fatalf("covered %d EVEX vector extract register encodings, want 36864", count) + } +} + +func TestDecodedX86VEXTRACTCompleteEVEXMemoryAndInvalidForms(t *testing.T) { + tests := []struct { + opcode byte + width64 bool + op Op + outputBytes int + vectorBits byte + }{ + {opcode: 0x19, op: "VEXTRACTF32X4", outputBytes: 16, vectorBits: 1}, + {opcode: 0x19, width64: true, op: "VEXTRACTF64X2", outputBytes: 16, vectorBits: 2}, + {opcode: 0x1b, op: "VEXTRACTF32X8", outputBytes: 32, vectorBits: 2}, + {opcode: 0x1b, width64: true, op: "VEXTRACTF64X4", outputBytes: 32, vectorBits: 2}, + {opcode: 0x39, op: "VEXTRACTI32X4", outputBytes: 16, vectorBits: 1}, + {opcode: 0x39, width64: true, op: "VEXTRACTI64X2", outputBytes: 16, vectorBits: 2}, + {opcode: 0x3b, op: "VEXTRACTI32X8", outputBytes: 32, vectorBits: 2}, + {opcode: 0x3b, width64: true, op: "VEXTRACTI64X4", outputBytes: 32, vectorBits: 2}, + } + for _, test := range tests { + code := encodeX86EVEXTRACT(test.opcode, 0xff, test.width64, test.vectorBits, 2, false, 21, 0) + code[5] = 0x68 + code = append(code[:6], append([]byte{0x7f}, code[6:]...)...) + got, length, ok, err := decodedX86VEXTRACT128Instruction(code, 64) + wantSourcePrefix := [...]string{"", "Y", "Z"}[test.vectorBits] + wantDestination := fmt.Sprintf("%d(AX)", test.outputBytes*127) + if err != nil || !ok || length != len(code) || got.Op != test.op || len(got.Args) != 4 || got.Args[0].Imm != 0xff || got.Args[1].String() != fmt.Sprintf("%s21", wantSourcePrefix) || got.Args[2].String() != "K2" || got.Args[3].String() != wantDestination { + t.Fatalf("decode %x = %+v, length=%d ok=%v err=%v; want %s $255, %s21, K2, %s", code, got, length, ok, err, test.op, wantSourcePrefix, wantDestination) + } + } + + valid := encodeX86EVEXTRACT(0x3b, 1, true, 2, 1, false, 0, 0) + for name, mutate := range map[string]func([]byte){ + "used vvvv": func(code []byte) { code[2] &^= 0x08 }, + "broadcast": func(code []byte) { code[3] |= 0x10 }, + "reserved length": func(code []byte) { code[3] |= 0x20 }, + } { + code := append([]byte(nil), valid...) + mutate(code) + if _, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(code, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, code, matched, decodeErr) + } + } + memoryZero := encodeX86EVEXTRACT(0x3b, 1, true, 2, 1, true, 0, 0) + memoryZero[5] = 0x00 + memoryZero = append(memoryZero[:6], append([]byte{0, 0, 0, 0}, memoryZero[6:]...)...) + if _, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(memoryZero, 64); !matched || decodeErr == nil { + t.Fatalf("zeroing memory encoding returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vinserti128_test.go b/amd64_raw_vinserti128_test.go new file mode 100644 index 000000000..122fbff64 --- /dev/null +++ b/amd64_raw_vinserti128_test.go @@ -0,0 +1,226 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86VINSERT128(opcode, immediate byte, firstSource, secondSource, destination int) []byte { + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-firstSource/8)<<5 | 3) + vexBits := byte((^secondSource&15)<<3 | 1<<2 | 1) + modRM := byte(0xc0 | (destination&7)<<3 | firstSource&7) + return []byte{0xc4, vex0, vexBits, opcode, modRM, immediate} +} + +func encodeX86VEXPackedBroadcast(opcode byte, wide bool, source, destination int) []byte { + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source/8)<<5 | 2) + vexBits := byte(0x79) + if wide { + vexBits |= 1 << 2 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + return []byte{0xc4, vex0, vexBits, opcode, modRM} +} + +func TestDecodeX86RawDirectiveGroupReportedInfectiousVINSERTI128(t *testing.T) { + code := []byte{ + 0xc4, 0xe3, 0x4d, 0x38, 0xf6, 0x01, + 0xc4, 0xe3, 0x45, 0x38, 0xff, 0x01, + 0xc4, 0x62, 0x7d, 0x78, 0xc5, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "infectious VINSERTI128 sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := []string{ + "VINSERTI128 $1, X6, Y6, Y6", + "VINSERTI128 $1, X7, Y7, Y7", + "VPBROADCASTB X5, Y8", + } + if len(decoded) != len(want) { + t.Fatalf("decoded %d instructions as %#v, want %d", len(decoded), decoded, len(want)) + } + for index := range want { + if !strings.HasPrefix(decoded[index].Raw, want[index]+" ") { + t.Fatalf("instruction %d decoded as %q, want %q", index, decoded[index].Raw, want[index]) + } + } +} + +func TestDecodedX86RawVINSERTI128Go127OracleCases(t *testing.T) { + for _, test := range []struct { + code []byte + want string + }{ + {code: []byte{0xc4, 0xe3, 0x05, 0x38, 0x13, 0x07}, want: "VINSERTI128 $7, 0(BX), Y15, Y2"}, + {code: []byte{0xc4, 0xc3, 0x05, 0x38, 0x13, 0x07}, want: "VINSERTI128 $7, 0(R11), Y15, Y2"}, + {code: []byte{0xc4, 0xe3, 0x05, 0x38, 0xd2, 0x07}, want: "VINSERTI128 $7, X2, Y15, Y2"}, + {code: []byte{0xc4, 0x43, 0x05, 0x38, 0xdb, 0x07}, want: "VINSERTI128 $7, X11, Y15, Y11"}, + } { + instruction, length, ok, err := decodedX86VINSERT128Instruction(test.code, 64) + if !ok || err != nil || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as (%#v, %d, %v, %v), want %q", test.code, instruction, length, ok, err, test.want) + } + } +} + +func TestDecodedX86RawVINSERT128CompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, wantOp := range map[byte]Op{0x18: "VINSERTF128", 0x38: "VINSERTI128"} { + for firstSource := 0; firstSource < 16; firstSource++ { + for secondSource := 0; secondSource < 16; secondSource++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(firstSource*16 + destination) + code := encodeX86VINSERT128(opcode, immediate, firstSource, secondSource, destination) + got, length, ok, err := decodedX86VINSERT128Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{ + fmt.Sprintf("$%d", immediate), + fmt.Sprintf("X%d", firstSource), + fmt.Sprintf("Y%d", secondSource), + fmt.Sprintf("Y%d", destination), + } + if got.Op != wantOp || len(got.Args) != len(wantArgs) { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + for index := range wantArgs { + if got.Args[index].String() != wantArgs[index] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d VINSERT128 register encodings, want 8192", count) + } +} + +func TestDecodedX86RawVINSERT128MemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x03, 0x15, 0x38, 0x64, 0x8b, 0x20, 0xff} + got, length, ok, err := decodedX86VINSERT128Instruction(code, 64) + wantMemory := MemRef{Segment: FS, Base: "R11", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VINSERTI128" || len(got.Args) != 4 || got.Args[0].Imm != 0xff || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) || got.Args[2].String() != "Y13" || got.Args[3].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VINSERT128(0x38, 1, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc4}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5]}, + encodeX86VINSERT128(0x39, 1, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VINSERT128Instruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + for name, mutate := range map[string]func([]byte){ + "wrong pp": func(code []byte) { code[2] &^= 1 }, + "VEX.W": func(code []byte) { code[2] |= 0x80 }, + "VEX.128": func(code []byte) { code[2] &^= 4 }, + "address override": func(code []byte) { copy(code[1:], code); code[0] = 0x67 }, + } { + code := append([]byte(nil), valid...) + if name == "address override" { + code = append(code, 0) + } + mutate(code) + if _, _, matched, decodeErr := decodedX86VINSERT128Instruction(code, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, code, matched, decodeErr) + } + } + truncatedImmediate := valid[:len(valid)-1] + if _, _, matched, decodeErr := decodedX86VINSERT128Instruction(truncatedImmediate, 64); !matched || decodeErr == nil { + t.Fatalf("truncated immediate returned ok=%v err=%v", matched, decodeErr) + } + if _, _, matched, decodeErr := decodedX86VINSERT128Instruction(valid, 32); !matched || decodeErr == nil { + t.Fatalf("386 VINSERT128 returned ok=%v err=%v", matched, decodeErr) + } +} + +func TestDecodedX86RawVEXPackedBroadcastCompleteRegisterFamily(t *testing.T) { + operations := map[byte]Op{ + 0x78: "VPBROADCASTB", + 0x79: "VPBROADCASTW", + 0x58: "VPBROADCASTD", + 0x59: "VPBROADCASTQ", + } + count := 0 + for opcode, wantOp := range operations { + for _, wide := range []bool{false, true} { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86VEXPackedBroadcast(opcode, wide, source, destination) + got, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + prefix := "X" + if wide { + prefix = "Y" + } + wantArgs := []string{fmt.Sprintf("X%d", source), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 2 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 2048 { + t.Fatalf("covered %d VEX packed-broadcast register encodings, want 2048", count) + } +} + +func TestDecodedX86RawVEXPackedBroadcastMemoryAndInvalidForms(t *testing.T) { + for opcode, wantOp := range map[byte]Op{0x78: "VPBROADCASTB", 0x79: "VPBROADCASTW", 0x58: "VPBROADCASTD", 0x59: "VPBROADCASTQ"} { + code := []byte{0x65, 0xc4, 0x02, 0x7d, opcode, 0x64, 0x8b, 0x20} + got, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + wantMemory := MemRef{Segment: GS, Base: "R11", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != wantOp || len(got.Args) != 2 || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) || got.Args[1].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + } + + valid := encodeX86VEXPackedBroadcast(0x78, true, 0, 0) + for _, invalid := range [][]byte{ + {0xc4}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXPackedBroadcast(0x77, true, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + for name, mutate := range map[string]func([]byte){ + "wrong pp": func(code []byte) { code[2] &^= 1 }, + "VEX.W": func(code []byte) { code[2] |= 0x80 }, + "used vvvv": func(code []byte) { code[2] &^= 0x08 }, + } { + code := append([]byte(nil), valid...) + mutate(code) + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(code, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, code, matched, decodeErr) + } + } + ripRelative := []byte{0xc4, 0xe2, 0x7d, 0x78, 0x05, 0, 0, 0, 0} + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(ripRelative, 64); !matched || decodeErr == nil { + t.Fatalf("RIP-relative encoding returned ok=%v err=%v", matched, decodeErr) + } + extended := encodeX86VEXPackedBroadcast(0x78, true, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("386 extended-register form returned ok=%v err=%v", matched, decodeErr) + } + valid386 := encodeX86VEXPackedBroadcast(0x78, true, 7, 7) + got, length, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(valid386, 32) + if decodeErr != nil || !matched || length != len(valid386) || got.Raw != "VPBROADCASTB X7, Y7" { + t.Fatalf("386 base-register form decoded as %+v, length=%d ok=%v err=%v", got, length, matched, decodeErr) + } +} diff --git a/amd64_raw_vmov_half_memory.go b/amd64_raw_vmov_half_memory.go new file mode 100644 index 000000000..61bc8ca44 --- /dev/null +++ b/amd64_raw_vmov_half_memory.go @@ -0,0 +1,92 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27 uses one _yvmovhpd operand table for all four VEX/EVEX half-vector +// memory moves. Register-only 0F 12/16 forms are the separate VMOVHLPS and +// VMOVLHPS family and are decoded before this function. +func decodedX86RawVectorHalfMemoryInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || p.mapNumber != 1 || (p.pp != 0 && p.pp != 1) { + return Instr{}, 0, false, nil + } + if p.opcode != 0x12 && p.opcode != 0x13 && p.opcode != 0x16 && p.opcode != 0x17 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("vector half memory move: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.vectorLength != 0 || p.mask != 0 || p.zero || p.broadcast { + return fail("invalid vector length, mask, zeroing, or broadcast") + } + if p.evex && (!p.fixed || p.w != (p.pp == 1)) { + return fail("invalid EVEX fixed or W bit") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if code[p.modRM]>>6 == 3 { + if p.pp == 0 { + return Instr{}, 0, false, nil + } + return fail("Go's VMOVHPD/VMOVLPD forms require memory") + } + + var memory Operand + var consumed int + var err error + if p.evex { + memory, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", 8) + } else { + memory, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + if memory.Kind != OpMem { + return fail("expected ModRM memory operand") + } + + op := Op("VMOVLPS") + if p.pp == 1 { + op = "VMOVLPD" + } + if p.opcode == 0x16 || p.opcode == 0x17 { + if p.pp == 0 { + op = "VMOVHPS" + } else { + op = "VMOVHPD" + } + } + store := p.opcode == 0x13 || p.opcode == 0x17 + vector := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (vector >= 8 || p.upper >= 8) { + return fail("extended vector register in 32-bit mode") + } + register := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", vector))} + args := []Operand{memory, {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}, register} + if store { + if p.upper != 0 { + return fail("store reserves the VEX/EVEX vvvv field") + } + args = []Operand{register, memory} + } + printed := make([]string, len(args)) + for i, arg := range args { + printed[i] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_vmov_half_memory_test.go b/amd64_raw_vmov_half_memory_test.go new file mode 100644 index 000000000..837bcc515 --- /dev/null +++ b/amd64_raw_vmov_half_memory_test.go @@ -0,0 +1,220 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVectorHalfMemory(op Op, encoding string, vector, upper int, store, wig bool) []byte { + high := strings.Contains(string(op), "H") + pd := strings.HasSuffix(string(op), "PD") + opcode := byte(0x12) + if high { + opcode = 0x16 + } + if store { + opcode++ + upper = 0 + } + pp := byte(0) + if pd { + pp = 1 + } + modRM := byte((vector&7)<<3 | 2) + rExt := byte(1 - ((vector >> 3) & 1)) + switch encoding { + case "VEX2": + p1 := rExt<<7 | byte(^upper&15)<<3 | pp + return []byte{0xc5, p1, opcode, modRM} + case "VEX3": + p0 := rExt<<7 | 1 // SIB index and base are R13 and R12. + p1 := byte(^upper&15)<<3 | pp + if wig { + p1 |= 0x80 + } + return []byte{0xc4, p0, p1, opcode, (modRM & 0x38) | 0x44, 0xac, 2} + case "EVEX": + r2Ext := byte(1 - ((vector >> 4) & 1)) + p0 := rExt<<7 | r2Ext<<4 | 1 + p1 := byte(^upper&15)<<3 | 4 | pp + if pd { + p1 |= 0x80 + } + p2 := byte((1 - ((upper >> 4) & 1)) << 3) + return []byte{0x62, p0, p1, p2, opcode, (modRM & 0x38) | 0x44, 0xac, 2} + default: + panic("unknown half-vector encoding") + } +} + +func TestTranslateRawVMOVHPDPathtracerRegression(t *testing.T) { + const source = ` +TEXT rawVMOVHPD(SB),$0-0 + MOVQ $0, DX + LONG $0x0217f9c5 // VMOVHPD X0, (DX) + RET +` + file, signatures := x86UnboundVoidFormatSource(t, source, "rawVMOVHPD", "amd64", "x86_64-unknown-linux-gnu") + file, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + ll := x86MoveHalfFormatObjectUnit(t, file, signatures, "amd64", "x86_64-unknown-linux-gnu") + if !strings.Contains(ll, "extractelement <2 x i64>") { + t.Fatalf("raw VMOVHPD did not store the high lane:\n%s", ll) + } +} + +func TestDecodedX86RawVectorHalfMemoryCompleteGo127Family(t *testing.T) { + for _, op := range []Op{"VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD"} { + for _, store := range []bool{false, true} { + for _, encoding := range []string{"VEX2", "VEX3", "EVEX"} { + vector, upper := 3, 4 + if encoding == "VEX3" { + vector, upper = 11, 12 + } + if encoding == "EVEX" { + vector, upper = 20, 21 + } + code := encodeX86RawVectorHalfMemory(op, encoding, vector, upper, store, encoding == "VEX3") + got, length, recognized, err := decodedX86RawVectorHalfMemoryInstruction(code, 64) + if err != nil || !recognized || length != len(code) || got.Op != op { + t.Fatalf("decode %s %s store=%v bytes=%x: got=%+v length=%d recognized=%v err=%v", op, encoding, store, code, got, length, recognized, err) + } + memoryIndex := 0 + registerIndex := 2 + if store { + memoryIndex, registerIndex = 1, 0 + } + if got.Args[memoryIndex].Kind != OpMem || got.Args[registerIndex].Reg != Reg(fmt.Sprintf("X%d", vector)) { + t.Fatalf("decoded wrong memory/register operands for %x: %+v", code, got.Args) + } + memory := got.Args[memoryIndex].Mem + if encoding == "VEX2" && (memory.Base != DX || memory.Off != 0) { + t.Fatalf("decoded wrong VEX2 memory operand for %x: %+v", code, memory) + } + if encoding == "VEX3" && (memory.Base != "R12" || memory.Index != "R13" || memory.Scale != 4 || memory.Off != 2) { + t.Fatalf("decoded wrong VEX3 memory operand for %x: %+v", code, memory) + } + if encoding == "EVEX" && (memory.Base != "R12" || memory.Index != "R13" || memory.Scale != 4 || memory.Off != 16) { + t.Fatalf("decoded wrong EVEX memory operand for %x: %+v", code, memory) + } + if !store && got.Args[1].Reg != Reg(fmt.Sprintf("X%d", upper)) { + t.Fatalf("decoded wrong upper-lane register for %x: %+v", code, got.Args) + } + } + } + } +} + +func TestDecodedX86RawVectorHalfMemoryRejectsInvalidForms(t *testing.T) { + valid := encodeX86RawVectorHalfMemory("VMOVHPD", "EVEX", 20, 21, false, false) + for _, test := range []struct { + name string + code []byte + }{ + {"missing ModRM", valid[:5]}, + {"wrong fixed bit", append([]byte(nil), valid...)}, + {"wrong W bit", append([]byte(nil), valid...)}, + {"vector length", append([]byte(nil), valid...)}, + {"mask", append([]byte(nil), valid...)}, + {"zeroing", append([]byte(nil), valid...)}, + {"broadcast", append([]byte(nil), valid...)}, + {"register operand", append([]byte(nil), valid...)}, + {"RIP relative", append([]byte(nil), valid...)}, + } { + switch test.name { + case "wrong fixed bit": + test.code[2] &^= 4 + case "wrong W bit": + test.code[2] &^= 0x80 + case "vector length": + test.code[3] |= 0x20 + case "mask": + test.code[3] |= 1 + case "zeroing": + test.code[3] |= 0x80 + case "broadcast": + test.code[3] |= 0x10 + case "register operand": + test.code[5] |= 0xc0 + case "RIP relative": + test.code[5] = 0x05 + } + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawVectorHalfMemoryInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid half-vector encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + store := encodeX86RawVectorHalfMemory("VMOVHPD", "VEX2", 3, 0, true, false) + store[1] &^= 0x08 // A non-reserved VEX.vvvv field is not a Go store form. + if _, _, recognized, err := decodedX86RawVectorHalfMemoryInstruction(store, 64); !recognized || err == nil { + t.Fatalf("accepted store with non-reserved vvvv: recognized=%v err=%v", recognized, err) + } + extended := encodeX86RawVectorHalfMemory("VMOVHPD", "VEX3", 11, 12, false, false) + if _, _, recognized, err := decodedX86RawVectorHalfMemoryInstruction(extended, 32); !recognized || err == nil { + t.Fatalf("accepted 386 extended register form: recognized=%v err=%v", recognized, err) + } +} + +func TestTranslateRawX86VectorHalfMemoryLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawHalfMemory(SB),$0-0\n") + if target.goarch == "386" { + source.WriteString("\tMOVL $0, DX\n") + } else { + source.WriteString("\tMOVQ $0, DX\n") + } + for _, op := range []Op{"VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD"} { + for _, store := range []bool{false, true} { + encodings := []string{"VEX2"} + if target.goarch == "amd64" { + encodings = append(encodings, "VEX3", "EVEX") + } + for _, encoding := range encodings { + vector, upper := 3, 4 + if encoding == "VEX3" { + vector, upper = 11, 12 + } + if encoding == "EVEX" { + vector, upper = 20, 21 + } + for _, value := range encodeX86RawVectorHalfMemory(op, encoding, vector, upper, store, false) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + file, signatures := x86UnboundVoidFormatSource(t, source.String(), "rawHalfMemory", target.goarch, target.triple) + file, err := normalizeX86RawFile(file, target.goarch) + if err != nil { + t.Fatal(err) + } + ll := x86MoveHalfFormatObjectUnit(t, file, signatures, target.goarch, target.triple) + for _, want := range []string{"extractelement <2 x i64>", "insertelement <2 x i64>"} { + if !strings.Contains(ll, want) { + t.Fatalf("half-vector lowering omitted %q", want) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-half-memory.ll", "raw-half-memory.o", ll) + }) + } +} diff --git a/amd64_raw_vmovq_test.go b/amd64_raw_vmovq_test.go new file mode 100644 index 000000000..9b8fb18ea --- /dev/null +++ b/amd64_raw_vmovq_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateRawVEXVMOVDWeaviateRegression(t *testing.T) { + const source = `TEXT rawVMOVD(SB), $0-0 + LONG $0x7e79c1c4 + BYTE $0xc7 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawVMOVD": {Name: "rawVMOVD", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "extractelement <4 x i32>") { + t.Fatalf("raw VMOVD lowering omitted scalar extraction:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-vmovd.ll", "amd64-raw-vex-vmovd.o", ir) +} + +func TestDecodedX86VMOVQInstructionCompleteVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + want string + }{ + {name: "x_to_gp", code: []byte{0xc4, 0xe1, 0xf9, 0x7e, 0xc8}, want: "VMOVQ X1, AX"}, + {name: "x_to_x", code: []byte{0xc4, 0xe1, 0x79, 0xd6, 0xc8}, want: "VMOVQ X1, X0"}, + {name: "gp_to_x", code: []byte{0xc4, 0xe1, 0xf9, 0x6e, 0xc8}, want: "VMOVQ AX, X1"}, + {name: "x_to_x_f3", code: []byte{0xc5, 0xfa, 0x7e, 0xc8}, want: "VMOVQ X0, X1"}, + {name: "memory_to_extended_x", code: []byte{0xc5, 0x7a, 0x7e, 0x5e, 0x20}, want: "VMOVQ 32(SI), X11"}, + {name: "extended_x_to_sib_memory", code: []byte{0xc4, 0x21, 0xf9, 0x7e, 0x94, 0x8c, 0x78, 0x56, 0x34, 0x12}, want: "VMOVQ X10, 305419896(SP)(R9*4)"}, + {name: "reported_weaviate_vmovd", code: []byte{0xc4, 0xc1, 0x79, 0x7e, 0xc7}, want: "VMOVD X0, R15"}, + {name: "vmovd_extended_gp_to_x", code: []byte{0xc4, 0xc1, 0x79, 0x6e, 0xcf}, want: "VMOVD R15, X1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VMOVQInstruction(test.code, 64) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(test.code) { + t.Fatalf("length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("instruction = %q, want %q", instruction.Raw, test.want) + } + }) + } +} + +func encodeX86VEXVMOVDQ(opcode, pp byte, vex3, width64 bool, vector, rm int) []byte { + modRM := byte(0xc0 | (vector&7)<<3 | rm&7) + if !vex3 { + vex1 := byte((1-vector/8)<<7 | 15<<3 | int(pp)) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-vector/8)<<7 | 1<<6 | (1-rm/8)<<5 | 1) + vex1 := byte(15<<3 | int(pp)) + if width64 { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestDecodedX86VMOVDQCompleteVEXRegisterFamily(t *testing.T) { + tests := []struct { + opcode byte + pp byte + width64 bool + op Op + reverse bool + vectorRM bool + vex2 bool + }{ + {opcode: 0x7e, pp: 1, op: "VMOVD", vex2: true}, + {opcode: 0x6e, pp: 1, op: "VMOVD", reverse: true, vex2: true}, + {opcode: 0x7e, pp: 1, width64: true, op: "VMOVQ"}, + {opcode: 0xd6, pp: 1, op: "VMOVQ", vectorRM: true, vex2: true}, + {opcode: 0x6e, pp: 1, width64: true, op: "VMOVQ", reverse: true}, + {opcode: 0x7e, pp: 2, op: "VMOVQ", reverse: true, vectorRM: true, vex2: true}, + } + count := 0 + for _, test := range tests { + for _, vex3 := range []bool{false, true} { + if !vex3 && !test.vex2 { + continue + } + rmLimit := 16 + if !vex3 { + rmLimit = 8 + } + for vector := 0; vector < 16; vector++ { + for rm := 0; rm < rmLimit; rm++ { + code := encodeX86VEXVMOVDQ(test.opcode, test.pp, vex3, test.width64, vector, rm) + got, length, ok, err := decodedX86VMOVQInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + rmName := fmt.Sprintf("R%d", rm) + if rm < 8 { + rmName = [...]string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"}[rm] + } + if test.vectorRM { + rmName = fmt.Sprintf("X%d", rm) + } + wantArgs := []string{fmt.Sprintf("X%d", vector), rmName} + if test.reverse { + wantArgs[0], wantArgs[1] = wantArgs[1], wantArgs[0] + } + if got.Op != test.op || len(got.Args) != 2 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, test.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 2048 { + t.Fatalf("covered %d VEX VMOVD/VMOVQ register encodings, want 2048", count) + } +} + +func TestDecodedX86VMOVDQCompleteGo127EVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + want string + }{ + {name: "d high x to gp", code: []byte{0x62, 0x61, 0x7d, 0x08, 0x7e, 0xf8}, want: "VMOVD X31, AX"}, + {name: "d gp to high x", code: []byte{0x62, 0x61, 0x7d, 0x08, 0x6e, 0xf8}, want: "VMOVD AX, X31"}, + {name: "q high x to gp", code: []byte{0x62, 0x61, 0xfd, 0x08, 0x7e, 0xf8}, want: "VMOVQ X31, AX"}, + {name: "q x to high x", code: []byte{0x62, 0x91, 0xfd, 0x08, 0xd6, 0xc7}, want: "VMOVQ X0, X31"}, + {name: "q gp to high x", code: []byte{0x62, 0x61, 0xfd, 0x08, 0x6e, 0xf8}, want: "VMOVQ AX, X31"}, + {name: "q high x to x alternate", code: []byte{0x62, 0xf1, 0xfe, 0x08, 0x7e, 0xc7}, want: "VMOVQ X7, X0"}, + {name: "d high x to compressed memory", code: []byte{0x62, 0x61, 0x7d, 0x08, 0x7e, 0x78, 0x7f}, want: "VMOVD X31, 508(AX)"}, + {name: "q compressed memory to high x", code: []byte{0x62, 0x61, 0xfd, 0x08, 0x6e, 0x78, 0x7f}, want: "VMOVQ 1016(AX), X31"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VMOVQInstruction(test.code, 64) + if err != nil { + t.Fatal(err) + } + if !ok || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as %+v length=%d ok=%v, want %q", test.code, instruction, length, ok, test.want) + } + }) + } +} + +func TestDecodedX86VMOVQInstructionRejectsInvalidVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc5, 0xfa, 0x7e}, mode: 64}, + {name: "vex_l_one", code: []byte{0xc5, 0xfe, 0x7e, 0xc8}, mode: 64}, + {name: "used_vvvv", code: []byte{0xc5, 0xf2, 0x7e, 0xc8}, mode: 64}, + {name: "w_one_in_386", code: []byte{0xc4, 0xe1, 0xf9, 0x6e, 0xc8}, mode: 32}, + {name: "extended_register_in_386", code: []byte{0xc5, 0x7a, 0x7e, 0xc0}, mode: 32}, + {name: "rip_relative", code: []byte{0xc5, 0xfa, 0x7e, 0x05, 0, 0, 0, 0}, mode: 64}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VMOVQInstruction(test.code, test.mode) + if !ok { + t.Fatal("VMOVQ-family encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} diff --git a/amd64_raw_vmovw_test.go b/amd64_raw_vmovw_test.go new file mode 100644 index 000000000..9cc1b2bf4 --- /dev/null +++ b/amd64_raw_vmovw_test.go @@ -0,0 +1,390 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Intel AVX512-FP16 section 5.61: EVEX.128.66.MAP5.WIG 6E/7E. +// The ModRM.reg operand is always X; ModRM.r/m is GP or m16, never X. +func rawVMOVW(load, wide bool, vector, general int, memory bool) []byte { + p0 := byte(0xf5) + p0 &^= byte((vector>>3&1)<<7 | (vector>>4&1)<<4 | (general>>3&1)<<5) + p1 := byte(0x7d) + if wide { + p1 |= 0x80 + } + opcode := byte(0x7e) + if load { + opcode = 0x6e + } + modRM := byte(0xc0 | (vector&7)<<3 | general&7) + if memory { + modRM = 0x40 | byte(vector&7)<<3 | byte(general&7) + } + code := []byte{0x62, p0, p1, 0x08, opcode, modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestRawVMOVWCompleteFormats(t *testing.T) { + for _, mode := range []int{32, 64} { + vectors, generals := 8, 8 + if mode == 64 { + vectors, generals = 32, 16 + } + for _, load := range []bool{false, true} { + for _, wide := range []bool{false, true} { + for vector := 0; vector < vectors; vector++ { + for general := 0; general < generals; general++ { + code := rawVMOVW(load, wide, vector, general, false) + got, n, ok, err := decodedX86VMOVQInstruction(code, mode) + if err != nil || !ok || n != len(code) || got.Op != "VMOVW" || !got.x86Encoded { + t.Fatalf("mode %d %x: %+v, n=%d, matched=%v, err=%v", mode, code, got, n, ok, err) + } + xIndex := 0 + if load { + xIndex = 1 + } + wantGeneral, _ := decodedX86GeneralRegister(general) + if got.Args[xIndex].String() != fmt.Sprintf("X%d", vector) || got.Args[1-xIndex].Kind != OpReg || + got.Args[1-xIndex].Reg != wantGeneral { + t.Fatalf("%x: wrong operands %+v", code, got.Args) + } + } + } + code := rawVMOVW(load, wide, 1, 0, true) + got, _, ok, err := decodedX86VMOVQInstruction(code, mode) + if err != nil || !ok { + t.Fatalf("%x: matched=%v, err=%v", code, ok, err) + } + memoryIndex := 1 + if load { + memoryIndex = 0 + } + if got.Args[memoryIndex].String() != "2(AX)" { + t.Fatalf("%x: expected two-byte displacement scaling, got %+v", code, got.Args) + } + } + } + } +} + +func TestRawVMOVWInvalidFormats(t *testing.T) { + for _, change := range []struct { + index int + bits byte + }{ + {2, 4}, {2, 8}, {3, 8}, {3, 1}, {3, 0x80}, {3, 0x10}, {3, 0x20}, {3, 0x40}, {1, 0x40}, + } { + code := rawVMOVW(true, false, 0, 1, false) + code[change.index] ^= change.bits + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid VMOVW", map[string]bool{}); err == nil { + t.Fatalf("accepted reserved VMOVW %x", code) + } + } + for _, code := range [][]byte{rawVMOVW(true, false, 8, 0, false), rawVMOVW(false, true, 0, 8, false)} { + if _, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 high VMOVW", map[string]bool{}); err == nil { + t.Fatalf("accepted 386 extended register %x", code) + } + } +} + +func TestRawVMOVWLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + type form struct { + name string + code []byte + load, memory bool + } + var forms []form + var complete []byte + for _, load := range []bool{false, true} { + for _, wide := range []bool{false, true} { + for _, memory := range []bool{false, true} { + code := rawVMOVW(load, wide, 1, 0, memory) + forms = append(forms, form{fmt.Sprintf("rawWordForm%d", len(forms)), code, load, memory}) + complete = append(complete, code...) + } + } + } + var source strings.Builder + // Retain the original eight-form, sequential function. Independent + // one-form functions prevent an unrelated transfer from satisfying a + // width, native-memory effect or GP-zero-extension assertion. + source.WriteString("TEXT rawWordMove(SB),4,$0-0\n") + source.WriteString(x86RawDisplacementDirectives(complete)) + source.WriteString("RET\n") + for _, f := range forms { + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", f.name) + source.WriteString(x86RawDisplacementDirectives(f.code)) + source.WriteString("RET\n") + } + for _, arch := range []string{"amd64", "386"} { + // BYTE acceptance alone is not an encoding oracle: require the exact + // contiguous stock-Go object text, including the original final RET. + x86HandoffOriginalBytes(t, arch, append(append([]byte(nil), complete...), 0xc3)) + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + // Keep the original unknown GP/X/m16 inputs as format-only + // objects and public Context controls. Initialized declarations + // and executed guard-page checks have independent test owners. + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.arch, target.triple) + ir = strings.ReplaceAll(ir, "@InstructionFormat0(", "@rawWordMove(") + for index, f := range forms { + ir = strings.ReplaceAll(ir, fmt.Sprintf("@InstructionFormat%d(", index+1), "@"+f.name+"(") + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := parseIRModuleInContext(ctx, ir) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + if err := rawVMOVWFunctionEffects(module, "rawWordMove", 2, 2, 2); err != nil { + t.Fatal(err) + } + mode := 64 + if target.arch == "386" { + mode = 32 + } + for _, f := range forms { + t.Run(f.name, func(t *testing.T) { + ins, size, known, err := decodedX86VMOVQInstruction(f.code, mode) + if err != nil || !known || size != len(f.code) || ins.Op != "VMOVW" || !ins.x86Encoded { + t.Fatalf("raw-only decoder receipt changed: %+v size=%d known=%t err=%v", ins, size, known, err) + } + properties, known, err := x86GPSIMDTransferProperties(ins, target.arch) + if err != nil || !known || properties.bits != 16 || !properties.spec.rawOnly || properties.mmx || + properties.destinationX != f.load || properties.sourceX == f.load || (properties.memory >= 0) != f.memory || + properties.memoryWrite != (f.memory && !f.load) { + t.Fatalf("16-bit raw-only transfer role changed: %+v known=%t err=%v", properties, known, err) + } + loads, stores, extensions := 0, 0, 0 + if f.memory { + if f.load { + loads = 1 + } else { + stores = 1 + } + } else if !f.load { + extensions = 1 + } + if err := rawVMOVWFunctionEffects(module, f.name, loads, stores, extensions); err != nil { + t.Fatal(err) + } + }) + } + rawVMOVWStoreCheckerMutations(t, ctx, module) + compileLLVMToObject(t, llc, target.triple, "raw-vmovw.ll", "raw-vmovw.o", ir) + }) + } +} + +func rawVMOVWFunctionEffects(module llvm.Module, name string, wantLoads, wantStores, wantExtensions int) error { + function := module.NamedFunction(name) + if function.IsNil() { + return fmt.Errorf("missing raw VMOVW function %s", name) + } + feature := false + for _, attribute := range function.GetFunctionAttributes() { + if attribute.IsString() && attribute.GetStringKind() == "target-features" { + for _, item := range strings.Split(attribute.GetStringValue(), ",") { + feature = feature || item == "+avx512fp16" + } + } + } + if !feature { + return fmt.Errorf("%s lost its private raw VMOVW ISA attribute", name) + } + loads, stores, extensions := 0, 0, 0 + for block := function.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for ins := block.FirstInstruction(); !ins.IsNil(); ins = llvm.NextInstruction(ins) { + if !ins.IsAZExtInst().IsNil() && ins.Operand(0).Type().TypeKind() == llvm.IntegerTypeKind && + ins.Operand(0).Type().IntTypeWidth() == 16 && ins.Type().IntTypeWidth() == 64 { + extensions++ + } + var typ llvm.Type + var pointer llvm.Value + load := !ins.IsALoadInst().IsNil() + if load { + typ, pointer = ins.Type(), ins.Operand(0) + } else if !ins.IsAStoreInst().IsNil() { + typ, pointer = ins.Operand(0).Type(), ins.Operand(1) + } else { + continue + } + if x86GPSIMDStoreTargetsBackingAlloca(pointer) { + continue // Virtual register banks are not original native memory. + } + if typ.TypeKind() != llvm.IntegerTypeKind || typ.IntTypeWidth() != 16 || !ins.IsVolatile() || + ins.Alignment() != 1 || pointer.IsAIntToPtrInst().IsNil() { + return fmt.Errorf("%s lost exact native m16 type/effect/address: %s", name, ins.String()) + } + if load { + loads++ + } else { + stores++ + } + } + } + if loads != wantLoads || stores != wantStores || extensions != wantExtensions { + return fmt.Errorf("%s m16 loads/stores/GP zero-extensions=%d/%d/%d, want=%d/%d/%d", + name, loads, stores, extensions, wantLoads, wantStores, wantExtensions) + } + return nil +} + +func rawVMOVWStoreCheckerMutations(t *testing.T, ctx llvm.Context, module llvm.Module) { + t.Helper() + function := module.NamedFunction("rawWordForm1") // X1 -> m16, W=0. + var store llvm.Value + for block := function.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for ins := block.FirstInstruction(); !ins.IsNil(); ins = llvm.NextInstruction(ins) { + if !ins.IsAStoreInst().IsNil() && !x86GPSIMDStoreTargetsBackingAlloca(ins.Operand(1)) { + if !store.IsNil() { + t.Fatal("mutation fixture has multiple native stores") + } + store = ins + } + } + } + if store.IsNil() { + t.Fatal("mutation fixture has no native store") + } + original := function.String() + for _, mutation := range []struct{ name, replacement string }{ + {"volatility", strings.Replace(store.String(), "store volatile", "store", 1)}, + {"width", "store volatile i32 1" + store.String()[strings.IndexByte(store.String(), ','):]}, + {"alignment", strings.Replace(store.String(), "align 1", "align 2", 1)}, + {"missing", "; removed original native m16 store"}, + } { + t.Run("checker-"+mutation.name, func(t *testing.T) { + if strings.Count(original, store.String()) != 1 || strings.Count(module.String(), original) != 1 { + t.Fatal("mutation does not bind the exact one-form function/store") + } + changed := strings.Replace(module.String(), original, strings.Replace(original, store.String(), mutation.replacement, 1), 1) + candidate, err := parseIRModuleInContext(ctx, changed) + if !candidate.IsNil() { + defer candidate.Dispose() + } + if err != nil { + t.Fatal("mutation must remain valid LLVM IR", err) + } + if err := llvm.VerifyModule(candidate, llvm.ReturnStatusAction); err != nil { + t.Fatal("mutation must remain a valid LLVM module", err) + } + if err := rawVMOVWFunctionEffects(candidate, "rawWordForm1", 0, 1, 0); err == nil { + t.Fatal("native m16 mutation escaped the per-function checker") + } + }) + } +} + +func TestRawVMOVWPortableRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source strings.Builder + source.WriteString("TEXT wordMove(SB),4,$0-24\nMOVQ input+0(FP), AX\nMOVQ out+8(FP), SI\nMOVQ vector+16(FP), DI\nVMOVDQU64 (DI), Z0\nMOVQ $-1, CX\n") + for _, code := range [][]byte{ + {0x62, 0xf5, 0x7d, 0x08, 0x6e, 0x00}, // m16 -> X0; no wider memory access. + {0x62, 0xf5, 0xfd, 0x08, 0x7e, 0xc1}, // X0 -> CX; WIG, zero-extend. + } { + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + source.WriteString("MOVQ CX, (SI)\nVMOVDQU64 Z0, 8(SI)\nMOVQ $-1, CX\n") + for _, code := range [][]byte{ + {0x62, 0xf5, 0x7d, 0x08, 0x6e, 0xc1}, // CX -> X0, truncate and clear all upper lanes. + {0x62, 0xf5, 0xfd, 0x08, 0x7e, 0x00}, // X0 -> m16; exactly two bytes. + } { + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + source.WriteString("VMOVDQU64 Z0, 72(SI)\nRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: testTargetTriple(runtime.GOOS, runtime.GOARCH), + Sigs: map[string]FuncSig{"wordMove": { + Name: "wordMove", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Attrs: "nounwind", + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + mainC := "#include \n#include \n#include \n#include \n" + x86TestGuardPagesC + ` +extern void wordMove(void *, void *, const void *); +int main(void) { + if (setup_guard()) return 1; + uint64_t output[17], vector[8]; + memset(vector, 0xff, sizeof(vector)); + for (int offset = 2; offset <= 3; offset++) { + unsigned char *pointer = guard + page - offset; + uint16_t input = 0x8123; + memcpy(pointer, &input, 2); + wordMove(pointer, output, vector); + if (output[0] != input || output[1] != input || output[9] != 0xffff) return 2; + for (int lane = 2; lane < 9; lane++) if (output[lane] != 0) return 3; + for (int lane = 10; lane < 17; lane++) if (output[lane] != 0) return 4; + memcpy(&input, pointer, 2); + if (input != 0xffff) return 5; + } + return free_guard(); +} +` + compileAndRunRuntimeTest(t, llc, clang, "vmovw_portable", ir, mainC) +} + +func TestRawVMOVWRIPLiteralAndNamedRejection(t *testing.T) { + code := []byte{0x62, 0xf5, 0x7d, 8, 0x6e, 5, 1, 0, 0, 0, 0xc3, 0x23, 0x81} + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "word RIP literal", map[string]bool{}) + if err != nil || len(got) != 2 || !got[0].x86RIPLiteral || got[0].Args[0].Kind != OpImm || got[0].Args[0].Imm != 0x8123 { + t.Fatalf("literal: %+v, err=%v", got, err) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated word pool", map[string]bool{}); err == nil { + t.Fatal("accepted truncated word pool") + } + const source = "TEXT namedWordMove(SB),4,$0-0\nVMOVW X0, AX\nRET\n" + for _, arch := range []string{"amd64", "386"} { + goAssemblerOracleOutput(t, arch, source, "", false) + t.Run(arch+"/named-three-APIs", func(t *testing.T) { + x86RequireZeroMaskAPIOutcome(t, arch, strings.Replace(source, "namedWordMove", "f", 1), false) + }) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedWordMove": {Name: "namedWordMove", Ret: Void}}, + }); err == nil { + t.Fatal("accepted VMOVW absent from Go's named assembler table") + } +} diff --git a/amd64_raw_vnni_test.go b/amd64_raw_vnni_test.go new file mode 100644 index 000000000..29b03b3db --- /dev/null +++ b/amd64_raw_vnni_test.go @@ -0,0 +1,125 @@ +package plan9asm + +import "testing" + +func TestDecodedX86VNNIInstructionCompleteVEXFamily(t *testing.T) { + for _, test := range []struct { + opcode byte + op Op + }{ + {0x50, "VPDPBUSD"}, + {0x51, "VPDPBUSDS"}, + {0x52, "VPDPWSSD"}, + {0x53, "VPDPWSSDS"}, + } { + t.Run(string(test.op), func(t *testing.T) { + code := []byte{0xc4, 0xe2, 0x71, test.opcode, 0xc2} + instruction, length, ok, err := decodedX86VNNIInstruction(code, 64) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(code) { + t.Fatalf("length = %d, want %d", length, len(code)) + } + want := string(test.op) + " X2, X1, X0" + if instruction.Op != test.op || instruction.Raw != want { + t.Fatalf("instruction = %#v, want %q", instruction, want) + } + }) + } + + for _, test := range []struct { + name string + code []byte + mode int + want string + }{ + { + name: "y_extended_registers", + code: []byte{0xc4, 0x42, 0x15, 0x50, 0xdc}, + mode: 64, + want: "VPDPBUSD Y12, Y13, Y11", + }, + { + name: "x_memory_sib_disp8", + code: []byte{0xc4, 0x62, 0x11, 0x50, 0x5c, 0x88, 0x20}, + mode: 64, + want: "VPDPBUSD 32(AX)(CX*4), X13, X11", + }, + { + name: "y_memory_negative_disp8", + code: []byte{0xc4, 0x62, 0x65, 0x51, 0x64, 0x35, 0xef}, + mode: 64, + want: "VPDPBUSDS -17(BP)(SI*1), Y3, Y12", + }, + { + name: "x_memory_extended_address", + code: []byte{0xc4, 0x82, 0x39, 0x52, 0x74, 0x7e, 0x07}, + mode: 64, + want: "VPDPWSSD 7(R14)(R15*2), X8, X6", + }, + { + name: "y_386_memory", + code: []byte{0xc4, 0xe2, 0x65, 0x53, 0x64, 0x35, 0xef}, + mode: 32, + want: "VPDPWSSDS -17(BP)(SI*1), Y3, Y4", + }, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VNNIInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode = (%#v, %d, %v), want %q", instruction, length, ok, test.want) + } + }) + } +} + +func TestDecodedX86VNNIInstructionRejectsInvalidVEXForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc4, 0xe2, 0x71, 0x50}, mode: 64}, + {name: "vex_pp_zero", code: []byte{0xc4, 0xe2, 0x70, 0x50, 0xc2}, mode: 64}, + {name: "vex_w_one", code: []byte{0xc4, 0xe2, 0xf1, 0x50, 0xc2}, mode: 64}, + {name: "address_override", code: []byte{0x67, 0xc4, 0xe2, 0x71, 0x50, 0xc2}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x42, 0x15, 0x50, 0xdc}, mode: 32}, + {name: "rip_relative", code: []byte{0xc4, 0xe2, 0x71, 0x50, 0x05, 0, 0, 0, 0}, mode: 64}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VNNIInstruction(test.code, test.mode) + if !ok { + t.Fatal("VNNI encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVNNIReportedSequence(t *testing.T) { + code := []byte{ + 0xc4, 0x42, 0x15, 0x50, 0xdc, + 0xc4, 0x62, 0x7d, 0x50, 0xc7, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "go-pherence VEX VNNI sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 { + t.Fatalf("decoded %d instructions, want 2: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VPDPBUSD" { + t.Fatalf("decoded op = %s, want VPDPBUSD", instruction.Op) + } + } +} diff --git a/amd64_raw_vpadd_test.go b/amd64_raw_vpadd_test.go new file mode 100644 index 000000000..75be16de6 --- /dev/null +++ b/amd64_raw_vpadd_test.go @@ -0,0 +1,233 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXPackedIntegerArithmetic(opcode byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | 1) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestDecodedX86PackedIntegerAddCompleteVEXRegisterFamily(t *testing.T) { + count := 0 + for opcode, properties := range decodedX86PackedIntegerArithmeticOps { + if !strings.HasPrefix(string(properties.op), "VPADD") { + continue + } + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedIntegerArithmetic(opcode, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != properties.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, properties.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + if count != 163840 { + t.Fatalf("covered %d VEX packed integer add register encodings, want 163840", count) + } +} + +func TestDecodedX86PackedIntegerSubtractCompleteVEXRegisterFamily(t *testing.T) { + subtractOps := map[byte]struct { + op Op + laneBytes int + }{ + 0xf8: {op: "VPSUBB", laneBytes: 1}, + 0xf9: {op: "VPSUBW", laneBytes: 2}, + 0xfa: {op: "VPSUBD", laneBytes: 4}, + 0xfb: {op: "VPSUBQ", laneBytes: 8}, + 0xe8: {op: "VPSUBSB", laneBytes: 1}, + 0xe9: {op: "VPSUBSW", laneBytes: 2}, + 0xd8: {op: "VPSUBUSB", laneBytes: 1}, + 0xd9: {op: "VPSUBUSW", laneBytes: 2}, + } + count := 0 + for opcode, properties := range subtractOps { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedIntegerArithmetic(opcode, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != properties.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, properties.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + if count != 163840 { + t.Fatalf("covered %d VEX packed integer subtract register encodings, want 163840", count) + } +} + +func TestDecodedX86PackedIntegerArithmeticCompleteEVEXRegisterFamily(t *testing.T) { + count := 0 + for opcode, properties := range decodedX86PackedIntegerArithmeticOps { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + for destination := 0; destination < 32; destination++ { + for source1 := 0; source1 < 32; source1++ { + for source2 := 0; source2 < 32; source2++ { + p0 := byte((1-(destination>>3)&1)<<7 | (1-(source2>>4)&1)<<6 | (1-(source2>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 1) + p1 := byte((^source1&15)<<3 | 1<<2 | 1) + if properties.laneBytes == 8 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^source1>>4&1)<<3 | 1) + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + code := []byte{0x62, p0, p1, p2, opcode, modRM} + got, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), "K1", fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != properties.op || len(got.Args) != 4 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, properties.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + if count != 1572864 { + t.Fatalf("covered %d EVEX packed integer arithmetic encodings, want 1572864", count) + } +} + +func TestTranslateRawVPSUBDWeaviateRegression(t *testing.T) { + const source = `TEXT rawPackedSubtract(SB), $0-0 + LONG $0xd5faedc5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawPackedSubtract": {Name: "rawPackedSubtract", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "sub <8 x i32>") { + t.Fatalf("raw VPSUBD lowering omitted vector subtraction:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-packed-subtract.ll", "amd64-raw-packed-subtract.o", ir) +} + +func TestDecodedX86PackedIntegerArithmeticCompleteGo127Family(t *testing.T) { + // Go 1.27 uses _yvandnpd for all sixteen VPADD/VPSUB variants. These LLVM 22 + // encodings cover every opcode plus VEX/EVEX X/Y/Z, high registers, + // memory, masking, zeroing, D/Q broadcast, and compressed disp8. + tests := []struct { + name string + code []byte + want string + }{ + {name: "vex byte x high", code: []byte{0xc5, 0x79, 0xfc, 0xc1}, want: "VPADDB X1, X0, X8"}, + {name: "vex qword y high memory", code: []byte{0xc4, 0x41, 0x6d, 0xd4, 0x4c, 0x24, 0x20}, want: "VPADDQ 32(R12), Y2, Y9"}, + {name: "byte x high masked", code: []byte{0x62, 0xa1, 0x55, 0x01, 0xfc, 0xf4}, want: "VPADDB X20, X21, K1, X22"}, + {name: "word y memory zeroing", code: []byte{0x62, 0xd1, 0x75, 0xaa, 0xfd, 0x50, 0x03}, want: "VPADDW.Z 96(R8), Y1, K2, Y2"}, + {name: "dword reported minio", code: []byte{0x62, 0x51, 0x45, 0x48, 0xfe, 0xc4}, want: "VPADDD Z12, Z7, Z8"}, + {name: "qword z broadcast zeroing", code: []byte{0x62, 0xf1, 0xf5, 0xdb, 0xd4, 0x50, 0x7f}, want: "VPADDQ.BCST.Z 1016(AX), Z1, K3, Z2"}, + {name: "signed byte z high", code: []byte{0x62, 0x31, 0x7d, 0x40, 0xec, 0xc9}, want: "VPADDSB Z17, Z16, Z9"}, + {name: "signed word x high memory masked", code: []byte{0x62, 0x41, 0x3d, 0x04, 0xed, 0x4c, 0x24, 0x02}, want: "VPADDSW 32(R12), X24, K4, X25"}, + {name: "unsigned byte y high masked", code: []byte{0x62, 0xa1, 0x55, 0x21, 0xdc, 0xf4}, want: "VPADDUSB Y20, Y21, K1, Y22"}, + {name: "unsigned word z memory zeroing", code: []byte{0x62, 0xf1, 0x75, 0xcb, 0xdd, 0x50, 0x7f}, want: "VPADDUSW.Z 8128(AX), Z1, K3, Z2"}, + {name: "subtract dword weaviate", code: []byte{0xc5, 0xed, 0xfa, 0xd5}, want: "VPSUBD Y5, Y2, Y2"}, + {name: "subtract qword broadcast zeroing", code: []byte{0x62, 0xf1, 0xf5, 0xdb, 0xfb, 0x50, 0x7f}, want: "VPSUBQ.BCST.Z 1016(AX), Z1, K3, Z2"}, + {name: "subtract unsigned word z memory zeroing", code: []byte{0x62, 0xf1, 0x75, 0xcb, 0xd9, 0x50, 0x7f}, want: "VPSUBUSW.Z 8128(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(test.code, 64) + if !ok { + t.Fatal("VPADD encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPADDD(t *testing.T) { + code := []byte{0x62, 0x51, 0x45, 0x48, 0xfe, 0xc4} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPADDD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPADDD" || !strings.HasPrefix(decoded[0].Raw, "VPADDD Z12, Z7, Z8 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpermd_test.go b/amd64_raw_vpermd_test.go new file mode 100644 index 000000000..fd64e92c0 --- /dev/null +++ b/amd64_raw_vpermd_test.go @@ -0,0 +1,303 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawEVEXVariableDwordPermute(opcode, vectorBits, mask int, zeroing, broadcast, memory bool) []byte { + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // data register 3, destination register 2. + if memory { + modRM = 0x50 // disp8(AX) data, destination register 2. + } + code := []byte{0x62, 0xf2, 0x75, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodedX86VariableDwordPermuteInstructionCompleteVEXFamily(t *testing.T) { + for _, test := range []struct { + opcode byte + op Op + }{ + {0x36, "VPERMD"}, + {0x16, "VPERMPS"}, + } { + t.Run(string(test.op), func(t *testing.T) { + code := []byte{0xc4, 0xe2, 0x25, test.opcode, 0xd6} + instruction, length, ok, err := decodedX86VariableDwordPermuteInstruction(code, 64) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(code) { + t.Fatalf("length = %d, want %d", length, len(code)) + } + want := string(test.op) + " Y6, Y11, Y2" + if instruction.Op != test.op || instruction.Raw != want { + t.Fatalf("instruction = %#v, want %q", instruction, want) + } + }) + } + + for _, test := range []struct { + name string + code []byte + mode int + want string + }{ + { + name: "extended_registers", + code: []byte{0xc4, 0x42, 0x05, 0x36, 0xdb}, + mode: 64, + want: "VPERMD Y11, Y15, Y11", + }, + { + name: "memory_sib_disp8", + code: []byte{0xc4, 0x62, 0x15, 0x36, 0x5c, 0x88, 0x20}, + mode: 64, + want: "VPERMD 32(AX)(CX*4), Y13, Y11", + }, + { + name: "memory_negative_disp8", + code: []byte{0xc4, 0x62, 0x65, 0x16, 0x64, 0x35, 0xef}, + mode: 64, + want: "VPERMPS -17(BP)(SI*1), Y3, Y12", + }, + { + name: "386_registers", + code: []byte{0xc4, 0xe2, 0x65, 0x36, 0xe5}, + mode: 32, + want: "VPERMD Y5, Y3, Y4", + }, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VariableDwordPermuteInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode = (%#v, %d, %v), want %q", instruction, length, ok, test.want) + } + }) + } +} + +func TestDecodeX86RawEVEXVPERMDSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VPERMD Z2, Z0, Z3. + code := []byte{0x62, 0xf2, 0x7d, 0x48, 0x36, 0xda, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "cumMinInt32 AVX512 VPERMD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPERMD" || + decoded[0].Args[0].Reg != "Z2" || decoded[0].Args[1].Reg != "Z0" || + decoded[0].Args[2].Reg != "Z3" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawEVEXVariableDwordPermuteCompleteGoForms(t *testing.T) { + for _, family := range []struct { + opcode int + op Op + }{ + {0x36, "VPERMD"}, + {0x16, "VPERMPS"}, + } { + for vectorIndex, vector := range []string{"Y", "Z"} { + vectorBits := vectorIndex + 1 + for _, variant := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "memory", memory: true}, + {name: "zeroing memory", mask: 7, zeroing: true, memory: true}, + {name: "broadcast memory", broadcast: true, memory: true}, + {name: "broadcast zeroing memory", mask: 1, zeroing: true, broadcast: true, memory: true}, + } { + name := fmt.Sprintf("%s/%s/%s", family.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawEVEXVariableDwordPermute(family.opcode, vectorBits, variant.mask, variant.zeroing, variant.broadcast, variant.memory) + got, length, ok, err := decodedX86EVEXVariableDwordPermuteInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := family.op + if variant.broadcast { + wantOp += ".BCST" + } + if variant.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[1].String() != vector+"1" || + got.Args[len(got.Args)-1].String() != vector+"2" || + !variant.memory && got.Args[0].String() != vector+"3" || + len(got.Args) != 3+map[bool]int{true: 1}[variant.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXVariableDwordPermuteRIPLiteral(t *testing.T) { + for _, family := range []struct { + opcode int + op Op + }{ + {0x36, "VPERMD"}, + {0x16, "VPERMPS"}, + } { + for vectorBits := 1; vectorBits <= 2; vectorBits++ { + for _, broadcast := range []bool{false, true} { + code := encodeX86RawEVEXVariableDwordPermute(family.opcode, vectorBits, 1, false, broadcast, false) + code[5] = 0x15 // RIP-relative first source. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << vectorBits + if broadcast { + literalWidth = 4 + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/width%d/broadcast%t", family.op, vectorBits, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != literalWidth { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXVariableDwordPermuteRejectsInvalidForms(t *testing.T) { + base := encodeX86RawEVEXVariableDwordPermute(0x36, 2, 0, false, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[2] |= 0x80 }, // Go's EVEX.W is zero + func(code []byte) { code[3] &^= 0x60 }, // no X-width EVEX form + func(code []byte) { code[3] |= 0x80 }, // zeroing without mask + func(code []byte) { code[3] |= 0x10 }, // broadcast requires memory + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86EVEXVariableDwordPermuteInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + if _, _, ok, err := decodedX86EVEXVariableDwordPermuteInstruction( + encodeX86RawEVEXVariableDwordPermute(0x16, 2, 1, false, false, false), 32, + ); !ok || err == nil { + t.Fatalf("accepted 386 masked form: ok=%v err=%v", ok, err) + } +} + +func TestTranslateX86RawEVEXVariableDwordPermuteLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawVariableDwordPermute(SB),$0-0\n") + for _, opcode := range []int{0x36, 0x16} { + for _, vectorBits := range []int{1, 2} { + mask := 1 + if target.goarch == "386" { + mask = 0 + } + for _, memory := range []bool{false, true} { + code := encodeX86RawEVEXVariableDwordPermute(opcode, vectorBits, mask, false, false, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + ir := x86UnboundVoidRawScalarFormatObject(t, source.String(), target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "raw-vpermd.ll", "raw-vpermd.o", ir) + }) + } +} + +func TestDecodedX86VariableDwordPermuteInstructionRejectsInvalidVEXForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc4, 0xe2, 0x25, 0x36}, mode: 64}, + {name: "vex_l_zero", code: []byte{0xc4, 0xe2, 0x21, 0x36, 0xd6}, mode: 64}, + {name: "vex_pp_zero", code: []byte{0xc4, 0xe2, 0x24, 0x36, 0xd6}, mode: 64}, + {name: "vex_w_one", code: []byte{0xc4, 0xe2, 0xa5, 0x36, 0xd6}, mode: 64}, + {name: "address_override", code: []byte{0x67, 0xc4, 0xe2, 0x25, 0x36, 0xd6}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x42, 0x05, 0x36, 0xdb}, mode: 32}, + {name: "rip_relative", code: []byte{0xc4, 0xe2, 0x25, 0x36, 0x05, 0, 0, 0, 0}, mode: 64}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VariableDwordPermuteInstruction(test.code, test.mode) + if !ok { + t.Fatal("VPERMD/VPERMPS encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVPERMDReportedSequence(t *testing.T) { + code := []byte{ + 0xc4, 0xe2, 0x25, 0x36, 0xd6, + 0xc4, 0xe2, 0x7d, 0x36, 0xd6, + 0xc4, 0xe2, 0x35, 0x36, 0xd6, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "go-pherence VEX VPERMD sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 3 { + t.Fatalf("decoded %d instructions, want 3: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VPERMD" { + t.Fatalf("decoded op = %s, want VPERMD", instruction.Op) + } + } +} diff --git a/amd64_raw_vpermi2_test.go b/amd64_raw_vpermi2_test.go new file mode 100644 index 000000000..53087be30 --- /dev/null +++ b/amd64_raw_vpermi2_test.go @@ -0,0 +1,51 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86VPERMI2CompleteGo127Family(t *testing.T) { + // Go 1.27 shares _yvblendmpd across VPERMI2B/W/D/Q/PS/PD. These + // LLVM-verified bytes cover all six opcodes, X/Y/Z widths, high registers, + // memory, masking, zeroing, broadcast, and compressed disp8 scaling. + tests := []struct { + name string + code []byte + want string + }{ + {name: "b x high masked", code: []byte{0x62, 0xa2, 0x55, 0x01, 0x75, 0xf4}, want: "VPERMI2B X20, X21, K1, X22"}, + {name: "w y memory zeroing", code: []byte{0x62, 0xd2, 0xf5, 0xaa, 0x75, 0x50, 0x03}, want: "VPERMI2W.Z 96(R8), Y1, K2, Y2"}, + {name: "d z high", code: []byte{0x62, 0x32, 0x7d, 0x40, 0x76, 0xc9}, want: "VPERMI2D Z17, Z16, Z9"}, + {name: "d z broadcast zeroing", code: []byte{0x62, 0xf2, 0x75, 0xdb, 0x76, 0x50, 0x7f}, want: "VPERMI2D.BCST.Z 508(AX), Z1, K3, Z2"}, + {name: "q reported minio", code: []byte{0x62, 0x22, 0xb5, 0x48, 0x76, 0xf2}, want: "VPERMI2Q Z18, Z9, Z30"}, + {name: "q y high memory masked", code: []byte{0x62, 0x42, 0xbd, 0x24, 0x76, 0x4c, 0x24, 0x02}, want: "VPERMI2Q 64(R12), Y24, K4, Y25"}, + {name: "ps x high masked", code: []byte{0x62, 0xa2, 0x55, 0x01, 0x77, 0xf4}, want: "VPERMI2PS X20, X21, K1, X22"}, + {name: "pd z broadcast zeroing", code: []byte{0x62, 0xf2, 0xf5, 0xdb, 0x77, 0x50, 0x7f}, want: "VPERMI2PD.BCST.Z 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86IndexedPermuteInstruction(test.code, 64) + if !ok { + t.Fatal("VPERMI2 encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPERMI2Q(t *testing.T) { + code := []byte{0x62, 0x22, 0xb5, 0x48, 0x76, 0xf2} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPERMI2Q", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPERMI2Q" || !strings.HasPrefix(decoded[0].Raw, "VPERMI2Q Z18, Z9, Z30 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpermq_test.go b/amd64_raw_vpermq_test.go new file mode 100644 index 000000000..61ca796fc --- /dev/null +++ b/amd64_raw_vpermq_test.go @@ -0,0 +1,79 @@ +package plan9asm + +import "testing" + +func TestDecodedX86VPERMQInstructionCompleteVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + want string + }{ + {name: "register", code: []byte{0xc4, 0xe3, 0xfd, 0x00, 0xc9, 0x39}, mode: 64, want: "VPERMQ $57, Y1, Y1"}, + {name: "extended_registers", code: []byte{0xc4, 0x43, 0xfd, 0x00, 0xca, 0xff}, mode: 64, want: "VPERMQ $255, Y10, Y9"}, + {name: "memory_disp8", code: []byte{0xc4, 0xe3, 0xfd, 0x00, 0x4e, 0x20, 0x01}, mode: 64, want: "VPERMQ $1, 32(SI), Y1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VPERMQInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(test.code) { + t.Fatalf("length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("instruction = %q, want %q", instruction.Raw, test.want) + } + }) + } +} + +func TestDecodedX86VPERMQInstructionRejectsInvalidVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc4, 0xe3, 0xfd, 0x00}, mode: 64}, + {name: "truncated_immediate", code: []byte{0xc4, 0xe3, 0xfd, 0x00, 0xc9}, mode: 64}, + {name: "vex_l_zero", code: []byte{0xc4, 0xe3, 0xf9, 0x00, 0xc9, 0x39}, mode: 64}, + {name: "used_vvvv", code: []byte{0xc4, 0xe3, 0xed, 0x00, 0xc9, 0x39}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x43, 0xfd, 0x00, 0xca, 0xff}, mode: 32}, + {name: "rip_relative", code: []byte{0xc4, 0xe3, 0xfd, 0x00, 0x0d, 0, 0, 0, 0, 0x39}, mode: 64}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VPERMQInstruction(test.code, test.mode) + if !ok { + t.Fatal("VPERMQ encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVPERMQSequence(t *testing.T) { + code := []byte{ + 0xc4, 0xe3, 0xfd, 0x00, 0xc9, 0x39, + 0xc4, 0xe3, 0xfd, 0x00, 0xd2, 0x4e, + 0xc4, 0xe3, 0xfd, 0x00, 0xdb, 0x93, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VPERMQ raw sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 3 { + t.Fatalf("decoded %d instructions, want 3: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VPERMQ" { + t.Fatalf("decoded op = %s, want VPERMQ", instruction.Op) + } + } +} diff --git a/amd64_raw_vpinsr_evex.go b/amd64_raw_vpinsr_evex.go new file mode 100644 index 000000000..1a938d5e4 --- /dev/null +++ b/amd64_raw_vpinsr_evex.go @@ -0,0 +1,66 @@ +package plan9asm + +import "fmt" + +// Go 1.27's _yvpinsrb table has one EVEX.128 row for each scalar width. +// VEX forms use decodedX86VPINSRInstruction; the EVEX row extends X +// base/destination registers to X31 and scales disp8 by the scalar width. +func decodedX86EVEXVPINSRInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86EVEXVPINSRInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXVPINSRInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.pp != 1 { + return Instr{}, 0, false, nil + } + op := Op("") + laneBytes := 0 + switch { + case p.mapNumber == 3 && p.opcode == 0x20 && !p.w: + op, laneBytes = "VPINSRB", 1 + case p.mapNumber == 1 && p.opcode == 0xc4 && !p.w: + op, laneBytes = "VPINSRW", 2 + case p.mapNumber == 3 && p.opcode == 0x22 && !p.w: + op, laneBytes = "VPINSRD", 4 + case p.mapNumber == 3 && p.opcode == 0x22 && p.w: + op, laneBytes = "VPINSRQ", 8 + default: + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("raw EVEX %s: %s", op, message) + } + if mode != 64 { + return fail("Go assembler exposes this EVEX form only on amd64") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.vectorLength != 0 || p.mask != 0 || p.zero || p.broadcast { + return fail("invalid EVEX fixed, length, mask, zeroing or broadcast bits") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + source, consumed, err := memory.scalar( + code[p.modRM:], mode, p.b, p.x, p.segment, laneBytes, + ) + if err != nil { + return Instr{}, 0, true, err + } + immediateIndex := p.modRM + consumed + if len(code) <= immediateIndex { + return fail("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + base := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", int(modRM>>3&7)+p.r*8))} + args := []Operand{immediate, source, base, destination} + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), source.String(), base.String(), destination.String()), + }, immediateIndex + 1, true, nil +} diff --git a/amd64_raw_vpinsr_evex_test.go b/amd64_raw_vpinsr_evex_test.go new file mode 100644 index 000000000..fd2e4b37d --- /dev/null +++ b/amd64_raw_vpinsr_evex_test.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +var x86RawEVEXVPINSRForms = []struct { + op Op + mapNumber int + opcode byte + width64 bool + laneBytes int +}{ + {"VPINSRB", 3, 0x20, false, 1}, + {"VPINSRW", 1, 0xc4, false, 2}, + {"VPINSRD", 3, 0x22, false, 4}, + {"VPINSRQ", 3, 0x22, true, 8}, +} + +func x86RawEVEXVPINSRGoForms() string { + var source strings.Builder + source.WriteString("TEXT rawVPINSR(SB),4,$0-0\n") + for _, form := range x86RawEVEXVPINSRForms { + fmt.Fprintf(&source, "\t%s $5, AX, X20, X21\n", form.op) + fmt.Fprintf(&source, "\t%s $255, %d(BX), X20, X21\n", form.op, form.laneBytes) + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestDecodeX86RawEVEXVPINSRSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VPINSRB $5, 1(SI)(R9*1), X6, X16. + code := []byte{0x62, 0xa3, 0x4d, 0x08, 0x20, 0x44, 0x0e, 0x01, 0x05, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "bitshuffleU8 AVX512 VPINSRB", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPINSRB" || + decoded[0].Args[0].Imm != 5 || decoded[0].Args[2].Reg != "X6" || + decoded[0].Args[3].Reg != "X16" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawEVEXVPINSRCompleteGoForms(t *testing.T) { + if len(x86RawEVEXVPINSRForms) != 4 { + t.Fatalf("EVEX VPINSR grammar has %d entries, want four", len(x86RawEVEXVPINSRForms)) + } + source := x86RawEVEXVPINSRGoForms() + code := assembleX87ControlBytes(t, "amd64", source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d=%+v, want %+v", index, got, want[index]) + } + } +} + +func TestDecodeX86RawEVEXVPINSRRejectsInvalidForms(t *testing.T) { + base := []byte{0x62, 0xa3, 0x4d, 0x08, 0x20, 0x44, 0x0e, 0x01, 0x05} + for _, variant := range []struct { + name string + mutate func([]byte) []byte + }{ + {"mask", func(code []byte) []byte { code[3] |= 1; return code }}, + {"zeroing", func(code []byte) []byte { code[3] |= 0x80; return code }}, + {"broadcast", func(code []byte) []byte { code[3] |= 0x10; return code }}, + {"vector length", func(code []byte) []byte { code[3] |= 0x20; return code }}, + {"address override", func(code []byte) []byte { return append([]byte{0x67}, code...) }}, + {"truncated immediate", func(code []byte) []byte { return code[:len(code)-1] }}, + } { + t.Run(variant.name, func(t *testing.T) { + code := variant.mutate(append([]byte(nil), base...)) + if _, _, matched, err := decodedX86EVEXVPINSRInstruction(code, 64); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", code, matched, err) + } + }) + } + if _, _, matched, err := decodedX86EVEXVPINSRInstruction(base, 32); !matched || err == nil { + t.Fatalf("accepted 386 EVEX VPINSR: matched=%v err=%v", matched, err) + } +} + +func TestTranslateX86RawEVEXVPINSRLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + source := x86RawEVEXVPINSRGoForms() + code := assembleX87ControlBytes(t, "amd64", source) + file := x86RawUnprovedReturnFormProbeFile(t, "amd64", "rawVPINSR", code) + // The original rows omit their GP/vector/address inputs. Their + // public Context and independent physical objects are separate gates. + ir := x86UnboundVoidRawProbeObject(t, file, "amd64", triple) + compileLLVMToObject(t, llc, triple, "raw-vpinsr.ll", "raw-vpinsr.o", ir) + }) + } +} diff --git a/amd64_raw_vpinsr_test.go b/amd64_raw_vpinsr_test.go new file mode 100644 index 000000000..56b13e983 --- /dev/null +++ b/amd64_raw_vpinsr_test.go @@ -0,0 +1,74 @@ +package plan9asm + +import "testing" + +func TestDecodedX86VPINSRInstructionCompleteVEXFamily(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + want string + }{ + {name: "byte_register", code: []byte{0xc4, 0xe3, 0x79, 0x20, 0xc1, 0x02}, mode: 64, want: "VPINSRB $2, CX, X0, X0"}, + {name: "word_two_byte_vex", code: []byte{0xc5, 0xf9, 0xc4, 0xc1, 0x02}, mode: 64, want: "VPINSRW $2, CX, X0, X0"}, + {name: "dword_register", code: []byte{0xc4, 0xe3, 0x79, 0x22, 0xc1, 0x02}, mode: 64, want: "VPINSRD $2, CX, X0, X0"}, + {name: "qword_register", code: []byte{0xc4, 0xe3, 0xf9, 0x22, 0xc1, 0x02}, mode: 64, want: "VPINSRQ $2, CX, X0, X0"}, + {name: "qword_memory_disp8", code: []byte{0xc4, 0x63, 0xa1, 0x22, 0x5e, 0x28, 0x01}, mode: 64, want: "VPINSRQ $1, 40(SI), X11, X11"}, + {name: "extended_registers", code: []byte{0xc4, 0x43, 0x31, 0x22, 0xca, 0x01}, mode: 64, want: "VPINSRD $1, R10, X9, X9"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VPINSRInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(test.code) { + t.Fatalf("length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("instruction = %q, want %q", instruction.Raw, test.want) + } + }) + } +} + +func TestDecodedX86VPINSRInstructionRejectsInvalidVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc4, 0xe3, 0xf9, 0x22}, mode: 64}, + {name: "truncated_immediate", code: []byte{0xc4, 0xe3, 0xf9, 0x22, 0xc1}, mode: 64}, + {name: "vex_l_one", code: []byte{0xc4, 0xe3, 0xfd, 0x22, 0xc1, 0x02}, mode: 64}, + {name: "vex_pp_zero", code: []byte{0xc4, 0xe3, 0xf8, 0x22, 0xc1, 0x02}, mode: 64}, + {name: "qword_in_386", code: []byte{0xc4, 0xe3, 0xf9, 0x22, 0xc1, 0x02}, mode: 32}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x43, 0x31, 0x22, 0xca, 0x01}, mode: 32}, + {name: "rip_relative", code: []byte{0xc4, 0xe3, 0xf9, 0x22, 0x05, 0, 0, 0, 0, 0x02}, mode: 64}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VPINSRInstruction(test.code, test.mode) + if !ok { + t.Fatal("VPINSR encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVPINSRQSequence(t *testing.T) { + code := []byte{0xc4, 0x63, 0xa1, 0x22, 0x5e, 0x28, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VPINSRQ raw sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPINSRQ" { + t.Fatalf("decoded = %#v, want one VPINSRQ", decoded) + } +} diff --git a/amd64_raw_vpror_test.go b/amd64_raw_vpror_test.go new file mode 100644 index 000000000..ec8393ff6 --- /dev/null +++ b/amd64_raw_vpror_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86PackedImmediateRotateCompleteGo127Family(t *testing.T) { + // Go 1.27's _yvprold table covers the four EVEX immediate rotate + // instructions across X/Y/Z, memory, masks, zeroing, broadcast, high + // registers, and compressed disp8. LLVM 22 verified these bytes. + tests := []struct { + name string + code []byte + want string + }{ + {name: "left dword x high masked", code: []byte{0x62, 0xb1, 0x4d, 0x01, 0x72, 0xcc, 0x06}, want: "VPROLD $6, X20, K1, X22"}, + {name: "left qword y memory zeroing", code: []byte{0x62, 0xd1, 0xed, 0xaa, 0x72, 0x48, 0x03, 0x07}, want: "VPROLQ.Z $7, 96(R8), K2, Y2"}, + {name: "right dword reported minio", code: []byte{0x62, 0xf1, 0x2d, 0x48, 0x72, 0xc4, 0x06}, want: "VPRORD $6, Z4, Z10"}, + {name: "right dword z broadcast high masked", code: []byte{0x62, 0xd1, 0x35, 0x5c, 0x72, 0x44, 0x24, 0x7f, 0x44}, want: "VPRORD.BCST $68, 508(R12), K4, Z9"}, + {name: "right qword z broadcast zeroing", code: []byte{0x62, 0xf1, 0xed, 0xdb, 0x72, 0x40, 0x7f, 0x09}, want: "VPRORQ.BCST.Z $9, 1016(AX), K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86PackedImmediateRotateInstruction(test.code, 64) + if !ok { + t.Fatal("packed immediate rotate encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPRORD(t *testing.T) { + code := []byte{0x62, 0xf1, 0x2d, 0x48, 0x72, 0xc4, 0x06} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPRORD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPRORD" || !strings.HasPrefix(decoded[0].Raw, "VPRORD $6, Z4, Z10 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpshift_test.go b/amd64_raw_vpshift_test.go new file mode 100644 index 000000000..b4fd57876 --- /dev/null +++ b/amd64_raw_vpshift_test.go @@ -0,0 +1,142 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodedX86PackedLogicalShiftCompleteGo127Family(t *testing.T) { + // Go 1.27's _yvpslld table covers six logical shifts with both immediate + // and XMM/m128 uniform-count encodings. These LLVM 22 bytes also cover + // VEX/EVEX, every width, masks, zeroing, broadcast, high registers, and + // compressed disp8. + tests := []struct { + name string + code []byte + want string + }{ + {name: "vex immediate right dword", code: []byte{0xc5, 0xb5, 0x72, 0xd2, 0x07}, want: "VPSRLD $7, Y2, Y9"}, + {name: "vex count left qword memory", code: []byte{0xc4, 0x41, 0x6d, 0xf3, 0x4c, 0x24, 0x20}, want: "VPSLLQ 32(R12), Y2, Y9"}, + {name: "immediate left word", code: []byte{0x62, 0xb1, 0x4d, 0x01, 0x71, 0xf4, 0x06}, want: "VPSLLW $6, X20, K1, X22"}, + {name: "immediate left dword", code: []byte{0x62, 0xd1, 0x6d, 0xaa, 0x72, 0x70, 0x03, 0x07}, want: "VPSLLD.Z $7, 96(R8), K2, Y2"}, + {name: "immediate left qword", code: []byte{0x62, 0xb1, 0xb5, 0x48, 0x73, 0xf1, 0x44}, want: "VPSLLQ $68, Z17, Z9"}, + {name: "immediate right word", code: []byte{0x62, 0xd1, 0x35, 0x24, 0x71, 0x54, 0x24, 0x02, 0x0a}, want: "VPSRLW $10, 64(R12), K4, Y25"}, + {name: "immediate right dword reported minio", code: []byte{0x62, 0x91, 0x05, 0x48, 0x72, 0xd6, 0x0a}, want: "VPSRLD $10, Z30, Z15"}, + {name: "immediate right qword broadcast", code: []byte{0x62, 0xf1, 0xed, 0xdb, 0x73, 0x50, 0x7f, 0x09}, want: "VPSRLQ.BCST.Z $9, 1016(AX), K3, Z2"}, + {name: "count left word", code: []byte{0x62, 0xa1, 0x55, 0x01, 0xf1, 0xf4}, want: "VPSLLW X20, X21, K1, X22"}, + {name: "count left dword memory", code: []byte{0x62, 0xd1, 0x75, 0xaa, 0xf2, 0x50, 0x03}, want: "VPSLLD.Z 48(R8), Y1, K2, Y2"}, + {name: "count left qword", code: []byte{0x62, 0x31, 0xfd, 0x40, 0xf3, 0xc9}, want: "VPSLLQ X17, Z16, Z9"}, + {name: "count right word memory", code: []byte{0x62, 0x41, 0x3d, 0x04, 0xd1, 0x4c, 0x24, 0x02}, want: "VPSRLW 32(R12), X24, K4, X25"}, + {name: "count right dword", code: []byte{0x62, 0xa1, 0x55, 0x21, 0xd2, 0xf4}, want: "VPSRLD X20, Y21, K1, Y22"}, + {name: "count right qword memory", code: []byte{0x62, 0xf1, 0xf5, 0xcb, 0xd3, 0x50, 0x7f}, want: "VPSRLQ.Z 2032(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86PackedUniformShiftInstruction(test.code, 64) + if !ok { + t.Fatal("packed logical shift encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodedX86PackedArithmeticShiftStreamVByteRegression(t *testing.T) { + code := []byte{0xc5, 0xfd, 0x72, 0xe0, 0x1f} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "streamvbyte VPSRAD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPSRAD" || !strings.HasPrefix(decoded[0].Raw, "VPSRAD $31, Y0, Y0 ") { + t.Fatalf("decoded streamvbyte VPSRAD as %#v", decoded) + } +} + +func TestDecodedX86PackedArithmeticShiftCompleteGo127Family(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + {"VEX immediate word", []byte{0xc5, 0xe9, 0x71, 0xe1, 0x03}, "VPSRAW $3, X1, X2"}, + {"VEX immediate dword", []byte{0xc5, 0xed, 0x72, 0xe1, 0x07}, "VPSRAD $7, Y1, Y2"}, + {"EVEX immediate qword", []byte{0x62, 0xf1, 0xed, 0x48, 0x72, 0xe1, 0x07}, "VPSRAQ $7, Z1, Z2"}, + {"VEX packed count word", []byte{0xc5, 0xed, 0xe1, 0xd9}, "VPSRAW X1, Y2, Y3"}, + {"EVEX packed count dword", []byte{0x62, 0xf1, 0x6d, 0x48, 0xe2, 0xd9}, "VPSRAD X1, Z2, Z3"}, + {"EVEX packed count qword", []byte{0x62, 0xf1, 0xed, 0x48, 0xe2, 0xd9}, "VPSRAQ X1, Z2, Z3"}, + {"EVEX dword broadcast zeroing", []byte{0x62, 0xf1, 0x6d, 0xbb, 0x72, 0x20, 0x07}, "VPSRAD.BCST.Z $7, 0(AX), K3, Y2"}, + {"EVEX qword broadcast high register", []byte{0x62, 0xf1, 0xed, 0xd3, 0x72, 0x60, 0x01, 0x07}, "VPSRAQ.BCST.Z $7, 8(AX), K3, Z18"}, + {"EVEX qword high register packed count", []byte{0x62, 0xa1, 0xd5, 0xc1, 0xe2, 0xf4}, "VPSRAQ.Z X20, Z21, K1, Z22"}, + {"EVEX word memory zeroing", []byte{0x62, 0xd1, 0x35, 0xaa, 0x71, 0x60, 0x01, 0x03}, "VPSRAW.Z $3, 32(R8), K2, Y9"}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, recognized, err := decodedX86PackedUniformShiftInstruction(test.code, 64) + if err != nil || !recognized || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode %x: raw=%q length=%d recognized=%v err=%v; want %q", test.code, instruction.Raw, length, recognized, err, test.want) + } + }) + } +} + +func TestTranslateRawPackedArithmeticShiftLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + encodings := [][]byte{ + {0xc5, 0xe9, 0x71, 0xe1, 0x03}, + {0xc5, 0xed, 0x72, 0xe1, 0x07}, + {0x62, 0xf1, 0xed, 0x48, 0x72, 0xe1, 0x07}, + {0xc5, 0xed, 0xe1, 0xd9}, + {0x62, 0xf1, 0x6d, 0x48, 0xe2, 0xd9}, + {0x62, 0xf1, 0xed, 0x48, 0xe2, 0xd9}, + } + encodings = append(encodings, []byte{0x62, 0xf1, 0x6d, 0xbb, 0x72, 0x20, 0x07}) + var source strings.Builder + source.WriteString("TEXT packedArithmeticShiftRaw(SB), $0-0\n") + for _, code := range encodings { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir := x86PackedShiftRawFormatObject(t, file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"packedArithmeticShiftRaw": {Name: "packedArithmeticShiftRaw", Ret: Void}}, + }) + compileLLVMToObject(t, llc, target.triple, "raw-arithmetic-shift.ll", "raw-arithmetic-shift.o", ir) + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPSRLD(t *testing.T) { + code := []byte{0x62, 0x91, 0x05, 0x48, 0x72, 0xd6, 0x0a} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPSRLD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPSRLD" || !strings.HasPrefix(decoded[0].Raw, "VPSRLD $10, Z30, Z15 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpshufb_literal_test.go b/amd64_raw_vpshufb_literal_test.go new file mode 100644 index 000000000..ab6713b2c --- /dev/null +++ b/amd64_raw_vpshufb_literal_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawVPSHUFBLiteral(evex bool, width, mask int, zeroing bool) []byte { + code := encodeRawScalarMove(evex, 1, 0, 0, 0, 0, mask, zeroing) + code[1] = code[1]&0xf0 | 2 + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*23+7)) + } + return code +} + +func TestDecodeX86RawVPSHUFBRIPDataCompleteFamily(t *testing.T) { + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + masks := []int{0} + if evex { + masks = append(masks, 7) + } + for _, mask := range masks { + name := fmt.Sprintf("evex%t/%d/mask%d", evex, width, mask) + t.Run(name, func(t *testing.T) { + code := x86RawVPSHUFBLiteral(evex, width, mask, mask != 0) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op("VPSHUFB") + if mask != 0 { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } +} + +func TestDecodeX86RawVPSHUFBRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawVPSHUFBLiteral(true, 64, 7, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x65}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVPSHUFBRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, form := range []struct { + evex bool + width int + mask int + }{ + {false, 16, 0}, {false, 32, 0}, + {true, 16, 0}, {true, 32, 7}, {true, 64, 7}, + } { + code := x86RawVPSHUFBLiteral(form.evex, form.width, form.mask, form.mask != 0) + name := fmt.Sprintf("vpshufbLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + pools = append(pools, code[len(code)-form.width:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + again, err := normalizeX86RawFile(normalized, "amd64") + if err != nil { + t.Fatal(err) + } + if len(again.Data) != len(pools) { + t.Fatalf("second normalization materialized %d constants, want %d", len(again.Data), len(pools)) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "<16 x i32> ") { + t.Fatal("missing lane-local and high-bit-zero byte shuffle") + } + compileLLVMToObject(t, llc, triple, "vpshufb-literal.ll", "vpshufb-literal.o", ir) + }) + } +} diff --git a/amd64_raw_vpshufb_test.go b/amd64_raw_vpshufb_test.go new file mode 100644 index 000000000..fb8282dda --- /dev/null +++ b/amd64_raw_vpshufb_test.go @@ -0,0 +1,50 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86VPSHUFBCompleteGo127Family(t *testing.T) { + // Go 1.27's _yvandnpd table gives VPSHUFB VEX X/Y forms and EVEX + // X/Y/Z forms, including high registers, memory, K masking, zeroing, + // and EVEX compressed disp8 scaling. LLVM 22 verified these bytes. + tests := []struct { + name string + code []byte + want string + }{ + {name: "vex x high destination", code: []byte{0xc4, 0x62, 0x79, 0x00, 0xc1}, want: "VPSHUFB X1, X0, X8"}, + {name: "vex y high memory", code: []byte{0xc4, 0x42, 0x6d, 0x00, 0x4c, 0x24, 0x20}, want: "VPSHUFB 32(R12), Y2, Y9"}, + {name: "evex x high masked", code: []byte{0x62, 0xa2, 0x55, 0x01, 0x00, 0xf4}, want: "VPSHUFB X20, X21, K1, X22"}, + {name: "evex y memory zeroing", code: []byte{0x62, 0xd2, 0x75, 0xaa, 0x00, 0x50, 0x03}, want: "VPSHUFB.Z 96(R8), Y1, K2, Y2"}, + {name: "evex z high", code: []byte{0x62, 0x32, 0x7d, 0x40, 0x00, 0xc9}, want: "VPSHUFB Z17, Z16, Z9"}, + {name: "evex z compressed memory zeroing", code: []byte{0x62, 0xf2, 0x75, 0xcb, 0x00, 0x50, 0x7f}, want: "VPSHUFB.Z 8128(AX), Z1, K3, Z2"}, + {name: "reported minio", code: []byte{0x62, 0xc2, 0x7d, 0x40, 0x00, 0xc3}, want: "VPSHUFB Z11, Z16, Z16"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VPSHUFBInstruction(test.code, 64) + if !ok { + t.Fatal("VPSHUFB encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPSHUFB(t *testing.T) { + code := []byte{0x62, 0xc2, 0x7d, 0x40, 0x00, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPSHUFB", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPSHUFB" || !strings.HasPrefix(decoded[0].Raw, "VPSHUFB Z11, Z16, Z16 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpternlog_test.go b/amd64_raw_vpternlog_test.go new file mode 100644 index 000000000..99149643e --- /dev/null +++ b/amd64_raw_vpternlog_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86RawImmediateThreeVectorSourceLocalRIPData(t *testing.T) { + for _, family := range []struct { + opcode byte + name string + }{{0x25, "VPTERNLOG"}, {0x03, "VALIGN"}} { + for _, qword := range []bool{false, true} { + for vectorLength := 0; vectorLength < 3; vectorLength++ { + for _, broadcast := range []bool{false, true} { + p1 := byte(0x7d) + laneWidth := 4 + suffix := "D" + if qword { + p1 |= 0x80 + laneWidth = 8 + suffix = "Q" + } + p2 := byte(0x08 | vectorLength<<5) + width := 16 << vectorLength + if broadcast { + p2 |= 0x10 + width = laneWidth + } + code := []byte{0x62, 0xf3, p1, p2, family.opcode, 0x05, 1, 0, 0, 0, 0xca, 0xc3} + for value := 0; value < width; value++ { + code = append(code, byte(value)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "immediate three-vector RIP", map[string]bool{}) + if err != nil { + t.Fatalf("%s%s width %d broadcast=%t: %v", family.name, suffix, width, broadcast, err) + } + if len(decoded) != 2 || len(decoded[0].x86RIPLiteralData) != width { + t.Fatalf("%s%s width %d broadcast=%t: %#v", family.name, suffix, width, broadcast, decoded) + } + } + } + } + } +} + +func TestDecodedX86TernaryLogicCompleteGo127Family(t *testing.T) { + // Go 1.27 shares _yvalignd between VPTERNLOGD/Q. These LLVM 22 bytes + // cover X/Y/Z, high registers, memory, masks, zeroing, broadcast, and + // compressed disp8 for both lane widths. + tests := []struct { + name string + code []byte + want string + }{ + {name: "dword x high masked", code: []byte{0x62, 0xa3, 0x55, 0x01, 0x25, 0xf4, 0xca}, want: "VPTERNLOGD $202, X20, X21, K1, X22"}, + {name: "qword y memory zeroing", code: []byte{0x62, 0xd3, 0xf5, 0xaa, 0x25, 0x50, 0x03, 0x96}, want: "VPTERNLOGQ.Z $150, 96(R8), Y1, K2, Y2"}, + {name: "dword reported minio", code: []byte{0x62, 0x73, 0x55, 0x48, 0x25, 0xce, 0xca}, want: "VPTERNLOGD $202, Z6, Z5, Z9"}, + {name: "dword z broadcast high masked", code: []byte{0x62, 0x53, 0x7d, 0x54, 0x25, 0x4c, 0x24, 0x7f, 0x44}, want: "VPTERNLOGD.BCST $68, 508(R12), Z16, K4, Z9"}, + {name: "qword z broadcast zeroing", code: []byte{0x62, 0xf3, 0xf5, 0xdb, 0x25, 0x50, 0x7f, 0x09}, want: "VPTERNLOGQ.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86TernaryLogicInstruction(test.code, 64) + if !ok { + t.Fatal("ternary-logic encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPTERNLOGD(t *testing.T) { + code := []byte{0x62, 0x73, 0x55, 0x48, 0x25, 0xce, 0xca} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPTERNLOGD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPTERNLOGD" || !strings.HasPrefix(decoded[0].Raw, "VPTERNLOGD $202, Z6, Z5, Z9 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpunpck_test.go b/amd64_raw_vpunpck_test.go new file mode 100644 index 000000000..166dffb4d --- /dev/null +++ b/amd64_raw_vpunpck_test.go @@ -0,0 +1,103 @@ +package plan9asm + +import "testing" + +func TestDecodedX86VPUNPCKInstructionCompleteVEXFamily(t *testing.T) { + for _, test := range []struct { + opcode byte + op Op + }{ + {0x60, "VPUNPCKLBW"}, + {0x61, "VPUNPCKLWD"}, + {0x62, "VPUNPCKLDQ"}, + {0x6c, "VPUNPCKLQDQ"}, + {0x68, "VPUNPCKHBW"}, + {0x69, "VPUNPCKHWD"}, + {0x6a, "VPUNPCKHDQ"}, + {0x6d, "VPUNPCKHQDQ"}, + } { + t.Run(string(test.op), func(t *testing.T) { + code := []byte{0xc5, 0xe9, test.opcode, 0xcb} + instruction, length, ok, err := decodedX86VPUNPCKInstruction(code, 64) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(code) { + t.Fatalf("length = %d, want %d", length, len(code)) + } + if instruction.Op != test.op || instruction.Raw != string(test.op)+" X3, X2, X1" { + t.Fatalf("instruction = %#v", instruction) + } + }) + } + + for _, test := range []struct { + name string + code []byte + want string + }{ + {name: "y_memory", code: []byte{0xc5, 0xed, 0x6d, 0x4e, 0x20}, want: "VPUNPCKHQDQ 32(SI), Y2, Y1"}, + {name: "extended_registers", code: []byte{0xc4, 0x41, 0x41, 0x6d, 0xf7}, want: "VPUNPCKHQDQ X15, X7, X14"}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VPUNPCKInstruction(test.code, 64) + if err != nil { + t.Fatal(err) + } + if !ok || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode = (%#v, %d, %v), want %q", instruction, length, ok, test.want) + } + }) + } +} + +func TestDecodedX86VPUNPCKInstructionRejectsInvalidVEXForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc5, 0xe9, 0x6d}, mode: 64}, + {name: "vex_pp_zero", code: []byte{0xc5, 0xe8, 0x6d, 0xcb}, mode: 64}, + {name: "vex_w_one", code: []byte{0xc4, 0xe1, 0xe9, 0x6d, 0xcb}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x41, 0x41, 0x6d, 0xf7}, mode: 32}, + {name: "rip_relative", code: []byte{0xc5, 0xe9, 0x6d, 0x0d, 0, 0, 0, 0}, mode: 64}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VPUNPCKInstruction(test.code, test.mode) + if !ok { + t.Fatal("VPUNPCK encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVPUNPCKSequence(t *testing.T) { + code := []byte{ + 0xc4, 0xc1, 0x41, 0x6d, 0xf7, + 0xc5, 0x41, 0x6c, 0xff, + 0xc4, 0xc1, 0x11, 0x6d, 0xff, + 0xc5, 0x61, 0x6c, 0xfb, + 0xc4, 0xc1, 0x69, 0x6d, 0xd7, + 0xc4, 0x41, 0x09, 0x6c, 0xfe, + 0xc4, 0xc1, 0x61, 0x6d, 0xdf, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VPUNPCK raw sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 7 { + t.Fatalf("decoded %d instructions, want 7: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VPUNPCKHQDQ" && instruction.Op != "VPUNPCKLQDQ" { + t.Fatalf("decoded unexpected op %s", instruction.Op) + } + } +} diff --git a/amd64_raw_vshuff_blocks_test.go b/amd64_raw_vshuff_blocks_test.go new file mode 100644 index 000000000..9a508d672 --- /dev/null +++ b/amd64_raw_vshuff_blocks_test.go @@ -0,0 +1,51 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86Shuffle128BitBlocksCompleteGo127Family(t *testing.T) { + // Go 1.27 shares _yvshuff32x4 across these four instructions. Its + // EVEX-only rows cover Y/Z register or memory sources, high registers, + // K masking, zeroing, scalar broadcast, and compressed disp8 scaling. + tests := []struct { + name string + code []byte + want string + }{ + {name: "f32 y high masked", code: []byte{0x62, 0xa3, 0x55, 0x21, 0x23, 0xf4, 0x06}, want: "VSHUFF32X4 $6, Y20, Y21, K1, Y22"}, + {name: "f32 z memory zeroing", code: []byte{0x62, 0xd3, 0x75, 0xca, 0x23, 0x50, 0x03, 0x07}, want: "VSHUFF32X4.Z $7, 192(R8), Z1, K2, Z2"}, + {name: "f64 reported minio", code: []byte{0x62, 0x23, 0x8d, 0x40, 0x23, 0xc0, 0xee}, want: "VSHUFF64X2 $238, Z16, Z30, Z24"}, + {name: "f64 z broadcast zeroing", code: []byte{0x62, 0xf3, 0xf5, 0xdb, 0x23, 0x50, 0x7f, 0x09}, want: "VSHUFF64X2.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + {name: "i32 y high memory masked", code: []byte{0x62, 0x43, 0x3d, 0x24, 0x43, 0x4c, 0x24, 0x02, 0x0a}, want: "VSHUFI32X4 $10, 64(R12), Y24, K4, Y25"}, + {name: "i32 z high", code: []byte{0x62, 0x33, 0x7d, 0x40, 0x43, 0xc9, 0x44}, want: "VSHUFI32X4 $68, Z17, Z16, Z9"}, + {name: "i64 y high masked", code: []byte{0x62, 0xa3, 0xd5, 0x21, 0x43, 0xf4, 0x06}, want: "VSHUFI64X2 $6, Y20, Y21, K1, Y22"}, + {name: "i64 z broadcast zeroing", code: []byte{0x62, 0xf3, 0xf5, 0xdb, 0x43, 0x50, 0x7f, 0x09}, want: "VSHUFI64X2.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86Shuffle128BitBlocksInstruction(test.code, 64) + if !ok { + t.Fatal("shuffle-block encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVSHUFF64X2(t *testing.T) { + code := []byte{0x62, 0x23, 0x8d, 0x40, 0x23, 0xc0, 0xee} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VSHUFF64X2", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VSHUFF64X2" || !strings.HasPrefix(decoded[0].Raw, "VSHUFF64X2 $238, Z16, Z30, Z24 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vshufps_test.go b/amd64_raw_vshufps_test.go new file mode 100644 index 000000000..b43c7973b --- /dev/null +++ b/amd64_raw_vshufps_test.go @@ -0,0 +1,136 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86VSHUFPSInstructionCompleteVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + want string + }{ + {name: "x_register", code: []byte{0xc5, 0x78, 0xc6, 0xc1, 0x44}, mode: 64, want: "VSHUFPS $68, X1, X0, X8"}, + {name: "y_register", code: []byte{0xc5, 0x6c, 0xc6, 0xcb, 0x44}, mode: 64, want: "VSHUFPS $68, Y3, Y2, Y9"}, + // Retain the old negative's exact bytes as the ISA's valid WIG form. + {name: "vex_w_one", code: []byte{0xc4, 0xe1, 0xbc, 0xc6, 0xc1, 0x44}, mode: 64, want: "VSHUFPS $68, Y1, Y8, Y0"}, + {name: "extended_registers", code: []byte{0xc4, 0x41, 0x3c, 0xc6, 0xd1, 0xdd}, mode: 64, want: "VSHUFPS $221, Y9, Y8, Y10"}, + {name: "memory_disp8", code: []byte{0xc5, 0xec, 0xc6, 0x56, 0x20, 0x01}, mode: 64, want: "VSHUFPS $1, 32(SI), Y2, Y2"}, + {name: "pd extended registers", code: []byte{0xc4, 0x41, 0x3d, 0xc6, 0xd1, 0xdd}, mode: 64, want: "VSHUFPD $221, Y9, Y8, Y10"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VSHUFPSInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(test.code) { + t.Fatalf("length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("instruction = %q, want %q", instruction.Raw, test.want) + } + }) + } +} + +func TestDecodedX86PackedFloatShuffleCompleteEVEXFamily(t *testing.T) { + // Go 1.27 assigns VSHUFPS and VSHUFPD the shared + // _yvgf2p8affineinvqb table. Its EVEX rows cover X/Y/Z register or + // memory sources, high registers, K masking, zeroing, and scalar + // broadcast. The byte sequences below are verified by LLVM 22. + tests := []struct { + name string + code []byte + want string + }{ + {name: "ps x high masked", code: []byte{0x62, 0xa1, 0x54, 0x01, 0xc6, 0xf4, 0x06}, want: "VSHUFPS $6, X20, X21, K1, X22"}, + {name: "ps y memory zeroing", code: []byte{0x62, 0xd1, 0x74, 0xaa, 0xc6, 0x50, 0x03, 0x07}, want: "VSHUFPS.Z $7, 96(R8), Y1, K2, Y2"}, + {name: "ps z high", code: []byte{0x62, 0x31, 0x7c, 0x40, 0xc6, 0xc9, 0x44}, want: "VSHUFPS $68, Z17, Z16, Z9"}, + {name: "ps z broadcast zeroing", code: []byte{0x62, 0xf1, 0x74, 0xdb, 0xc6, 0x50, 0x7f, 0x09}, want: "VSHUFPS.BCST.Z $9, 508(AX), Z1, K3, Z2"}, + {name: "pd x high masked", code: []byte{0x62, 0xa1, 0xd5, 0x01, 0xc6, 0xf4, 0x06}, want: "VSHUFPD $6, X20, X21, K1, X22"}, + {name: "pd y memory zeroing", code: []byte{0x62, 0xd1, 0xf5, 0xaa, 0xc6, 0x50, 0x03, 0x07}, want: "VSHUFPD.Z $7, 96(R8), Y1, K2, Y2"}, + {name: "pd z high", code: []byte{0x62, 0x31, 0xfd, 0x40, 0xc6, 0xc9, 0x44}, want: "VSHUFPD $68, Z17, Z16, Z9"}, + {name: "pd z broadcast zeroing", code: []byte{0x62, 0xf1, 0xf5, 0xdb, 0xc6, 0x50, 0x7f, 0x09}, want: "VSHUFPD.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VSHUFPSInstruction(test.code, 64) + if !ok { + t.Fatal("packed float shuffle encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) { + t.Fatalf("decoded length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("decoded %x as %q, want %q", test.code, instruction.Raw, test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioEVEXVSHUFPS(t *testing.T) { + code := []byte{0x62, 0x31, 0x7c, 0x40, 0xc6, 0xc9, 0x44} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio EVEX VSHUFPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VSHUFPS" || !strings.HasPrefix(decoded[0].Raw, "VSHUFPS $68, Z17, Z16, Z9 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} + +func TestDecodedX86VSHUFPSInstructionRejectsInvalidVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc5, 0x7c, 0xc6}, mode: 64}, + {name: "truncated_immediate", code: []byte{0xc5, 0x7c, 0xc6, 0xc1}, mode: 64}, + {name: "vex_pp_f3", code: []byte{0xc5, 0x7e, 0xc6, 0xc1, 0x44}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc5, 0x78, 0xc6, 0xc1, 0x44}, mode: 32}, + {name: "rip_relative", code: []byte{0xc5, 0xfc, 0xc6, 0x05, 0, 0, 0, 0, 0x44}, mode: 64}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VSHUFPSInstruction(test.code, test.mode) + if !ok { + t.Fatal("VSHUFPS encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVSHUFPSSequence(t *testing.T) { + code := []byte{ + 0xc5, 0x7c, 0xc6, 0xc1, 0x44, + 0xc5, 0xfc, 0xc6, 0xc1, 0xee, + 0xc5, 0x6c, 0xc6, 0xcb, 0x44, + 0xc5, 0xec, 0xc6, 0xd3, 0xee, + 0xc4, 0xc1, 0x3c, 0xc6, 0xd9, 0xdd, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VSHUFPS raw sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 5 { + t.Fatalf("decoded %d instructions, want 5: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VSHUFPS" { + t.Fatalf("decoded op = %s, want VSHUFPS", instruction.Op) + } + } +} diff --git a/amd64_raw_waitpkg.go b/amd64_raw_waitpkg.go new file mode 100644 index 000000000..ea3d5d3f5 --- /dev/null +++ b/amd64_raw_waitpkg.go @@ -0,0 +1,81 @@ +package plan9asm + +import "fmt" + +// WAITPKG shares the 0F AE group with fences, CLWB and FSGSBASE. Mandatory +// prefix plus mod=3 and /6 select the family; REX.B selects the GP register. +// See Intel SDM volume 2B (UMONITOR, UMWAIT, TPAUSE), not x86asm's legacy +// MFENCE decoding of these bytes. +func decodedX86WaitPackageInstruction(code []byte, mode int) (Instr, int, bool, error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + var repeat, segment, rex byte + operandOverride, addressOverride, locked := false, false, false + for i < len(code) { + switch code[i] { + case 0xf2, 0xf3: + repeat, rex = code[i], 0 + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65: + segment, rex = code[i], 0 + case 0x66: + operandOverride, rex = true, 0 + case 0x67: + addressOverride, rex = true, 0 + case 0xf0: + locked, rex = true, 0 + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + continue + } + goto opcode + } + i++ + } + +opcode: + prefix := repeat + if prefix == 0 && operandOverride { + prefix = 0x66 + } + if prefix == 0 || len(code) < i+2 || code[i] != 0x0f || code[i+1] != 0xae { + return Instr{}, 0, false, nil + } + if len(code) < i+3 { + return Instr{}, 0, true, fmt.Errorf("truncated WAITPKG ModRM") + } + modrm := code[i+2] + if modrm&0x38 != 0x30 || modrm>>6 != 3 { + // In particular, 66 0F AE /6 with a memory operand is CLWB. + return Instr{}, 0, false, nil + } + if locked || i+3 > 15 { + return Instr{}, 0, true, fmt.Errorf("invalid WAITPKG LOCK prefix or instruction length") + } + var op Op + for name, spec := range amd64ImplicitSystemSpecs { + if spec.rawPrefix == prefix { + op = name + break + } + } + register, ok := decodedX86GeneralRegister(int(modrm&7) + int(rex&1)*8) + if !ok || op == "" { + return Instr{}, 0, true, fmt.Errorf("invalid WAITPKG register or mandatory prefix") + } + ins := Instr{ + Op: op, Args: []Operand{{Kind: OpReg, Reg: register}}, + Raw: fmt.Sprintf("%s %s", op, register), x86Encoded: true, + } + if amd64ImplicitSystemSpecs[op].operand == amd64ImplicitSystemAddressGP { + ins.x86AddressBits = mode + if addressOverride { + ins.x86AddressBits /= 2 + } + ins.x86SegmentPrefix = segment + } + return ins, i + 3, true, nil +} diff --git a/amd64_raw_waitpkg_test.go b/amd64_raw_waitpkg_test.go new file mode 100644 index 000000000..e4f61e538 --- /dev/null +++ b/amd64_raw_waitpkg_test.go @@ -0,0 +1,176 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +// Intel SDM volume 2B: UMONITOR/UMWAIT/TPAUSE share 0F AE /6, mod=3. +// Go 1.27 uses ywrfsbase's single Yrl operand for all three spellings. +func TestX86RawWaitPackageFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + for _, form := range []struct { + op Op + prefix byte + }{{"UMONITOR", 0xf3}, {"UMWAIT", 0xf2}, {"TPAUSE", 0x66}} { + t.Run(target.triple+"/"+string(form.op), func(t *testing.T) { + code := []byte{form.prefix} + if target.arch == "amd64" { + // ixl-go's UMONITOR uses REX.W; W does not select its width. + code = append(code, 0x48) + } + code = append(code, 0x0f, 0xae, 0xf0) + var source strings.Builder + source.WriteString("TEXT waitpkg(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + var ir string + if form.op == "UMONITOR" { + // Address-monitoring retains its separate existing contract. + ir, err = Translate(file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"waitpkg": {Name: "waitpkg", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + } else { + ir = x86UnboundVoidRawScalarFormatObject(t, source.String(), target.arch, target.triple) + } + if !strings.Contains(ir, strings.ToLower(string(form.op))) { + t.Fatalf("decoded %x as another instruction", code) + } + compileLLVMToObject(t, llc, target.triple, "waitpkg.ll", "waitpkg.o", ir) + }) + } + } +} + +func TestX86WaitPackageClearsOtherFlags(t *testing.T) { + for _, op := range []string{"UMWAIT", "TPAUSE"} { + source := "TEXT flags(SB),$0-0\n" + op + " BX\nRET\n" + ir := x86UnboundVoidScalarFormatObject(t, source, "flags", "amd64", "x86_64-unknown-linux-gnu") + _, after, ok := strings.Cut(ir, "asm sideeffect") + if !ok { + t.Fatal("no hardware wait operation") + } + for _, flag := range []string{"z", "slt", "pf", "of"} { + if !strings.Contains(after, "store i1 false, ptr %flags_"+flag) { + t.Errorf("%s did not clear %s after capturing carry", op, flag) + } + } + } +} + +func TestX86RawWaitPackageRegisterGrammar(t *testing.T) { + for _, mode := range []int{32, 64} { + registerCount := 8 + if mode == 64 { + registerCount = 16 + } + for _, form := range []struct { + op Op + prefix byte + }{{"UMONITOR", 0xf3}, {"UMWAIT", 0xf2}, {"TPAUSE", 0x66}} { + for register := 0; register < registerCount; register++ { + for _, ignoredREX := range []byte{0, 2, 4, 8, 14} { + code := []byte{form.prefix, 0x67} + if mode == 64 { + code = append(code, 0x40|ignoredREX|byte(register>>3)) + } + code = append(code, 0x0f, 0xae, 0xf0|byte(register&7)) + ins, length, ok, err := decodedX86WaitPackageInstruction(code, mode) + wantRegister, _ := decodedX86GeneralRegister(register) + if err != nil || !ok || length != len(code) || ins.Op != form.op || + len(ins.Args) != 1 || ins.Args[0].Reg != wantRegister { + t.Fatalf("mode=%d bytes=%x: %+v, length=%d, ok=%v, err=%v", mode, code, ins, length, ok, err) + } + if form.op == "UMONITOR" && ins.x86AddressBits != mode/2 { + t.Fatalf("UMONITOR address size = %d, want %d", ins.x86AddressBits, mode/2) + } + } + } + } + } + for _, other := range [][]byte{ + {0x0f, 0xae, 0xf0}, // MFENCE + {0x66, 0x0f, 0xae, 0x30}, // CLWB (AX) + {0xf3, 0x0f, 0xae, 0xc0}, // RDFSBASEL AX + {0xf3, 0x0f, 0xae, 0x30}, // a memory encoding, not WAITPKG + {0xf2, 0x0f, 0xae, 0x30}, // a memory encoding, not WAITPKG + } { + if _, _, ok, _ := decodedX86WaitPackageInstruction(other, 64); ok { + t.Fatalf("claimed another 0F AE family: %x", other) + } + } + for _, invalid := range [][]byte{ + {0xf3, 0x0f, 0xae}, + {0xf0, 0xf3, 0x0f, 0xae, 0xf0}, + append(bytes.Repeat([]byte{0x67}, 12), 0xf3, 0x0f, 0xae, 0xf0), + } { + if _, err := decodeX86RawDirectiveGroup(invalid, 64, 0, "invalid WAITPKG", map[string]bool{}); err == nil { + t.Fatalf("accepted invalid WAITPKG encoding %x", invalid) + } + } +} + +func TestX86RawWaitPackageMonitorAddressOverrides(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + arch, triple string + mode int + }{ + {"amd64", "x86_64-unknown-linux-gnu", 64}, + {"amd64", "x86_64-apple-darwin", 64}, + {"amd64", "x86_64-pc-windows-msvc", 64}, + {"386", "i386-unknown-linux-gnu", 32}, + {"386", "i686-pc-windows-msvc", 32}, + } { + for _, segment := range []byte{0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65} { + t.Run(fmt.Sprintf("%s/%02x", target.triple, segment), func(t *testing.T) { + source := fmt.Sprintf(`TEXT monitor(SB),$0-0 + BYTE $0x%02x + BYTE $0x67; BYTE $0xf3; BYTE $0x0f; BYTE $0xae; BYTE $0xf3 + RET +`, segment) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"monitor": {Name: "monitor", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, fmt.Sprintf(".byte 0x%02x; umonitor", segment)) || + !strings.Contains(ir, fmt.Sprintf("zext i%d", target.mode/2)) { + t.Fatal("lost monitor segment or narrow address semantics") + } + compileLLVMToObject(t, llc, target.triple, "monitor.ll", "monitor.o", ir) + }) + } + } +} diff --git a/amd64_raw_x87_control_test.go b/amd64_raw_x87_control_test.go new file mode 100644 index 000000000..187e54987 --- /dev/null +++ b/amd64_raw_x87_control_test.go @@ -0,0 +1,425 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawX87AliasFamilyMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + wantTables := map[Op]string{"FCLEX": "ynone", "FINIT": "ynone", "FSAVE": "ysvrs_om", "FSTCW": "ysvrs_om", "FSTENV": "ysvrs_om", "FSTSW": "ystsw"} + if len(x86RawX87ControlAliases) != len(wantTables) { + t.Fatalf("raw x87 control grammar has %d aliases, want %d", len(x86RawX87ControlAliases), len(wantTables)) + } + for decoded, goOp := range x86RawX87ControlAliases { + if decoded.String() != "FN"+string(goOp)[1:] || !isX87Op(goOp) { + t.Fatalf("raw %s does not resolve to the Go x87 lowerer for %s", decoded, goOp) + } + row := regexp.MustCompile(`\{A` + string(goOp) + `,\s*(\w+),`).FindSubmatch(data) + if len(row) != 2 || string(row[1]) != wantTables[goOp] { + t.Fatalf("Go %s row = %q, want operand table %s", goOp, row, wantTables[goOp]) + } + } +} + +func TestX86RawX87EnvironmentAddressSpaces(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + source.WriteString("TEXT x87segments(SB),4,$0-0\n") + for _, opcode := range [][]byte{{0xdd, 0x30}, {0xd9, 0x38}, {0xd9, 0x30}, {0xdd, 0x38}} { + for _, segment := range []byte{0x64, 0x65} { + for _, b := range append([]byte{segment}, opcode...) { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"x87segments": {Name: "x87segments", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "x87-segments.ll", "x87-segments.o", ir) + } +} + +func TestX86RawX87DoesNotWidenLegacyEnvironmentFormat(t *testing.T) { + for _, code := range [][]byte{{0x66, 0xdd, 0x30}, {0x66, 0xd9, 0x30}} { + inst, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + if _, err := decodedX86GoSyntax(inst, code); err == nil { + t.Fatalf("silently treated 16-bit environment %x as the Go 32-bit environment form", code) + } + } +} + +// asm6.go's FCLEX/FINIT/FSAVE/FSTCW/FSTENV/FSTSW rows encode the +// no-wait forms, despite omitting Intel's N in their Go mnemonic. +func x87NoWaitFixtures() []struct { + code []byte + text string +} { + return []struct { + code []byte + text string + }{ + {[]byte{0xdb, 0xe2}, "FCLEX"}, + {[]byte{0xdb, 0xe3}, "FINIT"}, + {[]byte{0xdd, 0x30}, "FSAVE (AX)"}, + {[]byte{0xd9, 0x38}, "FSTCW (AX)"}, + {[]byte{0xd9, 0x30}, "FSTENV (AX)"}, + {[]byte{0xdd, 0x38}, "FSTSW (AX)"}, + {[]byte{0xdf, 0xe0}, "FSTSW AX"}, + {[]byte{0xdd, 0x74, 0x8b, 0xf9}, "FSAVE -7(BX)(CX*4)"}, + {[]byte{0xd9, 0x7c, 0x8b, 0xf9}, "FSTCW -7(BX)(CX*4)"}, + {[]byte{0xd9, 0x74, 0x8b, 0xf9}, "FSTENV -7(BX)(CX*4)"}, + {[]byte{0xdd, 0x7c, 0x8b, 0xf9}, "FSTSW -7(BX)(CX*4)"}, + } +} + +func TestX86RawX87NoWaitCompleteGoForms(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + t.Run(goarch, func(t *testing.T) { + var named, raw strings.Builder + named.WriteString("TEXT x87control(SB),4,$0-0\n") + raw.WriteString("TEXT x87control(SB),4,$0-0\n") + var expectedBytes []byte + for _, test := range x87NoWaitFixtures() { + fmt.Fprintln(&named, test.text) + expectedBytes = append(expectedBytes, test.code...) + for _, b := range test.code { + fmt.Fprintf(&raw, "BYTE $%#02x\n", b) + } + decoded, err := decodeX86RawDirectives(rawX86Function(test.code), goarch) + if err != nil { + t.Fatal(err) + } + want, err := Parse(ArchAMD64, "TEXT f(SB),$0-0\n"+test.text+"\n") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != want.Funcs[0].Instrs[1].Op || !reflect.DeepEqual(decoded.Instrs[0].Args, want.Funcs[0].Instrs[1].Args) { + t.Fatalf("decoded %x as %+v, want %s", test.code, decoded.Instrs, test.text) + } + } + named.WriteString("RET\n") + raw.WriteString("RET\n") + expectedBytes = append(expectedBytes, 0xc3) + code := assembleX87ControlBytes(t, goarch, named.String()) + if !bytes.HasPrefix(code, expectedBytes) { + t.Fatalf("Go %s encoded no-wait family as %x, want prefix %x", goarch, code, expectedBytes) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (goarch == "386") != strings.HasPrefix(triple, "i") { + continue + } + for _, mode := range []X87Mode{X87Auto, X87Software} { + var irs []string + for _, source := range []string{named.String(), raw.String()} { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: goarch, TargetTriple: triple, X87Mode: mode, Sigs: map[string]FuncSig{"x87control": {Name: "x87control", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + irs = append(irs, ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, ir := range irs { + compileLLVMToObject(t, llc, triple, "x87-control.ll", "x87-control.o", ir) + } + } + } + }) + } +} + +func assembleX87ControlBytes(t *testing.T, goarch, source string) []byte { + t.Helper() + dir := t.TempDir() + name := filepath.Join(dir, "control.s") + if err := os.WriteFile(name, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-S", "-o", filepath.Join(dir, "control.o"), name) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+goarch) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Go assembler: %v\n%s", err, out) + } + var code []byte + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") || len(fields[1]) != 2 { + continue + } + offset, err := strconv.ParseUint(fields[0][2:], 16, 32) + if err != nil || int(offset) != len(code) { + t.Fatalf("noncontiguous assembler bytes: %s", line) + } + for i, field := range fields[1:] { + if i >= 16 || len(field) != 2 { + break + } + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + break + } + code = append(code, byte(value)) + } + } + return code +} + +func TestX86RawX87ClearExceptionRuntimeSemantics(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; all-host x86 object tests remain required") + } + const source = `TEXT rawClear(SB),$0-0 +BYTE $0xdb; BYTE $0xe2 +RET +TEXT namedClear(SB),$0-0 +FCLEX +RET +TEXT rawInit(SB),$0-0 +BYTE $0xdb; BYTE $0xe3 +RET +TEXT namedInit(SB),$0-0 +FINIT +RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for _, name := range []string{"rawClear", "namedClear", "rawInit", "namedInit"} { + sigs[name] = FuncSig{Name: name, Ret: Void} + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple, prefix = "x86_64-apple-macosx", []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void rawClear(void), namedClear(void), rawInit(void), namedInit(void); +static void seed(void) { + uint16_t cw=0x077f; + __asm__ volatile("fninit; fldcw %0; fldz; fldz; fdivp; fstp %%st(0)" : : "m"(cw) : "st"); +} +static uint16_t sw(void) { uint16_t v; __asm__ volatile("fnstsw %0":"=a"(v)); return v; } +static uint16_t cw(void) { uint16_t v; __asm__ volatile("fnstcw %0":"=m"(v)); return v; } +int main(void) { + void (*f[])(void)={rawClear,namedClear,rawInit,namedInit}; + for (int i=0;i<4;i++) { + seed(); + if (!(sw()&1) || cw()!=0x077f) return 20; + f[i](); + if ((sw()&0x80ff) || cw()!=(i<2 ? 0x077f : 0x037f)) { + fprintf(stderr,"x87 operation %d did not update hardware: sw=%x cw=%x\n",i,sw(),cw()); + __asm__ volatile("fninit"); return 1; + } + } + __asm__ volatile("fninit"); return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_x87_clear", triple, ir, mainC, prefix) +} + +func TestX86RawX87SoftwareClearPreservesAllStatusFields(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("hardware status oracle requires amd64 or Rosetta") + } + const source = `TEXT clearModeledStatus(SB),4,$0-16 +MOVQ env+0(FP),BX +FLDENV (BX) +BYTE $0xdb; BYTE $0xe2 +XORL AX,AX +FSTSW AX +MOVQ AX,ret+8(FP) +RET +` + // The original source has no source-authenticated Tag/TOP shape at RET. + // Keep it as a Context control. The executed counterpart differs only by + // a final FINIT after the captured result, not by an invented entry fact. + _, err := x86X87InitializedWordAPI(t, source, "func clearModeledStatus(env *byte) uint64", + "amd64", "x86_64-unknown-linux-gnu", "text", X87Software) + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "logical x87 value stack is not balanced at RET") { + t.Fatal("complete original source borrowed an unknown empty value stack", err) + } + bounded := strings.Replace(source, "MOVQ AX,ret+8(FP)\nRET\n", "MOVQ AX,ret+8(FP)\nFINIT\nRET\n", 1) + if bounded == source { + t.Fatal("bounded runtime counterpart lost its exact final-reset anchor") + } + file, err := Parse(ArchAMD64, bounded) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple, prefix = "x86_64-apple-macosx", []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, X87Mode: X87Software, Sigs: map[string]FuncSig{ + "clearModeledStatus": {Name: "clearModeledStatus", Args: []LLVMType{Ptr}, Ret: I64, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}, Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }}, + }}) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern uint64_t clearModeledStatus(const void *); +int main(void) { + struct { unsigned char bytes[28]; } env={{0}}; + uint16_t control=0x037f, tags=0xffff; + memcpy(env.bytes,&control,2); memcpy(env.bytes+8,&tags,2); + for (unsigned v=0;v<=0xffff;v++) { + uint16_t status=v, expected; + memcpy(env.bytes+4,&status,2); + __asm__ volatile("fldenv %1; fnclex; fnstsw %0" : "=a"(expected) : "m"(env) : "memory"); + uint64_t got=clearModeledStatus(&env); + if (got!=expected) { + fprintf(stderr,"status %x: got %llx, hardware %x\n",v,(unsigned long long)got,expected); + __asm__ volatile("fninit"); return 1; + } + } + __asm__ volatile("fninit"); return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "x87_software_clear", triple, ir, mainC, prefix) +} + +func TestX86X87WordStoresToFPParametersAndResults(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source strings.Builder + sigs := make(map[string]FuncSig) + for _, op := range []string{"FSTCW", "FSTSW"} { + for _, kind := range []string{"param", "result"} { + name := op + kind + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-10\nFINIT\n", name) + if kind == "param" { + fmt.Fprintf(&source, "%s input+0(FP)\nMOVW input+0(FP),AX\nMOVW AX,ret+8(FP)\n", op) + } else { + fmt.Fprintf(&source, "%s ret+8(FP)\n", op) + } + source.WriteString("RET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{I16}, Ret: I16, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I16, Index: 0, Field: -1}}, Results: []FrameSlot{{Offset: 8, Type: I16, Index: 0, Field: -1}}, + }} + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern uint16_t FSTCWparam(uint16_t),FSTCWresult(uint16_t),FSTSWparam(uint16_t),FSTSWresult(uint16_t); +int main(void) { + return FSTCWparam(0xffff)!=0x037f || FSTCWresult(0xffff)!=0x037f || + FSTSWparam(0xffff)!=0 || FSTSWresult(0xffff)!=0; +} +` + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc", "x86_64-w64-windows-gnu"} { + goarch := "amd64" + if strings.HasPrefix(triple, "i") { + goarch = "386" + } + for _, mode := range []X87Mode{X87Auto, X87Software} { + t.Run(fmt.Sprintf("%s/%d", triple, mode), func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: goarch, TargetTriple: triple, X87Mode: mode, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "x87-fp.ll", "x87-fp.o", ir) + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && triple == "x86_64-apple-darwin" && rosettaAvailable() + hostOS := map[string]string{"darwin": "apple-darwin", "linux": "linux-gnu", "windows": "windows-msvc"}[runtime.GOOS] + if crossRosetta || (goarch == runtime.GOARCH && hostOS != "" && strings.Contains(triple, hostOS)) { + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + runtimeTriple := testTargetTriple(runtime.GOOS, goarch) + // Windows CI provides MinGW, not the MSVC CRT. Retain the + // MSVC object check above, but regenerate the runtime module + // for the actual linker ABI instead of requesting MSVC libs. + runtimeIR, err := Translate(file, Options{Goarch: goarch, TargetTriple: runtimeTriple, X87Mode: mode, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "x87_fp", runtimeTriple, runtimeIR, mainC, prefix) + } + }) + } + } +} + +func TestX86X87WordStoresRejectUnknownFPSlots(t *testing.T) { + for _, op := range []string{"FSTCW", "FSTSW"} { + file, err := Parse(ArchAMD64, "TEXT bad(SB),4,$0-0\n"+op+" unknown+24(FP)\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, arch := range []string{"386", "amd64"} { + if _, err := Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("%s accepted %s store to an undeclared FP slot", arch, op) + } + } + } +} diff --git a/amd64_register_bool_result.go b/amd64_register_bool_result.go new file mode 100644 index 000000000..ffe3f256e --- /dev/null +++ b/amd64_register_bool_result.go @@ -0,0 +1,183 @@ +package plan9asm + +import "strings" + +func x86RegisterBoolResultNeedsProof(fn Func, sig FuncSig, arch string) bool { + if arch != "amd64" || !strings.HasSuffix(fn.Sym, "") { + return false + } + if sig.Ret == I1 { + return true + } + for _, slot := range sig.Frame.Results { + if slot.Type == I1 { + return true + } + } + return false +} + +// Canonicality is a second must-fact beside byte initialization. This helper +// does not establish reads, FLAGS, memory or entry authority; the caller first +// executes the independent complete typed source transfer and then intersects +// these facts with its actual initialized GP bytes. Unmodeled effects revoke +// facts, never invent a value. Original Op/Raw/certificate identity is retained. +func x86RegisterCanonicalBytesAfter(original Instr, before [16]uint8, arch string) [16]uint8 { + if next, known := x86SegmentQueryCanonicalBytesAfter(original, before, arch); known { + return next + } + ins := x86SourceMoveInstruction(original, arch) + next := before + if _, known, err := x86DuplicateMoveProperties(ins.Op, ins, arch); known && err == nil && x86ResultNamedPredicateInstruction(ins) { + return next // No GP write; vector results cannot create canonical bool facts. + } + read := func(operand Operand, width int64) uint8 { + if operand.Kind == OpImm && operand.ImmRaw == "" && !operand.ImmIsFloat { + return fixedLocalConstantBooleans(operand.Imm, width) + } + if operand.Kind == OpReg { + if root, _, valid := x86ResultGP(operand.Reg, width, arch); valid { + return before[root] >> fixedLocalBooleanShift(operand.Reg) & uint8((uint16(1)<1 || logical!=(raw&255) || !canaries(observed)){fprintf(stderr,\"row=%d a=%%llx b=%%llx raw=%%llx expected=%%llx LLVM=%%u\\n\",(unsigned long long)a,(unsigned long long)b,(unsigned long long)raw,(unsigned long long)expected,logical);return 1;}}\n", index, row.expected, index, index) + } + // Noncanonical bytes are genuinely executed only by the raw observer. + // They must remain Context for every logical translator API, not be + // compared after lossy truncation to i1 (2->0, 3->1). + for offset, value := range []int{2, 3, 255} { + index := len(rows) + offset + original := amd64RegisterBoolSource(fmt.Sprintf("MOVB $%d,AX", value)) + originals.WriteString(amd64RegisterBoolOriginalAdapter(t, execution.triple, index, original)) + fmt.Fprintf(&checks, "{uint64_t observed[7]={0},raw=Original%d(a,b,observed);if(raw!=((a&~UINT64_C(255))|%d) || !canaries(observed))return 2;}\n", index, value) + } + driver := `#include +#include +` + originals.String() + prototypes.String() + ` +static int canaries(const uint64_t *p) { + return p[0]==UINT64_C(0x5a5a5a5a5a5a5a5a)&&p[1]==p[0]&& + p[2]==UINT64_C(0x6b6b6b6b6b6b6b6b)&&p[3]==p[2]&& + p[4]==UINT64_C(0x123456789abcdef0)&&p[5]==UINT64_C(0xfedcba9876543210)&& + p[6]==UINT64_C(0x876543210fedcba9); +} +int main(void) { + const uint64_t inputs[]={0,1,2,3,UINT64_C(0x8000000000000000),UINT64_C(0x7fffffffffffffff),UINT64_MAX}; + unsigned seen[16]={0}; + for(unsigned sample=0;sample<7;sample++) for(uint64_t b=0;b<4;b++) { + uint64_t a=inputs[sample],r=a-b; + unsigned cf=a>63,of=((a^b)&(a^r))>>63; + uint8_t parity=r;parity^=parity>>4;parity^=parity>>2;parity^=parity>>1; + unsigned pf=(parity&1)==0; + unsigned want[]={!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf}; + for(unsigned i=0;i<16;i++)seen[i]|=1U<>63!=0;of:=((a^b)&(a^r))>>63!=0 + parity:=byte(r);parity^=parity>>4;parity^=parity>>2;parity^=parity>>1;pf:=parity&1==0 + want:=[]bool{!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf} + for i,row:=range rows { + expected:=uint64(0);if want[i]{expected=1} + if row.call(a,b)!=want[i]{t.Fatalf("canonical Go bool caller row %d",i)} + raw,adjacent,input:=row.observe(a,b) + if raw!=0x5a5a5a5a5a5a5a00|expected||adjacent!=0x6b6b6b6b6b6b6b6b||input!=a{t.Fatalf("raw bool byte/canaries row %d: %x %x %x",i,raw,adjacent,input)} + seen[i]|=1<(SB),4,$0-17\n" + body + "\nRET\n" +} + +func TestAMD64RegisterBoolCanonicalResultThreeAPIs(t *testing.T) { + for _, body := range []string{ + "MOVB $0,AX", "MOVB $1,AX", "MOVW $257,AX", "MOVL $257,AX", "MOVQ $257,AX", + "MOVD $257,AX", "MOVDQ2Q $257,AX", "XORQ AX,AX", "ANDQ $1,AX", "ANDB $1,AX", + "CMPQ AX,BX\nSETEQ AX", "CMPQ AX,BX\nSETNE BX\nMOVB BX,AX", + "CMPQ AX,BX\nSETEQ BX\nMOVBQZX BX,AX", + "MOVB $1,CH\nMOVB CH,AL", "MOVW $257,CX\nMOVB CH,AL", + "MOVB $0,CX\nMOVB $1,DX\nORB DX,CX\nMOVB CX,AX", + } { + t.Run(body, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, amd64RegisterBoolSource(body), "func view(a,b uint64) bool\n", "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil || ir == "" { + t.Errorf("%s: proven canonical low result byte rejected: %v", api, err) + } + } + }) + } + for _, op := range strings.Fields("SETCC SETCS SETEQ SETGE SETGT SETHI SETLE SETLS SETLT SETMI SETNE SETOC SETOS SETPC SETPL SETPS") { + t.Run(op, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, amd64RegisterBoolSource("CMPQ AX,BX\n"+op+" AX"), "func view(a,b uint64) bool\n", "amd64", "x86_64-unknown-linux-gnu", api); err != nil { + t.Errorf("%s: actual Go SETcc canonical byte rejected: %v", api, err) + } + } + }) + } +} + +func TestAMD64RegisterBoolInitializedIsNotCanonical(t *testing.T) { + for name, body := range map[string]string{ + "two": "MOVB $2,AX", "three": "MOVB $3,AX", "ff": "MOVB $255,AX", + "linear-two": "MOVQ $2,AX", "linear-three": "MOVL $3,AX", "linear-ff": "MOVQ $255,AX", + "incoming-integer": "MOVB AX,AX", "unknown-flags": "SETEQ AX", "wrong-byte": "MOVB $1,AH", + "widened-unknown-argument": "MOVQ $0,BX\nMOVB $1,AX\nCMPQ AX,BX\nSETEQ AX", + "noncanonical-overwrite": "MOVB $1,AX\nINCB AX", + "vector-overwrite": "MOVB $1,AX\nPXOR X0,X0\nMOVQ X0,AX", + "one-noncanonical-edge": "CMPQ AX,BX\nJE done\nMOVB $2,AX\nJMP exit\ndone:\nMOVB $1,AX\nexit:", + "raw": "BYTE $0x90\nMOVB $1,AX", "FP-result": "MOVB $1,ret+16(FP)", + } { + t.Run(name, func(t *testing.T) { + declaration := "func view(a,b uint64) bool\n" + if name == "widened-unknown-argument" { + declaration = "func view(a uint8,b uint64) bool\n" + body = strings.ReplaceAll(body, "MOVB $1,AX\n", "") + } + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, amd64RegisterBoolSource(body), declaration, "amd64", "x86_64-unknown-linux-gnu", api); !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s: arbitrary/unknown/noncanonical result gained an i1 transport: %v", api, err) + } + } + }) + } +} + +func TestAMD64RegisterBoolDoesNotInitializeUpperGPBytes(t *testing.T) { + for _, move := range []string{"MOVB", "MOVW"} { + pkg := mustGoPackage(t, "registerbool", "package registerbool\nfunc view() uint64\n") + sig, err := DeriveGoFuncABI0Signature("view", pkg.Types.Scope().Lookup("view").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, "TEXT ·view(SB),4,$0-8\n"+move+" $1,AX\nRET\n") + if err != nil { + t.Fatal(err) + } + consumer := &amd64Ctx{fpScalarSource: file.Funcs[0], sig: sig, goarch: "amd64"} + if err := consumer.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s: a canonical byte manufactured unspecified upper result bytes: %v", move, err) + } + } + for _, declaration := range []string{"func view(a bool) bool\n", "func view(a uint64) (bool,bool)\n"} { + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, amd64RegisterBoolSource("MOVB $1,AX"), declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if api == "Go" && declaration == "func view(a bool) bool\n" { + // A complete GoModule now supplies its own call-local typed + // argument plan. It initializes only the declared low byte; + // the raw APIs and multiple-result boundary remain unchanged. + // VirtualBoolPortableValues separately executes both bool values. + if err != nil { + t.Errorf("complete typed bool argument lost its declaration binding: %v", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s: unbound bool argument/aggregate result was inferred: %v", api, err) + } + } + } +} + +func TestAMD64RegisterBoolCanonicalTypedTransferFamilies(t *testing.T) { + check := func(body string, canonical bool) { + t.Helper() + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, amd64RegisterBoolSource(body), "func view(a,b uint64) bool\n", "amd64", "x86_64-unknown-linux-gnu", api) + if canonical && err != nil || !canonical && !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s canonical=%v: %s: %v", api, canonical, body, err) + } + } + } + // The complete official source extension family, including the unusual + // word-read ymb rows, composes with physical per-byte facts, not type/name + // exemptions. A known low byte is retained; sign/zero extension cannot + // make an arbitrary source low byte canonical. + for op := range amd64ScalarExtensionMoveSpecs { + t.Run(string(op), func(t *testing.T) { + check("MOVQ $257,CX\n"+string(op)+" CX,AX", true) + check("MOVQ $258,CX\n"+string(op)+" CX,AX", false) + }) + } + for _, suffix := range []string{"B", "W", "L", "Q"} { + for _, op := range []string{"AND", "OR", "XOR"} { + t.Run(op+suffix, func(t *testing.T) { + check("MOVQ $1,CX\n"+op+suffix+" CX,AX", op == "AND") + check("MOVQ $1,CX\nMOVQ $0,AX\n"+op+suffix+" CX,AX", true) + check("MOVQ $2,CX\nMOVQ $6,AX\n"+op+suffix+" CX,AX", false) + }) + } + } + for op := range x86ConditionalMoveSpecs { + t.Run(string(op), func(t *testing.T) { + check("CMPQ AX,BX\nMOVQ $1,CX\nMOVQ $0,AX\n"+string(op)+" CX,AX", true) + check("CMPQ AX,BX\nMOVQ $2,CX\nMOVQ $0,AX\n"+string(op)+" CX,AX", false) + check("CMPQ AX,BX\nMOVQ $1,CX\nMOVQ $2,AX\n"+string(op)+" CX,AX", false) + check("MOVQ $1,CX\nMOVQ $0,AX\n"+string(op)+" CX,AX", false) + }) + } + check("CMPQ AX,BX\nJE yes\nMOVB $0,AX\nJMP done\nyes:\nMOVB $1,AX\ndone:", true) + check("MOVB $1,AH\nMOVB AH,R8\nMOVB R8,AX", false) // Actual REX is not AH. + check("MOVB $1,CH\nMOVBQSX CH,AX", false) // Actual REX is not CH. +} + +func TestAMD64RegisterBoolCanonicalFactJoinsAndIdentity(t *testing.T) { + first := x86ResultState{} + first.gp[0], first.canonicalGP[0] = 255, 255 + second := first + second.canonicalGP[0] = 254 + if joined := first.intersect(second); joined.gp[0] != 255 || joined.canonicalGP[0] != 254 { + t.Fatal("canonicality must intersect independently of initialized bytes", joined) + } + for _, alias := range []string{"MOVQ", "MOVD", "MOVDQ2Q"} { + file, err := Parse(ArchAMD64, amd64RegisterBoolSource(alias+" $257,AX")) + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + facts := x86RegisterCanonicalBytesAfter(original, [16]uint8{}, "amd64") + if facts[0] != 255 || original.Op != Op(alias) || original.Raw != file.Funcs[0].Instrs[1].Raw { + t.Fatal("source-only dispatch must preserve original identity and exact canonical bytes", alias, facts) + } + } +} diff --git a/amd64_register_result.go b/amd64_register_result.go new file mode 100644 index 000000000..ef0a2aedd --- /dev/null +++ b/amd64_register_result.go @@ -0,0 +1,207 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" +) + +func amd64IntegerRegisterResultBytes(typ LLVMType) int64 { + switch typ { + case I8: + return 1 + case I16: + return 2 + case I32: + return 4 + case I64: + return 8 + } + return 0 +} + +// This restriction-only selector reads the declared return type, never Frame +// validity or the instruction stream. A missing/malformed layout must still +// reach the same cached complete-source check as an ordinary valid tuple. +func amd64RegisterTupleResultNeedsProof(fn Func, sig FuncSig, arch string) bool { + if arch != "amd64" || !strings.HasSuffix(fn.Sym, "") { + return false + } + fields, tuple := parseLiteralStructFields(sig.Ret) + if !tuple || len(fields) < 2 { + return false + } + for _, field := range fields { + if amd64IntegerRegisterResultBytes(field) == 0 { + return false + } + } + return true +} + +// A flattened LLVM return does not retain Go's whole-value assignment rule. +// Consume the already selected declaration once: structs recurse, arrays with +// more than one element need stack transport, and every complete result must +// fit the remaining numerical register bank. This grants no storage access. +func goAMD64NumericalRegisterResults(fn *types.Func) bool { + sig, ok := fn.Type().(*types.Signature) + if !ok || sig.Recv() != nil || sig.Variadic() { + return false + } + var leaves func(types.Type) (int, bool) + leaves = func(typ types.Type) (int, bool) { + switch typ := typ.Underlying().(type) { + case *types.Basic: + return 1, typ.Info()&types.IsInteger != 0 + case *types.Array: + if typ.Len() == 0 { + return 0, true + } + if typ.Len() == 1 { + return leaves(typ.Elem()) + } + case *types.Struct: + count := 0 + for index := 0; index < typ.NumFields(); index++ { + n, valid := leaves(typ.Field(index).Type()) + if !valid || n > 9-count { + return 0, false + } + count += n + } + return count, true + } + return 0, false + } + used := 0 + for index := 0; index < sig.Results().Len(); index++ { + n, valid := leaves(sig.Results().At(index).Type()) + if !valid || n > 9-used { + return false // Do not partially assign a result that belongs on the stack. + } + used += n + } + return true +} + +// This lowering-only query observes the already modeled scalar register entry, +// not ABI0 FP fields. The source must explicitly select ABIInternal. It grants +// neither physical Go entry/closure/TLS state nor incoming spill-frame backing. +// See Go's cmd/compile/abi-internal.md, amd64 integer register assignment. +func checkAMD64RegisterResultInitialization(fn Func, sig FuncSig, transfer x86ResultSourceTransfer) error { + fail := func(reason string) error { + return fmt.Errorf("%w: amd64 register result initialization: %s", ErrProbeNeedsContext, reason) + } + registers := [...]Reg{AX, BX, CX, DI, SI, "R8", "R9", "R10", "R11"} + if !strings.HasSuffix(fn.Sym, "") || len(sig.ArgRegs) != 0 || + sig.ARMEntry != nil || sig.ARM64GoRegisterABI != nil || sig.ARM64ClosureABI != nil || + sig.WASMContext != "" || sig.WASMNative || sig.amd64GoArguments == nil && len(sig.Args) > len(registers) || + len(fn.X86RawText) != 0 || fn.x86SourceStack != nil || fn.x86FixedLocal != nil { + return fail("requires a complete ordinary scalar register entry") + } + for _, ins := range fn.Instrs { + if ins.x86Encoded || ins.x86SourceRaw != nil || ins.x86RawNearReturn != nil || + ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 { + return fail("raw/native source needs its separate entry contract") + } + } + resultWidth := amd64IntegerRegisterResultBytes(sig.Ret) + if sig.Ret == I1 { + resultWidth = 1 + } + var resultWidths []int64 + if sig.Ret == Void { + if len(sig.Frame.Results) != 0 { + return fail("Void entry cannot declare a result transport") + } + } else if fields, tuple := parseLiteralStructFields(sig.Ret); tuple { + if len(fields) < 2 || len(fields) > len(registers) || len(sig.Frame.Results) != len(fields) { + return fail("requires one complete flat integer register tuple") + } + resultWidths = make([]int64, len(fields)) + for index, typ := range fields { + result := sig.Frame.Results[index] + bytes := amd64IntegerRegisterResultBytes(typ) + if bytes == 0 || result.Type != typ || result.Index != index || result.Field != -1 || len(result.Fields) != 0 { + return fail("tuple result transport differs from its integer signature") + } + resultWidths[index] = bytes + } + } else { + if resultWidth == 0 || len(sig.Frame.Results) != 1 { + return fail("requires one complete integer register result") + } + result := sig.Frame.Results[0] + if result.Type != sig.Ret || result.Index != 0 || result.Field != -1 || len(result.Fields) != 0 { + return fail("result transport differs from the scalar signature") + } + resultWidths = []int64{resultWidth} + } + initial := x86ResultState{} + if entry := sig.amd64GoArguments; entry != nil { + if !entry.valid(fn, sig, "amd64") { + return fail("typed argument carrier lost its live original compiler/source binding") + } + var valid bool + initial, valid = amd64GoArgumentInitialState(entry.plan) + if !valid { + return fail("invalid declared typed argument carrier bytes") + } + } else { + for index, typ := range sig.Args { + bytes := amd64IntegerRegisterResultBytes(typ) + if typ == Ptr { + bytes = 8 + } + if bytes == 0 { + return fail("aggregate, floating, boolean or spilled argument needs its own register contract") + } + reg, mask, valid := x86ResultGP(registers[index], bytes, "amd64") + if !valid { + return fail("invalid integer register assignment") + } + if typ == Ptr { + initial.pointerGP[reg] = true // Opaque identity, not numerical bytes or bounds. + initial.callerAddressGP[reg] = true + } else { + initial.gp[reg] = mask // Narrow inputs do not initialize the unspecified upper bytes. + } + } + } + // No FP/AUTO ownership is imported from a classic signature or TEXT size. + // The register role must close separately before gaining any stack storage. + if sig.Ret == I1 { + original := transfer + transfer = func(pc int, ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) bool { + if isX86RawDirective(ins) { + return false + } + before := state.canonicalGP + if !original(pc, ins, state, owned, resultOffset, arch) { + return false + } + state.canonicalGP = x86RegisterCanonicalBytesAfter(ins, before, arch) + for index := range state.gp { + state.canonicalGP[index] &= state.gp[index] + } + return true + } + } + return checkX86ResultSourceFlow(fn, initial, nil, 0, "amd64", transfer, + func(state x86ResultState, pc int) error { + // The complete source flow still runs; only the result at RET is optional. + if sig.Ret == Void { + return nil + } + for index, bytes := range resultWidths { + reg, mask, _ := x86ResultGP(registers[index], bytes, "amd64") + if state.gp[reg]&mask != mask { + return fail(fmt.Sprintf("%s result bytes are not initialized on every path to RET %d", registers[index], pc)) + } + if sig.Ret == I1 && state.canonicalGP[reg]&mask != mask { + return fail(fmt.Sprintf("AX bool result byte is not canonical zero/one on every path to RET %d", pc)) + } + } + return nil + }) +} diff --git a/amd64_register_result_runtime_test.go b/amd64_register_result_runtime_test.go new file mode 100644 index 000000000..29a4408e9 --- /dev/null +++ b/amd64_register_result_runtime_test.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func amd64RegisterResultMatrix(t *testing.T) (source, declarations, checks string) { + t.Helper() + var asm, decl, calls strings.Builder + count := 0 + for _, family := range []struct { + suffix string + width int + }{{"B", 1}, {"W", 2}, {"D", 4}, {"Q", 8}} { + for _, prefix := range []string{"", "V"} { + for lane := 0; lane < 16/family.width; lane++ { + name := fmt.Sprint("lane", count) + asm.WriteString(strings.ReplaceAll(amd64RegisterLaneFixture(prefix+"PINSR"+family.suffix, lane), "·view", "·"+name)) + fmt.Fprintf(&decl, "func %s(value,seed uint64) uint64\n", name) + fmt.Fprintf(&calls, "if(check(%s,%d,%d)){fprintf(stderr,\"%s failed\\n\");return 1;}\n", name, family.width, lane, name) + count++ + } + } + } + if count != 60 { + t.Fatal("incomplete register-entry scalar insertion matrix", count) + } + return asm.String(), decl.String(), calls.String() +} + +func requireAMD64RegisterResultGoObject(t *testing.T, source string) { + t.Helper() + dir := t.TempDir() + input, object := filepath.Join(dir, "source.s"), filepath.Join(dir, "source.o") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + // The shared flag probe handles Go 1.20's explicit -compiling-runtime. + args := append(arm64ClosureGoAsmArgs(t), "-o", object, input) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", args...) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), true, "amd64"); failure != nil { + t.Fatalf("original Go register-entry object: %v\n%s", failure, output) + } +} + +func TestAMD64RegisterResultThreeTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + source, declarations, _ := amd64RegisterResultMatrix(t) + requireAMD64RegisterResultGoObject(t, source) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + for _, api := range []string{"text", "module", "Go"} { + t.Run(triple+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + x86ResultInitializationObject(t, llc, triple, text, "register-result") + } + }) + } + } +} + +func TestCrossLinuxRuntimeMatrixAMD64RegisterResultScalarLanes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + source, declarations, checks := amd64RegisterResultMatrix(t) + var header strings.Builder + header.WriteString("#include \n#include \n#include \n") + for index := 0; index < 60; index++ { + fmt.Fprintf(&header, "extern uint64_t lane%d(uint64_t,uint64_t);\n", index) + } + const reference = ` +static int check(uint64_t (*fn)(uint64_t,uint64_t), unsigned width, unsigned lane) { + const uint64_t values[] = {0, UINT64_MAX, UINT64_C(0x8070605040302010), + UINT64_C(0xfedcba9876543210)}; + for (unsigned a = 0; a < 4; a++) { + for (unsigned b = 0; b < 4; b++) { + uint8_t bytes[16] = {0}; + uint64_t low, high; + memcpy(bytes, &values[b], 8); + memcpy(bytes + width * lane, &values[a], width); + memcpy(&low, bytes, 8); + memcpy(&high, bytes + 8, 8); + uint64_t got = fn(values[a], values[b]); + if (got != (low ^ high)) { + fprintf(stderr, "width=%u lane=%u a=%u b=%u got=%llx want=%llx\n", + width, lane, a, b, (unsigned long long)got, (unsigned long long)(low ^ high)); + return 1; + } + } + } + return 0; +} +` + main := header.String() + reference + "int main(void){\n" + checks + "return 0;}\n" + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "register-results", triple, text, main, prefix) + } + }) + } + t.Log("60 register-entry forms × 16 independent byte inputs × 3 APIs × O0/O2; portable lowering, not a physical Go call bridge") +} diff --git a/amd64_register_result_test.go b/amd64_register_result_test.go new file mode 100644 index 000000000..f082834cf --- /dev/null +++ b/amd64_register_result_test.go @@ -0,0 +1,153 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "strings" + "testing" +) + +func amd64RegisterLaneFixture(op string, lane int) string { + destination := "X0" + insert := fmt.Sprintf("%s $%d,AX,X0", op, lane) + if strings.HasPrefix(op, "V") { + destination = "X1" + insert += ",X1" + } + return fmt.Sprintf("TEXT ·view(SB),4,$0-24\nMOVQ BX,X0\n%s\nPEXTRQ $1,%s,CX\nMOVQ %s,AX\nXORQ CX,AX\nRET\n", + insert, destination, destination) +} + +// The explicit register entry has no initialized ABI0 FP fields. This is the +// same seed/insert pattern as the SDK hash routines, without AES or globals. +func TestAMD64RegisterResultScalarLaneThreeAPIs(t *testing.T) { + for _, op := range []string{"PINSRB", "PINSRW", "PINSRD", "PINSRQ", "VPINSRB", "VPINSRW", "VPINSRD", "VPINSRQ"} { + t.Run(op, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, amd64RegisterLaneFixture(op, 255), + "func view(value,seed uint64) uint64\n", "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil || ir == "" { + t.Errorf("%s: register result incorrectly requires ABI0 result storage: %v", api, err) + } + } + }) + } +} + +func TestAMD64RegisterResultKeepsUnknownInputsAndSourceRoles(t *testing.T) { + const declaration = "func view(value,seed uint64) uint64\n" + source := amd64RegisterLaneFixture("PINSRQ", 1) + for name, bad := range map[string]string{ + "ABI0": strings.ReplaceAll(source, "", ""), + "unknown input": strings.ReplaceAll(source, "$1,AX,X0", "$1,R12,X0"), + "unknown return": strings.ReplaceAll(source, "RET", "MOVQ R12,AX\nRET"), + "ABI0 input": strings.ReplaceAll(source, "MOVQ BX,X0", "MOVQ seed+8(FP),X0"), + "ABI0 result": strings.ReplaceAll(source, "RET", "MOVQ AX,ret+16(FP)\nRET"), + "hidden g": strings.ReplaceAll(source, "$1,AX,X0", "$1,R14,X0"), + "hidden closure": strings.ReplaceAll(source, "$1,AX,X0", "$1,DX,X0"), + "physical stack": strings.ReplaceAll(source, "MOVQ BX,X0", "MOVQ (SP),X0"), + "unbound call": strings.ReplaceAll(source, "RET", "CALL ·Unknown(SB)\nRET"), + "path loses value": strings.ReplaceAll(source, "RET", "CMPQ BX,$0\nJE done\nMOVQ R12,AX\ndone:\nRET"), + } { + t.Run(name, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, bad, declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s: unbound entry/input/result was accepted: %v", api, err) + } + } + }) + } + t.Run("modeled timestamp", func(t *testing.T) { + // Its actual AX/DX output is numerical, not an unknown source effect. + known := strings.ReplaceAll(source, "RET", "RDTSC\nRET") + for _, api := range []string{"text", "module", "Go"} { + if ir, err := x86ScalarFPBitViewAPI(t, known, declaration, "amd64", "x86_64-unknown-linux-gnu", api); err != nil || ir == "" { + t.Errorf("%s: modeled timestamp return rejected: %v", api, err) + } + } + }) +} + +func TestAMD64RegisterResultIntegerWidthsAndEveryArgumentRegister(t *testing.T) { + registers := []string{"AX", "BX", "CX", "DI", "SI", "R8", "R9", "R10", "R11"} + for _, width := range []struct{ typ, insert, extract string }{ + {"uint8", "PINSRB", "PEXTRB"}, {"uint16", "PINSRW", "PEXTRW"}, + {"uint32", "PINSRD", "PEXTRD"}, {"uint64", "PINSRQ", "PEXTRQ"}, + } { + for index, reg := range registers { + t.Run(fmt.Sprintf("%s/%s", width.typ, reg), func(t *testing.T) { + parameters := strings.TrimSuffix(strings.Repeat(width.typ+",", index+1), ",") + declaration := fmt.Sprintf("func view(%s) %s\n", parameters, width.typ) + source := fmt.Sprintf("TEXT ·view(SB),4,$0-80\n%s $0,%s,X0\n%s $0,X0,AX\nRET\n", + width.insert, reg, width.extract) + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source, declaration, "amd64", "x86_64-unknown-linux-gnu", api); err != nil { + t.Errorf("%s: initialized selected scalar bytes rejected: %v", api, err) + } + } + }) + } + } +} + +func TestAMD64RegisterResultRejectsUnmodeledTransport(t *testing.T) { + source := amd64RegisterLaneFixture("PINSRQ", 1) + for _, declaration := range []string{ + "func view(value uint8,seed uint64) uint64", // Unknown upper argument bytes. + "func view(value uint16,seed uint64) uint64", + "func view(value uint32,seed uint64) uint64", + "func view(value *uint64,seed uint64) uint64", // Pointer is not numeric. + "func view(value [2]uint64,seed uint64) uint64", + "func view(value float64,seed uint64) uint64", + "func view(value bool,seed uint64) uint64", + "func view(value,seed uint64) *uint64", + "func view(value,seed uint64) bool", + "func view(" + strings.TrimSuffix(strings.Repeat("uint64,", 10), ",") + ") uint64", + } { + t.Run(declaration, func(t *testing.T) { + pkg := mustGoPackage(t, "registerresult", "package registerresult\n"+declaration) + sig, err := DeriveGoFuncABI0Signature("view", pkg.Types.Scope().Lookup("view").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx := &amd64Ctx{fpScalarSource: file.Funcs[0], sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unmodeled transport gained register initialization:", err) + } + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("register role escaped into the standalone ABI0 observer:", err) + } + }) + } +} + +func TestAMD64RegisterResultCompleteTupleKeepsSeparateABI0Observer(t *testing.T) { + // This original source already initializes both numerical result registers: + // AX holds value^seed and BX retains the complete second input. Admission + // by the paired register observer does not initialize either ABI0 FP slot. + const declaration = "func view(value,seed uint64) (uint64,uint64)" + source := amd64RegisterLaneFixture("PINSRQ", 1) + pkg := mustGoPackage(t, "registerresult", "package registerresult\n"+declaration) + sig, err := DeriveGoFuncABI0Signature("view", pkg.Types.Scope().Lookup("view").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewPackageAPI(t, source, pkg, "amd64", "x86_64-unknown-linux-gnu", api); err != nil { + t.Errorf("%s: the existing complete numerical tuple was rejected: %v", api, err) + } + } + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("register result role escaped into the standalone ABI0 observer:", err) + } +} diff --git a/amd64_register_tuple_result_test.go b/amd64_register_tuple_result_test.go new file mode 100644 index 000000000..b361e129e --- /dev/null +++ b/amd64_register_tuple_result_test.go @@ -0,0 +1,350 @@ +package plan9asm + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "go/types" + "strings" + "testing" +) + +type amd64RegisterTupleRuntimeCase struct { + name, declaration string + results []string +} + +func amd64RegisterTupleRuntimeCases() []amd64RegisterTupleRuntimeCase { + return []amd64RegisterTupleRuntimeCase{ + {name: "nine", results: []string{"uint64", "uint64", "uint64", "uint64", "uint64", "uint64", "uint64", "uint64", "uint64"}}, + {name: "widths", results: []string{"uint8", "uint16", "uint32"}}, + {name: "struct", results: []string{"uint64", "uint64", "uint64"}, declaration: "func tuple(seed uint64) (struct{ A,B uint64 },uint64)"}, + {name: "array-one", results: []string{"uint64", "uint64", "uint64"}, declaration: "func tuple(seed uint64) ([1]uint64,uint64,uint64)"}, + {name: "array-zero", results: []string{"uint64", "uint64", "uint64"}, declaration: "func tuple(seed uint64) ([0]uint64,uint64,uint64,uint64)"}, + } +} + +func amd64RegisterTupleRuntimeInputs(t *testing.T, test amd64RegisterTupleRuntimeCase, api string) (string, string) { + t.Helper() + source, declaration := amd64RegisterTupleFixture(test.results) + if test.declaration != "" { + declaration = test.declaration + } + pkg := mustGoPackage(t, "tupleruntime", "package tupleruntime\n"+declaration) + sig, err := DeriveGoFuncABI0Signature("tuple", pkg.Types.Scope().Lookup("tuple").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + ir, err := x86ScalarFPBitViewPackageAPI(t, source, pkg, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal(err) + } + fields, tuple := parseLiteralStructFields(sig.Ret) + if !tuple || len(fields) != len(test.results) || len(sig.Args) != 1 || sig.Args[0] != I64 { + t.Fatalf("runtime shim differs from the actual declared LLVM signature: %+v", sig) + } + var shim strings.Builder + fmt.Fprintf(&shim, "\ndefine void @observe_tuple(ptr %%out, i64 %%seed) {\n %%value = call %s @tuple(i64 %%seed)\n", sig.Ret) + var widths []string + for index, typ := range fields { + bytes := amd64IntegerRegisterResultBytes(typ) + if bytes == 0 { + t.Fatal("non-numerical tuple shim field", typ) + } + widths = append(widths, fmt.Sprint(8*bytes)) + fmt.Fprintf(&shim, " %%v%d = extractvalue %s %%value, %d\n", index, sig.Ret, index) + value := fmt.Sprintf("%%v%d", index) + if typ != I64 { + fmt.Fprintf(&shim, " %%wide%d = zext %s %s to i64\n", index, typ, value) + value = fmt.Sprintf("%%wide%d", index) + } + fmt.Fprintf(&shim, " %%p%d = getelementptr i64, ptr %%out, i64 %d\n store i64 %s, ptr %%p%d, align 8\n", index, index, value, index) + } + shim.WriteString(" ret void\n}\n") + main := strings.ReplaceAll(amd64RegisterTupleValueOracle, "@WIDTHS@", strings.Join(widths, ",")) + main = strings.ReplaceAll(main, "@COUNT@", fmt.Sprint(len(fields))) + return ir + shim.String(), main +} + +const amd64RegisterTupleValueOracle = ` +#include +#include +extern void observe_tuple(uint64_t *, uint64_t); +int main(void) { + const uint64_t guard=UINT64_C(0xabcdef7654321098); + const unsigned widths[]={@WIDTHS@}; + volatile uint64_t seed=UINT64_C(0xfedcba9876543210); + for(unsigned iteration=0;iteration<32;iteration++) { + uint64_t out[11]; + for(unsigned i=0;i<11;i++) out[i]=guard; + if(iteration==0) seed=0; + else if(iteration==1) seed=UINT64_MAX; + else seed=seed*UINT64_C(6364136223846793005)+1; + observe_tuple(out+1,seed); + if(out[0]!=guard) return 2; + for(unsigned field=0;field<@COUNT@;field++) { + uint64_t expected=seed+UINT64_C(28800)+field; + if(widths[field]<64) expected&=(UINT64_C(1)<"} + for name, sig := range map[string]FuncSig{ + "no frame": {Ret: "{ i64, i64, i64 }"}, + "wrong frame type": {Ret: "{ i64, i64, i64 }", Frame: FrameLayout{Results: []FrameSlot{{Type: Ptr}}}}, + "custom registers": {Ret: "{ i64, i64, i64 }", ArgRegs: []Reg{AX}}, + } { + t.Run(name, func(t *testing.T) { + if !x86TypedFPResultNeedsProof(fn, sig, "amd64") { + t.Fatal("integer tuple skipped the common audit because its Frame/role was invalid") + } + }) + } + for _, arch := range []string{"386", "arm", "arm64", "wasm"} { + if x86TypedFPResultNeedsProof(fn, FuncSig{Ret: "{ i64, i64, i64 }"}, arch) { + t.Fatal("AMD64 register tuple restriction changed another architecture:", arch) + } + } + for _, name := range []string{"tuple", "tuple"} { + if x86TypedFPResultNeedsProof(Func{Sym: name}, FuncSig{Ret: "{ i64, i64, i64 }"}, "amd64") { + t.Fatal("register tuple restriction inferred a missing selector:", name) + } + } +} diff --git a/amd64_register_tuple_runtime_linux_amd64_test.go b/amd64_register_tuple_runtime_linux_amd64_test.go new file mode 100644 index 000000000..1de1f77bd --- /dev/null +++ b/amd64_register_tuple_runtime_linux_amd64_test.go @@ -0,0 +1,34 @@ +//go:build linux && amd64 && go1.27 + +package plan9asm + +import ( + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixAMD64RegisterTupleSDKKernel(t *testing.T) { + TestAMD64RegisterTupleResultSDKOriginalGoObject(t) + execution := x86FloatCompareExecutionTools(t) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir := amd64RegisterTupleSDKIR(t, api) + for _, optimized := range []bool{false, true} { + body, label := ir, "IR-O0" + if optimized { + body, label = x86MulDivRoundOptimize(t, ir), "IR-O2" + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(label+"/backend"+level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "sdk-register-tuple", body, amd64RegisterTupleKernelOracle, level) + const expected = "SDK_REGISTER_TUPLE_KERNEL_PASS cases=64 fields=192" + if strings.TrimSpace(string(output)) != expected { + t.Fatalf("independent kernel/errno/tuple/canary oracle: %q", output) + } + t.Log("64 calls and 192 independently expected scalar fields; C uses only a void/pointer output shim") + }) + } + } + }) + } +} diff --git a/amd64_register_tuple_sdk_go127_test.go b/amd64_register_tuple_sdk_go127_test.go new file mode 100644 index 000000000..6ee53d284 --- /dev/null +++ b/amd64_register_tuple_sdk_go127_test.go @@ -0,0 +1,201 @@ +//go:build go1.27 + +package plan9asm + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" +) + +// Read the actual selected SDK declaration and assembly. No guessed ManualSig +// replaces its source-owned seven inputs and three register results. +func amd64RegisterTupleSDKSource(t *testing.T) (GoPackage, string, []byte) { + t.Helper() + const path = "internal/runtime/syscall/linux" + root := runtime.GOROOT() + directory := filepath.Join(root, "src", filepath.FromSlash(path)) + declaration, err := os.ReadFile(filepath.Join(directory, "syscall_linux.go")) + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "syscall_linux.go", declaration, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + var selected *ast.FuncDecl + for _, item := range file.Decls { + function, ok := item.(*ast.FuncDecl) + if ok && function.Name.Name == "Syscall6" { + if selected != nil || function.Recv != nil || function.Body != nil { + t.Fatal("SDK register entry declaration is not unique/bodyless") + } + selected = function + } + } + if selected == nil { + t.Fatal("selected SDK register entry declaration is absent") + } + // This selected declaration uses only built-in uintptr types. Keeping the + // original FuncType avoids importing or rechecking unrelated SDK bodies. + view := &ast.File{Name: file.Name, Decls: []ast.Decl{selected}} + config := types.Config{Sizes: types.SizesFor("gc", "amd64")} + typed, err := config.Check(path, fset, []*ast.File{view}, nil) + if err != nil { + t.Fatal(err) + } + assembly, err := os.ReadFile(filepath.Join(directory, "asm_linux_amd64.s")) + if err != nil { + t.Fatal(err) + } + expanded, err := PreprocessAssemblySource(string(assembly), AssemblyPreprocessOptions{ + FileName: "asm_linux_amd64.s", + ReadInclude: func(parent, name string) (string, []byte, error) { + if name != "textflag.h" { + return "", nil, fmt.Errorf("unexpected SDK include %q", name) + } + name = filepath.Join(root, "pkg", "include", name) + data, err := os.ReadFile(name) + return name, data, err + }, + }) + if err != nil { + t.Fatal(err) + } + t.Logf("actual SDK=%s declarationSHA256=%x assemblySHA256=%x", root, sha256.Sum256(declaration), sha256.Sum256(assembly)) + return GoPackage{Path: path, Types: typed, Syntax: []*ast.File{view}}, expanded, assembly +} + +func TestAMD64RegisterTupleResultSDKOriginalGoObject(t *testing.T) { + pkg, _, source := amd64RegisterTupleSDKSource(t) + dir := t.TempDir() + input, object := filepath.Join(dir, "source.s"), filepath.Join(dir, "source.o") + if err := os.WriteFile(input, source, 0600); err != nil { + t.Fatal(err) + } + root := runtime.GOROOT() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(root, "bin", "go"), "tool", "asm", + "-p", pkg.Path, "-I", filepath.Join(root, "pkg", "include"), "-o", object, input) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + data, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, data, err, ctx.Err(), true, "amd64"); failure != nil { + t.Fatalf("actual original SDK assembly: %v\n%s", failure, data) + } +} + +func amd64RegisterTupleSDKIR(t *testing.T, api string) string { + t.Helper() + pkg, source, _ := amd64RegisterTupleSDKSource(t) + ir, err := x86ScalarFPBitViewPackageAPI(t, source, pkg, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal(err) + } + // C sees a void function and an output pointer. The internal LLVM tuple is + // extracted here; its three-word return is never mistaken for the C ABI. + return ir + ` +define void @observe_sdk_tuple(ptr %out, i64 %n, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6) { + %r = call { i64, i64, i64 } @Syscall6(i64 %n, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6) + %r0 = extractvalue { i64, i64, i64 } %r, 0 + %r1 = extractvalue { i64, i64, i64 } %r, 1 + %r2 = extractvalue { i64, i64, i64 } %r, 2 + %p1 = getelementptr i64, ptr %out, i64 1 + %p2 = getelementptr i64, ptr %out, i64 2 + store i64 %r0, ptr %out, align 8 + store i64 %r1, ptr %p1, align 8 + store i64 %r2, ptr %p2, align 8 + ret void +} +` +} + +func TestAMD64RegisterTupleResultSDKThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir := amd64RegisterTupleSDKIR(t, api) + for _, optimized := range []bool{false, true} { + body := ir + if optimized { + body = x86MulDivRoundOptimize(t, body) + } + x86ResultInitializationObject(t, llc, "x86_64-unknown-linux-gnu", body, "sdk-register-tuple") + } + t.Logf("SDK_REGISTER_TUPLE_IR_BASE64=%s", base64.StdEncoding.EncodeToString([]byte(ir))) + }) + } +} + +const amd64RegisterTupleKernelOracle = ` +#include +#include +#include +#include + +extern void observe_sdk_tuple(uint64_t *, uint64_t, uint64_t, uint64_t, + uint64_t, uint64_t, uint64_t, uint64_t); + +static void reference(uint64_t result[3], uint64_t number, uint64_t a3) { + uint64_t ax, dx; + register uint64_t r10 __asm__("r10") = 0; + register uint64_t r8 __asm__("r8") = 0; + register uint64_t r9 __asm__("r9") = 0; + __asm__ volatile("syscall" + : "=a"(ax), "=d"(dx) + : "0"(number), "D"(UINT64_C(0)), "S"(UINT64_C(0)), "1"(a3), + "r"(r10), "r"(r8), "r"(r9) + : "rcx", "r11", "cc", "memory"); + if (ax >= (uint64_t)-4095) { + result[0] = UINT64_MAX; + result[1] = 0; + result[2] = -ax; + } else { + result[0] = ax; + result[1] = dx; + result[2] = 0; + } +} + +int main(void) { + const uint64_t guard = UINT64_C(0x9acefedb01234567); + const uint64_t numbers[] = {SYS_getpid, UINT64_MAX}; + unsigned cases = 0; + for (unsigned iteration = 0; iteration < 32; ++iteration) { + const uint64_t a3 = UINT64_C(0x7263548190abcdef) ^ iteration; + for (unsigned which = 0; which < 2; ++which) { + uint64_t expected[3]; + uint64_t output[] = {guard, 13, 29, 47, guard}; + errno = EDOM; + reference(expected, numbers[which], a3); + if (errno != EDOM) return 2; + observe_sdk_tuple(output + 1, numbers[which], 0, 0, a3, 0, 0, 0); + if (errno != EDOM || output[0] != guard || output[4] != guard) return 3; + for (unsigned field = 0; field < 3; ++field) { + if (output[field + 1] != expected[field]) { + fprintf(stderr, "case=%u field=%u got=%llx want=%llx\n", which, field, + (unsigned long long)output[field + 1], (unsigned long long)expected[field]); + return 4; + } + } + if (which == 1 && expected[2] != ENOSYS) return 5; + ++cases; + } + } + printf("SDK_REGISTER_TUPLE_KERNEL_PASS cases=%u fields=%u\n", cases, 3 * cases); + return cases == 64 ? 0 : 6; +} +` diff --git a/amd64_register_tuple_values_linux_amd64_test.go b/amd64_register_tuple_values_linux_amd64_test.go new file mode 100644 index 000000000..e77d28896 --- /dev/null +++ b/amd64_register_tuple_values_linux_amd64_test.go @@ -0,0 +1,35 @@ +//go:build linux && amd64 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixAMD64RegisterTupleValues(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + for _, test := range amd64RegisterTupleRuntimeCases() { + for _, api := range []string{"text", "module", "Go"} { + t.Run(test.name+"/"+api, func(t *testing.T) { + ir, main := amd64RegisterTupleRuntimeInputs(t, test, api) + for _, optimized := range []bool{false, true} { + body, label := ir, "IR-O0" + if optimized { + body, label = x86MulDivRoundOptimize(t, ir), "IR-O2" + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(label+"/backend"+level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "register-tuple-values", body, main, level) + expected := fmt.Sprintf("REGISTER_TUPLE_TRANSLATED_PASS fields=%d calls=32", len(test.results)) + if strings.TrimSpace(string(output)) != expected { + t.Fatalf("independent tuple values/truncation/canaries: %q", output) + } + }) + } + } + }) + } + } +} diff --git a/amd64_register_void_test.go b/amd64_register_void_test.go new file mode 100644 index 000000000..c41a0a1ed --- /dev/null +++ b/amd64_register_void_test.go @@ -0,0 +1,274 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// This derives an ordinary language signature, not a compiler/native entry, +// a physical frame, a pointer extent or any initialized memory contents. +func amd64RegisterVoidSource(t *testing.T, declaration, body string) (Func, FuncSig, string) { + t.Helper() + _, sig, source := x86ScalarFPVoidSource(t, declaration, body, "amd64") + source = strings.Replace(source, "·view(SB)", "·view(SB)", 1) + file, err := Parse(ArchAMD64, source) + if err != nil || len(file.Funcs) != 1 { + t.Fatal("ordinary register-void fixture lost its complete source", err) + } + return file.Funcs[0], sig, source +} + +func TestAMD64RegisterVoidKeepsDeclaredInputsAndFullFlow(t *testing.T) { + registers := []Reg{AX, BX, CX, DI, SI, "R8", "R9", "R10", "R11"} + cases := 0 + for _, width := range []struct { + typ, move string + bytes int64 + }{ + {"uint8", "MOVB", 1}, {"uint16", "MOVW", 2}, + {"uint32", "MOVL", 4}, {"uint64", "MOVQ", 8}, + } { + for index, register := range registers { + t.Run(width.typ+"/"+string(register), func(t *testing.T) { + declaration := "func view(" + strings.TrimSuffix(strings.Repeat(width.typ+",", index+1), ",") + ")\n" + body := fmt.Sprintf("%s %s,R12", width.move, register) + fn, sig, _ := amd64RegisterVoidSource(t, declaration, body) + seen, shapes := 0, 0 + err := checkAMD64RegisterResultInitialization(fn, sig, + func(_ int, ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) bool { + if ins.Raw != body { + t.Fatal("unexpected source effect in the input counterexample", ins.Raw) + } + if state.shapeOnly { + shapes++ + return x86ScalarFPResultTransfer(ins, state, owned, resultOffset, arch) + } + seen++ + for carrier, reg := range registers { + position, _, valid := x86ResultGP(reg, width.bytes, arch) + if !valid { + t.Fatal("invalid original argument carrier", reg) + } + want := uint8(0) + if carrier <= index { + want = uint8(1<") + case "manual-registers": + sig.ArgRegs = []Reg{AX} + case "result-without-slot": + sig.Ret = I64 + case "slot-on-Void": + sig.Frame.Results = append(sig.Frame.Results, sig.Frame.Params[0]) + case "raw": + fn.Instrs[1].x86Encoded = true + case "physical-address-mode": + fn.Instrs[1].x86AddressBits = 32 + case "WASM-entry": + sig.WASMContext = "go" + } + calls := 0 + err := checkAMD64RegisterResultInitialization(fn, sig, + func(_ int, ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) bool { + calls++ + return x86ScalarFPResultTransfer(ins, state, owned, resultOffset, arch) + }) + if !errors.Is(err, ErrProbeNeedsContext) || calls != 0 { + t.Fatal("Void admitted contradictory result/entry metadata", calls, err) + } + }) + } +} diff --git a/amd64_scalar_arithmetic_native.go b/amd64_scalar_arithmetic_native.go new file mode 100644 index 000000000..228f4a0d2 --- /dev/null +++ b/amd64_scalar_arithmetic_native.go @@ -0,0 +1,81 @@ +package plan9asm + +import "fmt" + +// The existing whole Go-table grammar supplies operand width, encoded byte +// aliases and carry roles. This query selects only its memory-to-register +// arithmetic/logical rows; it supplies no memory or entry capability. +func x86ScalarArithmeticNativeReadProperties(ins Instr, arch string) (x86LockScalarRMWForm, bool, error) { + form, known, err := x86ScalarArithmeticReadProperties(ins, arch) + if !known || err != nil { + return form, known, err + } + return form, x86OrdinaryNativeMemoryOperand(form.source, arch), nil +} + +func x86ScalarArithmeticReadProperties(ins Instr, arch string) (x86LockScalarRMWForm, bool, error) { + form, known, err := x86LockScalarRMWProperties(ins.Op, ins, arch) + if !known { + return form, false, nil + } + if err != nil { + return form, true, err + } + switch form.spec.stem { + case "ADD", "SUB", "ADC", "SBB", "AND", "OR", "XOR": + default: + return form, false, nil + } + if form.source.Kind != OpMem || form.source.Mem.Sym != "" || + form.destination.Kind != OpReg { + return form, false, nil + } + return form, true, nil +} + +func (c *amd64Ctx) evalScalarArithmeticSource(ins Instr, typ LLVMType) (string, Operand, error) { + form, known, err := x86ScalarArithmeticReadProperties(ins, c.goarch) + if err != nil { + return "", Operand{}, err + } + if !known || !c.nativeMemoryOperand(ins, 0) { + value, err := c.evalIntSized(ins.Args[0], typ) + return value, ins.Args[1], err + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + form.destination.Reg != ins.Args[1].Reg && (form.destination.Reg == SP || form.destination.Reg == BP) { + return "", Operand{}, fmt.Errorf("%w: scalar arithmetic needs its original native address/register contract", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.source) + if err != nil { + return "", Operand{}, err + } + value := c.newTmp() + // Preserve the fault and exact read even when all numerical results and + // flags are discarded. Do not assert alignment, bounds or pointee validity. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", value, typ, pointerType, pointer) + return "%" + value, form.destination, nil +} + +// Only the lowering paired with the exact volatile source load may consume +// its numerical continuation. The destination and incoming carry still need +// every source byte/flag. Public declaration and caller observers stay strict. +func x86LoweredScalarArithmeticNativeRead(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) (bool, bool) { + form, known, err := x86ScalarArithmeticNativeReadProperties(ins, arch) + if !known || err != nil { + return false, known + } + width := int64(form.spec.bits / 8) + index, _, valid := x86ResultGP(form.destination.Reg, width, arch) + if !valid || state.pointerGP[index] || + form.spec.carryInput && state.flags&x86ByteFlagCF == 0 || + !x86ResultRead(form.destination, width, state, owned, arch) || + !x86ResultReadWithNativeGPContinuation(ins, form.source, width, state, owned, arch) { + return false, true + } + if !x86ResultStore(form.destination, width, state, owned, resultOffset, arch) { + return false, true + } + state.flags = state.flags&^form.spec.flagsKilled | form.spec.flagsDefined + return true, true +} diff --git a/amd64_scalar_arithmetic_native_result_test.go b/amd64_scalar_arithmetic_native_result_test.go new file mode 100644 index 000000000..1b91f7228 --- /dev/null +++ b/amd64_scalar_arithmetic_native_result_test.go @@ -0,0 +1,264 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +// Keep this inventory independent of the production descriptors. These are +// all four widths of Go's add/subtract, carry/borrow and logical families. +var x86ScalarArithmeticNativeNames = []string{ + "ADDB", "ADDW", "ADDL", "ADDQ", + "SUBB", "SUBW", "SUBL", "SUBQ", + "ADCB", "ADCW", "ADCL", "ADCQ", + "SBBB", "SBBW", "SBBL", "SBBQ", + "ANDB", "ANDW", "ANDL", "ANDQ", + "ORB", "ORW", "ORL", "ORQ", + "XORB", "XORW", "XORL", "XORQ", +} + +func TestX86ScalarArithmeticNativeSparseBytesFlagsAndOrigins(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, op := range x86ScalarArithmeticNativeNames { + width := map[byte]int64{'B': 1, 'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + if arch == "386" && width == 8 { + continue + } + t.Run(arch+"/"+op, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, op+" -1(DX)(BX*1),CX") + base, _, _ := x86ResultGP(DX, 4, arch) + index, _, _ := x86ResultGP(BX, 4, arch) + destination, mask, _ := x86ResultGP(CX, width, arch) + for seed := 0; seed < 256; seed++ { + for flags := uint8(0); flags <= x86ByteFlagsAll; flags++ { + state := x86ResultState{flags: flags} + state.pointerGP[base], state.gp[index], state.gp[destination] = true, 0xff, uint8(seed) + want := state.clone() + accepted := uint8(seed)&mask == mask && + (!strings.HasPrefix(op, "ADC") && !strings.HasPrefix(op, "SBB") || flags&x86ByteFlagCF != 0) + if accepted { + want.flags = x86ByteFlagsAll + if strings.HasPrefix(op, "AND") || strings.HasPrefix(op, "OR") || strings.HasPrefix(op, "XOR") { + want.flags &^= x86ByteFlagAF // Logical AF is undefined, not preserved. + } + if width == 4 { + want.gp[destination] = 0xff + } + } + if got, known := x86LoweredScalarArithmeticNativeRead(ins, &state, nil, 0, arch); !known || got != accepted || !reflect.DeepEqual(state, want) { + t.Fatal("arithmetic changed byte/FLAGS roles", seed, flags, got, accepted, state, want) + } + } + } + for _, origin := range []string{"decoded", "raw", "raw return", "address", "unbound address", "segment", "pointer destination"} { + changed := ins + state := x86ResultState{flags: x86ByteFlagsAll} + state.pointerGP[base], state.gp[index], state.gp[destination] = true, 0xff, 0xff + switch origin { + case "decoded": + changed.x86Encoded = true + case "raw": + changed.x86SourceRaw = new(x86SourceRawInstruction) + case "raw return": + changed.x86RawNearReturn = new(x86RawNearReturnForm) + case "address": + changed.x86AddressBits = 32 + case "unbound address": + changed.x86UnboundAddressOverride = true + case "segment": + changed.x86SegmentPrefix = 0x64 + case "pointer destination": + state.pointerGP[destination] = true + } + before := state.clone() + if accepted, known := x86LoweredScalarArithmeticNativeRead(changed, &state, nil, 0, arch); !known || accepted || !reflect.DeepEqual(state, before) { + t.Fatal("unbound origin acquired native arithmetic", origin, state) + } + } + }) + } + } +} + +func TestX86ScalarArithmeticNativeOriginalREXByteAliases(t *testing.T) { + for _, row := range []struct { + op string + byte byte + }{{"ADDB", 0x02}, {"SUBB", 0x2a}, {"ADCB", 0x12}, {"SBBB", 0x1a}, + {"ANDB", 0x22}, {"ORB", 0x0a}, {"XORB", 0x32}} { + for ordinal, reg := range []string{"AH", "CH", "DH", "BH"} { + for _, address := range []string{"(DX)", "(R8)", "(DX)(R9*2)"} { + instruction := row.op + " " + address + "," + reg + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+instruction+"\nRET\n") + want := []byte{row.byte, byte(0x22 + ordinal*8), 0xc3} + effective := Reg(reg) + if address == "(R8)" { + want = []byte{0x41, row.byte, byte(0x20 + ordinal*8), 0xc3} + } else if address == "(DX)(R9*2)" { + want = []byte{0x42, row.byte, byte(0x24 + ordinal*8), 0x4a, 0xc3} + } + if address != "(DX)" { + effective = []Reg{SP, BP, SI, DI}[ordinal] + } + if !reflect.DeepEqual(code, want) { + t.Fatalf("original REX/ModRM view changed: %s: %x != %x", instruction, code, want) + } + ins := x86ScalarLaneInstruction(t, instruction) + form, known, err := x86ScalarArithmeticNativeReadProperties(ins, "amd64") + if !known || err != nil || form.destination.Reg != effective { + t.Fatal("typed destination differs from actual Go encoding", instruction, form, err) + } + if address != "(DX)" && ordinal < 2 { + c, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Name: "example.Value", Ret: Void}, nil) + before := output.String() + if _, _, err := c.evalScalarArithmeticSource(ins, I8); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatal("arithmetic manufactured a physical SP/BP alias", err) + } + } + } + } + } +} + +func TestX86ScalarArithmeticNativeResultsThreeAPIFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, op := range x86ScalarArithmeticNativeNames { + if target.arch == "386" && strings.HasSuffix(op, "Q") { + goAssemblerOracleOutput(t, target.arch, "TEXT f(SB),4,$0-0\n"+op+" (DX),CX\nRET\n", "", false) + continue + } + for _, address := range []string{"(DX)", "-1(DX)(BX*1)"} { + body := "CMPL BX,BX\n" + op + " " + address + ",CX" + asm, decl := x86ExtensionNativeFixture(fmt.Sprintf("F%d", count), target.arch, body, false) + source.WriteString(asm) + declarations.WriteString(strings.ReplaceAll(decl, "*[4]byte", "*[16]byte")) + count++ + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("native arithmetic lost its paired numerical continuation", api, err) + } + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + if strings.Count(text, "load volatile i") != count { + t.Fatal("native arithmetic lost its exact observable reads", api, optimized, count) + } + compileLLVMToObject(t, llc, target.triple, "arithmetic-native.ll", "arithmetic-native.o", text) + } + } + }) + } +} + +func TestX86ScalarArithmeticDeadNativeReadsSurviveOptimization(t *testing.T) { + for _, op := range x86ScalarArithmeticNativeNames { + t.Run(op, func(t *testing.T) { + source, declaration := x86ExtensionNativeFixture("Value", "amd64", "CMPL BX,BX\n"+op+" (DX),CX", true) + goAssemblerOracleOutput(t, "amd64", source, "", true) + width := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[op[len(op)-1]] + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal(err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + if strings.Count(text, fmt.Sprintf("load volatile i%d,", width)) != 1 { + t.Errorf("%s dead native read must retain exactly %d bits", api, width) + } + } + } + }) + } +} + +func TestX86ScalarArithmeticNativeRetainsPublicAndSourceBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word, move, triple := 8, "MOVQ", "x86_64-unknown-linux-gnu" + if arch == "386" { + word, move, triple = 4, "MOVL", "i386-unknown-linux-gnu" + } + for _, op := range x86ScalarArithmeticNativeNames { + if arch == "386" && strings.HasSuffix(op, "Q") { + continue + } + t.Run(arch+"/"+op, func(t *testing.T) { + body := fmt.Sprintf("%s p+0(FP),DX\nMOVL $0,CX\nCMPL CX,CX\n%s (DX),CX\nMOVL CX,ret0+%d(FP)\nRET", move, op, word) + _, file, sig, _ := x86ResultInitializationFixture(t, arch, "func Value(p *[16]byte) uint32", body) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unpaired declaration check acquired native storage", err) + } + } + for name, body := range map[string]string{ + "unknown base": op + " (DI),CX", + "numeric base": "MOVL $42,DX\n" + op + " (DX),CX", + "unknown index": op + " (DX)(SI*1),CX", + "pointer index": op + " (DX)(DX*1),CX", + "unknown result": op + " (DX),DI\n" + move + " DI,CX", + "pointer result": op + " (DX),DX\n" + move + " DX,CX", + "native store": op + " CX,(DX)", + "unknown store data": op + " DI,(DX)", + "unknown store base": op + " CX,(DI)", + "destroyed pointer": "MOVL $42,DX\n" + op + " (DX),CX", + } { + t.Run(name, func(t *testing.T) { + source, declaration := x86ExtensionNativeFixture("Value", arch, "CMPL BX,BX\n"+body, false) + goAssemblerOracleOutput(t, arch, source, "", true) + if name == "numeric base" || name == "destroyed pointer" { + x86RequireNumericNativeSourceContext(t, source, declaration, arch) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, arch, triple, api) + if name == "numeric base" || name == "destroyed pointer" { + bits := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[op[len(op)-1]] + if err != nil || strings.Count(ir, fmt.Sprintf("load volatile i%d,", bits)) != 1 { + t.Fatal("complete NUM address lost its exact scalar arithmetic read", api, err) + } + x86NumericNativeAddressObject(t, ir, triple) + continue + } + if name == "native store" { + // The paired RMW consumer now models this complete + // read/write. Preserve its exact former negative and + // require independent missing-input controls above. + if err != nil || ir == "" { + t.Fatal("complete paired native RMW lost its inputs", api, err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + if strings.Count(text, "load volatile") != 1 || strings.Count(text, "store volatile") != 1 { + t.Fatal("native RMW lost its exact observable read/write", api) + } + compileLLVMToObject(t, llc, triple, "paired-rmw.ll", "paired-rmw.o", text) + } + continue + } + if ir != "" || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native arithmetic acquired an unproved source effect", api, err) + } + } + }) + } + }) + } + } +} diff --git a/amd64_scalar_arithmetic_native_runtime_unix_test.go b/amd64_scalar_arithmetic_native_runtime_unix_test.go new file mode 100644 index 000000000..a7179b0e3 --- /dev/null +++ b/amd64_scalar_arithmetic_native_runtime_unix_test.go @@ -0,0 +1,209 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ScalarArithmeticNativeResultsAndFaults(t *testing.T) { + x86ScalarArithmeticNativeRuntime(t, false) +} + +func TestCrossLinuxRuntimeMatrixX86ScalarArithmeticNativeFlagsAndAliases(t *testing.T) { + x86ScalarArithmeticNativeRuntime(t, true) +} + +func x86ScalarArithmeticNativeRuntime(t *testing.T, flagsAndAliases bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, originals, rows strings.Builder + count := 0 + appendKernel := func(op string, width, carry int, dead, flags bool, alias string) { + name, original := fmt.Sprintf("F%d", count), fmt.Sprintf("Original%d", count) + comparison := "CMPL BX,BX\n" + nativeComparison := "MOVL $1,R10\nCMPL R10,R10\n" + if carry != 0 { + comparison = "CMPL BX,$2\n" + nativeComparison = "MOVL $1,R10\nCMPL R10,$2\n" + } + body := comparison + op + " -1(DX)(BX*1),CX" + native := "MOVQ $0x87654321fedcba98,AX\n" + nativeComparison + op + " (DI),AX\n" + mode := 0 // value result + if flags { + body += "\n" + x86ScalarArithmeticNativeFlagObserver + "\nMOVQ AX,CX" + native += x86ScalarArithmeticNativeFlagObserver + "\n" + mode = 1 + } + if alias != "" { + effective := "SI" + if alias == "BH" { + effective = "DI" + } + body = "MOVQ DX,R8\nMOVQ $0x87654321fedcba98," + effective + "\n" + comparison + op + " (R8)," + alias + "\nMOVQ " + effective + ",CX" + native = "MOVQ DI,R8\nMOVQ $0x87654321fedcba98," + effective + "\n" + nativeComparison + op + " (R8)," + alias + "\nMOVQ " + effective + ",AX\n" + } + if dead { + body += "\nMOVQ $0x87654321fedcba98,CX" + native += "MOVQ $0x87654321fedcba98,AX\n" + } + assembly, declaration := x86ExtensionNativeFixture(name, "amd64", body, false) + source.WriteString(assembly) + declarations.WriteString(strings.ReplaceAll(declaration, "*[4]byte", "*[16]byte")) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+native+"RET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\nextern uint64_t %s(const uint8_t*);\n", name, original) + operation, discarded := 0, 0 + if strings.HasPrefix(op, "SUB") || strings.HasPrefix(op, "SBB") { + operation = 1 + } else if strings.HasPrefix(op, "AND") { + operation = 2 + } else if strings.HasPrefix(op, "OR") { + operation = 3 + } else if strings.HasPrefix(op, "XOR") { + operation = 4 + } + if dead { + discarded = 1 + } + effectiveCarry := 0 + if strings.HasPrefix(op, "ADC") || strings.HasPrefix(op, "SBB") { + effectiveCarry = carry + } + fmt.Fprintf(&rows, "{%s,%s,%d,%d,%d,%d,%d},\n", name, original, width, operation, effectiveCarry, discarded, mode) + count++ + } + for _, op := range x86ScalarArithmeticNativeNames { + width := map[byte]int{'B': 1, 'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + for carry := 0; carry < 2; carry++ { + if flagsAndAliases { + appendKernel(op, width, carry, false, true, "") + if width == 1 { + for _, alias := range []string{"DH", "BH"} { + appendKernel(op, width, carry, false, false, alias) + } + } + } else { + for _, dead := range []bool{false, true} { + appendKernel(op, width, carry, dead, false, "") + } + } + } + } + wantCount := 112 + if flagsAndAliases { + wantCount = 84 + } + if count != wantCount { + t.Fatal("incomplete arithmetic/native state matrix", count, wantCount) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + // Reuse only the independent process/guard-page machinery. The arithmetic + // reference below does not consume a production descriptor or lowering. + start := strings.Index(x86ExtensionNativeRuntimeC, "static uint64_t reference(") + end := strings.Index(x86ExtensionNativeRuntimeC, "static int fault(") + if start < 0 || end <= start { + t.Fatal("missing complete fault-harness boundary") + } + driverTemplate := x86ExtensionNativeRuntimeC[:start] + x86ScalarArithmeticNativeReferenceC + "\n" + x86ExtensionNativeRuntimeC[end:] + driverTemplate = strings.NewReplacer( + "unsigned width, destination, sign, dead;", "unsigned width, operation, carry, dead, flags;", + "uint8_t bytes[8];", "uint8_t bytes[16];", + "const uint8_t *p=bytes+(pattern&1);", ` + uint8_t *p=bytes+(pattern&1); + uint64_t mask=k.width==8?UINT64_MAX:(UINT64_C(1)<<(k.width*8))-1; + uint64_t sign=UINT64_C(1)<<(k.width*8-1), a=seed&mask; + const uint64_t edges[]={0,1,mask,mask-1,sign,sign-1,a,a-1,a+1,~a,-a,-a-1}; + if(pattern>(8*b)); + }`, + "28 extension kernels", fmt.Sprintf("%d scalar arithmetic kernels", count), + ).Replace(driverTemplate) + driver := fmt.Sprintf(driverTemplate, prototypes.String(), originals.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text, level := ir, "-O0" + if optimized { + text, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + output := execution.compileAndRunLLVM(t, "scalar-arithmetic-native", text, driver, level) + want := fmt.Sprintf("%d scalar arithmetic kernels: numerical %d, fault %d, tripwires 2; failures 0\n", count, count*128, count*12) + if string(output) != want { + t.Fatal("original Go/LLVM arithmetic differs", string(output), want) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } +} + +// Capture every SETcc-observable status bit before combining them with +// flag-writing arithmetic. MOV/SETcc preserve the original flags. +const x86ScalarArithmeticNativeFlagObserver = `MOVL $0,AX +MOVL $0,SI +MOVL $0,DI +MOVL $0,R10 +MOVL $0,R11 +SETCS AL +SETEQ SIB +SETMI DIB +SETOS R10B +SETPS R11B +SHLL $1,SI +SHLL $2,DI +SHLL $3,R10 +SHLL $4,R11 +ORL SI,AX +ORL DI,AX +ORL R10,AX +ORL R11,AX` + +const x86ScalarArithmeticNativeReferenceC = ` +static uint64_t reference(Kernel k, const uint8_t *p) { + if(k.dead) return seed; + uint64_t input=0; + for(unsigned b=0;b>bit)&1; + return cf|((value==0)<<1)|(((value&sign)!=0)<<2)|(of<<3)|((ones%%2==0)<<4); + } + if(k.width<4) return (seed&~mask)|value; + return value; +} +` diff --git a/amd64_scalar_bitcount.go b/amd64_scalar_bitcount.go new file mode 100644 index 000000000..0fdc22f28 --- /dev/null +++ b/amd64_scalar_bitcount.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ScalarBitCountKind uint8 + +const ( + x86PopulationCount x86ScalarBitCountKind = iota + x86LeadingZeroCount + x86TrailingZeroCount +) + +type x86ScalarBitCountSpec struct { + bits int + kind x86ScalarBitCountKind +} + +// Complete Go yml_rl/ycrc32l scalar count rows. Shape, lowering and numerical +// byte effects share this table; it grants no native memory/caller permission. +var x86ScalarBitCountSpecs = map[Op]x86ScalarBitCountSpec{ + "POPCNTW": {16, x86PopulationCount}, "POPCNTL": {32, x86PopulationCount}, "POPCNTQ": {64, x86PopulationCount}, + "LZCNTW": {16, x86LeadingZeroCount}, "LZCNTL": {32, x86LeadingZeroCount}, "LZCNTQ": {64, x86LeadingZeroCount}, + "TZCNTW": {16, x86TrailingZeroCount}, "TZCNTL": {32, x86TrailingZeroCount}, "TZCNTQ": {64, x86TrailingZeroCount}, +} + +func x86ScalarBitCountProperties(op Op, ins Instr, arch string) (x86ScalarBitCountSpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86ScalarBitCountSpecs[Op(name)] + if !known { + return spec, false, nil + } + fail := func() (x86ScalarBitCountSpec, bool, error) { + return spec, true, fmt.Errorf("%s %s outside Go scalar bit-count grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || arch == "386" && spec.bits == 64 || dotted || + len(ins.Args) != 2 || ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return fail() + } + source, destination := ins.Args[0], ins.Args[1] + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, arch) { + return fail() + } + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, arch) { + return fail() + } + } else if !isAMD64MemoryOperand(source) || + source.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(source.Sym), "$") { + return fail() + } + if source.Kind == OpMem && !x86MemoryRegistersValidForArch(source.Mem, arch) { + return fail() + } + return spec, true, nil +} + +func x86TypedScalarBitCountByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + spec, known, err := x86ScalarBitCountProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + defined := x86ByteFlagCF | x86ByteFlagZF + if spec.kind == x86PopulationCount { + defined = x86ByteFlagsAll + } + width := int64(spec.bits / 8) + return x86TypedByteEffect{ + operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: width, read: true}, + {index: 1, operand: ins.Args[1], width: width, write: true}, + }, + flagsDefined: defined, + flagsKilled: x86ByteFlagsAll, + }, true, nil +} + +func (c *amd64Ctx) lowerScalarBitCount(op Op, ins Instr) (bool, bool, error) { + spec, known, err := x86ScalarBitCountProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + typ := amd64IntegerTypeForBits(spec.bits) + var value string + if isAMD64MemoryOperand(ins.Args[0]) { + bytes, readErr := c.loadX86PackedShiftBytes(ins.Args[0], spec.bits/8, true) + err = readErr + if err == nil { + integer := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", integer, spec.bits/8, bytes, typ) + value = "%" + integer + } + } else { + value, err = c.evalIntSized(ins.Args[0], typ) + } + if err != nil { + return true, false, err + } + count := c.newTmp() + if spec.kind == x86PopulationCount { + fmt.Fprintf(c.b, " %%%s = call %s @llvm.ctpop.%s(%s %s)\n", count, typ, typ, typ, value) + } else { + intrinsic := "cttz" + if spec.kind == x86LeadingZeroCount { + intrinsic = "ctlz" + } + fmt.Fprintf(c.b, " %%%s = call %s @llvm.%s.%s(%s %s, i1 false)\n", count, typ, intrinsic, typ, typ, value) + } + inputZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", inputZero, typ, value) + if spec.kind == x86PopulationCount { + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", inputZero, c.flagsZSlot) + c.clearX86StatusFlags(x86StatusAll &^ x86StatusZF) + } else { + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", inputZero, c.flagsCFSlot) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, 0\n", zero, typ, count) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + } + return true, false, c.storeRegSized(ins.Args[1].Reg, typ, "%"+count) +} + +// Only the paired exact volatile emitter supplies native numerical bytes. +// Widths, writes and defined/killed FLAGS remain the complete typed effect; +// standalone queries and callers acquire no pointer or memory permission. +func x86LoweredScalarBitCountNativeRead(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) (bool, bool) { + effect, known, err := x86TypedScalarBitCountByteEffect(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + if ins.Args[0].Kind != OpMem || ins.Args[0].Mem.Sym != "" { + return false, false + } + for _, operand := range effect.operands { + if operand.read && !x86ResultReadWithNativeContinuation(ins, operand.operand, operand.width, state, owned, arch, true) { + return false, true + } + } + for _, operand := range effect.operands { + if operand.write && !x86ResultStore(operand.operand, operand.width, state, owned, resultOffset, arch) { + return false, true + } + } + state.flags = state.flags&^effect.flagsKilled | effect.flagsDefined + return true, true +} diff --git a/amd64_scalar_bitcount_test.go b/amd64_scalar_bitcount_test.go new file mode 100644 index 000000000..02b8db39c --- /dev/null +++ b/amd64_scalar_bitcount_test.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86ScalarBitCountCompleteGoRows(t *testing.T) { + table, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := []string{ + "{ALZCNTL, yml_rl, Pf3, opBytes{0xbd}}", + "{ALZCNTQ, yml_rl, Pfw, opBytes{0xbd}}", + "{ALZCNTW, yml_rl, Pef3, opBytes{0xbd}}", + "{APOPCNTW, yml_rl, Pef3, opBytes{0xb8}}", + "{APOPCNTL, yml_rl, Pf3, opBytes{0xb8}}", + "{APOPCNTQ, yml_rl, Pfw, opBytes{0xb8}}", + "{ATZCNTW, ycrc32l, Pe, opBytes{0xf3, 0x0f, 0xbc, 0}}", + "{ATZCNTL, ycrc32l, Px, opBytes{0xf3, 0x0f, 0xbc, 0}}", + "{ATZCNTQ, ycrc32l, Pw, opBytes{0xf3, 0x0f, 0xbc, 0}}", + } + if len(x86ScalarBitCountSpecs) != len(rows) { + t.Fatal("incomplete scalar count family") + } + for _, row := range append(rows, + "var yml_rl = []ytab{\n\t{Zm_r, 1, argList{Yml, Yrl}},\n}", + "var ycrc32l = []ytab{\n\t{Zlitm_r, 0, argList{Yml, Yrl}},\n}") { + if !strings.Contains(string(table), row) { + t.Fatal("complete original Go encoding row changed:", row) + } + } + for _, family := range []string{"POPCNT", "LZCNT", "TZCNT"} { + for _, suffix := range []string{"W", "L", "Q"} { + op := Op(family + suffix) + for _, arch := range []string{"amd64", "386"} { + for _, input := range []string{"AX", "AL", "X0", "$3", "$table(SB)", "4(AX)(CX*2)", "table(SB)"} { + for _, output := range []string{"BX", "AH", "X1", "4(BX)"} { + source := fmt.Sprintf("TEXT probe(SB),4,$0-0\n%s %s,%s\nRET\n", op, input, output) + valid := !(arch == "386" && suffix == "Q") && output == "BX" && + (input == "AX" || input == "4(AX)(CX*2)" || input == "table(SB)") + goAssemblerOracleOutput(t, arch, source, "", valid) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, known, err := x86ScalarBitCountProperties(op, file.Funcs[0].Instrs[1], arch) + if !known || (err == nil) != valid { + t.Fatalf("%s %s %s,%s: known=%t error=%v", arch, op, input, output, known, err) + } + } + } + } + } + } +} + +func TestX86ScalarBitCountFiveTargetThreeAPIObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86OrderingHintTargets { + word, typ, move := 8, "uint64", "MOVQ" + if target.arch == "386" { + word, typ, move = 4, "uint32", "MOVL" + } + var declarations, source strings.Builder + declarations.WriteString("package ordering\n") + for _, family := range []string{"POPCNT", "LZCNT", "TZCNT"} { + for _, suffix := range []string{"W", "L", "Q"} { + if target.arch == "386" && suffix == "Q" { + continue // Independently Go-rejected above, not a skipped test. + } + op := family + suffix + name := strings.ToLower(op) + fmt.Fprintf(&declarations, "func %s(x %s) %s\n", name, typ, typ) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s x+0(FP),AX\n%s $0,BX\n%s AX,BX\n%s BX,ret0+%d(FP)\nRET\n", + name, 2*word, move, move, op, move, word) + } + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + dir := t.TempDir() + input, object := filepath.Join(dir, "count.ll"), filepath.Join(dir, "count.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+target.triple, "-filetype=obj", input, "-o", object)) + } + }) + } + } +} diff --git a/amd64_scalar_bitfield_forms.go b/amd64_scalar_bitfield_forms.go new file mode 100644 index 000000000..3e5bd3b33 --- /dev/null +++ b/amd64_scalar_bitfield_forms.go @@ -0,0 +1,77 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ScalarBitFieldKind uint8 + +const ( + x86BitFieldExtract x86ScalarBitFieldKind = iota + x86BitFieldZeroHigh +) + +type x86ScalarBitFieldSpec struct { + kind x86ScalarBitFieldKind + bits int + controlBytes int64 + flagsDefined uint8 +} + +// BEXTR and BZHI share the complete Go _ybextrl grammar, not their control +// bytes or FLAGS. W selects a qword only in mode64; mode32 keeps the spelling. +var x86ScalarBitFieldSpecs = map[Op]x86ScalarBitFieldSpec{ + "BEXTRL": {x86BitFieldExtract, 32, 2, x86ByteFlagCF | x86ByteFlagOF | x86ByteFlagZF}, + "BEXTRQ": {x86BitFieldExtract, 64, 2, x86ByteFlagCF | x86ByteFlagOF | x86ByteFlagZF}, + "BZHIL": {x86BitFieldZeroHigh, 32, 1, x86ByteFlagCF | x86ByteFlagOF | x86ByteFlagZF | x86ByteFlagSF}, + "BZHIQ": {x86BitFieldZeroHigh, 64, 1, x86ByteFlagCF | x86ByteFlagOF | x86ByteFlagZF | x86ByteFlagSF}, +} + +type x86ScalarBitFieldForm struct { + bits int + control, source, destination Operand + spec x86ScalarBitFieldSpec +} + +func x86ScalarBitFieldProperties(op Op, ins Instr, arch string) (x86ScalarBitFieldForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86ScalarBitFieldSpecs[Op(name)] + form := x86ScalarBitFieldForm{bits: spec.bits, spec: spec} + if !known { + return form, false, nil + } + if arch == "386" { + form.bits = 32 + } + fail := func() (x86ScalarBitFieldForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go scalar bit-field grammar: %q", arch, op, ins.Raw) + } + if arch != "386" && arch != "amd64" || dotted || len(ins.Args) != 3 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return fail() + } + form.control, form.source, form.destination = ins.Args[0], ins.Args[1], ins.Args[2] + for _, operand := range []Operand{form.control, form.destination} { + if operand.Kind != OpReg || !isX86YrlRegisterForArch(operand.Reg, arch) { + return fail() + } + } + if form.source.Kind == OpReg { + if !isX86YrlRegisterForArch(form.source.Reg, arch) { + return fail() + } + } else if !isAMD64MemoryOperand(form.source) || !x86PackedShiftMemoryValid(form.source, arch) || + x86NamedMultiplyMemoryError(form.source, arch) != nil { + return fail() + } + return form, true, nil +} + +func x86ScalarBitFieldFamilyProperties(op Op, ins Instr, arch string, kind x86ScalarBitFieldKind) (x86ScalarBitFieldForm, bool, error) { + form, known, err := x86ScalarBitFieldProperties(op, ins, arch) + if !known || form.spec.kind != kind { + return form, false, nil + } + return form, true, err +} diff --git a/amd64_scalar_bitscan.go b/amd64_scalar_bitscan.go new file mode 100644 index 000000000..8c2118205 --- /dev/null +++ b/amd64_scalar_bitscan.go @@ -0,0 +1,37 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ScalarBitScanSpec struct { + bits int + reverse bool +} + +// All six Go yml_rl rows. Zero input cannot independently initialize the +// destination: current Intel SDM preserves it, with a historical dword upper +// half exception. Only a whole native opcode retains that target's behavior. +var x86ScalarBitScanSpecs = map[Op]x86ScalarBitScanSpec{ + "BSFW": {16, false}, "BSFL": {32, false}, "BSFQ": {64, false}, + "BSRW": {16, true}, "BSRL": {32, true}, "BSRQ": {64, true}, +} + +func x86ScalarBitScanProperties(op Op, ins Instr, arch string) (x86ScalarBitScanSpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86ScalarBitScanSpecs[Op(name)] + if !known { + return spec, false, nil + } + // This is exactly the same complete Yml-to-Yrl operand grammar as the + // registered count family. Reuse only its shape, never its defined value + // or status flags, and preserve the original source instruction identity. + shape := ins + shape.Op = Op(fmt.Sprintf("POPCNT%s", name[len(name)-1:])) + _, _, err := x86ScalarBitCountProperties(shape.Op, shape, arch) + if dotted || ins.Op != "" && !strings.EqualFold(string(ins.Op), name) || err != nil { + return spec, true, fmt.Errorf("%s %s outside Go scalar bit-scan grammar: %q", arch, op, ins.Raw) + } + return spec, true, nil +} diff --git a/amd64_scalar_bitscan_execution_driver_test.go b/amd64_scalar_bitscan_execution_driver_test.go new file mode 100644 index 000000000..c429bf16f --- /dev/null +++ b/amd64_scalar_bitscan_execution_driver_test.go @@ -0,0 +1,382 @@ +package plan9asm + +import ( + "crypto/sha256" + "errors" + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86BitScanOriginalExecutionDriver(t *testing.T, fixture x86BitScanExecutionFixture) string { + t.Helper() + typ := "uint64" + if fixture.arch == "386" { + typ = "uint32" + } + var registrations, inputs, priors strings.Builder + for _, row := range fixture.rows { + public, observe := "Guard"+row.name, "nil" + if row.role >= 3 { + public, observe = "nil", row.name + } + fmt.Fprintf(®istrations, "{%s,%s,%d},\n", public, observe, row.role) + } + for _, value := range x86BitScanExecutionValues { + fmt.Fprintf(&inputs, "0x%x,", value) + } + for _, prior := range x86BitScanExecutionPriors { + fmt.Fprintf(&priors, "0x%x,", prior) + } + source := `package main + import "fmt" +` + fixture.declarations + fixture.observerDeclarations + fixture.callerDeclarations + ` +type word = ` + typ + ` +//go:noinline +func snapshot(p *[4]word) [4]word { return *p } +func main() { + rows:=[]struct{public func(*[4]word,word,word,uint32,uint32);observe func(*[4]word,word,word,word);role int}{ +` + registrations.String() + `} + inputs:=[]uint64{` + inputs.String() + `} + priors:=[]uint64{` + priors.String() + `} + for _,x:=range inputs {for _,prior:=range priors {for status:=0;status<64;status++ { + seed:=uint64(0) + for bit,position:=range []uint{0,2,4,6,7,11}{seed|=uint64(status>>bit&1)<= 3 { + public, observe = "NULL", row.name + fmt.Fprintf(&declarations, "extern void %s(word*,word,word,word);\n", row.name) + } else { + fmt.Fprintf(&declarations, "extern word %s(word,word,uint32_t,uint32_t);\n", row.name) + } + fmt.Fprintf(®istrations, "{%s,%s,%d},\n", public, observe, row.role) + } + for _, value := range x86BitScanExecutionValues { + fmt.Fprintf(&inputs, "UINT64_C(0x%x),", value) + } + for _, prior := range x86BitScanExecutionPriors { + fmt.Fprintf(&priors, "UINT64_C(0x%x),", prior) + } + return `#include +#include + +typedef ` + typ + ` word; + +static __attribute__((always_inline)) inline uintptr_t stack_pointer(void) { + uintptr_t value; + __asm__ volatile("` + stack + `" : "=r"(value) : : "memory"); + return value; +} + +` + declarations.String() + ` +static const struct { + word (*public)(word, word, uint32_t, uint32_t); + void (*observe)(word *, word, word, word); + int role; +} rows[] = { +` + registrations.String() + `}; + +int main(int argc, char **argv) { + if (argc != 2) + return 1; + + FILE *reference = fopen(argv[1], "r"); + if (!reference) + return 2; + + const uint64_t inputs[] = {` + inputs.String() + `}, + priors[] = {` + priors.String() + `}; + unsigned checks = 0; + + for (unsigned x = 0; x < sizeof(inputs) / sizeof(inputs[0]); x++) + for (unsigned p = 0; p < sizeof(priors) / sizeof(priors[0]); p++) + for (unsigned status = 0; status < 64; status++) { + uint64_t seed = 0; + const unsigned positions[] = {0, 2, 4, 6, 7, 11}; + for (unsigned bit = 0; bit < 6; bit++) + seed |= ((status >> bit) & 1u) << positions[bit]; + + for (unsigned r = 0; r < sizeof(rows) / sizeof(rows[0]); r++) { + const word left = (word)(UINT64_C(0x1122334455667788) ^ seed), + right = (word)(UINT64_C(0x8877665544332211) ^ priors[p]); + volatile word guards[] = {left, 0, 0, right}; + word value = 0, flags = 0; + const uintptr_t stackBefore = stack_pointer(); + + if (rows[r].role < 3) + value = rows[r].public((word)inputs[x], (word)priors[p], + (uint32_t)seed, (uint32_t)priors[p]); + else { + rows[r].observe((word *)guards, (word)inputs[x], + (word)priors[p], (word)seed); + value = guards[1]; + flags = guards[2] & 2261u; + } + + const uintptr_t stackAfter = stack_pointer(); + unsigned long long expectedValue = 0, expectedFlags = 0; + if (fscanf(reference, "%llx %llx", &expectedValue, &expectedFlags) != 2 || + (uint64_t)value != expectedValue || + (uint64_t)flags != expectedFlags || + guards[0] != left || guards[3] != right || + stackBefore != stackAfter) { + fprintf(stderr, + "bit-scan row=%u x=%llx prior=%llx seed=%llx value=%llx flags=%llx original=%llx/%llx SP=%llx/%llx or result-array boundary mismatch\n", + r, (unsigned long long)inputs[x], + (unsigned long long)priors[p], (unsigned long long)seed, + (unsigned long long)value, (unsigned long long)flags, + expectedValue, expectedFlags, + (unsigned long long)stackBefore, + (unsigned long long)stackAfter); + fclose(reference); + return 3; + } + checks++; + } + } + + unsigned long long extraValue, extraFlags; + if (fscanf(reference, "%llx %llx", &extraValue, &extraFlags) != EOF) { + fclose(reference); + return 4; + } + if (fclose(reference)) + return 5; + + printf("bit-scan original/LLVM NUM+six-status+canaries checks=%u\n", checks); + return 0; +} +` +} + +func x86BitScanExecuteOriginalAndLLVM(t *testing.T, arch string, execution x86FloatCompareExecution) { + t.Helper() + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + if execution.triple != triple || len(execution.prefix) != 0 || execution.preserveArgv0 { + t.Fatal("native bit-scan oracle needs exact direct Linux transport") + } + fixture := x86BitScanExecutionFixtureFor(t, arch) + dir := t.TempDir() + sources := map[string]string{ + "go.mod": "module bitscanexec\n\ngo 1.20\n", + "main.go": x86BitScanOriginalExecutionDriver(t, fixture), + "public_" + arch + ".s": fixture.public, + "status_" + arch + ".s": fixture.observers, + "callers_" + arch + ".s": fixture.callers, + } + for name, source := range sources { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + t.Logf("original complete source=%s SHA256=%x", name, sha256.Sum256([]byte(source))) + } + original := filepath.Join(dir, "original") + driver, driverHash := x86LEAActualSDKDriver(t) + command := exec.Command(driver, "build", "-p=1", "-o", original, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH="+arch, "CGO_ENABLED=0", "GOFLAGS=", "GOMAXPROCS=2") + x86FloatCompareCaptureCommand(t, command) + verifyOriginal := func() { + for name, source := range sources { + actual, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil || string(actual) != source { + t.Fatal("complete original input changed", name, err) + } + } + if actualDriver, actualHash := x86LEAActualSDKDriver(t); actualDriver != driver || actualHash != driverHash { + t.Fatal("original source driver changed") + } + } + run := func(executable string, arguments ...string) []byte { + if arch != "386" { + return execution.run(t, executable, arguments...) + } + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal("native bit-scan i386 needs a complete static ELF", err) + } + return x87IA32RunDirect(t, sealed, arguments, os.Environ()) + } + verifyOriginal() + observations := run(original) + if err := x86BitScanExecutionCheckRecords(fixture, observations); err != nil { + t.Fatal("original hardware destination/ZF/whole observation failed", err) + } + reference := filepath.Join(dir, "original.records") + if err := os.WriteFile(reference, observations, 0600); err != nil { + t.Fatal(err) + } + verifyRecords := func() { + actual, err := os.ReadFile(reference) + if err != nil || string(actual) != string(observations) { + t.Fatal("original records changed around LLVM observation", err) + } + } + count := len(fixture.rows) * len(x86BitScanExecutionValues) * len(x86BitScanExecutionPriors) * 64 + want := fmt.Sprintf("bit-scan original/LLVM NUM+six-status+canaries checks=%d\n", count) + t.Logf("native original Go arch=%s records=%d driver=%x output=%x", arch, count, driverHash, sha256.Sum256(observations)) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, fixture.public, fixture.declarations, arch, triple, api) + if err != nil || ir == "" { + t.Fatal("complete original numerical source failed", err) + } + ir = x86BitScanExecutionStatusShim(t, fixture, ir) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, input) + } + executable := execution.compileLLVM(t, "bitscan", input, x86BitScanLLVMExecutionDriver(fixture), level) + verifyRecords() + if output := run(executable, reference); string(output) != want { + t.Fatal("native original-Go/LLVM destination/status/canary disagreement", string(output)) + } + verifyRecords() + verifyOriginal() + }) + } + }) + } + verifyOriginal() + verifyRecords() +} + +func TestX86ScalarBitScanExecutionPhysicalStatusShimKeepsPublicRoles(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + fixture := x86BitScanExecutionFixtureFor(t, arch) + // Parse the complete original Go driver without building or executing. + x86BitScanOriginalExecutionDriver(t, fixture) + for _, api := range []string{"text", "module", "Go"} { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + ir, err := x86ScalarFPBitViewAPI(t, fixture.public, fixture.declarations, arch, triple, api) + if err != nil || ir == "" { + t.Fatal(err) + } + shim := x86BitScanExecutionStatusShim(t, fixture, ir) + ctx := llvm.NewContext() + module, err := parseIRModuleInContext(ctx, shim) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if err != nil { + t.Fatal("test-private complete shim IR malformed", err) + } + if strings.Contains(ir, "@Observe") || strings.Count(shim, "define void @Observe") != len(fixture.rows)*2/5 { + t.Fatal("physical observer leaked into ordinary public NUM module") + } + } + } +} + +func TestX86ScalarBitScanExecutionStatusObservationDoesNotAuthorizePublicFlags(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + fixture := x86BitScanExecutionFixtureFor(t, arch) + typ, move, word, triple := "uint64", "MOVQ", 8, "x86_64-unknown-linux-gnu" + if arch == "386" { + typ, move, word, triple = "uint32", "MOVL", 4, "i386-unknown-linux-gnu" + } + for _, row := range fixture.rows { + if row.role != 0 { + continue + } + for _, consumer := range []string{"SETCS ret+RESULT(FP)", "SETPS ret+RESULT(FP)", "SETMI ret+RESULT(FP)", "SETOS ret+RESULT(FP)", "LAHF\nMOVB AH,ret+RESULT(FP)"} { + declaration := fmt.Sprintf("func view(x,prior %s) byte\n", typ) + body := fmt.Sprintf("%s x+0(FP),AX\n%s prior+%d(FP),BX\n%s AX,BX\n%s", move, move, word, row.op, consumer) + source := x86ResultGPSIMDAdmissionSource(t, arch, declaration, body) + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+row.op+"/"+consumer+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, arch, triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("test-only physical flag shim granted public nonportable flag permission", err) + } + }) + } + } + } + } +} diff --git a/amd64_scalar_bitscan_execution_fixture_test.go b/amd64_scalar_bitscan_execution_fixture_test.go new file mode 100644 index 000000000..338d1472b --- /dev/null +++ b/amd64_scalar_bitscan_execution_fixture_test.go @@ -0,0 +1,348 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "math/bits" + "strconv" + "strings" + "testing" +) + +type x86BitScanExecutionRow struct { + name, op string + role int // Prior, Alias, ZeroFlag, original-only Observe, ObserveAlias. +} + +type x86BitScanExecutionFixture struct { + arch, public, declarations, observers, observerDeclarations string + callers, callerDeclarations string + rows []x86BitScanExecutionRow +} + +var x86BitScanExecutionValues = []uint64{0, 1, 7, 128, 255, 0x10000, 0x80000000, 0x8000000000000000, ^uint64(0)} +var x86BitScanExecutionPriors = []uint64{0, ^uint64(0), 0x1122334455667788, 0x8877665544332211} + +func x86BitScanExecutionSeed(index int) uint64 { + var result uint64 + for bit, position := range []uint{0, 2, 4, 6, 7, 11} { + result |= uint64(index>>bit&1) << position + } + return result +} + +func x86BitScanExecutionFixtureFor(t *testing.T, arch string) x86BitScanExecutionFixture { + t.Helper() + source, declarations := x86BitScanOriginalObjectFixture(t, arch) + fixture := x86BitScanExecutionFixture{arch: arch, public: source, declarations: declarations, + callers: "#include \"funcdata.h\"\n"} + word, typ, suffix := 8, "uint64", "Q" + if arch == "386" { + word, typ, suffix = 4, "uint32", "L" + } + for _, op := range []string{"BSFW", "BSFL", "BSFQ", "BSRW", "BSRL", "BSRQ"} { + if arch == "386" && strings.HasSuffix(op, "Q") { + continue // No Go mode32 Q instruction exists; its grammar stays rejected. + } + for role, prefix := range []string{"Prior", "Alias", "ZeroFlag", "Observe", "ObserveAlias"} { + name := prefix + op + fixture.rows = append(fixture.rows, x86BitScanExecutionRow{name, op, role}) + if role < 3 { + declaration, caller := x86BitScanExecutionGuardCallerFor(t, arch, name) + fixture.callerDeclarations += declaration + fixture.callers += caller + continue + } + fixture.observerDeclarations += fmt.Sprintf("func %s(out *[4]%s,x,prior,status %s)\n", name, typ, typ) + destination := "BX" + if role == 4 { + destination = "AX" + } + // Independent original-only physical observation. Neither this + // FLAGS seed nor its pointer writes enters the public translator. + fixture.observers += fmt.Sprintf(`TEXT ·%s(SB),4,$0-%d +MOV%s out+0(FP),DI +MOV%s x+%d(FP),AX +MOV%s prior+%d(FP),BX +MOV%s status+%d(FP),CX +AND%s $2261,CX +OR%s $2,CX +PUSH%s CX +POPF%s +%s AX,%s +PUSHF%s +POP%s DX +MOV%s %s,%d(DI) +MOV%s DX,%d(DI) +RET +`, name, 4*word, suffix, suffix, word, suffix, 2*word, suffix, 3*word, + suffix, suffix, suffix, suffix, op, destination, suffix, suffix, + suffix, destination, word, suffix, 2*word) + } + } + return fixture +} + +// Original-only callers check the actual ABI0 outgoing arguments and two +// adjacent post-result slots. They are not translated source or caller proof. +func x86BitScanExecutionGuardCallerFor(t *testing.T, arch, name string) (string, string) { + t.Helper() + word, typ, suffix := 8, "uint64", "Q" + if arch == "386" { + word, typ, suffix = 4, "uint32", "L" + } + declaration := fmt.Sprintf("func Guard%s(out *[4]%s,x,prior %s,a,b uint32)\n", name, typ, typ) + pkg := mustGoPackage(t, "caller", "package caller\n"+declaration) + sig, err := DeriveGoFuncABI0Signature("Guard"+name, pkg.Types.Scope().Lookup("Guard"+name).(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil || len(sig.Frame.Params) != 5 || len(sig.Frame.Results) != 0 || sig.Frame.Params[1].Offset != int64(word) || sig.Frame.Params[2].Offset != int64(2*word) || sig.Frame.Params[3].Offset != int64(3*word) || sig.Frame.Params[4].Offset != int64(3*word+4) { + t.Fatal("original caller lost its complete actual Go ABI0 layout", err) + } + result := 2*word + 8 + arguments := []struct { + name string + input, output int + suffix string + }{{"x", word, 0, suffix}, {"prior", 2 * word, word, suffix}, {"a", 3 * word, 2 * word, "L"}, {"b", 3*word + 4, 2*word + 4, "L"}} + var assembly strings.Builder + fmt.Fprintf(&assembly, "TEXT ·Guard%s(SB),4,$%d-%d\nNO_LOCAL_POINTERS\n", name, result+3*word, 3*word+8) + for _, argument := range arguments { + fmt.Fprintf(&assembly, "MOV%s %s+%d(FP),AX\nMOV%s AX,%d(SP)\n", argument.suffix, argument.name, argument.input, argument.suffix, argument.output) + } + for index, value := range []int{0x77a7a7a7, 0x2468ace, 0x13579bdf} { + fmt.Fprintf(&assembly, "MOV%s $%#x,%d(SP)\n", suffix, value, result+index*word) + } + fmt.Fprintf(&assembly, "CALL ·%s(SB)\nMOV%s out+0(FP),DI\nMOV%s $1,BX\n", name, suffix, suffix) + for index, value := range []int{0x2468ace, 0x13579bdf} { + fmt.Fprintf(&assembly, "MOV%s $%#x,AX\nMOV%s %d(SP),CX\nCMP%s AX,CX\nJNE broken\n", suffix, value, suffix, result+(index+1)*word, suffix) + } + for _, argument := range arguments { + fmt.Fprintf(&assembly, "MOV%s %s+%d(FP),AX\nMOV%s %d(SP),CX\nCMP%s AX,CX\nJNE broken\n", argument.suffix, argument.name, argument.input, argument.suffix, argument.output, argument.suffix) + } + fmt.Fprintf(&assembly, "JMP guarded\nbroken:\nMOV%s $0,BX\nguarded:\nMOV%s BX,%d(DI)\nMOV%s %d(SP),AX\nMOV%s AX,%d(DI)\nRET\n", suffix, suffix, 2*word, suffix, result, suffix, word) + return declaration, assembly.String() +} + +func x86BitScanExecutionWord(value uint64, arch string) uint64 { + if arch == "386" { + return uint64(uint32(value)) + } + return value +} + +// An independent numeric oracle. The historical dword-zero high half has two +// allowed physical observations; only actual original-Go records choose one. +// Other five status flags have no invented mathematical expectation. +func x86BitScanExecutionAccepts(row x86BitScanExecutionRow, arch string, x, prior, got, flags uint64) bool { + x, prior = x86BitScanExecutionWord(x, arch), x86BitScanExecutionWord(prior, arch) + if got != x86BitScanExecutionWord(got, arch) || flags&^uint64(2261) != 0 { + return false + } + width := map[byte]int{'W': 16, 'L': 32, 'Q': 64}[row.op[len(row.op)-1]] + input := x + if width < 64 { + input &= uint64(1)<= 3 && (flags>>6&1 == 1) != (input == 0) { + return false + } + want := prior + if input != 0 { + want = uint64(bits.TrailingZeros64(input)) + if strings.HasPrefix(row.op, "BSR") { + want = uint64(63 - bits.LeadingZeros64(input)) + } + if width == 16 { + want |= prior &^ 0xffff + } + } + if row.role == 2 && input == 0 { + want ^= uint64(1) << map[string]uint{"amd64": 63, "386": 31}[arch] + } + if got == want { + return true + } + if input == 0 && width == 32 && arch == "amd64" { + old := prior & 0xffffffff + if row.role == 2 { + old ^= uint64(1) << 63 + } + return got == old + } + return false +} + +func x86BitScanExecutionVisit(fixture x86BitScanExecutionFixture, visit func(x, prior, seed uint64, row x86BitScanExecutionRow) error) error { + for _, x := range x86BitScanExecutionValues { + for _, prior := range x86BitScanExecutionPriors { + for status := 0; status < 64; status++ { + for _, row := range fixture.rows { + if err := visit(x, prior, x86BitScanExecutionSeed(status), row); err != nil { + return err + } + } + } + } + } + return nil +} + +func x86BitScanExecutionCheckRecords(fixture x86BitScanExecutionFixture, output []byte) error { + lines := strings.Split(strings.TrimSuffix(string(output), "\n"), "\n") + index := 0 + err := x86BitScanExecutionVisit(fixture, func(x, prior, seed uint64, row x86BitScanExecutionRow) error { + if index >= len(lines) { + return fmt.Errorf("missing original physical observation %d", index) + } + fields := strings.Fields(lines[index]) + if len(fields) != 2 { + return fmt.Errorf("invalid original physical observation %d", index) + } + value, valueErr := strconv.ParseUint(fields[0], 16, 64) + flags, flagsErr := strconv.ParseUint(fields[1], 16, 64) + if valueErr != nil || flagsErr != nil || !x86BitScanExecutionAccepts(row, fixture.arch, x, prior, value, flags) { + return fmt.Errorf("original %s x=%x prior=%x seed=%x observation %d: %q", row.name, x, prior, seed, index, lines[index]) + } + index++ + return nil + }) + if err == nil && index != len(lines) { + err = fmt.Errorf("unexpected original observation count %d (want %d)", len(lines), index) + } + return err +} + +func TestX86ScalarBitScanExecutionOracleDoesNotInventZeroOrFlags(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + fixture := x86BitScanExecutionFixtureFor(t, arch) + wantNames := 6 + if arch == "386" { + wantNames = 4 + } + if len(fixture.rows) != 5*wantNames || strings.Count(fixture.public, "TEXT ·") != 3*wantNames || strings.Count(fixture.observers, "TEXT ·") != 2*wantNames { + t.Fatal("complete original public/physical source inventory missing") + } + if strings.Count(fixture.callers, "TEXT ·Guard") != 3*wantNames || strings.Count(fixture.callers, "CALL ·") != 3*wantNames || strings.Count(fixture.callers, "NO_LOCAL_POINTERS") != 3*wantNames || strings.Count(fixture.callers, "0x2468ace") != 6*wantNames || strings.Count(fixture.callers, "0x13579bdf") != 6*wantNames { + t.Fatal("actual ABI0 outgoing argument/result-adjacent guard inventory missing") + } + if strings.Contains(fixture.public, "PUSH") || strings.Contains(fixture.public, "POPF") || strings.Count(fixture.observers, "PUSH"+map[string]string{"amd64": "Q", "386": "L"}[arch]) != 2*wantNames { + t.Fatal("test-only physical FLAGS observation leaked into numerical source") + } + for _, row := range fixture.rows { + prior := x86BitScanExecutionWord(0x1122334455667788, arch) + if row.role == 1 || row.role == 4 { + continue // Its prior is the actual source, separately covered below. + } + flags, good := uint64(0), prior + if row.role >= 3 { + flags = 1 << 6 + } + if row.role == 2 { + good ^= uint64(1) << map[string]uint{"amd64": 63, "386": 31}[arch] + } + if !x86BitScanExecutionAccepts(row, arch, 0, prior, good, flags) { + t.Fatal("known zero prior was not retained", row) + } + for _, invented := range []uint64{16, 32, 64, ^uint64(0)} { + if x86BitScanExecutionAccepts(row, arch, 0, prior, invented, flags) { + t.Fatal("old software zero result became an allowed physical observation", row, invented) + } + } + if row.role >= 3 { + for seed := 0; seed < 64; seed++ { + other := x86BitScanExecutionSeed(seed) | 1<<6 + if !x86BitScanExecutionAccepts(row, arch, 0, prior, good, other) || x86BitScanExecutionAccepts(row, arch, 0, prior, good, other&^(1<<6)) { + t.Fatal("oracle guessed nonportable status bits or ignored portable ZF") + } + } + } + } + } +} + +func TestX86ScalarBitScanExecutionOracleCompleteWidthsAliasesAndRecords(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + fixture := x86BitScanExecutionFixtureFor(t, arch) + for _, row := range fixture.rows { + width := map[byte]int{'W': 16, 'L': 32, 'Q': 64}[row.op[len(row.op)-1]] + for _, x := range []uint64{0, 1, 7, 128, 255, 1 << (width - 1)} { + prior := x86BitScanExecutionWord(0x1122334455667788, arch) + if row.role == 1 || row.role == 4 { + prior = x86BitScanExecutionWord(x, arch) + } + want := prior + if x != 0 { + want = uint64(bits.TrailingZeros64(x)) + if strings.HasPrefix(row.op, "BSR") { + want = uint64(63 - bits.LeadingZeros64(x)) + } + if width == 16 { + want |= prior &^ 0xffff + } + } + if row.role == 2 && x == 0 { + want ^= uint64(1) << map[string]uint{"amd64": 63, "386": 31}[arch] + } + flags := uint64(0) + if row.role >= 3 && x == 0 { + flags = 1 << 6 + } + if !x86BitScanExecutionAccepts(row, arch, x, prior, want, flags) || x86BitScanExecutionAccepts(row, arch, x, prior, want^0x8000, flags) { + t.Fatal("scan width/prior/alias numeric model changed", arch, row, x) + } + } + if arch == "amd64" && width == 32 { + prior := uint64(0x1122334455667788) + x := uint64(0) + if row.role == 1 || row.role == 4 { + x, prior = 1<<32, 1<<32 // A zero dword with live high alias bytes. + } + flags, full, low := uint64(0), prior, prior&0xffffffff + if row.role >= 3 { + flags = 1 << 6 + } + if row.role == 2 { + full, low = full^(1<<63), low^(1<<63) + } + if !x86BitScanExecutionAccepts(row, arch, x, prior, full, flags) || !x86BitScanExecutionAccepts(row, arch, x, prior, low, flags) || x86BitScanExecutionAccepts(row, arch, x, prior, low|1<<40, flags) { + t.Fatal("oracle guessed old/new CPU dword-zero upper behavior", row) + } + } + } + // A complete synthetic record stream certifies the reader's checking, + // not execution. Give the nonportable five flags arbitrary values. + one := x86BitScanExecutionFixture{arch: arch, rows: []x86BitScanExecutionRow{{"ObserveBSFQ", "BSFQ", 3}}} + if arch == "386" { + one.rows[0].name, one.rows[0].op = "ObserveBSFL", "BSFL" + } + var records strings.Builder + err := x86BitScanExecutionVisit(one, func(x, prior, seed uint64, row x86BitScanExecutionRow) error { + x, prior = x86BitScanExecutionWord(x, arch), x86BitScanExecutionWord(prior, arch) + value, flags := prior, seed&^(1<<6) + if x == 0 { + flags |= 1 << 6 + } else { + value = uint64(bits.TrailingZeros64(x)) + } + fmt.Fprintf(&records, "%016x %04x\n", value, flags) + return nil + }) + if err != nil || x86BitScanExecutionCheckRecords(one, []byte(records.String())) != nil { + t.Fatal("bounded complete record reader rejected numeric/ZF observations", err) + } + lines := strings.Split(strings.TrimSuffix(records.String(), "\n"), "\n") + for _, bad := range []string{ + strings.Join(lines[:len(lines)-1], "\n") + "\n", + records.String() + "0 0\n", + "not-a-record\n" + strings.Join(lines[1:], "\n") + "\n", + "0000000000000040 0040\n" + strings.Join(lines[1:], "\n") + "\n", + "0000000000000000 0000\n" + strings.Join(lines[1:], "\n") + "\n", + } { + if x86BitScanExecutionCheckRecords(one, []byte(bad)) == nil { + t.Fatal("missing/extra/invalid/invented-zero/ZF record escaped validation") + } + } + } +} diff --git a/amd64_scalar_bitscan_execution_linux_amd64_test.go b/amd64_scalar_bitscan_execution_linux_amd64_test.go new file mode 100644 index 000000000..128a2ea33 --- /dev/null +++ b/amd64_scalar_bitscan_execution_linux_amd64_test.go @@ -0,0 +1,36 @@ +//go:build linux && amd64 + +package plan9asm + +import ( + "os/exec" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86BitScanNativeAMD64OriginalGoAndLLVM(t *testing.T) { + x87IA32RequireNativeHost(t) + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("native bit-scan owner requires LLVM22 llc/clang") + } + execution := x86FloatCompareExecution{llc: llc, triple: "x86_64-unknown-linux-gnu", + compiler: []string{clang, "-target", "x86_64-unknown-linux-gnu"}} + x86BitScanExecuteOriginalAndLLVM(t, "amd64", execution) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixX86BitScanNative386OriginalGoAndLLVM(t *testing.T) { + x87IA32RequireNativeHost(t) + llc := findLLVM22Tool("llc") + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if llc == "" || err != nil { + t.Fatal("native i386 bit-scan owner requires LLVM22/i686 compiler", err) + } + execution := x86FloatCompareExecution{llc: llc, triple: "i386-unknown-linux-gnu", + // Keep one fixed outgoing C frame so the SP observation does not + // mistake GCC's deferred caller-argument cleanup for an unbalanced + // compiler helper. This changes no translated Go source/ABI role. + compiler: []string{compiler, "-static", "-no-pie", "-maccumulate-outgoing-args"}} + x86BitScanExecuteOriginalAndLLVM(t, "386", execution) + x87IA32RequireNativeHost(t) +} diff --git a/amd64_scalar_bitscan_execution_object_test.go b/amd64_scalar_bitscan_execution_object_test.go new file mode 100644 index 000000000..02ad3cd71 --- /dev/null +++ b/amd64_scalar_bitscan_execution_object_test.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// Compile the exact native fixtures on every development host. This checks +// original ABI0 callers, physical observers and test-private LLVM shims before +// the Linux owners execute them; object success never stands in for execution. +func TestX86ScalarBitScanExecutionCompleteOriginalGoLinksAndLLVMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("complete native bit-scan fixtures require LLVM22 llc") + } + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + fixture := x86BitScanExecutionFixtureFor(t, arch) + dir := t.TempDir() + sources := map[string]string{ + "go.mod": "module bitscanexec\n\ngo 1.20\n", + "main.go": x86BitScanOriginalExecutionDriver(t, fixture), + "public_" + arch + ".s": fixture.public, + "status_" + arch + ".s": fixture.observers, + "callers_" + arch + ".s": fixture.callers, + } + for name, source := range sources { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + t.Logf("original complete input=%s SHA256=%x", name, sha256.Sum256([]byte(source))) + } + driver, driverHash := x86LEAActualSDKDriver(t) + verify := func() { + for name, source := range sources { + actual, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil || string(actual) != source { + t.Fatal("complete original fixture changed", name, err) + } + } + actualDriver, actualHash := x86LEAActualSDKDriver(t) + if actualDriver != driver || actualHash != driverHash { + t.Fatal("original fixture compiler changed") + } + } + original := filepath.Join(dir, "original") + command := exec.Command(driver, "build", "-p=1", "-o", original, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", + "GOOS=linux", "GOARCH="+arch, "CGO_ENABLED=0", "GOFLAGS=", "GOMAXPROCS=2") + verify() + x86FloatCompareCaptureCommand(t, command) + verify() + info, err := os.Lstat(original) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatal("original complete Go executable is missing or invalid", err) + } + + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, fixture.public, fixture.declarations, arch, triple, api) + if err != nil || ir == "" { + t.Fatal("complete original numerical fixture failed", err) + } + ir = x86BitScanExecutionStatusShim(t, fixture, ir) + for _, optimize := range []bool{false, true} { + input := ir + if optimize { + input = x86MulDivRoundOptimize(t, input) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR_O2=%t/backend%s", optimize, level), func(t *testing.T) { + x86PackedStringCompileObject(t, llc, triple, input, level) + }) + } + } + }) + } + verify() + }) + } +} diff --git a/amd64_scalar_bitscan_execution_policy_test.go b/amd64_scalar_bitscan_execution_policy_test.go new file mode 100644 index 000000000..dc3ff64e9 --- /dev/null +++ b/amd64_scalar_bitscan_execution_policy_test.go @@ -0,0 +1,153 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "strconv" + "strings" + "testing" +) + +func TestX86ScalarBitScanRuntimeNativeOwnersCannotEmulateOrSkip(t *testing.T) { + const path = "amd64_scalar_bitscan_execution_linux_amd64_test.go" + source, err := os.ReadFile(path) + if err != nil { + t.Fatal("required original-Go/LLVM bit-scan execution owners absent", err) + } + if err := x86BitScanNativeOwnerPolicy(source); err != nil { + t.Fatal(err) + } +} + +func x86BitScanNativeOwnerPolicy(source []byte) error { + source = normalizeNativeOwnerPolicySource(source) + if !strings.HasPrefix(string(source), "//go:build linux && amd64\n") { + return fmt.Errorf("required bit-scan counterparts lost their native Linux owner") + } + file, err := parser.ParseFile(token.NewFileSet(), "native-owner.go", source, 0) + if err != nil { + return err + } + roots := map[string]string{ + "TestCrossLinuxRuntimeMatrixX86BitScanNativeAMD64OriginalGoAndLLVM": "amd64", + "TestCrossLinuxRuntimeMatrixX86BitScanNative386OriginalGoAndLLVM": "386", + } + for _, declaration := range file.Decls { + fn, ok := declaration.(*ast.FuncDecl) + if !ok { + continue + } + arch, required := roots[fn.Name.Name] + if !required { + continue + } + gates, executions := 0, 0 + var gatePositions []token.Pos + var executionPosition token.Pos + var invalid error + topLevel := make(map[*ast.CallExpr]bool) + for _, statement := range fn.Body.List { + if expression, ok := statement.(*ast.ExprStmt); ok { + if call, ok := expression.X.(*ast.CallExpr); ok { + topLevel[call] = true + } + } + } + ast.Inspect(fn.Body, func(node ast.Node) bool { + switch value := node.(type) { + case *ast.CallExpr: + if name, ok := value.Fun.(*ast.Ident); ok { + if name.Name == "x87IA32RequireNativeHost" { + gates++ + gatePositions = append(gatePositions, value.Pos()) + if !topLevel[value] { + invalid = fmt.Errorf("native observation must run at top level") + } + } + if name.Name == "x86BitScanExecuteOriginalAndLLVM" { + executions++ + executionPosition = value.Pos() + if !topLevel[value] { + invalid = fmt.Errorf("native execution must run at top level") + } + if len(value.Args) != 3 { + invalid = fmt.Errorf("runtime oracle lost its complete architecture") + return false + } + literal, ok := value.Args[1].(*ast.BasicLit) + if !ok || literal.Value != strconv.Quote(arch) { + invalid = fmt.Errorf("native owner selected another architecture") + } + } + } + if name, ok := value.Fun.(*ast.SelectorExpr); ok && strings.HasPrefix(name.Sel.Name, "Skip") { + invalid = fmt.Errorf("missing native tools or failed execution cannot skip") + } + case *ast.KeyValueExpr: + if name, ok := value.Key.(*ast.Ident); ok && (name.Name == "prefix" || name.Name == "preserveArgv0") { + invalid = fmt.Errorf("native owner inserted a guest/emulator transport") + } + case *ast.ReturnStmt: + invalid = fmt.Errorf("required native owner cannot return before observation") + case *ast.BasicLit: + if value.Kind == token.STRING { + literal, err := strconv.Unquote(value.Value) + if err != nil || strings.Contains(literal, "qemu") || literal == "-exec" { + invalid = fmt.Errorf("native owner inserted an execution fallback: %v", err) + } + } + } + return true + }) + if invalid != nil { + return invalid + } + if gates != 2 || executions != 1 { + return fmt.Errorf("native before/after observation or whole execution absent: %s", fn.Name.Name) + } + if !(gatePositions[0] < executionPosition && executionPosition < gatePositions[1]) { + return fmt.Errorf("native observation must surround the actual execution: %s", fn.Name.Name) + } + delete(roots, fn.Name.Name) + } + if len(roots) != 0 { + return fmt.Errorf("required native denominator incomplete: %v", roots) + } + return nil +} + +func TestX86ScalarBitScanRuntimeNativePolicyRejectsMutation(t *testing.T) { + source, err := os.ReadFile("amd64_scalar_bitscan_execution_linux_amd64_test.go") + if err != nil { + t.Fatal(err) + } + const call = "\tx86BitScanExecuteOriginalAndLLVM(t, \"amd64\", execution)" + const gate = "\tx87IA32RequireNativeHost(t)" + text := string(normalizeNativeOwnerPolicySource(source)) + for name, mutation := range map[string]string{ + "missing after": strings.Replace(text, call+"\n"+gate, call, 1), + "both before": strings.Replace(text, call+"\n"+gate, gate+"\n"+call, 1), + "both after": strings.Replace(strings.Replace(text, gate+"\n", "", 1), call+"\n"+gate, call+"\n"+gate+"\n"+gate, 1), + "conditional after": strings.Replace(text, call+"\n"+gate, call+"\n\tif false {\n"+gate+"\n\t}", 1), + "conditional run": strings.Replace(text, call, "\tif false {\n"+call+"\n\t}", 1), + "early return": strings.Replace(text, call, "\treturn\n"+call, 1), + "skip": strings.Replace(text, call, "\tt.Skip(\"missing tool\")\n"+call, 1), + "prefix": strings.Replace(text, "{llc: llc,", "{prefix: []string{\"guest\"}, llc: llc,", 1), + "preserve argv": strings.Replace(text, "{llc: llc,", "{preserveArgv0: true, llc: llc,", 1), + "wrong arch": strings.Replace(text, call, strings.Replace(call, "amd64", "386", 1), 1), + "wrong native tags": strings.Replace(text, "linux && amd64", "darwin && amd64", 1), + "missing root": strings.Replace(text, "TestCrossLinuxRuntimeMatrixX86BitScanNativeAMD64OriginalGoAndLLVM", "NoRequiredAMD64Root", 1), + } { + t.Run(name, func(t *testing.T) { + if mutation == text { + t.Fatal("policy mutation did not change actual native owner source") + } + if x86BitScanNativeOwnerPolicy([]byte(mutation)) == nil { + t.Fatal("invalid native lifecycle/transport owner accepted") + } + }) + } +} diff --git a/amd64_scalar_bitscan_native.go b/amd64_scalar_bitscan_native.go new file mode 100644 index 000000000..fcb273227 --- /dev/null +++ b/amd64_scalar_bitscan_native.go @@ -0,0 +1,127 @@ +package plan9asm + +import ( + "fmt" + "sort" + "strings" +) + +func x86BitScanHelperOps(file *File, arch string) []Op { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + seen := make(map[Op]bool) + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + if _, known, err := x86ScalarBitScanProperties(ins.Op, ins, arch); known && err == nil { + seen[Op(strings.ToUpper(string(ins.Op)))] = true + } + } + } + ops := make([]Op, 0, len(seen)) + for op := range seen { + ops = append(ops, op) + } + sort.Slice(ops, func(i, j int) bool { return ops[i] < ops[j] }) + return ops +} + +func x86BitScanNamespace(file *File, arch string, resolve func(string) string, sigs map[string]FuncSig) string { + if len(x86BitScanHelperOps(file, arch)) == 0 { + return "" + } + return x86CompilerHelperNamespace(file, resolve, sigs, "bitscan") +} + +func x86BitScanHelperName(namespace, arch string, op Op) string { + return namespace + arch + "_" + strings.ToLower(string(op)) +} + +func emitX86BitScanHelpers(b *strings.Builder, file *File, arch, namespace string) { + if namespace == "" { + return + } + word, push, pop := I64, "pushq", "popq" + if arch == "386" { + word, push, pop = I32, "pushl", "popl" + } + for _, op := range x86BitScanHelperOps(file, arch) { + spec := x86ScalarBitScanSpecs[op] + typ := amd64IntegerTypeForBits(spec.bits) + modifier := map[int]string{16: "w", 32: "k", 64: "q"}[spec.bits] + mnemonic := strings.ToLower(string(op)) + name := x86BitScanHelperName(namespace, arch, op) + fmt.Fprintf(b, "define internal { %s, %s } @%s(%s %%source, %s %%prior, %s %%status) noinline noredzone {\nentry:\n", word, word, name, typ, word, word) + // Seed only virtual status representatives, never privileged FLAGS, + // direction/ID or a claimed physical caller-entry identity. The source + // observer still permits only portable ZF after a scan. Unknown other + // bits stay unobservable; a zero seed does not establish their validity. + fmt.Fprintf(b, " %%masked = and %s %%status, %d\n %%seed = or %s %%masked, 2\n", word, x86StatusAll, word) + assembly := fmt.Sprintf("%s $4; popf%s; %s ${3:%s}, ${0:%s}; pushf%s; %s $1", push, strings.TrimPrefix(pop, "pop"), mnemonic, modifier, modifier, strings.TrimPrefix(pop, "pop"), pop) + // The full native prior is tied to the full native output. In + // particular no software r32 writeback chooses the dword-zero upper + // behavior. Balanced temporary stack use lives only in this compiler + // helper; noinline+noredzone protects callers' red zones and source SP. + fmt.Fprintf(b, " %%result = call { %s, %s } asm sideeffect %q, \"=r,=r,0,r,r,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s %%prior, %s %%source, %s %%seed)\n", word, word, assembly, word, typ, word) + fmt.Fprintf(b, " ret { %s, %s } %%result\n}\n\n", word, word) + } +} + +func (c *amd64Ctx) lowerScalarBitScan(op Op, ins Instr) (bool, bool, error) { + arch := c.goarch + if arch == "" { + arch = "amd64" // The existing direct emitter's shape default. + } + spec, known, err := x86ScalarBitScanProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + if c.bitScanNS == "" || !x86ScalarFloatIntegerNativeTarget(arch, c.targetTriple) { + return true, false, fmt.Errorf("%w: bit scan needs its compiler-owned bridge and matching explicit x86 target", ErrProbeNeedsContext) + } + typ, word := amd64IntegerTypeForBits(spec.bits), I64 + if arch == "386" { + word = I32 + } + var source string + if ins.Args[0].Kind == OpReg { + source, err = c.evalIntSized(ins.Args[0], typ) + } else { + // Retain each existing source role and exact-width volatile native + // read; FP/AUTO/SB observations do not become native storage authority. + var bytes string + bytes, err = c.loadX86PackedShiftBytes(ins.Args[0], spec.bits/8, true) + if err == nil { + name := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", name, spec.bits/8, bytes, typ) + source = "%" + name + } + } + if err != nil { + return true, false, err + } + prior, err := c.loadReg(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + status := c.packX86Flags() + if word == I32 { + prior, status = c.truncI64(prior, I32), c.truncI64(status, I32) + } + result, value, flags := c.newTmp(), c.newTmp(), c.newTmp() + name := x86BitScanHelperName(c.bitScanNS, arch, Op(strings.ToUpper(string(op)))) + returnType := fmt.Sprintf("{ %s, %s }", word, word) + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, %s %s)\n", result, returnType, name, typ, source, word, prior, word, status) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 0\n %%%s = extractvalue %s %%%s, 1\n", value, returnType, result, flags, returnType, result) + for _, flag := range c.x86StatusFlagSlots() { + shift, bit := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %d\n %%%s = trunc %s %%%s to i1\n store i1 %%%s, ptr %s\n", shift, word, flags, flag.bit, bit, word, shift, bit, flag.slot) + } + output := "%" + value + if word == I32 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, output) + output = "%" + wide + } + return true, false, c.storeReg(ins.Args[1].Reg, output) +} diff --git a/amd64_scalar_bitscan_object_test.go b/amd64_scalar_bitscan_object_test.go new file mode 100644 index 000000000..7fd9b1160 --- /dev/null +++ b/amd64_scalar_bitscan_object_test.go @@ -0,0 +1,100 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" + "testing" +) + +// Complete source-mode cohorts, not cropped functions. Every supported name +// retains an independent six-status ADD producer, prior destination, full-word +// output, source/destination alias and portable ZF consumer. Q rows have no +// legal mode32 transport; their named grammar rejection is checked separately. +func x86BitScanOriginalObjectFixture(t *testing.T, arch string) (source, declarations string) { + t.Helper() + move, typ, word := "MOVQ", "uint64", int64(8) + if arch == "386" { + move, typ, word = "MOVL", "uint32", 4 + } + for _, op := range []string{"BSFW", "BSFL", "BSFQ", "BSRW", "BSRL", "BSRQ"} { + if arch == "386" && strings.HasSuffix(op, "Q") { + continue + } + for _, role := range []string{"Prior", "Alias", "ZeroFlag"} { + name := role + op + declaration := fmt.Sprintf("func %s(x,prior %s,a,b uint32) %s\n", name, typ, typ) + pkg := mustGoPackage(t, "scans", "package scans\n"+declaration) + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil || len(sig.Frame.Params) != 4 || len(sig.Frame.Results) != 1 { + t.Fatal("complete original typed scan layout unavailable", err) + } + if sig.Frame.Params[1].Offset != word || sig.Frame.Params[2].Offset != 2*word || sig.Frame.Params[3].Offset != 2*word+4 || sig.Frame.Results[0].Offset != 2*word+8 { + t.Fatal("actual Go scan ABI0 field layout changed", sig.Frame) + } + destination := "BX" + if role == "Alias" { + destination = "AX" + } + result := sig.Frame.Results[0].Offset + body := fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n%s x+0(FP),AX\n%s prior+%d(FP),BX\nMOVL a+%d(FP),CX\nMOVL b+%d(FP),DX\nADDL DX,CX\n%s AX,%s\n", name, result+word, move, move, word, 2*word, 2*word+4, op, destination) + if role == "ZeroFlag" { + // Read only portable ZF, then combine its byte into an ordinary + // numeric result. This grants neither bool nor other FLAGS roles. + body += fmt.Sprintf("SETEQ DL\nMOVBLZX DL,CX\nSHL%s $%d,CX\nXOR%s CX,%s\n", strings.TrimPrefix(move, "MOV"), word*8-1, strings.TrimPrefix(move, "MOV"), destination) + } + body += fmt.Sprintf("%s %s,ret+%d(FP)\nRET\n", move, destination, result) + source += body + declarations += declaration + } + } + return source, declarations +} + +func TestX86ScalarBitScanCompleteOriginalGoAndLLVMFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("complete bit-scan objects require LLVM22 llc") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + source, declarations := x86BitScanOriginalObjectFixture(t, target.arch) + count := 18 + if target.arch == "386" { + count = 12 + } + if strings.Count(source, "TEXT ·") != count || strings.Count(source, "ADDL DX,CX") != count { + t.Fatal("complete source/status producer inventory changed") + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("complete original scan cohort failed", err) + } + for _, optimizeIR := range []bool{false, true} { + input := ir + if optimizeIR { + input = x86MulDivRoundOptimize(t, input) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR_O2=%t/backend%s", optimizeIR, level), func(t *testing.T) { + x86PackedStringCompileObject(t, llc, target.triple, input, level) + }) + } + } + }) + } + }) + } +} + +func TestX86ScalarBitScanCompleteObjectFixtureNumericalRoles(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source, declarations := x86BitScanOriginalObjectFixture(t, arch) + // Pure/IR counterpart only. Actual Go and LLVM objects belong to the + // separate required five-target root and are never inferred here. + x86ResultGPSIMDAdmissionAPIs(t, arch, source, declarations, true) + } +} diff --git a/amd64_scalar_compare_fault_runtime_unix_test.go b/amd64_scalar_compare_fault_runtime_unix_test.go new file mode 100644 index 000000000..f686707fc --- /dev/null +++ b/amd64_scalar_compare_fault_runtime_unix_test.go @@ -0,0 +1,101 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ScalarCompareDeadReadFaults(t *testing.T) { + t.Run("scalar-sqrt", x86ScalarSqrtDeadReadFaultRuntime) + execution := x86FloatCompareExecutionTools(t) + var source, declarations, native, prototypes, pairs strings.Builder + for index, op := range x86ScalarCompareNames("amd64") { + name, original := fmt.Sprintf("dead%d", index), fmt.Sprintf("original%d", index) + // TEST $0 still reads its entire original memory operand. Neither its + // tautological flags nor a later independent result may suppress faults. + instruction := x86ScalarCompareInstruction(op, "(AX)", "$0") + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-8\nMOVQ p+0(FP),AX\n%s\nRET\n", name, instruction) + fmt.Fprintf(&declarations, "func %s(p *byte)\n", name) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+x86ScalarCompareInstruction(op, "(DI)", "$0")+"\nRET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*),%s(uint8_t*);\n", name, original) + fmt.Fprintf(&pairs, "{%s,%s,%d},\n", original, name, []int{1, 2, 4, 8}[index%4]) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + main := ` +#include +#include +#include +#include +#include +#include +#include +` + prototypes.String() + native.String() + ` +typedef void (*program)(uint8_t*); +static const struct { program original,translated; int width; } pairs[]={ +` + pairs.String() + `}; +static int outcome(program fn,uint8_t *p) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + fn(p); _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS)) return 1; + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + uint8_t *data=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(data==MAP_FAILED || mprotect(data+page,page,PROT_NONE)) return 2; + for(unsigned i=0;i 1<<32-1) { + return fail("requires a resolved Yi32 integer in its original operand position") + } + form.args[index].Imm = amd64ScalarAddSubImmediateInt64(operand.Imm, form.bits) + immediates++ + case OpReg: + allowed := isX86YrlRegisterForArch(operand.Reg, arch) + if form.bits == 8 { + allowed = isGoYmbRegisterForArch(operand.Reg, arch) + } + if !allowed { + return fail("uses a register outside its Go Yrb/Yrl class") + } + registers++ + default: + if operand.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(operand.Sym), "$") { + // Non-shared 386's Yi32 includes relocated symbol addresses, + // but Go can emit the address relocation only for the L width. + // This syntax does not make the address numerical in a result proof. + if arch != "386" || index != immediatePosition { + return fail("has no relocated Yi32 row in this mode/position") + } + address := Operand{Kind: OpSym, Sym: strings.TrimPrefix(strings.TrimSpace(operand.Sym), "$")} + if _, _, valid := x86ScalarCompareSBAddress(address, arch); !valid { + return fail("has an unresolved relocated immediate") + } + if form.bits != 32 { + return form, true, fmt.Errorf("%w: Go %s short relocated immediate has no original relocation sink: %q", ErrProbeNeedsContext, op, ins.Raw) + } + immediates++ + continue + } + tls := x86SourceTLSMemoryRole(operand, arch) + if form.memory >= 0 || !tls && !x86ScalarCompareMemoryValid(operand, arch) { + return fail("requires at most one encodable Ymb/Yml memory operand") + } + form.memory = index + needsTLS = needsTLS || tls + } + } + if registers == 0 && (form.memory < 0 || immediates != 1) { + return fail("does not match any complete Go compare/test row") + } + if needsTLS { + // Check the complete row first: a TLS annotation cannot make a bad + // register, immediate direction, or second memory operand valid. + return form, true, fmt.Errorf("%w: %s %s TLS memory needs its actual Go OS/arch/shared progedit, relocation and native TLS address contract: %q", ErrProbeNeedsContext, arch, op, ins.Raw) + } + if arch == "amd64" && form.bits == 8 && x86CompareExchangeHasREXByteView(x86CompareExchangeForm{ + source: form.args[0], destination: form.args[1], + }) { + for index, operand := range form.args { + if operand.Kind == OpReg && (operand.Reg == AH || operand.Reg == CH || operand.Reg == DH || operand.Reg == BH) { + form.args[index].Reg = amd64YmbEffectiveRegister(operand.Reg, 16) + } + } + } + return form, true, nil +} + +func x86ScalarCompareMemoryValid(operand Operand, arch string) bool { + if operand.Kind == OpFP { + return true // Its separate logical frame consumer validates the byte span. + } + if operand.Kind == OpSym { + if strings.HasPrefix(strings.TrimSpace(operand.Sym), "$") { + return false + } + if _, _, valid := x86ScalarCompareSBAddress(operand, arch); valid { + return true + } + address, err := parseInt(strings.TrimSpace(operand.Sym)) + return err == nil && (address == int64(int32(address)) || arch == "386" && address == int64(uint32(address))) + } + if operand.Kind != OpMem { + return false + } + memory := operand.Mem + if x86SymbolIndexedMemoryError(memory, arch) != nil || unresolvedX86MemoryOffsetError(memory) != nil || + memory.IndexExt != "" || memory.Segment != "" && memory.Segment != FS && memory.Segment != GS { + return false + } + if memory.Sym != "" { + var valid bool + _, memory, valid = x86ScalarCompareSBAddress(operand, arch) + if !valid { + return false + } + } + return x86MemoryRegistersValidForArch(memory, arch) && + (memory.Off == int64(int32(memory.Off)) || arch == "386" && memory.Off == int64(uint32(memory.Off))) && + (memory.Index == "" && memory.Scale == 0 || memory.Index != "" && memory.Index != SP && + (memory.Scale == 1 || memory.Scale == 2 || memory.Scale == 4 || memory.Scale == 8)) +} + +// Unlike the same-file DATA reader this recognizes an address, including a +// negative displacement, without claiming an allocated or initialized extent. +func x86ScalarCompareSBAddress(operand Operand, arch string) (string, MemRef, bool) { + memory, valid := x86SBMemoryAddressRole(operand, arch) + if !valid || memory.OffRaw != "" || memory.IndexExt != "" { + return "", MemRef{}, false + } + base, offset, valid := parseSBRef(memory.Sym) + if !valid || base == "" || strings.IndexAny(base, "$+(),[] \t") >= 0 || + memory.Off > 0 && offset > math.MaxInt64-memory.Off || + memory.Off < 0 && offset < math.MinInt64-memory.Off { + return "", MemRef{}, false + } + offset += memory.Off + if offset != int64(int32(offset)) && !(arch == "386" && offset == int64(uint32(offset))) { + return "", MemRef{}, false + } + return base, memory, true +} diff --git a/amd64_scalar_compare_forms_test.go b/amd64_scalar_compare_forms_test.go new file mode 100644 index 000000000..ffdc22b51 --- /dev/null +++ b/amd64_scalar_compare_forms_test.go @@ -0,0 +1,351 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86ScalarCompareCompleteActualGoTables(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for table, body := range map[string]string{ + "ycmpb": `{Z_ib, 1, argList{Yal, Yi32}}, {Zm_ibo, 2, argList{Ymb, Yi32}}, +{Zm_r, 1, argList{Ymb, Yrb}}, {Zr_m, 1, argList{Yrb, Ymb}},`, + "ycmpl": `{Zm_ibo, 2, argList{Yml, Yi8}}, {Z_il, 1, argList{Yax, Yi32}}, +{Zm_ilo, 2, argList{Yml, Yi32}}, {Zm_r, 1, argList{Yml, Yrl}}, {Zr_m, 1, argList{Yrl, Yml}},`, + "yxorb": `{Zib_, 1, argList{Yi32, Yal}}, {Zibo_m, 2, argList{Yi32, Ymb}}, +{Zr_m, 1, argList{Yrb, Ymb}}, {Zm_r, 1, argList{Ymb, Yrb}},`, + "ytestl": `{Zil_, 1, argList{Yi32, Yax}}, {Zilo_m, 2, argList{Yi32, Yml}}, +{Zr_m, 1, argList{Yrl, Yml}}, {Zm_r, 1, argList{Yml, Yrl}},`, + } { + start := strings.Index(string(data), "var "+table+" = []ytab{") + if start < 0 { + t.Fatal("actual Go omitted complete comparison table", table) + } + actual := strings.SplitN(strings.SplitN(string(data)[start:], "\n}", 2)[0], "[]ytab{", 2)[1] + got, err := x86SignedPackedFloatGoOpcodeTokens(actual) + want, wantErr := x86SignedPackedFloatGoOpcodeTokens(body) + if err != nil || wantErr != nil || !reflect.DeepEqual(got, want) { + t.Fatal("actual Go comparison table changed", table) + } + } +} + +// Independent rows for ycmpb/ycmpl/yxorb/ytestl. Accumulator, general register, +// both register/memory directions and all immediate encoding transitions. +func x86ScalarCompareOracleRows(op, arch string) []string { + rows := []string{"AX,CX", "CX,AX", "AX,8(DX)", "8(DX),AX", "AX,data-3(SB)", "data+3(SB),AX", + "8(FS),AX", "AX,8(GS)", "-7(DX)(CX*2),AX", "0,AX"} + for _, immediate := range []string{"0", "1", "127", "128", "255", "256", "-128", "-129", "32768", "65535", "2147483647", "2147483648", "4294967295", "-2147483648"} { + for _, value := range []string{"AX", "CX", "8(DX)", "data(SB)"} { + operands := value + ",$" + immediate + if strings.HasPrefix(op, "TEST") { + operands = "$" + immediate + "," + value + } + rows = append(rows, operands) + } + } + if strings.HasSuffix(op, "B") { + rows = append(rows, "AH,BL", "CH,DH", "BP,SI", "DI,AX") + if arch == "amd64" { + rows = append(rows, "CH,R8B", "R9B,BH", "DIB,8(R12)") + } + } else { + rows = append(rows, "SP,BP", "DI,SI") + if arch == "amd64" { + rows = append(rows, "R8,R15", "16(R12)(R9*8),R11") + } + } + if arch == "386" { + rows = append(rows, "data(SB)(SP),AX", "AX,data(SB)(CX*4)") + if strings.HasSuffix(op, "L") { + rows = append(rows, strings.TrimPrefix(x86ScalarCompareInstruction(op, "AX", "$data(SB)"), op+" ")) + } + } + return rows +} + +func TestX86ScalarCompareCompleteGoForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, deadRegisters, deadDeclarations strings.Builder + var initialized, unknownMemory []x86OriginalGPInitializedFormat + privateOnly := 0 + index := 0 + for _, op := range x86ScalarCompareNames(target.arch) { + for _, row := range x86ScalarCompareOracleRows(op, target.arch) { + body := op + " " + row + ins := x86PureGPUnknownInstruction(t, body) + form, known, err := x86ScalarCompareProperties(ins.Op, ins, target.arch) + if !known || err != nil { + t.Fatalf("typed shape %s: %v", body, err) + } + name := fmt.Sprintf("form%d", index) + original := fmt.Sprintf("TEXT ·%s(SB),4,$0-0\n%s\nRET\n", name, body) + source.WriteString(original) + if form.memory >= 0 { + memory, partner := form.args[form.memory], form.args[1-form.memory] + hardwareOnly := memory.Kind == OpMem && memory.Mem.Segment != "" && + memory.Mem.Base == "" && memory.Mem.Index == "" && partner.Kind == OpImm + if !hardwareOnly { + unknownMemory = append(unknownMemory, x86OriginalGPInitializedFormat{ + source: original, declaration: "func " + name + "()", + }) + } + } else if x86ScalarCompareOrdinaryDeadRegisters(form, target.arch) { + deadRegisters.WriteString(original) + fmt.Fprintf(&deadDeclarations, "func %s()\n", name) + } + fixture, reason := x86ScalarCompareOriginalInitializedFormat(target.arch, name, body, form) + if reason == "" { + initialized = append(initialized, fixture) + } else { + privateOnly++ + t.Logf("original compare row %d keeps its independent role: %s: %s", index, body, reason) + } + index++ + } + } + want := map[string]int{"amd64": 566, "386": 426}[target.arch] + if index != want { + t.Fatal("complete original compare inventory changed", index, want) + } + wantInitialized := map[string]int{"amd64": 420, "386": 306}[target.arch] + if len(initialized) != wantInitialized || privateOnly != want-wantInitialized { + t.Fatal("original compare input partition changed", len(initialized), privateOnly, wantInitialized) + } + file, signatures := x86UnboundVoidFormatSources(t, source.String(), target.arch, target.triple) + ir := x86SingleInstructionFormatObjectUnits(t, file, signatures, target.arch, target.triple, + func(c *amd64Ctx, instruction Instr) (bool, bool, error) { + return c.lowerCmpBt(instruction.Op, instruction) + }) + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + for row, fixture := range unknownMemory { + t.Run(fmt.Sprintf("unbound-memory/row%d", row), func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, fixture.source, fixture.declaration, target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("unbound original comparison acquired numerical inputs", api, err) + } + } + }) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run("dead-registers/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, deadRegisters.String(), deadDeclarations.String(), target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("ordinary dead comparison registers changed domain", err) + } + }) + } + x86OriginalGPInitializedFormatObjects(t, initialized, target.arch, target.triple) + t.Logf("%d unchanged Go/private LLVM comparison formats; %d initialized counterparts; %d independent physical/static/address roles", + index, len(initialized), privateOnly) + }) + } +} + +func x86ScalarCompareOrdinaryDeadRegisters(form x86ScalarCompareForm, arch string) bool { + for _, operand := range form.args { + if operand.Kind == OpImm { + continue + } + if operand.Kind != OpReg || x86OriginalGPPhysicalRegister(operand.Reg, arch) { + return false + } + } + return true +} + +func x86ScalarCompareOriginalInitializedFormat(arch, name, body string, form x86ScalarCompareForm) (x86OriginalGPInitializedFormat, string) { + // Both compare operands are reads. Reuse the test's ordinary two-input + // producer builder; this does not project a compare into a product BT form. + first, second := form.args[0], form.args[1] + if form.memory == 0 || second.Kind == OpImm { + first, second = second, first + } + for _, operand := range []Operand{first, second} { + if operand.Kind == OpSym && strings.HasPrefix(operand.Sym, "$") { + return x86OriginalGPInitializedFormat{}, "relocated immediate retains its source-bound numerical role" + } + } + for _, operand := range []*Operand{&first, &second} { + if operand.Kind == OpReg { + if base, _, alias := amd64ByteAlias(operand.Reg); alias { + operand.Reg = base + } + } + } + return x86BitTestOriginalInitializedFormat(arch, name, body, x86BitTestForm{index: first, destination: second}) +} + +func TestX86ScalarCompareMalformedForms(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, op := range x86ScalarCompareNames(arch) { + bad := []string{"(AX),(CX)", "$1,$2", "X0,AX", "AX", "AX,CX,DX", "AX,K1"} + if strings.HasPrefix(op, "CMP") { + bad = append(bad, "$1,AX") + } else { + bad = append(bad, "AX,$1") + } + if strings.HasSuffix(op, "B") && arch == "386" { + bad = append(bad, "SP,AX") + } else if !strings.HasSuffix(op, "B") { + bad = append(bad, "AL,AX") + } + if arch == "amd64" { + bad = append(bad, strings.TrimPrefix(x86ScalarCompareInstruction(op, "AX", "$4294967296"), op+" ")) + } + for _, row := range bad { + body := op + " " + row + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+body+"\nRET\n", "", false) + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86ScalarCompareProperties(ins.Op, ins, arch); !known || err == nil { + t.Errorf("invalid Go form admitted: %s %s", arch, body) + } + } + } + } +} + +func TestX86ScalarCompareShortRelocationBackendDefect(t *testing.T) { + for _, op := range []string{"CMPB", "CMPW", "TESTB", "TESTW"} { + t.Run(op, func(t *testing.T) { + body := x86ScalarCompareInstruction(op, "AX", "$data(SB)") + original := "TEXT f(SB),4,$0-0\n" + body + "\nRET\n" + dir := t.TempDir() + source, object := filepath.Join(dir, "forms.s"), filepath.Join(dir, "forms.o") + if err := os.WriteFile(source, []byte(original), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "asm", "-o", object, source) + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH=386", "GOTOOLCHAIN=local") + output, err := runtimeOracleCombinedOutput(ctx, command) + exit, exited := err.(*exec.ExitError) + if ctx.Err() != nil || !exited || exit.ProcessState == nil || !exit.ProcessState.Exited() || + exit.ExitCode() != 1 || string(output) != "asm: need reloc for $data(SB)\nasm: reloc\n" { + t.Fatalf("unknown short relocation backend observation: %v; context %v\n%s", err, ctx.Err(), output) + } + if goAssemblerOracleOutcome(source, object, output, err, ctx.Err(), false, "386") == nil { + t.Fatal("backend defect became an illegal-source certificate") + } + if data, err := os.ReadFile(source); err != nil || string(data) != original { + t.Fatal("original source changed during backend observation", err) + } + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86ScalarCompareProperties(ins.Op, ins, "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("missing original bytes became an ordinary numerical form: %v", err) + } + t.Log("GO_ASSEMBLER_DEFECT_OBSERVATION: short immediate relocation has no sink; not source rejection or a compiled Go object") + }) + } +} + +func TestX86ScalarCompareConditionalReadBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + ctx := &amd64Ctx{goarch: arch, resolve: func(s string) string { return s }} + for _, op := range x86ScalarCompareNames(arch) { + ins := x86PureGPUnknownInstruction(t, x86ScalarCompareInstruction(op, "-7(DX)", "AX")) + form, _, err := x86ScalarCompareProperties(ins.Op, ins, arch) + if err != nil { + t.Fatal(err) + } + full := x86ResultState{flags: x86ByteFlagsAll} + full.gp[0] = 0xff + full.pointerGP[2] = true + before := full.clone() + if accepted, known := ctx.scalarCompareNativeResultTransfer(ins, &full, nil, arch); !known || !accepted { + t.Fatal("bounded-free external read rejected", arch, op) + } + before.flags = x86ByteFlagsAll + if form.test { + before.flags &^= x86ByteFlagAF + } + if !reflect.DeepEqual(full, before) { + t.Fatal("compare acquired more than its defined flags", arch, op) + } + for b := 0; b < form.bits/8; b++ { + state := before.clone() + state.gp[0] &^= 1 << uint(b) + original := state.clone() + if accepted, _ := ctx.scalarCompareNativeResultTransfer(ins, &state, nil, arch); accepted || !reflect.DeepEqual(state, original) { + t.Fatal("uninitialized register byte gained flags", arch, op, b) + } + } + for name, mutate := range map[string]func(*Instr){ + "address-size": func(i *Instr) { i.x86AddressBits = 32 }, + "unbound-size": func(i *Instr) { i.x86UnboundAddressOverride = true }, + "raw": func(i *Instr) { i.x86Encoded = true }, + "float": func(i *Instr) { i.Args[1-form.memory] = Operand{Kind: OpImm, ImmIsFloat: true} }, + "unresolved": func(i *Instr) { i.Args[1-form.memory] = Operand{Kind: OpImm, ImmRaw: "not_defined"} }, + } { + changed := ins + changed.Args = append([]Operand(nil), ins.Args...) + mutate(&changed) + state := before.clone() + if accepted, _ := ctx.scalarCompareNativeResultTransfer(changed, &state, nil, arch); accepted { + t.Fatal("unbound read accepted", arch, op, name) + } + } + } + } +} + +func TestX86ScalarCompareNativeSourceAddressRoles(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + ctx := &amd64Ctx{resolve: func(s string) string { return s }, sigs: map[string]FuncSig{"function": {Name: "aliasedFunction"}}, + continuations: map[string]x86Continuation{"continuation": {}}} + state := x86ResultState{} + state.pointerGP[0], state.gp[1] = true, 0xff + for _, address := range []string{"-7(AX)", "8(AX)(CX*2)"} { + operand := x86PureGPUnknownInstruction(t, "CMPL "+address+",$1").Args[0] + if !ctx.scalarCompareNativeAddress(operand, &state, arch) { + t.Fatal("valid address role rejected", arch, address) + } + } + for _, address := range []string{"external-7(SB)", "external+7(SB)", "function(SB)", "aliasedFunction(SB)", "continuation(SB)", "$external(SB)", "(FS)", "(GS)", "(DX)", "(AX)(DX*4)", "0", "(SP)"} { + operand := x86PureGPUnknownInstruction(t, "CMPL "+address+",$1").Args[0] + if ctx.scalarCompareNativeAddress(operand, &state, arch) { + t.Fatal("unbound address role accepted", arch, address) + } + } + if arch == "386" { + for _, address := range []string{"external(SB)(SP)", "external(SB)(FS)", "external(SB)(X15)", "external(SB)(CX*4)"} { + operand := x86PureGPUnknownInstruction(t, "CMPL "+address+",$1").Args[0] + if ctx.scalarCompareNativeAddress(operand, &state, arch) { + t.Fatal("named address acquired source-layout authority", address) + } + if !x86ScalarCompareMemoryValid(operand, arch) { + t.Fatal("Go ignored base or active index lost its syntax", address) + } + } + operand := x86PureGPUnknownInstruction(t, "CMPL external(SB)(DX*4),$1").Args[0] + if ctx.scalarCompareNativeAddress(operand, &state, arch) { + t.Fatal("active unknown named index treated as ignored base") + } + } + } + ins := x86PureGPUnknownInstruction(t, "CMPB AH,R8B") + var ir strings.Builder + ctx := &amd64Ctx{goarch: "amd64", b: &ir} + if _, _, err := ctx.lowerCmpBt(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || ir.Len() != 0 { + t.Fatalf("unbound REX SP alias emitted code: %v", err) + } +} diff --git a/amd64_scalar_compare_native_read.go b/amd64_scalar_compare_native_read.go new file mode 100644 index 000000000..4381239c4 --- /dev/null +++ b/amd64_scalar_compare_native_read.go @@ -0,0 +1,55 @@ +package plan9asm + +// The paired compare/test emitter performs a volatile native read. If it +// continues, the comparison's inputs are numerical and its defined flags are +// available. This does not assert that an address is backed, in bounds, aligned +// or writable; no pointer identity, memory bytes or reusable proof is produced. +func (c *amd64Ctx) scalarCompareNativeResultTransfer(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + if ins.x86NumericalRead != nil { + return false, false // The immutable source-layout certificate owns this read. + } + form, known, err := x86ScalarCompareProperties(ins.Op, ins, arch) + if !known { + return false, false + } + if err != nil || ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + ins.x86Encoded || ins.x86SourceRaw != nil || ins.x86RawNearReturn != nil { + return false, true + } + if form.memory < 0 { + // The shared ordinary source transfer owns register-only effects, + // including separately authenticated caller/literal address predicates. + // A native-memory retry must not intercept those independent roles. + return false, false + } + for index, operand := range form.args { + if index != form.memory || !x86OrdinaryNativeMemoryOperand(operand, arch) { + if operand.Kind == OpReg && !state.shapeOnly { + register, _, valid := x86ResultGP(operand.Reg, int64(form.bits/8), arch) + if valid && state.pointerGP[register] { + return false, true + } + } + if !x86ResultRead(operand, int64(form.bits/8), state, owned, arch) { + return false, true + } + continue + } + if !x86ResultReadWithNativeGPContinuation(ins, operand, int64(form.bits/8), state, owned, arch) { + return false, true + } + } + state.flags = x86ByteFlagsAll + if form.test { + state.flags &^= x86ByteFlagAF + } + return true, true +} + +func (c *amd64Ctx) scalarCompareNativeAddress(operand Operand, state *x86ResultState, arch string) bool { + // This rule deliberately excludes unbound TLS/segments, absolute numerical + // addresses, unknown GP bases and virtual frame addresses. External SB needs + // its existing actual-source numerical certificate: a volatile read alone + // cannot establish that an LLVM global preserves original Go field layout. + return x86ResultIncomingPointerReadAddressIdentity(operand, state, arch) +} diff --git a/amd64_scalar_compare_native_test.go b/amd64_scalar_compare_native_test.go new file mode 100644 index 000000000..0eb9c42d0 --- /dev/null +++ b/amd64_scalar_compare_native_test.go @@ -0,0 +1,216 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func x86ScalarCompareNames(arch string) []string { + var names []string + for _, stem := range []string{"CMP", "TEST"} { + for _, suffix := range []string{"B", "W", "L", "Q"} { + if arch != "386" || suffix != "Q" { + names = append(names, stem+suffix) + } + } + } + return names +} + +func x86ScalarCompareInstruction(op, memory, other string) string { + if strings.HasPrefix(op, "TEST") { + return op + " " + other + "," + memory + } + return op + " " + memory + "," + other +} + +func x86ScalarCompareResultSource(name, body string) string { + // PEXTRD, unlike the older MOVL X0,AX compatibility path, actually invokes + // the private scalar-result observer. Both branches initialize every byte. + return fmt.Sprintf("TEXT ·%s(SB),4,$0-4\n%s\nJE equal\nMOVL $7,AX\nMOVL AX,X0\nJMP done\nequal:\nPXOR X0,X0\ndone:\nPEXTRD $0,X0,AX\nMOVL AX,ret+0(FP)\nRET\n", name, body) +} + +func TestX86ScalarCompareSourceDataResultFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc is required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + source.WriteString("GLOBL external(SB),8,$16\n") + for index, op := range x86ScalarCompareNames(target.arch) { + name := fmt.Sprintf("compare%d", index) + source.WriteString(x86ScalarCompareResultSource(name, x86ScalarCompareInstruction(op, "external+3(SB)", "$1"))) + fmt.Fprintf(&declarations, "func %s() uint32\n", name) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "compare.ll", "compare.o", ir) + }) + } + }) + } +} + +func TestX86ScalarCompareUnboundExternalResultStaysContext(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, op := range x86ScalarCompareNames(target.arch) { + for _, offset := range []int{-7, 0, 3} { + source := x86ScalarCompareResultSource("view", x86ScalarCompareInstruction(op, fmt.Sprintf("external%+d(SB)", offset), "$1")) + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, source, "var external [32]byte\nfunc view() uint32", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("external source-layout guard bypassed: %s/%s/%s/%d: %v", target.triple, api, op, offset, err) + } + } + } + } + } +} + +func x86ScalarComparePointerSource(arch, name, setup, compare, tail string) string { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + source := x86ScalarCompareResultSource(name, move+" p+0(FP),DX\n"+setup+compare+"\n"+tail) + source = strings.Replace(source, "$0-4", fmt.Sprintf("$0-%d", word+4), 1) + return strings.ReplaceAll(source, "ret+0(FP)", fmt.Sprintf("ret+%d(FP)", word)) +} + +func TestX86ScalarCompareIncomingPointerResultFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + for index, op := range x86ScalarCompareNames(target.arch) { + name := fmt.Sprintf("view%d", index) + compare := x86ScalarCompareInstruction(op, "-7(DX)(CX*2)", "$1") + source.WriteString(x86ScalarComparePointerSource(target.arch, name, "MOVL $4,CX\n", compare, "")) + fmt.Fprintf(&declarations, "func %s(p *byte) uint32\n", name) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "compare-pointer.ll", "compare-pointer.o", ir) + }) + } + }) + } +} + +func TestX86ScalarCompareIncomingPointerDoesNotGrantOtherEffects(t *testing.T) { + for _, target := range x86RawReturnTargets { + move := "MOVQ" + if target.arch == "386" { + move = "MOVL" + } + for _, op := range x86ScalarCompareNames(target.arch) { + for _, row := range []struct{ name, setup, address, partner, tail string }{ + {"unknown-index", "", "(DX)(CX*2)", "$1", ""}, + {"opaque-index", move + " p+0(FP),CX\n", "(DX)(CX*2)", "$1", ""}, + {"numeric-not-pointer", move + " $1234,DX\n", "(DX)", "$1", ""}, + {"unknown-numerical-input", "", "(DX)", "AX", ""}, + {"opaque-numerical-input", move + " p+0(FP),AX\n", "(DX)", "AX", ""}, + {"no-reusable-backing-credit", "", "(DX)", "$1", "MOVL (DX),DX\nMOVL (DX),AX\n"}, + {"no-write-data-credit", "", "(DX)", "$1", "MOVL BX,(DX)\n"}, + {"no-segment-credit", "", "(FS)", "$1", ""}, + {"unknown-segment-offset", "", "8(CX)(FS*2)", "$1", ""}, + {"opaque-segment-offset", move + " p+0(FP),CX\n", "8(CX)(FS*2)", "$1", ""}, + } { + source := x86ScalarComparePointerSource(target.arch, "view", row.setup, + x86ScalarCompareInstruction(op, row.address, row.partner), row.tail) + paired := row.name == "numeric-not-pointer" || row.name == "no-reusable-backing-credit" + if paired { + x86RequireNumericNativeSourceContext(t, source, "func view(p *byte) uint32", target.arch) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *byte) uint32", target.arch, target.triple, api) + if paired { + bits := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[op[len(op)-1]] + reads := 1 + if row.name == "no-reusable-backing-credit" { + reads = 3 + } + if err != nil || !strings.Contains(ir, fmt.Sprintf("load volatile i%d,", bits)) || strings.Count(ir, "load volatile") != reads || + row.name == "no-reusable-backing-credit" && strings.Count(ir, "load volatile i32,") < 2 { + t.Fatalf("independent NUM reads lost their widths or effects %s/%s/%s/%s: %v", target.triple, op, row.name, api, err) + } + x86NumericNativeAddressObject(t, ir, target.triple) + continue + } + if row.name == "no-segment-credit" { + // Keep the original tuple. Its absolute hardware base + // needs no incoming GP numerical or pointer authority. + if err != nil || !strings.Contains(ir, "ptr addrspace(257)") || !strings.Contains(ir, "load volatile") { + t.Fatalf("original hardware-segment read was lost %s/%s/%s: %v", target.triple, op, api, err) + } + continue + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("conditional read expanded authority %s/%s/%s/%s: %v", target.triple, op, row.name, api, err) + } + } + } + if strings.HasPrefix(op, "TEST") { + source := x86ScalarComparePointerSource(target.arch, "view", "", + x86ScalarCompareInstruction(op, "(DX)", "$1"), "LAHF\n") + if _, err := x86ScalarFPBitViewAPI(t, source, "func view(p *byte) uint32", target.arch, target.triple, "text"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("TEST undefined AF became observable: %s/%s: %v", target.triple, op, err) + } + } + } + } +} + +// The compare's continuation grants no credit to another read. Ordinary MOVL +// now has its own paired volatile load and numerical effect; both operations +// must survive even though the MOVL result is dead. Overwriting DX revokes its +// pointer identity; a later complete NUM read needs its own paired effect. +func TestX86ScalarCompareAndMoveRetainIndependentNativeReads(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + names := x86ScalarCompareNames(target.arch) + for index, op := range names { + name := fmt.Sprintf("CompareMove%d", index) + compare := x86ScalarCompareInstruction(op, "(DX)", "$1") + source.WriteString(x86ScalarComparePointerSource(target.arch, name, "", compare, "MOVL (DX),AX\n")) + fmt.Fprintf(&declarations, "func %s(p *byte) uint32\n", name) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("separately paired native reads rejected", api, err) + } + for level, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + if got := strings.Count(text, "load volatile"); got < 2*len(names) { + t.Fatal("a native read borrowed earlier backing or was discarded", api, level, got) + } + x86ResultInitializationObject(t, llc, target.triple, text, fmt.Sprintf("compare-move-%s-O%d", api, level*2)) + } + } + }) + } +} diff --git a/amd64_scalar_compare_runtime_test.go b/amd64_scalar_compare_runtime_test.go new file mode 100644 index 000000000..523a8780e --- /dev/null +++ b/amd64_scalar_compare_runtime_test.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ScalarCompareFlags(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, native, prototypes, pairs strings.Builder + count := 0 + for _, op := range x86ScalarCompareNames("amd64") { + rows := []string{"(DX),CX", "CX,(DX)"} + for _, immediate := range []string{"0", "127", "128", "255", "32768", "65535", "2147483648", "4294967295", "-2147483648"} { + rows = append(rows, strings.TrimPrefix(x86ScalarCompareInstruction(op, "(DX)", "$"+immediate), op+" ")) + } + if strings.HasSuffix(op, "B") { + rows = append(rows, "CH,(DX)", "(DX),CH", "CL,CH", "CH,CL", "BH,R8B", "R8B,DH") + } + for _, row := range rows { + name, original := fmt.Sprintf("compare%d", count), fmt.Sprintf("original%d", count) + mask := 0x8d5 // CF, PF, AF, ZF, SF, OF only. + if strings.HasPrefix(op, "TEST") { + mask &^= 0x10 // TEST leaves AF undefined; never invent an oracle value. + } + body := op + " " + row + setup, originalSetup := "", "" + if strings.Contains(row, "R8B") { + setup = "MOVQ x+8(FP),DI\nMOVQ x+8(FP),SI\nMOVQ x+8(FP),R8\nNOTQ R8\n" + originalSetup = "MOVQ SI,DI\nMOVQ SI,R8\nNOTQ R8\n" + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-24\nMOVQ p+0(FP),DX\nMOVQ x+8(FP),CX\n%s%s\nPUSHFQ\nPOPQ AX\nANDQ $%d,AX\nMOVQ AX,ret+16(FP)\nRET\n", name, setup, body, mask) + fmt.Fprintf(&declarations, "func %s(p *byte,x uint64) uint64\n", name) + originalBody := strings.ReplaceAll(body, "DX", "DI") + code := x86PackedShiftActualGoBytes(t, "amd64", fmt.Sprintf("TEXT f(SB),4,$0-0\nMOVQ SI,CX\n%s%s\nPUSHFQ\nPOPQ AX\nANDQ $%d,AX\nRET\n", originalSetup, originalBody, mask)) + if len(code) < 9 || code[len(code)-1] != 0xc3 { + t.Fatal("original Go flags program is incomplete", body, code) + } + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint8_t*,uint64_t);\n", name) + prototypes.WriteString(x86OriginalSysVDeclaration("uint64_t", original, "uint8_t*,uint64_t")) + fmt.Fprintf(&pairs, "{%s,%s},\n", original, name) + count++ + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + main := ` +#include +#include +#include +` + prototypes.String() + native.String() + x86OriginalSysVPointer("uint64_t", "original_program", "uint8_t*,uint64_t") + ` +typedef uint64_t (*program)(uint8_t*,uint64_t); +static const struct { original_program original; program translated; } pairs[]={ +` + pairs.String() + `}; +int main(void) { + const uint64_t patterns[]={0,1,15,16,127,128,255,256,32767,32768,65535,65536, + UINT64_C(0x7fffffff),UINT64_C(0x80000000),UINT64_C(0xffffffff), + UINT64_C(0x7fffffffffffffff),UINT64_C(0x8000000000000000), + UINT64_C(0xffffffffffffffff),UINT64_C(0x55aa0123cdef9876)}; + for(unsigned i=0;i